Merge branch 'bvl-i18n-validation' into 'master'

Updates to `.po` & `.pot` linting

Closes #46096 et #39292

See merge request gitlab-org/gitlab-ce!18779

app/assets/javascripts/ide/components/commit_sidebar/list_collapsed.vue

@@ -38,14 +38,17 @@ export default {
       return this.modifiedFilesLength ? 'multi-file-modified' : '';
     },
     additionsTooltip() {
-      return sprintf(n__('1 %{type} addition', '%d %{type} additions', this.addedFilesLength), {
+      return sprintf(n__('1 %{type} addition', '%{count} %{type} additions', this.addedFilesLength), {
         type: this.title.toLowerCase(),
+        count: this.addedFilesLength,
       });
     },
     modifiedTooltip() {
       return sprintf(
-        n__('1 %{type} modification', '%d %{type} modifications', this.modifiedFilesLength),
-        { type: this.title.toLowerCase() },
+        n__('1 %{type} modification', '%{count} %{type} modifications', this.modifiedFilesLength), {
+          type: this.title.toLowerCase(),
+          count: this.modifiedFilesLength,
+        },
       );
     },
     titleTooltip() {

lib/gitlab/i18n.rb

@@ -21,7 +21,8 @@ module Gitlab
       'nl_NL' => 'Nederlands',
       'tr_TR' => 'Türkçe',
       'id_ID' => 'Bahasa Indonesia',
-      'fil_PH' => 'Filipino'
+      'fil_PH' => 'Filipino',
+      'pl_PL' => 'Polski'
     }.freeze
 
     def available_locales

lib/gitlab/i18n/metadata_entry.rb

@@ -3,16 +3,25 @@ module Gitlab
     class MetadataEntry
       attr_reader :entry_data
 
+      # Avoid testing too many plurals if `nplurals` was incorrectly set.
+      # Based on info on https://www.gnu.org/software/gettext/manual/html_node/Plural-forms.html
+      # which mentions special cases for numbers ending in 2 digits
+      MAX_FORMS_TO_TEST = 101
+
       def initialize(entry_data)
         @entry_data = entry_data
       end
 
-      def expected_plurals
+      def expected_forms
         return nil unless plural_information
 
         plural_information['nplurals'].to_i
       end
 
+      def forms_to_test
+        @forms_to_test ||= [expected_forms, MAX_FORMS_TO_TEST].compact.min
+      end
+
       private
 
       def plural_information

lib/gitlab/i18n/po_linter.rb

 module Gitlab
   module I18n
     class PoLinter
+      include Gitlab::Utils::StrongMemoize
+
       attr_reader :po_path, :translation_entries, :metadata_entry, :locale
 
       VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze
@@ -34,7 +36,7 @@ module Gitlab
         end
 
         @translation_entries = entries.map do |entry_data|
-          Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_plurals)
+          Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_forms)
         end
 
         nil
@@ -48,7 +50,7 @@ module Gitlab
 
         translation_entries.each do |entry|
           errors_for_entry = validate_entry(entry)
-          errors[join_message(entry.msgid)] = errors_for_entry if errors_for_entry.any?
+          errors[entry.msgid] = errors_for_entry if errors_for_entry.any?
         end
 
         errors
@@ -62,6 +64,7 @@ module Gitlab
         validate_newlines(errors, entry)
         validate_number_of_plurals(errors, entry)
         validate_unescaped_chars(errors, entry)
+        validate_translation(errors, entry)
 
         errors
       end
@@ -81,35 +84,39 @@ module Gitlab
       end
 
       def validate_number_of_plurals(errors, entry)
-        return unless metadata_entry&.expected_plurals
+        return unless metadata_entry&.expected_forms
         return unless entry.translated?
 
-        if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_plurals
-          errors << "should have #{metadata_entry.expected_plurals} "\
-                    "#{'translations'.pluralize(metadata_entry.expected_plurals)}"
+        if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_forms
+          errors << "should have #{metadata_entry.expected_forms} "\
+                    "#{'translations'.pluralize(metadata_entry.expected_forms)}"
         end
       end
 
       def validate_newlines(errors, entry)
-        if entry.msgid_contains_newlines?
+        if entry.msgid_has_multiple_lines?
           errors << 'is defined over multiple lines, this breaks some tooling.'
         end
 
-        if entry.plural_id_contains_newlines?
+        if entry.plural_id_has_multiple_lines?
           errors << 'plural is defined over multiple lines, this breaks some tooling.'
         end
 
-        if entry.translations_contain_newlines?
+        if entry.translations_have_multiple_lines?
           errors << 'has translations defined over multiple lines, this breaks some tooling.'
         end
       end
 
       def validate_variables(errors, entry)
         if entry.has_singular_translation?
+          validate_variables_in_message(errors, entry.msgid, entry.msgid)
+
           validate_variables_in_message(errors, entry.msgid, entry.singular_translation)
         end
 
         if entry.has_plural?
+          validate_variables_in_message(errors, entry.plural_id, entry.plural_id)
+
           entry.plural_translations.each do |translation|
             validate_variables_in_message(errors, entry.plural_id, translation)
           end
@@ -117,41 +124,98 @@ module Gitlab
       end
 
       def validate_variables_in_message(errors, message_id, message_translation)
-        message_id = join_message(message_id)
         required_variables = message_id.scan(VARIABLE_REGEX)
 
         validate_unnamed_variables(errors, required_variables)
-        validate_translation(errors, message_id, required_variables)
         validate_variable_usage(errors, message_translation, required_variables)
       end
 
-      def validate_translation(errors, message_id, used_variables)
+      def validate_translation(errors, entry)
+        Gitlab::I18n.with_locale(locale) do
+          if entry.has_plural?
+            translate_plural(entry)
+          else
+            translate_singular(entry)
+          end
+        end
+
+      # `sprintf` could raise an `ArgumentError` when invalid passing something
+      # other than a Hash when using named variables
+      #
+      # `sprintf` could raise `TypeError` when passing a wrong type when using
+      # unnamed variables
+      #
+      # FastGettext::Translation could raise `RuntimeError` (raised as a string),
+      # or as subclassess `NoTextDomainConfigured` & `InvalidFormat`
+      #
+      # `FastGettext::Translation` could raise `ArgumentError` as subclassess
+      # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter`
+      rescue ArgumentError, TypeError, RuntimeError => e
+        errors << "Failure translating to #{locale}: #{e.message}"
+      end
+
+      def translate_singular(entry)
+        used_variables = entry.msgid.scan(VARIABLE_REGEX)
         variables = fill_in_variables(used_variables)
 
-        begin
-          Gitlab::I18n.with_locale(locale) do
-            translated = if message_id.include?('|')
-                           FastGettext::Translation.s_(message_id)
-                         else
-                           FastGettext::Translation._(message_id)
-                         end
+        translation = if entry.msgid.include?('|')
+                        FastGettext::Translation.s_(entry.msgid)
+                      else
+                        FastGettext::Translation._(entry.msgid)
+                      end
 
-            translated % variables
+        translation % variables if used_variables.any?
+      end
+
+      def translate_plural(entry)
+        used_variables = entry.plural_id.scan(VARIABLE_REGEX)
+        variables = fill_in_variables(used_variables)
+
+        numbers_covering_all_plurals.map do |number|
+          translation = FastGettext::Translation.n_(entry.msgid, entry.plural_id, number)
+
+          translation % variables if used_variables.any?
+        end
+      end
+
+      def numbers_covering_all_plurals
+        @numbers_covering_all_plurals ||= calculate_numbers_covering_all_plurals
+      end
+
+      def calculate_numbers_covering_all_plurals
+        required_numbers = []
+        discovered_indexes = []
+        counter = 0
+
+        while discovered_indexes.size < metadata_entry.forms_to_test && counter < Gitlab::I18n::MetadataEntry::MAX_FORMS_TO_TEST
+          index_for_count = index_for_pluralization(counter)
+
+          unless discovered_indexes.include?(index_for_count)
+            discovered_indexes << index_for_count
+            required_numbers << counter
           end
 
-        # `sprintf` could raise an `ArgumentError` when invalid passing something
-        # other than a Hash when using named variables
-        #
-        # `sprintf` could raise `TypeError` when passing a wrong type when using
-        # unnamed variables
-        #
-        # FastGettext::Translation could raise `RuntimeError` (raised as a string),
-        # or as subclassess `NoTextDomainConfigured` & `InvalidFormat`
-        #
-        # `FastGettext::Translation` could raise `ArgumentError` as subclassess
-        # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter`
-        rescue ArgumentError, TypeError, RuntimeError => e
-          errors << "Failure translating to #{locale} with #{variables}: #{e.message}"
+          counter += 1
+        end
+
+        required_numbers
+      end
+
+      def index_for_pluralization(counter)
+        # This calls the C function that defines the pluralization rule, it can
+        # return a boolean (`false` represents 0, `true` represents 1) or an integer
+        # that specifies the plural form to be used for the given number
+        pluralization_result = Gitlab::I18n.with_locale(locale) do
+          FastGettext.pluralisation_rule.call(counter)
+        end
+
+        case pluralization_result
+        when false
+          0
+        when true
+          1
+        else
+          pluralization_result
         end
       end
 
@@ -172,14 +236,18 @@ module Gitlab
       end
 
       def validate_unnamed_variables(errors, variables)
-        if  variables.size > 1 && variables.any? { |variable_name| unnamed_variable?(variable_name) }
+        unnamed_variables, named_variables = variables.partition { |name| unnamed_variable?(name) }
+
+        if unnamed_variables.any? && named_variables.any?
+          errors << 'is combining named variables with unnamed variables'
+        end
+
+        if unnamed_variables.size > 1
           errors << 'is combining multiple unnamed variables'
         end
       end
 
       def validate_variable_usage(errors, translation, required_variables)
-        translation = join_message(translation)
-
         # We don't need to validate when the message is empty.
         # In this case we fall back to the default, which has all the the
         # required variables.
@@ -205,10 +273,6 @@ module Gitlab
       def validate_flags(errors, entry)
         errors << "is marked #{entry.flag}" if entry.flag
       end
-
-      def join_message(message)
-        Array(message).join
-      end
     end
   end
 end

lib/gitlab/i18n/translation_entry.rb

@@ -11,11 +11,11 @@ module Gitlab
       end
 
       def msgid
-        entry_data[:msgid]
+        @msgid ||= Array(entry_data[:msgid]).join
       end
 
       def plural_id
-        entry_data[:msgid_plural]
+        @plural_id ||= Array(entry_data[:msgid_plural]).join
       end
 
       def has_plural?
@@ -23,12 +23,11 @@ module Gitlab
       end
 
       def singular_translation
-        all_translations.first if has_singular_translation?
+        all_translations.first.to_s if has_singular_translation?
       end
 
       def all_translations
-        @all_translations ||= entry_data.fetch_values(*translation_keys)
-                                .reject(&:empty?)
+        @all_translations ||= translation_entries.map { |translation| Array(translation).join }
       end
 
       def translated?
@@ -54,16 +53,16 @@ module Gitlab
         nplurals > 1 || !has_plural?
       end
 
-      def msgid_contains_newlines?
-        msgid.is_a?(Array)
+      def msgid_has_multiple_lines?
+        entry_data[:msgid].is_a?(Array)
       end
 
-      def plural_id_contains_newlines?
-        plural_id.is_a?(Array)
+      def plural_id_has_multiple_lines?
+        entry_data[:msgid_plural].is_a?(Array)
       end
 
-      def translations_contain_newlines?
-        all_translations.any? { |translation| translation.is_a?(Array) }
+      def translations_have_multiple_lines?
+        translation_entries.any? { |translation| translation.is_a?(Array) }
       end
 
       def msgid_contains_unescaped_chars?
@@ -84,6 +83,11 @@ module Gitlab
 
       private
 
+      def translation_entries
+        @translation_entries ||= entry_data.fetch_values(*translation_keys)
+                                   .reject(&:empty?)
+      end
+
       def translation_keys
         @translation_keys ||= entry_data.keys.select { |key| key.to_s =~ /\Amsgstr(\[\d+\])?\z/ }
       end

lib/tasks/gettext.rake

@@ -50,6 +50,32 @@ namespace :gettext do
     end
   end
 
+  task :updated_check do
+    # Removing all pre-translated files speeds up `gettext:find` as the
+    # files don't need to be merged.
+    `rm locale/*/gitlab.po`
+
+    # `gettext:find` writes touches to temp files to `stderr` which would cause
+    # `static-analysis` to report failures. We can ignore these
+    silence_stream(STDERR) { Rake::Task['gettext:find'].invoke }
+
+    changed_files = `git diff --name-only`.lines.map(&:strip)
+
+    # reset the locale folder for potential next tasks
+    `git checkout -- locale`
+
+    if changed_files.include?('locale/gitlab.pot')
+      raise <<~MSG
+        Newly translated strings found, please add them to `gitlab.pot` by running:
+
+          bundle exec rake gettext:find; git checkout -- locale/*/gitlab.po;
+
+        Then commit and push the resulting changes to `locale/gitlab.pot`.
+
+      MSG
+    end
+  end
+
   def report_errors_for_file(file, errors_for_file)
     puts "Errors in `#{file}`:"
 

lib/tasks/lint.rake

@@ -27,6 +27,7 @@ unless Rails.env.production?
         scss_lint
         flay
         gettext:lint
+        gettext:updated_check
         lint:static_verification
       ].each do |task|
         pid = Process.fork do

locale/gitlab.pot

@@ -8,8 +8,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: gitlab 1.0.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2018-06-12 18:57+1000\n"
-"PO-Revision-Date: 2018-06-12 18:57+1000\n"
+"POT-Creation-Date: 2018-06-13 14:05+0200\n"
+"PO-Revision-Date: 2018-06-13 14:05+0200\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
 "Language: \n"
@@ -133,12 +133,12 @@ msgid "- show less"
 msgstr ""
 
 msgid "1 %{type} addition"
-msgid_plural "%d %{type} additions"
+msgid_plural "%{count} %{type} additions"
 msgstr[0] ""
 msgstr[1] ""
 
 msgid "1 %{type} modification"
-msgid_plural "%d %{type} modifications"
+msgid_plural "%{count} %{type} modifications"
 msgstr[0] ""
 msgstr[1] ""
 
@@ -2252,10 +2252,10 @@ msgstr ""
 msgid "Gitaly"
 msgstr ""
 
-msgid "Gitaly|Address"
+msgid "Gitaly Servers"
 msgstr ""
 
-msgid "Gitaly Servers"
+msgid "Gitaly|Address"
 msgstr ""
 
 msgid "Go Back"
@@ -2419,6 +2419,15 @@ msgstr ""
 msgid "If your HTTP repository is not publicly accessible, add authentication information to the URL: <code>https://username:password@gitlab.company.com/group/project.git</code>."
 msgstr ""
 
+msgid "ImageDiffViewer|2-up"
+msgstr ""
+
+msgid "ImageDiffViewer|Onion skin"
+msgstr ""
+
+msgid "ImageDiffViewer|Swipe"
+msgstr ""
+
 msgid "Import"
 msgstr ""
 

spec/lib/gitlab/i18n/metadata_entry_spec.rb

 require 'spec_helper'
 
 describe Gitlab::I18n::MetadataEntry do
-  describe '#expected_plurals' do
+  describe '#expected_forms' do
     it 'returns the number of plurals' do
       data = {
         msgid: "",
@@ -22,7 +22,7 @@ describe Gitlab::I18n::MetadataEntry do
       }
       entry = described_class.new(data)
 
-      expect(entry.expected_plurals).to eq(2)
+      expect(entry.expected_forms).to eq(2)
     end
 
     it 'returns 0 for the POT-metadata' do
@@ -45,7 +45,7 @@ describe Gitlab::I18n::MetadataEntry do
       }
       entry = described_class.new(data)
 
-      expect(entry.expected_plurals).to eq(0)
+      expect(entry.expected_forms).to eq(0)
     end
   end
 end

spec/lib/gitlab/i18n/translation_entry_spec.rb

@@ -109,7 +109,7 @@ describe Gitlab::I18n::TranslationEntry do
       data = { msgid: %w(hello world) }
       entry = described_class.new(data, 2)
 
-      expect(entry.msgid_contains_newlines?).to be_truthy
+      expect(entry.msgid_has_multiple_lines?).to be_truthy
     end
   end
 
@@ -118,7 +118,7 @@ describe Gitlab::I18n::TranslationEntry do
       data = { msgid_plural: %w(hello world) }
       entry = described_class.new(data, 2)
 
-      expect(entry.plural_id_contains_newlines?).to be_truthy
+      expect(entry.plural_id_has_multiple_lines?).to be_truthy
     end
   end
 
@@ -127,7 +127,7 @@ describe Gitlab::I18n::TranslationEntry do
       data = { msgstr: %w(hello world) }
       entry = described_class.new(data, 2)
 
-      expect(entry.translations_contain_newlines?).to be_truthy
+      expect(entry.translations_have_multiple_lines?).to be_truthy
     end
   end