Automatic merge of gitlab-org/gitlab-ce master

app/models/programming_language.rb

@@ -3,4 +3,10 @@
 class ProgrammingLanguage < ActiveRecord::Base
   validates :name, presence: true
   validates :color, allow_blank: false, color: true
+
+  # Returns all programming languages which match the given name (case
+  # insensitively).
+  scope :with_name_case_insensitive, ->(name) do
+    where(arel_table[:name].matches(sanitize_sql_like(name)))
+  end
 end

app/models/project.rb

@@ -389,6 +389,16 @@ class Project < ActiveRecord::Base
     with_project_feature.where(project_features: { access_level_attribute => level })
   }
 
+  # Picks projects which use the given programming language
+  scope :with_programming_language, ->(language_name) do
+    lang_id_query = ProgrammingLanguage
+        .with_name_case_insensitive(language_name)
+        .select(:id)
+
+    joins(:repository_languages)
+        .where(repository_languages: { programming_language_id: lang_id_query })
+  end
+
   scope :with_builds_enabled, -> { with_feature_enabled(:builds) }
   scope :with_issues_enabled, -> { with_feature_enabled(:issues) }
   scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) }

changelogs/unreleased/46448-add-timestamps-for-each-stage-of-gitlab-rake-gitlab-backup-restore.yml

+---
+title: Display timestamps to messages printed by gitlab:backup:restore rake tasks
+merge_request:
+author: Will Chandler
+type: changed

changelogs/unreleased/51759-filter-by-language.yml

+---
+title: Add `with_programming_language` filter for projects to API
+merge_request: 24377
+author: Dylan MacKenzie
+type: added

doc/api/projects.md

@@ -54,6 +54,7 @@ GET /projects
 | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only) |
 | `with_issues_enabled` | boolean | no | Limit by enabled issues feature |
 | `with_merge_requests_enabled` | boolean | no | Limit by enabled merge requests feature |
+| `with_programming_language` | string | no | Limit by projects which use the given programming language |
 | `wiki_checksum_failed` | boolean | no | Limit projects where the wiki checksum calculation has failed _([Introduced][ee-6137] in [GitLab Premium][eep] 11.2)_ |
 | `repository_checksum_failed` | boolean | no | Limit projects where the repository checksum calculation has failed _([Introduced][ee-6137] in [GitLab Premium][eep] 11.2)_ |
 | `min_access_level` | integer | no | Limit by current user minimal [access level](members.md) |
@@ -281,6 +282,7 @@ GET /users/:user_id/projects
 | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only) |
 | `with_issues_enabled` | boolean | no | Limit by enabled issues feature |
 | `with_merge_requests_enabled` | boolean | no | Limit by enabled merge requests feature |
+| `with_programming_language` | string | no | Limit by projects which use the given programming language |
 | `min_access_level` | integer | no | Limit by current user minimal [access level](members.md) |
 
 ```json

lib/api/projects.rb

@@ -25,6 +25,9 @@ module API
         projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
         projects = projects.with_statistics if params[:statistics]
 
+        lang = params[:with_programming_language]
+        projects = projects.with_programming_language(lang) if lang
+
         projects
       end
 
@@ -93,6 +96,7 @@ module API
         optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of'
         optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature'
         optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature'
+        optional :with_programming_language, type: String, desc: 'Limit to repositories which use the given programming language'
         optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Limit by minimum access level of authenticated user'
 
         use :optional_filter_params_ee

lib/tasks/gitlab/backup.rake

@@ -47,9 +47,9 @@ namespace :gitlab do
 
           # Drop all tables Load the schema to ensure we don't have any newer tables
           # hanging out from a failed upgrade
-          progress.puts 'Cleaning the database ... '.color(:blue)
+          puts_time 'Cleaning the database ... '.color(:blue)
           Rake::Task['gitlab:db:drop_tables'].invoke
-          progress.puts 'done'.color(:green)
+          puts_time 'done'.color(:green)
           Rake::Task['gitlab:backup:db:restore'].invoke
         rescue Gitlab::TaskAbortedByUserError
           puts "Quitting...".color(:red)
@@ -72,165 +72,169 @@ namespace :gitlab do
 
     namespace :repo do
       task create: :gitlab_environment do
-        progress.puts "Dumping repositories ...".color(:blue)
+        puts_time "Dumping repositories ...".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("repositories")
-          progress.puts "[SKIPPED]".color(:cyan)
+          puts_time "[SKIPPED]".color(:cyan)
         else
           Backup::Repository.new(progress).dump
-          progress.puts "done".color(:green)
+          puts_time "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        progress.puts "Restoring repositories ...".color(:blue)
+        puts_time "Restoring repositories ...".color(:blue)
         Backup::Repository.new(progress).restore
-        progress.puts "done".color(:green)
+        puts_time "done".color(:green)
       end
     end
 
     namespace :db do
       task create: :gitlab_environment do
-        progress.puts "Dumping database ... ".color(:blue)
+        puts_time "Dumping database ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("db")
-          progress.puts "[SKIPPED]".color(:cyan)
+          puts_time "[SKIPPED]".color(:cyan)
         else
           Backup::Database.new(progress).dump
-          progress.puts "done".color(:green)
+          puts_time "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        progress.puts "Restoring database ... ".color(:blue)
+        puts_time "Restoring database ... ".color(:blue)
         Backup::Database.new(progress).restore
-        progress.puts "done".color(:green)
+        puts_time "done".color(:green)
       end
     end
 
     namespace :builds do
       task create: :gitlab_environment do
-        progress.puts "Dumping builds ... ".color(:blue)
+        puts_time "Dumping builds ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("builds")
-          progress.puts "[SKIPPED]".color(:cyan)
+          puts_time "[SKIPPED]".color(:cyan)
         else
           Backup::Builds.new(progress).dump
-          progress.puts "done".color(:green)
+          puts_time "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        progress.puts "Restoring builds ... ".color(:blue)
+        puts_time "Restoring builds ... ".color(:blue)
         Backup::Builds.new(progress).restore
-        progress.puts "done".color(:green)
+        puts_time "done".color(:green)
       end
     end
 
     namespace :uploads do
       task create: :gitlab_environment do
-        progress.puts "Dumping uploads ... ".color(:blue)
+        puts_time "Dumping uploads ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("uploads")
-          progress.puts "[SKIPPED]".color(:cyan)
+          puts_time "[SKIPPED]".color(:cyan)
         else
           Backup::Uploads.new(progress).dump
-          progress.puts "done".color(:green)
+          puts_time "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        progress.puts "Restoring uploads ... ".color(:blue)
+        puts_time "Restoring uploads ... ".color(:blue)
         Backup::Uploads.new(progress).restore
-        progress.puts "done".color(:green)
+        puts_time "done".color(:green)
       end
     end
 
     namespace :artifacts do
       task create: :gitlab_environment do
-        progress.puts "Dumping artifacts ... ".color(:blue)
+        puts_time "Dumping artifacts ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
-          progress.puts "[SKIPPED]".color(:cyan)
+          puts_time "[SKIPPED]".color(:cyan)
         else
           Backup::Artifacts.new(progress).dump
-          progress.puts "done".color(:green)
+          puts_time "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        progress.puts "Restoring artifacts ... ".color(:blue)
+        puts_time "Restoring artifacts ... ".color(:blue)
         Backup::Artifacts.new(progress).restore
-        progress.puts "done".color(:green)
+        puts_time "done".color(:green)
       end
     end
 
     namespace :pages do
       task create: :gitlab_environment do
-        progress.puts "Dumping pages ... ".color(:blue)
+        puts_time "Dumping pages ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("pages")
-          progress.puts "[SKIPPED]".color(:cyan)
+          puts_time "[SKIPPED]".color(:cyan)
         else
           Backup::Pages.new(progress).dump
-          progress.puts "done".color(:green)
+          puts_time "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        progress.puts "Restoring pages ... ".color(:blue)
+        puts_time "Restoring pages ... ".color(:blue)
         Backup::Pages.new(progress).restore
-        progress.puts "done".color(:green)
+        puts_time "done".color(:green)
       end
     end
 
     namespace :lfs do
       task create: :gitlab_environment do
-        progress.puts "Dumping lfs objects ... ".color(:blue)
+        puts_time "Dumping lfs objects ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("lfs")
-          progress.puts "[SKIPPED]".color(:cyan)
+          puts_time "[SKIPPED]".color(:cyan)
         else
           Backup::Lfs.new(progress).dump
-          progress.puts "done".color(:green)
+          puts_time "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        progress.puts "Restoring lfs objects ... ".color(:blue)
+        puts_time "Restoring lfs objects ... ".color(:blue)
         Backup::Lfs.new(progress).restore
-        progress.puts "done".color(:green)
+        puts_time "done".color(:green)
       end
     end
 
     namespace :registry do
       task create: :gitlab_environment do
-        progress.puts "Dumping container registry images ... ".color(:blue)
+        puts_time "Dumping container registry images ... ".color(:blue)
 
         if Gitlab.config.registry.enabled
           if ENV["SKIP"] && ENV["SKIP"].include?("registry")
-            progress.puts "[SKIPPED]".color(:cyan)
+            puts_time "[SKIPPED]".color(:cyan)
           else
             Backup::Registry.new(progress).dump
-            progress.puts "done".color(:green)
+            puts_time "done".color(:green)
           end
         else
-          progress.puts "[DISABLED]".color(:cyan)
+          puts_time "[DISABLED]".color(:cyan)
         end
       end
 
       task restore: :gitlab_environment do
-        progress.puts "Restoring container registry images ... ".color(:blue)
+        puts_time "Restoring container registry images ... ".color(:blue)
 
         if Gitlab.config.registry.enabled
           Backup::Registry.new(progress).restore
-          progress.puts "done".color(:green)
+          puts_time "done".color(:green)
         else
-          progress.puts "[DISABLED]".color(:cyan)
+          puts_time "[DISABLED]".color(:cyan)
         end
       end
     end
 
+    def puts_time(msg)
+      progress.puts "#{Time.now} -- #{msg}"
+    end
+
     def progress
       if ENV['CRON']
         # We need an object we can say 'puts' and 'print' to; let's use a

spec/requests/api/projects_spec.rb

@@ -49,6 +49,27 @@ describe API::Projects do
     namespace: user4.namespace)
   end
 
+  shared_context 'with language detection' do
+    let(:ruby) { create(:programming_language, name: 'Ruby') }
+    let(:javascript) { create(:programming_language, name: 'JavaScript') }
+    let(:html) { create(:programming_language, name: 'HTML') }
+
+    let(:mock_repo_languages) do
+      {
+        project => { ruby => 0.5, html => 0.5 },
+        project3 => { html => 0.7, javascript => 0.3 }
+      }
+    end
+
+    before do
+      mock_repo_languages.each do |proj, lang_shares|
+        lang_shares.each do |lang, share|
+          create(:repository_language, project: proj, programming_language: lang, share: share)
+        end
+      end
+    end
+  end
+
   describe 'GET /projects' do
     shared_examples_for 'projects response' do
       it 'returns an array of projects' do
@@ -344,6 +365,19 @@ describe API::Projects do
         end
       end
 
+      context 'and using the programming language filter' do
+        include_context 'with language detection'
+
+        it 'filters case-insensitively by programming language' do
+          get api('/projects', user), params: { with_programming_language: 'javascript' }
+
+          expect(response).to have_gitlab_http_status(200)
+          expect(response).to include_pagination_headers
+          expect(json_response).to be_an Array
+          expect(json_response.map { |p| p['id'] }).to contain_exactly(project3.id)
+        end
+      end
+
       context 'and using sorting' do
         it 'returns the correct order when sorted by id' do
           get api('/projects', user), params: { order_by: 'id', sort: 'desc' }
@@ -755,6 +789,19 @@ describe API::Projects do
       expect(json_response).to be_an Array
       expect(json_response.map { |project| project['id'] }).to contain_exactly(private_project1.id)
     end
+
+    context 'and using the programming language filter' do
+      include_context 'with language detection'
+
+      it 'filters case-insensitively by programming language' do
+        get api('/projects', user), params: { with_programming_language: 'ruby' }
+
+        expect(response).to have_gitlab_http_status(200)
+        expect(response).to include_pagination_headers
+        expect(json_response).to be_an Array
+        expect(json_response.map { |p| p['id'] }).to contain_exactly(project.id)
+      end
+    end
   end
 
   describe 'POST /projects/user/:id' do

spec/tasks/gitlab/backup_rake_spec.rb

@@ -71,21 +71,29 @@ describe 'gitlab:app namespace rake task' do
         end.to raise_error(SystemExit)
       end
 
-      it 'invokes restoration on match' do
-        allow(YAML).to receive(:load_file)
-          .and_return({ gitlab_version: gitlab_version })
-
-        expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:db:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:repo:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:builds:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:uploads:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:pages:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:lfs:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:registry:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
-        expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
+      context 'restore with matching gitlab version' do
+        before do
+          allow(YAML).to receive(:load_file)
+            .and_return({ gitlab_version: gitlab_version })
+          expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
+          expect(Rake::Task['gitlab:backup:db:restore']).to receive(:invoke)
+          expect(Rake::Task['gitlab:backup:repo:restore']).to receive(:invoke)
+          expect(Rake::Task['gitlab:backup:builds:restore']).to receive(:invoke)
+          expect(Rake::Task['gitlab:backup:uploads:restore']).to receive(:invoke)
+          expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive(:invoke)
+          expect(Rake::Task['gitlab:backup:pages:restore']).to receive(:invoke)
+          expect(Rake::Task['gitlab:backup:lfs:restore']).to receive(:invoke)
+          expect(Rake::Task['gitlab:backup:registry:restore']).to receive(:invoke)
+          expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
+        end
+
+        it 'invokes restoration on match' do
+          expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
+        end
+
+        it 'prints timestamps on messages' do
+          expect { run_rake_task('gitlab:backup:restore') }.to output(/.*\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}\s[-+]\d{4}\s--\s.*/).to_stdout
+        end
       end
     end