Automatic merge of gitlab-org/gitlab master

app/assets/javascripts/token_access/index.js

@@ -6,7 +6,7 @@ import TokenAccess from './components/token_access.vue';
 Vue.use(VueApollo);
 
 const apolloProvider = new VueApollo({
-  defaultClient: createDefaultClient(),
+  defaultClient: createDefaultClient({}, { assumeImmutableResults: true }),
 });
 
 export const initTokenAccess = (containerId = 'js-ci-token-access-app') => {

app/services/ci/stuck_builds/drop_running_service.rb

+# frozen_string_literal: true
+
+module Ci
+  module StuckBuilds
+    class DropRunningService
+      include DropHelpers
+
+      BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour
+
+      def execute
+        Gitlab::AppLogger.info "#{self.class}: Cleaning running, timed-out builds"
+
+        drop(running_timed_out_builds, failure_reason: :stuck_or_timeout_failure)
+      end
+
+      private
+
+      def running_timed_out_builds
+        Ci::Build.running.updated_at_before(BUILD_RUNNING_OUTDATED_TIMEOUT.ago)
+      end
+    end
+  end
+end

app/services/ci/stuck_builds/drop_service.rb

@@ -5,7 +5,6 @@ module Ci
     class DropService
       include DropHelpers
 
-      BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour
       BUILD_PENDING_OUTDATED_TIMEOUT = 1.day
       BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour
       BUILD_PENDING_STUCK_TIMEOUT = 1.hour
@@ -14,8 +13,6 @@ module Ci
       def execute
         Gitlab::AppLogger.info "#{self.class}: Cleaning stuck builds"
 
-        drop(running_timed_out_builds, failure_reason: :stuck_or_timeout_failure)
-
         drop(
           pending_builds(BUILD_PENDING_OUTDATED_TIMEOUT.ago),
           failure_reason: :stuck_or_timeout_failure
@@ -50,13 +47,6 @@ module Ci
           BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago
         )
       end
-
-      def running_timed_out_builds
-        Ci::Build.running.where( # rubocop: disable CodeReuse/ActiveRecord
-          'ci_builds.updated_at < ?',
-          BUILD_RUNNING_OUTDATED_TIMEOUT.ago
-        )
-      end
     end
   end
 end

app/workers/all_queues.yml

@@ -228,6 +228,15 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: cronjob:ci_stuck_builds_drop_running
+  :worker_name: Ci::StuckBuilds::DropRunningWorker
+  :feature_category: :continuous_integration
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: cronjob:container_expiration_policy
   :worker_name: ContainerExpirationPolicyWorker
   :feature_category: :container_registry

app/workers/ci/stuck_builds/drop_running_worker.rb

+# frozen_string_literal: true
+
+module Ci
+  module StuckBuilds
+    class DropRunningWorker
+      include ApplicationWorker
+
+      idempotent!
+
+      # rubocop:disable Scalability/CronWorkerContext
+      # This is an instance-wide cleanup query, so there's no meaningful
+      # scope to consider this in the context of.
+      include CronjobQueue
+      # rubocop:enable Scalability/CronWorkerContext
+
+      data_consistency :always
+
+      feature_category :continuous_integration
+
+      EXCLUSIVE_LEASE_KEY = 'ci_stuck_builds_drop_running_worker_lease'
+
+      def perform
+        return unless try_obtain_lease
+
+        begin
+          Ci::StuckBuilds::DropRunningService.new.execute
+        ensure
+          remove_lease
+        end
+      end
+
+      private
+
+      def try_obtain_lease
+        @uuid = Gitlab::ExclusiveLease.new(EXCLUSIVE_LEASE_KEY, timeout: 30.minutes).try_obtain
+      end
+
+      def remove_lease
+        Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid)
+      end
+    end
+  end
+end

app/workers/stuck_ci_jobs_worker.rb

@@ -17,6 +17,8 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker
   EXCLUSIVE_LEASE_KEY = 'stuck_ci_builds_worker_lease'
 
   def perform
+    Ci::StuckBuilds::DropRunningWorker.perform_in(20.minutes)
+
     return unless try_obtain_lease
 
     Ci::StuckBuilds::DropService.new.execute

doc/operations/incident_management/alerts.md

@@ -53,6 +53,8 @@ immediately identify which alerts you should prioritize investigating:
 
 Alerts contain one of the following icons:
 
+<!-- vale gitlab.SubstitutionWarning = NO -->
+
 | Severity | Icon                    | Color (hexadecimal) |
 |----------|-------------------------|---------------------|
 | Critical | **{severity-critical}** | `#8b2615`           |
@@ -62,6 +64,8 @@ Alerts contain one of the following icons:
 | Info     | **{severity-info}**     | `#418cd8`           |
 | Unknown  | **{severity-unknown}**  | `#bababa`           |
 
+<!-- vale gitlab.SubstitutionWarning = YES -->
+
 ## Alert details page
 
 Navigate to the Alert details view by visiting the [Alert list](alerts.md)

doc/user/application_security/coverage_fuzzing/index.md

@@ -10,7 +10,7 @@ type: reference, howto
 Coverage-guided fuzzing sends random inputs to an instrumented version of your application in an
 effort to cause unexpected behavior. Such behavior indicates a bug that you should address.
 GitLab allows you to add coverage-guided fuzz testing to your pipelines. This helps you discover
-bugs and potential security issues that other QA processes may miss. 
+bugs and potential security issues that other QA processes may miss.
 
 We recommend that you use fuzz testing in addition to the other security scanners in [GitLab Secure](../index.md)
 and your own test processes. If you're using [GitLab CI/CD](../../../ci/index.md),
@@ -248,6 +248,8 @@ which shows an overview of all the security vulnerabilities in your groups, proj
 Clicking the vulnerability opens a modal that provides additional information about the
 vulnerability:
 
+<!-- vale gitlab.Acronyms = NO -->
+
 - Status: The vulnerability's status. As with any type of vulnerability, a coverage fuzzing
   vulnerability can be Detected, Confirmed, Dismissed, or Resolved.
 - Project: The project in which the vulnerability exists.
@@ -261,3 +263,5 @@ vulnerability:
 - Scanner: The scanner that detected the vulnerability (for example, Coverage Fuzzing).
 - Scanner Provider: The engine that did the scan. For Coverage Fuzzing, this can be any of the
   engines listed in [Supported fuzzing engines and languages](#supported-fuzzing-engines-and-languages).
+
+<!-- vale gitlab.Acronyms = YES -->

doc/user/application_security/sast/index.md

@@ -27,6 +27,8 @@ analysis are available in the [security dashboards](../security_dashboard/index.
 
 The results are sorted by the priority of the vulnerability:
 
+<!-- vale gitlab.SubstitutionWarning = NO -->
+
 1. Critical
 1. High
 1. Medium
@@ -34,6 +36,8 @@ The results are sorted by the priority of the vulnerability:
 1. Info
 1. Unknown
 
+<!-- vale gitlab.SubstitutionWarning = YES -->
+
 A pipeline consists of multiple jobs, including SAST and DAST scanning. If any job fails to finish
 for any reason, the security dashboard does not show SAST scanner output. For example, if the SAST
 job finishes but the DAST job fails, the security dashboard does not show SAST results. On failure,

doc/user/application_security/vulnerability_report/index.md

@@ -45,6 +45,8 @@ From the Vulnerability Report you can:
 
 You can filter the vulnerabilities table by:
 
+<!-- vale gitlab.SubstitutionWarning = NO -->
+
 | Filter   | Available options |
 |:---------|:------------------|
 | Status   | Detected, Confirmed, Dismissed, Resolved. |
@@ -53,6 +55,8 @@ You can filter the vulnerabilities table by:
 | Project  | For more details, see [Project filter](#project-filter). |
 | Activity | For more details, see [Activity filter](#activity-filter). |
 
+<!-- vale gitlab.SubstitutionWarning = YES -->
+
 ### Filter the list of vulnerabilities
 
 To filter the list of vulnerabilities:

doc/user/project/import/gitea.md

 ---
-type: reference, howto
 stage: Manage
 group: Import
 info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
@@ -10,30 +9,30 @@ info: To determine the technical writer assigned to the Stage/Group associated w
 Import your projects from Gitea to GitLab with minimal effort.
 
 NOTE:
-This requires Gitea `v1.0.0` or newer.
+This requires Gitea `v1.0.0` or later.
 
 The Gitea importer can import:
 
-- Repository description (GitLab 8.15+)
-- Git repository data (GitLab 8.15+)
-- Issues (GitLab 8.15+)
-- Pull requests (GitLab 8.15+)
-- Milestones (GitLab 8.15+)
-- Labels (GitLab 8.15+)
+- Repository description
+- Git repository data
+- Issues
+- Pull requests
+- Milestones
+- Labels
 
 When importing, repository public access is retained. If a repository is private in Gitea, it's
 created as private in GitLab as well.
 
 ## How it works
 
-Since Gitea is currently not an OAuth provider, author/assignee cannot be mapped
-to users in your GitLab instance. This means that the project creator (most of
-the times the current user that started the import process) is set as the author,
-but a reference on the issue about the original Gitea author is kept.
+Because Gitea isn't an OAuth provider, author/assignee can't be mapped to users
+in your GitLab instance. This means the project creator (usually the user that
+started the import process) is set as the author. A reference, however, is kept
+on the issue about the original Gitea author.
 
-The importer creates any new namespaces (groups) if they don't exist or in
-the case the namespace is taken, the repository is imported under the user's
-namespace that started the import process.
+The importer creates any new namespaces (groups) if they don't exist. If the
+namespace is taken, the repository is imported under the user's namespace
+that started the import process.
 
 ## Import your Gitea repositories
 
@@ -41,7 +40,7 @@ The importer page is visible when you create a new project.
 
 ![New project page on GitLab](img/import_projects_from_new_project_page.png)
 
-Click the **Gitea** link and the import authorization process starts.
+Select the **Gitea** link to start the import authorization process.
 
 ![New Gitea project import](img/import_projects_from_gitea_new_import.png)
 
@@ -52,13 +51,13 @@ GitLab access your repositories:
 
 1. Go to `https://your-gitea-instance/user/settings/applications` (replace
    `your-gitea-instance` with the host of your Gitea instance).
-1. Click **Generate New Token**.
+1. Select **Generate New Token**.
 1. Enter a token description.
-1. Click **Generate Token**.
+1. Select **Generate Token**.
 1. Copy the token hash.
 1. Go back to GitLab and provide the token to the Gitea importer.
-1. Hit the **List Your Gitea Repositories** button and wait while GitLab reads
-   your repositories' information. Once done, you are taken to the importer
+1. Select **List Your Gitea Repositories** and wait while GitLab reads
+   your repositories' information. After it's done, GitLab displays the importer
    page to select the repositories to import.
 
 ### Select which repositories to import
@@ -66,19 +65,19 @@ GitLab access your repositories:
 After you've authorized access to your Gitea repositories, you are
 redirected to the Gitea importer page.
 
-From there, you can see the import statuses of your Gitea repositories.
+From there, you can view the import statuses of your Gitea repositories:
 
-- Those that are being imported show a _started_ status,
-- those already successfully imported are green with a _done_ status,
-- whereas those that are not yet imported have an **Import** button on the
+- Those that are being imported show a _started_ status.
+- Those already successfully imported are green with a _done_ status.
+- Those that aren't yet imported have an **Import** button on the
   right side of the table.
 
 You also can:
 
-- Import all your Gitea projects in one go by hitting **Import all projects** in
-  the upper left corner.
-- Filter projects by name. If filter is applied, hitting **Import all projects**
-  only imports matched projects.
+- Import all of your Gitea projects in one go by selecting **Import all projects**
+  in the upper left corner.
+- Filter projects by name. If filter is applied, selecting **Import all projects**
+  imports only matched projects.
 
 ![Gitea importer page](img/import_projects_from_gitea_importer_v12_3.png)
 

qa/qa/specs/features/browser_ui/1_manage/group/bulk_import_group_spec.rb

@@ -58,8 +58,8 @@ module QA
         testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1806',
         issue_1: 'https://gitlab.com/gitlab-org/gitlab/-/issues/331252',
         issue_2: 'https://gitlab.com/gitlab-org/gitlab/-/issues/333678',
-        # mostly impacts testing as it makes small groups import slower
-        issue_3: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332351'
+        issue_3: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332351',
+        except: { job: 'instance-image-slow-network' }
       ) do
         Page::Group::BulkImport.perform do |import_page|
           import_page.import_group(imported_group.path, imported_group.sandbox.path)

spec/services/ci/stuck_builds/drop_running_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::StuckBuilds::DropRunningService do
+  let!(:runner) { create :ci_runner }
+  let!(:job) { create :ci_build, runner: runner }
+  let(:created_at) { }
+  let(:updated_at) { }
+
+  subject(:service) { described_class.new }
+
+  before do
+    job_attributes = { status: status }
+    job_attributes[:created_at] = created_at if created_at
+    job_attributes[:updated_at] = updated_at if updated_at
+    job.update!(job_attributes)
+  end
+
+  context 'when job is running' do
+    let(:status) { 'running' }
+
+    context 'when job was updated_at more than an hour ago' do
+      let(:updated_at) { 2.hours.ago }
+
+      it_behaves_like 'job is dropped'
+    end
+
+    context 'when job was updated in less than 1 hour ago' do
+      let(:updated_at) { 30.minutes.ago }
+
+      it_behaves_like 'job is unchanged'
+    end
+  end
+
+  %w(success skipped failed canceled scheduled pending).each do |status|
+    context "when job is #{status}" do
+      let(:status) { status }
+      let(:updated_at) { 2.days.ago }
+
+      context 'when created_at is the same as updated_at' do
+        let(:created_at) { 2.days.ago }
+
+        it_behaves_like 'job is unchanged'
+      end
+
+      context 'when created_at is before updated_at' do
+        let(:created_at) { 3.days.ago }
+
+        it_behaves_like 'job is unchanged'
+      end
+    end
+  end
+
+  context 'for deleted project' do
+    let(:status) { 'running' }
+    let(:updated_at) { 2.days.ago }
+
+    before do
+      job.project.update!(pending_delete: true)
+    end
+
+    it_behaves_like 'job is dropped'
+  end
+end

spec/services/ci/stuck_builds/drop_service_spec.rb

@@ -17,48 +17,6 @@ RSpec.describe Ci::StuckBuilds::DropService do
     job.update!(job_attributes)
   end
 
-  shared_examples 'job is dropped' do
-    it 'changes status' do
-      expect(service).to receive(:drop).exactly(3).times.and_call_original
-      expect(service).to receive(:drop_stuck).exactly(:once).and_call_original
-
-      service.execute
-      job.reload
-
-      expect(job).to be_failed
-      expect(job).to be_stuck_or_timeout_failure
-    end
-
-    context 'when job have data integrity problem' do
-      it "does drop the job and logs the reason" do
-        job.update_columns(yaml_variables: '[{"key" => "value"}]')
-
-        expect(Gitlab::ErrorTracking).to receive(:track_exception)
-                                          .with(anything, a_hash_including(build_id: job.id))
-                                          .once
-                                          .and_call_original
-
-        service.execute
-        job.reload
-
-        expect(job).to be_failed
-        expect(job).to be_data_integrity_failure
-      end
-    end
-  end
-
-  shared_examples 'job is unchanged' do
-    it 'does not change status' do
-      expect(service).to receive(:drop).exactly(3).times.and_call_original
-      expect(service).to receive(:drop_stuck).exactly(:once).and_call_original
-
-      service.execute
-      job.reload
-
-      expect(job.status).to eq(status)
-    end
-  end
-
   context 'when job is pending' do
     let(:status) { 'pending' }
 
@@ -195,7 +153,7 @@ RSpec.describe Ci::StuckBuilds::DropService do
     context 'when job was updated_at more than an hour ago' do
       let(:updated_at) { 2.hours.ago }
 
-      it_behaves_like 'job is dropped'
+      it_behaves_like 'job is unchanged'
     end
 
     context 'when job was updated in less than 1 hour ago' do
@@ -238,7 +196,7 @@ RSpec.describe Ci::StuckBuilds::DropService do
       job.project.update!(pending_delete: true)
     end
 
-    it_behaves_like 'job is dropped'
+    it_behaves_like 'job is unchanged'
   end
 
   describe 'drop stale scheduled builds' do

spec/support/shared_examples/ci/stuck_builds_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'job is dropped' do
+  it 'changes status' do
+    service.execute
+    job.reload
+
+    expect(job).to be_failed
+    expect(job).to be_stuck_or_timeout_failure
+  end
+
+  context 'when job has data integrity problem' do
+    it 'drops the job and logs the reason' do
+      job.update_columns(yaml_variables: '[{"key" => "value"}]')
+
+      expect(Gitlab::ErrorTracking)
+        .to receive(:track_exception)
+        .with(anything, a_hash_including(build_id: job.id))
+        .once
+        .and_call_original
+
+      service.execute
+      job.reload
+
+      expect(job).to be_failed
+      expect(job).to be_data_integrity_failure
+    end
+  end
+end
+
+RSpec.shared_examples 'job is unchanged' do
+  it 'does not change status' do
+    service.execute
+    job.reload
+
+    expect(job.status).to eq(status)
+  end
+end

spec/workers/ci/stuck_builds/drop_running_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::StuckBuilds::DropRunningWorker do
+  include ExclusiveLeaseHelpers
+
+  let(:worker_lease_key)  { Ci::StuckBuilds::DropRunningWorker::EXCLUSIVE_LEASE_KEY }
+  let(:worker_lease_uuid) { SecureRandom.uuid }
+  let(:worker2)           { described_class.new }
+
+  subject(:worker) { described_class.new }
+
+  before do
+    stub_exclusive_lease(worker_lease_key, worker_lease_uuid)
+  end
+
+  describe '#perform' do
+    it_behaves_like 'an idempotent worker'
+
+    it 'executes an instance of Ci::StuckBuilds::DropRunningService' do
+      expect_next_instance_of(Ci::StuckBuilds::DropRunningService) do |service|
+        expect(service).to receive(:execute).exactly(:once)
+      end
+
+      worker.perform
+    end
+
+    context 'with an exclusive lease' do
+      it 'does not execute concurrently' do
+        expect(worker).to receive(:remove_lease).exactly(:once)
+        expect(worker2).not_to receive(:remove_lease)
+
+        worker.perform
+
+        stub_exclusive_lease_taken(worker_lease_key)
+
+        worker2.perform
+      end
+
+      it 'can execute in sequence' do
+        expect(worker).to receive(:remove_lease).at_least(:once)
+        expect(worker2).to receive(:remove_lease).at_least(:once)
+
+        worker.perform
+        worker2.perform
+      end
+
+      it 'cancels exclusive leases after worker perform' do
+        expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
+
+        worker.perform
+      end
+
+      context 'when the DropRunningService fails' do
+        it 'ensures cancellation of the exclusive lease' do
+          expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
+
+          allow_next_instance_of(Ci::StuckBuilds::DropRunningService) do |service|
+            allow(service).to receive(:execute) do
+              raise 'The query timed out'
+            end
+          end
+
+          expect { worker.perform }.to raise_error(/The query timed out/)
+        end
+      end
+    end
+  end
+end

spec/workers/stuck_ci_jobs_worker_spec.rb

@@ -16,7 +16,13 @@ RSpec.describe StuckCiJobsWorker do
   end
 
   describe '#perform' do
-    it 'executes an instance of Ci::StuckBuildsDropService' do
+    it 'enqueues a Ci::StuckBuilds::DropRunningWorker job' do
+      expect(Ci::StuckBuilds::DropRunningWorker).to receive(:perform_in).with(20.minutes).exactly(:once)
+
+      worker.perform
+    end
+
+    it 'executes an instance of Ci::StuckBuilds::DropService' do
       expect_next_instance_of(Ci::StuckBuilds::DropService) do |service|
         expect(service).to receive(:execute).exactly(:once)
       end