Commit 2c9d1abf authored by GitLab Bot's avatar GitLab Bot

Automatic merge of gitlab-org/gitlab master

parents 7e307738 72148118
......@@ -6,7 +6,7 @@ import TokenAccess from './components/token_access.vue';
Vue.use(VueApollo);
const apolloProvider = new VueApollo({
defaultClient: createDefaultClient(),
defaultClient: createDefaultClient({}, { assumeImmutableResults: true }),
});
export const initTokenAccess = (containerId = 'js-ci-token-access-app') => {
......
# frozen_string_literal: true
module Ci
module StuckBuilds
class DropRunningService
include DropHelpers
BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour
def execute
Gitlab::AppLogger.info "#{self.class}: Cleaning running, timed-out builds"
drop(running_timed_out_builds, failure_reason: :stuck_or_timeout_failure)
end
private
def running_timed_out_builds
Ci::Build.running.updated_at_before(BUILD_RUNNING_OUTDATED_TIMEOUT.ago)
end
end
end
end
......@@ -5,7 +5,6 @@ module Ci
class DropService
include DropHelpers
BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour
BUILD_PENDING_OUTDATED_TIMEOUT = 1.day
BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour
BUILD_PENDING_STUCK_TIMEOUT = 1.hour
......@@ -14,8 +13,6 @@ module Ci
def execute
Gitlab::AppLogger.info "#{self.class}: Cleaning stuck builds"
drop(running_timed_out_builds, failure_reason: :stuck_or_timeout_failure)
drop(
pending_builds(BUILD_PENDING_OUTDATED_TIMEOUT.ago),
failure_reason: :stuck_or_timeout_failure
......@@ -50,13 +47,6 @@ module Ci
BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago
)
end
def running_timed_out_builds
Ci::Build.running.where( # rubocop: disable CodeReuse/ActiveRecord
'ci_builds.updated_at < ?',
BUILD_RUNNING_OUTDATED_TIMEOUT.ago
)
end
end
end
end
......@@ -228,6 +228,15 @@
:weight: 1
:idempotent: true
:tags: []
- :name: cronjob:ci_stuck_builds_drop_running
:worker_name: Ci::StuckBuilds::DropRunningWorker
:feature_category: :continuous_integration
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
:tags: []
- :name: cronjob:container_expiration_policy
:worker_name: ContainerExpirationPolicyWorker
:feature_category: :container_registry
......
# frozen_string_literal: true
module Ci
module StuckBuilds
class DropRunningWorker
include ApplicationWorker
idempotent!
# rubocop:disable Scalability/CronWorkerContext
# This is an instance-wide cleanup query, so there's no meaningful
# scope to consider this in the context of.
include CronjobQueue
# rubocop:enable Scalability/CronWorkerContext
data_consistency :always
feature_category :continuous_integration
EXCLUSIVE_LEASE_KEY = 'ci_stuck_builds_drop_running_worker_lease'
def perform
return unless try_obtain_lease
begin
Ci::StuckBuilds::DropRunningService.new.execute
ensure
remove_lease
end
end
private
def try_obtain_lease
@uuid = Gitlab::ExclusiveLease.new(EXCLUSIVE_LEASE_KEY, timeout: 30.minutes).try_obtain
end
def remove_lease
Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid)
end
end
end
end
......@@ -17,6 +17,8 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker
EXCLUSIVE_LEASE_KEY = 'stuck_ci_builds_worker_lease'
def perform
Ci::StuckBuilds::DropRunningWorker.perform_in(20.minutes)
return unless try_obtain_lease
Ci::StuckBuilds::DropService.new.execute
......
......@@ -53,6 +53,8 @@ immediately identify which alerts you should prioritize investigating:
Alerts contain one of the following icons:
<!-- vale gitlab.SubstitutionWarning = NO -->
| Severity | Icon | Color (hexadecimal) |
|----------|-------------------------|---------------------|
| Critical | **{severity-critical}** | `#8b2615` |
......@@ -62,6 +64,8 @@ Alerts contain one of the following icons:
| Info | **{severity-info}** | `#418cd8` |
| Unknown | **{severity-unknown}** | `#bababa` |
<!-- vale gitlab.SubstitutionWarning = YES -->
## Alert details page
Navigate to the Alert details view by visiting the [Alert list](alerts.md)
......
......@@ -248,6 +248,8 @@ which shows an overview of all the security vulnerabilities in your groups, proj
Clicking the vulnerability opens a modal that provides additional information about the
vulnerability:
<!-- vale gitlab.Acronyms = NO -->
- Status: The vulnerability's status. As with any type of vulnerability, a coverage fuzzing
vulnerability can be Detected, Confirmed, Dismissed, or Resolved.
- Project: The project in which the vulnerability exists.
......@@ -261,3 +263,5 @@ vulnerability:
- Scanner: The scanner that detected the vulnerability (for example, Coverage Fuzzing).
- Scanner Provider: The engine that did the scan. For Coverage Fuzzing, this can be any of the
engines listed in [Supported fuzzing engines and languages](#supported-fuzzing-engines-and-languages).
<!-- vale gitlab.Acronyms = YES -->
......@@ -27,6 +27,8 @@ analysis are available in the [security dashboards](../security_dashboard/index.
The results are sorted by the priority of the vulnerability:
<!-- vale gitlab.SubstitutionWarning = NO -->
1. Critical
1. High
1. Medium
......@@ -34,6 +36,8 @@ The results are sorted by the priority of the vulnerability:
1. Info
1. Unknown
<!-- vale gitlab.SubstitutionWarning = YES -->
A pipeline consists of multiple jobs, including SAST and DAST scanning. If any job fails to finish
for any reason, the security dashboard does not show SAST scanner output. For example, if the SAST
job finishes but the DAST job fails, the security dashboard does not show SAST results. On failure,
......
......@@ -45,6 +45,8 @@ From the Vulnerability Report you can:
You can filter the vulnerabilities table by:
<!-- vale gitlab.SubstitutionWarning = NO -->
| Filter | Available options |
|:---------|:------------------|
| Status | Detected, Confirmed, Dismissed, Resolved. |
......@@ -53,6 +55,8 @@ You can filter the vulnerabilities table by:
| Project | For more details, see [Project filter](#project-filter). |
| Activity | For more details, see [Activity filter](#activity-filter). |
<!-- vale gitlab.SubstitutionWarning = YES -->
### Filter the list of vulnerabilities
To filter the list of vulnerabilities:
......
---
type: reference, howto
stage: Manage
group: Import
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
......@@ -10,30 +9,30 @@ info: To determine the technical writer assigned to the Stage/Group associated w
Import your projects from Gitea to GitLab with minimal effort.
NOTE:
This requires Gitea `v1.0.0` or newer.
This requires Gitea `v1.0.0` or later.
The Gitea importer can import:
- Repository description (GitLab 8.15+)
- Git repository data (GitLab 8.15+)
- Issues (GitLab 8.15+)
- Pull requests (GitLab 8.15+)
- Milestones (GitLab 8.15+)
- Labels (GitLab 8.15+)
- Repository description
- Git repository data
- Issues
- Pull requests
- Milestones
- Labels
When importing, repository public access is retained. If a repository is private in Gitea, it's
created as private in GitLab as well.
## How it works
Since Gitea is currently not an OAuth provider, author/assignee cannot be mapped
to users in your GitLab instance. This means that the project creator (most of
the times the current user that started the import process) is set as the author,
but a reference on the issue about the original Gitea author is kept.
Because Gitea isn't an OAuth provider, author/assignee can't be mapped to users
in your GitLab instance. This means the project creator (usually the user that
started the import process) is set as the author. A reference, however, is kept
on the issue about the original Gitea author.
The importer creates any new namespaces (groups) if they don't exist or in
the case the namespace is taken, the repository is imported under the user's
namespace that started the import process.
The importer creates any new namespaces (groups) if they don't exist. If the
namespace is taken, the repository is imported under the user's namespace
that started the import process.
## Import your Gitea repositories
......@@ -41,7 +40,7 @@ The importer page is visible when you create a new project.
![New project page on GitLab](img/import_projects_from_new_project_page.png)
Click the **Gitea** link and the import authorization process starts.
Select the **Gitea** link to start the import authorization process.
![New Gitea project import](img/import_projects_from_gitea_new_import.png)
......@@ -52,13 +51,13 @@ GitLab access your repositories:
1. Go to `https://your-gitea-instance/user/settings/applications` (replace
`your-gitea-instance` with the host of your Gitea instance).
1. Click **Generate New Token**.
1. Select **Generate New Token**.
1. Enter a token description.
1. Click **Generate Token**.
1. Select **Generate Token**.
1. Copy the token hash.
1. Go back to GitLab and provide the token to the Gitea importer.
1. Hit the **List Your Gitea Repositories** button and wait while GitLab reads
your repositories' information. Once done, you are taken to the importer
1. Select **List Your Gitea Repositories** and wait while GitLab reads
your repositories' information. After it's done, GitLab displays the importer
page to select the repositories to import.
### Select which repositories to import
......@@ -66,19 +65,19 @@ GitLab access your repositories:
After you've authorized access to your Gitea repositories, you are
redirected to the Gitea importer page.
From there, you can see the import statuses of your Gitea repositories.
From there, you can view the import statuses of your Gitea repositories:
- Those that are being imported show a _started_ status,
- those already successfully imported are green with a _done_ status,
- whereas those that are not yet imported have an **Import** button on the
- Those that are being imported show a _started_ status.
- Those already successfully imported are green with a _done_ status.
- Those that aren't yet imported have an **Import** button on the
right side of the table.
You also can:
- Import all your Gitea projects in one go by hitting **Import all projects** in
the upper left corner.
- Filter projects by name. If filter is applied, hitting **Import all projects**
only imports matched projects.
- Import all of your Gitea projects in one go by selecting **Import all projects**
in the upper left corner.
- Filter projects by name. If filter is applied, selecting **Import all projects**
imports only matched projects.
![Gitea importer page](img/import_projects_from_gitea_importer_v12_3.png)
......
......@@ -58,8 +58,8 @@ module QA
testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1806',
issue_1: 'https://gitlab.com/gitlab-org/gitlab/-/issues/331252',
issue_2: 'https://gitlab.com/gitlab-org/gitlab/-/issues/333678',
# mostly impacts testing as it makes small groups import slower
issue_3: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332351'
issue_3: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332351',
except: { job: 'instance-image-slow-network' }
) do
Page::Group::BulkImport.perform do |import_page|
import_page.import_group(imported_group.path, imported_group.sandbox.path)
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Ci::StuckBuilds::DropRunningService do
let!(:runner) { create :ci_runner }
let!(:job) { create :ci_build, runner: runner }
let(:created_at) { }
let(:updated_at) { }
subject(:service) { described_class.new }
before do
job_attributes = { status: status }
job_attributes[:created_at] = created_at if created_at
job_attributes[:updated_at] = updated_at if updated_at
job.update!(job_attributes)
end
context 'when job is running' do
let(:status) { 'running' }
context 'when job was updated_at more than an hour ago' do
let(:updated_at) { 2.hours.ago }
it_behaves_like 'job is dropped'
end
context 'when job was updated in less than 1 hour ago' do
let(:updated_at) { 30.minutes.ago }
it_behaves_like 'job is unchanged'
end
end
%w(success skipped failed canceled scheduled pending).each do |status|
context "when job is #{status}" do
let(:status) { status }
let(:updated_at) { 2.days.ago }
context 'when created_at is the same as updated_at' do
let(:created_at) { 2.days.ago }
it_behaves_like 'job is unchanged'
end
context 'when created_at is before updated_at' do
let(:created_at) { 3.days.ago }
it_behaves_like 'job is unchanged'
end
end
end
context 'for deleted project' do
let(:status) { 'running' }
let(:updated_at) { 2.days.ago }
before do
job.project.update!(pending_delete: true)
end
it_behaves_like 'job is dropped'
end
end
......@@ -17,48 +17,6 @@ RSpec.describe Ci::StuckBuilds::DropService do
job.update!(job_attributes)
end
shared_examples 'job is dropped' do
it 'changes status' do
expect(service).to receive(:drop).exactly(3).times.and_call_original
expect(service).to receive(:drop_stuck).exactly(:once).and_call_original
service.execute
job.reload
expect(job).to be_failed
expect(job).to be_stuck_or_timeout_failure
end
context 'when job have data integrity problem' do
it "does drop the job and logs the reason" do
job.update_columns(yaml_variables: '[{"key" => "value"}]')
expect(Gitlab::ErrorTracking).to receive(:track_exception)
.with(anything, a_hash_including(build_id: job.id))
.once
.and_call_original
service.execute
job.reload
expect(job).to be_failed
expect(job).to be_data_integrity_failure
end
end
end
shared_examples 'job is unchanged' do
it 'does not change status' do
expect(service).to receive(:drop).exactly(3).times.and_call_original
expect(service).to receive(:drop_stuck).exactly(:once).and_call_original
service.execute
job.reload
expect(job.status).to eq(status)
end
end
context 'when job is pending' do
let(:status) { 'pending' }
......@@ -195,7 +153,7 @@ RSpec.describe Ci::StuckBuilds::DropService do
context 'when job was updated_at more than an hour ago' do
let(:updated_at) { 2.hours.ago }
it_behaves_like 'job is dropped'
it_behaves_like 'job is unchanged'
end
context 'when job was updated in less than 1 hour ago' do
......@@ -238,7 +196,7 @@ RSpec.describe Ci::StuckBuilds::DropService do
job.project.update!(pending_delete: true)
end
it_behaves_like 'job is dropped'
it_behaves_like 'job is unchanged'
end
describe 'drop stale scheduled builds' do
......
# frozen_string_literal: true
RSpec.shared_examples 'job is dropped' do
it 'changes status' do
service.execute
job.reload
expect(job).to be_failed
expect(job).to be_stuck_or_timeout_failure
end
context 'when job has data integrity problem' do
it 'drops the job and logs the reason' do
job.update_columns(yaml_variables: '[{"key" => "value"}]')
expect(Gitlab::ErrorTracking)
.to receive(:track_exception)
.with(anything, a_hash_including(build_id: job.id))
.once
.and_call_original
service.execute
job.reload
expect(job).to be_failed
expect(job).to be_data_integrity_failure
end
end
end
RSpec.shared_examples 'job is unchanged' do
it 'does not change status' do
service.execute
job.reload
expect(job.status).to eq(status)
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Ci::StuckBuilds::DropRunningWorker do
include ExclusiveLeaseHelpers
let(:worker_lease_key) { Ci::StuckBuilds::DropRunningWorker::EXCLUSIVE_LEASE_KEY }
let(:worker_lease_uuid) { SecureRandom.uuid }
let(:worker2) { described_class.new }
subject(:worker) { described_class.new }
before do
stub_exclusive_lease(worker_lease_key, worker_lease_uuid)
end
describe '#perform' do
it_behaves_like 'an idempotent worker'
it 'executes an instance of Ci::StuckBuilds::DropRunningService' do
expect_next_instance_of(Ci::StuckBuilds::DropRunningService) do |service|
expect(service).to receive(:execute).exactly(:once)
end
worker.perform
end
context 'with an exclusive lease' do
it 'does not execute concurrently' do
expect(worker).to receive(:remove_lease).exactly(:once)
expect(worker2).not_to receive(:remove_lease)
worker.perform
stub_exclusive_lease_taken(worker_lease_key)
worker2.perform
end
it 'can execute in sequence' do
expect(worker).to receive(:remove_lease).at_least(:once)
expect(worker2).to receive(:remove_lease).at_least(:once)
worker.perform
worker2.perform
end
it 'cancels exclusive leases after worker perform' do
expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
worker.perform
end
context 'when the DropRunningService fails' do
it 'ensures cancellation of the exclusive lease' do
expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid)
allow_next_instance_of(Ci::StuckBuilds::DropRunningService) do |service|
allow(service).to receive(:execute) do
raise 'The query timed out'
end
end
expect { worker.perform }.to raise_error(/The query timed out/)
end
end
end
end
end
......@@ -16,7 +16,13 @@ RSpec.describe StuckCiJobsWorker do
end
describe '#perform' do
it 'executes an instance of Ci::StuckBuildsDropService' do
it 'enqueues a Ci::StuckBuilds::DropRunningWorker job' do
expect(Ci::StuckBuilds::DropRunningWorker).to receive(:perform_in).with(20.minutes).exactly(:once)
worker.perform
end
it 'executes an instance of Ci::StuckBuilds::DropService' do
expect_next_instance_of(Ci::StuckBuilds::DropService) do |service|
expect(service).to receive(:execute).exactly(:once)
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment