Automatic merge of gitlab-org/gitlab master

.gitlab/ci/frontend.gitlab-ci.yml

@@ -107,7 +107,7 @@ update-storybook-yarn-cache:
     - run_timed_command "gem install knapsack --no-document"
     - run_timed_command "scripts/gitaly-test-spawn"
     - source ./scripts/rspec_helpers.sh
-    - rspec_paralellized_job "--tag frontend_fixture"
+    - rspec_paralellized_job
   artifacts:
     name: frontend-fixtures
     expire_in: 31d

db/migrate/20210623163342_add_index_to_compliance_management_frameworks_pipeline_configuration.rb

+# frozen_string_literal: true
+
+class AddIndexToComplianceManagementFrameworksPipelineConfiguration < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_compliance_frameworks_id_where_frameworks_not_null'
+
+  def up
+    add_concurrent_index :compliance_management_frameworks, :id, name: INDEX_NAME, where: 'pipeline_configuration_full_path IS NOT NULL'
+  end
+
+  def down
+    remove_concurrent_index_by_name :compliance_management_frameworks, INDEX_NAME
+  end
+end

db/schema_migrations/20210623163342

+6732401a959a7a691b8729e2bfb38e010dd4da4cd04418aada22946c42fdd6dc
\ No newline at end of file

db/structure.sql

@@ -23110,6 +23110,8 @@ CREATE INDEX index_clusters_on_user_id ON clusters USING btree (user_id);
 
 CREATE UNIQUE INDEX index_commit_user_mentions_on_note_id ON commit_user_mentions USING btree (note_id);
 
+CREATE INDEX index_compliance_frameworks_id_where_frameworks_not_null ON compliance_management_frameworks USING btree (id) WHERE (pipeline_configuration_full_path IS NOT NULL);
+
 CREATE INDEX index_composer_cache_files_where_namespace_id_is_null ON packages_composer_cache_files USING btree (id) WHERE (namespace_id IS NULL);
 
 CREATE INDEX index_container_expiration_policies_on_next_run_at_and_enabled ON container_expiration_policies USING btree (next_run_at, enabled);

doc/development/usage_ping/dictionary.md

@@ -16332,6 +16332,18 @@ Status: `data_available`
 
 Tiers: `free`
 
+### `usage_activity_by_stage.manage.compliance_frameworks_with_pipeline`
+
+Count of compliance frameworks that have a pipeline configuration
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_all/20210622123800_compliance_frameworks_with_pipeline.yml)
+
+Group: `compliance`
+
+Status: `implemented`
+
+Tiers: `ultimate`
+
 ### `usage_activity_by_stage.manage.custom_compliance_frameworks`
 
 Total count of all custom compliance framework labels
@@ -18398,6 +18410,18 @@ Status: `data_available`
 
 Tiers: `free`
 
+### `usage_activity_by_stage_monthly.manage.compliance_frameworks_with_pipeline`
+
+Count of compliance frameworks that have a pipeline configuration
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_28d/20210622091519_compliance_frameworks_with_pipeline.yml)
+
+Group: `compliance`
+
+Status: `implemented`
+
+Tiers: `ultimate`
+
 ### `usage_activity_by_stage_monthly.manage.custom_compliance_frameworks`
 
 Monthly count of all custom compliance framework labels

doc/user/project/releases/index.md

@@ -14,8 +14,11 @@ To introduce a checkpoint in your source code history, you can assign a
 However, in most cases, your users need more than just the raw source code.
 They need compiled objects or other assets output by your CI/CD system.
 
-A GitLab *Release* is a snapshot of the source, build output, artifacts, and other metadata
-associated with a released version of your code.
+A GitLab Release can be:
+
+- A snapshot of the source code of your repository.
+- [Generic packages](../../packages/generic_packages/index.md) created from job artifacts.
+- Other metadata associated with a released version of your code.
 
 You can create a GitLab release on any branch. When you create a release:
 
@@ -273,14 +276,28 @@ Release note descriptions are unrelated. Description supports [Markdown](../../m
 A release contains the following types of assets:
 
 - [Source code](#source-code)
-- [Permanent links to release assets](#permanent-links-to-release-assets)
+- [Link](#links)
 
 #### Source code
 
 GitLab automatically generates `zip`, `tar.gz`, `tar.bz2`, and `tar`
 archived source code from the given Git tag. These are read-only assets.
 
-#### Permanent links to release assets
+#### Links
+
+A link is any URL which can point to whatever you like: documentation, built
+binaries, or other related materials. These can be both internal or external
+links from your GitLab instance.
+Each link as an asset has the following attributes:
+
+| Attribute   | Description                            | Required |
+| ----        | -----------                            | ---      |
+| `name`      | The name of the link.                  | Yes      |
+| `url`       | The URL to download a file.            | Yes      |
+| `filepath`  | The redirect link to the `url`. See [this section](#permanent-links-to-release-assets) for more information. | No       |
+| `link_type` | The content kind of what users can download via `url`. See [this section](#link-types) for more information. | No       |
+
+##### Permanent links to release assets
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27300) in GitLab 12.9.
 
@@ -289,20 +306,6 @@ GitLab always redirects this URL to the actual asset
 location, so even if the assets move to a different location, you can continue
 to use the same URL. This is defined during [link creation](../../../api/releases/links.md#create-a-link) or [updating](../../../api/releases/links.md#update-a-link).
 
-Each asset has a `name`, a `url` of the *actual* asset location, and optionally,
-`filepath` and `link_type` parameters.
-
-A `filepath` creates a URL pointing to the asset for the Release.
-
-The `link_type` parameter accepts one of the following four values:
-
-- `runbook`
-- `package`
-- `image`
-- `other` (default)
-
-This field has no effect on the URL and it's only used for visual purposes in the Releases page of your project.
-
 The format of the URL is:
 
 ```plaintext
@@ -329,13 +332,104 @@ https://gitlab.com/gitlab-org/gitlab-runner/releases/v11.9.0-rc2/downloads/binar
 
 The physical location of the asset can change at any time and the direct link remains unchanged.
 
-### Links
+##### Link Types
 
-A link is any URL which can point to whatever you like: documentation, built
-binaries, or other related materials. These can be both internal or external
-links from your GitLab instance.
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207257) in GitLab 13.1.
 
 The four types of links are "Runbook," "Package," "Image," and "Other."
+The `link_type` parameter accepts one of the following four values:
+
+- `runbook`
+- `package`
+- `image`
+- `other` (default)
+
+This field has no effect on the URL and it's only used for visual purposes in the Releases page of your project.
+
+##### Use a generic package for attaching binaries
+
+You can use [generic packages](../../packages/generic_packages/index.md)
+to store any artifacts from a release or tag pipeline,
+that can also be used for attaching binary files to an individual release entry.
+You basically need to:
+
+1. [Push the artifacts to the Generic Package Registry](../../packages/generic_packages/index.md#publish-a-package-file).
+1. [Attach the package link to the release](#links).
+
+The following example generates release assets, publishes them
+as a generic package, and then creates a release:
+
+```yaml
+stages:
+  - build
+  - upload
+  - release
+
+variables:
+  # Package version can only contain numbers (0-9), and dots (.).
+  # Must be in the format of X.Y.Z, i.e. should match /\A\d+\.\d+\.\d+\z/ regular expresion.
+  # See https://docs.gitlab.com/ee/user/packages/generic_packages/#publish-a-package-file
+  PACKAGE_VERSION: "1.2.3"
+  DARWIN_AMD64_BINARY: "myawesomerelease-darwin-amd64-${PACKAGE_VERSION}"
+  LINUX_AMD64_BINARY: "myawesomerelease-linux-amd64-${PACKAGE_VERSION}"
+  PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/myawesomerelease/${PACKAGE_VERSION}"
+
+build:
+  stage: build
+  image: alpine:latest
+  rules:
+    - if: $CI_COMMIT_TAG
+  script:
+    - mkdir bin
+    - echo "Mock binary for ${DARWIN_AMD64_BINARY}" > bin/${DARWIN_AMD64_BINARY}
+    - echo "Mock binary for ${LINUX_AMD64_BINARY}" > bin/${LINUX_AMD64_BINARY}
+  artifacts:
+    paths:
+      - bin/
+
+upload:
+  stage: upload
+  image: curlimages/curl:latest
+  rules:
+    - if: $CI_COMMIT_TAG
+  script:
+    - |
+      curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${DARWIN_AMD64_BINARY} ${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}
+    - |
+      curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${LINUX_AMD64_BINARY} ${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}
+
+release:
+  # Caution, as of 2021-02-02 these assets links require a login, see:
+  # https://gitlab.com/gitlab-org/gitlab/-/issues/299384
+  stage: release
+  image: registry.gitlab.com/gitlab-org/release-cli:latest
+  rules:
+    - if: $CI_COMMIT_TAG
+  script:
+    - |
+      release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \
+        --assets-link "{\"name\":\"${DARWIN_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}\"}" \
+        --assets-link "{\"name\":\"${LINUX_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}\"}"
+```
+
+PowerShell users may need to escape the double quote `"` inside a JSON
+string with a `` ` `` (back tick) for `--assets-link` and `ConvertTo-Json`
+before passing on to the `release-cli`.
+For example:
+
+```yaml
+release:
+  script:
+    - $env:asset = "{`"name`":`"MyFooAsset`",`"url`":`"https://gitlab.com/upack/artifacts/download/$env:UPACK_GROUP/$env:UPACK_NAME/$($env:GitVersion_SemVer)?contentOnly=zip`"}"
+    - $env:assetjson = $env:asset | ConvertTo-Json
+    - release-cli create --name $CI_COMMIT_TAG --description "Release $CI_COMMIT_TAG" --ref $CI_COMMIT_TAG --tag-name $CI_COMMIT_TAG --assets-link=$env:assetjson
+```
+
+NOTE:
+Directly attaching [job artifacts](../../../ci/pipelines/job_artifacts.md)
+links to a release is not recommended, because artifacts are ephemeral and
+are used to pass data in the same pipeline. This means there's a risk that
+they could either expire or someone might manually delete them.
 
 ## Release evidence
 

ee/app/assets/javascripts/security_configuration/api_fuzzing/graphql/provider.js

@@ -4,4 +4,11 @@ import createDefaultClient from '~/lib/graphql';
 
 Vue.use(VueApollo);
 
-export const apolloProvider = new VueApollo({ defaultClient: createDefaultClient() });
+export const apolloProvider = new VueApollo({
+  defaultClient: createDefaultClient(
+    {},
+    {
+      assumeImmutableResults: true,
+    },
+  ),
+});

ee/app/assets/javascripts/security_configuration/corpus_management/corpus_management_bundle.js

@@ -15,6 +15,7 @@ export default () => {
   }
 
   const defaultClient = createDefaultClient(resolvers, {
+    assumeImmutableResults: true,
     cacheConfig: {
       dataIdFromObject: (object) => {
         return object.id || defaultDataIdFromObject(object);

ee/config/metrics/counts_28d/20210622091519_compliance_frameworks_with_pipeline.yml

+---
+key_path: usage_activity_by_stage_monthly.manage.compliance_frameworks_with_pipeline
+name: "count_distinct_montlpipeline_containing_compliance_management_frameworks"
+description: "Count of compliance frameworks that have a pipeline configuration"
+product_section: dev
+product_stage: manage
+product_group: compliance
+product_category: "Compliance management"
+value_type: number
+status: implemented
+milestone: "14.1"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/329897
+time_frame: 28d
+data_source: database
+data_category: Optional
+distribution:
+- ee
+tier:
+- ultimate

ee/config/metrics/counts_all/20210622123800_compliance_frameworks_with_pipeline.yml

+---
+key_path: usage_activity_by_stage.manage.compliance_frameworks_with_pipeline
+name: "count_distinct_pipeline_containing_compliance_management_frameworks"
+description: "Count of compliance frameworks that have a pipeline configuration"
+product_section: dev
+product_stage: manage
+product_group: compliance
+product_category: "Compliance management"
+value_type: number
+status: implemented
+milestone: "14.1"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64601
+time_frame: all
+data_source: database
+data_category: Optional
+distribution:
+  - ee
+tier:
+  - ultimate

ee/lib/ee/gitlab/usage_data.rb

@@ -308,6 +308,7 @@ module EE
             value_stream_management_customized_group_stages: count(::Analytics::CycleAnalytics::GroupStage.where(custom: true)),
             projects_with_compliance_framework: count(::ComplianceManagement::ComplianceFramework::ProjectSettings),
             custom_compliance_frameworks: count(::ComplianceManagement::Framework),
+            compliance_frameworks_with_pipeline: count(::ComplianceManagement::Framework.where.not(pipeline_configuration_full_path: nil)),
             ldap_servers: ldap_available_servers.size,
             ldap_group_sync_enabled: ldap_config_present_for_any_provider?(:group_base),
             ldap_admin_sync_enabled: ldap_config_present_for_any_provider?(:admin_group),

ee/spec/factories/compliance_management/frameworks.rb

@@ -12,5 +12,9 @@ FactoryBot.define do
     trait :sox do
       name { 'SOX' }
     end
+
+    trait :with_pipeline do
+      pipeline_configuration_full_path { 'compliance.gitlab-ci.yml@test-project' }
+    end
   end
 end

ee/spec/lib/ee/gitlab/usage_data_spec.rb

@@ -415,6 +415,7 @@ RSpec.describe Gitlab::UsageData do
         create(:cycle_analytics_group_stage)
         create(:compliance_framework_project_setting)
         create(:compliance_framework)
+        create(:compliance_framework, :with_pipeline)
       end
 
       expect(described_class.usage_activity_by_stage_manage({})).to include(
@@ -422,7 +423,8 @@ RSpec.describe Gitlab::UsageData do
         ldap_users: 2,
         value_stream_management_customized_group_stages: 2,
         projects_with_compliance_framework: 2,
-        custom_compliance_frameworks: 4,
+        custom_compliance_frameworks: 6,
+        compliance_frameworks_with_pipeline: 2,
         ldap_servers: 2,
         ldap_group_sync_enabled: true,
         ldap_admin_sync_enabled: true,
@@ -433,7 +435,8 @@ RSpec.describe Gitlab::UsageData do
         ldap_users: 1,
         value_stream_management_customized_group_stages: 2,
         projects_with_compliance_framework: 2,
-        custom_compliance_frameworks: 4,
+        custom_compliance_frameworks: 6,
+        compliance_frameworks_with_pipeline: 2,
         ldap_servers: 2,
         ldap_group_sync_enabled: true,
         ldap_admin_sync_enabled: true,

spec/fast_spec_helper.rb

@@ -24,6 +24,8 @@ ActiveSupport::Dependencies.autoload_paths << 'ee/lib'
 ActiveSupport::XmlMini.backend = 'Nokogiri'
 
 RSpec.configure do |config|
-  config.filter_run focus: true
-  config.run_all_when_everything_filtered = true
+  unless ENV['CI']
+    config.filter_run focus: true
+    config.run_all_when_everything_filtered = true
+  end
 end

spec/spec_helper.rb

@@ -76,9 +76,6 @@ require_relative '../tooling/quality/test_level'
 quality_level = Quality::TestLevel.new
 
 RSpec.configure do |config|
-  config.filter_run focus: true
-  config.run_all_when_everything_filtered = true
-
   config.use_transactional_fixtures = true
   config.use_instantiated_fixtures = false
   config.fixture_path = Rails.root
@@ -113,6 +110,8 @@ RSpec.configure do |config|
   end
 
   unless ENV['CI']
+    config.filter_run focus: true
+    config.run_all_when_everything_filtered = true
     # Re-run failures locally with `--only-failures`
     config.example_status_persistence_file_path = './spec/examples.txt'
   end