Merge branch '326065-add-api-fuzzing.latest.gitlab-ci.yml' into 'master'

Remove stages definition from latest API Fuzzing CI template

See merge request gitlab-org/gitlab!59154

ee/changelogs/unreleased/326065-add-api-fuzzing-latest-gitlab-ci-yml.yml

+---
+title: Remove default stages definition from latest API Fuzzing CI template
+merge_request: 59154
+author:
+type: changed

ee/spec/lib/gitlab/ci/templates/api_fuzzing_latest_gitlab_ci_yaml_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'API-Fuzzing.latest.gitlab-ci.yml' do
+  subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('API-Fuzzing.latest') }
+
+  specify { expect(template).not_to be_nil }
+
+  describe 'the template file' do
+    let(:template_filename) { Rails.root.join("lib/gitlab/ci/templates/" + template.full_name) }
+    let(:contents) { File.read(template_filename) }
+    let(:production_registry) { 'registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine' }
+    let(:staging_registry) { 'registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing-src:${FUZZAPI_VERSION}-engine' }
+
+    # Make sure future changes to the template use the production container registry.
+    #
+    # The API Fuzzing template is developed against a dev container registry.
+    # The registry is switched when releasing new versions. The difference in
+    # names between development and production is also quite small making it
+    # easy to miss during review.
+    it 'uses the production repository' do
+      expect( contents.include?(production_registry) ).to be true
+    end
+
+    it 'doesn\'t use the staging repository' do
+      expect( contents.include?(staging_registry) ).to be false
+    end
+  end
+
+  describe 'the created pipeline' do
+    let(:default_branch) { 'master' }
+    let(:pipeline_branch) { default_branch }
+    let(:project) { create(:project, :custom_repo, files: { 'README.txt' => '' }) }
+    let(:user) { project.owner }
+    let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_branch ) }
+    let(:pipeline) { service.execute!(:push) }
+    let(:build_names) { pipeline.builds.pluck(:name) }
+
+    context 'when no stages' do
+      before do
+        stub_ci_pipeline_yaml_file(template.content)
+
+        allow_next_instance_of(Ci::BuildScheduleWorker) do |worker|
+          allow(worker).to receive(:perform).and_return(true)
+        end
+
+        allow(project).to receive(:default_branch).and_return(default_branch)
+      end
+
+      context 'when project has no stages' do
+        it 'includes no jobs' do
+          expect(build_names).to be_empty
+        end
+      end
+    end
+
+    context 'when stages includes fuzz' do
+      let(:ci_pipeline_yaml) { "stages: [\"fuzz\"]\n" }
+
+      before do
+        stub_ci_pipeline_yaml_file(ci_pipeline_yaml + template.content)
+
+        allow_next_instance_of(Ci::BuildScheduleWorker) do |worker|
+          allow(worker).to receive(:perform).and_return(true)
+        end
+
+        allow(project).to receive(:default_branch).and_return(default_branch)
+      end
+
+      context 'when project has no license' do
+        before do
+          create(:ci_variable, project: project, key: 'FUZZAPI_HAR', value: 'testing.har')
+          create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
+        end
+
+        it 'includes job to display error' do
+          expect(build_names).to match_array(%w[apifuzzer_fuzz_unlicensed])
+        end
+      end
+
+      context 'when project has Ultimate license' do
+        let(:license) { create(:license, plan: License::ULTIMATE_PLAN) }
+
+        before do
+          allow(License).to receive(:current).and_return(license)
+        end
+
+        context 'by default' do
+          it 'includes a job' do
+            expect(build_names).to match_array(%w[apifuzzer_fuzz])
+          end
+        end
+
+        context 'when configured with HAR' do
+          before do
+            create(:ci_variable, project: project, key: 'FUZZAPI_HAR', value: 'testing.har')
+            create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
+          end
+
+          it 'includes job' do
+            expect(build_names).to match_array(%w[apifuzzer_fuzz])
+          end
+        end
+
+        context 'when configured with OpenAPI' do
+          before do
+            create(:ci_variable, project: project, key: 'FUZZAPI_OPENAPI', value: 'testing.json')
+            create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
+          end
+
+          it 'includes job' do
+            expect(build_names).to match_array(%w[apifuzzer_fuzz])
+          end
+        end
+
+        context 'when configured with Postman' do
+          before do
+            create(:ci_variable, project: project, key: 'FUZZAPI_POSTMAN_COLLECTION', value: 'testing.json')
+            create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
+          end
+
+          it 'includes job' do
+            expect(build_names).to match_array(%w[apifuzzer_fuzz])
+          end
+        end
+
+        context 'when API_FUZZING_DISABLED=1' do
+          before do
+            create(:ci_variable, project: project, key: 'API_FUZZING_DISABLED', value: '1')
+            create(:ci_variable, project: project, key: 'FUZZAPI_HAR', value: 'testing.har')
+            create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
+          end
+
+          it 'includes no jobs' do
+            expect { pipeline }.to raise_error(Ci::CreatePipelineService::CreateError)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml

+# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/
+
+# Configure the scanning tool through the environment variables.
+# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables
+# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
+
+variables:
+    FUZZAPI_PROFILE: Quick
+    FUZZAPI_VERSION: latest
+    FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml
+    FUZZAPI_TIMEOUT: 30
+    FUZZAPI_REPORT: gl-api-fuzzing-report.json
+    FUZZAPI_REPORT_ASSET_PATH: assets
+    #
+    FUZZAPI_D_NETWORK: testing-net
+    #
+    # Wait up to 5 minutes for API Fuzzer and target url to become
+    # available (non 500 response to HTTP(s))
+    FUZZAPI_SERVICE_START_TIMEOUT: "300"
+    #
+    FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine
+    #
+
+apifuzzer_fuzz_unlicensed:
+    stage: fuzz
+    allow_failure: true
+    rules:
+        - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null'
+        - when: never
+    script:
+        - |
+            echo "Error: Your GitLab project is not licensed for API Fuzzing."
+        - exit 1
+
+apifuzzer_fuzz:
+    stage: fuzz
+    image:
+        name: $FUZZAPI_IMAGE
+        entrypoint: ["/bin/bash", "-l", "-c"]
+    variables:
+        FUZZAPI_PROJECT: $CI_PROJECT_PATH
+        FUZZAPI_API: http://localhost:80
+        FUZZAPI_NEW_REPORT: 1
+        FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log
+        TZ: America/Los_Angeles
+    allow_failure: true
+    rules:
+        - if: $FUZZAPI_D_TARGET_IMAGE
+          when: never
+        - if: $FUZZAPI_D_WORKER_IMAGE
+          when: never
+        - if: $API_FUZZING_DISABLED
+          when: never
+        - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
+                $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+          when: never
+        - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
+    script:
+        #
+        # Validate options
+        - |
+            if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \
+                echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \
+                echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \
+                exit 1; \
+            fi
+        #
+        # Run user provided pre-script
+        - sh -c "$FUZZAPI_PRE_SCRIPT"
+        #
+        # Make sure asset path exists
+        - mkdir -p $FUZZAPI_REPORT_ASSET_PATH
+        #
+        # Start API Security background process
+        - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER &
+        - APISEC_PID=$!
+        #
+        # Start scanning
+        - worker-entry
+        #
+        # Run user provided post-script
+        - sh -c "$FUZZAPI_POST_SCRIPT"
+        #
+        # Shutdown API Security
+        - kill $APISEC_PID
+        - wait $APISEC_PID
+        #
+    artifacts:
+        when: always
+        paths:
+            - $FUZZAPI_REPORT_ASSET_PATH
+            - $FUZZAPI_REPORT
+            - $FUZZAPI_LOG_SCANNER
+        reports:
+            api_fuzzing: $FUZZAPI_REPORT
+
+apifuzzer_fuzz_dnd:
+    stage: fuzz
+    image: docker:19.03.12
+    variables:
+        DOCKER_DRIVER: overlay2
+        DOCKER_TLS_CERTDIR: ""
+        FUZZAPI_PROJECT: $CI_PROJECT_PATH
+        FUZZAPI_API: http://apifuzzer:80
+    allow_failure: true
+    rules:
+        - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null
+          when: never
+        - if: $API_FUZZING_DISABLED
+          when: never
+        - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
+                $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+          when: never
+        - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
+    services:
+        - docker:19.03.12-dind
+    script:
+        #
+        #
+        - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+        #
+        - docker network create --driver bridge $FUZZAPI_D_NETWORK
+        #
+        # Run user provided pre-script
+        - sh -c "$FUZZAPI_PRE_SCRIPT"
+        #
+        # Make sure asset path exists
+        - mkdir -p $FUZZAPI_REPORT_ASSET_PATH
+        #
+        # Start peach testing engine container
+        - |
+            docker run -d \
+            --name apifuzzer \
+            --network $FUZZAPI_D_NETWORK \
+            -e Proxy:Port=8000 \
+            -e TZ=America/Los_Angeles \
+            -e GITLAB_FEATURES \
+            -p 80:80 \
+            -p 8000:8000 \
+            -p 514:514 \
+            --restart=no \
+            $FUZZAPI_IMAGE \
+            dotnet /peach/Peach.Web.dll
+        #
+        # Start target container
+        - |
+            if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \
+                docker run -d \
+                    --name target \
+                    --network $FUZZAPI_D_NETWORK \
+                    $FUZZAPI_D_TARGET_ENV \
+                    $FUZZAPI_D_TARGET_PORTS \
+                    $FUZZAPI_D_TARGET_VOLUME \
+                    --restart=no \
+                    $FUZZAPI_D_TARGET_IMAGE \
+                ; fi
+        #
+        # Start worker container if provided
+        - |
+            if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \
+                echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \
+                docker run \
+                    --name worker \
+                    --network $FUZZAPI_D_NETWORK \
+                    -e FUZZAPI_API=http://apifuzzer:80 \
+                    -e FUZZAPI_PROJECT \
+                    -e FUZZAPI_PROFILE \
+                    -e FUZZAPI_CONFIG \
+                    -e FUZZAPI_REPORT \
+                    -e FUZZAPI_REPORT_ASSET_PATH \
+                    -e FUZZAPI_NEW_REPORT=1 \
+                    -e FUZZAPI_HAR \
+                    -e FUZZAPI_OPENAPI \
+                    -e FUZZAPI_POSTMAN_COLLECTION \
+                    -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \
+                    -e FUZZAPI_TARGET_URL \
+                    -e FUZZAPI_OVERRIDES_FILE \
+                    -e FUZZAPI_OVERRIDES_ENV \
+                    -e FUZZAPI_OVERRIDES_CMD \
+                    -e FUZZAPI_OVERRIDES_INTERVAL \
+                    -e FUZZAPI_TIMEOUT \
+                    -e FUZZAPI_VERBOSE \
+                    -e FUZZAPI_SERVICE_START_TIMEOUT \
+                    -e FUZZAPI_HTTP_USERNAME \
+                    -e FUZZAPI_HTTP_PASSWORD \
+                    -e CI_PROJECT_URL \
+                    -e CI_JOB_ID \
+                    -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \
+                    $FUZZAPI_D_WORKER_ENV \
+                    $FUZZAPI_D_WORKER_PORTS \
+                    $FUZZAPI_D_WORKER_VOLUME \
+                    --restart=no \
+                    $FUZZAPI_D_WORKER_IMAGE \
+                ; fi
+        #
+        # Start API Fuzzing provided worker if no other worker present
+        - |
+            if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \
+                if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \
+                    echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \
+                    echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \
+                    exit 1; \
+                fi; \
+                docker run \
+                    --name worker \
+                    --network $FUZZAPI_D_NETWORK \
+                    -e TZ=America/Los_Angeles \
+                    -e FUZZAPI_API=http://apifuzzer:80 \
+                    -e FUZZAPI_PROJECT \
+                    -e FUZZAPI_PROFILE \
+                    -e FUZZAPI_CONFIG \
+                    -e FUZZAPI_REPORT \
+                    -e FUZZAPI_REPORT_ASSET_PATH \
+                    -e FUZZAPI_NEW_REPORT=1 \
+                    -e FUZZAPI_HAR \
+                    -e FUZZAPI_OPENAPI \
+                    -e FUZZAPI_POSTMAN_COLLECTION \
+                    -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \
+                    -e FUZZAPI_TARGET_URL \
+                    -e FUZZAPI_OVERRIDES_FILE \
+                    -e FUZZAPI_OVERRIDES_ENV \
+                    -e FUZZAPI_OVERRIDES_CMD \
+                    -e FUZZAPI_OVERRIDES_INTERVAL \
+                    -e FUZZAPI_TIMEOUT \
+                    -e FUZZAPI_VERBOSE \
+                    -e FUZZAPI_SERVICE_START_TIMEOUT \
+                    -e FUZZAPI_HTTP_USERNAME \
+                    -e FUZZAPI_HTTP_PASSWORD \
+                    -e CI_PROJECT_URL \
+                    -e CI_JOB_ID \
+                    -v $CI_PROJECT_DIR:/app \
+                    -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \
+                    -p 81:80 \
+                    -p 8001:8000 \
+                    -p 515:514 \
+                    --restart=no \
+                    $FUZZAPI_IMAGE \
+                    worker-entry \
+                ; fi
+        #
+        # Propagate exit code from api fuzzing scanner (if any)
+        - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi
+        #
+        # Run user provided post-script
+        - sh -c "$FUZZAPI_POST_SCRIPT"
+        #
+    after_script:
+        #
+        # Shutdown all containers
+        - echo "Stopping all containers"
+        - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi
+        - docker stop worker
+        - docker stop apifuzzer
+        #
+        # Save docker logs
+        - docker logs apifuzzer &> gl-api_fuzzing-logs.log
+        - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi
+        - docker logs worker &> gl-api_fuzzing-worker-logs.log
+        #
+    artifacts:
+        when: always
+        paths:
+            - ./gl-api_fuzzing*.log
+            - ./gl-api_fuzzing*.zip
+            - $FUZZAPI_REPORT_ASSET_PATH
+            - $FUZZAPI_REPORT
+        reports:
+            api_fuzzing: $FUZZAPI_REPORT
+
+# end