Merge branch 'enable-dast-fullscan-nightly' into 'master'

Enable dast nightly fullscan

See merge request gitlab-org/gitlab!38253

.gitlab-ci.yml

@@ -7,6 +7,7 @@ stages:
   - post-test
   - review-prepare
   - review
+  - dast
   - qa
   - post-qa
   - pages
@@ -104,3 +105,4 @@ include:
   - local: .gitlab/ci/yaml.gitlab-ci.yml
   - local: .gitlab/ci/releases.gitlab-ci.yml
   - local: .gitlab/ci/notify.gitlab-ci.yml
+  - local: .gitlab/ci/dast.gitlab-ci.yml

.gitlab/ci/dast.gitlab-ci.yml

+.dast_conf:
+  tags:
+    - prm
+  # For scheduling dast job
+  extends:
+    - .reports:schedule-dast
+  image:
+    name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION"
+  resource_group: dast_scan
+  variables:
+    DAST_USERNAME_FIELD: "user[login]"
+    DAST_PASSWORD_FIELD: "user[password]"
+    DAST_FULL_SCAN_ENABLED: "true"
+    DAST_SPIDER_MINS: 0
+    # TBD pin to a version
+    DAST_VERSION: 1.22.1
+    # -Xmx is used to set the JVM memory to 6GB to prevent DAST OutOfMemoryError.
+    DAST_ZAP_CLI_OPTIONS: "-Xmx6144m"
+    DAST_RULES: "41,42,43,10027,10032,10041,10042,10045,10047,10052,10053,10057,10061,10096,10097,10104,10106,20012,20014,20015,20016,20017,20018,40019,40020,40021,40024,40025,40027,40029,40032,90001,90019,10109,10026,10028,10029,10030,10031,10033,10034,10035,10036,10038,10039,10043,10044,10048,10050,10051,10058,10062,10095,10107,10108,30003,40013,40022,40023,40028,90021,90023,90024,90025,90027,90028,10003,50003,0,2,3,6,7,10010,10011,10015,10017,10019,10020,10021,10023,10024,10025,10037,10040,10054,10055,10056,10098,10105,10202,20019,30001,30002,40003,40008,40009,40012,40014,40016,40017,40018,50000,50001,90011,90020,90022,90033"
+  before_script:
+    - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"'
+    - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"'
+    - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"'
+    # Below three lines can be removed once https://gitlab.com/gitlab-org/gitlab/-/issues/230687 is fixed
+    - mkdir -p /zap/xml
+    - 'sed -i "84 s/true/false/" /zap/xml/config.xml'
+    - cat /zap/xml/config.xml
+    # Help pages are excluded from scan as they are static pages.
+    # profile/two_factor_auth is excluded from scan to prevent 2FA from being turned on from user profile, which will reduce coverage.
+    - 'export DAST_AUTH_EXCLUDE_URLS="${DAST_WEBSITE}/help/.*,${DAST_WEBSITE}/profile/two_factor_auth,${DAST_WEBSITE}/users/sign_out"'
+    - enable_rule () { read all_rules; rule=$1; echo $all_rules | sed -r "s/(,)?$rule(,)?/\1-1\2/" ; }
+    # Sort ids in DAST_RULES ascendingly, which is required when using DAST_RULES as argument to enable_rule
+    - 'DAST_RULES=$(echo $DAST_RULES | tr "," "\n" | sort -n | paste -sd ",")'
+  needs: ["review-deploy"]
+  stage: dast
+  # Default job timeout set to 90m and dast rules needs 2h to so that it won't timeout.
+  timeout: 2h
+  artifacts:
+    paths:
+      - gl-dast-report.json  # GitLab-specific
+    reports:
+      dast: gl-dast-report.json
+    expire_in: 1 week  # GitLab-specific
+
+# DAST scan with a subset of Release scan rules.
+DAST-fullscan-ruleset1:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user1"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10019 | enable_rule 10020 | enable_rule 10021 | enable_rule 10023 | enable_rule 10024 | enable_rule 10025 | enable_rule 10037 | enable_rule 10040 | enable_rule 10054 | enable_rule 10055 | enable_rule 10056)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with a subset of Release scan rules.
+DAST-fullscan-ruleset2:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user2"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 90011 | enable_rule 90020 | enable_rule 90022 | enable_rule 90033)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with a subset of Release scan rules.
+DAST-fullscan-ruleset3:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user3"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40016 | enable_rule 40017 | enable_rule 50000 | enable_rule 50001)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with a subset of Release scan rules.
+DAST-fullscan-ruleset4:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user4"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 0 | enable_rule 2 | enable_rule 3 | enable_rule 7 )
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with a subset of Release scan rules.
+DAST-fullscan-ruleset5:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user5"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10010 | enable_rule 10011 | enable_rule 10015 | enable_rule 10017 | enable_rule 10019)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with a subset of Release scan rules.
+DAST-fullscan-ruleset6:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user6"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 30001 | enable_rule 40009)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed
+# DAST scan with a subset of Beta scan rules.
+# DAST-fullscan-ruleset7:
+#   extends:
+#     - .dast_conf
+#   variables:
+#     DAST_USERNAME: "user7"
+#   script:
+#     - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10098 | enable_rule 10105 | enable_rule 10202 | enable_rule 30002 | enable_rule 40003 | enable_rule 40008 | enable_rule 40009)
+#     - echo $DAST_EXCLUDE_RULES
+#     - /analyze -t $DAST_WEBSITE -d
+
+# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed
+# Below jobs runs DAST scans with one time consuming scan rule. These scan rules are disabled in above jobs so that those jobs won't timeout.
+# DAST scan with rule - 20019 External Redirect
+# DAST-fullscan-rule-20019:
+#   extends:
+#     - .dast_conf
+#   variables:
+#     DAST_USERNAME: "user8"
+#   script:
+#     - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 20019)
+#     - echo $DAST_EXCLUDE_RULES
+#     - /analyze -t $DAST_WEBSITE -d
+
+# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed
+# DAST scan with rule -  10107 Httpoxy - Proxy Header Misuse - Active/beta
+# DAST-fullscan-rule-10107:
+#   extends:
+#     - .dast_conf
+#   variables:
+#     DAST_USERNAME: "user9"
+#   script:
+#     - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10107)
+#     - echo $DAST_EXCLUDE_RULES
+#     - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with rule - 90020 Remote OS Command Injection
+DAST-fullscan-rule-90020:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user10"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 90020)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with rule - 40018 SQL Injection - Active/release
+DAST-fullscan-rule-40018:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user11"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40018)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with rule - 40014 Cross Site Scripting (Persistent) - Active/release
+DAST-fullscan-rule-40014:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user12"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40014)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with rule - 6 Path travesal
+DAST-fullscan-rule-6:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user13"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 6)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with rule - 40012 Cross Site Scripting (Reflected)
+DAST-fullscan-rule-40012:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user14"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40012)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d

.gitlab/ci/reports.gitlab-ci.yml

@@ -145,45 +145,3 @@ dependency_scanning:
     reports:
       dependency_scanning: gl-dependency-scanning-report.json
     expire_in: 1 week  # GitLab-specific
-
-# Temporarily disabling review apps
-## We need to duplicate this job's definition because it seems it's impossible to
-## override an included `only.refs`.
-## See https://gitlab.com/gitlab-org/gitlab/issues/31371.
-# dast:
-#   extends:
-#     - .default-retry
-#     - .reports:rules:dast
-#   # This is needed so that manual jobs with needs don't block the pipeline.
-#   # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979.
-#   dependencies: ["review-deploy"]
-#   stage: qa  # GitLab-specific
-#   image:
-#     name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION"
-#   variables:
-#     # To be done in a later iteration
-#     # DAST_USERNAME: "root"
-#     # DAST_USERNAME_FIELD: "user[login]"
-#     # DAST_PASSWORD_FIELD: "user[passowrd]"
-#     DAST_VERSION: 1
-#   script:
-#     - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"'
-#     # To be done in a later iteration
-#     # - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"'
-#     # - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"'
-#     - /analyze -t $DAST_WEBSITE
-#   timeout: 4h
-#   artifacts:
-#     paths:
-#       - gl-dast-report.json  # GitLab-specific
-#     reports:
-#       dast: gl-dast-report.json
-#     expire_in: 1 week  # GitLab-specific
-
-# To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
-# schedule:dast:
-#   extends:
-#     - dast
-#     - .reports:schedule-dast
-#   variables:
-#     DAST_FULL_SCAN_ENABLED: "true"

.gitlab/ci/review.gitlab-ci.yml

@@ -77,6 +77,11 @@ review-deploy:
     # to have to manually start the jobs in sequence, so we do it for them.
     - '[ -z $CI_JOB_MANUAL ] || play_job "review-qa-smoke"'
     - '[ -z $CI_JOB_MANUAL ] || play_job "review-performance"'
+  after_script:
+    # Run seed-dast-test-data.sh only when DAST_RUN is set to true. This is to pupulate review app with data for DAST scan.
+    # Set DAST_RUN to true when jobs are manually scheduled.
+    - if [ "$DAST_RUN" == "true" ]; then source scripts/review_apps/seed-dast-test-data.sh; TRACE=1 trigger_proj_user_creation; fi
+
   artifacts:
     paths: [environment_url.txt]
     expire_in: 2 days

.gitlab/ci/rules.gitlab-ci.yml

@@ -645,6 +645,7 @@
     - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/'
       when: never
     - <<: *if-dot-com-gitlab-org-schedule
+      allow_failure: true
 
 ################
 # Review rules #
@@ -665,6 +666,8 @@
 
 .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise:
   rules:
+    - if: '$DAST_RUN == "true"' # Skip this job when DAST is run
+      when: never
     - <<: *if-not-ee
       when: never
     - <<: *if-dot-com-gitlab-org-merge-request

scripts/review_apps/seed-dast-test-data.sh

+[[ "$TRACE" ]] && set -x
+
+function create_user() {
+    local user="${1}"
+
+    # API details at https://docs.gitlab.com/ee/api/users.html#user-creation
+    #
+    # We set "can_create_group=false" because we don't want the DAST user to create groups.
+    # Otherwise, the DAST user likely creates a group and enables 2FA for all group members,
+    # which leads to the DAST scan getting "stuck" on the 2FA set up page.
+    # Once https://gitlab.com/gitlab-org/gitlab/-/issues/231447 is resolved, we can use 
+    # DAST_AUTH_EXCLUDE_URLS instead to prevent DAST from enabling 2FA.
+    curl --silent --show-error --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" \
+        --data "email=${user}@example.com" \
+        --data "name=${user}" \
+        --data "username=${user}" \
+        --data "password=${REVIEW_APPS_ROOT_PASSWORD}" \
+        --data "skip_confirmation=true" \
+        --data "can_create_group=false" \
+        "${CI_ENVIRONMENT_URL}/api/v4/users" > /tmp/user.json
+
+    [[ "$TRACE" ]] && cat /tmp/user.json >&2
+
+    jq .id /tmp/user.json
+}
+
+function create_project_for_user() {
+    local userid="${1}"
+
+    # API details at https://docs.gitlab.com/ee/api/projects.html#create-project-for-user
+    curl --silent --show-error --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" \
+        --data "user_id=${userid}" \
+        --data "name=awesome-test-project-${userid}" \
+        --data "visibility=private" \
+        "${CI_ENVIRONMENT_URL}/api/v4/projects/user/${userid}" > /tmp/project.json
+
+    [[ "$TRACE" ]] && cat /tmp/project.json >&2
+}
+
+function trigger_proj_user_creation(){
+    local u1=$(create_user "user1")
+    create_project_for_user $u1
+    local u2=$(create_user "user2")
+    create_project_for_user $u2
+    local u3=$(create_user "user3")
+    create_project_for_user $u3
+    local u4=$(create_user "user4")
+    create_project_for_user $u4
+    local u5=$(create_user "user5")
+    create_project_for_user $u5
+    local u6=$(create_user "user6")
+    create_project_for_user $u6
+    local u7=$(create_user "user7")
+    create_project_for_user $u7
+    local u8=$(create_user "user8")
+    create_project_for_user $u8
+    local u9=$(create_user "user9")
+    create_project_for_user $u9
+    local u10=$(create_user "user10")
+    create_project_for_user $u10
+    local u11=$(create_user "user11")
+    create_project_for_user $u11
+    local u12=$(create_user "user12")
+    create_project_for_user $u12
+    local u13=$(create_user "user13")
+    create_project_for_user $u13
+    local u14=$(create_user "user14")
+    create_project_for_user $u14
+}