Automatic merge of gitlab-org/gitlab-ce master

changelogs/unreleased/47234-composable-auto-devops.yml

+---
+title: Split Auto-DevOps.gitlab-ci.yml into reusable templates
+merge_request: 26520
+author:
+type: changed

doc/topics/autodevops/index.md

@@ -699,6 +699,21 @@ renaming `.staging` to `staging`. Then make sure to uncomment the `when` key of
 the `production` job to turn it into a manual action instead of deploying
 automatically.
 
+### Using components of Auto-DevOps
+
+If you only require a subset of the features offered by Auto-DevOps, you can include
+individual Auto-DevOps jobs into your own `.gitlab-ci.yml`.
+
+For example, to make use of [Auto Build](#auto-build), you can add the following to
+your `.gitlab-ci.yml`:
+
+```yaml
+include:
+  - template: Jobs/Build.gitlab-ci.yml
+```
+
+Consult the [Auto DevOps template] for information on available jobs.
+
 ### PostgreSQL database support
 
 In order to support applications that require a database,

doc/user/project/clusters/index.md

@@ -314,12 +314,6 @@ install it manually.
 
 ## Installing applications
 
-NOTE: **Note:**
-Before starting the installation of applications, make sure that time is synchronized
-between your GitLab server and your Kubernetes cluster. Otherwise, installation could fail
-and you may get errors like `Error: remote error: tls: bad certificate`
-in the `stdout` of pods created by GitLab in your Kubernetes cluster.
-
 GitLab provides a one-click install for various applications which can
 be added directly to your configured cluster. Those applications are
 needed for [Review Apps](../../../ci/review_apps/index.md) and
@@ -378,6 +372,29 @@ Upgrades will reset values back to the values built into the `runner`
 chart plus the values set by
 [`values.yaml`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/vendor/runner/values.yaml)
 
+### Troubleshooting applications
+
+Applications can fail with the following error:
+
+```text
+Error: remote error: tls: bad certificate
+```
+
+To avoid installation errors:
+
+- Before starting the installation of applications, make sure that time is synchronized
+  between your GitLab server and your Kubernetes cluster.
+- Ensure certificates are not out of sync.  When installing applications, GitLab expects a new cluster with no previous installation of Tiller.
+
+  You can confirm that the certificates match via `kubectl`:
+
+  ```sh
+  kubectl get configmaps/values-content-configuration-ingress -n gitlab-managed-apps -o \
+  "jsonpath={.data['cert\.pem']}" | base64 -d > a.pem
+  kubectl get secrets/tiller-secret -n gitlab-managed-apps -o "jsonpath={.data['ca\.crt']}" | base64 -d > b.pem
+  diff a.pem b.pem
+  ```
+
 ## Getting the external endpoint
 
 NOTE: **Note:**

lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml

+performance:
+  stage: performance
+  image: docker:stable
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - |
+      if ! docker info &>/dev/null; then
+        if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+          export DOCKER_HOST='tcp://localhost:2375'
+        fi
+      fi
+    - export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
+    - mkdir gitlab-exporter
+    - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js
+    - mkdir sitespeed-results
+    - |
+      if [ -f .gitlab-urls.txt ]
+      then
+        sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt
+        docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt
+      else
+        docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL"
+      fi
+    - mv sitespeed-results/data/performance.json performance.json
+  artifacts:
+    paths:
+    - performance.json
+    - sitespeed-results/
+  only:
+    refs:
+      - branches
+      - tags
+    kubernetes: active
+  except:
+    variables:
+      - $PERFORMANCE_DISABLED

lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml

+build:
+  stage: build
+  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image/master:stable"
+  services:
+    - docker:stable-dind
+  script:
+    - |
+      if [[ -z "$CI_COMMIT_TAG" ]]; then
+        export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG}
+        export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA}
+      else
+        export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE}
+        export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG}
+      fi
+    - /build/build.sh
+  only:
+    - branches
+    - tags

lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml

+code_quality:
+  stage: test
+  image: docker:stable
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - export CQ_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
+    - |
+      if ! docker info &>/dev/null; then
+        if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+          export DOCKER_HOST='tcp://localhost:2375'
+        fi
+      fi
+    - |
+      docker run --env SOURCE_CODE="$PWD" \
+            --volume "$PWD":/code \
+            --volume /var/run/docker.sock:/var/run/docker.sock \
+            "registry.gitlab.com/gitlab-org/security-products/codequality:$CQ_VERSION" /code
+  artifacts:
+    paths: [gl-code-quality-report.json]
+  only:
+    - branches
+    - tags
+  except:
+    variables:
+      - $CODE_QUALITY_DISABLED

lib/gitlab/ci/templates/Jobs/DAST.gitlab-ci.yml

+dast:
+  stage: dast
+  image: docker:stable
+  variables:
+    DOCKER_DRIVER: overlay2
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
+    - export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
+    - |
+      if ! docker info &>/dev/null; then
+        if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+          export DOCKER_HOST='tcp://localhost:2375'
+        fi
+      fi
+    - |
+      function dast_run() {
+        docker run \
+        --env DAST_TARGET_AVAILABILITY_TIMEOUT \
+        --volume "$PWD:/output" \
+        --volume /var/run/docker.sock:/var/run/docker.sock \
+        -w /output \
+        "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \
+        /analyze -t $DAST_WEBSITE \
+        "$@"
+      }
+    - |
+      if [ -n "$DAST_AUTH_URL" ]
+      then
+        dast_run \
+          --auth-url $DAST_AUTH_URL \
+          --auth-username $DAST_USERNAME \
+          --auth-password $DAST_PASSWORD \
+          --auth-username-field $DAST_USERNAME_FIELD \
+          --auth-password-field $DAST_PASSWORD_FIELD
+      else
+        dast_run
+      fi
+  artifacts:
+    reports:
+      dast: gl-dast-report.json
+  only:
+    refs:
+      - branches
+      - tags
+    variables:
+      - $GITLAB_FEATURES =~ /\bdast\b/
+  except:
+    refs:
+      - master
+    variables:
+      - $DAST_DISABLED

lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml

+test:
+  services:
+    - postgres:latest
+  variables:
+    POSTGRES_DB: test
+  stage: test
+  image: gliderlabs/herokuish:latest
+  script:
+    - |
+      if [ -z ${KUBERNETES_PORT+x} ]; then
+        DB_HOST=postgres
+      else
+        DB_HOST=localhost
+      fi
+    - export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}"
+    - cp -R . /tmp/app
+    - /bin/herokuish buildpack test
+  only:
+    - branches
+    - tags
+  except:
+    variables:
+      - $TEST_DISABLED

lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml

@@ -28,6 +28,12 @@ container_scanning:
     - docker:stable-dind
   script:
     - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then { export DOCKER_SERVICE="localhost" ; export DOCKER_HOST="tcp://${DOCKER_SERVICE}:2375" ; } fi
+    - |
+      if [[ -n "$CI_REGISTRY_USER" ]]; then
+        echo "Logging to GitLab Container Registry with CI credentials..."
+        docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
+        echo ""
+      fi
     - docker run -d --name db arminc/clair-db:latest
     - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:${CLAIR_LOCAL_SCAN_VERSION}
     - apk add -U wget ca-certificates
@@ -36,7 +42,6 @@ container_scanning:
     - mv clair-scanner_linux_amd64 clair-scanner
     - chmod +x clair-scanner
     - touch clair-whitelist.yml
-    - while( ! wget -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; done
     - retries=0
     - echo "Waiting for clair daemon to start"
     - while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done

lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml

@@ -4,6 +4,9 @@
 # List of the variables: https://gitlab.com/gitlab-org/security-products/dast#settings
 # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
 
+include:
+  - template: Jobs/DAST.gitlab-ci.yml
+
 variables:
   DAST_WEBSITE: http://example.com # Please edit to be your website to scan for vulnerabilities
 
@@ -14,46 +17,10 @@ stages:
   - dast
 
 dast:
-  stage: dast
-  image: docker:stable
-  variables:
-    DOCKER_DRIVER: overlay2
-  allow_failure: true
-  services:
-    - docker:stable-dind
-  script:
-    - export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
-    - |
-      function dast_run() {
-        docker run \
-        --env DAST_TARGET_AVAILABILITY_TIMEOUT \
-        --volume "$PWD:/output" \
-        --volume /var/run/docker.sock:/var/run/docker.sock \
-        -w /output \
-        "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \
-        /analyze -t $DAST_WEBSITE \
-        "$@"
-      }
-    - |
-      if [ -n "$DAST_AUTH_URL" ]
-      then
-        dast_run \
-          --auth-url $DAST_AUTH_URL \
-          --auth-username $DAST_USERNAME \
-          --auth-password $DAST_PASSWORD \
-          --auth-username-field $DAST_USERNAME_FIELD \
-          --auth-password-field $DAST_PASSWORD_FIELD
-      else
-        dast_run
-      fi
-  artifacts:
-    reports:
-      dast: gl-dast-report.json
   only:
     refs:
       - branches
-    variables:
-      - $GITLAB_FEATURES =~ /\bdast\b/
   except:
+    refs: [] # Override default from template
     variables:
       - $DAST_DISABLED

lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml

@@ -14,6 +14,12 @@ dependency_scanning:
     - docker:stable-dind
   script:
     - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
+    - |
+      if ! docker info &>/dev/null; then
+        if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+          export DOCKER_HOST='tcp://localhost:2375'
+        fi
+      fi
     - |
       docker run \
         --env DS_ANALYZER_IMAGES \

lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml

@@ -14,6 +14,12 @@ sast:
     - docker:stable-dind
   script:
     - export SAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
+    - |
+      if ! docker info &>/dev/null; then
+        if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+          export DOCKER_HOST='tcp://localhost:2375'
+        fi
+      fi
     - |
       docker run \
         --env SAST_ANALYZER_IMAGES \

spec/lib/gitlab/ci/templates/templates_spec.rb

@@ -4,6 +4,7 @@ require 'spec_helper'
 
 describe "CI YML Templates" do
   ABSTRACT_TEMPLATES = %w[Serverless].freeze
+  PROJECT_DEPENDENT_TEMPLATES = %w[Auto-DevOps].freeze
 
   def self.concrete_templates
     Gitlab::Template::GitlabCiYmlTemplate.all.reject do |template|
@@ -20,7 +21,10 @@ describe "CI YML Templates" do
   describe 'concrete templates with CI/CD jobs' do
     concrete_templates.each do |template|
       it "#{template.name} template should be valid" do
-        expect { Gitlab::Ci::YamlProcessor.new(template.content) }
+        # Trigger processing of included files
+        project = create(:project, :test_repo) if PROJECT_DEPENDENT_TEMPLATES.include?(template.name)
+
+        expect { Gitlab::Ci::YamlProcessor.new(template.content, project: project) }
           .not_to raise_error
       end
     end