Merge branch 'cache-public-assets' into 'master'

Cache Webpack assets in frontend assets compilation jobs

See merge request gitlab-org/gitlab!26008

.gitlab/ci/frontend.gitlab-ci.yml

@@ -2,6 +2,8 @@
   cache:
     paths:
       - vendor/ruby/
+      - public/assets/webpack/
+      - assets-hash.txt
       - .yarn-cache/
       - tmp/cache/assets/sprockets
       - tmp/cache/babel-loader
@@ -28,18 +30,24 @@
     DOCKER_DRIVER: overlay2
     DOCKER_HOST: tcp://docker:2375
   cache:
-    key: "assets-compile:production:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:tmp_cache_webpack:v9"
+    key: "assets-compile:production:v1"
   artifacts:
     name: webpack-report
     expire_in: 31d
     paths:
       - webpack-report/
-      - public/assets/
+      - assets-compile.log
+      - public/assets/application-*.css
+      - public/assets/application-*.css.gz
+    when: always
   script:
     - node --version
     - retry yarn install --frozen-lockfile --production --cache-folder .yarn-cache --prefer-offline
     - free -m
-    - retry bundle exec rake gitlab:assets:compile
+    - time bin/rake gitlab:assets:compile > assets-compile.log 2>&1
+    # TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists
+    # We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines
+    # https://gitlab.com/gitlab-org/gitlab/issues/208389
     - time scripts/build_assets_image
     - scripts/clean-old-cached-assets
     - rm -f /etc/apt/sources.list.d/google*.list  # We don't need to update Chrome here
@@ -71,7 +79,7 @@ gitlab:assets:compile pull-cache:
     - node --version
     - retry yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline
     - free -m
-    - retry bundle exec rake gitlab:assets:compile
+    - time bin/rake gitlab:assets:compile > assets-compile.log 2>&1
     - scripts/clean-old-cached-assets
   variables:
     SETUP_DB: "false"
@@ -79,12 +87,13 @@ gitlab:assets:compile pull-cache:
     NODE_OPTIONS: --max_old_space_size=3584
     WEBPACK_VENDOR_DLL: "true"
   cache:
-    key: "assets-compile:v9"
+    key: "assets-compile:test:v1"
   artifacts:
     expire_in: 7d
     paths:
       - node_modules
       - public/assets
+      - assets-compile.log
 
 compile-assets pull-push-cache:
   extends:
@@ -100,7 +109,7 @@ compile-assets pull-push-cache as-if-foss:
     - .as-if-foss
   cache:
     policy: pull-push
-    key: "assets-compile:v9:foss"
+    key: "assets-compile:test:as-if-foss:v1"
 
 compile-assets pull-cache:
   extends:
@@ -116,7 +125,7 @@ compile-assets pull-cache as-if-foss:
     - .as-if-foss
   cache:
     policy: pull
-    key: "assets-compile:v9:foss"
+    key: "assets-compile:test:as-if-foss:v1"
 
 .frontend-fixtures-base:
   extends:

.gitlab/ci/qa.gitlab-ci.yml

@@ -55,5 +55,9 @@ package-and-qa:
   extends:
     - .package-and-qa-base
     - .qa:rules:package-and-qa
-  needs: ["build-qa-image", "gitlab:assets:compile pull-cache"]
+  needs:
+    - job: build-qa-image
+      artifacts: false
+    - job: gitlab:assets:compile pull-cache
+      artifacts: false
   allow_failure: true

lib/tasks/gitlab/assets.rake

+# frozen_string_literal: true
+
+require 'fileutils'
+
+module Tasks
+  module Gitlab
+    module Assets
+      FOSS_ASSET_FOLDERS = %w[app/assets app/views fixtures/emojis vendor/assets/javascripts].freeze
+      EE_ASSET_FOLDERS = %w[ee/app/assets ee/app/views].freeze
+      JS_ASSET_PATTERNS = %w[*.js config/**/*.js].freeze
+      JS_ASSET_FILES = %w[package.json yarn.lock].freeze
+      MASTER_MD5_HASH_FILE = 'master-assets-hash.txt'
+      HEAD_MD5_HASH_FILE = 'assets-hash.txt'
+      PUBLIC_ASSETS_WEBPACK_DIR = 'public/assets/webpack'
+
+      def self.md5_of_assets_impacting_webpack_compilation
+        start_time = Time.now
+        asset_files = assets_impacting_webpack_compilation
+        puts "Generating the MD5 hash for #{assets_impacting_webpack_compilation.size} Webpack-related assets..."
+
+        asset_file_md5s = asset_files.map do |asset_file|
+          Digest::MD5.file(asset_file).hexdigest
+        end
+
+        Digest::MD5.hexdigest(asset_file_md5s.join).tap { |md5| puts "=> MD5 generated in #{Time.now - start_time}: #{md5}" }
+      end
+
+      def self.assets_impacting_webpack_compilation
+        assets_folders = FOSS_ASSET_FOLDERS
+        assets_folders += EE_ASSET_FOLDERS if ::Gitlab.ee?
+
+        asset_files = Dir.glob(JS_ASSET_PATTERNS)
+        asset_files += JS_ASSET_FILES
+
+        assets_folders.each do |folder|
+          asset_files.concat(Dir.glob(["#{folder}/**/*.*"]))
+        end
+
+        asset_files
+      end
+      private_class_method :assets_impacting_webpack_compilation
+    end
+  end
+end
+
 namespace :gitlab do
   namespace :assets do
     desc 'GitLab | Assets | Compile all frontend assets'
@@ -8,9 +53,35 @@ namespace :gitlab do
         yarn:check
         gettext:po_to_json
         rake:assets:precompile
-        webpack:compile
+        gitlab:assets:compile_webpack_if_needed
         gitlab:assets:fix_urls
-      ].each(&Gitlab::TaskHelpers.method(:invoke_and_time_task))
+      ].each(&::Gitlab::TaskHelpers.method(:invoke_and_time_task))
+    end
+
+    desc 'GitLab | Assets | Compile all Webpack assets'
+    task :compile_webpack_if_needed do
+      FileUtils.mv(Tasks::Gitlab::Assets::HEAD_MD5_HASH_FILE, Tasks::Gitlab::Assets::MASTER_MD5_HASH_FILE, force: true)
+
+      master_assets_md5 =
+        if File.exist?(Tasks::Gitlab::Assets::MASTER_MD5_HASH_FILE)
+          File.read(Tasks::Gitlab::Assets::MASTER_MD5_HASH_FILE)
+        else
+          'missing!'
+        end
+
+      head_assets_md5 = Tasks::Gitlab::Assets.md5_of_assets_impacting_webpack_compilation.tap do |md5|
+        File.write(Tasks::Gitlab::Assets::HEAD_MD5_HASH_FILE, md5)
+      end
+
+      puts "Webpack assets MD5 for `master`: #{master_assets_md5}"
+      puts "Webpack assets MD5 for `HEAD`: #{head_assets_md5}"
+
+      public_assets_webpack_dir_exists = Dir.exist?(Tasks::Gitlab::Assets::PUBLIC_ASSETS_WEBPACK_DIR)
+
+      if head_assets_md5 != master_assets_md5 || !public_assets_webpack_dir_exists
+        FileUtils.rm_r(Tasks::Gitlab::Assets::PUBLIC_ASSETS_WEBPACK_DIR) if public_assets_webpack_dir_exists
+        Rake::Task['webpack:compile'].invoke
+      end
     end
 
     desc 'GitLab | Assets | Clean up old compiled frontend assets'