Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/frontend.gitlab-ci.yml

@@ -4,6 +4,8 @@
       - vendor/ruby/
       - .yarn-cache/
       - tmp/cache/assets/sprockets
+      - tmp/cache/babel-loader
+      - tmp/cache/vue-loader
 
 .gitlab:assets:compile-metadata:
   extends:
@@ -28,7 +30,7 @@
     DOCKER_DRIVER: overlay2
     DOCKER_HOST: tcp://docker:2375
   cache:
-    key: "assets-compile:production:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6"
+    key: "assets-compile:production:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:tmp_cache_webpack:v7"
   artifacts:
     name: webpack-report
     expire_in: 31d
@@ -84,7 +86,7 @@ gitlab:assets:compile pull-cache:
     # we override the max_old_space_size to prevent OOM errors
     NODE_OPTIONS: --max_old_space_size=3584
   cache:
-    key: "assets-compile:v7"
+    key: "assets-compile:v8"
   artifacts:
     expire_in: 7d
     paths:
@@ -106,7 +108,7 @@ compile-assets pull-push-cache foss:
       - master
   cache:
     policy: pull-push
-    key: "assets-compile:v7:foss"
+    key: "assets-compile:v8:foss"
 
 compile-assets pull-cache:
   extends: .compile-assets-metadata
@@ -117,7 +119,7 @@ compile-assets pull-cache foss:
   extends: [".compile-assets-metadata", ".only-ee-as-if-foss"]
   cache:
     policy: pull
-    key: "assets-compile:v7:foss"
+    key: "assets-compile:v8:foss"
 
 .only-code-frontend-job-base:
   extends:

app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue

@@ -3,7 +3,7 @@ import { createNamespacedHelpers, mapState, mapActions } from 'vuex';
 import _ from 'underscore';
 import { GlFormInput, GlFormCheckbox } from '@gitlab/ui';
 import { sprintf, s__ } from '~/locale';
-import ClusterFormDropdown from './cluster_form_dropdown.vue';
+import ClusterFormDropdown from '~/create_cluster/components/cluster_form_dropdown.vue';
 import { KUBERNETES_VERSIONS } from '../constants';
 import LoadingButton from '~/vue_shared/components/loading_button.vue';
 

app/assets/javascripts/create_cluster/eks_cluster/store/index.js

@@ -4,7 +4,7 @@ import * as getters from './getters';
 import mutations from './mutations';
 import state from './state';
 
-import clusterDropdownStore from './cluster_dropdown';
+import clusterDropdownStore from '~/create_cluster/store/cluster_dropdown';
 
 import {
   fetchRoles,

changelogs/unreleased/remove-feature-flag-import-graceful-failures.yml

+---
+title: Remove feature flag for import graceful failures
+merge_request:
+author:
+type: other

changelogs/unreleased/sh-fix-sidekiq-timestamps.yml

+---
+title: Make Sidekiq timestamps consistently ISO 8601
+merge_request: 22750
+author:
+type: fixed

doc/user/packages/npm_registry/index.md

@@ -195,7 +195,7 @@ info Visit https://yarnpkg.com/en/docs/cli/install for documentation about this
 ```
 
 In this case, try adding this to your `.npmrc` file (and replace `<your_oauth_token>`
-with your with your OAuth or personal access token):
+with your OAuth or personal access token):
 
 ```text
 //gitlab.com/api/v4/projects/:_authToken=<your_oauth_token>

lib/gitlab/import_export/relation_tree_restorer.rb

@@ -75,9 +75,6 @@ module Gitlab
 
         save_id_mapping(relation_key, data_hash, relation_object)
       rescue => e
-        # re-raise if not enabled
-        raise e unless Feature.enabled?(:import_graceful_failures, @importable.group, default_enabled: true)
-
         log_import_failure(relation_key, relation_index, e)
       end
 

lib/gitlab/sidekiq_logging/json_formatter.rb

@@ -3,6 +3,8 @@
 module Gitlab
   module SidekiqLogging
     class JSONFormatter
+      TIMESTAMP_FIELDS = %w[created_at enqueued_at started_at retried_at failed_at completed_at].freeze
+
       def call(severity, timestamp, progname, data)
         output = {
           severity: severity,
@@ -13,11 +15,27 @@ module Gitlab
         when String
           output[:message] = data
         when Hash
+          convert_to_iso8601!(data)
           output.merge!(data)
         end
 
         output.to_json + "\n"
       end
+
+      private
+
+      def convert_to_iso8601!(payload)
+        TIMESTAMP_FIELDS.each do |key|
+          value = payload[key]
+          payload[key] = format_time(value) if value.present?
+        end
+      end
+
+      def format_time(timestamp)
+        return timestamp unless timestamp.is_a?(Numeric)
+
+        Time.at(timestamp).utc.iso8601(3)
+      end
     end
   end
 end

lib/gitlab/sidekiq_logging/structured_logger.rb

@@ -6,8 +6,6 @@ require 'active_record/log_subscriber'
 module Gitlab
   module SidekiqLogging
     class StructuredLogger
-      START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze
-      DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze
       MAXIMUM_JOB_ARGUMENTS_LENGTH = 10.kilobytes
 
       def call(job, queue)
@@ -65,8 +63,6 @@ module Gitlab
           payload['job_status'] = 'done'
         end
 
-        convert_to_iso8601(payload, DONE_TIMESTAMP_FIELDS)
-
         payload['db_duration'] = ActiveRecord::LogSubscriber.runtime
         payload['db_duration_s'] = payload['db_duration'] / 1000
 
@@ -79,7 +75,7 @@ module Gitlab
         # ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0)
         # supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime
         payload['cpu_s'] = time[:cputime].round(6) if time[:cputime] > 0
-        payload['completed_at'] = Time.now.utc
+        payload['completed_at'] = Time.now.utc.to_f
       end
 
       def parse_job(job)
@@ -91,17 +87,9 @@ module Gitlab
         job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS']
         job['args'] = limited_job_args(job['args']) if job['args']
 
-        convert_to_iso8601(job, START_TIMESTAMP_FIELDS)
-
         job
       end
 
-      def convert_to_iso8601(payload, keys)
-        keys.each do |key|
-          payload[key] = format_time(payload[key]) if payload[key]
-        end
-      end
-
       def elapsed(t0)
         t1 = get_time
         {
@@ -121,12 +109,6 @@ module Gitlab
         Gitlab::Metrics::System.monotonic_time
       end
 
-      def format_time(timestamp)
-        return timestamp if timestamp.is_a?(String)
-
-        Time.at(timestamp).utc.iso8601(3)
-      end
-
       def limited_job_args(args)
         return unless args.is_a?(Array)
 

spec/frontend/create_cluster/eks_cluster/components/cluster_form_dropdown_spec.js → spec/frontend/create_cluster/components/cluster_form_dropdown_spec.js

@@ -2,7 +2,7 @@ import { shallowMount } from '@vue/test-utils';
 import $ from 'jquery';
 
 import { GlIcon } from '@gitlab/ui';
-import ClusterFormDropdown from '~/create_cluster/eks_cluster/components/cluster_form_dropdown.vue';
+import ClusterFormDropdown from '~/create_cluster/components/cluster_form_dropdown.vue';
 import DropdownButton from '~/vue_shared/components/dropdown/dropdown_button.vue';
 import DropdownSearchInput from '~/vue_shared/components/dropdown/dropdown_search_input.vue';
 

spec/frontend/create_cluster/eks_cluster/components/eks_cluster_configuration_form_spec.js

@@ -5,7 +5,7 @@ import { GlFormCheckbox } from '@gitlab/ui';
 
 import EksClusterConfigurationForm from '~/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue';
 import eksClusterFormState from '~/create_cluster/eks_cluster/store/state';
-import clusterDropdownStoreState from '~/create_cluster/eks_cluster/store/cluster_dropdown/state';
+import clusterDropdownStoreState from '~/create_cluster/store/cluster_dropdown/state';
 
 const localVue = createLocalVue();
 localVue.use(Vuex);

spec/frontend/create_cluster/eks_cluster/store/cluster_dropdown/actions_spec.js → spec/frontend/create_cluster/store/cluster_dropdown/actions_spec.js

 import testAction from 'helpers/vuex_action_helper';
 
-import createState from '~/create_cluster/eks_cluster/store/cluster_dropdown/state';
-import * as types from '~/create_cluster/eks_cluster/store/cluster_dropdown/mutation_types';
-import actionsFactory from '~/create_cluster/eks_cluster/store/cluster_dropdown/actions';
+import createState from '~/create_cluster/store/cluster_dropdown/state';
+import * as types from '~/create_cluster/store/cluster_dropdown/mutation_types';
+import actionsFactory from '~/create_cluster/store/cluster_dropdown/actions';
 
 describe('Cluster dropdown Store Actions', () => {
   const items = [{ name: 'item 1' }];

spec/frontend/create_cluster/eks_cluster/store/cluster_dropdown/mutations_spec.js → spec/frontend/create_cluster/store/cluster_dropdown/mutations_spec.js

@@ -2,9 +2,9 @@ import {
   REQUEST_ITEMS,
   RECEIVE_ITEMS_SUCCESS,
   RECEIVE_ITEMS_ERROR,
-} from '~/create_cluster/eks_cluster/store/cluster_dropdown/mutation_types';
-import createState from '~/create_cluster/eks_cluster/store/cluster_dropdown/state';
-import mutations from '~/create_cluster/eks_cluster/store/cluster_dropdown/mutations';
+} from '~/create_cluster/store/cluster_dropdown/mutation_types';
+import createState from '~/create_cluster/store/cluster_dropdown/state';
+import mutations from '~/create_cluster/store/cluster_dropdown/mutations';
 
 describe('Cluster dropdown store mutations', () => {
   let state;

spec/lib/gitlab/sidekiq_logging/json_formatter_spec.rb

@@ -3,28 +3,53 @@
 require 'spec_helper'
 
 describe Gitlab::SidekiqLogging::JSONFormatter do
-  let(:hash_input) { { foo: 1, bar: 'test' } }
   let(:message) { 'This is a test' }
-  let(:timestamp) { Time.now }
+  let(:now) { Time.now }
+  let(:timestamp) { now.utc.to_f }
+  let(:timestamp_iso8601) { now.iso8601(3) }
+
+  describe 'with a Hash' do
+    let(:hash_input) do
+      {
+        foo: 1,
+        'bar' => 'test',
+        'created_at' => timestamp,
+        'enqueued_at' => timestamp,
+        'started_at' => timestamp,
+        'retried_at' => timestamp,
+        'failed_at' => timestamp,
+        'completed_at' => timestamp_iso8601
+      }
+    end
 
-  it 'wraps a Hash' do
-    result = subject.call('INFO', timestamp, 'my program', hash_input)
+    it 'properly formats timestamps into ISO 8601 form' do
+      result = subject.call('INFO', now, 'my program', hash_input)
 
       data = JSON.parse(result)
-    expected_output = hash_input.stringify_keys
-    expected_output['severity'] = 'INFO'
-    expected_output['time'] = timestamp.utc.iso8601(3)
+      expected_output = hash_input.stringify_keys.merge!(
+        {
+          'severity' => 'INFO',
+          'time' => timestamp_iso8601,
+          'created_at' => timestamp_iso8601,
+          'enqueued_at' => timestamp_iso8601,
+          'started_at' => timestamp_iso8601,
+          'retried_at' => timestamp_iso8601,
+          'failed_at' => timestamp_iso8601,
+          'completed_at' => timestamp_iso8601
+        }
+      )
 
       expect(data).to eq(expected_output)
     end
+  end
 
   it 'wraps a String' do
-    result = subject.call('DEBUG', timestamp, 'my string', message)
+    result = subject.call('DEBUG', now, 'my string', message)
 
     data = JSON.parse(result)
     expected_output = {
       severity: 'DEBUG',
-      time: timestamp.utc.iso8601(3),
+      time: timestamp_iso8601,
       message: message
     }
 

spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb

@@ -30,8 +30,8 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
         'message' => 'TestWorker JID-da883554ee4fe414012f5f42: start',
         'job_status' => 'start',
         'pid' => Process.pid,
-        'created_at' => created_at.iso8601(3),
-        'enqueued_at' => created_at.iso8601(3),
+        'created_at' => created_at.to_f,
+        'enqueued_at' => created_at.to_f,
         'scheduling_latency_s' => scheduling_latency_s
       )
     end
@@ -40,7 +40,7 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
         'message' => 'TestWorker JID-da883554ee4fe414012f5f42: done: 0.0 sec',
         'job_status' => 'done',
         'duration' => 0.0,
-        'completed_at' => timestamp.iso8601(3),
+        'completed_at' => timestamp.to_f,
         'cpu_s' => 1.111112,
         'db_duration' => 0,
         'db_duration_s' => 0
@@ -227,17 +227,17 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
   describe '#add_time_keys!' do
     let(:time) { { duration: 0.1231234, cputime: 1.2342345 } }
     let(:payload) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status' } }
-    let(:current_utc_time) { '2019-09-23 10:00:58 UTC' }
-    let(:payload_with_time_keys) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status', 'duration' => 0.123123, 'cpu_s' => 1.234235, 'completed_at' => current_utc_time } }
+    let(:current_utc_time) { Time.now.utc }
+    let(:payload_with_time_keys) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status', 'duration' => 0.123123, 'cpu_s' => 1.234235, 'completed_at' => current_utc_time.to_f } }
 
     subject { described_class.new }
 
     it 'update payload correctly' do
-      expect(Time).to receive_message_chain(:now, :utc).and_return(current_utc_time)
-
+      Timecop.freeze(current_utc_time) do
         subject.send(:add_time_keys!, time, payload)
 
         expect(payload).to eq(payload_with_time_keys)
       end
     end
+  end
 end

spec/tasks/gitlab/import_export/import_rake_spec.rb

@@ -76,11 +76,6 @@ describe 'gitlab:import_export:import rake task', :sidekiq do
     let(:not_imported_message) { /Total number of not imported relations: 1/ }
     let(:error) { /Validation failed: Notes is invalid/ }
 
-    context 'when import_graceful_failures feature flag is enabled' do
-      before do
-        stub_feature_flags(import_graceful_failures: true)
-      end
-
     it 'performs project import successfully' do
       expect { subject }.to output(not_imported_message).to_stdout
       expect { subject }.not_to raise_error
@@ -90,23 +85,4 @@ describe 'gitlab:import_export:import rake task', :sidekiq do
       expect(project.import_state.status).to eq('finished')
     end
   end
-
-    context 'when import_graceful_failures feature flag is disabled' do
-      before do
-        stub_feature_flags(import_graceful_failures: false)
-      end
-
-      it 'fails project import with an error' do
-        # Catch exit call, and raise exception instead
-        expect_any_instance_of(GitlabProjectImport).to receive(:exit)
-          .with(1).and_raise(SystemExit)
-
-        expect { subject }.to raise_error(SystemExit).and output(error).to_stdout
-
-        expect(project.merge_requests).to be_empty
-        expect(project.import_state.last_error).to match(error)
-        expect(project.import_state.status).to eq('failed')
-      end
-    end
-  end
 end