Expand dashboard schema validation to full list

In order to provide user with complete information we need to expand
dashboard validation to return exhaustive list of errors

app/graphql/resolvers/metrics/dashboard_resolver.rb

@@ -14,7 +14,8 @@ module Resolvers
       def resolve(**args)
         return unless environment
 
-        ::PerformanceMonitoring::PrometheusDashboard.find_for(project: environment.project, user: context[:current_user], path: args[:path], options: { environment: environment })
+        ::PerformanceMonitoring::PrometheusDashboard
+          .find_for(project: environment.project, user: context[:current_user], path: args[:path], options: { environment: environment })
       end
     end
   end

app/graphql/types/metrics/dashboard_type.rb

@@ -16,6 +16,13 @@ module Types
       field :annotations, Types::Metrics::Dashboards::AnnotationType.connection_type, null: true,
             description: 'Annotations added to the dashboard',
             resolver: Resolvers::Metrics::Dashboards::AnnotationResolver
+
+      # In order to maintain backward compatibility we need to return NULL when there are no warnings
+      # and dashboard validation returns an empty array when there are no issues.
+      def schema_validation_warnings
+        warnings = object.schema_validation_warnings
+        warnings unless warnings.empty?
+      end
     end
     # rubocop: enable Graphql/AuthorizeTypes
   end

app/models/blob_viewer/metrics_dashboard_yml.rb

@@ -25,20 +25,30 @@ module BlobViewer
     private
 
     def parse_blob_data
-      yaml = ::Gitlab::Config::Loader::Yaml.new(blob.data).load_raw!
+      if Feature.enabled?(:metrics_dashboard_exhaustive_validations, project)
+        exhaustive_metrics_dashboard_validation
+      else
+        old_metrics_dashboard_validation
+      end
+    end
 
+    def old_metrics_dashboard_validation
+      yaml = ::Gitlab::Config::Loader::Yaml.new(blob.data).load_raw!
       ::PerformanceMonitoring::PrometheusDashboard.from_json(yaml)
-      nil
+      []
     rescue Gitlab::Config::Loader::FormatError => error
-      wrap_yml_syntax_error(error)
+      ["YAML syntax: #{error.message}"]
     rescue ActiveModel::ValidationError => invalid
-      invalid.model.errors
+      invalid.model.errors.messages.map { |messages| messages.join(': ') }
     end
 
-    def wrap_yml_syntax_error(error)
-      ::PerformanceMonitoring::PrometheusDashboard.new.errors.tap do |errors|
-        errors.add(:'YAML syntax', error.message)
-      end
+    def exhaustive_metrics_dashboard_validation
+      yaml = ::Gitlab::Config::Loader::Yaml.new(blob.data).load_raw!
+      Gitlab::Metrics::Dashboard::Validator
+        .errors(yaml, dashboard_path: blob.path, project: project)
+        .map(&:message)
+    rescue Gitlab::Config::Loader::FormatError => error
+      [error.message]
     end
   end
 end

app/models/performance_monitoring/prometheus_dashboard.rb

@@ -53,14 +53,23 @@ module PerformanceMonitoring
     # This method is planned to be refactored as a part of https://gitlab.com/gitlab-org/gitlab/-/issues/219398
     # implementation. For new existing logic was reused to faster deliver MVC
     def schema_validation_warnings
+      return run_custom_validation.map(&:message) if Feature.enabled?(:metrics_dashboard_exhaustive_validations, environment&.project)
+
       self.class.from_json(reload_schema)
-      nil
+      []
+    rescue Gitlab::Metrics::Dashboard::Errors::LayoutError => error
+      [error.message]
     rescue ActiveModel::ValidationError => exception
       exception.model.errors.map { |attr, error| "#{attr}: #{error}" }
     end
 
     private
 
+    def run_custom_validation
+      Gitlab::Metrics::Dashboard::Validator
+        .errors(reload_schema, dashboard_path: path, project: environment&.project)
+    end
+
     # dashboard finder methods are somehow limited, #find includes checking if
     # user is authorised to view selected dashboard, but modifies schema, which in some cases may
     # cause false positives returned from validation, and #find_raw does not authorise users

app/views/projects/blob/viewers/_metrics_dashboard_yml.html.haml

@@ -5,7 +5,7 @@
   = icon('warning fw')
   = _('Metrics Dashboard YAML definition is invalid:')
   %ul
-    - viewer.errors.messages.each do |error|
-      %li= error.join(': ')
+    - viewer.errors.each do |error|
+      %li= error
 
 = link_to _('Learn more'), help_page_path('operations/metrics/dashboards/index.md')

config/feature_flags/development/metrics_dashboard_exhaustive_validations.yml

+---
+name: metrics_dashboard_exhaustive_validations
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40103
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/241697
+group: group::apm
+type: development
+default_enabled: false
\ No newline at end of file

lib/gitlab/metrics/dashboard/validator.rb

@@ -4,24 +4,22 @@ module Gitlab
   module Metrics
     module Dashboard
       module Validator
-        DASHBOARD_SCHEMA_PATH = 'lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json'.freeze
+        DASHBOARD_SCHEMA_PATH = Rails.root.join(*%w[lib gitlab metrics dashboard validator schemas dashboard.json]).freeze
 
         class << self
           def validate(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil)
-            errors = _validate(content, schema_path, dashboard_path: dashboard_path, project: project)
-            errors.empty?
+            errors(content, schema_path, dashboard_path: dashboard_path, project: project).empty?
           end
 
           def validate!(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil)
-            errors = _validate(content, schema_path, dashboard_path: dashboard_path, project: project)
+            errors = errors(content, schema_path, dashboard_path: dashboard_path, project: project)
             errors.empty? || raise(errors.first)
           end
 
-          private
-
-          def _validate(content, schema_path, dashboard_path: nil, project: nil)
-            client = Validator::Client.new(content, schema_path, dashboard_path: dashboard_path, project: project)
-            client.execute
+          def errors(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil)
+            Validator::Client
+              .new(content, schema_path, dashboard_path: dashboard_path, project: project)
+              .execute
           end
         end
       end

lib/gitlab/metrics/dashboard/validator/client.rb

@@ -46,7 +46,7 @@ module Gitlab
 
           def validate_against_schema
             schemer.validate(content).map do |error|
-              Errors::SchemaValidationError.new(error)
+              ::Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError.new(error)
             end
           end
         end

lib/gitlab/metrics/dashboard/validator/schemas/panel.json

@@ -4,7 +4,7 @@
   "properties": {
     "type": {
       "type": "string",
-      "enum": ["area-chart", "anomaly-chart", "bar", "column", "stacked-column", "single-stat", "heatmap"],
+      "enum": ["area-chart", "line-chart", "anomaly-chart", "bar", "column", "stacked-column", "single-stat", "heatmap", "gauge"],
       "default": "area-chart"
     },
     "title": { "type": "string" },

spec/features/projects/blobs/blob_show_spec.rb

@@ -564,35 +564,76 @@ RSpec.describe 'File blob', :js do
         file_path: '.gitlab/dashboards/custom-dashboard.yml',
         file_content: file_content
       ).execute
-
-      visit_blob('.gitlab/dashboards/custom-dashboard.yml')
     end
 
-    context 'valid dashboard file' do
-      let(:file_content) { File.read(Rails.root.join('config/prometheus/common_metrics.yml')) }
+    context 'with metrics_dashboard_exhaustive_validations feature flag off' do
+      before do
+        stub_feature_flags(metrics_dashboard_exhaustive_validations: false)
+        visit_blob('.gitlab/dashboards/custom-dashboard.yml')
+      end
 
-      it 'displays an auxiliary viewer' do
-        aggregate_failures do
-          # shows that dashboard yaml is valid
-          expect(page).to have_content('Metrics Dashboard YAML definition is valid.')
+      context 'valid dashboard file' do
+        let(:file_content) { File.read(Rails.root.join('config/prometheus/common_metrics.yml')) }
+
+        it 'displays an auxiliary viewer' do
+          aggregate_failures do
+            # shows that dashboard yaml is valid
+            expect(page).to have_content('Metrics Dashboard YAML definition is valid.')
+
+            # shows a learn more link
+            expect(page).to have_link('Learn more')
+          end
+        end
+      end
+
+      context 'invalid dashboard file' do
+        let(:file_content) { "dashboard: 'invalid'" }
 
-          # shows a learn more link
-          expect(page).to have_link('Learn more')
+        it 'displays an auxiliary viewer' do
+          aggregate_failures do
+            # shows that dashboard yaml is invalid
+            expect(page).to have_content('Metrics Dashboard YAML definition is invalid:')
+            expect(page).to have_content("panel_groups: should be an array of panel_groups objects")
+
+            # shows a learn more link
+            expect(page).to have_link('Learn more')
+          end
         end
       end
     end
 
-    context 'invalid dashboard file' do
-      let(:file_content) { "dashboard: 'invalid'" }
+    context 'with metrics_dashboard_exhaustive_validations feature flag on' do
+      before do
+        stub_feature_flags(metrics_dashboard_exhaustive_validations: true)
+        visit_blob('.gitlab/dashboards/custom-dashboard.yml')
+      end
 
-      it 'displays an auxiliary viewer' do
-        aggregate_failures do
-          # shows that dashboard yaml is invalid
-          expect(page).to have_content('Metrics Dashboard YAML definition is invalid:')
-          expect(page).to have_content("panel_groups: should be an array of panel_groups objects")
+      context 'valid dashboard file' do
+        let(:file_content) { File.read(Rails.root.join('config/prometheus/common_metrics.yml')) }
 
-          # shows a learn more link
-          expect(page).to have_link('Learn more')
+        it 'displays an auxiliary viewer' do
+          aggregate_failures do
+            # shows that dashboard yaml is valid
+            expect(page).to have_content('Metrics Dashboard YAML definition is valid.')
+
+            # shows a learn more link
+            expect(page).to have_link('Learn more')
+          end
+        end
+      end
+
+      context 'invalid dashboard file' do
+        let(:file_content) { "dashboard: 'invalid'" }
+
+        it 'displays an auxiliary viewer' do
+          aggregate_failures do
+            # shows that dashboard yaml is invalid
+            expect(page).to have_content('Metrics Dashboard YAML definition is invalid:')
+            expect(page).to have_content("root is missing required keys: panel_groups")
+
+            # shows a learn more link
+            expect(page).to have_link('Learn more')
+          end
         end
       end
     end

spec/fixtures/lib/gitlab/metrics/dashboard/broken_yml_syntax.yml

+dashboard: 'Test Dashboard'
+ panel_groups:
+  - group: Group B
+    panels:
+      - title: "Super Chart B"
+        type: "area-chart"
+        y_label: "y_label"
+        metrics:
+          - id: metric_b
+            query_range: 'query'
+            unit: unit
+            label: Legend Label
+- group: Group A

spec/lib/gitlab/metrics/dashboard/validator/errors_spec.rb

@@ -34,6 +34,17 @@ RSpec.describe Gitlab::Metrics::Dashboard::Validator::Errors do
 
           it { is_expected.to eq 'root is missing required keys: one' }
         end
+
+        context 'when there is type mismatch' do
+          %w(null string boolean integer number array object).each do |expected_type|
+            context "on type: #{expected_type}" do
+              let(:type) { expected_type }
+              let(:details) { nil }
+
+              it { is_expected.to eq "'property_name' at root is not of type: #{expected_type}" }
+            end
+          end
+        end
       end
 
       context 'for nested object' do
@@ -52,8 +63,6 @@ RSpec.describe Gitlab::Metrics::Dashboard::Validator::Errors do
               let(:type) { expected_type }
               let(:details) { nil }
 
-              subject { described_class.new(error_hash).message }
-
               it { is_expected.to eq "'property_name' at /nested_objects/0 is not of type: #{expected_type}" }
             end
           end

spec/lib/gitlab/metrics/dashboard/validator_spec.rb

@@ -143,4 +143,56 @@ RSpec.describe Gitlab::Metrics::Dashboard::Validator do
       end
     end
   end
+
+  describe '#errors' do
+    context 'valid dashboard schema' do
+      it 'returns no errors' do
+        expect(described_class.errors(valid_dashboard)).to eq []
+      end
+
+      context 'with duplicate metric_ids' do
+        it 'returns errors' do
+          expect(described_class.errors(duplicate_id_dashboard)).to eq [Gitlab::Metrics::Dashboard::Validator::Errors::DuplicateMetricIds.new]
+        end
+      end
+
+      context 'with dashboard_path and project' do
+        subject { described_class.errors(valid_dashboard, dashboard_path: 'test/path.yml', project: project) }
+
+        context 'with no conflicting metric identifiers in db' do
+          it { is_expected.to eq [] }
+        end
+
+        context 'with metric identifier present in current dashboard' do
+          before do
+            create(:prometheus_metric,
+                   identifier:     'metric_a1',
+                   dashboard_path: 'test/path.yml',
+                   project:        project
+                  )
+          end
+
+          it { is_expected.to eq [] }
+        end
+
+        context 'with metric identifier present in another dashboard' do
+          before do
+            create(:prometheus_metric,
+                   identifier:     'metric_a1',
+                   dashboard_path: 'some/other/dashboard/path.yml',
+                   project:        project
+                  )
+          end
+
+          it { is_expected.to eq [Gitlab::Metrics::Dashboard::Validator::Errors::DuplicateMetricIds.new] }
+        end
+      end
+    end
+
+    context 'invalid dashboard schema' do
+      it 'returns collection of validation errors' do
+        expect(described_class.errors(invalid_dashboard)).to all be_kind_of(Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError)
+      end
+    end
+  end
 end

spec/models/blob_viewer/metrics_dashboard_yml_spec.rb

@@ -9,119 +9,228 @@ RSpec.describe BlobViewer::MetricsDashboardYml do
   let_it_be(:project) { create(:project, :repository) }
   let(:blob) { fake_blob(path: '.gitlab/dashboards/custom-dashboard.yml', data: data) }
   let(:sha) { sample_commit.id }
+  let(:data) { fixture_file('lib/gitlab/metrics/dashboard/sample_dashboard.yml') }
 
   subject(:viewer) { described_class.new(blob) }
 
-  context 'when the definition is valid' do
-    let(:data) { File.read(Rails.root.join('config/prometheus/common_metrics.yml')) }
+  context 'with metrics_dashboard_exhaustive_validations feature flag on' do
+    before do
+      stub_feature_flags(metrics_dashboard_exhaustive_validations: true)
+    end
+
+    context 'when the definition is valid' do
+      before do
+        allow(Gitlab::Metrics::Dashboard::Validator).to receive(:errors).and_return([])
+      end
+
+      describe '#valid?' do
+        it 'calls prepare! on the viewer' do
+          expect(viewer).to receive(:prepare!)
+
+          viewer.valid?
+        end
+
+        it 'processes dashboard yaml and returns true', :aggregate_failures do
+          yml = ::Gitlab::Config::Loader::Yaml.new(data).load_raw!
+
+          expect_next_instance_of(::Gitlab::Config::Loader::Yaml, data) do |loader|
+            expect(loader).to receive(:load_raw!).and_call_original
+          end
+          expect(Gitlab::Metrics::Dashboard::Validator)
+            .to receive(:errors)
+            .with(yml, dashboard_path: '.gitlab/dashboards/custom-dashboard.yml', project: project)
+            .and_call_original
+          expect(viewer.valid?).to be true
+        end
+      end
+
+      describe '#errors' do
+        it 'returns empty array' do
+          expect(viewer.errors).to eq []
+        end
+      end
+    end
+
+    context 'when definition is invalid' do
+      let(:error) { ::Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError.new }
+      let(:data) do
+        <<~YAML
+          dashboard:
+        YAML
+      end
+
+      before do
+        allow(Gitlab::Metrics::Dashboard::Validator).to receive(:errors).and_return([error])
+      end
 
-    describe '#valid?' do
-      it 'calls prepare! on the viewer' do
-        allow(PerformanceMonitoring::PrometheusDashboard).to receive(:from_json)
+      describe '#valid?' do
+        it 'returns false' do
+          expect(viewer.valid?).to be false
+        end
+      end
 
-        expect(viewer).to receive(:prepare!)
+      describe '#errors' do
+        it 'returns validation errors' do
+          expect(viewer.errors).to eq ["Dashboard failed schema validation"]
+        end
+      end
+    end
 
-        viewer.valid?
+    context 'when YAML syntax is invalid' do
+      let(:data) do
+        <<~YAML
+          dashboard: 'empty metrics'
+           panel_groups:
+          - group: 'Group Title'
+        YAML
       end
 
-      it 'returns true', :aggregate_failures do
-        yml = ::Gitlab::Config::Loader::Yaml.new(data).load_raw!
+      describe '#valid?' do
+        it 'returns false' do
+          expect(Gitlab::Metrics::Dashboard::Validator).not_to receive(:errors)
+          expect(viewer.valid?).to be false
+        end
+      end
 
-        expect_next_instance_of(::Gitlab::Config::Loader::Yaml, data) do |loader|
-          expect(loader).to receive(:load_raw!).and_call_original
+      describe '#errors' do
+        it 'returns validation errors' do
+          expect(viewer.errors).to all be_kind_of String
         end
-        expect(PerformanceMonitoring::PrometheusDashboard)
-          .to receive(:from_json)
-          .with(yml)
-          .and_call_original
-        expect(viewer.valid?).to be_truthy
       end
     end
 
-    describe '#errors' do
-      it 'returns nil' do
-        allow(PerformanceMonitoring::PrometheusDashboard).to receive(:from_json)
+    context 'when YAML loader raises error' do
+      let(:data) do
+        <<~YAML
+          large yaml file
+        YAML
+      end
+
+      before do
+        allow(::Gitlab::Config::Loader::Yaml).to receive(:new)
+          .and_raise(::Gitlab::Config::Loader::Yaml::DataTooLargeError, 'The parsed YAML is too big')
+      end
 
-        expect(viewer.errors).to be nil
+      it 'is invalid' do
+        expect(Gitlab::Metrics::Dashboard::Validator).not_to receive(:errors)
+        expect(viewer.valid?).to be false
+      end
+
+      it 'returns validation errors' do
+        expect(viewer.errors).to eq ['The parsed YAML is too big']
       end
     end
   end
 
-  context 'when definition is invalid' do
-    let(:error) { ActiveModel::ValidationError.new(PerformanceMonitoring::PrometheusDashboard.new.tap(&:validate)) }
-    let(:data) do
-      <<~YAML
-        dashboard:
-      YAML
+  context 'with metrics_dashboard_exhaustive_validations feature flag off' do
+    before do
+      stub_feature_flags(metrics_dashboard_exhaustive_validations: false)
     end
 
-    describe '#valid?' do
-      it 'returns false' do
-        expect(PerformanceMonitoring::PrometheusDashboard)
-          .to receive(:from_json).and_raise(error)
+    context 'when the definition is valid' do
+      describe '#valid?' do
+        before do
+          allow(PerformanceMonitoring::PrometheusDashboard).to receive(:from_json)
+        end
+
+        it 'calls prepare! on the viewer' do
+          expect(viewer).to receive(:prepare!)
 
-        expect(viewer.valid?).to be_falsey
+          viewer.valid?
+        end
+
+        it 'processes dashboard yaml and returns true', :aggregate_failures do
+          yml = ::Gitlab::Config::Loader::Yaml.new(data).load_raw!
+
+          expect_next_instance_of(::Gitlab::Config::Loader::Yaml, data) do |loader|
+            expect(loader).to receive(:load_raw!).and_call_original
+          end
+          expect(PerformanceMonitoring::PrometheusDashboard)
+            .to receive(:from_json)
+                  .with(yml)
+                  .and_call_original
+          expect(viewer.valid?).to be true
+        end
+      end
+
+      describe '#errors' do
+        it 'returns empty array' do
+          expect(viewer.errors).to eq []
+        end
       end
     end
 
-    describe '#errors' do
-      it 'returns validation errors' do
-        allow(PerformanceMonitoring::PrometheusDashboard)
-          .to receive(:from_json).and_raise(error)
+    context 'when definition is invalid' do
+      let(:error) { ActiveModel::ValidationError.new(PerformanceMonitoring::PrometheusDashboard.new.tap(&:validate)) }
+      let(:data) do
+        <<~YAML
+        dashboard:
+        YAML
+      end
+
+      describe '#valid?' do
+        it 'returns false' do
+          expect(PerformanceMonitoring::PrometheusDashboard)
+            .to receive(:from_json).and_raise(error)
 
-        expect(viewer.errors).to be error.model.errors
+          expect(viewer.valid?).to be false
+        end
+      end
+
+      describe '#errors' do
+        it 'returns validation errors' do
+          allow(PerformanceMonitoring::PrometheusDashboard)
+            .to receive(:from_json).and_raise(error)
+
+          expect(viewer.errors).to eq error.model.errors.messages.map { |messages| messages.join(': ') }
+        end
       end
     end
-  end
 
-  context 'when YAML syntax is invalid' do
-    let(:data) do
-      <<~YAML
+    context 'when YAML syntax is invalid' do
+      let(:data) do
+        <<~YAML
         dashboard: 'empty metrics'
          panel_groups:
         - group: 'Group Title'
-      YAML
-    end
-
-    describe '#valid?' do
-      it 'returns false' do
-        expect(PerformanceMonitoring::PrometheusDashboard).not_to receive(:from_json)
-        expect(viewer.valid?).to be_falsey
+        YAML
       end
-    end
 
-    describe '#errors' do
-      it 'returns validation errors' do
-        yaml_wrapped_errors = { 'YAML syntax': ["(<unknown>): did not find expected key while parsing a block mapping at line 1 column 1"] }
+      describe '#valid?' do
+        it 'returns false' do
+          expect(PerformanceMonitoring::PrometheusDashboard).not_to receive(:from_json)
+          expect(viewer.valid?).to be false
+        end
+      end
 
-        expect(viewer.errors).to be_kind_of ActiveModel::Errors
-        expect(viewer.errors.messages).to eql(yaml_wrapped_errors)
+      describe '#errors' do
+        it 'returns validation errors' do
+          expect(viewer.errors).to eq ["YAML syntax: (<unknown>): did not find expected key while parsing a block mapping at line 1 column 1"]
+        end
       end
     end
-  end
 
-  context 'when YAML loader raises error' do
-    let(:data) do
-      <<~YAML
+    context 'when YAML loader raises error' do
+      let(:data) do
+        <<~YAML
         large yaml file
-      YAML
-    end
-
-    before do
-      allow(::Gitlab::Config::Loader::Yaml).to receive(:new)
-        .and_raise(::Gitlab::Config::Loader::Yaml::DataTooLargeError, 'The parsed YAML is too big')
-    end
+        YAML
+      end
 
-    it 'is invalid' do
-      expect(PerformanceMonitoring::PrometheusDashboard).not_to receive(:from_json)
-      expect(viewer.valid?).to be(false)
-    end
+      before do
+        allow(::Gitlab::Config::Loader::Yaml).to(
+          receive(:new).and_raise(::Gitlab::Config::Loader::Yaml::DataTooLargeError, 'The parsed YAML is too big')
+        )
+      end
 
-    it 'returns validation errors' do
-      yaml_wrapped_errors = { 'YAML syntax': ["The parsed YAML is too big"] }
+      it 'is invalid' do
+        expect(PerformanceMonitoring::PrometheusDashboard).not_to receive(:from_json)
+        expect(viewer.valid?).to be false
+      end
 
-      expect(viewer.errors).to be_kind_of(ActiveModel::Errors)
-      expect(viewer.errors.messages).to eq(yaml_wrapped_errors)
+      it 'returns validation errors' do
+        expect(viewer.errors).to eq ["YAML syntax: The parsed YAML is too big"]
+      end
     end
   end
 end

spec/models/performance_monitoring/prometheus_dashboard_spec.rb

@@ -219,20 +219,93 @@ RSpec.describe PerformanceMonitoring::PrometheusDashboard do
   end
 
   describe '#schema_validation_warnings' do
-    context 'when schema is valid' do
-      it 'returns nil' do
-        expect(described_class).to receive(:from_json)
-        expect(described_class.new.schema_validation_warnings).to be_nil
+    let(:environment) { create(:environment, project: project) }
+    let(:path) { '.gitlab/dashboards/test.yml' }
+    let(:project) { create(:project, :repository, :custom_repo, files: { path => dashboard_schema.to_yaml }) }
+
+    subject(:schema_validation_warnings) { described_class.new(dashboard_schema.merge(path: path, environment: environment)).schema_validation_warnings }
+
+    before do
+      allow(Gitlab::Metrics::Dashboard::Finder).to receive(:find_raw).with(project, dashboard_path: path).and_call_original
+    end
+
+    context 'metrics_dashboard_exhaustive_validations is on' do
+      before do
+        stub_feature_flags(metrics_dashboard_exhaustive_validations: true)
+      end
+
+      context 'when schema is valid' do
+        let(:dashboard_schema) { YAML.safe_load(fixture_file('lib/gitlab/metrics/dashboard/sample_dashboard.yml')) }
+
+        it 'returns empty array' do
+          expect(Gitlab::Metrics::Dashboard::Validator).to receive(:errors).with(dashboard_schema, dashboard_path: path, project: project).and_return([])
+
+          expect(schema_validation_warnings).to eq []
+        end
+      end
+
+      context 'when schema is invalid' do
+        let(:dashboard_schema) { YAML.safe_load(fixture_file('lib/gitlab/metrics/dashboard/dashboard_missing_panel_groups.yml')) }
+
+        it 'returns array with errors messages' do
+          error = ::Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError.new
+
+          expect(Gitlab::Metrics::Dashboard::Validator).to receive(:errors).with(dashboard_schema, dashboard_path: path, project: project).and_return([error])
+
+          expect(schema_validation_warnings).to eq [error.message]
+        end
+      end
+
+      context 'when YAML has wrong syntax' do
+        let(:project) { create(:project, :repository, :custom_repo, files: { path => fixture_file('lib/gitlab/metrics/dashboard/broken_yml_syntax.yml') }) }
+
+        subject(:schema_validation_warnings) { described_class.new(path: path, environment: environment).schema_validation_warnings }
+
+        it 'returns array with errors messages' do
+          expect(Gitlab::Metrics::Dashboard::Validator).not_to receive(:errors)
+
+          expect(schema_validation_warnings).to eq ['Invalid yaml']
+        end
       end
     end
 
-    context 'when schema is invalid' do
-      it 'returns array with errors messages' do
-        instance = described_class.new
-        instance.errors.add(:test, 'test error')
+    context 'metrics_dashboard_exhaustive_validations is off' do
+      before do
+        stub_feature_flags(metrics_dashboard_exhaustive_validations: false)
+      end
+
+      context 'when schema is valid' do
+        let(:dashboard_schema) { YAML.safe_load(fixture_file('lib/gitlab/metrics/dashboard/sample_dashboard.yml')) }
+
+        it 'returns empty array' do
+          expect(described_class).to receive(:from_json).with(dashboard_schema)
+
+          expect(schema_validation_warnings).to eq []
+        end
+      end
+
+      context 'when schema is invalid' do
+        let(:dashboard_schema) { YAML.safe_load(fixture_file('lib/gitlab/metrics/dashboard/dashboard_missing_panel_groups.yml')) }
+
+        it 'returns array with errors messages' do
+          instance = described_class.new
+          instance.errors.add(:test, 'test error')
+
+          expect(described_class).to receive(:from_json).and_raise(ActiveModel::ValidationError.new(instance))
+          expect(described_class.new.schema_validation_warnings).to eq ['test: test error']
+        end
+      end
 
-        expect(described_class).to receive(:from_json).and_raise(ActiveModel::ValidationError.new(instance))
-        expect(described_class.new.schema_validation_warnings).to eq ['test: test error']
+      context 'when YAML has wrong syntax' do
+        let(:project) { create(:project, :repository, :custom_repo, files: { path => fixture_file('lib/gitlab/metrics/dashboard/broken_yml_syntax.yml') }) }
+
+        subject(:schema_validation_warnings) { described_class.new(path: path, environment: environment).schema_validation_warnings }
+
+        it 'returns array with errors messages' do
+          expect(described_class).not_to receive(:from_json)
+
+          expect(schema_validation_warnings).to eq ['Invalid yaml']
+        end
       end
     end
   end

spec/requests/api/graphql/metrics/dashboard_query_spec.rb

@@ -7,7 +7,7 @@ RSpec.describe 'Getting Metrics Dashboard' do
 
   let_it_be(:current_user) { create(:user) }
   let(:project) { create(:project) }
-  let!(:environment) { create(:environment, project: project) }
+  let(:environment) { create(:environment, project: project) }
 
   let(:query) do
     graphql_query_for(
@@ -25,73 +25,156 @@ RSpec.describe 'Getting Metrics Dashboard' do
     )
   end
 
-  context 'for anonymous user' do
+  context 'with metrics_dashboard_exhaustive_validations feature flag off' do
     before do
-      post_graphql(query, current_user: current_user)
+      stub_feature_flags(metrics_dashboard_exhaustive_validations: false)
     end
 
-    context 'requested dashboard is available' do
-      let(:path) { 'config/prometheus/common_metrics.yml' }
+    context 'for anonymous user' do
+      before do
+        post_graphql(query, current_user: current_user)
+      end
+
+      context 'requested dashboard is available' do
+        let(:path) { 'config/prometheus/common_metrics.yml' }
+
+        it_behaves_like 'a working graphql query'
+
+        it 'returns nil' do
+          dashboard = graphql_data.dig('project', 'environments', 'nodes')
+
+          expect(dashboard).to be_nil
+        end
+      end
+    end
+
+    context 'for user with developer access' do
+      before do
+        project.add_developer(current_user)
+        post_graphql(query, current_user: current_user)
+      end
+
+      context 'requested dashboard is available' do
+        let(:path) { 'config/prometheus/common_metrics.yml' }
+
+        it_behaves_like 'a working graphql query'
+
+        it 'returns metrics dashboard' do
+          dashboard = graphql_data.dig('project', 'environments', 'nodes', 0, 'metricsDashboard')
+
+          expect(dashboard).to eql("path" => path, "schemaValidationWarnings" => nil)
+        end
+
+        context 'invalid dashboard' do
+          let(:path) { '.gitlab/dashboards/metrics.yml' }
+          let(:project) { create(:project, :repository, :custom_repo, namespace: current_user.namespace, files: { path => "---\ndashboard: 'test'" }) }
+
+          it 'returns metrics dashboard' do
+            dashboard = graphql_data.dig('project', 'environments', 'nodes', 0, 'metricsDashboard')
+
+            expect(dashboard).to eql("path" => path, "schemaValidationWarnings" => ["panel_groups: should be an array of panel_groups objects"])
+          end
+        end
+
+        context 'empty dashboard' do
+          let(:path) { '.gitlab/dashboards/metrics.yml' }
+          let(:project) { create(:project, :repository, :custom_repo, namespace: current_user.namespace, files: { path => "" }) }
+
+          it 'returns metrics dashboard' do
+            dashboard = graphql_data.dig('project', 'environments', 'nodes', 0, 'metricsDashboard')
 
-      it_behaves_like 'a working graphql query'
+            expect(dashboard).to eql("path" => path, "schemaValidationWarnings" => ["dashboard: can't be blank", "panel_groups: should be an array of panel_groups objects"])
+          end
+        end
+      end
+
+      context 'requested dashboard can not be found' do
+        let(:path) { 'config/prometheus/i_am_not_here.yml' }
 
-      it 'returns nil' do
-        dashboard = graphql_data.dig('project', 'environments', 'nodes')
+        it_behaves_like 'a working graphql query'
 
-        expect(dashboard).to be_nil
+        it 'returns nil' do
+          dashboard = graphql_data.dig('project', 'environments', 'nodes', 0, 'metricsDashboard')
+
+          expect(dashboard).to be_nil
+        end
       end
     end
   end
 
-  context 'for user with developer access' do
+  context 'with metrics_dashboard_exhaustive_validations feature flag on' do
     before do
-      project.add_developer(current_user)
-      post_graphql(query, current_user: current_user)
+      stub_feature_flags(metrics_dashboard_exhaustive_validations: true)
     end
 
-    context 'requested dashboard is available' do
-      let(:path) { 'config/prometheus/common_metrics.yml' }
+    context 'for anonymous user' do
+      before do
+        post_graphql(query, current_user: current_user)
+      end
+
+      context 'requested dashboard is available' do
+        let(:path) { 'config/prometheus/common_metrics.yml' }
+
+        it_behaves_like 'a working graphql query'
 
-      it_behaves_like 'a working graphql query'
+        it 'returns nil' do
+          dashboard = graphql_data.dig('project', 'environments', 'nodes')
 
-      it 'returns metrics dashboard' do
-        dashboard = graphql_data.dig('project', 'environments', 'nodes')[0]['metricsDashboard']
+          expect(dashboard).to be_nil
+        end
+      end
+    end
 
-        expect(dashboard).to eql("path" => path, "schemaValidationWarnings" => nil)
+    context 'for user with developer access' do
+      before do
+        project.add_developer(current_user)
+        post_graphql(query, current_user: current_user)
       end
 
-      context 'invalid dashboard' do
-        let(:path) { '.gitlab/dashboards/metrics.yml' }
-        let(:project) { create(:project, :repository, :custom_repo, namespace: current_user.namespace, files: { path => "---\ndashboard: 'test'" }) }
+      context 'requested dashboard is available' do
+        let(:path) { 'config/prometheus/common_metrics.yml' }
+
+        it_behaves_like 'a working graphql query'
 
         it 'returns metrics dashboard' do
           dashboard = graphql_data.dig('project', 'environments', 'nodes', 0, 'metricsDashboard')
 
-          expect(dashboard).to eql("path" => path, "schemaValidationWarnings" => ["panel_groups: should be an array of panel_groups objects"])
+          expect(dashboard).to eql("path" => path, "schemaValidationWarnings" => nil)
         end
-      end
 
-      context 'empty dashboard' do
-        let(:path) { '.gitlab/dashboards/metrics.yml' }
-        let(:project) { create(:project, :repository, :custom_repo, namespace: current_user.namespace, files: { path => "" }) }
+        context 'invalid dashboard' do
+          let(:path) { '.gitlab/dashboards/metrics.yml' }
+          let(:project) { create(:project, :repository, :custom_repo, namespace: current_user.namespace, files: { path => "---\ndashboard: 'test'" }) }
 
-        it 'returns metrics dashboard' do
-          dashboard = graphql_data.dig('project', 'environments', 'nodes', 0, 'metricsDashboard')
+          it 'returns metrics dashboard' do
+            dashboard = graphql_data.dig('project', 'environments', 'nodes', 0, 'metricsDashboard')
 
-          expect(dashboard).to eql("path" => path, "schemaValidationWarnings" => ["dashboard: can't be blank", "panel_groups: should be an array of panel_groups objects"])
+            expect(dashboard).to eql("path" => path, "schemaValidationWarnings" => ["root is missing required keys: panel_groups"])
+          end
+        end
+
+        context 'empty dashboard' do
+          let(:path) { '.gitlab/dashboards/metrics.yml' }
+          let(:project) { create(:project, :repository, :custom_repo, namespace: current_user.namespace, files: { path => "" }) }
+
+          it 'returns metrics dashboard' do
+            dashboard = graphql_data.dig('project', 'environments', 'nodes', 0, 'metricsDashboard')
+
+            expect(dashboard).to eql("path" => path, "schemaValidationWarnings" => ["root is missing required keys: dashboard, panel_groups"])
+          end
         end
       end
-    end
 
-    context 'requested dashboard can not be found' do
-      let(:path) { 'config/prometheus/i_am_not_here.yml' }
+      context 'requested dashboard can not be found' do
+        let(:path) { 'config/prometheus/i_am_not_here.yml' }
 
-      it_behaves_like 'a working graphql query'
+        it_behaves_like 'a working graphql query'
 
-      it 'returns nil' do
-        dashboard = graphql_data.dig('project', 'environments', 'nodes')[0]['metricsDashboard']
+        it 'returns nil' do
+          dashboard = graphql_data.dig('project', 'environments', 'nodes', 0, 'metricsDashboard')
 
-        expect(dashboard).to be_nil
+          expect(dashboard).to be_nil
+        end
       end
     end
   end