Merge remote-tracking branch 'origin/master' into ce-to-ee-2018-01-16

CHANGELOG-EE.md

@@ -344,6 +344,10 @@ Please view this file on the master branch, on stable branches it's out of date.
 - Add group issue boards.
 - Ports style changes fixed in a conflict in ce to ee upstream to master for new projects page.
 
+## 9.5.10 (2017-11-08)
+
+- [SECURITY] Ensure GitLab Geo JSON web tokens expire after 2 minutes.
+
 ## 9.5.9 (2017-10-16)
 
 - [SECURITY] Prevent Related Issues from leaking confidential issues.

Gemfile

@@ -243,6 +243,9 @@ gem 'charlock_holmes', '~> 0.7.5'
 # Faster JSON
 gem 'oj', '~> 2.17.4'
 
+# Faster blank
+gem 'fast_blank'
+
 # Parse time & duration
 gem 'chronic', '~> 0.10.2'
 gem 'chronic_duration', '~> 0.10.6'

Gemfile.lock

@@ -231,6 +231,7 @@ GEM
     faraday_middleware-multi_json (0.0.6)
       faraday_middleware
       multi_json
+    fast_blank (1.0.0)
     fast_gettext (1.4.0)
     ffaker (2.4.0)
     ffi (1.9.18)
@@ -1069,6 +1070,7 @@ DEPENDENCIES
   factory_bot_rails (~> 4.8.2)
   faraday (~> 0.12)
   faraday_middleware-aws-signers-v4
+  fast_blank
   ffaker (~> 2.4)
   flay (~> 2.8.0)
   flipper (~> 0.11.0)

app/assets/javascripts/dispatcher.js

@@ -31,7 +31,6 @@ import MiniPipelineGraph from './mini_pipeline_graph_dropdown';
 import UserCallout from './user_callout';
 import ShortcutsWiki from './shortcuts_wiki';
 import BlobViewer from './blob/viewer/index';
-import AutoWidthDropdownSelect from './issuable/auto_width_dropdown_select';
 import UsersSelect from './users_select';
 import GfmAutoComplete from './gfm_auto_complete';
 import Star from './star';
@@ -289,7 +288,6 @@ import initLDAPGroupsSelect from 'ee/ldap_groups_select'; // eslint-disable-line
           new LabelsSelect();
           new MilestoneSelect();
           new IssuableTemplateSelectors();
-          new AutoWidthDropdownSelect($('.js-target-branch-select')).init();
 
           initApprovals();
           break;
@@ -624,6 +622,8 @@ import initLDAPGroupsSelect from 'ee/ldap_groups_select'; // eslint-disable-line
             .catch(fail);
           break;
         case 'projects:clusters:show':
+        case 'projects:clusters:update':
+        case 'projects:clusters:destroy':
           import(/* webpackChunkName: "clusters" */ './clusters/clusters_bundle')
             .then(cluster => new cluster.default()) // eslint-disable-line new-cap
             .catch((err) => {

app/assets/javascripts/issuable/auto_width_dropdown_select.js

@@ -11,6 +11,14 @@ class AutoWidthDropdownSelect {
     const dropdownClass = this.dropdownClass;
     this.$selectElement.select2({
       dropdownCssClass: dropdownClass,
+      ...AutoWidthDropdownSelect.selectOptions(this.dropdownClass),
+    });
+
+    return this;
+  }
+
+  static selectOptions(dropdownClass) {
+    return {
       dropdownCss() {
         let resultantWidth = 'auto';
         const $dropdown = $(`.${dropdownClass}`);
@@ -29,9 +37,7 @@ class AutoWidthDropdownSelect {
           maxWidth: offsetParentWidth,
         };
       },
-    });
-
-    return this;
+    };
   }
 }
 

app/assets/javascripts/issuable_form.js

@@ -6,6 +6,7 @@ import Autosave from './autosave';
 import UsersSelect from './users_select';
 import GfmAutoComplete from './gfm_auto_complete';
 import ZenMode from './zen_mode';
+import AutoWidthDropdownSelect from './issuable/auto_width_dropdown_select';
 import { parsePikadayDate, pikadayToString } from './lib/utils/datefix';
 import groupsSelect from './groups_select';
 
@@ -48,6 +49,12 @@ export default class IssuableForm {
       });
       calendar.setDate(parsePikadayDate($issuableDueDate.val()));
     }
+
+    this.$targetBranchSelect = $('.js-target-branch-select', this.form);
+
+    if (this.$targetBranchSelect.length) {
+      this.initTargetBranchDropdown();
+    }
   }
 
   initAutosave() {
@@ -106,4 +113,37 @@ export default class IssuableForm {
   addWip() {
     this.titleField.val(`WIP: ${(this.titleField.val())}`);
   }
+
+  initTargetBranchDropdown() {
+    this.$targetBranchSelect.select2({
+      ...AutoWidthDropdownSelect.selectOptions('js-target-branch-select'),
+      ajax: {
+        url: this.$targetBranchSelect.data('endpoint'),
+        dataType: 'JSON',
+        quietMillis: 250,
+        data(search) {
+          return {
+            search,
+          };
+        },
+        results(data) {
+          return {
+            // `data` keys are translated so we can't just access them with a string based key
+            results: data[Object.keys(data)[0]].map(name => ({
+              id: name,
+              text: name,
+            })),
+          };
+        },
+      },
+      initSelection(el, callback) {
+        const val = el.val();
+
+        callback({
+          id: val,
+          text: val,
+        });
+      },
+    });
+  }
 }

app/views/projects/merge_requests/creations/_new_submit.html.haml

@@ -15,7 +15,7 @@
   = f.hidden_field :source_project_id
   = f.hidden_field :source_branch
   = f.hidden_field :target_project_id
-  = f.hidden_field :target_branch
+  = f.hidden_field :target_branch, id: ''
 
 .mr-compare.merge-request.js-merge-request-new-submit{ 'data-mr-submit-action': "#{j params[:tab].presence || 'new'}" }
   - if @commits.empty?

app/views/shared/issuable/form/_branch_chooser.html.haml

@@ -15,11 +15,10 @@
   = form.label :target_branch, class: 'control-label'
   .col-sm-10.target-branch-select-dropdown-container
     .issuable-form-select-holder
-      = form.select(:target_branch, issuable.target_branches,
-        { include_blank: true },
+      = form.hidden_field(:target_branch,
         { class: 'target_branch js-target-branch-select ref-name',
           disabled: issuable.new_record?,
-          data: { placeholder: "Select branch" }})
+          data: { placeholder: "Select branch", endpoint: refs_project_path(@project, sort: 'updated_desc', find: 'branches') }})
     - if issuable.new_record?
        
       = link_to 'Change branches', mr_change_branches_path(issuable)

changelogs/unreleased-ee/4378-fix-cluster-js-not-running-on-update-page.yml

+---
+title: Fix JavaScript bundle running on Cluster update/destroy pages
+merge_request: 4112
+author:
+type: fixed

changelogs/unreleased-ee/4395-geo-disaster-recovery.yml

+---
+title: Geo - Add a rake task to update Geo primary node URL
+merge_request: 4097
+author:
+type: fixed

changelogs/unreleased-ee/tc-geo-fdw-configurable.yml

+---
+title: Make it possible to enable/disable PostgreSQL FDW for Geo
+merge_request: 4020
+author:
+type: added

changelogs/unreleased-ee/tc-geo-nodes-more-api.yml

+---
+title: Add more endpoints for Geo Nodes API
+merge_request: 3923
+author:
+type: added

changelogs/unreleased/16468-add-fast-blank.yml

+---
+title: "Add fast-blank"
+merge_request: 16468
+author:
+type: performance

changelogs/unreleased/merge-request-target-branch-perf.yml

+---
+title: Improve performance of target branch dropdown
+merge_request:
+author:
+type: performance

config/database_geo.yml.postgresql

@@ -9,6 +9,7 @@ production:
   username: git
   password: "secure password"
   host: localhost
+  fdw: true
 
 #
 # Development specific
@@ -21,6 +22,7 @@ development:
   username: postgres
   password: "secure password"
   host: localhost
+  fdw: true
 
 #
 # Staging specific
@@ -33,6 +35,7 @@ staging:
   username: git
   password: "secure password"
   host: localhost
+  fdw: true
 
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
@@ -45,3 +48,4 @@ test: &test
   username: postgres
   password:
   host: localhost
+  fdw: true

doc/api/geo_nodes.md

@@ -65,6 +65,60 @@ Example response:
 }
 ```
 
+## Edit a Geo node
+
+Updates an existing Geo secondary node. The primary node cannot be edited.
+
+```
+PUT /geo_nodes/:id
+```
+
+| Attribute            | Type    | Required | Description                                                               |
+|----------------------|---------|----------|---------------------------------------------------------------------------|
+| `id`                 | integer | yes      | The ID of the Geo node.                                                   |
+| `enabled`            | boolean | no       | Flag indicating if the Geo node is enabled.                               |
+| `url`                | string  | no       | The URL to connect to the Geo node.                                       |
+| `files_max_capacity` | integer | no       | Control the maximum concurrency of LFS/attachment backfill for this node. |
+| `repos_max_capacity` | integer | no       | Control the maximum concurrency of repository backfill for this node.     |
+
+Example response:
+
+```json
+{
+  "id": 1,
+  "url": "https://primary.example.com/",
+  "primary": true,
+  "enabled": true,
+  "current": true,
+  "files_max_capacity": 10,
+  "repos_max_capacity": 25,
+  "clone_protocol": "http"
+}
+```
+
+## Repair a Geo node
+
+To repair the OAuth authentication of a Geo node.
+
+```
+PUT /geo_nodes/:id/repair
+```
+
+Example response:
+
+```json
+{
+  "id": 1,
+  "url": "https://primary.example.com/",
+  "primary": true,
+  "enabled": true,
+  "current": true,
+  "files_max_capacity": 10,
+  "repos_max_capacity": 25,
+  "clone_protocol": "http"
+}
+```
+
 ## Retrieve status about all secondary Geo nodes
 
 ```

doc/gitlab-geo/database.md

@@ -39,9 +39,9 @@ recover. See below for more details.
 
 The following guide assumes that:
 
-- You are using Omnibus and therefore you are using PostgreSQL 9.6 or later 
-  which includes the  [`pg_basebackup` tool][pgback] and improved 
-  [Foreign Data Wrapper][FDW] support. 
+- You are using Omnibus and therefore you are using PostgreSQL 9.6 or later
+  which includes the  [`pg_basebackup` tool][pgback] and improved
+  [Foreign Data Wrapper][FDW] support.
 - You have a primary node already set up (the GitLab server you are
   replicating from), running Omnibus' PostgreSQL (or equivalent version), and
   you have a new secondary server set up with the same versions of the OS,
@@ -67,22 +67,22 @@ The following guide assumes that:
     ```
 
     This command will use your defined `external_url` in `/etc/gitlab/gitlab.rb`.
-    
+
 1. GitLab 10.4 and up only: Do the following to make sure the `gitlab` database user has a password defined
 
     Generate a MD5 hash of the desired password:
-   
+
     ```bash
     gitlab-ctl pg-password-md5 gitlab
     # Enter password: mypassword
     # Confirm password: mypassword
     # fca0b89a972d69f00eb3ec98a5838484
     ```
-   
+
     Edit `/etc/gitlab/gitlab.rb`:
-   
+
     ```ruby
-    # Fill with the hash generated by `gitlab-ctl pg-password-md5 gitlab` 
+    # Fill with the hash generated by `gitlab-ctl pg-password-md5 gitlab`
     postgresql['sql_user_password'] = 'fca0b89a972d69f00eb3ec98a5838484'
 
     # If you have HA setup, this must be present in all nodes as well
@@ -304,25 +304,25 @@ because we have not yet configured the secondary server. This is the next step.
     connections. The certificate can only be replicated by someone with access
     to the private key, which is **only** present on the primary node.
 
-1. Configure PostgreSQL to listen on network interfaces on secondary
+1. Configure PostreSQL to enable FDW support
 
     This step is similar to how we configured the primary instance.
-    We need to enable this, even if using a single node, to enable FDW support.
-    
+    We need to enable this, to enable FDW support, even if using a single node.
+
     Edit `/etc/gitlab/gitlab.rb` and add the following, replacing the IP
     addresses with addresses appropriate to your network configuration:
-    
+
     ```ruby
-    geo_primary_role['enable'] = true
-    
     # Secondary addresses
-    # - replace '5.6.7.8' with the secondary public address
+    # - replace '5.6.7.8' with the secondary private address
     postgresql['listen_address'] = '5.6.7.8'
     postgresql['trust_auth_cidr_addresses'] = ['127.0.0.1/32','5.6.7.8/32']
-    postgresql['md5_auth_cidr_addresses'] = ['5.6.7.8/32']
-    
+
     # gitlab database user's password (defined previously)
     gitlab_rails['db_password'] = 'mypassword'
+
+    # enable fdw for the geo tracking database
+    geo_secondary['db_fdw'] = true
     ```
 
 1. Test that the `gitlab-psql` user can connect to the primary's database:
@@ -477,4 +477,5 @@ Read the [troubleshooting document](troubleshooting.md).
 [pgback]: http://www.postgresql.org/docs/9.2/static/app-pgbasebackup.html
 [external postgresql]: #external-postgresql-instances
 [tracking]: database_source.md#enable-tracking-database-on-the-secondary-server
+[FDW]: https://www.postgresql.org/docs/9.6/static/postgres-fdw.html
 [toc]: README.md#using-omnibus-gitlab

doc/gitlab-geo/database_source.md

@@ -33,9 +33,8 @@ recover. See below for more details.
 
 The following guide assumes that:
 
-- You are using PostgreSQL 9.6 or later
-  which includes the
-  [`pg_basebackup` tool][pgback] and improved [Foreign Data Wrapper][FDW] support. 
+- You are using PostgreSQL 9.6 or later which includes the
+  [`pg_basebackup` tool][pgback] and improved [Foreign Data Wrapper][FDW] support.
 - You have a primary node already set up (the GitLab server you are
   replicating from), running PostgreSQL 9.6 or later, and
   you have a new secondary server set up with the same versions of the OS,
@@ -291,6 +290,9 @@ node.
     sudo -u postgres psql -h $GEO_DB_HOST -d $GEO_DB_NAME -p $GEO_DB_PORT -c "GRANT USAGE ON FOREIGN SERVER gitlab_secondary TO $(GEO_DB_USER);"
     ```
 
+    And edit the content of `database_geo.yml` and to add `fdw: true` to
+    the  `production:` block.
+
 ### Step 4. Initiate the replication process
 
 Below we provide a script that connects the database on the secondary node to

doc/gitlab-geo/disaster-recovery.md

@@ -9,7 +9,7 @@ fail-over with minimal effort, in a disaster situation.
 
 See [current limitations](README.md#current-limitations) for more information.
 
-## Promoting a secondary geo replica
+### Step 1. Promoting a secondary geo replica
 
 > **Warning:** Disaster Recovery does not yet support systems with multiple
 > secondary geo replicas (e.g. one primary and two or more secondaries).
@@ -42,20 +42,6 @@ It does not enable GitLab Geo on the newly promoted primary.
     sudo -i
     ```
 
-1. Optional: Update the primary domain's DNS record.
-    
-    Updating the DNS records for the primary domain to point to the secondary
-    will prevent the need to update all references to the primary domain to the
-    secondary domain, like changing Git remotes and API URLs.
-
-    After updating the primary domain's DNS records to point to the secondary,
-    edit `/etc/gitlab/gitlab.rb` on the the secondary to reflect the new URL:
-
-    ```
-    # Change the existing external_url configuration
-    external_url 'https://gitlab.example.com'
-    ```
-
 1. Edit `/etc/gitlab/gitlab.rb` to reflect its new status as primary.
 
     Remove the following line:
@@ -79,10 +65,49 @@ It does not enable GitLab Geo on the newly promoted primary.
    previously for the secondary.
 1. Success! The secondary has now been promoted to primary.
 
+### Step 2. (Optional) Updating the primary domain's DNS record
+
+Updating the DNS records for the primary domain to point to the secondary
+will prevent the need to update all references to the primary domain to the
+secondary domain, like changing Git remotes and API URLs.
+
+1. SSH in to your **secondary** and login as root:
+
+    ```
+    sudo -i
+    ```
+
+1. Update the primary domain's DNS record.
+
+    After updating the primary domain's DNS records to point to the secondary,
+    edit `/etc/gitlab/gitlab.rb` on the the secondary to reflect the new URL:
+
+    ```
+    # Change the existing external_url configuration
+    external_url 'https://gitlab.example.com'
+    ```
+
+1. Reconfigure the secondary node for the change to take effect:
+
+    ```
+    gitlab-ctl reconfigure
+    ```
+
+1. Execute the command below to update the newly promoted primary node URL:
+
+    ```
+    gitlab-rake geo:update_primary_node_url
+    ```
+
+    This command will use the changed `external_url` configuration defined
+    in `/etc/gitlab/gitlab.rb`.
+
+1. Verify you can connect to the newly promoted primary using the primary URL.
+
     If you updated the DNS records for the primary domain, these changes may
     not have yet propagated depending on the previous DNS records TTL.
 
-## Add secondary geo replicas to a promoted primary
+### Step 3. (Optional) Add secondary geo replicas to a promoted primary
 
 Promoting a secondary to primary using the process above does not enable
 GitLab Geo on the new primary.

doc/workflow/issue_weight.md

 # Issue Weight
 
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/76)
+in [GitLab Enterprise Edition Starter](https://about.gitlab.com/gitlab-ee/) 8.3.
+
 When you have a lot of issues, it can be hard to get an overview.
 By adding a weight to each issue, you can get a better idea of how much time,
 value or complexity a given issue has or will cost.

doc/workflow/repository_mirroring.md

@@ -309,6 +309,23 @@ It may be possible to implement a locking mechanism using the server-side
 `pre-receive` hook to prevent the race condition. Read about [configuring
 custom Git hooks][hooks] on the GitLab server.
 
+### Mirroring with Perforce via GitFusion
+
+> **Warning:** Bidirectional mirroring should not be used as a permanent
+> configuration. There is no bidirectional mirroring without conflicts.
+> Refer to [Migrating from Perforce Helix][perforce] for alternative migration
+> approaches.
+
+GitFusion provides a Git interface to Perforce which can be used by GitLab to
+bidirectionally mirror projects with GitLab. This may be useful in some
+situations when migrating from Perforce to GitLab where overlapping Perforce
+workspaces cannot be migrated simultaneously to GitLab.
+
+If using mirroring with Perforce you should only mirror protected branches.
+Perforce will reject any pushes that rewrite history. It is recommended that
+only the fewest number of branches are mirrored due to the performance
+limitations of GitFusion.
+
 [ee-51]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/51
 [ee-2551]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/2551
 [ee-3117]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3117
@@ -320,3 +337,4 @@ custom Git hooks][hooks] on the GitLab server.
 [deploy-key]: ../ssh/README.md#deploy-keys
 [webhook]: ../user/project/integrations/webhooks.html#push-events
 [pull-api]: ../api/projects.html#start-the-pull-mirroring-process-for-a-project
+[perforce]: ../user/project/import/perforce.html

ee/app/controllers/admin/geo_nodes_controller.rb

@@ -44,9 +44,9 @@ class Admin::GeoNodesController < Admin::ApplicationController
   end
 
   def repair
-    if @node.primary? || !@node.missing_oauth_application?
+    if !@node.missing_oauth_application?
       flash[:notice] = "This node doesn't need to be repaired."
-    elsif @node.save
+    elsif @node.repair
       flash[:notice] = 'Node Authentication was successfully repaired.'
     else
       flash[:alert] = 'There was a problem repairing Node Authentication.'

ee/app/models/geo_node.rb

@@ -26,6 +26,8 @@ class GeoNode < ActiveRecord::Base
 
   before_validation :ensure_access_keys!
 
+  alias_method :repair, :save # the `update_dependents_attributes` hook will take care of it
+
   scope :with_url_prefix, ->(prefix) { where('url LIKE ?', "#{prefix}%") }
 
   attr_encrypted :secret_access_key,

ee/app/serializers/geo_node_status_entity.rb

-class GeoNodeStatusEntity < Grape::Entity
-  include ActionView::Helpers::NumberHelper
-
-  expose :geo_node_id
-
-  expose :healthy?, as: :healthy
-  expose :health do |node|
-    node.healthy? ? 'Healthy' : node.health
-  end
-  expose :health_status
-  expose :missing_oauth_application, as: :missing_oauth_application
-
-  expose :attachments_count
-  expose :attachments_synced_count
-  expose :attachments_failed_count
-  expose :attachments_synced_in_percentage do |node|
-    number_to_percentage(node.attachments_synced_in_percentage, precision: 2)
-  end
-
-  expose :db_replication_lag_seconds
-
-  expose :lfs_objects_count
-  expose :lfs_objects_synced_count
-  expose :lfs_objects_failed_count
-  expose :lfs_objects_synced_in_percentage do |node|
-    number_to_percentage(node.lfs_objects_synced_in_percentage, precision: 2)
-  end
-
-  expose :job_artifacts_count
-  expose :job_artifacts_synced_count
-  expose :job_artifacts_failed_count
-  expose :job_artifacts_synced_in_percentage do |node|
-    number_to_percentage(node.job_artifacts_synced_in_percentage, precision: 2)
-  end
-
-  expose :repositories_count
-  expose :repositories_failed_count
-  expose :repositories_synced_count
-  expose :repositories_synced_in_percentage do |node|
-    number_to_percentage(node.repositories_synced_in_percentage, precision: 2)
-  end
-
-  expose :wikis_count
-  expose :wikis_failed_count
-  expose :wikis_synced_count
-  expose :wikis_synced_in_percentage do |node|
-    number_to_percentage(node.wikis_synced_in_percentage, precision: 2)
-  end
-
-  expose :replication_slots_count
-  expose :replication_slots_used_count
-  expose :replication_slots_used_in_percentage do |node|
-    number_to_percentage(node.replication_slots_used_in_percentage, precision: 2)
-  end
-  expose :replication_slots_max_retained_wal_bytes
-
-  expose :last_event_id
-  expose :last_event_timestamp
-  expose :cursor_last_event_id
-  expose :cursor_last_event_timestamp
-
-  expose :last_successful_status_check_timestamp
-
-  expose :version
-  expose :revision
-
-  expose :namespaces, using: NamespaceEntity
-
-  # We load GeoNodeStatus data in two ways:
-  #
-  # 1. Directly by asking a Geo node via an API call
-  # 2. Via cached state in the database
-  #
-  # We don't yet cached the state of the shard information in the database, so if
-  # we don't have this information omit from the serialization entirely.
-  expose :storage_shards, using: StorageShardEntity, if: ->(status, options) do
-    status.storage_shards.present?
-  end
-
-  expose :storage_shards_match?, as: :storage_shards_match, if: -> (status, options) do
-    Gitlab::Geo.primary? && status.storage_shards.present?
-  end
-
-  private
-
-  def namespaces
-    object.geo_node.namespaces
-  end
-
-  def missing_oauth_application
-    object.geo_node.missing_oauth_application?
-  end
-
-  def version
-    Gitlab::VERSION
-  end
-
-  def revision
-    Gitlab::REVISION
-  end
-end

ee/app/serializers/geo_node_status_serializer.rb

 class GeoNodeStatusSerializer < BaseSerializer
-  entity GeoNodeStatusEntity
+  entity API::Entities::GeoNodeStatus
 end

ee/lib/gitlab/geo.rb

@@ -65,14 +65,22 @@ module Gitlab
       ::License.feature_available?(:geo)
     end
 
-    def self.fdw?
-      self.cache_value(:geo_fdw?) do
-        ::Geo::BaseRegistry.connection.execute(
+    def self.fdw_capable?
+      self.cache_value(:geo_fdw_capable?) do
+        ::Geo::TrackingBase.connection.execute(
           "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '#{FDW_SCHEMA}' AND table_type = 'FOREIGN TABLE'"
         ).first.fetch('count').to_i.positive?
       end
     end
 
+    def self.fdw?
+      return false unless self.fdw_capable?
+
+      # FDW is enabled by default, disable it by setting `fdw: false` in config/database_geo.yml
+      value = Rails.configuration.geo_database['fdw']
+      value.nil? ? true : value
+    end
+
     def self.fdw_table(table_name)
       FDW_SCHEMA + ".#{table_name}"
     end

ee/lib/gitlab/geo/geo_tasks.rb

@@ -15,6 +15,26 @@ module Gitlab
         end
       end
 
+      def update_primary_geo_node_url
+        node = Gitlab::Geo.primary_node
+
+        unless node.present?
+          $stdout.puts 'This is not a primary node'.color(:red)
+          exit 1
+        end
+
+        $stdout.puts "Updating primary Geo node with URL #{node.url} ..."
+
+        if node.update(url: GeoNode.current_node_url)
+          puts "#{node.url} is now the primary Geo node URL".color(:green)
+          $stdout.puts "#{node.url} is now the primary Geo node URL".color(:green)
+        else
+          puts "Error saving Geo node:\n#{node.errors.full_messages.join("\n")}".color(:red)
+          $stdout.puts "Error saving Geo node:\n#{node.errors.full_messages.join("\n")}".color(:red)
+          exit 1
+        end
+      end
+
       def refresh_foreign_tables!
         sql = <<~SQL
             DROP SCHEMA IF EXISTS gitlab_secondary CASCADE;

lib/api/entities.rb

@@ -1130,6 +1130,8 @@ module API
     end
 
     class GeoNode < Grape::Entity
+      include ::API::Helpers::RelatedResourcesHelpers
+
       expose :id
       expose :url
       expose :primary?, as: :primary
@@ -1142,6 +1144,129 @@ module API
       expose :clone_protocol do |_record, _options|
         'http'
       end
+
+      expose :_links do
+        expose :self do |geo_node|
+          expose_url api_v4_geo_nodes_path(id: geo_node.id)
+        end
+
+        expose :repair do |geo_node|
+          expose_url api_v4_geo_nodes_repair_path(id: geo_node.id)
+        end
+      end
+    end
+
+    class GeoNodeStatus < Grape::Entity
+      include ::API::Helpers::RelatedResourcesHelpers
+      include ActionView::Helpers::NumberHelper
+
+      expose :geo_node_id
+
+      expose :healthy?, as: :healthy
+      expose :health do |node|
+        node.healthy? ? 'Healthy' : node.health
+      end
+      expose :health_status
+      expose :missing_oauth_application
+
+      expose :attachments_count
+      expose :attachments_synced_count
+      expose :attachments_failed_count
+      expose :attachments_synced_in_percentage do |node|
+        number_to_percentage(node.attachments_synced_in_percentage, precision: 2)
+      end
+
+      expose :db_replication_lag_seconds
+
+      expose :lfs_objects_count
+      expose :lfs_objects_synced_count
+      expose :lfs_objects_failed_count
+      expose :lfs_objects_synced_in_percentage do |node|
+        number_to_percentage(node.lfs_objects_synced_in_percentage, precision: 2)
+      end
+
+      expose :job_artifacts_count
+      expose :job_artifacts_synced_count
+      expose :job_artifacts_failed_count
+      expose :job_artifacts_synced_in_percentage do |node|
+        number_to_percentage(node.job_artifacts_synced_in_percentage, precision: 2)
+      end
+
+      expose :repositories_count
+      expose :repositories_failed_count
+      expose :repositories_synced_count
+      expose :repositories_synced_in_percentage do |node|
+        number_to_percentage(node.repositories_synced_in_percentage, precision: 2)
+      end
+
+      expose :wikis_count
+      expose :wikis_failed_count
+      expose :wikis_synced_count
+      expose :wikis_synced_in_percentage do |node|
+        number_to_percentage(node.wikis_synced_in_percentage, precision: 2)
+      end
+
+      expose :replication_slots_count
+      expose :replication_slots_used_count
+      expose :replication_slots_used_in_percentage do |node|
+        number_to_percentage(node.replication_slots_used_in_percentage, precision: 2)
+      end
+      expose :replication_slots_max_retained_wal_bytes
+
+      expose :last_event_id
+      expose :last_event_timestamp
+      expose :cursor_last_event_id
+      expose :cursor_last_event_timestamp
+
+      expose :last_successful_status_check_timestamp
+
+      expose :version
+      expose :revision
+
+      expose :namespaces, using: NamespaceBasic
+
+      # We load GeoNodeStatus data in two ways:
+      #
+      # 1. Directly by asking a Geo node via an API call
+      # 2. Via cached state in the database
+      #
+      # We don't yet cached the state of the shard information in the database, so if
+      # we don't have this information omit from the serialization entirely.
+      expose :storage_shards, using: StorageShardEntity, if: ->(status, options) do
+        status.storage_shards.present?
+      end
+
+      expose :storage_shards_match?, as: :storage_shards_match, if: -> (status, options) do
+        Gitlab::Geo.primary? && status.storage_shards.present?
+      end
+
+      expose :_links do
+        expose :self do |geo_node_status|
+          expose_url api_v4_geo_nodes_status_path(id: geo_node_status.geo_node_id)
+        end
+
+        expose :node do |geo_node_status|
+          expose_url api_v4_geo_nodes_path(id: geo_node_status.geo_node_id)
+        end
+      end
+
+      private
+
+      def namespaces
+        object.geo_node.namespaces
+      end
+
+      def missing_oauth_application
+        object.geo_node.missing_oauth_application?
+      end
+
+      def version
+        Gitlab::VERSION
+      end
+
+      def revision
+        Gitlab::REVISION
+      end
     end
 
     class PersonalAccessToken < Grape::Entity

lib/api/geo.rb

@@ -36,7 +36,7 @@ module API
         authenticate_by_gitlab_geo_node_token!
 
         status = ::GeoNodeStatus.current_node_status
-        present status, with: GeoNodeStatusEntity
+        present status, with: Entities::GeoNodeStatus
       end
     end
 

lib/api/geo_nodes.rb

@@ -2,6 +2,7 @@ module API
   class GeoNodes < Grape::API
     include PaginationParams
     include APIGuard
+    include ::Gitlab::Utils::StrongMemoize
 
     before { authenticated_as_admin! }
 
@@ -25,12 +26,12 @@ module API
       # Example request:
       #   GET /geo_nodes/status
       desc 'Get status for all Geo nodes' do
-        success GeoNodeStatusEntity
+        success Entities::GeoNodeStatus
       end
       get '/status' do
         status = GeoNodeStatus.all
 
-        present paginate(status), with: GeoNodeStatusEntity
+        present paginate(status), with: Entities::GeoNodeStatus
       end
 
       # Get project registry failures for the current Geo node
@@ -55,49 +56,95 @@ module API
         present project_registries, with: ::GeoProjectRegistryEntity
       end
 
-      # Get all Geo node information
-      #
-      # Example request:
-      #   GET /geo_nodes/:id
-      desc 'Get a single GeoNode' do
-        success Entities::GeoNode
-      end
-      params do
-        requires :id, type: Integer, desc: 'The ID of the node'
-      end
-      get ':id' do
-        node = GeoNode.find_by(id: params[:id])
-
-        not_found!('GeoNode') unless node
-
-        present node, with: Entities::GeoNode
-      end
-
-      # Get Geo metrics for a single node
-      #
-      # Example request:
-      #   GET /geo_nodes/:id/status
-      desc 'Get metrics for a single Geo node' do
-        success Entities::GeoNode
-      end
-      params do
-        requires :id, type: Integer, desc: 'The ID of the node'
-      end
-      get ':id/status' do
-        geo_node = GeoNode.find(params[:id])
-
-        not_found('Geo node not found') unless geo_node
+      route_param :id, type: Integer, desc: 'The ID of the node' do
+        helpers do
+          def geo_node
+            strong_memoize(:geo_node) { GeoNode.find(params[:id]) }
+          end
 
-        status =
-          if geo_node.current?
-            GeoNodeStatus.current_node_status
+          def geo_node_status
+            strong_memoize(:geo_node_status) do
+              if geo_node.current?
+                GeoNodeStatus.current_node_status
+              else
+                geo_node.status
+              end
+            end
+          end
+        end
+
+        # Get all Geo node information
+        #
+        # Example request:
+        #   GET /geo_nodes/:id
+        desc 'Get a single GeoNode' do
+          success Entities::GeoNode
+        end
+        get do
+          not_found!('GeoNode') unless geo_node
+
+          present geo_node, with: Entities::GeoNode
+        end
+
+        # Get Geo metrics for a single node
+        #
+        # Example request:
+        #   GET /geo_nodes/:id/status
+        desc 'Get metrics for a single Geo node' do
+          success Entities::GeoNodeStatus
+        end
+        get 'status' do
+          not_found!('GeoNode') unless geo_node
+
+          not_found!('Status for Geo node not found') unless geo_node_status
+
+          present geo_node_status, with: Entities::GeoNodeStatus
+        end
+
+        # Repair authentication of the Geo node
+        #
+        # Example request:
+        #   POST /geo_nodes/:id/repair
+        desc 'Repair authentication of the Geo node' do
+          success Entities::GeoNodeStatus
+        end
+        post 'repair' do
+          not_found!('GeoNode') unless geo_node
+
+          if !geo_node.missing_oauth_application? || geo_node.repair
+            status 200
+            present geo_node_status, with: Entities::GeoNodeStatus
           else
-            geo_node.status
+            render_validation_error!(geo_node)
           end
-
-        not_found!('Status for Geo node not found') unless status
-
-        present status, with: ::GeoNodeStatusEntity
+        end
+
+        # Edit an existing Geo node
+        #
+        # Example request:
+        #   PUT /geo_nodes/:id
+        desc 'Edit an existing Geo secondary node' do
+          success Entities::GeoNode
+        end
+        params do
+          optional :enabled, type: Boolean, desc: 'Flag indicating if the Geo node is enabled'
+          optional :url, type: String, desc: 'The URL to connect to the Geo node'
+          optional :files_max_capacity, type: Integer, desc: 'Control the maximum concurrency of LFS/attachment backfill for this secondary node'
+          optional :repos_max_capacity, type: Integer, desc: 'Control the maximum concurrency of repository backfill for this secondary node'
+        end
+        put do
+          not_found!('GeoNode') unless geo_node
+
+          update_params = declared_params(include_missing: false)
+
+          if geo_node.primary?
+            forbidden!('Primary node cannot be edited')
+          elsif geo_node.update_attributes(update_params)
+            present geo_node, with: Entities::GeoNode
+          else
+            render_validation_error!(geo_node)
+          end
+        end
       end
     end
   end

lib/api/jobs.rb

@@ -38,6 +38,7 @@ module API
         builds = user_project.builds.order('id DESC')
         builds = filter_builds(builds, params[:scope])
 
+        builds = builds.preload(:user, :job_artifacts_archive, :runner, pipeline: :project)
         present paginate(builds), with: Entities::Job
       end
 

lib/api/v3/builds.rb

@@ -36,6 +36,7 @@ module API
           builds = user_project.builds.order('id DESC')
           builds = filter_builds(builds, params[:scope])
 
+          builds = builds.preload(:user, :job_artifacts_archive, :runner, pipeline: :project)
           present paginate(builds), with: ::API::V3::Entities::Build
         end
 

lib/tasks/geo.rake

@@ -194,4 +194,11 @@ namespace :geo do
       current_node.update!(primary: true)
     end
   end
+
+  desc 'Update Geo primary node URL'
+  task update_primary_node_url: :environment do
+    abort GEO_LICENSE_ERROR_TEXT unless Gitlab::Geo.license_allows?
+
+    Gitlab::Geo::GeoTasks.update_primary_geo_node_url
+  end
 end

package.json

@@ -111,7 +111,6 @@
     "karma-sourcemap-loader": "^0.3.7",
     "karma-webpack": "^2.0.4",
     "nodemon": "^1.11.0",
-    "prettier": "1.9.2",
     "webpack-dev-server": "^2.6.1"
   }
 }

scripts/add-code-formatters

-#!/bin/sh
-
-# Check if file exists with -f. Check if in in the gdk rook directory.
-if [ ! -f ../GDK_ROOT ]; then
-  echo "Please run script from gitlab (e.g. gitlab-development-kit/gitlab) root directory."
-  exit 1
-fi
-
-PRECOMMIT=$(git rev-parse --git-dir)/hooks/pre-commit
-
-# Check if symlink exists with -L. Check if script was already installed.
-if [ -L $PRECOMMIT ]; then
-  echo "Pre-commit script already installed."
-  exit 1
-fi
-
-ln -s ./pre-commit $PRECOMMIT
-echo "Pre-commit script installed successfully"

scripts/pre-commit

-#!/bin/sh
-
-# Check if file exists with -f. Check if in in the gdk rook directory.
-if [ ! -f ../GDK_ROOT ]; then
-  echo "Please run pre-commit from gitlab (e.g. gitlab-development-kit/gitlab) root directory."
-  exit 1
-fi
-
-jsfiles=$(git diff --cached --name-only --diff-filter=ACM "*.js" | tr '\n' ' ')
-[ -z "$jsfiles" ] && exit 0
-
-# Prettify all staged .js files
-echo "$jsfiles" | xargs ./node_modules/.bin/prettier --write
-
-# Add back the modified/prettified files to staging
-echo "$jsfiles" | xargs git add
-
-exit 0

scripts/prepare_postgres_fdw.sh

@@ -8,3 +8,8 @@ CREATE SCHEMA gitlab_secondary;
 IMPORT FOREIGN SCHEMA public FROM SERVER gitlab_secondary INTO gitlab_secondary;
 GRANT USAGE ON FOREIGN SERVER gitlab_secondary TO current_user;
 EOF
+
+# Ensure the FDW setting is enabled
+sed -i '/fdw:/d' config/database_geo.yml
+sed -i '/gitlabhq_geo_test/a\
+\ \ fdw: true' config/database_geo.yml

spec/fixtures/api/schemas/geo_node.json → spec/ee/fixtures/api/schemas/public_api/v4/geo_node.json

@@ -8,7 +8,8 @@
     "current",
     "files_max_capacity",
     "repos_max_capacity",
-    "clone_protocol"
+    "clone_protocol",
+    "_links"
   ],
   "properties" : {
     "id": { "type": "integer" },
@@ -18,7 +19,16 @@
     "current": { "type": "boolean" },
     "files_max_capacity": { "type": "integer" },
     "repos_max_capacity": { "type": "integer" },
-    "clone_protocol": { "type": ["string"] }
+    "clone_protocol": { "type": ["string"] },
+    "_links": {
+      "type": "object",
+      "required": ["self", "repair"],
+      "properties" : {
+        "self": { "type": "string" },
+        "repair": { "type": "string" }
+      },
+      "additionalProperties": false
+    }
   },
   "additionalProperties": false
 }

spec/fixtures/api/schemas/geo_node_status.json → spec/ee/fixtures/api/schemas/public_api/v4/geo_node_status.json

@@ -32,7 +32,8 @@
     "cursor_last_event_timestamp",
     "namespaces",
     "version",
-    "revision"
+    "revision",
+    "_links"
   ],
   "properties" : {
     "geo_node_id": { "type": "integer" },
@@ -74,7 +75,16 @@
     "storage_shards": { "type": "array" },
     "storage_shards_match": { "type": "boolean" },
     "version": { "type": ["string"] },
-    "revision": { "type": ["string"] }
+    "revision": { "type": ["string"] },
+    "_links": {
+      "type": "object",
+      "required": ["self", "node"],
+      "properties" : {
+        "self": { "type": "string" },
+        "node": { "type": "string" }
+      },
+      "additionalProperties": false
+    }
   },
   "additionalProperties": false
 }

spec/ee/spec/controllers/admin/geo_nodes_controller_spec.rb

@@ -267,7 +267,7 @@ describe Admin::GeoNodesController, :postgresql do
       it 'returns the status' do
         get :status, id: geo_node, format: :json
 
-        expect(response).to match_response_schema('geo_node_status')
+        expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
       end
     end
   end

spec/ee/spec/lib/gitlab/geo_spec.rb

@@ -18,24 +18,57 @@ describe Gitlab::Geo, :geo do
     end
   end
 
-  describe 'fdw?' do
+  describe 'fdw_capable?' do
     let(:fdw_check) { "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'gitlab_secondary' AND table_type = 'FOREIGN TABLE'" }
 
     before do
-      allow(::Geo::BaseRegistry.connection).to receive(:execute).with(anything).and_call_original
+      allow(::Geo::TrackingBase.connection).to receive(:execute).with(anything).and_call_original
     end
 
     it 'returns true when PostgreSQL FDW is enabled' do
-      expect(::Geo::BaseRegistry.connection).to receive(:execute).with(fdw_check).and_return([{ 'count' => 1 }])
+      expect(::Geo::TrackingBase.connection).to receive(:execute).with(fdw_check).and_return([{ 'count' => 1 }])
+
+      expect(described_class.fdw_capable?).to be_truthy
+    end
+
+    it 'returns false when PostgreSQL FDW is not enabled' do
+      expect(::Geo::TrackingBase.connection).to receive(:execute).with(fdw_check).and_return([{ 'count' => 0 }])
 
-      expect(described_class.fdw?).to be_truthy
+      expect(described_class.fdw_capable?).to be_falsey
     end
+  end
 
+  describe 'fdw?' do
     it 'returns false when PostgreSQL FDW is not enabled' do
-      expect(::Geo::BaseRegistry.connection).to receive(:execute).with(fdw_check).and_return([{ 'count' => 0 }])
+      allow(::Geo::TrackingBase.connection).to receive(:execute).and_return([{ 'count' => 0 }])
+      allow(Rails.configuration).to receive(:geo_database).and_return('fdw' => true)
 
       expect(described_class.fdw?).to be_falsey
     end
+
+    context 'with fdw capable' do
+      before do
+        allow(described_class).to receive(:fdw_capable?).and_return(true)
+      end
+
+      it 'returns true by default' do
+        allow(Rails.configuration).to receive(:geo_database).and_return('fdw' => nil)
+
+        expect(described_class.fdw?).to be_truthy
+      end
+
+      it 'returns false if configured in `config/database_geo.yml`' do
+        allow(Rails.configuration).to receive(:geo_database).and_return('fdw' => false)
+
+        expect(described_class.fdw?).to be_falsey
+      end
+
+      it 'returns true if configured in `config/database_geo.yml`' do
+        allow(Rails.configuration).to receive(:geo_database).and_return('fdw' => true)
+
+        expect(described_class.fdw?).to be_truthy
+      end
+    end
   end
 
   describe 'primary?' do

spec/ee/spec/models/geo_node_spec.rb

@@ -78,6 +78,17 @@ describe GeoNode, type: :model do
     end
   end
 
+  describe '#repair' do
+    it 'creates an oauth application for a Geo secondary node' do
+      stub_current_geo_node(node)
+      node.update_attribute(:oauth_application, nil)
+
+      node.repair
+
+      expect(node.oauth_application).to be_present
+    end
+  end
+
   describe '#current?' do
     it 'returns true when node is the current node' do
       node = described_class.new(url: described_class.current_node_url)

spec/factories/geo_node_statuses.rb

 FactoryBot.define do
   factory :geo_node_status do
-    sequence(:id)
     geo_node
     storage_shards { StorageShard.all }
 

spec/ee/spec/serializers/geo_node_status_entity_spec.rb → spec/lib/api/entities/geo_node_status_spec.rb

 require 'spec_helper'
 
-describe GeoNodeStatusEntity, :postgresql do
+describe API::Entities::GeoNodeStatus, :postgresql do
   include ::EE::GeoHelpers
 
   let(:geo_node_status) { build(:geo_node_status) }
@@ -11,38 +11,6 @@ describe GeoNodeStatusEntity, :postgresql do
 
   before { stub_primary_node }
 
-  it { is_expected.to have_key(:geo_node_id) }
-  it { is_expected.to have_key(:healthy) }
-  it { is_expected.to have_key(:health) }
-  it { is_expected.to have_key(:attachments_count) }
-  it { is_expected.to have_key(:attachments_failed_count) }
-  it { is_expected.to have_key(:attachments_synced_count) }
-  it { is_expected.to have_key(:attachments_synced_in_percentage) }
-  it { is_expected.to have_key(:lfs_objects_count) }
-  it { is_expected.to have_key(:lfs_objects_failed_count) }
-  it { is_expected.to have_key(:lfs_objects_synced_count) }
-  it { is_expected.to have_key(:lfs_objects_synced_in_percentage) }
-  it { is_expected.to have_key(:job_artifacts_count) }
-  it { is_expected.to have_key(:job_artifacts_failed_count) }
-  it { is_expected.to have_key(:job_artifacts_synced_count) }
-  it { is_expected.to have_key(:job_artifacts_synced_in_percentage) }
-  it { is_expected.to have_key(:repositories_count) }
-  it { is_expected.to have_key(:repositories_failed_count) }
-  it { is_expected.to have_key(:repositories_synced_count)}
-  it { is_expected.to have_key(:repositories_synced_in_percentage) }
-  it { is_expected.to have_key(:wikis_count) }
-  it { is_expected.to have_key(:wikis_failed_count) }
-  it { is_expected.to have_key(:wikis_synced_count)}
-  it { is_expected.to have_key(:wikis_synced_in_percentage) }
-  it { is_expected.to have_key(:replication_slots_count) }
-  it { is_expected.to have_key(:replication_slots_used_count)}
-  it { is_expected.to have_key(:replication_slots_used_in_percentage) }
-  it { is_expected.to have_key(:replication_slots_max_retained_wal_bytes) }
-  it { is_expected.to have_key(:last_successful_status_check_timestamp) }
-  it { is_expected.to have_key(:namespaces) }
-  it { is_expected.to have_key(:storage_shards) }
-  it { is_expected.to have_key(:storage_shards_match) }
-
   describe '#healthy' do
     context 'when node is healthy' do
       it 'returns true' do

spec/migrations/fix_wrongly_renamed_routes_spec.rb

 require 'spec_helper'
 require Rails.root.join('db', 'post_migrate', '20170518231126_fix_wrongly_renamed_routes.rb')
 
-describe FixWronglyRenamedRoutes, truncate: true do
+describe FixWronglyRenamedRoutes, :truncate, :migration do
   let(:subject) { described_class.new }
+  let(:namespaces_table) { table(:namespaces) }
+  let(:projects_table) { table(:projects) }
+  let(:routes_table) { table(:routes) }
   let(:broken_namespace) do
-    namespace = create(:group, name: 'apiis')
-    namespace.route.update_attribute(:path, 'api0is')
-    namespace
+    namespaces_table.create!(name: 'apiis', path: 'apiis').tap do |namespace|
+      routes_table.create!(source_type: 'Namespace', source_id: namespace.id, name: 'api0is', path: 'api0is')
+    end
   end
+  let(:broken_namespace_route) { routes_table.where(source_type: 'Namespace', source_id: broken_namespace.id).first }
 
   describe '#wrongly_renamed' do
     it "includes routes that have names that don't match their namespace" do
       broken_namespace
-      _other_namespace = create(:group, name: 'api0')
+      other_namespace = namespaces_table.create!(name: 'api0', path: 'api0')
+      routes_table.create!(source_type: 'Namespace', source_id: other_namespace.id, name: 'api0', path: 'api0')
 
       expect(subject.wrongly_renamed.map(&:id))
-        .to contain_exactly(broken_namespace.route.id)
+        .to contain_exactly(broken_namespace_route.id)
     end
   end
 
   describe "#paths_and_corrections" do
     it 'finds the wrong path and gets the correction from the namespace' do
       broken_namespace
-      namespace = create(:group, name: 'uploads-test')
-      namespace.route.update_attribute(:path, 'uploads0-test')
+      namespaces_table.create!(name: 'uploads-test', path: 'uploads-test').tap do |namespace|
+        routes_table.create!(source_type: 'Namespace', source_id: namespace.id, name: 'uploads-test', path: 'uploads0-test')
+      end
 
       expected_result = [
         { 'namespace_path' => 'apiis', 'path' => 'api0is' },
@@ -36,38 +42,45 @@ describe FixWronglyRenamedRoutes, truncate: true do
 
   describe '#routes_in_namespace_query' do
     it 'includes only the required routes' do
-      namespace = create(:group, path: 'hello')
-      project = create(:project, namespace: namespace)
-      _other_namespace = create(:group, path: 'hello0')
-
-      result = Route.where(subject.routes_in_namespace_query('hello'))
-
-      expect(result).to contain_exactly(namespace.route, project.route)
+      namespace = namespaces_table.create!(name: 'hello', path: 'hello')
+      namespace_route = routes_table.create!(source_type: 'Namespace', source_id: namespace.id, name: 'hello', path: 'hello')
+      project = projects_table.new(name: 'my-project', path: 'my-project', namespace_id: namespace.id).tap do |project|
+        project.save!(validate: false)
+      end
+      routes_table.create!(source_type: 'Project', source_id: project.id, name: 'my-project', path: 'hello/my-project')
+      _other_namespace = namespaces_table.create!(name: 'hello0', path: 'hello0')
+
+      result = routes_table.where(subject.routes_in_namespace_query('hello'))
+      project_route = routes_table.where(source_type: 'Project', source_id: project.id).first
+
+      expect(result).to contain_exactly(namespace_route, project_route)
     end
   end
 
   describe '#up' do
-    let(:broken_project) do
-      project = create(:project, namespace: broken_namespace, path: 'broken-project')
-      project.route.update_attribute(:path, 'api0is/broken-project')
-      project
-    end
-
     it 'renames incorrectly named routes' do
-      broken_project
+      broken_project =
+        projects_table.new(name: 'broken-project', path: 'broken-project', namespace_id: broken_namespace.id).tap do |project|
+          project.save!(validate: false)
+          routes_table.create!(source_type: 'Project', source_id: project.id, name: 'broken-project', path: 'api0is/broken-project')
+        end
 
       subject.up
 
-      expect(broken_project.route.reload.path).to eq('apiis/broken-project')
-      expect(broken_namespace.route.reload.path).to eq('apiis')
+      broken_project_route = routes_table.where(source_type: 'Project', source_id: broken_project.id).first
+
+      expect(broken_project_route.path).to eq('apiis/broken-project')
+      expect(broken_namespace_route.reload.path).to eq('apiis')
     end
 
     it "doesn't touch namespaces that look like something that should be renamed" do
-      namespace = create(:group, path: 'api0')
+      namespaces_table.create!(name: 'apiis', path: 'apiis')
+      namespace = namespaces_table.create!(name: 'hello', path: 'api0')
+      namespace_route = routes_table.create!(source_type: 'Namespace', source_id: namespace.id, name: 'hello', path: 'api0')
 
       subject.up
 
-      expect(namespace.route.reload.path).to eq('api0')
+      expect(namespace_route.reload.path).to eq('api0')
     end
   end
 end

spec/requests/api/geo_nodes_spec.rb

@@ -6,20 +6,19 @@ describe API::GeoNodes, :geo, api: true do
 
   set(:primary) { create(:geo_node, :primary) }
   set(:secondary) { create(:geo_node) }
-  set(:another_secondary) { create(:geo_node) }
+  set(:secondary_status) { create(:geo_node_status, :healthy, geo_node: secondary) }
 
-  set(:secondary_status) { create(:geo_node_status, :healthy, geo_node_id: secondary.id) }
-  set(:another_secondary_status) { create(:geo_node_status, :healthy, geo_node_id: another_secondary.id) }
+  let(:unexisting_node_id) { GeoNode.maximum(:id).to_i.succ }
 
-  let(:admin) { create(:admin) }
-  let(:user) { create(:user) }
+  set(:admin) { create(:admin) }
+  set(:user) { create(:user) }
 
   describe 'GET /geo_nodes' do
     it 'retrieves the Geo nodes if admin is logged in' do
       get api("/geo_nodes", admin)
 
       expect(response).to have_gitlab_http_status(200)
-      expect(response).to match_response_schema('geo_nodes')
+      expect(response).to match_response_schema('public_api/v4/geo_nodes', dir: 'ee')
     end
 
     it 'denies access if not admin' do
@@ -34,7 +33,15 @@ describe API::GeoNodes, :geo, api: true do
       get api("/geo_nodes/#{primary.id}", admin)
 
       expect(response).to have_gitlab_http_status(200)
-      expect(response).to match_response_schema('geo_node')
+      expect(response).to match_response_schema('public_api/v4/geo_node', dir: 'ee')
+
+      links = json_response['_links']
+      expect(links['self']).to end_with("/api/v4/geo_nodes/#{primary.id}")
+      expect(links['repair']).to end_with("/api/v4/geo_nodes/#{primary.id}/repair")
+    end
+
+    it_behaves_like '404 response' do
+      let(:request) { get api("/geo_nodes/#{unexisting_node_id}", admin) }
     end
 
     it 'denies access if not admin' do
@@ -49,7 +56,7 @@ describe API::GeoNodes, :geo, api: true do
       get api("/geo_nodes/status", admin)
 
       expect(response).to have_gitlab_http_status(200)
-      expect(response).to match_response_schema('geo_node_statuses')
+      expect(response).to match_response_schema('public_api/v4/geo_node_statuses', dir: 'ee')
     end
 
     it 'denies access if not admin' do
@@ -68,7 +75,11 @@ describe API::GeoNodes, :geo, api: true do
       get api("/geo_nodes/#{secondary.id}/status", admin)
 
       expect(response).to have_gitlab_http_status(200)
-      expect(response).to match_response_schema('geo_node_status')
+      expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
+
+      links = json_response['_links']
+      expect(links['self']).to end_with("/api/v4/geo_nodes/#{secondary.id}/status")
+      expect(links['node']).to end_with("/api/v4/geo_nodes/#{secondary.id}")
     end
 
     it 'fetches the current node status' do
@@ -80,14 +91,82 @@ describe API::GeoNodes, :geo, api: true do
       get api("/geo_nodes/#{secondary.id}/status", admin)
 
       expect(response).to have_gitlab_http_status(200)
-      expect(response).to match_response_schema('geo_node_status')
+      expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
+    end
+
+    it_behaves_like '404 response' do
+      let(:request) { get api("/geo_nodes/#{unexisting_node_id}/status", admin) }
     end
 
     it 'denies access if not admin' do
-      get api('/geo_nodes', user)
+      get api("/geo_nodes/#{secondary.id}/status", user)
+
+      expect(response).to have_gitlab_http_status(403)
+    end
+  end
+
+  describe 'POST /geo_nodes/:id/repair' do
+    it_behaves_like '404 response' do
+      let(:request) { post api("/geo_nodes/#{unexisting_node_id}/status", admin) }
+    end
+
+    it 'denies access if not admin' do
+      post api("/geo_nodes/#{secondary.id}/repair", user)
+
+      expect(response).to have_gitlab_http_status(403)
+    end
+
+    it 'returns 200 for the primary node' do
+      post api("/geo_nodes/#{primary.id}/repair", admin)
+
+      expect(response).to have_gitlab_http_status(200)
+      expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
+    end
+
+    it 'returns 200 when node does not need repairing' do
+      allow_any_instance_of(GeoNode).to receive(:missing_oauth_application?).and_return(false)
+
+      post api("/geo_nodes/#{secondary.id}/repair", admin)
+
+      expect(response).to have_gitlab_http_status(200)
+      expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
+    end
+
+    it 'repairs a secondary with oauth application missing' do
+      allow_any_instance_of(GeoNode).to receive(:missing_oauth_application?).and_return(true)
+
+      post api("/geo_nodes/#{secondary.id}/repair", admin)
+
+      expect(response).to have_gitlab_http_status(200)
+      expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
+    end
+  end
+
+  describe 'PUT /geo_nodes/:id' do
+    it_behaves_like '404 response' do
+      let(:request) { get api("/geo_nodes/#{unexisting_node_id}/status", admin) }
+    end
+
+    it 'denies access if not admin' do
+      put api("/geo_nodes/#{secondary.id}", user), {}
 
       expect(response).to have_gitlab_http_status(403)
     end
+
+    it 'updates the parameters' do
+      params = {
+        enabled: false,
+        url: 'https://updated.example.com/',
+        files_max_capacity: 33,
+        repos_max_capacity: 44
+      }.stringify_keys
+
+      put api("/geo_nodes/#{secondary.id}", admin), params
+
+      expect(response).to have_gitlab_http_status(200)
+      expect(response).to match_response_schema('public_api/v4/geo_node', dir: 'ee')
+      expect(json_response).to include(params)
+    end
   end
 
   describe 'GET /geo_nodes/current/failures/:type' do
@@ -101,7 +180,7 @@ describe API::GeoNodes, :geo, api: true do
       get api("/geo_nodes/current/failures", admin)
 
       expect(response).to have_gitlab_http_status(200)
-      expect(response).to match_response_schema('geo_project_registry')
+      expect(response).to match_response_schema('public_api/v4/geo_project_registry', dir: 'ee')
     end
 
     it 'does not show any registry when there is no failure' do

spec/requests/api/geo_spec.rb

@@ -185,7 +185,7 @@ describe API::Geo do
         get api('/geo/status'), nil, request.headers
 
         expect(response).to have_gitlab_http_status(200)
-        expect(response).to match_response_schema('geo_node_status')
+        expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
       end
     end
 
@@ -199,7 +199,7 @@ describe API::Geo do
         get api('/geo/status'), nil, request.headers
 
         expect(response).to have_gitlab_http_status(200)
-        expect(response).to match_response_schema('geo_node_status')
+        expect(response).to match_response_schema('public_api/v4/geo_node_status', dir: 'ee')
       end
     end
   end

spec/requests/api/jobs_spec.rb

@@ -29,9 +29,12 @@ describe API::Jobs do
   describe 'GET /projects/:id/jobs' do
     let(:query) { Hash.new }
 
-    before do
+    before do |example|
       job
-      get api("/projects/#{project.id}/jobs", api_user), query
+
+      unless example.metadata[:skip_before_request]
+        get api("/projects/#{project.id}/jobs", api_user), query
+      end
     end
 
     context 'authorized user' do
@@ -56,6 +59,23 @@ describe API::Jobs do
         expect(json_job['pipeline']['status']).to eq job.pipeline.status
       end
 
+      it 'avoids N+1 queries', skip_before_request: true do
+        first_build = create(:ci_build, :artifacts, pipeline: pipeline)
+        first_build.runner = create(:ci_runner)
+        first_build.user = create(:user)
+        first_build.save
+
+        control_count = ActiveRecord::QueryRecorder.new { go }.count
+
+        second_pipeline = create(:ci_empty_pipeline, project: project, sha: project.commit.id, ref: project.default_branch)
+        second_build = create(:ci_build, :artifacts, pipeline: second_pipeline)
+        second_build.runner = create(:ci_runner)
+        second_build.user = create(:user)
+        second_build.save
+
+        expect { go }.not_to exceed_query_limit(control_count)
+      end
+
       context 'filter project with one scope element' do
         let(:query) { { 'scope' => 'pending' } }
 
@@ -88,6 +108,10 @@ describe API::Jobs do
         expect(response).to have_gitlab_http_status(401)
       end
     end
+
+    def go
+      get api("/projects/#{project.id}/jobs", api_user), query
+    end
   end
 
   describe 'GET /projects/:id/pipelines/:pipeline_id/jobs' do

spec/requests/api/v3/builds_spec.rb

@@ -13,11 +13,13 @@ describe API::V3::Builds do
   describe 'GET /projects/:id/builds ' do
     let(:query) { '' }
 
-    before do
+    before do |example|
       build
       create(:ci_build, :skipped, pipeline: pipeline)
 
-      get v3_api("/projects/#{project.id}/builds?#{query}", api_user)
+      unless example.metadata[:skip_before_request]
+        get v3_api("/projects/#{project.id}/builds?#{query}", api_user)
+      end
     end
 
     context 'authorized user' do
@@ -41,6 +43,23 @@ describe API::V3::Builds do
         expect(json_build['pipeline']['status']).to eq build.pipeline.status
       end
 
+      it 'avoids N+1 queries', skip_before_request: true do
+        first_build = create(:ci_build, :artifacts, pipeline: pipeline)
+        first_build.runner = create(:ci_runner)
+        first_build.user = create(:user)
+        first_build.save
+
+        control_count = ActiveRecord::QueryRecorder.new { go }.count
+
+        second_pipeline = create(:ci_empty_pipeline, project: project, sha: project.commit.id, ref: project.default_branch)
+        second_build = create(:ci_build, :artifacts, pipeline: second_pipeline)
+        second_build.runner = create(:ci_runner)
+        second_build.user = create(:user)
+        second_build.save
+
+        expect { go }.not_to exceed_query_limit(control_count)
+      end
+
       context 'filter project with one scope element' do
         let(:query) { 'scope=pending' }
 
@@ -85,6 +104,10 @@ describe API::V3::Builds do
         expect(response).to have_gitlab_http_status(401)
       end
     end
+
+    def go
+      get v3_api("/projects/#{project.id}/builds?#{query}", api_user)
+    end
   end
 
   describe 'GET /projects/:id/repository/commits/:sha/builds' do

spec/tasks/geo_rake_spec.rb

 require 'rake_helper'
 
 describe 'geo rake tasks' do
+  include ::EE::GeoHelpers
+
   before do
     Rake.application.rake_require 'tasks/geo'
+    stub_licensed_features(geo: true)
   end
 
   describe 'set_primary_node task' do
     before do
-      expect(Gitlab::Geo).to receive(:license_allows?).and_return(true)
       stub_config_setting(protocol: 'https')
     end
 
@@ -26,14 +28,10 @@ describe 'geo rake tasks' do
   end
 
   describe 'set_secondary_as_primary task' do
-    include ::EE::GeoHelpers
-
     let!(:current_node) { create(:geo_node) }
     let!(:primary_node) { create(:geo_node, :primary) }
 
     before do
-      expect(Gitlab::Geo).to receive(:license_allows?).and_return(true)
-
       stub_current_geo_node(current_node)
     end
 
@@ -44,4 +42,19 @@ describe 'geo rake tasks' do
       expect(GeoNode.count).to eq(1)
     end
   end
+
+  describe 'update_primary_node_url task' do
+    let(:primary_node) { create(:geo_node, :primary, url: 'https://secondary.geo.example.com') }
+
+    before do
+      allow(GeoNode).to receive(:current_node_url).and_return('https://primary.geo.example.com')
+      stub_current_geo_node(primary_node)
+    end
+
+    it 'updates Geo primary node URL' do
+      run_rake_task('geo:update_primary_node_url')
+
+      expect(primary_node.reload.url).to eq 'https://primary.geo.example.com/'
+    end
+  end
 end

yarn.lock

@@ -5175,10 +5175,6 @@ preserve@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/preserve/-/preserve-0.2.0.tgz#815ed1f6ebc65926f865b310c0713bcb3315ce4b"
 
-prettier@1.9.2:
-  version "1.9.2"
-  resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.9.2.tgz#96bc2132f7a32338e6078aeb29727178c6335827"
-
 prettier@^1.7.0:
   version "1.8.2"
   resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.8.2.tgz#bff83e7fd573933c607875e5ba3abbdffb96aeb8"