Merge remote-tracking branch 'ee/master' into ce-to-ee-2017-12-14

* ee/master:
  Added Authorized Keys specific checks for Geo
  Simplify database replication instructions and make dryer
  Add SAST docs for EE
  Combine ssh docs and rename the doc
  Docs: update documentation guidelines
  Replace gl.utils.formatDate in place of exported function
  Add deprecated page back in
  Docs: add indexes for monitoring and performance monitoring
  Add docs for commit diff discussion in merge requests
  Import gitlab_projects.rb from gitlab-shell
  Show clear message when set-geo-primary-node was successful
  fix formatting of parameters for new group and transfer project to group

app/assets/javascripts/geo_nodes.js

@@ -3,7 +3,7 @@ import axios from 'axios';
 import SmartInterval from '~/smart_interval';
 import { s__ } from '~/locale';
 import { parseSeconds, stringifyTime } from './lib/utils/pretty_time';
-import { timeIntervalInWords } from './lib/utils/datetime_utility';
+import { formatDate, timeIntervalInWords } from './lib/utils/datetime_utility';
 import timeago from './vue_shared/mixins/timeago';
 
 const healthyClass = 'geo-node-healthy';
@@ -115,7 +115,7 @@ class GeoNodeStatus {
     let eventDate = notAvailable;
 
     if (eventTimestamp && eventTimestamp > 0) {
-      eventDate = gl.utils.formatDate(new Date(eventTimestamp * 1000));
+      eventDate = formatDate(new Date(eventTimestamp * 1000));
     }
 
     if (eventId) {

changelogs/unreleased-ee/4090-authorized-keys-check.yml

+---
+title: 'Geo: Added Authorized Keys specific checks'
+merge_request: 3728
+author:
+type: added

changelogs/unreleased-ee/tc-geo-set_primary_node-success.yml

+---
+title: Show clear message when set-geo-primary-node was successful
+merge_request: 3768
+author:
+type: changed

changelogs/unreleased/41016-import-gitlab-shell-projects.yml

+---
+title: Import some code and functionality from gitlab-shell to improve subprocess
+  handling
+merge_request:
+author:
+type: other

doc/README.md

@@ -107,6 +107,7 @@ Manage your [repositories](user/project/repository/index.md) from the UI (user i
   - [Work In Progress Merge Requests](user/project/merge_requests/work_in_progress_merge_requests.md)
   - [Merge Request discussion resolution](user/discussions/index.md#moving-a-single-discussion-to-a-new-issue): Resolve discussions, move discussions in a merge request to an issue, only allow merge requests to be merged if all discussions are resolved.
   - **(EES/EEP)** [Merge Request approval](user/project/merge_requests/merge_request_approvals.md): Make sure every merge request is approved by one or more people before getting merged.
+  - **(EEU)** [Static Application Security Testing](user/project/merge_requests/sast.md): Scan your code for vulnerabilities and display the results in merge requests.
   - [Checkout merge requests locally](user/project/merge_requests/index.md#checkout-merge-requests-locally)
   - [Cherry-pick](user/project/merge_requests/cherry_pick_changes.md)
 - [Milestones](user/project/milestones/index.md): Organize issues and merge requests into a cohesive group, optionally setting a due date.

doc/administration/operations/fast_ssh_key_lookup.md

+# Fast lookup of authorized SSH keys in the database
+
+Regular SSH operations become slow as the number of users grows because OpenSSH
+searches for a key to authorize a user via a linear search. In the worst case,
+such as when the user is not authorized to access GitLab, OpenSSH will scan the
+entire file to search for a key. This can take significant time and disk I/O,
+which will delay users attempting to push or pull to a repository. Making
+matters worse, if users add or remove keys frequently, the operating system may
+not be able to cache the `authorized_keys` file, which causes the disk to be
+accessed repeatedly.
+
+GitLab Shell solves this by providing a way to authorize SSH users via a fast,
+indexed lookup in the GitLab database. This page describes how to enable the fast
+lookup of authorized SSH keys.
+
+> **Warning:** OpenSSH version 6.9+ is required because
+`AuthorizedKeysCommand` must be able to accept a fingerprint. These
+instructions will break installations using older versions of OpenSSH, such as
+those included with CentOS 6 as of September 2017. If you want to use this
+feature for CentOS 6, follow [the instructions on how to build and install a custom OpenSSH package](#compiling-a-custom-version-of-openssh-for-centos-6) before continuing.
+
+
+## Fast lookup is required for GitLab Geo
+
+By default, GitLab manages an `authorized_keys` file, which contains all the
+public SSH keys for users allowed to access GitLab. However, to maintain a
+single source of truth, [Geo](../../gitlab-geo/README.md) needs to be configured to perform SSH fingerprint
+lookups via database lookup.
+
+As part of [setting up GitLab Geo](../../gitlab-geo/README.md#setup-instructions),
+you will be required to follow the steps outlined below for both the primary and
+secondary nodes, but note that the `Write to "authorized keys" file` checkbox
+only needs to be unchecked on the primary node since it will be reflected
+automatically on the secondary if database replication is working.
+
+## Setting up fast lookup via GitLab Shell
+
+GitLab Shell provides a way to authorize SSH users via a fast, indexed lookup
+to the GitLab database. GitLab Shell uses the fingerprint of the SSH key to
+check whether the user is authorized to access GitLab.
+
+Create the directory `/opt/gitlab-shell` first:
+
+```bash
+sudo mkdir -p /opt/gitlab-shell
+```
+
+Create this file at `/opt/gitlab-shell/authorized_keys`:
+
+```
+#!/bin/bash
+
+if [[ "$1" == "git" ]]; then
+  /opt/gitlab/embedded/service/gitlab-shell/bin/authorized_keys $2
+fi
+```
+
+Set appropriate ownership and permissions:
+
+```
+sudo chown root:git /opt/gitlab-shell/authorized_keys
+sudo chmod 0650 /opt/gitlab-shell/authorized_keys
+```
+
+Add the following to `/etc/ssh/sshd_config` or to `/assets/sshd_config` if you
+are using Omnibus Docker:
+
+```
+AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k
+AuthorizedKeysCommandUser git
+```
+
+Reload OpenSSH:
+
+```bash
+# Debian or Ubuntu installations
+sudo service ssh reload
+
+# CentOS installations
+sudo service sshd reload
+```
+
+Confirm that SSH is working by removing your user's SSH key in the UI, adding a
+new one, and attempting to pull a repo.
+
+> **Warning:** Do not disable writes until SSH is confirmed to be working
+perfectly because the file will quickly become out-of-date.
+
+In the case of lookup failures (which are not uncommon), the `authorized_keys`
+file will still be scanned. So git SSH performance will still be slow for many
+users as long as a large file exists.
+
+You can disable any more writes to the `authorized_keys` file by unchecking
+`Write to "authorized_keys" file` in the Application Settings of your GitLab
+installation.
+
+![Write to authorized keys setting](img/write_to_authorized_keys_setting.png)
+
+Again, confirm that SSH is working by removing your user's SSH key in the UI,
+adding a new one, and attempting to pull a repo.
+
+Then you can backup and delete your `authorized_keys` file for best performance.
+
+## How to go back to using the `authorized_keys` file
+
+This is a brief overview. Please refer to the above instructions for more context.
+
+1. [Rebuild the `authorized_keys` file](../raketasks/maintenance.md#rebuild-authorized_keys-file)
+1. Enable writes to the `authorized_keys` file in Application Settings
+1. Remove the `AuthorizedKeysCommand` lines from `/etc/ssh/sshd_config` or from `/assets/sshd_config` if you are using Omnibus Docker.
+1. Reload sshd: `sudo service sshd reload`
+1. Remove the `/opt/gitlab-shell/authorized_keys` file
+
+## Compiling a custom version of OpenSSH for CentOS 6
+
+Building a custom version of OpenSSH is not necessary for Ubuntu 16.04 users,
+since Ubuntu 16.04 ships with OpenSSH 7.2.
+
+It is also unnecessary for CentOS 7.4 users, as that version ships with
+OpenSSH 7.4. If you are using CentOS 7.0 - 7.3, we strongly recommend that you
+upgrade to CentOS 7.4 instead of following this procedure. This should be as
+simple as running `yum update`.
+
+CentOS 6 users must build their own OpenSSH package to enable SSH lookups via
+the database. The following instructions can be used to build OpenSSH 7.5:
+
+1. First, download the package and install the required packages:
+
+    ```
+    sudo su -
+    cd /tmp
+    curl --remote-name https://mirrors.evowise.com/pub/OpenBSD/OpenSSH/portable/openssh-7.5p1.tar.gz
+    tar xzvf openssh-7.5p1.tar.gz
+    yum install rpm-build gcc make wget openssl-devel krb5-devel pam-devel libX11-devel xmkmf libXt-devel
+    ```
+
+3. Prepare the build by copying files to the right place:
+
+    ```
+    mkdir -p /root/rpmbuild/{SOURCES,SPECS}
+    cp ./openssh-7.5p1/contrib/redhat/openssh.spec /root/rpmbuild/SPECS/
+    cp openssh-7.5p1.tar.gz /root/rpmbuild/SOURCES/
+    cd /root/rpmbuild/SPECS
+    ```
+
+3. Next, set the spec settings properly:
+
+    ```
+    sed -i -e "s/%define no_gnome_askpass 0/%define no_gnome_askpass 1/g" openssh.spec
+    sed -i -e "s/%define no_x11_askpass 0/%define no_x11_askpass 1/g" openssh.spec
+    sed -i -e "s/BuildPreReq/BuildRequires/g" openssh.spec
+    ```
+
+3. Build the RPMs:
+
+    ```
+    rpmbuild -bb openssh.spec
+    ```
+
+4. Ensure the RPMs were built:
+
+    ```
+    ls -al /root/rpmbuild/RPMS/x86_64/
+    ```
+
+    You should see something as the following:
+
+    ```
+    total 1324
+    drwxr-xr-x. 2 root root   4096 Jun 20 19:37 .
+    drwxr-xr-x. 3 root root     19 Jun 20 19:37 ..
+    -rw-r--r--. 1 root root 470828 Jun 20 19:37 openssh-7.5p1-1.x86_64.rpm
+    -rw-r--r--. 1 root root 490716 Jun 20 19:37 openssh-clients-7.5p1-1.x86_64.rpm
+    -rw-r--r--. 1 root root  17020 Jun 20 19:37 openssh-debuginfo-7.5p1-1.x86_64.rpm
+    -rw-r--r--. 1 root root 367516 Jun 20 19:37 openssh-server-7.5p1-1.x86_64.rpm
+    ```
+
+5. Install the packages. OpenSSH packages will replace `/etc/pam.d/sshd`
+   with its own version, which may prevent users from logging in, so be sure
+   that the file is backed up and restored after installation:
+
+    ```
+    timestamp=$(date +%s)
+    cp /etc/pam.d/sshd pam-ssh-conf-$timestamp
+    rpm -Uvh /root/rpmbuild/RPMS/x86_64/*.rpm
+    yes | cp pam-ssh-conf-$timestamp /etc/pam.d/sshd
+    ```
+
+6. Verify the installed version. In another window, attempt to login to the server:
+
+    ```
+    ssh -v <your-centos-machine>
+    ```
+
+    You should see a line that reads: "debug1: Remote protocol version 2.0, remote software version OpenSSH_7.5"
+
+    If not, you may need to restart sshd (e.g. `systemctl restart sshd.service`).
+
+7.  *IMPORTANT!* Open a new SSH session to your server before exiting to make
+    sure everything is working! If you need to downgrade, simple install the
+    older package:
+
+    ```
+    # Only run this if you run into a problem logging in
+    yum downgrade openssh-server openssh openssh-clients
+    ```

doc/administration/operations/index.md

@@ -15,5 +15,4 @@ that to prioritize important jobs.
 to restart Sidekiq.
 - **(EES/EEP)** [Extra Sidekiq operations](extra_sidekiq_processes.md): Configure an extra set of Sidekiq processes to ensure certain queues always have dedicated workers, no matter the amount of jobs that need to be processed.
 - [Unicorn](unicorn.md): Understand Unicorn and unicorn-worker-killer.
-- **(EES/EEP)** [Speed up SSH operations](speed_up_ssh.md): Authorize SSH users via a fast, indexed lookup to the GitLab database.
-- [Unicorn](unicorn.md): Understand Unicorn and unicorn-worker-killer.
+- **(EES/EEP)** [Speed up SSH operations](fast_ssh_key_lookup.md): Authorize SSH users via a fast, indexed lookup to the GitLab database.

doc/administration/operations/speed_up_ssh.md

-# Speed up SSH operations
-
->
-- [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/250) in GitLab Enterprise Edition 8.7.
-- Available in GitLab Enterprise Edition Starter.
-
-## The problem
-
-SSH operations become slow as the number of users grows.
-
-## The reason
-
-OpenSSH searches for a key to authorize a user via a linear search. In the worst case, such as when the user is not authorized to access GitLab, OpenSSH will scan the entire file to search for a key. This can take significant time and disk I/O, which will delay users attempting to push or pull to a repository. Making matters worse, if users add or remove keys frequently, the operating system may not be able to cache the authorized_keys file, which causes the disk to be accessed repeatedly.
-
-## The solution
-
-GitLab Shell provides a way to authorize SSH users via a fast, indexed lookup to the GitLab database. GitLab Shell uses the fingerprint of the SSH key to check whether the user is authorized to access GitLab.
-
-> **Warning:** OpenSSH version 6.9+ is required because
-`AuthorizedKeysCommand` must be able to accept a fingerprint. These
-instructions will break installations using older versions of OpenSSH, such as
-those included with CentOS 6 as of September 2017. If you want to use this
-feature for CentOS 6, follow [the instructions on how to build and install a custom OpenSSH package]
-(#compiling-a-custom-version-of-openssh-for-centos-6) before continuing.
-
-Create the directory `/opt/gitlab-shell` first:
-
-```bash
-sudo mkdir -p /opt/gitlab-shell
-```
-
-Create this file at `/opt/gitlab-shell/authorized_keys`:
-
-```
-#!/bin/bash
-
-if [[ "$1" == "git" ]]; then
-  /opt/gitlab/embedded/service/gitlab-shell/bin/authorized_keys $2
-fi
-```
-
-Set appropriate ownership and permissions:
-
-```
-sudo chown root:git /opt/gitlab-shell/authorized_keys
-sudo chmod 0650 /opt/gitlab-shell/authorized_keys
-```
-
-Add the following to `/etc/ssh/sshd_config` or to `/assets/sshd_config` if you are using Omnibus Docker:
-
-```
-AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k
-AuthorizedKeysCommandUser git
-```
-
-Reload OpenSSH:
-
-```bash
-# Debian or Ubuntu installations
-sudo service ssh reload
-
-# CentOS installations
-sudo service sshd reload
-```
-
-Confirm that SSH is working by removing your user's SSH key in the UI, adding a new one, and attempting to pull a repo.
-
-> **Warning:** Do not disable writes until SSH is confirmed to be working perfectly because the file will quickly become out-of-date.
-
-In the case of lookup failures (which are not uncommon), the `authorized_keys` file will still be scanned. So git SSH performance will still be slow for many users as long as a large file exists.
-
-You can disable any more writes to the `authorized_keys` file by unchecking `Write to "authorized_keys" file` in the Application Settings of your GitLab installation.
-
-![Write to authorized keys setting](img/write_to_authorized_keys_setting.png)
-
-Again, confirm that SSH is working by removing your user's SSH key in the UI, adding a new one, and attempting to pull a repo.
-
-Then you can backup and delete your `authorized_keys` file for best performance.
-
-## How to go back to using the `authorized_keys` file
-
-This is a brief overview. Please refer to the above instructions for more context.
-
-1. [Rebuild the `authorized_keys` file](../raketasks/maintenance.md#rebuild-authorized_keys-file)
-1. Enable writes to the `authorized_keys` file in Application Settings
-1. Remove the `AuthorizedKeysCommand` lines from `/etc/ssh/sshd_config` or from `/assets/sshd_config` if you are using Omnibus Docker.
-1. Reload sshd: `sudo service sshd reload`
-1. Remove the `/opt/gitlab-shell/authorized_keys` file
-
-## Compiling a custom version of OpenSSH for CentOS 6
-
-Building a custom version of OpenSSH is not necessary for Ubuntu 16.04 users,
-since Ubuntu 16.04 ships with OpenSSH 7.2.
-
-It is also unnecessary for CentOS 7.4 users, as that version ships with
-OpenSSH 7.4. If you are using CentOS 7.0 - 7.3, we strongly recommend that you
-upgrade to CentOS 7.4 instead of following this procedure. This should be as
-simple as running `yum update`.
-
-CentOS 6 users must build their own OpenSSH package to enable SSH lookups via
-the database. The following instructions can be used to build OpenSSH 7.5:
-
-1. First, download the package and install the required packages:
-
-    ```
-    sudo su -
-    cd /tmp
-    curl --remote-name https://mirrors.evowise.com/pub/OpenBSD/OpenSSH/portable/openssh-7.5p1.tar.gz
-    tar xzvf openssh-7.5p1.tar.gz
-    yum install rpm-build gcc make wget openssl-devel krb5-devel pam-devel libX11-devel xmkmf libXt-devel
-    ```
-
-3. Prepare the build by copying files to the right place:
-
-    ```
-    mkdir -p /root/rpmbuild/{SOURCES,SPECS}
-    cp ./openssh-7.5p1/contrib/redhat/openssh.spec /root/rpmbuild/SPECS/
-    cp openssh-7.5p1.tar.gz /root/rpmbuild/SOURCES/
-    cd /root/rpmbuild/SPECS
-    ```
-
-3. Next, set the spec settings properly:
-
-    ```
-    sed -i -e "s/%define no_gnome_askpass 0/%define no_gnome_askpass 1/g" openssh.spec
-    sed -i -e "s/%define no_x11_askpass 0/%define no_x11_askpass 1/g" openssh.spec
-    sed -i -e "s/BuildPreReq/BuildRequires/g" openssh.spec
-    ```
-
-3. Build the RPMs:
-
-    ```
-    rpmbuild -bb openssh.spec
-    ```
-
-4. Ensure the RPMs were built:
-
-    ```
-    ls -al /root/rpmbuild/RPMS/x86_64/
-    ```
-
-    You should see something as the following:
-
-    ```
-    total 1324
-    drwxr-xr-x. 2 root root   4096 Jun 20 19:37 .
-    drwxr-xr-x. 3 root root     19 Jun 20 19:37 ..
-    -rw-r--r--. 1 root root 470828 Jun 20 19:37 openssh-7.5p1-1.x86_64.rpm
-    -rw-r--r--. 1 root root 490716 Jun 20 19:37 openssh-clients-7.5p1-1.x86_64.rpm
-    -rw-r--r--. 1 root root  17020 Jun 20 19:37 openssh-debuginfo-7.5p1-1.x86_64.rpm
-    -rw-r--r--. 1 root root 367516 Jun 20 19:37 openssh-server-7.5p1-1.x86_64.rpm
-    ```
-
-5. Install the packages. OpenSSH packages will replace `/etc/pam.d/sshd`
-   with its own version, which may prevent users from logging in, so be sure
-   that the file is backed up and restored after installation:
-
-    ```
-    timestamp=$(date +%s)
-    cp /etc/pam.d/sshd pam-ssh-conf-$timestamp
-    rpm -Uvh /root/rpmbuild/RPMS/x86_64/*.rpm
-    yes | cp pam-ssh-conf-$timestamp /etc/pam.d/sshd
-    ```
-
-6. Verify the installed version. In another window, attempt to login to the server:
-
-    ```
-    ssh -v <your-centos-machine>
-    ```
-
-    You should see a line that reads: "debug1: Remote protocol version 2.0, remote software version OpenSSH_7.5"
-
-    If not, you may need to restart sshd (e.g. `systemctl restart sshd.service`).
-
-7.  *IMPORTANT!* Open a new SSH session to your server before exiting to make
-    sure everything is working! If you need to downgrade, simple install the
-    older package:
-
-    ```
-    # Only run this if you run into a problem logging in
-    yum downgrade openssh-server openssh openssh-clients
-    ```
+This document was moved to [another location](fast_ssh_key_lookup.md).

doc/api/groups.md

@@ -366,16 +366,16 @@ POST /groups
 
 Parameters:
 
-- `name` (required) - The name of the group
-- `path` (required) - The path of the group
-- `description` (optional) - The group's description
-- `membership_lock` (optional, boolean) - Prevent adding new members to project membership within this group
-- `share_with_group_lock` (optional, boolean) - Prevent sharing a project with another group within this group
-- `visibility` (optional) - The group's visibility. Can be `private`, `internal`, or `public`.
-- `lfs_enabled` (optional)      - Enable/disable Large File Storage (LFS) for the projects in this group
-- `request_access_enabled` (optional) - Allow users to request member access.
-- `parent_id` (optional) - The parent group id for creating nested group.
-- `shared_runners_minutes_limit` (optional) - (admin-only) Pipeline minutes quota for this group
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `name` | string | yes | The name of the group |
+| `path` | string | yes | The path of the group |
+| `description` | string | no | The group's description |
+| `visibility` | string | no | The group's visibility. Can be `private`, `internal`, or `public`. |
+| `lfs_enabled` | boolean | no | Enable/disable Large File Storage (LFS) for the projects in this group |
+| `request_access_enabled` | boolean | no | Allow users to request member access. |
+| `parent_id` | integer | no | The parent group id for creating nested group. |
+| `shared_runners_minutes_limit` | integer | no | (admin-only) Pipeline minutes quota for this group. |
 
 ## Transfer project to group
 
@@ -387,8 +387,10 @@ POST  /groups/:id/projects/:project_id
 
 Parameters:
 
-- `id` (required) - The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user
-- `project_id` (required) - The ID or path of a project
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user |
+| `project_id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) |
 
 ## Update group
 

doc/ci/examples/README.md

@@ -58,6 +58,10 @@ Apart from those, here is an collection of tutorials and guides on setting up yo
 
 - [Analyze code quality with the Code Climate CLI](code_climate.md)
 
+### Static Application Security Testing (SAST)
+
+- [Scan your code for vulnerabilities](sast.md)
+
 ### Other
 
 - [Using `dpl` as deployment tool](deployment/README.md)

doc/ci/examples/sast.md

+# Static application security testing with GitLab CI/CD
+
+NOTE: **Note:**
+In order to use this tool, a [GitLab Enterprise Edition Ultimate][ee] license
+is needed.
+
+This example shows how to run
+[Static Application Security Testing (SAST)](https://en.wikipedia.org/wiki/Static_program_analysis)
+on your project's source code by using GitLab CI/CD.
+
+All you need is a GitLab Runner with the Docker executor (the shared Runners on
+GitLab.com will work fine). You can then add a new job to `.gitlab-ci.yml`,
+called `sast`:
+
+```yaml
+sast:
+  image: registry.gitlab.com/gitlab-org/gl-sast:latest
+  script:
+    - /app/bin/run .
+  artifacts:
+    paths: [gl-sast-report.json]
+```
+
+Behind the scenes, the [gl-sast Docker image](https://gitlab.com/gitlab-org/gl-sast)
+is used to detect the language/framework and in turn runs the matching scan tool.
+
+The above example will create a `sast` job in your CI pipeline and will allow
+you to download and analyze the report artifact in JSON format.
+
+The results are sorted by the priority of the vulnerability:
+
+1. High
+1. Medium
+1. Low
+1. Unknown
+1. Everything else
+
+TIP: **Tip:**
+Starting with GitLab Enterprise Edition Ultimate 10.3, this information will
+be automatically extracted and shown right in the merge request widget. To do
+so, the CI job must be named `sast` and the artifact path must be
+`gl-sast-report.json`.
+[Learn more on application security testing results shown in merge requests](../../user/project/merge_requests/sast.md).
+
+## Supported languages and frameworks
+
+The following languages and frameworks are supported.
+
+| Language / framework | Scan tool |
+| -------------------- | --------- |
+| JavaScript    | [Retire.js](https://retirejs.github.io/retire.js)
+| Python        | [bandit](https://github.com/openstack/bandit) |
+| Ruby          | [bundler-audit](https://github.com/rubysec/bundler-audit) |
+| Ruby on Rails | [brakeman](https://brakemanscanner.org) |
+
+[ee]: https://about.gitlab.com/gitlab-ee/

doc/development/README.md

@@ -81,10 +81,9 @@ comments: false
 
 ## Documentation guides
 
-- [Documentation styleguide](doc_styleguide.md): Use this styleguide if you are
-  contributing to the documentation.
 - [Writing documentation](writing_documentation.md)
-  - [Distinction between general documentation and technical articles](writing_documentation.md#distinction-between-general-documentation-and-technical-articles)
+- [Documentation styleguide](doc_styleguide.md)
+- [Markdown](../user/markdown.md)
 
 ## Internationalization (i18n) guides
 

doc/gitlab-geo/README.md

@@ -86,7 +86,7 @@ current version of OpenSSH:
 
 Note that CentOS 6 and 7.0 ship with an old version of OpenSSH that do not
 support a feature that Geo requires. See the [documentation on GitLab Geo SSH
-access](ssh.md) for more details.
+access](../administration/operations/fast_ssh_key_lookup.md) for more details.
 
 ### LDAP
 
@@ -145,8 +145,8 @@ If you installed GitLab using the Omnibus packages (highly recommended):
    Geo node to unlock GitLab Geo.
 1. [Setup the database replication](database.md) (`primary (read-write) <->
    secondary (read-only)` topology).
-1. [Lookup authorized SSH keys in the database](../administration/operations/speed_up_ssh.html),
-   do this step for both primary AND secondary nodes.
+1. [Configure fast lookup of authorized SSH keys in the database](../administration/operations/fast_ssh_key_lookup.md),
+   this step is required and needs to be done on both the primary AND secondary nodes.
 1. [Configure GitLab](configuration.md) to set the primary and secondary nodes.
 1. Optional: [Configure a secondary LDAP server](../administration/auth/ldap.md)
    for the secondary. See [notes on LDAP](#ldap).
@@ -165,7 +165,7 @@ If you installed GitLab from source:
    Geo node to unlock GitLab Geo.
 1. [Setup the database replication](database_source.md) (`primary (read-write)
    <-> secondary (read-only)` topology).
-1. [Lookup authorized SSH keys in the database](../administration/operations/speed_up_ssh.html),
+1. [Configure fast lookup of authorized SSH keys in the database](../administration/operations/fast_ssh_key_lookup.md),
    do this step for both primary AND secondary nodes.
 1. [Configure GitLab](configuration_source.md) to set the primary and secondary
    nodes.

doc/gitlab-geo/after_setup.md

+This document was moved to [another location](using_a_geo_server.md).

doc/gitlab-geo/ssh.md

-# GitLab Geo SSH access
-
-By default, GitLab manages an `authorized_keys` file, which contains all the
-public SSH keys for users allowed to access GitLab. However, to maintain a
-single source of truth, Geo needs to be configured to perform SSH fingerprint
-lookups via database lookup. This approach is also much faster than scanning a
-file.
-
->**Note:**
-GitLab 10.0 and higher require database lookups for SSH keys.
-
-Note this feature is only available on operating systems that support OpenSSH
-6.9 and above. For CentOS 6, see the [instructions on building custom
-version of OpenSSH for your server]
-(../administration/operations/speed_up_ssh.html#compiling-a-custom-version-of-openssh-for-centos-6).
-
-For both primary AND secondary nodes, follow the instructions on
-[looking up authorized SSH keys in the database](../administration/operations/speed_up_ssh.html).
-
-Note that the 'Write to "authorized keys" file' checkbox only needs
-to be unchecked on the primary node since it will be reflected automatically
-in the secondary if database replication is working.
+This document was moved to [another location](../administration/operations/fast_ssh_key_lookup.md).

doc/gitlab-geo/updating_the_geo_nodes.md

@@ -93,7 +93,7 @@ for existing repositories was added in GitLab 10.1.
 ## Upgrading to GitLab 10.0
 
 Since GitLab 10.0, we require all **Geo** systems to [use SSH key lookups via
-the database](ssh.md) to avoid having to maintain consistency of the
+the database](../administration/operations/fast_ssh_key_lookup.md) to avoid having to maintain consistency of the
 `authorized_keys` file for SSH access. Failing to do this will prevent users
 from being able to clone via SSH.
 

doc/topics/autodevops/index.md

@@ -19,6 +19,7 @@ project in an easy and automatic way:
 1. [Auto Build](#auto-build)
 1. [Auto Test](#auto-test)
 1. [Auto Code Quality](#auto-code-quality)
+1. [Auto SAST (Static Application Security Testing)](#auto-sast)
 1. [Auto Review Apps](#auto-review-apps)
 1. [Auto Deploy](#auto-deploy)
 1. [Auto Monitoring](#auto-monitoring)
@@ -198,6 +199,18 @@ out. In GitLab Enterprise Edition Starter, differences between the source and
 target branches are
 [shown in the merge request widget](../../user/project/merge_requests/code_quality_diff.md).
 
+### Auto SAST
+
+> Introduced in [GitLab Enterprise Edition Ultimate][ee] 10.3.
+
+Static Application Security Testing (SAST) uses the
+[gl-sast Docker image](https://gitlab.com/gitlab-org/gl-sast) to run static
+analysis on the current code and checks for potential security issues. Once the
+report is created, it's uploaded as an artifact which you can later download and
+check out.
+
+Any security warnings are also [shown in the merge request widget](../../user/project/merge_requests/sast.md).
+
 ### Auto Review Apps
 
 NOTE: **Note:**
@@ -536,3 +549,4 @@ curl --data "value=true" --header "PRIVATE-TOKEN: personal_access_token" https:/
 [postgresql]: https://www.postgresql.org/
 [Auto DevOps template]: https://gitlab.com/gitlab-org/gitlab-ci-yml/blob/master/Auto-DevOps.gitlab-ci.yml
 [GitLab Omnibus Helm Chart]: ../../install/kubernetes/gitlab_omnibus.md
+[ee]: https://about.gitlab.com/gitlab-ee/

doc/user/index.md

@@ -55,6 +55,7 @@ and [Multiple Issue Boards](https://docs.gitlab.com/ee/user/project/issue_board.
 - [Lock files](https://docs.gitlab.com/ee/user/project/file_lock.html) to prevent conflicts
 - View of the current health and status of each CI environment running on Kubernetes with [Deploy Boards](https://docs.gitlab.com/ee/user/project/deploy_boards.html)
 - Leverage your continuous delivery method with [Canary Deployments](https://docs.gitlab.com/ee/user/project/canary_deployments.html)
+- Scan your code for vulnerabilities and [display them in merge requests](project/merge_requests/sast.md).
 
 You can also [integrate](project/integrations/project_services.md) GitLab with numerous third-party applications, such as Mattermost, Microsoft Teams, HipChat, Trello, Slack, Bamboo CI, JIRA, and a lot more.
 

doc/user/project/merge_requests/index.md

@@ -199,6 +199,17 @@ can show the Code Climate report right in the merge request widget area.
 
 [Read more about Code Quality reports.](code_quality_diff.md)
 
+## Static Application Security Testing
+
+> Introduced in [GitLab Enterprise Edition Ultimate][products] 10.3.
+
+If you are using [GitLab CI/CD][ci], you can analyze your source code for known
+vulnerabilities using Static Application Security Testing (SAST).
+Going a step further, GitLab can show the vulnerability report right in the
+merge request widget area.
+
+[Read more about Static Application Security Testing reports.](sast.md)
+
 ## Live preview with Review Apps
 
 If you configured [Review Apps](https://about.gitlab.com/features/review-apps/) for your project,

doc/user/project/merge_requests/sast.md

+# Static Application Security Testing (SAST)
+
+> [Introduced][ee-3775] in [GitLab Enterprise Edition Ultimate][ee] 10.3.
+
+## Overview
+
+If you are using [GitLab CI/CD][ci], you can analyze your source code for known
+vulnerabilities using Static Application Security Testing (SAST).
+
+Going a step further, GitLab can show the vulnerability list right in the merge
+request widget area:
+
+![SAST Widget](img/sast.png)
+
+## Use cases
+
+- Your application is using an external (open source) library, locked to a
+  specific version (e.g., via `Gemfile.lock`) and the version is known to be
+  vulnerable.
+- Your code has a potentially dangerous attribute in a class, or unsafe code
+  that can lead to unintended code execution.
+
+## How it works
+
+In order for the report to show in the merge request, you need to specify a
+`sast` job (exact name) that will analyze the code and upload the resulting
+`gl-sast-report.json` file as an artifact. GitLab will then check this file and
+show the information inside the merge request.
+
+This JSON file needs to be the only artifact file for the job. If you try
+to also include other files, it will break the vulnerability display in the
+merge request.
+
+For more information on how the `sast` job should look like, check the
+example on [analyzing a project's code for vulnerabilities][cc-docs].
+
+[ee-3775]: https://gitlab.com/gitlab-org/gitlab-ee/issues/3775
+[ee]: https://about.gitlab.com/gitlab-ee/
+[ci]: ../../../ci/README.md
+[cc-docs]: ../../../ci/examples/sast.md

ee/lib/gitlab/geo/geo_tasks.rb

@@ -5,10 +5,14 @@ module Gitlab
 
       def set_primary_geo_node
         node = GeoNode.new(primary: true, url: GeoNode.current_node_url)
-        $stdout.puts "Saving primary GeoNode with URL #{node.url}".color(:green)
+        $stdout.puts "Saving primary Geo node with URL #{node.url} ..."
         node.save
 
-        $stdout.puts "Error saving GeoNode:\n#{node.errors.full_messages.join("\n")}".color(:red) unless node.persisted?
+        if node.persisted?
+          $stdout.puts "#{node.url} is now the primary Geo node".color(:green)
+        else
+          $stdout.puts "Error saving Geo node:\n#{node.errors.full_messages.join("\n")}".color(:red)
+        end
       end
 
       def refresh_foreign_tables!

ee/lib/system_check/geo/authorized_keys_check.rb

+module SystemCheck
+  module Geo
+    class AuthorizedKeysCheck < ::SystemCheck::BaseCheck
+      set_name 'OpenSSH configured to use AuthorizedKeysCommand'
+
+      AUTHORIZED_KEYS_DOCS = 'doc/administration/operations/speed_up_ssh.md'.freeze
+      OPENSSH_AUTHORIZED_KEYS_CMD_REGEXP = %r{
+        ^AuthorizedKeysCommand # line starts with
+        \s+                    # one space or more
+        (?<quote>['"]?)        # detect optional quotes
+        (?<content>[^#'"]+)    # content should be at least 1 char, non quotes or start-comment symbol
+        \k<quote>              # boundary for command, backtracks the same detected quote, or none
+        \s*                    # optional any amount of space character
+        (?:\#.*)?$             # optional start-comment symbol followed by optionally any character until end of line
+      }x
+      OPENSSH_AUTHORIZED_KEYS_USER_REGEXP = %r{
+        ^AuthorizedKeysCommandUser # line starts with
+        \s+                        # one space or more
+        (?<quote>['"]?)            # detect optional quotes
+        (?<content>[^#'"]+)        # content should be at least 1 char, non quotes or start-comment symbol
+        \k<quote>                  # boundary for command, backtracks the same detected quote, or none
+        \s*                        # optional any amount of space character
+        (?:\#.*)?$                 # optional start-comment symbol followed by optionally any character until end of line
+      }x
+      OPENSSH_EXPECTED_COMMAND = '/opt/gitlab-shell/authorized_keys %u %k'.freeze
+
+      def multi_check
+        unless openssh_config_exists?
+          print_failure("Cannot find OpenSSH configuration file at: #{openssh_config_path}")
+
+          if in_docker?
+            try_fixing_it(
+              'If you are not using our official docker containers,',
+              'make sure you have OpenSSH server installed and configured correctly on this system'
+            )
+
+            for_more_information(AUTHORIZED_KEYS_DOCS)
+          else
+            try_fixing_it(
+              'Make sure you have OpenSSH server installed on this system'
+            )
+          end
+
+          return
+        end
+
+        unless openssh_config_readable?
+          print_skipped('Cannot access OpenSSH configuration file')
+
+          try_fixing_it(
+            'This is expected if you are using SELinux. You may want to check configuration manually'
+          )
+
+          for_more_information(AUTHORIZED_KEYS_DOCS)
+          return
+        end
+
+        authorized_keys_command = extract_authorized_keys_command
+        unless authorized_keys_command
+          print_failure('OpenSSH configuration file does not contain a AuthorizedKeysCommand')
+
+          try_fixing_it(
+            'Change your OpenSSH configuration file pointing to the correct command'
+          )
+
+          for_more_information(AUTHORIZED_KEYS_DOCS)
+          return
+        end
+
+        unless openssh_is_expected_command?(authorized_keys_command)
+          print_warning('OpenSSH configuration file points to a different AuthorizedKeysCommand')
+
+          try_fixing_it(
+            "We were expecting AuthorizedKeysCommand to be: #{OPENSSH_EXPECTED_COMMAND}",
+            "but instead it is: #{authorized_keys_command}",
+            'If you made a custom command, make sure it behaves according to GitLab\'s Documentation'
+          )
+
+          for_more_information(AUTHORIZED_KEYS_DOCS)
+          # this check should not block the others
+        end
+
+        authorized_keys_command_path = openssh_extract_command_path(authorized_keys_command)
+        unless File.file?(authorized_keys_command_path)
+          print_failure("Cannot find configured AuthorizedKeysCommand: #{authorized_keys_command_path}")
+
+          try_fixing_it(
+            'You need to create the file and add the correct content to it'
+          )
+
+          for_more_information(AUTHORIZED_KEYS_DOCS)
+          return
+        end
+
+        authorized_keys_command_user = extract_authorized_keys_command_user
+        unless authorized_keys_command_user
+          print_failure('OpenSSH configuration file does not contain a AuthorizedKeysCommandUser')
+
+          try_fixing_it(
+            'Change your OpenSSH configuration file pointing to the correct user'
+          )
+
+          for_more_information(AUTHORIZED_KEYS_DOCS)
+          return
+        end
+
+        unless authorized_keys_command_user == gitlab_user
+          print_warning('OpenSSH configuration file points to a different AuthorizedKeysCommandUser')
+
+          try_fixing_it(
+            "We were expecting AuthorizedKeysCommandUser to be: #{gitlab_user}",
+            "but instead it is: #{authorized_keys_command_user}",
+            'Fix your OpenSSH configuration file pointing to the correct user'
+          )
+
+          for_more_information(AUTHORIZED_KEYS_DOCS)
+          return
+        end
+
+        $stdout.puts 'yes'.color(:green)
+        true
+      end
+
+      def extract_authorized_keys_command
+        extract_openssh_config(OPENSSH_AUTHORIZED_KEYS_CMD_REGEXP)
+      end
+
+      def extract_authorized_keys_command_user
+        extract_openssh_config(OPENSSH_AUTHORIZED_KEYS_USER_REGEXP)
+      end
+
+      def openssh_config_path
+        @openssh_config_path ||= begin
+          if in_docker?
+            '/assets/sshd_config' # path in our official docker containers
+          else
+            '/etc/ssh/sshd_config'
+          end
+        end
+      end
+
+      private
+
+      def print_skipped(reason)
+        $stdout.puts 'skipped'.color(:magenta)
+
+        $stdout.puts '  Reason:'.color(:blue)
+        $stdout.puts "  #{reason}"
+      end
+
+      def print_warning(reason)
+        $stdout.puts 'warning'.color(:magenta)
+
+        $stdout.puts '  Reason:'.color(:blue)
+        $stdout.puts "  #{reason}"
+      end
+
+      def print_failure(reason)
+        $stdout.puts 'no'.color(:red)
+
+        $stdout.puts '  Reason:'.color(:blue)
+        $stdout.puts "  #{reason}"
+      end
+
+      def openssh_config_exists?
+        File.file?(openssh_config_path)
+      end
+
+      def openssh_config_readable?
+        File.readable?(openssh_config_path)
+      end
+
+      def openssh_extract_command_path(cmd_with_params)
+        cmd_with_params.split(' ').first
+      end
+
+      def openssh_is_expected_command?(authorized_keys_command)
+        authorized_keys_command.squeeze(' ') == OPENSSH_EXPECTED_COMMAND
+      end
+
+      def in_docker?
+        File.file?('/.dockerenv')
+      end
+
+      def gitlab_user
+        Gitlab.config.gitlab.user
+      end
+
+      def extract_openssh_config(regexp)
+        return false unless openssh_config_exists? && openssh_config_readable?
+
+        File.open(openssh_config_path) do |f|
+          f.each_line do |line|
+            if (match = line.match(regexp))
+              raw_content = match[:content]
+              # remove linebreak, and lead and trailing spaces
+              return raw_content.chomp.strip
+            end
+          end
+        end
+
+        nil
+      end
+    end
+  end
+end

ee/lib/system_check/geo/authorized_keys_flag_check.rb

+module SystemCheck
+  module Geo
+    class AuthorizedKeysFlagCheck < ::SystemCheck::BaseCheck
+      set_name 'GitLab configured to disable writing to authorized_keys file'
+
+      def check?
+        !Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled
+      end
+
+      def show_error
+        try_fixing_it(
+          "You need to disable `Write to authorized_keys file` in GitLab's Admin panel"
+        )
+
+        for_more_information(AUTHORIZED_KEYS_DOCS)
+      end
+    end
+  end
+end

lib/gitlab/git/gitlab_projects.rb

+module Gitlab
+  module Git
+    class GitlabProjects
+      include Gitlab::Git::Popen
+
+      # Absolute path to directory where repositories are stored.
+      # Example: /home/git/repositories
+      attr_reader :shard_path
+
+      # Relative path is a directory name for repository with .git at the end.
+      # Example: gitlab-org/gitlab-test.git
+      attr_reader :repository_relative_path
+
+      # Absolute path to the repository.
+      # Example: /home/git/repositorities/gitlab-org/gitlab-test.git
+      attr_reader :repository_absolute_path
+
+      # This is the path at which the gitlab-shell hooks directory can be found.
+      # It's essential for integration between git and GitLab proper. All new
+      # repositories should have their hooks directory symlinked here.
+      attr_reader :global_hooks_path
+
+      attr_reader :logger
+
+      def initialize(shard_path, repository_relative_path, global_hooks_path:, logger:)
+        @shard_path = shard_path
+        @repository_relative_path = repository_relative_path
+
+        @logger = logger
+        @global_hooks_path = global_hooks_path
+        @repository_absolute_path = File.join(shard_path, repository_relative_path)
+        @output = StringIO.new
+      end
+
+      def output
+        io = @output.dup
+        io.rewind
+        io.read
+      end
+
+      def rm_project
+        logger.info "Removing repository <#{repository_absolute_path}>."
+        FileUtils.rm_rf(repository_absolute_path)
+      end
+
+      # Move repository from one directory to another
+      #
+      # Example:  gitlab/gitlab-ci.git -> randx/six.git
+      #
+      # Won't work if target namespace directory does not exist
+      #
+      def mv_project(new_path)
+        new_absolute_path = File.join(shard_path, new_path)
+
+        # verify that the source repo exists
+        unless File.exist?(repository_absolute_path)
+          logger.error "mv-project failed: source path <#{repository_absolute_path}> does not exist."
+          return false
+        end
+
+        # ...and that the target repo does not exist
+        if File.exist?(new_absolute_path)
+          logger.error "mv-project failed: destination path <#{new_absolute_path}> already exists."
+          return false
+        end
+
+        logger.info "Moving repository from <#{repository_absolute_path}> to <#{new_absolute_path}>."
+        FileUtils.mv(repository_absolute_path, new_absolute_path)
+      end
+
+      # Import project via git clone --bare
+      # URL must be publicly cloneable
+      def import_project(source, timeout)
+        # Skip import if repo already exists
+        return false if File.exist?(repository_absolute_path)
+
+        masked_source = mask_password_in_url(source)
+
+        logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
+        cmd = %W(git clone --bare -- #{source} #{repository_absolute_path})
+
+        success = run_with_timeout(cmd, timeout, nil)
+
+        unless success
+          logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
+          FileUtils.rm_rf(repository_absolute_path)
+          return false
+        end
+
+        Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)
+
+        # The project was imported successfully.
+        # Remove the origin URL since it may contain password.
+        remove_origin_in_repo
+
+        true
+      end
+
+      def fork_repository(new_shard_path, new_repository_relative_path)
+        from_path = repository_absolute_path
+        to_path = File.join(new_shard_path, new_repository_relative_path)
+
+        # The repository cannot already exist
+        if File.exist?(to_path)
+          logger.error "fork-repository failed: destination repository <#{to_path}> already exists."
+          return false
+        end
+
+        # Ensure the namepsace / hashed storage directory exists
+        FileUtils.mkdir_p(File.dirname(to_path), mode: 0770)
+
+        logger.info "Forking repository from <#{from_path}> to <#{to_path}>."
+        cmd = %W(git clone --bare --no-local -- #{from_path} #{to_path})
+
+        run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path)
+      end
+
+      def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil)
+        tags_option = tags ? '--tags' : '--no-tags'
+
+        logger.info "Fetching remote #{name} for repository #{repository_absolute_path}."
+        cmd = %W(git fetch #{name} --prune --quiet)
+        cmd << '--force' if force
+        cmd << tags_option
+
+        setup_ssh_auth(ssh_key, known_hosts) do |env|
+          success = run_with_timeout(cmd, timeout, repository_absolute_path, env)
+
+          unless success
+            logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed."
+          end
+
+          success
+        end
+      end
+
+      def push_branches(remote_name, timeout, force, branch_names)
+        logger.info "Pushing branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}"
+        cmd = %w(git push)
+        cmd << '--force' if force
+        cmd += %W(-- #{remote_name}).concat(branch_names)
+
+        success = run_with_timeout(cmd, timeout, repository_absolute_path)
+
+        unless success
+          logger.error("Pushing branches to remote #{remote_name} failed.")
+        end
+
+        success
+      end
+
+      def delete_remote_branches(remote_name, branch_names)
+        branches = branch_names.map { |branch_name| ":#{branch_name}" }
+
+        logger.info "Pushing deleted branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}"
+        cmd = %W(git push -- #{remote_name}).concat(branches)
+
+        success = run(cmd, repository_absolute_path)
+
+        unless success
+          logger.error("Pushing deleted branches to remote #{remote_name} failed.")
+        end
+
+        success
+      end
+
+      protected
+
+      def run(*args)
+        output, exitstatus = popen(*args)
+        @output << output
+
+        exitstatus&.zero?
+      end
+
+      def run_with_timeout(*args)
+        output, exitstatus = popen_with_timeout(*args)
+        @output << output
+
+        exitstatus&.zero?
+      rescue Timeout::Error
+        @output.puts('Timed out')
+
+        false
+      end
+
+      def mask_password_in_url(url)
+        result = URI(url)
+        result.password = "*****" unless result.password.nil?
+        result.user = "*****" unless result.user.nil? # it's needed for oauth access_token
+        result
+      rescue
+        url
+      end
+
+      def remove_origin_in_repo
+        cmd = %w(git remote rm origin)
+        run(cmd, repository_absolute_path)
+      end
+
+      # Builds a small shell script that can be used to execute SSH with a set of
+      # custom options.
+      #
+      # Options are expanded as `'-oKey="Value"'`, so SSH will correctly interpret
+      # paths with spaces in them. We trust the user not to embed single or double
+      # quotes in the key or value.
+      def custom_ssh_script(options = {})
+        args = options.map { |k, v| %Q{'-o#{k}="#{v}"'} }.join(' ')
+
+        [
+          "#!/bin/sh",
+          "exec ssh #{args} \"$@\""
+        ].join("\n")
+      end
+
+      # Known hosts data and private keys can be passed to gitlab-shell in the
+      # environment. If present, this method puts them into temporary files, writes
+      # a script that can substitute as `ssh`, setting the options to respect those
+      # files, and yields: { "GIT_SSH" => "/tmp/myScript" }
+      def setup_ssh_auth(key, known_hosts)
+        options = {}
+
+        if key
+          key_file = Tempfile.new('gitlab-shell-key-file')
+          key_file.chmod(0o400)
+          key_file.write(key)
+          key_file.close
+
+          options['IdentityFile'] = key_file.path
+          options['IdentitiesOnly'] = 'yes'
+        end
+
+        if known_hosts
+          known_hosts_file = Tempfile.new('gitlab-shell-known-hosts')
+          known_hosts_file.chmod(0o400)
+          known_hosts_file.write(known_hosts)
+          known_hosts_file.close
+
+          options['StrictHostKeyChecking'] = 'yes'
+          options['UserKnownHostsFile'] = known_hosts_file.path
+        end
+
+        return yield({}) if options.empty?
+
+        script = Tempfile.new('gitlab-shell-ssh-wrapper')
+        script.chmod(0o755)
+        script.write(custom_ssh_script(options))
+        script.close
+
+        yield('GIT_SSH' => script.path)
+      ensure
+        key_file&.close!
+        known_hosts_file&.close!
+        script&.close!
+      end
+    end
+  end
+end

lib/gitlab/git/repository.rb

@@ -18,7 +18,6 @@ module Gitlab
         GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE
       ].freeze
       SEARCH_CONTEXT_LINES = 3
-      GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
       REBASE_WORKTREE_PREFIX = 'rebase'.freeze
       SQUASH_WORKTREE_PREFIX = 'squash'.freeze
       GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
@@ -40,10 +39,31 @@ module Gitlab
           repo = Rugged::Repository.init_at(repo_path, bare)
           repo.close
 
-          if symlink_hooks_to.present?
-            hooks_path = File.join(repo_path, 'hooks')
-            FileUtils.rm_rf(hooks_path)
-            FileUtils.ln_s(symlink_hooks_to, hooks_path)
+          create_hooks(repo_path, symlink_hooks_to) if symlink_hooks_to.present?
+
+          true
+        end
+
+        def create_hooks(repo_path, global_hooks_path)
+          local_hooks_path = File.join(repo_path, 'hooks')
+          real_local_hooks_path = :not_found
+
+          begin
+            real_local_hooks_path = File.realpath(local_hooks_path)
+          rescue Errno::ENOENT
+            # real_local_hooks_path == :not_found
+          end
+
+          # Do nothing if hooks already exist
+          unless real_local_hooks_path == File.realpath(global_hooks_path)
+            # Move the existing hooks somewhere safe
+            FileUtils.mv(
+              local_hooks_path,
+              "#{local_hooks_path}.old.#{Time.now.to_i}"
+            ) if File.exist?(local_hooks_path)
+
+            # Create the hooks symlink
+            FileUtils.ln_sf(global_hooks_path, local_hooks_path)
           end
 
           true

lib/gitlab/shell.rb

@@ -66,7 +66,7 @@ module Gitlab
     # Init new repository
     #
     # storage - project's storage name
-    # name - project path with namespace
+    # name - project disk path
     #
     # Ex.
     #   add_repository("/path/to/storage", "gitlab/gitlab-ci")
@@ -94,17 +94,21 @@ module Gitlab
     # Import repository
     #
     # storage - project's storage path
-    # name - project path with namespace
+    # name - project disk path
+    # url - URL to import from
     #
     # Ex.
-    #   import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git")
+    #   import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git")
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def import_repository(storage, name, url)
       # The timeout ensures the subprocess won't hang forever
-      cmd = [gitlab_shell_projects_path, 'import-project',
-             storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"]
-      gitlab_shell_fast_execute_raise_error(cmd)
+      cmd = gitlab_projects(storage, "#{name}.git")
+      success = cmd.import_project(url, git_timeout)
+
+      raise Error, cmd.output unless success
+
+      success
     end
 
     # Fetch remote for repository
@@ -132,16 +136,15 @@ module Gitlab
 
     # Move repository
     # storage - project's storage path
-    # path - project path with namespace
-    # new_path - new project path with namespace
+    # path - project disk path
+    # new_path - new project disk path
     #
     # Ex.
     #   mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def mv_repository(storage, path, new_path)
-      gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project',
-                                 storage, "#{path}.git", "#{new_path}.git"])
+      gitlab_projects(storage, "#{path}.git").mv_project("#{new_path}.git")
     end
 
     # Fork repository to new path
@@ -155,30 +158,21 @@ module Gitlab
     #
     # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one.
     def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path)
-      gitlab_shell_fast_execute(
-        [
-          gitlab_shell_projects_path,
-          'fork-repository',
-          forked_from_storage,
-          "#{forked_from_disk_path}.git",
-          forked_to_storage,
-          "#{forked_to_disk_path}.git"
-        ]
-      )
+      gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git")
+        .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git")
     end
 
     # Remove repository from file system
     #
     # storage - project's storage path
-    # name - project path with namespace
+    # name - project disk path
     #
     # Ex.
     #   remove_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def remove_repository(storage, name)
-      gitlab_shell_fast_execute([gitlab_shell_projects_path,
-                                 'rm-project', storage, "#{name}.git"])
+      gitlab_projects(storage, "#{name}.git").rm_project
     end
 
     # Add new key to gitlab-shell
@@ -370,56 +364,43 @@ module Gitlab
       end
     end
 
-    # Create (if necessary) and link the secret token file
-    def generate_and_link_secret_token
-      secret_file = Gitlab.config.gitlab_shell.secret_file
-      unless File.size?(secret_file)
-        # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
-        token = SecureRandom.hex(16)
-        File.write(secret_file, token)
-      end
-
-      link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret')
-      if File.exist?(gitlab_shell_path) && !File.exist?(link_path)
-        FileUtils.symlink(secret_file, link_path)
-      end
-    end
-
     # Push branch to remote repository
     #
-    # project_name - project's name with namespace
+    # storage - project's storage path
+    # project_name - project's disk path
     # remote_name - remote name
-    # branch_name - remote branch name
+    # branch_names - remote branch names to push
+    # forced - should we use --force flag
     #
     # Ex.
-    #   push_remote_branches('upstream', 'feature')
+    #   push_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test' 'upstream', ['feature'])
     #
     def push_remote_branches(storage, project_name, remote_name, branch_names, forced: true)
-      args = [gitlab_shell_projects_path, 'push-branches', storage, "#{project_name}.git", remote_name, '600']
-      args << '--force' if forced
-      args += [*branch_names]
+      cmd = gitlab_projects(storage, "#{project_name}.git")
 
-      output, status = Popen.popen(args)
-      raise Error, output unless status.zero?
+      success = cmd.push_branches(remote_name, git_timeout, forced, branch_names)
 
-      true
+      raise Error, cmd.output unless success
+
+      success
     end
 
-    # Delete branch from remote repository
-    #
-    # project_name - project's name with namespace
+    # storage - project's storage path
+    # project_name - project's disk path
     # remote_name - remote name
-    # branch_name - remote branch name
+    # branch_names - remote branch names
     #
     # Ex.
-    #   delete_remote_branches('upstream', 'feature')
+    #   delete_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test', 'upstream', ['feature'])
     #
     def delete_remote_branches(storage, project_name, remote_name, branch_names)
-      args = [gitlab_shell_projects_path, 'delete-remote-branches', storage, "#{project_name}.git", remote_name, *branch_names]
-      output, status = Popen.popen(args)
-      raise Error, output unless status.zero?
+      cmd = gitlab_projects(storage, "#{project_name}.git")
 
-      true
+      success = cmd.delete_remote_branches(remote_name, branch_names)
+
+      raise Error, cmd.output unless success
+
+      success
     end
 
     protected
@@ -460,24 +441,35 @@ module Gitlab
 
     private
 
-    def local_fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false)
-      args = [gitlab_shell_projects_path, 'fetch-remote', storage, name, remote, "#{Gitlab.config.gitlab_shell.git_timeout}"]
-      args << '--force' if forced
-      args << '--no-tags' if no_tags
+    def gitlab_projects(shard_path, disk_path)
+      Gitlab::Git::GitlabProjects.new(
+        shard_path,
+        disk_path,
+        global_hooks_path: Gitlab.config.gitlab_shell.hooks_path,
+        logger: Rails.logger
+      )
+    end
 
-      vars = {}
+    def local_fetch_remote(storage_path, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false)
+      vars = { force: forced, tags: !no_tags }
 
       if ssh_auth&.ssh_import?
         if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
-          vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key
+          vars[:ssh_key] = ssh_auth.ssh_private_key
         end
 
         if ssh_auth.ssh_known_hosts.present?
-          vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts
+          vars[:known_hosts] = ssh_auth.ssh_known_hosts
         end
       end
 
-      gitlab_shell_fast_execute_raise_error(args, vars)
+      cmd = gitlab_projects(storage_path, repository_relative_path)
+
+      success = cmd.fetch_remote(remote, git_timeout, vars)
+
+      raise Error, cmd.output unless success
+
+      success
     end
 
     def gitlab_shell_fast_execute(cmd)
@@ -513,6 +505,10 @@ module Gitlab
       Gitlab::GitalyClient::NamespaceService.new(storage)
     end
 
+    def git_timeout
+      Gitlab.config.gitlab_shell.git_timeout
+    end
+
     def gitaly_migrate(method, &block)
       Gitlab::GitalyClient.migrate(method, &block)
     rescue GRPC::NotFound, GRPC::BadStatus => e

lib/tasks/gitlab/check.rake

@@ -464,7 +464,9 @@ namespace :gitlab do
         SystemCheck::Geo::HttpConnectionCheck,
         SystemCheck::Geo::HTTPCloneEnabledCheck,
         SystemCheck::Geo::ClocksSynchronizationCheck,
-        SystemCheck::App::GitUserDefaultSSHConfigCheck
+        SystemCheck::App::GitUserDefaultSSHConfigCheck,
+        SystemCheck::Geo::AuthorizedKeysCheck,
+        SystemCheck::Geo::AuthorizedKeysFlagCheck
       ]
 
       SystemCheck.run('Geo', checks)

spec/ee/fixtures/system_check/sshd_config

+# Package generated configuration file
+# See the sshd_config(5) manpage for details
+
+# What ports, IPs and protocols we listen for
+Port 22
+# Use these options to restrict which interfaces/protocols sshd will bind to
+#ListenAddress ::
+#ListenAddress 0.0.0.0
+Protocol 2
+# HostKeys for protocol version 2
+HostKey /etc/ssh/ssh_host_rsa_key
+HostKey /etc/ssh/ssh_host_dsa_key
+HostKey /etc/ssh/ssh_host_ecdsa_key
+HostKey /etc/ssh/ssh_host_ed25519_key
+#Privilege Separation is turned on for security
+UsePrivilegeSeparation yes
+
+# Lifetime and size of ephemeral version 1 server key
+KeyRegenerationInterval 3600
+ServerKeyBits 1024
+
+# Logging
+SyslogFacility AUTH
+LogLevel INFO
+
+# Authentication:
+LoginGraceTime 120
+PermitRootLogin yes
+StrictModes yes
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+#AuthorizedKeysFile	%h/.ssh/authorized_keys
+#AuthorizedKeysCommand /opt/gitlab-shell/invalid_authorized_keys %u %k
+AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k
+AuthorizedKeysCommandUser git
+
+# Don't read the user's ~/.rhosts and ~/.shosts files
+IgnoreRhosts yes
+# For this to work you will also need host keys in /etc/ssh_known_hosts
+RhostsRSAAuthentication no
+# similar for protocol version 2
+HostbasedAuthentication no
+# Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication
+#IgnoreUserKnownHosts yes
+
+# To enable empty passwords, change to yes (NOT RECOMMENDED)
+PermitEmptyPasswords no
+
+# Change to yes to enable challenge-response passwords (beware issues with
+# some PAM modules and threads)
+ChallengeResponseAuthentication no
+
+# Change to no to disable tunnelled clear text passwords
+PasswordAuthentication yes
+
+# Kerberos options
+#KerberosAuthentication no
+#KerberosGetAFSToken no
+#KerberosOrLocalPasswd yes
+#KerberosTicketCleanup yes
+
+# GSSAPI options
+#GSSAPIAuthentication no
+#GSSAPICleanupCredentials yes
+
+X11Forwarding yes
+X11DisplayOffset 10
+PrintMotd no
+PrintLastLog yes
+TCPKeepAlive yes
+#UseLogin no
+
+#MaxStartups 10:30:60
+#Banner /etc/issue.net
+
+# Allow client to pass locale environment variables
+AcceptEnv LANG LC_*
+
+Subsystem sftp /usr/lib/openssh/sftp-server
+
+# Set this to 'yes' to enable PAM authentication, account processing,
+# and session processing. If this is enabled, PAM authentication will
+# be allowed through the ChallengeResponseAuthentication and
+# PasswordAuthentication.  Depending on your PAM configuration,
+# PAM authentication via ChallengeResponseAuthentication may bypass
+# the setting of "PermitRootLogin yes
+# If you just want the PAM account and session checks to run without
+# PAM authentication, then enable this but set PasswordAuthentication
+# and ChallengeResponseAuthentication to 'no'.
+UsePAM yes

spec/ee/fixtures/system_check/sshd_config_invalid_command

+# Package generated configuration file
+# See the sshd_config(5) manpage for details
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+#AuthorizedKeysFile	%h/.ssh/authorized_keys
+AuthorizedKeysCommand "/opt/gitlab-shell/invalid_authorized_keys   %u      %k"         # comment
+#AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k
+AuthorizedKeysCommandUser anotheruser  #comment with more stuff#

spec/ee/fixtures/system_check/sshd_config_invalid_user

+# Package generated configuration file
+# See the sshd_config(5) manpage for details
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+#AuthorizedKeysFile	%h/.ssh/authorized_keys
+AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k          # comment
+AuthorizedKeysCommandUser anotheruser  #comment with more stuff#

spec/ee/fixtures/system_check/sshd_config_no_command

+# Package generated configuration file
+# See the sshd_config(5) manpage for details
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+# AuthorizedKeysFile	%h/.ssh/authorized_keys
+# AuthorizedKeysCommand /opt/gitlab-shell/invalid_authorized_keys %u %k
+# AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k
+# AuthorizedKeysCommandUser git

spec/ee/fixtures/system_check/sshd_config_no_user

+# Package generated configuration file
+# See the sshd_config(5) manpage for details
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+#AuthorizedKeysFile	%h/.ssh/authorized_keys
+#AuthorizedKeysCommand /opt/gitlab-shell/invalid_authorized_keys %u %k
+AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k
+#AuthorizedKeysCommandUser git

spec/ee/spec/lib/gitlab/geo/geo_tasks_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Geo::GeoTasks do
+  describe '.set_primary_geo_node' do
+    before do
+      allow(GeoNode).to receive(:current_node_url).and_return('https://primary.geo.example.com')
+    end
+
+    it 'sets the primary node' do
+      expect { subject.set_primary_geo_node }.to output(/https:\/\/primary.geo.example.com\/ is now the primary Geo node/).to_stdout
+    end
+
+    it 'returns error when there is already a Primary node' do
+      create(:geo_node, :primary)
+
+      expect { subject.set_primary_geo_node }.to output(/Error saving Geo node:/).to_stdout
+    end
+  end
+end

spec/ee/spec/lib/system_check/geo/authorized_keys_check_spec.rb

+require 'spec_helper'
+require 'rake_helper'
+
+describe SystemCheck::Geo::AuthorizedKeysCheck do
+  describe '#multi_check' do
+    subject { described_class.new }
+
+    before do
+      allow(File).to receive(:file?).and_call_original # provides a default behavior when mocking
+      allow(File).to receive(:file?).with('/opt/gitlab-shell/authorized_keys') { true }
+    end
+
+    context 'OpenSSH config file' do
+      context 'in docker' do
+        it 'fails when config file does not exist' do
+          allow(subject).to receive(:in_docker?) { true }
+          allow(File).to receive(:file?).with('/assets/sshd_config') { false }
+
+          expect_failure('Cannot find OpenSSH configuration file at: /assets/sshd_config')
+
+          subject.multi_check
+        end
+      end
+
+      it 'fails when config file does not exist' do
+        allow(subject).to receive(:in_docker?) { false }
+        allow(File).to receive(:file?).with('/etc/ssh/sshd_config') { false }
+
+        expect_failure('Cannot find OpenSSH configuration file at: /etc/ssh/sshd_config')
+
+        subject.multi_check
+      end
+
+      it 'skips when config file is not readable' do
+        override_sshd_config('system_check/sshd_config')
+        allow(File).to receive(:readable?).with(expand_fixture_ee_path('system_check/sshd_config')) { false }
+
+        expect_skipped('Cannot access OpenSSH configuration file')
+
+        subject.multi_check
+      end
+    end
+
+    context 'AuthorizedKeysCommand' do
+      it 'fails when config file does not contain the AuthorizedKeysCommand' do
+        override_sshd_config('system_check/sshd_config_no_command')
+
+        expect_failure('OpenSSH configuration file does not contain a AuthorizedKeysCommand')
+
+        subject.multi_check
+      end
+
+      it 'warns when config file does not contain the correct AuthorizedKeysCommand' do
+        override_sshd_config('system_check/sshd_config_invalid_command')
+
+        expect_warning('OpenSSH configuration file points to a different AuthorizedKeysCommand')
+
+        subject.multi_check
+      end
+
+      it 'fails when cannot find referred authorized keys file on disk' do
+        override_sshd_config('system_check/sshd_config')
+        allow(subject).to receive(:extract_authorized_keys_command) { '/tmp/nonexistent/authorized_keys' }
+
+        expect_failure('Cannot find configured AuthorizedKeysCommand: /tmp/nonexistent/authorized_keys')
+
+        subject.multi_check
+      end
+    end
+
+    context 'AuthorizedKeysCommandUser' do
+      it 'fails when config file does not contain the AuthorizedKeysCommandUser' do
+        override_sshd_config('system_check/sshd_config_no_user')
+
+        expect_failure('OpenSSH configuration file does not contain a AuthorizedKeysCommandUser')
+
+        subject.multi_check
+      end
+
+      it 'fails when config file does not contain the correct AuthorizedKeysCommandUser' do
+        override_sshd_config('system_check/sshd_config_invalid_user')
+
+        expect_warning('OpenSSH configuration file points to a different AuthorizedKeysCommandUser')
+
+        subject.multi_check
+      end
+    end
+
+    it 'succeed when all conditions are met' do
+      override_sshd_config('system_check/sshd_config')
+      allow(subject).to receive(:gitlab_user) { 'git' }
+
+      result = subject.multi_check
+      expect($stdout.string).to include('yes')
+      expect(result).to be_truthy
+    end
+  end
+
+  describe '#extract_authorized_keys_command' do
+    it 'returns false when no command is available' do
+      override_sshd_config('system_check/sshd_config_no_command')
+
+      expect(subject.extract_authorized_keys_command).to be_falsey
+    end
+
+    it 'returns correct (uncommented) command' do
+      override_sshd_config('system_check/sshd_config')
+
+      expect(subject.extract_authorized_keys_command).to eq('/opt/gitlab-shell/authorized_keys %u %k')
+    end
+
+    it 'returns command without comments and without quotes' do
+      override_sshd_config('system_check/sshd_config_invalid_command')
+
+      expect(subject.extract_authorized_keys_command).to eq('/opt/gitlab-shell/invalid_authorized_keys   %u      %k')
+    end
+  end
+
+  describe '#extract_authorized_keys_command_user' do
+    it 'returns false when no command user is available' do
+      override_sshd_config('system_check/sshd_config_no_command')
+
+      expect(subject.extract_authorized_keys_command_user).to be_falsey
+    end
+
+    it 'returns correct (uncommented) command' do
+      override_sshd_config('system_check/sshd_config')
+
+      expect(subject.extract_authorized_keys_command_user).to eq('git')
+    end
+
+    it 'returns command without comments' do
+      override_sshd_config('system_check/sshd_config_invalid_command')
+
+      expect(subject.extract_authorized_keys_command_user).to eq('anotheruser')
+    end
+  end
+
+  describe '#openssh_config_path' do
+    context 'when in docker container' do
+      it 'returns /assets/sshd_config' do
+        allow(subject).to receive(:in_docker?) { true }
+
+        expect(subject.openssh_config_path).to eq('/assets/sshd_config')
+      end
+    end
+
+    context 'when not in docker container' do
+      it 'returns /etc/ssh/sshd_config' do
+        allow(subject).to receive(:in_docker?) { false }
+
+        expect(subject.openssh_config_path).to eq('/etc/ssh/sshd_config')
+      end
+    end
+  end
+
+  def expect_failure(reason)
+    expect(subject).to receive(:print_failure).with(reason)
+  end
+
+  def expect_warning(reason)
+    expect(subject).to receive(:print_warning).with(reason)
+  end
+
+  def expect_skipped(reason)
+    expect(subject).to receive(:print_skipped).with(reason)
+  end
+
+  def override_sshd_config(relative_path)
+    allow(subject).to receive(:openssh_config_path) { expand_fixture_ee_path(relative_path) }
+  end
+end

spec/ee/spec/lib/system_check/geo/authorized_keys_flag_check_spec.rb

+require 'spec_helper'
+require 'rake_helper'
+
+describe SystemCheck::Geo::AuthorizedKeysFlagCheck do
+  before do
+    silence_output
+  end
+
+  describe '#check?' do
+    it 'fails when write to authorized_keys still enabled' do
+      stub_application_setting(authorized_keys_enabled: true)
+
+      expect(subject.check?).to be_falsey
+    end
+
+    it 'succeed when write to authorized_keys is disabled' do
+      stub_application_setting(authorized_keys_enabled: false)
+
+      expect(subject.check?).to be_truthy
+    end
+  end
+end

spec/lib/gitlab/git/gitlab_projects_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Git::GitlabProjects do
+  after do
+    TestEnv.clean_test_path
+  end
+
+  let(:project) { create(:project, :repository) }
+
+  if $VERBOSE
+    let(:logger) { Logger.new(STDOUT) }
+  else
+    let(:logger) { double('logger').as_null_object }
+  end
+
+  let(:tmp_repos_path) { TestEnv.repos_path }
+  let(:repo_name) { project.disk_path + '.git' }
+  let(:tmp_repo_path) { File.join(tmp_repos_path, repo_name) }
+  let(:gl_projects) { build_gitlab_projects(tmp_repos_path, repo_name) }
+
+  describe '#initialize' do
+    it { expect(gl_projects.shard_path).to eq(tmp_repos_path) }
+    it { expect(gl_projects.repository_relative_path).to eq(repo_name) }
+    it { expect(gl_projects.repository_absolute_path).to eq(File.join(tmp_repos_path, repo_name)) }
+    it { expect(gl_projects.logger).to eq(logger) }
+  end
+
+  describe '#mv_project' do
+    let(:new_repo_path) { File.join(tmp_repos_path, 'repo.git') }
+
+    it 'moves a repo directory' do
+      expect(File.exist?(tmp_repo_path)).to be_truthy
+
+      message = "Moving repository from <#{tmp_repo_path}> to <#{new_repo_path}>."
+      expect(logger).to receive(:info).with(message)
+
+      expect(gl_projects.mv_project('repo.git')).to be_truthy
+
+      expect(File.exist?(tmp_repo_path)).to be_falsy
+      expect(File.exist?(new_repo_path)).to be_truthy
+    end
+
+    it "fails if the source path doesn't exist" do
+      expect(logger).to receive(:error).with("mv-project failed: source path <#{tmp_repos_path}/bad-src.git> does not exist.")
+
+      result = build_gitlab_projects(tmp_repos_path, 'bad-src.git').mv_project('repo.git')
+      expect(result).to be_falsy
+    end
+
+    it 'fails if the destination path already exists' do
+      FileUtils.mkdir_p(File.join(tmp_repos_path, 'already-exists.git'))
+
+      message = "mv-project failed: destination path <#{tmp_repos_path}/already-exists.git> already exists."
+      expect(logger).to receive(:error).with(message)
+
+      expect(gl_projects.mv_project('already-exists.git')).to be_falsy
+    end
+  end
+
+  describe '#rm_project' do
+    it 'removes a repo directory' do
+      expect(File.exist?(tmp_repo_path)).to be_truthy
+      expect(logger).to receive(:info).with("Removing repository <#{tmp_repo_path}>.")
+
+      expect(gl_projects.rm_project).to be_truthy
+
+      expect(File.exist?(tmp_repo_path)).to be_falsy
+    end
+  end
+
+  describe '#push_branches' do
+    let(:remote_name) { 'remote-name' }
+    let(:branch_name) { 'master' }
+    let(:cmd) { %W(git push -- #{remote_name} #{branch_name}) }
+    let(:force) { false }
+
+    subject { gl_projects.push_branches(remote_name, 600, force, [branch_name]) }
+
+    it 'executes the command' do
+      stub_spawn(cmd, 600, tmp_repo_path, success: true)
+
+      is_expected.to be_truthy
+    end
+
+    it 'fails' do
+      stub_spawn(cmd, 600, tmp_repo_path, success: false)
+
+      is_expected.to be_falsy
+    end
+
+    context 'with --force' do
+      let(:cmd) { %W(git push --force -- #{remote_name} #{branch_name}) }
+      let(:force) { true }
+
+      it 'executes the command' do
+        stub_spawn(cmd, 600, tmp_repo_path, success: true)
+
+        is_expected.to be_truthy
+      end
+    end
+  end
+
+  describe '#fetch_remote' do
+    let(:remote_name) { 'remote-name' }
+    let(:branch_name) { 'master' }
+    let(:force) { false }
+    let(:tags) { true }
+    let(:args) { { force: force, tags: tags }.merge(extra_args) }
+    let(:extra_args) { {} }
+    let(:cmd) { %W(git fetch #{remote_name} --prune --quiet --tags) }
+
+    subject { gl_projects.fetch_remote(remote_name, 600, args) }
+
+    def stub_tempfile(name, filename, opts = {})
+      chmod = opts.delete(:chmod)
+      file = StringIO.new
+
+      allow(file).to receive(:close!)
+      allow(file).to receive(:path).and_return(name)
+
+      expect(Tempfile).to receive(:new).with(filename).and_return(file)
+      expect(file).to receive(:chmod).with(chmod) if chmod
+
+      file
+    end
+
+    context 'with default args' do
+      it 'executes the command' do
+        stub_spawn(cmd, 600, tmp_repo_path, {}, success: true)
+
+        is_expected.to be_truthy
+      end
+
+      it 'fails' do
+        stub_spawn(cmd, 600, tmp_repo_path, {}, success: false)
+
+        is_expected.to be_falsy
+      end
+    end
+
+    context 'with --force' do
+      let(:force) { true }
+      let(:cmd) { %W(git fetch #{remote_name} --prune --quiet --force --tags) }
+
+      it 'executes the command with forced option' do
+        stub_spawn(cmd, 600, tmp_repo_path, {}, success: true)
+
+        is_expected.to be_truthy
+      end
+    end
+
+    context 'with --no-tags' do
+      let(:tags) { false }
+      let(:cmd) { %W(git fetch #{remote_name} --prune --quiet --no-tags) }
+
+      it 'executes the command' do
+        stub_spawn(cmd, 600, tmp_repo_path, {}, success: true)
+
+        is_expected.to be_truthy
+      end
+    end
+
+    describe 'with an SSH key' do
+      let(:extra_args) { { ssh_key: 'SSH KEY' } }
+
+      it 'sets GIT_SSH to a custom script' do
+        script = stub_tempfile('scriptFile', 'gitlab-shell-ssh-wrapper', chmod: 0o755)
+        key = stub_tempfile('/tmp files/keyFile', 'gitlab-shell-key-file', chmod: 0o400)
+
+        stub_spawn(cmd, 600, tmp_repo_path, { 'GIT_SSH' => 'scriptFile' }, success: true)
+
+        is_expected.to be_truthy
+
+        expect(script.string).to eq("#!/bin/sh\nexec ssh '-oIdentityFile=\"/tmp files/keyFile\"' '-oIdentitiesOnly=\"yes\"' \"$@\"")
+        expect(key.string).to eq('SSH KEY')
+      end
+    end
+
+    describe 'with known_hosts data' do
+      let(:extra_args) { { known_hosts: 'KNOWN HOSTS' } }
+
+      it 'sets GIT_SSH to a custom script' do
+        script = stub_tempfile('scriptFile', 'gitlab-shell-ssh-wrapper', chmod: 0o755)
+        key = stub_tempfile('/tmp files/knownHosts', 'gitlab-shell-known-hosts', chmod: 0o400)
+
+        stub_spawn(cmd, 600, tmp_repo_path, { 'GIT_SSH' => 'scriptFile' }, success: true)
+
+        is_expected.to be_truthy
+
+        expect(script.string).to eq("#!/bin/sh\nexec ssh '-oStrictHostKeyChecking=\"yes\"' '-oUserKnownHostsFile=\"/tmp files/knownHosts\"' \"$@\"")
+        expect(key.string).to eq('KNOWN HOSTS')
+      end
+    end
+  end
+
+  describe '#import_project' do
+    let(:project) { create(:project) }
+    let(:import_url) { TestEnv.factory_repo_path_bare }
+    let(:cmd) { %W(git clone --bare -- #{import_url} #{tmp_repo_path}) }
+    let(:timeout) { 600 }
+
+    subject { gl_projects.import_project(import_url, timeout) }
+
+    context 'success import' do
+      it 'imports a repo' do
+        expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
+
+        message = "Importing project from <#{import_url}> to <#{tmp_repo_path}>."
+        expect(logger).to receive(:info).with(message)
+
+        is_expected.to be_truthy
+
+        expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_truthy
+      end
+    end
+
+    context 'already exists' do
+      it "doesn't import" do
+        FileUtils.mkdir_p(tmp_repo_path)
+
+        is_expected.to be_falsy
+      end
+    end
+
+    context 'timeout' do
+      it 'does not import a repo' do
+        stub_spawn_timeout(cmd, timeout, nil)
+
+        message = "Importing project from <#{import_url}> to <#{tmp_repo_path}> failed."
+        expect(logger).to receive(:error).with(message)
+
+        is_expected.to be_falsy
+
+        expect(gl_projects.output).to eq("Timed out\n")
+        expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
+      end
+    end
+  end
+
+  describe '#fork_repository' do
+    let(:dest_repos_path) { tmp_repos_path }
+    let(:dest_repo_name) { File.join('@hashed', 'aa', 'bb', 'xyz.git') }
+    let(:dest_repo) { File.join(dest_repos_path, dest_repo_name) }
+    let(:dest_namespace) { File.dirname(dest_repo) }
+
+    subject { gl_projects.fork_repository(dest_repos_path, dest_repo_name) }
+
+    before do
+      FileUtils.mkdir_p(dest_repos_path)
+    end
+
+    after do
+      FileUtils.rm_rf(dest_repos_path)
+    end
+
+    it 'forks the repository' do
+      message = "Forking repository from <#{tmp_repo_path}> to <#{dest_repo}>."
+      expect(logger).to receive(:info).with(message)
+
+      is_expected.to be_truthy
+
+      expect(File.exist?(dest_repo)).to be_truthy
+      expect(File.exist?(File.join(dest_repo, 'hooks', 'pre-receive'))).to be_truthy
+      expect(File.exist?(File.join(dest_repo, 'hooks', 'post-receive'))).to be_truthy
+    end
+
+    it 'does not fork if a project of the same name already exists' do
+      # create a fake project at the intended destination
+      FileUtils.mkdir_p(dest_repo)
+
+      # trying to fork again should fail as the repo already exists
+      message = "fork-repository failed: destination repository <#{dest_repo}> already exists."
+      expect(logger).to receive(:error).with(message)
+
+      is_expected.to be_falsy
+    end
+
+    context 'different storages' do
+      let(:dest_repos_path) { File.join(File.dirname(tmp_repos_path), 'alternative') }
+
+      it 'forks the repo' do
+        is_expected.to be_truthy
+
+        expect(File.exist?(dest_repo)).to be_truthy
+        expect(File.exist?(File.join(dest_repo, 'hooks', 'pre-receive'))).to be_truthy
+        expect(File.exist?(File.join(dest_repo, 'hooks', 'post-receive'))).to be_truthy
+      end
+    end
+  end
+
+  def build_gitlab_projects(*args)
+    described_class.new(
+      *args,
+      global_hooks_path: Gitlab.config.gitlab_shell.hooks_path,
+      logger: logger
+    )
+  end
+
+  def stub_spawn(*args, success: true)
+    exitstatus = success ? 0 : nil
+    expect(gl_projects).to receive(:popen_with_timeout).with(*args)
+      .and_return(["output", exitstatus])
+  end
+
+  def stub_spawn_timeout(*args)
+    expect(gl_projects).to receive(:popen_with_timeout).with(*args)
+      .and_raise(Timeout::Error)
+  end
+end

spec/lib/gitlab/git/repository_spec.rb

@@ -19,6 +19,51 @@ describe Gitlab::Git::Repository, seed_helper: true do
 
   let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
 
+  describe '.create_hooks' do
+    let(:repo_path) { File.join(TestEnv.repos_path, 'hook-test.git') }
+    let(:hooks_dir) { File.join(repo_path, 'hooks') }
+    let(:target_hooks_dir) { Gitlab.config.gitlab_shell.hooks_path }
+    let(:existing_target) { File.join(repo_path, 'foobar') }
+
+    before do
+      FileUtils.rm_rf(repo_path)
+      FileUtils.mkdir_p(repo_path)
+    end
+
+    context 'hooks is a directory' do
+      let(:existing_file) { File.join(hooks_dir, 'my-file') }
+
+      before do
+        FileUtils.mkdir_p(hooks_dir)
+        FileUtils.touch(existing_file)
+        described_class.create_hooks(repo_path, target_hooks_dir)
+      end
+
+      it { expect(File.readlink(hooks_dir)).to eq(target_hooks_dir) }
+      it { expect(Dir[File.join(repo_path, "hooks.old.*/my-file")].count).to eq(1) }
+    end
+
+    context 'hooks is a valid symlink' do
+      before do
+        FileUtils.mkdir_p existing_target
+        File.symlink(existing_target, hooks_dir)
+        described_class.create_hooks(repo_path, target_hooks_dir)
+      end
+
+      it { expect(File.readlink(hooks_dir)).to eq(target_hooks_dir) }
+    end
+
+    context 'hooks is a broken symlink' do
+      before do
+        FileUtils.rm_f(existing_target)
+        File.symlink(existing_target, hooks_dir)
+        described_class.create_hooks(repo_path, target_hooks_dir)
+      end
+
+      it { expect(File.readlink(hooks_dir)).to eq(target_hooks_dir) }
+    end
+  end
+
   describe "Respond to" do
     subject { repository }
 

spec/support/fixture_helpers.rb

@@ -5,9 +5,19 @@ module FixtureHelpers
     File.read(expand_fixture_path(filename))
   end
 
+  def fixture_file_ee(filename)
+    return '' if filename.blank?
+
+    File.read(expand_fixture_ee_path(filename))
+  end
+
   def expand_fixture_path(filename)
     File.expand_path(Rails.root.join('spec/fixtures/', filename))
   end
+
+  def expand_fixture_ee_path(filename)
+    File.expand_path(Rails.root.join('spec/ee/fixtures/', filename))
+  end
 end
 
 RSpec.configure do |config|

spec/support/stub_env.rb

@@ -17,6 +17,7 @@ module StubENV
 
   def add_stubbed_value(key, value)
     allow(ENV).to receive(:[]).with(key).and_return(value)
+    allow(ENV).to receive(:key?).with(key).and_return(true)
     allow(ENV).to receive(:fetch).with(key).and_return(value)
     allow(ENV).to receive(:fetch).with(key, anything()) do |_, default_val|
       value || default_val
@@ -29,6 +30,7 @@ module StubENV
 
   def init_stub
     allow(ENV).to receive(:[]).and_call_original
+    allow(ENV).to receive(:key?).and_call_original
     allow(ENV).to receive(:fetch).and_call_original
     add_stubbed_value(STUBBED_KEY, true)
   end