Merge remote-tracking branch 'ee/master' into ce-to-ee-2017-12-14

* ee/master:
  Added Authorized Keys specific checks for Geo
  Simplify database replication instructions and make dryer
  Add SAST docs for EE
  Combine ssh docs and rename the doc
  Docs: update documentation guidelines
  Replace gl.utils.formatDate in place of exported function
  Add deprecated page back in
  Docs: add indexes for monitoring and performance monitoring
  Add docs for commit diff discussion in merge requests
  Import gitlab_projects.rb from gitlab-shell
  Show clear message when set-geo-primary-node was successful
  fix formatting of parameters for new group and transfer project to group

app/assets/javascripts/geo_nodes.js

@@ -3,7 +3,7 @@ import axios from 'axios';
 import SmartInterval from '~/smart_interval';
 import { s__ } from '~/locale';
 import { parseSeconds, stringifyTime } from './lib/utils/pretty_time';
-import { timeIntervalInWords } from './lib/utils/datetime_utility';
+import { formatDate, timeIntervalInWords } from './lib/utils/datetime_utility';
 import timeago from './vue_shared/mixins/timeago';
 
 const healthyClass = 'geo-node-healthy';
@@ -115,7 +115,7 @@ class GeoNodeStatus {
     let eventDate = notAvailable;
 
     if (eventTimestamp && eventTimestamp > 0) {
-      eventDate = gl.utils.formatDate(new Date(eventTimestamp * 1000));
+      eventDate = formatDate(new Date(eventTimestamp * 1000));
     }
 
     if (eventId) {

changelogs/unreleased-ee/4090-authorized-keys-check.yml

+---
+title: 'Geo: Added Authorized Keys specific checks'
+merge_request: 3728
+author:
+type: added

changelogs/unreleased-ee/tc-geo-set_primary_node-success.yml

+---
+title: Show clear message when set-geo-primary-node was successful
+merge_request: 3768
+author:
+type: changed

changelogs/unreleased/41016-import-gitlab-shell-projects.yml

+---
+title: Import some code and functionality from gitlab-shell to improve subprocess
+  handling
+merge_request:
+author:
+type: other

doc/README.md

@@ -107,6 +107,7 @@ Manage your [repositories](user/project/repository/index.md) from the UI (user i
   - [Work In Progress Merge Requests](user/project/merge_requests/work_in_progress_merge_requests.md)
   - [Merge Request discussion resolution](user/discussions/index.md#moving-a-single-discussion-to-a-new-issue): Resolve discussions, move discussions in a merge request to an issue, only allow merge requests to be merged if all discussions are resolved.
   - **(EES/EEP)** [Merge Request approval](user/project/merge_requests/merge_request_approvals.md): Make sure every merge request is approved by one or more people before getting merged.
+  - **(EEU)** [Static Application Security Testing](user/project/merge_requests/sast.md): Scan your code for vulnerabilities and display the results in merge requests.
   - [Checkout merge requests locally](user/project/merge_requests/index.md#checkout-merge-requests-locally)
   - [Cherry-pick](user/project/merge_requests/cherry_pick_changes.md)
 - [Milestones](user/project/milestones/index.md): Organize issues and merge requests into a cohesive group, optionally setting a due date.

doc/administration/operations/fast_ssh_key_lookup.md

+# Fast lookup of authorized SSH keys in the database
+
+Regular SSH operations become slow as the number of users grows because OpenSSH
+searches for a key to authorize a user via a linear search. In the worst case,
+such as when the user is not authorized to access GitLab, OpenSSH will scan the
+entire file to search for a key. This can take significant time and disk I/O,
+which will delay users attempting to push or pull to a repository. Making
+matters worse, if users add or remove keys frequently, the operating system may
+not be able to cache the `authorized_keys` file, which causes the disk to be
+accessed repeatedly.
+
+GitLab Shell solves this by providing a way to authorize SSH users via a fast,
+indexed lookup in the GitLab database. This page describes how to enable the fast
+lookup of authorized SSH keys.
+
+> **Warning:** OpenSSH version 6.9+ is required because
+`AuthorizedKeysCommand` must be able to accept a fingerprint. These
+instructions will break installations using older versions of OpenSSH, such as
+those included with CentOS 6 as of September 2017. If you want to use this
+feature for CentOS 6, follow [the instructions on how to build and install a custom OpenSSH package](#compiling-a-custom-version-of-openssh-for-centos-6) before continuing.
+
+
+## Fast lookup is required for GitLab Geo
+
+By default, GitLab manages an `authorized_keys` file, which contains all the
+public SSH keys for users allowed to access GitLab. However, to maintain a
+single source of truth, [Geo](../../gitlab-geo/README.md) needs to be configured to perform SSH fingerprint
+lookups via database lookup.
+
+As part of [setting up GitLab Geo](../../gitlab-geo/README.md#setup-instructions),
+you will be required to follow the steps outlined below for both the primary and
+secondary nodes, but note that the `Write to "authorized keys" file` checkbox
+only needs to be unchecked on the primary node since it will be reflected
+automatically on the secondary if database replication is working.
+
+## Setting up fast lookup via GitLab Shell
+
+GitLab Shell provides a way to authorize SSH users via a fast, indexed lookup
+to the GitLab database. GitLab Shell uses the fingerprint of the SSH key to
+check whether the user is authorized to access GitLab.
+
+Create the directory `/opt/gitlab-shell` first:
+
+```bash
+sudo mkdir -p /opt/gitlab-shell
+```
+
+Create this file at `/opt/gitlab-shell/authorized_keys`:
+
+```
+#!/bin/bash
+
+if [[ "$1" == "git" ]]; then
+  /opt/gitlab/embedded/service/gitlab-shell/bin/authorized_keys $2
+fi
+```
+
+Set appropriate ownership and permissions:
+
+```
+sudo chown root:git /opt/gitlab-shell/authorized_keys
+sudo chmod 0650 /opt/gitlab-shell/authorized_keys
+```
+
+Add the following to `/etc/ssh/sshd_config` or to `/assets/sshd_config` if you
+are using Omnibus Docker:
+
+```
+AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k
+AuthorizedKeysCommandUser git
+```
+
+Reload OpenSSH:
+
+```bash
+# Debian or Ubuntu installations
+sudo service ssh reload
+
+# CentOS installations
+sudo service sshd reload
+```
+
+Confirm that SSH is working by removing your user's SSH key in the UI, adding a
+new one, and attempting to pull a repo.
+
+> **Warning:** Do not disable writes until SSH is confirmed to be working
+perfectly because the file will quickly become out-of-date.
+
+In the case of lookup failures (which are not uncommon), the `authorized_keys`
+file will still be scanned. So git SSH performance will still be slow for many
+users as long as a large file exists.
+
+You can disable any more writes to the `authorized_keys` file by unchecking
+`Write to "authorized_keys" file` in the Application Settings of your GitLab
+installation.
+
+![Write to authorized keys setting](img/write_to_authorized_keys_setting.png)
+
+Again, confirm that SSH is working by removing your user's SSH key in the UI,
+adding a new one, and attempting to pull a repo.
+
+Then you can backup and delete your `authorized_keys` file for best performance.
+
+## How to go back to using the `authorized_keys` file
+
+This is a brief overview. Please refer to the above instructions for more context.
+
+1. [Rebuild the `authorized_keys` file](../raketasks/maintenance.md#rebuild-authorized_keys-file)
+1. Enable writes to the `authorized_keys` file in Application Settings
+1. Remove the `AuthorizedKeysCommand` lines from `/etc/ssh/sshd_config` or from `/assets/sshd_config` if you are using Omnibus Docker.
+1. Reload sshd: `sudo service sshd reload`
+1. Remove the `/opt/gitlab-shell/authorized_keys` file
+
+## Compiling a custom version of OpenSSH for CentOS 6
+
+Building a custom version of OpenSSH is not necessary for Ubuntu 16.04 users,
+since Ubuntu 16.04 ships with OpenSSH 7.2.
+
+It is also unnecessary for CentOS 7.4 users, as that version ships with
+OpenSSH 7.4. If you are using CentOS 7.0 - 7.3, we strongly recommend that you
+upgrade to CentOS 7.4 instead of following this procedure. This should be as
+simple as running `yum update`.
+
+CentOS 6 users must build their own OpenSSH package to enable SSH lookups via
+the database. The following instructions can be used to build OpenSSH 7.5:
+
+1. First, download the package and install the required packages:
+
+    ```
+    sudo su -
+    cd /tmp
+    curl --remote-name https://mirrors.evowise.com/pub/OpenBSD/OpenSSH/portable/openssh-7.5p1.tar.gz
+    tar xzvf openssh-7.5p1.tar.gz
+    yum install rpm-build gcc make wget openssl-devel krb5-devel pam-devel libX11-devel xmkmf libXt-devel
+    ```
+
+3. Prepare the build by copying files to the right place:
+
+    ```
+    mkdir -p /root/rpmbuild/{SOURCES,SPECS}
+    cp ./openssh-7.5p1/contrib/redhat/openssh.spec /root/rpmbuild/SPECS/
+    cp openssh-7.5p1.tar.gz /root/rpmbuild/SOURCES/
+    cd /root/rpmbuild/SPECS
+    ```
+
+3. Next, set the spec settings properly:
+
+    ```
+    sed -i -e "s/%define no_gnome_askpass 0/%define no_gnome_askpass 1/g" openssh.spec
+    sed -i -e "s/%define no_x11_askpass 0/%define no_x11_askpass 1/g" openssh.spec
+    sed -i -e "s/BuildPreReq/BuildRequires/g" openssh.spec
+    ```
+
+3. Build the RPMs:
+
+    ```
+    rpmbuild -bb openssh.spec
+    ```
+
+4. Ensure the RPMs were built:
+
+    ```
+    ls -al /root/rpmbuild/RPMS/x86_64/
+    ```
+
+    You should see something as the following:
+
+    ```
+    total 1324
+    drwxr-xr-x. 2 root root   4096 Jun 20 19:37 .
+    drwxr-xr-x. 3 root root     19 Jun 20 19:37 ..
+    -rw-r--r--. 1 root root 470828 Jun 20 19:37 openssh-7.5p1-1.x86_64.rpm
+    -rw-r--r--. 1 root root 490716 Jun 20 19:37 openssh-clients-7.5p1-1.x86_64.rpm
+    -rw-r--r--. 1 root root  17020 Jun 20 19:37 openssh-debuginfo-7.5p1-1.x86_64.rpm
+    -rw-r--r--. 1 root root 367516 Jun 20 19:37 openssh-server-7.5p1-1.x86_64.rpm
+    ```
+
+5. Install the packages. OpenSSH packages will replace `/etc/pam.d/sshd`
+   with its own version, which may prevent users from logging in, so be sure
+   that the file is backed up and restored after installation:
+
+    ```
+    timestamp=$(date +%s)
+    cp /etc/pam.d/sshd pam-ssh-conf-$timestamp
+    rpm -Uvh /root/rpmbuild/RPMS/x86_64/*.rpm
+    yes | cp pam-ssh-conf-$timestamp /etc/pam.d/sshd
+    ```
+
+6. Verify the installed version. In another window, attempt to login to the server:
+
+    ```
+    ssh -v <your-centos-machine>
+    ```
+
+    You should see a line that reads: "debug1: Remote protocol version 2.0, remote software version OpenSSH_7.5"
+
+    If not, you may need to restart sshd (e.g. `systemctl restart sshd.service`).
+
+7.  *IMPORTANT!* Open a new SSH session to your server before exiting to make
+    sure everything is working! If you need to downgrade, simple install the
+    older package:
+
+    ```
+    # Only run this if you run into a problem logging in
+    yum downgrade openssh-server openssh openssh-clients
+    ```

doc/administration/operations/index.md

@@ -15,5 +15,4 @@ that to prioritize important jobs.
 to restart Sidekiq.
 - **(EES/EEP)** [Extra Sidekiq operations](extra_sidekiq_processes.md): Configure an extra set of Sidekiq processes to ensure certain queues always have dedicated workers, no matter the amount of jobs that need to be processed.
 - [Unicorn](unicorn.md): Understand Unicorn and unicorn-worker-killer.
-- **(EES/EEP)** [Speed up SSH operations](speed_up_ssh.md): Authorize SSH users via a fast, indexed lookup to the GitLab database.
-- [Unicorn](unicorn.md): Understand Unicorn and unicorn-worker-killer.
+- **(EES/EEP)** [Speed up SSH operations](fast_ssh_key_lookup.md): Authorize SSH users via a fast, indexed lookup to the GitLab database.

doc/administration/operations/speed_up_ssh.md

-# Speed up SSH operations
-
->
-- [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/250) in GitLab Enterprise Edition 8.7.
-- Available in GitLab Enterprise Edition Starter.
-
-## The problem
-
-SSH operations become slow as the number of users grows.
-
-## The reason
-
-OpenSSH searches for a key to authorize a user via a linear search. In the worst case, such as when the user is not authorized to access GitLab, OpenSSH will scan the entire file to search for a key. This can take significant time and disk I/O, which will delay users attempting to push or pull to a repository. Making matters worse, if users add or remove keys frequently, the operating system may not be able to cache the authorized_keys file, which causes the disk to be accessed repeatedly.
-
-## The solution
-
-GitLab Shell provides a way to authorize SSH users via a fast, indexed lookup to the GitLab database. GitLab Shell uses the fingerprint of the SSH key to check whether the user is authorized to access GitLab.
-
-> **Warning:** OpenSSH version 6.9+ is required because
-`AuthorizedKeysCommand` must be able to accept a fingerprint. These
-instructions will break installations using older versions of OpenSSH, such as
-those included with CentOS 6 as of September 2017. If you want to use this
-feature for CentOS 6, follow [the instructions on how to build and install a custom OpenSSH package]
-(#compiling-a-custom-version-of-openssh-for-centos-6) before continuing.
-
-Create the directory `/opt/gitlab-shell` first:
-
-```bash
-sudo mkdir -p /opt/gitlab-shell
-```
-
-Create this file at `/opt/gitlab-shell/authorized_keys`:
-
-```
-#!/bin/bash
-
-if [[ "$1" == "git" ]]; then
-  /opt/gitlab/embedded/service/gitlab-shell/bin/authorized_keys $2
-fi
-```
-
-Set appropriate ownership and permissions:
-
-```
-sudo chown root:git /opt/gitlab-shell/authorized_keys
-sudo chmod 0650 /opt/gitlab-shell/authorized_keys
-```
-
-Add the following to `/etc/ssh/sshd_config` or to `/assets/sshd_config` if you are using Omnibus Docker:
-
-```
-AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k
-AuthorizedKeysCommandUser git
-```
-
-Reload OpenSSH:
-
-```bash
-# Debian or Ubuntu installations
-sudo service ssh reload
-
-# CentOS installations
-sudo service sshd reload
-```
-
-Confirm that SSH is working by removing your user's SSH key in the UI, adding a new one, and attempting to pull a repo.
-
-> **Warning:** Do not disable writes until SSH is confirmed to be working perfectly because the file will quickly become out-of-date.
-
-In the case of lookup failures (which are not uncommon), the `authorized_keys` file will still be scanned. So git SSH performance will still be slow for many users as long as a large file exists.
-
-You can disable any more writes to the `authorized_keys` file by unchecking `Write to "authorized_keys" file` in the Application Settings of your GitLab installation.
-
-![Write to authorized keys setting](img/write_to_authorized_keys_setting.png)
-
-Again, confirm that SSH is working by removing your user's SSH key in the UI, adding a new one, and attempting to pull a repo.
-
-Then you can backup and delete your `authorized_keys` file for best performance.
-
-## How to go back to using the `authorized_keys` file
-
-This is a brief overview. Please refer to the above instructions for more context.
-
-1. [Rebuild the `authorized_keys` file](../raketasks/maintenance.md#rebuild-authorized_keys-file)
-1. Enable writes to the `authorized_keys` file in Application Settings
-1. Remove the `AuthorizedKeysCommand` lines from `/etc/ssh/sshd_config` or from `/assets/sshd_config` if you are using Omnibus Docker.
-1. Reload sshd: `sudo service sshd reload`
-1. Remove the `/opt/gitlab-shell/authorized_keys` file
-
-## Compiling a custom version of OpenSSH for CentOS 6
-
-Building a custom version of OpenSSH is not necessary for Ubuntu 16.04 users,
-since Ubuntu 16.04 ships with OpenSSH 7.2.
-
-It is also unnecessary for CentOS 7.4 users, as that version ships with
-OpenSSH 7.4. If you are using CentOS 7.0 - 7.3, we strongly recommend that you
-upgrade to CentOS 7.4 instead of following this procedure. This should be as
-simple as running `yum update`.
-
-CentOS 6 users must build their own OpenSSH package to enable SSH lookups via
-the database. The following instructions can be used to build OpenSSH 7.5:
-
-1. First, download the package and install the required packages:
-
-    ```
-    sudo su -
-    cd /tmp
-    curl --remote-name https://mirrors.evowise.com/pub/OpenBSD/OpenSSH/portable/openssh-7.5p1.tar.gz
-    tar xzvf openssh-7.5p1.tar.gz
-    yum install rpm-build gcc make wget openssl-devel krb5-devel pam-devel libX11-devel xmkmf libXt-devel
-    ```
-
-3. Prepare the build by copying files to the right place:
-
-    ```
-    mkdir -p /root/rpmbuild/{SOURCES,SPECS}
-    cp ./openssh-7.5p1/contrib/redhat/openssh.spec /root/rpmbuild/SPECS/
-    cp openssh-7.5p1.tar.gz /root/rpmbuild/SOURCES/
-    cd /root/rpmbuild/SPECS
-    ```
-
-3. Next, set the spec settings properly:
-
-    ```
-    sed -i -e "s/%define no_gnome_askpass 0/%define no_gnome_askpass 1/g" openssh.spec
-    sed -i -e "s/%define no_x11_askpass 0/%define no_x11_askpass 1/g" openssh.spec
-    sed -i -e "s/BuildPreReq/BuildRequires/g" openssh.spec
-    ```
-
-3. Build the RPMs:
-
-    ```
-    rpmbuild -bb openssh.spec
-    ```
-
-4. Ensure the RPMs were built:
-
-    ```
-    ls -al /root/rpmbuild/RPMS/x86_64/
-    ```
-
-    You should see something as the following:
-
-    ```
-    total 1324
-    drwxr-xr-x. 2 root root   4096 Jun 20 19:37 .
-    drwxr-xr-x. 3 root root     19 Jun 20 19:37 ..
-    -rw-r--r--. 1 root root 470828 Jun 20 19:37 openssh-7.5p1-1.x86_64.rpm
-    -rw-r--r--. 1 root root 490716 Jun 20 19:37 openssh-clients-7.5p1-1.x86_64.rpm
-    -rw-r--r--. 1 root root  17020 Jun 20 19:37 openssh-debuginfo-7.5p1-1.x86_64.rpm
-    -rw-r--r--. 1 root root 367516 Jun 20 19:37 openssh-server-7.5p1-1.x86_64.rpm
-    ```
-
-5. Install the packages. OpenSSH packages will replace `/etc/pam.d/sshd`
-   with its own version, which may prevent users from logging in, so be sure
-   that the file is backed up and restored after installation:
-
-    ```
-    timestamp=$(date +%s)
-    cp /etc/pam.d/sshd pam-ssh-conf-$timestamp
-    rpm -Uvh /root/rpmbuild/RPMS/x86_64/*.rpm
-    yes | cp pam-ssh-conf-$timestamp /etc/pam.d/sshd
-    ```
-
-6. Verify the installed version. In another window, attempt to login to the server:
-
-    ```
-    ssh -v <your-centos-machine>
-    ```
-
-    You should see a line that reads: "debug1: Remote protocol version 2.0, remote software version OpenSSH_7.5"
-
-    If not, you may need to restart sshd (e.g. `systemctl restart sshd.service`).
-
-7.  *IMPORTANT!* Open a new SSH session to your server before exiting to make
-    sure everything is working! If you need to downgrade, simple install the
-    older package:
-
-    ```
-    # Only run this if you run into a problem logging in
-    yum downgrade openssh-server openssh openssh-clients
-    ```
+This document was moved to [another location](fast_ssh_key_lookup.md).

doc/api/groups.md

@@ -366,16 +366,16 @@ POST /groups
 
 Parameters:
 
-- `name` (required) - The name of the group
-- `path` (required) - The path of the group
-- `description` (optional) - The group's description
-- `membership_lock` (optional, boolean) - Prevent adding new members to project membership within this group
-- `share_with_group_lock` (optional, boolean) - Prevent sharing a project with another group within this group
-- `visibility` (optional) - The group's visibility. Can be `private`, `internal`, or `public`.
-- `lfs_enabled` (optional)      - Enable/disable Large File Storage (LFS) for the projects in this group
-- `request_access_enabled` (optional) - Allow users to request member access.
-- `parent_id` (optional) - The parent group id for creating nested group.
-- `shared_runners_minutes_limit` (optional) - (admin-only) Pipeline minutes quota for this group
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `name` | string | yes | The name of the group |
+| `path` | string | yes | The path of the group |
+| `description` | string | no | The group's description |
+| `visibility` | string | no | The group's visibility. Can be `private`, `internal`, or `public`. |
+| `lfs_enabled` | boolean | no | Enable/disable Large File Storage (LFS) for the projects in this group |
+| `request_access_enabled` | boolean | no | Allow users to request member access. |
+| `parent_id` | integer | no | The parent group id for creating nested group. |
+| `shared_runners_minutes_limit` | integer | no | (admin-only) Pipeline minutes quota for this group. |
 
 ## Transfer project to group
 
@@ -387,8 +387,10 @@ POST  /groups/:id/projects/:project_id
 
 Parameters:
 
-- `id` (required) - The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user
-- `project_id` (required) - The ID or path of a project
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user |
+| `project_id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) |
 
 ## Update group
 

doc/ci/examples/README.md

@@ -58,6 +58,10 @@ Apart from those, here is an collection of tutorials and guides on setting up yo
 
 - [Analyze code quality with the Code Climate CLI](code_climate.md)
 
+### Static Application Security Testing (SAST)
+
+- [Scan your code for vulnerabilities](sast.md)
+
 ### Other
 
 - [Using `dpl` as deployment tool](deployment/README.md)

doc/ci/examples/sast.md

+# Static application security testing with GitLab CI/CD
+
+NOTE: **Note:**
+In order to use this tool, a [GitLab Enterprise Edition Ultimate][ee] license
+is needed.
+
+This example shows how to run
+[Static Application Security Testing (SAST)](https://en.wikipedia.org/wiki/Static_program_analysis)
+on your project's source code by using GitLab CI/CD.
+
+All you need is a GitLab Runner with the Docker executor (the shared Runners on
+GitLab.com will work fine). You can then add a new job to `.gitlab-ci.yml`,
+called `sast`:
+
+```yaml
+sast:
+  image: registry.gitlab.com/gitlab-org/gl-sast:latest
+  script:
+    - /app/bin/run .
+  artifacts:
+    paths: [gl-sast-report.json]
+```
+
+Behind the scenes, the [gl-sast Docker image](https://gitlab.com/gitlab-org/gl-sast)
+is used to detect the language/framework and in turn runs the matching scan tool.
+
+The above example will create a `sast` job in your CI pipeline and will allow
+you to download and analyze the report artifact in JSON format.
+
+The results are sorted by the priority of the vulnerability:
+
+1. High
+1. Medium
+1. Low
+1. Unknown
+1. Everything else
+
+TIP: **Tip:**
+Starting with GitLab Enterprise Edition Ultimate 10.3, this information will
+be automatically extracted and shown right in the merge request widget. To do
+so, the CI job must be named `sast` and the artifact path must be
+`gl-sast-report.json`.
+[Learn more on application security testing results shown in merge requests](../../user/project/merge_requests/sast.md).
+
+## Supported languages and frameworks
+
+The following languages and frameworks are supported.
+
+| Language / framework | Scan tool |
+| -------------------- | --------- |
+| JavaScript    | [Retire.js](https://retirejs.github.io/retire.js)
+| Python        | [bandit](https://github.com/openstack/bandit) |
+| Ruby          | [bundler-audit](https://github.com/rubysec/bundler-audit) |
+| Ruby on Rails | [brakeman](https://brakemanscanner.org) |
+
+[ee]: https://about.gitlab.com/gitlab-ee/

doc/development/README.md

@@ -81,10 +81,9 @@ comments: false
 
 ## Documentation guides
 
-- [Documentation styleguide](doc_styleguide.md): Use this styleguide if you are
-  contributing to the documentation.
 - [Writing documentation](writing_documentation.md)
-  - [Distinction between general documentation and technical articles](writing_documentation.md#distinction-between-general-documentation-and-technical-articles)
+- [Documentation styleguide](doc_styleguide.md)
+- [Markdown](../user/markdown.md)
 
 ## Internationalization (i18n) guides
 

doc/gitlab-geo/README.md

@@ -86,7 +86,7 @@ current version of OpenSSH:
 
 Note that CentOS 6 and 7.0 ship with an old version of OpenSSH that do not
 support a feature that Geo requires. See the [documentation on GitLab Geo SSH
-access](ssh.md) for more details.
+access](../administration/operations/fast_ssh_key_lookup.md) for more details.
 
 ### LDAP
 
@@ -145,8 +145,8 @@ If you installed GitLab using the Omnibus packages (highly recommended):
    Geo node to unlock GitLab Geo.
 1. [Setup the database replication](database.md) (`primary (read-write) <->
    secondary (read-only)` topology).
-1. [Lookup authorized SSH keys in the database](../administration/operations/speed_up_ssh.html),
-   do this step for both primary AND secondary nodes.
+1. [Configure fast lookup of authorized SSH keys in the database](../administration/operations/fast_ssh_key_lookup.md),
+   this step is required and needs to be done on both the primary AND secondary nodes.
 1. [Configure GitLab](configuration.md) to set the primary and secondary nodes.
 1. Optional: [Configure a secondary LDAP server](../administration/auth/ldap.md)
    for the secondary. See [notes on LDAP](#ldap).
@@ -165,7 +165,7 @@ If you installed GitLab from source:
    Geo node to unlock GitLab Geo.
 1. [Setup the database replication](database_source.md) (`primary (read-write)
    <-> secondary (read-only)` topology).
-1. [Lookup authorized SSH keys in the database](../administration/operations/speed_up_ssh.html),
+1. [Configure fast lookup of authorized SSH keys in the database](../administration/operations/fast_ssh_key_lookup.md),
    do this step for both primary AND secondary nodes.
 1. [Configure GitLab](configuration_source.md) to set the primary and secondary
    nodes.

doc/gitlab-geo/after_setup.md

+This document was moved to [another location](using_a_geo_server.md).

doc/gitlab-geo/database_source.md

@@ -6,12 +6,12 @@ using the Omnibus GitLab packages, follow the
 [**database replication for Omnibus GitLab**](database.md) guide.
 
 >**Note:**
-Stages of the setup process must be completed in the documented order.
+The stages of the setup process must be completed in the documented order.
 Before attempting the steps in this stage, [complete all prior stages][toc].
 
 This document describes the minimal steps you have to take in order to
-replicate your GitLab database into another server. You may have to change
-some values according to your database setup, how big it is, etc.
+replicate your primary GitLab database to a secondary node's database. You may
+have to change some values according to your database setup, how big it is, etc.
 
 You are encouraged to first read through all the steps before executing them
 in your testing/production environment.
@@ -26,19 +26,19 @@ connect to secondary database servers (which are read-only too).
 In many databases documentation you will see "primary" being referenced as "master"
 and "secondary" as either "slave" or "standby" server (read-only).
 
-Since GitLab 9.4: We recommend using [PostgreSQL replication
+We recommend using [PostgreSQL replication
 slots](https://medium.com/@tk512/replication-slots-in-postgresql-b4b03d277c75)
 to ensure the primary retains all the data necessary for the secondaries to
 recover. See below for more details.
 
-### Prerequisites
-
 The following guide assumes that:
 
-- You are using PostgreSQL 9.6 or later which includes the [`pg_basebackup` tool][pgback].
-- You have a primary server already set up (the GitLab server you are
-  replicating from), and you have a new secondary server set up on the same OS
-  and PostgreSQL version. Also make sure the GitLab version is the same on all nodes.
+- You are using PostgreSQL 9.6 or later
+  which includes the [`pg_basebackup` tool][pgback].
+- You have a primary node already set up (the GitLab server you are
+  replicating from), running PostgreSQL 9.6 or later, and
+  you have a new secondary server set up with the same versions of the OS,
+  PostgreSQL, and GitLab on all nodes.
 - The IP of the primary server for our examples will be `1.2.3.4`, whereas the
   secondary's IP will be `5.6.7.8`. Note that the primary and secondary servers
   **must** be able to communicate over these addresses. These IP addresses can either
@@ -48,7 +48,7 @@ The following guide assumes that:
 
 1. SSH into your GitLab **primary** server and login as root:
 
-    ```
+    ```bash
     sudo -i
     ```
 
@@ -58,13 +58,14 @@ The following guide assumes that:
     bundle exec rake geo:set_primary_node
     ```
 
-1. Create a replication user named `gitlab_replicator`:
+1. Create a [replication user](https://wiki.postgresql.org/wiki/Streaming_Replication) named `gitlab_replicator`:
 
     ```bash
     sudo -u postgres psql -c "CREATE USER gitlab_replicator REPLICATION ENCRYPTED PASSWORD 'thepassword';"
     ```
 
 1. Set up TLS support for the PostgreSQL primary server
+
     > **Warning**: Only skip this step if you **know** that PostgreSQL traffic
     > between the primary and secondary will be secured through some other
     > means, e.g., a known-safe physical network path or a site-to-site VPN that
@@ -185,7 +186,7 @@ The following guide assumes that:
     zone, but when the respective times are converted to UTC time, the clocks
     must be synchronized to within 60 seconds of each other.
 
-    If you are using Ubuntu, verify NTP sync is enabled:
+    Verify NTP sync is enabled, using:
 
     ```bash
     timedatectl status | grep 'NTP synchronized'
@@ -196,57 +197,15 @@ The following guide assumes that:
 
 ### Step 2. Add the secondary GitLab node
 
-To prevent the secondary geo node trying to act as the primary once the
-database is replicated, the secondary geo node must be configured on the
-primary before the database is replicated.
-
-1. Visit the **primary** node's **Admin Area ➔ Geo Nodes**
-   (`/admin/geo_nodes`) in your browser.
-1. Add the secondary node by providing its full URL. **Do NOT** check the box
-   'This is a primary node'.
-1. Optionally, choose which namespaces should be replicated by the
-   secondary node. Leave blank to replicate all. Read more in
-   [selective replication](#selective-replication).
-1. Click the **Add node** button.
-
-The new secondary geo node will have the status **Unhealthy**. This is expected
-because we have not yet configured the secondary server. This is the next step.
+Follow the steps in ["add the secondary GitLab node"](database.md#step-2-add-the-secondary-gitlab-node).
 
 ### Step 3. Configure the secondary server
 
-1. SSH into your GitLab **secondary** server and login as root:
-
-    ```
-    sudo -i
-    ```
-
-1. Set up PostgreSQL TLS verification on the secondary
-
-    Copy the generated `server.crt` file onto the secondary server from the
-    primary, then install it in the right place:
-
-    ```bash
-    install -D -o postgres -g postgres -m 0400 -T server.crt ~postgres/.postgresql/root.crt
-    ```
-
-    PostgreSQL will now only recognize that exact certificate when verifying TLS
-    connections. The certificate can only be replicated by someone with access
-    to the private key, which is **only** present on the primary node.
-
-1. Test that the remote connection to the primary server works:
-
-    ```
-    sudo -u postgres psql --list -U gitlab_replicator -d "dbname=gitlabhq_production sslmode=verify-ca" -W -h 1.2.3.4
-    ```
-
-    When prompted enter the password you set in the first step for the
-    `gitlab_replicator` user. If all worked correctly, you should see the
-    database prompt.
-
-    A failure to connect here indicates that the TLS or networking configuration
-    is incorrect. Ensure that you've used the correct certificates and IP
-    addresses throughout. If you have a firewall, ensure that the secondary is
-    permitted to access the primary on port 5432.
+Follow the first steps in ["configure the secondary server"](database.md#step-3-configure-the-secondary-server),
+but note that since you are installing from source, the username and
+group listed as `gitlab-psql` in those steps should be replaced by `postgres`
+instead. After completing the "Test that the remote connection to the
+primary server works" step, continue here:
 
 1. Edit `postgresql.conf` to configure the secondary for streaming replication
    (for Debian/Ubuntu that would be `/etc/postgresql/9.*/main/postgresql.conf`):
@@ -261,45 +220,26 @@ because we have not yet configured the secondary server. This is the next step.
 
 1. Restart PostgreSQL for the changes to take effect.
 
-1. Optional since GitLab 9.1, and required for GitLab 10.0 or higher:
-   [Enable tracking database on the secondary server](#enable-tracking-database-on-the-secondary-server)
+1. Verify that clock synchronization is enabled, using:
 
-1. Otherwise, continue to [initiate the replication process](#step-3-initiate-the-replication-process).
+    ```bash
+    timedatectl status | grep 'NTP synchronized'
+    ```
 
 #### Enable tracking database on the secondary server
 
 Geo secondary nodes use a tracking database to keep track of replication status
-and recover automatically from some replication issues.
-
-It is added in GitLab 9.1, and since GitLab 10.0 it is required.
-
-1. Verify that clock synchronization is enabled.
-
-    >**Important:**
-    For Geo to work correctly, all nodes must have their clocks
-    synchronized. It is not required for all nodes to be set to the same time
-    zone, but when the respective times are converted to UTC time, the clocks
-    must be synchronized to within 60 seconds of each other.
+and recover automatically from some replication issues. Follow the steps below to create
+the tracking database.
 
-    If you are using Ubuntu, verify NTP sync is enabled:
+1. On the secondary node, run the following command to create `database_geo.yml` with the
+information of your secondary PostgreSQL instance:
 
     ```bash
-    timedatectl status | grep 'NTP synchronized'
-    ```
-
-    Refer to your Linux distribution documentation to setup clock
-    synchronization. This can easily be done using any NTP-compatible daemon.
-
-1. Create `database_geo.yml` with the information of your secondary PostgreSQL
-   database.  Note that GitLab will set up another database instance separate
-   from the primary, since this is where the secondary will track its internal
-   state:
-
-    ```
     sudo cp /home/git/gitlab/config/database_geo.yml.postgresql /home/git/gitlab/config/database_geo.yml
     ```
 
-1. Edit the content of `database_geo.yml` in `production:` like the example below:
+1. Edit the content of `database_geo.yml` in `production:` as in the example below:
 
     ```yaml
     #
@@ -315,19 +255,20 @@ It is added in GitLab 9.1, and since GitLab 10.0 it is required.
       host: /var/opt/gitlab/geo-postgresql
     ```
 
-1. Create the database `gitlabhq_geo_production` in that PostgreSQL
-   instance.
+1. Create the database `gitlabhq_geo_production` on the PostgreSQL instance of the secondary
+node.
 
 1. Set up the Geo tracking database:
 
-    ```
+    ```bash
     bundle exec rake geo:db:migrate
     ```
 
 ### Step 4. Initiate the replication process
 
-Below we provide a script that connects to the primary server, replicates the
-database and creates the needed files for replication.
+Below we provide a script that connects the database on the secondary node to
+the database on the primary node, replicates the database, and creates the
+needed files for streaming replication.
 
 The directories used are the defaults for Debian/Ubuntu. If you have changed
 any defaults, configure it as you see fit replacing the directories and paths.
@@ -413,7 +354,7 @@ The replication process is now over.
 
 ## MySQL replication
 
-We don't support MySQL replication for GitLab Geo.
+MySQL replication is not supported for GitLab Geo.
 
 ## Troubleshooting
 

doc/gitlab-geo/ssh.md

-# GitLab Geo SSH access
-
-By default, GitLab manages an `authorized_keys` file, which contains all the
-public SSH keys for users allowed to access GitLab. However, to maintain a
-single source of truth, Geo needs to be configured to perform SSH fingerprint
-lookups via database lookup. This approach is also much faster than scanning a
-file.
-
->**Note:**
-GitLab 10.0 and higher require database lookups for SSH keys.
-
-Note this feature is only available on operating systems that support OpenSSH
-6.9 and above. For CentOS 6, see the [instructions on building custom
-version of OpenSSH for your server]
-(../administration/operations/speed_up_ssh.html#compiling-a-custom-version-of-openssh-for-centos-6).
-
-For both primary AND secondary nodes, follow the instructions on
-[looking up authorized SSH keys in the database](../administration/operations/speed_up_ssh.html).
-
-Note that the 'Write to "authorized keys" file' checkbox only needs
-to be unchecked on the primary node since it will be reflected automatically
-in the secondary if database replication is working.
+This document was moved to [another location](../administration/operations/fast_ssh_key_lookup.md).

doc/gitlab-geo/updating_the_geo_nodes.md

@@ -93,7 +93,7 @@ for existing repositories was added in GitLab 10.1.
 ## Upgrading to GitLab 10.0
 
 Since GitLab 10.0, we require all **Geo** systems to [use SSH key lookups via
-the database](ssh.md) to avoid having to maintain consistency of the
+the database](../administration/operations/fast_ssh_key_lookup.md) to avoid having to maintain consistency of the
 `authorized_keys` file for SSH access. Failing to do this will prevent users
 from being able to clone via SSH.
 

doc/topics/autodevops/index.md

@@ -19,6 +19,7 @@ project in an easy and automatic way:
 1. [Auto Build](#auto-build)
 1. [Auto Test](#auto-test)
 1. [Auto Code Quality](#auto-code-quality)
+1. [Auto SAST (Static Application Security Testing)](#auto-sast)
 1. [Auto Review Apps](#auto-review-apps)
 1. [Auto Deploy](#auto-deploy)
 1. [Auto Monitoring](#auto-monitoring)
@@ -198,6 +199,18 @@ out. In GitLab Enterprise Edition Starter, differences between the source and
 target branches are
 [shown in the merge request widget](../../user/project/merge_requests/code_quality_diff.md).
 
+### Auto SAST
+
+> Introduced in [GitLab Enterprise Edition Ultimate][ee] 10.3.
+
+Static Application Security Testing (SAST) uses the
+[gl-sast Docker image](https://gitlab.com/gitlab-org/gl-sast) to run static
+analysis on the current code and checks for potential security issues. Once the
+report is created, it's uploaded as an artifact which you can later download and
+check out.
+
+Any security warnings are also [shown in the merge request widget](../../user/project/merge_requests/sast.md).
+
 ### Auto Review Apps
 
 NOTE: **Note:**
@@ -536,3 +549,4 @@ curl --data "value=true" --header "PRIVATE-TOKEN: personal_access_token" https:/
 [postgresql]: https://www.postgresql.org/
 [Auto DevOps template]: https://gitlab.com/gitlab-org/gitlab-ci-yml/blob/master/Auto-DevOps.gitlab-ci.yml
 [GitLab Omnibus Helm Chart]: ../../install/kubernetes/gitlab_omnibus.md
+[ee]: https://about.gitlab.com/gitlab-ee/

doc/user/index.md

@@ -55,6 +55,7 @@ and [Multiple Issue Boards](https://docs.gitlab.com/ee/user/project/issue_board.
 - [Lock files](https://docs.gitlab.com/ee/user/project/file_lock.html) to prevent conflicts
 - View of the current health and status of each CI environment running on Kubernetes with [Deploy Boards](https://docs.gitlab.com/ee/user/project/deploy_boards.html)
 - Leverage your continuous delivery method with [Canary Deployments](https://docs.gitlab.com/ee/user/project/canary_deployments.html)
+- Scan your code for vulnerabilities and [display them in merge requests](project/merge_requests/sast.md).
 
 You can also [integrate](project/integrations/project_services.md) GitLab with numerous third-party applications, such as Mattermost, Microsoft Teams, HipChat, Trello, Slack, Bamboo CI, JIRA, and a lot more.
 

doc/user/project/merge_requests/index.md

@@ -199,6 +199,17 @@ can show the Code Climate report right in the merge request widget area.
 
 [Read more about Code Quality reports.](code_quality_diff.md)
 
+## Static Application Security Testing
+
+> Introduced in [GitLab Enterprise Edition Ultimate][products] 10.3.
+
+If you are using [GitLab CI/CD][ci], you can analyze your source code for known
+vulnerabilities using Static Application Security Testing (SAST).
+Going a step further, GitLab can show the vulnerability report right in the
+merge request widget area.
+
+[Read more about Static Application Security Testing reports.](sast.md)
+
 ## Live preview with Review Apps
 
 If you configured [Review Apps](https://about.gitlab.com/features/review-apps/) for your project,

doc/user/project/merge_requests/sast.md

+# Static Application Security Testing (SAST)
+
+> [Introduced][ee-3775] in [GitLab Enterprise Edition Ultimate][ee] 10.3.
+
+## Overview
+
+If you are using [GitLab CI/CD][ci], you can analyze your source code for known
+vulnerabilities using Static Application Security Testing (SAST).
+
+Going a step further, GitLab can show the vulnerability list right in the merge
+request widget area:
+
+![SAST Widget](img/sast.png)
+
+## Use cases
+
+- Your application is using an external (open source) library, locked to a
+  specific version (e.g., via `Gemfile.lock`) and the version is known to be
+  vulnerable.
+- Your code has a potentially dangerous attribute in a class, or unsafe code
+  that can lead to unintended code execution.
+
+## How it works
+
+In order for the report to show in the merge request, you need to specify a
+`sast` job (exact name) that will analyze the code and upload the resulting
+`gl-sast-report.json` file as an artifact. GitLab will then check this file and
+show the information inside the merge request.
+
+This JSON file needs to be the only artifact file for the job. If you try
+to also include other files, it will break the vulnerability display in the
+merge request.
+
+For more information on how the `sast` job should look like, check the
+example on [analyzing a project's code for vulnerabilities][cc-docs].
+
+[ee-3775]: https://gitlab.com/gitlab-org/gitlab-ee/issues/3775
+[ee]: https://about.gitlab.com/gitlab-ee/
+[ci]: ../../../ci/README.md
+[cc-docs]: ../../../ci/examples/sast.md

ee/lib/gitlab/geo/geo_tasks.rb

@@ -5,10 +5,14 @@ module Gitlab
 
       def set_primary_geo_node
         node = GeoNode.new(primary: true, url: GeoNode.current_node_url)
-        $stdout.puts "Saving primary GeoNode with URL #{node.url}".color(:green)
+        $stdout.puts "Saving primary Geo node with URL #{node.url} ..."
         node.save
 
-        $stdout.puts "Error saving GeoNode:\n#{node.errors.full_messages.join("\n")}".color(:red) unless node.persisted?
+        if node.persisted?
+          $stdout.puts "#{node.url} is now the primary Geo node".color(:green)
+        else
+          $stdout.puts "Error saving Geo node:\n#{node.errors.full_messages.join("\n")}".color(:red)
+        end
       end
 
       def refresh_foreign_tables!

ee/lib/system_check/geo/authorized_keys_check.rb

+module SystemCheck
+  module Geo
+    class AuthorizedKeysCheck < ::SystemCheck::BaseCheck
+      set_name 'OpenSSH configured to use AuthorizedKeysCommand'
+
+      AUTHORIZED_KEYS_DOCS = 'doc/administration/operations/speed_up_ssh.md'.freeze
+      OPENSSH_AUTHORIZED_KEYS_CMD_REGEXP = %r{
+        ^AuthorizedKeysCommand # line starts with
+        \s+                    # one space or more
+        (?<quote>['"]?)        # detect optional quotes
+        (?<content>[^#'"]+)    # content should be at least 1 char, non quotes or start-comment symbol
+        \k<quote>              # boundary for command, backtracks the same detected quote, or none
+        \s*                    # optional any amount of space character
+        (?:\#.*)?$             # optional start-comment symbol followed by optionally any character until end of line
+      }x
+      OPENSSH_AUTHORIZED_KEYS_USER_REGEXP = %r{
+        ^AuthorizedKeysCommandUser # line starts with
+        \s+                        # one space or more
+        (?<quote>['"]?)            # detect optional quotes
+        (?<content>[^#'"]+)        # content should be at least 1 char, non quotes or start-comment symbol
+        \k<quote>                  # boundary for command, backtracks the same detected quote, or none
+        \s*                        # optional any amount of space character
+        (?:\#.*)?$                 # optional start-comment symbol followed by optionally any character until end of line
+      }x
+      OPENSSH_EXPECTED_COMMAND = '/opt/gitlab-shell/authorized_keys %u %k'.freeze
+
+      def multi_check
+        unless openssh_config_exists?
+          print_failure("Cannot find OpenSSH configuration file at: #{openssh_config_path}")
+
+          if in_docker?
+            try_fixing_it(
+              'If you are not using our official docker containers,',
+              'make sure you have OpenSSH server installed and configured correctly on this system'
+            )
+
+            for_more_information(AUTHORIZED_KEYS_DOCS)
+          else
+            try_fixing_it(
+              'Make sure you have OpenSSH server installed on this system'
+            )
+          end
+
+          return
+        end
+
+        unless openssh_config_readable?
+          print_skipped('Cannot access OpenSSH configuration file')
+
+          try_fixing_it(
+            'This is expected if you are using SELinux. You may want to check configuration manually'
+          )
+
+          for_more_information(AUTHORIZED_KEYS_DOCS)
+          return
+        end
+
+        authorized_keys_command = extract_authorized_keys_command
+        unless authorized_keys_command
+          print_failure('OpenSSH configuration file does not contain a AuthorizedKeysCommand')
+
+          try_fixing_it(
+            'Change your OpenSSH configuration file pointing to the correct command'
+          )
+
+          for_more_information(AUTHORIZED_KEYS_DOCS)
+          return
+        end
+
+        unless openssh_is_expected_command?(authorized_keys_command)
+          print_warning('OpenSSH configuration file points to a different AuthorizedKeysCommand')
+
+          try_fixing_it(
+            "We were expecting AuthorizedKeysCommand to be: #{OPENSSH_EXPECTED_COMMAND}",
+            "but instead it is: #{authorized_keys_command}",
+            'If you made a custom command, make sure it behaves according to GitLab\'s Documentation'
+          )
+
+          for_more_information(AUTHORIZED_KEYS_DOCS)
+          # this check should not block the others
+        end
+
+        authorized_keys_command_path = openssh_extract_command_path(authorized_keys_command)
+        unless File.file?(authorized_keys_command_path)
+          print_failure("Cannot find configured AuthorizedKeysCommand: #{authorized_keys_command_path}")
+
+          try_fixing_it(
+            'You need to create the file and add the correct content to it'
+          )
+
+          for_more_information(AUTHORIZED_KEYS_DOCS)
+          return
+        end
+
+        authorized_keys_command_user = extract_authorized_keys_command_user
+        unless authorized_keys_command_user
+          print_failure('OpenSSH configuration file does not contain a AuthorizedKeysCommandUser')
+
+          try_fixing_it(
+            'Change your OpenSSH configuration file pointing to the correct user'
+          )
+
+          for_more_information(AUTHORIZED_KEYS_DOCS)
+          return
+        end
+
+        unless authorized_keys_command_user == gitlab_user
+          print_warning('OpenSSH configuration file points to a different AuthorizedKeysCommandUser')
+
+          try_fixing_it(
+            "We were expecting AuthorizedKeysCommandUser to be: #{gitlab_user}",
+            "but instead it is: #{authorized_keys_command_user}",
+            'Fix your OpenSSH configuration file pointing to the correct user'
+          )
+
+          for_more_information(AUTHORIZED_KEYS_DOCS)
+          return
+        end
+
+        $stdout.puts 'yes'.color(:green)
+        true
+      end
+
+      def extract_authorized_keys_command
+        extract_openssh_config(OPENSSH_AUTHORIZED_KEYS_CMD_REGEXP)
+      end
+
+      def extract_authorized_keys_command_user
+        extract_openssh_config(OPENSSH_AUTHORIZED_KEYS_USER_REGEXP)
+      end
+
+      def openssh_config_path
+        @openssh_config_path ||= begin
+          if in_docker?
+            '/assets/sshd_config' # path in our official docker containers
+          else
+            '/etc/ssh/sshd_config'
+          end
+        end
+      end
+
+      private
+
+      def print_skipped(reason)
+        $stdout.puts 'skipped'.color(:magenta)
+
+        $stdout.puts '  Reason:'.color(:blue)
+        $stdout.puts "  #{reason}"
+      end
+
+      def print_warning(reason)
+        $stdout.puts 'warning'.color(:magenta)
+
+        $stdout.puts '  Reason:'.color(:blue)
+        $stdout.puts "  #{reason}"
+      end
+
+      def print_failure(reason)
+        $stdout.puts 'no'.color(:red)
+
+        $stdout.puts '  Reason:'.color(:blue)
+        $stdout.puts "  #{reason}"
+      end
+
+      def openssh_config_exists?
+        File.file?(openssh_config_path)
+      end
+
+      def openssh_config_readable?
+        File.readable?(openssh_config_path)
+      end
+
+      def openssh_extract_command_path(cmd_with_params)
+        cmd_with_params.split(' ').first
+      end
+
+      def openssh_is_expected_command?(authorized_keys_command)
+        authorized_keys_command.squeeze(' ') == OPENSSH_EXPECTED_COMMAND
+      end
+
+      def in_docker?
+        File.file?('/.dockerenv')
+      end
+
+      def gitlab_user
+        Gitlab.config.gitlab.user
+      end
+
+      def extract_openssh_config(regexp)
+        return false unless openssh_config_exists? && openssh_config_readable?
+
+        File.open(openssh_config_path) do |f|
+          f.each_line do |line|
+            if (match = line.match(regexp))
+              raw_content = match[:content]
+              # remove linebreak, and lead and trailing spaces
+              return raw_content.chomp.strip
+            end
+          end
+        end
+
+        nil
+      end
+    end
+  end
+end

ee/lib/system_check/geo/authorized_keys_flag_check.rb

+module SystemCheck
+  module Geo
+    class AuthorizedKeysFlagCheck < ::SystemCheck::BaseCheck
+      set_name 'GitLab configured to disable writing to authorized_keys file'
+
+      def check?
+        !Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled
+      end
+
+      def show_error
+        try_fixing_it(
+          "You need to disable `Write to authorized_keys file` in GitLab's Admin panel"
+        )
+
+        for_more_information(AUTHORIZED_KEYS_DOCS)
+      end
+    end
+  end
+end

lib/gitlab/git/gitlab_projects.rb

+module Gitlab
+  module Git
+    class GitlabProjects
+      include Gitlab::Git::Popen
+
+      # Absolute path to directory where repositories are stored.
+      # Example: /home/git/repositories
+      attr_reader :shard_path
+
+      # Relative path is a directory name for repository with .git at the end.
+      # Example: gitlab-org/gitlab-test.git
+      attr_reader :repository_relative_path
+
+      # Absolute path to the repository.
+      # Example: /home/git/repositorities/gitlab-org/gitlab-test.git
+      attr_reader :repository_absolute_path
+
+      # This is the path at which the gitlab-shell hooks directory can be found.
+      # It's essential for integration between git and GitLab proper. All new
+      # repositories should have their hooks directory symlinked here.
+      attr_reader :global_hooks_path
+
+      attr_reader :logger
+
+      def initialize(shard_path, repository_relative_path, global_hooks_path:, logger:)
+        @shard_path = shard_path
+        @repository_relative_path = repository_relative_path
+
+        @logger = logger
+        @global_hooks_path = global_hooks_path
+        @repository_absolute_path = File.join(shard_path, repository_relative_path)
+        @output = StringIO.new
+      end
+
+      def output
+        io = @output.dup
+        io.rewind
+        io.read
+      end
+
+      def rm_project
+        logger.info "Removing repository <#{repository_absolute_path}>."
+        FileUtils.rm_rf(repository_absolute_path)
+      end
+
+      # Move repository from one directory to another
+      #
+      # Example:  gitlab/gitlab-ci.git -> randx/six.git
+      #
+      # Won't work if target namespace directory does not exist
+      #
+      def mv_project(new_path)
+        new_absolute_path = File.join(shard_path, new_path)
+
+        # verify that the source repo exists
+        unless File.exist?(repository_absolute_path)
+          logger.error "mv-project failed: source path <#{repository_absolute_path}> does not exist."
+          return false
+        end
+
+        # ...and that the target repo does not exist
+        if File.exist?(new_absolute_path)
+          logger.error "mv-project failed: destination path <#{new_absolute_path}> already exists."
+          return false
+        end
+
+        logger.info "Moving repository from <#{repository_absolute_path}> to <#{new_absolute_path}>."
+        FileUtils.mv(repository_absolute_path, new_absolute_path)
+      end
+
+      # Import project via git clone --bare
+      # URL must be publicly cloneable
+      def import_project(source, timeout)
+        # Skip import if repo already exists
+        return false if File.exist?(repository_absolute_path)
+
+        masked_source = mask_password_in_url(source)
+
+        logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
+        cmd = %W(git clone --bare -- #{source} #{repository_absolute_path})
+
+        success = run_with_timeout(cmd, timeout, nil)
+
+        unless success
+          logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
+          FileUtils.rm_rf(repository_absolute_path)
+          return false
+        end
+
+        Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)
+
+        # The project was imported successfully.
+        # Remove the origin URL since it may contain password.
+        remove_origin_in_repo
+
+        true
+      end
+
+      def fork_repository(new_shard_path, new_repository_relative_path)
+        from_path = repository_absolute_path
+        to_path = File.join(new_shard_path, new_repository_relative_path)
+
+        # The repository cannot already exist
+        if File.exist?(to_path)
+          logger.error "fork-repository failed: destination repository <#{to_path}> already exists."
+          return false
+        end
+
+        # Ensure the namepsace / hashed storage directory exists
+        FileUtils.mkdir_p(File.dirname(to_path), mode: 0770)
+
+        logger.info "Forking repository from <#{from_path}> to <#{to_path}>."
+        cmd = %W(git clone --bare --no-local -- #{from_path} #{to_path})
+
+        run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path)
+      end
+
+      def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil)
+        tags_option = tags ? '--tags' : '--no-tags'
+
+        logger.info "Fetching remote #{name} for repository #{repository_absolute_path}."
+        cmd = %W(git fetch #{name} --prune --quiet)
+        cmd << '--force' if force
+        cmd << tags_option
+
+        setup_ssh_auth(ssh_key, known_hosts) do |env|
+          success = run_with_timeout(cmd, timeout, repository_absolute_path, env)
+
+          unless success
+            logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed."
+          end
+
+          success
+        end
+      end
+
+      def push_branches(remote_name, timeout, force, branch_names)
+        logger.info "Pushing branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}"
+        cmd = %w(git push)
+        cmd << '--force' if force
+        cmd += %W(-- #{remote_name}).concat(branch_names)
+
+        success = run_with_timeout(cmd, timeout, repository_absolute_path)
+
+        unless success
+          logger.error("Pushing branches to remote #{remote_name} failed.")
+        end
+
+        success
+      end
+
+      def delete_remote_branches(remote_name, branch_names)
+        branches = branch_names.map { |branch_name| ":#{branch_name}" }
+
+        logger.info "Pushing deleted branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}"
+        cmd = %W(git push -- #{remote_name}).concat(branches)
+
+        success = run(cmd, repository_absolute_path)
+
+        unless success
+          logger.error("Pushing deleted branches to remote #{remote_name} failed.")
+        end
+
+        success
+      end
+
+      protected
+
+      def run(*args)
+        output, exitstatus = popen(*args)
+        @output << output
+
+        exitstatus&.zero?
+      end
+
+      def run_with_timeout(*args)
+        output, exitstatus = popen_with_timeout(*args)
+        @output << output
+
+        exitstatus&.zero?
+      rescue Timeout::Error
+        @output.puts('Timed out')
+
+        false
+      end
+
+      def mask_password_in_url(url)
+        result = URI(url)
+        result.password = "*****" unless result.password.nil?
+        result.user = "*****" unless result.user.nil? # it's needed for oauth access_token
+        result
+      rescue
+        url
+      end
+
+      def remove_origin_in_repo
+        cmd = %w(git remote rm origin)
+        run(cmd, repository_absolute_path)
+      end
+
+      # Builds a small shell script that can be used to execute SSH with a set of
+      # custom options.
+      #
+      # Options are expanded as `'-oKey="Value"'`, so SSH will correctly interpret
+      # paths with spaces in them. We trust the user not to embed single or double
+      # quotes in the key or value.
+      def custom_ssh_script(options = {})
+        args = options.map { |k, v| %Q{'-o#{k}="#{v}"'} }.join(' ')
+
+        [
+          "#!/bin/sh",
+          "exec ssh #{args} \"$@\""
+        ].join("\n")
+      end
+
+      # Known hosts data and private keys can be passed to gitlab-shell in the
+      # environment. If present, this method puts them into temporary files, writes
+      # a script that can substitute as `ssh`, setting the options to respect those
+      # files, and yields: { "GIT_SSH" => "/tmp/myScript" }
+      def setup_ssh_auth(key, known_hosts)
+        options = {}
+
+        if key
+          key_file = Tempfile.new('gitlab-shell-key-file')
+          key_file.chmod(0o400)
+          key_file.write(key)
+          key_file.close
+
+          options['IdentityFile'] = key_file.path
+          options['IdentitiesOnly'] = 'yes'
+        end
+
+        if known_hosts
+          known_hosts_file = Tempfile.new('gitlab-shell-known-hosts')
+          known_hosts_file.chmod(0o400)
+          known_hosts_file.write(known_hosts)
+          known_hosts_file.close
+
+          options['StrictHostKeyChecking'] = 'yes'
+          options['UserKnownHostsFile'] = known_hosts_file.path
+        end
+
+        return yield({}) if options.empty?
+
+        script = Tempfile.new('gitlab-shell-ssh-wrapper')
+        script.chmod(0o755)
+        script.write(custom_ssh_script(options))
+        script.close
+
+        yield('GIT_SSH' => script.path)
+      ensure
+        key_file&.close!
+        known_hosts_file&.close!
+        script&.close!
+      end
+    end
+  end
+end

lib/gitlab/git/repository.rb

@@ -18,7 +18,6 @@ module Gitlab
         GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE
       ].freeze
       SEARCH_CONTEXT_LINES = 3
-      GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
       REBASE_WORKTREE_PREFIX = 'rebase'.freeze
       SQUASH_WORKTREE_PREFIX = 'squash'.freeze
       GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
@@ -40,10 +39,31 @@ module Gitlab
           repo = Rugged::Repository.init_at(repo_path, bare)
           repo.close
 
-          if symlink_hooks_to.present?
-            hooks_path = File.join(repo_path, 'hooks')
-            FileUtils.rm_rf(hooks_path)
-            FileUtils.ln_s(symlink_hooks_to, hooks_path)
+          create_hooks(repo_path, symlink_hooks_to) if symlink_hooks_to.present?
+
+          true
+        end
+
+        def create_hooks(repo_path, global_hooks_path)
+          local_hooks_path = File.join(repo_path, 'hooks')
+          real_local_hooks_path = :not_found
+
+          begin
+            real_local_hooks_path = File.realpath(local_hooks_path)
+          rescue Errno::ENOENT
+            # real_local_hooks_path == :not_found
+          end
+
+          # Do nothing if hooks already exist
+          unless real_local_hooks_path == File.realpath(global_hooks_path)
+            # Move the existing hooks somewhere safe
+            FileUtils.mv(
+              local_hooks_path,
+              "#{local_hooks_path}.old.#{Time.now.to_i}"
+            ) if File.exist?(local_hooks_path)
+
+            # Create the hooks symlink
+            FileUtils.ln_sf(global_hooks_path, local_hooks_path)
           end
 
           true

lib/gitlab/shell.rb

@@ -66,7 +66,7 @@ module Gitlab
     # Init new repository
     #
     # storage - project's storage name
-    # name - project path with namespace
+    # name - project disk path
     #
     # Ex.
     #   add_repository("/path/to/storage", "gitlab/gitlab-ci")
@@ -94,17 +94,21 @@ module Gitlab
     # Import repository
     #
     # storage - project's storage path
-    # name - project path with namespace
+    # name - project disk path
+    # url - URL to import from
     #
     # Ex.
-    #   import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git")
+    #   import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git")
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def import_repository(storage, name, url)
       # The timeout ensures the subprocess won't hang forever
-      cmd = [gitlab_shell_projects_path, 'import-project',
-             storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"]
-      gitlab_shell_fast_execute_raise_error(cmd)
+      cmd = gitlab_projects(storage, "#{name}.git")
+      success = cmd.import_project(url, git_timeout)
+
+      raise Error, cmd.output unless success
+
+      success
     end
 
     # Fetch remote for repository
@@ -132,16 +136,15 @@ module Gitlab
 
     # Move repository
     # storage - project's storage path
-    # path - project path with namespace
-    # new_path - new project path with namespace
+    # path - project disk path
+    # new_path - new project disk path
     #
     # Ex.
     #   mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def mv_repository(storage, path, new_path)
-      gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project',
-                                 storage, "#{path}.git", "#{new_path}.git"])
+      gitlab_projects(storage, "#{path}.git").mv_project("#{new_path}.git")
     end
 
     # Fork repository to new path
@@ -155,30 +158,21 @@ module Gitlab
     #
     # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one.
     def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path)
-      gitlab_shell_fast_execute(
-        [
-          gitlab_shell_projects_path,
-          'fork-repository',
-          forked_from_storage,
-          "#{forked_from_disk_path}.git",
-          forked_to_storage,
-          "#{forked_to_disk_path}.git"
-        ]
-      )
+      gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git")
+        .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git")
     end
 
     # Remove repository from file system
     #
     # storage - project's storage path
-    # name - project path with namespace
+    # name - project disk path
     #
     # Ex.
     #   remove_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def remove_repository(storage, name)
-      gitlab_shell_fast_execute([gitlab_shell_projects_path,
-                                 'rm-project', storage, "#{name}.git"])
+      gitlab_projects(storage, "#{name}.git").rm_project
     end
 
     # Add new key to gitlab-shell
@@ -370,56 +364,43 @@ module Gitlab
       end
     end
 
-    # Create (if necessary) and link the secret token file
-    def generate_and_link_secret_token
-      secret_file = Gitlab.config.gitlab_shell.secret_file
-      unless File.size?(secret_file)
-        # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
-        token = SecureRandom.hex(16)
-        File.write(secret_file, token)
-      end
-
-      link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret')
-      if File.exist?(gitlab_shell_path) && !File.exist?(link_path)
-        FileUtils.symlink(secret_file, link_path)
-      end
-    end
-
     # Push branch to remote repository
     #
-    # project_name - project's name with namespace
+    # storage - project's storage path
+    # project_name - project's disk path
     # remote_name - remote name
-    # branch_name - remote branch name
+    # branch_names - remote branch names to push
+    # forced - should we use --force flag
     #
     # Ex.
-    #   push_remote_branches('upstream', 'feature')
+    #   push_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test' 'upstream', ['feature'])
     #
     def push_remote_branches(storage, project_name, remote_name, branch_names, forced: true)
-      args = [gitlab_shell_projects_path, 'push-branches', storage, "#{project_name}.git", remote_name, '600']
-      args << '--force' if forced
-      args += [*branch_names]
+      cmd = gitlab_projects(storage, "#{project_name}.git")
 
-      output, status = Popen.popen(args)
-      raise Error, output unless status.zero?
+      success = cmd.push_branches(remote_name, git_timeout, forced, branch_names)
 
-      true
+      raise Error, cmd.output unless success
+
+      success
     end
 
-    # Delete branch from remote repository
-    #
-    # project_name - project's name with namespace
+    # storage - project's storage path
+    # project_name - project's disk path
     # remote_name - remote name
-    # branch_name - remote branch name
+    # branch_names - remote branch names
     #
     # Ex.
-    #   delete_remote_branches('upstream', 'feature')
+    #   delete_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test', 'upstream', ['feature'])
     #
     def delete_remote_branches(storage, project_name, remote_name, branch_names)
-      args = [gitlab_shell_projects_path, 'delete-remote-branches', storage, "#{project_name}.git", remote_name, *branch_names]
-      output, status = Popen.popen(args)
-      raise Error, output unless status.zero?
+      cmd = gitlab_projects(storage, "#{project_name}.git")
 
-      true
+      success = cmd.delete_remote_branches(remote_name, branch_names)
+
+      raise Error, cmd.output unless success
+
+      success
     end
 
     protected
@@ -460,24 +441,35 @@ module Gitlab
 
     private
 
-    def local_fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false)
-      args = [gitlab_shell_projects_path, 'fetch-remote', storage, name, remote, "#{Gitlab.config.gitlab_shell.git_timeout}"]
-      args << '--force' if forced
-      args << '--no-tags' if no_tags
+    def gitlab_projects(shard_path, disk_path)
+      Gitlab::Git::GitlabProjects.new(
+        shard_path,
+        disk_path,
+        global_hooks_path: Gitlab.config.gitlab_shell.hooks_path,
+        logger: Rails.logger
+      )
+    end
 
-      vars = {}
+    def local_fetch_remote(storage_path, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false)
+      vars = { force: forced, tags: !no_tags }
 
       if ssh_auth&.ssh_import?
         if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
-          vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key
+          vars[:ssh_key] = ssh_auth.ssh_private_key
         end
 
         if ssh_auth.ssh_known_hosts.present?
-          vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts
+          vars[:known_hosts] = ssh_auth.ssh_known_hosts
         end
       end
 
-      gitlab_shell_fast_execute_raise_error(args, vars)
+      cmd = gitlab_projects(storage_path, repository_relative_path)
+
+      success = cmd.fetch_remote(remote, git_timeout, vars)
+
+      raise Error, cmd.output unless success
+
+      success
     end
 
     def gitlab_shell_fast_execute(cmd)
@@ -513,6 +505,10 @@ module Gitlab
       Gitlab::GitalyClient::NamespaceService.new(storage)
     end
 
+    def git_timeout
+      Gitlab.config.gitlab_shell.git_timeout
+    end
+
     def gitaly_migrate(method, &block)
       Gitlab::GitalyClient.migrate(method, &block)
     rescue GRPC::NotFound, GRPC::BadStatus => e

lib/tasks/gitlab/check.rake

@@ -464,7 +464,9 @@ namespace :gitlab do
         SystemCheck::Geo::HttpConnectionCheck,
         SystemCheck::Geo::HTTPCloneEnabledCheck,
         SystemCheck::Geo::ClocksSynchronizationCheck,
-        SystemCheck::App::GitUserDefaultSSHConfigCheck
+        SystemCheck::App::GitUserDefaultSSHConfigCheck,
+        SystemCheck::Geo::AuthorizedKeysCheck,
+        SystemCheck::Geo::AuthorizedKeysFlagCheck
       ]
 
       SystemCheck.run('Geo', checks)

spec/ee/fixtures/system_check/sshd_config

+# Package generated configuration file
+# See the sshd_config(5) manpage for details
+
+# What ports, IPs and protocols we listen for
+Port 22
+# Use these options to restrict which interfaces/protocols sshd will bind to
+#ListenAddress ::
+#ListenAddress 0.0.0.0
+Protocol 2
+# HostKeys for protocol version 2
+HostKey /etc/ssh/ssh_host_rsa_key
+HostKey /etc/ssh/ssh_host_dsa_key
+HostKey /etc/ssh/ssh_host_ecdsa_key
+HostKey /etc/ssh/ssh_host_ed25519_key
+#Privilege Separation is turned on for security
+UsePrivilegeSeparation yes
+
+# Lifetime and size of ephemeral version 1 server key
+KeyRegenerationInterval 3600
+ServerKeyBits 1024
+
+# Logging
+SyslogFacility AUTH
+LogLevel INFO
+
+# Authentication:
+LoginGraceTime 120
+PermitRootLogin yes
+StrictModes yes
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+#AuthorizedKeysFile	%h/.ssh/authorized_keys
+#AuthorizedKeysCommand /opt/gitlab-shell/invalid_authorized_keys %u %k
+AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k
+AuthorizedKeysCommandUser git
+
+# Don't read the user's ~/.rhosts and ~/.shosts files
+IgnoreRhosts yes
+# For this to work you will also need host keys in /etc/ssh_known_hosts
+RhostsRSAAuthentication no
+# similar for protocol version 2
+HostbasedAuthentication no
+# Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication
+#IgnoreUserKnownHosts yes
+
+# To enable empty passwords, change to yes (NOT RECOMMENDED)
+PermitEmptyPasswords no
+
+# Change to yes to enable challenge-response passwords (beware issues with
+# some PAM modules and threads)
+ChallengeResponseAuthentication no
+
+# Change to no to disable tunnelled clear text passwords
+PasswordAuthentication yes
+
+# Kerberos options
+#KerberosAuthentication no
+#KerberosGetAFSToken no
+#KerberosOrLocalPasswd yes
+#KerberosTicketCleanup yes
+
+# GSSAPI options
+#GSSAPIAuthentication no
+#GSSAPICleanupCredentials yes
+
+X11Forwarding yes
+X11DisplayOffset 10
+PrintMotd no
+PrintLastLog yes
+TCPKeepAlive yes
+#UseLogin no
+
+#MaxStartups 10:30:60
+#Banner /etc/issue.net
+
+# Allow client to pass locale environment variables
+AcceptEnv LANG LC_*
+
+Subsystem sftp /usr/lib/openssh/sftp-server
+
+# Set this to 'yes' to enable PAM authentication, account processing,
+# and session processing. If this is enabled, PAM authentication will
+# be allowed through the ChallengeResponseAuthentication and
+# PasswordAuthentication.  Depending on your PAM configuration,
+# PAM authentication via ChallengeResponseAuthentication may bypass
+# the setting of "PermitRootLogin yes
+# If you just want the PAM account and session checks to run without
+# PAM authentication, then enable this but set PasswordAuthentication
+# and ChallengeResponseAuthentication to 'no'.
+UsePAM yes

spec/ee/fixtures/system_check/sshd_config_invalid_command

+# Package generated configuration file
+# See the sshd_config(5) manpage for details
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+#AuthorizedKeysFile	%h/.ssh/authorized_keys
+AuthorizedKeysCommand "/opt/gitlab-shell/invalid_authorized_keys   %u      %k"         # comment
+#AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k
+AuthorizedKeysCommandUser anotheruser  #comment with more stuff#

spec/ee/fixtures/system_check/sshd_config_invalid_user

+# Package generated configuration file
+# See the sshd_config(5) manpage for details
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+#AuthorizedKeysFile	%h/.ssh/authorized_keys
+AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k          # comment
+AuthorizedKeysCommandUser anotheruser  #comment with more stuff#

spec/ee/fixtures/system_check/sshd_config_no_command

+# Package generated configuration file
+# See the sshd_config(5) manpage for details
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+# AuthorizedKeysFile	%h/.ssh/authorized_keys
+# AuthorizedKeysCommand /opt/gitlab-shell/invalid_authorized_keys %u %k
+# AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k
+# AuthorizedKeysCommandUser git

spec/ee/fixtures/system_check/sshd_config_no_user

+# Package generated configuration file
+# See the sshd_config(5) manpage for details
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+#AuthorizedKeysFile	%h/.ssh/authorized_keys
+#AuthorizedKeysCommand /opt/gitlab-shell/invalid_authorized_keys %u %k
+AuthorizedKeysCommand /opt/gitlab-shell/authorized_keys %u %k
+#AuthorizedKeysCommandUser git

spec/ee/spec/lib/gitlab/geo/geo_tasks_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Geo::GeoTasks do
+  describe '.set_primary_geo_node' do
+    before do
+      allow(GeoNode).to receive(:current_node_url).and_return('https://primary.geo.example.com')
+    end
+
+    it 'sets the primary node' do
+      expect { subject.set_primary_geo_node }.to output(/https:\/\/primary.geo.example.com\/ is now the primary Geo node/).to_stdout
+    end
+
+    it 'returns error when there is already a Primary node' do
+      create(:geo_node, :primary)
+
+      expect { subject.set_primary_geo_node }.to output(/Error saving Geo node:/).to_stdout
+    end
+  end
+end

spec/ee/spec/lib/system_check/geo/authorized_keys_check_spec.rb

+require 'spec_helper'
+require 'rake_helper'
+
+describe SystemCheck::Geo::AuthorizedKeysCheck do
+  describe '#multi_check' do
+    subject { described_class.new }
+
+    before do
+      allow(File).to receive(:file?).and_call_original # provides a default behavior when mocking
+      allow(File).to receive(:file?).with('/opt/gitlab-shell/authorized_keys') { true }
+    end
+
+    context 'OpenSSH config file' do
+      context 'in docker' do
+        it 'fails when config file does not exist' do
+          allow(subject).to receive(:in_docker?) { true }
+          allow(File).to receive(:file?).with('/assets/sshd_config') { false }
+
+          expect_failure('Cannot find OpenSSH configuration file at: /assets/sshd_config')
+
+          subject.multi_check
+        end
+      end
+
+      it 'fails when config file does not exist' do
+        allow(subject).to receive(:in_docker?) { false }
+        allow(File).to receive(:file?).with('/etc/ssh/sshd_config') { false }
+
+        expect_failure('Cannot find OpenSSH configuration file at: /etc/ssh/sshd_config')
+
+        subject.multi_check
+      end
+
+      it 'skips when config file is not readable' do
+        override_sshd_config('system_check/sshd_config')
+        allow(File).to receive(:readable?).with(expand_fixture_ee_path('system_check/sshd_config')) { false }
+
+        expect_skipped('Cannot access OpenSSH configuration file')
+
+        subject.multi_check
+      end
+    end
+
+    context 'AuthorizedKeysCommand' do
+      it 'fails when config file does not contain the AuthorizedKeysCommand' do
+        override_sshd_config('system_check/sshd_config_no_command')
+
+        expect_failure('OpenSSH configuration file does not contain a AuthorizedKeysCommand')
+
+        subject.multi_check
+      end
+
+      it 'warns when config file does not contain the correct AuthorizedKeysCommand' do
+        override_sshd_config('system_check/sshd_config_invalid_command')
+
+        expect_warning('OpenSSH configuration file points to a different AuthorizedKeysCommand')
+
+        subject.multi_check
+      end
+
+      it 'fails when cannot find referred authorized keys file on disk' do
+        override_sshd_config('system_check/sshd_config')
+        allow(subject).to receive(:extract_authorized_keys_command) { '/tmp/nonexistent/authorized_keys' }
+
+        expect_failure('Cannot find configured AuthorizedKeysCommand: /tmp/nonexistent/authorized_keys')
+
+        subject.multi_check
+      end
+    end
+
+    context 'AuthorizedKeysCommandUser' do
+      it 'fails when config file does not contain the AuthorizedKeysCommandUser' do
+        override_sshd_config('system_check/sshd_config_no_user')
+
+        expect_failure('OpenSSH configuration file does not contain a AuthorizedKeysCommandUser')
+
+        subject.multi_check
+      end
+
+      it 'fails when config file does not contain the correct AuthorizedKeysCommandUser' do
+        override_sshd_config('system_check/sshd_config_invalid_user')
+
+        expect_warning('OpenSSH configuration file points to a different AuthorizedKeysCommandUser')
+
+        subject.multi_check
+      end
+    end
+
+    it 'succeed when all conditions are met' do
+      override_sshd_config('system_check/sshd_config')
+      allow(subject).to receive(:gitlab_user) { 'git' }
+
+      result = subject.multi_check
+      expect($stdout.string).to include('yes')
+      expect(result).to be_truthy
+    end
+  end
+
+  describe '#extract_authorized_keys_command' do
+    it 'returns false when no command is available' do
+      override_sshd_config('system_check/sshd_config_no_command')
+
+      expect(subject.extract_authorized_keys_command).to be_falsey
+    end
+
+    it 'returns correct (uncommented) command' do
+      override_sshd_config('system_check/sshd_config')
+
+      expect(subject.extract_authorized_keys_command).to eq('/opt/gitlab-shell/authorized_keys %u %k')
+    end
+
+    it 'returns command without comments and without quotes' do
+      override_sshd_config('system_check/sshd_config_invalid_command')
+
+      expect(subject.extract_authorized_keys_command).to eq('/opt/gitlab-shell/invalid_authorized_keys   %u      %k')
+    end
+  end
+
+  describe '#extract_authorized_keys_command_user' do
+    it 'returns false when no command user is available' do
+      override_sshd_config('system_check/sshd_config_no_command')
+
+      expect(subject.extract_authorized_keys_command_user).to be_falsey
+    end
+
+    it 'returns correct (uncommented) command' do
+      override_sshd_config('system_check/sshd_config')
+
+      expect(subject.extract_authorized_keys_command_user).to eq('git')
+    end
+
+    it 'returns command without comments' do
+      override_sshd_config('system_check/sshd_config_invalid_command')
+
+      expect(subject.extract_authorized_keys_command_user).to eq('anotheruser')
+    end
+  end
+
+  describe '#openssh_config_path' do
+    context 'when in docker container' do
+      it 'returns /assets/sshd_config' do
+        allow(subject).to receive(:in_docker?) { true }
+
+        expect(subject.openssh_config_path).to eq('/assets/sshd_config')
+      end
+    end
+
+    context 'when not in docker container' do
+      it 'returns /etc/ssh/sshd_config' do
+        allow(subject).to receive(:in_docker?) { false }
+
+        expect(subject.openssh_config_path).to eq('/etc/ssh/sshd_config')
+      end
+    end
+  end
+
+  def expect_failure(reason)
+    expect(subject).to receive(:print_failure).with(reason)
+  end
+
+  def expect_warning(reason)
+    expect(subject).to receive(:print_warning).with(reason)
+  end
+
+  def expect_skipped(reason)
+    expect(subject).to receive(:print_skipped).with(reason)
+  end
+
+  def override_sshd_config(relative_path)
+    allow(subject).to receive(:openssh_config_path) { expand_fixture_ee_path(relative_path) }
+  end
+end

spec/ee/spec/lib/system_check/geo/authorized_keys_flag_check_spec.rb

+require 'spec_helper'
+require 'rake_helper'
+
+describe SystemCheck::Geo::AuthorizedKeysFlagCheck do
+  before do
+    silence_output
+  end
+
+  describe '#check?' do
+    it 'fails when write to authorized_keys still enabled' do
+      stub_application_setting(authorized_keys_enabled: true)
+
+      expect(subject.check?).to be_falsey
+    end
+
+    it 'succeed when write to authorized_keys is disabled' do
+      stub_application_setting(authorized_keys_enabled: false)
+
+      expect(subject.check?).to be_truthy
+    end
+  end
+end

spec/lib/gitlab/git/gitlab_projects_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Git::GitlabProjects do
+  after do
+    TestEnv.clean_test_path
+  end
+
+  let(:project) { create(:project, :repository) }
+
+  if $VERBOSE
+    let(:logger) { Logger.new(STDOUT) }
+  else
+    let(:logger) { double('logger').as_null_object }
+  end
+
+  let(:tmp_repos_path) { TestEnv.repos_path }
+  let(:repo_name) { project.disk_path + '.git' }
+  let(:tmp_repo_path) { File.join(tmp_repos_path, repo_name) }
+  let(:gl_projects) { build_gitlab_projects(tmp_repos_path, repo_name) }
+
+  describe '#initialize' do
+    it { expect(gl_projects.shard_path).to eq(tmp_repos_path) }
+    it { expect(gl_projects.repository_relative_path).to eq(repo_name) }
+    it { expect(gl_projects.repository_absolute_path).to eq(File.join(tmp_repos_path, repo_name)) }
+    it { expect(gl_projects.logger).to eq(logger) }
+  end
+
+  describe '#mv_project' do
+    let(:new_repo_path) { File.join(tmp_repos_path, 'repo.git') }
+
+    it 'moves a repo directory' do
+      expect(File.exist?(tmp_repo_path)).to be_truthy
+
+      message = "Moving repository from <#{tmp_repo_path}> to <#{new_repo_path}>."
+      expect(logger).to receive(:info).with(message)
+
+      expect(gl_projects.mv_project('repo.git')).to be_truthy
+
+      expect(File.exist?(tmp_repo_path)).to be_falsy
+      expect(File.exist?(new_repo_path)).to be_truthy
+    end
+
+    it "fails if the source path doesn't exist" do
+      expect(logger).to receive(:error).with("mv-project failed: source path <#{tmp_repos_path}/bad-src.git> does not exist.")
+
+      result = build_gitlab_projects(tmp_repos_path, 'bad-src.git').mv_project('repo.git')
+      expect(result).to be_falsy
+    end
+
+    it 'fails if the destination path already exists' do
+      FileUtils.mkdir_p(File.join(tmp_repos_path, 'already-exists.git'))
+
+      message = "mv-project failed: destination path <#{tmp_repos_path}/already-exists.git> already exists."
+      expect(logger).to receive(:error).with(message)
+
+      expect(gl_projects.mv_project('already-exists.git')).to be_falsy
+    end
+  end
+
+  describe '#rm_project' do
+    it 'removes a repo directory' do
+      expect(File.exist?(tmp_repo_path)).to be_truthy
+      expect(logger).to receive(:info).with("Removing repository <#{tmp_repo_path}>.")
+
+      expect(gl_projects.rm_project).to be_truthy
+
+      expect(File.exist?(tmp_repo_path)).to be_falsy
+    end
+  end
+
+  describe '#push_branches' do
+    let(:remote_name) { 'remote-name' }
+    let(:branch_name) { 'master' }
+    let(:cmd) { %W(git push -- #{remote_name} #{branch_name}) }
+    let(:force) { false }
+
+    subject { gl_projects.push_branches(remote_name, 600, force, [branch_name]) }
+
+    it 'executes the command' do
+      stub_spawn(cmd, 600, tmp_repo_path, success: true)
+
+      is_expected.to be_truthy
+    end
+
+    it 'fails' do
+      stub_spawn(cmd, 600, tmp_repo_path, success: false)
+
+      is_expected.to be_falsy
+    end
+
+    context 'with --force' do
+      let(:cmd) { %W(git push --force -- #{remote_name} #{branch_name}) }
+      let(:force) { true }
+
+      it 'executes the command' do
+        stub_spawn(cmd, 600, tmp_repo_path, success: true)
+
+        is_expected.to be_truthy
+      end
+    end
+  end
+
+  describe '#fetch_remote' do
+    let(:remote_name) { 'remote-name' }
+    let(:branch_name) { 'master' }
+    let(:force) { false }
+    let(:tags) { true }
+    let(:args) { { force: force, tags: tags }.merge(extra_args) }
+    let(:extra_args) { {} }
+    let(:cmd) { %W(git fetch #{remote_name} --prune --quiet --tags) }
+
+    subject { gl_projects.fetch_remote(remote_name, 600, args) }
+
+    def stub_tempfile(name, filename, opts = {})
+      chmod = opts.delete(:chmod)
+      file = StringIO.new
+
+      allow(file).to receive(:close!)
+      allow(file).to receive(:path).and_return(name)
+
+      expect(Tempfile).to receive(:new).with(filename).and_return(file)
+      expect(file).to receive(:chmod).with(chmod) if chmod
+
+      file
+    end
+
+    context 'with default args' do
+      it 'executes the command' do
+        stub_spawn(cmd, 600, tmp_repo_path, {}, success: true)
+
+        is_expected.to be_truthy
+      end
+
+      it 'fails' do
+        stub_spawn(cmd, 600, tmp_repo_path, {}, success: false)
+
+        is_expected.to be_falsy
+      end
+    end
+
+    context 'with --force' do
+      let(:force) { true }
+      let(:cmd) { %W(git fetch #{remote_name} --prune --quiet --force --tags) }
+
+      it 'executes the command with forced option' do
+        stub_spawn(cmd, 600, tmp_repo_path, {}, success: true)
+
+        is_expected.to be_truthy
+      end
+    end
+
+    context 'with --no-tags' do
+      let(:tags) { false }
+      let(:cmd) { %W(git fetch #{remote_name} --prune --quiet --no-tags) }
+
+      it 'executes the command' do
+        stub_spawn(cmd, 600, tmp_repo_path, {}, success: true)
+
+        is_expected.to be_truthy
+      end
+    end
+
+    describe 'with an SSH key' do
+      let(:extra_args) { { ssh_key: 'SSH KEY' } }
+
+      it 'sets GIT_SSH to a custom script' do
+        script = stub_tempfile('scriptFile', 'gitlab-shell-ssh-wrapper', chmod: 0o755)
+        key = stub_tempfile('/tmp files/keyFile', 'gitlab-shell-key-file', chmod: 0o400)
+
+        stub_spawn(cmd, 600, tmp_repo_path, { 'GIT_SSH' => 'scriptFile' }, success: true)
+
+        is_expected.to be_truthy
+
+        expect(script.string).to eq("#!/bin/sh\nexec ssh '-oIdentityFile=\"/tmp files/keyFile\"' '-oIdentitiesOnly=\"yes\"' \"$@\"")
+        expect(key.string).to eq('SSH KEY')
+      end
+    end
+
+    describe 'with known_hosts data' do
+      let(:extra_args) { { known_hosts: 'KNOWN HOSTS' } }
+
+      it 'sets GIT_SSH to a custom script' do
+        script = stub_tempfile('scriptFile', 'gitlab-shell-ssh-wrapper', chmod: 0o755)
+        key = stub_tempfile('/tmp files/knownHosts', 'gitlab-shell-known-hosts', chmod: 0o400)
+
+        stub_spawn(cmd, 600, tmp_repo_path, { 'GIT_SSH' => 'scriptFile' }, success: true)
+
+        is_expected.to be_truthy
+
+        expect(script.string).to eq("#!/bin/sh\nexec ssh '-oStrictHostKeyChecking=\"yes\"' '-oUserKnownHostsFile=\"/tmp files/knownHosts\"' \"$@\"")
+        expect(key.string).to eq('KNOWN HOSTS')
+      end
+    end
+  end
+
+  describe '#import_project' do
+    let(:project) { create(:project) }
+    let(:import_url) { TestEnv.factory_repo_path_bare }
+    let(:cmd) { %W(git clone --bare -- #{import_url} #{tmp_repo_path}) }
+    let(:timeout) { 600 }
+
+    subject { gl_projects.import_project(import_url, timeout) }
+
+    context 'success import' do
+      it 'imports a repo' do
+        expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
+
+        message = "Importing project from <#{import_url}> to <#{tmp_repo_path}>."
+        expect(logger).to receive(:info).with(message)
+
+        is_expected.to be_truthy
+
+        expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_truthy
+      end
+    end
+
+    context 'already exists' do
+      it "doesn't import" do
+        FileUtils.mkdir_p(tmp_repo_path)
+
+        is_expected.to be_falsy
+      end
+    end
+
+    context 'timeout' do
+      it 'does not import a repo' do
+        stub_spawn_timeout(cmd, timeout, nil)
+
+        message = "Importing project from <#{import_url}> to <#{tmp_repo_path}> failed."
+        expect(logger).to receive(:error).with(message)
+
+        is_expected.to be_falsy
+
+        expect(gl_projects.output).to eq("Timed out\n")
+        expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
+      end
+    end
+  end
+
+  describe '#fork_repository' do
+    let(:dest_repos_path) { tmp_repos_path }
+    let(:dest_repo_name) { File.join('@hashed', 'aa', 'bb', 'xyz.git') }
+    let(:dest_repo) { File.join(dest_repos_path, dest_repo_name) }
+    let(:dest_namespace) { File.dirname(dest_repo) }
+
+    subject { gl_projects.fork_repository(dest_repos_path, dest_repo_name) }
+
+    before do
+      FileUtils.mkdir_p(dest_repos_path)
+    end
+
+    after do
+      FileUtils.rm_rf(dest_repos_path)
+    end
+
+    it 'forks the repository' do
+      message = "Forking repository from <#{tmp_repo_path}> to <#{dest_repo}>."
+      expect(logger).to receive(:info).with(message)
+
+      is_expected.to be_truthy
+
+      expect(File.exist?(dest_repo)).to be_truthy
+      expect(File.exist?(File.join(dest_repo, 'hooks', 'pre-receive'))).to be_truthy
+      expect(File.exist?(File.join(dest_repo, 'hooks', 'post-receive'))).to be_truthy
+    end
+
+    it 'does not fork if a project of the same name already exists' do
+      # create a fake project at the intended destination
+      FileUtils.mkdir_p(dest_repo)
+
+      # trying to fork again should fail as the repo already exists
+      message = "fork-repository failed: destination repository <#{dest_repo}> already exists."
+      expect(logger).to receive(:error).with(message)
+
+      is_expected.to be_falsy
+    end
+
+    context 'different storages' do
+      let(:dest_repos_path) { File.join(File.dirname(tmp_repos_path), 'alternative') }
+
+      it 'forks the repo' do
+        is_expected.to be_truthy
+
+        expect(File.exist?(dest_repo)).to be_truthy
+        expect(File.exist?(File.join(dest_repo, 'hooks', 'pre-receive'))).to be_truthy
+        expect(File.exist?(File.join(dest_repo, 'hooks', 'post-receive'))).to be_truthy
+      end
+    end
+  end
+
+  def build_gitlab_projects(*args)
+    described_class.new(
+      *args,
+      global_hooks_path: Gitlab.config.gitlab_shell.hooks_path,
+      logger: logger
+    )
+  end
+
+  def stub_spawn(*args, success: true)
+    exitstatus = success ? 0 : nil
+    expect(gl_projects).to receive(:popen_with_timeout).with(*args)
+      .and_return(["output", exitstatus])
+  end
+
+  def stub_spawn_timeout(*args)
+    expect(gl_projects).to receive(:popen_with_timeout).with(*args)
+      .and_raise(Timeout::Error)
+  end
+end

spec/lib/gitlab/git/repository_spec.rb

@@ -19,6 +19,51 @@ describe Gitlab::Git::Repository, seed_helper: true do
 
   let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
 
+  describe '.create_hooks' do
+    let(:repo_path) { File.join(TestEnv.repos_path, 'hook-test.git') }
+    let(:hooks_dir) { File.join(repo_path, 'hooks') }
+    let(:target_hooks_dir) { Gitlab.config.gitlab_shell.hooks_path }
+    let(:existing_target) { File.join(repo_path, 'foobar') }
+
+    before do
+      FileUtils.rm_rf(repo_path)
+      FileUtils.mkdir_p(repo_path)
+    end
+
+    context 'hooks is a directory' do
+      let(:existing_file) { File.join(hooks_dir, 'my-file') }
+
+      before do
+        FileUtils.mkdir_p(hooks_dir)
+        FileUtils.touch(existing_file)
+        described_class.create_hooks(repo_path, target_hooks_dir)
+      end
+
+      it { expect(File.readlink(hooks_dir)).to eq(target_hooks_dir) }
+      it { expect(Dir[File.join(repo_path, "hooks.old.*/my-file")].count).to eq(1) }
+    end
+
+    context 'hooks is a valid symlink' do
+      before do
+        FileUtils.mkdir_p existing_target
+        File.symlink(existing_target, hooks_dir)
+        described_class.create_hooks(repo_path, target_hooks_dir)
+      end
+
+      it { expect(File.readlink(hooks_dir)).to eq(target_hooks_dir) }
+    end
+
+    context 'hooks is a broken symlink' do
+      before do
+        FileUtils.rm_f(existing_target)
+        File.symlink(existing_target, hooks_dir)
+        described_class.create_hooks(repo_path, target_hooks_dir)
+      end
+
+      it { expect(File.readlink(hooks_dir)).to eq(target_hooks_dir) }
+    end
+  end
+
   describe "Respond to" do
     subject { repository }
 

spec/support/fixture_helpers.rb

@@ -5,9 +5,19 @@ module FixtureHelpers
     File.read(expand_fixture_path(filename))
   end
 
+  def fixture_file_ee(filename)
+    return '' if filename.blank?
+
+    File.read(expand_fixture_ee_path(filename))
+  end
+
   def expand_fixture_path(filename)
     File.expand_path(Rails.root.join('spec/fixtures/', filename))
   end
+
+  def expand_fixture_ee_path(filename)
+    File.expand_path(Rails.root.join('spec/ee/fixtures/', filename))
+  end
 end
 
 RSpec.configure do |config|

spec/support/stub_env.rb

@@ -17,6 +17,7 @@ module StubENV
 
   def add_stubbed_value(key, value)
     allow(ENV).to receive(:[]).with(key).and_return(value)
+    allow(ENV).to receive(:key?).with(key).and_return(true)
     allow(ENV).to receive(:fetch).with(key).and_return(value)
     allow(ENV).to receive(:fetch).with(key, anything()) do |_, default_val|
       value || default_val
@@ -29,6 +30,7 @@ module StubENV
 
   def init_stub
     allow(ENV).to receive(:[]).and_call_original
+    allow(ENV).to receive(:key?).and_call_original
     allow(ENV).to receive(:fetch).and_call_original
     add_stubbed_value(STUBBED_KEY, true)
   end