Merge branch 'ci/api-runners' into 'master'

Add runners API

References #4264

See merge request !2640

CHANGELOG

@@ -65,6 +65,7 @@ v 8.5.0 (unreleased)
   - Ability to see and sort on vote count from Issues and MR lists
   - Fix builds scheduler when first build in stage was allowed to fail
   - User project limit is reached notice is hidden if the projects limit is zero 
+  - Add API support for managing runners and project's runners
 
 v 8.4.4
   - Update omniauth-saml gem to 1.4.2

app/models/ci/runner.rb

@@ -22,6 +22,7 @@ module Ci
     extend Ci::Model
 
     LAST_CONTACT_TIME = 5.minutes.ago
+    AVAILABLE_SCOPES = ['specific', 'shared', 'active', 'paused', 'online']
     
     has_many :builds, class_name: 'Ci::Build'
     has_many :runner_projects, dependent: :destroy, class_name: 'Ci::RunnerProject'
@@ -38,6 +39,11 @@ module Ci
     scope :online, ->() { where('contacted_at > ?', LAST_CONTACT_TIME) }
     scope :ordered, ->() { order(id: :desc) }
 
+    scope :owned_or_shared, ->(project_id) do
+      joins('LEFT JOIN ci_runner_projects ON ci_runner_projects.runner_id = ci_runners.id')
+        .where("ci_runner_projects.gl_project_id = :project_id OR ci_runners.is_shared = true", project_id: project_id)
+    end
+
     acts_as_taggable
 
     def self.search(query)

doc/api/README.md

@@ -32,6 +32,7 @@ following locations:
 - [Builds](builds.md)
 - [Build triggers](build_triggers.md)
 - [Build Variables](build_variables.md)
+- [Runners](runners.md)
 
 ## Authentication
 

doc/api/runners.md

+# Runners API
+
+## List owned runners
+
+Get a list of specific runners available to the user.
+
+```
+GET /runners
+GET /runners?scope=active
+```
+
+| Attribute | Type    | Required | Description         |
+|-----------|---------|----------|---------------------|
+| `scope`   | string  | no       | The scope of specific runners to show, one of: `active`, `paused`, `online`; showing all runners if none provided |
+
+```
+curl -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v3/runners"
+```
+
+Example response:
+
+```json
+[
+    {
+        "active": true,
+        "description": "test-1-20150125",
+        "id": 6,
+        "is_shared": false,
+        "name": null
+    },
+    {
+        "active": true,
+        "description": "test-2-20150125",
+        "id": 8,
+        "is_shared": false,
+        "name": null
+    }
+]
+```
+
+## List all runners
+
+Get a list of all runners in the GitLab instance (specific and shared). Access
+is restricted to users with `admin` privileges.
+
+```
+GET /runners/all
+GET /runners/all?scope=online
+```
+
+| Attribute | Type    | Required | Description         |
+|-----------|---------|----------|---------------------|
+| `scope`   | string  | no       | The scope of runners to show, one of: `specific`, `shared`, `active`, `paused`, `online`; showing all runners if none provided |
+
+```
+curl -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v3/runners/all"
+```
+
+Example response:
+
+```json
+[
+    {
+        "active": true,
+        "description": "shared-runner-1",
+        "id": 1,
+        "is_shared": true,
+        "name": null
+    },
+    {
+        "active": true,
+        "description": "shared-runner-2",
+        "id": 3,
+        "is_shared": true,
+        "name": null
+    },
+    {
+        "active": true,
+        "description": "test-1-20150125",
+        "id": 6,
+        "is_shared": false,
+        "name": null
+    },
+    {
+        "active": true,
+        "description": "test-2-20150125",
+        "id": 8,
+        "is_shared": false,
+        "name": null
+    }
+]
+```
+
+## Get runner's details
+
+Get details of a runner.
+
+```
+GET /runners/:id
+```
+
+| Attribute | Type    | Required | Description         |
+|-----------|---------|----------|---------------------|
+| `id`      | integer | yes      | The ID of a runner  |
+
+```
+curl -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v3/runners/6"
+```
+
+Example response:
+
+```json
+{
+    "active": true,
+    "architecture": null,
+    "description": "test-1-20150125",
+    "id": 6,
+    "is_shared": false,
+    "contacted_at": "2016-01-25T16:39:48.066Z",
+    "name": null,
+    "platform": null,
+    "projects": [
+        {
+            "id": 1,
+            "name": "GitLab Community Edition",
+            "name_with_namespace": "GitLab.org / GitLab Community Edition",
+            "path": "gitlab-ce",
+            "path_with_namespace": "gitlab-org/gitlab-ce"
+        }
+    ],
+    "token": "205086a8e3b9a2b818ffac9b89d102",
+    "revision": null,
+    "tag_list": [
+        "ruby",
+        "mysql"
+    ],
+    "version": null
+}
+```
+
+## Update runner's details
+
+Update details of a runner.
+
+```
+PUT /runners/:id
+```
+
+| Attribute     | Type    | Required | Description         |
+|---------------|---------|----------|---------------------|
+| `id`          | integer | yes      | The ID of a runner  |
+| `description` | string  | no       | The description of a runner |
+| `active`      | boolean | no       | The state of a runner; can be set to `true` or `false` |
+| `tag_list`    | array   | no       | The list of tags for a runner; put array of tags, that should be finally assigned to a runner |
+
+```
+curl -X PUT -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v3/runners/6" -F "description=test-1-20150125-test" -F "tag_list=ruby,mysql,tag1,tag2"
+```
+
+Example response:
+
+```json
+{
+    "active": true,
+    "architecture": null,
+    "description": "test-1-20150125-test",
+    "id": 6,
+    "is_shared": false,
+    "contacted_at": "2016-01-25T16:39:48.066Z",
+    "name": null,
+    "platform": null,
+    "projects": [
+        {
+            "id": 1,
+            "name": "GitLab Community Edition",
+            "name_with_namespace": "GitLab.org / GitLab Community Edition",
+            "path": "gitlab-ce",
+            "path_with_namespace": "gitlab-org/gitlab-ce"
+        }
+    ],
+    "token": "205086a8e3b9a2b818ffac9b89d102",
+    "revision": null,
+    "tag_list": [
+        "ruby",
+        "mysql",
+        "tag1",
+        "tag2"
+    ],
+    "version": null
+}
+```
+
+## Remove a runner
+
+Remove a runner.
+
+```
+DELETE /runners/:id
+```
+
+| Attribute | Type    | Required | Description         |
+|-----------|---------|----------|---------------------|
+| `id`      | integer | yes      | The ID of a runner  |
+
+```
+curl -X DELETE -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v3/runners/6"
+```
+
+Example response:
+
+```json
+{
+    "active": true,
+    "description": "test-1-20150125-test",
+    "id": 6,
+    "is_shared": false,
+    "name": null,
+}
+```
+
+## List project's runners
+
+List all runners (specific and shared) available in the project. Shared runners
+are listed if at least one shared runner is defined **and** shared runners
+usage is enabled in the project's settings.
+
+```
+GET /projects/:id/runners
+```
+
+| Attribute | Type    | Required | Description         |
+|-----------|---------|----------|---------------------|
+| `id`      | integer | yes      | The ID of a project |
+
+```
+curl -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v3/projects/9/runners"
+```
+
+Example response:
+
+```json
+[
+    {
+        "active": true,
+        "description": "test-2-20150125",
+        "id": 8,
+        "is_shared": false,
+        "name": null
+    },
+    {
+        "active": true,
+        "description": "development_runner",
+        "id": 5,
+        "is_shared": true,
+        "name": null
+    }
+]
+```
+
+## Enable a runner in project
+
+Enable an available specific runner in the project.
+
+```
+POST /projects/:id/runners
+```
+
+| Attribute   | Type    | Required | Description         |
+|-------------|---------|----------|---------------------|
+| `id`        | integer | yes      | The ID of a project |
+| `runner_id` | integer | yes      | The ID of a runner  |
+
+```
+curl -X POST -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v3/project/9/runners" -F "runner_id=9"
+```
+
+Example response:
+
+```json
+{
+    "active": true,
+    "description": "test-2016-02-01",
+    "id": 9,
+    "is_shared": false,
+    "name": null
+}
+```
+
+## Disable a runner from project
+
+Disable a specific runner from the project. It works only if the project isn't
+the only project associated with the specified runner. If so, an error is
+returned. Use the [Remove a runner](#remove-a-runner) call instead.
+
+```
+DELETE /projects/:id/runners/:runner_id
+```
+
+| Attribute   | Type    | Required | Description         |
+|-------------|---------|----------|---------------------|
+| `id`        | integer | yes      | The ID of a project |
+| `runner_id` | integer | yes      | The ID of a runner  |
+
+```
+curl -X DELETE -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v3/project/9/runners/9"
+```
+
+Example response:
+
+```json
+{
+    "active": true,
+    "description": "test-2016-02-01",
+    "id": 9,
+    "is_shared": false,
+    "name": null
+}
+```

doc/ci/api/runners.md

 # Runners API
 
+_**Note:** This API is intended to be used only by Runners as their own
+communication channel. For the consumer API see the
+[new Runners API](../../api/runners.md)._
+
 ## Runners
 
 ### Retrieve all runners
@@ -74,4 +78,4 @@ Returns:
   "updated_at" : "2015-02-26T11:39:39.232Z",
   "description" : "awesome runner"
 }
-```
\ No newline at end of file
+```

lib/api/api.rb

@@ -56,5 +56,6 @@ module API
     mount Triggers
     mount Builds
     mount Variables
+    mount Runners
   end
 end

lib/api/entities.rb

@@ -49,7 +49,7 @@ module API
       expose :enable_ssl_verification
     end
 
-    class ForkedFromProject < Grape::Entity
+    class BasicProjectDetails < Grape::Entity
       expose :id
       expose :name, :name_with_namespace
       expose :path, :path_with_namespace
@@ -67,7 +67,7 @@ module API
       expose :shared_runners_enabled
       expose :creator_id
       expose :namespace
-      expose :forked_from_project, using: Entities::ForkedFromProject, if: lambda{ |project, options| project.forked? }
+      expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? }
       expose :avatar_url
       expose :star_count, :forks_count
       expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? }
@@ -377,6 +377,20 @@ module API
       expose :name
     end
 
+    class RunnerDetails < Runner
+      expose :tag_list
+      expose :version, :revision, :platform, :architecture
+      expose :contacted_at
+      expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? }
+      expose :projects, with: Entities::BasicProjectDetails do |runner, options|
+        if options[:current_user].is_admin?
+          runner.projects
+        else
+          options[:current_user].authorized_projects.where(id: runner.projects)
+        end
+      end
+    end
+
     class Build < Grape::Entity
       expose :id, :status, :stage, :name, :ref, :tag, :coverage
       expose :created_at, :started_at, :finished_at

lib/api/runners.rb

+module API
+  # Runners API
+  class Runners < Grape::API
+    before { authenticate! }
+
+    resource :runners do
+      # Get runners available for user
+      #
+      # Example Request:
+      #   GET /runners
+      get do
+        runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: ['specific', 'shared'])
+        present paginate(runners), with: Entities::Runner
+      end
+
+      # Get all runners - shared and specific
+      #
+      # Example Request:
+      #   GET /runners/all
+      get 'all' do
+        authenticated_as_admin!
+        runners = filter_runners(Ci::Runner.all, params[:scope])
+        present paginate(runners), with: Entities::Runner
+      end
+
+      # Get runner's details
+      #
+      # Parameters:
+      #   id (required) - The ID of ther runner
+      # Example Request:
+      #   GET /runners/:id
+      get ':id' do
+        runner = get_runner(params[:id])
+        authenticate_show_runner!(runner)
+
+        present runner, with: Entities::RunnerDetails, current_user: current_user
+      end
+
+      # Update runner's details
+      #
+      # Parameters:
+      #   id (required) - The ID of ther runner
+      #   description (optional) - Runner's description
+      #   active (optional) - Runner's status
+      #   tag_list (optional) - Array of tags for runner
+      # Example Request:
+      #   PUT /runners/:id
+      put ':id' do
+        runner = get_runner(params[:id])
+        authenticate_update_runner!(runner)
+
+        attrs = attributes_for_keys [:description, :active, :tag_list]
+        if runner.update(attrs)
+          present runner, with: Entities::RunnerDetails, current_user: current_user
+        else
+          render_validation_error!(runner)
+        end
+      end
+
+      # Remove runner
+      #
+      # Parameters:
+      #   id (required) - The ID of ther runner
+      # Example Request:
+      #   DELETE /runners/:id
+      delete ':id' do
+        runner = get_runner(params[:id])
+        authenticate_delete_runner!(runner)
+        runner.destroy!
+
+        present runner, with: Entities::Runner
+      end
+    end
+
+    resource :projects do
+      before { authorize_admin_project }
+
+      # Get runners available for project
+      #
+      # Example Request:
+      #   GET /projects/:id/runners
+      get ':id/runners' do
+        runners = filter_runners(Ci::Runner.owned_or_shared(user_project.id), params[:scope])
+        present paginate(runners), with: Entities::Runner
+      end
+
+      # Enable runner for project
+      #
+      # Parameters:
+      #   id (required) - The ID of the project
+      #   runner_id (required) - The ID of the runner
+      # Example Request:
+      #   POST /projects/:id/runners/:runner_id
+      post ':id/runners' do
+        required_attributes! [:runner_id]
+
+        runner = get_runner(params[:runner_id])
+        authenticate_enable_runner!(runner)
+        Ci::RunnerProject.create(runner: runner, project: user_project)
+
+        present runner, with: Entities::Runner
+      end
+
+      # Disable project's runner
+      #
+      # Parameters:
+      #   id (required) - The ID of the project
+      #   runner_id (required) - The ID of the runner
+      # Example Request:
+      #   DELETE /projects/:id/runners/:runner_id
+      delete ':id/runners/:runner_id' do
+        runner_project = user_project.runner_projects.find_by(runner_id: params[:runner_id])
+        not_found!('Runner') unless runner_project
+
+        runner = runner_project.runner
+        forbidden!("Only one project associated with the runner. Please remove the runner instead") if runner.projects.count == 1
+
+        runner_project.destroy
+
+        present runner, with: Entities::Runner
+      end
+    end
+
+    helpers do
+      def filter_runners(runners, scope, options = {})
+        return runners unless scope.present?
+
+        available_scopes = ::Ci::Runner::AVAILABLE_SCOPES
+        if options[:without]
+          available_scopes = available_scopes - options[:without]
+        end
+
+        if (available_scopes & [scope]).empty?
+          render_api_error!('Scope contains invalid value', 400)
+        end
+
+        runners.send(scope)
+      end
+
+      def get_runner(id)
+        runner = Ci::Runner.find(id)
+        not_found!('Runner') unless runner
+        runner
+      end
+
+      def authenticate_show_runner!(runner)
+        return if runner.is_shared || current_user.is_admin?
+        forbidden!("No access granted") unless user_can_access_runner?(runner)
+      end
+
+      def authenticate_update_runner!(runner)
+        return if current_user.is_admin?
+        forbidden!("Runner is shared") if runner.is_shared?
+        forbidden!("No access granted") unless user_can_access_runner?(runner)
+      end
+
+      def authenticate_delete_runner!(runner)
+        return if current_user.is_admin?
+        forbidden!("Runner is shared") if runner.is_shared?
+        forbidden!("Runner associated with more than one project") if runner.projects.count > 1
+        forbidden!("No access granted") unless user_can_access_runner?(runner)
+      end
+
+      def authenticate_enable_runner!(runner)
+        forbidden!("Runner is shared") if runner.is_shared?
+        return if current_user.is_admin?
+        forbidden!("No access granted") unless user_can_access_runner?(runner)
+      end
+
+      def user_can_access_runner?(runner)
+        current_user.ci_authorized_runners.exists?(runner.id)
+      end
+    end
+  end
+end

spec/factories/ci/runners.rb

@@ -25,14 +25,12 @@ FactoryGirl.define do
       "My runner#{n}"
     end
 
-    platform "darwin"
+    platform  "darwin"
+    is_shared false
+    active    true
 
-    factory :ci_shared_runner do
+    trait :shared do
       is_shared true
     end
-
-    factory :ci_specific_runner do
-      is_shared false
-    end
   end
 end

spec/features/runners_spec.rb

@@ -17,10 +17,10 @@ describe "Runners" do
       @project3 = FactoryGirl.create :empty_project
       @project3.team << [user, :developer]
 
-      @shared_runner = FactoryGirl.create :ci_shared_runner
-      @specific_runner = FactoryGirl.create :ci_specific_runner
-      @specific_runner2 = FactoryGirl.create :ci_specific_runner
-      @specific_runner3 = FactoryGirl.create :ci_specific_runner
+      @shared_runner = FactoryGirl.create :ci_runner, :shared
+      @specific_runner = FactoryGirl.create :ci_runner
+      @specific_runner2 = FactoryGirl.create :ci_runner
+      @specific_runner3 = FactoryGirl.create :ci_runner
       @project.runners << @specific_runner
       @project2.runners << @specific_runner2
       @project3.runners << @specific_runner3
@@ -84,7 +84,7 @@ describe "Runners" do
     before do
       @project = FactoryGirl.create :empty_project
       @project.team << [user, :master]
-      @specific_runner = FactoryGirl.create :ci_specific_runner
+      @specific_runner = FactoryGirl.create :ci_runner
       @project.runners << @specific_runner
     end
 

spec/models/build_spec.rb

@@ -243,7 +243,7 @@ describe Ci::Build, models: true do
   end
 
   describe :can_be_served? do
-    let(:runner) { FactoryGirl.create :ci_specific_runner }
+    let(:runner) { FactoryGirl.create :ci_runner }
 
     before { build.project.runners << runner }
 
@@ -285,7 +285,7 @@ describe Ci::Build, models: true do
     end
 
     context 'if there are runner' do
-      let(:runner) { FactoryGirl.create :ci_specific_runner }
+      let(:runner) { FactoryGirl.create :ci_runner }
 
       before do
         build.project.runners << runner
@@ -322,7 +322,7 @@ describe Ci::Build, models: true do
         it { is_expected.to be_truthy }
 
         context "and there are specific runner" do
-          let(:runner) { FactoryGirl.create :ci_specific_runner, contacted_at: 1.second.ago }
+          let(:runner) { FactoryGirl.create :ci_runner, contacted_at: 1.second.ago }
 
           before do
             build.project.runners << runner

spec/models/ci/runner_spec.rb

@@ -39,7 +39,7 @@ describe Ci::Runner, models: true do
 
   describe :assign_to do
     let!(:project) { FactoryGirl.create :empty_project }
-    let!(:shared_runner) { FactoryGirl.create(:ci_shared_runner) }
+    let!(:shared_runner) { FactoryGirl.create(:ci_runner, :shared) }
 
     before { shared_runner.assign_to(project) }
 
@@ -52,15 +52,15 @@ describe Ci::Runner, models: true do
     subject { Ci::Runner.online }
 
     before do
-      @runner1 = FactoryGirl.create(:ci_shared_runner, contacted_at: 1.year.ago)
-      @runner2 = FactoryGirl.create(:ci_shared_runner, contacted_at: 1.second.ago)
+      @runner1 = FactoryGirl.create(:ci_runner, :shared, contacted_at: 1.year.ago)
+      @runner2 = FactoryGirl.create(:ci_runner, :shared, contacted_at: 1.second.ago)
     end
 
     it { is_expected.to eq([@runner2])}
   end
 
   describe :online? do
-    let(:runner) { FactoryGirl.create(:ci_shared_runner) }
+    let(:runner) { FactoryGirl.create(:ci_runner, :shared) }
 
     subject { runner.online? }
 
@@ -84,7 +84,7 @@ describe Ci::Runner, models: true do
   end
 
   describe :status do
-    let(:runner) { FactoryGirl.create(:ci_shared_runner, contacted_at: 1.second.ago) }
+    let(:runner) { FactoryGirl.create(:ci_runner, :shared, contacted_at: 1.second.ago) }
 
     subject { runner.status }
 
@@ -115,7 +115,7 @@ describe Ci::Runner, models: true do
 
   describe "belongs_to_one_project?" do
     it "returns false if there are two projects runner assigned to" do
-      runner = FactoryGirl.create(:ci_specific_runner)
+      runner = FactoryGirl.create(:ci_runner)
       project = FactoryGirl.create(:empty_project)
       project1 = FactoryGirl.create(:empty_project)
       project.runners << runner
@@ -125,7 +125,7 @@ describe Ci::Runner, models: true do
     end
 
     it "returns true" do
-      runner = FactoryGirl.create(:ci_specific_runner)
+      runner = FactoryGirl.create(:ci_runner)
       project = FactoryGirl.create(:empty_project)
       project.runners << runner
 

spec/models/project_spec.rb

@@ -519,8 +519,8 @@ describe Project, models: true do
 
   describe :any_runners do
     let(:project) { create(:empty_project, shared_runners_enabled: shared_runners_enabled) }
-    let(:specific_runner) { create(:ci_specific_runner) }
-    let(:shared_runner) { create(:ci_shared_runner) }
+    let(:specific_runner) { create(:ci_runner) }
+    let(:shared_runner) { create(:ci_runner, :shared) }
 
     context 'for shared runners disabled' do
       let(:shared_runners_enabled) { false }