Merge branch 'ee-42086-project-fetch-statistics-api-http-only' into 'master'

CE Port 23596 Add project http fetch statistics API

See merge request gitlab-org/gitlab-ee!9740

app/controllers/projects/git_http_controller.rb

@@ -20,6 +20,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController
 
   # POST /foo/bar.git/git-upload-pack (git pull)
   def git_upload_pack
+    enqueue_fetch_statistics_update
+
     render_ok
   end
 
@@ -67,6 +69,13 @@ class Projects::GitHttpController < Projects::GitHttpClientController
     render plain: exception.message, status: :service_unavailable
   end
 
+  def enqueue_fetch_statistics_update
+    return if wiki?
+    return unless project.daily_statistics_enabled?
+
+    ProjectDailyStatisticsWorker.perform_async(project.id)
+  end
+
   def access
     @access ||= access_klass.new(access_actor, project,
       'http', authentication_abilities: authentication_abilities,

app/finders/projects/daily_statistics_finder.rb

+# frozen_string_literal: true
+
+module Projects
+  class DailyStatisticsFinder
+    attr_reader :project
+
+    def initialize(project)
+      @project = project
+    end
+
+    def fetches
+      ProjectDailyStatistic.of_project(project)
+        .of_last_30_days
+        .sorted_by_date_desc
+    end
+
+    def total_fetch_count
+      fetches.sum_fetch_count
+    end
+  end
+end

app/models/project.rb

@@ -639,6 +639,10 @@ class Project < ActiveRecord::Base
     auto_devops&.enabled.nil? && !(Gitlab::CurrentSettings.auto_devops_enabled? || Feature.enabled?(:force_autodevops_on_by_default, self))
   end
 
+  def daily_statistics_enabled?
+    Feature.enabled?(:project_daily_statistics, self, default_enabled: true)
+  end
+
   def empty_repo?
     repository.empty?
   end

app/models/project_daily_statistic.rb

+# frozen_string_literal: true
+
+class ProjectDailyStatistic < ActiveRecord::Base
+  belongs_to :project
+
+  scope :of_project, -> (project) { where(project: project) }
+  scope :of_last_30_days, -> { where('date >= ?', 29.days.ago.utc.to_date) }
+  scope :sorted_by_date_desc, -> { order(project_id: :desc, date: :desc) }
+  scope :sum_fetch_count, -> { sum(:fetch_count) }
+end

app/policies/project_policy.rb

@@ -278,6 +278,7 @@ class ProjectPolicy < BasePolicy
     enable :admin_cluster
     enable :create_environment_terminal
     enable :destroy_release
+    enable :daily_statistics
   end
 
   rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror

app/services/projects/fetch_statistics_increment_service.rb

+# frozen_string_literal: true
+
+module Projects
+  class FetchStatisticsIncrementService
+    attr_reader :project
+
+    def initialize(project)
+      @project = project
+    end
+
+    def execute
+      increment_fetch_count_sql = <<~SQL
+        INSERT INTO #{table_name} (project_id, date, fetch_count)
+        VALUES (#{project.id}, '#{Date.today}', 1)
+      SQL
+
+      increment_fetch_count_sql += if Gitlab::Database.postgresql?
+                                     "ON CONFLICT (project_id, date) DO UPDATE SET fetch_count = #{table_name}.fetch_count + 1"
+                                   else
+                                     "ON DUPLICATE KEY UPDATE fetch_count = #{table_name}.fetch_count + 1"
+                                   end
+
+      ActiveRecord::Base.connection.execute(increment_fetch_count_sql)
+    end
+
+    private
+
+    def table_name
+      ProjectDailyStatistic.table_name
+    end
+  end
+end

app/workers/all_queues.yml

@@ -146,3 +146,4 @@
 - repository_cleanup
 - delete_stored_files
 - import_issues_csv
+- project_daily_statistics

app/workers/project_daily_statistics_worker.rb

+# frozen_string_literal: true
+
+class ProjectDailyStatisticsWorker
+  include ApplicationWorker
+
+  def perform(project_id)
+    project = Project.find_by_id(project_id)
+
+    return unless project&.repository&.exists?
+
+    Projects::FetchStatisticsIncrementService.new(project).execute
+  end
+end

changelogs/unreleased/42086-project-fetch-statistics-api-http-only.yml

+---
+title: Add project fetch statistics
+merge_request: 23596
+author: Jacopo Beschi @jacopo-beschi
+type: added

config/sidekiq_queues.yml

@@ -85,6 +85,7 @@
   - [repository_cleanup, 1]
   - [delete_stored_files, 1]
   - [remote_mirror_notification, 2]
+  - [project_daily_statistics, 1]
   - [import_issues_csv, 2]
   - [chat_notification, 2]
 

db/migrate/20181205171941_create_project_daily_statistics.rb

+# frozen_string_literal: true
+
+class CreateProjectDailyStatistics < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :project_daily_statistics, id: :bigserial do |t|
+      t.integer :project_id, null: false
+      t.integer :fetch_count, null: false
+      t.date :date
+
+      t.index [:project_id, :date], unique: true, order: { date: :desc }
+      t.foreign_key :projects, on_delete: :cascade
+    end
+  end
+end

db/schema.rb

@@ -2213,6 +2213,13 @@ ActiveRecord::Schema.define(version: 20190220150130) do
     t.index ["project_id", "key"], name: "index_project_custom_attributes_on_project_id_and_key", unique: true, using: :btree
   end
 
+  create_table "project_daily_statistics", id: :bigserial, force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.integer "fetch_count", null: false
+    t.date "date"
+    t.index ["project_id", "date"], name: "index_project_daily_statistics_on_project_id_and_date", unique: true, order: { date: :desc }, using: :btree
+  end
+
   create_table "project_deploy_tokens", force: :cascade do |t|
     t.integer "project_id", null: false
     t.integer "deploy_token_id", null: false
@@ -3488,6 +3495,7 @@ ActiveRecord::Schema.define(version: 20190220150130) do
   add_foreign_key "project_auto_devops", "projects", on_delete: :cascade
   add_foreign_key "project_ci_cd_settings", "projects", name: "fk_24c15d2f2e", on_delete: :cascade
   add_foreign_key "project_custom_attributes", "projects", on_delete: :cascade
+  add_foreign_key "project_daily_statistics", "projects", on_delete: :cascade
   add_foreign_key "project_deploy_tokens", "deploy_tokens", on_delete: :cascade
   add_foreign_key "project_deploy_tokens", "projects", on_delete: :cascade
   add_foreign_key "project_error_tracking_settings", "projects", on_delete: :cascade

doc/api/project_statistics.md

+# Project statistics API
+
+Every API call to [project](../user/project/index.md) statistics must be authenticated.
+
+## Get the statistics of the last 30 days
+
+Retrieving the statistics requires write access to the repository.
+Currently only HTTP fetches statistics are returned.
+Fetches statistics includes both clones and pulls count and are HTTP only, SSH fetches are not included.
+
+```
+GET /projects/:id/statistics
+```
+
+| Attribute  | Type   | Required | Description |
+| ---------- | ------ | -------- | ----------- |
+| `id `      | integer / string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) |
+
+Example response:
+
+```json
+{
+  "fetches": {
+    "total": 50,
+    "days": [
+      {
+        "count": 10,
+        "date": "2018-01-10"
+      },
+      {
+        "count": 10,
+        "date": "2018-01-09"
+      },
+      {
+        "count": 10,
+        "date": "2018-01-08"
+      },
+      {
+        "count": 10,
+        "date": "2018-01-07"
+      },
+      {
+        "count": 10,
+        "date": "2018-01-06"
+      }
+    ]
+  }
+}
+```

doc/user/permissions.md

@@ -110,6 +110,7 @@ The following table depicts the various user permission levels in a project.
 | Force push to protected branches [^4] |         |            |             |          |        |
 | Remove protected branches [^4]        |         |            |             |          |        |
 | View project Audit Events             |         |            |             | ✓        | ✓      |
+| View project statistics               |         |            |             | ✓        | ✓      |
 
 ## Project features permissions
 

doc/user/project/index.md

@@ -8,7 +8,7 @@ Your projects can be [available](../../public_access/public_access.md)
 publicly, internally, or privately, at your choice. GitLab does not limit
 the number of private projects you create.
 
-## Project's features
+## Project features
 
 When you create a project in GitLab, you'll have access to a large number of
 [features](https://about.gitlab.com/features/):
@@ -96,7 +96,7 @@ When you create a project in GitLab, you'll have access to a large number of
 - [Code owners](code_owners.md): specify code owners for certain files **[STARTER]**
 - [License Management](merge_requests/license_management.md): approve and blacklist licenses for projects. **[ULTIMATE]**
 
-### Project's integrations
+### Project integrations
 
 [Integrate your project](integrations/index.md) with Jira, Mattermost,
 Kubernetes, Slack, and a lot more.
@@ -137,7 +137,7 @@ as a CI/CD project.
 
 Read through the documentation on [CI/CD for external repositories](../../ci/ci_cd_for_external_repos/index.md).
 
-## Project's members
+## Project members
 
 Learn how to [add members to your projects](members/index.md).
 
@@ -191,3 +191,23 @@ password <personal_access_token>
 
 To quickly access a project from the GitLab UI using the project ID,
 visit the `/projects/:id` URL in your browser or other tool accessing the project.
+
+## Project APIs
+
+There are numerous [APIs](../../api/README.md) to use with your projects:
+
+- [Badges](../../api/project_badges.md)
+- [Clusters](../../api/project_clusters.md)
+- [Discussions](../../api/discussions.md)
+- [General](../../api/projects.md)
+- [Import/export](../../api/project_import_export.md)
+- [Issue Board](../../api/boards.md)
+- [Labels](../../api/labels.md)
+- [Markdown](../../api/markdown.md)
+- [Merge Requests](../../api/merge_requests.md)
+- [Milestones](../../api/milestones.md)
+- [Services](../../api/services.md)
+- [Snippets](../../api/project_snippets.md)
+- [Templates](../../api/project_templates.md)
+- [Traffic](../../api/project_statistics.md)
+- [Variables](../../api/project_level_variables.md)

lib/api/api.rb

@@ -151,6 +151,7 @@ module API
     mount ::API::Projects
     mount ::API::ProjectSnapshots
     mount ::API::ProjectSnippets
+    mount ::API::ProjectStatistics
     mount ::API::ProjectTemplates
     mount ::API::ProtectedBranches
     mount ::API::ProtectedTags

lib/api/entities.rb

@@ -300,6 +300,18 @@ module API
       expose :build_artifacts_size, as: :job_artifacts_size
     end
 
+    class ProjectDailyFetches < Grape::Entity
+      expose :fetch_count, as: :count
+      expose :date
+    end
+
+    class ProjectDailyStatistics < Grape::Entity
+      expose :fetches do
+        expose :total_fetch_count, as: :total
+        expose :fetches, as: :days, using: ProjectDailyFetches
+      end
+    end
+
     class Member < Grape::Entity
       expose :user, merge: true, using: UserBasic
       expose :access_level

lib/api/project_statistics.rb

+# frozen_string_literal: true
+
+module API
+  class ProjectStatistics < Grape::API
+    before do
+      authenticate!
+      not_found! unless user_project.daily_statistics_enabled?
+      authorize! :daily_statistics, user_project
+    end
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      desc 'Get the list of project fetch statistics for the last 30 days'
+      get ":id/statistics" do
+        statistic_finder = ::Projects::DailyStatisticsFinder.new(user_project)
+
+        present statistic_finder, with: Entities::ProjectDailyStatistics
+      end
+    end
+  end
+end

spec/factories/project_daily_statistics.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :project_daily_statistic do
+    project
+    fetch_count 1
+  end
+end

spec/models/project_daily_statistic_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe ProjectDailyStatistic do
+  it { is_expected.to belong_to(:project) }
+end

spec/models/project_spec.rb

@@ -2603,6 +2603,18 @@ describe Project do
     end
   end
 
+  describe '#daily_statistics_enabled?' do
+    it { is_expected.to be_daily_statistics_enabled }
+
+    context 'when :project_daily_statistics is disabled for the project' do
+      before do
+        stub_feature_flags(project_daily_statistics: { thing: subject, enabled: false })
+      end
+
+      it { is_expected.not_to be_daily_statistics_enabled }
+    end
+  end
+
   describe '#change_head' do
     let(:project) { create(:project, :repository) }
 

spec/policies/project_policy_spec.rb

@@ -49,6 +49,7 @@ describe ProjectPolicy do
       admin_project_member admin_note admin_wiki admin_project
       admin_commit_status admin_build admin_container_image
       admin_pipeline admin_environment admin_deployment destroy_release add_cluster
+      daily_statistics
     ]
   end
 

spec/requests/api/project_statistics_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe API::ProjectStatistics do
+  let(:maintainer) { create(:user) }
+  let(:public_project) { create(:project, :public) }
+
+  before do
+    public_project.add_maintainer(maintainer)
+  end
+
+  describe 'GET /projects/:id/statistics' do
+    let!(:fetch_statistics1) { create(:project_daily_statistic, project: public_project, fetch_count: 30, date: 29.days.ago) }
+    let!(:fetch_statistics2) { create(:project_daily_statistic, project: public_project, fetch_count: 4, date: 3.days.ago) }
+    let!(:fetch_statistics3) { create(:project_daily_statistic, project: public_project, fetch_count: 3, date: 2.days.ago) }
+    let!(:fetch_statistics4) { create(:project_daily_statistic, project: public_project, fetch_count: 2, date: 1.day.ago) }
+    let!(:fetch_statistics5) { create(:project_daily_statistic, project: public_project, fetch_count: 1, date: Date.today) }
+    let!(:fetch_statistics_other_project) { create(:project_daily_statistic, project: create(:project), fetch_count: 29, date: 29.days.ago) }
+
+    it 'returns the fetch statistics of the last 30 days' do
+      get api("/projects/#{public_project.id}/statistics", maintainer)
+
+      expect(response).to have_gitlab_http_status(200)
+      fetches = json_response['fetches']
+      expect(fetches['total']).to eq(40)
+      expect(fetches['days'].length).to eq(5)
+      expect(fetches['days'].first).to eq({ 'count' => fetch_statistics5.fetch_count, 'date' => fetch_statistics5.date.to_s })
+      expect(fetches['days'].last).to eq({ 'count' => fetch_statistics1.fetch_count, 'date' => fetch_statistics1.date.to_s })
+    end
+
+    it 'excludes the fetch statistics older than 30 days' do
+      create(:project_daily_statistic, fetch_count: 31, project: public_project, date: 30.days.ago)
+
+      get api("/projects/#{public_project.id}/statistics", maintainer)
+
+      expect(response).to have_gitlab_http_status(200)
+      fetches = json_response['fetches']
+      expect(fetches['total']).to eq(40)
+      expect(fetches['days'].length).to eq(5)
+      expect(fetches['days'].last).to eq({ 'count' => fetch_statistics1.fetch_count, 'date' => fetch_statistics1.date.to_s })
+    end
+
+    it 'responds with 403 when the user is not a maintainer of the repository' do
+      developer = create(:user)
+      public_project.add_developer(developer)
+
+      get api("/projects/#{public_project.id}/statistics", developer)
+
+      expect(response).to have_gitlab_http_status(403)
+      expect(json_response['message']).to eq('403 Forbidden')
+    end
+
+    it 'responds with 404 when daily_statistics_enabled? is false' do
+      stub_feature_flags(project_daily_statistics: { thing: public_project, enabled: false })
+
+      get api("/projects/#{public_project.id}/statistics", maintainer)
+
+      expect(response).to have_gitlab_http_status(404)
+    end
+  end
+end

spec/services/projects/fetch_statistics_increment_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+module Projects
+  describe FetchStatisticsIncrementService do
+    let(:project) { create(:project) }
+
+    describe '#execute' do
+      subject { described_class.new(project).execute }
+
+      it 'creates a new record for today with count == 1' do
+        expect { subject }.to change { ProjectDailyStatistic.count }.by(1)
+        created_stat = ProjectDailyStatistic.last
+
+        expect(created_stat.fetch_count).to eq(1)
+        expect(created_stat.project).to eq(project)
+        expect(created_stat.date).to eq(Date.today)
+      end
+
+      it "doesn't increment previous days statistics" do
+        yesterday_stat = create(:project_daily_statistic, fetch_count: 5, project: project, date: 1.day.ago)
+
+        expect { subject }.not_to change { yesterday_stat.reload.fetch_count }
+      end
+
+      context 'when the record already exists for today' do
+        let!(:project_daily_stat) { create(:project_daily_statistic, fetch_count: 5, project: project, date: Date.today) }
+
+        it 'increments the today record count by 1' do
+          expect { subject }.to change { project_daily_stat.reload.fetch_count }.to(6)
+        end
+      end
+    end
+  end
+end

spec/workers/project_daily_statistics_worker_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe ProjectDailyStatisticsWorker, '#perform' do
+  let(:worker) { described_class.new }
+  let(:project) { create(:project) }
+
+  describe '#perform' do
+    context 'with a non-existing project' do
+      it 'does nothing' do
+        expect(Projects::FetchStatisticsIncrementService).not_to receive(:new)
+
+        worker.perform(-1)
+      end
+    end
+
+    context 'with an existing project without a repository' do
+      it 'does nothing' do
+        expect(Projects::FetchStatisticsIncrementService).not_to receive(:new)
+
+        worker.perform(project.id)
+      end
+    end
+
+    it 'calls daily_statistics_service with the given project' do
+      project = create(:project, :repository)
+
+      expect_next_instance_of(Projects::FetchStatisticsIncrementService, project) do |service|
+        expect(service).to receive(:execute)
+      end
+
+      worker.perform(project.id)
+    end
+  end
+end