From 5ae9a44aa17c8929627cc450f936cd960c143e25 Mon Sep 17 00:00:00 2001 From: Jacopo Date: Thu, 13 Dec 2018 20:26:56 +0100 Subject: Add project http fetch statistics API The API get projects/:id/traffic/fetches allows user with write access to the repository to get the number of clones for the last 30 days. --- app/controllers/projects/git_http_controller.rb | 9 ++++ app/finders/projects/daily_statistics_finder.rb | 21 ++++++++ app/models/project.rb | 4 ++ app/models/project_daily_statistic.rb | 10 ++++ app/policies/project_policy.rb | 1 + .../projects/fetch_statistics_increment_service.rb | 32 +++++++++++ app/workers/all_queues.yml | 1 + app/workers/project_daily_statistics_worker.rb | 13 +++++ ...2086-project-fetch-statistics-api-http-only.yml | 5 ++ config/sidekiq_queues.yml | 1 + ...181205171941_create_project_daily_statistics.rb | 18 +++++++ db/schema.rb | 8 +++ doc/api/project_statistics.md | 49 +++++++++++++++++ doc/user/permissions.md | 1 + doc/user/project/index.md | 26 +++++++-- lib/api/api.rb | 1 + lib/api/entities.rb | 12 +++++ lib/api/project_statistics.rb | 23 ++++++++ spec/factories/project_daily_statistics.rb | 8 +++ spec/models/project_daily_statistic_spec.rb | 7 +++ spec/models/project_spec.rb | 12 +++++ spec/policies/project_policy_spec.rb | 1 + spec/requests/api/project_statistics_spec.rb | 62 ++++++++++++++++++++++ .../fetch_statistics_increment_service_spec.rb | 36 +++++++++++++ .../project_daily_statistics_worker_spec.rb | 35 ++++++++++++ 25 files changed, 393 insertions(+), 3 deletions(-) create mode 100644 app/finders/projects/daily_statistics_finder.rb create mode 100644 app/models/project_daily_statistic.rb create mode 100644 app/services/projects/fetch_statistics_increment_service.rb create mode 100644 app/workers/project_daily_statistics_worker.rb create mode 100644 changelogs/unreleased/42086-project-fetch-statistics-api-http-only.yml create mode 100644 db/migrate/20181205171941_create_project_daily_statistics.rb create mode 100644 doc/api/project_statistics.md create mode 100644 lib/api/project_statistics.rb create mode 100644 spec/factories/project_daily_statistics.rb create mode 100644 spec/models/project_daily_statistic_spec.rb create mode 100644 spec/requests/api/project_statistics_spec.rb create mode 100644 spec/services/projects/fetch_statistics_increment_service_spec.rb create mode 100644 spec/workers/project_daily_statistics_worker_spec.rb diff --git a/app/controllers/projects/git_http_controller.rb b/app/controllers/projects/git_http_controller.rb index 30e436365de..0c5328fc941 100644 --- a/app/controllers/projects/git_http_controller.rb +++ b/app/controllers/projects/git_http_controller.rb @@ -20,6 +20,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController # POST /foo/bar.git/git-upload-pack (git pull) def git_upload_pack + enqueue_fetch_statistics_update + render_ok end @@ -67,6 +69,13 @@ class Projects::GitHttpController < Projects::GitHttpClientController render plain: exception.message, status: :service_unavailable end + def enqueue_fetch_statistics_update + return if wiki? + return unless project.daily_statistics_enabled? + + ProjectDailyStatisticsWorker.perform_async(project.id) + end + def access @access ||= access_klass.new(access_actor, project, 'http', authentication_abilities: authentication_abilities, diff --git a/app/finders/projects/daily_statistics_finder.rb b/app/finders/projects/daily_statistics_finder.rb new file mode 100644 index 00000000000..912c23107bc --- /dev/null +++ b/app/finders/projects/daily_statistics_finder.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Projects + class DailyStatisticsFinder + attr_reader :project + + def initialize(project) + @project = project + end + + def fetches + ProjectDailyStatistic.of_project(project) + .of_last_30_days + .sorted_by_date_desc + end + + def total_fetch_count + fetches.sum_fetch_count + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index c72d3a3b725..82ef95d975a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -629,6 +629,10 @@ class Project < ActiveRecord::Base auto_devops&.enabled.nil? && !(Gitlab::CurrentSettings.auto_devops_enabled? || Feature.enabled?(:force_autodevops_on_by_default, self)) end + def daily_statistics_enabled? + Feature.enabled?(:project_daily_statistics, self, default_enabled: true) + end + def empty_repo? repository.empty? end diff --git a/app/models/project_daily_statistic.rb b/app/models/project_daily_statistic.rb new file mode 100644 index 00000000000..ff115dd010f --- /dev/null +++ b/app/models/project_daily_statistic.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class ProjectDailyStatistic < ActiveRecord::Base + belongs_to :project + + scope :of_project, -> (project) { where(project: project) } + scope :of_last_30_days, -> { where('date >= ?', 29.days.ago.utc.to_date) } + scope :sorted_by_date_desc, -> { order(project_id: :desc, date: :desc) } + scope :sum_fetch_count, -> { sum(:fetch_count) } +end diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index cadbc5ae009..403ad85c5c4 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -278,6 +278,7 @@ class ProjectPolicy < BasePolicy enable :admin_cluster enable :create_environment_terminal enable :destroy_release + enable :daily_statistics end rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror diff --git a/app/services/projects/fetch_statistics_increment_service.rb b/app/services/projects/fetch_statistics_increment_service.rb new file mode 100644 index 00000000000..8644e6bf313 --- /dev/null +++ b/app/services/projects/fetch_statistics_increment_service.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Projects + class FetchStatisticsIncrementService + attr_reader :project + + def initialize(project) + @project = project + end + + def execute + increment_fetch_count_sql = <<~SQL + INSERT INTO #{table_name} (project_id, date, fetch_count) + VALUES (#{project.id}, '#{Date.today}', 1) + SQL + + increment_fetch_count_sql += if Gitlab::Database.postgresql? + "ON CONFLICT (project_id, date) DO UPDATE SET fetch_count = #{table_name}.fetch_count + 1" + else + "ON DUPLICATE KEY UPDATE fetch_count = #{table_name}.fetch_count + 1" + end + + ActiveRecord::Base.connection.execute(increment_fetch_count_sql) + end + + private + + def table_name + ProjectDailyStatistic.table_name + end + end +end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index d0fc130b04f..337f39b2091 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -146,3 +146,4 @@ - repository_cleanup - delete_stored_files - import_issues_csv +- project_daily_statistics diff --git a/app/workers/project_daily_statistics_worker.rb b/app/workers/project_daily_statistics_worker.rb new file mode 100644 index 00000000000..101f5c28459 --- /dev/null +++ b/app/workers/project_daily_statistics_worker.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class ProjectDailyStatisticsWorker + include ApplicationWorker + + def perform(project_id) + project = Project.find_by_id(project_id) + + return unless project&.repository&.exists? + + Projects::FetchStatisticsIncrementService.new(project).execute + end +end diff --git a/changelogs/unreleased/42086-project-fetch-statistics-api-http-only.yml b/changelogs/unreleased/42086-project-fetch-statistics-api-http-only.yml new file mode 100644 index 00000000000..f2c4f88b746 --- /dev/null +++ b/changelogs/unreleased/42086-project-fetch-statistics-api-http-only.yml @@ -0,0 +1,5 @@ +--- +title: Add project fetch statistics +merge_request: 23596 +author: Jacopo Beschi @jacopo-beschi +type: added diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 90cd787d5ac..1f40363e126 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -85,5 +85,6 @@ - [repository_cleanup, 1] - [delete_stored_files, 1] - [remote_mirror_notification, 2] + - [project_daily_statistics, 1] - [import_issues_csv, 2] - [chat_notification, 2] diff --git a/db/migrate/20181205171941_create_project_daily_statistics.rb b/db/migrate/20181205171941_create_project_daily_statistics.rb new file mode 100644 index 00000000000..c9e2a1e1aa7 --- /dev/null +++ b/db/migrate/20181205171941_create_project_daily_statistics.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class CreateProjectDailyStatistics < ActiveRecord::Migration[5.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + create_table :project_daily_statistics, id: :bigserial do |t| + t.integer :project_id, null: false + t.integer :fetch_count, null: false + t.date :date + + t.index [:project_id, :date], unique: true, order: { date: :desc } + t.foreign_key :projects, on_delete: :cascade + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 1651a24f412..3a2bab1c57e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1578,6 +1578,13 @@ ActiveRecord::Schema.define(version: 20190204115450) do t.index ["project_id", "key"], name: "index_project_custom_attributes_on_project_id_and_key", unique: true, using: :btree end + create_table "project_daily_statistics", id: :bigserial, force: :cascade do |t| + t.integer "project_id", null: false + t.integer "fetch_count", null: false + t.date "date" + t.index ["project_id", "date"], name: "index_project_daily_statistics_on_project_id_and_date", unique: true, order: { date: :desc }, using: :btree + end + create_table "project_deploy_tokens", force: :cascade do |t| t.integer "project_id", null: false t.integer "deploy_token_id", null: false @@ -2461,6 +2468,7 @@ ActiveRecord::Schema.define(version: 20190204115450) do add_foreign_key "project_auto_devops", "projects", on_delete: :cascade add_foreign_key "project_ci_cd_settings", "projects", name: "fk_24c15d2f2e", on_delete: :cascade add_foreign_key "project_custom_attributes", "projects", on_delete: :cascade + add_foreign_key "project_daily_statistics", "projects", on_delete: :cascade add_foreign_key "project_deploy_tokens", "deploy_tokens", on_delete: :cascade add_foreign_key "project_deploy_tokens", "projects", on_delete: :cascade add_foreign_key "project_error_tracking_settings", "projects", on_delete: :cascade diff --git a/doc/api/project_statistics.md b/doc/api/project_statistics.md new file mode 100644 index 00000000000..34d73abfcbf --- /dev/null +++ b/doc/api/project_statistics.md @@ -0,0 +1,49 @@ +# Project statistics API + +Every API call to [project](../user/project/index.md) statistics must be authenticated. + +## Get the statistics of the last 30 days + +Retrieving the statistics requires write access to the repository. +Currently only HTTP fetches statistics are returned. +Fetches statistics includes both clones and pulls count and are HTTP only, SSH fetches are not included. + +``` +GET /projects/:id/statistics +``` + +| Attribute | Type | Required | Description | +| ---------- | ------ | -------- | ----------- | +| `id ` | integer / string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | + +Example response: + +```json +{ + "fetches": { + "total": 50, + "days": [ + { + "count": 10, + "date": "2018-01-10" + }, + { + "count": 10, + "date": "2018-01-09" + }, + { + "count": 10, + "date": "2018-01-08" + }, + { + "count": 10, + "date": "2018-01-07" + }, + { + "count": 10, + "date": "2018-01-06" + } + ] + } +} +``` diff --git a/doc/user/permissions.md b/doc/user/permissions.md index 74a966f3a17..5fc24e175ae 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -111,6 +111,7 @@ The following table depicts the various user permission levels in a project. | Force push to protected branches [^4] | | | | | | | Remove protected branches [^4] | | | | | | | View project Audit Events | | | | ✓ | ✓ | +| View project statistics | | | | ✓ | ✓ | ## Project features permissions diff --git a/doc/user/project/index.md b/doc/user/project/index.md index 6a1aadf058e..4a1b164bf37 100644 --- a/doc/user/project/index.md +++ b/doc/user/project/index.md @@ -8,7 +8,7 @@ Your projects can be [available](../../public_access/public_access.md) publicly, internally, or privately, at your choice. GitLab does not limit the number of private projects you create. -## Project's features +## Project features When you create a project in GitLab, you'll have access to a large number of [features](https://about.gitlab.com/features/): @@ -82,7 +82,7 @@ your code blocks, overriding GitLab's default choice of language. the source, build output, and other metadata or artifacts associated with a released version of your code. -### Project's integrations +### Project integrations [Integrate your project](integrations/index.md) with Jira, Mattermost, Kubernetes, Slack, and a lot more. @@ -116,7 +116,7 @@ Read through the documentation on [project settings](settings/index.md). - [Export a project from GitLab](settings/import_export.md#exporting-a-project-and-its-data) - [Importing and exporting projects between GitLab instances](settings/import_export.md) -## Project's members +## Project members Learn how to [add members to your projects](members/index.md). @@ -170,3 +170,23 @@ password To quickly access a project from the GitLab UI using the project ID, visit the `/projects/:id` URL in your browser or other tool accessing the project. + +## Project APIs + +There are numerous [APIs](../../api/README.md) to use with your projects: + +- [Badges](../../api/project_badges.md) +- [Clusters](../../api/project_clusters.md) +- [Discussions](../../api/discussions.md) +- [General](../../api/projects.md) +- [Import/export](../../api/project_import_export.md) +- [Issue Board](../../api/boards.md) +- [Labels](../../api/labels.md) +- [Markdown](../../api/markdown.md) +- [Merge Requests](../../api/merge_requests.md) +- [Milestones](../../api/milestones.md) +- [Services](../../api/services.md) +- [Snippets](../../api/project_snippets.md) +- [Templates](../../api/project_templates.md) +- [Traffic](../../api/project_statistics.md) +- [Variables](../../api/project_level_variables.md) diff --git a/lib/api/api.rb b/lib/api/api.rb index 4dd1b459554..bf8ddba6f0d 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -141,6 +141,7 @@ module API mount ::API::Projects mount ::API::ProjectSnapshots mount ::API::ProjectSnippets + mount ::API::ProjectStatistics mount ::API::ProjectTemplates mount ::API::ProtectedBranches mount ::API::ProtectedTags diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9199f898ea0..c13cb389947 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -300,6 +300,18 @@ module API expose :build_artifacts_size, as: :job_artifacts_size end + class ProjectDailyFetches < Grape::Entity + expose :fetch_count, as: :count + expose :date + end + + class ProjectDailyStatistics < Grape::Entity + expose :fetches do + expose :total_fetch_count, as: :total + expose :fetches, as: :days, using: ProjectDailyFetches + end + end + class Member < Grape::Entity expose :user, merge: true, using: UserBasic expose :access_level diff --git a/lib/api/project_statistics.rb b/lib/api/project_statistics.rb new file mode 100644 index 00000000000..2f73785f72d --- /dev/null +++ b/lib/api/project_statistics.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module API + class ProjectStatistics < Grape::API + before do + authenticate! + not_found! unless user_project.daily_statistics_enabled? + authorize! :daily_statistics, user_project + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get the list of project fetch statistics for the last 30 days' + get ":id/statistics" do + statistic_finder = ::Projects::DailyStatisticsFinder.new(user_project) + + present statistic_finder, with: Entities::ProjectDailyStatistics + end + end + end +end diff --git a/spec/factories/project_daily_statistics.rb b/spec/factories/project_daily_statistics.rb new file mode 100644 index 00000000000..7e4142fa401 --- /dev/null +++ b/spec/factories/project_daily_statistics.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :project_daily_statistic do + project + fetch_count 1 + end +end diff --git a/spec/models/project_daily_statistic_spec.rb b/spec/models/project_daily_statistic_spec.rb new file mode 100644 index 00000000000..86210af15d8 --- /dev/null +++ b/spec/models/project_daily_statistic_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe ProjectDailyStatistic do + it { is_expected.to belong_to(:project) } +end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 1f9088c2e6b..4a218cdc24f 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2335,6 +2335,18 @@ describe Project do end end + describe '#daily_statistics_enabled?' do + it { is_expected.to be_daily_statistics_enabled } + + context 'when :project_daily_statistics is disabled for the project' do + before do + stub_feature_flags(project_daily_statistics: { thing: subject, enabled: false }) + end + + it { is_expected.not_to be_daily_statistics_enabled } + end + end + describe '#change_head' do let(:project) { create(:project, :repository) } diff --git a/spec/policies/project_policy_spec.rb b/spec/policies/project_policy_spec.rb index 93a468f585b..997bdc82af6 100644 --- a/spec/policies/project_policy_spec.rb +++ b/spec/policies/project_policy_spec.rb @@ -49,6 +49,7 @@ describe ProjectPolicy do admin_project_member admin_note admin_wiki admin_project admin_commit_status admin_build admin_container_image admin_pipeline admin_environment admin_deployment destroy_release add_cluster + daily_statistics ] end diff --git a/spec/requests/api/project_statistics_spec.rb b/spec/requests/api/project_statistics_spec.rb new file mode 100644 index 00000000000..184d0a72c37 --- /dev/null +++ b/spec/requests/api/project_statistics_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe API::ProjectStatistics do + let(:maintainer) { create(:user) } + let(:public_project) { create(:project, :public) } + + before do + public_project.add_maintainer(maintainer) + end + + describe 'GET /projects/:id/statistics' do + let!(:fetch_statistics1) { create(:project_daily_statistic, project: public_project, fetch_count: 30, date: 29.days.ago) } + let!(:fetch_statistics2) { create(:project_daily_statistic, project: public_project, fetch_count: 4, date: 3.days.ago) } + let!(:fetch_statistics3) { create(:project_daily_statistic, project: public_project, fetch_count: 3, date: 2.days.ago) } + let!(:fetch_statistics4) { create(:project_daily_statistic, project: public_project, fetch_count: 2, date: 1.day.ago) } + let!(:fetch_statistics5) { create(:project_daily_statistic, project: public_project, fetch_count: 1, date: Date.today) } + let!(:fetch_statistics_other_project) { create(:project_daily_statistic, project: create(:project), fetch_count: 29, date: 29.days.ago) } + + it 'returns the fetch statistics of the last 30 days' do + get api("/projects/#{public_project.id}/statistics", maintainer) + + expect(response).to have_gitlab_http_status(200) + fetches = json_response['fetches'] + expect(fetches['total']).to eq(40) + expect(fetches['days'].length).to eq(5) + expect(fetches['days'].first).to eq({ 'count' => fetch_statistics5.fetch_count, 'date' => fetch_statistics5.date.to_s }) + expect(fetches['days'].last).to eq({ 'count' => fetch_statistics1.fetch_count, 'date' => fetch_statistics1.date.to_s }) + end + + it 'excludes the fetch statistics older than 30 days' do + create(:project_daily_statistic, fetch_count: 31, project: public_project, date: 30.days.ago) + + get api("/projects/#{public_project.id}/statistics", maintainer) + + expect(response).to have_gitlab_http_status(200) + fetches = json_response['fetches'] + expect(fetches['total']).to eq(40) + expect(fetches['days'].length).to eq(5) + expect(fetches['days'].last).to eq({ 'count' => fetch_statistics1.fetch_count, 'date' => fetch_statistics1.date.to_s }) + end + + it 'responds with 403 when the user is not a maintainer of the repository' do + developer = create(:user) + public_project.add_developer(developer) + + get api("/projects/#{public_project.id}/statistics", developer) + + expect(response).to have_gitlab_http_status(403) + expect(json_response['message']).to eq('403 Forbidden') + end + + it 'responds with 404 when daily_statistics_enabled? is false' do + stub_feature_flags(project_daily_statistics: { thing: public_project, enabled: false }) + + get api("/projects/#{public_project.id}/statistics", maintainer) + + expect(response).to have_gitlab_http_status(404) + end + end +end diff --git a/spec/services/projects/fetch_statistics_increment_service_spec.rb b/spec/services/projects/fetch_statistics_increment_service_spec.rb new file mode 100644 index 00000000000..fcfb138aad6 --- /dev/null +++ b/spec/services/projects/fetch_statistics_increment_service_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'spec_helper' + +module Projects + describe FetchStatisticsIncrementService do + let(:project) { create(:project) } + + describe '#execute' do + subject { described_class.new(project).execute } + + it 'creates a new record for today with count == 1' do + expect { subject }.to change { ProjectDailyStatistic.count }.by(1) + created_stat = ProjectDailyStatistic.last + + expect(created_stat.fetch_count).to eq(1) + expect(created_stat.project).to eq(project) + expect(created_stat.date).to eq(Date.today) + end + + it "doesn't increment previous days statistics" do + yesterday_stat = create(:project_daily_statistic, fetch_count: 5, project: project, date: 1.day.ago) + + expect { subject }.not_to change { yesterday_stat.reload.fetch_count } + end + + context 'when the record already exists for today' do + let!(:project_daily_stat) { create(:project_daily_statistic, fetch_count: 5, project: project, date: Date.today) } + + it 'increments the today record count by 1' do + expect { subject }.to change { project_daily_stat.reload.fetch_count }.to(6) + end + end + end + end +end diff --git a/spec/workers/project_daily_statistics_worker_spec.rb b/spec/workers/project_daily_statistics_worker_spec.rb new file mode 100644 index 00000000000..8640add99e5 --- /dev/null +++ b/spec/workers/project_daily_statistics_worker_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe ProjectDailyStatisticsWorker, '#perform' do + let(:worker) { described_class.new } + let(:project) { create(:project) } + + describe '#perform' do + context 'with a non-existing project' do + it 'does nothing' do + expect(Projects::FetchStatisticsIncrementService).not_to receive(:new) + + worker.perform(-1) + end + end + + context 'with an existing project without a repository' do + it 'does nothing' do + expect(Projects::FetchStatisticsIncrementService).not_to receive(:new) + + worker.perform(project.id) + end + end + + it 'calls daily_statistics_service with the given project' do + project = create(:project, :repository) + + expect_next_instance_of(Projects::FetchStatisticsIncrementService, project) do |service| + expect(service).to receive(:execute) + end + + worker.perform(project.id) + end + end +end -- cgit v1.2.3