From 1eeef229aae5affdce415c2364858e8efc64f4b5 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 3 Apr 2020 03:07:58 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- .../georgekoltsov-add-metrics-to-importers.yml | 5 + .../update-auto-build-image-with-cnb-support.yml | 5 + danger/roulette/Dangerfile | 45 ++++++-- doc/README.md | 2 +- doc/ci/junit_test_reports.md | 15 +++ doc/ci/multi_project_pipelines.md | 2 - doc/ci/pipelines/index.md | 4 +- doc/ci/pipelines/job_artifacts.md | 2 +- doc/ci/pipelines/settings.md | 2 +- doc/topics/autodevops/index.md | 30 +++++- doc/user/project/clusters/add_remove_clusters.md | 6 +- doc/user/project/clusters/runbooks/index.md | 4 +- lib/gitlab/bitbucket_import/importer.rb | 114 +++++++++++---------- lib/gitlab/bitbucket_import/metrics.rb | 41 ++++++++ lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml | 2 +- lib/gitlab/danger/helper.rb | 6 +- lib/gitlab/import/metrics.rb | 60 +++++++++++ spec/lib/gitlab/bitbucket_import/importer_spec.rb | 68 ++++++++++++ spec/lib/gitlab/import/metrics_spec.rb | 56 ++++++++++ spec/services/projects/import_service_spec.rb | 20 +++- 20 files changed, 407 insertions(+), 82 deletions(-) create mode 100644 changelogs/unreleased/georgekoltsov-add-metrics-to-importers.yml create mode 100644 changelogs/unreleased/update-auto-build-image-with-cnb-support.yml create mode 100644 lib/gitlab/bitbucket_import/metrics.rb create mode 100644 lib/gitlab/import/metrics.rb create mode 100644 spec/lib/gitlab/import/metrics_spec.rb diff --git a/changelogs/unreleased/georgekoltsov-add-metrics-to-importers.yml b/changelogs/unreleased/georgekoltsov-add-metrics-to-importers.yml new file mode 100644 index 00000000000..12a8cabb4a7 --- /dev/null +++ b/changelogs/unreleased/georgekoltsov-add-metrics-to-importers.yml @@ -0,0 +1,5 @@ +--- +title: Add Bitbucket Importer metrics +merge_request: 27524 +author: +type: other diff --git a/changelogs/unreleased/update-auto-build-image-with-cnb-support.yml b/changelogs/unreleased/update-auto-build-image-with-cnb-support.yml new file mode 100644 index 00000000000..517a127a0e7 --- /dev/null +++ b/changelogs/unreleased/update-auto-build-image-with-cnb-support.yml @@ -0,0 +1,5 @@ +--- +title: Add initial support for Cloud Native Buildpacks in Auto DevOps builds +merge_request: 28165 +author: +type: added diff --git a/danger/roulette/Dangerfile b/danger/roulette/Dangerfile index d07d652e601..417b5889bcf 100644 --- a/danger/roulette/Dangerfile +++ b/danger/roulette/Dangerfile @@ -38,24 +38,32 @@ MARKDOWN NO_REVIEWER = 'No reviewer available'.freeze NO_MAINTAINER = 'No maintainer available'.freeze -def spin_for_category(team, project, category, branch_name) - random = roulette.new_random(branch_name) - labels = gitlab.mr_labels +Spin = Struct.new(:reviewer, :maintainer) + +def spin_role_for_category(team, role, project, category) + team.select do |member| + member.public_send("#{role}?", project, category, gitlab.mr_labels) # rubocop:disable GitlabSecurity/PublicSend + end +end +def spin_for_category(team, project, category, branch_name) reviewers, traintainers, maintainers = - %i[reviewer? traintainer? maintainer?].map do |kind| - team.select do |member| - member.public_send(kind, project, category, labels) # rubocop:disable GitlabSecurity/PublicSend - end + %i[reviewer traintainer maintainer].map do |role| + spin_role_for_category(team, role, project, category) end # TODO: take CODEOWNERS into account? # https://gitlab.com/gitlab-org/gitlab/issues/26723 # Make traintainers have triple the chance to be picked as a reviewer + random = roulette.new_random(branch_name) reviewer = roulette.spin_for_person(reviewers + traintainers + traintainers, random: random) maintainer = roulette.spin_for_person(maintainers, random: random) + Spin.new(reviewer, maintainer) +end + +def markdown_row_for_category(category, reviewer, maintainer) "| #{helper.label_for_category(category)} | #{reviewer&.markdown_name || NO_REVIEWER} | #{maintainer&.markdown_name || NO_MAINTAINER} |" end @@ -85,8 +93,29 @@ if changes.any? && !gitlab.mr_labels.include?('CSS cleanup') project = helper.project_name unknown = changes.fetch(:unknown, []) + spin_per_category = categories.each_with_object({}) do |category, memo| + memo[category] = spin_for_category(team, project, category, canonical_branch_name) + end + + rows = spin_per_category.map do |category, spin| + reviewer = spin.reviewer + maintainer = spin.maintainer + + case category + when :test + if reviewer.nil? + # Fetch an already picked backend reviewer, or pick one otherwise + reviewer = spin_per_category[:backend]&.reviewer || spin_for_category(team, project, :backend, canonical_branch_name).reviewer + end + when :engineering_productivity + if maintainer.nil? + # Fetch an already picked backend maintainer, or pick one otherwise + maintainer = spin_per_category[:backend]&.maintainer || spin_for_category(team, project, :backend, canonical_branch_name).maintainer + end + end - rows = categories.map { |category| spin_for_category(team, project, category, canonical_branch_name) } + markdown_row_for_category(category, reviewer, maintainer) + end markdown(MESSAGE) markdown(CATEGORY_TABLE_HEADER + rows.join("\n")) unless rows.empty? diff --git a/doc/README.md b/doc/README.md index 3a3e93244c4..8c6f76306c4 100644 --- a/doc/README.md +++ b/doc/README.md @@ -306,7 +306,7 @@ The following documentation relates to the DevOps **Configure** stage: | Configure Topics | Description | |:-----------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------| | [Auto DevOps](topics/autodevops/index.md) | Automatically employ a complete DevOps lifecycle. | -| [Create Kubernetes clusters](user/project/clusters/add_remove_clusters.md#add-new-cluster) | Use Kubernetes and GitLab. | +| [Create Kubernetes clusters](user/project/clusters/add_remove_clusters.md#create-new-cluster) | Use Kubernetes and GitLab. | | [Executable Runbooks](user/project/clusters/runbooks/index.md) | Documented procedures that explain how to carry out particular processes. | | [GitLab ChatOps](ci/chatops/README.md) | Interact with CI/CD jobs through chat services. | | [Installing Applications](user/project/clusters/index.md#installing-applications) | Deploy Helm, Ingress, and Prometheus on Kubernetes. | diff --git a/doc/ci/junit_test_reports.md b/doc/ci/junit_test_reports.md index 78c9965aa08..313ade0887e 100644 --- a/doc/ci/junit_test_reports.md +++ b/doc/ci/junit_test_reports.md @@ -151,6 +151,21 @@ java: - target/failsafe-reports/TEST-*.xml ``` +### Python example + +This example uses pytest with the `--junitxml=report.xml` flag to format the output +for JUnit: + +```yaml +pytest: + stage: test + script: + - pytest --junitxml=report.xml + artifacts: + reports: + junit: report.xml +``` + ### C/C++ example There are a few tools that can produce JUnit reports in C/C++. diff --git a/doc/ci/multi_project_pipelines.md b/doc/ci/multi_project_pipelines.md index 7c79bf350b9..14277b6b55b 100644 --- a/doc/ci/multi_project_pipelines.md +++ b/doc/ci/multi_project_pipelines.md @@ -10,8 +10,6 @@ type: reference You can set up [GitLab CI/CD](README.md) across multiple projects, so that a pipeline in one project can trigger a pipeline in another project. -## Overview - GitLab CI/CD is a powerful continuous integration tool that works not only per project, but also across projects with multi-project pipelines. diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index 28c587f39bb..18f5c5f6827 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -12,8 +12,6 @@ Watch our ["Mastering continuous software development"](https://about.gitlab.com/webcast/mastering-ci-cd/) webcast to see a comprehensive demo of GitLab CI/CD pipeline. -## Introduction - Pipelines are the top-level component of continuous integration, delivery, and deployment. Pipelines comprise: @@ -33,7 +31,7 @@ If you have a [mirrored repository that GitLab pulls from](../../user/project/re you may need to enable pipeline triggering in your project's **Settings > Repository > Pull from a remote repository > Trigger pipelines for mirror updates**. -### Simple pipeline example +## Simple pipeline example As an example, imagine a pipeline consisting of four stages, executed in the following order: diff --git a/doc/ci/pipelines/job_artifacts.md b/doc/ci/pipelines/job_artifacts.md index d44df8defcf..ed791ea9c4a 100644 --- a/doc/ci/pipelines/job_artifacts.md +++ b/doc/ci/pipelines/job_artifacts.md @@ -3,7 +3,7 @@ disqus_identifier: 'https://docs.gitlab.com/ee/user/project/pipelines/job_artifa type: reference, howto --- -# Introduction to job artifacts +# Job artifacts > - Introduced in GitLab 8.2 and GitLab Runner 0.7.0. > - Starting with GitLab 8.4 and GitLab Runner 1.0, the artifacts archive format changed to `ZIP`, and it is now possible to browse its contents, with the added ability of downloading the files separately. diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 48862986114..b1c5ee6e24f 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -3,7 +3,7 @@ disqus_identifier: 'https://docs.gitlab.com/ee/user/project/pipelines/settings.h type: reference, howto --- -# Pipelines settings +# Pipeline settings To reach the pipelines settings navigate to your project's **Settings > CI/CD**. diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index bf58a2f7510..cc6d7d65115 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -109,7 +109,7 @@ To make full use of Auto DevOps, you will need: To enable deployments, you will need: 1. A [Kubernetes 1.12+ cluster](../../user/project/clusters/index.md) for the project. The easiest - way is to add a [new cluster using the GitLab UI](../../user/project/clusters/add_remove_clusters.md#add-new-cluster). + way is to create a [new cluster using the GitLab UI](../../user/project/clusters/add_remove_clusters.md#create-new-cluster). For Kubernetes 1.16+ clusters, there is some additional configuration for [Auto Deploy for Kubernetes 1.16+](#kubernetes-116). 1. NGINX Ingress. You can deploy it to your Kubernetes cluster by installing the [GitLab-managed app for Ingress](../../user/clusters/applications.md#ingress), @@ -404,6 +404,33 @@ If Auto Build fails despite the project meeting the buildpack requirements, set a project variable `TRACE=true` to enable verbose logging, which may help to troubleshoot. +#### Auto Build using Cloud Native Buildpacks (beta) + +> Introduced in [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28165). + +Auto Build supports building your application using [Cloud Native Buildpacks](https://buildpacks.io) +through the [`pack` command](https://github.com/buildpacks/pack). To use Cloud Native Buildpacks, +set the CI variable `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` to a non-empty value. + +Cloud Native Buildpacks (CNBs) are an evolution of Heroku buildpacks, and +will eventually supersede Herokuish-based builds within Auto DevOps. For more +information, see [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/212692). + +Builds using Cloud Native Buildpacks support the same options as builds using +Heroku buildpacks, with the following caveats: + +- The buildpack must be a Cloud Native Buildpack. A Heroku buildpack can be + converted to a Cloud Native Buildpack using Heroku's + [`cnb-shim`](https://github.com/heroku/cnb-shim). +- `BUILDPACK_URL` must be in a form + [supported by `pack`](https://buildpacks.io/docs/app-developer-guide/specific-buildpacks/). +- The `/bin/herokuish` command is not present in the resulting image, and prefixing + commands with `/bin/herokuish procfile exec` is no longer required (nor possible). + +NOTE: **Note**: Auto Test still uses Herokuish, as test suite detection is not +yet part of the Cloud Native Buildpack specification. For more information, see +[this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/212689). + ### Auto Test Auto Test automatically runs the appropriate tests for your application using @@ -1162,6 +1189,7 @@ applications. |-----------------------------------------|------------------------------------| | `ADDITIONAL_HOSTS` | Fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. | | `_ADDITIONAL_HOSTS` | For a specific environment, the fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. This takes precedence over `ADDITIONAL_HOSTS`. | +| `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` | When set to a non-empty value and no `Dockerfile` is present, Auto Build builds your application using Cloud Native Buildpacks instead of Herokuish. [More details](#auto-build-using-cloud-native-buildpacks-beta). | | `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` | Extra arguments to be passed to the `docker build` command. Note that using quotes will not prevent word splitting. [More details](#passing-arguments-to-docker-build). | | `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` | A [comma-separated list of CI variable names](#passing-secrets-to-docker-build) to be passed to the `docker build` command as secrets. | | `AUTO_DEVOPS_CHART` | Helm Chart used to deploy your apps. Defaults to the one [provided by GitLab](https://gitlab.com/gitlab-org/charts/auto-deploy-app). | diff --git a/doc/user/project/clusters/add_remove_clusters.md b/doc/user/project/clusters/add_remove_clusters.md index 7f7978d1089..dce273ce602 100644 --- a/doc/user/project/clusters/add_remove_clusters.md +++ b/doc/user/project/clusters/add_remove_clusters.md @@ -14,7 +14,7 @@ Google Kubernetes Engine Integration. All you have to do is [follow this link](h ## Before you begin -Before [adding a Kubernetes cluster](#add-new-cluster) using GitLab, you need: +Before [adding a Kubernetes cluster](#create-new-cluster) using GitLab, you need: - GitLab itself. Either: - A GitLab.com [account](https://about.gitlab.com/pricing/#gitlab-com). @@ -127,9 +127,9 @@ If you don't want to use GitLab Runner in privileged mode, either: 1. Installing a Runner [using `docker+machine`](https://docs.gitlab.com/runner/executors/docker_machine.html). -## Add new cluster +## Create new cluster -New clusters can be added using GitLab for: +New clusters can be created using GitLab for: - [Google Kubernetes Engine (GKE)](add_gke_clusters.md). - [Amazon Elastic Kubernetes Service (EKS)](add_eks_clusters.md). diff --git a/doc/user/project/clusters/runbooks/index.md b/doc/user/project/clusters/runbooks/index.md index ad3d675f158..5575cd1d2d4 100644 --- a/doc/user/project/clusters/runbooks/index.md +++ b/doc/user/project/clusters/runbooks/index.md @@ -35,7 +35,7 @@ for an overview of how this is accomplished in GitLab!** To create an executable runbook, you will need: 1. **Kubernetes** - A Kubernetes cluster is required to deploy the rest of the applications. - The simplest way to get started is to add a cluster using one of [GitLab's integrations](../add_remove_clusters.md#add-new-cluster). + The simplest way to get started is to add a cluster using one of [GitLab's integrations](../add_remove_clusters.md#create-new-cluster). 1. **Helm Tiller** - Helm is a package manager for Kubernetes and is required to install all the other applications. It is installed in its own pod inside the cluster which can run the Helm CLI in a safe environment. @@ -60,7 +60,7 @@ the components outlined above and the preloaded demo runbook. ### 1. Add a Kubernetes cluster -Follow the steps outlined in [Add new cluster](../add_remove_clusters.md#add-new-cluster) +Follow the steps outlined in [Create new cluster](../add_remove_clusters.md#create-new-cluster) to add a Kubernetes cluster to your project. ### 2. Install Helm Tiller, Ingress, and JupyterHub diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 5af839d8a32..d8f9105d66d 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -3,6 +3,8 @@ module Gitlab module BitbucketImport class Importer + include Gitlab::BitbucketImport::Metrics + LABELS = [{ title: 'bug', color: '#FF0000' }, { title: 'enhancement', color: '#428BCA' }, { title: 'proposal', color: '#69D100' }, @@ -83,38 +85,42 @@ module Gitlab errors << { type: :wiki, errors: e.message } end - # rubocop: disable CodeReuse/ActiveRecord def import_issues return unless repo.issues_enabled? create_labels client.issues(repo).each do |issue| - description = '' - description += @formatter.author_line(issue.author) unless find_user_id(issue.author) - description += issue.description - - label_name = issue.kind - milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil - - gitlab_issue = project.issues.create!( - iid: issue.iid, - title: issue.title, - description: description, - state_id: Issue.available_states[issue.state], - author_id: gitlab_user_id(project, issue.author), - milestone: milestone, - created_at: issue.created_at, - updated_at: issue.updated_at - ) - - gitlab_issue.labels << @labels[label_name] - - import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? - rescue StandardError => e - errors << { type: :issue, iid: issue.iid, errors: e.message } + import_issue(issue) end end + + # rubocop: disable CodeReuse/ActiveRecord + def import_issue(issue) + description = '' + description += @formatter.author_line(issue.author) unless find_user_id(issue.author) + description += issue.description + + label_name = issue.kind + milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil + + gitlab_issue = project.issues.create!( + iid: issue.iid, + title: issue.title, + description: description, + state_id: Issue.available_states[issue.state], + author_id: gitlab_user_id(project, issue.author), + milestone: milestone, + created_at: issue.created_at, + updated_at: issue.updated_at + ) + + gitlab_issue.labels << @labels[label_name] + + import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? + rescue StandardError => e + errors << { type: :issue, iid: issue.iid, errors: e.message } + end # rubocop: enable CodeReuse/ActiveRecord def import_issue_comments(issue, gitlab_issue) @@ -159,37 +165,41 @@ module Gitlab pull_requests = client.pull_requests(repo) pull_requests.each do |pull_request| - description = '' - description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) - description += pull_request.description - - source_branch_sha = pull_request.source_branch_sha - target_branch_sha = pull_request.target_branch_sha - source_branch_sha = project.repository.commit(source_branch_sha)&.sha || source_branch_sha - target_branch_sha = project.repository.commit(target_branch_sha)&.sha || target_branch_sha - - merge_request = project.merge_requests.create!( - iid: pull_request.iid, - title: pull_request.title, - description: description, - source_project: project, - source_branch: pull_request.source_branch_name, - source_branch_sha: source_branch_sha, - target_project: project, - target_branch: pull_request.target_branch_name, - target_branch_sha: target_branch_sha, - state: pull_request.state, - author_id: gitlab_user_id(project, pull_request.author), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - ) - - import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? - rescue StandardError => e - store_pull_request_error(pull_request, e) + import_pull_request(pull_request) end end + def import_pull_request(pull_request) + description = '' + description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) + description += pull_request.description + + source_branch_sha = pull_request.source_branch_sha + target_branch_sha = pull_request.target_branch_sha + source_branch_sha = project.repository.commit(source_branch_sha)&.sha || source_branch_sha + target_branch_sha = project.repository.commit(target_branch_sha)&.sha || target_branch_sha + + merge_request = project.merge_requests.create!( + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: project, + source_branch: pull_request.source_branch_name, + source_branch_sha: source_branch_sha, + target_project: project, + target_branch: pull_request.target_branch_name, + target_branch_sha: target_branch_sha, + state: pull_request.state, + author_id: gitlab_user_id(project, pull_request.author), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + ) + + import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? + rescue StandardError => e + store_pull_request_error(pull_request, e) + end + def import_pull_request_comments(pull_request, merge_request) comments = client.pull_request_comments(repo, pull_request.iid) diff --git a/lib/gitlab/bitbucket_import/metrics.rb b/lib/gitlab/bitbucket_import/metrics.rb new file mode 100644 index 00000000000..25e2d9b211e --- /dev/null +++ b/lib/gitlab/bitbucket_import/metrics.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Metrics + extend ActiveSupport::Concern + + IMPORTER = :bitbucket_importer + + included do + prepend Gitlab::Import::Metrics + + Gitlab::Import::Metrics.measure(:execute, metrics: { + "#{IMPORTER}_imported_projects": { + type: :counter, + description: 'The number of imported Bitbucket projects' + }, + "#{IMPORTER}_total_duration_seconds": { + type: :histogram, + labels: { importer: IMPORTER }, + description: 'Total time spent importing Bitbucket projects, in seconds' + } + }) + + Gitlab::Import::Metrics.measure(:import_issue, metrics: { + "#{IMPORTER}_imported_issues": { + type: :counter, + description: 'The number of imported Bitbucket issues' + } + }) + + Gitlab::Import::Metrics.measure(:import_pull_request, metrics: { + "#{IMPORTER}_imported_pull_requests": { + type: :counter, + description: 'The number of imported Bitbucket pull requests' + } + }) + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index bb0de9df8bf..ceaa8115c3d 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,6 +1,6 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image/master:stable" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.2.0" variables: DOCKER_TLS_CERTDIR: "" services: diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 6bb46b1730f..aa2737262be 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -100,6 +100,7 @@ module Gitlab test: "~test ~Quality for `spec/features/*`", engineering_productivity: '~"Engineering Productivity" for CI, Danger' }.freeze + # First-match win, so be sure to put more specific regex at the top... CATEGORIES = { %r{\Adoc/} => :none, # To reinstate roulette for documentation, set to `:docs`. %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :none, # To reinstate roulette for documentation, set to `:docs`. @@ -145,9 +146,8 @@ module Gitlab %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, %r{\A(ee/)?(bin|config|generator_templates|lib|rubocop)/} => :backend, %r{\A(ee/)?spec/features/} => :test, - %r{\A(ee/)?spec/(?!javascripts|frontend)[^/]+} => :backend, - %r{\A(ee/)?vendor/(?!assets)[^/]+} => :backend, - %r{\A(ee/)?vendor/(languages\.yml|licenses\.csv)\z} => :backend, + %r{\A(ee/)?spec/} => :backend, + %r{\A(ee/)?vendor/} => :backend, %r{\A(Gemfile|Gemfile.lock|Rakefile)\z} => :backend, %r{\A[A-Z_]+_VERSION\z} => :backend, %r{\A\.rubocop(_todo)?\.yml\z} => :backend, diff --git a/lib/gitlab/import/metrics.rb b/lib/gitlab/import/metrics.rb new file mode 100644 index 00000000000..76638a8cf86 --- /dev/null +++ b/lib/gitlab/import/metrics.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +# Prepend `Gitlab::Import::Metrics` to a class in order +# to measure and emit `Gitlab::Metrics` metrics of specified methods. +# +# @example +# class Importer +# prepend Gitlab::Import::Metrics +# +# Gitlab::ImportExport::Metrics.measure :execute, metrics: { +# importer_counter: { +# type: :counter, +# description: 'counter' +# }, +# importer_histogram: { +# type: :histogram, +# labels: { importer: 'importer' }, +# description: 'histogram' +# } +# } +# +# def execute +# ... +# end +# end +# +# Each call to `#execute` increments `importer_counter` as well as +# measures `#execute` duration and reports histogram `importer_histogram` +module Gitlab + module Import + module Metrics + def self.measure(method_name, metrics:) + define_method "#{method_name}" do |*args| + start_time = Time.zone.now + + result = super(*args) + + end_time = Time.zone.now + + report_measurement_metrics(metrics, end_time - start_time) + + result + end + end + + def report_measurement_metrics(metrics, duration) + metrics.each do |metric_name, metric_value| + case metric_value[:type] + when :counter + Gitlab::Metrics.counter(metric_name, metric_value[:description]).increment + when :histogram + Gitlab::Metrics.histogram(metric_name, metric_value[:description]).observe(metric_value[:labels], duration) + else + nil + end + end + end + end + end +end diff --git a/spec/lib/gitlab/bitbucket_import/importer_spec.rb b/spec/lib/gitlab/bitbucket_import/importer_spec.rb index b95175efc0c..b3c1f86c5ee 100644 --- a/spec/lib/gitlab/bitbucket_import/importer_spec.rb +++ b/spec/lib/gitlab/bitbucket_import/importer_spec.rb @@ -87,6 +87,7 @@ describe Gitlab::BitbucketImport::Importer do values: sample_issues_statuses } end + let(:counter) { double('counter', increment: true) } subject { described_class.new(project) } @@ -213,6 +214,24 @@ describe Gitlab::BitbucketImport::Importer do expect(merge_request_diff.start_commit_sha).to eq target_branch_sha end end + + context 'metrics' do + before do + allow(Gitlab::Metrics).to receive(:counter) { counter } + allow(pull_request).to receive(:raw).and_return('hello world') + end + + it 'counts imported pull requests' do + expect(Gitlab::Metrics).to receive(:counter).with( + :bitbucket_importer_imported_pull_requests, + 'The number of imported Bitbucket pull requests' + ) + + expect(counter).to receive(:increment) + + subject.execute + end + end end context 'issues statuses' do @@ -339,5 +358,54 @@ describe Gitlab::BitbucketImport::Importer do expect(importer.errors).to be_empty end end + + context 'metrics' do + before do + allow(Gitlab::Metrics).to receive(:counter) { counter } + end + + it 'counts imported issues' do + expect(Gitlab::Metrics).to receive(:counter).with( + :bitbucket_importer_imported_issues, + 'The number of imported Bitbucket issues' + ) + + expect(counter).to receive(:increment) + + subject.execute + end + end + end + + describe '#execute' do + context 'metrics' do + let(:histogram) { double(:histogram) } + + before do + allow(subject).to receive(:import_wiki) + allow(subject).to receive(:import_issues) + allow(subject).to receive(:import_pull_requests) + + allow(Gitlab::Metrics).to receive(:counter) { counter } + allow(Gitlab::Metrics).to receive(:histogram) { histogram } + end + + it 'counts and measures duration of imported projects' do + expect(Gitlab::Metrics).to receive(:counter).with( + :bitbucket_importer_imported_projects, + 'The number of imported Bitbucket projects' + ) + + expect(Gitlab::Metrics).to receive(:histogram).with( + :bitbucket_importer_total_duration_seconds, + 'Total time spent importing Bitbucket projects, in seconds' + ) + + expect(counter).to receive(:increment) + expect(histogram).to receive(:observe).with({ importer: described_class::IMPORTER }, anything) + + subject.execute + end + end end end diff --git a/spec/lib/gitlab/import/metrics_spec.rb b/spec/lib/gitlab/import/metrics_spec.rb new file mode 100644 index 00000000000..0799d19fcef --- /dev/null +++ b/spec/lib/gitlab/import/metrics_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Import::Metrics do + let(:importer_stub) do + Class.new do + prepend Gitlab::Import::Metrics + + Gitlab::Import::Metrics.measure :execute, metrics: { + importer_counter: { + type: :counter, + description: 'description' + }, + importer_histogram: { + type: :histogram, + labels: { importer: 'importer' }, + description: 'description' + } + } + + def execute + true + end + end + end + + subject { importer_stub.new.execute } + + describe '#execute' do + let(:counter) { double(:counter) } + let(:histogram) { double(:histogram) } + + it 'increments counter metric' do + expect(Gitlab::Metrics) + .to receive(:counter) + .with(:importer_counter, 'description') + .and_return(counter) + + expect(counter).to receive(:increment) + + subject + end + + it 'measures method duration and reports histogram metric' do + expect(Gitlab::Metrics) + .to receive(:histogram) + .with(:importer_histogram, 'description') + .and_return(histogram) + + expect(histogram).to receive(:observe).with({ importer: 'importer' }, anything) + + subject + end + end +end diff --git a/spec/services/projects/import_service_spec.rb b/spec/services/projects/import_service_spec.rb index 1e9ac40128a..af8118f9b11 100644 --- a/spec/services/projects/import_service_spec.rb +++ b/spec/services/projects/import_service_spec.rb @@ -123,8 +123,13 @@ describe Projects::ImportService do it 'succeeds if repository import is successful' do expect(project.repository).to receive(:import_repository).and_return(true) - expect_any_instance_of(Gitlab::BitbucketImport::Importer).to receive(:execute).and_return(true) - expect_any_instance_of(Projects::LfsPointers::LfsImportService).to receive(:execute).and_return(status: :success) + expect_next_instance_of(Gitlab::BitbucketImport::Importer) do |importer| + expect(importer).to receive(:execute).and_return(true) + end + + expect_next_instance_of(Projects::LfsPointers::LfsImportService) do |service| + expect(service).to receive(:execute).and_return(status: :success) + end result = subject.execute @@ -147,8 +152,15 @@ describe Projects::ImportService do error_message = 'error message' expect(project.repository).to receive(:import_repository).and_return(true) - expect_any_instance_of(Gitlab::BitbucketImport::Importer).to receive(:execute).and_return(true) - expect_any_instance_of(Projects::LfsPointers::LfsImportService).to receive(:execute).and_return(status: :error, message: error_message) + + expect_next_instance_of(Gitlab::BitbucketImport::Importer) do |importer| + expect(importer).to receive(:execute).and_return(true) + end + + expect_next_instance_of(Projects::LfsPointers::LfsImportService) do |service| + expect(service).to receive(:execute).and_return(status: :error, message: error_message) + end + expect(Gitlab::AppLogger).to receive(:error).with("The Lfs import process failed. #{error_message}") subject.execute -- cgit v1.2.3