diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb | 10 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb | 10 | ||||
| -rw-r--r-- | lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml | 77 | ||||
| -rw-r--r-- | lib/gitlab/ci/templates/Beta/Auto-DevOps.gitlab-ci.yml | 163 | ||||
| -rw-r--r-- | lib/gitlab/database/batch_count.rb | 33 | ||||
| -rw-r--r-- | lib/gitlab/usage_data.rb | 4 |
6 files changed, 106 insertions, 191 deletions
diff --git a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb b/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb index 8d19a73dfc3..4947e2eb879 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb @@ -23,15 +23,7 @@ module Gitlab private def template_name - if beta_enabled? - 'Beta/Auto-DevOps' - else - 'Auto-DevOps' - end - end - - def beta_enabled? - Feature.enabled?(:auto_devops_beta, project, default_enabled: true) + 'Auto-DevOps' end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb b/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb index c72b5f18424..5e4bb84360c 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb @@ -23,15 +23,7 @@ module Gitlab private def template_name - if beta_enabled? - 'Beta/Auto-DevOps' - else - 'Auto-DevOps' - end - end - - def beta_enabled? - Feature.enabled?(:auto_devops_beta, project, default_enabled: true) + 'Auto-DevOps' end end end diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 5a7642d24ee..a9f29bda9b9 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -72,6 +72,83 @@ stages: - performance - cleanup +workflow: + rules: + - if: '$BUILDPACK_URL || $AUTO_DEVOPS_EXPLICITLY_ENABLED == "1"' + + - exists: + - Dockerfile + + # https://github.com/heroku/heroku-buildpack-clojure + - exists: + - project.clj + + # https://github.com/heroku/heroku-buildpack-go + - exists: + - go.mod + - Gopkg.mod + - Godeps/Godeps.json + - vendor/vendor.json + - glide.yaml + - src/**/*.go + + # https://github.com/heroku/heroku-buildpack-gradle + - exists: + - gradlew + - build.gradle + - settings.gradle + + # https://github.com/heroku/heroku-buildpack-java + - exists: + - pom.xml + - pom.atom + - pom.clj + - pom.groovy + - pom.rb + - pom.scala + - pom.yaml + - pom.yml + + # https://github.com/heroku/heroku-buildpack-multi + - exists: + - .buildpacks + + # https://github.com/heroku/heroku-buildpack-nodejs + - exists: + - package.json + + # https://github.com/heroku/heroku-buildpack-php + - exists: + - composer.json + - index.php + + # https://github.com/heroku/heroku-buildpack-play + # TODO: detect script excludes some scala files + - exists: + - '**/conf/application.conf' + + # https://github.com/heroku/heroku-buildpack-python + # TODO: detect script checks that all of these exist, not any + - exists: + - requirements.txt + - setup.py + - Pipfile + + # https://github.com/heroku/heroku-buildpack-ruby + - exists: + - Gemfile + + # https://github.com/heroku/heroku-buildpack-scala + - exists: + - '*.sbt' + - project/*.scala + - .sbt/*.scala + - project/build.properties + + # https://github.com/dokku/buildpack-nginx + - exists: + - .static + include: - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Beta/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Beta/Auto-DevOps.gitlab-ci.yml deleted file mode 100644 index 2c5035705ac..00000000000 --- a/lib/gitlab/ci/templates/Beta/Auto-DevOps.gitlab-ci.yml +++ /dev/null @@ -1,163 +0,0 @@ -# Auto DevOps - BETA do not use -# This CI/CD configuration provides a standard pipeline for -# * building a Docker image (using a buildpack if necessary), -# * storing the image in the container registry, -# * running tests from a buildpack, -# * running code quality analysis, -# * creating a review app for each topic branch, -# * and continuous deployment to production -# -# Test jobs may be disabled by setting environment variables: -# * test: TEST_DISABLED -# * code_quality: CODE_QUALITY_DISABLED -# * license_management: LICENSE_MANAGEMENT_DISABLED -# * performance: PERFORMANCE_DISABLED -# * sast: SAST_DISABLED -# * dependency_scanning: DEPENDENCY_SCANNING_DISABLED -# * container_scanning: CONTAINER_SCANNING_DISABLED -# * dast: DAST_DISABLED -# * review: REVIEW_DISABLED -# * stop_review: REVIEW_DISABLED -# -# In order to deploy, you must have a Kubernetes cluster configured either -# via a project integration, or via group/project variables. -# KUBE_INGRESS_BASE_DOMAIN must also be set on the cluster settings, -# as a variable at the group or project level, or manually added below. -# -# Continuous deployment to production is enabled by default. -# If you want to deploy to staging first, set STAGING_ENABLED environment variable. -# If you want to enable incremental rollout, either manual or time based, -# set INCREMENTAL_ROLLOUT_MODE environment variable to "manual" or "timed". -# If you want to use canary deployments, set CANARY_ENABLED environment variable. -# -# If Auto DevOps fails to detect the proper buildpack, or if you want to -# specify a custom buildpack, set a project variable `BUILDPACK_URL` to the -# repository URL of the buildpack. -# e.g. BUILDPACK_URL=https://github.com/heroku/heroku-buildpack-ruby.git#v142 -# If you need multiple buildpacks, add a file to your project called -# `.buildpacks` that contains the URLs, one on each line, in order. -# Note: Auto CI does not work with multiple buildpacks yet - -image: alpine:latest - -variables: - # KUBE_INGRESS_BASE_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. - # KUBE_INGRESS_BASE_DOMAIN: domain.example.com - - POSTGRES_USER: user - POSTGRES_PASSWORD: testing-password - POSTGRES_ENABLED: "true" - POSTGRES_DB: $CI_ENVIRONMENT_SLUG - POSTGRES_VERSION: 9.6.2 - - DOCKER_DRIVER: overlay2 - - ROLLOUT_RESOURCE_TYPE: deployment - - DOCKER_TLS_CERTDIR: "" # https://gitlab.com/gitlab-org/gitlab-runner/issues/4501 - -stages: - - build - - test - - deploy # dummy stage to follow the template guidelines - - review - - dast - - staging - - canary - - production - - incremental rollout 10% - - incremental rollout 25% - - incremental rollout 50% - - incremental rollout 100% - - performance - - cleanup - -workflow: - rules: - - if: '$BUILDPACK_URL || $AUTO_DEVOPS_EXPLICITLY_ENABLED == "1"' - - - exists: - - Dockerfile - - # https://github.com/heroku/heroku-buildpack-clojure - - exists: - - project.clj - - # https://github.com/heroku/heroku-buildpack-go - - exists: - - go.mod - - Gopkg.mod - - Godeps/Godeps.json - - vendor/vendor.json - - glide.yaml - - src/**/*.go - - # https://github.com/heroku/heroku-buildpack-gradle - - exists: - - gradlew - - build.gradle - - settings.gradle - - # https://github.com/heroku/heroku-buildpack-java - - exists: - - pom.xml - - pom.atom - - pom.clj - - pom.groovy - - pom.rb - - pom.scala - - pom.yaml - - pom.yml - - # https://github.com/heroku/heroku-buildpack-multi - - exists: - - .buildpacks - - # https://github.com/heroku/heroku-buildpack-nodejs - - exists: - - package.json - - # https://github.com/heroku/heroku-buildpack-php - - exists: - - composer.json - - index.php - - # https://github.com/heroku/heroku-buildpack-play - # TODO: detect script excludes some scala files - - exists: - - '**/conf/application.conf' - - # https://github.com/heroku/heroku-buildpack-python - # TODO: detect script checks that all of these exist, not any - - exists: - - requirements.txt - - setup.py - - Pipfile - - # https://github.com/heroku/heroku-buildpack-ruby - - exists: - - Gemfile - - # https://github.com/heroku/heroku-buildpack-scala - - exists: - - '*.sbt' - - project/*.scala - - .sbt/*.scala - - project/build.properties - - # https://github.com/dokku/buildpack-nginx - - exists: - - .static - -include: - - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml - - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml - - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml - - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml - - template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml - - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml - - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml - - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/License-Management.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml - - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 8972b77abef..e0f6b0f9eee 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -4,12 +4,18 @@ # Implements a distinct and ordinary batch counter # Needs indexes on the column below to calculate max, min and range queries # For larger tables just set use higher batch_size with index optimization +# +# In order to not use a possible complex time consuming query when calculating min and max for batch_distinct_count +# the start and finish can be sent specifically +# # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 +# # Examples: # extend ::Gitlab::Database::BatchCount # batch_count(User.active) # batch_count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) # batch_distinct_count(::Project, :creator_id) +# batch_distinct_count(::Project.with_active_services.service_desk_enabled.where(time_period), start: ::User.minimum(:id), finish: ::User.maximum(:id)) module Gitlab module Database module BatchCount @@ -17,8 +23,8 @@ module Gitlab BatchCounter.new(relation, column: column).count(batch_size: batch_size) end - def batch_distinct_count(relation, column = nil, batch_size: nil) - BatchCounter.new(relation, column: column).count(mode: :distinct, batch_size: batch_size) + def batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) + BatchCounter.new(relation, column: column).count(mode: :distinct, batch_size: batch_size, start: start, finish: finish) end class << self @@ -31,9 +37,10 @@ module Gitlab MIN_REQUIRED_BATCH_SIZE = 1_250 MAX_ALLOWED_LOOPS = 10_000 SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep - # Each query should take <<500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 - DEFAULT_DISTINCT_BATCH_SIZE = 10_000 - DEFAULT_BATCH_SIZE = 100_000 + + # Each query should take < 500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 + DEFAULT_DISTINCT_BATCH_SIZE = 100_000 + DEFAULT_BATCH_SIZE = 10_000 def initialize(relation, column: nil) @relation = relation @@ -46,15 +53,15 @@ module Gitlab start > finish end - def count(batch_size: nil, mode: :itself) + def count(batch_size: nil, mode: :itself, start: nil, finish: nil) raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? raise "The mode #{mode.inspect} is not supported" unless [:itself, :distinct].include?(mode) # non-distinct have better performance batch_size ||= mode == :distinct ? DEFAULT_DISTINCT_BATCH_SIZE : DEFAULT_BATCH_SIZE - start = @relation.minimum(@column) || 0 - finish = @relation.maximum(@column) || 0 + start = actual_start(start) + finish = actual_finish(finish) raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 return FALLBACK if unwanted_configuration?(finish, batch_size, start) @@ -84,6 +91,16 @@ module Gitlab # rubocop:disable GitlabSecurity/PublicSend @relation.select(@column).public_send(mode).where(@column => start..(finish - 1)).count end + + private + + def actual_start(start) + start || @relation.minimum(@column) || 0 + end + + def actual_finish(finish) + finish || @relation.maximum(@column) || 0 + end end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 8c0da1ba999..c131566380e 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -240,9 +240,9 @@ module Gitlab fallback end - def distinct_count(relation, column = nil, fallback: -1, batch: true) + def distinct_count(relation, column = nil, fallback: -1, batch: true, start: nil, finish: nil) if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true) - Gitlab::Database::BatchCount.batch_distinct_count(relation, column) + Gitlab::Database::BatchCount.batch_distinct_count(relation, column, start: start, finish: finish) else relation.distinct_count_by(column) end |