diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-11-20 01:11:55 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-11-20 01:11:55 +0300 |
| commit | 5a8431feceba47fd8e1804d9aa1b1730606b71d5 (patch) | |
| tree | e5df8e0ceee60f4af8093f5c4c2f934b8abced05 /lib | |
| parent | 4d477238500c347c6553d335d920bedfc5a46869 (diff) | |
Add latest changes from gitlab-org/gitlab@12-5-stable-ee
Diffstat (limited to 'lib')
237 files changed, 4050 insertions, 1446 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index d71f0c38ce6..a2bdb76b834 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -21,6 +21,7 @@ module API Gitlab::GrapeLogging::Loggers::ClientEnvLogger.new, Gitlab::GrapeLogging::Loggers::RouteLogger.new, Gitlab::GrapeLogging::Loggers::UserLogger.new, + Gitlab::GrapeLogging::Loggers::ExceptionLogger.new, Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new, Gitlab::GrapeLogging::Loggers::PerfLogger.new, Gitlab::GrapeLogging::Loggers::CorrelationIdLogger.new @@ -112,6 +113,7 @@ module API mount ::API::Files mount ::API::GroupBoards mount ::API::GroupClusters + mount ::API::GroupExport mount ::API::GroupLabels mount ::API::GroupMilestones mount ::API::Groups diff --git a/lib/api/branches.rb b/lib/api/branches.rb index f8f79ab6f5a..054242dca4c 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -32,7 +32,7 @@ module API use :filter_params end get ':id/repository/branches' do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42329') + user_project.preload_protected_branches repository = user_project.repository diff --git a/lib/api/commits.rb b/lib/api/commits.rb index ffff40141de..63a7fdfa3ab 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -169,7 +169,7 @@ module API not_found! 'Commit' unless commit - raw_diffs = ::Kaminari.paginate_array(commit.raw_diffs.to_a) + raw_diffs = ::Kaminari.paginate_array(commit.diffs(expanded: true).diffs.to_a) present paginate(raw_diffs), with: Entities::Diff end @@ -223,7 +223,7 @@ module API present user_project.repository.commit(result[:result]), with: Entities::Commit else - render_api_error!(result[:message], 400) + error!(result.slice(:message, :error_code), 400, header) end end @@ -257,7 +257,7 @@ module API present user_project.repository.commit(result[:result]), with: Entities::Commit else - render_api_error!(result[:message], 400) + error!(result.slice(:message, :error_code), 400, header) end end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index da882547071..f97200f20b9 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -17,7 +17,7 @@ module API end params do use :pagination - optional :order_by, type: String, values: %w[id iid created_at ref], default: 'id', desc: 'Return deployments ordered by `id` or `iid` or `created_at` or `ref`' + optional :order_by, type: String, values: %w[id iid created_at updated_at ref], default: 'id', desc: 'Return deployments ordered by `id` or `iid` or `created_at` or `updated_at` or `ref`' optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 91811efacd7..9617f1a8acf 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -307,6 +307,7 @@ module API expose :only_allow_merge_if_pipeline_succeeds expose :request_access_enabled expose :only_allow_merge_if_all_discussions_are_resolved + expose :remove_source_branch_after_merge expose :printing_merge_request_link_enabled expose :merge_method expose :statistics, using: 'API::Entities::ProjectStatistics', if: -> (project, options) { @@ -488,11 +489,11 @@ module API end expose :developers_can_push do |repo_branch, options| - options[:project].protected_branches.developers_can?(:push, repo_branch.name) + ::ProtectedBranch.developers_can?(:push, repo_branch.name, protected_refs: options[:project].protected_branches) end expose :developers_can_merge do |repo_branch, options| - options[:project].protected_branches.developers_can?(:merge, repo_branch.name) + ::ProtectedBranch.developers_can?(:merge, repo_branch.name, protected_refs: options[:project].protected_branches) end expose :can_push do |repo_branch, options| @@ -754,6 +755,7 @@ module API end expose :diff_head_sha, as: :sha expose :merge_commit_sha + expose :squash_commit_sha expose :discussion_locked expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch @@ -776,6 +778,10 @@ module API expose :squash expose :task_completion_status + + expose :cannot_be_merged?, as: :has_conflicts + + expose :mergeable_discussions_state?, as: :blocking_discussions_resolved end class MergeRequest < MergeRequestBasic @@ -1248,6 +1254,7 @@ module API # let's not expose the secret key in a response attributes.delete(:asset_proxy_secret_key) + attributes.delete(:eks_secret_access_key) attributes end @@ -1290,7 +1297,11 @@ module API end class Release < Grape::Entity - expose :name + include ::API::Helpers::Presentable + + expose :name do |release, _| + can_download_code? ? release.name : "Release-#{release.id}" + end expose :tag, as: :tag_name, if: ->(_, _) { can_download_code? } expose :description expose :description_html do |entity| @@ -1302,8 +1313,8 @@ module API expose :commit, using: Entities::Commit, if: ->(_, _) { can_download_code? } expose :upcoming_release?, as: :upcoming_release expose :milestones, using: Entities::Milestone, if: -> (release, _) { release.milestones.present? } - expose :commit_path, if: ->(_, _) { can_download_code? } - expose :tag_path, if: ->(_, _) { can_download_code? } + expose :commit_path, expose_nil: false + expose :tag_path, expose_nil: false expose :assets do expose :assets_count, as: :count do |release, _| assets_to_exclude = can_download_code? ? [] : [:sources] @@ -1315,8 +1326,9 @@ module API end end expose :_links do - expose :merge_requests_url, if: -> (_) { release_mr_issue_urls_available? } - expose :issues_url, if: -> (_) { release_mr_issue_urls_available? } + expose :merge_requests_url, expose_nil: false + expose :issues_url, expose_nil: false + expose :edit_url, expose_nil: false end private @@ -1324,36 +1336,6 @@ module API def can_download_code? Ability.allowed?(options[:current_user], :download_code, object.project) end - - def commit_path - return unless object.commit - - Gitlab::Routing.url_helpers.project_commit_path(project, object.commit.id) - end - - def tag_path - Gitlab::Routing.url_helpers.project_tag_path(project, object.tag) - end - - def merge_requests_url - Gitlab::Routing.url_helpers.project_merge_requests_url(project, params_for_issues_and_mrs) - end - - def issues_url - Gitlab::Routing.url_helpers.project_issues_url(project, params_for_issues_and_mrs) - end - - def params_for_issues_and_mrs - { scope: 'all', state: 'opened', release_tag: object.tag } - end - - def release_mr_issue_urls_available? - ::Feature.enabled?(:release_mr_issue_urls, project) - end - - def project - @project ||= object.project - end end class Tag < Grape::Entity @@ -1699,6 +1681,7 @@ module API expose :verified?, as: :verified expose :verification_code, as: :verification_code expose :enabled_until + expose :auto_ssl_enabled expose :certificate, as: :certificate_expiration, @@ -1714,6 +1697,7 @@ module API expose :verified?, as: :verified expose :verification_code, as: :verification_code expose :enabled_until + expose :auto_ssl_enabled expose :certificate, if: ->(pages_domain, _) { pages_domain.certificate? }, @@ -1737,7 +1721,12 @@ module API class Blob < Grape::Entity expose :basename expose :data - expose :filename + expose :path + # TODO: :filename was renamed to :path but both still return the full path, + # in the future we can only return the filename here without the leading + # directory path. + # https://gitlab.com/gitlab-org/gitlab/issues/34521 + expose :filename, &:path expose :id expose :ref expose :startline @@ -1813,6 +1802,7 @@ module API expose :user, using: Entities::UserBasic expose :platform_kubernetes, using: Entities::Platform::Kubernetes expose :provider_gcp, using: Entities::Provider::Gcp + expose :management_project, using: Entities::ProjectIdentity end class ClusterProject < Cluster diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb index a70ac63cc6e..abfe10b7fa1 100644 --- a/lib/api/group_clusters.rb +++ b/lib/api/group_clusters.rb @@ -84,6 +84,7 @@ module API requires :cluster_id, type: Integer, desc: 'The cluster ID' optional :name, type: String, desc: 'Cluster name' optional :domain, type: String, desc: 'Cluster base domain' + optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do optional :api_url, type: String, desc: 'URL to access the Kubernetes API' optional :token, type: String, desc: 'Token to authenticate against Kubernetes' diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb index fd24662cc9a..7f95b411b36 100644 --- a/lib/api/group_container_repositories.rb +++ b/lib/api/group_container_repositories.rb @@ -23,9 +23,11 @@ module API end get ':id/registry/repositories' do repositories = ContainerRepositoriesFinder.new( - id: user_group.id, container_type: :group + user: current_user, subject: user_group ).execute + track_event('list_repositories') + present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags] end end diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb new file mode 100644 index 00000000000..8025a16e191 --- /dev/null +++ b/lib/api/group_export.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module API + class GroupExport < Grape::API + before do + authorize! :admin_group, user_group + end + + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: { id: %r{[^/]+} } do + desc 'Download export' do + detail 'This feature was introduced in GitLab 12.5.' + end + get ':id/export/download' do + if user_group.export_file_exists? + present_carrierwave_file!(user_group.export_file) + else + render_api_error!('404 Not found or has expired', 404) + end + end + + desc 'Start export' do + detail 'This feature was introduced in GitLab 12.5.' + end + post ':id/export' do + GroupExportWorker.perform_async(current_user.id, user_group.id, params) + + accepted! + end + end + end +end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 19c29847ce3..49b86489a8b 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -9,6 +9,7 @@ module API GITLAB_SHARED_SECRET_HEADER = "Gitlab-Shared-Secret" SUDO_PARAM = :sudo API_USER_ENV = 'gitlab.api.user' + API_EXCEPTION_ENV = 'gitlab.api.exception' def declared_params(options = {}) options = { include_parent_namespaces: false }.merge(options) @@ -387,6 +388,9 @@ module API Gitlab::Sentry.track_acceptable_exception(exception, extra: params) end + # This is used with GrapeLogging::Loggers::ExceptionLogger + env[API_EXCEPTION_ENV] = exception + # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 trace = exception.backtrace @@ -451,6 +455,17 @@ module API end end + def track_event(action = action_name, **args) + category = args.delete(:category) || self.options[:for].name + raise "invalid category" unless category + + ::Gitlab::Tracking.event(category, action.to_s, **args) + rescue => error + Rails.logger.warn( # rubocop:disable Gitlab/RailsLogger + "Tracking event failed for action: #{action}, category: #{category}, message: #{error.message}" + ) + end + protected def project_finder_params_ce @@ -464,6 +479,8 @@ module API finder_params[:user] = params.delete(:user) if params[:user] finder_params[:custom_attributes] = params[:custom_attributes] if params[:custom_attributes] finder_params[:min_access_level] = params[:min_access_level] if params[:min_access_level] + finder_params[:id_after] = params[:id_after] if params[:id_after] + finder_params[:id_before] = params[:id_before] if params[:id_before] finder_params end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 4c575381d30..dfac777e4a1 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -140,7 +140,8 @@ module API { repository: repository.gitaly_repository, address: Gitlab::GitalyClient.address(project.repository_storage), - token: Gitlab::GitalyClient.token(project.repository_storage) + token: Gitlab::GitalyClient.token(project.repository_storage), + features: Feature::Gitaly.server_feature_flags } end end diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 71bbc218f94..9c5b355e823 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -4,254 +4,7 @@ module API module Helpers module Pagination def paginate(relation) - strategy = if params[:pagination] == 'keyset' && Feature.enabled?('api_keyset_pagination') - KeysetPaginationStrategy - else - DefaultPaginationStrategy - end - - strategy.new(self).paginate(relation) - end - - class Base - private - - def per_page - @per_page ||= params[:per_page] - end - - def base_request_uri - @base_request_uri ||= URI.parse(request.url).tap do |uri| - uri.host = Gitlab.config.gitlab.host - uri.port = Gitlab.config.gitlab.port - end - end - - def build_page_url(query_params:) - base_request_uri.tap do |uri| - uri.query = query_params - end.to_s - end - - def page_href(next_page_params = {}) - query_params = params.merge(**next_page_params, per_page: per_page).to_query - - build_page_url(query_params: query_params) - end - end - - class KeysetPaginationInfo - attr_reader :relation, :request_context - - def initialize(relation, request_context) - # This is because it's rather complex to support multiple values with possibly different sort directions - # (and we don't need this in the API) - if relation.order_values.size > 1 - raise "Pagination only supports ordering by a single column." \ - "The following columns were given: #{relation.order_values.map { |v| v.expr.name }}" - end - - @relation = relation - @request_context = request_context - end - - def fields - keys.zip(values).reject { |_, v| v.nil? }.to_h - end - - def column_for_order_by(relation) - relation.order_values.first&.expr&.name - end - - # Sort direction (`:asc` or `:desc`) - def sort - @sort ||= if order_by_primary_key? - # Default order is by id DESC - :desc - else - # API defaults to DESC order if param `sort` not present - request_context.params[:sort]&.to_sym || :desc - end - end - - # Do we only sort by primary key? - def order_by_primary_key? - keys.size == 1 && keys.first == primary_key - end - - def primary_key - relation.model.primary_key.to_sym - end - - def sort_ascending? - sort == :asc - end - - # Build hash of request parameters for a given record (relevant to pagination) - def params_for(record) - return {} unless record - - keys.each_with_object({}) do |key, h| - h["ks_prev_#{key}".to_sym] = record.attributes[key.to_s] - end - end - - private - - # All values present in request parameters that correspond to #keys. - def values - @values ||= keys.map do |key| - request_context.params["ks_prev_#{key}".to_sym] - end - end - - # All keys relevant to pagination. - # This always includes the primary key. Optionally, the `order_by` key is prepended. - def keys - @keys ||= [column_for_order_by(relation), primary_key].compact.uniq - end - end - - class KeysetPaginationStrategy < Base - attr_reader :request_context - delegate :params, :header, :request, to: :request_context - - def initialize(request_context) - @request_context = request_context - end - - # rubocop: disable CodeReuse/ActiveRecord - def paginate(relation) - pagination = KeysetPaginationInfo.new(relation, request_context) - - paged_relation = relation.limit(per_page) - - if conds = conditions(pagination) - paged_relation = paged_relation.where(*conds) - end - - # In all cases: sort by primary key (possibly in addition to another sort column) - paged_relation = paged_relation.order(pagination.primary_key => pagination.sort) - - add_default_pagination_headers - - if last_record = paged_relation.last - next_page_params = pagination.params_for(last_record) - add_navigation_links(next_page_params) - end - - paged_relation - end - # rubocop: enable CodeReuse/ActiveRecord - - private - - def conditions(pagination) - fields = pagination.fields - - return if fields.empty? - - placeholder = fields.map { '?' } - - comp = if pagination.sort_ascending? - '>' - else - '<' - end - - [ - # Row value comparison: - # (A, B) < (a, b) <=> (A < a) OR (A = a AND B < b) - # <=> A <= a AND ((A < a) OR (A = a AND B < b)) - "(#{fields.keys.join(',')}) #{comp} (#{placeholder.join(',')})", - *fields.values - ] - end - - def add_default_pagination_headers - header 'X-Per-Page', per_page.to_s - end - - def add_navigation_links(next_page_params) - header 'X-Next-Page', page_href(next_page_params) - header 'Link', link_for('next', next_page_params) - end - - def link_for(rel, next_page_params) - %(<#{page_href(next_page_params)}>; rel="#{rel}") - end - end - - class DefaultPaginationStrategy < Base - attr_reader :request_context - delegate :params, :header, :request, to: :request_context - - def initialize(request_context) - @request_context = request_context - end - - def paginate(relation) - paginate_with_limit_optimization(add_default_order(relation)).tap do |data| - add_pagination_headers(data) - end - end - - private - - def paginate_with_limit_optimization(relation) - pagination_data = relation.page(params[:page]).per(params[:per_page]) - return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) - return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit) - - limited_total_count = pagination_data.total_count_with_limit - if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT - # The call to `total_count_with_limit` memoizes `@arel` because of a call to `references_eager_loaded_tables?` - # We need to call `reset` because `without_count` relies on `@arel` being unmemoized - pagination_data.reset.without_count - else - pagination_data - end - end - - def add_default_order(relation) - if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? - relation = relation.order(:id) # rubocop: disable CodeReuse/ActiveRecord - end - - relation - end - - def add_pagination_headers(paginated_data) - header 'X-Per-Page', paginated_data.limit_value.to_s - header 'X-Page', paginated_data.current_page.to_s - header 'X-Next-Page', paginated_data.next_page.to_s - header 'X-Prev-Page', paginated_data.prev_page.to_s - header 'Link', pagination_links(paginated_data) - - return if data_without_counts?(paginated_data) - - header 'X-Total', paginated_data.total_count.to_s - header 'X-Total-Pages', total_pages(paginated_data).to_s - end - - def pagination_links(paginated_data) - [].tap do |links| - links << %(<#{page_href(page: paginated_data.prev_page)}>; rel="prev") if paginated_data.prev_page - links << %(<#{page_href(page: paginated_data.next_page)}>; rel="next") if paginated_data.next_page - links << %(<#{page_href(page: 1)}>; rel="first") - - links << %(<#{page_href(page: total_pages(paginated_data))}>; rel="last") unless data_without_counts?(paginated_data) - end.join(', ') - end - - def total_pages(paginated_data) - # Ensure there is in total at least 1 page - [paginated_data.total_pages, 1].max - end - - def data_without_counts?(paginated_data) - paginated_data.is_a?(Kaminari::PaginatableWithoutCount) - end + ::Gitlab::Pagination::OffsetPagination.new(self).paginate(relation) end end end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 94619204274..47b1f037eb8 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -30,6 +30,7 @@ module API optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' + optional :remove_source_branch_after_merge, type: Boolean, desc: 'Remove the source branch by default after merge' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' @@ -94,6 +95,7 @@ module API :path, :printing_merge_request_link_enabled, :public_builds, + :remove_source_branch_after_merge, :repository_access_level, :request_access_enabled, :resolve_outdated_diff_discussions, @@ -109,7 +111,6 @@ module API :jobs_enabled, :merge_requests_enabled, :wiki_enabled, - :jobs_enabled, :snippets_enabled ] end diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index d9a22484c1f..c70f2f3e2c8 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -77,7 +77,7 @@ module API response_with_status(**payload) when ::Gitlab::GitAccessResult::CustomAction - response_with_status(code: 300, message: check_result.message, payload: check_result.payload) + response_with_status(code: 300, payload: check_result.payload, gl_console_messages: check_result.console_messages) else response_with_status(code: 500, success: false, message: UNKNOWN_CHECK_RESULT_ERROR) end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 1436238c5cf..6e10414def4 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -296,9 +296,12 @@ module API end get ':id/merge_requests/:merge_request_iid/commits' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - commits = ::Kaminari.paginate_array(merge_request.commits) - present paginate(commits), with: Entities::Commit + commits = + paginate(merge_request.merge_request_diff.merge_request_diff_commits) + .map { |commit| Commit.from_hash(commit.to_hash, merge_request.project) } + + present commits, with: Entities::Commit end desc 'Show the merge request changes' do @@ -404,7 +407,8 @@ module API merge_params = HashWithIndifferentAccess.new( commit_message: params[:merge_commit_message], squash_commit_message: params[:squash_commit_message], - should_remove_source_branch: params[:should_remove_source_branch] + should_remove_source_branch: params[:should_remove_source_branch], + sha: params[:sha] || merge_request.diff_head_sha ) if merge_when_pipeline_succeeds && merge_request.head_pipeline && merge_request.head_pipeline.active? @@ -455,6 +459,8 @@ module API status :accepted present rebase_in_progress: merge_request.rebase_in_progress? + rescue ::MergeRequest::RebaseLockTimeout => e + render_api_error!(e.message, 409) end desc 'List issues that will be closed on merge' do diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index ec2fe8270b7..2d02a4e624c 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -92,8 +92,10 @@ module API requires :domain, type: String, desc: 'The domain' # rubocop:disable Scalability/FileUploads # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 - optional :certificate, allow_blank: false, types: [File, String], desc: 'The certificate', as: :user_provided_certificate - optional :key, allow_blank: false, types: [File, String], desc: 'The key', as: :user_provided_key + optional :certificate, types: [File, String], desc: 'The certificate', as: :user_provided_certificate + optional :key, types: [File, String], desc: 'The key', as: :user_provided_key + optional :auto_ssl_enabled, allow_blank: false, type: Boolean, default: false, + desc: "Enables automatic generation of SSL certificates issued by Let's Encrypt for custom domains." # rubocop:enable Scalability/FileUploads all_or_none_of :user_provided_certificate, :user_provided_key end @@ -116,14 +118,16 @@ module API requires :domain, type: String, desc: 'The domain' # rubocop:disable Scalability/FileUploads # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 - optional :certificate, allow_blank: false, types: [File, String], desc: 'The certificate', as: :user_provided_certificate - optional :key, allow_blank: false, types: [File, String], desc: 'The key', as: :user_provided_key + optional :certificate, types: [File, String], desc: 'The certificate', as: :user_provided_certificate + optional :key, types: [File, String], desc: 'The key', as: :user_provided_key + optional :auto_ssl_enabled, allow_blank: true, type: Boolean, + desc: "Enables automatic generation of SSL certificates issued by Let's Encrypt for custom domains." # rubocop:enable Scalability/FileUploads end put ":id/pages/domains/:domain", requirements: PAGES_DOMAINS_ENDPOINT_REQUIREMENTS do authorize! :update_pages, user_project - pages_domain_params = declared(params, include_parent_namespaces: false) + pages_domain_params = declared(params, include_parent_namespaces: false, include_missing: false) # Remove empty private key if certificate is not empty. if pages_domain_params[:user_provided_certificate] && !pages_domain_params[:user_provided_key] diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index 45c800d7d1e..8e35914f48a 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -88,6 +88,7 @@ module API requires :cluster_id, type: Integer, desc: 'The cluster ID' optional :name, type: String, desc: 'Cluster name' optional :domain, type: String, desc: 'Cluster base domain' + optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do optional :api_url, type: String, desc: 'URL to access the Kubernetes API' optional :token, type: String, desc: 'Token to authenticate against Kubernetes' diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index 2a05974509a..2b33069e324 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -24,9 +24,11 @@ module API end get ':id/registry/repositories' do repositories = ContainerRepositoriesFinder.new( - id: user_project.id, container_type: :project + user: current_user, subject: user_project ).execute + track_event( 'list_repositories') + present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags] end @@ -40,6 +42,7 @@ module API authorize_admin_container_image! DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) + track_event('delete_repository') status :accepted end @@ -56,6 +59,8 @@ module API authorize_read_container_image! tags = Kaminari.paginate_array(repository.tags) + track_event('list_tags') + present paginate(tags), with: Entities::ContainerRegistry::Tag end @@ -77,6 +82,8 @@ module API CleanupContainerRepositoryWorker.perform_async(current_user.id, repository.id, declared_params.except(:repository_id)) + track_event('delete_tag_bulk') + status :accepted end @@ -111,6 +118,8 @@ module API .execute(repository) if result[:status] == :success + track_event('delete_tag') + status :ok else status :bad_request diff --git a/lib/api/projects.rb b/lib/api/projects.rb index d2dacafe7f9..669def2b63c 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -61,6 +61,8 @@ module API optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' optional :with_programming_language, type: String, desc: 'Limit to repositories which use the given programming language' optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Limit by minimum access level of authenticated user' + optional :id_after, type: Integer, desc: 'Limit results to projects with IDs greater than the specified ID' + optional :id_before, type: Integer, desc: 'Limit results to projects with IDs less than the specified ID' use :optional_filter_params_ee end @@ -69,7 +71,8 @@ module API optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' optional :import_url, type: String, desc: 'URL from which the project is imported' optional :template_name, type: String, desc: "Name of template from which to create project" - mutually_exclusive :import_url, :template_name + optional :template_project_id, type: Integer, desc: "Project ID of template from which to create project" + mutually_exclusive :import_url, :template_name, :template_project_id end def load_projects diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 4238529142c..3f600ef4a04 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -45,7 +45,7 @@ module API end params do requires :tag_name, type: String, desc: 'The name of the tag', as: :tag - requires :name, type: String, desc: 'The name of the release' + optional :name, type: String, desc: 'The name of the release' requires :description, type: String, desc: 'The release notes' optional :ref, type: String, desc: 'The commit sha or branch name' optional :assets, type: Hash do diff --git a/lib/api/settings.rb b/lib/api/settings.rb index c90ba0c9b5d..5362b3060c1 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -42,6 +42,7 @@ module API optional :asset_proxy_whitelist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically whitelisted.' optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" + optional :default_ci_config_path, type: String, desc: 'The instance default CI configuration path for new projects' optional :default_project_creation, type: Integer, values: ::Gitlab::Access.project_creation_values, desc: 'Determine if developers can create projects in the group' optional :default_branch_protection, type: Integer, values: ::Gitlab::Access.protection_values, desc: 'Determine if developers can push to master' optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility' @@ -52,6 +53,12 @@ module API optional :domain_blacklist_enabled, type: Boolean, desc: 'Enable domain blacklist for sign ups' optional :domain_blacklist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' optional :domain_whitelist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' + optional :eks_integration_enabled, type: Boolean, desc: 'Enable integration with Amazon EKS' + given eks_integration_enabled: -> (val) { val } do + requires :eks_account_id, type: String, desc: 'Amazon account ID for EKS integration' + requires :eks_access_key_id, type: String, desc: 'Access key ID for the EKS integration IAM user' + requires :eks_secret_access_key, type: String, desc: 'Secret access key for the EKS integration IAM user' + end optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.' optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.' @@ -129,16 +136,22 @@ module API optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application' optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled' + optional :sourcegraph_enabled, type: Boolean, desc: 'Enable Sourcegraph' + optional :sourcegraph_public_only, type: Boolean, desc: 'Only allow public projects to communicate with Sourcegraph' + given sourcegraph_enabled: ->(val) { val } do + requires :sourcegraph_url, type: String, desc: 'The configured Sourcegraph instance URL' + end optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins' optional :local_markdown_version, type: Integer, desc: 'Local markdown version, increase this value when any cached markdown should be invalidated' optional :allow_local_requests_from_hooks_and_services, type: Boolean, desc: 'Deprecated: Use :allow_local_requests_from_web_hooks_and_services instead. Allow requests to the local network from hooks and services.' # support legacy names, can be removed in v5 optional :snowplow_enabled, type: Grape::API::Boolean, desc: 'Enable Snowplow tracking' + optional :snowplow_iglu_registry_url, type: String, desc: 'The Snowplow base Iglu Schema Registry URL to use for custom context and self describing events' given snowplow_enabled: ->(val) { val } do requires :snowplow_collector_hostname, type: String, desc: 'The Snowplow collector hostname' optional :snowplow_cookie_domain, type: String, desc: 'The Snowplow cookie domain' - optional :snowplow_site_id, type: String, desc: 'The Snowplow site name / application ic' + optional :snowplow_app_id, type: String, desc: 'The Snowplow site name / application id' end ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb index daa9598a204..693c20cb73a 100644 --- a/lib/api/sidekiq_metrics.rb +++ b/lib/api/sidekiq_metrics.rb @@ -36,7 +36,8 @@ module API { processed: stats.processed, failed: stats.failed, - enqueued: stats.enqueued + enqueued: stats.enqueued, + dead: stats.dead_size } end end diff --git a/lib/banzai/filter/inline_grafana_metrics_filter.rb b/lib/banzai/filter/inline_grafana_metrics_filter.rb new file mode 100644 index 00000000000..321580b532f --- /dev/null +++ b/lib/banzai/filter/inline_grafana_metrics_filter.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # HTML filter that inserts a placeholder element for each + # reference to a grafana dashboard. + class InlineGrafanaMetricsFilter < Banzai::Filter::InlineEmbedsFilter + # Placeholder element for the frontend to use as an + # injection point for charts. + def create_element(params) + begin_loading_dashboard(params[:url]) + + doc.document.create_element( + 'div', + class: 'js-render-metrics', + 'data-dashboard-url': metrics_dashboard_url(params) + ) + end + + def embed_params(node) + query_params = Gitlab::Metrics::Dashboard::Url.parse_query(node['href']) + return unless [:panelId, :from, :to].all? do |param| + query_params.include?(param) + end + + { url: node['href'], start: query_params[:from], end: query_params[:to] } + end + + # Selects any links with an href contains the configured + # grafana domain for the project + def xpath_search + return unless grafana_url.present? + + %(descendant-or-self::a[starts-with(@href, '#{grafana_url}')]) + end + + private + + def project + context[:project] + end + + def grafana_url + project&.grafana_integration&.grafana_url + end + + def metrics_dashboard_url(params) + Gitlab::Routing.url_helpers.project_grafana_api_metrics_dashboard_url( + project, + embedded: true, + grafana_url: params[:url], + start: format_time(params[:start]), + end: format_time(params[:end]) + ) + end + + # Formats a timestamp from Grafana for compatibility with + # parsing in JS via `new Date(timestamp)` + # + # @param time [String] Represents miliseconds since epoch + def format_time(time) + Time.at(time.to_i / 1000).utc.strftime('%FT%TZ') + end + + # Fetches a dashboard and caches the result for the + # FE to fetch quickly while rendering charts + def begin_loading_dashboard(url) + ::Gitlab::Metrics::Dashboard::Finder.find( + project, + embedded: true, + grafana_url: url + ) + end + end + end +end diff --git a/lib/banzai/filter/inline_metrics_redactor_filter.rb b/lib/banzai/filter/inline_metrics_redactor_filter.rb index 4d8a5028898..e84ba83e03e 100644 --- a/lib/banzai/filter/inline_metrics_redactor_filter.rb +++ b/lib/banzai/filter/inline_metrics_redactor_filter.rb @@ -8,14 +8,17 @@ module Banzai include Gitlab::Utils::StrongMemoize METRICS_CSS_CLASS = '.js-render-metrics' + URL = Gitlab::Metrics::Dashboard::Url + + Embed = Struct.new(:project_path, :permission) # Finds all embeds based on the css class the FE # uses to identify the embedded content, removing # only unnecessary nodes. def call nodes.each do |node| - path = paths_by_node[node] - user_has_access = user_access_by_path[path] + embed = embeds_by_node[node] + user_has_access = user_access_by_embed[embed] node.remove unless user_has_access end @@ -30,40 +33,69 @@ module Banzai end # Returns all nodes which the FE will identify as - # a metrics dashboard placeholder element + # a metrics embed placeholder element # # @return [Nokogiri::XML::NodeSet] def nodes @nodes ||= doc.css(METRICS_CSS_CLASS) end - # Maps a node to the full path of a project. + # Maps a node to key properties of an embed. # Memoized so we only need to run the regex to get # the project full path from the url once per node. # - # @return [Hash<Nokogiri::XML::Node, String>] - def paths_by_node - strong_memoize(:paths_by_node) do - nodes.each_with_object({}) do |node, paths| - paths[node] = path_for_node(node) + # @return [Hash<Nokogiri::XML::Node, Embed>] + def embeds_by_node + strong_memoize(:embeds_by_node) do + nodes.each_with_object({}) do |node, embeds| + embed = Embed.new + url = node.attribute('data-dashboard-url').to_s + + set_path_and_permission(embed, url, URL.regex, :read_environment) + set_path_and_permission(embed, url, URL.grafana_regex, :read_project) unless embed.permission + + embeds[node] = embed if embed.permission end end end - # Gets a project's full_path from the dashboard url - # in the placeholder node. The FE will use the attr - # `data-dashboard-url`, so we want to check against that - # attribute directly in case a user has manually - # created a metrics element (rather than supporting - # an alternate attr in InlineMetricsFilter). + # Attempts to determine the path and permission attributes + # of a url based on expected dashboard url formats and + # sets the attributes on an Embed object # - # @return [String] - def path_for_node(node) - url = node.attribute('data-dashboard-url').to_s - - Gitlab::Metrics::Dashboard::Url.regex.match(url) do |m| + # @param embed [Embed] + # @param url [String] + # @param regex [RegExp] + # @param permission [Symbol] + def set_path_and_permission(embed, url, regex, permission) + return unless path = regex.match(url) do |m| "#{$~[:namespace]}/#{$~[:project]}" end + + embed.project_path = path + embed.permission = permission + end + + # Returns a mapping representing whether the current user + # has permission to view the embed for the project. + # Determined in a batch + # + # @return [Hash<Embed, Boolean>] + def user_access_by_embed + strong_memoize(:user_access_by_embed) do + unique_embeds.each_with_object({}) do |embed, access| + project = projects_by_path[embed.project_path] + + access[embed] = Ability.allowed?(user, embed.permission, project) + end + end + end + + # Returns a unique list of embeds + # + # @return [Array<Embed>] + def unique_embeds + embeds_by_node.values.uniq end # Maps a project's full path to a Project object. @@ -74,22 +106,17 @@ module Banzai def projects_by_path strong_memoize(:projects_by_path) do Project.eager_load(:route, namespace: [:route]) - .where_full_path_in(paths_by_node.values.uniq) + .where_full_path_in(unique_project_paths) .index_by(&:full_path) end end - # Returns a mapping representing whether the current user - # has permission to view the metrics for the project. - # Determined in a batch + # Returns a list of the full_paths of every project which + # has an embed in the doc # - # @return [Hash<Project, Boolean>] - def user_access_by_path - strong_memoize(:user_access_by_path) do - projects_by_path.each_with_object({}) do |(path, project), access| - access[path] = Ability.allowed?(user, :read_environment, project) - end - end + # @return [Array<String>] + def unique_project_paths + embeds_by_node.values.map(&:project_path).uniq end end end diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb index ed82fbc1f94..98987ee2019 100644 --- a/lib/banzai/filter/video_link_filter.rb +++ b/lib/banzai/filter/video_link_filter.rb @@ -15,7 +15,7 @@ module Banzai end def extra_element_attrs - { width: "100%" } + { width: "400" } end end end diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb index 82b99d3de4a..90edc7010f4 100644 --- a/lib/banzai/pipeline/ascii_doc_pipeline.rb +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -10,6 +10,9 @@ module Banzai Filter::SyntaxHighlightFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, + Filter::ColorFilter, + Filter::ImageLazyLoadFilter, + Filter::ImageLinkFilter, Filter::AsciiDocPostProcessingFilter ] end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 08e27257fdf..f6c12cdb53b 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -30,6 +30,7 @@ module Banzai Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, Filter::InlineMetricsFilter, + Filter::InlineGrafanaMetricsFilter, Filter::TableOfContentsFilter, Filter::AutolinkFilter, Filter::ExternalLinkFilter, diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb index a498c9bc213..8d9de2dbc7d 100644 --- a/lib/bitbucket/representation/pull_request.rb +++ b/lib/bitbucket/representation/pull_request.rb @@ -16,9 +16,10 @@ module Bitbucket end def state - if raw['state'] == 'MERGED' + case raw['state'] + when 'MERGED' 'merged' - elsif raw['state'] == 'DECLINED' + when 'DECLINED', 'SUPERSEDED' 'closed' else 'opened' diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 92861c567a8..bc0347f6ea1 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -51,7 +51,7 @@ module ContainerRegistry def upload_blob(name, content, digest) upload = faraday.post("/v2/#{name}/blobs/uploads/") - return unless upload.success? + return upload unless upload.success? location = URI(upload.headers['location']) diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index d99a209dc87..9e9df88373a 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -74,7 +74,14 @@ module DeclarativePolicy next unless klass.name begin - policy_class = "#{klass.name}Policy".constantize + klass_name = + if subject_class.respond_to?(:declarative_policy_class) + subject_class.declarative_policy_class + else + "#{klass.name}Policy" + end + + policy_class = klass_name.constantize # NOTE: the < operator here tests whether policy_class # inherits from Base. We can't use #is_a? because that diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index 81f8ba5c8c3..0ac2d017e1a 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -7,7 +7,6 @@ class Feature # Server feature flags should use '_' to separate words. SERVER_FEATURE_FLAGS = %w[ - cache_invalidator inforef_uploadpack_cache get_all_lfs_pointers_go ].freeze @@ -20,7 +19,7 @@ class Feature default_on = DEFAULT_ON_FLAGS.include?(feature_flag) Feature.enabled?("gitaly_#{feature_flag}", default_enabled: default_on) - rescue ActiveRecord::NoDatabaseError + rescue ActiveRecord::NoDatabaseError, PG::ConnectionBad false end diff --git a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb index 15cdd25e711..568104cb30b 100644 --- a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb +++ b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb @@ -5,7 +5,7 @@ require 'rails/generators' module Rails class PostDeploymentMigrationGenerator < Rails::Generators::NamedBase def create_migration_file - timestamp = Time.now.strftime('%Y%m%d%H%M%S') + timestamp = Time.now.utc.strftime('%Y%m%d%H%M%S') template "migration.rb", "db/post_migrate/#{timestamp}_#{file_name}.rb" end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index ad8e693ccbc..0e6db54eb46 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -47,6 +47,18 @@ module Gitlab Gitlab.config.gitlab.url == COM_URL || gl_subdomain? end + def self.canary? + Gitlab::Utils.to_boolean(ENV['CANARY']) + end + + def self.com_and_canary? + com? && canary? + end + + def self.com_but_not_canary? + com? && !canary? + end + def self.org? Gitlab.config.gitlab.url == 'https://dev.gitlab.org' end diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb index 33cbe1a62ef..9ea20a4d6a4 100644 --- a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb +++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb @@ -68,3 +68,5 @@ module Gitlab end end end + +Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder') diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb index 0c0f737f2c9..05b16672912 100644 --- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb @@ -12,6 +12,8 @@ module Gitlab class DataCollector include Gitlab::Utils::StrongMemoize + delegate :serialized_records, to: :records_fetcher + def initialize(stage:, params: {}) @stage = stage @params = params diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index 90d03142b2a..2662aa38d6b 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -130,3 +130,5 @@ module Gitlab end end end + +Gitlab::Analytics::CycleAnalytics::RecordsFetcher.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::RecordsFetcher') diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb index 58572446de6..f6e22044142 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -47,27 +47,29 @@ module Gitlab ] }.freeze - def [](identifier) + def self.[](identifier) events.find { |e| e.identifier.to_s.eql?(identifier.to_s) } || raise(KeyError) end # hash for defining ActiveRecord enum: identifier => number - def to_enum - ENUM_MAPPING.each_with_object({}) { |(k, v), hash| hash[k.identifier] = v } + def self.to_enum + enum_mapping.each_with_object({}) { |(k, v), hash| hash[k.identifier] = v } end - # will be overridden in EE with custom events - def pairing_rules + def self.pairing_rules PAIRING_RULES end - # will be overridden in EE with custom events - def events + def self.events EVENTS end - module_function :[], :to_enum, :pairing_rules, :events + def self.enum_mapping + ENUM_MAPPING + end end end end end + +Gitlab::Analytics::CycleAnalytics::StageEvents.prepend_if_ee('::EE::Gitlab::Analytics::CycleAnalytics::StageEvents') diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb index 6af1b90bccc..9f0ca80ba50 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics module StageEvents - class CodeStageStart < SimpleStageEvent + class CodeStageStart < StageEvent def self.name s_("CycleAnalyticsEvent|Issue first mentioned in a commit") end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb index 8c9a80740a9..a159580b7bd 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics module StageEvents - class IssueCreated < SimpleStageEvent + class IssueCreated < StageEvent def self.name s_("CycleAnalyticsEvent|Issue created") end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb index fe7f2d85f8b..a3b7fa16daf 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics module StageEvents - class IssueFirstMentionedInCommit < SimpleStageEvent + class IssueFirstMentionedInCommit < MetricsBasedStageEvent def self.name s_("CycleAnalyticsEvent|Issue first mentioned in a commit") end @@ -20,12 +20,6 @@ module Gitlab def timestamp_projection issue_metrics_table[:first_mentioned_in_commit_at] end - - # rubocop: disable CodeReuse/ActiveRecord - def apply_query_customization(query) - query.joins(:metrics) - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb index 77e4092b9ab..0ea98e82ecc 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics module StageEvents - class IssueStageEnd < SimpleStageEvent + class IssueStageEnd < MetricsBasedStageEvent def self.name PlanStageStart.name end @@ -26,7 +26,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def apply_query_customization(query) - query.joins(:metrics).where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + super.where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb index 7059c425b8f..013e068e479 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics module StageEvents - class MergeRequestCreated < SimpleStageEvent + class MergeRequestCreated < StageEvent def self.name s_("CycleAnalyticsEvent|Merge request created") end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb index 3d7482eaaf0..654d0befbc3 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics module StageEvents - class MergeRequestFirstDeployedToProduction < SimpleStageEvent + class MergeRequestFirstDeployedToProduction < MetricsBasedStageEvent def self.name s_("CycleAnalyticsEvent|Merge request first deployed to production") end @@ -23,7 +23,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def apply_query_customization(query) - query.joins(:metrics).where(timestamp_projection.gteq(mr_table[:created_at])) + super.where(timestamp_projection.gteq(mr_table[:created_at])) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb index 36bb4d6fc8d..a0b1c12756f 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics module StageEvents - class MergeRequestLastBuildFinished < SimpleStageEvent + class MergeRequestLastBuildFinished < MetricsBasedStageEvent def self.name s_("CycleAnalyticsEvent|Merge request last build finish time") end @@ -20,12 +20,6 @@ module Gitlab def timestamp_projection mr_metrics_table[:latest_build_finished_at] end - - # rubocop: disable CodeReuse/ActiveRecord - def apply_query_customization(query) - query.joins(:metrics) - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb index 468d9899cc7..da3b5cdfaa4 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics module StageEvents - class MergeRequestLastBuildStarted < SimpleStageEvent + class MergeRequestLastBuildStarted < MetricsBasedStageEvent def self.name s_("CycleAnalyticsEvent|Merge request last build start time") end @@ -20,12 +20,6 @@ module Gitlab def timestamp_projection mr_metrics_table[:latest_build_started_at] end - - # rubocop: disable CodeReuse/ActiveRecord - def apply_query_customization(query) - query.joins(:metrics) - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb index 82ecaf1cd6b..e67a6f7eea6 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics module StageEvents - class MergeRequestMerged < SimpleStageEvent + class MergeRequestMerged < MetricsBasedStageEvent def self.name s_("CycleAnalyticsEvent|Merge request merged") end @@ -20,12 +20,6 @@ module Gitlab def timestamp_projection mr_metrics_table[:merged_at] end - - # rubocop: disable CodeReuse/ActiveRecord - def apply_query_customization(query) - query.joins(:metrics) - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb new file mode 100644 index 00000000000..4ca8745abe4 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class MetricsBasedStageEvent < StageEvent + # rubocop: disable CodeReuse/ActiveRecord + def apply_query_customization(query) + query.joins(:metrics) + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb index 7ece7d62faa..37168a1fb0f 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics module StageEvents - class PlanStageStart < SimpleStageEvent + class PlanStageStart < MetricsBasedStageEvent def self.name s_("CycleAnalyticsEvent|Issue first associated with a milestone or issue first added to a board") end @@ -26,8 +26,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def apply_query_customization(query) - query - .joins(:metrics) + super .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil)) end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb index 607371a32e8..b249f6874e7 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics module StageEvents - class ProductionStageEnd < SimpleStageEvent + class ProductionStageEnd < StageEvent def self.name PlanStageStart.name end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/simple_stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/simple_stage_event.rb deleted file mode 100644 index 253c489d822..00000000000 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/simple_stage_event.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Analytics - module CycleAnalytics - module StageEvents - # Represents a simple event that usually refers to one database column and does not require additional user input - class SimpleStageEvent < StageEvent - end - end - end - end -end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb index aa392140eb5..667d6def414 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -35,6 +35,10 @@ module Gitlab query end + def label_based? + false + end + private attr_reader :params diff --git a/lib/gitlab/auth/ip_rate_limiter.rb b/lib/gitlab/auth/ip_rate_limiter.rb index 74d359bcd28..acb46abb6f3 100644 --- a/lib/gitlab/auth/ip_rate_limiter.rb +++ b/lib/gitlab/auth/ip_rate_limiter.rb @@ -21,11 +21,12 @@ module Gitlab end def register_fail! + return false if trusted_ip? + # Allow2Ban.filter will return false if this IP has not failed too often yet @banned = Rack::Attack::Allow2Ban.filter(ip, config) do - # If we return false here, the failure for this IP is ignored by Allow2Ban - # If we return true here, the count for the IP is incremented. - ip_can_be_banned? + # We return true to increment the count for this IP + true end end @@ -33,20 +34,16 @@ module Gitlab @banned end + def trusted_ip? + trusted_ips.any? { |netmask| netmask.include?(ip) } + end + private def config Gitlab.config.rack_attack.git_basic_auth end - def ip_can_be_banned? - !trusted_ip? - end - - def trusted_ip? - trusted_ips.any? { |netmask| netmask.include?(ip) } - end - def trusted_ips strong_memoize(:trusted_ips) do config.ip_whitelist.map do |proxy| diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index eb1d0925c55..4bc0ceedae7 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -21,6 +21,14 @@ module Gitlab Gitlab.config.ldap.enabled end + def self.sign_in_enabled? + enabled? && !prevent_ldap_sign_in? + end + + def self.prevent_ldap_sign_in? + Gitlab.config.ldap.prevent_ldap_sign_in + end + def self.servers Gitlab.config.ldap['servers']&.values || [] end diff --git a/lib/gitlab/background_migration/legacy_upload_mover.rb b/lib/gitlab/background_migration/legacy_upload_mover.rb index c9e47f210be..1879a6c5427 100644 --- a/lib/gitlab/background_migration/legacy_upload_mover.rb +++ b/lib/gitlab/background_migration/legacy_upload_mover.rb @@ -18,6 +18,7 @@ module Gitlab def execute return unless upload + return unless upload.model_type == 'Note' if !project # if we don't have models associated with the upload we can not move it diff --git a/lib/gitlab/background_migration/legacy_uploads_migrator.rb b/lib/gitlab/background_migration/legacy_uploads_migrator.rb index a9d38a27e0c..f7cadb9b00d 100644 --- a/lib/gitlab/background_migration/legacy_uploads_migrator.rb +++ b/lib/gitlab/background_migration/legacy_uploads_migrator.rb @@ -14,7 +14,7 @@ module Gitlab include Database::MigrationHelpers def perform(start_id, end_id) - Upload.where(id: start_id..end_id, uploader: 'AttachmentUploader').find_each do |upload| + Upload.where(id: start_id..end_id, uploader: 'AttachmentUploader', model_type: 'Note').find_each do |upload| LegacyUploadMover.new(upload).execute end end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index f5fb33f1660..23e8be4a9ab 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -176,7 +176,7 @@ module Gitlab self.table_name = 'projects' def self.find_by_full_path(path) - order_sql = "(CASE WHEN routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" + order_sql = Arel.sql("(CASE WHEN routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)") where_full_path_in(path).reorder(order_sql).take end diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index 8d25b66af9c..cbda3808b86 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -22,11 +22,11 @@ module Gitlab start_offset = @state.offset - @state.set_current_line!(style: Style.new(@state.inherited_style)) + @state.new_line!( + style: Style.new(@state.inherited_style)) stream.each_line do |line| - s = StringScanner.new(line) - convert_line(s) + consume_line(line) end # This must be assigned before flushing the current line @@ -52,26 +52,41 @@ module Gitlab private - def convert_line(scanner) - until scanner.eos? - - if scanner.scan(Gitlab::Regex.build_trace_section_regex) - handle_section(scanner) - elsif scanner.scan(/\e([@-_])(.*?)([@-~])/) - handle_sequence(scanner) - elsif scanner.scan(/\e(([@-_])(.*?)?)?$/) - break - elsif scanner.scan(/</) - @state.current_line << '<' - elsif scanner.scan(/\r?\n/) - # we advance the offset of the next current line - # so it does not start from \n - flush_current_line(advance_offset: scanner.matched_size) - else - @state.current_line << scanner.scan(/./m) - end - - @state.offset += scanner.matched_size + def consume_line(line) + scanner = StringScanner.new(line) + + consume_token(scanner) until scanner.eos? + end + + def consume_token(scanner) + if scan_token(scanner, Gitlab::Regex.build_trace_section_regex, consume: false) + handle_section(scanner) + elsif scan_token(scanner, /\e([@-_])(.*?)([@-~])/) + handle_sequence(scanner) + elsif scan_token(scanner, /\e(([@-_])(.*?)?)?$/) + # stop scanning + scanner.terminate + elsif scan_token(scanner, /\r?\n/) + flush_current_line + elsif scan_token(scanner, /\r/) + # drop last line + @state.current_line.clear! + elsif scan_token(scanner, /.[^\e\r\ns]*/m) + # this is a join from all previous tokens and first letters + # it always matches at least one character `.` + # it matches everything that is not start of: + # `\e`, `<`, `\r`, `\n`, `s` (for section_start) + @state.current_line << scanner[0] + else + raise 'invalid parser state' + end + end + + def scan_token(scanner, match, consume: true) + scanner.scan(match).tap do |result| + # we need to move offset as soon + # as we match the token + @state.offset += scanner.matched_size if consume && result end end @@ -96,32 +111,50 @@ module Gitlab section_name = sanitize_section_name(section) if action == "start" - handle_section_start(section_name, timestamp) + handle_section_start(scanner, section_name, timestamp) elsif action == "end" - handle_section_end(section_name, timestamp) + handle_section_end(scanner, section_name, timestamp) + else + raise 'unsupported action' end end - def handle_section_start(section, timestamp) - flush_current_line unless @state.current_line.empty? + def handle_section_start(scanner, section, timestamp) + # We make a new line for new section + flush_current_line + @state.open_section(section, timestamp) + + # we need to consume match after handling + # the open of section, as we want the section + # marker to be refresh on incremental update + @state.offset += scanner.matched_size end - def handle_section_end(section, timestamp) + def handle_section_end(scanner, section, timestamp) return unless @state.section_open?(section) - flush_current_line unless @state.current_line.empty? + # We flush the content to make the end + # of section to be a new line + flush_current_line + @state.close_section(section, timestamp) - # ensure that section end is detached from the last - # line in the section + # we need to consume match before handling + # as we want the section close marker + # not to be refreshed on incremental update + @state.offset += scanner.matched_size + + # this flushes an empty line with `section_duration` flush_current_line end - def flush_current_line(advance_offset: 0) - @lines << @state.current_line.to_h + def flush_current_line + unless @state.current_line.empty? + @lines << @state.current_line.to_h + end - @state.set_current_line!(advance_offset: advance_offset) + @state.new_line! end def sanitize_section_name(section) diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index 173fb1df88e..21aa1f84353 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -47,12 +47,17 @@ module Gitlab @current_segment.text << data end + def clear! + @segments.clear + @current_segment = Segment.new(style: style) + end + def style @current_segment.style end def empty? - @segments.empty? && @current_segment.empty? + @segments.empty? && @current_segment.empty? && @section_duration.nil? end def update_style(ansi_commands) diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index db7a9035b8b..7e1a8102a35 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -46,9 +46,9 @@ module Gitlab @open_sections.key?(section) end - def set_current_line!(style: nil, advance_offset: 0) + def new_line!(style: nil) new_line = Line.new( - offset: @offset + advance_offset, + offset: @offset, style: style || @current_line.style, sections: @open_sections.keys ) diff --git a/lib/gitlab/ci/ansi2json/style.rb b/lib/gitlab/ci/ansi2json/style.rb index 2739ffdfa5d..77f61178b37 100644 --- a/lib/gitlab/ci/ansi2json/style.rb +++ b/lib/gitlab/ci/ansi2json/style.rb @@ -15,14 +15,10 @@ module Gitlab end def update(ansi_commands) - command = ansi_commands.shift - return unless command - - if changes = Gitlab::Ci::Ansi2json::Parser.new(command, ansi_commands).changes - apply_changes(changes) - end + # treat e\[m as \e[0m + ansi_commands = ['0'] if ansi_commands.empty? - update(ansi_commands) + evaluate_stack_command(ansi_commands) end def set? @@ -50,6 +46,17 @@ module Gitlab private + def evaluate_stack_command(ansi_commands) + command = ansi_commands.shift + return unless command + + if changes = Gitlab::Ci::Ansi2json::Parser.new(command, ansi_commands).changes + apply_changes(changes) + end + + evaluate_stack_command(ansi_commands) + end + def apply_changes(changes) case when changes[:reset] diff --git a/lib/gitlab/ci/build/context/base.rb b/lib/gitlab/ci/build/context/base.rb new file mode 100644 index 00000000000..02b97ea76e9 --- /dev/null +++ b/lib/gitlab/ci/build/context/base.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Context + class Base + attr_reader :pipeline + + def initialize(pipeline) + @pipeline = pipeline + end + + def variables + raise NotImplementedError + end + + protected + + def pipeline_attributes + { + pipeline: pipeline, + project: pipeline.project, + user: pipeline.user, + ref: pipeline.ref, + tag: pipeline.tag, + trigger_request: pipeline.legacy_trigger, + protected: pipeline.protected_ref? + } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb new file mode 100644 index 00000000000..dfd86d3ad72 --- /dev/null +++ b/lib/gitlab/ci/build/context/build.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Context + class Build < Base + include Gitlab::Utils::StrongMemoize + + attr_reader :attributes + + def initialize(pipeline, attributes = {}) + super(pipeline) + + @attributes = attributes + end + + def variables + strong_memoize(:variables) do + # This is a temporary piece of technical debt to allow us access + # to the CI variables to evaluate rules before we persist a Build + # with the result. We should refactor away the extra Build.new, + # but be able to get CI Variables directly from the Seed::Build. + stub_build.scoped_variables_hash + end + end + + private + + def stub_build + ::Ci::Build.new(build_attributes) + end + + def build_attributes + attributes.merge(pipeline_attributes) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/context/global.rb b/lib/gitlab/ci/build/context/global.rb new file mode 100644 index 00000000000..fdd3ac358d5 --- /dev/null +++ b/lib/gitlab/ci/build/context/global.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Context + class Global < Base + include Gitlab::Utils::StrongMemoize + + def initialize(pipeline, yaml_variables:) + super(pipeline) + + @yaml_variables = yaml_variables.to_a + end + + def variables + strong_memoize(:variables) do + # This is a temporary piece of technical debt to allow us access + # to the CI variables to evaluate workflow:rules + # with the result. We should refactor away the extra Build.new, + # but be able to get CI Variables directly from the Seed::Build. + stub_build.scoped_variables_hash + .reject { |key, _value| key =~ /\ACI_(JOB|BUILD)/ } + end + end + + private + + def stub_build + ::Ci::Build.new(build_attributes) + end + + def build_attributes + pipeline_attributes.merge( + yaml_variables: @yaml_variables) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/changes.rb b/lib/gitlab/ci/build/policy/changes.rb index 9c705a1cd3e..9ae4198bbf7 100644 --- a/lib/gitlab/ci/build/policy/changes.rb +++ b/lib/gitlab/ci/build/policy/changes.rb @@ -9,7 +9,7 @@ module Gitlab @globs = Array(globs) end - def satisfied_by?(pipeline, seed) + def satisfied_by?(pipeline, context) return true if pipeline.modified_paths.nil? pipeline.modified_paths.any? do |path| diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb index 4c7dc947cd0..4e8693724e5 100644 --- a/lib/gitlab/ci/build/policy/kubernetes.rb +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -11,7 +11,7 @@ module Gitlab end end - def satisfied_by?(pipeline, seed = nil) + def satisfied_by?(pipeline, context = nil) pipeline.has_kubernetes_active? end end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index c3005303fd8..afe0ccb361e 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -9,7 +9,7 @@ module Gitlab @patterns = Array(refs) end - def satisfied_by?(pipeline, seed = nil) + def satisfied_by?(pipeline, context = nil) @patterns.any? do |pattern| pattern, path = pattern.split('@', 2) diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb index ceb5210cfb5..1394340ce1f 100644 --- a/lib/gitlab/ci/build/policy/specification.rb +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -17,7 +17,7 @@ module Gitlab @spec = spec end - def satisfied_by?(pipeline, seed = nil) + def satisfied_by?(pipeline, context = nil) raise NotImplementedError end end diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb index e9c8864123f..7b1ce6330f0 100644 --- a/lib/gitlab/ci/build/policy/variables.rb +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -9,8 +9,8 @@ module Gitlab @expressions = Array(expressions) end - def satisfied_by?(pipeline, seed) - variables = seed.scoped_variables_hash + def satisfied_by?(pipeline, context) + variables = context.variables statements = @expressions.map do |statement| ::Gitlab::Ci::Pipeline::Expression::Statement diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index 43399c74457..c705b6f86c7 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -13,17 +13,21 @@ module Gitlab options: { start_in: start_in }.compact }.compact end + + def pass? + self.when != 'never' + end end - def initialize(rule_hashes, default_when = 'on_success') + def initialize(rule_hashes, default_when:) @rule_list = Rule.fabricate_list(rule_hashes) @default_when = default_when end - def evaluate(pipeline, build) + def evaluate(pipeline, context) if @rule_list.nil? Result.new(@default_when) - elsif matched_rule = match_rule(pipeline, build) + elsif matched_rule = match_rule(pipeline, context) Result.new( matched_rule.attributes[:when] || @default_when, matched_rule.attributes[:start_in] @@ -35,8 +39,8 @@ module Gitlab private - def match_rule(pipeline, build) - @rule_list.find { |rule| rule.matches?(pipeline, build) } + def match_rule(pipeline, context) + @rule_list.find { |rule| rule.matches?(pipeline, context) } end end end diff --git a/lib/gitlab/ci/build/rules/rule.rb b/lib/gitlab/ci/build/rules/rule.rb index 8d52158c8d2..077e4d150fb 100644 --- a/lib/gitlab/ci/build/rules/rule.rb +++ b/lib/gitlab/ci/build/rules/rule.rb @@ -23,8 +23,8 @@ module Gitlab end end - def matches?(pipeline, build) - @clauses.all? { |clause| clause.satisfied_by?(pipeline, build) } + def matches?(pipeline, context) + @clauses.all? { |clause| clause.satisfied_by?(pipeline, context) } end end end diff --git a/lib/gitlab/ci/build/rules/rule/clause.rb b/lib/gitlab/ci/build/rules/rule/clause.rb index bf787fe95a6..6d4bbbb8c21 100644 --- a/lib/gitlab/ci/build/rules/rule/clause.rb +++ b/lib/gitlab/ci/build/rules/rule/clause.rb @@ -20,7 +20,7 @@ module Gitlab @spec = spec end - def satisfied_by?(pipeline, seed = nil) + def satisfied_by?(pipeline, context = nil) raise NotImplementedError end end diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index 81d2ee6c24c..728a66ca87f 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -8,7 +8,7 @@ module Gitlab @globs = Array(globs) end - def satisfied_by?(pipeline, seed) + def satisfied_by?(pipeline, context) return true if pipeline.modified_paths.nil? pipeline.modified_paths.any? do |path| diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb index 62f8371283f..85e77438f51 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -15,7 +15,7 @@ module Gitlab @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?)) end - def satisfied_by?(pipeline, seed) + def satisfied_by?(pipeline, context) paths = worktree_paths(pipeline) exact_matches?(paths) || pattern_matches?(paths) diff --git a/lib/gitlab/ci/build/rules/rule/clause/if.rb b/lib/gitlab/ci/build/rules/rule/clause/if.rb index 18c3b450f95..6143a736ca6 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/if.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/if.rb @@ -8,10 +8,9 @@ module Gitlab @expression = expression end - def satisfied_by?(pipeline, seed) - variables = seed.scoped_variables_hash - - ::Gitlab::Ci::Pipeline::Expression::Statement.new(@expression, variables).truthful? + def satisfied_by?(pipeline, context) + ::Gitlab::Ci::Pipeline::Expression::Statement.new( + @expression, context.variables).truthful? end end end diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 41613369ca2..9d8d7675234 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -12,7 +12,9 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[name untracked paths reports when expire_in].freeze + ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as].freeze + EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze + EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces" attributes ALLOWED_KEYS @@ -21,11 +23,18 @@ module Gitlab validations do validates :config, type: Hash validates :config, allowed_keys: ALLOWED_KEYS + validates :paths, presence: true, if: :expose_as_present? with_options allow_nil: true do validates :name, type: String validates :untracked, boolean: true validates :paths, array_of_strings: true + validates :paths, array_of_strings: { + with: /\A[^*]*\z/, + message: "can't contain '*' when used with 'expose_as'" + }, if: :expose_as_present? + validates :expose_as, type: String, length: { maximum: 100 }, if: :expose_as_present? + validates :expose_as, format: { with: EXPOSE_AS_REGEX, message: EXPOSE_AS_ERROR_MESSAGE }, if: :expose_as_present? validates :reports, type: Hash validates :when, inclusion: { in: %w[on_success on_failure always], @@ -41,6 +50,12 @@ module Gitlab @config[:reports] = reports_value if @config.key?(:reports) @config end + + def expose_as_present? + return false unless Feature.enabled?(:ci_expose_arbitrary_artifacts_in_mr, default_enabled: true) + + !@config[:expose_as].nil? + end end end end diff --git a/lib/gitlab/ci/config/entry/boolean.rb b/lib/gitlab/ci/config/entry/boolean.rb new file mode 100644 index 00000000000..10619ef9f8d --- /dev/null +++ b/lib/gitlab/ci/config/entry/boolean.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents the interrutible value. + # + class Boolean < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, boolean: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/commands.rb b/lib/gitlab/ci/config/entry/commands.rb index 02e368c1813..7a86fca3056 100644 --- a/lib/gitlab/ci/config/entry/commands.rb +++ b/lib/gitlab/ci/config/entry/commands.rb @@ -11,11 +11,11 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable validations do - validates :config, array_of_strings_or_string: true + validates :config, string_or_nested_array_of_strings: true end def value - Array(@config) + Array(@config).flatten(1) end end end diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb index 6200d7c7f87..83127bde6e4 100644 --- a/lib/gitlab/ci/config/entry/default.rb +++ b/lib/gitlab/ci/config/entry/default.rb @@ -11,11 +11,10 @@ module Gitlab # class Default < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable - - DuplicateError = Class.new(Gitlab::Config::Loader::FormatError) + include ::Gitlab::Config::Entry::Inheritable ALLOWED_KEYS = %i[before_script image services - after_script cache].freeze + after_script cache interruptible].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -41,31 +40,22 @@ module Gitlab description: 'Configure caching between build jobs.', inherit: true - helpers :before_script, :image, :services, :after_script, :cache - - def compose!(deps = nil) - super(self) + entry :interruptible, Entry::Boolean, + description: 'Set jobs interruptible default value.', + inherit: false - inherit!(deps) - end + helpers :before_script, :image, :services, :after_script, :cache, :interruptible private - def inherit!(deps) - return unless deps - - self.class.nodes.each do |key, factory| - next unless factory.inheritable? + def overwrite_entry(deps, key, current_entry) + inherited_entry = deps[key] - root_entry = deps[key] - next unless root_entry.specified? - - if self[key].specified? - raise DuplicateError, "#{key} is defined in top-level and `default:` entry" - end - - @entries[key] = root_entry + if inherited_entry.specified? && current_entry.specified? + raise InheritError, "#{key} is defined in top-level and `default:` entry" end + + inherited_entry unless current_entry.specified? end end end diff --git a/lib/gitlab/ci/config/entry/files.rb b/lib/gitlab/ci/config/entry/files.rb new file mode 100644 index 00000000000..d0d6a36d754 --- /dev/null +++ b/lib/gitlab/ci/config/entry/files.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents an array of file paths. + # + class Files < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, array_of_strings: true + validates :config, length: { + minimum: 1, + maximum: 2, + too_short: 'requires at least %{count} item', + too_long: 'has too many items (maximum is %{count})' + } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 07d5be86b1e..c75ae87a985 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -10,6 +10,7 @@ module Gitlab class Job < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Inheritable ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze ALLOWED_KEYS = %i[tags script only except rules type image services @@ -37,7 +38,6 @@ module Gitlab with_options allow_nil: true do validates :tags, array_of_strings: true validates :allow_failure, boolean: true - validates :interruptible, boolean: true validates :parallel, numericality: { only_integer: true, greater_than_or_equal_to: 2, less_than_or_equal_to: 50 } @@ -49,7 +49,6 @@ module Gitlab validates :timeout, duration: { limit: ChronicDuration.output(Project::MAX_BUILD_TIMEOUT) } validates :dependencies, array_of_strings: true - validates :needs, array_of_strings: true validates :extends, array_of_strings_or_string: true validates :rules, array_of_hashes: true end @@ -73,13 +72,16 @@ module Gitlab inherit: true entry :script, Entry::Commands, - description: 'Commands that will be executed in this job.' + description: 'Commands that will be executed in this job.', + inherit: false entry :stage, Entry::Stage, - description: 'Pipeline stage this job will be executed into.' + description: 'Pipeline stage this job will be executed into.', + inherit: false entry :type, Entry::Stage, - description: 'Deprecated: stage this job will be executed into.' + description: 'Deprecated: stage this job will be executed into.', + inherit: false entry :after_script, Entry::Script, description: 'Commands that will be executed when finishing job.', @@ -97,30 +99,50 @@ module Gitlab description: 'Services that will be used to execute this job.', inherit: true + entry :interruptible, Entry::Boolean, + description: 'Set jobs interruptible value.', + inherit: true + entry :only, Entry::Policy, description: 'Refs policy this job will be executed for.', - default: Entry::Policy::DEFAULT_ONLY + default: Entry::Policy::DEFAULT_ONLY, + inherit: false entry :except, Entry::Policy, - description: 'Refs policy this job will be executed for.' + description: 'Refs policy this job will be executed for.', + inherit: false entry :rules, Entry::Rules, - description: 'List of evaluable Rules to determine job inclusion.' + description: 'List of evaluable Rules to determine job inclusion.', + inherit: false, + metadata: { + allowed_when: %w[on_success on_failure always never manual delayed].freeze + } + + entry :needs, Entry::Needs, + description: 'Needs configuration for this job.', + metadata: { allowed_needs: %i[job] }, + inherit: false entry :variables, Entry::Variables, - description: 'Environment variables available for this job.' + description: 'Environment variables available for this job.', + inherit: false entry :artifacts, Entry::Artifacts, - description: 'Artifacts configuration for this job.' + description: 'Artifacts configuration for this job.', + inherit: false entry :environment, Entry::Environment, - description: 'Environment configuration for this job.' + description: 'Environment configuration for this job.', + inherit: false entry :coverage, Entry::Coverage, - description: 'Coverage configuration for this job.' + description: 'Coverage configuration for this job.', + inherit: false entry :retry, Entry::Retry, - description: 'Retry configuration for this job.' + description: 'Retry configuration for this job.', + inherit: false helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, @@ -155,8 +177,6 @@ module Gitlab @entries.delete(:except) end end - - inherit!(deps) end def name @@ -185,21 +205,8 @@ module Gitlab private - # We inherit config entries from `default:` - # if the entry has the `inherit: true` flag set - def inherit!(deps) - return unless deps - - self.class.nodes.each do |key, factory| - next unless factory.inheritable? - - default_entry = deps.default[key] - job_entry = self[key] - - if default_entry.specified? && !job_entry.specified? - @entries[key] = default_entry - end - end + def overwrite_entry(deps, key, current_entry) + deps.default[key] unless current_entry.specified? end def to_hash diff --git a/lib/gitlab/ci/config/entry/key.rb b/lib/gitlab/ci/config/entry/key.rb index 0c10967e629..f12f0919348 100644 --- a/lib/gitlab/ci/config/entry/key.rb +++ b/lib/gitlab/ci/config/entry/key.rb @@ -7,11 +7,48 @@ module Gitlab ## # Entry that represents a key. # - class Key < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable + class Key < ::Gitlab::Config::Entry::Simplifiable + strategy :SimpleKey, if: -> (config) { config.is_a?(String) || config.is_a?(Symbol) } + strategy :ComplexKey, if: -> (config) { config.is_a?(Hash) } - validations do - validates :config, key: true + class SimpleKey < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, key: true + end + + def self.default + 'default' + end + + def value + super.to_s + end + end + + class ComplexKey < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Configurable + + ALLOWED_KEYS = %i[files prefix].freeze + REQUIRED_KEYS = %i[files].freeze + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + validates :config, required_keys: REQUIRED_KEYS + end + + entry :files, Entry::Files, + description: 'Files that should be used to build the key' + entry :prefix, Entry::Prefix, + description: 'Prefix that is added to the final cache key' + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} should be a hash, a string or a symbol"] + end end def self.default diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb new file mode 100644 index 00000000000..b6db546d8ff --- /dev/null +++ b/lib/gitlab/ci/config/entry/need.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Need < ::Gitlab::Config::Entry::Simplifiable + strategy :Job, if: -> (config) { config.is_a?(String) } + + class Job < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, presence: true + validates :config, type: String + end + + def type + :job + end + + def value + { name: @config } + end + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def type + end + + def value + end + + def errors + ["#{location} has an unsupported type"] + end + end + end + end + end + end +end + +::Gitlab::Ci::Config::Entry::Need.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Need') diff --git a/lib/gitlab/ci/config/entry/needs.rb b/lib/gitlab/ci/config/entry/needs.rb new file mode 100644 index 00000000000..28452aaaa16 --- /dev/null +++ b/lib/gitlab/ci/config/entry/needs.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a set of needs dependencies. + # + class Needs < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, presence: true + + validate do + unless config.is_a?(Hash) || config.is_a?(Array) + errors.add(:config, 'can only be a Hash or an Array') + end + end + + validate on: :composed do + extra_keys = value.keys - opt(:allowed_needs) + if extra_keys.any? + errors.add(:config, "uses invalid types: #{extra_keys.join(', ')}") + end + end + end + + def compose!(deps = nil) + super(deps) do + [@config].flatten.each_with_index do |need, index| + @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Need) + .value(need) + .with(key: "need", parent: self, description: "need definition.") # rubocop:disable CodeReuse/ActiveRecord + .create! + end + + @entries.each_value do |entry| + entry.compose!(deps) + end + end + end + + def value + values = @entries.values.select(&:type) + values.group_by(&:type).transform_values do |values| + values.map(&:value) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/prefix.rb b/lib/gitlab/ci/config/entry/prefix.rb new file mode 100644 index 00000000000..3244ad6d611 --- /dev/null +++ b/lib/gitlab/ci/config/entry/prefix.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a key prefix. + # + class Prefix < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, key: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 07022ff7b54..25fb278d9b8 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable ALLOWED_KEYS = %i[default include before_script image services - after_script variables stages types cache].freeze + after_script variables stages types cache workflow].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -64,6 +64,9 @@ module Gitlab description: 'Configure caching between build jobs.', reserved: true + entry :workflow, Entry::Workflow, + description: 'List of evaluable rules to determine Pipeline status' + helpers :default, :jobs, :stages, :types, :variables delegate :before_script_value, diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb index 5d6d1c026e3..59e0ef583ae 100644 --- a/lib/gitlab/ci/config/entry/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -8,9 +8,9 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - CLAUSES = %i[if changes exists].freeze - ALLOWED_KEYS = %i[if changes exists when start_in].freeze - ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze + CLAUSES = %i[if changes exists].freeze + ALLOWED_KEYS = %i[if changes exists when start_in].freeze + ALLOWABLE_WHEN = %w[on_success on_failure always never manual delayed].freeze attributes :if, :changes, :exists, :when, :start_in @@ -25,7 +25,14 @@ module Gitlab with_options allow_nil: true do validates :if, expression: true validates :changes, :exists, array_of_strings: true, length: { maximum: 50 } - validates :when, allowed_values: { in: ALLOWED_WHEN } + validates :when, allowed_values: { in: ALLOWABLE_WHEN } + end + + validate do + validates_with Gitlab::Config::Entry::Validators::AllowedValuesValidator, + attributes: %i[when], + allow_nil: true, + in: opt(:allowed_when) end end diff --git a/lib/gitlab/ci/config/entry/script.rb b/lib/gitlab/ci/config/entry/script.rb index 9d25a82b521..285e18218b3 100644 --- a/lib/gitlab/ci/config/entry/script.rb +++ b/lib/gitlab/ci/config/entry/script.rb @@ -11,7 +11,11 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable validations do - validates :config, array_of_strings: true + validates :config, nested_array_of_strings: true + end + + def value + config.flatten(1) end end end diff --git a/lib/gitlab/ci/config/entry/workflow.rb b/lib/gitlab/ci/config/entry/workflow.rb new file mode 100644 index 00000000000..a51a3fbdcd2 --- /dev/null +++ b/lib/gitlab/ci/config/entry/workflow.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Workflow < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + + ALLOWED_KEYS = %i[rules].freeze + + validations do + validates :config, type: Hash + validates :config, allowed_keys: ALLOWED_KEYS + validates :config, presence: true + end + + entry :rules, Entry::Rules, + description: 'List of evaluable Rules to determine Pipeline status.', + metadata: { allowed_when: %w[always never] } + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 09f9bf5f69f..e714ef225f5 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -18,8 +18,8 @@ module Gitlab config[:dependencies] = expand_names(config[:dependencies]) end - if config[:needs] - config[:needs] = expand_names(config[:needs]) + if job_needs = config.dig(:needs, :job) + config[:needs][:job] = expand_needs(job_needs) end config @@ -36,6 +36,22 @@ module Gitlab end end + def expand_needs(job_needs) + return unless job_needs + + job_needs.flat_map do |job_need| + job_need_name = job_need[:name].to_sym + + if all_job_names = parallelized_jobs[job_need_name] + all_job_names.map do |job_name| + { name: job_name } + end + else + job_need + end + end + end + def parallelized_jobs strong_memoize(:parallelized_jobs) do @jobs_config.each_with_object({}) do |(job_name, config), hash| diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb index bab1c73e2f1..aabdf7ce47d 100644 --- a/lib/gitlab/ci/pipeline/chain/base.rb +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Chain class Base - attr_reader :pipeline, :command + attr_reader :pipeline, :command, :config delegate :project, :current_user, to: :command diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 899df81ea5c..9662209f88e 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -22,8 +22,6 @@ module Gitlab external_pull_request: @command.external_pull_request, variables_attributes: Array(@command.variables_attributes) ) - - @pipeline.set_config_source end def break? diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 58f89a6be5e..c2df419cca0 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -10,7 +10,9 @@ module Gitlab :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update + :chat_data, :allow_mirror_update, + # These attributes are set by Chains during processing: + :config_content, :config_processor, :stage_seeds ) do include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb new file mode 100644 index 00000000000..a8cd99b8e92 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content < Chain::Base + include Chain::Helpers + + def perform! + return if @command.config_content + + if content = content_from_repo + @command.config_content = content + @pipeline.config_source = :repository_source + # TODO: we should persist ci_config_path + # @pipeline.config_path = ci_config_path + elsif content = content_from_auto_devops + @command.config_content = content + @pipeline.config_source = :auto_devops_source + end + + unless @command.config_content + return error("Missing #{ci_config_path} file") + end + end + + def break? + @pipeline.errors.any? || @pipeline.persisted? + end + + private + + def content_from_repo + return unless project + return unless @pipeline.sha + return unless ci_config_path + + project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path) + rescue GRPC::NotFound, GRPC::Internal + nil + end + + def content_from_auto_devops + return unless project&.auto_devops_enabled? + + Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content + end + + def ci_config_path + project.ci_config_path.presence || '.gitlab-ci.yml' + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb new file mode 100644 index 00000000000..731b0fdb286 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Process < Chain::Base + include Chain::Helpers + + def perform! + raise ArgumentError, 'missing config content' unless @command.config_content + + @command.config_processor = ::Gitlab::Ci::YamlProcessor.new( + @command.config_content, { + project: project, + sha: @pipeline.sha, + user: current_user + } + ) + rescue Gitlab::Ci::YamlProcessor::ValidationError => ex + error(ex.message, config_error: true) + rescue => ex + Gitlab::Sentry.track_acceptable_exception(ex, extra: { + project_id: project.id, + sha: @pipeline.sha + }) + + error("Undefined error (#{Labkit::Correlation::CorrelationId.current_id})", + config_error: true) + end + + def break? + @pipeline.errors.any? || @pipeline.persisted? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb new file mode 100644 index 00000000000..0ee9485eebc --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class EvaluateWorkflowRules < Chain::Base + include ::Gitlab::Utils::StrongMemoize + include Chain::Helpers + + def perform! + return unless Feature.enabled?(:workflow_rules, @pipeline.project) + + unless workflow_passed? + error('Pipeline filtered out by workflow rules.') + end + end + + def break? + return false unless Feature.enabled?(:workflow_rules, @pipeline.project) + + !workflow_passed? + end + + private + + def workflow_passed? + strong_memoize(:workflow_passed) do + workflow_rules.evaluate(@pipeline, global_context).pass? + end + end + + def workflow_rules + Gitlab::Ci::Build::Rules.new( + workflow_config[:rules], default_when: 'always') + end + + def global_context + Gitlab::Ci::Build::Context::Global.new( + @pipeline, yaml_variables: workflow_config[:yaml_variables]) + end + + def workflow_config + @command.config_processor.workflow_attributes || {} + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 13eca5a9d28..3a40c7b167c 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -10,29 +10,12 @@ module Gitlab PopulateError = Class.new(StandardError) def perform! - # Allocate next IID. This operation must be outside of transactions of pipeline creations. - pipeline.ensure_project_iid! - - # Protect the pipeline. This is assigned in Populate instead of - # Build to prevent erroring out on ambiguous refs. - pipeline.protected = @command.protected_ref? - - ## - # Populate pipeline with block argument of CreatePipelineService#execute. - # - @command.seeds_block&.call(pipeline) - - ## - # Gather all runtime build/stage errors - # - if seeds_errors = pipeline.stage_seeds.flat_map(&:errors).compact.presence - return error(seeds_errors.join("\n"), config_error: true) - end + raise ArgumentError, 'missing stage seeds' unless @command.stage_seeds ## # Populate pipeline with all stages, and stages with builds. # - pipeline.stages = pipeline.stage_seeds.map(&:to_resource) + pipeline.stages = @command.stage_seeds.map(&:to_resource) if pipeline.stages.none? return error('No stages / jobs for this pipeline.') diff --git a/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb index 1e09b417311..9267c72efa4 100644 --- a/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb +++ b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb @@ -6,11 +6,13 @@ module Gitlab module Chain class RemoveUnwantedChatJobs < Chain::Base def perform! - return unless pipeline.config_processor && pipeline.chat? + raise ArgumentError, 'missing config processor' unless @command.config_processor + + return unless pipeline.chat? # When scheduling a chat pipeline we only want to run the build # that matches the chat command. - pipeline.config_processor.jobs.select! do |name, _| + @command.config_processor.jobs.select! do |name, _| name.to_s == command.chat_data[:command].to_s end end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb new file mode 100644 index 00000000000..2e177cfec7e --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class Seed < Chain::Base + include Chain::Helpers + include Gitlab::Utils::StrongMemoize + + def perform! + raise ArgumentError, 'missing config processor' unless @command.config_processor + + # Allocate next IID. This operation must be outside of transactions of pipeline creations. + pipeline.ensure_project_iid! + + # Protect the pipeline. This is assigned in Populate instead of + # Build to prevent erroring out on ambiguous refs. + pipeline.protected = @command.protected_ref? + + ## + # Populate pipeline with block argument of CreatePipelineService#execute. + # + @command.seeds_block&.call(pipeline) + + ## + # Gather all runtime build/stage errors + # + if stage_seeds_errors + return error(stage_seeds_errors.join("\n"), config_error: true) + end + + @command.stage_seeds = stage_seeds + end + + def break? + pipeline.errors.any? + end + + private + + def stage_seeds_errors + stage_seeds.flat_map(&:errors).compact.presence + end + + def stage_seeds + strong_memoize(:stage_seeds) do + seeds = stages_attributes.inject([]) do |previous_stages, attributes| + seed = Gitlab::Ci::Pipeline::Seed::Stage.new(pipeline, attributes, previous_stages) + previous_stages + [seed] + end + + seeds.select(&:included?) + end + end + + def stages_attributes + @command.config_processor.stages_attributes + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb deleted file mode 100644 index 28c38cc3d18..00000000000 --- a/lib/gitlab/ci/pipeline/chain/validate/config.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Validate - class Config < Chain::Base - include Chain::Helpers - - def perform! - unless @pipeline.config_processor - unless @pipeline.ci_yaml_file - return error("Missing #{@pipeline.ci_yaml_file_path} file") - end - - if @command.save_incompleted && @pipeline.has_yaml_errors? - @pipeline.drop!(:config_error) - end - - error(@pipeline.yaml_errors) - end - end - - def break? - @pipeline.errors.any? || @pipeline.persisted? - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index fc9c540088b..dce56b22666 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -28,7 +28,9 @@ module Gitlab @except = Gitlab::Ci::Build::Policy .fabricate(attributes.delete(:except)) @rules = Gitlab::Ci::Build::Rules - .new(attributes.delete(:rules)) + .new(attributes.delete(:rules), default_when: 'on_success') + @cache = Seed::Build::Cache + .new(pipeline, attributes.delete(:cache)) end def name @@ -38,7 +40,7 @@ module Gitlab def included? strong_memoize(:inclusion) do if @using_rules - included_by_rules? + rules_result.pass? elsif @using_only || @using_except all_of_only? && none_of_except? else @@ -59,6 +61,7 @@ module Gitlab @seed_attributes .deep_merge(pipeline_attributes) .deep_merge(rules_attributes) + .deep_merge(cache_attributes) end def bridge? @@ -80,26 +83,14 @@ module Gitlab end end - def scoped_variables_hash - strong_memoize(:scoped_variables_hash) do - # This is a temporary piece of technical debt to allow us access - # to the CI variables to evaluate rules before we persist a Build - # with the result. We should refactor away the extra Build.new, - # but be able to get CI Variables directly from the Seed::Build. - ::Ci::Build.new( - @seed_attributes.merge(pipeline_attributes) - ).scoped_variables_hash - end - end - private def all_of_only? - @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } + @only.all? { |spec| spec.satisfied_by?(@pipeline, evaluate_context) } end def none_of_except? - @except.none? { |spec| spec.satisfied_by?(@pipeline, self) } + @except.none? { |spec| spec.satisfied_by?(@pipeline, evaluate_context) } end def needs_errors @@ -141,13 +132,27 @@ module Gitlab } end - def included_by_rules? - rules_attributes[:when] != 'never' + def rules_attributes + return {} unless @using_rules + + rules_result.build_attributes end - def rules_attributes - strong_memoize(:rules_attributes) do - @using_rules ? @rules.evaluate(@pipeline, self).build_attributes : {} + def rules_result + strong_memoize(:rules_result) do + @rules.evaluate(@pipeline, evaluate_context) + end + end + + def evaluate_context + strong_memoize(:evaluate_context) do + Gitlab::Ci::Build::Context::Build.new(@pipeline, @seed_attributes) + end + end + + def cache_attributes + strong_memoize(:cache_attributes) do + @cache.build_attributes end end end diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb new file mode 100644 index 00000000000..7671035b896 --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Seed + class Build + class Cache + def initialize(pipeline, cache) + @pipeline = pipeline + local_cache = cache.to_h.deep_dup + @key = local_cache.delete(:key) + @paths = local_cache.delete(:paths) + @policy = local_cache.delete(:policy) + @untracked = local_cache.delete(:untracked) + + raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any? + end + + def build_attributes + { + options: { + cache: { + key: key_string, + paths: @paths, + policy: @policy, + untracked: @untracked + }.compact.presence + }.compact + } + end + + private + + def key_string + key_from_string || key_from_files + end + + def key_from_string + @key.to_s if @key.is_a?(String) || @key.is_a?(Symbol) + end + + def key_from_files + return unless @key.is_a?(Hash) + + [@key[:prefix], files_digest].select(&:present?).join('-') + end + + def files_digest + hash_of_the_latest_changes || 'default' + end + + def hash_of_the_latest_changes + return unless Feature.enabled?(:ci_file_based_cache, @pipeline.project, default_enabled: true) + + ids = files.map { |path| last_commit_id_for_path(path) } + ids = ids.compact.sort.uniq + + Digest::SHA1.hexdigest(ids.join('-')) if ids.any? + end + + def files + @key[:files] + .to_a + .select(&:present?) + .uniq + end + + def last_commit_id_for_path(path) + @pipeline.project.repository.last_commit_id_for_path(@pipeline.sha, path) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 961012c2cee..910d93f54ce 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -16,7 +16,9 @@ module Gitlab stale_schedule: 'stale schedule', job_execution_timeout: 'job execution timeout', archived_failure: 'archived failure', - unmet_prerequisites: 'unmet prerequisites' + unmet_prerequisites: 'unmet prerequisites', + scheduler_failure: 'scheduler failure', + data_integrity_failure: 'data integrity failure' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 3cdb7b5420c..a60b00b2ee8 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -18,7 +18,7 @@ code_quality: --env SOURCE_CODE="$PWD" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock - "registry.gitlab.com/gitlab-org/security-products/codequality:12-0-stable" /code + "registry.gitlab.com/gitlab-org/security-products/codequality:12-5-stable" /code artifacts: reports: codequality: gl-code-quality-report.json diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index ae2ff9992f9..7a672f910dd 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,8 +1,8 @@ -.auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.1.0" +.dast-auto-deploy: + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.6.0" dast_environment_deploy: - extends: .auto-deploy + extends: .dast-auto-deploy stage: review script: - auto-deploy check_kube_domain @@ -28,10 +28,10 @@ dast_environment_deploy: variables: - $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME - $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH - - $DAST_WEBSITE # we don't need to create a review app if a URL is already given + - $DAST_WEBSITE # we don't need to create a review app if a URL is already given stop_dast_environment: - extends: .auto-deploy + extends: .dast-auto-deploy stage: cleanup variables: GIT_STRATEGY: none diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index a8ec2d4781d..738be44d5f4 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.1.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.7.0" review: extends: .auto-deploy diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index f058468ed8e..ef2fc561201 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -9,16 +9,17 @@ container_scanning: name: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CS_MAJOR_VERSION entrypoint: [] variables: - # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here - # with a specific version to provide consistency for integration testing purposes - CLAIR_DB_IMAGE_TAG: latest - # Override this variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yaml` file. - # See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template + # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image + # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes + CLAIR_DB_IMAGE_TAG: "latest" + CLAIR_DB_IMAGE: "arminc/clair-db:$CLAIR_DB_IMAGE_TAG" + # Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml` + # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details GIT_STRATEGY: none allow_failure: true services: - - name: arminc/clair-db:$CLAIR_DB_IMAGE_TAG + - name: $CLAIR_DB_IMAGE alias: clair-vulnerabilities-db script: # the kubernetes executor currently ignores the Docker image entrypoint value, so the start.sh script must diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index c8930bc6263..4993d22d400 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -4,6 +4,12 @@ # List of the variables: https://gitlab.com/gitlab-org/security-products/dependency-scanning#settings # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +variables: + DS_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + DS_DEFAULT_ANALYZERS: "gemnasium, retire.js, gemnasium-python, gemnasium-maven, bundler-audit" + DS_MAJOR_VERSION: 2 + DS_DISABLE_DIND: "false" + dependency_scanning: stage: test image: docker:stable @@ -45,6 +51,7 @@ dependency_scanning: DS_PIP_DEPENDENCY_PATH \ PIP_INDEX_URL \ PIP_EXTRA_INDEX_URL \ + MAVEN_CLI_OPTS \ ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ @@ -61,3 +68,63 @@ dependency_scanning: except: variables: - $DEPENDENCY_SCANNING_DISABLED + - $DS_DISABLE_DIND == 'true' + +.analyzer: + extends: dependency_scanning + services: [] + except: + variables: + - $DS_DISABLE_DIND == 'false' + script: + - /analyzer run + +gemnasium-dependency_scanning: + extends: .analyzer + image: + name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium:$DS_MAJOR_VERSION" + only: + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby|javascript|php/ + +gemnasium-maven-dependency_scanning: + extends: .analyzer + image: + name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" + only: + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bjava\b/ + +gemnasium-python-dependency_scanning: + extends: .analyzer + image: + name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" + only: + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /python/ + +bundler-audit-dependency_scanning: + extends: .analyzer + image: + name: "$DS_ANALYZER_IMAGE_PREFIX/bundler-audit:$DS_MAJOR_VERSION" + only: + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /bundler-audit/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby/ + +retire-js-dependency_scanning: + extends: .analyzer + image: + name: "$DS_ANALYZER_IMAGE_PREFIX/retire.js:$DS_MAJOR_VERSION" + only: + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /retire.js/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/ diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index a0c2ab3aa26..c81b4efddbc 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -7,7 +7,7 @@ variables: SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex" - SAST_MAJOR_VERSION: 2 + SAST_ANALYZER_IMAGE_TAG: 2 SAST_DISABLE_DIND: "false" sast: @@ -35,45 +35,12 @@ sast: export DOCKER_HOST='tcp://localhost:2375' fi fi - - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage - function propagate_env_vars() { - CURRENT_ENV=$(printenv) - - for VAR_NAME; do - echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " - done - } + - | + printenv | grep -E '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | cut -d'=' -f1 | \ + (while IFS='\\n' read -r VAR; do unset -v "$VAR"; done; /bin/printenv > .env) - | docker run \ - $(propagate_env_vars \ - SAST_BANDIT_EXCLUDED_PATHS \ - SAST_ANALYZER_IMAGES \ - SAST_ANALYZER_IMAGE_PREFIX \ - SAST_ANALYZER_IMAGE_TAG \ - SAST_DEFAULT_ANALYZERS \ - SAST_PULL_ANALYZER_IMAGES \ - SAST_BRAKEMAN_LEVEL \ - SAST_FLAWFINDER_LEVEL \ - SAST_GITLEAKS_ENTROPY_LEVEL \ - SAST_GOSEC_LEVEL \ - SAST_EXCLUDED_PATHS \ - SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ - SAST_RUN_ANALYZER_TIMEOUT \ - SAST_JAVA_VERSION \ - ANT_HOME \ - ANT_PATH \ - GRADLE_PATH \ - JAVA_OPTS \ - JAVA_PATH \ - JAVA_8_VERSION \ - JAVA_11_VERSION \ - MAVEN_CLI_OPTS \ - MAVEN_PATH \ - MAVEN_REPO_PATH \ - SBT_PATH \ - FAIL_NEVER \ - ) \ + --env-file .env \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code @@ -94,7 +61,7 @@ sast: bandit-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/bandit:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -104,7 +71,7 @@ bandit-sast: brakeman-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -114,7 +81,7 @@ brakeman-sast: eslint-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -124,7 +91,7 @@ eslint-sast: flawfinder-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/flawfinder:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -134,7 +101,7 @@ flawfinder-sast: gosec-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/gosec:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -144,7 +111,7 @@ gosec-sast: nodejs-scan-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -154,7 +121,7 @@ nodejs-scan-sast: phpcs-security-audit-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/phpcs-security-audit:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -164,7 +131,7 @@ phpcs-security-audit-sast: pmd-apex-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/pmd-apex:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -174,7 +141,7 @@ pmd-apex-sast: secrets-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -183,7 +150,7 @@ secrets-sast: security-code-scan-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/security-code-scan:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -193,7 +160,7 @@ security-code-scan-sast: sobelow-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/sobelow:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -203,7 +170,7 @@ sobelow-sast: spotbugs-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/spotbugs:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -213,7 +180,7 @@ spotbugs-sast: tslint-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/tslint:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/tslint:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index f6a3abefcfb..833c545fc5b 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -39,15 +39,15 @@ module Gitlab when: job[:when] || 'on_success', environment: job[:environment_name], coverage_regex: job[:coverage], - yaml_variables: yaml_variables(name), - needs_attributes: job[:needs]&.map { |need| { name: need } }, + yaml_variables: transform_to_yaml_variables(job_variables(name)), + needs_attributes: job.dig(:needs, :job), interruptible: job[:interruptible], rules: job[:rules], + cache: job[:cache], options: { image: job[:image], services: job[:services], artifacts: job[:artifacts], - cache: job[:cache], dependencies: job[:dependencies], job_timeout: job[:timeout], before_script: job[:before_script], @@ -59,7 +59,7 @@ module Gitlab instance: job[:instance], start_in: job[:start_in], trigger: job[:trigger], - bridge_needs: job[:needs] + bridge_needs: job.dig(:needs, :bridge)&.first }.compact }.compact end @@ -83,6 +83,13 @@ module Gitlab end end + def workflow_attributes + { + rules: @config.dig(:workflow, :rules), + yaml_variables: transform_to_yaml_variables(@variables) + } + end + def self.validation_message(content, opts = {}) return 'Please provide content of .gitlab-ci.yml' if content.blank? @@ -118,20 +125,17 @@ module Gitlab end end - def yaml_variables(name) - variables = (@variables || {}) - .merge(job_variables(name)) + def job_variables(name) + job_variables = @jobs.dig(name.to_sym, :variables) - variables.map do |key, value| - { key: key.to_s, value: value, public: true } - end + @variables.to_h + .merge(job_variables.to_h) end - def job_variables(name) - job = @jobs[name.to_sym] - return {} unless job - - job[:variables] || {} + def transform_to_yaml_variables(variables) + variables.to_h.map do |key, value| + { key: key.to_s, value: value, public: true } + end end def validate_job_stage!(name, job) @@ -159,17 +163,19 @@ module Gitlab end def validate_job_needs!(name, job) - return unless job[:needs] + return unless job.dig(:needs, :job) stage_index = @stages.index(job[:stage]) - job[:needs].each do |need| - raise ValidationError, "#{name} job: undefined need: #{need}" unless @jobs[need.to_sym] + job.dig(:needs, :job).each do |need| + need_job_name = need[:name] + + raise ValidationError, "#{name} job: undefined need: #{need_job_name}" unless @jobs[need_job_name.to_sym] - needs_stage_index = @stages.index(@jobs[need.to_sym][:stage]) + needs_stage_index = @stages.index(@jobs[need_job_name.to_sym][:stage]) unless needs_stage_index.present? && needs_stage_index < stage_index - raise ValidationError, "#{name} job: need #{need} is not defined in prior stages" + raise ValidationError, "#{name} job: need #{need_job_name} is not defined in prior stages" end end end diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files.rb b/lib/gitlab/cleanup/orphan_job_artifact_files.rb index 1b01ca25559..020de45e5bf 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files.rb @@ -8,7 +8,8 @@ module Gitlab ABSOLUTE_ARTIFACT_DIR = ::JobArtifactUploader.root.freeze LOST_AND_FOUND = File.join(ABSOLUTE_ARTIFACT_DIR, '-', 'lost+found').freeze BATCH_SIZE = 500 - DEFAULT_NICENESS = 'Best-effort' + DEFAULT_NICENESS = 'best-effort' + VALID_NICENESS_LEVELS = %w{none realtime best-effort idle}.freeze attr_accessor :batch, :total_found, :total_cleaned attr_reader :limit, :dry_run, :niceness, :logger @@ -16,7 +17,7 @@ module Gitlab def initialize(limit: nil, dry_run: true, niceness: nil, logger: nil) @limit = limit @dry_run = dry_run - @niceness = niceness || DEFAULT_NICENESS + @niceness = (niceness || DEFAULT_NICENESS).downcase @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger @total_found = @total_cleaned = 0 @@ -35,7 +36,7 @@ module Gitlab clean_batch! - log_info("Processed #{total_found} job artifacts to find and clean #{total_cleaned} orphans.") + log_info("Processed #{total_found} job artifact(s) to find and cleaned #{total_cleaned} orphan(s).") end private @@ -75,7 +76,7 @@ module Gitlab def find_artifacts Open3.popen3(*find_command) do |stdin, stdout, stderr, status_thread| stdout.each_line do |line| - yield line + yield line.chomp end log_error(stderr.read.color(:red)) unless status_thread.value.success? @@ -99,7 +100,7 @@ module Gitlab cmd += %w[-type d] if ionice - raise ArgumentError, 'Invalid niceness' unless niceness.match?(/^\w[\w\-]*$/) + raise ArgumentError, 'Invalid niceness' unless VALID_NICENESS_LEVELS.include?(niceness) cmd.unshift(*%W[#{ionice} --class #{niceness}]) end diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index 294ffad02ce..2b3dc94fc5e 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -10,38 +10,39 @@ module Gitlab # # We have the following lifecycle events. # - # - on_master_start: + # - on_before_fork (on master process): # # Unicorn/Puma Cluster: This will be called exactly once, # on startup, before the workers are forked. This is # called in the PARENT/MASTER process. # - # Sidekiq/Puma Single: This is called immediately. + # Sidekiq/Puma Single: This is not called. # - # - on_before_fork: + # - on_master_start (on master process): # # Unicorn/Puma Cluster: This will be called exactly once, # on startup, before the workers are forked. This is # called in the PARENT/MASTER process. # - # Sidekiq/Puma Single: This is not called. + # Sidekiq/Puma Single: This is called immediately. # - # - on_worker_start: + # - on_before_blackout_period (on master process): # - # Unicorn/Puma Cluster: This is called in the worker process - # exactly once before processing requests. + # Unicorn/Puma Cluster: This will be called before a blackout + # period when performing graceful shutdown of master. + # This is called on `master` process. # - # Sidekiq/Puma Single: This is called immediately. + # Sidekiq/Puma Single: This is not called. # - # - on_before_phased_restart: + # - on_before_graceful_shutdown (on master process): # # Unicorn/Puma Cluster: This will be called before a graceful - # shutdown of workers starts happening. + # shutdown of workers starts happening, but after blackout period. # This is called on `master` process. # # Sidekiq/Puma Single: This is not called. # - # - on_before_master_restart: + # - on_before_master_restart (on master process): # # Unicorn: This will be called before a new master is spun up. # This is called on forked master before `execve` to become @@ -53,6 +54,13 @@ module Gitlab # # Sidekiq/Puma Single: This is not called. # + # - on_worker_start (on worker process): + # + # Unicorn/Puma Cluster: This is called in the worker process + # exactly once before processing requests. + # + # Sidekiq/Puma Single: This is called immediately. + # # Blocks will be executed in the order in which they are registered. # class LifecycleEvents @@ -75,9 +83,15 @@ module Gitlab end # Read the config/initializers/cluster_events_before_phased_restart.rb - def on_before_phased_restart(&block) + def on_before_blackout_period(&block) # Defer block execution - (@master_phased_restart ||= []) << block + (@master_blackout_period ||= []) << block + end + + # Read the config/initializers/cluster_events_before_phased_restart.rb + def on_before_graceful_shutdown(&block) + # Defer block execution + (@master_graceful_shutdown ||= []) << block end def on_before_master_restart(&block) @@ -97,27 +111,24 @@ module Gitlab # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.) # def do_worker_start - @worker_start_hooks&.each do |block| - block.call - end + call(@worker_start_hooks) end def do_before_fork - @before_fork_hooks&.each do |block| - block.call - end + call(@before_fork_hooks) end - def do_before_phased_restart - @master_phased_restart&.each do |block| - block.call - end + def do_before_graceful_shutdown + call(@master_blackout_period) + + blackout_seconds = ::Settings.shutdown.blackout_seconds.to_i + sleep(blackout_seconds) if blackout_seconds > 0 + + call(@master_graceful_shutdown) end def do_before_master_restart - @master_restart_hooks&.each do |block| - block.call - end + call(@master_restart_hooks) end # DEPRECATED @@ -132,6 +143,10 @@ module Gitlab private + def call(hooks) + hooks&.each(&:call) + end + def in_clustered_environment? # Sidekiq doesn't fork return false if Sidekiq.server? diff --git a/lib/gitlab/cluster/mixins/puma_cluster.rb b/lib/gitlab/cluster/mixins/puma_cluster.rb index e9157d9f1e4..106c2731c07 100644 --- a/lib/gitlab/cluster/mixins/puma_cluster.rb +++ b/lib/gitlab/cluster/mixins/puma_cluster.rb @@ -8,8 +8,12 @@ module Gitlab raise 'missing method Puma::Cluster#stop_workers' unless base.method_defined?(:stop_workers) end + # This looks at internal status of `Puma::Cluster` + # https://github.com/puma/puma/blob/v3.12.1/lib/puma/cluster.rb#L333 def stop_workers - Gitlab::Cluster::LifecycleEvents.do_before_phased_restart + if @status == :stop # rubocop:disable Gitlab/ModuleWithInstanceVariables + Gitlab::Cluster::LifecycleEvents.do_before_graceful_shutdown + end super end diff --git a/lib/gitlab/cluster/mixins/unicorn_http_server.rb b/lib/gitlab/cluster/mixins/unicorn_http_server.rb index 765fd0c2baa..440ed02a355 100644 --- a/lib/gitlab/cluster/mixins/unicorn_http_server.rb +++ b/lib/gitlab/cluster/mixins/unicorn_http_server.rb @@ -5,11 +5,26 @@ module Gitlab module Mixins module UnicornHttpServer def self.prepended(base) - raise 'missing method Unicorn::HttpServer#reexec' unless base.method_defined?(:reexec) + unless base.method_defined?(:reexec) && base.method_defined?(:stop) + raise 'missing method Unicorn::HttpServer#reexec or Unicorn::HttpServer#stop' + end end def reexec - Gitlab::Cluster::LifecycleEvents.do_before_phased_restart + Gitlab::Cluster::LifecycleEvents.do_before_graceful_shutdown + + super + end + + # The stop on non-graceful shutdown is executed twice: + # `#stop(false)` and `#stop`. + # + # The first stop will wipe-out all workers, so we need to check + # the flag and a list of workers + def stop(graceful = true) + if graceful && @workers.any? # rubocop:disable Gitlab/ModuleWithInstanceVariables + Gitlab::Cluster::LifecycleEvents.do_before_graceful_shutdown + end super end diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index a8440b63baa..92c799875b5 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -3,7 +3,12 @@ module Gitlab module Cluster class PumaWorkerKillerInitializer - def self.start(puma_options, puma_per_worker_max_memory_mb: 850, puma_master_max_memory_mb: 550) + def self.start( + puma_options, + puma_per_worker_max_memory_mb: 850, + puma_master_max_memory_mb: 550, + additional_puma_dev_max_memory_mb: 200 + ) require 'puma_worker_killer' PumaWorkerKiller.config do |config| @@ -14,7 +19,11 @@ module Gitlab # The Puma Worker Killer checks the total RAM used by both the master # and worker processes. # https://github.com/schneems/puma_worker_killer/blob/v0.1.0/lib/puma_worker_killer/puma_memory.rb#L57 - config.ram = puma_master_max_memory_mb + (worker_count * puma_per_worker_max_memory_mb) + # + # Additional memory is added when running in `development` + config.ram = puma_master_max_memory_mb + + (worker_count * puma_per_worker_max_memory_mb) + + (Rails.env.development? ? (1 + worker_count) * additional_puma_dev_max_memory_mb : 0) config.frequency = 20 # seconds diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb index b7ec4b7c4f8..bda84dc2cff 100644 --- a/lib/gitlab/config/entry/configurable.rb +++ b/lib/gitlab/config/entry/configurable.rb @@ -29,22 +29,24 @@ module Gitlab def compose!(deps = nil) return unless valid? - self.class.nodes.each do |key, factory| - # If we override the config type validation - # we can end with different config types like String - next unless config.is_a?(Hash) + super do + self.class.nodes.each do |key, factory| + # If we override the config type validation + # we can end with different config types like String + next unless config.is_a?(Hash) - factory - .value(config[key]) - .with(key: key, parent: self) + factory + .value(config[key]) + .with(key: key, parent: self) - entries[key] = factory.create! - end + entries[key] = factory.create! + end - yield if block_given? + yield if block_given? - entries.each_value do |entry| - entry.compose!(deps) + entries.each_value do |entry| + entry.compose!(deps) + end end end # rubocop: enable CodeReuse/ActiveRecord @@ -67,12 +69,13 @@ module Gitlab private # rubocop: disable CodeReuse/ActiveRecord - def entry(key, entry, description: nil, default: nil, inherit: nil, reserved: nil) + def entry(key, entry, description: nil, default: nil, inherit: nil, reserved: nil, metadata: {}) factory = ::Gitlab::Config::Entry::Factory.new(entry) .with(description: description) .with(default: default) .with(inherit: inherit) .with(reserved: reserved) + .metadata(metadata) (@nodes ||= {}).merge!(key.to_sym => factory) end diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb index 8f1f4a81bb5..7c5ffaa7621 100644 --- a/lib/gitlab/config/entry/factory.rb +++ b/lib/gitlab/config/entry/factory.rb @@ -9,10 +9,12 @@ module Gitlab class Factory InvalidFactory = Class.new(StandardError) - def initialize(entry) - @entry = entry + attr_reader :entry_class + + def initialize(entry_class) + @entry_class = entry_class @metadata = {} - @attributes = { default: entry.default } + @attributes = { default: entry_class.default } end def value(value) @@ -34,6 +36,10 @@ module Gitlab @attributes[:description] end + def inherit + @attributes[:inherit] + end + def inheritable? @attributes[:inherit] end @@ -52,7 +58,7 @@ module Gitlab if @value.nil? Entry::Unspecified.new(fabricate_unspecified) else - fabricate(@entry, @value) + fabricate(entry_class, @value) end end @@ -68,12 +74,12 @@ module Gitlab if default.nil? fabricate(Entry::Undefined) else - fabricate(@entry, default) + fabricate(entry_class, default) end end - def fabricate(entry, value = nil) - entry.new(value, @metadata) do |node| + def fabricate(entry_class, value = nil) + entry_class.new(value, @metadata) do |node| node.key = @attributes[:key] node.parent = @attributes[:parent] node.default = @attributes[:default] diff --git a/lib/gitlab/config/entry/inheritable.rb b/lib/gitlab/config/entry/inheritable.rb new file mode 100644 index 00000000000..91ca82e6338 --- /dev/null +++ b/lib/gitlab/config/entry/inheritable.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # Entry that represents an inheritable configs. + # + module Inheritable + InheritError = Class.new(Gitlab::Config::Loader::FormatError) + + def compose!(deps = nil, &blk) + super(deps, &blk) + + inherit!(deps) + end + + private + + # We inherit config entries from `default:` + # if the entry has the `inherit: true` flag set + def inherit!(deps) + return unless deps + + self.class.nodes.each do |key, factory| + next unless factory.inheritable? + + new_entry = overwrite_entry(deps, key, self[key]) + + entries[key] = new_entry if new_entry&.specified? + end + end + + def overwrite_entry(deps, key, current_entry) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/config/entry/node.rb b/lib/gitlab/config/entry/node.rb index e014f15fbd8..84d3409ed91 100644 --- a/lib/gitlab/config/entry/node.rb +++ b/lib/gitlab/config/entry/node.rb @@ -112,6 +112,10 @@ module Gitlab @aspects ||= [] end + def self.with_aspect(blk) + self.aspects.append(blk) + end + private attr_reader :entries diff --git a/lib/gitlab/config/entry/simplifiable.rb b/lib/gitlab/config/entry/simplifiable.rb index d58aba07d15..315f1947e2c 100644 --- a/lib/gitlab/config/entry/simplifiable.rb +++ b/lib/gitlab/config/entry/simplifiable.rb @@ -4,11 +4,11 @@ module Gitlab module Config module Entry class Simplifiable < SimpleDelegator - EntryStrategy = Struct.new(:name, :condition) + EntryStrategy = Struct.new(:name, :klass, :condition) attr_reader :subject - def initialize(config, **metadata) + def initialize(config, **metadata, &blk) unless self.class.const_defined?(:UnknownStrategy) raise ArgumentError, 'UndefinedStrategy not available!' end @@ -19,14 +19,13 @@ module Gitlab entry = self.class.entry_class(strategy) - @subject = entry.new(config, metadata) + @subject = entry.new(config, metadata, &blk) - yield(@subject) if block_given? super(@subject) end def self.strategy(name, **opts) - EntryStrategy.new(name, opts.fetch(:if)).tap do |strategy| + EntryStrategy.new(name, opts.dig(:class), opts.fetch(:if)).tap do |strategy| strategies.append(strategy) end end @@ -37,7 +36,7 @@ module Gitlab def self.entry_class(strategy) if strategy.present? - self.const_get(strategy.name, false) + strategy.klass || self.const_get(strategy.name, false) else self::UnknownStrategy end diff --git a/lib/gitlab/config/entry/validatable.rb b/lib/gitlab/config/entry/validatable.rb index 1c88c68c11c..45b852dc2e0 100644 --- a/lib/gitlab/config/entry/validatable.rb +++ b/lib/gitlab/config/entry/validatable.rb @@ -7,14 +7,27 @@ module Gitlab extend ActiveSupport::Concern def self.included(node) - node.aspects.append -> do - @validator = self.class.validator.new(self) - @validator.validate(:new) + node.with_aspect -> do + validate(:new) end end + def validator + @validator ||= self.class.validator.new(self) + end + + def validate(context = nil) + validator.validate(context) + end + + def compose!(deps = nil, &blk) + super(deps, &blk) + + validate(:composed) + end + def errors - @validator.messages + descendants.flat_map(&:errors) # rubocop:disable Gitlab/ModuleWithInstanceVariables + validator.messages + descendants.flat_map(&:errors) end class_methods do diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 374f929878e..d1c23c41d35 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -61,8 +61,15 @@ module Gitlab include LegacyValidationHelpers def validate_each(record, attribute, value) - unless validate_array_of_strings(value) - record.errors.add(attribute, 'should be an array of strings') + valid = validate_array_of_strings(value) + + record.errors.add(attribute, 'should be an array of strings') unless valid + + if valid && options[:with] + unless value.all? { |v| v =~ options[:with] } + message = options[:message] || 'contains elements that do not match the format' + record.errors.add(attribute, message) + end end end end @@ -221,6 +228,34 @@ module Gitlab end end + class NestedArrayOfStringsValidator < ArrayOfStringsOrStringValidator + def validate_each(record, attribute, value) + unless validate_nested_array_of_strings(value) + record.errors.add(attribute, 'should be an array containing strings and arrays of strings') + end + end + + private + + def validate_nested_array_of_strings(values) + values.is_a?(Array) && values.all? { |element| validate_array_of_strings_or_string(element) } + end + end + + class StringOrNestedArrayOfStringsValidator < NestedArrayOfStringsValidator + def validate_each(record, attribute, value) + unless validate_string_or_nested_array_of_strings(value) + record.errors.add(attribute, 'should be a string or an array containing strings and arrays of strings') + end + end + + private + + def validate_string_or_nested_array_of_strings(values) + validate_string(values) || validate_nested_array_of_strings(values) + end + end + class TypeValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) type = options[:with] diff --git a/lib/gitlab/cycle_analytics/group_stage_summary.rb b/lib/gitlab/cycle_analytics/group_stage_summary.rb index a1fc941495d..26eaaf7df83 100644 --- a/lib/gitlab/cycle_analytics/group_stage_summary.rb +++ b/lib/gitlab/cycle_analytics/group_stage_summary.rb @@ -3,18 +3,17 @@ module Gitlab module CycleAnalytics class GroupStageSummary - attr_reader :group, :from, :current_user, :options + attr_reader :group, :current_user, :options def initialize(group, options:) @group = group - @from = options[:from] @current_user = options[:current_user] @options = options end def data - [serialize(Summary::Group::Issue.new(group: group, from: from, current_user: current_user, options: options)), - serialize(Summary::Group::Deploy.new(group: group, from: from, options: options))] + [serialize(Summary::Group::Issue.new(group: group, current_user: current_user, options: options)), + serialize(Summary::Group::Deploy.new(group: group, options: options))] end private diff --git a/lib/gitlab/cycle_analytics/summary/group/base.rb b/lib/gitlab/cycle_analytics/summary/group/base.rb index 48d8164bde1..f1d20d5aefa 100644 --- a/lib/gitlab/cycle_analytics/summary/group/base.rb +++ b/lib/gitlab/cycle_analytics/summary/group/base.rb @@ -5,11 +5,10 @@ module Gitlab module Summary module Group class Base - attr_reader :group, :from, :options + attr_reader :group, :options - def initialize(group:, from:, options:) + def initialize(group:, options:) @group = group - @from = from @options = options end diff --git a/lib/gitlab/cycle_analytics/summary/group/deploy.rb b/lib/gitlab/cycle_analytics/summary/group/deploy.rb index 78d677cf558..11a9152cf0c 100644 --- a/lib/gitlab/cycle_analytics/summary/group/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/group/deploy.rb @@ -20,7 +20,8 @@ module Gitlab def find_deployments deployments = Deployment.joins(:project).merge(Project.inside_path(group.full_path)) deployments = deployments.where(projects: { id: options[:projects] }) if options[:projects] - deployments = deployments.where("deployments.created_at > ?", from) + deployments = deployments.where("deployments.created_at > ?", options[:from]) + deployments = deployments.where("deployments.created_at < ?", options[:to]) if options[:to] deployments.success.count end end diff --git a/lib/gitlab/cycle_analytics/summary/group/issue.rb b/lib/gitlab/cycle_analytics/summary/group/issue.rb index 9daae8531d8..4d5ee1d43ca 100644 --- a/lib/gitlab/cycle_analytics/summary/group/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/group/issue.rb @@ -5,11 +5,10 @@ module Gitlab module Summary module Group class Issue < Group::Base - attr_reader :group, :from, :current_user, :options + attr_reader :group, :current_user, :options - def initialize(group:, from:, current_user:, options:) + def initialize(group:, current_user:, options:) @group = group - @from = from @current_user = current_user @options = options end @@ -25,10 +24,19 @@ module Gitlab private def find_issues - issues = IssuesFinder.new(current_user, group_id: group.id, include_subgroups: true, created_after: from).execute + issues = IssuesFinder.new(current_user, finder_params).execute issues = issues.where(projects: { id: options[:projects] }) if options[:projects] issues.count end + + def finder_params + { + group_id: group.id, + include_subgroups: true, + created_after: options[:from], + created_before: options[:to] + }.compact + end end end end diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index 8a253893892..ddb9d907640 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -28,6 +28,10 @@ module Gitlab true end + def thread_name + self.class.name.demodulize.underscore + end + def start return unless enabled? @@ -35,7 +39,10 @@ module Gitlab break thread if thread? if start_working - @thread = Thread.new { run_thread } + @thread = Thread.new do + Thread.current.name = thread_name + run_thread + end end end end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index f22fc41a6d8..0e7e0c40a8a 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -93,8 +93,8 @@ module Gitlab docs: "~documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now. none: "", qa: "~QA", - test: "~test for `spec/features/*`", - engineering_productivity: "Engineering Productivity for CI config review" + test: "~test ~Quality for `spec/features/*`", + engineering_productivity: '~"Engineering Productivity" for CI, Danger' }.freeze CATEGORIES = { %r{\Adoc/} => :none, # To reinstate roulette for documentation, set to `:docs`. @@ -104,7 +104,7 @@ module Gitlab %r{\A(ee/)?public/} => :frontend, %r{\A(ee/)?spec/(javascripts|frontend)/} => :frontend, %r{\A(ee/)?vendor/assets/} => :frontend, - %r{\Ascripts/frontend/} => :frontend, + %r{\A(ee/)?scripts/frontend/} => :frontend, %r{(\A|/)( \.babelrc | \.eslintignore | @@ -130,14 +130,18 @@ module Gitlab %r{\A(app/models/project_authorization|app/services/users/refresh_authorized_projects_service)(/|\.rb)} => :database, %r{\Arubocop/cop/migration(/|\.rb)} => :database, + %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :engineering_productivity, + %r{Dangerfile\z} => :engineering_productivity, + %r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity, + %r{\A(ee/)?scripts/} => :engineering_productivity, + %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, - %r{\A(ee/)?(bin|config|danger|generator_templates|lib|rubocop|scripts)/} => :backend, + %r{\A(ee/)?(bin|config|generator_templates|lib|rubocop)/} => :backend, %r{\A(ee/)?spec/features/} => :test, %r{\A(ee/)?spec/(?!javascripts|frontend)[^/]+} => :backend, %r{\A(ee/)?vendor/(?!assets)[^/]+} => :backend, %r{\A(ee/)?vendor/(languages\.yml|licenses\.csv)\z} => :backend, - %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :engineering_productivity, - %r{\A(Dangerfile|Gemfile|Gemfile.lock|Procfile|Rakefile)\z} => :backend, + %r{\A(Gemfile|Gemfile.lock|Procfile|Rakefile)\z} => :backend, %r{\A[A-Z_]+_VERSION\z} => :backend, %r{\A\.rubocop(_todo)?\.yml\z} => :backend, diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index 5c2324836d7..e96f5177195 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -67,7 +67,10 @@ module Gitlab area && labels.any?("devops::#{area.downcase}") if kind == :reviewer when :engineering_productivity - role[/Engineering Productivity/] if kind == :reviewer + return false unless role[/Engineering Productivity/] + return true if kind == :reviewer + + capabilities(project).include?("#{kind} backend") else capabilities(project).include?("#{kind} #{category}") end diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index f11e032ab84..70587b3132a 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -6,11 +6,17 @@ module Gitlab extend self def build(deployment) + # Deployments will not have a deployable when created using the API. + deployable_url = + if deployment.deployable + Gitlab::UrlBuilder.build(deployment.deployable) + end + { object_kind: 'deployment', status: deployment.status, deployable_id: deployment.deployable_id, - deployable_url: Gitlab::UrlBuilder.build(deployment.deployable), + deployable_url: deployable_url, environment: deployment.environment.name, project: deployment.project.hook_attrs, short_sha: deployment.short_sha, diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index a83b03f540c..65cfd47e1e8 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -19,12 +19,25 @@ module Gitlab user_email: "john@example.com", user_avatar: "https://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=8://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=80", project_id: 15, + project: { + id: 15, + name: "gitlab", + description: "", + web_url: "http://test.example.com/gitlab/gitlab", + avatar_url: "https://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=8://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=80", + git_ssh_url: "git@test.example.com:gitlab/gitlab.git", + git_http_url: "http://test.example.com/gitlab/gitlab.git", + namespace: "gitlab", + visibility_level: 0, + path_with_namespace: "gitlab/gitlab", + default_branch: "master" + }, commits: [ { id: "c5feabde2d8cd023215af4d2ceeb7a64839fc428", message: "Add simple search to projects in public area", timestamp: "2013-05-13T18:18:08+00:00", - url: "https://test.example.com/gitlab/gitlabhq/commit/c5feabde2d8cd023215af4d2ceeb7a64839fc428", + url: "https://test.example.com/gitlab/gitlab/commit/c5feabde2d8cd023215af4d2ceeb7a64839fc428", author: { name: "Test User", email: "test@example.com" @@ -45,7 +58,20 @@ module Gitlab # user_name: String, # user_username: String, # user_email: String - # project_id: String, + # project_id: Fixnum, + # project: { + # id: Fixnum, + # name: String, + # description: String, + # web_url: String, + # avatar_url: String, + # git_ssh_url: String, + # git_http_url: String, + # namespace: String, + # visibility_level: Fixnum, + # path_with_namespace: String, + # default_branch: String + # } # repository: { # name: String, # url: String, diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index ae29546cdac..7ea7565f758 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -108,9 +108,7 @@ module Gitlab 'in the body of your migration class' end - if supports_drop_index_concurrently? - options = options.merge({ algorithm: :concurrently }) - end + options = options.merge({ algorithm: :concurrently }) unless index_exists?(table_name, column_name, options) Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" # rubocop:disable Gitlab/RailsLogger @@ -136,9 +134,7 @@ module Gitlab 'in the body of your migration class' end - if supports_drop_index_concurrently? - options = options.merge({ algorithm: :concurrently }) - end + options = options.merge({ algorithm: :concurrently }) unless index_exists_by_name?(table_name, index_name) Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, index_name: #{index_name}" # rubocop:disable Gitlab/RailsLogger @@ -150,13 +146,6 @@ module Gitlab end end - # Only available on Postgresql >= 9.2 - def supports_drop_index_concurrently? - version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i - - version >= 90200 - end - # Adds a foreign key with only minimal locking on the tables involved. # # This method only requires minimal locking @@ -966,7 +955,7 @@ into similar problems in the future (e.g. when new tables are created). table_name = model_class.quoted_table_name model_class.each_batch(of: batch_size) do |relation| - start_id, end_id = relation.pluck("MIN(#{table_name}.id), MAX(#{table_name}.id)").first + start_id, end_id = relation.pluck("MIN(#{table_name}.id)", "MAX(#{table_name}.id)").first if jobs.length >= BACKGROUND_MIGRATION_JOB_BUFFER_SIZE # Note: This code path generally only helps with many millions of rows diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 3e8a9b89998..cea25967801 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -66,11 +66,13 @@ module Gitlab def move_repositories(namespace, old_full_path, new_full_path) repo_shards_for_namespace(namespace).each do |repository_storage| # Ensure old directory exists before moving it - gitlab_shell.add_namespace(repository_storage, old_full_path) + Gitlab::GitalyClient::NamespaceService.allow do + gitlab_shell.add_namespace(repository_storage, old_full_path) - unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path) - message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}" - Rails.logger.error message # rubocop:disable Gitlab/RailsLogger + unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path) + message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}" + Rails.logger.error message # rubocop:disable Gitlab/RailsLogger + end end end end diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb index dfef158cc1d..8cd9694b741 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -21,7 +21,6 @@ module Gitlab :create_project, :save_project_id, :add_group_members, - :add_to_whitelist, :add_prometheus_manual_configuration def initialize @@ -126,28 +125,6 @@ module Gitlab end end - def add_to_whitelist(result) - return success(result) unless prometheus_enabled? - return success(result) unless prometheus_listen_address.present? - - uri = parse_url(internal_prometheus_listen_address_uri) - return error(_('Prometheus listen_address in config/gitlab.yml is not a valid URI')) unless uri - - application_settings.add_to_outbound_local_requests_whitelist([uri.normalized_host]) - response = application_settings.save - - if response - # Expire the Gitlab::CurrentSettings cache after updating the whitelist. - # This happens automatically in an after_commit hook, but in migrations, - # the after_commit hook only runs at the end of the migration. - Gitlab::CurrentSettings.expire_current_application_settings - success(result) - else - log_error("Could not add prometheus URL to whitelist, errors: %{errors}" % { errors: application_settings.errors.full_messages }) - error(_('Could not add prometheus URL to whitelist')) - end - end - def add_prometheus_manual_configuration(result) return success(result) unless prometheus_enabled? return success(result) unless prometheus_listen_address.present? @@ -176,19 +153,11 @@ module Gitlab end def prometheus_enabled? - Gitlab.config.prometheus.enable if Gitlab.config.prometheus - rescue Settingslogic::MissingSetting - log_error('prometheus.enable is not present in config/gitlab.yml') - - false + ::Gitlab::Prometheus::Internal.prometheus_enabled? end def prometheus_listen_address - Gitlab.config.prometheus.listen_address.to_s if Gitlab.config.prometheus - rescue Settingslogic::MissingSetting - log_error('Prometheus listen_address is not present in config/gitlab.yml') - - nil + ::Gitlab::Prometheus::Internal.listen_address end def instance_admins @@ -231,23 +200,7 @@ module Gitlab end def internal_prometheus_listen_address_uri - if prometheus_listen_address.starts_with?('0.0.0.0:') - # 0.0.0.0:9090 - port = ':' + prometheus_listen_address.split(':').second - 'http://localhost' + port - - elsif prometheus_listen_address.starts_with?(':') - # :9090 - 'http://localhost' + prometheus_listen_address - - elsif prometheus_listen_address.starts_with?('http') - # https://localhost:9090 - prometheus_listen_address - - else - # localhost:9090 - 'http://' + prometheus_listen_address - end + ::Gitlab::Prometheus::Internal.uri end def prometheus_service_attributes diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb index 4d27b706e1e..59a7c4a6660 100644 --- a/lib/gitlab/devise_failure.rb +++ b/lib/gitlab/devise_failure.rb @@ -2,6 +2,8 @@ module Gitlab class DeviseFailure < Devise::FailureApp + include ::SessionsHelper + # If the request format is not known, send a redirect instead of a 401 # response, since this is the outcome we're most likely to want def http_auth? @@ -9,5 +11,11 @@ module Gitlab request_format && super end + + def respond + limit_session_time + + super + end end end diff --git a/lib/gitlab/error_tracking/detailed_error.rb b/lib/gitlab/error_tracking/detailed_error.rb new file mode 100644 index 00000000000..225280a42f4 --- /dev/null +++ b/lib/gitlab/error_tracking/detailed_error.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class DetailedError + include ActiveModel::Model + + attr_accessor :count, + :culprit, + :external_base_url, + :external_url, + :first_release_last_commit, + :first_release_short_version, + :first_seen, + :frequency, + :id, + :last_release_last_commit, + :last_release_short_version, + :last_seen, + :message, + :project_id, + :project_name, + :project_slug, + :short_id, + :status, + :title, + :type, + :user_count + end + end +end diff --git a/lib/gitlab/error_tracking/error_event.rb b/lib/gitlab/error_tracking/error_event.rb new file mode 100644 index 00000000000..c6e0d82f868 --- /dev/null +++ b/lib/gitlab/error_tracking/error_event.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class ErrorEvent + include ActiveModel::Model + + attr_accessor :issue_id, :date_received, :stack_trace_entries + end + end +end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 3d14a8dde8d..efddda0ec65 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -3,8 +3,6 @@ module Gitlab module EtagCaching class Router - prepend_if_ee('EE::Gitlab::EtagCaching::Router') # rubocop: disable Cop/InjectEnterpriseEditionModule - Route = Struct.new(:regexp, :name) # We enable an ETag for every request matching the regex. # To match a regex the path needs to match the following: @@ -80,3 +78,5 @@ module Gitlab end end end + +Gitlab::EtagCaching::Router.prepend_if_ee('EE::Gitlab::EtagCaching::Router') diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 895755376ee..948f720b01b 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -14,13 +14,15 @@ module Gitlab signup_flow: { feature_toggle: :experimental_separate_sign_up_flow, environment: ::Gitlab.dev_env_or_com?, - enabled_ratio: 0.1 + enabled_ratio: 0.1, + tracking_category: 'Growth::Acquisition::Experiment::SignUpFlow' } }.freeze # Controller concern that checks if an experimentation_subject_id cookie is present and sets it if absent. # Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method - # to controllers and views. + # to controllers and views. It returns true when the experiment is enabled and the user is selected as part + # of the experimental group. # module ControllerConcern extend ActiveSupport::Concern @@ -36,22 +38,67 @@ module Gitlab cookies.permanent.signed[:experimentation_subject_id] = { value: SecureRandom.uuid, domain: :all, - secure: ::Gitlab.config.gitlab.https + secure: ::Gitlab.config.gitlab.https, + httponly: true } end def experiment_enabled?(experiment_key) - Experimentation.enabled?(experiment_key, experimentation_subject_index) + Experimentation.enabled_for_user?(experiment_key, experimentation_subject_index) || forced_enabled?(experiment_key) + end + + def track_experiment_event(experiment_key, action) + track_experiment_event_for(experiment_key, action) do |tracking_data| + ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), tracking_data) + end + end + + def frontend_experimentation_tracking_data(experiment_key, action) + track_experiment_event_for(experiment_key, action) do |tracking_data| + gon.push(tracking_data: tracking_data) + end end private + def experimentation_subject_id + cookies.signed[:experimentation_subject_id] + end + def experimentation_subject_index - experimentation_subject_id = cookies.signed[:experimentation_subject_id] return if experimentation_subject_id.blank? experimentation_subject_id.delete('-').hex % 100 end + + def track_experiment_event_for(experiment_key, action) + return unless Experimentation.enabled?(experiment_key) + + yield experimentation_tracking_data(experiment_key, action) + end + + def experimentation_tracking_data(experiment_key, action) + { + category: tracking_category(experiment_key), + action: action, + property: tracking_group(experiment_key), + label: experimentation_subject_id + } + end + + def tracking_category(experiment_key) + Experimentation.experiment(experiment_key).tracking_category + end + + def tracking_group(experiment_key) + return unless Experimentation.enabled?(experiment_key) + + experiment_enabled?(experiment_key) ? 'experimental_group' : 'control_group' + end + + def forced_enabled?(experiment_key) + params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s + end end class << self @@ -59,18 +106,20 @@ module Gitlab Experiment.new(EXPERIMENTS[key].merge(key: key)) end - def enabled?(experiment_key, experimentation_subject_index) + def enabled?(experiment_key) return false unless EXPERIMENTS.key?(experiment_key) experiment = experiment(experiment_key) + experiment.feature_toggle_enabled? && experiment.enabled_for_environment? + end - experiment.feature_toggle_enabled? && - experiment.enabled_for_environment? && - experiment.enabled_for_experimentation_subject?(experimentation_subject_index) + def enabled_for_user?(experiment_key, experimentation_subject_index) + enabled?(experiment_key) && + experiment(experiment_key).enabled_for_experimentation_subject?(experimentation_subject_index) end end - Experiment = Struct.new(:key, :feature_toggle, :environment, :enabled_ratio, keyword_init: true) do + Experiment = Struct.new(:key, :feature_toggle, :environment, :enabled_ratio, :tracking_category, keyword_init: true) do def feature_toggle_enabled? return Feature.enabled?(key, default_enabled: true) if feature_toggle.nil? diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index b5d308e462c..ce1370bab0f 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -7,7 +7,7 @@ module Gitlab image_name = if appearance.favicon.exists? appearance.favicon_path - elsif Gitlab::Utils.to_boolean(ENV['CANARY']) + elsif Gitlab.canary? 'favicon-yellow.png' elsif Rails.env.development? development_favicon diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index 3958814208c..ec9d2df613b 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -15,12 +15,12 @@ module Gitlab def find(query) query = Gitlab::Search::Query.new(query, encode_binary: true) do - filter :filename, matcher: ->(filter, blob) { blob.binary_filename =~ /#{filter[:regex_value]}$/i } - filter :path, matcher: ->(filter, blob) { blob.binary_filename =~ /#{filter[:regex_value]}/i } - filter :extension, matcher: ->(filter, blob) { blob.binary_filename =~ /\.#{filter[:regex_value]}$/i } + filter :filename, matcher: ->(filter, blob) { blob.binary_path =~ /#{filter[:regex_value]}$/i } + filter :path, matcher: ->(filter, blob) { blob.binary_path =~ /#{filter[:regex_value]}/i } + filter :extension, matcher: ->(filter, blob) { blob.binary_path =~ /\.#{filter[:regex_value]}$/i } end - files = find_by_filename(query.term) + find_by_content(query.term) + files = find_by_path(query.term) + find_by_content(query.term) files = query.filter_results(files) if query.filters.any? @@ -35,13 +35,14 @@ module Gitlab end end - def find_by_filename(query) - search_filenames(query).map do |filename| - Gitlab::Search::FoundBlob.new(blob_filename: filename, project: project, ref: ref, repository: repository) + def find_by_path(query) + search_paths(query).map do |path| + Gitlab::Search::FoundBlob.new(blob_path: path, project: project, ref: ref, repository: repository) end end - def search_filenames(query) + # Overriden in Gitlab::WikiFileFinder + def search_paths(query) repository.search_files_by_name(query, ref) end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 8fac3621df9..6210223917b 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -155,10 +155,6 @@ module Gitlab end end - def extract_signature(repository, commit_id) - repository.gitaly_commit_client.extract_signature(commit_id) - end - def extract_signature_lazily(repository, commit_id) BatchLoader.for(commit_id).batch(key: repository) do |commit_ids, loader, args| batch_signature_extraction(args[:key], commit_ids).each do |commit_id, signature_data| diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index b2c22898079..4971a18e270 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -25,9 +25,18 @@ module Gitlab InvalidRef = Class.new(StandardError) GitError = Class.new(StandardError) DeleteBranchError = Class.new(StandardError) - CreateTreeError = Class.new(StandardError) TagExistsError = Class.new(StandardError) ChecksumError = Class.new(StandardError) + class CreateTreeError < StandardError + attr_reader :error_code + + def initialize(error_code) + super(self.class.name) + + # The value coming from Gitaly is an uppercase String (e.g., "EMPTY") + @error_code = error_code.downcase.to_sym + end + end # Directory name of repo attr_reader :name diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index c1bcd8e934a..3025fc6bfdb 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -133,14 +133,6 @@ module Gitlab GollumSlug.generate(title, format) end - def page_formatted_data(title:, dir: nil, version: nil) - version = version&.id - - wrapped_gitaly_errors do - gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version) - end - end - private def gitaly_wiki_client diff --git a/lib/gitlab/git_access_result/custom_action.rb b/lib/gitlab/git_access_result/custom_action.rb index a05a4baed82..336f3405f72 100644 --- a/lib/gitlab/git_access_result/custom_action.rb +++ b/lib/gitlab/git_access_result/custom_action.rb @@ -3,7 +3,7 @@ module Gitlab module GitAccessResult class CustomAction - attr_reader :payload, :message + attr_reader :payload, :console_messages # Example of payload: # @@ -16,9 +16,9 @@ module Gitlab # } # } # - def initialize(payload, message) + def initialize(payload, console_messages) @payload = payload - @message = message + @console_messages = console_messages end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index be695e7e91a..5b47853b9c1 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -142,18 +142,39 @@ module Gitlab # kwargs.merge(deadline: Time.now + 10) # end # - def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout) - start = Gitlab::Metrics::System.monotonic_time - request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {} + def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block) + self.measure_timings(service, rpc, request) do + self.execute(storage, service, rpc, request, remote_storage: remote_storage, timeout: timeout, &block) + end + end + # This method is like GitalyClient.call but should be used with + # Gitaly streaming RPCs. It measures how long the the RPC took to + # produce the full response, not just the initial response. + def self.streaming_call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout) + self.measure_timings(service, rpc, request) do + response = self.execute(storage, service, rpc, request, remote_storage: remote_storage, timeout: timeout) + + yield(response) + end + end + + def self.execute(storage, service, rpc, request, remote_storage:, timeout:) enforce_gitaly_request_limits(:call) kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage) kwargs = yield(kwargs) if block_given? stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend + end + + def self.measure_timings(service, rpc, request) + start = Gitlab::Metrics::System.monotonic_time + + yield ensure duration = Gitlab::Metrics::System.monotonic_time - start + request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {} # Keep track, separately, for the performance bar self.query_time += duration diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index b0559729ff3..15318bc817a 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -200,8 +200,9 @@ module Gitlab to: to ) - response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request, timeout: GitalyClient.medium_timeout) - consume_commits_response(response) + GitalyClient.streaming_call(@repository.storage, :commit_service, :commits_between, request, timeout: GitalyClient.medium_timeout) do |response| + consume_commits_response(response) + end end def diff_stats(left_commit_sha, right_commit_sha) @@ -224,8 +225,9 @@ module Gitlab ) request.order = opts[:order].upcase if opts[:order].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout) - consume_commits_response(response) + GitalyClient.streaming_call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout) do |response| + consume_commits_response(response) + end end def list_commits_by_oid(oids) @@ -233,8 +235,9 @@ module Gitlab request = Gitaly::ListCommitsByOidRequest.new(repository: @gitaly_repo, oid: oids) - response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) - consume_commits_response(response) + GitalyClient.streaming_call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) do |response| + consume_commits_response(response) + end rescue GRPC::NotFound # If no repository is found, happens mainly during testing [] end @@ -249,8 +252,9 @@ module Gitlab offset: offset.to_i ) - response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout) - consume_commits_response(response) + GitalyClient.streaming_call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout) do |response| + consume_commits_response(response) + end end def languages(ref = nil) @@ -323,9 +327,9 @@ module Gitlab request.paths = encode_repeated(Array(options[:path])) if options[:path].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) - - consume_commits_response(response) + GitalyClient.streaming_call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) do |response| + consume_commits_response(response) + end end def filter_shas_with_signatures(shas) @@ -348,25 +352,6 @@ module Gitlab end end - def extract_signature(commit_id) - request = Gitaly::ExtractCommitSignatureRequest.new(repository: @gitaly_repo, commit_id: commit_id) - response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request, timeout: GitalyClient.fast_timeout) - - signature = +''.b - signed_text = +''.b - - response.each do |message| - signature << message.signature - signed_text << message.signed_text - end - - return if signature.blank? && signed_text.blank? - - [signature, signed_text] - rescue GRPC::InvalidArgument => ex - raise ArgumentError, ex - end - def get_commit_signatures(commit_ids) request = Gitaly::GetCommitSignaturesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request, timeout: GitalyClient.fast_timeout) diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb index 0be214f3035..dbcebec3aa2 100644 --- a/lib/gitlab/gitaly_client/namespace_service.rb +++ b/lib/gitlab/gitaly_client/namespace_service.rb @@ -3,14 +3,23 @@ module Gitlab module GitalyClient class NamespaceService - def initialize(storage) - @storage = storage + extend Gitlab::TemporarilyAllow + + NamespaceServiceAccessError = Class.new(StandardError) + ALLOW_KEY = :allow_namespace + + def self.allow + temporarily_allow(ALLOW_KEY) { yield } end - def exists?(name) - request = Gitaly::NamespaceExistsRequest.new(storage_name: @storage, name: name) + def self.denied? + !temporarily_allowed?(ALLOW_KEY) + end + + def initialize(storage) + raise NamespaceServiceAccessError if self.class.denied? - gitaly_client_call(:namespace_exists, request, timeout: GitalyClient.fast_timeout).exists + @storage = storage end def add(name) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 6e486c763da..61c5db4c4df 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -447,7 +447,7 @@ module Gitlab elsif response.commit_error.presence raise Gitlab::Git::CommitError, response.commit_error elsif response.create_tree_error.presence - raise Gitlab::Git::Repository::CreateTreeError, response.create_tree_error + raise Gitlab::Git::Repository::CreateTreeError, response.create_tree_error_code end Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 15e0d7349dd..9034edb6263 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -179,18 +179,6 @@ module Gitlab wiki_file end - def get_formatted_data(title:, dir: nil, version: nil) - request = Gitaly::WikiGetFormattedDataRequest.new( - repository: @gitaly_repo, - title: encode_binary(title), - revision: encode_binary(version), - directory: encode_binary(dir) - ) - - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request, timeout: GitalyClient.medium_timeout) - response.reduce([]) { |memo, msg| memo << msg.data }.join - end - private # If a block is given and the yielded value is truthy, iteration will be diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index f1e31a615a4..2616a19fdaa 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -42,9 +42,6 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend push_frontend_feature_flag(:suppress_ajax_navigation_errors, default_enabled: true) - - # Flag controls a GFM feature used across many routes. - push_frontend_feature_flag(:gfm_grafana_integration) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 32f61b1d65c..1dce26efc65 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -4,6 +4,10 @@ module Gitlab module Gpg extend self + CleanupError = Class.new(StandardError) + BG_CLEANUP_RUNTIME_S = 2 + FG_CLEANUP_RUNTIME_S = 0.5 + MUTEX = Mutex.new module CurrentKeyChain @@ -94,16 +98,55 @@ module Gitlab previous_dir = current_home_dir tmp_dir = Dir.mktmpdir GPGME::Engine.home_dir = tmp_dir + tmp_keychains_created.increment + yield ensure - # Ignore any errors when removing the tmp directory, as we may run into a + GPGME::Engine.home_dir = previous_dir + + begin + cleanup_tmp_dir(tmp_dir) + rescue CleanupError => e + # This means we left a GPG-agent process hanging. Logging the problem in + # sentry will make this more visible. + Gitlab::Sentry.track_exception(e, + issue_url: 'https://gitlab.com/gitlab-org/gitlab/issues/20918', + extra: { tmp_dir: tmp_dir }) + end + + tmp_keychains_removed.increment unless File.exist?(tmp_dir) + end + + def cleanup_tmp_dir(tmp_dir) + return FileUtils.remove_entry(tmp_dir, true) if Feature.disabled?(:gpg_cleanup_retries) + + # Retry when removing the tmp directory failed, as we may run into a # race condition: # The `gpg-agent` agent process may clean up some files as well while # `FileUtils.remove_entry` is iterating the directory and removing all # its contained files and directories recursively, which could raise an # error. - FileUtils.remove_entry(tmp_dir, true) - GPGME::Engine.home_dir = previous_dir + # Failing to remove the tmp directory could leave the `gpg-agent` process + # running forever. + Retriable.retriable(max_elapsed_time: cleanup_time, base_interval: 0.1) do + FileUtils.remove_entry(tmp_dir) if File.exist?(tmp_dir) + end + rescue => e + raise CleanupError, e + end + + def cleanup_time + Sidekiq.server? ? BG_CLEANUP_RUNTIME_S : FG_CLEANUP_RUNTIME_S + end + + def tmp_keychains_created + @tmp_keychains_created ||= Gitlab::Metrics.counter(:gpg_tmp_keychains_created_total, + 'The number of temporary GPG keychains created') + end + + def tmp_keychains_removed + @tmp_keychains_removed ||= Gitlab::Metrics.counter(:gpg_tmp_keychains_removed_total, + 'The number of temporary GPG keychains removed') end end end diff --git a/lib/gitlab/grape_logging/loggers/exception_logger.rb b/lib/gitlab/grape_logging/loggers/exception_logger.rb new file mode 100644 index 00000000000..022eb15d28d --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/exception_logger.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module GrapeLogging + module Loggers + class ExceptionLogger < ::GrapeLogging::Loggers::Base + def parameters(request, _) + # grape-logging attempts to pass the logger the exception + # (https://github.com/aserafin/grape_logging/blob/v1.7.0/lib/grape_logging/middleware/request_logger.rb#L63), + # but it appears that the rescue_all in api.rb takes + # precedence so the logger never sees it. We need to + # store and retrieve the exception from the environment. + exception = request.env[::API::Helpers::API_EXCEPTION_ENV] + + return {} unless exception.is_a?(Exception) + + data = { + exception: { + class: exception.class.to_s, + message: exception.message + } + } + + if exception.backtrace + data[:exception][:backtrace] = Gitlab::Profiler.clean_backtrace(exception.backtrace) + end + + data + end + end + end + end +end diff --git a/lib/gitlab/graphql/authorize/instrumentation.rb b/lib/gitlab/graphql/authorize/instrumentation.rb index 15ecc3b04f0..f9ff2b30eae 100644 --- a/lib/gitlab/graphql/authorize/instrumentation.rb +++ b/lib/gitlab/graphql/authorize/instrumentation.rb @@ -9,12 +9,16 @@ module Gitlab def instrument(_type, field) service = AuthorizeFieldService.new(field) - if service.authorizations? + if service.authorizations? && !resolver_skips_authorizations?(field) field.redefine { resolve(service.authorized_resolve) } else field end end + + def resolver_skips_authorizations?(field) + field.metadata[:resolver].try(:skip_authorizations?) + end end end end diff --git a/lib/gitlab/graphql/connections.rb b/lib/gitlab/graphql/connections.rb index fbccdfa7b08..38c7d98f37c 100644 --- a/lib/gitlab/graphql/connections.rb +++ b/lib/gitlab/graphql/connections.rb @@ -6,7 +6,11 @@ module Gitlab def self.use(_schema) GraphQL::Relay::BaseConnection.register_connection_implementation( ActiveRecord::Relation, - Gitlab::Graphql::Connections::KeysetConnection + Gitlab::Graphql::Connections::Keyset::Connection + ) + GraphQL::Relay::BaseConnection.register_connection_implementation( + Gitlab::Graphql::FilterableArray, + Gitlab::Graphql::Connections::FilterableArrayConnection ) end end diff --git a/lib/gitlab/graphql/connections/filterable_array_connection.rb b/lib/gitlab/graphql/connections/filterable_array_connection.rb new file mode 100644 index 00000000000..800f2c949c6 --- /dev/null +++ b/lib/gitlab/graphql/connections/filterable_array_connection.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Connections + # FilterableArrayConnection is useful especially for lazy-loaded values. + # It allows us to call a callback only on the slice of array being + # rendered in the "after loaded" phase. For example we can check + # permissions only on a small subset of items. + class FilterableArrayConnection < GraphQL::Relay::ArrayConnection + def paged_nodes + @filtered_nodes ||= nodes.filter_callback.call(super) + end + end + end + end +end diff --git a/lib/gitlab/graphql/connections/keyset/conditions/base_condition.rb b/lib/gitlab/graphql/connections/keyset/conditions/base_condition.rb new file mode 100644 index 00000000000..22728cc0b65 --- /dev/null +++ b/lib/gitlab/graphql/connections/keyset/conditions/base_condition.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Connections + module Keyset + module Conditions + class BaseCondition + def initialize(arel_table, names, values, operator, before_or_after) + @arel_table, @names, @values, @operator, @before_or_after = arel_table, names, values, operator, before_or_after + end + + def build + raise NotImplementedError + end + + private + + attr_reader :arel_table, :names, :values, :operator, :before_or_after + + def table_condition(attribute, value, operator) + case operator + when '>' + arel_table[attribute].gt(value) + when '<' + arel_table[attribute].lt(value) + when '=' + arel_table[attribute].eq(value) + when 'is_null' + arel_table[attribute].eq(nil) + when 'is_not_null' + arel_table[attribute].not_eq(nil) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/connections/keyset/conditions/not_null_condition.rb b/lib/gitlab/graphql/connections/keyset/conditions/not_null_condition.rb new file mode 100644 index 00000000000..3b56ddb996d --- /dev/null +++ b/lib/gitlab/graphql/connections/keyset/conditions/not_null_condition.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Connections + module Keyset + module Conditions + class NotNullCondition < BaseCondition + def build + conditions = [first_attribute_condition] + + # If there is only one order field, we can assume it + # does not contain NULLs, and don't need additional + # conditions + unless names.count == 1 + conditions << [second_attribute_condition, final_condition] + end + + conditions.join + end + + private + + # ex: "(relative_position > 23)" + def first_attribute_condition + <<~SQL + (#{table_condition(names.first, values.first, operator.first).to_sql}) + SQL + end + + # ex: " OR (relative_position = 23 AND id > 500)" + def second_attribute_condition + condition = <<~SQL + OR ( + #{table_condition(names.first, values.first, '=').to_sql} + AND + #{table_condition(names[1], values[1], operator[1]).to_sql} + ) + SQL + + condition + end + + # ex: " OR (relative_position IS NULL)" + def final_condition + if before_or_after == :after + <<~SQL + OR (#{table_condition(names.first, nil, 'is_null').to_sql}) + SQL + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/connections/keyset/conditions/null_condition.rb b/lib/gitlab/graphql/connections/keyset/conditions/null_condition.rb new file mode 100644 index 00000000000..71a74936d5d --- /dev/null +++ b/lib/gitlab/graphql/connections/keyset/conditions/null_condition.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Connections + module Keyset + module Conditions + class NullCondition < BaseCondition + def build + [first_attribute_condition, final_condition].join + end + + private + + # ex: "(relative_position IS NULL AND id > 500)" + def first_attribute_condition + condition = <<~SQL + ( + #{table_condition(names.first, nil, 'is_null').to_sql} + AND + #{table_condition(names[1], values[1], operator[1]).to_sql} + ) + SQL + + condition + end + + # ex: " OR (relative_position IS NOT NULL)" + def final_condition + if before_or_after == :before + <<~SQL + OR (#{table_condition(names.first, nil, 'is_not_null').to_sql}) + SQL + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/connections/keyset/connection.rb b/lib/gitlab/graphql/connections/keyset/connection.rb new file mode 100644 index 00000000000..c75ea206edb --- /dev/null +++ b/lib/gitlab/graphql/connections/keyset/connection.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +# Keyset::Connection provides cursor based pagination, to avoid using OFFSET. +# It basically sorts / filters using WHERE sorting_value > cursor. +# We do this for performance reasons (https://gitlab.com/gitlab-org/gitlab-foss/issues/45756), +# as well as for having stable pagination +# https://graphql-ruby.org/pro/cursors.html#whats-the-difference +# https://coderwall.com/p/lkcaag/pagination-you-re-probably-doing-it-wrong +# +# It currently supports sorting on two columns, but the last column must +# be the primary key. If it's not already included, an order on the +# primary key will be added automatically, like `order(id: :desc)` +# +# Issue.order(created_at: :asc).order(:id) +# Issue.order(due_date: :asc) +# +# You can also use `Gitlab::Database.nulls_last_order`: +# +# Issue.reorder(::Gitlab::Database.nulls_last_order('due_date', 'DESC')) +# +# It will tolerate non-attribute ordering, but only attributes determine the cursor. +# For example, this is legitimate: +# +# Issue.order('issues.due_date IS NULL').order(due_date: :asc).order(:id) +# +# but anything more complex has a chance of not working. +# +module Gitlab + module Graphql + module Connections + module Keyset + class Connection < GraphQL::Relay::BaseConnection + include Gitlab::Utils::StrongMemoize + + # TODO https://gitlab.com/gitlab-org/gitlab/issues/35104 + include Gitlab::Graphql::Connections::Keyset::LegacyKeysetConnection + + def cursor_from_node(node) + return legacy_cursor_from_node(node) if use_legacy_pagination? + + encoded_json_from_ordering(node) + end + + def sliced_nodes + return legacy_sliced_nodes if use_legacy_pagination? + + @sliced_nodes ||= + begin + OrderInfo.validate_ordering(ordered_nodes, order_list) + + sliced = ordered_nodes + sliced = slice_nodes(sliced, before, :before) if before.present? + sliced = slice_nodes(sliced, after, :after) if after.present? + + sliced + end + end + + def paged_nodes + # These are the nodes that will be loaded into memory for rendering + # So we're ok loading them into memory here as that's bound to happen + # anyway. Having them ready means we can modify the result while + # rendering the fields. + @paged_nodes ||= load_paged_nodes.to_a + end + + private + + def load_paged_nodes + if first && last + raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both") + end + + if last + sliced_nodes.last(limit_value) + else + sliced_nodes.limit(limit_value) # rubocop: disable CodeReuse/ActiveRecord + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def slice_nodes(sliced, encoded_cursor, before_or_after) + decoded_cursor = ordering_from_encoded_json(encoded_cursor) + builder = QueryBuilder.new(arel_table, order_list, decoded_cursor, before_or_after) + ordering = builder.conditions + + sliced.where(*ordering).where.not(id: decoded_cursor['id']) + end + # rubocop: enable CodeReuse/ActiveRecord + + def limit_value + @limit_value ||= [first, last, max_page_size].compact.min + end + + def ordered_nodes + strong_memoize(:order_nodes) do + unless nodes.primary_key.present? + raise ArgumentError.new('Relation must have a primary key') + end + + list = OrderInfo.build_order_list(nodes) + + # ensure there is a primary key ordering + if list&.last&.attribute_name != nodes.primary_key + nodes.order(arel_table[nodes.primary_key].desc) # rubocop: disable CodeReuse/ActiveRecord + else + nodes + end + end + end + + def order_list + strong_memoize(:order_list) do + OrderInfo.build_order_list(ordered_nodes) + end + end + + def arel_table + nodes.arel_table + end + + # Storing the current order values in the cursor allows us to + # make an intelligent decision on handling NULL values. + # Otherwise we would either need to fetch the record first, + # or fetch it in the SQL, significantly complicating it. + def encoded_json_from_ordering(node) + ordering = { 'id' => node[:id].to_s } + + order_list.each do |field| + field_name = field.attribute_name + ordering[field_name] = node[field_name].to_s + end + + encode(ordering.to_json) + end + + def ordering_from_encoded_json(cursor) + JSON.parse(decode(cursor)) + rescue JSON::ParserError + # for the transition period where a client might request using an + # old style cursor. Once removed, make it an error: + # raise Gitlab::Graphql::Errors::ArgumentError, "Please provide a valid cursor" + # TODO can be removed in next release + # https://gitlab.com/gitlab-org/gitlab/issues/32933 + field_name = order_list.first.attribute_name + + { field_name => decode(cursor) } + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/connections/keyset/legacy_keyset_connection.rb b/lib/gitlab/graphql/connections/keyset/legacy_keyset_connection.rb new file mode 100644 index 00000000000..baf900d1048 --- /dev/null +++ b/lib/gitlab/graphql/connections/keyset/legacy_keyset_connection.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +# TODO https://gitlab.com/gitlab-org/gitlab/issues/35104 +module Gitlab + module Graphql + module Connections + module Keyset + module LegacyKeysetConnection + def legacy_cursor_from_node(node) + encode(node[legacy_order_field].to_s) + end + + # rubocop: disable CodeReuse/ActiveRecord + def legacy_sliced_nodes + @sliced_nodes ||= + begin + sliced = nodes + + sliced = sliced.where(legacy_before_slice) if before.present? + sliced = sliced.where(legacy_after_slice) if after.present? + + sliced + end + end + # rubocop: enable CodeReuse/ActiveRecord + + private + + def use_legacy_pagination? + strong_memoize(:feature_disabled) do + Feature.disabled?(:graphql_keyset_pagination, default_enabled: true) + end + end + + def legacy_before_slice + if legacy_sort_direction == :asc + arel_table[legacy_order_field].lt(decode(before)) + else + arel_table[legacy_order_field].gt(decode(before)) + end + end + + def legacy_after_slice + if legacy_sort_direction == :asc + arel_table[legacy_order_field].gt(decode(after)) + else + arel_table[legacy_order_field].lt(decode(after)) + end + end + + def legacy_order_info + @legacy_order_info ||= nodes.order_values.first + end + + def legacy_order_field + @legacy_order_field ||= legacy_order_info&.expr&.name || nodes.primary_key + end + + def legacy_sort_direction + @legacy_order_direction ||= legacy_order_info&.direction || :desc + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/connections/keyset/order_info.rb b/lib/gitlab/graphql/connections/keyset/order_info.rb new file mode 100644 index 00000000000..4d85e8f79b7 --- /dev/null +++ b/lib/gitlab/graphql/connections/keyset/order_info.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Connections + module Keyset + class OrderInfo + attr_reader :attribute_name, :sort_direction + + def initialize(order_value) + if order_value.is_a?(String) + @attribute_name, @sort_direction = extract_nulls_last_order(order_value) + else + @attribute_name = order_value.expr.name + @sort_direction = order_value.direction + end + end + + def operator_for(before_or_after) + case before_or_after + when :before + sort_direction == :asc ? '<' : '>' + when :after + sort_direction == :asc ? '>' : '<' + end + end + + # Only allow specific node types + def self.build_order_list(relation) + order_list = relation.order_values.select do |value| + supported_order_value?(value) + end + + order_list.map { |info| OrderInfo.new(info) } + end + + def self.validate_ordering(relation, order_list) + if order_list.empty? + raise ArgumentError.new('A minimum of 1 ordering field is required') + end + + if order_list.count > 2 + raise ArgumentError.new('A maximum of 2 ordering fields are allowed') + end + + # make sure the last ordering field is non-nullable + attribute_name = order_list.last&.attribute_name + + if relation.columns_hash[attribute_name].null + raise ArgumentError.new("Column `#{attribute_name}` must not allow NULL") + end + + if order_list.last.attribute_name != relation.primary_key + raise ArgumentError.new("Last ordering field must be the primary key, `#{relation.primary_key}`") + end + end + + def self.supported_order_value?(order_value) + return true if order_value.is_a?(Arel::Nodes::Ascending) || order_value.is_a?(Arel::Nodes::Descending) + return false unless order_value.is_a?(String) + + tokens = order_value.downcase.split + + tokens.last(2) == %w(nulls last) && tokens.count == 4 + end + + private + + def extract_nulls_last_order(order_value) + tokens = order_value.downcase.split + + [tokens.first, (tokens[1] == 'asc' ? :asc : :desc)] + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/connections/keyset/query_builder.rb b/lib/gitlab/graphql/connections/keyset/query_builder.rb new file mode 100644 index 00000000000..e93c25d85fc --- /dev/null +++ b/lib/gitlab/graphql/connections/keyset/query_builder.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Connections + module Keyset + class QueryBuilder + def initialize(arel_table, order_list, decoded_cursor, before_or_after) + @arel_table, @order_list, @decoded_cursor, @before_or_after = arel_table, order_list, decoded_cursor, before_or_after + + if order_list.empty? + raise ArgumentError.new('No ordering scopes have been supplied') + end + end + + # Based on whether the main field we're ordering on is NULL in the + # cursor, we can more easily target our query condition. + # We assume that the last ordering field is unique, meaning + # it will not contain NULLs. + # We currently only support two ordering fields. + # + # Example of the conditions for + # relation: Issue.order(relative_position: :asc).order(id: :asc) + # after cursor: relative_position: 1500, id: 500 + # + # when cursor[relative_position] is not NULL + # + # ("issues"."relative_position" > 1500) + # OR ( + # "issues"."relative_position" = 1500 + # AND + # "issues"."id" > 500 + # ) + # OR ("issues"."relative_position" IS NULL) + # + # when cursor[relative_position] is NULL + # + # "issues"."relative_position" IS NULL + # AND + # "issues"."id" > 500 + # + def conditions + attr_names = order_list.map { |field| field.attribute_name } + attr_values = attr_names.map { |name| decoded_cursor[name] } + + if attr_names.count == 1 && attr_values.first.nil? + raise Gitlab::Graphql::Errors::ArgumentError.new('Before/after cursor invalid: `nil` was provided as only sortable value') + end + + if attr_names.count == 1 || attr_values.first.present? + Keyset::Conditions::NotNullCondition.new(arel_table, attr_names, attr_values, operators, before_or_after).build + else + Keyset::Conditions::NullCondition.new(arel_table, attr_names, attr_values, operators, before_or_after).build + end + end + + private + + attr_reader :arel_table, :order_list, :decoded_cursor, :before_or_after + + def operators + order_list.map { |field| field.operator_for(before_or_after) } + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/connections/keyset_connection.rb b/lib/gitlab/graphql/connections/keyset_connection.rb deleted file mode 100644 index 715963a44c1..00000000000 --- a/lib/gitlab/graphql/connections/keyset_connection.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module Connections - class KeysetConnection < GraphQL::Relay::BaseConnection - def cursor_from_node(node) - encode(node[order_field].to_s) - end - - # rubocop: disable CodeReuse/ActiveRecord - def sliced_nodes - @sliced_nodes ||= - begin - sliced = nodes - - sliced = sliced.where(before_slice) if before.present? - sliced = sliced.where(after_slice) if after.present? - - sliced - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def paged_nodes - # These are the nodes that will be loaded into memory for rendering - # So we're ok loading them into memory here as that's bound to happen - # anyway. Having them ready means we can modify the result while - # rendering the fields. - @paged_nodes ||= load_paged_nodes.to_a - end - - private - - def load_paged_nodes - if first && last - raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both") - end - - if last - sliced_nodes.last(limit_value) - else - sliced_nodes.limit(limit_value) # rubocop: disable CodeReuse/ActiveRecord - end - end - - def before_slice - if sort_direction == :asc - table[order_field].lt(decode(before)) - else - table[order_field].gt(decode(before)) - end - end - - def after_slice - if sort_direction == :asc - table[order_field].gt(decode(after)) - else - table[order_field].lt(decode(after)) - end - end - - def limit_value - @limit_value ||= [first, last, max_page_size].compact.min - end - - def table - nodes.arel_table - end - - def order_info - @order_info ||= nodes.order_values.first - end - - def order_field - @order_field ||= order_info&.expr&.name || nodes.primary_key - end - - def sort_direction - @order_direction ||= order_info&.direction || :desc - end - end - end - end -end diff --git a/lib/gitlab/graphql/filterable_array.rb b/lib/gitlab/graphql/filterable_array.rb new file mode 100644 index 00000000000..4909d291fd6 --- /dev/null +++ b/lib/gitlab/graphql/filterable_array.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + class FilterableArray < Array + attr_reader :filter_callback + + def initialize(filter_callback, *args) + super(args) + @filter_callback = filter_callback + end + end + end +end diff --git a/lib/gitlab/graphql/loaders/pipeline_for_sha_loader.rb b/lib/gitlab/graphql/loaders/pipeline_for_sha_loader.rb deleted file mode 100644 index 70344392138..00000000000 --- a/lib/gitlab/graphql/loaders/pipeline_for_sha_loader.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module Loaders - class PipelineForShaLoader - attr_accessor :project, :sha - - def initialize(project, sha) - @project, @sha = project, sha - end - - def find_last - BatchLoader::GraphQL.for(sha).batch(key: project) do |shas, loader, args| - pipelines = args[:key].ci_pipelines.latest_for_shas(shas) - - pipelines.each do |pipeline| - loader.call(pipeline.sha, pipeline) - end - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/master_check.rb b/lib/gitlab/health_checks/master_check.rb new file mode 100644 index 00000000000..057bce84ddd --- /dev/null +++ b/lib/gitlab/health_checks/master_check.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + # This check is registered on master, + # and validated by worker + class MasterCheck + extend SimpleAbstractCheck + + class << self + def register_master + # when we fork, we pass the read pipe to child + # child can then react on whether the other end + # of pipe is still available + @pipe_read, @pipe_write = IO.pipe + end + + def finish_master + close_read + close_write + end + + def register_worker + # fork needs to close the pipe + close_write + end + + private + + def close_read + @pipe_read&.close + @pipe_read = nil + end + + def close_write + @pipe_write&.close + @pipe_write = nil + end + + def metric_prefix + 'master_check' + end + + def successful?(result) + result + end + + def check + # the lack of pipe is a legitimate failure of check + return false unless @pipe_read + + @pipe_read.read_nonblock(1) + + true + rescue IO::EAGAINWaitReadable + # if it is blocked, it means that the pipe is still open + # and there's no data waiting on it + true + rescue EOFError + # the pipe is closed + false + end + end + end + end +end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index b2ac60fe825..516e7f54a6e 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -15,7 +15,7 @@ module Gitlab end def storage_path - File.join(Settings.shared['path'], 'tmp/project_exports') + File.join(Settings.shared['path'], 'tmp/gitlab_exports') end def import_upload_path(filename:) @@ -50,8 +50,8 @@ module Gitlab 'VERSION' end - def export_filename(project:) - basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.full_path.tr('/', '_')}" + def export_filename(exportable:) + basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{exportable.full_path.tr('/', '_')}" "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" end @@ -63,6 +63,14 @@ module Gitlab def reset_tokens? true end + + def group_filename + 'group.json' + end + + def group_config_file + Rails.root.join('lib/gitlab/import_export/group_import_export.yml') + end end end diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb index 6f4919ead4e..83c4bc47349 100644 --- a/lib/gitlab/import_export/config.rb +++ b/lib/gitlab/import_export/config.rb @@ -3,7 +3,8 @@ module Gitlab module ImportExport class Config - def initialize + def initialize(config: Gitlab::ImportExport.config_file) + @config = config @hash = parse_yaml @hash.deep_symbolize_keys! @ee_hash = @hash.delete(:ee) || {} @@ -50,7 +51,7 @@ module Gitlab end def parse_yaml - YAML.load_file(Gitlab::ImportExport.config_file) + YAML.load_file(@config) end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 05432f433e7..2fd12e3aa78 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -60,7 +60,7 @@ module Gitlab def copy_archive return if @archive_file - @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) + @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @project)) download_or_copy_upload(@project.import_export_upload.import_file, @archive_file) end diff --git a/lib/gitlab/import_export/group_import_export.yml b/lib/gitlab/import_export/group_import_export.yml new file mode 100644 index 00000000000..c1900350c86 --- /dev/null +++ b/lib/gitlab/import_export/group_import_export.yml @@ -0,0 +1,36 @@ +# Model relationships to be included in the group import/export +# +# This list _must_ only contain relationships that are available to both FOSS and +# Enterprise editions. EE specific relationships must be defined in the `ee` section further +# down below. +tree: + group: + - :milestones + - :badges + - labels: + - :priorities + - :boards + - members: + - :user + +included_attributes: + +excluded_attributes: + group: + - :runners_token + - :runners_token_encrypted + +methods: + labels: + - :type + badges: + - :type + +preloads: + +# EE specific relationships and settings to include. All of this will be merged +# into the previous structures if EE is used. +ee: + tree: + group: + - :epics diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb index de1629d0e28..b94839363df 100644 --- a/lib/gitlab/import_export/group_project_object_builder.rb +++ b/lib/gitlab/import_export/group_project_object_builder.rb @@ -49,11 +49,12 @@ module Gitlab ].compact end - # Returns Arel clause `"{table_name}"."project_id" = {project.id}` + # Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present + # For example: merge_request has :target_project_id, and we are searching by :iid # or, if group is present: # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}` def where_clause_base - clause = table[:project_id].eq(project.id) + clause = table[:project_id].eq(project.id) if project clause = clause.or(table[:group_id].eq(group.id)) if group clause @@ -103,6 +104,10 @@ module Gitlab klass == Milestone end + def merge_request? + klass == MergeRequest + end + # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: @@ -124,7 +129,7 @@ module Gitlab # Returns Arel clause for a particular model or `nil`. def where_clause_for_klass - # no-op + return attrs_to_arel(attributes.slice('iid')) if merge_request? end end end diff --git a/lib/gitlab/import_export/group_tree_saver.rb b/lib/gitlab/import_export/group_tree_saver.rb new file mode 100644 index 00000000000..8d2fb881cc0 --- /dev/null +++ b/lib/gitlab/import_export/group_tree_saver.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class GroupTreeSaver + attr_reader :full_path + + def initialize(group:, current_user:, shared:, params: {}) + @params = params + @current_user = current_user + @shared = shared + @group = group + @full_path = File.join(@shared.export_path, ImportExport.group_filename) + end + + def save + group_tree = serialize(@group, reader.group_tree) + tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) + + true + rescue => e + @shared.error(e) + false + end + + private + + def serialize(group, relations_tree) + group_tree = tree_saver.serialize(group, relations_tree) + + group.children.each do |child| + group_tree['children'] ||= [] + group_tree['children'] << serialize(child, relations_tree) + end + + group_tree + rescue => e + @shared.error(e) + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new( + shared: @shared, + config: Gitlab::ImportExport::Config.new( + config: Gitlab::ImportExport.group_config_file + ).to_h + ) + end + + def tree_saver + @tree_saver ||= RelationTreeSaver.new + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 141e73e6a47..1aafe5804c0 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -28,6 +28,7 @@ tree: - label: - :priorities - :issue_assignees + - :zoom_meetings - snippets: - :award_emoji - notes: @@ -147,6 +148,8 @@ excluded_attributes: - :emails_disabled - :max_pages_size - :max_artifacts_size + - :marked_for_deletion_at + - :marked_for_deletion_by_user_id namespaces: - :runners_token - :runners_token_encrypted diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 3fa5765fd4a..c401f96b5c1 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -15,7 +15,6 @@ module Gitlab @user = user @shared = shared @project = project - @saved = true end def restore @@ -33,7 +32,8 @@ module Gitlab ActiveRecord::Base.uncached do ActiveRecord::Base.no_touching do update_project_params! - create_relations + create_project_relations! + post_import! end end @@ -69,81 +69,75 @@ module Gitlab # in the DB. The structure and relationships between models are guessed from # the configuration yaml file too. # Finally, it updates each attribute in the newly imported project. - def create_relations - project_relations.each do |relation_key, relation_definition| - relation_key_s = relation_key.to_s - - if relation_definition.present? - create_sub_relations(relation_key_s, relation_definition, @tree_hash) - elsif @tree_hash[relation_key_s].present? - save_relation_hash(relation_key_s, @tree_hash[relation_key_s]) - end - end + def create_project_relations! + project_relations.each(&method( + :process_project_relation!)) + end + def post_import! @project.merge_requests.set_latest_merge_request_diff_ids! - - @saved end - def save_relation_hash(relation_key, relation_hash_batch) - relation_hash = create_relation(relation_key, relation_hash_batch) + def process_project_relation!(relation_key, relation_definition) + data_hashes = @tree_hash.delete(relation_key) + return unless data_hashes - remove_group_models(relation_hash) if relation_hash.is_a?(Array) + # we do not care if we process array or hash + data_hashes = [data_hashes] unless data_hashes.is_a?(Array) - @saved = false unless @project.append_or_update_attribute(relation_key, relation_hash) + # consume and remove objects from memory + while data_hash = data_hashes.shift + process_project_relation_item!(relation_key, relation_definition, data_hash) + end + end - save_id_mappings(relation_key, relation_hash_batch, relation_hash) + def process_project_relation_item!(relation_key, relation_definition, data_hash) + relation_object = build_relation(relation_key, relation_definition, data_hash) + return unless relation_object + return if group_model?(relation_object) - @project.reset + relation_object.project = @project + relation_object.save! + + save_id_mapping(relation_key, data_hash, relation_object) end # Older, serialized CI pipeline exports may only have a # merge_request_id and not the full hash of the merge request. To # import these pipelines, we need to preserve the mapping between # the old and new the merge request ID. - def save_id_mappings(relation_key, relation_hash_batch, relation_hash) + def save_id_mapping(relation_key, data_hash, relation_object) return unless relation_key == 'merge_requests' - relation_hash = Array(relation_hash) - - Array(relation_hash_batch).each_with_index do |raw_data, index| - merge_requests_mapping[raw_data['id']] = relation_hash[index]['id'] - end - end - - # Remove project models that became group models as we found them at group level. - # This no longer required saving them at the root project level. - # For example, in the case of an existing group label that matched the title. - def remove_group_models(relation_hash) - relation_hash.reject! do |value| - GROUP_MODELS.include?(value.class) && value.group_id - end - end - - def remove_feature_dependent_sub_relations!(_relation_item) - # no-op + merge_requests_mapping[data_hash['id']] = relation_object.id end def project_relations - @project_relations ||= reader.attributes_finder.find_relations_tree(:project) + @project_relations ||= + reader + .attributes_finder + .find_relations_tree(:project) + .deep_stringify_keys end def update_project_params! - Gitlab::Timeless.timeless(@project) do - project_params = @tree_hash.reject do |key, value| - project_relations.include?(key.to_sym) - end + project_params = @tree_hash.reject do |key, value| + project_relations.include?(key) + end - project_params = project_params.merge(present_project_override_params) + project_params = project_params.merge( + present_project_override_params) - # Cleaning all imported and overridden params - project_params = Gitlab::ImportExport::AttributeCleaner.clean( - relation_hash: project_params, - relation_class: Project, - excluded_keys: excluded_keys_for_relation(:project)) + # Cleaning all imported and overridden params + project_params = Gitlab::ImportExport::AttributeCleaner.clean( + relation_hash: project_params, + relation_class: Project, + excluded_keys: excluded_keys_for_relation(:project)) - @project.assign_attributes(project_params) - @project.drop_visibility_level! + @project.assign_attributes(project_params) + @project.drop_visibility_level! + + Gitlab::Timeless.timeless(@project) do @project.save! end end @@ -160,75 +154,61 @@ module Gitlab @project_override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {} end - # Given a relation hash containing one or more models and its relationships, - # loops through each model and each object from a model type and - # and assigns its correspondent attributes hash from +tree_hash+ - # Example: - # +relation_key+ issues, loops through the list of *issues* and for each individual - # issue, finds any subrelations such as notes, creates them and assign them back to the hash - # - # Recursively calls this method if the sub-relation is a hash containing more sub-relations - def create_sub_relations(relation_key, relation_definition, tree_hash, save: true) - return if tree_hash[relation_key].blank? - - tree_array = [tree_hash[relation_key]].flatten - - # Avoid keeping a possible heavy object in memory once we are done with it - while relation_item = tree_array.shift - remove_feature_dependent_sub_relations!(relation_item) - - # The transaction at this level is less speedy than one single transaction - # But we can't have it in the upper level or GC won't get rid of the AR objects - # after we save the batch. - Project.transaction do - process_sub_relation(relation_key, relation_definition, relation_item) - - # For every subrelation that hangs from Project, save the associated records altogether - # This effectively batches all records per subrelation item, only keeping those in memory - # We have to keep in mind that more batch granularity << Memory, but >> Slowness - if save - save_relation_hash(relation_key, [relation_item]) - tree_hash[relation_key].delete(relation_item) - end - end - end - - tree_hash.delete(relation_key) if save + def build_relations(relation_key, relation_definition, data_hashes) + data_hashes.map do |data_hash| + build_relation(relation_key, relation_definition, data_hash) + end.compact end - def process_sub_relation(relation_key, relation_definition, relation_item) - relation_definition.each do |sub_relation_key, sub_relation_definition| - # We just use author to get the user ID, do not attempt to create an instance. - next if sub_relation_key == :author + def build_relation(relation_key, relation_definition, data_hash) + # TODO: This is hack to not create relation for the author + # Rather make `RelationFactory#set_note_author` to take care of that + return data_hash if relation_key == 'author' - sub_relation_key_s = sub_relation_key.to_s + # create relation objects recursively for all sub-objects + relation_definition.each do |sub_relation_key, sub_relation_definition| + transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition) + end - # create dependent relations if present - if sub_relation_definition.present? - create_sub_relations(sub_relation_key_s, sub_relation_definition, relation_item, save: false) + Gitlab::ImportExport::RelationFactory.create( + relation_sym: relation_key.to_sym, + relation_hash: data_hash, + members_mapper: members_mapper, + merge_requests_mapping: merge_requests_mapping, + user: @user, + project: @project, + excluded_keys: excluded_keys_for_relation(relation_key)) + end + + def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition) + sub_data_hash = data_hash[sub_relation_key] + return unless sub_data_hash + + # if object is a hash we can create simple object + # as it means that this is 1-to-1 vs 1-to-many + sub_data_hash = + if sub_data_hash.is_a?(Array) + build_relations( + sub_relation_key, + sub_relation_definition, + sub_data_hash).presence + else + build_relation( + sub_relation_key, + sub_relation_definition, + sub_data_hash) end - # transform relation hash to actual object - sub_relation_hash = relation_item[sub_relation_key_s] - if sub_relation_hash.present? - relation_item[sub_relation_key_s] = create_relation(sub_relation_key, sub_relation_hash) - end + # persist object(s) or delete from relation + if sub_data_hash + data_hash[sub_relation_key] = sub_data_hash + else + data_hash.delete(sub_relation_key) end end - def create_relation(relation_key, relation_hash_list) - relation_array = [relation_hash_list].flatten.map do |relation_hash| - Gitlab::ImportExport::RelationFactory.create( - relation_sym: relation_key.to_sym, - relation_hash: relation_hash, - members_mapper: members_mapper, - merge_requests_mapping: merge_requests_mapping, - user: @user, - project: @project, - excluded_keys: excluded_keys_for_relation(relation_key)) - end.compact - - relation_hash_list.is_a?(Array) ? relation_array : relation_array.first + def group_model?(relation_object) + GROUP_MODELS.include?(relation_object.class) && relation_object.group_id end def reader @@ -241,5 +221,3 @@ module Gitlab end end end - -Gitlab::ImportExport::ProjectTreeRestorer.prepend_if_ee('::EE::Gitlab::ImportExport::ProjectTreeRestorer') diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 63c71105efe..386a4cfdfc6 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -3,25 +3,20 @@ module Gitlab module ImportExport class ProjectTreeSaver - include Gitlab::ImportExport::CommandLineUtil - attr_reader :full_path def initialize(project:, current_user:, shared:, params: {}) - @params = params - @project = project + @params = params + @project = project @current_user = current_user - @shared = shared - @full_path = File.join(@shared.export_path, ImportExport.project_filename) + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) end def save - mkdir_p(@shared.export_path) - - project_tree = serialize_project_tree + project_tree = tree_saver.serialize(@project, reader.project_tree) fix_project_tree(project_tree) - project_tree_json = JSON.generate(project_tree) - File.write(full_path, project_tree_json) + tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) true rescue => e @@ -43,16 +38,6 @@ module Gitlab RelationRenameService.add_new_associations(project_tree) end - def serialize_project_tree - if Feature.enabled?(:export_fast_serialize, default_enabled: true) - Gitlab::ImportExport::FastHashSerializer - .new(@project, reader.project_tree) - .execute - else - @project.as_json(reader.project_tree) - end - end - def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end @@ -74,6 +59,10 @@ module Gitlab GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) end + + def tree_saver + @tree_saver ||= RelationTreeSaver.new + end end end end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 9e81c6a3d07..1390770acef 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -5,24 +5,31 @@ module Gitlab class Reader attr_reader :tree, :attributes_finder - def initialize(shared:) - @shared = shared - - @attributes_finder = Gitlab::ImportExport::AttributesFinder.new( - config: ImportExport::Config.new.to_h) + def initialize(shared:, config: ImportExport::Config.new.to_h) + @shared = shared + @config = config + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(config: @config) end # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html # for outputting a project in JSON format, including its relations and sub relations. def project_tree - attributes_finder.find_root(:project) - rescue => e - @shared.error(e) - false + tree_by_key(:project) + end + + def group_tree + tree_by_key(:group) end def group_members_tree - attributes_finder.find_root(:group_members) + tree_by_key(:group_members) + end + + def tree_by_key(key) + attributes_finder.find_root(key) + rescue => e + @shared.error(e) + false end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index ae8025c52ef..ae6b3c161ce 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -38,10 +38,13 @@ module Gitlab IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature merge_request].freeze + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature merge_request ProjectCiCdSetting].freeze TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze + # This represents all relations that have unique key on `project_id` + UNIQUE_RELATIONS = %i[project_feature ProjectCiCdSetting].freeze + def self.create(*args) new(*args).create end @@ -274,7 +277,7 @@ module Gitlab end def setup_pipeline - @relation_hash.fetch('stages').each do |stage| + @relation_hash.fetch('stages', []).each do |stage| stage.statuses.each do |status| status.pipeline = imported_object end @@ -324,8 +327,7 @@ module Gitlab end def find_or_create_object! - return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature - return find_or_create_merge_request! if @relation_name == :merge_request + return relation_class.find_or_create_by(project_id: @project.id) if UNIQUE_RELATIONS.include?(@relation_name) # Can't use IDs as validation exists calling `group` or `project` attributes finder_hash = parsed_relation_hash.tap do |hash| @@ -336,11 +338,6 @@ module Gitlab GroupProjectObjectBuilder.build(relation_class, finder_hash) end - - def find_or_create_merge_request! - @project.merge_requests.find_by(iid: parsed_relation_hash['iid']) || - relation_class.new(parsed_relation_hash) - end end end end diff --git a/lib/gitlab/import_export/relation_rename_service.rb b/lib/gitlab/import_export/relation_rename_service.rb index 179bde5e21e..03aaa6aefc3 100644 --- a/lib/gitlab/import_export/relation_rename_service.rb +++ b/lib/gitlab/import_export/relation_rename_service.rb @@ -8,7 +8,7 @@ # The behavior of these renamed relationships should be transient and it should # only last one release until you completely remove the renaming from the list. # -# When importing, this class will check the project hash and: +# When importing, this class will check the hash and: # - if only the old relationship name is found, it will rename it with the new one # - if only the new relationship name is found, it will do nothing # - if it finds both, it will use the new relationship data diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/relation_tree_saver.rb new file mode 100644 index 00000000000..a0452071ccf --- /dev/null +++ b/lib/gitlab/import_export/relation_tree_saver.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class RelationTreeSaver + include Gitlab::ImportExport::CommandLineUtil + + def serialize(exportable, relations_tree) + if Feature.enabled?(:export_fast_serialize, default_enabled: true) + Gitlab::ImportExport::FastHashSerializer + .new(exportable, relations_tree) + .execute + else + exportable.as_json(relations_tree) + end + end + + def save(tree, dir_path, filename) + mkdir_p(dir_path) + + tree_json = JSON.generate(tree) + + File.write(File.join(dir_path, filename), tree_json) + end + end + end +end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index bea7a7cce65..ae82c380755 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -9,16 +9,16 @@ module Gitlab new(*args).save end - def initialize(project:, shared:) - @project = project - @shared = shared + def initialize(exportable:, shared:) + @exportable = exportable + @shared = shared end def save if compress_and_save remove_export_path - Rails.logger.info("Saved project export #{archive_file}") # rubocop:disable Gitlab/RailsLogger + Rails.logger.info("Saved #{@exportable.class} export #{archive_file}") # rubocop:disable Gitlab/RailsLogger save_upload else @@ -48,11 +48,11 @@ module Gitlab end def archive_file - @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) + @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @exportable)) end def save_upload - upload = ImportExportUpload.find_or_initialize_by(project: @project) + upload = initialize_upload File.open(archive_file) { |file| upload.export_file = file } @@ -62,6 +62,12 @@ module Gitlab def error_message "Unable to save #{archive_file} into #{@shared.export_path}." end + + def initialize_upload + exportable_kind = @exportable.class.name.downcase + + ImportExportUpload.find_or_initialize_by(Hash[exportable_kind, @exportable]) + end end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 02d46a1f498..2539a6828c3 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -23,21 +23,21 @@ module Gitlab module ImportExport class Shared - attr_reader :errors, :project + attr_reader :errors, :exportable, :logger LOCKS_DIRECTORY = 'locks' - def initialize(project) - @project = project - @errors = [] - @logger = Gitlab::Import::Logger.build + def initialize(exportable) + @exportable = exportable + @errors = [] + @logger = Gitlab::Import::Logger.build end def active_export_count Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) } end - # The path where the project metadata and repository bundle is saved + # The path where the exportable metadata and repository bundle (in case of project) is saved def export_path @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path) end @@ -84,11 +84,18 @@ module Gitlab end def relative_archive_path - @relative_archive_path ||= File.join(@project.disk_path, SecureRandom.hex) + @relative_archive_path ||= File.join(relative_base_path, SecureRandom.hex) end def relative_base_path - @project.disk_path + case exportable_type + when 'Project' + @exportable.disk_path + when 'Group' + @exportable.full_path + else + raise Gitlab::ImportExport::Error.new("Unsupported Exportable Type #{@exportable&.class}") + end end def log_error(details) @@ -100,17 +107,24 @@ module Gitlab end def log_base_data - { - importer: 'Import/Export', - import_jid: @project&.import_state&.jid, - project_id: @project&.id, - project_path: @project&.full_path + log = { + importer: 'Import/Export', + exportable_id: @exportable&.id, + exportable_path: @exportable&.full_path } + + log[:import_jid] = @exportable&.import_state&.jid if exportable_type == 'Project' + + log end def filtered_error_message(message) Projects::ImportErrorFilter.filter_message(message) end + + def exportable_type + @exportable.class.name + end end end end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index e6a5facb2a5..edaa9c645b4 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -21,5 +21,49 @@ module Gitlab payload[:rugged_duration_ms] = Gitlab::RuggedInstrumentation.query_time_ms end end + + # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the + # `enqueued_at` field or `created_at` field is available. + # + # * If the job doesn't contain sufficient information, returns nil + # * If the job has a start time in the future, returns 0 + # * If the job contains an invalid start time value, returns nil + # @param [Hash] job a Sidekiq job, represented as a hash + def self.queue_duration_for_job(job) + # Old gitlab-shell messages don't provide enqueued_at/created_at attributes + enqueued_at = job['enqueued_at'] || job['created_at'] + return unless enqueued_at + + enqueued_at_time = convert_to_time(enqueued_at) + return unless enqueued_at_time + + # Its possible that if theres clock-skew between two nodes + # this value may be less than zero. In that event, we record the value + # as zero. + [elapsed_by_absolute_time(enqueued_at_time), 0].max + end + + # Calculates the time in seconds, as a float, from + # the provided start time until now + # + # @param [Time] start + def self.elapsed_by_absolute_time(start) + (Time.now - start).to_f.round(6) + end + private_class_method :elapsed_by_absolute_time + + # Convert a representation of a time into a `Time` value + # + # @param time_value String, Float time representation, or nil + def self.convert_to_time(time_value) + return time_value if time_value.is_a?(Time) + return Time.iso8601(time_value) if time_value.is_a?(String) + return Time.at(time_value) if time_value.is_a?(Numeric) && time_value > 0 + rescue ArgumentError + # Swallow invalid dates. Better to loose some observability + # than bring all background processing down because of a date + # formatting bug in a client + end + private_class_method :convert_to_time end end diff --git a/lib/gitlab/kubernetes/config_maps/aws_node_auth.rb b/lib/gitlab/kubernetes/config_maps/aws_node_auth.rb new file mode 100644 index 00000000000..ef51cee09ca --- /dev/null +++ b/lib/gitlab/kubernetes/config_maps/aws_node_auth.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module ConfigMaps + class AwsNodeAuth + attr_reader :node_role + + def initialize(node_role) + @node_role = node_role + end + + def generate + Kubeclient::Resource.new( + metadata: metadata, + data: data + ) + end + + private + + def metadata + { + 'name' => 'aws-auth', + 'namespace' => 'kube-system' + } + end + + def data + { 'mapRoles' => instance_role_config(node_role) } + end + + def instance_role_config(role) + [{ + 'rolearn' => role, + 'username' => 'system:node:{{EC2PrivateDNSName}}', + 'groups' => [ + 'system:bootstrappers', + 'system:nodes' + ] + }].to_yaml + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 16ed0cb0f8e..b5181670b93 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -3,8 +3,8 @@ module Gitlab module Kubernetes module Helm - HELM_VERSION = '2.14.3' - KUBECTL_VERSION = '1.11.10' + HELM_VERSION = '2.16.1' + KUBECTL_VERSION = '1.13.12' NAMESPACE = 'gitlab-managed-apps' SERVICE_ACCOUNT = 'tiller' CLUSTER_ROLE_BINDING = 'tiller-admin' diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index f572bc43533..ccb053f507d 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -40,7 +40,7 @@ module Gitlab private def repository_update_command - 'helm repo update' if repository + 'helm repo update' end # Uses `helm upgrade --install` which means we can use this for both diff --git a/lib/gitlab/metrics/dashboard/errors.rb b/lib/gitlab/metrics/dashboard/errors.rb index d41bd2c43c7..264ea0488e7 100644 --- a/lib/gitlab/metrics/dashboard/errors.rb +++ b/lib/gitlab/metrics/dashboard/errors.rb @@ -9,6 +9,7 @@ module Gitlab module Errors DashboardProcessingError = Class.new(StandardError) PanelNotFoundError = Class.new(StandardError) + MissingIntegrationError = Class.new(StandardError) LayoutError = Class.new(DashboardProcessingError) MissingQueryError = Class.new(DashboardProcessingError) @@ -22,6 +23,10 @@ module Gitlab error("#{dashboard_path} could not be found.", :not_found) when PanelNotFoundError error(error.message, :not_found) + when ::Grafana::Client::Error + error(error.message, :service_unavailable) + when MissingIntegrationError + error('Proxy support for this API is not available currently', :bad_request) else raise error end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index 297f109ff81..268112f33a9 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -12,6 +12,7 @@ module Gitlab # @param project [Project] # @param user [User] # @param environment [Environment] + # @param options [Hash<Symbol,Any>] # @param options - embedded [Boolean] Determines whether the # dashboard is to be rendered as part of an # issue or location other than the primary @@ -31,6 +32,8 @@ module Gitlab # @param options - cluster [Cluster] # @param options - cluster_type [Symbol] The level of # cluster, one of [:admin, :project, :group] + # @param options - grafana_url [String] URL pointing + # to a grafana dashboard panel # @return [Hash] def find(project, user, options = {}) service_for(options) diff --git a/lib/gitlab/metrics/dashboard/processor.rb b/lib/gitlab/metrics/dashboard/processor.rb index bfdee76a818..9566e5afb9a 100644 --- a/lib/gitlab/metrics/dashboard/processor.rb +++ b/lib/gitlab/metrics/dashboard/processor.rb @@ -17,7 +17,10 @@ module Gitlab # Returns a new dashboard hash with the results of # running transforms on the dashboard. + # @return [Hash, nil] def process + return unless @dashboard + @dashboard.deep_symbolize_keys.tap do |dashboard| @sequence.each do |stage| stage.new(@project, dashboard, @params).transform! diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb index 10b686fbb81..aee7f6685ad 100644 --- a/lib/gitlab/metrics/dashboard/service_selector.rb +++ b/lib/gitlab/metrics/dashboard/service_selector.rb @@ -18,6 +18,7 @@ module Gitlab # @return [Gitlab::Metrics::Dashboard::Services::BaseService] def call(params) return SERVICES::CustomMetricEmbedService if custom_metric_embed?(params) + return SERVICES::GrafanaMetricEmbedService if grafana_metric_embed?(params) return SERVICES::DynamicEmbedService if dynamic_embed?(params) return SERVICES::DefaultEmbedService if params[:embedded] return SERVICES::SystemDashboardService if system_dashboard?(params[:dashboard_path]) @@ -40,6 +41,10 @@ module Gitlab SERVICES::CustomMetricEmbedService.valid_params?(params) end + def grafana_metric_embed?(params) + SERVICES::GrafanaMetricEmbedService.valid_params?(params) + end + def dynamic_embed?(params) SERVICES::DynamicEmbedService.valid_params?(params) end diff --git a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb index 188912bedb4..62479ed6de4 100644 --- a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb @@ -9,7 +9,7 @@ module Gitlab # find a corresponding database record. If found, # includes the record's id in the dashboard config. def transform! - common_metrics = ::PrometheusMetric.common + common_metrics = ::PrometheusMetricsFinder.new(common: true).execute for_metrics do |metric| metric_record = common_metrics.find { |m| m.identifier == metric[:id] } diff --git a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb new file mode 100644 index 00000000000..ce75c54d014 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb @@ -0,0 +1,224 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class GrafanaFormatter < BaseStage + include Gitlab::Utils::StrongMemoize + + CHART_TYPE = 'area-chart' + PROXY_PATH = 'api/v1/query_range' + + # Reformats the specified panel in the Gitlab + # dashboard-yml format + def transform! + InputFormatValidator.new( + grafana_dashboard, + datasource, + panel, + query_params + ).validate! + + new_dashboard = formatted_dashboard + + dashboard.clear + dashboard.merge!(new_dashboard) + end + + private + + def formatted_dashboard + { panel_groups: [{ panels: [formatted_panel] }] } + end + + def formatted_panel + { + title: panel[:title], + type: CHART_TYPE, + y_label: '', # Grafana panels do not include a Y-Axis label + metrics: panel[:targets].map.with_index do |target, idx| + formatted_metric(target, idx) + end + } + end + + def formatted_metric(metric, idx) + { + id: "#{metric[:legendFormat]}_#{idx}", + query_range: format_query(metric), + label: replace_variables(metric[:legendFormat]), + prometheus_endpoint_path: prometheus_endpoint_for_metric(metric) + }.compact + end + + # Panel specified by the url from the Grafana dashboard + def panel + strong_memoize(:panel) do + grafana_dashboard[:dashboard][:panels].find do |panel| + panel[:id].to_s == query_params[:panelId] + end + end + end + + # Grafana url query parameters. Includes information + # on which panel to select and time range. + def query_params + strong_memoize(:query_params) do + Gitlab::Metrics::Dashboard::Url.parse_query(grafana_url) + end + end + + # Endpoint which will return prometheus metric data + # for the metric + def prometheus_endpoint_for_metric(metric) + Gitlab::Routing.url_helpers.project_grafana_api_path( + project, + datasource_id: datasource[:id], + proxy_path: PROXY_PATH, + query: format_query(metric) + ) + end + + # Reformats query for compatibility with prometheus api. + def format_query(metric) + expression = remove_new_lines(metric[:expr]) + expression = replace_variables(expression) + expression = replace_global_variables(expression, metric) + + expression + end + + # Accomodates instance-defined Grafana variables. + # These are variables defined by users, and values + # must be provided in the query parameters. + def replace_variables(expression) + return expression unless grafana_dashboard[:dashboard][:templating] + + grafana_dashboard[:dashboard][:templating][:list] + .sort_by { |variable| variable[:name].length } + .each do |variable| + variable_value = query_params[:"var-#{variable[:name]}"] + + expression = expression.gsub("$#{variable[:name]}", variable_value) + expression = expression.gsub("[[#{variable[:name]}]]", variable_value) + expression = expression.gsub("{{#{variable[:name]}}}", variable_value) + end + + expression + end + + # Replaces Grafana global built-in variables with values. + # Only $__interval and $__from and $__to are supported. + # + # See https://grafana.com/docs/reference/templating/#global-built-in-variables + def replace_global_variables(expression, metric) + expression = expression.gsub('$__interval', metric[:interval]) if metric[:interval] + expression = expression.gsub('$__from', query_params[:from]) + expression = expression.gsub('$__to', query_params[:to]) + + expression + end + + # Removes new lines from expression. + def remove_new_lines(expression) + expression.gsub(/\R+/, '') + end + + # Grafana datasource object corresponding to the + # specified dashboard + def datasource + params[:datasource] + end + + # The specified Grafana dashboard + def grafana_dashboard + params[:grafana_dashboard] + end + + # The URL specifying which Grafana panel to embed + def grafana_url + params[:grafana_url] + end + end + + class InputFormatValidator + include ::Gitlab::Metrics::Dashboard::Errors + + attr_reader :grafana_dashboard, :datasource, :panel, :query_params + + UNSUPPORTED_GRAFANA_GLOBAL_VARS = %w( + $__interval_ms + $__timeFilter + $__name + $timeFilter + $interval + ).freeze + + def initialize(grafana_dashboard, datasource, panel, query_params) + @grafana_dashboard = grafana_dashboard + @datasource = datasource + @panel = panel + @query_params = query_params + end + + def validate! + validate_query_params! + validate_datasource! + validate_panel_type! + validate_variable_definitions! + validate_global_variables! + end + + private + + def validate_datasource! + return if datasource[:access] == 'proxy' && datasource[:type] == 'prometheus' + + raise_error 'Only Prometheus datasources with proxy access in Grafana are supported.' + end + + def validate_query_params! + return if [:panelId, :from, :to].all? { |param| query_params.include?(param) } + + raise_error 'Grafana query parameters must include panelId, from, and to.' + end + + def validate_panel_type! + return if panel[:type] == 'graph' && panel[:lines] + + raise_error 'Panel type must be a line graph.' + end + + def validate_variable_definitions! + return unless grafana_dashboard[:dashboard][:templating] + + return if grafana_dashboard[:dashboard][:templating][:list].all? do |variable| + query_params[:"var-#{variable[:name]}"].present? + end + + raise_error 'All Grafana variables must be defined in the query parameters.' + end + + def validate_global_variables! + return unless panel_contains_unsupported_vars? + + raise_error 'Prometheus must not include' + end + + def panel_contains_unsupported_vars? + panel[:targets].any? do |target| + UNSUPPORTED_GRAFANA_GLOBAL_VARS.any? do |variable| + target[:expr].include?(variable) + end + end + end + + def raise_error(message) + raise DashboardProcessingError.new(message) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb index 643be309992..c0f67d445f8 100644 --- a/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb @@ -9,7 +9,7 @@ module Gitlab # config. If there are no project-specific metrics, # this will have no effect. def transform! - project.prometheus_metrics.each do |project_metric| + PrometheusMetricsFinder.new(project: project).execute.each do |project_metric| group = find_or_create_panel_group(dashboard[:panel_groups], project_metric) panel = find_or_create_panel(group[:panels], project_metric) find_or_create_metric(panel[:metrics], project_metric) diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb index 94f8b2e02b1..712f769bbeb 100644 --- a/lib/gitlab/metrics/dashboard/url.rb +++ b/lib/gitlab/metrics/dashboard/url.rb @@ -14,17 +14,31 @@ module Gitlab def regex %r{ (?<url> - #{Regexp.escape(Gitlab.config.gitlab.url)} - \/#{Project.reference_pattern} + #{gitlab_pattern} + #{project_pattern} (?:\/\-)? \/environments \/(?<environment>\d+) \/metrics - (?<query> - \?[a-zA-Z0-9%.()+_=-]+ - (&[a-zA-Z0-9%.()+_=-]+)* - )? - (?<anchor>\#[a-z0-9_-]+)? + #{query_pattern} + #{anchor_pattern} + ) + }x + end + + # Matches dashboard urls for a Grafana embed. + # + # EX - https://<host>/<namespace>/<project>/grafana/metrics_dashboard + def grafana_regex + %r{ + (?<url> + #{gitlab_pattern} + #{project_pattern} + (?:\/\-)? + \/grafana + \/metrics_dashboard + #{query_pattern} + #{anchor_pattern} ) }x end @@ -45,6 +59,24 @@ module Gitlab def build_dashboard_url(*args) Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args) end + + private + + def gitlab_pattern + Regexp.escape(Gitlab.config.gitlab.url) + end + + def project_pattern + "\/#{Project.reference_pattern}" + end + + def query_pattern + '(?<query>\?[a-zA-Z0-9%.()+_=-]+(&[a-zA-Z0-9%.()+_=-]+)*)?' + end + + def anchor_pattern + '(?<anchor>\#[a-z0-9_-]+)?' + end end end end diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index 3940f6fa155..b6a27d8556a 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -20,6 +20,10 @@ module Gitlab def initialize super + # DEPRECATED: + # these `readiness_checks` are deprecated + # as presenting no value in a way how we run + # application: https://gitlab.com/gitlab-org/gitlab/issues/35343 self.readiness_checks = [ WebExporter::ExporterCheck.new(self), Gitlab::HealthChecks::PumaCheck, @@ -35,6 +39,10 @@ module Gitlab File.join(Rails.root, 'log', 'web_exporter.log') end + def mark_as_not_running! + @running = false + end + private def start_working @@ -43,24 +51,9 @@ module Gitlab end def stop_working - @running = false - wait_in_blackout_period if server && thread.alive? + mark_as_not_running! super end - - def wait_in_blackout_period - return unless blackout_seconds > 0 - - @server.logger.info( - message: 'starting blackout...', - duration_s: blackout_seconds) - - sleep(blackout_seconds) - end - - def blackout_seconds - settings['blackout_seconds'].to_i - end end end end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 085e28123a7..b57f9a19f8e 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -35,7 +35,7 @@ module Gitlab def self.initialize_http_request_duration_seconds HTTP_METHODS.each do |method, statuses| statuses.each do |status| - http_request_duration_seconds.get({ method: method, status: status.to_i }) + http_request_duration_seconds.get({ method: method, status: status.to_s }) end end end @@ -49,7 +49,7 @@ module Gitlab status, headers, body = @app.call(env) elapsed = Time.now.to_f - started - RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status }, elapsed) + RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status.to_s }, elapsed) [status, headers, body] rescue diff --git a/lib/gitlab/pagination/base.rb b/lib/gitlab/pagination/base.rb new file mode 100644 index 00000000000..90fa1f8d1ec --- /dev/null +++ b/lib/gitlab/pagination/base.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + class Base + private + + def per_page + @per_page ||= params[:per_page] + end + + def base_request_uri + @base_request_uri ||= URI.parse(request.url).tap do |uri| + uri.host = Gitlab.config.gitlab.host + uri.port = Gitlab.config.gitlab.port + end + end + + def build_page_url(query_params:) + base_request_uri.tap do |uri| + uri.query = query_params + end.to_s + end + + def page_href(next_page_params = {}) + query_params = params.merge(**next_page_params, per_page: per_page).to_query + + build_page_url(query_params: query_params) + end + end + end +end diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb new file mode 100644 index 00000000000..bf31f252a6b --- /dev/null +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + class OffsetPagination < Base + attr_reader :request_context + delegate :params, :header, :request, to: :request_context + + def initialize(request_context) + @request_context = request_context + end + + def paginate(relation) + paginate_with_limit_optimization(add_default_order(relation)).tap do |data| + add_pagination_headers(data) + end + end + + private + + def paginate_with_limit_optimization(relation) + pagination_data = relation.page(params[:page]).per(params[:per_page]) + return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) + return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit) + + limited_total_count = pagination_data.total_count_with_limit + if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT + # The call to `total_count_with_limit` memoizes `@arel` because of a call to `references_eager_loaded_tables?` + # We need to call `reset` because `without_count` relies on `@arel` being unmemoized + pagination_data.reset.without_count + else + pagination_data + end + end + + def add_default_order(relation) + if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? + relation = relation.order(:id) # rubocop: disable CodeReuse/ActiveRecord + end + + relation + end + + def add_pagination_headers(paginated_data) + header 'X-Per-Page', paginated_data.limit_value.to_s + header 'X-Page', paginated_data.current_page.to_s + header 'X-Next-Page', paginated_data.next_page.to_s + header 'X-Prev-Page', paginated_data.prev_page.to_s + header 'Link', pagination_links(paginated_data) + + return if data_without_counts?(paginated_data) + + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', total_pages(paginated_data).to_s + end + + def pagination_links(paginated_data) + [].tap do |links| + links << %(<#{page_href(page: paginated_data.prev_page)}>; rel="prev") if paginated_data.prev_page + links << %(<#{page_href(page: paginated_data.next_page)}>; rel="next") if paginated_data.next_page + links << %(<#{page_href(page: 1)}>; rel="first") + + links << %(<#{page_href(page: total_pages(paginated_data))}>; rel="last") unless data_without_counts?(paginated_data) + end.join(', ') + end + + def total_pages(paginated_data) + # Ensure there is in total at least 1 page + [paginated_data.total_pages, 1].max + end + + def data_without_counts?(paginated_data) + paginated_data.is_a?(Kaminari::PaginatableWithoutCount) + end + end + end +end diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb index a9270cd536e..4e5e2d4a6a9 100644 --- a/lib/gitlab/project_authorizations.rb +++ b/lib/gitlab/project_authorizations.rb @@ -57,7 +57,7 @@ module Gitlab private # Builds a recursive CTE that gets all the groups the current user has - # access to, including any nested groups. + # access to, including any nested groups and any shared groups. def recursive_cte cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) members = Member.arel_table @@ -68,20 +68,27 @@ module Gitlab .select([namespaces[:id], members[:access_level]]) .except(:order) + if Feature.enabled?(:share_group_with_group) + # Namespaces shared with any of the group + cte << Group.select([namespaces[:id], 'group_group_links.group_access AS access_level']) + .joins(join_group_group_links) + .joins(join_members_on_group_group_links) + end + # Sub groups of any groups the user is a member of. cte << Group.select([ namespaces[:id], greatest(members[:access_level], cte.table[:access_level], 'access_level') ]) .joins(join_cte(cte)) - .joins(join_members) + .joins(join_members_on_namespaces) .except(:order) cte end # Builds a LEFT JOIN to join optional memberships onto the CTE. - def join_members + def join_members_on_namespaces members = Member.arel_table namespaces = Namespace.arel_table @@ -94,6 +101,23 @@ module Gitlab Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) end + def join_group_group_links + group_group_links = GroupGroupLink.arel_table + namespaces = Namespace.arel_table + + cond = group_group_links[:shared_group_id].eq(namespaces[:id]) + Arel::Nodes::InnerJoin.new(group_group_links, Arel::Nodes::On.new(cond)) + end + + def join_members_on_group_group_links + group_group_links = GroupGroupLink.arel_table + members = Member.arel_table + + cond = group_group_links[:shared_with_group_id].eq(members[:source_id]) + .and(members[:user_id].eq(user.id)) + Arel::Nodes::InnerJoin.new(members, Arel::Nodes::On.new(cond)) + end + # Builds an INNER JOIN to join namespaces onto the CTE. def join_cte(cte) namespaces = Namespace.arel_table diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index fa1d1203842..279fc4aa375 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -53,7 +53,8 @@ module Gitlab ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'), ProjectTemplate.new('nfplainhtml', 'Netlify/Plain HTML', _('A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfplain-html', 'illustrations/logos/netlify.svg'), ProjectTemplate.new('nfgitbook', 'Netlify/GitBook', _('A GitBook site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfgitbook', 'illustrations/logos/netlify.svg'), - ProjectTemplate.new('nfhexo', 'Netlify/Hexo', _('A Hexo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhexo', 'illustrations/logos/netlify.svg') + ProjectTemplate.new('nfhexo', 'Netlify/Hexo', _('A Hexo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhexo', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('serverless_framework', 'Serverless Framework/JS', _('A basic page and serverless function that uses AWS Lambda, AWS API Gateway, and GitLab Pages'), 'https://gitlab.com/gitlab-org/project-templates/serverless-framework', 'illustrations/logos/serverless_framework.svg') ].freeze class << self diff --git a/lib/gitlab/prometheus/internal.rb b/lib/gitlab/prometheus/internal.rb new file mode 100644 index 00000000000..d59352119ba --- /dev/null +++ b/lib/gitlab/prometheus/internal.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Prometheus + class Internal + def self.uri + return if listen_address.blank? + + if listen_address.starts_with?('0.0.0.0:') + # 0.0.0.0:9090 + port = ':' + listen_address.split(':').second + 'http://localhost' + port + + elsif listen_address.starts_with?(':') + # :9090 + 'http://localhost' + listen_address + + elsif listen_address.starts_with?('http') + # https://localhost:9090 + listen_address + + else + # localhost:9090 + 'http://' + listen_address + end + end + + def self.listen_address + Gitlab.config.prometheus.listen_address.to_s if Gitlab.config.prometheus + rescue Settingslogic::MissingSetting + Gitlab::AppLogger.error('Prometheus listen_address is not present in config/gitlab.yml') + + nil + end + + def self.prometheus_enabled? + Gitlab.config.prometheus.enable if Gitlab.config.prometheus + rescue Settingslogic::MissingSetting + Gitlab::AppLogger.error('prometheus.enable is not present in config/gitlab.yml') + + false + end + end + end +end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index caf0d453b6f..1b6f7282eb3 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -11,13 +11,15 @@ module Gitlab validates :name, :priority, :metrics, presence: true def self.common_metrics - all_groups = ::PrometheusMetric.common.group_by(&:group_title).map do |name, metrics| - MetricGroup.new( - name: name, - priority: metrics.map(&:priority).max, - metrics: metrics.map(&:to_query_metric) - ) - end + all_groups = ::PrometheusMetricsFinder.new(common: true).execute + .group_by(&:group_title) + .map do |name, metrics| + MetricGroup.new( + name: name, + priority: metrics.map(&:priority).max, + metrics: metrics.map(&:to_query_metric) + ) + end all_groups.sort_by(&:priority).reverse end diff --git a/lib/gitlab/prometheus/queries/knative_invocation_query.rb b/lib/gitlab/prometheus/queries/knative_invocation_query.rb index 2691abe46d6..8873608c411 100644 --- a/lib/gitlab/prometheus/queries/knative_invocation_query.rb +++ b/lib/gitlab/prometheus/queries/knative_invocation_query.rb @@ -7,11 +7,14 @@ module Gitlab include QueryAdditionalMetrics def query(serverless_function_id) - PrometheusMetric - .find_by_identifier(:system_metrics_knative_function_invocation_count) - .to_query_metric.tap do |q| - q.queries[0][:result] = run_query(q.queries[0][:query_range], context(serverless_function_id)) - end + PrometheusMetricsFinder + .new(identifier: :system_metrics_knative_function_invocation_count, common: true) + .execute + .first + .to_query_metric + .tap do |q| + q.queries[0][:result] = run_query(q.queries[0][:query_range], context(serverless_function_id)) + end end protected diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 340ec75c5f1..942f90e8040 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -234,7 +234,7 @@ module Gitlab "#{comment} #{SHRUG}" end - desc _("Append the comment with %{TABLEFLIP}") % { tableflip: TABLEFLIP } + desc _("Append the comment with %{tableflip}") % { tableflip: TABLEFLIP } params '<Comment>' types Issuable substitution :tableflip do |comment| diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 404e0c31871..838aefb59f0 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -174,18 +174,14 @@ module Gitlab params '<Zoom URL>' types Issue condition do - zoom_link_service.can_add_link? + @zoom_service = zoom_link_service + @zoom_service.can_add_link? end parse_params do |link| - zoom_link_service.parse_link(link) + @zoom_service.parse_link(link) end command :zoom do |link| - result = zoom_link_service.add_link(link) - - if result.success? - @updates[:description] = result.payload[:description] - end - + result = @zoom_service.add_link(link) @execution_message[:zoom] = result.message end @@ -194,15 +190,11 @@ module Gitlab execution_message _('Zoom meeting removed') types Issue condition do - zoom_link_service.can_remove_link? + @zoom_service = zoom_link_service + @zoom_service.can_remove_link? end command :remove_zoom do - result = zoom_link_service.remove_link - - if result.success? - @updates[:description] = result.payload[:description] - end - + result = @zoom_service.remove_link @execution_message[:remove_zoom] = result.message end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index fa1615a5953..412d00c6939 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -25,6 +25,8 @@ module Gitlab if Sidekiq.server? # the pool will be used in a multi-threaded context size += Sidekiq.options[:concurrency] + elsif defined?(::Puma) + size += Puma.cli_config.options[:max_threads] end size diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 3d1f15c72ae..e3a434dfe35 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -120,13 +120,26 @@ module Gitlab @breakline_regex ||= /\r\n|\r|\n/ end + # https://docs.aws.amazon.com/general/latest/gr/acct-identifiers.html + def aws_account_id_regex + /\A\d{12}\z/ + end + + def aws_account_id_message + 'must be a 12-digit number' + end + # https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html def aws_arn_regex /\Aarn:\S+\z/ end def aws_arn_regex_message - "must be a valid Amazon Resource Name" + 'must be a valid Amazon Resource Name' + end + + def utc_date_regex + @utc_date_regex ||= /\A[0-9]{4}-[0-9]{2}-[0-9]{2}\z/.freeze end end end diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb index fa09ecbdf30..360239a84e4 100644 --- a/lib/gitlab/search/found_blob.rb +++ b/lib/gitlab/search/found_blob.rb @@ -8,20 +8,20 @@ module Gitlab include BlobLanguageFromGitAttributes include Gitlab::Utils::StrongMemoize - attr_reader :project, :content_match, :blob_filename + attr_reader :project, :content_match, :blob_path - FILENAME_REGEXP = /\A(?<ref>[^:]*):(?<filename>[^\x00]*)\x00/.freeze - CONTENT_REGEXP = /^(?<ref>[^:]*):(?<filename>[^\x00]*)\x00(?<startline>\d+)\x00/.freeze + PATH_REGEXP = /\A(?<ref>[^:]*):(?<path>[^\x00]*)\x00/.freeze + CONTENT_REGEXP = /^(?<ref>[^:]*):(?<path>[^\x00]*)\x00(?<startline>\d+)\x00/.freeze def self.preload_blobs(blobs) - to_fetch = blobs.select { |blob| blob.is_a?(self) && blob.blob_filename } + to_fetch = blobs.select { |blob| blob.is_a?(self) && blob.blob_path } to_fetch.each { |blob| blob.fetch_blob } end def initialize(opts = {}) @id = opts.fetch(:id, nil) - @binary_filename = opts.fetch(:filename, nil) + @binary_path = opts.fetch(:path, nil) @binary_basename = opts.fetch(:basename, nil) @ref = opts.fetch(:ref, nil) @startline = opts.fetch(:startline, nil) @@ -34,7 +34,7 @@ module Gitlab # Allow those to just pass project_id instead. @project_id = opts.fetch(:project_id, nil) @content_match = opts.fetch(:content_match, nil) - @blob_filename = opts.fetch(:blob_filename, nil) + @blob_path = opts.fetch(:blob_path, nil) @repository = opts.fetch(:repository, nil) end @@ -50,16 +50,16 @@ module Gitlab @startline ||= parsed_content[:startline] end - # binary_filename is used for running filters on all matches, - # for grepped results (which use content_match), we get - # filename from the beginning of the grepped result which is faster - # then parsing whole snippet - def binary_filename - @binary_filename ||= content_match ? search_result_filename : parsed_content[:binary_filename] + # binary_path is used for running filters on all matches. + # For grepped results (which use content_match), we get + # the path from the beginning of the grepped result which is faster + # than parsing the whole snippet + def binary_path + @binary_path ||= content_match ? search_result_path : parsed_content[:binary_path] end - def filename - @filename ||= encode_utf8(@binary_filename || parsed_content[:binary_filename]) + def path + @path ||= encode_utf8(@binary_path || parsed_content[:binary_path]) end def basename @@ -70,10 +70,6 @@ module Gitlab @data ||= encode_utf8(@binary_data || parsed_content[:binary_data]) end - def path - filename - end - def project_id @project_id || @project&.id end @@ -83,16 +79,16 @@ module Gitlab end def fetch_blob - path = [ref, blob_filename] - missing_blob = { binary_filename: blob_filename } + path = [ref, blob_path] + missing_blob = { binary_path: blob_path } BatchLoader.for(path).batch(default_value: missing_blob) do |refs, loader| Gitlab::Git::Blob.batch(repository, refs, blob_size_limit: 1024).each do |blob| # if the blob couldn't be fetched for some reason, - # show at least the blob filename + # show at least the blob path data = { id: blob.id, - binary_filename: blob.path, + binary_path: blob.path, binary_basename: path_without_extension(blob.path), ref: ref, startline: 1, @@ -107,8 +103,8 @@ module Gitlab private - def search_result_filename - content_match.match(FILENAME_REGEXP) { |matches| matches[:filename] } + def search_result_path + content_match.match(PATH_REGEXP) { |matches| matches[:path] } end def path_without_extension(path) @@ -119,7 +115,7 @@ module Gitlab strong_memoize(:parsed_content) do if content_match parse_search_result - elsif blob_filename + elsif blob_path fetch_blob else {} @@ -129,7 +125,7 @@ module Gitlab def parse_search_result ref = nil - filename = nil + path = nil basename = nil data = [] @@ -138,17 +134,17 @@ module Gitlab content_match.each_line.each_with_index do |line, index| prefix ||= line.match(CONTENT_REGEXP)&.tap do |matches| ref = matches[:ref] - filename = matches[:filename] + path = matches[:path] startline = matches[:startline] startline = startline.to_i - index - basename = path_without_extension(filename) + basename = path_without_extension(path) end data << line.sub(prefix.to_s, '') end { - binary_filename: filename, + binary_path: path, binary_basename: basename, ref: ref, startline: startline, diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 8e2f16271eb..f96346322db 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -14,7 +14,71 @@ end module Gitlab class Seeder + extend ActionView::Helpers::NumberHelper + + ESTIMATED_INSERT_PER_MINUTE = 2_000_000 + MASS_INSERT_ENV = 'MASS_INSERT' + + module ProjectSeed + extend ActiveSupport::Concern + + included do + scope :not_mass_generated, -> do + where.not("path LIKE '#{Gitlab::Seeder::Projects::MASS_INSERT_NAME_START}%'") + end + end + end + + module UserSeed + extend ActiveSupport::Concern + + included do + scope :not_mass_generated, -> do + where.not("username LIKE '#{Gitlab::Seeder::Users::MASS_INSERT_USERNAME_START}%'") + end + end + end + + def self.with_mass_insert(size, model) + humanized_model_name = model.is_a?(String) ? model : model.model_name.human.pluralize(size) + + if !ENV[MASS_INSERT_ENV] && !ENV['CI'] + puts "\nSkipping mass insertion for #{humanized_model_name}." + puts "Consider running the seed with #{MASS_INSERT_ENV}=1" + return + end + + humanized_size = number_with_delimiter(size) + estimative = estimated_time_message(size) + + puts "\nCreating #{humanized_size} #{humanized_model_name}." + puts estimative + + yield + + puts "\n#{number_with_delimiter(size)} #{humanized_model_name} created!" + end + + def self.estimated_time_message(size) + estimated_minutes = (size.to_f / ESTIMATED_INSERT_PER_MINUTE).round + humanized_minutes = 'minute'.pluralize(estimated_minutes) + + if estimated_minutes.zero? + "Rough estimated time: less than a minute ⏰" + else + "Rough estimated time: #{estimated_minutes} #{humanized_minutes} ⏰" + end + end + def self.quiet + # Disable database insertion logs so speed isn't limited by ability to print to console + old_logger = ActiveRecord::Base.logger + ActiveRecord::Base.logger = nil + + # Additional seed logic for models. + Project.include(ProjectSeed) + User.include(UserSeed) + mute_notifications mute_mailer @@ -23,6 +87,7 @@ module Gitlab yield SeedFu.quiet = false + ActiveRecord::Base.logger = old_logger puts "\nOK".color(:green) end diff --git a/lib/gitlab/serializer/pagination.rb b/lib/gitlab/serializer/pagination.rb index eb242cc7c20..bb7571dd66a 100644 --- a/lib/gitlab/serializer/pagination.rb +++ b/lib/gitlab/serializer/pagination.rb @@ -4,7 +4,6 @@ module Gitlab module Serializer class Pagination InvalidResourceError = Class.new(StandardError) - include ::API::Helpers::Pagination def initialize(request, response) @request = request @@ -13,13 +12,13 @@ module Gitlab def paginate(resource) if resource.respond_to?(:page) - super(resource) + ::Gitlab::Pagination::OffsetPagination.new(self).paginate(resource) else raise InvalidResourceError end end - # Methods needed by `API::Helpers::Pagination` + # Methods needed by `Gitlab::Pagination::OffsetPagination` # attr_reader :request diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 0d3e78c0a66..c449c6879bc 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -40,6 +40,11 @@ module Gitlab config = { socket_path: address.sub(/\Aunix:/, ''), storage: storages } config[:auth] = { token: 'secret' } if Rails.env.test? + + internal_socket_dir = File.join(gitaly_dir, 'internal_sockets') + FileUtils.mkdir(internal_socket_dir) unless File.exist?(internal_socket_dir) + config[:internal_socket_dir] = internal_socket_dir + config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } config[:bin_dir] = Gitlab.config.gitaly.client_path diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 125d0d1cfbb..28e5d0ba8f5 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -285,18 +285,6 @@ module Gitlab end end - # Check if such directory exists in repositories. - # - # Usage: - # exists?(storage, 'gitlab') - # exists?(storage, 'gitlab/cookies.git') - # - # rubocop: disable CodeReuse/ActiveRecord - def exists?(storage, dir_name) - Gitlab::GitalyClient::NamespaceService.new(storage).exists?(dir_name) - end - # rubocop: enable CodeReuse/ActiveRecord - def repository_exists?(storage, dir_name) Gitlab::Git::Repository.new(storage, dir_name, nil, nil).exists? rescue GRPC::Internal diff --git a/lib/gitlab/sidekiq_daemon/monitor.rb b/lib/gitlab/sidekiq_daemon/monitor.rb index a3d61c69ae1..0723b514c90 100644 --- a/lib/gitlab/sidekiq_daemon/monitor.rb +++ b/lib/gitlab/sidekiq_daemon/monitor.rb @@ -4,6 +4,7 @@ module Gitlab module SidekiqDaemon class Monitor < Daemon include ::Gitlab::Utils::StrongMemoize + extend ::Gitlab::Utils::Override NOTIFICATION_CHANNEL = 'sidekiq:cancel:notifications' CANCEL_DEADLINE = 24.hours.seconds @@ -24,6 +25,11 @@ module Gitlab @jobs_mutex = Mutex.new end + override :thread_name + def thread_name + "job_monitor" + end + def within_job(worker_class, jid, queue) jobs_mutex.synchronize do jobs[jid] = { worker_class: worker_class, thread: Thread.current, started_at: Gitlab::Metrics::System.monotonic_time } diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 853fb2777c3..ca9e3b8428c 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -36,11 +36,8 @@ module Gitlab payload['message'] = "#{base_message(payload)}: start" payload['job_status'] = 'start' - # Old gitlab-shell messages don't provide enqueued_at/created_at attributes - enqueued_at = payload['enqueued_at'] || payload['created_at'] - if enqueued_at - payload['scheduling_latency_s'] = elapsed_by_absolute_time(Time.iso8601(enqueued_at)) - end + scheduling_latency_s = ::Gitlab::InstrumentationHelper.queue_duration_for_job(payload) + payload['scheduling_latency_s'] = scheduling_latency_s if scheduling_latency_s payload end @@ -98,10 +95,6 @@ module Gitlab end end - def elapsed_by_absolute_time(start) - (Time.now.utc - start).to_f.round(6) - end - def elapsed(t0) t1 = get_time { diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb index 8af353d8674..bd819843bd4 100644 --- a/lib/gitlab/sidekiq_middleware/metrics.rb +++ b/lib/gitlab/sidekiq_middleware/metrics.rb @@ -9,43 +9,56 @@ module Gitlab def initialize @metrics = init_metrics + + @metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) end def call(_worker, job, queue) labels = create_labels(queue) + queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) + + @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration @metrics[:sidekiq_running_jobs].increment(labels, 1) if job['retry_count'].present? @metrics[:sidekiq_jobs_retried_total].increment(labels, 1) end + job_succeeded = false + monotonic_time_start = Gitlab::Metrics::System.monotonic_time job_thread_cputime_start = get_thread_cputime - - realtime = Benchmark.realtime do + begin yield - end + job_succeeded = true + ensure + monotonic_time_end = Gitlab::Metrics::System.monotonic_time + job_thread_cputime_end = get_thread_cputime + + monotonic_time = monotonic_time_end - monotonic_time_start + job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start - job_thread_cputime_end = get_thread_cputime - job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start - @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) + # sidekiq_running_jobs, sidekiq_jobs_failed_total should not include the job_status label + @metrics[:sidekiq_running_jobs].increment(labels, -1) + @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded - @metrics[:sidekiq_jobs_completion_seconds].observe(labels, realtime) - rescue Exception # rubocop: disable Lint/RescueException - @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) - raise - ensure - @metrics[:sidekiq_running_jobs].increment(labels, -1) + # job_status: done, fail match the job_status attribute in structured logging + labels[:job_status] = job_succeeded ? :done : :fail + @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) + @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time) + end end private def init_metrics { - sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), - sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), - sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :livesum) + sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), + sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), + sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all) } end diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index 079b5916566..239479f99d2 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -10,6 +10,7 @@ module Gitlab Gitlab::SlashCommands::IssueSearch, Gitlab::SlashCommands::IssueMove, Gitlab::SlashCommands::IssueClose, + Gitlab::SlashCommands::IssueComment, Gitlab::SlashCommands::Deploy, Gitlab::SlashCommands::Run ] diff --git a/lib/gitlab/slash_commands/issue_comment.rb b/lib/gitlab/slash_commands/issue_comment.rb new file mode 100644 index 00000000000..cbb9c41aab0 --- /dev/null +++ b/lib/gitlab/slash_commands/issue_comment.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + class IssueComment < IssueCommand + def self.match(text) + /\Aissue\s+comment\s+#{Issue.reference_prefix}?(?<iid>\d+)\n*(?<note_body>(.|\n)*)/.match(text) + end + + def self.help_message + 'issue comment <id> *`⇧ Shift`*+*`↵ Enter`* <comment>' + end + + def execute(match) + note_body = match[:note_body].to_s.strip + issue = find_by_iid(match[:iid]) + + return not_found unless issue + return access_denied unless can_create_note?(issue) + + note = create_note(issue: issue, note: note_body) + + if note.persisted? + presenter(note).present + else + presenter(note).display_errors + end + end + + private + + def can_create_note?(issue) + Ability.allowed?(current_user, :create_note, issue) + end + + def not_found + Gitlab::SlashCommands::Presenters::Access.new.not_found + end + + def access_denied + Gitlab::SlashCommands::Presenters::Access.new.generic_access_denied + end + + def create_note(issue:, note:) + note_params = { noteable: issue, note: note } + + Notes::CreateService.new(project, current_user, note_params).execute + end + + def presenter(note) + Gitlab::SlashCommands::Presenters::IssueComment.new(note) + end + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb index 9ce1bcfb37c..fbc3cf2e049 100644 --- a/lib/gitlab/slash_commands/presenters/access.rb +++ b/lib/gitlab/slash_commands/presenters/access.rb @@ -15,6 +15,10 @@ module Gitlab MESSAGE end + def generic_access_denied + ephemeral_response(text: 'You are not allowed to perform the given chatops command.') + end + def deactivated ephemeral_response(text: <<~MESSAGE) You are not allowed to perform the given chatops command since diff --git a/lib/gitlab/slash_commands/presenters/issue_comment.rb b/lib/gitlab/slash_commands/presenters/issue_comment.rb new file mode 100644 index 00000000000..cce71e23b21 --- /dev/null +++ b/lib/gitlab/slash_commands/presenters/issue_comment.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module Presenters + class IssueComment < Presenters::Base + include Presenters::NoteBase + + def present + ephemeral_response(new_note) + end + + private + + def new_note + { + attachments: [ + { + title: "#{issue.title} · #{issue.to_reference}", + title_link: resource_url, + author_name: author.name, + author_icon: author.avatar_url, + fallback: "New comment on #{issue.to_reference}: #{issue.title}", + pretext: pretext, + color: color, + fields: fields, + mrkdwn_in: [ + :title, + :pretext, + :fields + ] + } + ] + } + end + + def pretext + "I commented on an issue on #{author_profile_link}'s behalf: *#{issue.to_reference}* in #{project_link}" + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/note_base.rb b/lib/gitlab/slash_commands/presenters/note_base.rb new file mode 100644 index 00000000000..7758fc740de --- /dev/null +++ b/lib/gitlab/slash_commands/presenters/note_base.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module Presenters + module NoteBase + GREEN = '#38ae67' + + def color + GREEN + end + + def issue + resource.noteable + end + + def project + issue.project + end + + def project_link + "[#{project.full_name}](#{project.web_url})" + end + + def author + resource.author + end + + def author_profile_link + "[#{author.to_reference}](#{url_for(author)})" + end + + def fields + [ + { + title: 'Comment', + value: resource.note + } + ] + end + + private + + attr_reader :resource + end + end + end +end diff --git a/lib/gitlab/sourcegraph.rb b/lib/gitlab/sourcegraph.rb new file mode 100644 index 00000000000..d0f12c8364a --- /dev/null +++ b/lib/gitlab/sourcegraph.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + class Sourcegraph + class << self + def feature_conditional? + feature.conditional? + end + + def feature_available? + # The sourcegraph_bundle feature could be conditionally applied, so check if `!off?` + !feature.off? + end + + def feature_enabled?(thing = nil) + feature.enabled?(thing) + end + + private + + def feature + Feature.get(:sourcegraph) + end + end + end +end diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index f05592fc3a3..b15f2ca385a 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -29,7 +29,7 @@ module Gitlab end if fragments.any? - fragments.join("\n#{union_keyword}\n") + "(" + fragments.join(")\n#{union_keyword}\n(") + ")" else 'NULL' end diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 8532845f3cb..ac02ec635e4 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -158,15 +158,17 @@ module Gitlab end def checkout_or_clone_version(version:, repo:, target_dir:) - version = - if version.starts_with?("=") - version.sub(/\A=/, '') # tag or branch - else - "v#{version}" # tag - end - clone_repo(repo, target_dir) unless Dir.exist?(target_dir) - checkout_version(version, target_dir) + checkout_version(get_version(version), target_dir) + end + + # this function implements the same logic we have in omnibus for dealing with components version + def get_version(component_version) + # If not a valid version string following SemVer it is probably a branch name or a SHA + # commit of one of our own component so it doesn't need `v` prepended + return component_version unless /^\d+\.\d+\.\d+(-rc\d+)?$/.match?(component_version) + + "v#{component_version}" end def clone_repo(repo, target_dir) diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 2470685bc00..91e2ff0b10d 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -45,9 +45,10 @@ module Gitlab namespace: SNOWPLOW_NAMESPACE, hostname: Gitlab::CurrentSettings.snowplow_collector_hostname, cookie_domain: Gitlab::CurrentSettings.snowplow_cookie_domain, - app_id: Gitlab::CurrentSettings.snowplow_site_id, + app_id: Gitlab::CurrentSettings.snowplow_app_id, form_tracking: additional_features, - link_click_tracking: additional_features + link_click_tracking: additional_features, + iglu_registry_url: Gitlab::CurrentSettings.snowplow_iglu_registry_url }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym } end @@ -58,7 +59,7 @@ module Gitlab SnowplowTracker::AsyncEmitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname, protocol: 'https'), SnowplowTracker::Subject.new, SNOWPLOW_NAMESPACE, - Gitlab::CurrentSettings.snowplow_site_id + Gitlab::CurrentSettings.snowplow_app_id ) end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index cb492b69fec..b6effac25c6 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -13,7 +13,8 @@ module Gitlab end def uncached_data - license_usage_data.merge(system_usage_data) + license_usage_data + .merge(system_usage_data) .merge(features_usage_data) .merge(components_usage_data) .merge(cycle_analytics_usage_data) @@ -66,17 +67,23 @@ module Gitlab clusters_disabled: count(::Clusters::Cluster.disabled), project_clusters_disabled: count(::Clusters::Cluster.disabled.project_type), group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type), + clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled), clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), clusters_applications_helm: count(::Clusters::Applications::Helm.available), clusters_applications_ingress: count(::Clusters::Applications::Ingress.available), clusters_applications_cert_managers: count(::Clusters::Applications::CertManager.available), + clusters_applications_crossplane: count(::Clusters::Applications::Crossplane.available), clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.available), clusters_applications_runner: count(::Clusters::Applications::Runner.available), clusters_applications_knative: count(::Clusters::Applications::Knative.available), + clusters_applications_elastic_stack: count(::Clusters::Applications::ElasticStack.available), in_review_folder: count(::Environment.in_review_folder), + grafana_integrated_projects: count(GrafanaIntegration.enabled), groups: count(Group), issues: count(Issue), + issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue), + issues_using_zoom_quick_actions: count(ZoomMeeting.select(:issue_id).distinct), keys: count(Key), label_lists: count(List.label), lfs_objects: count(LfsObject), @@ -127,7 +134,9 @@ module Gitlab omniauth_enabled: Gitlab::Auth.omniauth_enabled?, prometheus_metrics_enabled: Gitlab::Metrics.prometheus_metrics_enabled?, reply_by_email_enabled: Gitlab::IncomingEmail.enabled?, - signup_enabled: Gitlab::CurrentSettings.allow_signup? + signup_enabled: Gitlab::CurrentSettings.allow_signup?, + web_ide_clientside_preview_enabled: Gitlab::CurrentSettings.web_ide_clientside_preview_enabled?, + ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity) } end @@ -165,10 +174,13 @@ module Gitlab types = { SlackService: :projects_slack_notifications_active, SlackSlashCommandsService: :projects_slack_slash_active, - PrometheusService: :projects_prometheus_active + PrometheusService: :projects_prometheus_active, + CustomIssueTrackerService: :projects_custom_issue_tracker_active, + JenkinsService: :projects_jenkins_active, + MattermostService: :projects_mattermost_active } - results = count(Service.unscoped.where(type: types.keys, active: true).group(:type), fallback: Hash.new(-1)) + results = count(Service.active.by_type(types.keys).group(:type), fallback: Hash.new(-1)) types.each_with_object({}) { |(klass, key), response| response[key] = results[klass.to_s] || 0 } .merge(jira_usage) end @@ -183,8 +195,8 @@ module Gitlab projects_jira_active: -1 } - Service.unscoped - .where(type: :JiraService, active: true) + Service.active + .by_type(:JiraService) .includes(:jira_tracker_data) .find_in_batches(batch_size: BATCH_SIZE) do |services| diff --git a/lib/gitlab/usage_data_counters/web_ide_counter.rb b/lib/gitlab/usage_data_counters/web_ide_counter.rb index 0718c1dd761..c012a6c96df 100644 --- a/lib/gitlab/usage_data_counters/web_ide_counter.rb +++ b/lib/gitlab/usage_data_counters/web_ide_counter.rb @@ -8,6 +8,7 @@ module Gitlab COMMITS_COUNT_KEY = 'WEB_IDE_COMMITS_COUNT' MERGE_REQUEST_COUNT_KEY = 'WEB_IDE_MERGE_REQUESTS_COUNT' VIEWS_COUNT_KEY = 'WEB_IDE_VIEWS_COUNT' + PREVIEW_COUNT_KEY = 'WEB_IDE_PREVIEWS_COUNT' class << self def increment_commits_count @@ -34,11 +35,22 @@ module Gitlab total_count(VIEWS_COUNT_KEY) end + def increment_previews_count + return unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? + + increment(PREVIEW_COUNT_KEY) + end + + def total_previews_count + total_count(PREVIEW_COUNT_KEY) + end + def totals { web_ide_commits: total_commits_count, web_ide_views: total_views_count, - web_ide_merge_requests: total_merge_requests_count + web_ide_merge_requests: total_merge_requests_count, + web_ide_previews: total_previews_count } end end diff --git a/lib/gitlab/utils/deep_size.rb b/lib/gitlab/utils/deep_size.rb index 562cf09e249..ed2ceb8af7c 100644 --- a/lib/gitlab/utils/deep_size.rb +++ b/lib/gitlab/utils/deep_size.rb @@ -25,6 +25,10 @@ module Gitlab !too_big? && !too_deep? end + def self.human_default_max_size + ActiveSupport::NumberHelper.number_to_human_size(DEFAULT_MAX_SIZE) + end + private def evaluate diff --git a/lib/gitlab/wiki_file_finder.rb b/lib/gitlab/wiki_file_finder.rb index e9be6db50da..a963cc7954f 100644 --- a/lib/gitlab/wiki_file_finder.rb +++ b/lib/gitlab/wiki_file_finder.rb @@ -12,12 +12,12 @@ module Gitlab private - def search_filenames(query) + def search_paths(query) safe_query = Regexp.escape(query.tr(' ', '-')) safe_query = Regexp.new(safe_query, Regexp::IGNORECASE) - filenames = repository.ls_files(ref) + paths = repository.ls_files(ref) - filenames.grep(safe_query) + paths.grep(safe_query) end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index db67e4fd479..713ca31bbc5 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -14,6 +14,7 @@ module Gitlab NOTIFICATION_CHANNEL = 'workhorse:notifications' ALLOWED_GIT_HTTP_ACTIONS = %w[git_receive_pack git_upload_pack info_refs].freeze DETECT_HEADER = 'Gitlab-Workhorse-Detect-Content-Type' + ARCHIVE_FORMATS = %w(zip tar.gz tar.bz2 tar).freeze include JwtAuthenticatable diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 9085835dee6..99029b54a69 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -12,6 +12,7 @@ module GoogleApi SCOPE = 'https://www.googleapis.com/auth/cloud-platform' LEAST_TOKEN_LIFE_TIME = 10.minutes CLUSTER_MASTER_AUTH_USERNAME = 'admin' + CLUSTER_IPV4_CIDR_BLOCK = '/16' class << self def session_key_for_token @@ -97,7 +98,8 @@ module GoogleApi enabled: legacy_abac }, ip_allocation_policy: { - use_ip_aliases: true + use_ip_aliases: true, + cluster_ipv4_cidr_block: CLUSTER_IPV4_CIDR_BLOCK }, addons_config: enable_addons.each_with_object({}) do |addon, hash| hash[addon] = { disabled: false } diff --git a/lib/grafana/client.rb b/lib/grafana/client.rb index 0765630f9bb..b419f79bace 100644 --- a/lib/grafana/client.rb +++ b/lib/grafana/client.rb @@ -11,6 +11,18 @@ module Grafana @token = token end + # @param uid [String] Unique identifier for a Grafana dashboard + def get_dashboard(uid:) + http_get("#{@api_url}/api/dashboards/uid/#{uid}") + end + + # @param name [String] Unique identifier for a Grafana datasource + def get_datasource(name:) + # CGI#escape formats strings such that the Grafana endpoint + # will not recognize the dashboard name. Preferring URI#escape. + http_get("#{@api_url}/api/datasources/name/#{URI.escape(name)}") # rubocop:disable Lint/UriEscapeUnescape + end + # @param datasource_id [String] Grafana ID for the datasource # @param proxy_path [String] Path to proxy - ex) 'api/v1/query_range' def proxy_datasource(datasource_id:, proxy_path:, query: {}) @@ -57,7 +69,7 @@ module Grafana def handle_response(response) return response if response.code == 200 - raise_error "Grafana response status code: #{response.code}" + raise_error "Grafana response status code: #{response.code}, Message: #{response.body}" end def raise_error(message) diff --git a/lib/prometheus/pid_provider.rb b/lib/prometheus/pid_provider.rb index e0f7e7e0a9e..228639357ac 100644 --- a/lib/prometheus/pid_provider.rb +++ b/lib/prometheus/pid_provider.rb @@ -6,7 +6,7 @@ module Prometheus def worker_id if Sidekiq.server? - 'sidekiq' + sidekiq_worker_id elsif defined?(Unicorn::Worker) unicorn_worker_id elsif defined?(::Puma) @@ -18,6 +18,14 @@ module Prometheus private + def sidekiq_worker_id + if worker = ENV['SIDEKIQ_WORKER_ID'] + "sidekiq_#{worker}" + else + 'sidekiq' + end + end + def unicorn_worker_id if matches = process_name.match(/unicorn.*worker\[([0-9]+)\]/) "unicorn_#{matches[1]}" diff --git a/lib/quality/kubernetes_client.rb b/lib/quality/kubernetes_client.rb index 190b48ba7cb..cc899bf9374 100644 --- a/lib/quality/kubernetes_client.rb +++ b/lib/quality/kubernetes_client.rb @@ -12,7 +12,16 @@ module Quality @namespace = namespace end - def cleanup(release_name:) + def cleanup(release_name:, wait: true) + selector = case release_name + when String + %(-l release="#{release_name}") + when Array + %(-l 'release in (#{release_name.join(', ')})') + else + raise ArgumentError, 'release_name must be a string or an array' + end + command = [ %(--namespace "#{namespace}"), 'delete', @@ -20,7 +29,8 @@ module Quality '--now', '--ignore-not-found', '--include-uninitialized', - %(-l release="#{release_name}") + %(--wait=#{wait}), + selector ] run_command(command) diff --git a/lib/sentry/client.rb b/lib/sentry/client.rb index 07cca1c8d1e..6191d69c870 100644 --- a/lib/sentry/client.rb +++ b/lib/sentry/client.rb @@ -4,6 +4,7 @@ module Sentry class Client Error = Class.new(StandardError) MissingKeysError = Class.new(StandardError) + ResponseInvalidSizeError = Class.new(StandardError) attr_accessor :url, :token @@ -12,9 +13,23 @@ module Sentry @token = token end + def issue_details(issue_id:) + issue = get_issue(issue_id: issue_id) + + map_to_detailed_error(issue) + end + + def issue_latest_event(issue_id:) + latest_event = get_issue_latest_event(issue_id: issue_id) + + map_to_event(latest_event) + end + def list_issues(issue_status:, limit:) issues = get_issues(issue_status: issue_status, limit: limit) + validate_size(issues) + handle_mapping_exceptions do map_to_errors(issues) end @@ -30,6 +45,12 @@ module Sentry private + def validate_size(issues) + return if Gitlab::Utils::DeepSize.new(issues).valid? + + raise Client::ResponseInvalidSizeError, "Sentry API response is too big. Limit is #{Gitlab::Utils::DeepSize.human_default_max_size}." + end + def handle_mapping_exceptions(&block) yield rescue KeyError => e @@ -61,6 +82,14 @@ module Sentry }) end + def get_issue(issue_id:) + http_get(issue_api_url(issue_id)) + end + + def get_issue_latest_event(issue_id:) + http_get(issue_latest_event_api_url(issue_id)) + end + def get_projects http_get(projects_api_url) end @@ -88,7 +117,7 @@ module Sentry raise_error "Sentry response status code: #{response.code}" end - response + response.parsed_response end def raise_error(message) @@ -102,6 +131,20 @@ module Sentry projects_url end + def issue_api_url(issue_id) + issue_url = URI(@url) + issue_url.path = "/api/0/issues/#{issue_id}/" + + issue_url + end + + def issue_latest_event_api_url(issue_id) + latest_event_url = URI(@url) + latest_event_url.path = "/api/0/issues/#{issue_id}/events/latest/" + + latest_event_url + end + def issues_api_url issues_url = URI(@url + '/issues/') issues_url.path.squeeze!('/') @@ -119,38 +162,87 @@ module Sentry def issue_url(id) issues_url = @url + "/issues/#{id}" - issues_url = ErrorTracking::ProjectErrorTrackingSetting.extract_sentry_external_url(issues_url) - uri = URI(issues_url) + parse_sentry_url(issues_url) + end + + def project_url + parse_sentry_url(@url) + end + + def parse_sentry_url(api_url) + url = ErrorTracking::ProjectErrorTrackingSetting.extract_sentry_external_url(api_url) + + uri = URI(url) uri.path.squeeze!('/') + # Remove trailing spaces + uri = uri.to_s.gsub(/\/\z/, '') - uri.to_s + uri end - def map_to_error(issue) - id = issue.fetch('id') + def map_to_event(event) + stack_trace = parse_stack_trace(event) - count = issue.fetch('count', nil) + Gitlab::ErrorTracking::ErrorEvent.new( + issue_id: event.dig('groupID'), + date_received: event.dig('dateReceived'), + stack_trace_entries: stack_trace + ) + end - frequency = issue.dig('stats', '24h') - message = issue.dig('metadata', 'value') + def parse_stack_trace(event) + exception_entry = event.dig('entries')&.detect { |h| h['type'] == 'exception' } + return unless exception_entry - external_url = issue_url(id) + exception_values = exception_entry.dig('data', 'values') + stack_trace_entry = exception_values&.detect { |h| h['stacktrace'].present? } + return unless stack_trace_entry + + stack_trace_entry.dig('stacktrace', 'frames') + end + + def map_to_detailed_error(issue) + Gitlab::ErrorTracking::DetailedError.new( + id: issue.fetch('id'), + first_seen: issue.fetch('firstSeen', nil), + last_seen: issue.fetch('lastSeen', nil), + title: issue.fetch('title', nil), + type: issue.fetch('type', nil), + user_count: issue.fetch('userCount', nil), + count: issue.fetch('count', nil), + message: issue.dig('metadata', 'value'), + culprit: issue.fetch('culprit', nil), + external_url: issue_url(issue.fetch('id')), + external_base_url: project_url, + short_id: issue.fetch('shortId', nil), + status: issue.fetch('status', nil), + frequency: issue.dig('stats', '24h'), + project_id: issue.dig('project', 'id'), + project_name: issue.dig('project', 'name'), + project_slug: issue.dig('project', 'slug'), + first_release_last_commit: issue.dig('firstRelease', 'lastCommit'), + last_release_last_commit: issue.dig('lastRelease', 'lastCommit'), + first_release_short_version: issue.dig('firstRelease', 'shortVersion'), + last_release_short_version: issue.dig('lastRelease', 'shortVersion') + ) + end + def map_to_error(issue) Gitlab::ErrorTracking::Error.new( - id: id, + id: issue.fetch('id'), first_seen: issue.fetch('firstSeen', nil), last_seen: issue.fetch('lastSeen', nil), title: issue.fetch('title', nil), type: issue.fetch('type', nil), user_count: issue.fetch('userCount', nil), - count: count, - message: message, + count: issue.fetch('count', nil), + message: issue.dig('metadata', 'value'), culprit: issue.fetch('culprit', nil), - external_url: external_url, + external_url: issue_url(issue.fetch('id')), short_id: issue.fetch('shortId', nil), status: issue.fetch('status', nil), - frequency: frequency, + frequency: issue.dig('stats', '24h'), project_id: issue.dig('project', 'id'), project_name: issue.dig('project', 'name'), project_slug: issue.dig('project', 'slug') diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index b1db4dc94a6..0488f26318a 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -5,6 +5,10 @@ namespace :dev do task setup: :environment do ENV['force'] = 'yes' Rake::Task["gitlab:setup"].invoke + + # Make sure DB statistics are up to date. + ActiveRecord::Base.connection.execute('ANALYZE') + Rake::Task["gitlab:shell:setup"].invoke end diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake index 902f22684ee..f8ce3cd46a8 100644 --- a/lib/tasks/gitlab/graphql.rake +++ b/lib/tasks/gitlab/graphql.rake @@ -2,10 +2,24 @@ return if Rails.env.production? +require 'graphql/rake_task' + namespace :gitlab do OUTPUT_DIR = Rails.root.join("doc/api/graphql/reference") TEMPLATES_DIR = 'lib/gitlab/graphql/docs/templates/' + # Defines tasks for dumping the GraphQL schema: + # - gitlab:graphql:schema:dump + # - gitlab:graphql:schema:idl + # - gitlab:graphql:schema:json + GraphQL::RakeTask.new( + schema_name: 'GitlabSchema', + dependencies: [:environment], + directory: OUTPUT_DIR, + idl_outfile: "gitlab_schema.graphql", + json_outfile: "gitlab_schema.json" + ) + namespace :graphql do desc 'GitLab | Generate GraphQL docs' task compile_docs: :environment do @@ -25,11 +39,20 @@ namespace :gitlab do if doc == renderer.contents puts "GraphQL documentation is up to date" else - puts '#' * 10 - puts '#' - puts '# GraphQL documentation is outdated! Please update it by running `bundle exec rake gitlab:graphql:compile_docs`.' - puts '#' - puts '#' * 10 + format_output('GraphQL documentation is outdated! Please update it by running `bundle exec rake gitlab:graphql:compile_docs`.') + abort + end + end + + desc 'GitLab | Check if GraphQL schemas are up to date' + task check_schema: :environment do + idl_doc = File.read(Rails.root.join(OUTPUT_DIR, 'gitlab_schema.graphql')) + json_doc = File.read(Rails.root.join(OUTPUT_DIR, 'gitlab_schema.json')) + + if idl_doc == GitlabSchema.to_definition && json_doc == GitlabSchema.to_json + puts "GraphQL schema is up to date" + else + format_output('GraphQL schema is outdated! Please update it by running `bundle exec rake gitlab:graphql:schema:dump`.') abort end end @@ -42,3 +65,12 @@ def render_options template: Rails.root.join(TEMPLATES_DIR, 'default.md.haml') } end + +def format_output(str) + heading = '#' * 10 + puts heading + puts '#' + puts "# #{str}" + puts '#' + puts heading +end diff --git a/lib/tasks/gitlab/seed.rake b/lib/tasks/gitlab/seed.rake index d76e38b73b5..d758280ba69 100644 --- a/lib/tasks/gitlab/seed.rake +++ b/lib/tasks/gitlab/seed.rake @@ -22,7 +22,7 @@ namespace :gitlab do [project] else - Project.find_each + Project.not_mass_generated.find_each end projects.each do |project| diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index abd47f018f1..a592015963d 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -43,7 +43,7 @@ namespace :gitlab do [ %w(bin/install) + repository_storage_paths_args, - %w(bin/compile) + %w(make build) ].each do |cmd| unless Kernel.system(*cmd) raise "command failed: #{cmd.join(' ')}" diff --git a/lib/tasks/gitlab/uploads/legacy.rake b/lib/tasks/gitlab/uploads/legacy.rake index 2eeb694d341..74db0060b8d 100644 --- a/lib/tasks/gitlab/uploads/legacy.rake +++ b/lib/tasks/gitlab/uploads/legacy.rake @@ -15,7 +15,7 @@ namespace :gitlab do batch_size = 5000 delay_interval = 5.minutes.to_i - Upload.where(uploader: 'AttachmentUploader').each_batch(of: batch_size) do |relation, index| + Upload.where(uploader: 'AttachmentUploader', model_type: 'Note').each_batch(of: batch_size) do |relation, index| start_id, end_id = relation.pluck('MIN(id), MAX(id)').first delay = index * delay_interval |