diff options
Diffstat (limited to 'lib')
46 files changed, 668 insertions, 249 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 4dd1b459554..bf8ddba6f0d 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -141,6 +141,7 @@ module API mount ::API::Projects mount ::API::ProjectSnapshots mount ::API::ProjectSnippets + mount ::API::ProjectStatistics mount ::API::ProjectTemplates mount ::API::ProtectedBranches mount ::API::ProtectedTags diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 7c035990fb0..18f15632f2b 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -300,6 +300,18 @@ module API expose :build_artifacts_size, as: :job_artifacts_size end + class ProjectDailyFetches < Grape::Entity + expose :fetch_count, as: :count + expose :date + end + + class ProjectDailyStatistics < Grape::Entity + expose :fetches do + expose :total_fetch_count, as: :total + expose :fetches, as: :days, using: ProjectDailyFetches + end + end + class Member < Grape::Entity expose :user, merge: true, using: UserBasic expose :access_level @@ -473,6 +485,12 @@ module API expose(:project_id) { |entity| entity&.project.try(:id) } expose :title, :description expose :state, :created_at, :updated_at + + # Avoids an N+1 query when metadata is included + def issuable_metadata(subject, options, method) + cached_subject = options.dig(:issuable_metadata, subject.id) + (cached_subject || subject).public_send(method) # rubocop: disable GitlabSecurity/PublicSend + end end class Diff < Grape::Entity @@ -518,54 +536,32 @@ module API class IssueBasic < ProjectEntity expose :closed_at expose :closed_by, using: Entities::UserBasic - expose :labels do |issue, options| + expose :labels do |issue| # Avoids an N+1 query since labels are preloaded issue.labels.map(&:title).sort end expose :milestone, using: Entities::Milestone expose :assignees, :author, using: Entities::UserBasic - expose :assignee, using: ::API::Entities::UserBasic do |issue, options| + expose :assignee, using: ::API::Entities::UserBasic do |issue| issue.assignees.first end - expose :user_notes_count - expose :upvotes do |issue, options| - if options[:issuable_metadata] - # Avoids an N+1 query when metadata is included - options[:issuable_metadata][issue.id].upvotes - else - issue.upvotes - end - end - expose :downvotes do |issue, options| - if options[:issuable_metadata] - # Avoids an N+1 query when metadata is included - options[:issuable_metadata][issue.id].downvotes - else - issue.downvotes - end - end + expose(:user_notes_count) { |issue, options| issuable_metadata(issue, options, :user_notes_count) } + expose(:merge_requests_count) { |issue, options| issuable_metadata(issue, options, :merge_requests_count) } + expose(:upvotes) { |issue, options| issuable_metadata(issue, options, :upvotes) } + expose(:downvotes) { |issue, options| issuable_metadata(issue, options, :downvotes) } expose :due_date expose :confidential expose :discussion_locked - expose :web_url do |issue, options| + expose :web_url do |issue| Gitlab::UrlBuilder.build(issue) end expose :time_stats, using: 'API::Entities::IssuableTimeStats' do |issue| issue end - - expose :merge_requests_count do |issue, options| - if options[:issuable_metadata] - # Avoids an N+1 query when metadata is included - options[:issuable_metadata][issue.id].merge_requests_count - else - issue.merge_requests_closing_issues.count - end - end end class Issue < IssueBasic @@ -659,23 +655,12 @@ module API MarkupHelper.markdown_field(entity, :description) end expose :target_branch, :source_branch - expose :upvotes do |merge_request, options| - if options[:issuable_metadata] - options[:issuable_metadata][merge_request.id].upvotes - else - merge_request.upvotes - end - end - expose :downvotes do |merge_request, options| - if options[:issuable_metadata] - options[:issuable_metadata][merge_request.id].downvotes - else - merge_request.downvotes - end - end + expose(:user_notes_count) { |merge_request, options| issuable_metadata(merge_request, options, :user_notes_count) } + expose(:upvotes) { |merge_request, options| issuable_metadata(merge_request, options, :upvotes) } + expose(:downvotes) { |merge_request, options| issuable_metadata(merge_request, options, :downvotes) } expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id - expose :labels do |merge_request, options| + expose :labels do |merge_request| # Avoids an N+1 query since labels are preloaded merge_request.labels.map(&:title).sort end @@ -693,7 +678,6 @@ module API end expose :diff_head_sha, as: :sha expose :merge_commit_sha - expose :user_notes_count expose :discussion_locked expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch @@ -701,7 +685,7 @@ module API # Deprecated expose :allow_collaboration, as: :allow_maintainer_to_push, if: -> (merge_request, _) { merge_request.for_fork? } - expose :web_url do |merge_request, options| + expose :web_url do |merge_request| Gitlab::UrlBuilder.build(merge_request) end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 94ed9ac6fb1..f43f4d961d6 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -54,6 +54,7 @@ module API optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' + optional :confidential, type: Boolean, desc: 'Filter confidential or public issues' use :pagination use :issues_params_ee diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 03f6684226f..44f1e81caf2 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -388,6 +388,31 @@ module API present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end + desc 'Merge a merge request to its default temporary merge ref path' + params do + optional :merge_commit_message, type: String, desc: 'Custom merge commit message' + end + put ':id/merge_requests/:merge_request_iid/merge_to_ref' do + merge_request = find_project_merge_request(params[:merge_request_iid]) + + authorize! :admin_merge_request, user_project + + merge_params = { + commit_message: params[:merge_commit_message] + } + + result = ::MergeRequests::MergeToRefService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) + + if result[:status] == :success + present result.slice(:commit_id), 200 + else + http_status = result[:http_status] || 400 + render_api_error!(result[:message], http_status) + end + end + desc 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' do success Entities::MergeRequest end diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index da31bcb8dac..ca24742b7a3 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -98,6 +98,23 @@ module API milestone_issuables_for(user_project, :merge_request) end + + desc 'Promote a milestone to group milestone' do + detail 'This feature was introduced in GitLab 11.9' + end + post ':id/milestones/:milestone_id/promote' do + begin + authorize! :admin_milestone, user_project + authorize! :admin_milestone, user_project.group + + milestone = user_project.milestones.find(params[:milestone_id]) + Milestones::PromoteService.new(user_project, current_user).execute(milestone) + + status(200) + rescue Milestones::PromoteService::PromoteMilestoneError => error + render_api_error!(error.message, 400) + end + end end end end diff --git a/lib/api/project_statistics.rb b/lib/api/project_statistics.rb new file mode 100644 index 00000000000..2f73785f72d --- /dev/null +++ b/lib/api/project_statistics.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module API + class ProjectStatistics < Grape::API + before do + authenticate! + not_found! unless user_project.daily_statistics_enabled? + authorize! :daily_statistics, user_project + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get the list of project fetch statistics for the last 30 days' + get ":id/statistics" do + statistic_finder = ::Projects::DailyStatisticsFinder.new(user_project) + + present statistic_finder, with: Entities::ProjectDailyStatistics + end + end + end +end diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb index d05ddad7466..119902a189c 100644 --- a/lib/api/project_templates.rb +++ b/lib/api/project_templates.rb @@ -36,7 +36,10 @@ module API optional :project, type: String, desc: 'The project name to use when expanding placeholders in the template. Only affects licenses' optional :fullname, type: String, desc: 'The full name of the copyright holder to use when expanding placeholders in the template. Only affects licenses' end - get ':id/templates/:type/:name', requirements: { name: /[\w\.-]+/ } do + # The regex is needed to ensure a period (e.g. agpl-3.0) + # isn't confused with a format type. We also need to allow encoded + # values (e.g. C%2B%2B for C++), so allow % and + as well. + get ':id/templates/:type/:name', requirements: { name: /[\w%.+-]+/ } do template = TemplateFinder .build(params[:type], user_project, name: params[:name]) .execute diff --git a/lib/api/runners.rb b/lib/api/runners.rb index f72b33605a7..f3fea463e7f 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -17,6 +17,7 @@ module API desc: 'The type of the runners to show' optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, desc: 'The status of the runners to show' + optional :tag_list, type: Array[String], desc: 'The tags of the runners to show' use :pagination end get do @@ -24,6 +25,7 @@ module API runners = filter_runners(runners, params[:scope], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + runners = runners.tagged_with(params[:tag_list]) if params[:tag_list] present paginate(runners), with: Entities::Runner end @@ -38,6 +40,7 @@ module API desc: 'The type of the runners to show' optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, desc: 'The status of the runners to show' + optional :tag_list, type: Array[String], desc: 'The tags of the runners to show' use :pagination end get 'all' do @@ -47,6 +50,7 @@ module API runners = filter_runners(runners, params[:scope]) runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + runners = runners.tagged_with(params[:tag_list]) if params[:tag_list] present paginate(runners), with: Entities::Runner end @@ -139,6 +143,7 @@ module API desc: 'The type of the runners to show' optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, desc: 'The status of the runners to show' + optional :tag_list, type: Array[String], desc: 'The tags of the runners to show' use :pagination end get ':id/runners' do @@ -146,6 +151,7 @@ module API runners = filter_runners(runners, params[:scope]) runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + runners = runners.tagged_with(params[:tag_list]) if params[:tag_list] present paginate(runners), with: Entities::Runner end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index e073450283b..f42ca5a9cd6 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -58,6 +58,10 @@ module Gitlab Rails.env.development? || org? || com? end + def self.ee? + Object.const_defined?(:License) + end + def self.process_name return 'sidekiq' if Sidekiq.server? return 'console' if defined?(Rails::Console) diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index 0e9bb5c94bb..df5f5ffc253 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -29,8 +29,8 @@ module Gitlab def matches_pattern?(pattern, pipeline) return true if pipeline.tag? && pattern == 'tags' return true if pipeline.branch? && pattern == 'branches' - return true if pipeline.source == pattern - return true if pipeline.source&.pluralize == pattern + return true if sanitized_source_name(pipeline) == pattern + return true if sanitized_source_name(pipeline)&.pluralize == pattern # patterns can be matched only when branch or tag is used # the pattern matching does not work for merge requests pipelines @@ -42,6 +42,10 @@ module Gitlab end end end + + def sanitized_source_name(pipeline) + @sanitized_source_name ||= pipeline&.source&.delete_suffix('_event') + end end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 5875479183e..15643fa03ac 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -84,7 +84,8 @@ module Gitlab Config::External::Processor.new(config, project: project, sha: sha || project.repository.root_ref_sha, - user: user).perform + user: user, + expandset: Set.new).perform end end end diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb index 09ecb5fdb99..2b5a59c078e 100644 --- a/lib/gitlab/ci/config/entry/global.rb +++ b/lib/gitlab/ci/config/entry/global.rb @@ -17,6 +17,9 @@ module Gitlab entry :image, Entry::Image, description: 'Docker image that will be used to execute jobs.' + entry :include, Entry::Includes, + description: 'List of external YAML files to include.' + entry :services, Entry::Services, description: 'Docker images that will be linked to the container.' diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb new file mode 100644 index 00000000000..f2f3dd84eda --- /dev/null +++ b/lib/gitlab/ci/config/entry/include.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a single include. + # + class Include < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + ALLOWED_KEYS = %i[local file remote template].freeze + + validations do + validates :config, hash_or_string: true + validates :config, allowed_keys: ALLOWED_KEYS + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/includes.rb b/lib/gitlab/ci/config/entry/includes.rb new file mode 100644 index 00000000000..82b2b1ccf4b --- /dev/null +++ b/lib/gitlab/ci/config/entry/includes.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a list of include. + # + class Includes < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: Array + end + + def self.aspects + super.append -> do + @config = Array.wrap(@config) + + @config.each_with_index do |config, i| + @entries[i] = ::Gitlab::Config::Entry::Factory.new(Entry::Include) + .value(config || {}) + .create! + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index a747886093c..2ffbb214a92 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -12,7 +12,7 @@ module Gitlab YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze - Context = Struct.new(:project, :sha, :user) + Context = Struct.new(:project, :sha, :user, :expandset) def initialize(params, context) @params = params @@ -43,13 +43,27 @@ module Gitlab end def to_hash - @hash ||= Gitlab::Config::Loader::Yaml.new(content).load! - rescue Gitlab::Config::Loader::FormatError - nil + expanded_content_hash end protected + def expanded_content_hash + return unless content_hash + + strong_memoize(:expanded_content_yaml) do + expand_includes(content_hash) + end + end + + def content_hash + strong_memoize(:content_yaml) do + Gitlab::Config::Loader::Yaml.new(content).load! + end + rescue Gitlab::Config::Loader::FormatError + nil + end + def validate! validate_location! validate_content! if errors.none? @@ -73,6 +87,14 @@ module Gitlab errors.push("Included file `#{location}` does not have valid YAML syntax!") end end + + def expand_includes(hash) + External::Processor.new(hash, **expand_context).perform + end + + def expand_context + { project: nil, sha: nil, user: nil, expandset: context.expandset } + end end end end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index 2535d178ba8..229a06451e8 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -31,6 +31,13 @@ module Gitlab def fetch_local_content context.project.repository.blob_data_at(context.sha, location) end + + def expand_context + super.merge( + project: context.project, + sha: context.sha, + user: context.user) + end end end end diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index e75540dbe5a..b828f77835c 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -64,6 +64,13 @@ module Gitlab project.commit(ref_name).try(:sha) end end + + def expand_context + super.merge( + project: project, + sha: sha, + user: context.user) + end end end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 108bfd5eb43..aff5c5b9651 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -7,6 +7,8 @@ module Gitlab class Mapper include Gitlab::Utils::StrongMemoize + MAX_INCLUDES = 50 + FILE_CLASSES = [ External::File::Remote, External::File::Template, @@ -14,25 +16,34 @@ module Gitlab External::File::Project ].freeze - AmbigiousSpecificationError = Class.new(StandardError) + Error = Class.new(StandardError) + AmbigiousSpecificationError = Class.new(Error) + DuplicateIncludesError = Class.new(Error) + TooManyIncludesError = Class.new(Error) + + def initialize(values, project:, sha:, user:, expandset:) + raise Error, 'Expanded needs to be `Set`' unless expandset.is_a?(Set) - def initialize(values, project:, sha:, user:) @locations = Array.wrap(values.fetch(:include, [])) @project = project @sha = sha @user = user + @expandset = expandset end def process + return [] if locations.empty? + locations .compact .map(&method(:normalize_location)) + .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) end private - attr_reader :locations, :project, :sha, :user + attr_reader :locations, :project, :sha, :user, :expandset # convert location if String to canonical form def normalize_location(location) @@ -51,6 +62,23 @@ module Gitlab end end + def verify_duplicates!(location) + if expandset.count >= MAX_INCLUDES + raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!" + end + + # We scope location to context, as this allows us to properly support + # relative incldues, and similarly looking relative in another project + # does not trigger duplicate error + scoped_location = location.merge( + context_project: project, + context_sha: sha) + + unless expandset.add?(scoped_location) + raise DuplicateIncludesError, "Include `#{location.to_json}` was already included!" + end + end + def select_first_matching(location) matching = FILE_CLASSES.map do |file_class| file_class.new(location, context) @@ -63,7 +91,7 @@ module Gitlab def context strong_memoize(:context) do - External::File::Base::Context.new(project, sha, user) + External::File::Base::Context.new(project, sha, user, expandset) end end end diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb index 69bc164a039..1dd2d42016a 100644 --- a/lib/gitlab/ci/config/external/processor.rb +++ b/lib/gitlab/ci/config/external/processor.rb @@ -7,11 +7,11 @@ module Gitlab class Processor IncludeError = Class.new(StandardError) - def initialize(values, project:, sha:, user:) + def initialize(values, project:, sha:, user:, expandset:) @values = values - @external_files = External::Mapper.new(values, project: project, sha: sha, user: user).process + @external_files = External::Mapper.new(values, project: project, sha: sha, user: user, expandset: expandset).process @content = {} - rescue External::Mapper::AmbigiousSpecificationError => e + rescue External::Mapper::Error => e raise IncludeError, e.message end diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 41632211374..164a4634d84 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -12,6 +12,8 @@ module Gitlab ref: @command.ref, sha: @command.sha, before_sha: @command.before_sha, + source_sha: @command.source_sha, + target_sha: @command.target_sha, tag: @command.tag_exists?, trigger_requests: Array(@command.trigger_request), user: @command.current_user, diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index e4ed1424865..7b77e86feae 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -7,7 +7,7 @@ module Gitlab module Chain Command = Struct.new( :source, :project, :current_user, - :origin_ref, :checkout_sha, :after_sha, :before_sha, + :origin_ref, :checkout_sha, :after_sha, :before_sha, :source_sha, :target_sha, :trigger_request, :schedule, :merge_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml new file mode 100644 index 00000000000..245e6bec60a --- /dev/null +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -0,0 +1,28 @@ +# This is a very simple template that mainly relies on FastLane to build and distribute your app. +# Read more about how to use this template on the blog post https://about.gitlab.com/2019/03/06/ios-publishing-with-gitlab-and-fastlane/ +# You will also need fastlane and signing configuration for this to work, along with a MacOS runner. +# These details are provided in the blog post. + +# Note that when you're using the shell executor for MacOS builds, the +# build and tests run as the identity of the runner logged in user, directly on +# the build host. This is less secure than using container executors, so please +# take a look at our security implications documentation at +# https://docs.gitlab.com/runner/security/#usage-of-shell-executor for additional +# detail on what to keep in mind in this scenario. + +stages: + - build + +variables: + LC_ALL: "en_US.UTF-8" + LANG: "en_US.UTF-8" + GIT_STRATEGY: clone + +build: + stage: build + script: + - bundle install + - bundle exec fastlane build + artifacts: + paths: + - ./*.ipa diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 5ed6427072a..f7d046600e8 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -49,6 +49,7 @@ module Gitlab Event.contributions.where(author_id: contributor.id) .where(created_at: date.beginning_of_day..date.end_of_day) .where(project_id: projects) + .with_associations end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index d3c86fdb629..d2b7ca015d4 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -123,6 +123,7 @@ module Gitlab # Files that don't fit into any category are marked with :none %r{\A(ee/)?changelogs/} => :none, + %r{\Alocale/gitlab\.pot\z} => :none, # Fallbacks in case the above patterns miss anything %r{\.rb\z} => :backend, diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 5863815ca85..c5688cda565 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -5,6 +5,7 @@ module Gitlab module Git class Commit include Gitlab::EncodingHelper + prepend Gitlab::Git::RuggedImpl::Commit extend Gitlab::Git::WrapsGitalyErrors attr_accessor :raw_commit, :head @@ -62,15 +63,19 @@ module Gitlab # This saves us an RPC round trip. return nil if commit_id.include?(':') - commit = wrapped_gitaly_errors do - repo.gitaly_commit_client.find_commit(commit_id) - end + commit = find_commit(repo, commit_id) decorate(repo, commit) if commit rescue Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository, ArgumentError nil end + def find_commit(repo, commit_id) + wrapped_gitaly_errors do + repo.gitaly_commit_client.find_commit(commit_id) + end + end + # Get last commit for HEAD # # Ex. @@ -185,6 +190,10 @@ module Gitlab @repository = repository @head = head + init_commit(raw_commit) + end + + def init_commit(raw_commit) case raw_commit when Hash init_from_hash(raw_commit) @@ -400,3 +409,5 @@ module Gitlab end end end + +Gitlab::Git::Commit.singleton_class.prepend Gitlab::Git::RuggedImpl::Commit::ClassMethods diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index eec91194949..47cfb483509 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -4,6 +4,7 @@ module Gitlab module Git class Ref include Gitlab::EncodingHelper + include Gitlab::Git::RuggedImpl::Ref # Branch or tag name # without "refs/tags|heads" prefix diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index aea132a3dd9..2bfff8397e8 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -11,6 +11,7 @@ module Gitlab include Gitlab::Git::WrapsGitalyErrors include Gitlab::EncodingHelper include Gitlab::Utils::StrongMemoize + include Gitlab::Git::RuggedImpl::Repository SEARCH_CONTEXT_LINES = 3 REV_LIST_COMMIT_LIMIT = 2_000 diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb new file mode 100644 index 00000000000..251802878c3 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Gitlab + module Git + module RuggedImpl + module Commit + module ClassMethods + extend ::Gitlab::Utils::Override + + def rugged_find(repo, commit_id) + obj = repo.rev_parse_target(commit_id) + + obj.is_a?(::Rugged::Commit) ? obj : nil + rescue ::Rugged::Error + nil + end + + override :find_commit + def find_commit(repo, commit_id) + if Feature.enabled?(:rugged_find_commit) + rugged_find(repo, commit_id) + else + super + end + end + end + + extend ::Gitlab::Utils::Override + + override :init_commit + def init_commit(raw_commit) + case raw_commit + when ::Rugged::Commit + init_from_rugged(raw_commit) + else + super + end + end + + def init_from_rugged(commit) + author = commit.author + committer = commit.committer + + @raw_commit = commit + @id = commit.oid + @message = commit.message + @authored_date = author[:time] + @committed_date = committer[:time] + @author_name = author[:name] + @author_email = author[:email] + @committer_name = committer[:name] + @committer_email = committer[:email] + @parent_ids = commit.parents.map(&:oid) + end + end + end + end +end +# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/git/rugged_impl/ref.rb b/lib/gitlab/git/rugged_impl/ref.rb new file mode 100644 index 00000000000..b553e82dc47 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/ref.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +module Gitlab + module Git + module RuggedImpl + module Ref + def self.dereference_object(object) + object = object.target while object.is_a?(::Rugged::Tag::Annotation) + + object + end + end + end + end +end diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb new file mode 100644 index 00000000000..135c47017b3 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Gitlab + module Git + module RuggedImpl + module Repository + FEATURE_FLAGS = %i(rugged_find_commit).freeze + + def alternate_object_directories + relative_object_directories.map { |d| File.join(path, d) } + end + + ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[ + GIT_OBJECT_DIRECTORY_RELATIVE + GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE + ].freeze + + def relative_object_directories + Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact + end + + def rugged + @rugged ||= ::Rugged::Repository.new(path, alternates: alternate_object_directories) + rescue ::Rugged::RepositoryError, ::Rugged::OSError + raise ::Gitlab::Git::Repository::NoRepository.new('no repository for such path') + end + + def cleanup + @rugged&.close + end + + # Return the object that +revspec+ points to. If +revspec+ is an + # annotated tag, then return the tag's target instead. + def rev_parse_target(revspec) + obj = rugged.rev_parse(revspec) + Ref.dereference_object(obj) + end + end + end + end +end +# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 5aeedb0f50d..869b835b61e 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -164,8 +164,6 @@ module Gitlab kwargs = yield(kwargs) if block_given? stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend - rescue GRPC::Unavailable => ex - handle_grpc_unavailable!(ex) ensure duration = Gitlab::Metrics::System.monotonic_time - start @@ -178,27 +176,6 @@ module Gitlab add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc) end - def self.handle_grpc_unavailable!(ex) - status = ex.to_status - raise ex unless status.details == 'Endpoint read failed' - - # There is a bug in grpc 1.8.x that causes a client process to get stuck - # always raising '14:Endpoint read failed'. The only thing that we can - # do to recover is to restart the process. - # - # See https://gitlab.com/gitlab-org/gitaly/issues/1029 - - if Sidekiq.server? - raise Gitlab::SidekiqMiddleware::Shutdown::WantShutdown.new(ex.to_s) - else - # SIGQUIT requests a Unicorn worker to shut down gracefully after the current request. - Process.kill('QUIT', Process.pid) - end - - raise ex - end - private_class_method :handle_grpc_unavailable! - def self.current_transaction_labels Gitlab::Metrics::Transaction.current&.labels || {} end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index a7e20d9429e..a08bfd0e25b 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -324,8 +324,8 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end - def search_files_by_content(ref, query, chunked_response: true) - request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query, chunked_response: chunked_response) + def search_files_by_content(ref, query) + request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request) search_results_from_response(response) @@ -340,18 +340,11 @@ module Gitlab gitaly_response.each do |message| next if message.nil? - # Old client will ignore :chunked_response flag - # and return messages with `matches` key. - # This code path will be removed post 12.0 release - if message.matches.any? - matches += message.matches - else - current_match << message.match_data - - if message.end_of_match - matches << current_match - current_match = +"" - end + current_match << message.match_data + + if message.end_of_match + matches << current_match + current_match = +"" end end diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 754cccb6b3f..78ef6bfc0ec 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -32,11 +32,19 @@ module Gitlab end def self.disk_access_denied? + return false if rugged_enabled? + !temporarily_allowed?(ALLOW_KEY) && GitalyClient.feature_enabled?(DISK_ACCESS_DENIED_FLAG) rescue false # Err on the side of caution, don't break gitlab for people end + def self.rugged_enabled? + Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.any? do |flag| + Feature.enabled?(flag) + end + end + def initialize(storage) raise InvalidConfigurationError, "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash) raise InvalidConfigurationError, INVALID_STORAGE_MESSAGE unless storage.has_key?('path') diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb index 87cf2c8b598..71ff7465d9b 100644 --- a/lib/gitlab/github_import/importer/milestones_importer.rb +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -42,6 +42,7 @@ module Gitlab description: milestone.description, project_id: project.id, state: state_for(milestone), + due_date: milestone.due_on&.to_date, created_at: milestone.created_at, updated_at: milestone.updated_at } diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 3235d3ccc4e..e00309e7946 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -25,6 +25,7 @@ module Gitlab gon.test_env = Rails.env.test? gon.suggested_label_colors = LabelsHelper.suggested_colors gon.first_day_of_week = current_user&.first_day_of_week || Gitlab::CurrentSettings.first_day_of_week + gon.ee = Gitlab.ee? if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index bf463077dcc..7046b4e2a43 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -13,10 +13,18 @@ module Gitlab # # @param [Integer] start first project id for the range # @param [Integer] finish last project id for the range - def bulk_schedule(start:, finish:) + def bulk_schedule_migration(start:, finish:) ::HashedStorage::MigratorWorker.perform_async(start, finish) end + # Schedule a range of projects to be bulk rolledback with #bulk_rollback asynchronously + # + # @param [Integer] start first project id for the range + # @param [Integer] finish last project id for the range + def bulk_schedule_rollback(start:, finish:) + ::HashedStorage::RollbackerWorker.perform_async(start, finish) + end + # Start migration of projects from specified range # # Flagging a project to be migrated is a synchronous action @@ -34,6 +42,23 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # Start rollback of projects from specified range + # + # Flagging a project to be rolled back is a synchronous action + # but the rollback runs through async jobs + # + # @param [Integer] start first project id for the range + # @param [Integer] finish last project id for the range + # rubocop: disable CodeReuse/ActiveRecord + def bulk_rollback(start:, finish:) + projects = build_relation(start, finish) + + projects.with_route.find_each(batch_size: BATCH_SIZE) do |project| + rollback(project) + end + end + # rubocop: enable CodeReuse/ActiveRecord + # Flag a project to be migrated to Hashed Storage # # @param [Project] project that will be migrated @@ -45,8 +70,15 @@ module Gitlab Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end + # Flag a project to be rolled-back to Legacy Storage + # + # @param [Project] project that will be rolled-back def rollback(project) - # TODO: implement rollback strategy + Rails.logger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..." + + project.rollback_to_legacy_storage! + rescue => err + Rails.logger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end private diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 38f552fab03..87a31a37e3f 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -24,7 +24,7 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def self.project_id_batches(&block) + def self.project_id_batches_migration(&block) Project.with_unmigrated_storage.in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches ids = relation.pluck(:id) @@ -34,6 +34,16 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord + def self.project_id_batches_rollback(&block) + Project.with_storage_feature(:repository).in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches + ids = relation.pluck(:id) + + yield ids.min, ids.max + end + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord def self.legacy_attachments_relation Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments]) JOIN projects diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb index 1adf83739ad..24daad638f4 100644 --- a/lib/gitlab/json_cache.rb +++ b/lib/gitlab/json_cache.rb @@ -71,7 +71,21 @@ module Gitlab end def parse_entry(raw, klass) - klass.new(raw) if valid_entry?(raw, klass) + return unless valid_entry?(raw, klass) + return klass.new(raw) unless klass.ancestors.include?(ActiveRecord::Base) + + # When the cached value is a persisted instance of ActiveRecord::Base in + # some cases a relation can return an empty collection becauses scope.none! + # is being applied on ActiveRecord::Associations::CollectionAssociation#scope + # when the new_record? method incorrectly returns false. + # + # See https://gitlab.com/gitlab-org/gitlab-ee/issues/9903#note_145329964 + attributes = klass.attributes_builder.build_from_database(raw, {}) + klass.allocate.init_with("attributes" => attributes, "new_record" => new_record?(raw, klass)) + end + + def new_record?(raw, klass) + raw.fetch(klass.primary_key, nil).blank? end def valid_entry?(raw, klass) diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 29c511524a2..9b6ff602fcd 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -29,6 +29,7 @@ module Gitlab ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'), ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'), ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/logos/dotnet.svg'), + ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development.'), 'https://gitlab.com/gitlab-org/project-templates/go-micro'), ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo'), ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll'), ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML.'), 'https://gitlab.com/pages/plain-html'), diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index 259345b8a9a..e7bfcb16582 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -48,6 +48,8 @@ module Gitlab def execute(context, arg) return if noop? || !available?(context) + count_commands_executed_in(context) + execute_block(action_block, context, arg) end @@ -73,6 +75,13 @@ module Gitlab private + def count_commands_executed_in(context) + return unless context.respond_to?(:commands_executed_count=) + + context.commands_executed_count ||= 0 + context.commands_executed_count += 1 + end + def execute_block(block, context, arg) if arg.present? parsed = parse_params(arg, context) diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index c7d8dfcd495..40b641b8317 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -340,16 +340,16 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + def hooks_path + File.join(gitlab_shell_path, 'hooks') + end + protected def gitlab_shell_path File.expand_path(Gitlab.config.gitlab_shell.path) end - def gitlab_shell_hooks_path - File.expand_path(Gitlab.config.gitlab_shell.hooks_path) - end - def gitlab_shell_user_home File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb new file mode 100644 index 00000000000..47333d257eb --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class MemoryKiller + # Default the RSS limit to 0, meaning the MemoryKiller is disabled + MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i + # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit + GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i + # Wait 30 seconds for running jobs to finish during graceful shutdown + SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + + # Create a mutex used to ensure there will be only one thread waiting to + # shut Sidekiq down + MUTEX = Mutex.new + + def call(worker, job, queue) + yield + + current_rss = get_rss + + return unless MAX_RSS > 0 && current_rss > MAX_RSS + + Thread.new do + # Return if another thread is already waiting to shut Sidekiq down + next unless MUTEX.try_lock + + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ + " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + + # Wait `GRACE_TIME` to give the memory intensive job time to finish. + # Then, tell Sidekiq to stop fetching new jobs. + wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs') + + # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. + # Then, tell Sidekiq to gracefully shut down by giving jobs a few more + # moments to finish, killing and requeuing them if they didn't, and + # then terminating itself. + wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') + + # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. + wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') + end + end + + private + + def get_rss + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) + return 0 unless status.zero? + + output.to_i + end + + def wait_and_signal(time, signal, explanation) + Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + sleep(time) + + Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + Process.kill(signal, pid) + end + + def pid + Process.pid + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb deleted file mode 100644 index 19f3be83bce..00000000000 --- a/lib/gitlab/sidekiq_middleware/shutdown.rb +++ /dev/null @@ -1,135 +0,0 @@ -# frozen_string_literal: true - -require 'mutex_m' - -module Gitlab - module SidekiqMiddleware - class Shutdown - extend Mutex_m - - # Default the RSS limit to 0, meaning the MemoryKiller is disabled - MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i - # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit - GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i - # Wait 30 seconds for running jobs to finish during graceful shutdown - SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i - - # This exception can be used to request that the middleware start shutting down Sidekiq - WantShutdown = Class.new(StandardError) - - ShutdownWithoutRaise = Class.new(WantShutdown) - private_constant :ShutdownWithoutRaise - - # For testing only, to avoid race conditions (?) in Rspec mocks. - attr_reader :trace - - # We store the shutdown thread in a class variable to ensure that there - # can be only one shutdown thread in the process. - def self.create_shutdown_thread - mu_synchronize do - break unless @shutdown_thread.nil? - - @shutdown_thread = Thread.new { yield } - end - end - - # For testing only: so we can wait for the shutdown thread to finish. - def self.shutdown_thread - mu_synchronize { @shutdown_thread } - end - - # For testing only: so that we can reset the global state before each test. - def self.clear_shutdown_thread - mu_synchronize { @shutdown_thread = nil } - end - - def initialize - @trace = Queue.new if Rails.env.test? - end - - def call(worker, job, queue) - shutdown_exception = nil - - begin - yield - check_rss! - rescue WantShutdown => ex - shutdown_exception = ex - end - - return unless shutdown_exception - - self.class.create_shutdown_thread do - do_shutdown(worker, job, shutdown_exception) - end - - raise shutdown_exception unless shutdown_exception.is_a?(ShutdownWithoutRaise) - end - - private - - def do_shutdown(worker, job, shutdown_exception) - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} shutting down because of #{shutdown_exception} after job "\ - "#{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" - - # Wait `GRACE_TIME` to give the memory intensive job time to finish. - # Then, tell Sidekiq to stop fetching new jobs. - wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs') - - # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. - # Then, tell Sidekiq to gracefully shut down by giving jobs a few more - # moments to finish, killing and requeuing them if they didn't, and - # then terminating itself. - wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') - - # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. - wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') - end - - def check_rss! - return unless MAX_RSS > 0 - - current_rss = get_rss - return unless current_rss > MAX_RSS - - raise ShutdownWithoutRaise.new("current RSS #{current_rss} exceeds maximum RSS #{MAX_RSS}") - end - - def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status.zero? - - output.to_i - end - - def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - sleep(time) - - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - kill(signal, pid) - end - - def pid - Process.pid - end - - def sleep(time) - if Rails.env.test? - @trace << [:sleep, time] - else - Kernel.sleep(time) - end - end - - def kill(signal, pid) - if Rails.env.test? - @trace << [:kill, signal, pid] - else - Process.kill(signal, pid) - end - end - end - end -end diff --git a/lib/tasks/gitlab/features.rake b/lib/tasks/gitlab/features.rake new file mode 100644 index 00000000000..d115961108e --- /dev/null +++ b/lib/tasks/gitlab/features.rake @@ -0,0 +1,24 @@ +namespace :gitlab do + namespace :features do + desc 'GitLab | Features | Enable direct Git access via Rugged for NFS' + task enable_rugged: :environment do + set_rugged_feature_flags(true) + puts 'All Rugged feature flags were enabled.' + end + + task disable_rugged: :environment do + set_rugged_feature_flags(false) + puts 'All Rugged feature flags were disabled.' + end + end + + def set_rugged_feature_flags(status) + Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.each do |flag| + if status + Feature.enable(flag) + else + Feature.disable(flag) + end + end + end +end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index e97d77d20e0..b8798fb3cfd 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -58,7 +58,7 @@ namespace :gitlab do puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab::Auth.omniauth_enabled? # check Gitolite version - gitlab_shell_version_file = "#{Gitlab.config.gitlab_shell.hooks_path}/../VERSION" + gitlab_shell_version_file = "#{Gitlab.config.gitlab_shell.path}/VERSION" if File.readable?(gitlab_shell_version_file) gitlab_shell_version = File.read(gitlab_shell_version_file) end @@ -72,7 +72,7 @@ namespace :gitlab do puts "- #{name}: \t#{repository_storage.legacy_disk_path}" end end - puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" + puts "GitLab Shell path:\t\t#{Gitlab.config.gitlab_shell.path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" end end diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index f9ce3e1d338..a2136ce1b92 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -36,8 +36,54 @@ namespace :gitlab do print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}" - helper.project_id_batches do |start, finish| - storage_migrator.bulk_schedule(start: start, finish: finish) + helper.project_id_batches_migration do |start, finish| + storage_migrator.bulk_schedule_migration(start: start, finish: finish) + + print '.' + end + + puts ' Done!' + end + + desc 'GitLab | Storage | Rollback existing projects to Legacy Storage' + task rollback_to_legacy: :environment do + if Gitlab::Database.read_only? + warn 'This task requires database write access. Exiting.' + + next + end + + storage_migrator = Gitlab::HashedStorage::Migrator.new + helper = Gitlab::HashedStorage::RakeHelper + + if helper.range_single_item? + project = Project.with_storage_feature(:repository).find_by(id: helper.range_from) + + unless project + warn "There are no projects that can be rolledback with ID=#{helper.range_from}" + + next + end + + puts "Enqueueing storage rollback of #{project.full_path} (ID=#{project.id})..." + storage_migrator.rollback(project) + + next + end + + hashed_projects_count = Project.with_storage_feature(:repository).count + + if hashed_projects_count == 0 + warn 'There are no projects that can have storage rolledback. Nothing to do!' + + next + end + + print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}" + + helper.project_id_batches_rollback do |start, finish| + puts "Start: #{start} FINISH: #{finish}" + storage_migrator.bulk_schedule_rollback(start: start, finish: finish) print '.' end |