diff options
Diffstat (limited to 'lib')
53 files changed, 1175 insertions, 573 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 4be7707d3e7..f858d9fa23d 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -62,6 +62,14 @@ module API expose :admin?, as: :is_admin end + class UserStatus < Grape::Entity + expose :emoji + expose :message + expose :message_html do |entity| + MarkupHelper.markdown_field(entity, :message) + end + end + class Email < Grape::Entity expose :id, :email end diff --git a/lib/api/helpers/headers_helpers.rb b/lib/api/helpers/headers_helpers.rb index cde51fccc62..c9c44e3c218 100644 --- a/lib/api/helpers/headers_helpers.rb +++ b/lib/api/helpers/headers_helpers.rb @@ -3,7 +3,11 @@ module API module HeadersHelpers def set_http_headers(header_data) header_data.each do |key, value| - header "X-Gitlab-#{key.to_s.split('_').collect(&:capitalize).join('-')}", value + if value.is_a?(Enumerable) + raise ArgumentError.new("Header value should be a string") + end + + header "X-Gitlab-#{key.to_s.split('_').collect(&:capitalize).join('-')}", value.to_s end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index a9803be9f69..516f25db15b 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -11,7 +11,8 @@ module API # # Params: # key_id - ssh key id for Git over SSH - # user_id - user id for Git over HTTP + # user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode + # username - user name for Git over SSH in keyless SSH cert mode # protocol - Git access protocol being used, e.g. HTTP or SSH # project - project full_path (not path on disk) # action - git action (git-upload-pack or git-receive-pack) @@ -28,6 +29,8 @@ module API Key.find_by(id: params[:key_id]) elsif params[:user_id] User.find_by(id: params[:user_id]) + elsif params[:username] + User.find_by_username(params[:username]) end protocol = params[:protocol] @@ -58,6 +61,7 @@ module API { status: true, gl_repository: gl_repository, + gl_id: Gitlab::GlId.gl_id(user), gl_username: user&.username, # This repository_path is a bogus value but gitlab-shell still requires @@ -71,10 +75,17 @@ module API post "/lfs_authenticate" do status 200 - key = Key.find(params[:key_id]) - key.update_last_used_at + if params[:key_id] + actor = Key.find(params[:key_id]) + actor.update_last_used_at + elsif params[:user_id] + actor = User.find_by(id: params[:user_id]) + raise ActiveRecord::RecordNotFound.new("No such user id!") unless actor + else + raise ActiveRecord::RecordNotFound.new("No key_id or user_id passed!") + end - token_handler = Gitlab::LfsToken.new(key) + token_handler = Gitlab::LfsToken.new(actor) { username: token_handler.actor_name, @@ -100,7 +111,7 @@ module API end # - # Discover user by ssh key or user id + # Discover user by ssh key, user id or username # get "/discover" do if params[:key_id] @@ -108,6 +119,8 @@ module API user = key.user elsif params[:user_id] user = User.find_by(id: params[:user_id]) + elsif params[:username] + user = User.find_by(username: params[:username]) end present user, with: Entities::UserSafe @@ -141,22 +154,30 @@ module API post '/two_factor_recovery_codes' do status 200 - key = Key.find_by(id: params[:key_id]) + if params[:key_id] + key = Key.find_by(id: params[:key_id]) - if key - key.update_last_used_at - else - break { 'success' => false, 'message' => 'Could not find the given key' } - end + if key + key.update_last_used_at + else + break { 'success' => false, 'message' => 'Could not find the given key' } + end - if key.is_a?(DeployKey) - break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } - end + if key.is_a?(DeployKey) + break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } + end + + user = key.user - user = key.user + unless user + break { success: false, message: 'Could not find a user for the given key' } + end + elsif params[:user_id] + user = User.find_by(id: params[:user_id]) - unless user - break { success: false, message: 'Could not find a user for the given key' } + unless user + break { success: false, message: 'Could not find the given user' } + end end unless user.two_factor_enabled? diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 25185d6edc8..bda05d1795b 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -162,6 +162,9 @@ module API desc: 'The IID of a merge request for which to resolve discussions' optional :discussion_to_resolve, type: String, desc: 'The ID of a discussion to resolve, also pass `merge_request_to_resolve_discussions_of`' + optional :iid, type: Integer, + desc: 'The internal ID of a project issue. Available only for admins and project owners.' + use :issue_params end post ':id/issues' do @@ -169,9 +172,10 @@ module API authorize! :create_issue, user_project - # Setting created_at time only allowed for admins and project owners + # Setting created_at time or iid only allowed for admins and project owners unless current_user.admin? || user_project.owner == current_user params.delete(:created_at) + params.delete(:iid) end issue_params = declared_params(include_missing: false) diff --git a/lib/api/members.rb b/lib/api/members.rb index 3d2220fed96..d23dd834c69 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -75,7 +75,10 @@ module API member = source.members.find_by(user_id: params[:user_id]) conflict!('Member already exists') if member - member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) + user = User.find_by_id(params[:user_id]) + not_found!('User') unless user + + member = source.add_user(user, params[:access_level], current_user: current_user, expires_at: params[:expires_at]) if !member not_allowed! # This currently can only be reached in EE diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 7adde79d6c3..5738bf220c6 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -321,7 +321,7 @@ module API post ':id/archive' do authorize!(:archive_project, user_project) - user_project.archive! + ::Projects::UpdateService.new(user_project, current_user, archived: true).execute present user_project, with: Entities::Project end @@ -332,7 +332,7 @@ module API post ':id/unarchive' do authorize!(:archive_project, user_project) - user_project.unarchive! + ::Projects::UpdateService.new(@project, current_user, archived: false).execute present user_project, with: Entities::Project end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index c7595493e11..c9931c2d603 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -80,7 +80,15 @@ module API params do requires :token, type: String, desc: %q(Runner's authentication token) optional :last_update, type: String, desc: %q(Runner's queue last_update token) - optional :info, type: Hash, desc: %q(Runner's metadata) + optional :info, type: Hash, desc: %q(Runner's metadata) do + optional :name, type: String, desc: %q(Runner's name) + optional :version, type: String, desc: %q(Runner's version) + optional :revision, type: String, desc: %q(Runner's revision) + optional :platform, type: String, desc: %q(Runner's platform) + optional :architecture, type: String, desc: %q(Runner's architecture) + optional :executor, type: String, desc: %q(Runner's executor) + optional :features, type: Hash, desc: %q(Runner's features) + end optional :session, type: Hash, desc: %q(Runner's session data) do optional :url, type: String, desc: %q(Session's url) optional :certificate, type: String, desc: %q(Session's certificate) diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 1ca7d23203b..897010217dc 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -20,116 +20,114 @@ module API success Entities::ApplicationSetting end params do + optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' + optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' + optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out' + optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues' + given akismet_enabled: ->(val) { val } do + requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com' + end + optional :clientside_sentry_enabled, type: Boolean, desc: 'Sentry can also be used for reporting and logging clientside exceptions. https://sentry.io/for/javascript/' + given clientside_sentry_enabled: ->(val) { val } do + requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name' + end + optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' + optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master' + optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility' optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility' + optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility' - optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility' - optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for groups, projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' - optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project manifest], - desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources' - optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' - optional :project_export_enabled, type: Boolean, desc: 'Enable project export' - optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' - optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' - optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' - optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.' - optional :user_oauth_applications, type: Boolean, desc: 'Allow users to register any application to use GitLab as an OAuth provider' - optional :user_default_external, type: Boolean, desc: 'Newly registered users will by default be external' - optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled' - optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up' - optional :domain_whitelist, type: String, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' optional :domain_blacklist_enabled, type: Boolean, desc: 'Enable domain blacklist for sign ups' given domain_blacklist_enabled: ->(val) { val } do requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end - optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :password_authentication_enabled_for_web, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' - optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 - mutually_exclusive :password_authentication_enabled_for_web, :password_authentication_enabled, :signin_enabled - optional :password_authentication_enabled_for_git, type: Boolean, desc: 'Flag indicating if password authentication is enabled for Git over HTTP(S)' - optional :performance_bar_allowed_group_path, type: String, desc: 'Path of the group that is allowed to toggle the performance bar.' - optional :performance_bar_allowed_group_id, type: String, desc: 'Depreated: Use :performance_bar_allowed_group_path instead. Path of the group that is allowed to toggle the performance bar.' # support legacy names, can be removed in v6 - optional :performance_bar_enabled, type: String, desc: 'Deprecated: Pass `performance_bar_allowed_group_path: nil` instead. Allow enabling the performance.' # support legacy names, can be removed in v6 - optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' - given require_two_factor_authentication: ->(val) { val } do - requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' - end - optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' - optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out' - optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application' + optional :domain_whitelist, type: String, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' + optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.' + optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' + optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.' + optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.' + optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.' + optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' optional :help_page_hide_commercial_content, type: Boolean, desc: 'Hide marketing-related entries from help' - optional :help_page_text, type: String, desc: 'Custom text displayed on the help page' optional :help_page_support_url, type: String, desc: 'Alternate support URL for help page' - optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects' - given shared_runners_enabled: ->(val) { val } do - requires :shared_runners_text, type: String, desc: 'Shared runners text ' + optional :help_page_text, type: String, desc: 'Custom text displayed on the help page' + optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' + optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' + given housekeeping_enabled: ->(val) { val } do + requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance." + requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run." + requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." + requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run." + end + optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.' + optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project manifest], + desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' + optional :koding_enabled, type: Boolean, desc: 'Enable Koding' + given koding_enabled: ->(val) { val } do + requires :koding_url, type: String, desc: 'The Koding team URL' end optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size for each job's artifacts" - optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" + optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' - optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' - optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics' optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics' given metrics_enabled: ->(val) { val } do requires :metrics_host, type: String, desc: 'The InfluxDB host' - requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB' - requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open' - requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out' requires :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.' - requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds' requires :metrics_packet_size, type: Integer, desc: 'The amount of points to store in a single UDP packet' + requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open' + requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB' + requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds' + requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out' end - optional :sidekiq_throttling_enabled, type: Boolean, desc: 'Enable Sidekiq Job Throttling' - given sidekiq_throttling_enabled: ->(val) { val } do - requires :sidekiq_throttling_queus, type: Array[String], desc: 'Choose which queues you wish to throttle' - requires :sidekiq_throttling_factor, type: Float, desc: 'The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 + optional :password_authentication_enabled_for_web, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' + mutually_exclusive :password_authentication_enabled_for_web, :password_authentication_enabled, :signin_enabled + optional :password_authentication_enabled_for_git, type: Boolean, desc: 'Flag indicating if password authentication is enabled for Git over HTTP(S)' + optional :performance_bar_allowed_group_id, type: String, desc: 'Deprecated: Use :performance_bar_allowed_group_path instead. Path of the group that is allowed to toggle the performance bar.' # support legacy names, can be removed in v6 + optional :performance_bar_allowed_group_path, type: String, desc: 'Path of the group that is allowed to toggle the performance bar.' + optional :performance_bar_enabled, type: String, desc: 'Deprecated: Pass `performance_bar_allowed_group_path: nil` instead. Allow enabling the performance.' # support legacy names, can be removed in v6 + optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML' + given plantuml_enabled: ->(val) { val } do + requires :plantuml_url, type: String, desc: 'The PlantUML server URL' end + optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' + optional :project_export_enabled, type: Boolean, desc: 'Enable project export' + optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics' optional :recaptcha_enabled, type: Boolean, desc: 'Helps prevent bots from creating accounts' given recaptcha_enabled: ->(val) { val } do requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha' requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha' end - optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues' - given akismet_enabled: ->(val) { val } do - requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com' + optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." + optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects' + optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' + given require_two_factor_authentication: ->(val) { val } do + requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' end - optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' + optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for groups, projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' + optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up' optional :sentry_enabled, type: Boolean, desc: 'Sentry is an error reporting and logging tool which is currently not shipped with GitLab, get it here: https://getsentry.com' given sentry_enabled: ->(val) { val } do requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name' end - optional :clientside_sentry_enabled, type: Boolean, desc: 'Sentry can also be used for reporting and logging clientside exceptions. https://sentry.io/for/javascript/' - given clientside_sentry_enabled: ->(val) { val } do - requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name' - end - optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects' - optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." - optional :koding_enabled, type: Boolean, desc: 'Enable Koding' - given koding_enabled: ->(val) { val } do - requires :koding_url, type: String, desc: 'The Koding team URL' - end - optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML' - given plantuml_enabled: ->(val) { val } do - requires :plantuml_url, type: String, desc: 'The PlantUML server URL' + optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.' + optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects' + given shared_runners_enabled: ->(val) { val } do + requires :shared_runners_text, type: String, desc: 'Shared runners text ' end - optional :version_check_enabled, type: Boolean, desc: 'Let GitLab inform you when an update is available.' - optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.' - optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.' - optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' - given housekeeping_enabled: ->(val) { val } do - requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance." - requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run." - requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run." - requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." + optional :sidekiq_throttling_enabled, type: Boolean, desc: 'Enable Sidekiq Job Throttling' + given sidekiq_throttling_enabled: ->(val) { val } do + requires :sidekiq_throttling_factor, type: Float, desc: 'The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.' + requires :sidekiq_throttling_queues, type: Array[String], desc: 'Choose which queues you wish to throttle' end + optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 + optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled' optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' - optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' - optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.' - optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.' - optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.' optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' + optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins' ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/api/users.rb b/lib/api/users.rb index e83887b3e9e..b0811bb4aad 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -121,6 +121,17 @@ module API present user, opts end + desc "Get the status of a user" + params do + requires :id_or_username, type: String, desc: 'The ID or username of the user' + end + get ":id_or_username/status" do + user = find_user(params[:id_or_username]) + not_found!('User') unless user && can?(current_user, :read_user, user) + + present user.status || {}, with: Entities::UserStatus + end + desc 'Create a user. Available only for admins.' do success Entities::UserPublic end @@ -740,6 +751,30 @@ module API present paginate(activities), with: Entities::UserActivity end + + desc 'Set the status of the current user' do + success Entities::UserStatus + end + params do + optional :emoji, type: String, desc: "The emoji to set on the status" + optional :message, type: String, desc: "The status message to set" + end + put "status" do + forbidden! unless can?(current_user, :update_user_status, current_user) + + if ::Users::SetStatusService.new(current_user, declared_params).execute + present current_user.status, with: Entities::UserStatus + else + render_validation_error!(current_user.status) + end + end + + desc 'get the status of the current user' do + success Entities::UserStatus + end + get 'status' do + present current_user.status || {}, with: Entities::UserStatus + end end end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index af762db517c..906ed498026 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -1,10 +1,7 @@ require 'yaml' -require_relative 'helper' module Backup class Repository - include Backup::Helper - attr_reader :progress def initialize(progress) @@ -42,131 +39,36 @@ module Backup end def prepare_directories - Gitlab.config.repositories.storages.each do |name, repository_storage| - delete_all_repositories(name, repository_storage) + Gitlab.config.repositories.storages.each do |name, _repository_storage| + Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories end end def backup_project(project) - gitaly_migrate(:repository_backup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - backup_project_gitaly(project) - else - backup_project_local(project) - end - end - - backup_custom_hooks(project) - rescue => e - progress_warn(project, e, 'Failed to backup repo') - end - - def backup_project_gitaly(project) path_to_project_bundle = path_to_bundle(project) Gitlab::GitalyClient::RepositoryService.new(project.repository) .create_bundle(path_to_project_bundle) - end - - def backup_project_local(project) - path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(project) - end - - path_to_project_bundle = path_to_bundle(project) - - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) - output, status = Gitlab::Popen.popen(cmd) - progress_warn(project, cmd.join(' '), output) unless status.zero? - end - - def delete_all_repositories(name, repository_storage) - gitaly_migrate(:delete_all_repositories, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories - else - local_delete_all_repositories(name, repository_storage) - end - end - end - - def local_delete_all_repositories(name, repository_storage) - path = repository_storage.legacy_disk_path - return unless File.exist?(path) - - bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) - FileUtils.mkdir_p(bk_repos_path, mode: 0700) - files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] - - begin - FileUtils.mv(files, bk_repos_path) - rescue Errno::EACCES - access_denied_error(path) - rescue Errno::EBUSY - resource_busy_error(path) - end - end - def local_restore_custom_hooks(project, dir) - path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(project) - end - cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) - output, status = Gitlab::Popen.popen(cmd) - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end - - def gitaly_restore_custom_hooks(project, dir) - custom_hooks_path = path_to_tars(project, dir) - Gitlab::GitalyClient::RepositoryService.new(project.repository) - .restore_custom_hooks(custom_hooks_path) + backup_custom_hooks(project) + rescue => e + progress_warn(project, e, 'Failed to backup repo') end - def local_backup_custom_hooks(project) - in_path(path_to_tars(project)) do |dir| - path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(project) - end - break unless File.exist?(File.join(path_to_project_repo, dir)) - - FileUtils.mkdir_p(path_to_tars(project)) - cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir}) - output, status = Gitlab::Popen.popen(cmd) - - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end - end + def backup_custom_hooks(project) + FileUtils.mkdir_p(project_backup_path(project)) - def gitaly_backup_custom_hooks(project) - FileUtils.mkdir_p(path_to_tars(project)) - custom_hooks_path = path_to_tars(project, 'custom_hooks') + custom_hooks_path = custom_hooks_tar(project) Gitlab::GitalyClient::RepositoryService.new(project.repository) .backup_custom_hooks(custom_hooks_path) end - def backup_custom_hooks(project) - gitaly_migrate(:backup_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_backup_custom_hooks(project) - else - local_backup_custom_hooks(project) - end - end - end - def restore_custom_hooks(project) - in_path(path_to_tars(project)) do |dir| - gitaly_migrate(:restore_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_restore_custom_hooks(project, dir) - else - local_restore_custom_hooks(project, dir) - end - end - end + return unless Dir.exist?(project_backup_path(project)) + return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none? + + custom_hooks_path = custom_hooks_tar(project) + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .restore_custom_hooks(custom_hooks_path) end def restore @@ -181,7 +83,8 @@ module Backup restore_repo_success = nil if File.exist?(path_to_project_bundle) begin - project.repository.create_from_bundle path_to_project_bundle + project.repository.create_from_bundle(path_to_project_bundle) + restore_custom_hooks(project) restore_repo_success = true rescue => e restore_repo_success = false @@ -197,8 +100,6 @@ module Backup progress.puts "[Failed] restoring #{project.full_path} repository".color(:red) end - restore_custom_hooks(project) - wiki = ProjectWiki.new(project) path_to_wiki_bundle = path_to_bundle(wiki) @@ -219,48 +120,28 @@ module Backup protected - def path_to_repo(project) - project.repository.path_to_repo - end - def path_to_bundle(project) File.join(backup_repos_path, project.disk_path + '.bundle') end - def path_to_tars(project, dir = nil) - path = File.join(backup_repos_path, project.disk_path) + def project_backup_path(project) + File.join(backup_repos_path, project.disk_path) + end - if dir - File.join(path, "#{dir}.tar") - else - path - end + def custom_hooks_tar(project) + File.join(project_backup_path(project), "custom_hooks.tar") end def backup_repos_path File.join(Gitlab.config.backup.path, 'repositories') end - def in_path(path) - return unless Dir.exist?(path) - - dir_entries = Dir.entries(path) - - if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar') - yield('custom_hooks') - end - end - def prepare FileUtils.rm_rf(backup_repos_path) FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.mkdir(backup_repos_path, mode: 0700) end - def silent - { err: '/dev/null', out: '/dev/null' } - end - private def progress_warn(project, cmd, output) @@ -273,18 +154,8 @@ module Backup project_or_wiki.repository.empty? end - def repository_storage_paths_args - Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } - end - def display_repo_path(project) project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path end - - def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - Gitlab::GitalyClient.migrate(method, status: status, &block) - rescue GRPC::NotFound, GRPC::BadStatus => e - raise Error, e - end end end diff --git a/lib/banzai/pipeline/emoji_pipeline.rb b/lib/banzai/pipeline/emoji_pipeline.rb new file mode 100644 index 00000000000..a1b522f4303 --- /dev/null +++ b/lib/banzai/pipeline/emoji_pipeline.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Banzai + module Pipeline + class EmojiPipeline < BasePipeline + # These filters will only perform sanitization of the content, preventing + # XSS, and replace emoji. + def self.filters + @filters ||= FilterArray[ + Filter::HtmlEntityFilter, + Filter::SanitizationFilter, + Filter::EmojiFilter + ] + end + end + end +end diff --git a/lib/feature.rb b/lib/feature.rb index d27b2b0e72f..09c5ef3ad94 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -46,6 +46,10 @@ class Feature get(key).enabled?(thing) end + def disabled?(key, thing = nil) + !enabled?(key, thing) + end + def enable(key, thing = true) get(key).enable(thing) end diff --git a/lib/gitlab/auth/activity.rb b/lib/gitlab/auth/activity.rb new file mode 100644 index 00000000000..761f0819c60 --- /dev/null +++ b/lib/gitlab/auth/activity.rb @@ -0,0 +1,78 @@ +module Gitlab + module Auth + ## + # Metrics and logging for user authentication activity. + # + class Activity + extend Gitlab::Utils::StrongMemoize + + COUNTERS = { + user_authenticated: 'Counter of successful authentication events', + user_unauthenticated: 'Counter of authentication failures', + user_not_found: 'Counter of failed log-ins when user is unknown', + user_password_invalid: 'Counter of failed log-ins with invalid password', + user_session_override: 'Counter of manual log-ins and sessions overrides', + user_session_destroyed: 'Counter of user sessions being destroyed', + user_two_factor_authenticated: 'Counter of two factor authentications', + user_sessionless_authentication: 'Counter of sessionless authentications', + user_blocked: 'Counter of sign in attempts when user is blocked' + }.freeze + + def initialize(opts) + @opts = opts + end + + def user_authentication_failed! + self.class.user_unauthenticated_counter_increment! + + case @opts[:message] + when :not_found_in_database + self.class.user_not_found_counter_increment! + when :invalid + self.class.user_password_invalid_counter_increment! + end + end + + def user_authenticated! + self.class.user_authenticated_counter_increment! + end + + def user_session_override! + self.class.user_session_override_counter_increment! + + case @opts[:message] + when :two_factor_authenticated + self.class.user_two_factor_authenticated_counter_increment! + when :sessionless_sign_in + self.class.user_sessionless_authentication_counter_increment! + end + end + + def user_blocked! + self.class.user_blocked_counter_increment! + end + + def user_session_destroyed! + self.class.user_session_destroyed_counter_increment! + end + + def self.each_counter + COUNTERS.each_pair do |metric, description| + yield "#{metric}_counter", metric, description + end + end + + each_counter do |counter, metric, description| + define_singleton_method(counter) do + strong_memoize(counter) do + Gitlab::Metrics.counter("gitlab_auth_#{metric}_total".to_sym, description) + end + end + + define_singleton_method("#{counter}_increment!") do + public_send(counter).increment # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end +end diff --git a/lib/gitlab/auth/blocked_user_tracker.rb b/lib/gitlab/auth/blocked_user_tracker.rb index 7609a7b04f6..50712d7eac2 100644 --- a/lib/gitlab/auth/blocked_user_tracker.rb +++ b/lib/gitlab/auth/blocked_user_tracker.rb @@ -2,35 +2,19 @@ module Gitlab module Auth class BlockedUserTracker - ACTIVE_RECORD_REQUEST_PARAMS = 'action_dispatch.request.request_parameters' - - def self.log_if_user_blocked(env) - message = env.dig('warden.options', :message) - - # Devise calls User#active_for_authentication? on the User model and then - # throws an exception to Warden with User#inactive_message: - # https://github.com/plataformatec/devise/blob/v4.2.1/lib/devise/hooks/activatable.rb#L8 - # - # Since Warden doesn't pass the user record to the failure handler, we - # need to do a database lookup with the username. We can limit the - # lookups to happen when the user was blocked by checking the inactive - # message passed along by Warden. - return unless message == User::BLOCKED_MESSAGE - - # Check for either LDAP or regular GitLab account logins - login = env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'username') || - env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login') - - return unless login.present? - - user = User.by_login(login) + def initialize(user, auth) + @user = user + @auth = auth + end - return unless user&.blocked? + def log_activity! + return unless @user.blocked? - Gitlab::AppLogger.info("Failed login for blocked user: user=#{user.username} ip=#{env['REMOTE_ADDR']}") - SystemHooksService.new.execute_hooks_for(user, :failed_login) + Gitlab::AppLogger.info <<~INFO + "Failed login for blocked user: user=#{@user.username} ip=#{@auth.request.ip}") + INFO - true + SystemHooksService.new.execute_hooks_for(@user, :failed_login) rescue TypeError end end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 4ca5a78e068..04aa6aab771 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -26,6 +26,12 @@ module Gitlab end end + # This is called from within a rake task only used by Admins, so allow writing + # to STDOUT + def self.log(message) + puts message # rubocop:disable Rails/Output + end + attr_reader :user, :project_name, :bare_repo delegate :log, to: :class @@ -59,11 +65,10 @@ module Gitlab import_type: 'bare_repository', namespace_id: group&.id).execute - if project.persisted? && mv_repo(project) + if project.persisted? && mv_repositories(project) log " * Created #{project.name} (#{project_full_path})".color(:green) project.write_repository_config - Gitlab::Git::Repository.create_hooks(project.repository.path_to_repo, Gitlab.config.gitlab_shell.hooks_path) ProjectCacheWorker.perform_async(project.id) else @@ -74,12 +79,11 @@ module Gitlab project end - def mv_repo(project) - storage_path = storage_path_for_shard(project.repository_storage) - FileUtils.mv(repo_path, project.repository.path_to_repo) + def mv_repositories(project) + mv_repo(bare_repo.repo_path, project.repository) if bare_repo.wiki_exists? - FileUtils.mv(wiki_path, File.join(storage_path, project.disk_path + '.wiki.git')) + mv_repo(bare_repo.wiki_path, project.wiki.repository) end true @@ -89,6 +93,11 @@ module Gitlab false end + def mv_repo(path, repository) + repository.create_from_bundle(bundle(path)) + FileUtils.rm_rf(path) + end + def storage_path_for_shard(shard) Gitlab.config.repositories.storages[shard].legacy_disk_path end @@ -101,10 +110,17 @@ module Gitlab Groups::NestedCreateService.new(user, group_path: group_path).execute end - # This is called from within a rake task only used by Admins, so allow writing - # to STDOUT - def self.log(message) - puts message # rubocop:disable Rails/Output + def bundle(repo_path) + # TODO: we could save some time and disk space by using + # `git bundle create - --all` and streaming the bundle directly to + # Gitaly, rather than writing it on disk first + bundle_path = "#{repo_path}.bundle" + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all) + output, status = Gitlab::Popen.popen(cmd) + + raise output unless status.zero? + + bundle_path end end end diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb index fe267248275..c0c666dfb7b 100644 --- a/lib/gitlab/bare_repository_import/repository.rb +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -1,5 +1,3 @@ -# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/953 -# module Gitlab module BareRepositoryImport class Repository diff --git a/lib/gitlab/ci/build/artifacts/gzip_file_adapter.rb b/lib/gitlab/ci/build/artifacts/gzip_file_adapter.rb new file mode 100644 index 00000000000..65f65cdce08 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/gzip_file_adapter.rb @@ -0,0 +1,46 @@ +module Gitlab + module Ci + module Build + module Artifacts + class GzipFileAdapter + attr_reader :stream + + InvalidStreamError = Class.new(StandardError) + + def initialize(stream) + raise InvalidStreamError, "Stream is required" unless stream + + @stream = stream + end + + def each_blob + stream.seek(0) + + until stream.eof? + gzip(stream) do |gz| + yield gz.read, gz.orig_name + unused = gz.unused&.length.to_i + # pos has already reached to EOF at the moment + # We rewind the pos to the top of unused files + # to read next gzip stream, to support multistream archives + # https://golang.org/src/compress/gzip/gunzip.go#L117 + stream.seek(-unused, IO::SEEK_CUR) + end + end + end + + private + + def gzip(stream, &block) + gz = Zlib::GzipReader.new(stream) + yield(gz) + rescue Zlib::Error => e + raise InvalidStreamError, e.message + ensure + gz&.finish + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb new file mode 100644 index 00000000000..a4eccc08dfc --- /dev/null +++ b/lib/gitlab/ci/parsers.rb @@ -0,0 +1,9 @@ +module Gitlab + module Ci + module Parsers + def self.fabricate!(file_type) + "Gitlab::Ci::Parsers::#{file_type.classify}".constantize.new + end + end + end +end diff --git a/lib/gitlab/ci/parsers/junit.rb b/lib/gitlab/ci/parsers/junit.rb new file mode 100644 index 00000000000..3c4668ec13b --- /dev/null +++ b/lib/gitlab/ci/parsers/junit.rb @@ -0,0 +1,69 @@ +module Gitlab + module Ci + module Parsers + class Junit + attr_reader :data + + JunitParserError = Class.new(StandardError) + + def parse!(xml_data, test_suite) + @data = Hash.from_xml(xml_data) + + each_suite do |testcases| + testcases.each do |testcase| + test_case = create_test_case(testcase) + test_suite.add_test_case(test_case) + end + end + rescue REXML::ParseException => e + raise JunitParserError, "XML parsing failed: #{e.message}" + rescue => e + raise JunitParserError, "JUnit parsing failed: #{e.message}" + end + + private + + def each_suite + testsuites.each do |testsuite| + yield testcases(testsuite) + end + end + + def testsuites + if data['testsuites'] + data['testsuites']['testsuite'] + else + [data['testsuite']] + end + end + + def testcases(testsuite) + if testsuite['testcase'].is_a?(Array) + testsuite['testcase'] + else + [testsuite['testcase']] + end + end + + def create_test_case(data) + if data['failure'] + status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED + system_output = data['failure'] + else + status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS + system_output = nil + end + + ::Gitlab::Ci::Reports::TestCase.new( + classname: data['classname'], + name: data['name'], + file: data['file'], + execution_time: data['time'], + status: status, + system_output: system_output + ) + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb new file mode 100644 index 00000000000..b4d08ed257f --- /dev/null +++ b/lib/gitlab/ci/reports/test_case.rb @@ -0,0 +1,32 @@ +module Gitlab + module Ci + module Reports + class TestCase + STATUS_SUCCESS = 'success'.freeze + STATUS_FAILED = 'failed'.freeze + STATUS_SKIPPED = 'skipped'.freeze + STATUS_ERROR = 'error'.freeze + STATUS_TYPES = [STATUS_SUCCESS, STATUS_FAILED, STATUS_SKIPPED, STATUS_ERROR].freeze + + attr_reader :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key + + def initialize(name:, classname:, execution_time:, status:, file: nil, system_output: nil, stack_trace: nil) + @name = name + @classname = classname + @file = file + @execution_time = execution_time.to_f + @status = status + @system_output = system_output + @stack_trace = stack_trace + @key = sanitize_key_name("#{classname}_#{name}") + end + + private + + def sanitize_key_name(key) + key.gsub(/[^0-9A-Za-z]/, '-') + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb new file mode 100644 index 00000000000..c6e732e68eb --- /dev/null +++ b/lib/gitlab/ci/reports/test_reports.rb @@ -0,0 +1,39 @@ +module Gitlab + module Ci + module Reports + class TestReports + attr_reader :test_suites + + def initialize + @test_suites = {} + end + + def get_suite(suite_name) + test_suites[suite_name] ||= TestSuite.new(suite_name) + end + + def total_time + test_suites.values.sum(&:total_time) + end + + def total_count + test_suites.values.sum(&:total_count) + end + + def total_status + if failed_count > 0 || error_count > 0 + TestCase::STATUS_FAILED + else + TestCase::STATUS_SUCCESS + end + end + + TestCase::STATUS_TYPES.each do |status_type| + define_method("#{status_type}_count") do + test_suites.values.sum { |suite| suite.public_send("#{status_type}_count") } # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_reports_comparer.rb b/lib/gitlab/ci/reports/test_reports_comparer.rb new file mode 100644 index 00000000000..c0943f5a51a --- /dev/null +++ b/lib/gitlab/ci/reports/test_reports_comparer.rb @@ -0,0 +1,38 @@ +module Gitlab + module Ci + module Reports + class TestReportsComparer + include Gitlab::Utils::StrongMemoize + + attr_reader :base_reports, :head_reports + + def initialize(base_reports, head_reports) + @base_reports = base_reports || TestReports.new + @head_reports = head_reports + end + + def suite_comparers + strong_memoize(:suite_comparers) do + head_reports.test_suites.map do |name, test_suite| + TestSuiteComparer.new(name, base_reports.get_suite(name), test_suite) + end + end + end + + def total_status + if suite_comparers.any? { |suite| suite.total_status == TestCase::STATUS_FAILED } + TestCase::STATUS_FAILED + else + TestCase::STATUS_SUCCESS + end + end + + %w(total_count resolved_count failed_count).each do |method| + define_method(method) do + suite_comparers.sum { |suite| suite.public_send(method) } # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb new file mode 100644 index 00000000000..b722d0ba735 --- /dev/null +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -0,0 +1,54 @@ +module Gitlab + module Ci + module Reports + class TestSuite + attr_reader :name + attr_reader :test_cases + attr_reader :total_time + + def initialize(name = nil) + @name = name + @test_cases = {} + @total_time = 0.0 + @duplicate_cases = [] + end + + def add_test_case(test_case) + @duplicate_cases << test_case if existing_key?(test_case) + + @test_cases[test_case.status] ||= {} + @test_cases[test_case.status][test_case.key] = test_case + @total_time += test_case.execution_time + end + + def total_count + test_cases.values.sum(&:count) + end + + def total_status + if failed_count > 0 || error_count > 0 + TestCase::STATUS_FAILED + else + TestCase::STATUS_SUCCESS + end + end + + TestCase::STATUS_TYPES.each do |status_type| + define_method("#{status_type}") do + test_cases[status_type] || {} + end + + define_method("#{status_type}_count") do + test_cases[status_type]&.length.to_i + end + end + + private + + def existing_key?(test_case) + @test_cases[test_case.status]&.key?(test_case.key) + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb new file mode 100644 index 00000000000..642aa593092 --- /dev/null +++ b/lib/gitlab/ci/reports/test_suite_comparer.rb @@ -0,0 +1,57 @@ +module Gitlab + module Ci + module Reports + class TestSuiteComparer + include Gitlab::Utils::StrongMemoize + + attr_reader :name, :base_suite, :head_suite + + def initialize(name, base_suite, head_suite) + @name = name + @base_suite = base_suite || TestSuite.new + @head_suite = head_suite + end + + def new_failures + strong_memoize(:new_failures) do + head_suite.failed.reject do |key, _| + base_suite.failed.include?(key) + end.values + end + end + + def existing_failures + strong_memoize(:existing_failures) do + head_suite.failed.select do |key, _| + base_suite.failed.include?(key) + end.values + end + end + + def resolved_failures + strong_memoize(:resolved_failures) do + head_suite.success.select do |key, _| + base_suite.failed.include?(key) + end.values + end + end + + def total_count + head_suite.total_count + end + + def total_status + head_suite.total_status + end + + def resolved_count + resolved_failures.count + end + + def failed_count + new_failures.count + existing_failures.count + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 155f4fc1343..703f0b9217b 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -4,12 +4,13 @@ module Gitlab module Build class Failed < Status::Extended REASONS = { - 'unknown_failure' => 'unknown failure', - 'script_failure' => 'script failure', - 'api_failure' => 'API failure', - 'stuck_or_timeout_failure' => 'stuck or timeout failure', - 'runner_system_failure' => 'runner system failure', - 'missing_dependency_failure' => 'missing dependency failure' + unknown_failure: 'unknown failure', + script_failure: 'script failure', + api_failure: 'API failure', + stuck_or_timeout_failure: 'stuck or timeout failure', + runner_system_failure: 'runner system failure', + missing_dependency_failure: 'missing dependency failure', + runner_unsupported: 'unsupported runner' }.freeze def status_tooltip @@ -31,7 +32,11 @@ module Gitlab end def description - "<br> (#{REASONS[subject.failure_reason]})" + "<br> (#{failure_reason_message})" + end + + def failure_reason_message + REASONS.fetch(subject.failure_reason.to_sym) end end end diff --git a/lib/gitlab/cleanup/project_upload_file_finder.rb b/lib/gitlab/cleanup/project_upload_file_finder.rb new file mode 100644 index 00000000000..2ee8b60e76a --- /dev/null +++ b/lib/gitlab/cleanup/project_upload_file_finder.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + class ProjectUploadFileFinder + FIND_BATCH_SIZE = 500 + ABSOLUTE_UPLOAD_DIR = FileUploader.root.freeze + EXCLUDED_SYSTEM_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/-/*".freeze + EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze + EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze + + # Paths are relative to the upload directory + def each_file_batch(batch_size: FIND_BATCH_SIZE, &block) + cmd = build_find_command(ABSOLUTE_UPLOAD_DIR) + + Open3.popen2(*cmd) do |stdin, stdout, status_thread| + yield_paths_in_batches(stdout, batch_size, &block) + + raise "Find command failed" unless status_thread.value.success? + end + end + + private + + def yield_paths_in_batches(stdout, batch_size, &block) + paths = [] + + stdout.each_line("\0") do |line| + paths << line.chomp("\0") + + if paths.size >= batch_size + yield(paths) + paths = [] + end + end + + yield(paths) if paths.any? + end + + def build_find_command(search_dir) + cmd = %W[find -L #{search_dir} + -type f + ! ( -path #{EXCLUDED_SYSTEM_UPLOADS_PATH} -prune ) + ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) + ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) + -print0] + + ionice = which_ionice + cmd = %W[#{ionice} -c Idle] + cmd if ionice + + log_msg = "find command: \"#{cmd.join(' ')}\"" + Rails.logger.info log_msg + + cmd + end + + def which_ionice + Gitlab::Utils.which('ionice') + rescue StandardError + # In this case, returning false is relatively safe, + # even though it isn't very nice + false + end + end + end +end diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb new file mode 100644 index 00000000000..f55ab535efe --- /dev/null +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + class ProjectUploads + LOST_AND_FOUND = File.join(ProjectUploadFileFinder::ABSOLUTE_UPLOAD_DIR, '-', 'project-lost-found') + + attr_reader :logger + + def initialize(logger: nil) + @logger = logger || Rails.logger + end + + def run!(dry_run: true) + logger.info "Looking for orphaned project uploads to clean up#{'. Dry run' if dry_run}..." + + each_orphan_file do |path, upload_path| + result = cleanup(path, upload_path, dry_run) + + logger.info result + end + end + + private + + def cleanup(path, upload_path, dry_run) + # This happened in staging: + # `find` returned a path on which `File.delete` raised `Errno::ENOENT` + return "Cannot find file: #{path}" unless File.exist?(path) + + correct_path = upload_path && find_correct_path(upload_path) + + if correct_path + move(path, correct_path, 'fix', dry_run) + else + move_to_lost_and_found(path, dry_run) + end + end + + # Accepts a path in the form of "#{hex_secret}/#{filename}" + def find_correct_path(upload_path) + upload = Upload.find_by(uploader: 'FileUploader', path: upload_path) + return unless upload && upload.local? && upload.model + + upload.absolute_path + rescue => e + logger.error e.message + + # absolute_path depends on a lot of code. If it doesn't work, then it + # it doesn't matter if the upload file is in the right place. Treat it + # as uncorrectable. + # I.e. the project record might be missing, which raises an exception. + nil + end + + def move_to_lost_and_found(path, dry_run) + new_path = path.sub(/\A#{ProjectUploadFileFinder::ABSOLUTE_UPLOAD_DIR}/, LOST_AND_FOUND) + + move(path, new_path, 'move to lost and found', dry_run) + end + + def move(path, new_path, prefix, dry_run) + action = "#{prefix} #{path} -> #{new_path}" + + if dry_run + "Can #{action}" + else + begin + FileUtils.mkdir_p(File.dirname(new_path)) + FileUtils.mv(path, new_path) + + "Did #{action}" + rescue => e + "Error during #{action}: #{e.inspect}" + end + end + end + + # Yields absolute paths of project upload files that are not in the + # uploads table + def each_orphan_file + ProjectUploadFileFinder.new.each_file_batch do |file_paths| + logger.debug "Processing batch of #{file_paths.size} project upload file paths, starting with #{file_paths.first}" + + file_paths.each do |path| + pup = ProjectUploadPath.from_path(path) + + yield(path, pup.upload_path) if pup.orphan? + end + end + end + + class ProjectUploadPath + PROJECT_FULL_PATH_REGEX = %r{\A#{FileUploader.root}/(.+)/(\h+/[^/]+)\z}.freeze + + attr_reader :full_path, :upload_path + + def initialize(full_path, upload_path) + @full_path = full_path + @upload_path = upload_path + end + + def self.from_path(path) + path_matched = path.match(PROJECT_FULL_PATH_REGEX) + return new(nil, nil) unless path_matched + + new(path_matched[1], path_matched[2]) + end + + def orphan? + return true if full_path.nil? || upload_path.nil? + + # It's possible to reduce to one query, but `where_full_path_in` is complex + !Upload.exists?(path: upload_path, model_id: project_id, model_type: 'Project', uploader: 'FileUploader') + end + + private + + def project_id + @project_id ||= Project.where_full_path_in([full_path]).pluck(:id) + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index eb02c7ac8e1..73151e4a4c5 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -39,19 +39,6 @@ module Gitlab ChecksumError = Class.new(StandardError) class << self - # Unlike `new`, `create` takes the repository path - def create(repo_path, bare: true, symlink_hooks_to: nil) - FileUtils.mkdir_p(repo_path, mode: 0770) - - # Equivalent to `git --git-path=#{repo_path} init [--bare]` - repo = Rugged::Repository.init_at(repo_path, bare) - repo.close - - create_hooks(repo_path, symlink_hooks_to) if symlink_hooks_to.present? - - true - end - def create_hooks(repo_path, global_hooks_path) local_hooks_path = File.join(repo_path, 'hooks') real_local_hooks_path = :not_found diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index 742118b76a8..e731e654f3c 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class GitPostReceive include Gitlab::Identifier @@ -14,10 +16,11 @@ module Gitlab end def changes_refs - return enum_for(:changes_refs) unless block_given? + return changes unless block_given? changes.each do |change| - oldrev, newrev, ref = change.strip.split(' ') + change.strip! + oldrev, newrev, ref = change.split(' ') yield oldrev, newrev, ref end @@ -26,13 +29,10 @@ module Gitlab private def deserialize_changes(changes) - changes = utf8_encode_changes(changes) - changes.lines + utf8_encode_changes(changes).each_line end def utf8_encode_changes(changes) - changes = changes.dup - changes.force_encoding('UTF-8') return changes if changes.valid_encoding? diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index fbe7d4ba1ae..41d58192818 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -166,7 +166,7 @@ module Gitlab except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) } ) - response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.fast_timeout) + response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.default_timeout) raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? end diff --git a/lib/gitlab/graphs/commits.rb b/lib/gitlab/graphs/commits.rb index 3caf9036459..c4ffc19df09 100644 --- a/lib/gitlab/graphs/commits.rb +++ b/lib/gitlab/graphs/commits.rb @@ -18,7 +18,7 @@ module Gitlab end def commit_per_day - @commit_per_day ||= @commits.size / (@duration + 1) + @commit_per_day ||= (@commits.size.to_f / (@duration + 1)).round(1) end def collect_data diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index 9251ed654cd..d11fcc6a3e3 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -30,7 +30,7 @@ module Gitlab end end - # Flag a project to me migrated + # Flag a project to be migrated # # @param [Object] project that will be migrated def migrate(project) diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2f163db936b..3adc44f8044 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -18,6 +18,21 @@ module Gitlab private + def download_or_copy_upload(uploader, upload_path) + if uploader.upload.local? + copy_files(uploader.path, upload_path) + else + download(uploader.url, upload_path) + end + end + + def download(url, upload_path) + File.open(upload_path, 'w') do |file| + # Download (stream) file from the uploader's location + IO.copy_stream(URI.parse(url).open, file) + end + end + def tar_with_options(archive:, dir:, options:) execute(%W(tar -#{options} #{archive} -C #{dir} .)) end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 4c411f4847e..7fd66b4e244 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -10,15 +10,18 @@ module Gitlab new(*args).import end - def initialize(archive_file:, shared:) + def initialize(project:, archive_file:, shared:) + @project = project @archive_file = archive_file @shared = shared end def import mkdir_p(@shared.export_path) + mkdir_p(@shared.archive_path) - remove_symlinks! + remove_symlinks + copy_archive wait_for_archived_file do decompress_archive @@ -27,7 +30,8 @@ module Gitlab @shared.error(e) false ensure - remove_symlinks! + remove_import_file + remove_symlinks end private @@ -51,7 +55,15 @@ module Gitlab result end - def remove_symlinks! + def copy_archive + return if @archive_file + + @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) + + download_or_copy_upload(@project.import_export_upload.import_file, @archive_file) + end + + def remove_symlinks extracted_files.each do |path| FileUtils.rm(path) if File.lstat(path).symlink? end @@ -59,6 +71,10 @@ module Gitlab true end + def remove_import_file + FileUtils.rm_rf(@archive_file) + end + def extracted_files Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) } end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index da3667faf7a..f69f98a78a3 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -107,6 +107,7 @@ excluded_attributes: - :storage_version - :remote_mirror_available_overridden - :description_html + - :repository_languages snippets: - :expired_at merge_request_diff: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 63cab07324a..4e179f63d8c 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -35,7 +35,8 @@ module Gitlab end def import_file - Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + Gitlab::ImportExport::FileImporter.import(project: @project, + archive_file: @archive_file, shared: @shared) end @@ -91,7 +92,14 @@ module Gitlab end def remove_import_file - FileUtils.rm_rf(@archive_file) + return unless Gitlab::ImportExport.object_storage? + + upload = @project.import_export_upload + + return unless upload&.import_file&.file + + upload.remove_import_file! + upload.save! end def overwrite_project diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 1110149712d..07875ebb56a 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -91,10 +91,7 @@ module Gitlab mkdir_p(File.join(uploads_export_path, secret)) - File.open(upload_path, 'w') do |file| - # Download (stream) file from the uploader's location - IO.copy_stream(URI.parse(upload.file.url).open, file) - end + download_or_copy_upload(upload, upload_path) end end end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index af9b880ef9e..45816bee176 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -22,24 +22,28 @@ module Gitlab class << self def options - @options ||= Hash[ImportTable.map { |importer| [importer.title, importer.name] }] + Hash[import_table.map { |importer| [importer.title, importer.name] }] end def values - @values ||= ImportTable.map(&:name) + import_table.map(&:name) end def importer_names - @importer_names ||= ImportTable.select(&:importer).map(&:name) + import_table.select(&:importer).map(&:name) end def importer(name) - ImportTable.find { |import_source| import_source.name == name }.importer + import_table.find { |import_source| import_source.name == name }.importer end def title(name) options.key(name) end + + def import_table + ImportTable + end end end end diff --git a/lib/gitlab/kubernetes/config_map.rb b/lib/gitlab/kubernetes/config_map.rb index 9e55dae137c..8a8a59a9cd4 100644 --- a/lib/gitlab/kubernetes/config_map.rb +++ b/lib/gitlab/kubernetes/config_map.rb @@ -1,15 +1,15 @@ module Gitlab module Kubernetes class ConfigMap - def initialize(name, files) + def initialize(name, values = "") @name = name - @files = files + @values = values end def generate resource = ::Kubeclient::Resource.new resource.metadata = metadata - resource.data = files + resource.data = { values: values } resource end @@ -19,7 +19,7 @@ module Gitlab private - attr_reader :name, :files + attr_reader :name, :values def metadata { diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 0f0588b8b23..530ccf88053 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -1,7 +1,7 @@ module Gitlab module Kubernetes module Helm - HELM_VERSION = '2.7.0'.freeze + HELM_VERSION = '2.7.2'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze end end diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index d65374cc23b..c4de9a398cc 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -9,7 +9,7 @@ module Gitlab def install(command) namespace.ensure_exists! - create_config_map(command) + create_config_map(command) if command.config_map? kubeclient.create_pod(command.pod_resource) end diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb index afcfd109de0..f9ebe53d6af 100644 --- a/lib/gitlab/kubernetes/helm/base_command.rb +++ b/lib/gitlab/kubernetes/helm/base_command.rb @@ -1,7 +1,13 @@ module Gitlab module Kubernetes module Helm - module BaseCommand + class BaseCommand + attr_reader :name + + def initialize(name) + @name = name + end + def pod_resource Gitlab::Kubernetes::Helm::Pod.new(self, namespace).generate end @@ -18,32 +24,16 @@ module Gitlab HEREDOC end - def pod_name - "install-#{name}" - end - - def config_map_resource - Gitlab::Kubernetes::ConfigMap.new(name, files).generate + def config_map? + false end - def file_names - files.keys - end - - def name - raise "Not implemented" - end - - def files - raise "Not implemented" + def pod_name + "install-#{name}" end private - def files_dir - "/data/helm/#{name}/config" - end - def namespace Gitlab::Kubernetes::Helm::NAMESPACE end diff --git a/lib/gitlab/kubernetes/helm/certificate.rb b/lib/gitlab/kubernetes/helm/certificate.rb deleted file mode 100644 index c344add82cd..00000000000 --- a/lib/gitlab/kubernetes/helm/certificate.rb +++ /dev/null @@ -1,72 +0,0 @@ -module Gitlab - module Kubernetes - module Helm - class Certificate - INFINITE_EXPIRY = 1000.years - SHORT_EXPIRY = 30.minutes - - attr_reader :key, :cert - - def key_string - @key.to_s - end - - def cert_string - @cert.to_pem - end - - def self.from_strings(key_string, cert_string) - key = OpenSSL::PKey::RSA.new(key_string) - cert = OpenSSL::X509::Certificate.new(cert_string) - new(key, cert) - end - - def self.generate_root - _issue(signed_by: nil, expires_in: INFINITE_EXPIRY, certificate_authority: true) - end - - def issue(expires_in: SHORT_EXPIRY) - self.class._issue(signed_by: self, expires_in: expires_in, certificate_authority: false) - end - - private - - def self._issue(signed_by:, expires_in:, certificate_authority:) - key = OpenSSL::PKey::RSA.new(4096) - public_key = key.public_key - - subject = OpenSSL::X509::Name.parse("/C=US") - - cert = OpenSSL::X509::Certificate.new - cert.subject = subject - - cert.issuer = signed_by&.cert&.subject || subject - - cert.not_before = Time.now - cert.not_after = expires_in.from_now - cert.public_key = public_key - cert.serial = 0x0 - cert.version = 2 - - if certificate_authority - extension_factory = OpenSSL::X509::ExtensionFactory.new - extension_factory.subject_certificate = cert - extension_factory.issuer_certificate = cert - cert.add_extension(extension_factory.create_extension('subjectKeyIdentifier', 'hash')) - cert.add_extension(extension_factory.create_extension('basicConstraints', 'CA:TRUE', true)) - cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true)) - end - - cert.sign(signed_by&.key || key, OpenSSL::Digest::SHA256.new) - - new(key, cert) - end - - def initialize(key, cert) - @key = key - @cert = cert - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb index a4546509515..a02e64561f6 100644 --- a/lib/gitlab/kubernetes/helm/init_command.rb +++ b/lib/gitlab/kubernetes/helm/init_command.rb @@ -1,16 +1,7 @@ module Gitlab module Kubernetes module Helm - class InitCommand - include BaseCommand - - attr_reader :name, :files - - def initialize(name:, files:) - @name = name - @files = files - end - + class InitCommand < BaseCommand def generate_script super + [ init_helm_command @@ -20,12 +11,7 @@ module Gitlab private def init_helm_command - tls_flags = "--tiller-tls" \ - " --tiller-tls-verify --tls-ca-cert #{files_dir}/ca.pem" \ - " --tiller-tls-cert #{files_dir}/cert.pem" \ - " --tiller-tls-key #{files_dir}/key.pem" - - "helm init #{tls_flags} >/dev/null" + "helm init >/dev/null" end end end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index c7d6a9c5b4d..d2133a6d65b 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -1,16 +1,14 @@ module Gitlab module Kubernetes module Helm - class InstallCommand - include BaseCommand + class InstallCommand < BaseCommand + attr_reader :name, :chart, :version, :repository, :values - attr_reader :name, :files, :chart, :version, :repository - - def initialize(name:, chart:, files:, version: nil, repository: nil) + def initialize(name, chart:, values:, version: nil, repository: nil) @name = name @chart = chart @version = version - @files = files + @values = values @repository = repository end @@ -22,6 +20,14 @@ module Gitlab ].compact.join("\n") end + def config_map? + true + end + + def config_map_resource + Gitlab::Kubernetes::ConfigMap.new(name, values).generate + end + private def init_command @@ -33,27 +39,14 @@ module Gitlab end def script_command - "helm install" \ - "#{optional_tls_flags} " \ - "#{chart} " \ - "--name #{name}" \ - "#{optional_version_flag} " \ - "--namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} " \ - "-f /data/helm/#{name}/config/values.yaml >/dev/null\n" + <<~HEREDOC + helm install #{chart} --name #{name}#{optional_version_flag} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null + HEREDOC end def optional_version_flag " --version #{version}" if version end - - def optional_tls_flags - return unless files.key?(:'ca.pem') - - " --tls" \ - " --tls-ca-cert #{files_dir}/ca.pem" \ - " --tls-cert #{files_dir}/cert.pem" \ - " --tls-key #{files_dir}/key.pem" - end end end end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index 6e5d3388405..1e12299eefd 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -10,8 +10,10 @@ module Gitlab def generate spec = { containers: [container_specification], restartPolicy: 'Never' } - spec[:volumes] = volumes_specification - spec[:containers][0][:volumeMounts] = volume_mounts_specification + if command.config_map? + spec[:volumes] = volumes_specification + spec[:containers][0][:volumeMounts] = volume_mounts_specification + end ::Kubeclient::Resource.new(metadata: metadata, spec: spec) end @@ -59,7 +61,7 @@ module Gitlab name: 'configuration-volume', configMap: { name: "values-content-configuration-#{command.name}", - items: command.file_names.map { |name| { key: name, path: name } } + items: [{ key: 'values', path: 'values.yaml' }] } } ] diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb new file mode 100644 index 00000000000..a41435fdb79 --- /dev/null +++ b/lib/gitlab/language_detection.rb @@ -0,0 +1,68 @@ +module Gitlab + class LanguageDetection + MAX_LANGUAGES = 5 + + def initialize(repository, repository_languages) + @repository = repository + @repository_languages = repository_languages + end + + def languages + detection.keys + end + + def language_color(name) + detection.dig(name, :color) + end + + # Newly detected languages, returned in a structure accepted by + # Gitlab::Database.bulk_insert + def insertions(programming_languages) + lang_to_id = programming_languages.map { |p| [p.name, p.id] }.to_h + + (languages - previous_language_names).map do |new_lang| + { + project_id: @repository.project.id, + share: detection[new_lang][:value], + programming_language_id: lang_to_id[new_lang] + } + end + end + + # updates analyses which records only require updating of their share + def updates + to_update = @repository_languages.select do |lang| + detection.key?(lang.name) && detection[lang.name][:value] != lang.share + end + + to_update.map do |lang| + { programming_language_id: lang.programming_language_id, share: detection[lang.name][:value] } + end + end + + # Returns the ids of the programming languages that do not occur in the detection + # as current repository languages + def deletions + @repository_languages.map do |repo_lang| + next if detection.key?(repo_lang.name) + + repo_lang.programming_language_id + end.compact + end + + private + + def previous_language_names + @previous_language_names ||= @repository_languages.map(&:name) + end + + def detection + @detection ||= + @repository + .languages + .first(MAX_LANGUAGES) + .map { |l| [l[:label], l] } + .to_h + end + end +end diff --git a/lib/gitlab/middleware/basic_health_check.rb b/lib/gitlab/middleware/basic_health_check.rb new file mode 100644 index 00000000000..f2a03217098 --- /dev/null +++ b/lib/gitlab/middleware/basic_health_check.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +# This middleware provides a health check that does not hit the database. Its purpose +# is to notify the prober that the application server is handling requests, but a 200 +# response does not signify that the database or other services are ready. +# +# See https://thisdata.com/blog/making-a-rails-health-check-that-doesnt-hit-the-database/ for +# more details. + +module Gitlab + module Middleware + class BasicHealthCheck + # This can't be frozen because Rails::Rack::Logger wraps the body + # rubocop:disable Style/MutableConstant + OK_RESPONSE = [200, { 'Content-Type' => 'text/plain' }, ["GitLab OK"]] + EMPTY_RESPONSE = [404, { 'Content-Type' => 'text/plain' }, [""]] + # rubocop:enable Style/MutableConstant + HEALTH_PATH = '/-/health' + + def initialize(app) + @app = app + end + + def call(env) + return @app.call(env) unless env['PATH_INFO'] == HEALTH_PATH + + request = Rack::Request.new(env) + + return OK_RESPONSE if client_ip_whitelisted?(request) + + EMPTY_RESPONSE + end + + def client_ip_whitelisted?(request) + ip_whitelist.any? { |e| e.include?(request.ip) } + end + + def ip_whitelist + @ip_whitelist ||= Settings.monitoring.ip_whitelist.map(&IPAddr.method(:new)) + end + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index e1a958c508a..0f26fcfe8cb 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -99,5 +99,9 @@ module Gitlab ) }mx end + + def jira_transition_id_regex + @jira_transition_id_regex ||= /\d+/ + end end end diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb new file mode 100644 index 00000000000..f24a01e6cf5 --- /dev/null +++ b/lib/gitlab/template_helper.rb @@ -0,0 +1,28 @@ +module Gitlab + module TemplateHelper + include Gitlab::Utils::StrongMemoize + + def prepare_template_environment(file) + return unless file + + if Gitlab::ImportExport.object_storage? + params[:import_export_upload] = ImportExportUpload.new(import_file: file) + else + FileUtils.mkdir_p(File.dirname(import_upload_path)) + FileUtils.copy_entry(file.path, import_upload_path) + + params[:import_source] = import_upload_path + end + end + + def import_upload_path + strong_memoize(:import_upload_path) do + Gitlab::ImportExport.import_upload_path(filename: tmp_filename) + end + end + + def tmp_filename + SecureRandom.hex + end + end +end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index a8acafa9cd9..e5b5f3548e4 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -385,14 +385,6 @@ namespace :gitlab do end end - namespace :repo do - desc "GitLab | Check the integrity of the repositories managed by GitLab" - task check: :gitlab_environment do - puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red) - Rake::Task["gitlab:git:fsck"].execute - end - end - namespace :orphans do desc 'Gitlab | Check for orphaned namespaces and repositories' task check: :gitlab_environment do @@ -422,23 +414,6 @@ namespace :gitlab do end end - namespace :user do - desc "GitLab | Check the integrity of a specific user's repositories" - task :check_repos, [:username] => :gitlab_environment do |t, args| - username = args[:username] || prompt("Check repository integrity for username? ".color(:blue)) - user = User.find_by(username: username) - if user - repo_dirs = user.authorized_projects.map do |p| - p.repository.path_to_repo - end - - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } - else - puts "\nUser '#{username}' not found".color(:red) - end - end - end - # Helper methods ########################## diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 5e07b12ee1c..a2feb074b1d 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -7,9 +7,8 @@ namespace :gitlab do desc "GitLab | Cleanup | Clean namespaces" task dirs: :gitlab_environment do warn_user_is_not_gitlab - remove_flag = ENV['REMOVE'] - namespaces = Namespace.pluck(:path) + namespaces = Namespace.pluck(:path) namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored Gitlab.config.repositories.storages.each do |name, repository_storage| git_base_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path } @@ -31,7 +30,7 @@ namespace :gitlab do end all_dirs.each do |dir_path| - if remove_flag + if remove? if FileUtils.rm_rf dir_path puts "Removed...#{dir_path}".color(:red) else @@ -43,7 +42,7 @@ namespace :gitlab do end end - unless remove_flag + unless remove? puts "To cleanup this directories run this command with REMOVE=true".color(:yellow) end end @@ -104,5 +103,37 @@ namespace :gitlab do puts "To block these users run this command with BLOCK=true".color(:yellow) end end + + desc "GitLab | Cleanup | Clean orphaned project uploads" + task project_uploads: :gitlab_environment do + warn_user_is_not_gitlab + + cleaner = Gitlab::Cleanup::ProjectUploads.new(logger: logger) + cleaner.run!(dry_run: dry_run?) + + if dry_run? + logger.info "To clean up these files run this command with DRY_RUN=false".color(:yellow) + end + end + + def remove? + ENV['REMOVE'] == 'true' + end + + def dry_run? + ENV['DRY_RUN'] != 'false' + end + + def logger + return @logger if defined?(@logger) + + @logger = if Rails.env.development? || Rails.env.production? + Logger.new(STDOUT).tap do |stdout_logger| + stdout_logger.extend(ActiveSupport::Logger.broadcast(Rails.logger)) + end + else + Rails.logger + end + end end end diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index cb4f7e5c8a8..8a53b51d4fe 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -1,87 +1,24 @@ namespace :gitlab do namespace :git do - desc "GitLab | Git | Repack" - task repack: :gitlab_environment do - failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo") - if failures.empty? - puts "Done".color(:green) - else - output_failures(failures) - end - end - - desc "GitLab | Git | Run garbage collection on all repos" - task gc: :gitlab_environment do - failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} gc --auto --quiet), "Garbage Collecting") - if failures.empty? - puts "Done".color(:green) - else - output_failures(failures) - end - end - - desc "GitLab | Git | Prune all repos" - task prune: :gitlab_environment do - failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} prune), "Git Prune") - if failures.empty? - puts "Done".color(:green) - else - output_failures(failures) - end - end - desc 'GitLab | Git | Check all repos integrity' task fsck: :gitlab_environment do - failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo| - check_config_lock(repo) - check_ref_locks(repo) - end - - if failures.empty? - puts "Done".color(:green) - else - output_failures(failures) - end - end - - def perform_git_cmd(cmd, message) - puts "Starting #{message} on all repositories" - failures = [] - all_repos do |repo| - if system(*cmd, chdir: repo) - puts "Performed #{message} at #{repo}" - else - failures << repo + Project.find_each(batch_size: 100) do |project| + begin + project.repository.fsck + + rescue => e + failures << "#{project.full_path} on #{project.repository_storage}: #{e}" end - yield(repo) if block_given? + puts "Performed integrity check for #{project.repository.full_path}" end - failures - end - - def output_failures(failures) - puts "The following repositories reported errors:".color(:red) - failures.each { |f| puts "- #{f}" } - end - - def check_config_lock(repo_dir) - config_exists = File.exist?(File.join(repo_dir, 'config.lock')) - config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) - - puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" - end - - def check_ref_locks(repo_dir) - lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock')) - - if lock_files.present? - puts "Ref lock files exist:".color(:red) - - lock_files.each { |lock_file| puts " #{lock_file}" } + if failures.empty? + puts "Done".color(:green) else - puts "No ref lock files exist".color(:green) + puts "The following repositories reported errors:".color(:red) + failures.each { |f| puts "- #{f}" } end end end |