diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-04-20 21:38:24 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-04-20 21:38:24 +0300 |
commit | 983a0bba5d2a042c4a3bbb22432ec192c7501d82 (patch) | |
tree | b153cd387c14ba23bd5a07514c7c01fddf6a78a0 /lib/gitlab | |
parent | a2bddee2cdb38673df0e004d5b32d9f77797de64 (diff) |
Add latest changes from gitlab-org/gitlab@12-10-stable-ee
Diffstat (limited to 'lib/gitlab')
45 files changed, 721 insertions, 273 deletions
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb index f6e22044142..5146f92f521 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -24,6 +24,13 @@ module Gitlab EVENTS = ENUM_MAPPING.keys.freeze + INTERNAL_EVENTS = [ + StageEvents::CodeStageStart, + StageEvents::IssueStageEnd, + StageEvents::PlanStageStart, + StageEvents::ProductionStageEnd + ].freeze + # Defines which start_event and end_event pairs are allowed PAIRING_RULES = { StageEvents::PlanStageStart => [ @@ -67,6 +74,11 @@ module Gitlab def self.enum_mapping ENUM_MAPPING end + + # Events that are specific to the 7 default stages + def self.internal_events + INTERNAL_EVENTS + end end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 60a50e97998..a3feda9bb59 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -25,6 +25,10 @@ module Gitlab Labkit::Context.push(application_context.to_lazy_hash) end + def self.current_context_include?(attribute_name) + Labkit::Context.current.to_h.include?(Labkit::Context.log_key(attribute_name)) + end + def initialize(**args) unknown_attributes = args.keys - APPLICATION_ATTRIBUTES.map(&:name) raise ArgumentError, "#{unknown_attributes} are not known keys" if unknown_attributes.any? diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb new file mode 100644 index 00000000000..491facd0a43 --- /dev/null +++ b/lib/gitlab/ci/jwt.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Jwt + NOT_BEFORE_TIME = 5 + DEFAULT_EXPIRE_TIME = 60 * 5 + + def self.for_build(build) + self.new(build, ttl: build.metadata_timeout).encoded + end + + def initialize(build, ttl: nil) + @build = build + @ttl = ttl + end + + def payload + custom_claims.merge(reserved_claims) + end + + def encoded + headers = { kid: kid, typ: 'JWT' } + + JWT.encode(payload, key, 'RS256', headers) + end + + private + + attr_reader :build, :ttl, :key_data + + def reserved_claims + now = Time.now.to_i + + { + jti: SecureRandom.uuid, + iss: Settings.gitlab.host, + iat: now, + nbf: now - NOT_BEFORE_TIME, + exp: now + (ttl || DEFAULT_EXPIRE_TIME), + sub: "job_#{build.id}" + } + end + + def custom_claims + { + namespace_id: namespace.id.to_s, + namespace_path: namespace.full_path, + project_id: project.id.to_s, + project_path: project.full_path, + user_id: user&.id.to_s, + user_login: user&.username, + user_email: user&.email, + pipeline_id: build.pipeline.id.to_s, + job_id: build.id.to_s, + ref: source_ref, + ref_type: ref_type, + ref_protected: build.protected.to_s + } + end + + def key + @key ||= OpenSSL::PKey::RSA.new(Rails.application.secrets.openid_connect_signing_key) + end + + def public_key + key.public_key + end + + def kid + public_key.to_jwk[:kid] + end + + def project + build.project + end + + def namespace + project.namespace + end + + def user + build.user + end + + def source_ref + build.pipeline.source_ref + end + + def ref_type + ::Ci::BuildRunnerPresenter.new(build).ref_type + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/factory.rb b/lib/gitlab/ci/status/bridge/factory.rb index 910de865483..5d397dba0de 100644 --- a/lib/gitlab/ci/status/bridge/factory.rb +++ b/lib/gitlab/ci/status/bridge/factory.rb @@ -5,6 +5,10 @@ module Gitlab module Status module Bridge class Factory < Status::Factory + def self.extended_statuses + [Status::Bridge::Failed] + end + def self.common_helpers Status::Bridge::Common end diff --git a/lib/gitlab/ci/status/bridge/failed.rb b/lib/gitlab/ci/status/bridge/failed.rb new file mode 100644 index 00000000000..de7446c238c --- /dev/null +++ b/lib/gitlab/ci/status/bridge/failed.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class Failed < Status::Build::Failed + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 6b72db951ed..3949b87bbda 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,6 +1,6 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.2.1" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.2.2" variables: DOCKER_TLS_CERTDIR: "" services: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index c6c8256b4bb..9bf0d31409a 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.12.1" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.13.0" review: extends: .auto-deploy diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 713b11c4d8f..54a29b04d39 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.13.1" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.15.0" environment: name: production variables: @@ -17,6 +17,8 @@ apply: ELASTIC_STACK_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/elastic-stack/values.yaml VAULT_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/vault/values.yaml CROSSPLANE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/crossplane/values.yaml + FLUENTD_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/fluentd/values.yaml + KNATIVE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/knative/values.yaml script: - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml only: diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 717e91b3ae5..0ecf37b37a3 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -57,6 +57,8 @@ dependency_scanning: PIP_EXTRA_INDEX_URL \ PIP_REQUIREMENTS_FILE \ MAVEN_CLI_OPTS \ + GRADLE_CLI_OPTS \ + SBT_CLI_OPTS \ BUNDLER_AUDIT_UPDATE_DISABLED \ BUNDLER_AUDIT_ADVISORY_DB_URL \ BUNDLER_AUDIT_ADVISORY_DB_REF_NAME \ diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 891fd8c1bb5..2b08d3c63bb 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -43,7 +43,7 @@ module Gitlab end def uncached_application_settings - return fake_application_settings unless connect_to_db? + return fake_application_settings if Gitlab::Runtime.rake? && !connect_to_db? current_settings = ::ApplicationSetting.current # If there are pending migrations, it's possible there are columns that diff --git a/lib/gitlab/cycle_analytics/group_stage_summary.rb b/lib/gitlab/cycle_analytics/group_stage_summary.rb index 26eaaf7df83..09b33d01846 100644 --- a/lib/gitlab/cycle_analytics/group_stage_summary.rb +++ b/lib/gitlab/cycle_analytics/group_stage_summary.rb @@ -12,14 +12,42 @@ module Gitlab end def data - [serialize(Summary::Group::Issue.new(group: group, current_user: current_user, options: options)), - serialize(Summary::Group::Deploy.new(group: group, options: options))] + [issue_stats, + deploy_stats, + deployment_frequency_stats] end private - def serialize(summary_object) - AnalyticsSummarySerializer.new.represent(summary_object) + def issue_stats + serialize( + Summary::Group::Issue.new( + group: group, current_user: current_user, options: options) + ) + end + + def deployments_summary + @deployments_summary ||= + Summary::Group::Deploy.new(group: group, options: options) + end + + def deploy_stats + serialize deployments_summary + end + + def deployment_frequency_stats + serialize( + Summary::Group::DeploymentFrequency.new( + deployments: deployments_summary.value, + group: group, + options: options), + with_unit: true + ) + end + + def serialize(summary_object, with_unit: false) + AnalyticsSummarySerializer.new.represent( + summary_object, with_unit: with_unit) end end end diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb index 9c75d4bb455..564feb0319f 100644 --- a/lib/gitlab/cycle_analytics/stage_summary.rb +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -14,6 +14,7 @@ module Gitlab summary = [issue_stats] summary << commit_stats if user_has_sufficient_access? summary << deploy_stats + summary << deployment_frequency_stats end private @@ -26,16 +27,32 @@ module Gitlab serialize(Summary::Commit.new(project: @project, from: @from, to: @to)) end + def deployments_summary + @deployments_summary ||= + Summary::Deploy.new(project: @project, from: @from, to: @to) + end + def deploy_stats - serialize(Summary::Deploy.new(project: @project, from: @from, to: @to)) + serialize deployments_summary + end + + def deployment_frequency_stats + serialize( + Summary::DeploymentFrequency.new( + deployments: deployments_summary.value, + from: @from, + to: @to), + with_unit: true + ) end def user_has_sufficient_access? @project.team.member?(@current_user, Gitlab::Access::REPORTER) end - def serialize(summary_object) - AnalyticsSummarySerializer.new.represent(summary_object) + def serialize(summary_object, with_unit: false) + AnalyticsSummarySerializer.new.represent( + summary_object, with_unit: with_unit) end end end diff --git a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb new file mode 100644 index 00000000000..436dc91bd6b --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + class DeploymentFrequency < Base + include SummaryHelper + + def initialize(deployments:, from:, to: nil, project: nil) + @deployments = deployments + + super(project: project, from: from, to: to) + end + + def title + _('Deployment Frequency') + end + + def value + @value ||= + frequency(@deployments, @from, @to || Time.now) + end + + def unit + _('per day') + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/group/deployment_frequency.rb b/lib/gitlab/cycle_analytics/summary/group/deployment_frequency.rb new file mode 100644 index 00000000000..9fbbbb5a1ec --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/group/deployment_frequency.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + module Group + class DeploymentFrequency < Group::Base + include GroupProjectsProvider + include SummaryHelper + + def initialize(deployments:, group:, options:) + @deployments = deployments + + super(group: group, options: options) + end + + def title + _('Deployment Frequency') + end + + def value + @value ||= + frequency(@deployments, options[:from], options[:to] || Time.now) + end + + def unit + _('per day') + end + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary_helper.rb b/lib/gitlab/cycle_analytics/summary_helper.rb new file mode 100644 index 00000000000..06abcd151d4 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary_helper.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module SummaryHelper + def frequency(count, from, to) + return count if count.zero? + + freq = (count / days(from, to)).round(1) + freq.zero? ? '0' : freq + end + + def days(from, to) + [(to.end_of_day - from.beginning_of_day).fdiv(1.day), 1].max + end + end + end +end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 8e699de8164..14facd6b1d4 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -45,7 +45,7 @@ module Gitlab target_branch: merge_request.target_branch, target_project_id: merge_request.target_project_id, state: merge_request.state, - merge_status: merge_request.merge_status, + merge_status: merge_request.public_merge_status, url: Gitlab::UrlBuilder.build(merge_request) } end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 3922f5c6683..cf5ff8ddb7b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -184,14 +184,16 @@ module Gitlab # short period of time. The key _is_ enforced for any newly created # data. - execute <<-EOF.strip_heredoc - ALTER TABLE #{source} - ADD CONSTRAINT #{options[:name]} - FOREIGN KEY (#{options[:column]}) - REFERENCES #{target} (id) - #{on_delete_statement(options[:on_delete])} - NOT VALID; - EOF + with_lock_retries do + execute <<-EOF.strip_heredoc + ALTER TABLE #{source} + ADD CONSTRAINT #{options[:name]} + FOREIGN KEY (#{options[:column]}) + REFERENCES #{target} (id) + #{on_delete_statement(options[:on_delete])} + NOT VALID; + EOF + end end # Validate the existing constraint. This can potentially take a very diff --git a/lib/gitlab/diff/formatters/text_formatter.rb b/lib/gitlab/diff/formatters/text_formatter.rb index 5b670b1f83b..728457b3139 100644 --- a/lib/gitlab/diff/formatters/text_formatter.rb +++ b/lib/gitlab/diff/formatters/text_formatter.rb @@ -6,10 +6,12 @@ module Gitlab class TextFormatter < BaseFormatter attr_reader :old_line attr_reader :new_line + attr_reader :line_range def initialize(attrs) @old_line = attrs[:old_line] @new_line = attrs[:new_line] + @line_range = attrs[:line_range] super(attrs) end @@ -23,7 +25,7 @@ module Gitlab end def to_h - super.merge(old_line: old_line, new_line: new_line) + super.merge(old_line: old_line, new_line: new_line, line_range: line_range) end def line_age diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 055eae2c0fd..01ec9798fe4 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -94,7 +94,11 @@ module Gitlab Gitlab::Redis::Cache.with do |redis| redis.pipelined do hash.each do |diff_file_id, highlighted_diff_lines_hash| - redis.hset(key, diff_file_id, highlighted_diff_lines_hash.to_json) + redis.hset( + key, + diff_file_id, + compose_data(highlighted_diff_lines_hash.to_json) + ) end # HSETs have to have their expiration date manually updated @@ -152,12 +156,45 @@ module Gitlab end results.map! do |result| - JSON.parse(result, symbolize_names: true) unless result.nil? + JSON.parse(extract_data(result), symbolize_names: true) unless result.nil? end file_paths.zip(results).to_h end + def compose_data(json_data) + if ::Feature.enabled?(:gzip_diff_cache, default_enabled: true) + # #compress returns ASCII-8BIT, so we need to force the encoding to + # UTF-8 before caching it in redis, else we risk encoding mismatch + # errors. + # + ActiveSupport::Gzip.compress(json_data).force_encoding("UTF-8") + else + json_data + end + rescue Zlib::GzipFile::Error + json_data + end + + def extract_data(data) + # Since when we deploy this code, we'll be dealing with an already + # populated cache full of data that isn't gzipped, we want to also + # check to see if the data is gzipped before we attempt to #decompress + # it, thus we check the first 2 bytes for "\x1F\x8B" to confirm it is + # a gzipped string. While a non-gzipped string will raise a + # Zlib::GzipFile::Error, which we're rescuing, we don't want to count + # on rescue for control flow. This check can be removed in the release + # after this change is released. + # + if ::Feature.enabled?(:gzip_diff_cache, default_enabled: true) && data[0..1] == "\x1F\x8B" + ActiveSupport::Gzip.decompress(data) + else + data + end + rescue Zlib::GzipFile::Error + data + end + def cacheable?(diff_file) diffable.present? && diff_file.text? && diff_file.diffable? end diff --git a/lib/gitlab/elasticsearch/logs.rb b/lib/gitlab/elasticsearch/logs.rb deleted file mode 100644 index 3b6d1d0286a..00000000000 --- a/lib/gitlab/elasticsearch/logs.rb +++ /dev/null @@ -1,154 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Elasticsearch - class Logs - InvalidCursor = Class.new(RuntimeError) - - # How many log lines to fetch in a query - LOGS_LIMIT = 500 - - def initialize(client) - @client = client - end - - def pod_logs(namespace, pod_name: nil, container_name: nil, search: nil, start_time: nil, end_time: nil, cursor: nil) - query = { bool: { must: [] } }.tap do |q| - filter_pod_name(q, pod_name) - filter_namespace(q, namespace) - filter_container_name(q, container_name) - filter_search(q, search) - filter_times(q, start_time, end_time) - end - - body = build_body(query, cursor) - response = @client.search body: body - - format_response(response) - end - - private - - def build_body(query, cursor = nil) - body = { - query: query, - # reverse order so we can query N-most recent records - sort: [ - { "@timestamp": { order: :desc } }, - { "offset": { order: :desc } } - ], - # only return these fields in the response - _source: ["@timestamp", "message", "kubernetes.pod.name"], - # fixed limit for now, we should support paginated queries - size: ::Gitlab::Elasticsearch::Logs::LOGS_LIMIT - } - - unless cursor.nil? - body[:search_after] = decode_cursor(cursor) - end - - body - end - - def filter_pod_name(query, pod_name) - # We can filter by "all pods" with a null pod_name - return if pod_name.nil? - - query[:bool][:must] << { - match_phrase: { - "kubernetes.pod.name" => { - query: pod_name - } - } - } - end - - def filter_namespace(query, namespace) - query[:bool][:must] << { - match_phrase: { - "kubernetes.namespace" => { - query: namespace - } - } - } - end - - def filter_container_name(query, container_name) - # A pod can contain multiple containers. - # By default we return logs from every container - return if container_name.nil? - - query[:bool][:must] << { - match_phrase: { - "kubernetes.container.name" => { - query: container_name - } - } - } - end - - def filter_search(query, search) - return if search.nil? - - query[:bool][:must] << { - simple_query_string: { - query: search, - fields: [:message], - default_operator: :and - } - } - end - - def filter_times(query, start_time, end_time) - return unless start_time || end_time - - time_range = { range: { :@timestamp => {} } }.tap do |tr| - tr[:range][:@timestamp][:gte] = start_time if start_time - tr[:range][:@timestamp][:lt] = end_time if end_time - end - - query[:bool][:filter] = [time_range] - end - - def format_response(response) - results = response.fetch("hits", {}).fetch("hits", []) - last_result = results.last - results = results.map do |hit| - { - timestamp: hit["_source"]["@timestamp"], - message: hit["_source"]["message"], - pod: hit["_source"]["kubernetes"]["pod"]["name"] - } - end - - # we queried for the N-most recent records but we want them ordered oldest to newest - { - logs: results.reverse, - cursor: last_result.nil? ? nil : encode_cursor(last_result["sort"]) - } - end - - # we want to hide the implementation details of the search_after parameter from the frontend - # behind a single easily transmitted value - def encode_cursor(obj) - obj.join(',') - end - - def decode_cursor(obj) - cursor = obj.split(',').map(&:to_i) - - unless valid_cursor(cursor) - raise InvalidCursor, "invalid cursor format" - end - - cursor - end - - def valid_cursor(cursor) - cursor.instance_of?(Array) && - cursor.length == 2 && - cursor.map {|i| i.instance_of?(Integer)}.reduce(:&) - end - end - end -end diff --git a/lib/gitlab/elasticsearch/logs/lines.rb b/lib/gitlab/elasticsearch/logs/lines.rb new file mode 100644 index 00000000000..fb32a6c9fcd --- /dev/null +++ b/lib/gitlab/elasticsearch/logs/lines.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: true + +module Gitlab + module Elasticsearch + module Logs + class Lines + InvalidCursor = Class.new(RuntimeError) + + # How many log lines to fetch in a query + LOGS_LIMIT = 500 + + def initialize(client) + @client = client + end + + def pod_logs(namespace, pod_name: nil, container_name: nil, search: nil, start_time: nil, end_time: nil, cursor: nil) + query = { bool: { must: [] } }.tap do |q| + filter_pod_name(q, pod_name) + filter_namespace(q, namespace) + filter_container_name(q, container_name) + filter_search(q, search) + filter_times(q, start_time, end_time) + end + + body = build_body(query, cursor) + response = @client.search body: body + + format_response(response) + end + + private + + def build_body(query, cursor = nil) + body = { + query: query, + # reverse order so we can query N-most recent records + sort: [ + { "@timestamp": { order: :desc } }, + { "offset": { order: :desc } } + ], + # only return these fields in the response + _source: ["@timestamp", "message", "kubernetes.pod.name"], + # fixed limit for now, we should support paginated queries + size: ::Gitlab::Elasticsearch::Logs::Lines::LOGS_LIMIT + } + + unless cursor.nil? + body[:search_after] = decode_cursor(cursor) + end + + body + end + + def filter_pod_name(query, pod_name) + # We can filter by "all pods" with a null pod_name + return if pod_name.nil? + + query[:bool][:must] << { + match_phrase: { + "kubernetes.pod.name" => { + query: pod_name + } + } + } + end + + def filter_namespace(query, namespace) + query[:bool][:must] << { + match_phrase: { + "kubernetes.namespace" => { + query: namespace + } + } + } + end + + def filter_container_name(query, container_name) + # A pod can contain multiple containers. + # By default we return logs from every container + return if container_name.nil? + + query[:bool][:must] << { + match_phrase: { + "kubernetes.container.name" => { + query: container_name + } + } + } + end + + def filter_search(query, search) + return if search.nil? + + query[:bool][:must] << { + simple_query_string: { + query: search, + fields: [:message], + default_operator: :and + } + } + end + + def filter_times(query, start_time, end_time) + return unless start_time || end_time + + time_range = { range: { :@timestamp => {} } }.tap do |tr| + tr[:range][:@timestamp][:gte] = start_time if start_time + tr[:range][:@timestamp][:lt] = end_time if end_time + end + + query[:bool][:filter] = [time_range] + end + + def format_response(response) + results = response.fetch("hits", {}).fetch("hits", []) + last_result = results.last + results = results.map do |hit| + { + timestamp: hit["_source"]["@timestamp"], + message: hit["_source"]["message"], + pod: hit["_source"]["kubernetes"]["pod"]["name"] + } + end + + # we queried for the N-most recent records but we want them ordered oldest to newest + { + logs: results.reverse, + cursor: last_result.nil? ? nil : encode_cursor(last_result["sort"]) + } + end + + # we want to hide the implementation details of the search_after parameter from the frontend + # behind a single easily transmitted value + def encode_cursor(obj) + obj.join(',') + end + + def decode_cursor(obj) + cursor = obj.split(',').map(&:to_i) + + unless valid_cursor(cursor) + raise InvalidCursor, "invalid cursor format" + end + + cursor + end + + def valid_cursor(cursor) + cursor.instance_of?(Array) && + cursor.length == 2 && + cursor.map {|i| i.instance_of?(Integer)}.reduce(:&) + end + end + end + end +end diff --git a/lib/gitlab/elasticsearch/logs/pods.rb b/lib/gitlab/elasticsearch/logs/pods.rb new file mode 100644 index 00000000000..66499ae956a --- /dev/null +++ b/lib/gitlab/elasticsearch/logs/pods.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Elasticsearch + module Logs + class Pods + # How many items to fetch in a query + PODS_LIMIT = 500 + CONTAINERS_LIMIT = 500 + + def initialize(client) + @client = client + end + + def pods(namespace) + body = build_body(namespace) + response = @client.search body: body + + format_response(response) + end + + private + + def build_body(namespace) + { + aggs: { + pods: { + aggs: { + containers: { + terms: { + field: 'kubernetes.container.name', + size: ::Gitlab::Elasticsearch::Logs::Pods::CONTAINERS_LIMIT + } + } + }, + terms: { + field: 'kubernetes.pod.name', + size: ::Gitlab::Elasticsearch::Logs::Pods::PODS_LIMIT + } + } + }, + query: { + bool: { + must: { + match_phrase: { + "kubernetes.namespace": namespace + } + } + } + }, + # don't populate hits, only the aggregation is needed + size: 0 + } + end + + def format_response(response) + results = response.dig("aggregations", "pods", "buckets") || [] + results.map do |bucket| + { + name: bucket["key"], + container_names: (bucket.dig("containers", "buckets") || []).map do |cbucket| + cbucket["key"] + end + } + end + end + end + end + end +end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index a6e49825fd0..b893d625f8d 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -186,7 +186,7 @@ module Gitlab return event unless CUSTOM_FINGERPRINTING.include?(ex.class.name) - event.fingerprint = ['{{ default }}', ex.class.name, ex.message] + event.fingerprint = [ex.class.name, ex.message] event end diff --git a/lib/gitlab/file_hook.rb b/lib/gitlab/file_hook.rb index 38c19ff506f..f23ef2921d7 100644 --- a/lib/gitlab/file_hook.rb +++ b/lib/gitlab/file_hook.rb @@ -3,16 +3,17 @@ module Gitlab module FileHook def self.any? - plugin_glob.any? { |entry| File.file?(entry) } + dir_glob.any? { |entry| File.file?(entry) } end def self.files - plugin_glob.select { |entry| File.file?(entry) } + dir_glob.select { |entry| File.file?(entry) } end - def self.plugin_glob - Dir.glob(Rails.root.join('plugins/*')) + def self.dir_glob + Dir.glob([Rails.root.join('file_hooks/*'), Rails.root.join('plugins/*')]) end + private_class_method :dir_glob def self.execute_all_async(data) args = files.map { |file| [file, data] } diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 3b9402da0dd..697c943b4ec 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -216,10 +216,6 @@ module Gitlab SafeRequestStore[:gitaly_query_time] = duration end - def self.query_time_ms - (self.query_time * 1000).round(2) - end - def self.current_transaction_labels Gitlab::Metrics::Transaction.current&.labels || {} end diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb index 045a341f2ed..ac149cadb5b 100644 --- a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb +++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb @@ -15,9 +15,9 @@ module Gitlab attributes = { time: datetime.utc.iso8601(3), severity: severity, - duration: time[:total], - db: time[:db], - view: time[:view] + duration_s: Gitlab::Utils.ms_to_round_sec(time[:total]), + db_duration_s: Gitlab::Utils.ms_to_round_sec(time[:db]), + view_duration_s: Gitlab::Utils.ms_to_round_sec(time[:view]) }.merge!(data) ::Lograge.formatter.call(attributes) << "\n" diff --git a/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb index 705e23adff2..fe741a5bbe8 100644 --- a/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb +++ b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb @@ -18,9 +18,9 @@ module Gitlab return {} unless proxy_start && start_time # Time in milliseconds since gitlab-workhorse started the request - duration = (start_time.to_f * 1_000 - proxy_start.to_f / 1_000_000).round(2) + duration = start_time.to_f * 1_000 - proxy_start.to_f / 1_000_000 - { 'queue_duration': duration } + { 'queue_duration_s': Gitlab::Utils.ms_to_round_sec(duration) } end end end diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb index 323e6727a9f..5d96a0f3c0a 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb @@ -3,7 +3,7 @@ module Gitlab module ImportExport module Group - class TreeRestorer + class LegacyTreeRestorer include Gitlab::Utils::StrongMemoize attr_reader :user diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index f9a6fdc05aa..6b066b800a5 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -37,10 +37,6 @@ module Gitlab ::RequestStore[REDIS_CALL_DETAILS] ||= [] end - def self.query_time_ms - (self.query_time * 1000).round(2) - end - def self.query_time ::RequestStore[REDIS_CALL_DURATION] || 0 end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 5d4e6a7bdef..308c3007720 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -4,28 +4,28 @@ module Gitlab module InstrumentationHelper extend self - KEYS = %i(gitaly_calls gitaly_duration rugged_calls rugged_duration_ms redis_calls redis_duration_ms).freeze + KEYS = %i(gitaly_calls gitaly_duration_s rugged_calls rugged_duration_s redis_calls redis_duration_s).freeze def add_instrumentation_data(payload) gitaly_calls = Gitlab::GitalyClient.get_request_count if gitaly_calls > 0 payload[:gitaly_calls] = gitaly_calls - payload[:gitaly_duration] = Gitlab::GitalyClient.query_time_ms + payload[:gitaly_duration_s] = Gitlab::GitalyClient.query_time.round(2) end rugged_calls = Gitlab::RuggedInstrumentation.query_count if rugged_calls > 0 payload[:rugged_calls] = rugged_calls - payload[:rugged_duration_ms] = Gitlab::RuggedInstrumentation.query_time_ms + payload[:rugged_duration_s] = Gitlab::RuggedInstrumentation.query_time.round(2) end redis_calls = Gitlab::Instrumentation::Redis.get_request_count if redis_calls > 0 payload[:redis_calls] = redis_calls - payload[:redis_duration_ms] = Gitlab::Instrumentation::Redis.query_time_ms + payload[:redis_duration_s] = Gitlab::Instrumentation::Redis.query_time.round(2) end end @@ -47,7 +47,7 @@ module Gitlab # Its possible that if theres clock-skew between two nodes # this value may be less than zero. In that event, we record the value # as zero. - [elapsed_by_absolute_time(enqueued_at_time), 0].max + [elapsed_by_absolute_time(enqueued_at_time), 0].max.round(2) end # Calculates the time in seconds, as a float, from diff --git a/lib/gitlab/jira_import.rb b/lib/gitlab/jira_import.rb index fe4351d9029..3f56094956a 100644 --- a/lib/gitlab/jira_import.rb +++ b/lib/gitlab/jira_import.rb @@ -34,6 +34,10 @@ module Gitlab cache_class.increment(self.failed_issues_counter_cache_key(project_id)) end + def self.issue_failures(project_id) + cache_class.read(self.failed_issues_counter_cache_key(project_id)).to_i + end + def self.get_issues_next_start_at(project_id) cache_class.read(self.jira_issues_next_page_cache_key(project_id)).to_i end diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb index 5fbdbbc08c1..5381812186d 100644 --- a/lib/gitlab/jira_import/base_importer.rb +++ b/lib/gitlab/jira_import/base_importer.rb @@ -6,10 +6,10 @@ module Gitlab attr_reader :project, :client, :formatter, :jira_project_key def initialize(project) - raise Projects::ImportService::Error, _('Jira import feature is disabled.') unless project.jira_issues_import_feature_flag_enabled? - raise Projects::ImportService::Error, _('Jira integration not configured.') unless project.jira_service&.active? + project.validate_jira_import_settings! @jira_project_key = project.latest_jira_import&.jira_project_key + raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key @project = project diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb new file mode 100644 index 00000000000..5ebda67e2ae --- /dev/null +++ b/lib/gitlab/json.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Json + class << self + def parse(*args) + adapter.parse(*args) + end + + def parse!(*args) + adapter.parse!(*args) + end + + def dump(*args) + adapter.dump(*args) + end + + def generate(*args) + adapter.generate(*args) + end + + def pretty_generate(*args) + adapter.pretty_generate(*args) + end + + private + + def adapter + ::JSON + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb index 2bcb428b25d..31cd21f17e0 100644 --- a/lib/gitlab/kubernetes/helm/base_command.rb +++ b/lib/gitlab/kubernetes/helm/base_command.rb @@ -25,11 +25,21 @@ module Gitlab end def service_account_resource - nil + return unless rbac? + + Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate end def cluster_role_binding_resource - nil + return unless rbac? + + subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }] + + Gitlab::Kubernetes::ClusterRoleBinding.new( + cluster_role_binding_name, + cluster_role_name, + subjects + ).generate end def file_names @@ -61,6 +71,14 @@ module Gitlab def service_account_name Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT end + + def cluster_role_binding_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING + end + + def cluster_role_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE + end end end end diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb index 88ed8572ffc..058f38f2c9c 100644 --- a/lib/gitlab/kubernetes/helm/init_command.rb +++ b/lib/gitlab/kubernetes/helm/init_command.rb @@ -24,24 +24,6 @@ module Gitlab @rbac end - def service_account_resource - return unless rbac? - - Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate - end - - def cluster_role_binding_resource - return unless rbac? - - subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }] - - Gitlab::Kubernetes::ClusterRoleBinding.new( - cluster_role_binding_name, - cluster_role_name, - subjects - ).generate - end - private def init_helm_command @@ -69,14 +51,6 @@ module Gitlab ['--service-account', service_account_name] end - - def cluster_role_binding_name - Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING - end - - def cluster_role_name - Gitlab::Kubernetes::Helm::CLUSTER_ROLE - end end end end diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index 5dbff7d9102..145d67d7101 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -18,7 +18,7 @@ module Gitlab user_id: event.payload[:user_id], username: event.payload[:username], ua: event.payload[:ua], - queue_duration: event.payload[:queue_duration] + queue_duration_s: event.payload[:queue_duration_s] } ::Gitlab::InstrumentationHelper.add_instrumentation_data(payload) @@ -28,7 +28,7 @@ module Gitlab payload[Labkit::Correlation::CorrelationId::LOG_KEY] = Labkit::Correlation::CorrelationId.current_id if cpu_s = Gitlab::Metrics::System.thread_cpu_duration(::Gitlab::RequestContext.instance.start_thread_cpu_time) - payload[:cpu_s] = cpu_s + payload[:cpu_s] = cpu_s.round(2) end # https://github.com/roidrage/lograge#logging-errors--exceptions diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index c6b898ab5aa..630788f1a8a 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -20,8 +20,10 @@ module Gitlab # Time in milliseconds since gitlab-workhorse started the request duration = Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000 trans.set(:rails_queue_duration, duration) - metric_rails_queue_duration_seconds.observe(trans.labels, duration / 1_000) - env[GITLAB_RAILS_QUEUE_DURATION_KEY] = duration.round(2) + + duration_s = Gitlab::Utils.ms_to_round_sec(duration) + metric_rails_queue_duration_seconds.observe(trans.labels, duration_s) + env[GITLAB_RAILS_QUEUE_DURATION_KEY] = duration_s end @app.call(env) diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index beeaeb70d51..38adfc03ea7 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -36,33 +36,36 @@ module Gitlab name == other.name && title == other.title end - TEMPLATES_TABLE = [ - ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'), - ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'), - ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'), - ProjectTemplate.new('iosswift', 'iOS (Swift)', _('A ready-to-go template for use with iOS Swift apps.'), 'https://gitlab.com/gitlab-org/project-templates/iosswift', 'illustrations/logos/swift.svg'), - ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/logos/dotnet.svg'), - ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps.'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'), - ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development.'), 'https://gitlab.com/gitlab-org/project-templates/go-micro'), - ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby.'), 'https://gitlab.com/pages/gatsby'), - ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo'), - ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll'), - ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML.'), 'https://gitlab.com/pages/plain-html'), - ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook'), - ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo'), - ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'), - ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'), - ProjectTemplate.new('nfplainhtml', 'Netlify/Plain HTML', _('A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfplain-html', 'illustrations/logos/netlify.svg'), - ProjectTemplate.new('nfgitbook', 'Netlify/GitBook', _('A GitBook site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfgitbook', 'illustrations/logos/netlify.svg'), - ProjectTemplate.new('nfhexo', 'Netlify/Hexo', _('A Hexo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhexo', 'illustrations/logos/netlify.svg'), - ProjectTemplate.new('salesforcedx', 'SalesforceDX', _('A project boilerplate for Salesforce App development with Salesforce Developer tools.'), 'https://gitlab.com/gitlab-org/project-templates/salesforcedx'), - ProjectTemplate.new('serverless_framework', 'Serverless Framework/JS', _('A basic page and serverless function that uses AWS Lambda, AWS API Gateway, and GitLab Pages'), 'https://gitlab.com/gitlab-org/project-templates/serverless-framework', 'illustrations/logos/serverless_framework.svg'), - ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab.'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management') - ].freeze + def self.localized_templates_table + [ + ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'), + ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'), + ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'), + ProjectTemplate.new('iosswift', 'iOS (Swift)', _('A ready-to-go template for use with iOS Swift apps.'), 'https://gitlab.com/gitlab-org/project-templates/iosswift', 'illustrations/logos/swift.svg'), + ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/logos/dotnet.svg'), + ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps.'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'), + ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development.'), 'https://gitlab.com/gitlab-org/project-templates/go-micro'), + ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby.'), 'https://gitlab.com/pages/gatsby'), + ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo'), + ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll'), + ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML.'), 'https://gitlab.com/pages/plain-html'), + ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook'), + ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo'), + ProjectTemplate.new('sse_middleman', 'Static Site Editor/Middleman', _('Middleman project with Static Site Editor support'), 'https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman'), + ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('nfplainhtml', 'Netlify/Plain HTML', _('A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfplain-html', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('nfgitbook', 'Netlify/GitBook', _('A GitBook site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfgitbook', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('nfhexo', 'Netlify/Hexo', _('A Hexo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhexo', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('salesforcedx', 'SalesforceDX', _('A project boilerplate for Salesforce App development with Salesforce Developer tools.'), 'https://gitlab.com/gitlab-org/project-templates/salesforcedx'), + ProjectTemplate.new('serverless_framework', 'Serverless Framework/JS', _('A basic page and serverless function that uses AWS Lambda, AWS API Gateway, and GitLab Pages'), 'https://gitlab.com/gitlab-org/project-templates/serverless-framework', 'illustrations/logos/serverless_framework.svg'), + ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab.'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management') + ].freeze + end class << self def all - TEMPLATES_TABLE + localized_templates_table end def find(name) diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index e9127095a0d..7c06698ffec 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -8,14 +8,49 @@ module Gitlab included do # MergeRequest only quick actions definitions - desc _('Merge (when the pipeline succeeds)') - explanation _('Merges this merge request when the pipeline succeeds.') - execution_message _('Scheduled to merge this merge request when the pipeline succeeds.') + desc do + if Feature.enabled?(:merge_orchestration_service, quick_action_target.project, default_enabled: true) + if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) + _("Merge automatically (%{strategy})") % { strategy: preferred_strategy.humanize } + else + _("Merge immediately") + end + else + _('Merge (when the pipeline succeeds)') + end + end + explanation do + if Feature.enabled?(:merge_orchestration_service, quick_action_target.project, default_enabled: true) + if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) + _("Schedules to merge this merge request (%{strategy}).") % { strategy: preferred_strategy.humanize } + else + _('Merges this merge request immediately.') + end + else + _('Merges this merge request when the pipeline succeeds.') + end + end + execution_message do + if Feature.enabled?(:merge_orchestration_service, quick_action_target.project, default_enabled: true) + if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) + _("Scheduled to merge this merge request (%{strategy}).") % { strategy: preferred_strategy.humanize } + else + _('Merged this merge request.') + end + else + _('Scheduled to merge this merge request when the pipeline succeeds.') + end + end types MergeRequest condition do - last_diff_sha = params && params[:merge_request_diff_head_sha] - quick_action_target.persisted? && - quick_action_target.mergeable_with_quick_action?(current_user, autocomplete_precheck: !last_diff_sha, last_diff_sha: last_diff_sha) + if Feature.enabled?(:merge_orchestration_service, quick_action_target.project, default_enabled: true) + quick_action_target.persisted? && + merge_orchestration_service.can_merge?(quick_action_target) + else + last_diff_sha = params && params[:merge_request_diff_head_sha] + quick_action_target.persisted? && + quick_action_target.mergeable_with_quick_action?(current_user, autocomplete_precheck: !last_diff_sha, last_diff_sha: last_diff_sha) + end end command :merge do @updates[:merge] = params[:merge_request_diff_head_sha] @@ -70,6 +105,14 @@ module Gitlab @updates[:target_branch] = branch_name if project.repository.branch_exists?(branch_name) end end + + def merge_orchestration_service + @merge_orchestration_service ||= MergeRequests::MergeOrchestrationService.new(project, current_user) + end + + def preferred_auto_merge_strategy(merge_request) + merge_orchestration_service.preferred_auto_merge_strategy(merge_request) + end end end end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index d74e64116ca..0473fa89a0d 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -194,3 +194,5 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord end end + +Gitlab::SearchResults.prepend_if_ee('EE::Gitlab::SearchResults') diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index af9072ea201..ea60190353e 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -50,27 +50,27 @@ module Gitlab message = base_message(payload) if job_exception - payload['message'] = "#{message}: fail: #{payload['duration']} sec" + payload['message'] = "#{message}: fail: #{payload['duration_s']} sec" payload['job_status'] = 'fail' payload['error_message'] = job_exception.message payload['error_class'] = job_exception.class.name else - payload['message'] = "#{message}: done: #{payload['duration']} sec" + payload['message'] = "#{message}: done: #{payload['duration_s']} sec" payload['job_status'] = 'done' end - payload['db_duration'] = ActiveRecord::LogSubscriber.runtime - payload['db_duration_s'] = payload['db_duration'] / 1000 + db_duration = ActiveRecord::LogSubscriber.runtime + payload['db_duration_s'] = Gitlab::Utils.ms_to_round_sec(db_duration) payload end def add_time_keys!(time, payload) - payload['duration'] = time[:duration].round(6) + payload['duration_s'] = time[:duration].round(2) # ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0) # supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime - payload['cpu_s'] = time[:cputime].round(6) if time[:cputime] > 0 + payload['cpu_s'] = time[:cputime].round(2) if time[:cputime] > 0 payload['completed_at'] = Time.now.utc.to_f end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 60618787b24..61ed2fe1a06 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -71,7 +71,7 @@ module Gitlab end def get_gitaly_time(job) - job.fetch(:gitaly_duration, 0) / 1000.0 + job.fetch(:gitaly_duration_s, 0) end end end diff --git a/lib/gitlab/signed_commit.rb b/lib/gitlab/signed_commit.rb index 809e0a3f034..7a154978938 100644 --- a/lib/gitlab/signed_commit.rb +++ b/lib/gitlab/signed_commit.rb @@ -4,6 +4,8 @@ module Gitlab class SignedCommit include Gitlab::Utils::StrongMemoize + delegate :id, to: :@commit + def initialize(commit) @commit = commit diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index 08de9df14f8..b60f0b78fef 100644 --- a/lib/gitlab/slash_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -91,7 +91,7 @@ module Gitlab title: "#{issue.title} ยท #{issue.to_reference}", title_link: resource_url, author_name: author.name, - author_icon: author.avatar_url, + author_icon: author.avatar_url(only_path: false), fallback: fallback_message, pretext: custom_pretext, text: text, diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index ad6b213bb50..2e8a3ca4242 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -122,6 +122,10 @@ module Gitlab bytes.to_f / Numeric::MEGABYTE end + def ms_to_round_sec(ms) + (ms.to_f / 1000).round(2) + end + # Used in EE # Accepts either an Array or a String and returns an array def ensure_array_from_string(string_or_array) |