diff options
author | Shinya Maeda <shinya@gitlab.com> | 2018-05-03 08:21:37 +0300 |
---|---|---|
committer | Shinya Maeda <shinya@gitlab.com> | 2018-05-03 08:21:37 +0300 |
commit | 90da0d4fe2a96248f864e003c1cf7f958f46c421 (patch) | |
tree | 3b40c7468d15aa7722a3104664838cccb747b25a /lib | |
parent | b2d841851ecdd4a850ca2dc8d7f2f5b3d0b5a2bb (diff) | |
parent | 30464bceaac08bf7a8415df7d3b1e0ddf6182762 (diff) |
Merge branch 'live-trace-v2' into live-trace-v2-efficient-destroy-all
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/background_migration/migrate_stage_index.rb | 47 | ||||
-rw-r--r-- | lib/gitlab/ci/cron_parser.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/seed/stage.rb | 1 | ||||
-rw-r--r-- | lib/gitlab/database/arel_methods.rb | 18 | ||||
-rw-r--r-- | lib/gitlab/database/migration_helpers.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb | 10 | ||||
-rw-r--r-- | lib/gitlab/git/repository.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/gon_helper.rb | 1 | ||||
-rw-r--r-- | lib/gitlab/redis/shared_state.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/shell.rb | 44 |
10 files changed, 91 insertions, 47 deletions
diff --git a/lib/gitlab/background_migration/migrate_stage_index.rb b/lib/gitlab/background_migration/migrate_stage_index.rb new file mode 100644 index 00000000000..f90f35a913d --- /dev/null +++ b/lib/gitlab/background_migration/migrate_stage_index.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class MigrateStageIndex + def perform(start_id, stop_id) + migrate_stage_index_sql(start_id.to_i, stop_id.to_i).tap do |sql| + ActiveRecord::Base.connection.execute(sql) + end + end + + private + + def migrate_stage_index_sql(start_id, stop_id) + if Gitlab::Database.postgresql? + <<~SQL + WITH freqs AS ( + SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds + WHERE stage_id BETWEEN #{start_id} AND #{stop_id} + AND stage_idx IS NOT NULL + GROUP BY stage_id, stage_idx + ), indexes AS ( + SELECT DISTINCT stage_id, last_value(stage_idx) + OVER (PARTITION BY stage_id ORDER BY freq ASC) AS index + FROM freqs + ) + + UPDATE ci_stages SET position = indexes.index + FROM indexes WHERE indexes.stage_id = ci_stages.id + AND ci_stages.position IS NULL; + SQL + else + <<~SQL + UPDATE ci_stages + SET position = + (SELECT stage_idx FROM ci_builds + WHERE ci_builds.stage_id = ci_stages.id + GROUP BY ci_builds.stage_idx ORDER BY COUNT(*) DESC LIMIT 1) + WHERE ci_stages.id BETWEEN #{start_id} AND #{stop_id} + AND ci_stages.position IS NULL + SQL + end + end + end + end +end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index 551483d0aaa..73f36735e35 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -6,7 +6,7 @@ module Gitlab def initialize(cron, cron_timezone = 'UTC') @cron = cron - @cron_timezone = ActiveSupport::TimeZone.find_tzinfo(cron_timezone).name + @cron_timezone = timezone_name(cron_timezone) end def next_time_from(time) @@ -24,6 +24,12 @@ module Gitlab private + def timezone_name(timezone) + ActiveSupport::TimeZone.find_tzinfo(timezone).name + rescue TZInfo::InvalidTimezoneIdentifier + timezone + end + # NOTE: # cron_timezone can only accept timezones listed in TZInfo::Timezone. # Aliases of Timezones from ActiveSupport::TimeZone are NOT accepted, diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index c101f30d6e8..2b58d9863a0 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -19,6 +19,7 @@ module Gitlab def attributes { name: @attributes.fetch(:name), + position: @attributes.fetch(:index), pipeline: @pipeline, project: @pipeline.project } end diff --git a/lib/gitlab/database/arel_methods.rb b/lib/gitlab/database/arel_methods.rb new file mode 100644 index 00000000000..d7e3ce08b32 --- /dev/null +++ b/lib/gitlab/database/arel_methods.rb @@ -0,0 +1,18 @@ +module Gitlab + module Database + module ArelMethods + private + + # In Arel 7.0.0 (Arel 7.1.4 is used in Rails 5.0) the `engine` parameter of `Arel::UpdateManager#initializer` + # was removed. + # Remove this file and inline this method when removing rails5? code. + def arel_update_manager + if Gitlab.rails5? + Arel::UpdateManager.new + else + Arel::UpdateManager.new(ActiveRecord::Base) + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 77079e5e72b..c21bae5e16b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1,6 +1,8 @@ module Gitlab module Database module MigrationHelpers + include Gitlab::Database::ArelMethods + BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time @@ -314,7 +316,7 @@ module Gitlab stop_arel = yield table, stop_arel if block_given? stop_row = exec_query(stop_arel.to_sql).to_hash.first - update_arel = Arel::UpdateManager.new(ActiveRecord::Base) + update_arel = arel_update_manager .table(table) .set([[table[column], value]]) .where(table[:id].gteq(start_id)) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 1a697396ff1..14de28a1d08 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -3,6 +3,8 @@ module Gitlab module RenameReservedPathsMigration module V1 class RenameBase + include Gitlab::Database::ArelMethods + attr_reader :paths, :migration delegate :update_column_in_batches, @@ -62,10 +64,10 @@ module Gitlab old_full_path, new_full_path) - update = Arel::UpdateManager.new(ActiveRecord::Base) - .table(routes) - .set([[routes[:path], replace_statement]]) - .where(Arel::Nodes::SqlLiteral.new(filter)) + update = arel_update_manager + .table(routes) + .set([[routes[:path], replace_statement]]) + .where(Arel::Nodes::SqlLiteral.new(filter)) execute(update.to_sql) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index de0044fc149..84d37f77fbb 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1569,7 +1569,8 @@ module Gitlab end def checksum - gitaly_migrate(:calculate_checksum) do |is_enabled| + gitaly_migrate(:calculate_checksum, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_repository_client.calculate_checksum else diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index a7e055ac444..c741dabe168 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -19,6 +19,7 @@ module Gitlab gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') gon.sprite_icons = IconsHelper.sprite_icon_path gon.sprite_file_icons = IconsHelper.sprite_file_icons_path + gon.emoji_sprites_css_path = ActionController::Base.helpers.stylesheet_path('emoji_sprites') gon.test_env = Rails.env.test? gon.suggested_label_colors = LabelsHelper.suggested_colors diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 10bec7a90da..e5a0fdae7ef 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -5,6 +5,8 @@ module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper SESSION_NAMESPACE = 'session:gitlab'.freeze + USER_SESSIONS_NAMESPACE = 'session:user:gitlab'.freeze + USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab'.freeze DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 156115f8a8f..4a691d640b3 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -294,17 +294,7 @@ module Gitlab # add_namespace("default", "gitlab") # def add_namespace(storage, name) - Gitlab::GitalyClient.migrate(:add_namespace, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| - if enabled - Gitlab::GitalyClient::NamespaceService.new(storage).add(name) - else - path = full_path(storage, name) - FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) - end - end - rescue Errno::EEXIST => e - Rails.logger.warn("Directory exists as a file: #{e} at: #{path}") + Gitlab::GitalyClient::NamespaceService.new(storage).add(name) rescue GRPC::InvalidArgument => e raise ArgumentError, e.message end @@ -316,14 +306,7 @@ module Gitlab # rm_namespace("default", "gitlab") # def rm_namespace(storage, name) - Gitlab::GitalyClient.migrate(:remove_namespace, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| - if enabled - Gitlab::GitalyClient::NamespaceService.new(storage).remove(name) - else - FileUtils.rm_r(full_path(storage, name), force: true) - end - end + Gitlab::GitalyClient::NamespaceService.new(storage).remove(name) rescue GRPC::InvalidArgument => e raise ArgumentError, e.message end @@ -335,17 +318,7 @@ module Gitlab # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # def mv_namespace(storage, old_name, new_name) - Gitlab::GitalyClient.migrate(:rename_namespace, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| - if enabled - Gitlab::GitalyClient::NamespaceService.new(storage) - .rename(old_name, new_name) - else - break false if exists?(storage, new_name) || !exists?(storage, old_name) - - FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) - end - end + Gitlab::GitalyClient::NamespaceService.new(storage).rename(old_name, new_name) rescue GRPC::InvalidArgument false end @@ -370,17 +343,8 @@ module Gitlab # exists?(storage, 'gitlab') # exists?(storage, 'gitlab/cookies.git') # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) - Gitlab::GitalyClient.migrate(:namespace_exists, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| - if enabled - Gitlab::GitalyClient::NamespaceService.new(storage) - .exists?(dir_name) - else - File.exist?(full_path(storage, dir_name)) - end - end + Gitlab::GitalyClient::NamespaceService.new(storage).exists?(dir_name) end protected |