diff options
Diffstat (limited to 'lib/gitlab')
5 files changed, 197 insertions, 11 deletions
diff --git a/lib/gitlab/database/partitioning/replace_table.rb b/lib/gitlab/database/partitioning/replace_table.rb new file mode 100644 index 00000000000..13178431703 --- /dev/null +++ b/lib/gitlab/database/partitioning/replace_table.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class ReplaceTable + DELIMITER = ";\n\n" + + attr_reader :original_table, :replacement_table, :replaced_table, :primary_key_column, + :sequence, :original_primary_key, :replacement_primary_key, :replaced_primary_key + + def initialize(original_table, replacement_table, replaced_table, primary_key_column) + @original_table = original_table + @replacement_table = replacement_table + @replaced_table = replaced_table + @primary_key_column = primary_key_column + + @sequence = default_sequence(original_table, primary_key_column) + @original_primary_key = default_primary_key(original_table) + @replacement_primary_key = default_primary_key(replacement_table) + @replaced_primary_key = default_primary_key(replaced_table) + end + + def perform + yield sql_to_replace_table if block_given? + + execute(sql_to_replace_table) + end + + private + + delegate :execute, :quote_table_name, :quote_column_name, to: :connection + def connection + @connection ||= ActiveRecord::Base.connection + end + + def default_sequence(table, column) + "#{table}_#{column}_seq" + end + + def default_primary_key(table) + "#{table}_pkey" + end + + def sql_to_replace_table + @sql_to_replace_table ||= [ + drop_default_sql(original_table, primary_key_column), + set_default_sql(replacement_table, primary_key_column, "nextval('#{quote_table_name(sequence)}'::regclass)"), + + change_sequence_owner_sql(sequence, replacement_table, primary_key_column), + + rename_table_sql(original_table, replaced_table), + rename_constraint_sql(replaced_table, original_primary_key, replaced_primary_key), + + rename_table_sql(replacement_table, original_table), + rename_constraint_sql(original_table, replacement_primary_key, original_primary_key) + ].join(DELIMITER) + end + + def drop_default_sql(table, column) + "ALTER TABLE #{quote_table_name(table)} ALTER COLUMN #{quote_column_name(column)} DROP DEFAULT" + end + + def set_default_sql(table, column, expression) + "ALTER TABLE #{quote_table_name(table)} ALTER COLUMN #{quote_column_name(column)} SET DEFAULT #{expression}" + end + + def change_sequence_owner_sql(sequence, table, column) + "ALTER SEQUENCE #{quote_table_name(sequence)} OWNED BY #{quote_table_name(table)}.#{quote_column_name(column)}" + end + + def rename_table_sql(old_name, new_name) + "ALTER TABLE #{quote_table_name(old_name)} RENAME TO #{quote_table_name(new_name)}" + end + + def rename_constraint_sql(table, old_name, new_name) + "ALTER TABLE #{quote_table_name(table)} RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)}" + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index f7b0306b769..6adbe046cb0 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -66,7 +66,10 @@ module Gitlab create_range_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key) create_daterange_partitions(partitioned_table_name, partition_column.name, min_date, max_date) end - create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key) + + with_lock_retries do + create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key) + end end # Clean up a partitioned copy of an existing table. First, deletes the database function and trigger that were @@ -81,13 +84,9 @@ module Gitlab assert_not_in_transaction_block(scope: ERROR_SCOPE) with_lock_retries do - trigger_name = make_sync_trigger_name(table_name) - drop_trigger(table_name, trigger_name) + drop_sync_trigger(table_name) end - function_name = make_sync_function_name(table_name) - drop_function(function_name) - partitioned_table_name = make_partitioned_table_name(table_name) drop_table(partitioned_table_name) end @@ -177,6 +176,52 @@ module Gitlab end end + # Replaces a non-partitioned table with its partitioned copy. This is the final step in a partitioning + # migration, which makes the partitioned table ready for use by the application. The partitioned copy should be + # replaced with the original table in such a way that it appears seamless to any database clients. The replaced + # table will be renamed to "#{replaced_table}_archived" + # + # **NOTE** This method should only be used after all other migration steps have completed successfully. + # There are several limitations to this method that MUST be handled before, or during, the swap migration: + # + # - Secondary indexes and foreign keys are not automatically recreated on the partitioned table. + # - Some types of constraints (UNIQUE and EXCLUDE) which rely on indexes, will not automatically be recreated + # on the partitioned table, since the underlying index will not be present. + # - Foreign keys referencing the original non-partitioned table, would also need to be updated to reference the + # partitioned table, but unfortunately this is not supported in PG11. + # - Views referencing the original table will not be automatically updated to reference the partitioned table. + # + # Example: + # + # replace_with_partitioned_table :audit_events + # + def replace_with_partitioned_table(table_name) + assert_table_is_allowed(table_name) + + partitioned_table_name = make_partitioned_table_name(table_name) + archived_table_name = make_archived_table_name(table_name) + primary_key_name = connection.primary_key(table_name) + + replace_table(table_name, partitioned_table_name, archived_table_name, primary_key_name) + end + + # Rolls back a migration that replaced a non-partitioned table with its partitioned copy. This can be used to + # restore the original non-partitioned table in the event of an unexpected issue. + # + # Example: + # + # rollback_replace_with_partitioned_table :audit_events + # + def rollback_replace_with_partitioned_table(table_name) + assert_table_is_allowed(table_name) + + partitioned_table_name = make_partitioned_table_name(table_name) + archived_table_name = make_archived_table_name(table_name) + primary_key_name = connection.primary_key(archived_table_name) + + replace_table(table_name, archived_table_name, partitioned_table_name, primary_key_name) + end + private def assert_table_is_allowed(table_name) @@ -190,6 +235,10 @@ module Gitlab tmp_table_name("#{table}_part") end + def make_archived_table_name(table) + "#{table}_archived" + end + def make_sync_function_name(table) object_name(table, 'table_sync_function') end @@ -270,12 +319,18 @@ module Gitlab function_name = make_sync_function_name(source_table_name) trigger_name = make_sync_trigger_name(source_table_name) - with_lock_retries do - create_sync_function(function_name, partitioned_table_name, unique_key) - create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table") + create_sync_function(function_name, partitioned_table_name, unique_key) + create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table") - create_sync_trigger(source_table_name, trigger_name, function_name) - end + create_sync_trigger(source_table_name, trigger_name, function_name) + end + + def drop_sync_trigger(source_table_name) + trigger_name = make_sync_trigger_name(source_table_name) + drop_trigger(source_table_name, trigger_name) + + function_name = make_sync_function_name(source_table_name) + drop_function(function_name) end def create_sync_function(name, partitioned_table_name, unique_key) @@ -358,6 +413,21 @@ module Gitlab end end end + + def replace_table(original_table_name, replacement_table_name, replaced_table_name, primary_key_name) + replace_table = Gitlab::Database::Partitioning::ReplaceTable.new(original_table_name, + replacement_table_name, replaced_table_name, primary_key_name) + + with_lock_retries do + drop_sync_trigger(original_table_name) + + replace_table.perform do |sql| + say("replace_table(\"#{sql}\")") + end + + create_trigger_to_sync_tables(original_table_name, replaced_table_name, primary_key_name) + end + end end end end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 803acef9a40..a5ace2be773 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -123,6 +123,7 @@ module Gitlab end extra = sanitize_request_parameters(extra) + inject_sql_query_into_extra(exception, extra) if sentry && Raven.configuration.server Raven.capture_exception(exception, tags: default_tags, extra: extra) @@ -149,6 +150,12 @@ module Gitlab filter.filter(parameters) end + def inject_sql_query_into_extra(exception, extra) + return unless exception.is_a?(ActiveRecord::StatementInvalid) + + extra[:sql] = PgQuery.normalize(exception.sql.to_s) + end + def sentry_dsn return unless Rails.env.production? || Rails.env.development? return unless Gitlab.config.sentry.enabled diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index e8839875109..b6d570a3d08 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -33,6 +33,9 @@ module Gitlab ISSUE_DUE_DATE_CHANGED = 'g_project_management_issue_due_date_changed' ISSUE_TIME_ESTIMATE_CHANGED = 'g_project_management_issue_time_estimate_changed' ISSUE_TIME_SPENT_CHANGED = 'g_project_management_issue_time_spent_changed' + ISSUE_COMMENT_ADDED = 'g_project_management_issue_comment_added' + ISSUE_COMMENT_EDITED = 'g_project_management_issue_comment_edited' + ISSUE_COMMENT_REMOVED = 'g_project_management_issue_comment_removed' class << self def track_issue_created_action(author:, time: Time.zone.now) @@ -147,6 +150,18 @@ module Gitlab track_unique_action(ISSUE_TIME_SPENT_CHANGED, author, time) end + def track_issue_comment_added_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_COMMENT_ADDED, author, time) + end + + def track_issue_comment_edited_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_COMMENT_EDITED, author, time) + end + + def track_issue_comment_removed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_COMMENT_REMOVED, author, time) + end + private def track_unique_action(action, author, time) diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events.yml index bc56c5d6d9b..023427946aa 100644 --- a/lib/gitlab/usage_data_counters/known_events.yml +++ b/lib/gitlab/usage_data_counters/known_events.yml @@ -303,3 +303,15 @@ category: issues_edit redis_slot: project_management aggregation: daily +- name: g_project_management_issue_comment_added + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_comment_edited + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_comment_removed + category: issues_edit + redis_slot: project_management + aggregation: daily |