diff options
author | Nick Thomas <nick@gitlab.com> | 2018-12-04 19:48:30 +0300 |
---|---|---|
committer | Nick Thomas <nick@gitlab.com> | 2018-12-04 19:48:30 +0300 |
commit | 80163b972a7bbbd203bb86e1d1474a592979e6d7 (patch) | |
tree | 43d11c56f1020cb0b7b9f20547edea280ae8af26 /lib | |
parent | 42a7d3b7fe22e193bab1edca0165ceafe1d6b7bd (diff) | |
parent | a1bd34e9c04c79488dc20ad0af08b0c94bffe675 (diff) |
Merge branch 'fix/gb/encrypt-runners-tokens' into 'master'
Encrypt runners tokens
Closes #51232 and #52931
See merge request gitlab-org/gitlab-ce!23412
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/runner.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/background_migration/encrypt_columns.rb | 19 | ||||
-rw-r--r-- | lib/gitlab/background_migration/encrypt_runners_tokens.rb | 32 | ||||
-rw-r--r-- | lib/gitlab/background_migration/models/encrypt_columns/namespace.rb | 28 | ||||
-rw-r--r-- | lib/gitlab/background_migration/models/encrypt_columns/project.rb | 28 | ||||
-rw-r--r-- | lib/gitlab/background_migration/models/encrypt_columns/runner.rb | 28 | ||||
-rw-r--r-- | lib/gitlab/background_migration/models/encrypt_columns/settings.rb | 37 | ||||
-rw-r--r-- | lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/crypto_helper.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/import_export/import_export.yml | 7 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_factory.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/utils.rb | 15 |
12 files changed, 198 insertions, 15 deletions
diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 2f15f3a7d76..c60d25b88cb 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -19,7 +19,6 @@ module API optional :tag_list, type: Array[String], desc: %q(List of Runner's tags) optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' end - # rubocop: disable CodeReuse/ActiveRecord post '/' do attributes = attributes_for_keys([:description, :active, :locked, :run_untagged, :tag_list, :maximum_timeout]) .merge(get_runner_details_from_request) @@ -28,10 +27,10 @@ module API if runner_registration_token_valid? # Create shared runner. Requires admin access attributes.merge(runner_type: :instance_type) - elsif project = Project.find_by(runners_token: params[:token]) + elsif project = Project.find_by_runners_token(params[:token]) # Create a specific runner for the project attributes.merge(runner_type: :project_type, projects: [project]) - elsif group = Group.find_by(runners_token: params[:token]) + elsif group = Group.find_by_runners_token(params[:token]) # Create a specific runner for the group attributes.merge(runner_type: :group_type, groups: [group]) else @@ -46,7 +45,6 @@ module API render_validation_error!(runner) end end - # rubocop: enable CodeReuse/ActiveRecord desc 'Deletes a registered Runner' do http_codes [[204, 'Runner was deleted'], [403, 'Forbidden']] diff --git a/lib/gitlab/background_migration/encrypt_columns.rb b/lib/gitlab/background_migration/encrypt_columns.rb index bd5f12276ab..b9ad8267e37 100644 --- a/lib/gitlab/background_migration/encrypt_columns.rb +++ b/lib/gitlab/background_migration/encrypt_columns.rb @@ -5,15 +5,17 @@ module Gitlab # EncryptColumn migrates data from an unencrypted column - `foo`, say - to # an encrypted column - `encrypted_foo`, say. # + # To avoid depending on a particular version of the model in app/, add a + # model to `lib/gitlab/background_migration/models/encrypt_columns` and use + # it in the migration that enqueues the jobs, so code can be shared. + # # For this background migration to work, the table that is migrated _has_ to # have an `id` column as the primary key. Additionally, the encrypted column # should be managed by attr_encrypted, and map to an attribute with the same # name as the unencrypted column (i.e., the unencrypted column should be - # shadowed). + # shadowed), unless you want to define specific methods / accessors in the + # temporary model in `/models/encrypt_columns/your_model.rb`. # - # To avoid depending on a particular version of the model in app/, add a - # model to `lib/gitlab/background_migration/models/encrypt_columns` and use - # it in the migration that enqueues the jobs, so code can be shared. class EncryptColumns def perform(model, attributes, from, to) model = model.constantize if model.is_a?(String) @@ -36,6 +38,10 @@ module Gitlab end end + def clear_migrated_values? + true + end + private # Build a hash of { attribute => encrypted column name } @@ -72,7 +78,10 @@ module Gitlab if instance.changed? instance.save! - instance.update_columns(to_clear) + + if clear_migrated_values? + instance.update_columns(to_clear) + end end end diff --git a/lib/gitlab/background_migration/encrypt_runners_tokens.rb b/lib/gitlab/background_migration/encrypt_runners_tokens.rb new file mode 100644 index 00000000000..91e559a8765 --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_runners_tokens.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # EncryptColumn migrates data from an unencrypted column - `foo`, say - to + # an encrypted column - `encrypted_foo`, say. + # + # We only create a subclass here because we want to isolate this migration + # (migrating unencrypted runner registration tokens to encrypted columns) + # from other `EncryptColumns` migration. This class name is going to be + # serialized and stored in Redis and later picked by Sidekiq, so we need to + # create a separate class name in order to isolate these migration tasks. + # + # We can solve this differently, see tech debt issue: + # + # https://gitlab.com/gitlab-org/gitlab-ce/issues/54328 + # + class EncryptRunnersTokens < EncryptColumns + def perform(model, from, to) + resource = "::Gitlab::BackgroundMigration::Models::EncryptColumns::#{model.to_s.capitalize}" + model = resource.constantize + attributes = model.encrypted_attributes.keys + + super(model, attributes, from, to) + end + + def clear_migrated_values? + false + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb b/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb new file mode 100644 index 00000000000..41f18979d76 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `runners_token` column in `namespaces` table. + # + class Namespace < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + def runners_token=(value) + self.runners_token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { runners_token: { attribute: :runners_token_encrypted } } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/project.rb b/lib/gitlab/background_migration/models/encrypt_columns/project.rb new file mode 100644 index 00000000000..bfeae14584d --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/project.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `runners_token` column in `projects` table. + # + class Project < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'projects' + self.inheritance_column = :_type_disabled + + def runners_token=(value) + self.runners_token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { runners_token: { attribute: :runners_token_encrypted } } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/runner.rb b/lib/gitlab/background_migration/models/encrypt_columns/runner.rb new file mode 100644 index 00000000000..14ddce4b147 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/runner.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `token` column in `ci_runners` table. + # + class Runner < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'ci_runners' + self.inheritance_column = :_type_disabled + + def token=(value) + self.token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { token: { attribute: :token_encrypted } } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/settings.rb b/lib/gitlab/background_migration/models/encrypt_columns/settings.rb new file mode 100644 index 00000000000..08ae35c0671 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/settings.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `runners_token` column in `application_settings` table. + # + class Settings < ActiveRecord::Base + include ::EachBatch + include ::CacheableAttributes + + self.table_name = 'application_settings' + self.inheritance_column = :_type_disabled + + after_commit do + ::ApplicationSetting.expire + end + + def runners_registration_token=(value) + self.runners_registration_token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { + runners_registration_token: { + attribute: :runners_registration_token_encrypted + } + } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb index bb76eb8ed48..ccd9d4c6d44 100644 --- a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb +++ b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb @@ -15,12 +15,12 @@ module Gitlab attr_encrypted :token, mode: :per_attribute_iv, algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_truncated + key: ::Settings.attr_encrypted_db_key_base_truncated attr_encrypted :url, mode: :per_attribute_iv, algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_truncated + key: ::Settings.attr_encrypted_db_key_base_truncated end end end diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb index 68d0b5d8f8a..87a03d9c58f 100644 --- a/lib/gitlab/crypto_helper.rb +++ b/lib/gitlab/crypto_helper.rb @@ -6,8 +6,8 @@ module Gitlab AES256_GCM_OPTIONS = { algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_truncated, - iv: Settings.attr_encrypted_db_key_base_truncated[0..11] + key: Settings.attr_encrypted_db_key_base_32, + iv: Settings.attr_encrypted_db_key_base_12 }.freeze def sha256(value) @@ -17,7 +17,7 @@ module Gitlab def aes256_gcm_encrypt(value) encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value)) - Base64.encode64(encrypted_token) + Base64.strict_encode64(encrypted_token) end def aes256_gcm_decrypt(value) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 8380e86c128..4f3298812a8 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -100,6 +100,7 @@ excluded_attributes: - :import_source - :mirror - :runners_token + - :runners_token_encrypted - :repository_storage - :repository_read_only - :lfs_enabled @@ -114,6 +115,9 @@ excluded_attributes: - :remote_mirror_available_overridden - :description_html - :repository_languages + namespaces: + - :runners_token + - :runners_token_encrypted project_import_state: - :last_error - :jid @@ -155,6 +159,9 @@ excluded_attributes: - :encrypted_token_iv - :encrypted_url - :encrypted_url_iv + runners: + - :token + - :token_encrypted services: - :template diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 097c7653754..9cc781ddf8d 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -10,6 +10,7 @@ module Gitlab triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', + runners: 'Ci::Runner', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', @@ -33,7 +34,7 @@ module Gitlab EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze - TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze + TOKEN_RESET_MODELS = %w[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze def self.create(*args) new(*args).create diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 9e59137a2c0..e0e8f598ba4 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -16,6 +16,21 @@ module Gitlab str.force_encoding(Encoding::UTF_8) end + def ensure_utf8_size(str, bytes:) + raise ArgumentError, 'Empty string provided!' if str.empty? + raise ArgumentError, 'Negative string size provided!' if bytes.negative? + + truncated = str.each_char.each_with_object(+'') do |char, object| + if object.bytesize + char.bytesize > bytes + break object + else + object.concat(char) + end + end + + truncated + ('0' * (bytes - truncated.bytesize)) + end + # Append path to host, making sure there's one single / in between def append_path(host, path) "#{host.to_s.sub(%r{\/+$}, '')}/#{path.to_s.sub(%r{^\/+}, '')}" |