Merge branch 'fix/gb/encrypt-runners-tokens' into 'master'

Encrypt runners tokens

Closes #51232 and #52931

See merge request gitlab-org/gitlab-ce!23412

12 files changed, 198 insertions, 15 deletions

diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index 2f15f3a7d76..c60d25b88cb 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -19,7 +19,6 @@ module API
         optional :tag_list, type: Array[String], desc: %q(List of Runner's tags)
         optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job'
       end
-      # rubocop: disable CodeReuse/ActiveRecord
       post '/' do
         attributes = attributes_for_keys([:description, :active, :locked, :run_untagged, :tag_list, :maximum_timeout])
           .merge(get_runner_details_from_request)
@@ -28,10 +27,10 @@ module API
           if runner_registration_token_valid?
             # Create shared runner. Requires admin access
             attributes.merge(runner_type: :instance_type)
-          elsif project = Project.find_by(runners_token: params[:token])
+          elsif project = Project.find_by_runners_token(params[:token])
             # Create a specific runner for the project
             attributes.merge(runner_type: :project_type, projects: [project])
-          elsif group = Group.find_by(runners_token: params[:token])
+          elsif group = Group.find_by_runners_token(params[:token])
             # Create a specific runner for the group
             attributes.merge(runner_type: :group_type, groups: [group])
           else
@@ -46,7 +45,6 @@ module API
           render_validation_error!(runner)
         end
       end
-      # rubocop: enable CodeReuse/ActiveRecord
 
       desc 'Deletes a registered Runner' do
         http_codes [[204, 'Runner was deleted'], [403, 'Forbidden']]
diff --git a/lib/gitlab/background_migration/encrypt_columns.rb b/lib/gitlab/background_migration/encrypt_columns.rb
index bd5f12276ab..b9ad8267e37 100644
--- a/lib/gitlab/background_migration/encrypt_columns.rb
+++ b/lib/gitlab/background_migration/encrypt_columns.rb
@@ -5,15 +5,17 @@ module Gitlab
     # EncryptColumn migrates data from an unencrypted column - `foo`, say - to
     # an encrypted column - `encrypted_foo`, say.
     #
+    # To avoid depending on a particular version of the model in app/, add a
+    # model to `lib/gitlab/background_migration/models/encrypt_columns` and use
+    # it in the migration that enqueues the jobs, so code can be shared.
+    #
     # For this background migration to work, the table that is migrated _has_ to
     # have an `id` column as the primary key. Additionally, the encrypted column
     # should be managed by attr_encrypted, and map to an attribute with the same
     # name as the unencrypted column (i.e., the unencrypted column should be
-    # shadowed).
+    # shadowed), unless you want to define specific methods / accessors in the
+    # temporary model in `/models/encrypt_columns/your_model.rb`.
     #
-    # To avoid depending on a particular version of the model in app/, add a
-    # model to `lib/gitlab/background_migration/models/encrypt_columns` and use
-    # it in the migration that enqueues the jobs, so code can be shared.
     class EncryptColumns
       def perform(model, attributes, from, to)
         model = model.constantize if model.is_a?(String)
@@ -36,6 +38,10 @@ module Gitlab
         end
       end
 
+      def clear_migrated_values?
+        true
+      end
+
       private
 
       # Build a hash of { attribute => encrypted column name }
@@ -72,7 +78,10 @@ module Gitlab
 
         if instance.changed?
           instance.save!
-          instance.update_columns(to_clear)
+
+          if clear_migrated_values?
+            instance.update_columns(to_clear)
+          end
         end
       end
 
diff --git a/lib/gitlab/background_migration/encrypt_runners_tokens.rb b/lib/gitlab/background_migration/encrypt_runners_tokens.rb
new file mode 100644
index 00000000000..91e559a8765
--- /dev/null
+++ b/lib/gitlab/background_migration/encrypt_runners_tokens.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # EncryptColumn migrates data from an unencrypted column - `foo`, say - to
+    # an encrypted column - `encrypted_foo`, say.
+    #
+    # We only create a subclass here because we want to isolate this migration
+    # (migrating unencrypted runner registration tokens to encrypted columns)
+    # from other `EncryptColumns` migration. This class name is going to be
+    # serialized and stored in Redis and later picked by Sidekiq, so we need to
+    # create a separate class name in order to isolate these migration tasks.
+    #
+    # We can solve this differently, see tech debt issue:
+    #
+    # https://gitlab.com/gitlab-org/gitlab-ce/issues/54328
+    #
+    class EncryptRunnersTokens < EncryptColumns
+      def perform(model, from, to)
+        resource = "::Gitlab::BackgroundMigration::Models::EncryptColumns::#{model.to_s.capitalize}"
+        model = resource.constantize
+        attributes = model.encrypted_attributes.keys
+
+        super(model, attributes, from, to)
+      end
+
+      def clear_migrated_values?
+        false
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb b/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb
new file mode 100644
index 00000000000..41f18979d76
--- /dev/null
+++ b/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module Models
+      module EncryptColumns
+        # This model is shared between synchronous and background migrations to
+        # encrypt the `runners_token` column in `namespaces` table.
+        #
+        class Namespace < ActiveRecord::Base
+          include ::EachBatch
+
+          self.table_name = 'namespaces'
+          self.inheritance_column = :_type_disabled
+
+          def runners_token=(value)
+            self.runners_token_encrypted =
+              ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value)
+          end
+
+          def self.encrypted_attributes
+            { runners_token: { attribute: :runners_token_encrypted } }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/models/encrypt_columns/project.rb b/lib/gitlab/background_migration/models/encrypt_columns/project.rb
new file mode 100644
index 00000000000..bfeae14584d
--- /dev/null
+++ b/lib/gitlab/background_migration/models/encrypt_columns/project.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module Models
+      module EncryptColumns
+        # This model is shared between synchronous and background migrations to
+        # encrypt the `runners_token` column in `projects` table.
+        #
+        class Project < ActiveRecord::Base
+          include ::EachBatch
+
+          self.table_name = 'projects'
+          self.inheritance_column = :_type_disabled
+
+          def runners_token=(value)
+            self.runners_token_encrypted =
+              ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value)
+          end
+
+          def self.encrypted_attributes
+            { runners_token: { attribute: :runners_token_encrypted } }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/models/encrypt_columns/runner.rb b/lib/gitlab/background_migration/models/encrypt_columns/runner.rb
new file mode 100644
index 00000000000..14ddce4b147
--- /dev/null
+++ b/lib/gitlab/background_migration/models/encrypt_columns/runner.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module Models
+      module EncryptColumns
+        # This model is shared between synchronous and background migrations to
+        # encrypt the `token` column in `ci_runners` table.
+        #
+        class Runner < ActiveRecord::Base
+          include ::EachBatch
+
+          self.table_name = 'ci_runners'
+          self.inheritance_column = :_type_disabled
+
+          def token=(value)
+            self.token_encrypted =
+              ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value)
+          end
+
+          def self.encrypted_attributes
+            { token: { attribute: :token_encrypted } }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/models/encrypt_columns/settings.rb b/lib/gitlab/background_migration/models/encrypt_columns/settings.rb
new file mode 100644
index 00000000000..08ae35c0671
--- /dev/null
+++ b/lib/gitlab/background_migration/models/encrypt_columns/settings.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module Models
+      module EncryptColumns
+        # This model is shared between synchronous and background migrations to
+        # encrypt the `runners_token` column in `application_settings` table.
+        #
+        class Settings < ActiveRecord::Base
+          include ::EachBatch
+          include ::CacheableAttributes
+
+          self.table_name = 'application_settings'
+          self.inheritance_column = :_type_disabled
+
+          after_commit do
+            ::ApplicationSetting.expire
+          end
+
+          def runners_registration_token=(value)
+            self.runners_registration_token_encrypted =
+              ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value)
+          end
+
+          def self.encrypted_attributes
+            {
+              runners_registration_token: {
+                attribute: :runners_registration_token_encrypted
+              }
+            }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb
index bb76eb8ed48..ccd9d4c6d44 100644
--- a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb
+++ b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb
@@ -15,12 +15,12 @@ module Gitlab
           attr_encrypted :token,
                          mode:      :per_attribute_iv,
                          algorithm: 'aes-256-gcm',
-                         key:       Settings.attr_encrypted_db_key_base_truncated
+                         key:       ::Settings.attr_encrypted_db_key_base_truncated
 
           attr_encrypted :url,
                          mode:      :per_attribute_iv,
                          algorithm: 'aes-256-gcm',
-                         key:       Settings.attr_encrypted_db_key_base_truncated
+                         key:       ::Settings.attr_encrypted_db_key_base_truncated
         end
       end
     end
diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb
index 68d0b5d8f8a..87a03d9c58f 100644
--- a/lib/gitlab/crypto_helper.rb
+++ b/lib/gitlab/crypto_helper.rb
@@ -6,8 +6,8 @@ module Gitlab
 
     AES256_GCM_OPTIONS = {
       algorithm: 'aes-256-gcm',
-      key: Settings.attr_encrypted_db_key_base_truncated,
-      iv: Settings.attr_encrypted_db_key_base_truncated[0..11]
+      key: Settings.attr_encrypted_db_key_base_32,
+      iv: Settings.attr_encrypted_db_key_base_12
     }.freeze
 
     def sha256(value)
@@ -17,7 +17,7 @@ module Gitlab
 
     def aes256_gcm_encrypt(value)
       encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value))
-      Base64.encode64(encrypted_token)
+      Base64.strict_encode64(encrypted_token)
     end
 
     def aes256_gcm_decrypt(value)
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 8380e86c128..4f3298812a8 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -100,6 +100,7 @@ excluded_attributes:
     - :import_source
     - :mirror
     - :runners_token
+    - :runners_token_encrypted
     - :repository_storage
     - :repository_read_only
     - :lfs_enabled
@@ -114,6 +115,9 @@ excluded_attributes:
     - :remote_mirror_available_overridden
     - :description_html
     - :repository_languages
+  namespaces:
+    - :runners_token
+    - :runners_token_encrypted
   project_import_state:
     - :last_error
     - :jid
@@ -155,6 +159,9 @@ excluded_attributes:
     - :encrypted_token_iv
     - :encrypted_url
     - :encrypted_url_iv
+  runners:
+    - :token
+    - :token_encrypted
   services:
     - :template
 
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 097c7653754..9cc781ddf8d 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -10,6 +10,7 @@ module Gitlab
                     triggers: 'Ci::Trigger',
                     pipeline_schedules: 'Ci::PipelineSchedule',
                     builds: 'Ci::Build',
+                    runners: 'Ci::Runner',
                     hooks: 'ProjectHook',
                     merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
                     push_access_levels: 'ProtectedBranch::PushAccessLevel',
@@ -33,7 +34,7 @@ module Gitlab
 
       EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze
 
-      TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze
+      TOKEN_RESET_MODELS = %w[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
 
       def self.create(*args)
         new(*args).create
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index 9e59137a2c0..e0e8f598ba4 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -16,6 +16,21 @@ module Gitlab
       str.force_encoding(Encoding::UTF_8)
     end
 
+    def ensure_utf8_size(str, bytes:)
+      raise ArgumentError, 'Empty string provided!' if str.empty?
+      raise ArgumentError, 'Negative string size provided!' if bytes.negative?
+
+      truncated = str.each_char.each_with_object(+'') do |char, object|
+        if object.bytesize + char.bytesize > bytes
+          break object
+        else
+          object.concat(char)
+        end
+      end
+
+      truncated + ('0' * (bytes - truncated.bytesize))
+    end
+
     # Append path to host, making sure there's one single / in between
     def append_path(host, path)
       "#{host.to_s.sub(%r{\/+$}, '')}/#{path.to_s.sub(%r{^\/+}, '')}"