Add latest changes from gitlab-org/gitlab@master

11 files changed, 280 insertions, 12 deletions

diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb
index 86dedc12fca..0fbd0e6be44 100644
--- a/lib/api/helpers/integrations_helpers.rb
+++ b/lib/api/helpers/integrations_helpers.rb
@@ -440,6 +440,32 @@ module API
             },
             chat_notification_events
           ].flatten,
+          'harbor' => [
+            {
+              required: true,
+              name: :url,
+              type: String,
+              desc: 'The base URL to the Harbor instance which is being linked to this GitLab project. For example, https://demo.goharbor.io.'
+            },
+            {
+              required: true,
+              name: :project_name,
+              type: String,
+              desc: 'The Project name to the Harbor instance. For example, testproject.'
+            },
+            {
+              required: true,
+              name: :username,
+              type: String,
+              desc: 'The username created from Harbor interface.'
+            },
+            {
+              required: true,
+              name: :password,
+              type: String,
+              desc: 'The password of the user.'
+            }
+          ],
           'irker' => [
             {
               required: true,
@@ -856,6 +882,7 @@ module API
           ::Integrations::ExternalWiki,
           ::Integrations::Flowdock,
           ::Integrations::HangoutsChat,
+          ::Integrations::Harbor,
           ::Integrations::Irker,
           ::Integrations::Jenkins,
           ::Integrations::Jira,
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 950119e155e..1b16873f737 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -231,12 +231,26 @@ module Gitlab
       ::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name)
     end
 
+    # This returns all matching schemas that a given connection can use
+    # Since the `ActiveRecord::Base` might change the connection (from main to ci)
+    # This does not look at literal connection names, but rather compares
+    # models that are holders for a given db_config_name
+    def self.gitlab_schemas_for_connection(connection)
+      connection_name = self.db_config_name(connection)
+      primary_model = self.database_base_models.fetch(connection_name)
+
+      self.schemas_to_base_models
+        .select { |_, models| models.include?(primary_model) }
+        .keys
+        .map!(&:to_sym)
+    end
+
     def self.db_config_for_connection(connection)
       return unless connection
 
-      # The LB connection proxy does not have a direct db_config
-      # that can be referenced
-      return if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy)
+      if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy)
+        return connection.load_balancer.configuration.primary_db_config
+      end
 
       # During application init we might receive `NullPool`
       return unless connection.respond_to?(:pool) &&
diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb
index a950a63c575..e9846dd4e85 100644
--- a/lib/gitlab/database/async_indexes/migration_helpers.rb
+++ b/lib/gitlab/database/async_indexes/migration_helpers.rb
@@ -5,6 +5,8 @@ module Gitlab
     module AsyncIndexes
       module MigrationHelpers
         def unprepare_async_index(table_name, column_name, **options)
+          Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
+
           return unless async_index_creation_available?
 
           index_name = options[:name] || index_name(table_name, column_name)
@@ -15,6 +17,8 @@ module Gitlab
         end
 
         def unprepare_async_index_by_name(table_name, index_name, **options)
+          Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
+
           return unless async_index_creation_available?
 
           PostgresAsyncIndex.find_by(name: index_name).try do |async_index|
@@ -32,6 +36,8 @@ module Gitlab
         # If the requested index has already been created, it is not stored in the table for
         # asynchronous creation.
         def prepare_async_index(table_name, column_name, **options)
+          Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
+
           return unless async_index_creation_available?
 
           index_name = options[:name] || index_name(table_name, column_name)
diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb
index 7adf6ba6afb..d7ea614e2af 100644
--- a/lib/gitlab/database/gitlab_schema.rb
+++ b/lib/gitlab/database/gitlab_schema.rb
@@ -95,6 +95,10 @@ module Gitlab
       def self.tables_to_schema
         @tables_to_schema ||= YAML.load_file(Rails.root.join('lib/gitlab/database/gitlab_schemas.yml'))
       end
+
+      def self.schema_names
+        @schema_names ||= self.tables_to_schema.values.to_set
+      end
     end
   end
 end
diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb
index e16db5af8ce..6517923d23e 100644
--- a/lib/gitlab/database/load_balancing.rb
+++ b/lib/gitlab/database/load_balancing.rb
@@ -47,6 +47,8 @@ module Gitlab
       # Returns the role (primary/replica) of the database the connection is
       # connecting to.
       def self.db_role_for_connection(connection)
+        return ROLE_UNKNOWN if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy)
+
         db_config = Database.db_config_for_connection(connection)
         return ROLE_UNKNOWN unless db_config
 
diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb
index 63444ebe169..86b3afaa47b 100644
--- a/lib/gitlab/database/load_balancing/configuration.rb
+++ b/lib/gitlab/database/load_balancing/configuration.rb
@@ -94,6 +94,10 @@ module Gitlab
           end
         end
 
+        def primary_db_config
+          primary_model_or_model_if_enabled.connection_db_config
+        end
+
         def replica_db_config
           @model.connection_db_config
         end
diff --git a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb
new file mode 100644
index 00000000000..b4e31565c60
--- /dev/null
+++ b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module MigrationHelpers
+      module RestrictGitlabSchema
+        extend ActiveSupport::Concern
+
+        MigrationSkippedError = Class.new(StandardError)
+
+        included do
+          class_attribute :allowed_gitlab_schemas
+        end
+
+        class_methods do
+          def restrict_gitlab_migration(gitlab_schema:)
+            unless Gitlab::Database::GitlabSchema.schema_names.include?(gitlab_schema)
+              raise "Unknown 'gitlab_schema: #{gitlab_schema}' specified. It needs to be one of: " \
+                "#{Gitlab::Database::GitlabSchema.schema_names.to_a}"
+            end
+
+            self.allowed_gitlab_schemas = [gitlab_schema]
+          end
+        end
+
+        def migrate(direction)
+          if unmatched_schemas.any?
+            # TODO: Today skipping migration would raise an exception.
+            # Ideally, skipped migration should be ignored (not loaded), or softly ignored.
+            # Read more in: https://gitlab.com/gitlab-org/gitlab/-/issues/355014
+            raise MigrationSkippedError, "Current migration is skipped since it modifies "\
+              "'#{self.class.allowed_gitlab_schemas}' which is outside of '#{allowed_schemas_for_connection}'"
+          end
+
+          Gitlab::Database::QueryAnalyzer.instance.within([validator_class]) do
+            validator_class.allowed_gitlab_schemas = self.allowed_gitlab_schemas
+
+            super
+          end
+        end
+
+        private
+
+        def validator_class
+          Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas
+        end
+
+        def unmatched_schemas
+          (self.allowed_gitlab_schemas || []) - allowed_schemas_for_connection
+        end
+
+        def allowed_schemas_for_connection
+          Gitlab::Database.gitlab_schemas_for_connection(connection)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb
index 2736f9d18dc..0c78dda734c 100644
--- a/lib/gitlab/database/query_analyzer.rb
+++ b/lib/gitlab/database/query_analyzer.rb
@@ -30,13 +30,17 @@ module Gitlab
         end
       end
 
-      def within
+      def within(user_analyzers = nil)
         # Due to singleton nature of analyzers
         # only an outer invocation of the `.within`
         # is allowed to initialize them
-        return yield if already_within?
+        if already_within?
+          raise 'Query analyzers are already defined, cannot re-define them.' if user_analyzers
 
-        begin!
+          return yield
+        end
+
+        begin!(user_analyzers || all_analyzers)
 
         begin
           yield
@@ -61,21 +65,21 @@ module Gitlab
           next if analyzer.suppressed? && !analyzer.requires_tracking?(parsed)
 
           analyzer.analyze(parsed)
-        rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e
+        rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e
           # We catch all standard errors to prevent validation errors to introduce fatal errors in production
           Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
         end
       end
 
       # Enable query analyzers
-      def begin!
-        analyzers = all_analyzers.select do |analyzer|
+      def begin!(analyzers = all_analyzers)
+        analyzers = analyzers.select do |analyzer|
           if analyzer.enabled?
             analyzer.begin!
 
             true
           end
-        rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e
+        rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e
           Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
 
           false
@@ -88,7 +92,7 @@ module Gitlab
       def end!
         enabled_analyzers.select do |analyzer|
           analyzer.end!
-        rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e
+        rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e
           Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
         end
 
diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb
new file mode 100644
index 00000000000..ab40ba5d59b
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb
@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module QueryAnalyzers
+      class RestrictAllowedSchemas < Base
+        UnsupportedSchemaError = Class.new(QueryAnalyzerError)
+        DDLNotAllowedError = Class.new(UnsupportedSchemaError)
+        DMLNotAllowedError = Class.new(UnsupportedSchemaError)
+        DMLAccessDeniedError = Class.new(UnsupportedSchemaError)
+
+        IGNORED_SCHEMAS = %i[gitlab_shared].freeze
+
+        class << self
+          def enabled?
+            true
+          end
+
+          def allowed_gitlab_schemas
+            self.context[:allowed_gitlab_schemas]
+          end
+
+          def allowed_gitlab_schemas=(value)
+            self.context[:allowed_gitlab_schemas] = value
+          end
+
+          def analyze(parsed)
+            # If list of schemas is empty, we allow only DDL changes
+            if self.dml_mode?
+              self.restrict_to_dml_only(parsed)
+            else
+              self.restrict_to_ddl_only(parsed)
+            end
+          end
+
+          def require_ddl_mode!(message = "")
+            return unless self.context
+
+            self.raise_dml_not_allowed_error(message) if self.dml_mode?
+          end
+
+          def require_dml_mode!(message = "")
+            return unless self.context
+
+            self.raise_ddl_not_allowed_error(message) if self.ddl_mode?
+          end
+
+          private
+
+          def restrict_to_ddl_only(parsed)
+            tables = self.dml_tables(parsed)
+            schemas = self.dml_schemas(tables)
+
+            if schemas.any?
+              self.raise_dml_not_allowed_error("Modifying of '#{tables}' (#{schemas.to_a}) with '#{parsed.sql}'")
+            end
+          end
+
+          def restrict_to_dml_only(parsed)
+            if parsed.pg.ddl_tables.any?
+              self.raise_ddl_not_allowed_error("Modifying of '#{parsed.pg.ddl_tables}' with '#{parsed.sql}'")
+            end
+
+            if parsed.pg.ddl_functions.any?
+              self.raise_ddl_not_allowed_error("Modifying of '#{parsed.pg.ddl_functions}' with '#{parsed.sql}'")
+            end
+
+            tables = self.dml_tables(parsed)
+            schemas = self.dml_schemas(tables)
+
+            if (schemas - self.allowed_gitlab_schemas).any?
+              raise DMLAccessDeniedError, "Select/DML queries (SELECT/UPDATE/DELETE) do access '#{tables}' (#{schemas.to_a}) " \
+                "which is outside of list of allowed schemas: '#{self.allowed_gitlab_schemas}'."
+            end
+          end
+
+          def dml_mode?
+            self.allowed_gitlab_schemas&.any?
+          end
+
+          def ddl_mode?
+            !self.dml_mode?
+          end
+
+          def dml_tables(parsed)
+            parsed.pg.select_tables + parsed.pg.dml_tables
+          end
+
+          def dml_schemas(tables)
+            extra_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables)
+            extra_schemas.subtract(IGNORED_SCHEMAS)
+            extra_schemas
+          end
+
+          def raise_dml_not_allowed_error(message)
+            raise DMLNotAllowedError, "Select/DML queries (SELECT/UPDATE/DELETE) are disallowed in the DDL (structure) mode. #{message}"
+          end
+
+          def raise_ddl_not_allowed_error(message)
+            raise DDLNotAllowedError, "DDL queries (structure) are disallowed in the Select/DML (SELECT/UPDATE/DELETE) mode. #{message}"
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/harbor/client.rb b/lib/gitlab/harbor/client.rb
new file mode 100644
index 00000000000..06142ae2b40
--- /dev/null
+++ b/lib/gitlab/harbor/client.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Harbor
+    class Client
+      Error = Class.new(StandardError)
+      ConfigError = Class.new(Error)
+
+      attr_reader :integration
+
+      def initialize(integration)
+        raise ConfigError, 'Please check your integration configuration.' unless integration
+
+        @integration = integration
+      end
+
+      def ping
+        options = { headers: headers.merge!('Accept': 'text/plain') }
+        response = Gitlab::HTTP.get(url('ping'), options)
+
+        { success: response.success? }
+      end
+
+      private
+
+      def url(path)
+        Gitlab::Utils.append_path(base_url, path)
+      end
+
+      def base_url
+        Gitlab::Utils.append_path(integration.url, '/api/v2.0/')
+      end
+
+      def headers
+        auth = Base64.strict_encode64("#{integration.username}:#{integration.password}")
+        {
+          'Content-Type': 'application/json',
+          'Authorization': "Basic #{auth}"
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb
index 1350d75b216..82c2b3297c1 100644
--- a/lib/gitlab/integrations/sti_type.rb
+++ b/lib/gitlab/integrations/sti_type.rb
@@ -5,7 +5,7 @@ module Gitlab
     class StiType < ActiveRecord::Type::String
       NAMESPACED_INTEGRATIONS = Set.new(%w(
         Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog
-        Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker Jenkins Jira Mattermost
+        Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Harbor Irker Jenkins Jira Mattermost
         MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker
         Prometheus Pushover Redmine Shimo Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao
       )).freeze