Add latest changes from gitlab-org/gitlab@master

13 files changed, 195 insertions, 125 deletions

diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS
index f64f8682425..718664c8e3e 100644
--- a/.gitlab/CODEOWNERS
+++ b/.gitlab/CODEOWNERS
@@ -1495,6 +1495,10 @@ ee/lib/ee/api/entities/project.rb
 ^[DiffMatchPatch gem]
 /vendor/gems/diff_match_patch @garyh
 
+^[Gitlab Housekeeper gem] @DylanGriffith @rymai @tigerwnz
+/gems/gitlab-housekeeper/
+/keeps/
+
 # JiHu GitLab rules. See https://gitlab.com/gitlab-jh/gitlab-jh-enablement/-/issues/213#note_1024367528
 
 [JH Frontend] @jihulab/maintainers/frontend
diff --git a/db/migrate/20240119110912_timestamp_for_sbom_source_packages.rb b/db/migrate/20240119110912_timestamp_for_sbom_source_packages.rb
new file mode 100644
index 00000000000..101209c33a7
--- /dev/null
+++ b/db/migrate/20240119110912_timestamp_for_sbom_source_packages.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class TimestampForSbomSourcePackages < Gitlab::Database::Migration[2.2]
+  milestone '16.9'
+
+  def up
+    add_timestamps_with_timezone(:sbom_source_packages, null: false, default: -> { 'NOW()' })
+  end
+
+  def down
+    remove_timestamps(:sbom_source_packages)
+  end
+end
diff --git a/db/post_migrate/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads.rb b/db/post_migrate/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads.rb
index fe939b4aaa0..ab62d1b98b0 100644
--- a/db/post_migrate/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads.rb
+++ b/db/post_migrate/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class QueueBackfillHasMergeRequestOfVulnerabilityReads < Gitlab::Database::Migration[2.1]
-  MIGRATION_NAME = 'BackfillHasMergeRequestOfVulnerabilityReads'
+  MIGRATION = 'BackfillHasMergeRequestOfVulnerabilityReads'
   DELAY_INTERVAL = 2.minutes
   BATCH_SIZE = 10_000
   SUB_BATCH_SIZE = 200
@@ -11,7 +11,7 @@ class QueueBackfillHasMergeRequestOfVulnerabilityReads < Gitlab::Database::Migra
 
   def up
     queue_batched_background_migration(
-      MIGRATION_NAME,
+      MIGRATION,
       :vulnerability_reads,
       :vulnerability_id,
       job_interval: DELAY_INTERVAL,
@@ -21,6 +21,6 @@ class QueueBackfillHasMergeRequestOfVulnerabilityReads < Gitlab::Database::Migra
   end
 
   def down
-    delete_batched_background_migration(MIGRATION_NAME, :vulnerability_reads, :vulnerability_id, [])
+    delete_batched_background_migration(MIGRATION, :vulnerability_reads, :vulnerability_id, [])
   end
 end
diff --git a/db/schema_migrations/20240119110912 b/db/schema_migrations/20240119110912
new file mode 100644
index 00000000000..b8dbb8f8525
--- /dev/null
+++ b/db/schema_migrations/20240119110912
@@ -0,0 +1 @@
+d0d41713de056bea00554a559170a72b346df472f786faab5bb7906a522fcf7d
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index aad301f401c..c7651cd9dee 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -23597,6 +23597,8 @@ CREATE TABLE sbom_source_packages (
     id bigint NOT NULL,
     name text NOT NULL,
     purl_type smallint NOT NULL,
+    created_at timestamp with time zone DEFAULT now() NOT NULL,
+    updated_at timestamp with time zone DEFAULT now() NOT NULL,
     CONSTRAINT check_8fba79abed CHECK ((char_length(name) <= 255))
 );
 
diff --git a/doc/user/application_security/policies/img/experimental-features-policies.png b/doc/user/application_security/policies/img/experimental-features-policies.png
new file mode 100644
index 00000000000..5d1ecd0b80a
--- /dev/null
+++ b/doc/user/application_security/policies/img/experimental-features-policies.png
diff --git a/doc/user/application_security/policies/scan-execution-policies.md b/doc/user/application_security/policies/scan-execution-policies.md
index 9a6f7581876..26a98432d47 100644
--- a/doc/user/application_security/policies/scan-execution-policies.md
+++ b/doc/user/application_security/policies/scan-execution-policies.md
@@ -12,11 +12,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w
 > - Support for custom CI variables in the Scan Execution Policies editor [introduced](https://gitlab.com/groups/gitlab-org/-/epics/9566) in GitLab 16.2.
 > - Enforcement of scan execution policies on projects with an existing GitLab CI/CD configuration [introduced](https://gitlab.com/groups/gitlab-org/-/epics/6880) in GitLab 16.2 [with a flag](../../../administration/feature_flags.md) named `scan_execution_policy_pipelines`. Feature flag `scan_execution_policy_pipelines` removed in GitLab 16.5.
 
-FLAG:
-On self-managed GitLab, this feature is enabled by default. To disable it, ask an
-administrator to [disable the feature flag](../../../administration/feature_flags.md) named
-`scan_execution_policy_pipelines`. On GitLab.com, this feature is enabled.
-
 Group, subgroup, or project owners can use scan execution policies to require that security scans
 run on a specified schedule or with the project pipeline. The security scan runs with multiple
 project pipelines if you define the policy at a group or subgroup level. GitLab injects the required
@@ -331,14 +326,13 @@ These experimental features have limitations:
    without a `.gitlab-ci.yml` is not supported.
 1. The pipeline execution action cannot be used with a scheduled trigger type.
 
-### Pipeline execution policy action
+To enable these experimental features, a Group owner or administrator must toggle the experimental features by visiting `Settings > General > Permissions and group features`.
 
-> The `custom` scan action type was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/126457) in GitLab 16.4 [with a flag](../../../administration/feature_flags.md) named `compliance_pipeline_in_policies`.
+![Enabling experimental security policy features](img/experimental-features-policies.png)
+ 
+Have feedback on our experimental features? We'd love to hear it! Please share your thoughts in our [feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/434425).
 
-FLAG:
-On self-managed GitLab, by default this feature is available.
-To hide the feature, an administrator can [disable the feature flag](../../../administration/feature_flags.md) named `compliance_pipeline_in_policies`.
-On GitLab.com, this feature is available.
+### Pipeline execution policy action
 
 The pipeline execution policy action introduces a new scan action type into
 scan execution policies for creating and enforcing custom CI in your target
@@ -402,14 +396,6 @@ In this example a `test job` is injected into the `test` stage of the pipeline,
 
 ### Security policy scopes
 
-> The `policy_scope` field was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/135398) in GitLab 16.7 [with a flag](../../../administration/feature_flags.md) named `security_policies_policy_scope`.
-
-FLAG:
-On self-managed GitLab, by default this feature is available. To hide the feature,
-an administrator can [disable the feature flag](../../../administration/feature_flags.md)
-named `security_policies_policy_scope`.
-On GitLab.com, this feature is available.
-
 Security policy enforcement depends first on establishing a link between the group, subgroup, or
 project on which you want to enforce policies, and the security policy project that contains the
 policies. For example, if you are linking policies to a group, a group owner must create the link to
diff --git a/doc/user/application_security/policies/scan-result-policies.md b/doc/user/application_security/policies/scan-result-policies.md
index c91b21cae69..5e1541ce536 100644
--- a/doc/user/application_security/policies/scan-result-policies.md
+++ b/doc/user/application_security/policies/scan-result-policies.md
@@ -361,16 +361,13 @@ We have identified in [epic 11020](https://gitlab.com/groups/gitlab-org/-/epics/
 
 ### Security policy scopes
 
-> The `policy_scope` field was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/135398) in GitLab 16.7 [with a flag](../../../administration/feature_flags.md) named `security_policies_policy_scope`.
+To enable these experimental features, a Group owner or administrator must toggle the experimental features by visiting `Settings > General > Permissions and group features`.
 
-FLAG:
-On self-managed GitLab, by default this feature is available. To hide the feature,
-an administrator can [disable the feature flag](../../../administration/feature_flags.md)
-named `security_policies_policy_scope`.
-On GitLab.com, this feature is available.
+![Enabling experimental security policy features](img/experimental-features-policies.png)
+
+Have feedback on our experimental features? We'd love to hear it! Please share your thoughts in our [feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/434425).
 
-Security policy enforcement depends first on establishing a link between the group, subgroup, or
-project on which you want to enforce policies, and the security policy project that contains the
+Security policy enforcement depends first on establishing a link between the group, subgroup, or project on which you want to enforce policies, and the security policy project that contains the
 policies. For example, if you are linking policies to a group, a group owner must create the link to
 the security policy project. Then, all policies in the security policy project are inherited by all
 projects in the group.
@@ -433,14 +430,9 @@ scan_result_policy:
 
 ### Merge request rules widget shows a scan result policy is invalid or duplicated **(ULTIMATE SELF)**
 
-On GitLab self-managed from 15.0 to 16.4, the most likely cause is that the project was exported from a
-group and imported into another, and had scan result policy rules. These rules are stored in a
-separate project to the one that was exported. As a result, the project contains policy rules that
-reference entities that don't exist in the imported project's group. The result is policy rules that
-are invalid, duplicated, or both.
+On GitLab self-managed from 15.0 to 16.4, the most likely cause is that the project was exported from a group and imported into another, and had scan result policy rules. These rules are stored in a separate project to the one that was exported. As a result, the project contains policy rules that reference entities that don't exist in the imported project's group. The result is policy rules that are invalid, duplicated, or both.
 
-To remove all invalid scan result policy rules from a GitLab instance, an administrator can run
-the following script in the [Rails console](../../../administration/operations/rails_console.md).
+To remove all invalid scan result policy rules from a GitLab instance, an administrator can run the following script in the [Rails console](../../../administration/operations/rails_console.md).
 
 ```ruby
 Project.joins(:approval_rules).where(approval_rules: { report_type: %i[scan_finding license_scanning] }).where.not(approval_rules: { security_orchestration_policy_configuration_id: nil }).find_in_batches.flat_map do |batch|
diff --git a/doc/user/clusters/agent/gitops/agent.md b/doc/user/clusters/agent/gitops/agent.md
index 2eba00f0c78..f1fe3f98f55 100644
--- a/doc/user/clusters/agent/gitops/agent.md
+++ b/doc/user/clusters/agent/gitops/agent.md
@@ -14,6 +14,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
 
 WARNING:
 This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/406545) in GitLab 16.2. You should use the [Flux integration](../gitops.md) for GitOps.
+See [Migrate from legacy GitOps to Flux](migrate_to_flux.md).
 
 This diagram shows the repositories and main actors in a GitOps deployment:
 
@@ -37,89 +38,6 @@ sequenceDiagram
 
 For details, view the [architecture documentation](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/architecture.md#high-level-architecture).
 
-## Migrate to Flux
-
-You should migrate your legacy GitOps deployments so they can be
-deployed with Flux. To migrate, configure Flux to deploy manifests
-with Kustomize. If you don't have a `kustomization.yaml` file in
-the given path, it is generated automatically.
-
-Prerequisites:
-
-- A configuration like:
-
-  ```yaml
-  manifest_projects:
-  - id: my-group/my-project
-    default_namespace: production
-    paths:
-    - glob: 'environments/production/**/*.yaml'
-  ```
-
-- A [Flux installation](https://fluxcd.io/flux/installation/) with manifests in `environments/flux-system`.
-- You use a deploy token to access GitLab.
-- Your cluster can access GitLab over HTTPS.
-
-To migrate:
-
-1. Create a file called `environments/flux-system/production.yaml` with the following contents:
-
-   ```yaml
-   # This manifest was generated by flux. DO NOT EDIT.
-   ---
-   apiVersion: source.toolkit.fluxcd.io/v1
-   kind: GitRepository
-   metadata:
-     name: production
-     namespace: flux-system
-   spec:
-     interval: 1m0s
-     ref:
-       branch: main
-     secretRef:
-       name: gitlab-deploy-token
-     url: https://gitlab.example.com/my-group/my-project.git
-   ---
-   apiVersion: kustomize.toolkit.fluxcd.io/v1
-   kind: Kustomization
-   metadata:
-     name: production
-     namespace: flux-system
-   spec:
-     interval: 10m0s
-     path: ./environments/production
-     prune: true
-     sourceRef:
-       kind: GitRepository
-       name: production
-   ```
-
-1. Optional. Because `agentk` uses the `default_namespace` by default, you might need to add a `kustomization.yaml` file to `/environments/production/` and list the relevant resources.
-   For example:
-
-   ```yaml
-   apiVersion: kustomize.config.k8s.io/v1beta1
-   kind: Kustomization
-   namespace: default
-   resources:
-   - relative/path/to-your/resource-1.yaml
-   - relative/path/to-your/resource-2.yaml
-   ```
-
-   When you commit the `kustomization.yaml` file to the repository, a reconciliation with Flux and `agentk` is triggered.
-   Because `agentk` can't handle the Kustomization file, it logs errors when you commit the file.
-
-1. To ensure Flux has taken over the management of the resource, check for the resource in the `status.inventory` value of the `production` Flux Kustomization object:
-
-   ```shell
-   kubectl get kustomization production -n flux-system -o json | jq '.status.inventory.entries'
-   ```
-
-1. Remove the entry from the `manifest_projects` list.
-
-After you migrate, your GitOps deployments deploy with Flux.
-To get the most out of your Flux integration, see the [Flux Kustomization CRD](https://fluxcd.io/flux/components/kustomize/kustomization/) and [GitLab Flux documentation](../gitops.md).
-
 ## GitOps workflow steps
 
 To update a Kubernetes cluster by using GitOps, complete the following steps.
diff --git a/doc/user/clusters/agent/gitops/migrate_to_flux.md b/doc/user/clusters/agent/gitops/migrate_to_flux.md
new file mode 100644
index 00000000000..ba97612ac6a
--- /dev/null
+++ b/doc/user/clusters/agent/gitops/migrate_to_flux.md
@@ -0,0 +1,148 @@
+---
+stage: Deploy
+group: Environments
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments
+---
+
+# Migrate from legacy GitOps to Flux **(FREE ALL)**
+
+Most users can migrate from their [legacy agent-based GitOps solution](agent.md)
+to Flux without additional work or downtime. In most cases, Flux can
+take over existing workloads without any restarts.
+
+## Example GitOps configuration
+
+Your legacy GitOps setup might contain an agent configuration like:
+
+```yaml
+gitops:
+  manifest_projects:
+  - id: <your-group>/<your-repository>
+    paths:
+    - glob: 'manifests/*.yaml'
+```
+
+The `manifests` directory referenced in the `paths.glob` might have two
+manifests. One manifest defines a `Namespace`:
+
+```yaml
+# /manifests/namespace.yaml
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: production
+```
+
+And the other manifest defines a `Deployment`:
+
+```yaml
+# /manifests/deployment.yaml
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  namespace: production
+  labels:
+    app: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.14.2
+        ports:
+        - containerPort: 80
+```
+
+The topics on this page use this configuration to
+demonstrate a migration to Flux.
+
+## Disable legacy GitOps functionality in the agent
+
+When the GitOps configuration is removed, the agent
+doesn't delete any running workloads it applied.
+To remove the GitOps functionality from your agent:
+
+- Delete the `gitops` section from the agent configuration file.
+
+You still need a functional agent,
+so don't delete your entire `config.yaml` file.
+
+If you have multiple items under `gitops.manifest_projects` or under the `paths` list, you can migrate one part at a time by removing only the specific project or path.
+
+## Bootstrap Flux
+
+Prerequisites:
+
+- You disabled the GitOps functionality in your agent.
+- You installed the Flux CLI in a terminal with access to your cluster.
+
+To bootstrap Flux:
+
+- In your terminal, run the `flux bootstrap gitlab` command. For example:
+
+  ```shell
+  flux bootstrap gitlab \
+  --owner=<your-group> \
+  --repository=<your-repository> \
+  --branch=main \
+  --path=manifests/ \
+  --deploy-token-auth
+  ```
+
+Flux is installed on your cluster, and the necessary
+Flux configuration files are committed to `manifests/flux-system`,
+which syncs Flux and the entire `manifests` directory.
+
+Because the workloads (the `Namespace` and `Deployment` manifests)
+are already declared in the `manifests` directory, there is
+no extra work involved.
+
+For more information about configuring Flux with GitLab, see
+[Tutorial: Set up Flux for GitOps](flux_tutorial.md).
+
+## Troubleshooting
+
+### `flux bootstrap` doesn't reconcile manifests correctly
+
+The `flux bootstrap` command creates a `kustomizations.kustomize.toolkit.fluxcd.io`
+resource that points to the `manifests` directory. 
+This resource applies to all the Kubernetes manifests in the directory,
+without requiring a [Kustomization file](https://kubectl.docs.kubernetes.io/references/kustomize/glossary/#kustomization).
+
+This process might not work with your configuration.
+To troubleshoot, review the Flux Kustomization status for potential issues:
+
+```shell
+kubectl get kustomizations.kustomize.toolkit.fluxcd.io -n flux-system
+```
+
+### Use a `default_namespace` in the agent configuration
+
+You might encounter an issue if your legacy agent-based GitOps setup
+refers to a `default_namespace` in the agent configuration, but omits this
+namespace in the manifests itself. This causes an error where
+your bootstrapped Flux doesn't know that your existing manifests are applied
+to the `default_namespace`.
+
+To solve this issue, you can either:
+
+- Set the namespace manually in your previously existing resource YAML.
+- Move your resources into a dedicated directory, and point Flux at it with `kustomize.toolkit.fluxcd.io/Kustomization`, where `spec.targetNamespace` specifies the namespace.
+- Move the resources into a subdirectory and add a `kustomization.yaml` file that sets the `spec.namespace` property.
+
+If you prefer to move the resources outside the path already configured for Flux,
+you should use `kustomize.toolkit.fluxcd.io/Kustomization`.
+If you prefer to move the resources into a subdirectory of a path already watched by
+Flux, you should use a `kustomize.config.k8s.io/Kustomization`.
diff --git a/keeps/overdue_finalize_background_migration.rb b/keeps/overdue_finalize_background_migration.rb
index 5cfeca0b41b..035db201b48 100644
--- a/keeps/overdue_finalize_background_migration.rb
+++ b/keeps/overdue_finalize_background_migration.rb
@@ -23,7 +23,7 @@ module Keeps
   #   -k Keeps::OverdueFinalizeBackgroundMigration
   # ```
   class OverdueFinalizeBackgroundMigration < ::Gitlab::Housekeeper::Keep
-    CUTOFF_MILESTONE = '16.4'
+    CUTOFF_MILESTONE = '16.8' # Only finalize migrations added before this
 
     def initialize; end
 
@@ -45,6 +45,7 @@ module Keeps
         identifiers = [self.class.name.demodulize, job_name]
 
         last_migration_file = last_migration_for_job(job_name)
+        next unless last_migration_file
 
         # rubocop:disable Gitlab/DocUrl -- Not running inside rails application
         description = <<~MARKDOWN
@@ -110,14 +111,19 @@ module Keeps
     end
 
     def last_migration_for_job(job_name)
-      result = ::Gitlab::Housekeeper::Shell.execute('git', 'grep', '--name-only', "MIGRATION = .#{job_name}.").chomp
-      result = result.each_line.select do |file|
+      files = ::Gitlab::Housekeeper::Shell.execute('git', 'grep', '--name-only', "MIGRATION = .#{job_name}.")
+        .each_line.map(&:chomp)
+
+      result = files.select do |file|
         File.read(file).include?('queue_batched_background_migration')
       end.max
 
       raise "Could not find migration for #{job_name}" unless result.present?
 
       result
+    rescue ::Gitlab::Housekeeper::Shell::Error
+      # `git grep` returns an error status code if it finds no results
+      nil
     end
 
     def add_ensure_call_to_migration(file, queue_method_node, job_name)
diff --git a/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb b/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb
index 844c3b54587..c7eac42c0e0 100644
--- a/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb
+++ b/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb
@@ -166,7 +166,7 @@ RSpec.describe Gitlab::Metrics::Exporter::BaseExporter, feature_category: :cloud
       end
 
       describe '#start' do
-        it "doesn't start running server" do
+        it "doesn't start running server", quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/438765' do
           expect(::WEBrick::HTTPServer).not_to receive(:new)
 
           exporter.start
diff --git a/spec/migrations/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads_spec.rb b/spec/migrations/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads_spec.rb
index 7214e0114d4..519e81cc933 100644
--- a/spec/migrations/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads_spec.rb
+++ b/spec/migrations/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads_spec.rb
@@ -4,7 +4,7 @@ require 'spec_helper'
 require_migration!
 
 RSpec.describe QueueBackfillHasMergeRequestOfVulnerabilityReads, feature_category: :database do
-  let!(:batched_migration) { described_class::MIGRATION_NAME }
+  let!(:batched_migration) { described_class::MIGRATION }
 
   it 'schedules a new batched migration' do
     reversible_migration do |migration|