diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2024-01-11 09:12:48 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2024-01-11 09:12:48 +0300 |
commit | 826d6628ca045013b9d19ec5cb4d02ac81b76c68 (patch) | |
tree | 176fe063a69cce5534d72558c505fd943d243935 | |
parent | 988f8190b39847793faba06375973f8d4a024426 (diff) |
Add latest changes from gitlab-org/gitlab@master
-rw-r--r-- | db/docs/sbom_source_packages.yml | 10 | ||||
-rw-r--r-- | db/migrate/20231221033539_create_sbom_source_packages_table.rb | 42 | ||||
-rw-r--r-- | db/schema_migrations/20231221033539 | 1 | ||||
-rw-r--r-- | db/structure.sql | 29 | ||||
-rw-r--r-- | doc/administration/job_artifacts_troubleshooting.md | 14 | ||||
-rw-r--r-- | lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | qa/qa/runtime/browser.rb | 15 | ||||
-rw-r--r-- | spec/lib/gitlab/database/no_new_tables_with_gitlab_main_schema_spec.rb | 3 |
10 files changed, 105 insertions, 15 deletions
diff --git a/db/docs/sbom_source_packages.yml b/db/docs/sbom_source_packages.yml new file mode 100644 index 00000000000..2c0df1df88e --- /dev/null +++ b/db/docs/sbom_source_packages.yml @@ -0,0 +1,10 @@ +--- +table_name: sbom_source_packages +classes: +- Sbom::SourcePackage +feature_categories: +- dependency_management +description: Tracks Source Package of an SBOM Occurrence +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/140539 +milestone: '16.8' +gitlab_schema: gitlab_main diff --git a/db/migrate/20231221033539_create_sbom_source_packages_table.rb b/db/migrate/20231221033539_create_sbom_source_packages_table.rb new file mode 100644 index 00000000000..6e2389baf8e --- /dev/null +++ b/db/migrate/20231221033539_create_sbom_source_packages_table.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateSbomSourcePackagesTable < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + milestone '16.8' + + SBOM_SOURCE_PACKAGES_INDEX_NAME = 'idx_sbom_source_packages_on_name_and_purl_type' + SBOM_OCCURRENCES_SOURCE_PACKAGE_ID_AND_ID_INDEX_NAME = 'index_sbom_source_packages_on_source_package_id_and_id' + + def up + with_lock_retries do + add_column :sbom_occurrences, :source_package_id, :bigint, if_not_exists: true + end + + create_table :sbom_source_packages, if_not_exists: true do |t| + t.text :name, null: false, limit: 255 + t.integer :purl_type, limit: 2, null: false + t.index [:name, :purl_type], unique: true, name: SBOM_SOURCE_PACKAGES_INDEX_NAME + end + + add_concurrent_index :sbom_occurrences, [:source_package_id, :id], + name: SBOM_OCCURRENCES_SOURCE_PACKAGE_ID_AND_ID_INDEX_NAME + + add_concurrent_foreign_key :sbom_occurrences, :sbom_source_packages, + column: :source_package_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key_if_exists( + :sbom_occurrences, + column: :source_package_id, + on_delete: :cascade + ) + remove_column :sbom_occurrences, :source_package_id, if_exists: true + drop_table :sbom_source_packages, if_exists: true + end + end +end diff --git a/db/schema_migrations/20231221033539 b/db/schema_migrations/20231221033539 new file mode 100644 index 00000000000..6056121cac1 --- /dev/null +++ b/db/schema_migrations/20231221033539 @@ -0,0 +1 @@ +4493149bdc2db628180768717ebea209665e2311c6ce08269a6985ac643017c2
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 46631ec83c1..d7fe66a0086 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -23495,6 +23495,7 @@ CREATE TABLE sbom_occurrences ( vulnerabilities jsonb DEFAULT '[]'::jsonb, highest_severity smallint, vulnerability_count integer DEFAULT 0 NOT NULL, + source_package_id bigint, CONSTRAINT check_3f2d2c7ffc CHECK ((char_length(package_manager) <= 255)), CONSTRAINT check_9b29021fa8 CHECK ((char_length(component_name) <= 255)), CONSTRAINT check_bd1367d4c1 CHECK ((char_length(input_file_path) <= 255)) @@ -23526,6 +23527,22 @@ CREATE SEQUENCE sbom_occurrences_vulnerabilities_id_seq ALTER SEQUENCE sbom_occurrences_vulnerabilities_id_seq OWNED BY sbom_occurrences_vulnerabilities.id; +CREATE TABLE sbom_source_packages ( + id bigint NOT NULL, + name text NOT NULL, + purl_type smallint NOT NULL, + CONSTRAINT check_8fba79abed CHECK ((char_length(name) <= 255)) +); + +CREATE SEQUENCE sbom_source_packages_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE sbom_source_packages_id_seq OWNED BY sbom_source_packages.id; + CREATE TABLE sbom_sources ( id bigint NOT NULL, created_at timestamp with time zone NOT NULL, @@ -27687,6 +27704,8 @@ ALTER TABLE ONLY sbom_occurrences ALTER COLUMN id SET DEFAULT nextval('sbom_occu ALTER TABLE ONLY sbom_occurrences_vulnerabilities ALTER COLUMN id SET DEFAULT nextval('sbom_occurrences_vulnerabilities_id_seq'::regclass); +ALTER TABLE ONLY sbom_source_packages ALTER COLUMN id SET DEFAULT nextval('sbom_source_packages_id_seq'::regclass); + ALTER TABLE ONLY sbom_sources ALTER COLUMN id SET DEFAULT nextval('sbom_sources_id_seq'::regclass); ALTER TABLE ONLY scan_result_policies ALTER COLUMN id SET DEFAULT nextval('scan_result_policies_id_seq'::regclass); @@ -30335,6 +30354,9 @@ ALTER TABLE ONLY sbom_occurrences ALTER TABLE ONLY sbom_occurrences_vulnerabilities ADD CONSTRAINT sbom_occurrences_vulnerabilities_pkey PRIMARY KEY (id); +ALTER TABLE ONLY sbom_source_packages + ADD CONSTRAINT sbom_source_packages_pkey PRIMARY KEY (id); + ALTER TABLE ONLY sbom_sources ADD CONSTRAINT sbom_sources_pkey PRIMARY KEY (id); @@ -32306,6 +32328,8 @@ CREATE INDEX idx_repository_states_outdated_checksums ON project_repository_stat CREATE INDEX idx_sbom_occurrences_on_project_id_and_source_id ON sbom_occurrences USING btree (project_id, source_id); +CREATE UNIQUE INDEX idx_sbom_source_packages_on_name_and_purl_type ON sbom_source_packages USING btree (name, purl_type); + CREATE UNIQUE INDEX idx_security_scans_on_build_and_scan_type ON security_scans USING btree (build_id, scan_type); CREATE INDEX idx_security_scans_on_scan_type ON security_scans USING btree (scan_type); @@ -35188,6 +35212,8 @@ CREATE UNIQUE INDEX index_sbom_occurrences_on_uuid ON sbom_occurrences USING btr CREATE INDEX index_sbom_occurrences_vulnerabilities_on_vulnerability_id ON sbom_occurrences_vulnerabilities USING btree (vulnerability_id); +CREATE INDEX index_sbom_source_packages_on_source_package_id_and_id ON sbom_occurrences USING btree (source_package_id, id); + CREATE UNIQUE INDEX index_sbom_sources_on_source_type_and_source ON sbom_sources USING btree (source_type, source); CREATE UNIQUE INDEX index_scan_result_policies_on_position_in_configuration ON scan_result_policies USING btree (security_orchestration_policy_configuration_id, project_id, orchestration_policy_idx, rule_idx); @@ -38734,6 +38760,9 @@ ALTER TABLE ONLY fork_network_members ALTER TABLE ONLY work_item_colors ADD CONSTRAINT fk_b15b0912d0 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE SET NULL; +ALTER TABLE ONLY sbom_occurrences + ADD CONSTRAINT fk_b1b65d8d17 FOREIGN KEY (source_package_id) REFERENCES sbom_source_packages(id) ON DELETE CASCADE; + ALTER TABLE ONLY vulnerabilities ADD CONSTRAINT fk_b1de915a15 FOREIGN KEY (author_id) REFERENCES users(id) ON DELETE SET NULL; diff --git a/doc/administration/job_artifacts_troubleshooting.md b/doc/administration/job_artifacts_troubleshooting.md index b8605ff94bf..6cdde87cc1d 100644 --- a/doc/administration/job_artifacts_troubleshooting.md +++ b/doc/administration/job_artifacts_troubleshooting.md @@ -272,8 +272,11 @@ To change the number of job artifacts listed, change the number in `limit(50)`. WARNING: These commands remove data permanently from database and storage. Before running them, we highly recommend seeking guidance from a Support Engineer, or running them in a test environment with a backup of the instance ready to be restored, just in case. -If you need to manually remove job artifacts associated with multiple jobs while -**retaining their job logs**, this can be done from the [Rails console](operations/rails_console.md): +You can manually remove job artifacts associated with multiple completed jobs while +**retaining their job logs** from the [Rails console](operations/rails_console.md). +A completed job is any job with the status of success, failed, canceled, or skipped. + +To delete jobs completed before a specific date: 1. Select jobs to be deleted: @@ -326,8 +329,11 @@ If you need to manually remove job artifacts associated with multiple jobs while WARNING: These commands remove data permanently from both the database and from disk. Before running them, we highly recommend seeking guidance from a Support Engineer, or running them in a test environment with a backup of the instance ready to be restored, just in case. -If you need to manually remove **all** job artifacts associated with multiple jobs, -**including job logs**, this can be done from the [Rails console](operations/rails_console.md): +You can manually remove job artifacts associated with multiple completed jobs while +**retaining their job logs** from the [Rails console](operations/rails_console.md). +A completed job is any job with the status of success, failed, canceled, or skipped. + +To delete jobs completed before a specific date: 1. Select the jobs to be deleted: diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 467d4629010..6f8bed32796 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.1' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 0744a5a6321..52367cfe97d 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.1' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index b3e44dbf4be..06dc91a8bbc 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.1' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/qa/qa/runtime/browser.rb b/qa/qa/runtime/browser.rb index 1fad0b76645..a0949a0daf4 100644 --- a/qa/qa/runtime/browser.rb +++ b/qa/qa/runtime/browser.rb @@ -69,17 +69,18 @@ module QA chrome_options = { args: %w[no-sandbox] } # Run headless by default unless WEBDRIVER_HEADLESS is false - if QA::Runtime::Env.webdriver_headless? - chrome_options[:args] << 'headless' - - # Chrome documentation says this flag is needed for now - # https://developers.google.com/web/updates/2017/04/headless-chrome#cli - chrome_options[:args] << 'disable-gpu' - end + chrome_options[:args] << 'headless=new' if QA::Runtime::Env.webdriver_headless? # Disable /dev/shm use in CI. See https://gitlab.com/gitlab-org/gitlab/issues/4252 chrome_options[:args] << 'disable-dev-shm-usage' if QA::Runtime::Env.disable_dev_shm? + # Allows chrome to consider all actions as secure when no ssl is used + Runtime::Scenario.attributes[:gitlab_address].tap do |address| + next unless address.start_with?('http://') + + chrome_options[:args] << "unsafely-treat-insecure-origin-as-secure=#{address}" + end + # Set chrome default download path # TODO: Set for remote grid as well once Sauce Labs tests are deprecated and Options.chrome is added # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112258 diff --git a/spec/lib/gitlab/database/no_new_tables_with_gitlab_main_schema_spec.rb b/spec/lib/gitlab/database/no_new_tables_with_gitlab_main_schema_spec.rb index d1d7aa12c46..4fc62c6cc74 100644 --- a/spec/lib/gitlab/database/no_new_tables_with_gitlab_main_schema_spec.rb +++ b/spec/lib/gitlab/database/no_new_tables_with_gitlab_main_schema_spec.rb @@ -12,7 +12,8 @@ RSpec.describe 'new tables with gitlab_main schema', feature_category: :cell do # Specific tables can be exempted from this requirement, and such tables must be added to the `exempted_tables` list. let!(:exempted_tables) do [ - "audit_events_instance_amazon_s3_configurations" # https://gitlab.com/gitlab-org/gitlab/-/issues/431327 + "audit_events_instance_amazon_s3_configurations", # https://gitlab.com/gitlab-org/gitlab/-/issues/431327 + "sbom_source_packages" # https://gitlab.com/gitlab-org/gitlab/-/issues/437718 ] end |