diff options
20 files changed, 1558 insertions, 1236 deletions
diff --git a/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml b/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml index c2447519268..767dc8433d2 100644 --- a/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml +++ b/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml @@ -99,7 +99,6 @@ cache-gems: gdk config set postgresql.replica_2.enabled true &&\ gdk config set load_balancing.enabled true &&\ gdk reconfigure &&\ - echo 'recovery_min_apply_delay = 5s' >> postgresql-replica-2/data/postgresql.conf &&\ gdk restart" gdk-qa-smoke: diff --git a/db/post_migrate/20230519023720_swap_ci_pipeline_variables_pk_with_bigint.rb b/db/post_migrate/20230519023720_swap_ci_pipeline_variables_pk_with_bigint.rb new file mode 100644 index 00000000000..6c088cd12a9 --- /dev/null +++ b/db/post_migrate/20230519023720_swap_ci_pipeline_variables_pk_with_bigint.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +class SwapCiPipelineVariablesPkWithBigint < Gitlab::Database::Migration[2.1] + include Gitlab::Database::MigrationHelpers::ConvertToBigint + disable_ddl_transaction! + + TABLE_NAME = 'ci_pipeline_variables' + + def up + swap + end + + def down + swap(stepping_down: true) + end + + private + + def swap(stepping_down: false) + # Prepare the names we need below + primary_key_constraint_name = "#{TABLE_NAME}_pkey" + sequence_name = "#{TABLE_NAME}_id_seq" + bigint_primary_key_index_name = "index_#{TABLE_NAME}_on_id_convert_to_bigint" + temp_name = quote_column_name(:id_tmp) + id_name = quote_column_name(:id) + id_convert_to_bigint_name = quote_column_name(:id_convert_to_bigint) + function_name = quote_table_name( + Gitlab::Database::UnidirectionalCopyTrigger.on_table( + TABLE_NAME, connection: Ci::ApplicationRecord.connection + ).name(:id, :id_convert_to_bigint) + ) + + # 2. Create indexes using the bigint columns that match the existing indexes using the integer column + # NOTE: this index is already created in: + # - https://gitlab.com/gitlab-org/gitlab/-/merge_requests/120946 + # - https://gitlab.com/gitlab-org/gitlab/-/merge_requests/120950 + # Therefore, this won't have any effect for `up` step, but will be used for `down` step. + add_concurrent_index TABLE_NAME, :id_convert_to_bigint, unique: true, name: bigint_primary_key_index_name + + # 4. Inside a transaction, swap the columns + # When stepping up, it will swap the bigint column as the primary key and the int column as `bigint` + # When stepping down, it will swap the int column as the primary key and the bigint column as `bigint` + with_lock_retries(raise_on_exhaustion: true) do + # a. Lock the tables involved. + execute "LOCK TABLE #{TABLE_NAME} IN ACCESS EXCLUSIVE MODE" + + # b. Rename the columns to swap names + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{id_name} TO #{temp_name}" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{id_convert_to_bigint_name} TO #{id_name}" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{temp_name} TO #{id_convert_to_bigint_name}" + + # c. Reset the trigger function + execute "ALTER FUNCTION #{function_name} RESET ALL" + + # d. Swap the defaults + execute "ALTER SEQUENCE #{sequence_name} OWNED BY #{TABLE_NAME}.id" + change_column_default TABLE_NAME, :id, -> { "nextval('#{sequence_name}'::regclass)" } + change_column_default TABLE_NAME, :id_convert_to_bigint, 0 + + # e. Swap the PK constraint + execute "ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT #{primary_key_constraint_name} CASCADE" + rename_index TABLE_NAME, bigint_primary_key_index_name, primary_key_constraint_name + execute <<~SQL + ALTER TABLE #{TABLE_NAME} + ADD CONSTRAINT #{primary_key_constraint_name} PRIMARY KEY + USING INDEX #{primary_key_constraint_name} + SQL + end + + return unless stepping_down + + # For stepping down, we will need to recreate the index after the swap + add_concurrent_index TABLE_NAME, :id_convert_to_bigint, unique: true, name: bigint_primary_key_index_name + end +end diff --git a/db/schema_migrations/20230519023720 b/db/schema_migrations/20230519023720 new file mode 100644 index 00000000000..13e5eeaff1e --- /dev/null +++ b/db/schema_migrations/20230519023720 @@ -0,0 +1 @@ +fe0db2f640801d6086d5199364bfdf6f868cc53eda46c6e0896b49915775b652
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 6576dc0615d..587311dad88 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -13646,7 +13646,7 @@ CREATE SEQUENCE ci_pipeline_schedules_id_seq ALTER SEQUENCE ci_pipeline_schedules_id_seq OWNED BY ci_pipeline_schedules.id; CREATE TABLE ci_pipeline_variables ( - id integer NOT NULL, + id_convert_to_bigint integer DEFAULT 0 NOT NULL, key character varying NOT NULL, value text, encrypted_value text, @@ -13656,7 +13656,7 @@ CREATE TABLE ci_pipeline_variables ( variable_type smallint DEFAULT 1 NOT NULL, partition_id bigint DEFAULT 100 NOT NULL, raw boolean DEFAULT false NOT NULL, - id_convert_to_bigint bigint DEFAULT 0 NOT NULL + id bigint NOT NULL ); CREATE SEQUENCE ci_pipeline_variables_id_seq @@ -30466,8 +30466,6 @@ CREATE INDEX index_ci_pipeline_schedules_on_owner_id_and_id_and_active ON ci_pip CREATE INDEX index_ci_pipeline_schedules_on_project_id ON ci_pipeline_schedules USING btree (project_id); -CREATE UNIQUE INDEX index_ci_pipeline_variables_on_id_convert_to_bigint ON ci_pipeline_variables USING btree (id_convert_to_bigint); - CREATE UNIQUE INDEX index_ci_pipeline_variables_on_pipeline_id_and_key ON ci_pipeline_variables USING btree (pipeline_id, key); CREATE INDEX index_ci_pipelines_config_on_pipeline_id ON ci_pipelines_config USING btree (pipeline_id); diff --git a/doc/administration/audit_event_streaming.md b/doc/administration/audit_event_streaming.md index 2831996106f..d6a6725210d 100644 --- a/doc/administration/audit_event_streaming.md +++ b/doc/administration/audit_event_streaming.md @@ -1,1226 +1,11 @@ --- -stage: Govern -group: Compliance -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: 'audit_event_streaming/index.md' +remove_date: '2023-09-12' --- -# Audit event streaming **(ULTIMATE)** +This document was moved to [another location](audit_event_streaming/index.md). -> - API [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.5 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events_namespace`. Disabled by default. -> - API [Enabled on GitLab.com and by default on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/338939) in GitLab 14.7. -> - API [Feature flag `ff_external_audit_events_namespace`](https://gitlab.com/gitlab-org/gitlab/-/issues/349588) removed in GitLab 14.8. -> - UI [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336411) in GitLab 14.9. -> - [Subgroup events recording](https://gitlab.com/gitlab-org/gitlab/-/issues/366878) fixed in GitLab 15.2. -> - Custom HTTP headers API [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361216) in GitLab 15.1 [with a flag](feature_flags.md) named `streaming_audit_event_headers`. Disabled by default. -> - Custom HTTP headers API [enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/362941) in GitLab 15.2. -> - Custom HTTP headers API [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/366524) in GitLab 15.3. [Feature flag `streaming_audit_event_headers`](https://gitlab.com/gitlab-org/gitlab/-/issues/362941) removed. -> - Custom HTTP headers UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361630) in GitLab 15.2 [with a flag](feature_flags.md) named `custom_headers_streaming_audit_events_ui`. Disabled by default. -> - Custom HTTP headers UI [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) in GitLab 15.3. [Feature flag `custom_headers_streaming_audit_events_ui`](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) removed. -> - [Improved user experience](https://gitlab.com/gitlab-org/gitlab/-/issues/367963) in GitLab 15.3. -> - User-specified verification token API support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360813) in GitLab 15.4. -> - Event type filters API [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344845) in GitLab 15.7. -> - APIs for instance level streaming destinations [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335175) in GitLab 16.0 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events`. Disabled by default. -> - UI for instance level streaming destinations [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events`. Disabled by default. - -Users can set a streaming destination for a top-level group or instance to receive all audit events about the group, subgroups, and -projects as structured JSON. - -Top-level group owners and instance administrators can manage their audit logs in third-party systems. Any service that can receive -structured JSON data can be used as the streaming destination. - -Each streaming destination can have up to 20 custom HTTP headers included with each streamed event. - -NOTE: -GitLab can stream a single event more than once to the same destination. Use the `id` key in the payload to deduplicate incoming data. - -## Add a new streaming destination - -WARNING: -Streaming destinations receive **all** audit event data, which could include sensitive information. Make sure you trust the streaming destination. - -### Use the GitLab UI - -#### Top-level group streaming destinations - -Users with the Owner role for a group can add streaming destinations for it: - -1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. -1. Select **Secure > Audit events**. -1. On the main area, select **Streams** tab. -1. Select **Add streaming destination** to show the section for adding destinations. -1. Enter the destination URL to add. -1. Optional. Locate the **Custom HTTP headers** table. -1. Ignore the **Active** checkbox because it isn't functional. To track progress on adding functionality to the - **Active** checkbox, see [issue 367509](https://gitlab.com/gitlab-org/gitlab/-/issues/367509). -1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to - 20 headers per streaming destination. -1. After all headers have been filled out, select **Add** to add the new streaming destination. - -#### Instance streaming destinations - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named -`ff_external_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. - -Prerequisites: - -- Administrator access on the instance. - -To add a streaming destination for an instance: - -1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). -1. Select **Admin Area**. -1. On the left sidebar, select **Monitoring > Audit events**. -1. On the main area, select **Streams** tab. -1. Select **Add streaming destination** to show the section for adding destinations. -1. Enter the destination URL to add. -1. Select **Add** to add the new streaming destination. - -### Use the API - -#### Top-level group streaming destinations - -To enable streaming and add a destination, users with the Owner role for a group must use the -`externalAuditEventDestinationCreate` mutation in the GraphQL API. - -```graphql -mutation { - externalAuditEventDestinationCreate(input: { destinationUrl: "https://mydomain.io/endpoint/ingest", groupPath: "my-group" } ) { - errors - externalAuditEventDestination { - id - destinationUrl - verificationToken - group { - name - } - } - } -} -``` - -Group owners can also optionally specify their own verification token (instead of the default GitLab-generated one) using the GraphQL `auditEventsStreamingHeadersCreate` -mutation. Verification token length must be within 16 to 24 characters and trailing whitespace are not trimmed. GitLab recommends setting a cryptographically random and unique value. For example: - -```graphql -mutation { - externalAuditEventDestinationCreate(input: { destinationUrl: "https://mydomain.io/endpoint/ingest", groupPath: "my-group", verificationToken: "unique-random-verification-token-here" } ) { - errors - externalAuditEventDestination { - id - destinationUrl - verificationToken - group { - name - } - } - } -} -``` - -Event streaming is enabled if: - -- The returned `errors` object is empty. -- The API responds with `200 OK`. - -Group owners can add an HTTP header using the GraphQL `auditEventsStreamingHeadersCreate` mutation. You can retrieve the destination ID -by [listing all the streaming destinations](#use-the-api-1) for the group or from the mutation above. - -```graphql -mutation { - auditEventsStreamingHeadersCreate(input: { destinationId: "gid://gitlab/AuditEvents::ExternalAuditEventDestination/24601", key: "foo", value: "bar" }) { - errors - } -} -``` - -The header is created if the returned `errors` object is empty. - -#### Instance streaming destinations - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335175) in GitLab 16.0 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named -`ff_external_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. - -Prerequisites: - -- Administrator access on the instance. - -To enable streaming and add a destination, use the -`instanceExternalAuditEventDestinationCreate` mutation in the GraphQL API. - -```graphql -mutation { - instanceExternalAuditEventDestinationCreate(input: { destinationUrl: "https://mydomain.io/endpoint/ingest"}) { - errors - instanceExternalAuditEventDestination { - destinationUrl - id - verificationToken - } - } -} -``` - -Event streaming is enabled if: - -- The returned `errors` object is empty. -- The API responds with `200 OK`. - -## List streaming destinations - -Users with the Owner role for a group can list streaming destinations. - -### Use the GitLab UI - -#### For top-level group streaming destinations - -To list the streaming destinations for a top-level group: - -1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. -1. Select **Secure > Audit events**. -1. On the main area, select **Streams** tab. -1. To the right of the item, select **Edit** (**{pencil}**) to see all the custom HTTP headers. - -#### For instance streaming destinations - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named -`ff_external_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. - -Prerequisites: - -- Administrator access on the instance. - -To list the streaming destinations for an instance: - -1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). -1. Select **Admin Area**. -1. On the left sidebar, select **Monitoring > Audit events**. -1. On the main area, select **Streams** tab. - -### Use the API - -#### Top-level group streaming destinations - -Users with the Owner role for a group can view a list of streaming destinations at any time using the -`externalAuditEventDestinations` query type. - -```graphql -query { - group(fullPath: "my-group") { - id - externalAuditEventDestinations { - nodes { - destinationUrl - verificationToken - id - headers { - nodes { - key - value - id - } - } - eventTypeFilters - } - } - } -} -``` - -If the resulting list is empty, then audit streaming is not enabled for that group. - -#### Instance streaming destinations - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335175) in GitLab 16.0 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named -`ff_external_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. - -Prerequisites: - -- Administrator access on the instance. - -To view a list of streaming destinations for an instance, use the -`instanceExternalAuditEventDestinations` query type. - -```graphql -query { - instanceExternalAuditEventDestinations { - nodes { - id - destinationUrl - verificationToken - } - } -} -``` - -If the resulting list is empty, then audit streaming is not enabled for the instance. - -You need the ID values returned by this query for the update and delete mutations. - -## Update streaming destinations - -Users with the Owner role for a group and instance administrators can update streaming destinations for the group and the instance respectively. - -### Use the GitLab UI - -#### Top-level group streaming destinations - -To update a streaming destinations custom HTTP headers: - -1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. -1. Select **Secure > Audit events**. -1. On the main area, select **Streams** tab. -1. To the right of the item, select **Edit** (**{pencil}**). -1. Locate the **Custom HTTP headers** table. -1. Locate the header that you wish to update. -1. Ignore the **Active** checkbox because it isn't functional. To track progress on adding functionality to the - **Active** checkbox, see [issue 367509](https://gitlab.com/gitlab-org/gitlab/-/issues/367509). -1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to - 20 headers per streaming destination. -1. Select **Save** to update the streaming destination. - -### Use the API - -#### Top-level group streaming destinations - -Users with the Owner role for a group can update streaming destinations custom HTTP headers using the -`auditEventsStreamingHeadersUpdate` mutation type. You can retrieve the custom HTTP headers ID -by [listing all the custom HTTP headers](#use-the-api-1) for the group. - -```graphql -mutation { - externalAuditEventDestinationDestroy(input: { id: destination }) { - errors - } -} -``` - -Streaming destination is updated if: - -- The returned `errors` object is empty. -- The API responds with `200 OK`. - -Group owners can remove an HTTP header using the GraphQL `auditEventsStreamingHeadersDestroy` mutation. You can retrieve the header ID -by [listing all the custom HTTP headers](#use-the-api-1) for the group. - -```graphql -mutation { - auditEventsStreamingHeadersDestroy(input: { headerId: "gid://gitlab/AuditEvents::Streaming::Header/1" }) { - errors - } -} -``` - -The header is deleted if the returned `errors` object is empty. - -#### Instance streaming destinations - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335175) in GitLab 16.0 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named -`ff_external_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. - -Prerequisites: - -- Administrator access on the instance. - -To update streaming destinations for an instance, use the -`instanceExternalAuditEventDestinationUpdate` mutation type. You can retrieve the destination ID -by [listing all the external destinations](#use-the-api-1) for the instance. - -```graphql -mutation { - instanceExternalAuditEventDestinationUpdate(input: { id: "gid://gitlab/AuditEvents::InstanceExternalAuditEventDestination/1", destinationUrl: "https://www.new-domain.com/webhook"}) { - errors - instanceExternalAuditEventDestination { - destinationUrl - id - verificationToken - } - } -} -``` - -Streaming destination is updated if: - -- The returned `errors` object is empty. -- The API responds with `200 OK`. - -## Delete streaming destinations - -Users with the Owner role for a group and instance administrators can delete streaming destinations. - -When the last destination is successfully deleted, streaming is disabled for the group or the instance. - -### Use the GitLab UI - -#### Top-level group streaming destinations - -To delete a streaming destination: - -1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. To the right of the item, select **Delete** (**{remove}**). - -To delete only the custom HTTP headers for a streaming destination: - -1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. To the right of the item, **Edit** (**{pencil}**). -1. Locate the **Custom HTTP headers** table. -1. Locate the header that you wish to remove. -1. To the right of the header, select **Delete** (**{remove}**). -1. Select **Save** to update the streaming destination. - -#### Instance streaming destinations - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named -`ff_external_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. - -Prerequisites: - -- Administrator access on the instance. - -To delete the streaming destinations for an instance: - -1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). -1. Select **Admin Area**. -1. On the left sidebar, select **Monitoring > Audit events**. -1. On the main area, select the **Streams** tab. -1. To the right of the item, select **Delete** (**{remove}**). - -### Use the API - -#### Top-level group streaming destinations - -Users with the Owner role for a group can delete streaming destinations using the -`externalAuditEventDestinationDestroy` mutation type. You can retrieve the destinations ID -by [listing all the streaming destinations](#use-the-api-1) for the group. - -```graphql -mutation { - externalAuditEventDestinationDestroy(input: { id: destination }) { - errors - } -} -``` - -Streaming destination is deleted if: - -- The returned `errors` object is empty. -- The API responds with `200 OK`. - -Group owners can remove an HTTP header using the GraphQL `auditEventsStreamingHeadersDestroy` mutation. You can retrieve the header ID -by [listing all the custom HTTP headers](#use-the-api-1) for the group. - -```graphql -mutation { - auditEventsStreamingHeadersDestroy(input: { headerId: "gid://gitlab/AuditEvents::Streaming::Header/1" }) { - errors - } -} -``` - -The header is deleted if the returned `errors` object is empty. - -#### Instance streaming destinations - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335175) in GitLab 16.0 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named -`ff_external_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. - -Prerequisites: - -- Administrator access on the instance. - -To deleted streaming destinations, use the -`instanceExternalAuditEventDestinationDestroy` mutation type. You can retrieve the destinations ID -by [listing all the streaming destinations](#use-the-api-1) for the instance. - -```graphql -mutation { - instanceExternalAuditEventDestinationDestroy(input: { id: "gid://gitlab/AuditEvents::InstanceExternalAuditEventDestination/1" }) { - errors - } -} -``` - -Streaming destination is deleted if: - -- The returned `errors` object is empty. -- The API responds with `200 OK`. - -## Verify event authenticity - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345424) in GitLab 14.8. - -Each streaming destination has a unique verification token (`verificationToken`) that can be used to verify the authenticity of the event. This -token is either specified by the Owner or generated automatically when the event destination is created and cannot be changed. - -Each streamed event contains the verification token in the `X-Gitlab-Event-Streaming-Token` HTTP header that can be verified against -the destination's value when [listing streaming destinations](#list-streaming-destinations). - -### Use the GitLab UI - -#### Top-level group streaming destinations - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360814) in GitLab 15.2. - -Users with the Owner role for a group can list streaming destinations and see the verification tokens: - -1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams**. -1. View the verification token on the right side of each item. - -#### Instance streaming destinations - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named -`ff_external_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. - -Prerequisites: - -- Administrator access on the instance. - -To list streaming destinations for an instance and see the verification tokens: - -1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). -1. Select **Admin Area**. -1. On the left sidebar, select **Monitoring > Audit events**. -1. On the main area, select the **Streams**. -1. View the verification token on the right side of each item. - -## Override default content type header - -By default, streaming destinations use a `content-type` header of `application/x-www-form-urlencoded`. However, you might want to set the `content-type` header to something else. -For example ,`application/json`. - -To override the `content-type` header default value for either a top-level group streaming destination or an instance streaming destination, use either the -[GitLab UI](#use-the-gitlab-ui-2) or using the [GraphQL API](#use-the-api-2). - -## Event type filters - -> - Event type filters API [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344845) in GitLab 15.7. -> - Event type filtering in the UI with a defined list of audit event types [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/413581) in GitLab 16.1. - -When this feature is enabled for a group, you can use the GitLab UI or an API to permit users to filter streamed audit events per destination. -If the feature is enabled with no filters, the destination receives all audit events. - -A streaming destination that has an event type filter set has a **filtered** (**{filter}**) label. - -### Use the GitLab UI - -To update a streaming destination's event filters: - -1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. -1. Select **Secure > Audit events**. -1. On the main area, select the **Streams** tab. -1. To the right of the item, select **Edit** (**{pencil}**). -1. Select **Filter by stream event**. -1. Select the dropdown list and select or unselect the required event types. -1. Select **Save** to update the event filters. - -### Use the API to add an event type filter - -Prerequisites: - -- You must have the Owner role for the group. - -You can add a list of event type filters using the `auditEventsStreamingDestinationEventsAdd` query type: - -```graphql -mutation { - auditEventsStreamingDestinationEventsAdd(input: { - destinationId: "gid://gitlab/AuditEvents::ExternalAuditEventDestination/1", - eventTypeFilters: ["list of event type filters"]}){ - errors - eventTypeFilters - } -} -``` - -Event type filters are added if: - -- The returned `errors` object is empty. -- The API responds with `200 OK`. - -### Use the API to remove an event type filter - -Prerequisites: - -- You must have the Owner role for the group. - -You can remove a list of event type filters using the `auditEventsStreamingDestinationEventsRemove` query type: - -```graphql -mutation { - auditEventsStreamingDestinationEventsRemove(input: { - destinationId: "gid://gitlab/AuditEvents::ExternalAuditEventDestination/1", - eventTypeFilters: ["list of event type filters"] - }){ - errors - } -} -``` - -Event type filters are removed if: - -- The returned `errors` object is empty. -- The API responds with `200 OK`. - -## Payload schema - -> Documentation for an audit event streaming schema was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358149) in GitLab 15.3. - -Streamed audit events have a predictable schema in the body of the response. - -| Field | Description | Notes | -|------------------|------------------------------------------------------------|-----------------------------------------------------------------------------------| -| `author_id` | User ID of the user who triggered the event | | -| `author_name` | Human-readable name of the author that triggered the event | Helpful when the author no longer exists | -| `created_at` | Timestamp when event was triggered | | -| `details` | JSON object containing additional metadata | Has no defined schema but often contains additional information about an event | -| `entity_id` | ID of the audit event's entity | | -| `entity_path` | Full path of the entity affected by the auditable event | | -| `entity_type` | String representation of the type of entity | Acceptable values include `User`, `Group`, and `Key`. This list is not exhaustive | -| `event_type` | String representation of the type of audit event | | -| `id` | Unique identifier for the audit event | Can be used for deduplication if required | -| `ip_address` | IP address of the host used to trigger the event | | -| `target_details` | Additional details about the target | | -| `target_id` | ID of the audit event's target | | -| `target_type` | String representation of the target's type | | - -### JSON payload schema - -```json -{ - "properties": { - "id": { - "type": "string" - }, - "author_id": { - "type": "integer" - }, - "author_name": { - "type": "string" - }, - "details": {}, - "ip_address": { - "type": "string" - }, - "entity_id": { - "type": "integer" - }, - "entity_path": { - "type": "string" - }, - "entity_type": { - "type": "string" - }, - "event_type": { - "type": "string" - }, - "target_id": { - "type": "integer" - }, - "target_type": { - "type": "string" - }, - "target_details": { - "type": "string" - }, - }, - "type": "object" -} -``` - -## Audit event streaming on Git operations - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `audit_event_streaming_git_operations`. Disabled by default. -> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/357211) in GitLab 15.0. -> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/357211) in GitLab 15.1 by default. -> - [Added `details.author_class` field](https://gitlab.com/gitlab-org/gitlab/-/issues/363876) in GitLab 15.3. -> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101583) in GitLab 15.6. Feature flag `audit_event_streaming_git_operations` removed. - -Streaming audit events can be sent when authenticated users push, pull, or clone a project's remote Git repositories: - -- [Using SSH](../user/ssh.md). -- Using HTTP or HTTPS. -- Using the **Download** button (**{download}**) in GitLab UI. - -Audit events are not captured for users that are not signed in. For example, when downloading a public project. - -To configure streaming audit events for Git operations, see [Add a new streaming destination](#add-a-new-streaming-destination). - -### Headers - -> `X-Gitlab-Audit-Event-Type` [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86881) in GitLab 15.0. - -Headers are formatted as follows: - -```plaintext -POST /logs HTTP/1.1 -Host: <DESTINATION_HOST> -Content-Type: application/x-www-form-urlencoded -X-Gitlab-Event-Streaming-Token: <DESTINATION_TOKEN> -X-Gitlab-Audit-Event-Type: repository_git_operation -``` - -### Example payloads for SSH events - -Fetch: - -```json -{ - "id": 1, - "author_id": 1, - "entity_id": 29, - "entity_type": "Project", - "details": { - "author_name": "Administrator", - "author_class": "User", - "target_id": 29, - "target_type": "Project", - "target_details": "example-project", - "custom_message": { - "protocol": "ssh", - "action": "git-upload-pack" - }, - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "Administrator", - "entity_path": "example-group/example-project", - "target_details": "example-project", - "created_at": "2022-02-23T06:21:05.283Z", - "target_type": "Project", - "target_id": 29, - "event_type": "repository_git_operation" -} -``` - -Push: - -```json -{ - "id": 1, - "author_id": 1, - "entity_id": 29, - "entity_type": "Project", - "details": { - "author_name": "Administrator", - "author_class": "User", - "target_id": 29, - "target_type": "Project", - "target_details": "example-project", - "custom_message": { - "protocol": "ssh", - "action": "git-receive-pack" - }, - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "Administrator", - "entity_path": "example-group/example-project", - "target_details": "example-project", - "created_at": "2022-02-23T06:23:08.746Z", - "target_type": "Project", - "target_id": 29, - "event_type": "repository_git_operation" -} -``` - -### Example payloads for SSH events with Deploy Key - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363876) in GitLab 15.3. - -Fetch: - -```json -{ - "id": 1, - "author_id": -3, - "entity_id": 29, - "entity_type": "Project", - "details": { - "author_name": "deploy-key-name", - "author_class": "DeployKey", - "target_id": 29, - "target_type": "Project", - "target_details": "example-project", - "custom_message": { - "protocol": "ssh", - "action": "git-upload-pack" - }, - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "deploy-key-name", - "entity_path": "example-group/example-project", - "target_details": "example-project", - "created_at": "2022-07-26T05:43:53.662Z", - "target_type": "Project", - "target_id": 29, - "event_type": "repository_git_operation" -} -``` - -### Example payloads for HTTP and HTTPS events - -Fetch: - -```json -{ - "id": 1, - "author_id": 1, - "entity_id": 29, - "entity_type": "Project", - "details": { - "author_name": "Administrator", - "author_class": "User", - "target_id": 29, - "target_type": "Project", - "target_details": "example-project", - "custom_message": { - "protocol": "http", - "action": "git-upload-pack" - }, - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "Administrator", - "entity_path": "example-group/example-project", - "target_details": "example-project", - "created_at": "2022-02-23T06:25:43.938Z", - "target_type": "Project", - "target_id": 29, - "event_type": "repository_git_operation" -} -``` - -Push: - -```json -{ - "id": 1, - "author_id": 1, - "entity_id": 29, - "entity_type": "Project", - "details": { - "author_name": "Administrator", - "author_class": "User", - "target_id": 29, - "target_type": "Project", - "target_details": "example-project", - "custom_message": { - "protocol": "http", - "action": "git-receive-pack" - }, - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "Administrator", - "entity_path": "example-group/example-project", - "target_details": "example-project", - "created_at": "2022-02-23T06:26:29.294Z", - "target_type": "Project", - "target_id": 29, - "event_type": "repository_git_operation" -} -``` - -### Example payloads for HTTP and HTTPS events with Deploy Token - -Fetch: - -```json -{ - "id": 1, - "author_id": -2, - "entity_id": 22, - "entity_type": "Project", - "details": { - "author_name": "deploy-token-name", - "author_class": "DeployToken", - "target_id": 22, - "target_type": "Project", - "target_details": "example-project", - "custom_message": { - "protocol": "http", - "action": "git-upload-pack" - }, - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "deploy-token-name", - "entity_path": "example-group/example-project", - "target_details": "example-project", - "created_at": "2022-07-26T05:46:25.850Z", - "target_type": "Project", - "target_id": 22, - "event_type": "repository_git_operation" -} -``` - -### Example payloads for events from GitLab UI download button - -Fetch: - -```json -{ - "id": 1, - "author_id": 99, - "entity_id": 29, - "entity_type": "Project", - "details": { - "custom_message": "Repository Download Started", - "author_name": "example_username", - "author_class": "User", - "target_id": 29, - "target_type": "Project", - "target_details": "example-group/example-project", - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "example_username", - "entity_path": "example-group/example-project", - "target_details": "example-group/example-project", - "created_at": "2022-02-23T06:27:17.873Z", - "target_type": "Project", - "target_id": 29, - "event_type": "repository_git_operation" -} -``` - -## Audit event streaming on merge request approval actions - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/271162) in GitLab 14.9. - -Stream audit events that relate to merge approval actions performed within a project. - -### Headers - -Headers are formatted as follows: - -```plaintext -POST /logs HTTP/1.1 -Host: <DESTINATION_HOST> -Content-Type: application/x-www-form-urlencoded -X-Gitlab-Event-Streaming-Token: <DESTINATION_TOKEN> -X-Gitlab-Audit-Event-Type: audit_operation -``` - -### Example payload - -```json -{ - "id": 1, - "author_id": 1, - "entity_id": 6, - "entity_type": "Project", - "details": { - "author_name": "example_username", - "target_id": 20, - "target_type": "MergeRequest", - "target_details": "merge request title", - "custom_message": "Approved merge request", - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "example_username", - "entity_path": "example-group/example-project", - "target_details": "merge request title", - "created_at": "2022-03-09T06:53:11.181Z", - "target_type": "MergeRequest", - "target_id": 20, - "event_type": "audit_operation" -} -``` - -## Audit event streaming on merge request create actions - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90911) in GitLab 15.2. - -Stream audit events that relate to merge request create actions using the `/logs` endpoint. - -Send API requests that contain the `X-Gitlab-Audit-Event-Type` header with value `merge_request_create`. GitLab responds with JSON payloads with an -`event_type` field set to `merge_request_create`. - -### Headers - -Headers are formatted as follows: - -```plaintext -POST /logs HTTP/1.1 -Host: <DESTINATION_HOST> -Content-Type: application/x-www-form-urlencoded -X-Gitlab-Audit-Event-Type: merge_request_create -X-Gitlab-Event-Streaming-Token: <DESTINATION_TOKEN> -``` - -### Example payload - -```json -{ - "id": 1, - "author_id": 1, - "entity_id": 24, - "entity_type": "Project", - "details": { - "author_name": "example_user", - "target_id": 132, - "target_type": "MergeRequest", - "target_details": "Update test.md", - "custom_message": "Added merge request", - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "Administrator", - "entity_path": "example-group/example-project", - "target_details": "Update test.md", - "created_at": "2022-07-04T00:19:22.675Z", - "target_type": "MergeRequest", - "target_id": 132, - "event_type": "merge_request_create" -} -``` - -## Audit event streaming on project fork actions - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90916) in GitLab 15.2. - -Stream audit events that relate to project fork actions using the `/logs` endpoint. - -Send API requests that contain the `X-Gitlab-Audit-Event-Type` header with value `project_fork_operation`. GitLab responds with JSON payloads with an -`event_type` field set to `project_fork_operation`. - -### Headers - -Headers are formatted as follows: - -```plaintext -POST /logs HTTP/1.1 -Host: <DESTINATION_HOST> -Content-Type: application/x-www-form-urlencoded -X-Gitlab-Audit-Event-Type: project_fork_operation -X-Gitlab-Event-Streaming-Token: <DESTINATION_TOKEN> -``` - -### Example payload - -```json -{ - "id": 1, - "author_id": 1, - "entity_id": 24, - "entity_type": "Project", - "details": { - "author_name": "example_username", - "target_id": 24, - "target_type": "Project", - "target_details": "example-project", - "custom_message": "Forked project to another-group/example-project-forked", - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "example_username", - "entity_path": "example-group/example-project", - "target_details": "example-project", - "created_at": "2022-06-30T03:43:35.384Z", - "target_type": "Project", - "target_id": 24, - "event_type": "project_fork_operation" -} -``` - -## Audit event streaming on project group link actions - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90955) in GitLab 15.2. - -Stream audit events that relate to project group link creation, updates, and deletion using the `/logs` endpoint. - -Send API requests that contain the `X-Gitlab-Audit-Event-Type` header with value of either: - -- `project_group_link_create`. -- `project_group_link_update`. -- `project_group_link_destroy`. - -GitLab responds with JSON payloads with an `event_type` field set to either: - -- `project_group_link_create`. -- `project_group_link_update`. -- `project_group_link_destroy`. - -### Example Headers - -Headers are formatted as follows: - -```plaintext -POST /logs HTTP/1.1 -Host: <DESTINATION_HOST> -Content-Type: application/x-www-form-urlencoded -X-Gitlab-Audit-Event-Type: project_group_link_create -X-Gitlab-Event-Streaming-Token: <DESTINATION_TOKEN> -``` - -### Example payload for project group link create - -```json -{ - "id": 1, - "author_id": 1, - "entity_id": 24, - "entity_type": "Project", - "details": { - "author_name": "example-user", - "target_id": 31, - "target_type": "Group", - "target_details": "another-group", - "custom_message": "Added project group link", - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "example-user", - "entity_path": "example-group/example-project", - "target_details": "another-group", - "created_at": "2022-07-04T00:43:09.318Z", - "target_type": "Group", - "target_id": 31, - "event_type": "project_group_link_create" -} -``` - -### Example payload for project group link update - -```json -{ - "id": 1, - "author_id": 1, - "entity_id": 24, - "entity_type": "Project", - "details": { - "author_name": "example-user", - "target_id": 31, - "target_type": "Group", - "target_details": "another-group", - "custom_message": "Changed project group link profile group_access from Developer to Guest", - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "example-user", - "entity_path": "example-group/example-project", - "target_details": "another-group", - "created_at": "2022-07-04T00:43:28.328Z", - "target_type": "Group", - "target_id": 31, - "event_type": "project_group_link_update" -} -``` - -### Example payload for project group link delete - -```json -{ - "id": 1, - "author_id": 1, - "entity_id": 24, - "entity_type": "Project", - "details": { - "author_name": "example-user", - "target_id": 31, - "target_type": "Group", - "target_details": "another-group", - "custom_message": "Removed project group link", - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "example-user", - "entity_path": "example-group/example-project", - "target_details": "another-group", - "created_at": "2022-07-04T00:42:56.279Z", - "target_type": "Group", - "target_id": 31, - "event_type": "project_group_link_destroy" -} -``` - -## Audit event streaming on invalid merge request approver state - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/374566) in GitLab 15.5. - -Stream audit events that relate to invalid merge request approver states within a project. - -### Headers - -Headers are formatted as follows: - -```plaintext -POST /logs HTTP/1.1 -Host: <DESTINATION_HOST> -Content-Type: application/x-www-form-urlencoded -X-Gitlab-Event-Streaming-Token: <DESTINATION_TOKEN> -X-Gitlab-Audit-Event-Type: audit_operation -``` - -### Example payload - -```json -{ - "id": 1, - "author_id": 1, - "entity_id": 6, - "entity_type": "Project", - "details": { - "author_name": "example_username", - "target_id": 20, - "target_type": "MergeRequest", - "target_details": { title: "Merge request title", iid: "Merge request iid", id: "Merge request id" }, - "custom_message": "Invalid merge request approver rules", - "ip_address": "127.0.0.1", - "entity_path": "example-group/example-project" - }, - "ip_address": "127.0.0.1", - "author_name": "example_username", - "entity_path": "example-group/example-project", - "target_details": "merge request title", - "created_at": "2022-03-09T06:53:11.181Z", - "target_type": "MergeRequest", - "target_id": 20, - "event_type": "audit_operation" -} -``` +<!-- This redirect file can be deleted after <2023-09-12>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/audit_event_streaming/examples.md b/doc/administration/audit_event_streaming/examples.md new file mode 100644 index 00000000000..d741e5fd60d --- /dev/null +++ b/doc/administration/audit_event_streaming/examples.md @@ -0,0 +1,578 @@ +--- +stage: Govern +group: Compliance +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Audit event streaming examples + +The following sections provide examples of audit event streaming. + +## Audit event streaming on Git operations + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.9 [with a flag](../feature_flags.md) named `audit_event_streaming_git_operations`. Disabled by default. +> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/357211) in GitLab 15.0. +> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/357211) in GitLab 15.1 by default. +> - [Added `details.author_class` field](https://gitlab.com/gitlab-org/gitlab/-/issues/363876) in GitLab 15.3. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101583) in GitLab 15.6. Feature flag `audit_event_streaming_git_operations` removed. + +Streaming audit events can be sent when authenticated users push, pull, or clone a project's remote Git repositories: + +- [Using SSH](../../user/ssh.md). +- Using HTTP or HTTPS. +- Using **Download** (**{download}**) in GitLab UI. + +Audit events are not captured for users that are not signed in. For example, when downloading a public project. + +To configure streaming audit events for Git operations, see [Add a new streaming destination](index.md#add-a-new-streaming-destination). + +### Headers + +> `X-Gitlab-Audit-Event-Type` [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86881) in GitLab 15.0. + +Headers are formatted as follows: + +```plaintext +POST /logs HTTP/1.1 +Host: <DESTINATION_HOST> +Content-Type: application/x-www-form-urlencoded +X-Gitlab-Event-Streaming-Token: <DESTINATION_TOKEN> +X-Gitlab-Audit-Event-Type: repository_git_operation +``` + +### Example payloads for SSH events + +Fetch: + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 29, + "entity_type": "Project", + "details": { + "author_name": "Administrator", + "author_class": "User", + "target_id": 29, + "target_type": "Project", + "target_details": "example-project", + "custom_message": { + "protocol": "ssh", + "action": "git-upload-pack" + }, + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "Administrator", + "entity_path": "example-group/example-project", + "target_details": "example-project", + "created_at": "2022-02-23T06:21:05.283Z", + "target_type": "Project", + "target_id": 29, + "event_type": "repository_git_operation" +} +``` + +Push: + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 29, + "entity_type": "Project", + "details": { + "author_name": "Administrator", + "author_class": "User", + "target_id": 29, + "target_type": "Project", + "target_details": "example-project", + "custom_message": { + "protocol": "ssh", + "action": "git-receive-pack" + }, + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "Administrator", + "entity_path": "example-group/example-project", + "target_details": "example-project", + "created_at": "2022-02-23T06:23:08.746Z", + "target_type": "Project", + "target_id": 29, + "event_type": "repository_git_operation" +} +``` + +### Example payloads for SSH events with Deploy Key + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363876) in GitLab 15.3. + +Fetch: + +```json +{ + "id": 1, + "author_id": -3, + "entity_id": 29, + "entity_type": "Project", + "details": { + "author_name": "deploy-key-name", + "author_class": "DeployKey", + "target_id": 29, + "target_type": "Project", + "target_details": "example-project", + "custom_message": { + "protocol": "ssh", + "action": "git-upload-pack" + }, + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "deploy-key-name", + "entity_path": "example-group/example-project", + "target_details": "example-project", + "created_at": "2022-07-26T05:43:53.662Z", + "target_type": "Project", + "target_id": 29, + "event_type": "repository_git_operation" +} +``` + +### Example payloads for HTTP and HTTPS events + +Fetch: + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 29, + "entity_type": "Project", + "details": { + "author_name": "Administrator", + "author_class": "User", + "target_id": 29, + "target_type": "Project", + "target_details": "example-project", + "custom_message": { + "protocol": "http", + "action": "git-upload-pack" + }, + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "Administrator", + "entity_path": "example-group/example-project", + "target_details": "example-project", + "created_at": "2022-02-23T06:25:43.938Z", + "target_type": "Project", + "target_id": 29, + "event_type": "repository_git_operation" +} +``` + +Push: + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 29, + "entity_type": "Project", + "details": { + "author_name": "Administrator", + "author_class": "User", + "target_id": 29, + "target_type": "Project", + "target_details": "example-project", + "custom_message": { + "protocol": "http", + "action": "git-receive-pack" + }, + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "Administrator", + "entity_path": "example-group/example-project", + "target_details": "example-project", + "created_at": "2022-02-23T06:26:29.294Z", + "target_type": "Project", + "target_id": 29, + "event_type": "repository_git_operation" +} +``` + +### Example payloads for HTTP and HTTPS events with Deploy Token + +Fetch: + +```json +{ + "id": 1, + "author_id": -2, + "entity_id": 22, + "entity_type": "Project", + "details": { + "author_name": "deploy-token-name", + "author_class": "DeployToken", + "target_id": 22, + "target_type": "Project", + "target_details": "example-project", + "custom_message": { + "protocol": "http", + "action": "git-upload-pack" + }, + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "deploy-token-name", + "entity_path": "example-group/example-project", + "target_details": "example-project", + "created_at": "2022-07-26T05:46:25.850Z", + "target_type": "Project", + "target_id": 22, + "event_type": "repository_git_operation" +} +``` + +### Example payloads for events from GitLab UI download button + +Fetch: + +```json +{ + "id": 1, + "author_id": 99, + "entity_id": 29, + "entity_type": "Project", + "details": { + "custom_message": "Repository Download Started", + "author_name": "example_username", + "author_class": "User", + "target_id": 29, + "target_type": "Project", + "target_details": "example-group/example-project", + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "example_username", + "entity_path": "example-group/example-project", + "target_details": "example-group/example-project", + "created_at": "2022-02-23T06:27:17.873Z", + "target_type": "Project", + "target_id": 29, + "event_type": "repository_git_operation" +} +``` + +## Audit event streaming on merge request approval actions + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/271162) in GitLab 14.9. + +Stream audit events that relate to merge approval actions performed in a project. + +### Headers + +Headers are formatted as follows: + +```plaintext +POST /logs HTTP/1.1 +Host: <DESTINATION_HOST> +Content-Type: application/x-www-form-urlencoded +X-Gitlab-Event-Streaming-Token: <DESTINATION_TOKEN> +X-Gitlab-Audit-Event-Type: audit_operation +``` + +### Example payload + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 6, + "entity_type": "Project", + "details": { + "author_name": "example_username", + "target_id": 20, + "target_type": "MergeRequest", + "target_details": "merge request title", + "custom_message": "Approved merge request", + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "example_username", + "entity_path": "example-group/example-project", + "target_details": "merge request title", + "created_at": "2022-03-09T06:53:11.181Z", + "target_type": "MergeRequest", + "target_id": 20, + "event_type": "audit_operation" +} +``` + +## Audit event streaming on merge request create actions + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90911) in GitLab 15.2. + +Stream audit events that relate to merge request create actions using the `/logs` endpoint. + +Send API requests that contain the `X-Gitlab-Audit-Event-Type` header with value `merge_request_create`. GitLab responds with JSON payloads with an +`event_type` field set to `merge_request_create`. + +### Headers + +Headers are formatted as follows: + +```plaintext +POST /logs HTTP/1.1 +Host: <DESTINATION_HOST> +Content-Type: application/x-www-form-urlencoded +X-Gitlab-Audit-Event-Type: merge_request_create +X-Gitlab-Event-Streaming-Token: <DESTINATION_TOKEN> +``` + +### Example payload + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 24, + "entity_type": "Project", + "details": { + "author_name": "example_user", + "target_id": 132, + "target_type": "MergeRequest", + "target_details": "Update test.md", + "custom_message": "Added merge request", + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "Administrator", + "entity_path": "example-group/example-project", + "target_details": "Update test.md", + "created_at": "2022-07-04T00:19:22.675Z", + "target_type": "MergeRequest", + "target_id": 132, + "event_type": "merge_request_create" +} +``` + +## Audit event streaming on project fork actions + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90916) in GitLab 15.2. + +Stream audit events that relate to project fork actions using the `/logs` endpoint. + +Send API requests that contain the `X-Gitlab-Audit-Event-Type` header with value `project_fork_operation`. GitLab responds with JSON payloads with an +`event_type` field set to `project_fork_operation`. + +### Headers + +Headers are formatted as follows: + +```plaintext +POST /logs HTTP/1.1 +Host: <DESTINATION_HOST> +Content-Type: application/x-www-form-urlencoded +X-Gitlab-Audit-Event-Type: project_fork_operation +X-Gitlab-Event-Streaming-Token: <DESTINATION_TOKEN> +``` + +### Example payload + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 24, + "entity_type": "Project", + "details": { + "author_name": "example_username", + "target_id": 24, + "target_type": "Project", + "target_details": "example-project", + "custom_message": "Forked project to another-group/example-project-forked", + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "example_username", + "entity_path": "example-group/example-project", + "target_details": "example-project", + "created_at": "2022-06-30T03:43:35.384Z", + "target_type": "Project", + "target_id": 24, + "event_type": "project_fork_operation" +} +``` + +## Audit event streaming on project group link actions + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90955) in GitLab 15.2. + +Stream audit events that relate to project group link creation, updates, and deletion using the `/logs` endpoint. + +Send API requests that contain the `X-Gitlab-Audit-Event-Type` header with value of either: + +- `project_group_link_create`. +- `project_group_link_update`. +- `project_group_link_destroy`. + +GitLab responds with JSON payloads with an `event_type` field set to either: + +- `project_group_link_create`. +- `project_group_link_update`. +- `project_group_link_destroy`. + +### Example Headers + +Headers are formatted as follows: + +```plaintext +POST /logs HTTP/1.1 +Host: <DESTINATION_HOST> +Content-Type: application/x-www-form-urlencoded +X-Gitlab-Audit-Event-Type: project_group_link_create +X-Gitlab-Event-Streaming-Token: <DESTINATION_TOKEN> +``` + +### Example payload for project group link create + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 24, + "entity_type": "Project", + "details": { + "author_name": "example-user", + "target_id": 31, + "target_type": "Group", + "target_details": "another-group", + "custom_message": "Added project group link", + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "example-user", + "entity_path": "example-group/example-project", + "target_details": "another-group", + "created_at": "2022-07-04T00:43:09.318Z", + "target_type": "Group", + "target_id": 31, + "event_type": "project_group_link_create" +} +``` + +### Example payload for project group link update + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 24, + "entity_type": "Project", + "details": { + "author_name": "example-user", + "target_id": 31, + "target_type": "Group", + "target_details": "another-group", + "custom_message": "Changed project group link profile group_access from Developer to Guest", + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "example-user", + "entity_path": "example-group/example-project", + "target_details": "another-group", + "created_at": "2022-07-04T00:43:28.328Z", + "target_type": "Group", + "target_id": 31, + "event_type": "project_group_link_update" +} +``` + +### Example payload for project group link delete + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 24, + "entity_type": "Project", + "details": { + "author_name": "example-user", + "target_id": 31, + "target_type": "Group", + "target_details": "another-group", + "custom_message": "Removed project group link", + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "example-user", + "entity_path": "example-group/example-project", + "target_details": "another-group", + "created_at": "2022-07-04T00:42:56.279Z", + "target_type": "Group", + "target_id": 31, + "event_type": "project_group_link_destroy" +} +``` + +## Audit event streaming on invalid merge request approver state + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/374566) in GitLab 15.5. + +Stream audit events that relate to invalid merge request approver states in a project. + +### Headers + +Headers are formatted as follows: + +```plaintext +POST /logs HTTP/1.1 +Host: <DESTINATION_HOST> +Content-Type: application/x-www-form-urlencoded +X-Gitlab-Event-Streaming-Token: <DESTINATION_TOKEN> +X-Gitlab-Audit-Event-Type: audit_operation +``` + +### Example payload + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 6, + "entity_type": "Project", + "details": { + "author_name": "example_username", + "target_id": 20, + "target_type": "MergeRequest", + "target_details": { title: "Merge request title", iid: "Merge request iid", id: "Merge request id" }, + "custom_message": "Invalid merge request approver rules", + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "example_username", + "entity_path": "example-group/example-project", + "target_details": "merge request title", + "created_at": "2022-03-09T06:53:11.181Z", + "target_type": "MergeRequest", + "target_id": 20, + "event_type": "audit_operation" +} +``` diff --git a/doc/administration/audit_event_streaming/graphql_api.md b/doc/administration/audit_event_streaming/graphql_api.md new file mode 100644 index 00000000000..f5a31f073dc --- /dev/null +++ b/doc/administration/audit_event_streaming/graphql_api.md @@ -0,0 +1,442 @@ +--- +stage: Govern +group: Compliance +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Audit event streaming GraphQL API **(ULTIMATE)** + +> - API [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.5 [with a flag](../feature_flags.md) named `ff_external_audit_events_namespace`. Disabled by default. +> - API [Enabled on GitLab.com and by default on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/338939) in GitLab 14.7. +> - API [Feature flag `ff_external_audit_events_namespace`](https://gitlab.com/gitlab-org/gitlab/-/issues/349588) removed in GitLab 14.8. +> - Custom HTTP headers API [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361216) in GitLab 15.1 [with a flag](../feature_flags.md) named `streaming_audit_event_headers`. Disabled by default. +> - Custom HTTP headers API [enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/362941) in GitLab 15.2. +> - Custom HTTP headers API [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/366524) in GitLab 15.3. [Feature flag `streaming_audit_event_headers`](https://gitlab.com/gitlab-org/gitlab/-/issues/362941) removed. +> - User-specified verification token API support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360813) in GitLab 15.4. +> - APIs for custom HTTP headers for instance level streaming destinations [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/404560) in GitLab 16.1 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default. + +Audit event streaming destinations can be maintained using a GraphQL API. + +## Add a new streaming destination + +Add a new streaming destination to top-level groups or an entire instance. + +WARNING: +Streaming destinations receive **all** audit event data, which could include sensitive information. Make sure you trust the streaming destination. + +### Top-level group streaming destinations + +Prerequisites: + +- Owner role for a top-level group. + +To enable streaming and add a destination to a top-level group, use the `externalAuditEventDestinationCreate` mutation. + +```graphql +mutation { + externalAuditEventDestinationCreate(input: { destinationUrl: "https://mydomain.io/endpoint/ingest", groupPath: "my-group" } ) { + errors + externalAuditEventDestination { + id + destinationUrl + verificationToken + group { + name + } + } + } +} +``` + +You can optionally specify your own verification token (instead of the default GitLab-generated one) using the GraphQL +`externalAuditEventDestinationCreate` +mutation. Verification token length must be within 16 to 24 characters and trailing whitespace are not trimmed. You +should set a cryptographically random and unique value. For example: + +```graphql +mutation { + externalAuditEventDestinationCreate(input: { destinationUrl: "https://mydomain.io/endpoint/ingest", groupPath: "my-group", verificationToken: "unique-random-verification-token-here" } ) { + errors + externalAuditEventDestination { + id + destinationUrl + verificationToken + group { + name + } + } + } +} +``` + +Event streaming is enabled if: + +- The returned `errors` object is empty. +- The API responds with `200 OK`. + +You can add an HTTP header using the GraphQL `auditEventsStreamingHeadersCreate` mutation. You can retrieve the +destination ID by [listing all the streaming destinations](#list-streaming-destinations) for the group or from the +mutation above. + +```graphql +mutation { + auditEventsStreamingHeadersCreate(input: { destinationId: "gid://gitlab/AuditEvents::ExternalAuditEventDestination/24601", key: "foo", value: "bar" }) { + errors + } +} +``` + +The header is created if the returned `errors` object is empty. + +### Instance streaming destinations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335175) in GitLab 16.0 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../feature_flags.md) named +`ff_external_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. + +Prerequisites: + +- Administrator access on the instance. + +To enable streaming and add a destination, use the +`instanceExternalAuditEventDestinationCreate` mutation in the GraphQL API. + +```graphql +mutation { + instanceExternalAuditEventDestinationCreate(input: { destinationUrl: "https://mydomain.io/endpoint/ingest"}) { + errors + instanceExternalAuditEventDestination { + destinationUrl + id + verificationToken + } + } +} +``` + +Event streaming is enabled if: + +- The returned `errors` object is empty. +- The API responds with `200 OK`. + +Instance administrators can add an HTTP header using the GraphQL `auditEventsStreamingInstanceHeadersCreate` mutation. You can retrieve the destination ID +by [listing all the streaming destinations](#list-streaming-destinations) for the instance or from the mutation above. + +```graphql +mutation { + auditEventsStreamingInstanceHeadersCreate(input: + { + destinationId: "gid://gitlab/AuditEvents::InstanceExternalAuditEventDestination/42", + key: "foo", + value: "bar" + }) { + errors + header { + id + key + value + } + } +} +``` + +The header is created if the returned `errors` object is empty. + +## List streaming destinations + +List new streaming destinations for top-level groups or an entire instance. + +### Top-level group streaming destinations + +Prerequisites: + +- Owner role for a top-level group. + +You can view a list of streaming destinations for a top-level group using the `externalAuditEventDestinations` query +type. + +```graphql +query { + group(fullPath: "my-group") { + id + externalAuditEventDestinations { + nodes { + destinationUrl + verificationToken + id + headers { + nodes { + key + value + id + } + } + eventTypeFilters + } + } + } +} +``` + +If the resulting list is empty, then audit streaming is not enabled for that group. + +### Instance streaming destinations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335175) in GitLab 16.0 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../feature_flags.md) named +`ff_external_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. + +Prerequisites: + +- Administrator access on the instance. + +To view a list of streaming destinations for an instance, use the +`instanceExternalAuditEventDestinations` query type. + +```graphql +query { + instanceExternalAuditEventDestinations { + nodes { + id + destinationUrl + verificationToken + headers { + nodes { + id + key + value + } + } + } + } +} +``` + +If the resulting list is empty, then audit streaming is not enabled for the instance. + +You need the ID values returned by this query for the update and delete mutations. + +## Update streaming destinations + +Update streaming destinations for a top-level group or an entire instance. + +### Top-level group streaming destinations + +Prerequisites: + +- Owner role for a top-level group. + +Users with the Owner role for a group can update streaming destinations' custom HTTP headers using the +`auditEventsStreamingHeadersUpdate` mutation type. You can retrieve the custom HTTP headers ID +by [listing all the custom HTTP headers](#list-streaming-destinations) for the group. + +```graphql +mutation { + externalAuditEventDestinationDestroy(input: { id: destination }) { + errors + } +} +``` + +Streaming destination is updated if: + +- The returned `errors` object is empty. +- The API responds with `200 OK`. + +Group owners can remove an HTTP header using the GraphQL `auditEventsStreamingHeadersDestroy` mutation. You can retrieve the header ID +by [listing all the custom HTTP headers](#list-streaming-destinations) for the group. + +```graphql +mutation { + auditEventsStreamingHeadersDestroy(input: { headerId: "gid://gitlab/AuditEvents::Streaming::Header/1" }) { + errors + } +} +``` + +The header is deleted if the returned `errors` object is empty. + +### Instance streaming destinations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335175) in GitLab 16.0 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../feature_flags.md) named +`ff_external_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. + +Prerequisites: + +- Administrator access on the instance. + +To update streaming destinations for an instance, use the +`instanceExternalAuditEventDestinationUpdate` mutation type. You can retrieve the destination ID +by [listing all the external destinations](#list-streaming-destinations) for the instance. + +```graphql +mutation { + instanceExternalAuditEventDestinationUpdate(input: { id: "gid://gitlab/AuditEvents::InstanceExternalAuditEventDestination/1", destinationUrl: "https://www.new-domain.com/webhook"}) { + errors + instanceExternalAuditEventDestination { + destinationUrl + id + verificationToken + } + } +} +``` + +Streaming destination is updated if: + +- The returned `errors` object is empty. +- The API responds with `200 OK`. + +Instance administrators can update streaming destinations custom HTTP headers using the +`auditEventsStreamingInstanceHeadersUpdate` mutation type. You can retrieve the custom HTTP headers ID +by [listing all the custom HTTP headers](#list-streaming-destinations) for the instance. + +```graphql +mutation { + auditEventsStreamingInstanceHeadersUpdate(input: { headerId: "gid://gitlab/AuditEvents::Streaming::InstanceHeader/2", key: "new-key", value: "new-value" }) { + errors + header { + id + key + value + } + } +} +``` + +The header is updated if the returned `errors` object is empty. + +## Delete streaming destinations + +Delete streaming destinations for a top-level group or an entire instance. + +When the last destination is successfully deleted, streaming is disabled for the group or the instance. + +### Top-level group streaming destinations + +Prerequisites: + +- Owner role for a top-level group. + +Users with the Owner role for a group can delete streaming destinations using the +`externalAuditEventDestinationDestroy` mutation type. You can retrieve the destinations ID +by [listing all the streaming destinations](#list-streaming-destinations) for the group. + +```graphql +mutation { + externalAuditEventDestinationDestroy(input: { id: destination }) { + errors + } +} +``` + +Streaming destination is deleted if: + +- The returned `errors` object is empty. +- The API responds with `200 OK`. + +Group owners can remove an HTTP header using the GraphQL `auditEventsStreamingHeadersDestroy` mutation. You can retrieve the header ID +by [listing all the custom HTTP headers](#list-streaming-destinations) for the group. + +```graphql +mutation { + auditEventsStreamingHeadersDestroy(input: { headerId: "gid://gitlab/AuditEvents::Streaming::Header/1" }) { + errors + } +} +``` + +The header is deleted if the returned `errors` object is empty. + +### Instance streaming destinations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335175) in GitLab 16.0 [with a flag](../feature_flags.md) named `ff_external_audit_events`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../feature_flags.md) named +`ff_external_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. + +Prerequisites: + +- Administrator access on the instance. + +To delete streaming destinations, use the +`instanceExternalAuditEventDestinationDestroy` mutation type. You can retrieve the destinations ID +by [listing all the streaming destinations](#list-streaming-destinations) for the instance. + +```graphql +mutation { + instanceExternalAuditEventDestinationDestroy(input: { id: "gid://gitlab/AuditEvents::InstanceExternalAuditEventDestination/1" }) { + errors + } +} +``` + +Streaming destination is deleted if: + +- The returned `errors` object is empty. +- The API responds with `200 OK`. + +## Event type filters + +> Event type filters API [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344845) in GitLab 15.7. + +When this feature is enabled for a group, you can use an API to permit users to filter streamed audit events per destination. +If the feature is enabled with no filters, the destination receives all audit events. + +A streaming destination that has an event type filter set has a **filtered** (**{filter}**) label. + +### Use the API to add an event type filter + +Prerequisites: + +- You must have the Owner role for the group. + +You can add a list of event type filters using the `auditEventsStreamingDestinationEventsAdd` query type: + +```graphql +mutation { + auditEventsStreamingDestinationEventsAdd(input: { + destinationId: "gid://gitlab/AuditEvents::ExternalAuditEventDestination/1", + eventTypeFilters: ["list of event type filters"]}){ + errors + eventTypeFilters + } +} +``` + +Event type filters are added if: + +- The returned `errors` object is empty. +- The API responds with `200 OK`. + +### Use the API to remove an event type filter + +Prerequisites: + +- You must have the Owner role for the group. + +You can remove a list of event type filters using the `auditEventsStreamingDestinationEventsRemove` query type: + +```graphql +mutation { + auditEventsStreamingDestinationEventsRemove(input: { + destinationId: "gid://gitlab/AuditEvents::ExternalAuditEventDestination/1", + eventTypeFilters: ["list of event type filters"] + }){ + errors + } +} +``` + +Event type filters are removed if: + +- The returned `errors` object is empty. +- The API responds with `200 OK`. diff --git a/doc/administration/audit_event_streaming/index.md b/doc/administration/audit_event_streaming/index.md new file mode 100644 index 00000000000..44c6cff7455 --- /dev/null +++ b/doc/administration/audit_event_streaming/index.md @@ -0,0 +1,302 @@ +--- +stage: Govern +group: Compliance +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Audit event streaming **(ULTIMATE)** + +> - UI [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336411) in GitLab 14.9. +> - [Subgroup events recording](https://gitlab.com/gitlab-org/gitlab/-/issues/366878) fixed in GitLab 15.2. +> - Custom HTTP headers UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361630) in GitLab 15.2 [with a flag](../feature_flags.md) named `custom_headers_streaming_audit_events_ui`. Disabled by default. +> - Custom HTTP headers UI [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) in GitLab 15.3. [Feature flag `custom_headers_streaming_audit_events_ui`](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) removed. +> - [Improved user experience](https://gitlab.com/gitlab-org/gitlab/-/issues/367963) in GitLab 15.3. + +Users can set a streaming destination for a top-level group or instance to receive all audit events about the group, subgroups, and +projects as structured JSON. + +Top-level group owners and instance administrators can manage their audit logs in third-party systems. Any service that can receive +structured JSON data can be used as the streaming destination. + +Each streaming destination can have up to 20 custom HTTP headers included with each streamed event. + +NOTE: +GitLab can stream a single event more than once to the same destination. Use the `id` key in the payload to deduplicate incoming data. + +## Add a new streaming destination + +Add a new streaming destination to top-level groups or an entire instance. + +WARNING: +Streaming destinations receive **all** audit event data, which could include sensitive information. Make sure you trust the streaming destination. + +### Top-level group streaming destinations + +Prerequisites: + +- Owner role for a top-level group. + +To add streaming destinations to a top-level group: + +1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. +1. Select **Secure > Audit events**. +1. On the main area, select **Streams** tab. +1. Select **Add streaming destination** to show the section for adding destinations. +1. Enter the destination URL to add. +1. Optional. Locate the **Custom HTTP headers** table. +1. Ignore the **Active** checkbox because it isn't functional. To track progress on adding functionality to the + **Active** checkbox, see [issue 367509](https://gitlab.com/gitlab-org/gitlab/-/issues/367509). +1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to + 20 headers per streaming destination. +1. After all headers have been filled out, select **Add** to add the new streaming destination. + +### Instance streaming destinations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../feature_flags.md) named `instance_streaming_audit_events`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../feature_flags.md) named +`instance_streaming_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. + +Prerequisites: + +- Administrator access on the instance. + +To add a streaming destination for an instance: + +1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). +1. Select **Admin Area**. +1. On the left sidebar, select **Monitoring > Audit Events**. +1. On the main area, select **Streams** tab. +1. Select **Add streaming destination** to show the section for adding destinations. +1. Enter the destination URL to add. +1. Select **Add** to add the new streaming destination. + +## List streaming destinations + +List new streaming destinations for top-level groups or an entire instance. + +### For top-level group streaming destinations + +Prerequisites: + +- Owner role for a group. + +To list the streaming destinations for a top-level group: + +1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. +1. Select **Secure > Audit events**. +1. On the main area, select **Streams** tab. +1. To the right of the item, select **Edit** (**{pencil}**) to see all the custom HTTP headers. + +### For instance streaming destinations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../feature_flags.md) named `instance_streaming_audit_events`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../feature_flags.md) named +`instance_streaming_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. + +Prerequisites: + +- Administrator access on the instance. + +To list the streaming destinations for an instance: + +1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). +1. Select **Admin Area**. +1. On the left sidebar, select **Monitoring > Audit Events**. +1. On the main area, select **Streams** tab. + +## Update streaming destinations + +Update streaming destinations for a top-level group or an entire instance. + +### Top-level group streaming destinations + +Prerequisites: + +- Owner role for a group. + +To update a streaming destination's custom HTTP headers: + +1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. +1. Select **Secure > Audit events**. +1. On the main area, select **Streams** tab. +1. To the right of the item, select **Edit** (**{pencil}**). +1. Locate the **Custom HTTP headers** table. +1. Locate the header that you wish to update. +1. Ignore the **Active** checkbox because it isn't functional. To track progress on adding functionality to the + **Active** checkbox, see [issue 367509](https://gitlab.com/gitlab-org/gitlab/-/issues/367509). +1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to + 20 headers per streaming destination. +1. Select **Save** to update the streaming destination. + +## Delete streaming destinations + +Delete streaming destinations for a top-level group or an entire instance. When the last destination is successfully +deleted, streaming is disabled for the group or the instance. + +### Top-level group streaming destinations + +Prerequisites: + +- Owner role for a group. + +To delete a streaming destination: + +1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. To the right of the item, select **Delete** (**{remove}**). + +To delete only the custom HTTP headers for a streaming destination: + +1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams** tab. +1. To the right of the item, **Edit** (**{pencil}**). +1. Locate the **Custom HTTP headers** table. +1. Locate the header that you wish to remove. +1. To the right of the header, select **Delete** (**{remove}**). +1. Select **Save** to update the streaming destination. + +### Instance streaming destinations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../feature_flags.md) named `instance_streaming_audit_events`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../feature_flags.md) named +`instance_streaming_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. + +Prerequisites: + +- Administrator access on the instance. + +To delete the streaming destinations for an instance: + +1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). +1. Select **Admin Area**. +1. On the left sidebar, select **Monitoring > Audit Events**. +1. On the main area, select the **Streams** tab. +1. To the right of the item, select **Delete** (**{remove}**). + +## Verify event authenticity + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345424) in GitLab 14.8. + +Each streaming destination has a unique verification token (`verificationToken`) that can be used to verify the authenticity of the event. This +token is either specified by the Owner or generated automatically when the event destination is created and cannot be changed. + +Each streamed event contains the verification token in the `X-Gitlab-Event-Streaming-Token` HTTP header that can be verified against +the destination's value when [listing streaming destinations](#list-streaming-destinations). + +### Top-level group streaming destinations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360814) in GitLab 15.2. + +Prerequisites: + +- Owner role for a group. + +To list streaming destinations and see the verification tokens: + +1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. +1. Select **Secure > Audit events**. +1. On the main area, select the **Streams**. +1. View the verification token on the right side of each item. + +### Instance streaming destinations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398107) in GitLab 16.1 [with a flag](../feature_flags.md) named `instance_streaming_audit_events`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../feature_flags.md) named +`instance_streaming_audit_events`. On GitLab.com, this feature is not available. The feature is not ready for production use. + +Prerequisites: + +- Administrator access on the instance. + +To list streaming destinations for an instance and see the verification tokens: + +1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). +1. Select **Admin Area**. +1. On the left sidebar, select **Monitoring > Audit Events**. +1. On the main area, select the **Streams**. +1. View the verification token on the right side of each item. + +## Override default content type header + +By default, streaming destinations use a `content-type` header of `application/x-www-form-urlencoded`. However, you +might want to set the `content-type` header to something else. For example ,`application/json`. + +To override the `content-type` header default value for either a top-level group streaming destination or an instance +streaming destination, use either the [GitLab UI](#update-streaming-destinations) or using the +[GraphQL API](graphql_api.md#update-streaming-destinations). + +## Payload schema + +> Documentation for an audit event streaming schema was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358149) in GitLab 15.3. + +Streamed audit events have a predictable schema in the body of the response. + +| Field | Description | Notes | +|------------------|------------------------------------------------------------|-----------------------------------------------------------------------------------| +| `author_id` | User ID of the user who triggered the event | | +| `author_name` | Human-readable name of the author that triggered the event | Helpful when the author no longer exists | +| `created_at` | Timestamp when event was triggered | | +| `details` | JSON object containing additional metadata | Has no defined schema but often contains additional information about an event | +| `entity_id` | ID of the audit event's entity | | +| `entity_path` | Full path of the entity affected by the auditable event | | +| `entity_type` | String representation of the type of entity | Acceptable values include `User`, `Group`, and `Key`. This list is not exhaustive | +| `event_type` | String representation of the type of audit event | | +| `id` | Unique identifier for the audit event | Can be used for deduplication if required | +| `ip_address` | IP address of the host used to trigger the event | | +| `target_details` | Additional details about the target | | +| `target_id` | ID of the audit event's target | | +| `target_type` | String representation of the target's type | | + +### JSON payload schema + +```json +{ + "properties": { + "id": { + "type": "string" + }, + "author_id": { + "type": "integer" + }, + "author_name": { + "type": "string" + }, + "details": {}, + "ip_address": { + "type": "string" + }, + "entity_id": { + "type": "integer" + }, + "entity_path": { + "type": "string" + }, + "entity_type": { + "type": "string" + }, + "event_type": { + "type": "string" + }, + "target_id": { + "type": "integer" + }, + "target_type": { + "type": "string" + }, + "target_details": { + "type": "string" + }, + }, + "type": "object" +} +``` diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index dec4b92b03c..0e8315e528a 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -231,6 +231,17 @@ To use an external Prometheus server: } ``` + You can also specify more than one IP address if you have multiple Prometheus servers: + + ```ruby + nginx['status']['options'] = { + "server_tokens" => "off", + "access_log" => "off", + "allow" => ["192.168.0.1", "192.168.0.2"] + "deny" => "all", + } + ``` + 1. To allow the Prometheus server to fetch from the [GitLab metrics](#gitlab-metrics) endpoint, add the Prometheus server IP address to the [monitoring IP allowlist](../ip_allowlist.md): diff --git a/doc/development/pages/dnsmasq.md b/doc/development/pages/dnsmasq.md new file mode 100644 index 00000000000..a14e215ccd7 --- /dev/null +++ b/doc/development/pages/dnsmasq.md @@ -0,0 +1,65 @@ +--- +type: reference, dev +stage: Plan +group: Knowledge +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +description: "dnsmasq configuration guidelines for GitLab Pages" +--- + +# Using `dnsmasq` to dynamically handle GitLab Pages subdomains + +You can use [`dnsmasq`](https://wiki.debian.org/dnsmasq) to test +GitLab Pages sites locally without having to configure each site on `/etc/hosts`. + +## Use `dnsmasq` on macOS + +To use `dnsmasq` on macOS: + +1. Install `dnsmasq`: + +```console +brew install dnsmasq +``` + +1. Set up the `*.test` domain lookup: + +```console +# Ensure the configuration directory exists +mkdir -p $(brew --prefix)/etc/ + +# Add `*.test` to the `127.0.0.1` lookup +echo 'address=/.test/127.0.0.1' >> $(brew --prefix)/etc/dnsmasq.conf + +# Start `dnsmasq` +sudo brew services start dnsmasq +``` + +1. Create a DNS resolver: + +```console +# Ensure the resolver directory exists +sudo mkdir -p /etc/resolver + +# Add the localhost address as a resolver for `.test` domains +echo "nameserver 127.0.0.1" | sudo tee /etc/resolver/test +``` + +You can now create a GitLab Pages site locally with a dynamic domain. +If you [configure GitLab Pages](index.md#configuring-gitlab-pages-with-gdk) and +create a `root/html` project, that project is accessible through `http://root.gdk.pages.test:3010/html`. + +## Troubleshooting + +For GitLab Runner, you must define `gdk.test` in `/etc/hosts`. +If you're using GitLab Runner locally, you must also configure `/etc/hosts`: + +```console +# Append GDK configuration in `/etc/hosts` +cat <<-EOF | sudo tee -a /etc/hosts + +## GDK +127.0.0.1 gdk.test +::1 gdk.test +# ---------------------------- +EOF +``` diff --git a/doc/development/pages/index.md b/doc/development/pages/index.md index 4769dbf427d..25c1ea7a864 100644 --- a/doc/development/pages/index.md +++ b/doc/development/pages/index.md @@ -37,7 +37,7 @@ for GitLab Pages, and then one entry for each page site: If instead of editing your `/etc/hosts` you'd prefer to use a DNS wildcard, you can use: - [`nip.io`](https://nip.io) -- [`dnsmasq`](https://wiki.debian.org/dnsmasq) +- [`dnsmasq`](dnsmasq.md) ## Configuring GitLab Pages without GDK diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 4dbb05de584..322cefc71d9 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -269,9 +269,13 @@ There are two different ways to add a new project to a group: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2534) in GitLab 10.5. > - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/25975) from GitLab Premium to GitLab Free in 11.10. -By default, users with at least the Developer role can create projects under a group. +By default, users with: -To change this setting for a specific group: +- At least the Developer role can create projects under a group. This default can be changed. +- At least the Maintainer role can fork projects into a group. This default prevents users with the Developer role from forking projects that + contain protected branches and cannot be changed. + +To change the role that can create projects under a group: 1. On the left sidebar, at the top, select **Search GitLab** (**{search}**) to find your group. 1. Select **Settings > General**. diff --git a/doc/user/permissions.md b/doc/user/permissions.md index 5781b109da9..0b02de59ab4 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -369,6 +369,7 @@ The following table lists group permissions available for each role: | View [Productivity analytics](analytics/productivity_analytics.md) | | ✓ | ✓ | ✓ | ✓ | | Create and edit [group wiki](project/wiki/group.md) pages | | | ✓ | ✓ | ✓ | | Create project in group | | | ✓ (2)(4) | ✓ (2) | ✓ (2) | +| Fork project into a group | | | | ✓ | ✓ | | Create/edit/delete group milestones | | ✓ | ✓ | ✓ | ✓ | | Create/edit/delete iterations | | ✓ | ✓ | ✓ | ✓ | | Create/edit/delete metrics dashboard annotations | | | ✓ | ✓ | ✓ | diff --git a/doc/user/project/deploy_keys/index.md b/doc/user/project/deploy_keys/index.md index ffe237825a7..5bd19fec0ba 100644 --- a/doc/user/project/deploy_keys/index.md +++ b/doc/user/project/deploy_keys/index.md @@ -56,7 +56,7 @@ As with all sensitive information, you should ensure only those who need access For human interactions, use credentials tied to users such as Personal Access Tokens. To help detect a potential secret leak, you can use the -[Audit Event](../../../administration/audit_event_streaming.md#example-payloads-for-ssh-events-with-deploy-key) feature. +[Audit Event](../../../administration/audit_event_streaming/examples.md#example-payloads-for-ssh-events-with-deploy-key) feature. ## View deploy keys diff --git a/doc/user/project/repository/code_suggestions.md b/doc/user/project/repository/code_suggestions.md index 932e081b091..dd1fa3f4c8b 100644 --- a/doc/user/project/repository/code_suggestions.md +++ b/doc/user/project/repository/code_suggestions.md @@ -70,6 +70,7 @@ Prerequisites: - Code Suggestions must be [enabled for the top-level group](../../group/manage.md#enable-code-suggestions). - Code Suggestions must be [enabled for your user account](#enable-code-suggestions-for-an-individual-user). +- You must be a GitLab team member. Code Suggestions work automatically in the [GitLab WebIDE](../../project/web_ide/index.md) if the above prerequisites are met. To disable Code Suggestions in the WebIDE, disable the user account setting. diff --git a/lefthook.yml b/lefthook.yml index 2c9e7532ec6..0d67d6521a2 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -97,6 +97,10 @@ pre-push: files: git diff --name-only --diff-filter=d $(git merge-base origin/master HEAD)..HEAD glob: '*.{rb}' run: bundle exec rake lint:static_verification + bundler-checksum-lint: + tags: backend bundler + glob: 'Gemfile.lock' + run: bundle exec bundler-checksum lint scripts: "merge_conflicts": diff --git a/spec/frontend/snippets/components/edit_spec.js b/spec/frontend/snippets/components/edit_spec.js index d17e20ac227..17862953920 100644 --- a/spec/frontend/snippets/components/edit_spec.js +++ b/spec/frontend/snippets/components/edit_spec.js @@ -113,7 +113,7 @@ describe('Snippet Edit app', () => { const triggerBlobActions = (actions) => findBlobActions().vm.$emit('actions', actions); const setUploadFilesHtml = (paths) => { - wrapper.vm.$el.innerHTML = paths + wrapper.element.innerHTML = paths .map((path) => `<input name="files[]" value="${path}">`) .join(''); }; diff --git a/vendor/gems/bundler-checksum/lib/bundler_checksum/command.rb b/vendor/gems/bundler-checksum/lib/bundler_checksum/command.rb index c6c71431538..b84abd932fb 100644 --- a/vendor/gems/bundler-checksum/lib/bundler_checksum/command.rb +++ b/vendor/gems/bundler-checksum/lib/bundler_checksum/command.rb @@ -3,19 +3,27 @@ module BundlerChecksum module Command autoload :Init, File.expand_path("command/init", __dir__) + autoload :Lint, File.expand_path("command/lint", __dir__) autoload :Verify, File.expand_path("command/verify", __dir__) autoload :Helper, File.expand_path("command/helper", __dir__) def self.execute(args) if args.empty? - $stderr.puts 'A command must be given [init,update,verify]' + $stderr.puts 'A command must be given [init,update,verify,lint]' end - if args.first == 'init' + case args.first + when 'init' Init.execute - elsif args.first == 'update' + when 'update' $stderr.puts 'Not implemented, please use init' - elsif args.first == 'verify' + when 'lint' + linted = Lint.execute + + unless linted + exit 1 + end + when 'verify' verified = Verify.execute unless verified diff --git a/vendor/gems/bundler-checksum/lib/bundler_checksum/command/lint.rb b/vendor/gems/bundler-checksum/lib/bundler_checksum/command/lint.rb new file mode 100644 index 00000000000..01da04ce27c --- /dev/null +++ b/vendor/gems/bundler-checksum/lib/bundler_checksum/command/lint.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'set' + +module BundlerChecksum::Command + module Lint + extend self + + def execute + linted = true + + Bundler.definition.resolve.sort_by(&:name).each do |spec| + next unless spec.source.is_a?(Bundler::Source::Rubygems) + + unless checksum_for?(spec.name) + $stderr.puts "ERROR: Missing checksum for gem `#{spec.name}`" + linted = false + end + end + + unless linted + $stderr.puts <<~MSG + + Please run `bundle exec bundler-checksum init` to add missing checksums. + MSG + end + + linted + end + + private + + def checksum_for?(name) + gems_with_checksums.include?(name) + end + + def gems_with_checksums + @gems_with_checksums ||= local_checksums.map { |hash| hash[:name] }.to_set + end + + def local_checksums + ::BundlerChecksum.checksums_from_file + end + end +end diff --git a/vendor/gems/bundler-checksum/test/project_with_checksum_lock/scripts/test b/vendor/gems/bundler-checksum/test/project_with_checksum_lock/scripts/test index bb256449226..9b6d83591f8 100755 --- a/vendor/gems/bundler-checksum/test/project_with_checksum_lock/scripts/test +++ b/vendor/gems/bundler-checksum/test/project_with_checksum_lock/scripts/test @@ -3,6 +3,9 @@ set -x set -e +# Ensure that each gem has a checksum entry +ruby -I ../../lib ../../bin/bundler-checksum lint + # Check there's no differences after re-initialising ruby -I ../../lib ../../bin/bundler-checksum init git diff --exit-code Gemfile.checksum |