diff options
24 files changed, 237 insertions, 38 deletions
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 5d079f57267..992df095813 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -393,6 +393,10 @@ module Ci newest_first(ref: ref).failed.take end + def self.jobs_count_in_alive_pipelines + created_after(24.hours.ago).alive.joins(:builds).count + end + # Returns a Hash containing the latest pipeline for every given # commit. # diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index e002039d226..78a60969ba7 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1020,6 +1020,7 @@ :idempotent: :tags: - :exclude_from_gitlab_com + - :needs_own_queue - :name: hashed_storage:hashed_storage_project_migrate :worker_name: HashedStorage::ProjectMigrateWorker :feature_category: :source_code_management @@ -1030,6 +1031,7 @@ :idempotent: :tags: - :exclude_from_gitlab_com + - :needs_own_queue - :name: hashed_storage:hashed_storage_project_rollback :worker_name: HashedStorage::ProjectRollbackWorker :feature_category: :source_code_management @@ -1040,6 +1042,7 @@ :idempotent: :tags: - :exclude_from_gitlab_com + - :needs_own_queue - :name: hashed_storage:hashed_storage_rollbacker :worker_name: HashedStorage::RollbackerWorker :feature_category: :source_code_management @@ -1050,6 +1053,7 @@ :idempotent: :tags: - :exclude_from_gitlab_com + - :needs_own_queue - :name: incident_management:clusters_applications_check_prometheus_health :worker_name: Clusters::Applications::CheckPrometheusHealthWorker :feature_category: :incident_management @@ -2028,7 +2032,8 @@ :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: [] + :tags: + - :needs_own_queue - :name: emails_on_push :worker_name: EmailsOnPushWorker :feature_category: :source_code_management @@ -2771,7 +2776,8 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :needs_own_queue - :name: snippets_schedule_bulk_repository_shard_moves :worker_name: Snippets::ScheduleBulkRepositoryShardMovesWorker :feature_category: :gitaly diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb index 5ebe6e7c691..1514897b2e4 100644 --- a/app/workers/email_receiver_worker.rb +++ b/app/workers/email_receiver_worker.rb @@ -11,6 +11,9 @@ class EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker urgency :high weight 2 + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1087#jobs-written-to-redis-without-passing-through-the-application + tags :needs_own_queue + attr_accessor :raw def perform(raw) diff --git a/app/workers/hashed_storage/migrator_worker.rb b/app/workers/hashed_storage/migrator_worker.rb index f8abdd4089e..80e86fd7814 100644 --- a/app/workers/hashed_storage/migrator_worker.rb +++ b/app/workers/hashed_storage/migrator_worker.rb @@ -10,7 +10,10 @@ module HashedStorage queue_namespace :hashed_storage feature_category :source_code_management - tags :exclude_from_gitlab_com + + # Gitlab::HashedStorage::Migrator#migration_pending? depends on the + # queue size of this worker. + tags :exclude_from_gitlab_com, :needs_own_queue # @param [Integer] start initial ID of the batch # @param [Integer] finish last ID of the batch diff --git a/app/workers/hashed_storage/project_migrate_worker.rb b/app/workers/hashed_storage/project_migrate_worker.rb index e4dcf828446..edddea55356 100644 --- a/app/workers/hashed_storage/project_migrate_worker.rb +++ b/app/workers/hashed_storage/project_migrate_worker.rb @@ -10,7 +10,10 @@ module HashedStorage queue_namespace :hashed_storage loggable_arguments 1 - tags :exclude_from_gitlab_com + + # Gitlab::HashedStorage::Migrator#migration_pending? depends on the + # queue size of this worker. + tags :exclude_from_gitlab_com, :needs_own_queue attr_reader :project_id diff --git a/app/workers/hashed_storage/project_rollback_worker.rb b/app/workers/hashed_storage/project_rollback_worker.rb index 4ad837602ed..c5841dbbb28 100644 --- a/app/workers/hashed_storage/project_rollback_worker.rb +++ b/app/workers/hashed_storage/project_rollback_worker.rb @@ -10,7 +10,10 @@ module HashedStorage queue_namespace :hashed_storage loggable_arguments 1 - tags :exclude_from_gitlab_com + + # Gitlab::HashedStorage::Migrator#rollback_pending? depends on the + # queue size of this worker. + tags :exclude_from_gitlab_com, :needs_own_queue attr_reader :project_id diff --git a/app/workers/hashed_storage/rollbacker_worker.rb b/app/workers/hashed_storage/rollbacker_worker.rb index 887e43faeba..90e48f0e37a 100644 --- a/app/workers/hashed_storage/rollbacker_worker.rb +++ b/app/workers/hashed_storage/rollbacker_worker.rb @@ -10,7 +10,10 @@ module HashedStorage queue_namespace :hashed_storage feature_category :source_code_management - tags :exclude_from_gitlab_com + + # Gitlab::HashedStorage::Migrator#rollback_pending? depends on the + # queue size of this worker. + tags :exclude_from_gitlab_com, :needs_own_queue # @param [Integer] start initial ID of the batch # @param [Integer] finish last ID of the batch diff --git a/app/workers/service_desk_email_receiver_worker.rb b/app/workers/service_desk_email_receiver_worker.rb index a77c21562b3..f546fce3e8a 100644 --- a/app/workers/service_desk_email_receiver_worker.rb +++ b/app/workers/service_desk_email_receiver_worker.rb @@ -8,6 +8,9 @@ class ServiceDeskEmailReceiverWorker < EmailReceiverWorker # rubocop:disable Sca feature_category :service_desk sidekiq_options retry: 3 + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1087#jobs-written-to-redis-without-passing-through-the-application + tags :needs_own_queue + def should_perform? ::Gitlab::ServiceDeskEmail.enabled? end diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index 2aa95a2b0f1..a0034cb623f 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -45,6 +45,7 @@ The following metrics are available: | `gitlab_ci_pipeline_size_builds` | Histogram | 13.1 | Total number of builds within a pipeline grouped by a pipeline source | `source` | | `job_waiter_started_total` | Counter | 12.9 | Number of batches of jobs started where a web request is waiting for the jobs to complete | `worker` | | `job_waiter_timeouts_total` | Counter | 12.9 | Number of batches of jobs that timed out where a web request is waiting for the jobs to complete | `worker` | +| `gitlab_ci_active_jobs` | Histogram | 14.2 | Count of active jobs when pipeline is created | | | `gitlab_database_transaction_seconds` | Histogram | 12.1 | Time spent in database transactions, in seconds | | | `gitlab_method_call_duration_seconds` | Histogram | 10.2 | Method calls real duration | `controller`, `action`, `module`, `method` | | `gitlab_page_out_of_bounds` | Counter | 12.8 | Counter for the PageLimiter pagination limit being hit | `controller`, `action`, `bot` | diff --git a/doc/administration/postgresql/pgbouncer.md b/doc/administration/postgresql/pgbouncer.md index 4f9056b9b50..981a87be887 100644 --- a/doc/administration/postgresql/pgbouncer.md +++ b/doc/administration/postgresql/pgbouncer.md @@ -200,6 +200,45 @@ Once you've performed the tasks or procedure, switch back to using PgBouncer: sudo gitlab-ctl reconfigure ``` +## Fine tuning + +PgBouncer's default settings suit the majority of installations. +In specific cases you may want to change the performance-specific and resource-specific variables to either increase possible +throughput or to limit resource utilization that could cause memory exhaustion on the database. + +You can find the parameters and respective documentation on the [official PgBouncer documentation](https://www.pgbouncer.org/config.html). +Listed below are the most relevant ones and their defaults on an Omnibus GitLab installation: + +- `pgbouncer['max_client_conn']` (default: `2048`, depends on server file descriptor limits) + This is the "frontend" pool in PgBouncer: connections from Rails to PgBouncer. +- `pgbouncer['default_pool_size']` (default: `100`) + This is the "backend" pool in PgBouncer: connections from PgBouncer to the database. + +The ideal number for `default_pool_size` must be enough to handle all provisioned services that need to access +the database. Each of the listed services below use the following formula to define database pool size: + +- `puma` : `max_threads + headroom` (default `14`) + - `max_threads` is configured via: `gitlab['puma']['max_threads']` (default: `4`) + - `headroom` can be configured via `DB_POOL_HEADROOM` env variable (default to `10`) +- `sidekiq` : `max_concurrency + 1 + headroom` (default: `61`) + - `max_concurrency` is configured via: `sidekiq['max_concurrency']` (default: `50`) + - `headroom` can be configured via `DB_POOL_HEADROOM` env variable (default to `10`) +- `geo-logcursor`: `1+headroom` (default: `11`) + - `headroom` can be configured via `DB_POOL_HEADROOM` env variable (default to `10`) + +To calculate the `default_pool_size`, multiply the number of instances of `puma`, `sidekiq` and `geo-logcursor` by the +number of connections each can consume as per listed above. The total will be the suggested `default_pool_size`. + +If you are using more than one PgBouncer with an internal Load Balancer, you may be able to divide the +`default_pool_size` by the number of instances to guarantee an evenly distributed load between them. + +The `pgbouncer['max_client_conn']` is the hard-limit of connections PgBouncer can accept. It's unlikely you will need +to change this. If you are hitting that limit, you may want to consider adding additional PgBouncers with an internal +Load Balancer. + +When setting up the limits for a PgBouncer that points to the Geo Tracking Database, +you can likely ignore `puma` from the equation, as it is only accessing that database sporadically. + ## Troubleshooting In case you are experiencing any issues connecting through PgBouncer, the first diff --git a/doc/user/group/index.md b/doc/user/group/index.md index af6b7e987b6..99bdbc27981 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -650,8 +650,9 @@ By default, projects in a group can be forked. Optionally, on [Premium](https://about.gitlab.com/pricing/) or higher tiers, you can prevent the projects in a group from being forked outside of the current top-level group. -Previously, this setting was available only for groups enforcing group managed -account. This setting will be removed from the SAML setting page, and migrated to the +Previously, this setting was available only for groups enforcing a +[Group Managed Account](saml_sso/group_managed_accounts.md) in SAML. +This setting will be removed from the SAML setting page, and migrated to the group settings page. In the interim period, both of these settings are taken into consideration. If even one is set to `true`, then the group does not allow outside forks. diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index c3c1728602c..8cc939b1ce6 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -97,6 +97,11 @@ module Gitlab .observe({ source: pipeline.source.to_s }, pipeline.total_size) end + def observe_jobs_count_in_alive_pipelines + metrics.active_jobs_histogram + .observe({ plan: project.actual_plan_name }, project.all_pipelines.jobs_count_in_alive_pipelines) + end + def increment_pipeline_failure_reason_counter(reason) metrics.pipeline_failure_reason_counter .increment(reason: (reason || :unknown_failure).to_s) diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index dc648568129..bbfc6759b35 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -22,6 +22,7 @@ module Gitlab @command.observe_creation_duration(Time.now - @start) @command.observe_pipeline_size(@pipeline) + @command.observe_jobs_count_in_alive_pipelines @pipeline end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 84b88374a7f..80b10639237 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -29,6 +29,15 @@ module Gitlab ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end + def self.active_jobs_histogram + name = :gitlab_ci_active_jobs + comment = 'Total amount of active jobs' + labels = { plan: nil } + buckets = [0, 200, 500, 1_000, 2_000, 5_000, 10_000] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + def self.pipeline_processing_events_counter name = :gitlab_ci_pipeline_processing_events_total comment = 'Total amount of pipeline processing events' diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 94487ec3be1..abfcba5a429 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -4130,9 +4130,18 @@ msgid_plural "ApprovalRuleSummary|%{count} approvals required from %{membersCoun msgstr[0] "" msgstr[1] "" +msgid "ApprovalRule|%{scanner} +%{additionalScanners} more" +msgstr "" + msgid "ApprovalRule|Add approvers" msgstr "" +msgid "ApprovalRule|All scanners" +msgstr "" + +msgid "ApprovalRule|Apply this approval rule to consider only the selected security scanners." +msgstr "" + msgid "ApprovalRule|Approval rules" msgstr "" @@ -4151,9 +4160,21 @@ msgstr "" msgid "ApprovalRule|Name" msgstr "" +msgid "ApprovalRule|Please select at least one security scanner" +msgstr "" + msgid "ApprovalRule|Rule name" msgstr "" +msgid "ApprovalRule|Security scanners" +msgstr "" + +msgid "ApprovalRule|Select All" +msgstr "" + +msgid "ApprovalRule|Select scanners" +msgstr "" + msgid "ApprovalRule|Target branch" msgstr "" diff --git a/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb index cc4aaffb0a4..83d47ae6819 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do let_it_be(:user) { create(:user) } let(:pipeline) { build_stubbed(:ci_pipeline) } - let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new } + let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new(project: project) } let(:first_step) { spy('first step') } let(:second_step) { spy('second step') } let(:sequence) { [first_step, second_step] } @@ -71,5 +71,20 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do expect(histogram).to have_received(:observe) .with({ source: 'push' }, 0) end + + it 'records active jobs by pipeline plan in a histogram' do + allow(command.metrics) + .to receive(:active_jobs_histogram) + .and_return(histogram) + + pipeline = create(:ci_pipeline, project: project, status: :running) + create(:ci_build, :finished, project: project, pipeline: pipeline) + create(:ci_build, :failed, project: project, pipeline: pipeline) + create(:ci_build, :running, project: project, pipeline: pipeline) + subject.build! + + expect(histogram).to have_received(:observe) + .with(hash_including(plan: project.actual_plan_name), 3) + end end end diff --git a/workhorse/config.toml.example b/workhorse/config.toml.example index cb29508a90b..27dc29ee078 100644 --- a/workhorse/config.toml.example +++ b/workhorse/config.toml.example @@ -1,5 +1,7 @@ # alt_document_root = '/home/git/public/assets' # shutdown_timeout = "60s" +# trusted_cidrs_for_x_forwarded_for = [] +# trusted_cidrs_for_propagation = [] [redis] URL = "unix:/home/git/gitlab/redis/redis.socket" diff --git a/workhorse/config_test.go b/workhorse/config_test.go index 36e43c23abd..658a352a333 100644 --- a/workhorse/config_test.go +++ b/workhorse/config_test.go @@ -30,6 +30,9 @@ func TestConfigFile(t *testing.T) { data := ` shutdown_timeout = "60s" +trusted_cidrs_for_x_forwarded_for = ["127.0.0.1/8", "192.168.0.1/8"] +trusted_cidrs_for_propagation = ["10.0.0.1/8"] + [redis] password = "redis password" [object_storage] @@ -51,6 +54,8 @@ max_scaler_procs = 123 require.Equal(t, "redis password", cfg.Redis.Password) require.Equal(t, "test provider", cfg.ObjectStorageCredentials.Provider) require.Equal(t, uint32(123), cfg.ImageResizerConfig.MaxScalerProcs, "image resizer max_scaler_procs") + require.Equal(t, []string{"127.0.0.1/8", "192.168.0.1/8"}, cfg.TrustedCIDRsForXForwardedFor) + require.Equal(t, []string{"10.0.0.1/8"}, cfg.TrustedCIDRsForPropagation) require.Equal(t, 60*time.Second, cfg.ShutdownTimeout.Duration) } diff --git a/workhorse/go.mod b/workhorse/go.mod index 1b2e1ec1339..3ce279f2ccc 100644 --- a/workhorse/go.mod +++ b/workhorse/go.mod @@ -7,7 +7,7 @@ require ( github.com/BurntSushi/toml v0.3.1 github.com/FZambia/sentinel v1.0.0 github.com/alecthomas/chroma v0.7.3 - github.com/aws/aws-sdk-go v1.36.1 + github.com/aws/aws-sdk-go v1.37.0 github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/disintegration/imaging v1.6.2 @@ -29,7 +29,7 @@ require ( github.com/smartystreets/goconvey v1.6.4 github.com/stretchr/testify v1.7.0 gitlab.com/gitlab-org/gitaly/v14 v14.0.0-rc1 - gitlab.com/gitlab-org/labkit v1.4.0 + gitlab.com/gitlab-org/labkit v1.6.0 gocloud.dev v0.21.1-0.20201223184910-5094f54ed8bb golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5 // indirect golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 @@ -37,5 +37,6 @@ require ( golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 golang.org/x/tools v0.1.0 google.golang.org/grpc v1.37.0 + gopkg.in/DataDog/dd-trace-go.v1 v1.31.0 // indirect honnef.co/go/tools v0.1.3 ) diff --git a/workhorse/go.sum b/workhorse/go.sum index 3f9b36b5b49..bb7f4f70fc4 100644 --- a/workhorse/go.sum +++ b/workhorse/go.sum @@ -19,6 +19,7 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= @@ -46,6 +47,8 @@ cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lU cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/stackdriver v0.13.4/go.mod h1:aXENhDJ1Y4lIg4EUaVTwzvYETVNZk10Pu26tevFKLUc= +contrib.go.opencensus.io/exporter/stackdriver v0.13.8 h1:lIFYmQsqejvlq+GobFUbC5F0prD5gvhP6r0gWLZRDq4= +contrib.go.opencensus.io/exporter/stackdriver v0.13.8/go.mod h1:huNtlWx75MwO7qMs0KrMxPZXzNNWebav1Sq/pm02JdQ= contrib.go.opencensus.io/integrations/ocsql v0.1.7/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= @@ -145,8 +148,9 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.36.1 h1:rDgSL20giXXu48Ycx6Qa4vWaNTVTltUl6vA73ObCSVk= github.com/aws/aws-sdk-go v1.36.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.37.0 h1:GzFnhOIsrGyQ69s7VgqtrG2BG8v7X7vwB3Xpbd/DBBk= +github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -158,6 +162,7 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0 h1:t/LhUZLVitR1Ow2YOnduCsavhwFUklBMoGVYUCqmCqk= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= @@ -309,8 +314,9 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -379,6 +385,7 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 h1:zIaiqGYDQwa4HVx5wGRTXbx38Pqxjemn4BP98wpzpXo= @@ -783,8 +790,8 @@ gitlab.com/gitlab-org/gitlab-shell v1.9.8-0.20201117050822-3f9890ef73dc/go.mod h gitlab.com/gitlab-org/labkit v0.0.0-20190221122536-0c3fc7cdd57c/go.mod h1:rYhLgfrbEcyfinG+R3EvKu6bZSsmwQqcXzLfHWSfUKM= gitlab.com/gitlab-org/labkit v0.0.0-20200908084045-45895e129029/go.mod h1:SNfxkfUwVNECgtmluVayv0GWFgEjjBs5AzgsowPQuo0= gitlab.com/gitlab-org/labkit v1.0.0/go.mod h1:nohrYTSLDnZix0ebXZrbZJjymRar8HeV2roWL5/jw2U= -gitlab.com/gitlab-org/labkit v1.4.0 h1:KZTEylusrFmqLXSzE5bHfBf7/xI2NLnsyoRgB7I7Oh8= -gitlab.com/gitlab-org/labkit v1.4.0/go.mod h1:4YbseTLUD7g4pPSylV57Hpyf7N3hbbxdx8K81//U/XM= +gitlab.com/gitlab-org/labkit v1.6.0 h1:Qgk+W+N0cujGBmZSjMqvM+4qIEjl7VgIK4nxlQO0RlA= +gitlab.com/gitlab-org/labkit v1.6.0/go.mod h1:1ZuVZpjSpCKUgjLx8P6jzkkQFxJI1thUKr6yKV3p0vY= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= @@ -796,6 +803,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -921,6 +929,7 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 h1:b0LrWgu8+q7z4J+0Y3Umo5q1dL7NXBkKBWkaVkAq17E= @@ -934,6 +943,7 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201203001011-0b49973bad19/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -949,6 +959,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1103,6 +1114,7 @@ golang.org/x/tools v0.0.0-20201203202102-a1a1cbeaa516/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1139,6 +1151,7 @@ google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQW google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.37.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= @@ -1196,6 +1209,8 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1245,6 +1260,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/DataDog/dd-trace-go.v1 v1.7.0/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fzwVCaGWylTe3tg= +gopkg.in/DataDog/dd-trace-go.v1 v1.30.0/go.mod h1:SnKViq44dv/0gjl9RpkP0Y2G3BJSRkp6eYdCSu39iI8= gopkg.in/DataDog/dd-trace-go.v1 v1.31.0 h1:ouY+DNlRTckk63TNh468tPWBC21qBZPniVQXQs0iq10= gopkg.in/DataDog/dd-trace-go.v1 v1.31.0/go.mod h1:SnKViq44dv/0gjl9RpkP0Y2G3BJSRkp6eYdCSu39iI8= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/workhorse/internal/config/config.go b/workhorse/internal/config/config.go index 9f214385f81..60cfd567f5d 100644 --- a/workhorse/internal/config/config.go +++ b/workhorse/internal/config/config.go @@ -85,25 +85,27 @@ type ImageResizerConfig struct { } type Config struct { - Redis *RedisConfig `toml:"redis"` - Backend *url.URL `toml:"-"` - CableBackend *url.URL `toml:"-"` - Version string `toml:"-"` - DocumentRoot string `toml:"-"` - DevelopmentMode bool `toml:"-"` - Socket string `toml:"-"` - CableSocket string `toml:"-"` - ProxyHeadersTimeout time.Duration `toml:"-"` - APILimit uint `toml:"-"` - APIQueueLimit uint `toml:"-"` - APIQueueTimeout time.Duration `toml:"-"` - APICILongPollingDuration time.Duration `toml:"-"` - ObjectStorageConfig ObjectStorageConfig `toml:"-"` - ObjectStorageCredentials ObjectStorageCredentials `toml:"object_storage"` - PropagateCorrelationID bool `toml:"-"` - ImageResizerConfig ImageResizerConfig `toml:"image_resizer"` - AltDocumentRoot string `toml:"alt_document_root"` - ShutdownTimeout TomlDuration `toml:"shutdown_timeout"` + Redis *RedisConfig `toml:"redis"` + Backend *url.URL `toml:"-"` + CableBackend *url.URL `toml:"-"` + Version string `toml:"-"` + DocumentRoot string `toml:"-"` + DevelopmentMode bool `toml:"-"` + Socket string `toml:"-"` + CableSocket string `toml:"-"` + ProxyHeadersTimeout time.Duration `toml:"-"` + APILimit uint `toml:"-"` + APIQueueLimit uint `toml:"-"` + APIQueueTimeout time.Duration `toml:"-"` + APICILongPollingDuration time.Duration `toml:"-"` + ObjectStorageConfig ObjectStorageConfig `toml:"-"` + ObjectStorageCredentials ObjectStorageCredentials `toml:"object_storage"` + PropagateCorrelationID bool `toml:"-"` + ImageResizerConfig ImageResizerConfig `toml:"image_resizer"` + AltDocumentRoot string `toml:"alt_document_root"` + ShutdownTimeout TomlDuration `toml:"shutdown_timeout"` + TrustedCIDRsForXForwardedFor []string `toml:"trusted_cidrs_for_x_forwarded_for"` + TrustedCIDRsForPropagation []string `toml:"trusted_cidrs_for_propagation"` } var DefaultImageResizerConfig = ImageResizerConfig{ diff --git a/workhorse/internal/upstream/upstream.go b/workhorse/internal/upstream/upstream.go index 0b46228a0a8..983b0516ee9 100644 --- a/workhorse/internal/upstream/upstream.go +++ b/workhorse/internal/upstream/upstream.go @@ -87,6 +87,12 @@ func newUpstream(cfg config.Config, accessLogger *logrus.Logger, routesCallback if cfg.PropagateCorrelationID { correlationOpts = append(correlationOpts, correlation.WithPropagation()) } + if cfg.TrustedCIDRsForPropagation != nil { + correlationOpts = append(correlationOpts, correlation.WithCIDRsTrustedForPropagation(cfg.TrustedCIDRsForPropagation)) + } + if cfg.TrustedCIDRsForXForwardedFor != nil { + correlationOpts = append(correlationOpts, correlation.WithCIDRsTrustedForXForwardedFor(cfg.TrustedCIDRsForXForwardedFor)) + } handler := correlation.InjectCorrelationID(&up, correlationOpts...) // TODO: move to LabKit https://gitlab.com/gitlab-org/gitlab/-/issues/324823 diff --git a/workhorse/main.go b/workhorse/main.go index e2afe5358cb..6e7b80bc8c6 100644 --- a/workhorse/main.go +++ b/workhorse/main.go @@ -152,6 +152,8 @@ func buildConfig(arg0 string, args []string) (*bootConfig, *config.Config, error cfg.ImageResizerConfig = cfgFromFile.ImageResizerConfig cfg.AltDocumentRoot = cfgFromFile.AltDocumentRoot cfg.ShutdownTimeout = cfgFromFile.ShutdownTimeout + cfg.TrustedCIDRsForXForwardedFor = cfgFromFile.TrustedCIDRsForXForwardedFor + cfg.TrustedCIDRsForPropagation = cfgFromFile.TrustedCIDRsForPropagation return boot, cfg, nil } diff --git a/workhorse/main_test.go b/workhorse/main_test.go index bbcfaa1b4ee..6e61e2fc65a 100644 --- a/workhorse/main_test.go +++ b/workhorse/main_test.go @@ -611,16 +611,53 @@ func TestPropagateCorrelationIdHeader(t *testing.T) { defer ts.Close() testCases := []struct { - desc string - propagateCorrelationID bool + desc string + propagateCorrelationID bool + xffHeader string + trustedCIDRsForPropagation []string + trustedCIDRsForXForwardedFor []string + propagationExpected bool }{ { desc: "propagateCorrelatedId is true", propagateCorrelationID: true, + propagationExpected: true, }, { desc: "propagateCorrelatedId is false", propagateCorrelationID: false, + propagationExpected: false, + }, + { + desc: "propagation with trusted propagation CIDR", + propagateCorrelationID: true, + // Assumes HTTP connection's RemoteAddr will be 127.0.0.1:x + trustedCIDRsForPropagation: []string{"127.0.0.1/8"}, + propagationExpected: true, + }, + { + desc: "propagation with trusted propagation and X-Forwarded-For CIDRs", + propagateCorrelationID: true, + // Assumes HTTP connection's RemoteAddr will be 127.0.0.1:x + xffHeader: "1.2.3.4, 127.0.0.1", + trustedCIDRsForPropagation: []string{"1.2.3.4/32"}, + trustedCIDRsForXForwardedFor: []string{"127.0.0.1/32", "192.168.0.1/32"}, + propagationExpected: true, + }, + { + desc: "propagation not active with invalid propagation CIDR", + propagateCorrelationID: true, + trustedCIDRsForPropagation: []string{"asdf"}, + propagationExpected: false, + }, + { + desc: "propagation with invalid X-Forwarded-For CIDR", + propagateCorrelationID: true, + // Assumes HTTP connection's RemoteAddr will be 127.0.0.1:x + xffHeader: "1.2.3.4, 127.0.0.1", + trustedCIDRsForPropagation: []string{"1.2.3.4/32"}, + trustedCIDRsForXForwardedFor: []string{"bad"}, + propagationExpected: false, }, } @@ -628,19 +665,27 @@ func TestPropagateCorrelationIdHeader(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { upstreamConfig := newUpstreamConfig(ts.URL) upstreamConfig.PropagateCorrelationID = tc.propagateCorrelationID + upstreamConfig.TrustedCIDRsForPropagation = tc.trustedCIDRsForPropagation + upstreamConfig.TrustedCIDRsForXForwardedFor = tc.trustedCIDRsForXForwardedFor ws := startWorkhorseServerWithConfig(upstreamConfig) defer ws.Close() resource := "/api/v3/projects/123/repository/not/special" propagatedRequestId := "Propagated-RequestId-12345678" - resp, _ := httpGet(t, ws.URL+resource, map[string]string{"X-Request-Id": propagatedRequestId}) + headers := map[string]string{"X-Request-Id": propagatedRequestId} + + if tc.xffHeader != "" { + headers["X-Forwarded-For"] = tc.xffHeader + } + + resp, _ := httpGet(t, ws.URL+resource, headers) requestIds := resp.Header["X-Request-Id"] require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource) require.Equal(t, 1, len(requestIds), "GET %q: One X-Request-Id present", resource) - if tc.propagateCorrelationID { + if tc.propagationExpected { require.Contains(t, requestIds, propagatedRequestId, "GET %q: Has X-Request-Id %s present", resource, propagatedRequestId) } else { require.NotContains(t, requestIds, propagatedRequestId, "GET %q: X-Request-Id not propagated") |