diff options
| -rw-r--r-- | app/models/clusters/applications/ingress.rb | 34 | ||||
| -rw-r--r-- | changelogs/unreleased/13055-show-the-sync-information-for-design-repositories.yml | 5 | ||||
| -rw-r--r-- | changelogs/unreleased/8558-use-custom-modsecurity-ingress-config.yml | 5 | ||||
| -rw-r--r-- | db/migrate/20191009100244_add_geo_design_repository_counters.rb | 16 | ||||
| -rw-r--r-- | db/schema.rb | 4 | ||||
| -rw-r--r-- | doc/administration/geo/replication/using_a_geo_server.md | 8 | ||||
| -rw-r--r-- | doc/api/geo_nodes.md | 12 | ||||
| -rw-r--r-- | lib/api/internal/base.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/analytics/cycle_analytics/base_query_builder.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/analytics/cycle_analytics/records_fetcher.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/git_access_result/custom_action.rb | 6 | ||||
| -rw-r--r-- | locale/gitlab.pot | 3 | ||||
| -rw-r--r-- | spec/models/clusters/applications/ingress_spec.rb | 18 | ||||
| -rw-r--r-- | spec/requests/api/internal/base_spec.rb | 7 | ||||
| -rw-r--r-- | vendor/ingress/modsecurity.conf | 273 |
15 files changed, 385 insertions, 12 deletions
diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb index 48712c8cc09..41f5ad6550e 100644 --- a/app/models/clusters/applications/ingress.rb +++ b/app/models/clusters/applications/ingress.rb @@ -78,12 +78,42 @@ module Clusters "controller" => { "config" => { "enable-modsecurity" => "true", - "enable-owasp-modsecurity-crs" => "true" - } + "enable-owasp-modsecurity-crs" => "true", + "modsecurity.conf" => modsecurity_config_content + }, + "extraVolumeMounts" => [ + { + "name" => "modsecurity-template-volume", + "mountPath" => "/etc/nginx/modsecurity/modsecurity.conf", + "subPath" => "modsecurity.conf" + } + ], + "extraVolumes" => [ + { + "name" => "modsecurity-template-volume", + "configMap" => { + "name" => "ingress-nginx-ingress-controller", + "items" => [ + { + "key" => "modsecurity.conf", + "path" => "modsecurity.conf" + } + ] + } + } + ] } } end + def modsecurity_config_content + File.read(modsecurity_config_file_path) + end + + def modsecurity_config_file_path + Rails.root.join('vendor', 'ingress', 'modsecurity.conf') + end + def content_values YAML.load_file(chart_values_file).deep_merge!(specification) end diff --git a/changelogs/unreleased/13055-show-the-sync-information-for-design-repositories.yml b/changelogs/unreleased/13055-show-the-sync-information-for-design-repositories.yml new file mode 100644 index 00000000000..2f807170f17 --- /dev/null +++ b/changelogs/unreleased/13055-show-the-sync-information-for-design-repositories.yml @@ -0,0 +1,5 @@ +--- +title: 'Geo: Add resigns-related fields to Geo Node Status table' +merge_request: 18379 +author: +type: other diff --git a/changelogs/unreleased/8558-use-custom-modsecurity-ingress-config.yml b/changelogs/unreleased/8558-use-custom-modsecurity-ingress-config.yml new file mode 100644 index 00000000000..180715e89d6 --- /dev/null +++ b/changelogs/unreleased/8558-use-custom-modsecurity-ingress-config.yml @@ -0,0 +1,5 @@ +--- +title: Add modsecurity template for ingress-controller +merge_request: 18485 +author: +type: changed diff --git a/db/migrate/20191009100244_add_geo_design_repository_counters.rb b/db/migrate/20191009100244_add_geo_design_repository_counters.rb new file mode 100644 index 00000000000..26387453f88 --- /dev/null +++ b/db/migrate/20191009100244_add_geo_design_repository_counters.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddGeoDesignRepositoryCounters < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + change_table :geo_node_statuses do |t| + t.column :design_repositories_count, :integer + t.column :design_repositories_synced_count, :integer + t.column :design_repositories_failed_count, :integer + t.column :design_repositories_registry_count, :integer + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 237f5ab4732..36f7bba904b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1650,6 +1650,10 @@ ActiveRecord::Schema.define(version: 2019_10_17_045817) do t.integer "container_repositories_synced_count" t.integer "container_repositories_failed_count" t.integer "container_repositories_registry_count" + t.integer "design_repositories_count" + t.integer "design_repositories_synced_count" + t.integer "design_repositories_failed_count" + t.integer "design_repositories_registry_count" t.index ["geo_node_id"], name: "index_geo_node_statuses_on_geo_node_id", unique: true end diff --git a/doc/administration/geo/replication/using_a_geo_server.md b/doc/administration/geo/replication/using_a_geo_server.md index 55c7e78da92..37982f2756c 100644 --- a/doc/administration/geo/replication/using_a_geo_server.md +++ b/doc/administration/geo/replication/using_a_geo_server.md @@ -10,8 +10,12 @@ Example of the output you will see when pushing to a **secondary** node: ```bash $ git push -> GitLab: You're pushing to a Geo secondary. -> GitLab: We'll help you by proxying this request to the primary: ssh://git@primary.geo/user/repo.git +remote: +remote: You're pushing to a Geo secondary. We'll help you by proxying this +remote: request to the primary: +remote: +remote: ssh://git@primary.geo/user/repo.git +remote: Everything up-to-date ``` diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index 5eba7f038ed..e44d69f1419 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -237,6 +237,10 @@ Example response: "container_repositories_synced_count": nil, "container_repositories_failed_count": nil, "container_repositories_synced_in_percentage": "0.00%", + "design_repositories_count": 3, + "design_repositories_synced_count": nil, + "design_repositories_failed_count": nil, + "design_repositories_synced_in_percentage": "0.00%", "projects_count": 41, "repositories_failed_count": nil, "repositories_synced_count": nil, @@ -304,6 +308,10 @@ Example response: "container_repositories_synced_count": nil, "container_repositories_failed_count": nil, "container_repositories_synced_in_percentage": "0.00%", + "design_repositories_count": 3, + "design_repositories_synced_count": nil, + "design_repositories_failed_count": nil, + "design_repositories_synced_in_percentage": "0.00%", "projects_count": 41, "repositories_failed_count": 1, "repositories_synced_count": 40, @@ -387,6 +395,10 @@ Example response: "container_repositories_synced_count": nil, "container_repositories_failed_count": nil, "container_repositories_synced_in_percentage": "0.00%", + "design_repositories_count": 3, + "design_repositories_synced_count": nil, + "design_repositories_failed_count": nil, + "design_repositories_synced_in_percentage": "0.00%", "projects_count": 41, "repositories_failed_count": 1, "repositories_synced_count": 40, diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index d9a22484c1f..c70f2f3e2c8 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -77,7 +77,7 @@ module API response_with_status(**payload) when ::Gitlab::GitAccessResult::CustomAction - response_with_status(code: 300, message: check_result.message, payload: check_result.payload) + response_with_status(code: 300, payload: check_result.payload, gl_console_messages: check_result.console_messages) else response_with_status(code: 500, success: false, message: UNKNOWN_CHECK_RESULT_ERROR) end diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb index 33cbe1a62ef..9ea20a4d6a4 100644 --- a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb +++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb @@ -68,3 +68,5 @@ module Gitlab end end end + +Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder') diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index 90d03142b2a..2662aa38d6b 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -130,3 +130,5 @@ module Gitlab end end end + +Gitlab::Analytics::CycleAnalytics::RecordsFetcher.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::RecordsFetcher') diff --git a/lib/gitlab/git_access_result/custom_action.rb b/lib/gitlab/git_access_result/custom_action.rb index a05a4baed82..336f3405f72 100644 --- a/lib/gitlab/git_access_result/custom_action.rb +++ b/lib/gitlab/git_access_result/custom_action.rb @@ -3,7 +3,7 @@ module Gitlab module GitAccessResult class CustomAction - attr_reader :payload, :message + attr_reader :payload, :console_messages # Example of payload: # @@ -16,9 +16,9 @@ module Gitlab # } # } # - def initialize(payload, message) + def initialize(payload, console_messages) @payload = payload - @message = message + @console_messages = console_messages end end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index e6146b9e520..083e8f2a7fd 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -7493,6 +7493,9 @@ msgstr "" msgid "GeoNodes|Data replication lag" msgstr "" +msgid "GeoNodes|Design repositories" +msgstr "" + msgid "GeoNodes|Does not match the primary storage configuration" msgstr "" diff --git a/spec/models/clusters/applications/ingress_spec.rb b/spec/models/clusters/applications/ingress_spec.rb index 9806b49d7a6..60b139a7610 100644 --- a/spec/models/clusters/applications/ingress_spec.rb +++ b/spec/models/clusters/applications/ingress_spec.rb @@ -156,6 +156,15 @@ describe Clusters::Applications::Ingress do it 'includes modsecurity core ruleset enablement' do expect(subject.values).to include("enable-owasp-modsecurity-crs: 'true'") end + + it 'includes modsecurity.conf content' do + expect(subject.values).to include('modsecurity.conf') + # Includes file content from Ingress#modsecurity_config_content + expect(subject.values).to include('SecAuditLog') + + expect(subject.values).to include('extraVolumes') + expect(subject.values).to include('extraVolumeMounts') + end end context 'when ingress_modsecurity is disabled' do @@ -172,6 +181,15 @@ describe Clusters::Applications::Ingress do it 'excludes modsecurity core ruleset enablement' do expect(subject.values).not_to include('enable-owasp-modsecurity-crs') end + + it 'excludes modsecurity.conf content' do + expect(subject.values).not_to include('modsecurity.conf') + # Excludes file content from Ingress#modsecurity_config_content + expect(subject.values).not_to include('SecAuditLog') + + expect(subject.values).not_to include('extraVolumes') + expect(subject.values).not_to include('extraVolumeMounts') + end end end end diff --git a/spec/requests/api/internal/base_spec.rb b/spec/requests/api/internal/base_spec.rb index 01a2e33c0d9..63b75b810dc 100644 --- a/spec/requests/api/internal/base_spec.rb +++ b/spec/requests/api/internal/base_spec.rb @@ -407,7 +407,6 @@ describe API::Internal::Base do context "custom action" do let(:access_checker) { double(Gitlab::GitAccess) } - let(:message) { 'CustomActionError message' } let(:payload) do { 'action' => 'geo_proxy_to_primary', @@ -418,8 +417,8 @@ describe API::Internal::Base do } } end - - let(:custom_action_result) { Gitlab::GitAccessResult::CustomAction.new(payload, message) } + let(:console_messages) { ['informational message'] } + let(:custom_action_result) { Gitlab::GitAccessResult::CustomAction.new(payload, console_messages) } before do project.add_guest(user) @@ -446,8 +445,8 @@ describe API::Internal::Base do expect(response).to have_gitlab_http_status(300) expect(json_response['status']).to be_truthy - expect(json_response['message']).to eql(message) expect(json_response['payload']).to eql(payload) + expect(json_response['gl_console_messages']).to eql(console_messages) expect(user.reload.last_activity_on).to be_nil end end diff --git a/vendor/ingress/modsecurity.conf b/vendor/ingress/modsecurity.conf new file mode 100644 index 00000000000..ee702a50ed5 --- /dev/null +++ b/vendor/ingress/modsecurity.conf @@ -0,0 +1,273 @@ +# -- GitLab Customization ---------------------------------------------- +# Based on https://github.com/SpiderLabs/ModSecurity/blob/v3.0.3/modsecurity.conf-recommended +# Our base modsecurity.conf includes some minor customization: +# - `SecRuleEngine` is disabled, defaulting to `DetectionOnly`. Overridable at project-level +# - `SecAuditLogType` is disabled, defaulting to `Serial`. Overridable at project-level +# - `SecStatusEngine` is disabled, to disallow usage reporting +# +# ---------------------------------------------------------------------------- + +# -- Rule engine initialization ---------------------------------------------- + +# Enable ModSecurity, attaching it to every transaction. Use detection +# only to start with, because that minimises the chances of post-installation +# disruption. +# +# SecRuleEngine DetectionOnly + + +# -- Request body handling --------------------------------------------------- + +# Allow ModSecurity to access request bodies. If you don't, ModSecurity +# won't be able to see any POST parameters, which opens a large security +# hole for attackers to exploit. +# +SecRequestBodyAccess On + + +# Enable XML request body parser. +# Initiate XML Processor in case of xml content-type +# +SecRule REQUEST_HEADERS:Content-Type "(?:application(?:/soap\+|/)|text/)xml" \ + "id:'200000',phase:1,t:none,t:lowercase,pass,nolog,ctl:requestBodyProcessor=XML" + +# Enable JSON request body parser. +# Initiate JSON Processor in case of JSON content-type; change accordingly +# if your application does not use 'application/json' +# +SecRule REQUEST_HEADERS:Content-Type "application/json" \ + "id:'200001',phase:1,t:none,t:lowercase,pass,nolog,ctl:requestBodyProcessor=JSON" + +# Maximum request body size we will accept for buffering. If you support +# file uploads then the value given on the first line has to be as large +# as the largest file you are willing to accept. The second value refers +# to the size of data, with files excluded. You want to keep that value as +# low as practical. +# +SecRequestBodyLimit 13107200 +SecRequestBodyNoFilesLimit 131072 + +# What do do if the request body size is above our configured limit. +# Keep in mind that this setting will automatically be set to ProcessPartial +# when SecRuleEngine is set to DetectionOnly mode in order to minimize +# disruptions when initially deploying ModSecurity. +# +SecRequestBodyLimitAction Reject + +# Verify that we've correctly processed the request body. +# As a rule of thumb, when failing to process a request body +# you should reject the request (when deployed in blocking mode) +# or log a high-severity alert (when deployed in detection-only mode). +# +SecRule REQBODY_ERROR "!@eq 0" \ +"id:'200002', phase:2,t:none,log,deny,status:400,msg:'Failed to parse request body.',logdata:'%{reqbody_error_msg}',severity:2" + +# By default be strict with what we accept in the multipart/form-data +# request body. If the rule below proves to be too strict for your +# environment consider changing it to detection-only. You are encouraged +# _not_ to remove it altogether. +# +SecRule MULTIPART_STRICT_ERROR "!@eq 0" \ +"id:'200003',phase:2,t:none,log,deny,status:400, \ +msg:'Multipart request body failed strict validation: \ +PE %{REQBODY_PROCESSOR_ERROR}, \ +BQ %{MULTIPART_BOUNDARY_QUOTED}, \ +BW %{MULTIPART_BOUNDARY_WHITESPACE}, \ +DB %{MULTIPART_DATA_BEFORE}, \ +DA %{MULTIPART_DATA_AFTER}, \ +HF %{MULTIPART_HEADER_FOLDING}, \ +LF %{MULTIPART_LF_LINE}, \ +SM %{MULTIPART_MISSING_SEMICOLON}, \ +IQ %{MULTIPART_INVALID_QUOTING}, \ +IP %{MULTIPART_INVALID_PART}, \ +IH %{MULTIPART_INVALID_HEADER_FOLDING}, \ +FL %{MULTIPART_FILE_LIMIT_EXCEEDED}'" + +# Did we see anything that might be a boundary? +# +# Here is a short description about the ModSecurity Multipart parser: the +# parser returns with value 0, if all "boundary-like" line matches with +# the boundary string which given in MIME header. In any other cases it returns +# with different value, eg. 1 or 2. +# +# The RFC 1341 descript the multipart content-type and its syntax must contains +# only three mandatory lines (above the content): +# * Content-Type: multipart/mixed; boundary=BOUNDARY_STRING +# * --BOUNDARY_STRING +# * --BOUNDARY_STRING-- +# +# First line indicates, that this is a multipart content, second shows that +# here starts a part of the multipart content, third shows the end of content. +# +# If there are any other lines, which starts with "--", then it should be +# another boundary id - or not. +# +# After 3.0.3, there are two kinds of types of boundary errors: strict and permissive. +# +# If multipart content contains the three necessary lines with correct order, but +# there are one or more lines with "--", then parser returns with value 2 (non-zero). +# +# If some of the necessary lines (usually the start or end) misses, or the order +# is wrong, then parser returns with value 1 (also a non-zero). +# +# You can choose, which one is what you need. The example below contains the +# 'strict' mode, which means if there are any lines with start of "--", then +# ModSecurity blocked the content. But the next, commented example contains +# the 'permissive' mode, then you check only if the necessary lines exists in +# correct order. Whit this, you can enable to upload PEM files (eg "----BEGIN.."), +# or other text files, which contains eg. HTTP headers. +# +# The difference is only the operator - in strict mode (first) the content blocked +# in case of any non-zero value. In permissive mode (second, commented) the +# content blocked only if the value is explicit 1. If it 0 or 2, the content will +# allowed. +# + +# +# See #1747 and #1924 for further information on the possible values for +# MULTIPART_UNMATCHED_BOUNDARY. +# +SecRule MULTIPART_UNMATCHED_BOUNDARY "@eq 1" \ + "id:'200004',phase:2,t:none,log,deny,msg:'Multipart parser detected a possible unmatched boundary.'" + + +# PCRE Tuning +# We want to avoid a potential RegEx DoS condition +# +SecPcreMatchLimit 1000 +SecPcreMatchLimitRecursion 1000 + +# Some internal errors will set flags in TX and we will need to look for these. +# All of these are prefixed with "MSC_". The following flags currently exist: +# +# MSC_PCRE_LIMITS_EXCEEDED: PCRE match limits were exceeded. +# +SecRule TX:/^MSC_/ "!@streq 0" \ + "id:'200005',phase:2,t:none,deny,msg:'ModSecurity internal error flagged: %{MATCHED_VAR_NAME}'" + + +# -- Response body handling -------------------------------------------------- + +# Allow ModSecurity to access response bodies. +# You should have this directive enabled in order to identify errors +# and data leakage issues. +# +# Do keep in mind that enabling this directive does increases both +# memory consumption and response latency. +# +SecResponseBodyAccess On + +# Which response MIME types do you want to inspect? You should adjust the +# configuration below to catch documents but avoid static files +# (e.g., images and archives). +# +SecResponseBodyMimeType text/plain text/html text/xml + +# Buffer response bodies of up to 512 KB in length. +SecResponseBodyLimit 524288 + +# What happens when we encounter a response body larger than the configured +# limit? By default, we process what we have and let the rest through. +# That's somewhat less secure, but does not break any legitimate pages. +# +SecResponseBodyLimitAction ProcessPartial + + +# -- Filesystem configuration ------------------------------------------------ + +# The location where ModSecurity stores temporary files (for example, when +# it needs to handle a file upload that is larger than the configured limit). +# +# This default setting is chosen due to all systems have /tmp available however, +# this is less than ideal. It is recommended that you specify a location that's private. +# +SecTmpDir /tmp/ + +# The location where ModSecurity will keep its persistent data. This default setting +# is chosen due to all systems have /tmp available however, it +# too should be updated to a place that other users can't access. +# +SecDataDir /tmp/ + + +# -- File uploads handling configuration ------------------------------------- + +# The location where ModSecurity stores intercepted uploaded files. This +# location must be private to ModSecurity. You don't want other users on +# the server to access the files, do you? +# +#SecUploadDir /opt/modsecurity/var/upload/ + +# By default, only keep the files that were determined to be unusual +# in some way (by an external inspection script). For this to work you +# will also need at least one file inspection rule. +# +#SecUploadKeepFiles RelevantOnly + +# Uploaded files are by default created with permissions that do not allow +# any other user to access them. You may need to relax that if you want to +# interface ModSecurity to an external program (e.g., an anti-virus). +# +#SecUploadFileMode 0600 + + +# -- Debug log configuration ------------------------------------------------- + +# The default debug log configuration is to duplicate the error, warning +# and notice messages from the error log. +# +#SecDebugLog /opt/modsecurity/var/log/debug.log +#SecDebugLogLevel 3 + + +# -- Audit log configuration ------------------------------------------------- + +# Log the transactions that are marked by a rule, as well as those that +# trigger a server error (determined by a 5xx or 4xx, excluding 404, +# level response status codes). +# +SecAuditEngine RelevantOnly +SecAuditLogRelevantStatus "^(?:5|4(?!04))" + +# Log everything we know about a transaction. +SecAuditLogParts ABIJDEFHZ + +# Use a single file for logging. This is much easier to look at, but +# assumes that you will use the audit log only ocassionally. +# +# SecAuditLogType Serial +SecAuditLog /var/log/modsec_audit.log + +# Specify the path for concurrent audit logging. +#SecAuditLogStorageDir /opt/modsecurity/var/audit/ + + +# -- Miscellaneous ----------------------------------------------------------- + +# Use the most commonly used application/x-www-form-urlencoded parameter +# separator. There's probably only one application somewhere that uses +# something else so don't expect to change this value. +# +SecArgumentSeparator & + +# Settle on version 0 (zero) cookies, as that is what most applications +# use. Using an incorrect cookie version may open your installation to +# evasion attacks (against the rules that examine named cookies). +# +SecCookieFormat 0 + +# Specify your Unicode Code Point. +# This mapping is used by the t:urlDecodeUni transformation function +# to properly map encoded data to your language. Properly setting +# these directives helps to reduce false positives and negatives. +# +SecUnicodeMapFile unicode.mapping 20127 + +# Improve the quality of ModSecurity by sharing information about your +# current ModSecurity version and dependencies versions. +# The following information will be shared: ModSecurity version, +# Web Server version, APR version, PCRE version, Lua version, Libxml2 +# version, Anonymous unique id for host. +# SecStatusEngine On + + |