diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-12-20 12:07:57 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-12-20 12:07:57 +0300 |
commit | 7881eb30eaa8b01dbcfe87faa09927c75c7d6e45 (patch) | |
tree | 298bc8d2c62b2f2c29cb8ecbcf3de3eaaa6466d9 /config | |
parent | 64b66e0cb6d1bfd27abf24e06653f00bddb60597 (diff) |
Add latest changes from gitlab-org/gitlab@12-6-stable-ee
Diffstat (limited to 'config')
33 files changed, 544 insertions, 431 deletions
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index a5486e450d4..5ac9b7ee6e5 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -181,6 +181,11 @@ production: &base mailbox: "inbox" # The IDLE command timeout. idle_timeout: 60 + # The log file path for the structured log file. + # Since `mail_room` is run independently of Rails, an absolute path is preferred. + # The default is 'log/mail_room_json.log' relative to the root of the Rails app. + # + # log_path: log/mail_room_json.log ## Build Artifacts artifacts: @@ -321,8 +326,8 @@ production: &base # external_https: ["1.1.1.1:443", "[2001::1]:443"] # If defined, enables custom domain and certificate support in GitLab Pages # File that contains the shared secret key for verifying access for gitlab-pages. - # Default is '.gitlab_pages_shared_secret' relative to Rails.root (i.e. root of the GitLab app). - # secret_file: /home/git/gitlab/.gitlab_pages_shared_secret + # Default is '.gitlab_pages_secret' relative to Rails.root (i.e. root of the GitLab app). + # secret_file: /home/git/gitlab/.gitlab_pages_secret ## Mattermost ## For enabling Add to Mattermost button @@ -366,6 +371,9 @@ production: &base # Send admin emails once a week admin_email_worker: cron: "0 0 * * 0" + # Send emails for personal tokens which are about to expire + personal_access_tokens_expiring_worker: + cron: "0 1 * * *" # Remove outdated repository archives repository_archive_cache_worker: diff --git a/config/helpers/vendor_dll_hash.js b/config/helpers/vendor_dll_hash.js new file mode 100644 index 00000000000..cfd7be66ad3 --- /dev/null +++ b/config/helpers/vendor_dll_hash.js @@ -0,0 +1,23 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); + +const CACHE_PATHS = [ + './config/webpack.config.js', + './config/webpack.vendor.config.js', + './package.json', + './yarn.lock', +]; + +const resolvePath = file => path.resolve(__dirname, '../..', file); +const readFile = file => fs.readFileSync(file); +const fileHash = buffer => + crypto + .createHash('md5') + .update(buffer) + .digest('hex'); + +module.exports = () => { + const fileBuffers = CACHE_PATHS.map(resolvePath).map(readFile); + return fileHash(Buffer.concat(fileBuffers)).substr(0, 12); +}; diff --git a/config/initializers/0_marginalia.rb b/config/initializers/0_marginalia.rb new file mode 100644 index 00000000000..f88a90854e3 --- /dev/null +++ b/config/initializers/0_marginalia.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'marginalia' + +::Marginalia::Comment.extend(::Gitlab::Marginalia::Comment) + +# Patch to modify 'Marginalia::ActiveRecordInstrumentation.annotate_sql' method with feature check. +# Orignal Marginalia::ActiveRecordInstrumentation is included to ActiveRecord::ConnectionAdapters::PostgreSQLAdapter in the Marginalia Railtie. +# Refer: https://github.com/basecamp/marginalia/blob/v1.8.0/lib/marginalia/railtie.rb#L67 +ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(Gitlab::Marginalia::ActiveRecordInstrumentation) + +Marginalia::Comment.components = [:application, :controller, :action, :correlation_id, :jid, :job_class, :line] + +Gitlab::Marginalia.set_application_name + +Gitlab::Marginalia.enable_sidekiq_instrumentation + +Gitlab::Marginalia.set_feature_cache diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index df4f49524bc..691e4339bf0 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -291,7 +291,7 @@ Settings.pages['url'] ||= Settings.__send__(:build_pages_url) Settings.pages['external_http'] ||= false unless Settings.pages['external_http'].present? Settings.pages['external_https'] ||= false unless Settings.pages['external_https'].present? Settings.pages['artifacts_server'] ||= Settings.pages['enabled'] if Settings.pages['artifacts_server'].nil? -Settings.pages['secret_file'] ||= Rails.root.join('.gitlab_pages_shared_secret') +Settings.pages['secret_file'] ||= Rails.root.join('.gitlab_pages_secret') # # Geo @@ -407,6 +407,9 @@ Settings.cron_jobs['repository_check_worker']['job_class'] = 'RepositoryCheck::D Settings.cron_jobs['admin_email_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['admin_email_worker']['cron'] ||= '0 0 * * 0' Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker' +Settings.cron_jobs['personal_access_tokens_expiring_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['personal_access_tokens_expiring_worker']['cron'] ||= '0 1 * * *' +Settings.cron_jobs['personal_access_tokens_expiring_worker']['job_class'] = 'PersonalAccessTokens::ExpiringWorker' Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker' @@ -466,9 +469,15 @@ Settings.cron_jobs['namespaces_prune_aggregation_schedules_worker']['cron'] ||= Settings.cron_jobs['namespaces_prune_aggregation_schedules_worker']['job_class'] = 'Namespaces::PruneAggregationSchedulesWorker' Gitlab.ee do + Settings.cron_jobs['adjourned_group_deletion_worker'] ||= Settingslogic.new({}) + Settings.cron_jobs['adjourned_group_deletion_worker']['cron'] ||= '0 3 * * *' + Settings.cron_jobs['adjourned_group_deletion_worker']['job_class'] = 'AdjournedGroupDeletionWorker' Settings.cron_jobs['clear_shared_runners_minutes_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['clear_shared_runners_minutes_worker']['cron'] ||= '0 0 1 * *' Settings.cron_jobs['clear_shared_runners_minutes_worker']['job_class'] = 'ClearSharedRunnersMinutesWorker' + Settings.cron_jobs['adjourned_projects_deletion_cron_worker'] ||= Settingslogic.new({}) + Settings.cron_jobs['adjourned_projects_deletion_cron_worker']['cron'] ||= '0 4 * * *' + Settings.cron_jobs['adjourned_projects_deletion_cron_worker']['job_class'] = 'AdjournedProjectsDeletionCronWorker' Settings.cron_jobs['geo_file_download_dispatch_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['geo_file_download_dispatch_worker']['cron'] ||= '*/1 * * * *' Settings.cron_jobs['geo_file_download_dispatch_worker']['job_class'] ||= 'Geo::FileDownloadDispatchWorker' @@ -645,6 +654,7 @@ Settings.rack_attack.git_basic_auth['ip_whitelist'] ||= %w{127.0.0.1} Settings.rack_attack.git_basic_auth['maxretry'] ||= 10 Settings.rack_attack.git_basic_auth['findtime'] ||= 1.minute Settings.rack_attack.git_basic_auth['bantime'] ||= 1.hour +Settings.rack_attack['admin_area_protected_paths_enabled'] ||= false # # Gitaly diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb index d40049970c1..c14ee1458bc 100644 --- a/config/initializers/7_prometheus_metrics.rb +++ b/config/initializers/7_prometheus_metrics.rb @@ -32,15 +32,8 @@ end Sidekiq.configure_server do |config| config.on(:startup) do - # webserver metrics are cleaned up in config.ru: `warmup` block - Prometheus::CleanupMultiprocDirService.new.execute - # In production, sidekiq is run in a multi-process setup where processes might interfere - # with each other cleaning up and reinitializing prometheus database files, which is why - # we're re-doing the work every time here. - # A cleaner solution would be to run the cleanup pre-fork, and the initialization once - # after all workers have forked, but I don't know how at this point. - ::Prometheus::Client.reinitialize_on_pid_change(force: true) - + # Do not clean the metrics directory here - the supervisor script should + # have already taken care of that Gitlab::Metrics::Exporter::SidekiqExporter.instance.start end end diff --git a/config/initializers/devise_dynamic_password_length_validation.rb b/config/initializers/devise_dynamic_password_length_validation.rb new file mode 100644 index 00000000000..03d613da929 --- /dev/null +++ b/config/initializers/devise_dynamic_password_length_validation.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# Discard the default Devise length validation from the `User` model. + +# This needs to be discarded because the length validation provided by Devise does not +# support dynamically checking for min and max lengths. + +# A new length validation has been added to the User model instead, to keep supporting +# dynamic password length validations, like: + +# validates :password, length: { maximum: proc { password_length.max }, minimum: proc { password_length.min } }, allow_blank: true + +def length_validator_supports_dynamic_length_checks?(validator) + validator.options[:minimum].is_a?(Proc) && + validator.options[:maximum].is_a?(Proc) +end + +# Get the in-built Devise validator on password length. +password_length_validator = User.validators_on(:password).find do |validator| + validator.kind == :length +end + +# This initializer can be removed as soon as https://github.com/plataformatec/devise/pull/5166 +# is merged into Devise. + +# TODO: Update Devise. Issue: https://gitlab.com/gitlab-org/gitlab/issues/118450 +if length_validator_supports_dynamic_length_checks?(password_length_validator) + raise "Devise now supports dynamic length checks, please remove the monkey patch in #{__FILE__}" +else + # discard the in-built length validator by always returning true + def password_length_validator.validate(*_) + true + end + + # add a custom password length validator with support for dynamic length validation. + User.class_eval do + validates :password, length: { maximum: proc { password_length.max }, minimum: proc { password_length.min } }, allow_blank: true + end +end diff --git a/config/initializers/direct_upload_support.rb b/config/initializers/direct_upload_support.rb index 32fc8c8bc69..0fc6e82207e 100644 --- a/config/initializers/direct_upload_support.rb +++ b/config/initializers/direct_upload_support.rb @@ -3,17 +3,35 @@ class DirectUploadsValidator ValidationError = Class.new(StandardError) - def verify!(object_store) + def verify!(uploader_type, object_store) return unless object_store.enabled return unless object_store.direct_upload - return if SUPPORTED_DIRECT_UPLOAD_PROVIDERS.include?(object_store.connection&.provider.to_s) - raise ValidationError, "Only #{SUPPORTED_DIRECT_UPLOAD_PROVIDERS.join(',')} are supported as a object storage provider when 'direct_upload' is used" + raise ValidationError, "Object storage is configured for '#{uploader_type}', but the 'connection' section is missing" unless object_store.key?('connection') + + provider = object_store.connection&.provider.to_s + + raise ValidationError, "No provider configured for '#{uploader_type}'. #{supported_provider_text}" if provider.blank? + + return if SUPPORTED_DIRECT_UPLOAD_PROVIDERS.include?(provider) + + raise ValidationError, "Object storage provider '#{provider}' is not supported " \ + "when 'direct_upload' is used for '#{uploader_type}'. #{supported_provider_text}" + end + + def supported_provider_text + "Only #{SUPPORTED_DIRECT_UPLOAD_PROVIDERS.join(', ')} are supported." end end DirectUploadsValidator.new.tap do |validator| - [Gitlab.config.artifacts, Gitlab.config.uploads, Gitlab.config.lfs].each do |uploader| - validator.verify!(uploader.object_store) + CONFIGS = { + artifacts: Gitlab.config.artifacts, + uploads: Gitlab.config.uploads, + lfs: Gitlab.config.lfs + }.freeze + + CONFIGS.each do |uploader_type, uploader| + validator.verify!(uploader_type, uploader.object_store) end end diff --git a/config/initializers/elastic_client_setup.rb b/config/initializers/elastic_client_setup.rb index a1abb29838b..f38b606b3a8 100644 --- a/config/initializers/elastic_client_setup.rb +++ b/config/initializers/elastic_client_setup.rb @@ -7,6 +7,17 @@ require 'gitlab/current_settings' Gitlab.ee do require 'elasticsearch/model' + ### Monkey patches + + Elasticsearch::Model::Response::Records.prepend GemExtensions::Elasticsearch::Model::Response::Records + Elasticsearch::Model::Adapter::Multiple::Records.prepend GemExtensions::Elasticsearch::Model::Adapter::Multiple::Records + Elasticsearch::Model::Indexing::InstanceMethods.prepend GemExtensions::Elasticsearch::Model::Indexing::InstanceMethods + Elasticsearch::Model::Adapter::ActiveRecord::Importing.prepend GemExtensions::Elasticsearch::Model::Adapter::ActiveRecord::Importing + Elasticsearch::Model::Client::InstanceMethods.prepend GemExtensions::Elasticsearch::Model::Client + Elasticsearch::Model::Client::ClassMethods.prepend GemExtensions::Elasticsearch::Model::Client + Elasticsearch::Model::ClassMethods.prepend GemExtensions::Elasticsearch::Model::Client + Elasticsearch::Model.singleton_class.prepend GemExtensions::Elasticsearch::Model::Client + ### Modified from elasticsearch-model/lib/elasticsearch/model.rb [ @@ -32,15 +43,4 @@ Gitlab.ee do target.respond_to?(:as_indexed_json) ? target.__send__(:as_indexed_json, options) : super end CODE - - ### Monkey patches - - Elasticsearch::Model::Response::Records.prepend GemExtensions::Elasticsearch::Model::Response::Records - Elasticsearch::Model::Adapter::Multiple::Records.prepend GemExtensions::Elasticsearch::Model::Adapter::Multiple::Records - Elasticsearch::Model::Indexing::InstanceMethods.prepend GemExtensions::Elasticsearch::Model::Indexing::InstanceMethods - Elasticsearch::Model::Adapter::ActiveRecord::Importing.prepend GemExtensions::Elasticsearch::Model::Adapter::ActiveRecord::Importing - Elasticsearch::Model::Client::InstanceMethods.prepend GemExtensions::Elasticsearch::Model::Client - Elasticsearch::Model::Client::ClassMethods.prepend GemExtensions::Elasticsearch::Model::Client - Elasticsearch::Model::ClassMethods.prepend GemExtensions::Elasticsearch::Model::Client - Elasticsearch::Model.singleton_class.prepend GemExtensions::Elasticsearch::Model::Client end diff --git a/config/initializers/forbid_sidekiq_in_transactions.rb b/config/initializers/forbid_sidekiq_in_transactions.rb index bb190af60b5..9bade443aae 100644 --- a/config/initializers/forbid_sidekiq_in_transactions.rb +++ b/config/initializers/forbid_sidekiq_in_transactions.rb @@ -29,7 +29,7 @@ module Sidekiq MSG rescue Sidekiq::Worker::EnqueueFromTransactionError => e ::Rails.logger.error(e.message) if ::Rails.env.production? - Gitlab::Sentry.track_exception(e) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end end diff --git a/config/initializers/hamlit.rb b/config/initializers/hamlit.rb index 51dbffeda05..b5bcae4bbfc 100644 --- a/config/initializers/hamlit.rb +++ b/config/initializers/hamlit.rb @@ -1,18 +1,4 @@ -module Hamlit - class TemplateHandler - def call(template) - Engine.new( - generator: Temple::Generators::RailsOutputBuffer, - attr_quote: '"' - ).call(template.source) - end - end -end - -ActionView::Template.register_template_handler( - :haml, - Hamlit::TemplateHandler.new -) +Hamlit::RailsTemplate.set_options(attr_quote: '"') Hamlit::Filters.remove_filter('coffee') Hamlit::Filters.remove_filter('coffeescript') diff --git a/config/initializers/rack_attack_git_basic_auth.rb b/config/initializers/rack_attack_git_basic_auth.rb deleted file mode 100644 index 71e5e2969ce..00000000000 --- a/config/initializers/rack_attack_git_basic_auth.rb +++ /dev/null @@ -1,14 +0,0 @@ -# Tell the Rack::Attack Rack middleware to maintain an IP blacklist. -# We update the blacklist in Gitlab::Auth::IpRateLimiter. -Rack::Attack.blocklist('Git HTTP Basic Auth') do |req| - rate_limiter = Gitlab::Auth::IpRateLimiter.new(req.ip) - - next false if !rate_limiter.enabled? || rate_limiter.trusted_ip? - - Rack::Attack::Allow2Ban.filter(req.ip, Gitlab.config.rack_attack.git_basic_auth) do - # This block only gets run if the IP was not already banned. - # Return false, meaning that we do not see anything wrong with the - # request at this time - false - end -end diff --git a/config/initializers/rack_attack_new.rb b/config/initializers/rack_attack_new.rb index 92a8bf79432..267d4c1eda9 100644 --- a/config/initializers/rack_attack_new.rb +++ b/config/initializers/rack_attack_new.rb @@ -1,11 +1,22 @@ +# Specs for this file can be found on: +# * spec/lib/gitlab/throttle_spec.rb +# * spec/requests/rack_attack_global_spec.rb module Gitlab::Throttle def self.settings Gitlab::CurrentSettings.current_application_settings end + # Returns true if we should use the Admin Area protected paths throttle def self.protected_paths_enabled? - !self.omnibus_protected_paths_present? && - self.settings.throttle_protected_paths_enabled? + return false if should_use_omnibus_protected_paths? + + self.settings.throttle_protected_paths_enabled? + end + + # To be removed in 13.0: https://gitlab.com/gitlab-org/gitlab/issues/29952 + def self.should_use_omnibus_protected_paths? + !Settings.rack_attack.admin_area_protected_paths_enabled && + self.omnibus_protected_paths_present? end def self.omnibus_protected_paths_present? @@ -102,11 +113,15 @@ class Rack::Attack class Request def unauthenticated? - !authenticated_user_id([:api, :rss, :ics]) + !(authenticated_user_id([:api, :rss, :ics]) || authenticated_runner_id) end def authenticated_user_id(request_formats) - Gitlab::Auth::RequestAuthenticator.new(self).user(request_formats)&.id + request_authenticator.user(request_formats)&.id + end + + def authenticated_runner_id + request_authenticator.runner&.id end def api_request? @@ -139,6 +154,10 @@ class Rack::Attack private + def request_authenticator + @request_authenticator ||= Gitlab::Auth::RequestAuthenticator.new(self) + end + def protected_paths Gitlab::CurrentSettings.current_application_settings.protected_paths end diff --git a/config/initializers/sentry.rb b/config/initializers/sentry.rb index 48daca3d254..a1eddd6a2c2 100644 --- a/config/initializers/sentry.rb +++ b/config/initializers/sentry.rb @@ -2,35 +2,4 @@ require 'gitlab/current_settings' -def configure_sentry - if Gitlab::Sentry.enabled? - Raven.configure do |config| - config.dsn = Gitlab.config.sentry.dsn - config.release = Gitlab.revision - config.current_environment = Gitlab.config.sentry.environment - - # Sanitize fields based on those sanitized from Rails. - config.sanitize_fields = Rails.application.config.filter_parameters.map(&:to_s) - # Sanitize authentication headers - config.sanitize_http_headers = %w[Authorization Private-Token] - config.tags = { program: Gitlab.process_name } - # Debugging for https://gitlab.com/gitlab-org/gitlab-foss/issues/57727 - config.before_send = lambda do |event, hint| - if ActiveModel::MissingAttributeError === hint[:exception] - columns_hash = ActiveRecord::Base - .connection - .schema_cache - .instance_variable_get(:@columns_hash) - .map { |k, v| [k, v.map(&:first)] } - .to_h - - event.extra.merge!(columns_hash) - end - - event - end - end - end -end - -configure_sentry if Rails.env.production? || Rails.env.development? +Gitlab::ErrorTracking.configure diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index b5d98399015..b4a1e0da41a 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -31,21 +31,17 @@ enable_json_logs = Gitlab.config.sidekiq.log_format == 'json' enable_sidekiq_memory_killer = ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.nonzero? use_sidekiq_daemon_memory_killer = ENV["SIDEKIQ_DAEMON_MEMORY_KILLER"].to_i.nonzero? use_sidekiq_legacy_memory_killer = !use_sidekiq_daemon_memory_killer +use_request_store = ENV.fetch('SIDEKIQ_REQUEST_STORE', 1).to_i.nonzero? Sidekiq.configure_server do |config| config.redis = queues_config_hash - config.server_middleware do |chain| - chain.add Gitlab::SidekiqMiddleware::Monitor - chain.add Gitlab::SidekiqMiddleware::Metrics if Settings.monitoring.sidekiq_exporter - chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] && !enable_json_logs - chain.add Gitlab::SidekiqMiddleware::MemoryKiller if enable_sidekiq_memory_killer && use_sidekiq_legacy_memory_killer - chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware unless ENV['SIDEKIQ_REQUEST_STORE'] == '0' - chain.add Gitlab::SidekiqMiddleware::BatchLoader - chain.add Gitlab::SidekiqMiddleware::CorrelationLogger - chain.add Gitlab::SidekiqMiddleware::InstrumentationLogger - chain.add Gitlab::SidekiqStatus::ServerMiddleware - end + config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator({ + metrics: Settings.monitoring.sidekiq_exporter, + arguments_logger: ENV['SIDEKIQ_LOG_ARGUMENTS'] && !enable_json_logs, + memory_killer: enable_sidekiq_memory_killer && use_sidekiq_legacy_memory_killer, + request_store: use_request_store + })) if enable_json_logs Sidekiq.logger.formatter = Gitlab::SidekiqLogging::JSONFormatter.new @@ -56,10 +52,7 @@ Sidekiq.configure_server do |config| config.error_handlers << Gitlab::SidekiqLogging::ExceptionHandler.new end - config.client_middleware do |chain| - chain.add Gitlab::SidekiqStatus::ClientMiddleware - chain.add Gitlab::SidekiqMiddleware::CorrelationInjector - end + config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator) config.on :startup do # Clear any connections that might have been obtained before starting @@ -125,8 +118,5 @@ end Sidekiq.configure_client do |config| config.redis = queues_config_hash - config.client_middleware do |chain| - chain.add Gitlab::SidekiqMiddleware::CorrelationInjector - chain.add Gitlab::SidekiqStatus::ClientMiddleware - end + config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator) end diff --git a/config/initializers/snowplow_tracker.rb b/config/initializers/snowplow_tracker.rb new file mode 100644 index 00000000000..3c730e03738 --- /dev/null +++ b/config/initializers/snowplow_tracker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# Gitlab.com uses Snowplow for identifying users and events. +# https://gitlab.com/gitlab-org/gitlab/issues/6329 +# +# SnowplowTracker write log into STDERR +# https://github.com/snowplow/snowplow-ruby-tracker/blob/39fcfa2be793f2e25e73087a9700abc93f43b5e8/lib/snowplow-tracker/emitters.rb#L23 +# `LOGGER = Logger.new(STDERR)` +# +# In puma.rb, if `stdout_redirect` specify stderr, Puma will overwrite STDERR in: +# https://github.com/puma/puma/blob/b41205f5cacbc2ad0060472bdce68ba636f42175/lib/puma/runner.rb#L134 +# `STDERR.reopen stderr, (append ? "a" : "w")` +# As a result, SnowplowTracker will log into Puma stderr, when Puma enabled. +# +# By default, SnowplowTracker uses default log formatter. +# When enable Puma, SnowplowTracker log is expected to be JSON format, as part of puma_stderr.log. +# Hence overwrite ::SnowplowTracker::LOGGER.formatter to JSON formatter + +if defined?(::Puma) && defined?(::SnowplowTracker::LOGGER) + ::SnowplowTracker::LOGGER.formatter = proc do |severity, datetime, progname, msg| + { severity: severity, timestamp: datetime.utc.iso8601(3), pid: $$, progname: progname, message: msg }.to_json << "\n" + end +end diff --git a/config/initializers/zz_metrics.rb b/config/initializers/zz_metrics.rb index bc28780cc77..5bbfb97277c 100644 --- a/config/initializers/zz_metrics.rb +++ b/config/initializers/zz_metrics.rb @@ -8,6 +8,8 @@ # # rubocop:disable Metrics/AbcSize def instrument_classes(instrumentation) + return if ENV['STATIC_VERIFICATION'] + instrumentation.instrument_instance_methods(Gitlab::Shell) instrumentation.instrument_methods(Gitlab::Git) @@ -88,8 +90,8 @@ def instrument_classes(instrumentation) instrumentation.instrument_instance_methods(Gitlab::Highlight) Gitlab.ee do - instrumentation.instrument_methods(Elasticsearch::Git::Repository) - instrumentation.instrument_instance_methods(Elasticsearch::Git::Repository) + instrumentation.instrument_instance_methods(Elastic::Latest::GitInstanceProxy) + instrumentation.instrument_instance_methods(Elastic::Latest::GitClassProxy) instrumentation.instrument_instance_methods(Search::GlobalService) instrumentation.instrument_instance_methods(Search::ProjectService) diff --git a/config/knative/api_groups.yml b/config/knative/api_groups.yml new file mode 100644 index 00000000000..4697086eb5a --- /dev/null +++ b/config/knative/api_groups.yml @@ -0,0 +1,11 @@ +--- + +- networking.istio.io +- rbac.istio.io +- authentication.istio.io +- config.istio.io + +- networking.internal.knative.dev +- serving.knative.dev +- caching.internal.knative.dev +- autoscaling.internal.knative.dev
\ No newline at end of file diff --git a/config/knative/api_resources.yml b/config/knative/api_resources.yml deleted file mode 100644 index 095f44ed799..00000000000 --- a/config/knative/api_resources.yml +++ /dev/null @@ -1,70 +0,0 @@ ---- - -- meshpolicies.authentication.istio.io -- policies.authentication.istio.io -- adapters.config.istio.io -- apikeys.config.istio.io -- attributemanifests.config.istio.io -- authorizations.config.istio.io -- bypasses.config.istio.io -- podautoscalers.autoscaling.internal.knative.dev -- builds.build.knative.dev -- buildtemplates.build.knative.dev -- clusterbuildtemplates.build.knative.dev -- images.caching.internal.knative.dev -- certificates.networking.internal.knative.dev -- clusteringresses.networking.internal.knative.dev -- serverlessservices.networking.internal.knative.dev -- configurations.serving.knative.dev -- revisions.serving.knative.dev -- routes.serving.knative.dev -- services.serving.knative.dev -- checknothings.config.istio.io -- circonuses.config.istio.io -- deniers.config.istio.io -- edges.config.istio.io -- fluentds.config.istio.io -- handlers.config.istio.io -- httpapispecbindings.config.istio.io -- httpapispecs.config.istio.io -- instances.config.istio.io -- kubernetesenvs.config.istio.io -- kuberneteses.config.istio.io -- listcheckers.config.istio.io -- listentries.config.istio.io -- logentries.config.istio.io -- memquotas.config.istio.io -- metrics.config.istio.io -- noops.config.istio.io -- opas.config.istio.io -- prometheuses.config.istio.io -- quotas.config.istio.io -- quotaspecbindings.config.istio.io -- quotaspecs.config.istio.io -- rbacs.config.istio.io -- redisquotas.config.istio.io -- reportnothings.config.istio.io -- rules.config.istio.io -- servicecontrolreports.config.istio.io -- servicecontrols.config.istio.io -- signalfxs.config.istio.io -- solarwindses.config.istio.io -- stackdrivers.config.istio.io -- statsds.config.istio.io -- stdios.config.istio.io -- templates.config.istio.io -- tracespans.config.istio.io -- destinationrules.networking.istio.io -- envoyfilters.networking.istio.io -- gateways.networking.istio.io -- serviceentries.networking.istio.io -- virtualservices.networking.istio.io -- rbacconfigs.rbac.istio.io -- servicerolebindings.rbac.istio.io -- serviceroles.rbac.istio.io -- cloudwatches.config.istio.io -- clusterrbacconfigs.rbac.istio.io -- dogstatsds.config.istio.io -- ingresses.networking.internal.knative.dev -- sidecars.networking.istio.io -- zipkins.config.istio.io diff --git a/config/mail_room.yml b/config/mail_room.yml index c3a5be8d38c..75024c2b2e1 100644 --- a/config/mail_room.yml +++ b/config/mail_room.yml @@ -13,6 +13,8 @@ :email: <%= config[:user].to_json %> :password: <%= config[:password].to_json %> :idle_timeout: <%= config[:idle_timeout].to_json %> + :logger: + :log_path: <%= config[:log_path].to_json %> :name: <%= config[:mailbox].to_json %> diff --git a/config/prometheus/common_metrics.yml b/config/prometheus/common_metrics.yml index 795243fab49..314ee44ed71 100644 --- a/config/prometheus/common_metrics.yml +++ b/config/prometheus/common_metrics.yml @@ -209,6 +209,6 @@ panel_groups: weight: 1 metrics: - id: system_metrics_knative_function_invocation_count - query_range: 'sum(ceil(rate(istio_requests_total{destination_service_namespace="%{kube_namespace}", destination_app=~"%{function_name}.*"}[1m])*60))' + query_range: 'sum(ceil(rate(istio_requests_total{destination_service_namespace="%{kube_namespace}", destination_service=~"%{function_name}.*"}[1m])*60))' label: invocations / minute unit: requests diff --git a/config/prometheus/pod_metrics.yml b/config/prometheus/pod_metrics.yml new file mode 100644 index 00000000000..29575ec543e --- /dev/null +++ b/config/prometheus/pod_metrics.yml @@ -0,0 +1,59 @@ +dashboard: 'Pod metrics' +priority: 10 +panel_groups: +- group: CPU metrics + panels: + - title: "CPU usage" + type: "line-chart" + y_label: "Cores per pod" + metrics: + - id: pod_cpu_usage_seconds_total + query_range: 'rate(container_cpu_usage_seconds_total{pod_name="{{pod_name}}",container_name="POD"}[5m])' + unit: "cores" + label: pod_name +- group: Memory metrics + panels: + - title: "Memory usage working set" + type: "line-chart" + y_label: "Working set memory (MiB)" + metrics: + - id: pod_memory_working_set + query_range: 'container_memory_working_set_bytes{pod_name="{{pod_name}}",container_name="POD"}/1024/1024' + unit: "MiB" + label: pod_name +- group: Network metrics + panels: + - title: "Network Receive (In)" + type: "line-chart" + y_label: "Received (KiB/sec)" + metrics: + - id: pod_network_receive + query_range: 'rate(container_network_receive_bytes_total{pod_name="{{pod_name}}",container_name="POD"}[5m])/1024' + unit: "KiB / sec" + label: pod_name + - title: "Network Transmit (Out)" + type: "line-chart" + y_label: "Transmitted (KiB/sec)" + metrics: + - id: pod_network_transmit + query_range: 'rate(container_network_transmit_bytes_total{pod_name="{{pod_name}}",container_name="POD"}[5m])/1024' + unit: "KiB / sec" + label: pod_name +- group: Disk metrics + panels: + - title: "Disk Reads" + type: "line-chart" + y_label: "Disk reads (KiB/sec)" + metrics: + - id: pod_disk_reads + query_range: 'rate(container_fs_reads_bytes_total{container_name="POD",pod_name="{{pod_name}}"}[5m])/1024' + unit: "KiB / sec" + label: pod_name + - title: "Disk Writes" + type: "line-chart" + y_label: "Disk writes (KiB/sec)" + metrics: + - id: pod_disk_writes + query_range: 'rate(container_fs_writes_bytes_total{container_name="POD",pod_name="{{pod_name}}"}[5m])/1024' + unit: "KiB / sec" + label: pod_name diff --git a/config/pseudonymizer.yml b/config/pseudonymizer.yml index 8bbc69dca7d..1f4ed9a8421 100644 --- a/config/pseudonymizer.yml +++ b/config/pseudonymizer.yml @@ -47,7 +47,6 @@ tables: epics: whitelist: - id - - milestone_id - group_id - author_id - assignee_id @@ -93,7 +92,7 @@ tables: - updated_at - description - milestone_id - - state + - state_id - updated_by_id - weight - due_date @@ -174,7 +173,7 @@ tables: - created_at - updated_at - milestone_id - - state + - state_id - merge_status - target_project_id - updated_by_id diff --git a/config/puma.example.development.rb b/config/puma.example.development.rb index 6f686437f88..ecbfac660c9 100644 --- a/config/puma.example.development.rb +++ b/config/puma.example.development.rb @@ -78,3 +78,11 @@ tag 'gitlab-puma-worker' # value is 60 seconds. # worker_timeout 60 + +# Use json formatter +require_relative "/home/git/gitlab/lib/gitlab/puma_logging/json_formatter" + +json_formatter = Gitlab::PumaLogging::JSONFormatter.new +log_formatter do |str| + json_formatter.call(str) +end diff --git a/config/puma.rb.example b/config/puma.rb.example index 10f255a87de..cd7adca157f 100644 --- a/config/puma.rb.example +++ b/config/puma.rb.example @@ -68,3 +68,11 @@ tag 'gitlab-puma-worker' # value is 60 seconds. # worker_timeout 60 + +# Use json formatter +require_relative "/home/git/gitlab/lib/gitlab/puma_logging/json_formatter" + +json_formatter = Gitlab::PumaLogging::JSONFormatter.new +log_formatter do |str| + json_formatter.call(str) +end
\ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index 9fb4d94f068..518cf985718 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -118,6 +118,8 @@ Rails.application.routes.draw do draw :trial draw :trial_registration draw :country + draw :country_state + draw :subscription end Gitlab.ee do @@ -144,11 +146,6 @@ Rails.application.routes.draw do post :create_gcp post :create_aws post :authorize_aws_role - delete :revoke_aws_role - - scope :aws do - get 'api/:resource', to: 'clusters#aws_proxy', as: :aws_proxy - end end member do @@ -166,6 +163,7 @@ Rails.application.routes.draw do end get :cluster_status, format: :json + delete :clear_cache end end end diff --git a/config/routes/group.rb b/config/routes/group.rb index 437c80b8c92..30671d4e0a1 100644 --- a/config/routes/group.rb +++ b/config/routes/group.rb @@ -1,8 +1,10 @@ # frozen_string_literal: true +# rubocop: disable Cop/PutGroupRoutesUnderScope resources :groups, only: [:index, :new, :create] do post :preview_markdown end +# rubocop: enable Cop/PutGroupRoutesUnderScope constraints(::Constraints::GroupUrlConstrainer.new) do scope(path: 'groups/*id', diff --git a/config/routes/instance_statistics.rb b/config/routes/instance_statistics.rb index 1102ef6b017..967255d5b82 100644 --- a/config/routes/instance_statistics.rb +++ b/config/routes/instance_statistics.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true namespace :instance_statistics do - root to: redirect('-/instance_statistics/conversational_development_index') + root to: redirect('-/instance_statistics/dev_ops_score') resources :cohorts, only: :index - resources :conversational_development_index, only: :index + resources :dev_ops_score, only: :index end diff --git a/config/routes/profile.rb b/config/routes/profile.rb index 403f430850e..fcf8812ee2e 100644 --- a/config/routes/profile.rb +++ b/config/routes/profile.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # for secondary email confirmations - uses the same confirmation controller as :users devise_for :emails, path: 'profile/emails', controllers: { confirmations: :confirmations } @@ -42,14 +44,6 @@ resource :profile, only: [:show, :update] do end end - Gitlab.ee do - resource :slack, only: [:edit] do - member do - get :slack_link - end - end - end - resources :chat_names, only: [:index, :new, :create, :destroy] do collection do delete :deny @@ -73,10 +67,5 @@ resource :profile, only: [:show, :update] do end resources :u2f_registrations, only: [:destroy] - - Gitlab.ee do - resources :pipeline_quota, only: [:index] - resources :billings, only: [:index] - end end end diff --git a/config/routes/project.rb b/config/routes/project.rb index 3f913683b00..26808de5b41 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -1,14 +1,12 @@ -resources :projects, only: [:index, :new, :create] +# frozen_string_literal: true -Gitlab.ee do - scope "/-/push_from_secondary/:geo_node_id" do - draw :git_http - end -end +# rubocop: disable Cop/PutProjectRoutesUnderScope +resources :projects, only: [:index, :new, :create] draw :git_http get '/projects/:id' => 'projects#resolve' +# rubocop: enable Cop/PutProjectRoutesUnderScope constraints(::Constraints::ProjectUrlConstrainer.new) do # If the route has a wildcard segment, the segment has a regex constraint, @@ -60,7 +58,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do get :trace, defaults: { format: 'json' } get :raw get :terminal - get '/terminal.ws/authorize', to: 'jobs#terminal_websocket_authorize', constraints: { format: nil } + get '/terminal.ws/authorize', to: 'jobs#terminal_websocket_authorize', format: false end resource :artifacts, only: [] do @@ -87,22 +85,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do resource :operations, only: [:show, :update] resource :integrations, only: [:show] - Gitlab.ee do - resource :slack, only: [:destroy, :edit, :update] do - get :slack_auth - end - end - resource :repository, only: [:show], controller: :repository do post :create_deploy_token, path: 'deploy_token/create' post :cleanup end end - Gitlab.ee do - resources :feature_flags - end - resources :autocomplete_sources, only: [] do collection do get 'members' @@ -171,6 +159,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do member do put :test end + + resources :hook_logs, only: [:show], controller: :service_hook_logs do + member do + post :retry + end + end end resources :boards, only: [:index, :show, :create, :update, :destroy], constraints: { id: /\d+/ } do @@ -179,7 +173,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end - resources :releases, only: [:index, :edit], param: :tag, constraints: { tag: %r{[^/]+} } + resources :releases, only: [:index, :edit], param: :tag, constraints: { tag: %r{[^/]+} } do + member do + get :evidence + end + end + resources :starrers, only: [:index] resources :forks, only: [:index, :new, :create] resources :group_links, only: [:index, :create, :update, :destroy], constraints: { id: /\d+/ } @@ -216,9 +215,69 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do get :production end end + + concerns :clusterable + + namespace :serverless do + scope :functions do + get '/:environment_id/:id', to: 'functions#show' + get '/:environment_id/:id/metrics', to: 'functions#metrics', as: :metrics + end + + resources :functions, only: [:index] + end + + resources :environments, except: [:destroy] do + member do + post :stop + post :cancel_auto_stop + get :terminal + get :metrics + get :additional_metrics + get :metrics_dashboard + get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', format: false + + get '/prometheus/api/v1/*proxy_path', to: 'environments/prometheus_api#proxy', as: :prometheus_api + + get '/sample_metrics', to: 'environments/sample_metrics#query' if ENV['USE_SAMPLE_METRICS'] + end + + collection do + get :metrics, action: :metrics_redirect + get :folder, path: 'folders/*id', constraints: { format: /(html|json)/ } + get :search + end + + resources :deployments, only: [:index] do + member do + get :metrics + get :additional_metrics + end + end + end + + resources :error_tracking, only: [:index], controller: :error_tracking do + collection do + get ':issue_id/details', + to: 'error_tracking#details', + as: 'details' + get ':issue_id/stack_trace', + to: 'error_tracking#stack_trace', + as: 'stack_trace' + post :list_projects + end + end + + # The wiki routing contains wildcard characters so + # its preferable to keep it below all other project routes + draw :wiki end # End of the /-/ scope. + # All new routes should go under /-/ scope. + # Look for scope '-' at the top of the file. + # rubocop: disable Cop/PutProjectRoutesUnderScope + # # Templates # @@ -263,16 +322,6 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do namespace :prometheus do resources :metrics, constraints: { id: %r{[^\/]+} }, only: [:index, :new, :create, :edit, :update, :destroy] do get :active_common, on: :collection - - Gitlab.ee do - post :validate_query, on: :collection - end - end - - Gitlab.ee do - resources :alerts, constraints: { id: /\d+/ }, only: [:index, :create, :show, :update, :destroy] do - post :notify, on: :collection - end end end @@ -285,15 +334,6 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do get :pipeline_status get :ci_environments_status post :toggle_subscription - - Gitlab.ee do - get :approvals - post :approvals, action: :approve - delete :approvals, action: :unapprove - - post :rebase - end - post :remove_wip post :assign_related_issues get :discussions, format: :json @@ -301,13 +341,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do get :test_reports get :exposed_artifacts - scope constraints: { format: nil }, action: :show do - get :commits, defaults: { tab: 'commits' } - get :pipelines, defaults: { tab: 'pipelines' } - get :diffs, defaults: { tab: 'diffs' } - end - - scope constraints: { format: 'json' }, as: :json do + scope constraints: ->(req) { req.format == :json }, as: :json do get :commits get :pipelines get :diffs, to: 'merge_requests/diffs#show' @@ -317,6 +351,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do get :cached_widget, to: 'merge_requests/content#cached_widget' end + scope action: :show do + get :commits, defaults: { tab: 'commits' } + get :pipelines, defaults: { tab: 'pipelines' } + get :diffs, defaults: { tab: 'diffs' } + end + get :diff_for_path, controller: 'merge_requests/diffs' scope controller: 'merge_requests/conflicts' do @@ -331,21 +371,6 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do post :bulk_update end - Gitlab.ee do - resources :approvers, only: :destroy - delete 'approvers', to: 'approvers#destroy_via_user_id', as: :approver_via_user_id - resources :approver_groups, only: :destroy - - scope module: :merge_requests do - resources :drafts, only: [:index, :update, :create, :destroy] do - collection do - post :publish - delete :discard - end - end - end - end - resources :discussions, only: [:show], constraints: { id: /\h{40}/ } do member do post :resolve @@ -360,37 +385,22 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do scope path: 'new', as: :new_merge_request do get '', action: :new - scope constraints: { format: nil }, action: :new do - get :diffs, defaults: { tab: 'diffs' } - get :pipelines, defaults: { tab: 'pipelines' } - end - - scope constraints: { format: 'json' }, as: :json do + scope constraints: ->(req) { req.format == :json }, as: :json do get :diffs get :pipelines end + scope action: :new do + get :diffs, defaults: { tab: 'diffs' } + get :pipelines, defaults: { tab: 'pipelines' } + end + get :diff_for_path get :branch_from get :branch_to end end - Gitlab.ee do - resources :path_locks, only: [:index, :destroy] do - collection do - post :toggle - end - end - - get '/service_desk' => 'service_desk#show', as: :service_desk - put '/service_desk' => 'service_desk#update', as: :service_desk_refresh - end - - Gitlab.ee do - resources :push_rules, constraints: { id: /\d+/ }, only: [:update] - end - resources :pipelines, only: [:index, :new, :create, :show] do collection do resource :pipelines_settings, path: 'settings', only: [:show, :update] @@ -409,11 +419,6 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do get :failures get :status get :test_report - - Gitlab.ee do - get :security - get :licenses - end end member do @@ -430,60 +435,6 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end - concerns :clusterable - - resources :environments, except: [:destroy] do - member do - post :stop - get :terminal - get :metrics - get :additional_metrics - get :metrics_dashboard - get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', constraints: { format: nil } - - get '/prometheus/api/v1/*proxy_path', to: 'environments/prometheus_api#proxy', as: :prometheus_api - - Gitlab.ee do - get :logs - get '/pods/(:pod_name)/containers/(:container_name)/logs', to: 'environments#k8s_pod_logs', as: :k8s_pod_logs - end - end - - collection do - get :metrics, action: :metrics_redirect - get :folder, path: 'folders/*id', constraints: { format: /(html|json)/ } - get :search - - Gitlab.ee do - get :logs, action: :logs_redirect - end - end - - resources :deployments, only: [:index] do - member do - get :metrics - get :additional_metrics - end - end - end - - Gitlab.ee do - resources :protected_environments, only: [:create, :update, :destroy], constraints: { id: /\d+/ } do - collection do - get 'search' - end - end - end - - namespace :serverless do - scope :functions do - get '/:environment_id/:id', to: 'functions#show' - get '/:environment_id/:id/metrics', to: 'functions#metrics', as: :metrics - end - - resources :functions, only: [:index] - end - draw :legacy_builds resources :hooks, only: [:index, :create, :edit, :update, :destroy], constraints: { id: /\d+/ } do @@ -517,14 +468,6 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end - Gitlab.ee do - namespace :security do - resource :dashboard, only: [:show], controller: :dashboard - end - - resources :vulnerability_feedback, only: [:index, :create, :update, :destroy], constraints: { id: /\d+/ } - end - get :issues, to: 'issues#calendar', constraints: lambda { |req| req.format == :ics } resources :issues, concerns: :awardable, constraints: { id: /\d+/ } do @@ -538,24 +481,11 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do get :realtime_changes post :create_merge_request get :discussions, format: :json - - Gitlab.ee do - get 'designs(/*vueroute)', to: 'issues#designs', as: :designs, format: false - end end collection do post :bulk_update post :import_csv - - Gitlab.ee do - post :export_csv - get :service_desk - end - end - - Gitlab.ee do - resources :issue_links, only: [:index, :create, :destroy], as: 'links', path: 'links' end end @@ -590,18 +520,11 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end - Gitlab.ee do - resources :approvers, only: :destroy - resources :approver_groups, only: :destroy - end - resources :runner_projects, only: [:create, :destroy] resources :badges, only: [:index] do collection do scope '*ref', constraints: { ref: Gitlab::PathRegex.git_reference_regex } do constraints format: /svg/ do - # Keep around until 10.0, see gitlab-org/gitlab-ce#35307 - get :build, to: "badges#pipeline" get :pipeline get :coverage end @@ -609,36 +532,32 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end - Gitlab.ee do - resources :audit_events, only: [:index] - end - - resources :error_tracking, only: [:index], controller: :error_tracking do - collection do - get ':issue_id/details', - to: 'error_tracking#details', - as: 'details' - get ':issue_id/stack_trace', - to: 'error_tracking#stack_trace', - as: 'stack_trace' - post :list_projects - end - end - scope :usage_ping, controller: :usage_ping do post :web_ide_clientside_preview end - # Since both wiki and repository routing contains wildcard characters + # The repository routing contains wildcard characters so # its preferable to keep it below all other project routes - draw :wiki draw :repository - Gitlab.ee do - resources :managed_licenses, only: [:index, :show, :new, :create, :edit, :update, :destroy] - end + # All new routes should go under /-/ scope. + # Look for scope '-' at the top of the file. + # rubocop: enable Cop/PutProjectRoutesUnderScope + + # Legacy routes. + # Introduced in 12.0. + # Should be removed with https://gitlab.com/gitlab-org/gitlab/issues/28848. + Gitlab::Routing.redirect_legacy_paths(self, :settings, :branches, :tags, + :network, :graphs, :autocomplete_sources, + :project_members, :deploy_keys, :deploy_tokens, + :labels, :milestones, :services, :boards, :releases, + :forks, :group_links, :import, :avatar, :mirror, + :cycle_analytics, :mattermost, :variables, :triggers, + :environments, :protected_environments, :error_tracking, + :serverless, :clusters, :audit_events, :wikis) end + # rubocop: disable Cop/PutProjectRoutesUnderScope resources(:projects, path: '/', constraints: { id: Gitlab::PathRegex.project_route_regex }, @@ -660,24 +579,6 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do put :new_issuable_address end end - end - - # Legacy routes. - # Introduced in 12.0. - # Should be removed with https://gitlab.com/gitlab-org/gitlab/issues/28848. - scope(path: '*namespace_id', - as: :namespace, - namespace_id: Gitlab::PathRegex.full_namespace_route_regex) do - scope(path: ':project_id', - constraints: { project_id: Gitlab::PathRegex.project_route_regex }, - module: :projects, - as: :project) do - Gitlab::Routing.redirect_legacy_paths(self, :settings, :branches, :tags, - :network, :graphs, :autocomplete_sources, - :project_members, :deploy_keys, :deploy_tokens, - :labels, :milestones, :services, :boards, :releases, - :forks, :group_links, :import, :avatar, :mirror, - :cycle_analytics, :mattermost, :variables, :triggers) - end + # rubocop: enable Cop/PutProjectRoutesUnderScope end end diff --git a/config/routes/user.rb b/config/routes/user.rb index 31af321d2b2..fe7a0aa3233 100644 --- a/config/routes/user.rb +++ b/config/routes/user.rb @@ -1,7 +1,4 @@ -Gitlab.ee do - get 'unsubscribes/:email', to: 'unsubscribes#show', as: :unsubscribe - post 'unsubscribes/:email', to: 'unsubscribes#create' -end +# frozen_string_literal: true # Allows individual providers to be directed to a chosen controller # Call from inside devise_scope @@ -30,10 +27,6 @@ devise_for :users, controllers: { omniauth_callbacks: :omniauth_callbacks, devise_scope :user do get '/users/auth/:provider/omniauth_error' => 'omniauth_callbacks#omniauth_error', as: :omniauth_error get '/users/almost_there' => 'confirmations#almost_there' - - Gitlab.ee do - get '/users/auth/kerberos_spnego/negotiate' => 'omniauth_kerberos_spnego#negotiate' - end end scope '-/users', module: :users do diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index b4be61d8a3d..68ad819d48b 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -123,3 +123,5 @@ - [refresh_license_compliance_checks, 2] - [design_management_new_version, 1] - [epics, 2] + - [personal_access_tokens, 1] + - [adjourned_project_deletion, 1] diff --git a/config/webpack.config.js b/config/webpack.config.js index 9c7a3f42c97..d85fa84c32f 100644 --- a/config/webpack.config.js +++ b/config/webpack.config.js @@ -1,3 +1,4 @@ +const fs = require('fs'); const path = require('path'); const glob = require('glob'); const webpack = require('webpack'); @@ -7,8 +8,10 @@ const CompressionPlugin = require('compression-webpack-plugin'); const MonacoWebpackPlugin = require('monaco-editor-webpack-plugin'); const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin; const CopyWebpackPlugin = require('copy-webpack-plugin'); +const vendorDllHash = require('./helpers/vendor_dll_hash'); const ROOT_PATH = path.resolve(__dirname, '..'); +const VENDOR_DLL = process.env.WEBPACK_VENDOR_DLL && process.env.WEBPACK_VENDOR_DLL !== 'false'; const CACHE_PATH = process.env.WEBPACK_CACHE_PATH || path.join(ROOT_PATH, 'tmp/cache'); const IS_PRODUCTION = process.env.NODE_ENV === 'production'; const IS_DEV_SERVER = process.env.WEBPACK_DEV_SERVER === 'true'; @@ -23,6 +26,7 @@ const NO_SOURCEMAPS = process.env.NO_SOURCEMAPS; const VUE_VERSION = require('vue/package.json').version; const VUE_LOADER_VERSION = require('vue-loader/package.json').version; +const WEBPACK_VERSION = require('webpack/package.json').version; const devtool = IS_PRODUCTION ? 'source-map' : 'cheap-module-eval-source-map'; @@ -102,6 +106,7 @@ const alias = { if (IS_EE) { Object.assign(alias, { ee: path.join(ROOT_PATH, 'ee/app/assets/javascripts'), + ee_component: path.join(ROOT_PATH, 'ee/app/assets/javascripts'), ee_empty_states: path.join(ROOT_PATH, 'ee/app/views/shared/empty_states'), ee_icons: path.join(ROOT_PATH, 'ee/app/views/shared/icons'), ee_images: path.join(ROOT_PATH, 'ee/app/assets/images'), @@ -110,6 +115,25 @@ if (IS_EE) { }); } +// if there is a compiled DLL with a matching hash string, use it +let dll; + +if (VENDOR_DLL && !IS_PRODUCTION) { + const dllHash = vendorDllHash(); + const dllCachePath = path.join(ROOT_PATH, `tmp/cache/webpack-dlls/${dllHash}`); + if (fs.existsSync(dllCachePath)) { + console.log(`Using vendor DLL found at: ${dllCachePath}`); + dll = { + manifestPath: path.join(dllCachePath, 'vendor.dll.manifest.json'), + cacheFrom: dllCachePath, + cacheTo: path.join(ROOT_PATH, `public/assets/webpack/dll.${dllHash}/`), + publicPath: `dll.${dllHash}/vendor.dll.bundle.js`, + }; + } else { + console.log(`Warning: No vendor DLL found at: ${dllCachePath}. DllPlugin disabled.`); + } +} + module.exports = { mode: IS_PRODUCTION ? 'production' : 'development', @@ -182,7 +206,7 @@ module.exports = { options: { limit: 2048 }, }, { - test: /\_worker\.js$/, + test: /_worker\.js$/, use: [ { loader: 'worker-loader', @@ -264,6 +288,11 @@ module.exports = { modules: false, assets: true, }); + + // tell our rails helper where to find the DLL files + if (dll) { + stats.dllAssets = dll.publicPath; + } return JSON.stringify(stats, null, 2); }, }), @@ -283,16 +312,28 @@ module.exports = { jQuery: 'jquery', }), - new webpack.NormalModuleReplacementPlugin(/^ee_component\/(.*)\.vue/, function(resource) { - if (Object.keys(module.exports.resolve.alias).indexOf('ee') >= 0) { - resource.request = resource.request.replace(/^ee_component/, 'ee'); - } else { + // reference our compiled DLL modules + dll && + new webpack.DllReferencePlugin({ + context: ROOT_PATH, + manifest: dll.manifestPath, + }), + + dll && + new CopyWebpackPlugin([ + { + from: dll.cacheFrom, + to: dll.cacheTo, + }, + ]), + + !IS_EE && + new webpack.NormalModuleReplacementPlugin(/^ee_component\/(.*)\.vue/, resource => { resource.request = path.join( ROOT_PATH, 'app/assets/javascripts/vue_shared/components/empty_component.js', ); - } - }), + }), new CopyWebpackPlugin([ { @@ -361,6 +402,21 @@ module.exports = { console.log(`Webpack heap size: ${toMB(memoryUsage)} MB`); + const webpackStatistics = { + memoryUsage, + date: Date.now(), // milliseconds + commitSHA: process.env.CI_COMMIT_SHA, + nodeVersion: process.versions.node, + webpackVersion: WEBPACK_VERSION, + }; + + console.log(webpackStatistics); + + fs.writeFileSync( + path.join(ROOT_PATH, 'webpack-dev-server.json'), + JSON.stringify(webpackStatistics), + ); + // exit in case we're running webpack-dev-server IS_DEV_SERVER && process.exit(); }); diff --git a/config/webpack.vendor.config.js b/config/webpack.vendor.config.js new file mode 100644 index 00000000000..bddbf067d7c --- /dev/null +++ b/config/webpack.vendor.config.js @@ -0,0 +1,71 @@ +const path = require('path'); +const webpack = require('webpack'); +const vendorDllHash = require('./helpers/vendor_dll_hash'); + +const ROOT_PATH = path.resolve(__dirname, '..'); + +const dllHash = vendorDllHash(); +const dllCachePath = path.join(ROOT_PATH, `tmp/cache/webpack-dlls/${dllHash}`); +const dllPublicPath = `/assets/webpack/dll.${dllHash}/`; + +module.exports = { + mode: 'development', + resolve: { + extensions: ['.js'], + }, + + context: ROOT_PATH, + + entry: { + vendor: [ + 'jquery', + 'pdfjs-dist/build/pdf', + 'pdfjs-dist/build/pdf.worker.min', + 'sql.js', + 'core-js', + 'echarts', + 'lodash', + 'underscore', + 'vuex', + 'pikaday', + 'vue/dist/vue.esm.js', + 'at.js', + 'jed', + 'mermaid', + 'katex', + 'three', + 'select2', + 'moment', + 'aws-sdk', + 'sanitize-html', + 'bootstrap/dist/js/bootstrap.js', + 'sortablejs/modular/sortable.esm.js', + 'popper.js', + 'apollo-client', + 'source-map', + 'mousetrap', + ], + }, + + output: { + path: dllCachePath, + publicPath: dllPublicPath, + filename: '[name].dll.bundle.js', + chunkFilename: '[name].dll.chunk.js', + library: '[name]_[hash]', + }, + + plugins: [ + new webpack.DllPlugin({ + path: path.join(dllCachePath, '[name].dll.manifest.json'), + name: '[name]_[hash]', + }), + ], + + node: { + fs: 'empty', // sqljs requires fs + setImmediate: false, + }, + + devtool: 'cheap-module-source-map', +}; |