diff options
Diffstat (limited to 'config')
50 files changed, 636 insertions, 299 deletions
diff --git a/config/application.rb b/config/application.rb index 76a2c47a750..921baa5d617 100644 --- a/config/application.rb +++ b/config/application.rb @@ -8,7 +8,7 @@ module Gitlab # This method is used for smooth upgrading from the current Rails 4.x to Rails 5.0. # https://gitlab.com/gitlab-org/gitlab-ce/issues/14286 def self.rails5? - ENV["RAILS5"].in?(%w[1 true]) + !%w[0 false].include?(ENV["RAILS5"]) end class Application < Rails::Application @@ -19,6 +19,7 @@ module Gitlab require_dependency Rails.root.join('lib/gitlab/request_context') require_dependency Rails.root.join('lib/gitlab/current_settings') require_dependency Rails.root.join('lib/gitlab/middleware/read_only') + require_dependency Rails.root.join('lib/gitlab/middleware/basic_health_check') # This needs to be loaded before DB connection is made # to make sure that all connections have NO_ZERO_DATE @@ -84,6 +85,7 @@ module Gitlab # - Any parameter ending with `token` # - Any parameter containing `password` # - Any parameter containing `secret` + # - Any parameter ending with `key` # - Two-factor tokens (:otp_attempt) # - Repo/Project Import URLs (:import_url) # - Build traces (:trace) @@ -91,15 +93,13 @@ module Gitlab # - GitLab Pages SSL cert/key info (:certificate, :encrypted_key) # - Webhook URLs (:hook) # - Sentry DSN (:sentry_dsn) - # - Deploy keys (:key) # - File content from Web Editor (:content) - config.filter_parameters += [/token$/, /password/, /secret/] + config.filter_parameters += [/token$/, /password/, /secret/, /key$/] config.filter_parameters += %i( certificate encrypted_key hook import_url - key otp_attempt sentry_dsn trace @@ -134,6 +134,7 @@ module Gitlab config.assets.precompile << "notify.css" config.assets.precompile << "mailers/*.css" config.assets.precompile << "page_bundles/ide.css" + config.assets.precompile << "page_bundles/xterm.css" config.assets.precompile << "performance_bar.css" config.assets.precompile << "lib/ace.js" config.assets.precompile << "test.css" @@ -143,7 +144,7 @@ module Gitlab config.assets.precompile << "errors.css" # Import gitlab-svgs directly from vendored directory - config.assets.paths << "#{config.root}/node_modules/@gitlab-org/gitlab-svgs/dist" + config.assets.paths << "#{config.root}/node_modules/@gitlab/svgs/dist" config.assets.precompile << "icons.svg" config.assets.precompile << "icons.json" config.assets.precompile << "illustrations/*.svg" @@ -157,9 +158,12 @@ module Gitlab config.action_view.sanitized_allowed_protocols = %w(smb) + # Nokogiri is significantly faster and uses less memory than REXML + ActiveSupport::XmlMini.backend = 'Nokogiri' + # This middleware needs to precede ActiveRecord::QueryCache and other middlewares that # connect to the database. - config.middleware.insert_after "Rails::Rack::Logger", "Gitlab::Middleware::BasicHealthCheck" + config.middleware.insert_after Rails::Rack::Logger, ::Gitlab::Middleware::BasicHealthCheck config.middleware.insert_after Warden::Manager, Rack::Attack @@ -196,7 +200,7 @@ module Gitlab config.cache_store = :redis_store, caching_config_hash - config.active_record.raise_in_transactional_callbacks = true + config.active_record.raise_in_transactional_callbacks = true unless rails5? config.active_job.queue_adapter = :sidekiq @@ -204,7 +208,7 @@ module Gitlab ENV['GITLAB_PATH_OUTSIDE_HOOK'] = ENV['PATH'] ENV['GIT_TERMINAL_PROMPT'] = '0' - # Gitlab Read-only middleware support + # GitLab Read-only middleware support config.middleware.insert_after ActionDispatch::Flash, ::Gitlab::Middleware::ReadOnly config.generators do |g| diff --git a/config/boot.rb b/config/boot.rb index 655c54ddb84..1aeacdabbad 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -1,10 +1,10 @@ def rails5? - %w[1 true].include?(ENV["RAILS5"]) + !%w[0 false].include?(ENV["RAILS5"]) end require 'rubygems' unless rails5? -gemfile = rails5? ? "Gemfile.rails5" : "Gemfile" +gemfile = rails5? ? "Gemfile" : "Gemfile.rails4" ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../#{gemfile}", __dir__) # Set up gems listed in the Gemfile. diff --git a/config/dependency_decisions.yml b/config/dependency_decisions.yml index dce1fc1bc45..84d47bd52ad 100644 --- a/config/dependency_decisions.yml +++ b/config/dependency_decisions.yml @@ -235,8 +235,9 @@ :why: https://github.com/component/inherit/blob/master/LICENSE :versions: [] :when: 2017-01-14 20:10:41.804804000 Z -- - :approve +- - :license - fsevents + - MIT - :who: Matt Lee :why: https://github.com/strongloop/fsevents/blob/master/LICENSE :versions: [] @@ -380,8 +381,9 @@ :why: https://github.com/Tjatse/ansi-html/blob/master/LICENSE :versions: [] :when: 2017-04-10 05:42:12.898178000 Z -- - :approve +- - :license - map-stream + - MIT - :who: Mike Greiling :why: https://github.com/dominictarr/map-stream/blob/master/LICENCE :versions: [] @@ -458,8 +460,9 @@ :why: CC0 1.0 - https://github.com/jonathantneal/svg4everybody/blob/master/LICENSE.md :versions: [] :when: 2017-09-13 17:31:16.425819400 Z -- - :approve - - "@gitlab-org/gitlab-svgs" +- - :license + - "@gitlab/svgs" + - MIT - :who: Tim Zallmann :why: Our own library - GitLab License https://gitlab.com/gitlab-org/gitlab-svgs :versions: [] @@ -528,8 +531,9 @@ :why: https://github.com/mafintosh/cyclist/blob/master/LICENSE :versions: [] :when: 2018-02-20 21:37:43.774978000 Z -- - :approve +- - :license - bitsyntax + - MIT - :who: Mike Greiling :why: https://github.com/squaremo/bitsyntax-js/blob/master/LICENSE-MIT :versions: [] @@ -541,32 +545,50 @@ :versions: [] :when: 2018-06-08 05:30:56.764116000 Z - - :approve - - "@gitlab-org/gitlab-ui" - - :who: Clement Ho - :why: Our own library - :versions: [] - :when: 2018-07-17 21:02:54.529227000 Z -- - :approve - lz-string - :who: Phil Hughes :why: https://github.com/pieroxy/lz-string/blob/master/LICENSE.txt :versions: [] :when: 2018-08-03 08:22:44.973457000 Z -- - :approve +- - :license - smooshpack + - LGPL - :who: Phil Hughes :why: https://github.com/CompuIves/codesandbox-client/blob/master/packages/sandpack/LICENSE.md :versions: [] :when: 2018-08-03 08:24:29.578991000 Z -- - :approve +- - :license - codesandbox-import-util-types + - LGPL - :who: Phil Hughes :why: https://github.com/codesandbox-app/codesandbox-importers/blob/master/packages/types/LICENSE :versions: [] :when: 2018-08-03 12:22:47.574421000 Z -- - :approve +- - :license - codesandbox-import-utils + - LGPL - :who: Phil Hughes :why: https://github.com/codesandbox-app/codesandbox-importers/blob/master/packages/import-utils/LICENSE :versions: [] :when: 2018-08-03 12:23:24.083046000 Z +- - :ignore_group + - devDependencies + - :who: Winnie Hellmann + :why: NPM packages used for development are not distributed with the final product + and are therefore exempt. + :versions: [] + :when: 2018-08-30 12:06:35.668181000 Z +- - :approve + - caniuse-lite + - :who: Mike Greiling + :why: CC-BY-4.0 license. Tool only used during build process, code is not present + in compiled/distributed product so attribution not needed. + :versions: [] + :when: 2018-10-02 19:23:11.221660000 Z +- - :approve + - node-releases + - :who: Mike Greiling + :why: CC-BY-4.0 license. Tool only used during build process, code is not present + in compiled/distributed product so attribution not needed. + :versions: [] + :when: 2018-10-02 19:23:54.840151000 Z diff --git a/config/environment.rb b/config/environment.rb index 5d35937f7c6..3a52656a2c1 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -1,10 +1,10 @@ # Load the rails application # Remove this condition when upgraded to rails 5.0. -if %w[1 true].include?(ENV["RAILS5"]) - require_relative 'application' -else +if %w[0 false].include?(ENV["RAILS5"]) require File.expand_path('application', __dir__) +else + require_relative 'application' end # Initialize the rails application diff --git a/config/environments/development.rb b/config/environments/development.rb index 23790b84e3c..494ddd72556 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -45,4 +45,6 @@ Rails.application.configure do # Do not log asset requests config.assets.quiet = true + + config.allow_concurrency = defined?(::Puma) end diff --git a/config/environments/production.rb b/config/environments/production.rb index 9941987929c..71195164e7a 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -83,5 +83,5 @@ Rails.application.configure do config.eager_load = true - config.allow_concurrency = false + config.allow_concurrency = defined?(::Puma) end diff --git a/config/environments/test.rb b/config/environments/test.rb index af1011a1ab1..072f93150a3 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -21,12 +21,12 @@ Rails.application.configure do if Gitlab.rails5? config.public_file_server.enabled = true + config.public_file_server.headers = { 'Cache-Control' => 'public, max-age=3600' } else config.serve_static_files = true + config.static_cache_control = "public, max-age=3600" end - config.static_cache_control = "public, max-age=3600" - # Show full error reports and disable caching config.consider_all_requests_local = true config.action_controller.perform_caching = false diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 561ff57c9fb..09e21b2c6f2 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -94,7 +94,7 @@ production: &base # This happens when the commit is pushed or merged into the default branch of a project. # When not specified the default issue_closing_pattern as specified below will be used. # Tip: you can test your closing pattern at http://rubular.com. - # issue_closing_pattern: '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing)|[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)' + # issue_closing_pattern: '\b((?:[Cc]los(?:e[sd]?|ing)|\b[Ff]ix(?:e[sd]|ing)?|\b[Rr]esolv(?:e[sd]?|ing)|\b[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)' ## Default project features settings default_projects_features: @@ -207,9 +207,14 @@ production: &base # endpoint: 'http://127.0.0.1:9000' # default: nil # path_style: true # Use 'host/bucket_name/object' instead of 'bucket_name.host/object' + ## Packages (maven repository so far) + packages: + enabled: false + ## GitLab Pages pages: enabled: false + access_control: false # The location where pages are stored (default: shared/pages). # path: shared/pages @@ -261,6 +266,9 @@ production: &base # once per hour you will have concurrent 'git fsck' jobs. repository_check_worker: cron: "20 * * * *" + # Archive live traces which have not been archived yet + ci_archive_traces_cron_worker: + cron: "17 * * * *" # Send admin emails once a week admin_email_worker: cron: "0 0 * * 0" @@ -444,7 +452,7 @@ production: &base ## OmniAuth settings omniauth: # Allow login via Twitter, Google, etc. using OmniAuth providers - enabled: false + # enabled: true # Uncomment this to automatically sign in with a specific omniauth provider's without # showing GitLab's sign-in page (default: show the GitLab sign-in page) @@ -583,7 +591,7 @@ production: &base gitaly: # Path to the directory containing Gitaly client executables. client_path: /home/git/gitaly/bin - # Default Gitaly authentication token. Can be overriden per storage. Can + # Default Gitaly authentication token. Can be overridden per storage. Can # be left blank when Gitaly is running locally on a Unix socket, which # is the normal way to deploy Gitaly. token: @@ -757,8 +765,8 @@ test: host: localhost port: 80 - # When you run tests we clone and setup gitlab-shell - # In order to setup it correctly you need to specify + # When you run tests we clone and set up gitlab-shell + # In order to set it up correctly you need to specify # your system username you use to run GitLab # user: YOUR_USERNAME pages: @@ -768,9 +776,6 @@ test: default: path: tmp/tests/repositories/ gitaly_address: unix:tmp/tests/gitaly/gitaly.socket - broken: - path: tmp/tests/non-existent-repositories - gitaly_address: unix:tmp/tests/gitaly/gitaly.socket gitaly: client_path: tmp/tests/gitaly @@ -792,7 +797,7 @@ test: project_key: PROJECT omniauth: - enabled: true + # enabled: true allow_single_sign_on: true external_providers: [] diff --git a/config/initializers/0_as_concern.rb b/config/initializers/0_as_concern.rb index 40232bd6252..ff132547225 100644 --- a/config/initializers/0_as_concern.rb +++ b/config/initializers/0_as_concern.rb @@ -1,25 +1,7 @@ -# This module is based on: https://gist.github.com/bcardarella/5735987 - -module Prependable - def prepend_features(base) - if base.instance_variable_defined?(:@_dependencies) - base.instance_variable_get(:@_dependencies) << self - false - else - return false if base < self - - super - base.singleton_class.send(:prepend, const_get('ClassMethods')) if const_defined?(:ClassMethods) - @_dependencies.each { |dep| base.send(:prepend, dep) } # rubocop:disable Gitlab/ModuleWithInstanceVariables - base.class_eval(&@_included_block) if instance_variable_defined?(:@_included_block) # rubocop:disable Gitlab/ModuleWithInstanceVariables - end - end -end +# frozen_string_literal: true module ActiveSupport module Concern - prepend Prependable - - alias_method :prepended, :included + prepend Gitlab::Patch::Prependable end end diff --git a/config/initializers/0_post_deployment_migrations.rb b/config/initializers/0_post_deployment_migrations.rb index 3d81b869b52..2d647f72840 100644 --- a/config/initializers/0_post_deployment_migrations.rb +++ b/config/initializers/0_post_deployment_migrations.rb @@ -1,14 +1,4 @@ # Post deployment migrations are included by default. This file must be loaded # before other initializers as Rails may otherwise memoize a list of migrations # excluding the post deployment migrations. -unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] - Rails.application.config.paths['db'].each do |db_path| - path = Rails.root.join(db_path, 'post_migrate').to_s - - Rails.application.config.paths['db/migrate'] << path - - # Rails memoizes migrations at certain points where it won't read the above - # path just yet. As such we must also update the following list of paths. - ActiveRecord::Migrator.migrations_paths << path - end -end +Gitlab::Database.add_post_migrate_path_to_rails diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 9ad55e21d11..bd02b85c7ce 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -45,7 +45,7 @@ if Settings.ldap['enabled'] || Rails.env.test? end Settings['omniauth'] ||= Settingslogic.new({}) -Settings.omniauth['enabled'] = false if Settings.omniauth['enabled'].nil? +Settings.omniauth['enabled'] = true if Settings.omniauth['enabled'].nil? Settings.omniauth['auto_sign_in_with_provider'] = false if Settings.omniauth['auto_sign_in_with_provider'].nil? Settings.omniauth['allow_single_sign_on'] = false if Settings.omniauth['allow_single_sign_on'].nil? Settings.omniauth['external_providers'] = [] if Settings.omniauth['external_providers'].nil? @@ -136,12 +136,12 @@ Settings.gitlab['signup_enabled'] ||= true if Settings.gitlab['signup_enabled']. Settings.gitlab['signin_enabled'] ||= true if Settings.gitlab['signin_enabled'].nil? Settings.gitlab['restricted_visibility_levels'] = Settings.__send__(:verify_constant_array, Gitlab::VisibilityLevel, Settings.gitlab['restricted_visibility_levels'], []) Settings.gitlab['username_changing_enabled'] = true if Settings.gitlab['username_changing_enabled'].nil? -Settings.gitlab['issue_closing_pattern'] = '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing)|[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?: *,? +and +| *,? *)?)|([A-Z][A-Z0-9_]+-\d+))+)' if Settings.gitlab['issue_closing_pattern'].nil? +Settings.gitlab['issue_closing_pattern'] = '\b((?:[Cc]los(?:e[sd]?|ing)|\b[Ff]ix(?:e[sd]|ing)?|\b[Rr]esolv(?:e[sd]?|ing)|\b[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?: *,? +and +| *,? *)?)|([A-Z][A-Z0-9_]+-\d+))+)' if Settings.gitlab['issue_closing_pattern'].nil? Settings.gitlab['default_projects_features'] ||= {} Settings.gitlab['webhook_timeout'] ||= 10 Settings.gitlab['max_attachment_size'] ||= 10 Settings.gitlab['session_expire_delay'] ||= 10080 -Settings.gitlab['unauthenticated_session_expire_delay'] ||= 1.hour.to_i +Settings.gitlab['unauthenticated_session_expire_delay'] ||= 2.hours.to_i Settings.gitlab.default_projects_features['issues'] = true if Settings.gitlab.default_projects_features['issues'].nil? Settings.gitlab.default_projects_features['merge_requests'] = true if Settings.gitlab.default_projects_features['merge_requests'].nil? Settings.gitlab.default_projects_features['wiki'] = true if Settings.gitlab.default_projects_features['wiki'].nil? @@ -200,6 +200,7 @@ Settings.registry['path'] = Settings.absolute(Settings.registry['path # Settings['pages'] ||= Settingslogic.new({}) Settings.pages['enabled'] = false if Settings.pages['enabled'].nil? +Settings.pages['access_control'] = false if Settings.pages['access_control'].nil? Settings.pages['path'] = Settings.absolute(Settings.pages['path'] || File.join(Settings.shared['path'], "pages")) Settings.pages['https'] = false if Settings.pages['https'].nil? Settings.pages['host'] ||= "example.com" diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb index 146c4b1e024..8052880cc3d 100644 --- a/config/initializers/7_prometheus_metrics.rb +++ b/config/initializers/7_prometheus_metrics.rb @@ -26,9 +26,25 @@ Sidekiq.configure_server do |config| end if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled? - unless Sidekiq.server? - Gitlab::Metrics::Samplers::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start + Gitlab::Cluster::LifecycleEvents.on_worker_start do + defined?(::Prometheus::Client.reinitialize_on_pid_change) && Prometheus::Client.reinitialize_on_pid_change + + unless Sidekiq.server? + Gitlab::Metrics::Samplers::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start + end + + Gitlab::Metrics::Samplers::RubySampler.initialize_instance(Settings.monitoring.ruby_sampler_interval).start end +end - Gitlab::Metrics::Samplers::RubySampler.initialize_instance(Settings.monitoring.ruby_sampler_interval).start +Gitlab::Cluster::LifecycleEvents.on_master_restart do + # The following is necessary to ensure stale Prometheus metrics don't + # accumulate over time. It needs to be done in this hook as opposed to + # inside an init script to ensure metrics files aren't deleted after new + # unicorn workers start after a SIGUSR2 is received. + prometheus_multiproc_dir = ENV['prometheus_multiproc_dir'] + if prometheus_multiproc_dir + old_metrics = Dir[File.join(prometheus_multiproc_dir, '*.db')] + FileUtils.rm_rf(old_metrics) + end end diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb index 4d8d35bf6cf..468f80939d7 100644 --- a/config/initializers/8_metrics.rb +++ b/config/initializers/8_metrics.rb @@ -3,7 +3,6 @@ # that we can stub it for testing, as it is only called when metrics are # enabled. # -# rubocop:disable Metrics/AbcSize def instrument_classes(instrumentation) instrumentation.instrument_instance_methods(Gitlab::Shell) @@ -48,16 +47,6 @@ def instrument_classes(instrumentation) instrumentation.instrument_methods(Premailer::Adapter::Nokogiri) instrumentation.instrument_instance_methods(Premailer::Adapter::Nokogiri) - [ - :Blame, :Branch, :BranchCollection, :Blob, :Commit, :Diff, :Repository, - :Tag, :TagCollection, :Tree - ].each do |name| - const = Rugged.const_get(name) - - instrumentation.instrument_methods(const) - instrumentation.instrument_instance_methods(const) - end - instrumentation.instrument_methods(Banzai::Renderer) instrumentation.instrument_methods(Banzai::Querying) @@ -101,7 +90,6 @@ def instrument_classes(instrumentation) # Needed for https://gitlab.com/gitlab-org/gitlab-ce/issues/30224#note_32306159 instrumentation.instrument_instance_method(MergeRequestDiff, :load_commits) end -# rubocop:enable Metrics/AbcSize # With prometheus enabled by default this breaks all specs # that stubs methods using `any_instance_of` for the models reloaded here. @@ -110,7 +98,11 @@ end # check: https://github.com/rspec/rspec-mocks#settings-mocks-or-stubs-on-any-instance-of-a-class # # Related issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/33587 -if Gitlab::Metrics.enabled? && !Rails.env.test? +# +# In development mode, we turn off eager loading when we're running +# `rails generate migration` because eager loading short-circuits the +# loading of our custom migration templates. +if Gitlab::Metrics.enabled? && !Rails.env.test? && !(Rails.env.development? && defined?(Rails::Generators)) require 'pathname' require 'influxdb' require 'connection_pool' @@ -170,7 +162,9 @@ if Gitlab::Metrics.enabled? && !Rails.env.test? GC::Profiler.enable - Gitlab::Metrics::Samplers::InfluxSampler.initialize_instance.start + Gitlab::Cluster::LifecycleEvents.on_worker_start do + Gitlab::Metrics::Samplers::InfluxSampler.initialize_instance.start + end module TrackNewRedisConnections def connect(*args) diff --git a/config/initializers/active_record_avoid_type_casting_in_uniqueness_validator.rb b/config/initializers/active_record_avoid_type_casting_in_uniqueness_validator.rb index d9418caf68b..ef4abb77bd7 100644 --- a/config/initializers/active_record_avoid_type_casting_in_uniqueness_validator.rb +++ b/config/initializers/active_record_avoid_type_casting_in_uniqueness_validator.rb @@ -21,8 +21,6 @@ # This bug was fixed in Rails 5.1 by https://github.com/rails/rails/pull/24745/commits/aa062318c451512035c10898a1af95943b1a3803 if Gitlab.rails5? - ActiveSupport::Deprecation.warn("#{__FILE__} is a monkey patch which must be removed when upgrading to Rails 5.1") - if Rails.version.start_with?("5.1") raise "Remove this monkey patch: #{__FILE__}" end diff --git a/config/initializers/active_record_lifecycle.rb b/config/initializers/active_record_lifecycle.rb new file mode 100644 index 00000000000..7fa37121efc --- /dev/null +++ b/config/initializers/active_record_lifecycle.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# Don't handle sidekiq configuration as it +# has its own special active record configuration here +if defined?(ActiveRecord::Base) && !Sidekiq.server? + Gitlab::Cluster::LifecycleEvents.on_worker_start do + ActiveSupport.on_load(:active_record) do + ActiveRecord::Base.establish_connection + + Rails.logger.debug("ActiveRecord connection established") + end + end +end + +if defined?(ActiveRecord::Base) + Gitlab::Cluster::LifecycleEvents.on_before_fork do + # the following is highly recommended for Rails + "preload_app true" + # as there's no need for the master process to hold a connection + ActiveRecord::Base.connection.disconnect! + + Rails.logger.debug("ActiveRecord connection disconnected") + end +end diff --git a/config/initializers/active_record_verbose_query_logs.rb b/config/initializers/active_record_verbose_query_logs.rb index 44f86fec7e0..1c5fbc8e830 100644 --- a/config/initializers/active_record_verbose_query_logs.rb +++ b/config/initializers/active_record_verbose_query_logs.rb @@ -47,7 +47,9 @@ module ActiveRecord end end - unless Gitlab.rails5? + if Rails.version.start_with?("5.2") + raise "Remove this monkey patch: #{__FILE__}" + else prepend(VerboseQueryLogs) unless Rails.env.production? end end diff --git a/config/initializers/asset_sync.rb b/config/initializers/asset_sync.rb deleted file mode 100644 index 7f3934853fa..00000000000 --- a/config/initializers/asset_sync.rb +++ /dev/null @@ -1,31 +0,0 @@ -AssetSync.configure do |config| - # Disable the asset_sync gem by default. If it is enabled, but not configured, - # asset_sync will cause the build to fail. - config.enabled = if ENV.has_key?('ASSET_SYNC_ENABLED') - ENV['ASSET_SYNC_ENABLED'] == 'true' - else - false - end - - # Pulled from https://github.com/AssetSync/asset_sync/blob/v2.2.0/lib/asset_sync/engine.rb#L15-L40 - # This allows us to disable asset_sync by default and configure through environment variables - # Updates to asset_sync gem should be checked - config.fog_provider = ENV['FOG_PROVIDER'] if ENV.has_key?('FOG_PROVIDER') - config.fog_directory = ENV['FOG_DIRECTORY'] if ENV.has_key?('FOG_DIRECTORY') - config.fog_region = ENV['FOG_REGION'] if ENV.has_key?('FOG_REGION') - - config.aws_access_key_id = ENV['ASSETS_AWS_ACCESS_KEY_ID'] if ENV.has_key?('ASSETS_AWS_ACCESS_KEY_ID') - config.aws_secret_access_key = ENV['ASSETS_AWS_SECRET_ACCESS_KEY'] if ENV.has_key?('ASSETS_AWS_SECRET_ACCESS_KEY') - config.aws_reduced_redundancy = ENV['AWS_REDUCED_REDUNDANCY'] == true if ENV.has_key?('AWS_REDUCED_REDUNDANCY') - - config.rackspace_username = ENV['RACKSPACE_USERNAME'] if ENV.has_key?('RACKSPACE_USERNAME') - config.rackspace_api_key = ENV['RACKSPACE_API_KEY'] if ENV.has_key?('RACKSPACE_API_KEY') - - config.google_storage_access_key_id = ENV['GOOGLE_STORAGE_ACCESS_KEY_ID'] if ENV.has_key?('GOOGLE_STORAGE_ACCESS_KEY_ID') - config.google_storage_secret_access_key = ENV['GOOGLE_STORAGE_SECRET_ACCESS_KEY'] if ENV.has_key?('GOOGLE_STORAGE_SECRET_ACCESS_KEY') - - config.existing_remote_files = ENV['ASSET_SYNC_EXISTING_REMOTE_FILES'] || "keep" - - config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION') - config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST') -end diff --git a/config/initializers/carrierwave_patch.rb b/config/initializers/carrierwave_patch.rb new file mode 100644 index 00000000000..35ffff03abe --- /dev/null +++ b/config/initializers/carrierwave_patch.rb @@ -0,0 +1,29 @@ +# This monkey patches CarrierWave 1.2.3 to make Google Cloud Storage work with +# extra query parameters: +# https://github.com/carrierwaveuploader/carrierwave/pull/2332/files +module CarrierWave + module Storage + class Fog < Abstract + class File + def authenticated_url(options = {}) + if %w(AWS Google Rackspace OpenStack).include?(@uploader.fog_credentials[:provider]) + # avoid a get by using local references + local_directory = connection.directories.new(key: @uploader.fog_directory) + local_file = local_directory.files.new(key: path) + expire_at = ::Fog::Time.now + @uploader.fog_authenticated_url_expiration + case @uploader.fog_credentials[:provider] + when 'AWS', 'Google' + local_file.url(expire_at, options) + when 'Rackspace' + connection.get_object_https_url(@uploader.fog_directory, path, expire_at, options) + when 'OpenStack' + connection.get_object_https_url(@uploader.fog_directory, path, expire_at) + else + local_file.url(expire_at) + end + end + end + end + end + end +end diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb index c41b2db722c..179e00cdbd0 100644 --- a/config/initializers/devise.rb +++ b/config/initializers/devise.rb @@ -71,7 +71,7 @@ Devise.setup do |config| # a value less than 10 in other environments. config.stretches = Rails.env.test? ? 1 : 10 - # Setup a pepper to generate the encrypted password. + # Set up a pepper to generate the encrypted password. # config.pepper = "2ef62d549c4ff98a5d3e0ba211e72cff592060247e3bbbb9f499af1222f876f53d39b39b823132affb32858168c79c1d7741d26499901b63c6030a42129924ef" # ==> Configuration for :confirmable diff --git a/config/initializers/fill_shards.rb b/config/initializers/fill_shards.rb new file mode 100644 index 00000000000..18e067c8854 --- /dev/null +++ b/config/initializers/fill_shards.rb @@ -0,0 +1,3 @@ +if Shard.connected? && !Gitlab::Database.read_only? + Shard.populate! +end diff --git a/config/initializers/fog_google_https_private_urls.rb b/config/initializers/fog_google_https_private_urls.rb index f92e623a5d2..682b1050c68 100644 --- a/config/initializers/fog_google_https_private_urls.rb +++ b/config/initializers/fog_google_https_private_urls.rb @@ -7,9 +7,9 @@ module Fog class GoogleXML class File < Fog::Model module MonkeyPatch - def url(expires) + def url(expires, options = {}) requires :key - collection.get_https_url(key, expires) + collection.get_https_url(key, expires, options) end end diff --git a/config/initializers/gettext_rails_i18n_patch.rb b/config/initializers/gettext_rails_i18n_patch.rb index 49551319435..c1342f48ebd 100644 --- a/config/initializers/gettext_rails_i18n_patch.rb +++ b/config/initializers/gettext_rails_i18n_patch.rb @@ -1,5 +1,6 @@ require 'gettext_i18n_rails/haml_parser' require 'gettext_i18n_rails_js/parser/javascript' +require 'json' VUE_TRANSLATE_REGEX = /((%[\w.-]+)(?:\s))?{{ (N|n|s)?__\((.*)\) }}/ @@ -36,6 +37,20 @@ module GettextI18nRailsJs ".vue" ].include? ::File.extname(file) end + + def collect_for(file) + gettext_messages_by_file[file] || [] + end + + private + + def gettext_messages_by_file + @gettext_messages_by_file ||= JSON.parse(load_messages) + end + + def load_messages + `node scripts/frontend/extract_gettext_all.js --all` + end end end end diff --git a/config/initializers/gollum.rb b/config/initializers/gollum.rb deleted file mode 100644 index ea9cc151a57..00000000000 --- a/config/initializers/gollum.rb +++ /dev/null @@ -1,28 +0,0 @@ -# WARNING changes in this file must be manually propagated to gitaly-ruby. -# -# https://gitlab.com/gitlab-org/gitaly/blob/master/ruby/lib/gitlab/gollum.rb - -module Gollum - GIT_ADAPTER = "rugged".freeze -end -require "gollum-lib" - -module Gollum - class Page - def text_data(encoding = nil) - data = if raw_data.respond_to?(:encoding) - raw_data.force_encoding(encoding || Encoding::UTF_8) - else - raw_data - end - - Gitlab::EncodingHelper.encode!(data) - end - end -end - -Rails.application.configure do - config.after_initialize do - Gollum::Page.per_page = Kaminari.config.default_per_page - end -end diff --git a/config/initializers/hipchat_client_patch.rb b/config/initializers/hipchat_client_patch.rb new file mode 100644 index 00000000000..aec265312bb --- /dev/null +++ b/config/initializers/hipchat_client_patch.rb @@ -0,0 +1,14 @@ +# This monkey patches the HTTParty used in https://github.com/hipchat/hipchat-rb. +module HipChat + class Client + connection_adapter ::Gitlab::ProxyHTTPConnectionAdapter + end + + class Room + connection_adapter ::Gitlab::ProxyHTTPConnectionAdapter + end + + class User + connection_adapter ::Gitlab::ProxyHTTPConnectionAdapter + end +end diff --git a/config/initializers/kubeclient.rb b/config/initializers/kubeclient.rb index 7f115268b37..2d9f439fdc0 100644 --- a/config/initializers/kubeclient.rb +++ b/config/initializers/kubeclient.rb @@ -13,4 +13,25 @@ class Kubeclient::Client ns_prefix = build_namespace_prefix(namespace) rest_client["#{ns_prefix}#{entity_name_plural}/#{name}:#{port}/proxy"].url end + + # Monkey patch to set `max_redirects: 0`, so that kubeclient + # does not follow redirects and expose internal services. + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/53158 + def create_rest_client(path = nil) + path ||= @api_endpoint.path + options = { + ssl_ca_file: @ssl_options[:ca_file], + ssl_cert_store: @ssl_options[:cert_store], + verify_ssl: @ssl_options[:verify_ssl], + ssl_client_cert: @ssl_options[:client_cert], + ssl_client_key: @ssl_options[:client_key], + proxy: @http_proxy_uri, + user: @auth_options[:username], + password: @auth_options[:password], + open_timeout: @timeouts[:open], + read_timeout: @timeouts[:read], + max_redirects: 0 + } + RestClient::Resource.new(@api_endpoint.merge(path).to_s, options) + end end diff --git a/config/initializers/lograge.rb b/config/initializers/lograge.rb index 1cf8a24e98c..840404e0ec0 100644 --- a/config/initializers/lograge.rb +++ b/config/initializers/lograge.rb @@ -22,7 +22,8 @@ unless Sidekiq.server? params: params, remote_ip: event.payload[:remote_ip], user_id: event.payload[:user_id], - username: event.payload[:username] + username: event.payload[:username], + ua: event.payload[:ua] } gitaly_calls = Gitlab::GitalyClient.get_request_count diff --git a/config/initializers/macos.rb b/config/initializers/macos.rb new file mode 100644 index 00000000000..f410af6ed47 --- /dev/null +++ b/config/initializers/macos.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +if /darwin/ =~ RUBY_PLATFORM + Gitlab::Cluster::LifecycleEvents.on_before_fork do + require 'fiddle' + + # Dynamically load Foundation.framework, ~implicitly~ initialising + # the Objective-C runtime before any forking happens in Unicorn + # + # From https://bugs.ruby-lang.org/issues/14009 + Fiddle.dlopen '/System/Library/Frameworks/Foundation.framework/Foundation' + end +end diff --git a/config/initializers/mysql_set_length_for_binary_indexes.rb b/config/initializers/mysql_set_length_for_binary_indexes.rb index de0bc5322aa..0445d8fcae2 100644 --- a/config/initializers/mysql_set_length_for_binary_indexes.rb +++ b/config/initializers/mysql_set_length_for_binary_indexes.rb @@ -2,13 +2,17 @@ # MySQL adapter apply a length of 20. Otherwise MySQL can't create an index on # binary columns. +# This module can be removed once a Rails 5 schema is used. +# It can't be wrapped in a check that checks Gitlab.rails5? because +# the old Rails 4 schema layout is still used module MysqlSetLengthForBinaryIndex def add_index(table_name, column_names, options = {}) + options[:length] ||= {} Array(column_names).each do |column_name| column = ActiveRecord::Base.connection.columns(table_name).find { |c| c.name == column_name } if column&.type == :binary - options[:length] = 20 + options[:length][column_name] = 20 end end @@ -19,3 +23,47 @@ end if defined?(ActiveRecord::ConnectionAdapters::Mysql2Adapter) ActiveRecord::ConnectionAdapters::Mysql2Adapter.send(:prepend, MysqlSetLengthForBinaryIndex) end + +module MysqlSetLengthForBinaryIndexAndIgnorePostgresOptionsForSchema + # This method is used in Rails 5 schema loading as t.index + def index(column_names, options = {}) + # Ignore indexes that use opclasses, + # also see config/initializers/mysql_ignore_postgresql_options.rb + if options[:opclasses] + warn "WARNING: index on columns #{column_names} uses unsupported option, skipping." + return + end + + # when running rails 4 with rails 5 schema, rails 4 doesn't support multiple + # indexes on the same set of columns. Mysql doesn't support partial indexes, so if + # an index already exists and we add another index, skip it if it's partial: + # see https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/21492#note_102821326 + if !Gitlab.rails5? && indexes[column_names] && options[:where] + warn "WARNING: index on columns #{column_names} already exists and partial index is not supported, skipping." + return + end + + options[:length] ||= {} + Array(column_names).each do |column_name| + column = columns.find { |c| c.name == column_name } + + if column&.type == :binary + options[:length][column_name] = 20 + end + end + + super(column_names, options) + end +end + +def mysql_adapter? + defined?(ActiveRecord::ConnectionAdapters::Mysql2Adapter) && ActiveRecord::Base.connection.is_a?(ActiveRecord::ConnectionAdapters::Mysql2Adapter) +end + +if Gitlab.rails5? + if defined?(ActiveRecord::ConnectionAdapters::MySQL::TableDefinition) + ActiveRecord::ConnectionAdapters::MySQL::TableDefinition.send(:prepend, MysqlSetLengthForBinaryIndexAndIgnorePostgresOptionsForSchema) + end +elsif mysql_adapter? && defined?(ActiveRecord::ConnectionAdapters::TableDefinition) + ActiveRecord::ConnectionAdapters::TableDefinition.send(:prepend, MysqlSetLengthForBinaryIndexAndIgnorePostgresOptionsForSchema) +end diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb index bc9b52ceef7..a6f43415ec5 100644 --- a/config/initializers/peek.rb +++ b/config/initializers/peek.rb @@ -18,7 +18,6 @@ Peek.into PEEK_DB_VIEW Peek.into Peek::Views::Gitaly Peek.into Peek::Views::Rblineprof Peek.into Peek::Views::Redis -Peek.into Peek::Views::Sidekiq Peek.into Peek::Views::GC # rubocop:disable Naming/ClassAndModuleCamelCase diff --git a/config/initializers/postgresql_opclasses_support.rb b/config/initializers/postgresql_opclasses_support.rb index 7b8afc78817..07b06629dea 100644 --- a/config/initializers/postgresql_opclasses_support.rb +++ b/config/initializers/postgresql_opclasses_support.rb @@ -144,7 +144,10 @@ module ActiveRecord [column, opclass] if opclass end.compact] - IndexDefinition.new(table_name, index_name, unique, column_names, [], orders, where, nil, using, opclasses) + index_attrs = [table_name, index_name, unique, column_names, [], orders, where, nil, using, opclasses] + index_attrs.insert(-2, nil) if Gitlab.rails5? # include index comment for Rails 5 + + IndexDefinition.new(*index_attrs) end end.compact end @@ -172,29 +175,42 @@ module ActiveRecord def indexes(table, stream) if (indexes = @connection.indexes(table)).any? add_index_statements = indexes.map do |index| - statement_parts = [ - "add_index #{remove_prefix_and_suffix(index.table).inspect}", - index.columns.inspect, - "name: #{index.name.inspect}", - ] - statement_parts << 'unique: true' if index.unique - - index_lengths = (index.lengths || []).compact - statement_parts << "length: #{Hash[index.columns.zip(index.lengths)].inspect}" if index_lengths.any? - - index_orders = index.orders || {} - statement_parts << "order: #{index.orders.inspect}" if index_orders.any? - statement_parts << "where: #{index.where.inspect}" if index.where - statement_parts << "using: #{index.using.inspect}" if index.using - statement_parts << "type: #{index.type.inspect}" if index.type - statement_parts << "opclasses: #{index.opclasses}" if index.opclasses.present? - - " #{statement_parts.join(', ')}" + table_name = remove_prefix_and_suffix(index.table).inspect + " add_index #{([table_name]+index_parts(index)).join(', ')}" end stream.puts add_index_statements.sort.join("\n") stream.puts end end + + def indexes_in_create(table, stream) + if (indexes = @connection.indexes(table)).any? + index_statements = indexes.map do |index| + " t.index #{index_parts(index).join(', ')}" + end + stream.puts index_statements.sort.join("\n") + end + end + + def index_parts(index) + index_parts = [ + index.columns.inspect, + "name: #{index.name.inspect}", + ] + index_parts << "unique: true" if index.unique + index_parts << "length: { #{format_options(index.lengths)} }" if index.lengths.present? + index_parts << "order: { #{format_options(index.orders)} }" if index.orders.present? + index_parts << "where: #{index.where.inspect}" if index.where + index_parts << "using: #{index.using.inspect}" if index.using + index_parts << "type: #{index.type.inspect}" if index.type + index_parts << "opclasses: #{index.opclasses.inspect}" if index.opclasses.present? + index_parts << "comment: #{index.comment.inspect}" if Gitlab.rails5? && index.comment + index_parts + end + + def format_options(options) + options.map { |key, value| "#{key}: #{value.inspect}" }.join(", ") + end end end diff --git a/config/initializers/rbtrace.rb b/config/initializers/rbtrace.rb new file mode 100644 index 00000000000..6a1b71bf4bd --- /dev/null +++ b/config/initializers/rbtrace.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +if ENV['ENABLE_RBTRACE'] + Gitlab::Cluster::LifecycleEvents.on_worker_start do + # Unicorn clears out signals before it forks, so rbtrace won't work + # unless it is enabled after the fork. + require 'rbtrace' + end +end diff --git a/config/initializers/routing_draw.rb b/config/initializers/routing_draw.rb index 25003cf0239..f0f74954eef 100644 --- a/config/initializers/routing_draw.rb +++ b/config/initializers/routing_draw.rb @@ -1,7 +1,3 @@ # Adds draw method into Rails routing -# It allows us to keep routing splitted into files -class ActionDispatch::Routing::Mapper - def draw(routes_name) - instance_eval(File.read(Rails.root.join("config/routes/#{routes_name}.rb"))) - end -end +# It allows us to keep routing split into files +ActionDispatch::Routing::Mapper.prepend Gitlab::Patch::DrawRoute diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 6f54bee4713..565efc858d1 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,3 +1,9 @@ +require 'sidekiq/web' + +# Disable the Sidekiq Rack session since GitLab already has its own session store. +# CSRF protection still works (https://github.com/mperham/sidekiq/commit/315504e766c4fd88a29b7772169060afc4c40329). +Sidekiq::Web.set :sessions, false + # Custom Queues configuration queues_config_hash = Gitlab::Redis::Queues.params queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE @@ -8,8 +14,6 @@ Sidekiq.default_worker_options = { retry: 3 } enable_json_logs = Gitlab.config.sidekiq.log_format == 'json' Sidekiq.configure_server do |config| - require 'rbtrace' if ENV['ENABLE_RBTRACE'] - config.redis = queues_config_hash config.server_middleware do |chain| @@ -34,6 +38,10 @@ Sidekiq.configure_server do |config| ActiveRecord::Base.clear_all_connections! end + if Feature.enabled?(:gitlab_sidekiq_reliable_fetcher) + Sidekiq::ReliableFetcher.setup_reliable_fetch!(config) + end + # Sidekiq-cron: load recurring jobs from gitlab.yml # UGLY Hack to get nested hash from settingslogic cron_jobs = JSON.parse(Gitlab.config.cron_jobs.to_json) @@ -49,14 +57,12 @@ Sidekiq.configure_server do |config| end Sidekiq::Cron::Job.load_from_hash! cron_jobs - Gitlab::SidekiqThrottler.execute! - Gitlab::SidekiqVersioning.install! - config = Gitlab::Database.config || + db_config = Gitlab::Database.config || Rails.application.config.database_configuration[Rails.env] - config['pool'] = Sidekiq.options[:concurrency] - ActiveRecord::Base.establish_connection(config) + db_config['pool'] = Sidekiq.options[:concurrency] + ActiveRecord::Base.establish_connection(db_config) Rails.logger.debug("Connection Pool size for Sidekiq Server is now: #{ActiveRecord::Base.connection.pool.instance_variable_get('@size')}") # Avoid autoload issue such as 'Mail::Parsers::AddressStruct' diff --git a/config/initializers/static_files.rb b/config/initializers/static_files.rb index 6c28686e69a..a0b8b68f3ef 100644 --- a/config/initializers/static_files.rb +++ b/config/initializers/static_files.rb @@ -1,17 +1,26 @@ app = Rails.application -if app.config.serve_static_files +if (Gitlab.rails5? && app.config.public_file_server.enabled) || app.config.serve_static_files # The `ActionDispatch::Static` middleware intercepts requests for static files # by checking if they exist in the `/public` directory. # We're replacing it with our `Gitlab::Middleware::Static` that does the same, # except ignoring `/uploads`, letting those go through to the GitLab Rails app. - app.config.middleware.swap( - ActionDispatch::Static, - Gitlab::Middleware::Static, - app.paths["public"].first, - app.config.static_cache_control - ) + if Gitlab.rails5? + app.config.middleware.swap( + ActionDispatch::Static, + Gitlab::Middleware::Static, + app.paths["public"].first, + headers: app.config.public_file_server.headers + ) + else + app.config.middleware.swap( + ActionDispatch::Static, + Gitlab::Middleware::Static, + app.paths["public"].first, + app.config.static_cache_control + ) + end # If webpack-dev-server is configured, proxy webpack's public directory # instead of looking for static assets diff --git a/config/initializers/warden.rb b/config/initializers/warden.rb index 33f55069c3e..1d2bb2bce0a 100644 --- a/config/initializers/warden.rb +++ b/config/initializers/warden.rb @@ -31,6 +31,11 @@ Rails.application.configure do |config| Warden::Manager.before_logout(scope: :user) do |user, auth, opts| user ||= auth.user + + # Rails CSRF protection may attempt to log out a user before that + # user even logs in + next unless user + activity = Gitlab::Auth::Activity.new(opts) tracker = Gitlab::Auth::BlockedUserTracker.new(user, auth) diff --git a/config/karma.config.js b/config/karma.config.js index 84810332dc2..e1d7c30b1c2 100644 --- a/config/karma.config.js +++ b/config/karma.config.js @@ -22,6 +22,13 @@ webpackConfig.optimization.splitChunks = false; // use quicker sourcemap option webpackConfig.devtool = 'cheap-inline-source-map'; +// set BABEL_ENV to indicate when we're running code coverage +webpackConfig.plugins.push( + new webpack.DefinePlugin({ + 'process.env.BABEL_ENV': JSON.stringify(process.env.BABEL_ENV || process.env.NODE_ENV || null), + }) +); + const specFilters = argumentsParser .option( '-f, --filter-spec [filter]', @@ -80,11 +87,12 @@ if (specFilters.length) { module.exports = function(config) { process.env.TZ = 'Etc/UTC'; - const progressReporter = process.env.CI ? 'mocha' : 'progress'; - const karmaConfig = { basePath: ROOT_PATH, browsers: ['ChromeHeadlessCustom'], + client: { + color: !process.env.CI, + }, customLaunchers: { ChromeHeadlessCustom: { base: 'ChromeHeadless', @@ -104,11 +112,19 @@ module.exports = function(config) { preprocessors: { 'spec/javascripts/**/*.js': ['webpack', 'sourcemap'], }, - reporters: [progressReporter], + reporters: ['progress'], webpack: webpackConfig, webpackMiddleware: { stats: 'errors-only' }, }; + if (process.env.CI) { + karmaConfig.reporters = ['mocha', 'junit']; + karmaConfig.junitReporter = { + outputFile: 'junit_karma.xml', + useBrowserName: false, + }; + } + if (process.env.BABEL_ENV === 'coverage' || process.env.NODE_ENV === 'coverage') { karmaConfig.reporters.push('coverage-istanbul'); karmaConfig.coverageIstanbulReporter = { diff --git a/config/locales/en.yml b/config/locales/en.yml index 795e5d4e6bc..0a43a1d9a6b 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -8,6 +8,8 @@ en: issue_link: source: Source issue target: Target issue + group: + path: Group URL errors: messages: label_already_exists_at_group_level: "already exists at group level for %{group}. Please choose another one." diff --git a/config/prometheus/additional_metrics.yml b/config/prometheus/common_metrics.yml index c994bad7865..52023a2e3cb 100644 --- a/config/prometheus/additional_metrics.yml +++ b/config/prometheus/common_metrics.yml @@ -7,7 +7,8 @@ - nginx_upstream_responses_total weight: 1 queries: - - query_range: 'sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) by (status_code)' + - id: response_metrics_nginx_ingress_throughput_status_code + query_range: 'sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) by (status_code)' unit: req / sec label: Status Code series: @@ -25,7 +26,8 @@ - nginx_upstream_response_msecs_avg weight: 1 queries: - - query_range: 'avg(nginx_upstream_response_msecs_avg{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"})' + - id: response_metrics_nginx_ingress_latency_pod_average + query_range: 'avg(nginx_upstream_response_msecs_avg{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"})' label: Pod average unit: ms - title: "HTTP Error Rate" @@ -34,7 +36,8 @@ - nginx_upstream_responses_total weight: 1 queries: - - query_range: 'sum(rate(nginx_upstream_responses_total{status_code="5xx", upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) / sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) * 100' + - id: response_metrics_nginx_ingress_http_error_rate + query_range: 'sum(rate(nginx_upstream_responses_total{status_code="5xx", upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) / sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) * 100' label: 5xx Errors unit: "%" - group: Response metrics (HA Proxy) @@ -46,10 +49,12 @@ - haproxy_frontend_http_requests_total weight: 1 queries: - - query_range: 'sum(rate(haproxy_frontend_http_requests_total{%{environment_filter}}[2m])) by (code)' + - id: response_metrics_ha_proxy_throughput_status_code + query_range: 'sum(rate(haproxy_frontend_http_requests_total{%{environment_filter}}[2m])) by (code)' unit: req / sec + label: Status Code series: - - label: code + - label: status_code when: - value: 2xx color: green @@ -63,7 +68,8 @@ - haproxy_frontend_http_responses_total weight: 1 queries: - - query_range: 'sum(rate(haproxy_frontend_http_responses_total{code="5xx",%{environment_filter}}[2m])) / sum(rate(haproxy_frontend_http_responses_total{%{environment_filter}}[2m]))' + - id: response_metrics_ha_proxy_http_error_rate + query_range: 'sum(rate(haproxy_frontend_http_responses_total{code="5xx",%{environment_filter}}[2m])) / sum(rate(haproxy_frontend_http_responses_total{%{environment_filter}}[2m]))' label: HTTP Errors unit: "%" - group: Response metrics (AWS ELB) @@ -75,7 +81,8 @@ - aws_elb_request_count_sum weight: 1 queries: - - query_range: 'sum(aws_elb_request_count_sum{%{environment_filter}}) / 60' + - id: response_metrics_aws_elb_throughput_requests + query_range: 'sum(aws_elb_request_count_sum{%{environment_filter}}) / 60' label: Total unit: req / sec - title: "Latency" @@ -84,7 +91,8 @@ - aws_elb_latency_average weight: 1 queries: - - query_range: 'avg(aws_elb_latency_average{%{environment_filter}}) * 1000' + - id: response_metrics_aws_elb_latency_average + query_range: 'avg(aws_elb_latency_average{%{environment_filter}}) * 1000' label: Average unit: ms - title: "HTTP Error Rate" @@ -94,7 +102,8 @@ - aws_elb_httpcode_backend_5_xx_sum weight: 1 queries: - - query_range: 'sum(aws_elb_httpcode_backend_5_xx_sum{%{environment_filter}}) / sum(aws_elb_request_count_sum{%{environment_filter}})' + - id: response_metrics_aws_elb_http_error_rate + query_range: 'sum(aws_elb_httpcode_backend_5_xx_sum{%{environment_filter}}) / sum(aws_elb_request_count_sum{%{environment_filter}})' label: HTTP Errors unit: "%" - group: Response metrics (NGINX) @@ -106,7 +115,8 @@ - nginx_server_requests weight: 1 queries: - - query_range: 'sum(rate(nginx_server_requests{server_zone!="*", server_zone!="_", %{environment_filter}}[2m])) by (code)' + - id: response_metrics_nginx_throughput_status_code + query_range: 'sum(rate(nginx_server_requests{server_zone!="*", server_zone!="_", %{environment_filter}}[2m])) by (code)' unit: req / sec label: Status Code series: @@ -124,7 +134,8 @@ - nginx_server_requestMsec weight: 1 queries: - - query_range: 'avg(nginx_server_requestMsec{%{environment_filter}})' + - id: response_metrics_nginx_latency + query_range: 'avg(nginx_server_requestMsec{%{environment_filter}})' label: Upstream unit: ms - title: "HTTP Error Rate" @@ -133,7 +144,8 @@ - nginx_server_requests weight: 1 queries: - - query_range: 'sum(rate(nginx_server_requests{code="5xx", %{environment_filter}}[2m]))' + - id: response_metrics_nginx_http_error_rate + query_range: 'sum(rate(nginx_server_requests{code="5xx", %{environment_filter}}[2m]))' label: HTTP Errors unit: "errors / sec" - group: System metrics (Kubernetes) @@ -145,7 +157,8 @@ - container_memory_usage_bytes weight: 4 queries: - - query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) /1024/1024/1024' + - id: system_metrics_kubernetes_container_memory_total + query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) /1024/1024/1024' label: Total unit: GB - title: "Core Usage (Total)" @@ -154,7 +167,8 @@ - container_cpu_usage_seconds_total weight: 3 queries: - - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job)' + - id: system_metrics_kubernetes_container_cores_total + query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job)' label: Total unit: "cores" - title: "Memory Usage (Pod average)" @@ -163,15 +177,39 @@ - container_memory_usage_bytes weight: 2 queries: - - query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024' + - id: system_metrics_kubernetes_container_memory_average + query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024' + label: Pod average + unit: MB + - title: "Canary: Memory Usage (Pod Average)" + y_label: "Memory Used per Pod" + required_metrics: + - container_memory_usage_bytes + weight: 2 + queries: + - id: system_metrics_kubernetes_container_memory_average_canary + query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024' label: Pod average unit: MB - - title: "Core Usage (Pod average)" + track: canary + - title: "Core Usage (Pod Average)" y_label: "Cores per Pod" required_metrics: - container_cpu_usage_seconds_total weight: 1 queries: - - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))' + - id: system_metrics_kubernetes_container_core_usage + query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))' label: Pod average - unit: "cores"
\ No newline at end of file + unit: "cores" + - title: "Canary: Core Usage (Pod Average)" + y_label: "Cores per Pod" + required_metrics: + - container_cpu_usage_seconds_total + weight: 1 + queries: + - id: system_metrics_kubernetes_container_core_usage_canary + query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))' + label: Pod average + unit: "cores" + track: canary diff --git a/config/puma.example.development.rb b/config/puma.example.development.rb new file mode 100644 index 00000000000..490c940077a --- /dev/null +++ b/config/puma.example.development.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +# ----------------------------------------------------------------------- +# This file is used by the GDK to generate a default config/puma.rb file +# Note that `/home/git` will be substituted for the actual GDK root +# directory when this file is generated +# ----------------------------------------------------------------------- + +# Load "path" as a rackup file. +# +# The default is "config.ru". +# +rackup 'config.ru' +pidfile '/home/git/gitlab/tmp/pids/puma.pid' +state_path '/home/git/gitlab/tmp/pids/puma.state' + +stdout_redirect '/home/git/gitlab/log/puma.stdout.log', + '/home/git/gitlab/log/puma.stderr.log', + true + +# Configure "min" to be the minimum number of threads to use to answer +# requests and "max" the maximum. +# +# The default is "0, 16". +# +threads 1, 4 + +# By default, workers accept all requests and queue them to pass to handlers. +# When false, workers accept the number of simultaneous requests configured. +# +# Queueing requests generally improves performance, but can cause deadlocks if +# the app is waiting on a request to itself. See https://github.com/puma/puma/issues/612 +# +# When set to false this may require a reverse proxy to handle slow clients and +# queue requests before they reach puma. This is due to disabling HTTP keepalive +queue_requests false + +# Bind the server to "url". "tcp://", "unix://" and "ssl://" are the only +# accepted protocols. +bind 'unix:///home/git/gitlab.socket' + +workers 2 + +require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events" +require_relative "/home/git/gitlab/lib/gitlab/cluster/puma_worker_killer_initializer" + +on_restart do + # Signal application hooks that we're about to restart + Gitlab::Cluster::LifecycleEvents.do_master_restart +end + +before_fork do + # Signal to the puma killer + Gitlab::Cluster::PumaWorkerKillerInitializer.start @config.options unless ENV['DISABLE_PUMA_WORKER_KILLER'] + + # Signal application hooks that we're about to fork + Gitlab::Cluster::LifecycleEvents.do_before_fork +end + +Gitlab::Cluster::LifecycleEvents.set_puma_options @config.options +on_worker_boot do + # Signal application hooks of worker start + Gitlab::Cluster::LifecycleEvents.do_worker_start +end + +# Preload the application before starting the workers; this conflicts with +# phased restart feature. (off by default) + +preload_app! + +tag 'gitlab-puma-worker' + +# Verifies that all workers have checked in to the master process within +# the given timeout. If not the worker process will be restarted. Default +# value is 60 seconds. +# +worker_timeout 60 diff --git a/config/routes.rb b/config/routes.rb index d16a587c5ee..484e05114be 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -27,6 +27,15 @@ Rails.application.routes.draw do authorizations: 'oauth/authorizations' end + # This is here so we can "reserve" the path for the Jira integration in GitLab EE + # Having a non-existent controller here does not affect the scope in any way since all possible routes + # get a 404 proc returned. It is written in this way to minimize merge conflicts with EE + scope path: '/login/oauth', controller: 'oauth/jira/authorizations', as: :oauth_jira do + match '*all', via: [:get, :post], to: proc { [404, {}, ['']] } + end + + draw :oauth + use_doorkeeper_openid_connect # Autocomplete @@ -49,7 +58,6 @@ Rails.application.routes.draw do # '/-/health' implemented by BasicHealthMiddleware get 'liveness' => 'health#liveness' get 'readiness' => 'health#readiness' - post 'storage_check' => 'health#storage_check' resources :metrics, only: [:index] mount Peek::Railtie => '/peek', as: 'peek_routes' @@ -72,11 +80,33 @@ Rails.application.routes.draw do get 'ide' => 'ide#index' get 'ide/*vueroute' => 'ide#index', format: false + draw :operations draw :instance_statistics + + if ENV['GITLAB_ENABLE_CHAOS_ENDPOINTS'] + get '/chaos/leakmem' => 'chaos#leakmem' + get '/chaos/cpuspin' => 'chaos#cpuspin' + get '/chaos/sleep' => 'chaos#sleep' + get '/chaos/kill' => 'chaos#kill' + end end - # Koding route - get 'koding' => 'koding#index' + concern :clusterable do + resources :clusters, only: [:index, :new, :show, :update, :destroy] do + collection do + post :create_user + post :create_gcp + end + + member do + scope :applications do + post '/:application', to: 'clusters/applications#create', as: :install_applications + end + + get :cluster_status, format: :json + end + end + end draw :api draw :sidekiq diff --git a/config/routes/admin.rb b/config/routes/admin.rb index 7ee960970f8..af333bdc748 100644 --- a/config/routes/admin.rb +++ b/config/routes/admin.rb @@ -59,7 +59,7 @@ namespace :admin do resources :hook_logs, only: [:show] do member do - get :retry + post :retry end end end @@ -69,10 +69,9 @@ namespace :admin do end resource :logs, only: [:show] - resource :health_check, controller: 'health_check', only: [:show] do - post :reset_storage_health - end + resource :health_check, controller: 'health_check', only: [:show] resource :background_jobs, controller: 'background_jobs', only: [:show] + resource :system_info, controller: 'system_info', only: [:show] resources :requests_profiles, only: [:index, :show], param: :name, constraints: { name: /.+\.html/ } @@ -106,10 +105,12 @@ namespace :admin do resource :application_settings, only: [:show, :update] do resources :services, only: [:index, :edit, :update] + get :usage_data - put :reset_runners_token + put :reset_registration_token put :reset_health_check_token put :clear_repository_check_states + get :integrations, :repository, :templates, :ci_cd, :reporting, :metrics_and_profiling, :network, :geo, :preferences end resources :labels diff --git a/config/routes/group.rb b/config/routes/group.rb index 25fbb38ba87..a0aeebe4b91 100644 --- a/config/routes/group.rb +++ b/config/routes/group.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + resources :groups, only: [:index, :new, :create] do post :preview_markdown end @@ -14,6 +16,9 @@ constraints(::Constraints::GroupUrlConstrainer.new) do get :projects, as: :projects_group get :activity, as: :activity_group put :transfer, as: :transfer_group + # TODO: Remove as part of refactor in https://gitlab.com/gitlab-org/gitlab-ce/issues/49693 + get 'shared', action: :show, as: :group_shared + get 'archived', action: :show, as: :group_archived end get '/', action: :show, as: :group_canonical @@ -24,8 +29,9 @@ constraints(::Constraints::GroupUrlConstrainer.new) do as: :group, constraints: { group_id: Gitlab::PathRegex.full_namespace_route_regex }) do namespace :settings do - resource :ci_cd, only: [:show], controller: 'ci_cd' - resources :badges, only: [:index] + resource :ci_cd, only: [:show], controller: 'ci_cd' do + put :reset_registration_token + end end resource :variables, only: [:show, :update] @@ -37,7 +43,7 @@ constraints(::Constraints::GroupUrlConstrainer.new) do post :toggle_subscription, on: :member end - resources :milestones, constraints: { id: %r{[^/]+} }, only: [:index, :show, :edit, :update, :new, :create] do + resources :milestones, constraints: { id: %r{[^/]+} } do member do get :merge_requests get :participants @@ -47,6 +53,8 @@ constraints(::Constraints::GroupUrlConstrainer.new) do resource :avatar, only: [:destroy] + concerns :clusterable + resources :group_members, only: [:index, :create, :update, :destroy], concerns: :access_requestable do post :resend_invite, on: :member delete :leave, on: :collection @@ -59,7 +67,6 @@ constraints(::Constraints::GroupUrlConstrainer.new) do end end - # On CE only index and show actions are needed resources :boards, only: [:index, :show] resources :runners, only: [:index, :edit, :update, :destroy, :show] do diff --git a/config/routes/instance_statistics.rb b/config/routes/instance_statistics.rb index 824ef47cda3..1102ef6b017 100644 --- a/config/routes/instance_statistics.rb +++ b/config/routes/instance_statistics.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true namespace :instance_statistics do - root to: redirect('/-/instance_statistics/conversational_development_index') + root to: redirect('-/instance_statistics/conversational_development_index') resources :cohorts, only: :index resources :conversational_development_index, only: :index diff --git a/config/routes/project.rb b/config/routes/project.rb index 0220e88c819..3f1ad90dfca 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -32,6 +32,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do get 'labels' get 'milestones' get 'commands' + get 'snippets' end end @@ -145,7 +146,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end - controller 'merge_requests/creations', path: 'merge_requests' do + scope path: 'merge_requests', controller: 'merge_requests/creations' do post '', action: :create, as: nil scope path: 'new', as: :new_merge_request do @@ -177,6 +178,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do resource :mirror, only: [:show, :update] do member do + get :ssh_host_keys, constraints: { format: :json } post :update_now end end @@ -205,20 +207,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end - resources :clusters, except: [:edit, :create] do - collection do - post :create_gcp - post :create_user - end - - member do - get :status, format: :json - - scope :applications do - post '/:application', to: 'clusters/applications#create', as: :install_applications - end - end - end + concerns :clusterable resources :environments, except: [:destroy] do member do @@ -275,6 +264,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do member do get :status post :cancel + post :unschedule post :retry post :play post :erase @@ -307,7 +297,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do resources :hook_logs, only: [:show] do member do - get :retry + post :retry end end end @@ -365,7 +355,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do get :discussions, format: :json end collection do - post :bulk_update + post :bulk_update end end @@ -437,12 +427,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do get :members, to: redirect("%{namespace_id}/%{project_id}/project_members") resource :ci_cd, only: [:show, :update], controller: 'ci_cd' do post :reset_cache + put :reset_registration_token end resource :integrations, only: [:show] resource :repository, only: [:show], controller: :repository do post :create_deploy_token, path: 'deploy_token/create' end - resources :badges, only: [:index] end # Since both wiki and repository routing contains wildcard characters diff --git a/config/routes/user.rb b/config/routes/user.rb index bc7df5e7584..e0ae264e2c0 100644 --- a/config/routes/user.rb +++ b/config/routes/user.rb @@ -45,6 +45,7 @@ scope(constraints: { username: Gitlab::PathRegex.root_namespace_route_regex }) d get :contributed, as: :contributed_projects get :snippets get :exists + get :activity get '/', to: redirect('%{username}'), as: nil end diff --git a/config/routes/wiki.rb b/config/routes/wiki.rb index cd3828b743c..1a07b1c206b 100644 --- a/config/routes/wiki.rb +++ b/config/routes/wiki.rb @@ -2,7 +2,7 @@ scope(controller: :wikis) do scope(path: 'wikis', as: :wikis) do get :git_access get :pages - get '/', to: redirect('/%{namespace_id}/%{project_id}/wikis/home') + get '/', to: redirect('%{namespace_id}/%{project_id}/wikis/home') post '/', to: 'wikis#create' end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index fb7738a5536..53e1c8778b6 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -29,6 +29,7 @@ - [pipeline_creation, 4] - [pipeline_default, 3] - [pipeline_cache, 3] + - [deployment, 3] - [pipeline_hooks, 2] - [gitlab_shell, 2] - [email_receiver, 2] @@ -46,6 +47,7 @@ - [project_service, 1] - [delete_user, 1] - [todos_destroyer, 1] + - [delete_container_repository, 1] - [delete_merged_branches, 1] - [authorized_projects, 1] - [expire_build_instance_artifacts, 1] @@ -78,3 +80,4 @@ - [create_note_diff_file, 1] - [delete_diff_files, 1] - [detect_repository_languages, 1] + - [auto_devops, 2] diff --git a/config/unicorn.rb.example b/config/unicorn.rb.example index 020e9a00d87..4637eb8bc6e 100644 --- a/config/unicorn.rb.example +++ b/config/unicorn.rb.example @@ -81,22 +81,16 @@ preload_app true # fast LAN. check_client_connection false +require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events" + before_exec do |server| - # The following is necessary to ensure stale Prometheus metrics don't - # accumulate over time. It needs to be done in this hook as opposed to - # inside an init script to ensure metrics files aren't deleted after new - # unicorn workers start after a SIGUSR2 is received. - if ENV['prometheus_multiproc_dir'] - old_metrics = Dir[File.join(ENV['prometheus_multiproc_dir'], '*.db')] - FileUtils.rm_rf(old_metrics) - end + # Signal application hooks that we're about to restart + Gitlab::Cluster::LifecycleEvents.do_master_restart end before_fork do |server, worker| - # the following is highly recommended for Rails + "preload_app true" - # as there's no need for the master process to hold a connection - defined?(ActiveRecord::Base) and - ActiveRecord::Base.connection.disconnect! + # Signal application hooks that we're about to fork + Gitlab::Cluster::LifecycleEvents.do_before_fork # The following is only recommended for memory/DB-constrained # installations. It is not needed if your system can house @@ -124,25 +118,10 @@ before_fork do |server, worker| end after_fork do |server, worker| - # Unicorn clears out signals before it forks, so rbtrace won't work - # unless it is enabled after the fork. - require 'rbtrace' if ENV['ENABLE_RBTRACE'] + # Signal application hooks of worker start + Gitlab::Cluster::LifecycleEvents.do_worker_start # per-process listener ports for debugging/admin/migrations # addr = "127.0.0.1:#{9293 + worker.nr}" # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true) - - # the following is *required* for Rails + "preload_app true", - defined?(ActiveRecord::Base) and - ActiveRecord::Base.establish_connection - - # reset prometheus client, this will cause any opened metrics files to be closed - defined?(::Prometheus::Client.reinitialize_on_pid_change) && - Prometheus::Client.reinitialize_on_pid_change - - # if preload_app is true, then you may also want to check and - # restart any other shared sockets/descriptors such as Memcached, - # and Redis. TokyoCabinet file handles are safe to reuse - # between any number of forked children (assuming your kernel - # correctly implements pread()/pwrite() system calls) end diff --git a/config/unicorn.rb.example.development b/config/unicorn.rb.example.development index 5712549a66d..f7541bb9d55 100644 --- a/config/unicorn.rb.example.development +++ b/config/unicorn.rb.example.development @@ -1,32 +1,61 @@ +# frozen_string_literal: true + +# ------------------------------------------------------------------------- +# This file is used by the GDK to generate a default config/unicorn.rb file +# Note that `/home/git` will be substituted for the actual GDK root +# directory when this file is generated +# ------------------------------------------------------------------------- + worker_processes 2 timeout 60 +listen '/home/git/gitlab.socket' + preload_app true check_client_connection false +require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events" + +before_exec do |server| + # Signal application hooks that we're about to restart + Gitlab::Cluster::LifecycleEvents.do_master_restart +end + before_fork do |server, worker| - # the following is highly recommended for Rails + "preload_app true" - # as there's no need for the master process to hold a connection - defined?(ActiveRecord::Base) and - ActiveRecord::Base.connection.disconnect! - - if /darwin/ =~ RUBY_PLATFORM - require 'fiddle' - - # Dynamically load Foundation.framework, ~implicitly~ initialising - # the Objective-C runtime before any forking happens in Unicorn - # - # From https://bugs.ruby-lang.org/issues/14009 - Fiddle.dlopen '/System/Library/Frameworks/Foundation.framework/Foundation' + # Signal application hooks that we're about to fork + Gitlab::Cluster::LifecycleEvents.do_before_fork + + # The following is only recommended for memory/DB-constrained + # installations. It is not needed if your system can house + # twice as many worker_processes as you have configured. + # + # This allows a new master process to incrementally + # phase out the old master process with SIGTTOU to avoid a + # thundering herd (especially in the "preload_app false" case) + # when doing a transparent upgrade. The last worker spawned + # will then kill off the old master process with a SIGQUIT. + old_pid = "#{server.config[:pid]}.oldbin" + if old_pid != server.pid + begin + sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU + Process.kill(sig, File.read(old_pid).to_i) + rescue Errno::ENOENT, Errno::ESRCH + end end + # + # Throttle the master from forking too quickly by sleeping. Due + # to the implementation of standard Unix signal handlers, this + # helps (but does not completely) prevent identical, repeated signals + # from being lost when the receiving process is busy. + # sleep 1 end after_fork do |server, worker| - # Unicorn clears out signals before it forks, so rbtrace won't work - # unless it is enabled after the fork. - require 'rbtrace' if ENV['ENABLE_RBTRACE'] + # Signal application hooks of worker start + Gitlab::Cluster::LifecycleEvents.do_worker_start - # the following is *required* for Rails + "preload_app true", - defined?(ActiveRecord::Base) and - ActiveRecord::Base.establish_connection + # per-process listener ports for debugging/admin/migrations + # addr = "127.0.0.1:#{9293 + worker.nr}" + # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true) end + diff --git a/config/webpack.config.js b/config/webpack.config.js index 583f05f2fb7..9ecae9790fd 100644 --- a/config/webpack.config.js +++ b/config/webpack.config.js @@ -8,7 +8,7 @@ const MonacoWebpackPlugin = require('monaco-editor-webpack-plugin'); const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin; const ROOT_PATH = path.resolve(__dirname, '..'); -const CACHE_PATH = path.join(ROOT_PATH, 'tmp/cache'); +const CACHE_PATH = process.env.WEBPACK_CACHE_PATH || path.join(ROOT_PATH, 'tmp/cache'); const IS_PRODUCTION = process.env.NODE_ENV === 'production'; const IS_DEV_SERVER = process.argv.join(' ').indexOf('webpack-dev-server') !== -1; const DEV_SERVER_HOST = process.env.DEV_SERVER_HOST || 'localhost'; |