diff options
Diffstat (limited to 'lib')
124 files changed, 3345 insertions, 403 deletions
diff --git a/lib/api/commits.rb b/lib/api/commits.rb index fcaff35459e..5aeffc8fb99 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -73,7 +73,26 @@ module API params do requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false requires :commit_message, type: String, desc: 'Commit message' - requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' + requires :actions, type: Array, desc: 'Actions to perform in commit' do + requires :action, type: String, desc: 'The action to perform, `create`, `delete`, `move`, `update`, `chmod`', values: %w[create update move delete chmod].freeze + requires :file_path, type: String, desc: 'Full path to the file. Ex. `lib/class.rb`' + given action: ->(action) { action == 'move' } do + requires :previous_path, type: String, desc: 'Original full path to the file being moved. Ex. `lib/class1.rb`' + end + given action: ->(action) { %w[create move].include? action } do + optional :content, type: String, desc: 'File content' + end + given action: ->(action) { action == 'update' } do + requires :content, type: String, desc: 'File content' + end + optional :encoding, type: String, desc: '`text` or `base64`', default: 'text', values: %w[text base64] + given action: ->(action) { %w[update move delete].include? action } do + optional :last_commit_id, type: String, desc: 'Last known file commit id' + end + given action: ->(action) { action == 'chmod' } do + requires :execute_filemode, type: Boolean, desc: 'When `true/false` enables/disables the execute flag on the file.' + end + end optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 0fec3dc3dc4..12c4340c1ba 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -53,7 +53,7 @@ module API class User < UserBasic expose :created_at, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } - expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization + expose :bio, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization end class UserActivity < Grape::Entity diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index e569fad8663..804f6fa9b73 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -44,10 +44,10 @@ module API params do requires :name, type: String, desc: 'The name of the protected branch' optional :push_access_level, type: Integer, - values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, + values: ProtectedBranch::PushAccessLevel.allowed_access_levels, desc: 'Access levels allowed to push (defaults: `40`, maintainer access level)' optional :merge_access_level, type: Integer, - values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, + values: ProtectedBranch::MergeAccessLevel.allowed_access_levels, desc: 'Access levels allowed to merge (defaults: `40`, maintainer access level)' end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/api/protected_tags.rb b/lib/api/protected_tags.rb index 219d51a2bc5..e406344e42d 100644 --- a/lib/api/protected_tags.rb +++ b/lib/api/protected_tags.rb @@ -47,7 +47,7 @@ module API params do requires :name, type: String, desc: 'The name of the protected tag' optional :create_access_level, type: Integer, default: Gitlab::Access::MAINTAINER, - values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, + values: ProtectedTag::CreateAccessLevel.allowed_access_levels, desc: 'Access levels allowed to create (defaults: `40`, maintainer access level)' end post ':id/protected_tags' do diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 30abd0b63e9..9bcdfc8cb15 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -11,10 +11,18 @@ module API params do optional :scope, type: String, values: Ci::Runner::AVAILABLE_STATUSES, desc: 'The scope of specific runners to show' + optional :type, type: String, values: Ci::Runner::AVAILABLE_TYPES, + desc: 'The type of the runners to show' + optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, + desc: 'The status of the runners to show' use :pagination end get do - runners = filter_runners(current_user.ci_owned_runners, params[:scope], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + runners = current_user.ci_owned_runners + runners = filter_runners(runners, params[:scope], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) + runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + present paginate(runners), with: Entities::Runner end @@ -24,11 +32,20 @@ module API params do optional :scope, type: String, values: Ci::Runner::AVAILABLE_SCOPES, desc: 'The scope of specific runners to show' + optional :type, type: String, values: Ci::Runner::AVAILABLE_TYPES, + desc: 'The type of the runners to show' + optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, + desc: 'The status of the runners to show' use :pagination end get 'all' do authenticated_as_admin! - runners = filter_runners(Ci::Runner.all, params[:scope]) + + runners = Ci::Runner.all + runners = filter_runners(runners, params[:scope]) + runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) + runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + present paginate(runners), with: Entities::Runner end @@ -116,10 +133,18 @@ module API params do optional :scope, type: String, values: Ci::Runner::AVAILABLE_SCOPES, desc: 'The scope of specific runners to show' + optional :type, type: String, values: Ci::Runner::AVAILABLE_TYPES, + desc: 'The type of the runners to show' + optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, + desc: 'The status of the runners to show' use :pagination end get ':id/runners' do - runners = filter_runners(Ci::Runner.owned_or_instance_wide(user_project.id), params[:scope]) + runners = Ci::Runner.owned_or_instance_wide(user_project.id) + runners = filter_runners(runners, params[:scope]) + runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) + runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + present paginate(runners), with: Entities::Runner end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 897010217dc..8d71bd9dff1 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -102,7 +102,7 @@ module API end optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects' - optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' + optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to set up Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' end @@ -117,11 +117,6 @@ module API given shared_runners_enabled: ->(val) { val } do requires :shared_runners_text, type: String, desc: 'Shared runners text ' end - optional :sidekiq_throttling_enabled, type: Boolean, desc: 'Enable Sidekiq Job Throttling' - given sidekiq_throttling_enabled: ->(val) { val } do - requires :sidekiq_throttling_factor, type: Float, desc: 'The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.' - requires :sidekiq_throttling_queues, type: Array[String], desc: 'Choose which queues you wish to throttle' - end optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application' optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled' diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index ad0806df8e6..4764f8e1e19 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -296,7 +296,7 @@ module Banzai # Returns projects for the given paths. def find_for_paths(paths) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? cache = refs_cache to_query = paths - cache.keys @@ -340,7 +340,7 @@ module Banzai end def refs_cache - RequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} + Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} end def parent_type diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index b4a7a44e109..8159dcfed72 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -97,9 +97,7 @@ module Banzai private def external_issues_cached(attribute) - return project.public_send(attribute) unless RequestStore.active? # rubocop:disable GitlabSecurity/PublicSend - - cached_attributes = RequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } + cached_attributes = Gitlab::SafeRequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? # rubocop:disable GitlabSecurity/PublicSend cached_attributes[project.id][attribute] end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 8e838d04bad..7acbc933adc 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -60,7 +60,11 @@ module Banzai path_parts.unshift(relative_url_root, project.full_path) end - path = Addressable::URI.escape(File.join(*path_parts)) + begin + path = Addressable::URI.escape(File.join(*path_parts)) + rescue Addressable::URI::InvalidURIError + return + end html_attr.value = if context[:only_path] diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 68752f5bb5a..3ab154a7b1c 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -166,7 +166,7 @@ module Banzai # objects that have not yet been queried. For objects that have already # been queried the object is returned from the cache. def collection_objects_for_ids(collection, ids) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? ids = ids.map(&:to_i) cache = collection_cache[collection_cache_key(collection)] to_query = ids - cache.keys @@ -248,7 +248,7 @@ module Banzai end def collection_cache - RequestStore[:banzai_collection_cache] ||= Hash.new do |hash, key| + Gitlab::SafeRequestStore[:banzai_collection_cache] ||= Hash.new do |hash, key| hash[key] = {} end end diff --git a/lib/banzai/request_store_reference_cache.rb b/lib/banzai/request_store_reference_cache.rb index 426131442a2..9a9704f9837 100644 --- a/lib/banzai/request_store_reference_cache.rb +++ b/lib/banzai/request_store_reference_cache.rb @@ -1,8 +1,8 @@ module Banzai module RequestStoreReferenceCache def cached_call(request_store_key, cache_key, path: []) - if RequestStore.active? - cache = RequestStore[request_store_key] ||= Hash.new do |hash, key| + if Gitlab::SafeRequestStore.active? + cache = Gitlab::SafeRequestStore[request_store_key] ||= Hash.new do |hash, key| hash[key] = Hash.new { |h, k| h[k] = {} } end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index f756a211a12..24fdcd6fbb1 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -1,76 +1,42 @@ -class EventFilter - attr_accessor :params - - class << self - def all - 'all' - end - - def push - 'push' - end - - def merged - 'merged' - end +# frozen_string_literal: true - def issue - 'issue' - end - - def comments - 'comments' - end - - def team - 'team' - end +class EventFilter + attr_accessor :filter + + ALL = 'all' + PUSH = 'push' + MERGED = 'merged' + ISSUE = 'issue' + COMMENTS = 'comments' + TEAM = 'team' + FILTERS = [ALL, PUSH, MERGED, ISSUE, COMMENTS, TEAM].freeze + + def initialize(filter) + # Split using comma to maintain backward compatibility Ex/ "filter1,filter2" + filter = filter.to_s.split(',')[0].to_s + @filter = FILTERS.include?(filter) ? filter : ALL end - def initialize(params) - @params = if params - params.dup - else - [] # EventFilter.default_filter - end + def active?(key) + filter == key.to_s end # rubocop: disable CodeReuse/ActiveRecord def apply_filter(events) - return events if params.blank? || params == EventFilter.all - - case params - when EventFilter.push + case filter + when PUSH events.where(action: Event::PUSHED) - when EventFilter.merged + when MERGED events.where(action: Event::MERGED) - when EventFilter.comments + when COMMENTS events.where(action: Event::COMMENTED) - when EventFilter.team + when TEAM events.where(action: [Event::JOINED, Event::LEFT, Event::EXPIRED]) - when EventFilter.issue + when ISSUE events.where(action: [Event::CREATED, Event::UPDATED, Event::CLOSED, Event::REOPENED]) - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def options(key) - filter = params.dup - - if filter.include? key - filter.delete key else - filter << key - end - - filter - end - - def active?(key) - if params.present? - params.include? key - else - key == EventFilter.all + events end end + # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index e8dbde176ef..e02d403f7b1 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -54,7 +54,7 @@ module ExtractsPath valid_refs = ref_names.select { |v| id.start_with?("#{v}/") } - if valid_refs.length == 0 + if valid_refs.empty? # No exact ref match, so just try our best pair = id.match(%r{([^/]+)(.*)}).captures else diff --git a/lib/feature.rb b/lib/feature.rb index f4b57376313..0e90ad9a333 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -28,11 +28,7 @@ class Feature end def persisted_names - if RequestStore.active? - RequestStore[:flipper_persisted_names] ||= FlipperFeature.feature_names - else - FlipperFeature.feature_names - end + Gitlab::SafeRequestStore[:flipper_persisted_names] ||= FlipperFeature.feature_names end def persisted?(feature) @@ -76,11 +72,7 @@ class Feature end def flipper - if RequestStore.active? - RequestStore[:flipper] ||= build_flipper_instance - else - @flipper ||= build_flipper_instance - end + @flipper ||= (Gitlab::SafeRequestStore[:flipper] ||= build_flipper_instance) end def build_flipper_instance diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index eeab7791643..f323d2e0f7a 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -92,12 +92,12 @@ module Gitlab if provider Gitlab::AppLogger.info( "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \ - "blocking Gitlab user \"#{user.name}\" (#{user.email})" + "blocking GitLab user \"#{user.name}\" (#{user.email})" ) else Gitlab::AppLogger.info( "Account is not provided by LDAP, " \ - "blocking Gitlab user \"#{user.name}\" (#{user.email})" + "blocking GitLab user \"#{user.name}\" (#{user.email})" ) end end @@ -107,7 +107,7 @@ module Gitlab Gitlab::AppLogger.info( "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \ - "unblocking Gitlab user \"#{user.name}\" (#{user.email})" + "unblocking GitLab user \"#{user.name}\" (#{user.email})" ) end end diff --git a/lib/gitlab/background_migration/populate_external_pipeline_source.rb b/lib/gitlab/background_migration/populate_external_pipeline_source.rb new file mode 100644 index 00000000000..036fe641757 --- /dev/null +++ b/lib/gitlab/background_migration/populate_external_pipeline_source.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateExternalPipelineSource + module Migratable + class Pipeline < ActiveRecord::Base + self.table_name = 'ci_pipelines' + + def self.sources + { + unknown: nil, + push: 1, + web: 2, + trigger: 3, + schedule: 4, + api: 5, + external: 6 + } + end + end + + class CommitStatus < ActiveRecord::Base + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled + + scope :has_pipeline, -> { where('ci_builds.commit_id=ci_pipelines.id') } + scope :of_type, -> (type) { where('type=?', type) } + end + end + + def perform(start_id, stop_id) + external_pipelines(start_id, stop_id) + .update_all(source: Migratable::Pipeline.sources[:external]) + end + + private + + def external_pipelines(start_id, stop_id) + Migratable::Pipeline.where(id: (start_id..stop_id)) + .where( + 'EXISTS (?) AND NOT EXISTS (?)', + Migratable::CommitStatus.of_type('GenericCommitStatus').has_pipeline.select(1), + Migratable::CommitStatus.of_type('Ci::Build').has_pipeline.select(1) + ) + end + end + end +end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb index 671b8e7e1b1..b96e161a5b6 100644 --- a/lib/gitlab/cache/request_cache.rb +++ b/lib/gitlab/cache/request_cache.rb @@ -26,8 +26,8 @@ module Gitlab define_method(method_name) do |*args| store = - if RequestStore.active? - RequestStore.store + if Gitlab::SafeRequestStore.active? + Gitlab::SafeRequestStore.store else ivar_name = # ! and ? cannot be used as ivar name "@cache_#{method_name.to_s.tr('!?', "\u2605\u2606")}" diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 5963f3eb90c..7bc482a6f48 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -9,7 +11,7 @@ module Gitlab include Validatable include Attributable - ALLOWED_KEYS = %i[junit].freeze + ALLOWED_KEYS = %i[junit sast dependency_scanning container_scanning dast].freeze attributes ALLOWED_KEYS @@ -19,6 +21,10 @@ module Gitlab with_options allow_nil: true do validates :junit, array_of_strings_or_string: true + validates :sast, array_of_strings_or_string: true + validates :dependency_scanning, array_of_strings_or_string: true + validates :container_scanning, array_of_strings_or_string: true + validates :dast, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb deleted file mode 100644 index a4eccc08dfc..00000000000 --- a/lib/gitlab/ci/parsers.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Gitlab - module Ci - module Parsers - def self.fabricate!(file_type) - "Gitlab::Ci::Parsers::#{file_type.classify}".constantize.new - end - end - end -end diff --git a/lib/gitlab/ci/parsers/junit.rb b/lib/gitlab/ci/parsers/junit.rb deleted file mode 100644 index d1c136f2009..00000000000 --- a/lib/gitlab/ci/parsers/junit.rb +++ /dev/null @@ -1,66 +0,0 @@ -module Gitlab - module Ci - module Parsers - class Junit - JunitParserError = Class.new(StandardError) - - def parse!(xml_data, test_suite) - root = Hash.from_xml(xml_data) - - all_cases(root) do |test_case| - test_case = create_test_case(test_case) - test_suite.add_test_case(test_case) - end - rescue REXML::ParseException => e - raise JunitParserError, "XML parsing failed: #{e.message}" - rescue => e - raise JunitParserError, "JUnit parsing failed: #{e.message}" - end - - private - - def all_cases(root, parent = nil, &blk) - return unless root.present? - - [root].flatten.compact.map do |node| - next unless node.is_a?(Hash) - - # we allow only one top-level 'testsuites' - all_cases(node['testsuites'], root, &blk) unless parent - - # we require at least one level of testsuites or testsuite - each_case(node['testcase'], &blk) if parent - - # we allow multiple nested 'testsuite' (eg. PHPUnit) - all_cases(node['testsuite'], root, &blk) - end - end - - def each_case(testcase, &blk) - return unless testcase.present? - - [testcase].flatten.compact.map(&blk) - end - - def create_test_case(data) - if data['failure'] - status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED - system_output = data['failure'] - else - status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS - system_output = nil - end - - ::Gitlab::Ci::Reports::TestCase.new( - classname: data['classname'], - name: data['name'], - file: data['file'], - execution_time: data['time'], - status: status, - system_output: system_output - ) - end - end - end - end -end diff --git a/lib/gitlab/ci/parsers/test.rb b/lib/gitlab/ci/parsers/test.rb new file mode 100644 index 00000000000..c6bc9662b07 --- /dev/null +++ b/lib/gitlab/ci/parsers/test.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Test + ParserNotFoundError = Class.new(StandardError) + + PARSERS = { + junit: ::Gitlab::Ci::Parsers::Test::Junit + }.freeze + + def self.fabricate!(file_type) + PARSERS.fetch(file_type.to_sym).new + rescue KeyError + raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb new file mode 100644 index 00000000000..5d7d9a751d8 --- /dev/null +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Test + class Junit + JunitParserError = Class.new(StandardError) + + def parse!(xml_data, test_suite) + root = Hash.from_xml(xml_data) + + all_cases(root) do |test_case| + test_case = create_test_case(test_case) + test_suite.add_test_case(test_case) + end + rescue REXML::ParseException => e + raise JunitParserError, "XML parsing failed: #{e.message}" + rescue => e + raise JunitParserError, "JUnit parsing failed: #{e.message}" + end + + private + + def all_cases(root, parent = nil, &blk) + return unless root.present? + + [root].flatten.compact.map do |node| + next unless node.is_a?(Hash) + + # we allow only one top-level 'testsuites' + all_cases(node['testsuites'], root, &blk) unless parent + + # we require at least one level of testsuites or testsuite + each_case(node['testcase'], &blk) if parent + + # we allow multiple nested 'testsuite' (eg. PHPUnit) + all_cases(node['testsuite'], root, &blk) + end + end + + def each_case(testcase, &blk) + return unless testcase.present? + + [testcase].flatten.compact.map(&blk) + end + + def create_test_case(data) + if data['failure'] + status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED + system_output = data['failure'] + else + status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS + system_output = nil + end + + ::Gitlab::Ci::Reports::TestCase.new( + classname: data['classname'], + name: data['name'], + file: data['file'], + execution_time: data['time'], + status: status, + system_output: system_output + ) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml new file mode 100644 index 00000000000..5f9d54ff574 --- /dev/null +++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml @@ -0,0 +1,51 @@ +# Read more about this script on this blog post https://about.gitlab.com/2016/11/30/setting-up-gitlab-ci-for-android-projects/, by Greyson Parrelli +image: openjdk:8-jdk + +variables: + ANDROID_COMPILE_SDK: "25" + ANDROID_BUILD_TOOLS: "24.0.0" + ANDROID_SDK_TOOLS: "24.4.1" + +before_script: + - apt-get --quiet update --yes + - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1 + - wget --quiet --output-document=android-sdk.tgz https://dl.google.com/android/android-sdk_r${ANDROID_SDK_TOOLS}-linux.tgz + - tar --extract --gzip --file=android-sdk.tgz + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter android-${ANDROID_COMPILE_SDK} + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter platform-tools + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter build-tools-${ANDROID_BUILD_TOOLS} + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter extra-android-m2repository + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter extra-google-google_play_services + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter extra-google-m2repository + - export ANDROID_HOME=$PWD/android-sdk-linux + - export PATH=$PATH:$PWD/android-sdk-linux/platform-tools/ + - chmod +x ./gradlew + +stages: + - build + - test + +build: + stage: build + script: + - ./gradlew assembleDebug + artifacts: + paths: + - app/build/outputs/ + +unitTests: + stage: test + script: + - ./gradlew test + +functionalTests: + stage: test + script: + - wget --quiet --output-document=android-wait-for-emulator https://raw.githubusercontent.com/travis-ci/travis-cookbooks/0f497eb71291b52a703143c5cd63a217c8766dc9/community-cookbooks/android-sdk/files/default/android-wait-for-emulator + - chmod +x android-wait-for-emulator + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter sys-img-x86-google_apis-${ANDROID_COMPILE_SDK} + - echo no | android-sdk-linux/tools/android create avd -n test -t android-${ANDROID_COMPILE_SDK} --abi google_apis/x86 + - android-sdk-linux/tools/emulator64-x86 -avd test -no-window -no-audio & + - ./android-wait-for-emulator + - adb shell input keyevent 82 + - ./gradlew cAT diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml new file mode 100644 index 00000000000..aebaf2d6982 --- /dev/null +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -0,0 +1,832 @@ +# Auto DevOps +# This CI/CD configuration provides a standard pipeline for +# * building a Docker image (using a buildpack if necessary), +# * storing the image in the container registry, +# * running tests from a buildpack, +# * running code quality analysis, +# * creating a review app for each topic branch, +# * and continuous deployment to production +# +# Test jobs may be disabled by setting environment variables: +# * test: TEST_DISABLED +# * code_quality: CODE_QUALITY_DISABLED +# * license_management: LICENSE_MANAGEMENT_DISABLED +# * performance: PERFORMANCE_DISABLED +# * sast: SAST_DISABLED +# * dependency_scanning: DEPENDENCY_SCANNING_DISABLED +# * container_scanning: CONTAINER_SCANNING_DISABLED +# * dast: DAST_DISABLED +# * review: REVIEW_DISABLED +# * stop_review: REVIEW_DISABLED +# +# In order to deploy, you must have a Kubernetes cluster configured either +# via a project integration, or via group/project variables. +# AUTO_DEVOPS_DOMAIN must also be set as a variable at the group or project +# level, or manually added below. +# +# Continuous deployment to production is enabled by default. +# If you want to deploy to staging first, or enable incremental rollouts, +# set STAGING_ENABLED or INCREMENTAL_ROLLOUT_ENABLED environment variables. +# If you want to use canary deployments, set CANARY_ENABLED environment variable. +# +# If Auto DevOps fails to detect the proper buildpack, or if you want to +# specify a custom buildpack, set a project variable `BUILDPACK_URL` to the +# repository URL of the buildpack. +# e.g. BUILDPACK_URL=https://github.com/heroku/heroku-buildpack-ruby.git#v142 +# If you need multiple buildpacks, add a file to your project called +# `.buildpacks` that contains the URLs, one on each line, in order. +# Note: Auto CI does not work with multiple buildpacks yet + +image: alpine:latest + +variables: + # AUTO_DEVOPS_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. + # AUTO_DEVOPS_DOMAIN: domain.example.com + + POSTGRES_USER: user + POSTGRES_PASSWORD: testing-password + POSTGRES_ENABLED: "true" + POSTGRES_DB: $CI_ENVIRONMENT_SLUG + + KUBERNETES_VERSION: 1.8.6 + HELM_VERSION: 2.10.0 + + DOCKER_DRIVER: overlay2 + +stages: + - build + - test + - review + - dast + - staging + - canary + - production + - performance + - cleanup + +build: + stage: build + image: docker:stable-git + services: + - docker:stable-dind + script: + - setup_docker + - build + only: + - branches + +test: + services: + - postgres:latest + variables: + POSTGRES_DB: test + stage: test + image: gliderlabs/herokuish:latest + script: + - setup_test_db + - cp -R . /tmp/app + - /bin/herokuish buildpack test + only: + - branches + except: + variables: + - $TEST_DISABLED + +code_quality: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - code_quality + artifacts: + paths: [gl-code-quality-report.json] + only: + - branches + except: + variables: + - $CODE_QUALITY_DISABLED + +license_management: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - license_management + artifacts: + paths: [gl-license-management-report.json] + only: + - branches + only: + variables: + - $GITLAB_FEATURES =~ /\blicense_management\b/ + except: + variables: + - $LICENSE_MANAGEMENT_DISABLED + +performance: + stage: performance + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - performance + artifacts: + paths: + - performance.json + - sitespeed-results/ + only: + refs: + - branches + kubernetes: active + except: + variables: + - $PERFORMANCE_DISABLED + +sast: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - sast + artifacts: + paths: [gl-sast-report.json] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bsast\b/ + except: + variables: + - $SAST_DISABLED + +dependency_scanning: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - dependency_scanning + artifacts: + paths: [gl-dependency-scanning-report.json] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + except: + variables: + - $DEPENDENCY_SCANNING_DISABLED + +container_scanning: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - container_scanning + artifacts: + paths: [gl-container-scanning-report.json] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bsast_container\b/ + except: + variables: + - $CONTAINER_SCANNING_DISABLED + +dast: + stage: dast + allow_failure: true + image: registry.gitlab.com/gitlab-org/security-products/zaproxy + variables: + POSTGRES_DB: "false" + script: + - dast + artifacts: + paths: [gl-dast-report.json] + only: + refs: + - branches + kubernetes: active + variables: + - $GITLAB_FEATURES =~ /\bdast\b/ + except: + refs: + - master + variables: + - $DAST_DISABLED + +review: + stage: review + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - install_tiller + - create_secret + - deploy + - persist_environment_url + environment: + name: review/$CI_COMMIT_REF_NAME + url: http://$CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN + on_stop: stop_review + artifacts: + paths: [environment_url.txt] + only: + refs: + - branches + kubernetes: active + except: + refs: + - master + variables: + - $REVIEW_DISABLED + +stop_review: + stage: cleanup + variables: + GIT_STRATEGY: none + script: + - install_dependencies + - delete + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + when: manual + allow_failure: true + only: + refs: + - branches + kubernetes: active + except: + refs: + - master + variables: + - $REVIEW_DISABLED + +# Keys that start with a dot (.) will not be processed by GitLab CI. +# Staging and canary jobs are disabled by default, to enable them +# remove the dot (.) before the job name. +# https://docs.gitlab.com/ee/ci/yaml/README.html#hidden-keys + +# Staging deploys are disabled by default since +# continuous deployment to production is enabled by default +# If you prefer to automatically deploy to staging and +# only manually promote to production, enable this job by setting +# STAGING_ENABLED. + +staging: + stage: staging + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - install_tiller + - create_secret + - deploy + environment: + name: staging + url: http://$CI_PROJECT_PATH_SLUG-staging.$AUTO_DEVOPS_DOMAIN + only: + refs: + - master + kubernetes: active + variables: + - $STAGING_ENABLED + +# Canaries are also disabled by default, but if you want them, +# and know what the downsides are, you can enable this by setting +# CANARY_ENABLED. + +canary: + stage: canary + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - install_tiller + - create_secret + - deploy canary + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + when: manual + only: + refs: + - master + kubernetes: active + variables: + - $CANARY_ENABLED + +.production: &production_template + stage: production + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - install_tiller + - create_secret + - deploy + - delete canary + - delete rollout + - persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + artifacts: + paths: [environment_url.txt] + +production: + <<: *production_template + only: + refs: + - master + kubernetes: active + except: + variables: + - $STAGING_ENABLED + - $CANARY_ENABLED + - $INCREMENTAL_ROLLOUT_ENABLED + +production_manual: + <<: *production_template + when: manual + allow_failure: false + only: + refs: + - master + kubernetes: active + variables: + - $STAGING_ENABLED + - $CANARY_ENABLED + except: + variables: + - $INCREMENTAL_ROLLOUT_ENABLED + +# This job implements incremental rollout on for every push to `master`. + +.rollout: &rollout_template + stage: production + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - install_tiller + - create_secret + - deploy rollout $ROLLOUT_PERCENTAGE + - scale stable $((100-ROLLOUT_PERCENTAGE)) + - delete canary + - persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + artifacts: + paths: [environment_url.txt] + +rollout 10%: + <<: *rollout_template + variables: + ROLLOUT_PERCENTAGE: 10 + when: manual + only: + refs: + - master + kubernetes: active + variables: + - $INCREMENTAL_ROLLOUT_ENABLED + +rollout 25%: + <<: *rollout_template + variables: + ROLLOUT_PERCENTAGE: 25 + when: manual + only: + refs: + - master + kubernetes: active + variables: + - $INCREMENTAL_ROLLOUT_ENABLED + +rollout 50%: + <<: *rollout_template + variables: + ROLLOUT_PERCENTAGE: 50 + when: manual + only: + refs: + - master + kubernetes: active + variables: + - $INCREMENTAL_ROLLOUT_ENABLED + +rollout 100%: + <<: *production_template + when: manual + allow_failure: false + only: + refs: + - master + kubernetes: active + variables: + - $INCREMENTAL_ROLLOUT_ENABLED + +# --------------------------------------------------------------------------- + +.auto_devops: &auto_devops | + # Auto DevOps variables and functions + [[ "$TRACE" ]] && set -x + auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} + export DATABASE_URL=${DATABASE_URL-$auto_database_url} + export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG + export CI_APPLICATION_TAG=$CI_COMMIT_SHA + export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID} + export TILLER_NAMESPACE=$KUBE_NAMESPACE + # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products + export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + + function registry_login() { + if [[ -n "$CI_REGISTRY_USER" ]]; then + echo "Logging to GitLab Container Registry with CI credentials..." + docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" + echo "" + fi + } + + function container_scanning() { + registry_login + + docker run -d --name db arminc/clair-db:latest + docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.1 + apk add -U wget ca-certificates + docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} + wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64 + mv clair-scanner_linux_amd64 clair-scanner + chmod +x clair-scanner + touch clair-whitelist.yml + retries=0 + echo "Waiting for clair daemon to start" + while( ! wget -T 10 -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done + ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-container-scanning-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true + } + + function code_quality() { + docker run --env SOURCE_CODE="$PWD" \ + --volume "$PWD":/code \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code + } + + function license_management() { + # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" + LICENSE_MANAGEMENT_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + + docker run --volume "$PWD:/code" \ + "registry.gitlab.com/gitlab-org/security-products/license-management:$LICENSE_MANAGEMENT_VERSION" analyze /code + } + + function sast() { + case "$CI_SERVER_VERSION" in + *-ee) + + # Deprecation notice for CONFIDENCE_LEVEL variable + if [ -z "$SAST_CONFIDENCE_LEVEL" -a "$CONFIDENCE_LEVEL" ]; then + SAST_CONFIDENCE_LEVEL="$CONFIDENCE_LEVEL" + echo "WARNING: CONFIDENCE_LEVEL is deprecated and MUST be replaced with SAST_CONFIDENCE_LEVEL" + fi + + docker run --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}" \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code + ;; + *) + echo "GitLab EE is required" + ;; + esac + } + + function dependency_scanning() { + case "$CI_SERVER_VERSION" in + *-ee) + docker run --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}" \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code + ;; + *) + echo "GitLab EE is required" + ;; + esac + } + + function get_replicas() { + track="${1:-stable}" + percentage="${2:-100}" + + env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' ) + env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' ) + + if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then + # for stable track get number of replicas from `PRODUCTION_REPLICAS` + eval new_replicas=\$${env_slug}_REPLICAS + if [[ -z "$new_replicas" ]]; then + new_replicas=$REPLICAS + fi + else + # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` + eval new_replicas=\$${env_track}_${env_slug}_REPLICAS + if [[ -z "$new_replicas" ]]; then + eval new_replicas=\${env_track}_REPLICAS + fi + fi + + replicas="${new_replicas:-1}" + replicas="$(($replicas * $percentage / 100))" + + # always return at least one replicas + if [[ $replicas -gt 0 ]]; then + echo "$replicas" + else + echo 1 + fi + } + + function deploy() { + track="${1-stable}" + percentage="${2:-100}" + name="$CI_ENVIRONMENT_SLUG" + + replicas="1" + service_enabled="true" + postgres_enabled="$POSTGRES_ENABLED" + + # if track is different than stable, + # re-use all attached resources + if [[ "$track" != "stable" ]]; then + name="$name-$track" + service_enabled="false" + postgres_enabled="false" + fi + + replicas=$(get_replicas "$track" "$percentage") + + if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then + secret_name='gitlab-registry' + else + secret_name='' + fi + + helm upgrade --install \ + --wait \ + --set service.enabled="$service_enabled" \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set image.repository="$CI_APPLICATION_REPOSITORY" \ + --set image.tag="$CI_APPLICATION_TAG" \ + --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ + --set application.track="$track" \ + --set application.database_url="$DATABASE_URL" \ + --set service.url="$CI_ENVIRONMENT_URL" \ + --set replicaCount="$replicas" \ + --set postgresql.enabled="$postgres_enabled" \ + --set postgresql.nameOverride="postgres" \ + --set postgresql.postgresUser="$POSTGRES_USER" \ + --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ + --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + + kubectl rollout status -n "$KUBE_NAMESPACE" -w "deployment/$name" + } + + function scale() { + track="${1-stable}" + percentage="${2-100}" + name="$CI_ENVIRONMENT_SLUG" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + replicas=$(get_replicas "$track" "$percentage") + + if [[ -n "$(helm ls -q "^$name$")" ]]; then + helm upgrade --reuse-values \ + --wait \ + --set replicaCount="$replicas" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + fi + } + + function install_dependencies() { + apk add -U openssl curl tar gzip bash ca-certificates git + curl -L -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub + curl -L -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk + apk add glibc-2.28-r0.apk + rm glibc-2.28-r0.apk + + curl "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx + mv linux-amd64/helm /usr/bin/ + helm version --client + + curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" + chmod +x /usr/bin/kubectl + kubectl version --client + } + + function setup_docker() { + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + } + + function setup_test_db() { + if [ -z ${KUBERNETES_PORT+x} ]; then + DB_HOST=postgres + else + DB_HOST=localhost + fi + export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}" + } + + function download_chart() { + if [[ ! -d chart ]]; then + auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app} + auto_chart_name=$(basename $auto_chart) + auto_chart_name=${auto_chart_name%.tgz} + auto_chart_name=${auto_chart_name%.tar.gz} + else + auto_chart="chart" + auto_chart_name="chart" + fi + + helm init --client-only + helm repo add gitlab https://charts.gitlab.io + if [[ ! -d "$auto_chart" ]]; then + helm fetch ${auto_chart} --untar + fi + if [ "$auto_chart_name" != "chart" ]; then + mv ${auto_chart_name} chart + fi + + helm dependency update chart/ + helm dependency build chart/ + } + + function ensure_namespace() { + kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" + } + + function check_kube_domain() { + if [ -z ${AUTO_DEVOPS_DOMAIN+x} ]; then + echo "In order to deploy or use Review Apps, AUTO_DEVOPS_DOMAIN variable must be set" + echo "You can do it in Auto DevOps project settings or defining a variable at group or project level" + echo "You can also manually add it in .gitlab-ci.yml" + false + else + true + fi + } + + function build() { + registry_login + + if [[ -f Dockerfile ]]; then + echo "Building Dockerfile-based application..." + docker build \ + --build-arg HTTP_PROXY="$HTTP_PROXY" \ + --build-arg http_proxy="$http_proxy" \ + --build-arg HTTPS_PROXY="$HTTPS_PROXY" \ + --build-arg https_proxy="$https_proxy" \ + --build-arg FTP_PROXY="$FTP_PROXY" \ + --build-arg ftp_proxy="$ftp_proxy" \ + --build-arg NO_PROXY="$NO_PROXY" \ + --build-arg no_proxy="$no_proxy" \ + -t "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" . + else + echo "Building Heroku-based application using gliderlabs/herokuish docker image..." + docker run -i \ + -e BUILDPACK_URL \ + -e HTTP_PROXY \ + -e http_proxy \ + -e HTTPS_PROXY \ + -e https_proxy \ + -e FTP_PROXY \ + -e ftp_proxy \ + -e NO_PROXY \ + -e no_proxy \ + --name="$CI_CONTAINER_NAME" -v "$(pwd):/tmp/app:ro" gliderlabs/herokuish /bin/herokuish buildpack build + docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" + docker rm "$CI_CONTAINER_NAME" >/dev/null + echo "" + + echo "Configuring $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG docker image..." + docker create --expose 5000 --env PORT=5000 --name="$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" /bin/herokuish procfile start web + docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" + docker rm "$CI_CONTAINER_NAME" >/dev/null + echo "" + fi + + echo "Pushing to GitLab Container Registry..." + docker push "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" + echo "" + } + + function install_tiller() { + echo "Checking Tiller..." + helm init --upgrade + kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy" + if ! helm version --debug; then + echo "Failed to init Tiller." + return 1 + fi + echo "" + } + + function create_secret() { + echo "Create secret..." + if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then + return + fi + + kubectl create secret -n "$KUBE_NAMESPACE" \ + docker-registry gitlab-registry \ + --docker-server="$CI_REGISTRY" \ + --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \ + --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \ + --docker-email="$GITLAB_USER_EMAIL" \ + -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f - + } + + function dast() { + export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + + mkdir /zap/wrk/ + /zap/zap-baseline.py -J gl-dast-report.json -t "$CI_ENVIRONMENT_URL" || true + cp /zap/wrk/gl-dast-report.json . + } + + function performance() { + export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + + mkdir gitlab-exporter + wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js + + mkdir sitespeed-results + + if [ -f .gitlab-urls.txt ] + then + sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt + else + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" + fi + + mv sitespeed-results/data/performance.json performance.json + } + + function persist_environment_url() { + echo $CI_ENVIRONMENT_URL > environment_url.txt + } + + function delete() { + track="${1-stable}" + name="$CI_ENVIRONMENT_SLUG" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + if [[ -n "$(helm ls -q "^$name$")" ]]; then + helm delete --purge "$name" + fi + } + +before_script: + - *auto_devops diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml new file mode 100644 index 00000000000..2d218b2e164 --- /dev/null +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -0,0 +1,35 @@ +# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options + +# you can delete this line if you're not using Docker +image: busybox:latest + +before_script: + - echo "Before script section" + - echo "For example you might run an update here or install a build dependency" + - echo "Or perhaps you might print out some debugging details" + +after_script: + - echo "After script section" + - echo "For example you might do some cleanup here" + +build1: + stage: build + script: + - echo "Do your build here" + +test1: + stage: test + script: + - echo "Do a test here" + - echo "For example run a test suite" + +test2: + stage: test + script: + - echo "Do another parallel test here" + - echo "For example run a lint test" + +deploy1: + stage: deploy + script: + - echo "Do your deploy here" diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml new file mode 100644 index 00000000000..c83c49d8c95 --- /dev/null +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -0,0 +1,26 @@ +# use the official gcc image, based on debian +# can use verions as well, like gcc:5.2 +# see https://hub.docker.com/_/gcc/ +image: gcc + +build: + stage: build + # instead of calling g++ directly you can also use some build toolkit like make + # install the necessary build tools when needed + # before_script: + # - apt update && apt -y install make autoconf + script: + - g++ helloworld.cpp -o mybinary + artifacts: + paths: + - mybinary + # depending on your build setup it's most likely a good idea to cache outputs to reduce the build time + # cache: + # paths: + # - "*.o" + +# run tests using the binary built before +test: + stage: test + script: + - ./runmytests.sh diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml new file mode 100644 index 00000000000..4d5b6484d6e --- /dev/null +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -0,0 +1,51 @@ +# This file uses Test Kitchen with the kitchen-dokken driver to +# perform functional testing. Doing so requires that your runner be a +# Docker runner configured for privileged mode. Please see +# https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode +# for help configuring your runner properly, or, if you want to switch +# to a different driver, see http://kitchen.ci/docs/drivers + +image: "chef/chefdk" +services: + - docker:dind + +variables: + DOCKER_HOST: "tcp://docker:2375" + KITCHEN_LOCAL_YAML: ".kitchen.dokken.yml" + +stages: + - lint + - unit + - functional + +foodcritic: + stage: lint + script: + - chef exec foodcritic . + +cookstyle: + stage: lint + script: + - chef exec cookstyle . + +chefspec: + stage: unit + script: + - chef exec rspec spec + +# Set up your test matrix here. Example: +#verify-centos-6: +# stage: functional +# before_script: +# - apt-get update +# - apt-get -y install rsync +# script: +# - kitchen verify default-centos-6 --destroy=always +# +#verify-centos-7: +# stage: functional +# before_script: +# - apt-get update +# - apt-get -y install rsync +# script: +# - kitchen verify default-centos-7 --destroy=always diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml new file mode 100644 index 00000000000..f066285b1ad --- /dev/null +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -0,0 +1,22 @@ +# Based on openjdk:8, already includes lein +image: clojure:lein-2.7.0 +# If you need to configure a database, add a `services` section here +# See https://docs.gitlab.com/ce/ci/services/postgres.html +# Make sure you configure the connection as well + +before_script: + # If you need to install any external applications, like a + # postgres client, you may want to uncomment the line below: + # + #- apt-get update -y + # + # Retrieve project dependencies + # Do this on before_script since it'll be shared between both test and + # any production sections a user adds + - lein deps + +test: + script: + # If you need to run any migrations or configure the database, this + # would be the point to do it. + - lein test diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml new file mode 100644 index 00000000000..36386a19fdc --- /dev/null +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -0,0 +1,36 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/crystallang/crystal/ +image: "crystallang/crystal:latest" + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# services: +# - mysql:latest +# - redis:latest +# - postgres:latest + +# variables: +# POSTGRES_DB: database_name + +# Cache shards in between builds +cache: + paths: + - lib + +# This is a basic example for a shard or script which doesn't use +# services such as redis or postgres +before_script: + - apt-get update -qq && apt-get install -y -qq libxml2-dev + - crystal -v # Print out Crystal version for debugging + - shards + +# If you are using built-in Crystal Spec. +spec: + script: + - crystal spec + +# If you are using minitest.cr +minitest: + script: + - crystal test/spec_test.cr # change to the file(s) you execute for tests diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml new file mode 100644 index 00000000000..57afcbbe8b5 --- /dev/null +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -0,0 +1,49 @@ +# Official framework image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/python +image: python:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - postgres:latest + +variables: + POSTGRES_DB: database_name + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - ~/.cache/pip/ + +# This is a basic example for a gem or script which doesn't use +# services such as redis or postgres +before_script: + - python -V # Print out python version for debugging + # Uncomment next line if your Django app needs a JS runtime: + # - apt-get update -q && apt-get install nodejs -yqq + - pip install -r requirements.txt + +# To get Django tests to work you may need to create a settings file using +# the following DATABASES: +# +# DATABASES = { +# 'default': { +# 'ENGINE': 'django.db.backends.postgresql_psycopg2', +# 'NAME': 'ci', +# 'USER': 'postgres', +# 'PASSWORD': 'postgres', +# 'HOST': 'postgres', +# 'PORT': '5432', +# }, +# } +# +# and then adding `--settings app.settings.ci` (or similar) to the test command + +test: + variables: + DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB" + script: + - python manage.py test diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml new file mode 100644 index 00000000000..eeefadaa019 --- /dev/null +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -0,0 +1,24 @@ +# Official docker image. +image: docker:latest + +services: + - docker:dind + +before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + +build-master: + stage: build + script: + - docker build --pull -t "$CI_REGISTRY_IMAGE" . + - docker push "$CI_REGISTRY_IMAGE" + only: + - master + +build: + stage: build + script: + - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" . + - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" + except: + - master diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml new file mode 100644 index 00000000000..cf9c731637c --- /dev/null +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -0,0 +1,18 @@ +image: elixir:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - redis:latest + - postgres:latest + +before_script: + - mix local.rebar --force + - mix local.hex --force + - mix deps.get + +mix: + script: + - mix test diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml new file mode 100644 index 00000000000..d572d7a1edc --- /dev/null +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -0,0 +1,35 @@ +image: golang:latest + +variables: + # Please edit to your GitLab project + REPO_NAME: gitlab.com/namespace/project + +# The problem is that to be able to use go get, one needs to put +# the repository in the $GOPATH. So for example if your gitlab domain +# is gitlab.com, and that your repository is namespace/project, and +# the default GOPATH being /go, then you'd need to have your +# repository in /go/src/gitlab.com/namespace/project +# Thus, making a symbolic link corrects this. +before_script: + - mkdir -p $GOPATH/src/$(dirname $REPO_NAME) + - ln -svf $CI_PROJECT_DIR $GOPATH/src/$REPO_NAME + - cd $GOPATH/src/$REPO_NAME + +stages: + - test + - build + +format: + stage: test + script: + - go fmt $(go list ./... | grep -v /vendor/) + - go vet $(go list ./... | grep -v /vendor/) + - go test -race $(go list ./... | grep -v /vendor/) + +compile: + stage: build + script: + - go build -race -ldflags "-extldflags '-static'" -o $CI_PROJECT_DIR/mybinary + artifacts: + paths: + - mybinary diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml new file mode 100644 index 00000000000..48d98dddfad --- /dev/null +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -0,0 +1,36 @@ +# This is the Gradle build system for JVM applications +# https://gradle.org/ +# https://github.com/gradle/gradle +image: gradle:alpine + +# Disable the Gradle daemon for Continuous Integration servers as correctness +# is usually a priority over speed in CI environments. Using a fresh +# runtime for each build is more reliable since the runtime is completely +# isolated from any previous builds. +variables: + GRADLE_OPTS: "-Dorg.gradle.daemon=false" + +before_script: + - export GRADLE_USER_HOME=`pwd`/.gradle + +build: + stage: build + script: gradle --build-cache assemble + cache: + key: "$CI_COMMIT_REF_NAME" + policy: push + paths: + - build + - .gradle + + +test: + stage: test + script: gradle check + cache: + key: "$CI_COMMIT_REF_NAME" + policy: pull + paths: + - build + - .gradle + diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml new file mode 100644 index 00000000000..7fc698d50cf --- /dev/null +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -0,0 +1,40 @@ +# This template uses the java:8 docker image because there isn't any +# official Grails image at this moment +# +# Grails Framework https://grails.org/ is a powerful Groovy-based web application framework for the JVM +# +# This yml works with Grails 3.x only +# Feel free to change GRAILS_VERSION version with your project version (3.0.1, 3.1.1,...) +# Feel free to change GRADLE_VERSION version with your gradle project version (2.13, 2.14,...) +# If you use Angular profile, this yml it's prepared to work with it + +image: java:8 + +variables: + GRAILS_VERSION: "3.1.9" + GRADLE_VERSION: "2.13" + +# We use SDKMan as tool for managing versions +before_script: + - apt-get update -qq && apt-get install -y -qq unzip + - curl -sSL https://get.sdkman.io | bash + - echo sdkman_auto_answer=true > /root/.sdkman/etc/config + - source /root/.sdkman/bin/sdkman-init.sh + - sdk install gradle $GRADLE_VERSION < /dev/null + - sdk use gradle $GRADLE_VERSION +# As it's not a good idea to version gradle.properties feel free to add your +# environments variable here + - echo grailsVersion=$GRAILS_VERSION > gradle.properties + - echo gradleWrapperVersion=2.14 >> gradle.properties +# refresh dependencies from your project + - ./gradlew --refresh-dependencies +# Be aware that if you are using Angular profile, +# Bower cannot be run as root if you don't allow it before. +# Feel free to remove next line if you are not using Bower + - echo {\"allow_root\":true} > /root/.bowerrc + +# This build job does the full grails pipeline +# (compile, test, integrationTest, war, assemble). +build: + script: + - ./gradlew build
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml new file mode 100644 index 00000000000..04c21b4725d --- /dev/null +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -0,0 +1,76 @@ +# This is an example .gitlab-ci.yml file to test (and optionally report the coverage +# results of) your [Julia][1] packages. Please refer to the [documentation][2] +# for more information about package development in Julia. +# +# Here, it is assumed that your Julia package is named `MyPackage`. Change it to +# whatever name you have given to your package. +# +# [1]: http://julialang.org/ +# [2]: https://docs.julialang.org/en/v1/manual/documentation/index.html + +# Below is the template to run your tests in Julia +.test_template: &test_definition + # Uncomment below if you would like to run the tests on specific references + # only, such as the branches `master`, `development`, etc. + # only: + # - master + # - development + script: + # Let's run the tests. Substitute `coverage = false` below, if you do not + # want coverage results. + - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage"); Pkg.test("MyPackage"; coverage = true)' + # Comment out below if you do not want coverage results. + - julia -e 'using Pkg; Pkg.add("Coverage"); + import MyPackage; cd(joinpath(dirname(pathof(MyPackage)), "..")); + using Coverage; cl, tl = get_summary(process_folder()); + println("(", cl/tl*100, "%) covered")' + +# Name a test and select an appropriate image. +# images comes from Docker hub +test:0.7: + image: julia:0.7 + <<: *test_definition + +test:1.0: + image: julia:1.0 + <<: *test_definition + +# Maybe you would like to test your package against the development branch: +# test:1.1-dev (not sure there is such an image in docker, so not tested yet): +# image: julia:v1.1-dev +# # ... allowing for failures, since we are testing against the development +# # branch: +# allow_failure: true +# <<: *test_definition + +# REMARK: Do not forget to enable the coverage feature for your project, if you +# are using code coverage reporting above. This can be done by +# +# - Navigating to the `CI/CD Pipelines` settings of your project, +# - Copying and pasting the default `Simplecov` regex example provided, i.e., +# `\(\d+.\d+\%\) covered` in the `test coverage parsing` textfield. + +# Example documentation deployment +pages: + image: julia:0.7 + stage: deploy + script: + - apt-get update -qq && apt-get install -y git # needed by Documenter + - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage");' # rebuild Julia (can be put somewhere else I'm sure + - julia -e 'using Pkg; import MyPackage; Pkg.add("Documenter")' # install Documenter + - julia --color=yes docs/make.jl # make documentation + - mv docs/build public # move to the directory picked up by Gitlab pages + artifacts: + paths: + - public + only: + - master + + +# WARNING: This template is using the `julia` images from [Docker +# Hub][3]. One can use custom Julia images and/or the official ones found +# in the same place. However, care must be taken to correctly locate the binary +# file (`/opt/julia/bin/julia` above), which is usually given on the image's +# description page. +# +# [3]: https://hub.docker.com/_/julia/ diff --git a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml new file mode 100644 index 00000000000..a4aed36889e --- /dev/null +++ b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml @@ -0,0 +1,11 @@ +# use docker image with latex preinstalled +# since there is no official latex image, use https://github.com/blang/latex-docker +# possible alternative: https://github.com/natlownes/docker-latex +image: blang/latex + +build: + script: + - latexmk -pdf + artifacts: + paths: + - "*.pdf" diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml new file mode 100644 index 00000000000..0688f77a1d2 --- /dev/null +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -0,0 +1,82 @@ +# Official framework image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/php +image: php:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + +variables: + MYSQL_DATABASE: project_name + MYSQL_ROOT_PASSWORD: secret + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - vendor/ + - node_modules/ + +# This is a basic example for a gem or script which doesn't use +# services such as redis or postgres +before_script: + # Update packages + - apt-get update -yqq + + # Upgrade to Node 7 + - curl -sL https://deb.nodesource.com/setup_7.x | bash - + + # Install dependencies + - apt-get install git nodejs libcurl4-gnutls-dev libicu-dev libmcrypt-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libpq-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev -yqq + + # Install php extensions + - docker-php-ext-install mbstring pdo_mysql curl json intl gd xml zip bz2 opcache + + # Install & enable Xdebug for code coverage reports + - pecl install xdebug + - docker-php-ext-enable xdebug + + # Install Composer and project dependencies. + - curl -sS https://getcomposer.org/installer | php + - php composer.phar install + + # Install Node dependencies. + # comment this out if you don't have a node dependency + - npm install + + # Copy over testing configuration. + # Don't forget to set the database config in .env.testing correctly + # DB_HOST=mysql + # DB_DATABASE=project_name + # DB_USERNAME=root + # DB_PASSWORD=secret + - cp .env.testing .env + + # Run npm build + # comment this out if you don't have a frontend build + # you can change this to to your frontend building script like + # npm run build + - npm run dev + + # Generate an application key. Re-cache. + - php artisan key:generate + - php artisan config:cache + + # Run database migrations. + - php artisan migrate + + # Run database seed + - php artisan db:seed + +test: + script: + # run laravel tests + - php vendor/bin/phpunit --coverage-text --colors=never + + # run frontend tests + # if you have any task for testing frontend + # set it in your package.json script + # comment this out if you don't have a frontend test + - npm test diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml new file mode 100644 index 00000000000..d61ff239e13 --- /dev/null +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -0,0 +1,102 @@ +--- +# Build JAVA applications using Apache Maven (http://maven.apache.org) +# For docker image tags see https://hub.docker.com/_/maven/ +# +# For general lifecycle information see https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html +# +# This template will build and test your projects as well as create the documentation. +# +# * Caches downloaded dependencies and plugins between invocation. +# * Verify but don't deploy merge requests. +# * Deploy built artifacts from master branch only. +# * Shows how to use multiple jobs in test stage for verifying functionality +# with multiple JDKs. +# * Uses site:stage to collect the documentation for multi-module projects. +# * Publishes the documentation for `master` branch. + +variables: + # This will supress any download for dependencies and plugins or upload messages which would clutter the console log. + # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work. + MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" + # As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used + # when running from the command line. + # `installAtEnd` and `deployAtEnd` are only effective with recent version of the corresponding plugins. + MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true" + +# Cache downloaded dependencies and plugins between builds. +# To keep cache across branches add 'key: "$CI_JOB_NAME"' +cache: + paths: + - .m2/repository + +# This will only validate and compile stuff and run e.g. maven-enforcer-plugin. +# Because some enforcer rules might check dependency convergence and class duplications +# we use `test-compile` here instead of `validate`, so the correct classpath is picked up. +.validate: &validate + stage: build + script: + - 'mvn $MAVEN_CLI_OPTS test-compile' + +# For merge requests do not `deploy` but only run `verify`. +# See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html +.verify: &verify + stage: test + script: + - 'mvn $MAVEN_CLI_OPTS verify site site:stage' + except: + - master + +# Validate merge requests using JDK7 +validate:jdk7: + <<: *validate + image: maven:3.3.9-jdk-7 + +# Validate merge requests using JDK8 +validate:jdk8: + <<: *validate + image: maven:3.3.9-jdk-8 + +# Verify merge requests using JDK7 +verify:jdk7: + <<: *verify + image: maven:3.3.9-jdk-7 + +# Verify merge requests using JDK8 +verify:jdk8: + <<: *verify + image: maven:3.3.9-jdk-8 + + +# For `master` branch run `mvn deploy` automatically. +# Here you need to decide whether you want to use JDK7 or 8. +# To get this working you need to define a volume while configuring your gitlab-ci-multi-runner. +# Mount your `settings.xml` as `/root/.m2/settings.xml` which holds your secrets. +# See https://maven.apache.org/settings.html +deploy:jdk8: + # Use stage test here, so the pages job may later pickup the created site. + stage: test + script: + - 'mvn $MAVEN_CLI_OPTS deploy site site:stage' + only: + - master + # Archive up the built documentation site. + artifacts: + paths: + - target/staging + image: maven:3.3.9-jdk-8 + + +pages: + image: busybox:latest + stage: deploy + script: + # Because Maven appends the artifactId automatically to the staging path if you did define a parent pom, + # you might need to use `mv target/staging/YOUR_ARTIFACT_ID public` instead. + - mv target/staging public + dependencies: + - deploy:jdk8 + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml new file mode 100644 index 00000000000..3585f99760f --- /dev/null +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -0,0 +1,42 @@ +# This is a simple gitlab continuous integration template (compatible with the shared runner provided on gitlab.com) +# using the official mono docker image to build a visual studio project. +# +# MyProject.sln +# MyProject\ +# MyProject\ +# MyProject.csproj (console application) +# MyProject.Test\ +# MyProject.Test.csproj (test library using nuget packages "NUnit" and "NUnit.ConsoleRunner") +# +# Please find the full example project here: +# https://gitlab.com/tobiaskoch/gitlab-ci-example-mono + +# see https://hub.docker.com/_/mono/ +image: mono:latest + +stages: + - test + - deploy + +before_script: + - nuget restore -NonInteractive + +release: + stage: deploy + only: + - master + artifacts: + paths: + - build/release/MyProject.exe + script: + # The output path is relative to the position of the csproj-file + - msbuild /p:Configuration="Release" /p:Platform="Any CPU" + /p:OutputPath="./../../build/release/" "MyProject.sln" + +debug: + stage: test + script: + # The output path is relative to the position of the csproj-file + - msbuild /p:Configuration="Debug" /p:Platform="Any CPU" + /p:OutputPath="./../../build/debug/" "MyProject.sln" + - mono packages/NUnit.ConsoleRunner.3.6.0/tools/nunit3-console.exe build/debug/MyProject.Test.dll
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml new file mode 100644 index 00000000000..41de1458582 --- /dev/null +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -0,0 +1,27 @@ +# Official framework image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/node/tags/ +image: node:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - redis:latest + - postgres:latest + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - node_modules/ + +test_async: + script: + - npm install + - node ./specs/start.js ./specs/async.spec.js + +test_db: + script: + - npm install + - node ./specs/start.js ./specs/db-postgres.spec.js diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml new file mode 100644 index 00000000000..290b9997084 --- /dev/null +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -0,0 +1,92 @@ +image: ayufan/openshift-cli + +stages: + - test + - review + - staging + - production + - cleanup + +variables: + OPENSHIFT_SERVER: openshift.default.svc.cluster.local + # OPENSHIFT_DOMAIN: apps.example.com + # Configure this variable in Secure Variables: + # OPENSHIFT_TOKEN: my.openshift.token + +test1: + stage: test + before_script: [] + script: + - echo run tests + +test2: + stage: test + before_script: [] + script: + - echo run tests + +.deploy: &deploy + before_script: + - oc login "$OPENSHIFT_SERVER" --token="$OPENSHIFT_TOKEN" --insecure-skip-tls-verify + - oc project "$CI_PROJECT_NAME-$CI_PROJECT_ID" 2> /dev/null || oc new-project "$CI_PROJECT_NAME-$CI_PROJECT_ID" + script: + - "oc get services $APP 2> /dev/null || oc new-app . --name=$APP --strategy=docker" + - "oc start-build $APP --from-dir=. --follow || sleep 3s && oc start-build $APP --from-dir=. --follow" + - "oc get routes $APP 2> /dev/null || oc expose service $APP --hostname=$APP_HOST" + +review: + <<: *deploy + stage: review + variables: + APP: review-$CI_COMMIT_REF_NAME + APP_HOST: $CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN + environment: + name: review/$CI_COMMIT_REF_NAME + url: http://$CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN + on_stop: stop-review + only: + - branches + except: + - master + +stop-review: + <<: *deploy + stage: cleanup + script: + - oc delete all -l "app=$APP" + when: manual + variables: + APP: review-$CI_COMMIT_REF_NAME + GIT_STRATEGY: none + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + only: + - branches + except: + - master + +staging: + <<: *deploy + stage: staging + variables: + APP: staging + APP_HOST: $CI_PROJECT_NAME-staging.$OPENSHIFT_DOMAIN + environment: + name: staging + url: http://$CI_PROJECT_NAME-staging.$OPENSHIFT_DOMAIN + only: + - master + +production: + <<: *deploy + stage: production + variables: + APP: production + APP_HOST: $CI_PROJECT_NAME.$OPENSHIFT_DOMAIN + when: manual + environment: + name: production + url: http://$CI_PROJECT_NAME.$OPENSHIFT_DOMAIN + only: + - master diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml new file mode 100644 index 00000000000..33f44ee9222 --- /dev/null +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -0,0 +1,36 @@ +# Select image from https://hub.docker.com/_/php/ +image: php:7.1.1 + +# Select what we should cache between builds +cache: + paths: + - vendor/ + +before_script: +- apt-get update -yqq +- apt-get install -yqq git libmcrypt-dev libpq-dev libcurl4-gnutls-dev libicu-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev +# Install PHP extensions +- docker-php-ext-install mbstring mcrypt pdo_pgsql curl json intl gd xml zip bz2 opcache +# Install & enable Xdebug for code coverage reports +- pecl install xdebug +- docker-php-ext-enable xdebug +# Install and run Composer +- curl -sS https://getcomposer.org/installer | php +- php composer.phar install + +# Bring in any services we need http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service +# See http://docs.gitlab.com/ce/ci/services/README.html for examples. +services: + - mysql:5.7 + +# Set any variables we need +variables: + # Configure mysql environment variables (https://hub.docker.com/r/_/mysql/) + MYSQL_DATABASE: mysql_database + MYSQL_ROOT_PASSWORD: mysql_strong_password + +# Run our tests +# If Xdebug was installed you can generate a coverage report and see code coverage metrics. +test: + script: + - vendor/bin/phpunit --configuration phpunit.xml --coverage-text --colors=never
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml new file mode 100644 index 00000000000..fa296057c72 --- /dev/null +++ b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml @@ -0,0 +1,26 @@ +image: + name: hashicorp/packer:1.0.4 + entrypoint: + - '/usr/bin/env' + - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + +before_script: + - packer --version + +stages: + - validate + - deploy + +validate: + stage: validate + script: + - find . -maxdepth 1 -name '*.json' -print0 | xargs -t0n1 packer validate + +build: + stage: deploy + environment: production + script: + - find . -maxdepth 1 -name '*.json' -print0 | xargs -t0n1 packer build + when: manual + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml new file mode 100644 index 00000000000..7fcc0b436b5 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -0,0 +1,16 @@ +# Full project: https://gitlab.com/pages/brunch +image: node:4.2.2 + +pages: + cache: + paths: + - node_modules/ + + script: + - npm install -g brunch + - brunch build --production + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml new file mode 100644 index 00000000000..791afdd23f1 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -0,0 +1,13 @@ +# Full project: https://gitlab.com/pages/doxygen +image: alpine + +pages: + script: + - apk update && apk add doxygen + - doxygen doxygen/Doxyfile + - mv doxygen/documentation/html/ public/ + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml new file mode 100644 index 00000000000..9df2a4797b2 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -0,0 +1,17 @@ +image: node:latest + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - node_modules/ + +pages: + script: + - yarn install + - ./node_modules/.bin/gatsby build --prefix-paths + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml new file mode 100644 index 00000000000..249a168aa33 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -0,0 +1,12 @@ +# Full project: https://gitlab.com/pages/plain-html +pages: + stage: deploy + script: + - mkdir .public + - cp -r * .public + - mv .public public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml new file mode 100644 index 00000000000..dd3ef149668 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -0,0 +1,16 @@ +# Full project: https://gitlab.com/pages/harp +image: node:4.2.2 + +pages: + cache: + paths: + - node_modules + + script: + - npm install -g harp + - harp compile ./ public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml new file mode 100644 index 00000000000..02d02250bbf --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -0,0 +1,16 @@ +# Full project: https://gitlab.com/pages/hexo +image: node:6.10.0 + +pages: + script: + - npm install + - ./node_modules/hexo/bin/hexo generate + artifacts: + paths: + - public + cache: + paths: + - node_modules + key: project + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml new file mode 100644 index 00000000000..b8cfb0f56f6 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -0,0 +1,17 @@ +# Full project: https://gitlab.com/pages/hugo +image: dettmering/hugo-build + +pages: + script: + - hugo + artifacts: + paths: + - public + only: + - master + +test: + script: + - hugo + except: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml new file mode 100644 index 00000000000..f5b40f2b9f1 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -0,0 +1,25 @@ +# Full project: https://gitlab.com/pages/hyde +image: python:2.7 + +cache: + paths: + - vendor/ + +test: + stage: test + script: + - pip install hyde + - hyde gen + except: + - master + +pages: + stage: deploy + script: + - pip install hyde + - hyde gen -d public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml new file mode 100644 index 00000000000..7abfaf53e8e --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml @@ -0,0 +1,32 @@ +# This template uses the java:8 docker image because there isn't any +# official JBake image at this moment +# +# JBake https://jbake.org/ is a Java based, open source, static site/blog generator for developers & designers +# +# This yml works with jBake 2.5.1 +# Feel free to change JBAKE_VERSION version +# +# HowTo at: https://jorge.aguilera.gitlab.io/howtojbake/ + +image: java:8 + +variables: + JBAKE_VERSION: 2.5.1 + + +# We use SDKMan as tool for managing versions +before_script: + - apt-get update -qq && apt-get install -y -qq unzip zip + - curl -sSL https://get.sdkman.io | bash + - echo sdkman_auto_answer=true > /root/.sdkman/etc/config + - source /root/.sdkman/bin/sdkman-init.sh + - sdk install jbake $JBAKE_VERSION < /dev/null + - sdk use jbake $JBAKE_VERSION + +# This build job produced the output directory of your site +pages: + script: + - jbake . public + artifacts: + paths: + - public
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml new file mode 100644 index 00000000000..37f50554036 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -0,0 +1,30 @@ +# Template project: https://gitlab.com/pages/jekyll +# Docs: https://docs.gitlab.com/ce/pages/ +image: ruby:2.3 + +variables: + JEKYLL_ENV: production + +before_script: +- bundle install + +test: + stage: test + script: + - bundle exec jekyll build -d test + artifacts: + paths: + - test + except: + - master + +pages: + stage: deploy + script: + - bundle exec jekyll build -d public + artifacts: + paths: + - public + only: + - master + diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml new file mode 100644 index 00000000000..0e5fb410a4e --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -0,0 +1,42 @@ +# Jigsaw is a simple static sites generator with Laravel's Blade. +# +# Full project: https://github.com/tightenco/jigsaw + +image: php:7.2 + +# These folders are cached between builds +cache: + paths: + - vendor/ + - node_modules/ + +before_script: + # Update packages + - apt-get update -yqq + + # Install dependencies + - apt-get install -yqq gnupg zlib1g-dev libpng-dev + + # Install Node 8 + - curl -sL https://deb.nodesource.com/setup_8.x | bash - + - apt-get install -yqq nodejs + + # Install php extensions + - docker-php-ext-install zip + + # Install Composer and project dependencies. + - curl -sS https://getcomposer.org/installer | php + - php composer.phar install + + # Install Node dependencies. + - npm install + +pages: + script: + - npm run production + - mv build_production public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml new file mode 100644 index 00000000000..c5c44a5d86c --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -0,0 +1,12 @@ +# Full project: https://gitlab.com/pages/hyde +image: python:2.7 + +pages: + script: + - pip install lektor + - lektor build --output-path public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml new file mode 100644 index 00000000000..50e8b7ccd46 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -0,0 +1,17 @@ +# Full project: https://gitlab.com/pages/metalsmith +image: node:4.2.2 + +pages: + cache: + paths: + - node_modules/ + + script: + - npm install -g metalsmith + - npm install + - make build + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml new file mode 100644 index 00000000000..9f4cc0574d6 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -0,0 +1,27 @@ +# Full project: https://gitlab.com/pages/middleman +image: ruby:2.3 + +cache: + paths: + - vendor + +test: + script: + - apt-get update -yqqq + - apt-get install -y nodejs + - bundle install --path vendor + - bundle exec middleman build + except: + - master + +pages: + script: + - apt-get update -yqqq + - apt-get install -y nodejs + - bundle install --path vendor + - bundle exec middleman build + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml new file mode 100644 index 00000000000..b469b316ba5 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -0,0 +1,12 @@ +# Full project: https://gitlab.com/pages/nanoc +image: ruby:2.3 + +pages: + script: + - bundle install -j4 + - nanoc + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml new file mode 100644 index 00000000000..4762ec9acfd --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -0,0 +1,15 @@ +# Full project: https://gitlab.com/pages/octopress +image: ruby:2.3 + +pages: + script: + - apt-get update -qq && apt-get install -qq nodejs + - bundle install -j4 + - bundle exec rake generate + - mv public .public + - mv .public/octopress public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml new file mode 100644 index 00000000000..c5f3154f587 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml @@ -0,0 +1,10 @@ +# Full project: https://gitlab.com/pages/pelican +image: python:2.7-alpine + +pages: + script: + - pip install -r requirements.txt + - pelican -s publishconf.py + artifacts: + paths: + - public/ diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml new file mode 100644 index 00000000000..2e0589de652 --- /dev/null +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -0,0 +1,51 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/python/tags/ +image: python:latest + +# Change pip's cache directory to be inside the project directory since we can +# only cache local items. +variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache" + +# Pip's cache doesn't store the python packages +# https://pip.pypa.io/en/stable/reference/pip_install/#caching +# +# If you want to also cache the installed packages, you have to install +# them in a virtualenv and cache it as well. +cache: + paths: + - .cache/pip + - venv/ + +before_script: + - python -V # Print out python version for debugging + - pip install virtualenv + - virtualenv venv + - source venv/bin/activate + +test: + script: + - python setup.py test + - pip install tox flake8 # you can also use tox + - tox -e py36,flake8 + +run: + script: + - python setup.py bdist_wheel + # an alternative approach is to install and run: + - pip install dist/* + # run the command here + artifacts: + paths: + - dist/*.whl + +pages: + script: + - pip install sphinx sphinx-rtd-theme + - cd doc ; make html + - mv build/html/ ../public/ + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml new file mode 100644 index 00000000000..93cb31f48c0 --- /dev/null +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -0,0 +1,54 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/ruby/tags/ +image: "ruby:2.5" + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - redis:latest + - postgres:latest + +variables: + POSTGRES_DB: database_name + +# Cache gems in between builds +cache: + paths: + - vendor/ruby + +# This is a basic example for a gem or script which doesn't use +# services such as redis or postgres +before_script: + - ruby -v # Print out ruby version for debugging + # Uncomment next line if your rails app needs a JS runtime: + # - apt-get update -q && apt-get install nodejs -yqq + - gem install bundler --no-ri --no-rdoc # Bundler is not installed with the image + - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby + +# Optional - Delete if not using `rubocop` +rubocop: + script: + - rubocop + +rspec: + script: + - rspec spec + +rails: + variables: + DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB" + script: + - rails db:migrate + - rails db:seed + - rails test + +# This deploy job uses a simple deploy flow to Heroku, other providers, e.g. AWS Elastic Beanstalk +# are supported too: https://github.com/travis-ci/dpl +deploy: + type: deploy + environment: production + script: + - gem install dpl + - dpl --provider=heroku --app=$HEROKU_APP_NAME --api-key=$HEROKU_PRODUCTION_KEY diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml new file mode 100644 index 00000000000..cab087c48c7 --- /dev/null +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -0,0 +1,23 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/rust/tags/ +image: "rust:latest" + +# Optional: Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +#services: +# - mysql:latest +# - redis:latest +# - postgres:latest + +# Optional: Install a C compiler, cmake and git into the container. +# You will often need this when you (or any of your dependencies) depends on C code. +#before_script: +#- apt-get update -yqq +#- apt-get install -yqq --no-install-recommends build-essential + +# Use cargo to test the project +test:cargo: + script: + - rustc --version && cargo --version # Print version info for debugging + - cargo test --all --verbose diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml new file mode 100644 index 00000000000..b4208ed9d7d --- /dev/null +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -0,0 +1,22 @@ +# Official Java image. Look for the different tagged releases at +# https://hub.docker.com/r/library/java/tags/ . A Java image is not required +# but an image with a JVM speeds up the build a bit. +image: java:8 + +before_script: + # Enable the usage of sources over https + - apt-get update -yqq + - apt-get install apt-transport-https -yqq + # Add keyserver for SBT + - echo "deb http://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list + - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + # Install SBT + - apt-get update -yqq + - apt-get install sbt -yqq + # Log the sbt version + - sbt sbt-version + +test: + script: + # Execute your project's tests + - sbt clean test diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml new file mode 100644 index 00000000000..ba8a802ba4f --- /dev/null +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -0,0 +1,30 @@ +# Lifted from: https://about.gitlab.com/2016/03/10/setting-up-gitlab-ci-for-ios-projects/ +# This file assumes an own GitLab CI runner, setup on a macOS system. +stages: + - build + - archive + +build_project: + stage: build + script: + - xcodebuild clean -project ProjectName.xcodeproj -scheme SchemeName | xcpretty + - xcodebuild test -project ProjectName.xcodeproj -scheme SchemeName -destination 'platform=iOS Simulator,name=iPhone 8,OS=11.3' | xcpretty -s + tags: + - ios_11-3 + - xcode_9-3 + - macos_10-13 + +archive_project: + stage: archive + script: + - xcodebuild clean archive -archivePath build/ProjectName -scheme SchemeName + - xcodebuild -exportArchive -exportFormat ipa -archivePath "build/ProjectName.xcarchive" -exportPath "build/ProjectName.ipa" -exportProvisioningProfile "ProvisioningProfileName" + only: + - master + artifacts: + paths: + - build/ProjectName.ipa + tags: + - ios_11-3 + - xcode_9-3 + - macos_10-13 diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml new file mode 100644 index 00000000000..7160fce26a8 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -0,0 +1,55 @@ +# Official image for Hashicorp's Terraform. It uses light image which is Alpine +# based as it is much lighter. +# +# Entrypoint is also needed as image by default set `terraform` binary as an +# entrypoint. +image: + name: hashicorp/terraform:light + entrypoint: + - '/usr/bin/env' + - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + +# Default output file for Terraform plan +variables: + PLAN: plan.tfplan + +cache: + paths: + - .terraform + +before_script: + - terraform --version + - terraform init + +stages: + - validate + - build + - deploy + +validate: + stage: validate + script: + - terraform validate + +plan: + stage: build + script: + - terraform plan -out=$PLAN + artifacts: + name: plan + paths: + - $PLAN + +# Separate apply job for manual launching Terraform as it can be destructive +# action. +apply: + stage: deploy + environment: + name: production + script: + - terraform apply -input=false $PLAN + dependencies: + - plan + when: manual + only: + - master diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml new file mode 100644 index 00000000000..fc3d4ecdbba --- /dev/null +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -0,0 +1,86 @@ +# The following script will work for any project that can be built from command line by msbuild +# It uses powershell shell executor, so you need to add the following line to your config.toml file +# (located in gitlab-runner.exe directory): +# shell = "powershell" +# +# The script is composed of 3 stages: build, test and deploy. +# +# The build stage restores NuGet packages and uses msbuild to build the exe and msi +# One major issue you'll find is that you can't build msi projects from command line +# if you use vdproj. There are workarounds building msi via devenv, but they rarely work +# The best solution is migrating your vdproj projects to WiX, as it can be build directly +# by msbuild. +# +# The test stage runs nunit from command line against Test project inside your solution +# It also saves the resulting TestResult.xml file +# +# The deploy stage copies the exe and msi from build stage to a network drive +# You need to have the network drive mapped as Local System user for gitlab-runner service to see it +# The best way to persist the mapping is via a scheduled task (see: https://stackoverflow.com/a/7867064/1288473), +# running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes + + +# place project specific paths in variables to make the rest of the script more generic +variables: + EXE_RELEASE_FOLDER: 'YourApp\bin\Release' + MSI_RELEASE_FOLDER: 'Setup\bin\Release' + TEST_FOLDER: 'Tests\bin\Release' + DEPLOY_FOLDER: 'P:\Projects\YourApp\Builds' + + NUGET_PATH: 'C:\NuGet\nuget.exe' + MSBUILD_PATH: 'C:\Program Files (x86)\MSBuild\14.0\Bin\msbuild.exe' + NUNIT_PATH: 'C:\Program Files (x86)\NUnit.org\nunit-console\nunit3-console.exe' + +stages: + - build + - test + - deploy + +build_job: + stage: build + only: + - tags # the build process will only be started by git tag commits + script: + - '& "$env:NUGET_PATH" restore' # restore Nuget dependencies + - '& "$env:MSBUILD_PATH" /p:Configuration=Release' # build the project + artifacts: + expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on + paths: + - '$env:EXE_RELEASE_FOLDER\YourApp.exe' # saving exe to copy to deploy folder + - '$env:MSI_RELEASE_FOLDER\YourApp Setup.msi' # saving msi to copy to deploy folder + - '$env:TEST_FOLDER\' # saving entire Test project so NUnit can run tests + +test_job: + stage: test + only: + - tags + script: + - '& "$env:NUNIT_PATH" ".\$env:TEST_FOLDER\Tests.dll"' # running NUnit tests + artifacts: + expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on + paths: + - '.\TestResult.xml' # saving NUnit results to copy to deploy folder + dependencies: + - build_job + +deploy_job: + stage: deploy + only: + - tags + script: + # Compose a folder for each release based on commit tag. + # Assuming your tag is Rev1.0.0.1, and your last commit message is 'First commit' + # the artifact files will be copied to: + # P:\Projects\YourApp\Builds\Rev1.0.0.1 - First commit\ + - '$commitSubject = git log -1 --pretty=%s' + - '$deployFolder = $($env:DEPLOY_FOLDER) + "\" + $($env:CI_BUILD_TAG) + " - " + $commitSubject + "\"' + + # xcopy takes care of recursively creating required folders + - 'xcopy /y ".\$env:EXE_RELEASE_FOLDER\YourApp.exe" "$deployFolder"' + - 'xcopy /y ".\$env:MSI_RELEASE_FOLDER\YourApp Setup.msi" "$deployFolder"' + - 'xcopy /y ".\TestResult.xml" "$deployFolder"' + + dependencies: + - build_job + - test_job +
\ No newline at end of file diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 3236abfa43f..1ffc2639237 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -30,8 +30,9 @@ module Gitlab note_events = event_counts(date_from, :merge_requests) .having(action: [Event::COMMENTED]) - union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) - events = Event.find_by_sql(union.to_sql).map(&:attributes) + events = Event + .from_union([repo_events, issue_events, mr_events, note_events]) + .map(&:attributes) @activity_dates = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| activities[event["date"]] += event["total_amount"] diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index c9dbd2d2e5f..de7c959e706 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -2,11 +2,7 @@ module Gitlab module CurrentSettings class << self def current_application_settings - if RequestStore.active? - RequestStore.fetch(:current_application_settings) { ensure_application_settings! } - else - ensure_application_settings! - end + Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } end def fake_application_settings(attributes = {}) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 8eacad078c8..42f9605f5ac 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -249,5 +249,21 @@ module Gitlab end private_class_method :database_version + + def self.add_post_migrate_path_to_rails(force: false) + return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force + + Rails.application.config.paths['db'].each do |db_path| + path = Rails.root.join(db_path, 'post_migrate').to_s + + unless Rails.application.config.paths['db/migrate'].include? path + Rails.application.config.paths['db/migrate'] << path + + # Rails memoizes migrations at certain points where it won't read the above + # path just yet. As such we must also update the following list of paths. + ActiveRecord::Migrator.migrations_paths << path + end + end + end end end diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index d32837f5793..7d334a79009 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -2,6 +2,8 @@ module Gitlab module Database # Model that can be used for querying permissions of a SQL user. class Grant < ActiveRecord::Base + include FromUnion + self.table_name = if Database.postgresql? 'information_schema.role_table_grants' @@ -42,9 +44,7 @@ module Gitlab .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") ] - union = SQL::Union.new(queries).to_sql - - Grant.from("(#{union}) privs").any? + Grant.from_union(queries, alias_as: 'privs').any? end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 7f012312819..30541ee3553 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1073,6 +1073,10 @@ into similar problems in the future (e.g. when new tables are created). connection.select_value(index_sql).to_i > 0 end + + def mysql_compatible_index_length + Gitlab::Database.mysql? ? 20 : nil + end end end end diff --git a/lib/gitlab/database/subquery.rb b/lib/gitlab/database/subquery.rb new file mode 100644 index 00000000000..36e4559b554 --- /dev/null +++ b/lib/gitlab/database/subquery.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Subquery + class << self + def self_join(relation) + t = relation.arel_table + t2 = if !Gitlab.rails5? + relation.arel.as('t2') + else + # Work around a bug in Rails 5, where LIMIT causes trouble + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/51729 + r = relation.limit(nil).arel + r.take(relation.limit_value) if relation.limit_value + r.as('t2') + end + + relation.unscoped.joins(t.join(t2).on(t[:id].eq(t2[:id])).join_sources.first) + end + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d2ca7a070fa..fb117baca9e 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -20,8 +20,9 @@ module Gitlab DiffViewer::Image ].sort_by { |v| v.binary? ? 0 : 1 }.freeze - def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil) + def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil, stats: nil) @diff = diff + @stats = stats @repository = repository @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs @@ -165,11 +166,11 @@ module Gitlab end def added_lines - diff_lines.count(&:added?) + @stats&.additions || diff_lines.count(&:added?) end def removed_lines - diff_lines.count(&:removed?) + @stats&.deletions || diff_lines.count(&:removed?) end def file_identifier diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 2acb0e43b69..b79ff771a2b 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -2,23 +2,27 @@ module Gitlab module Diff module FileCollection class Base + include Gitlab::Utils::StrongMemoize + attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs, :diffable delegate :count, :size, :real_size, to: :diff_files def self.default_options - ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false) + ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false, include_stats: true) end def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil) diff_options = self.class.default_options.merge(diff_options || {}) @diffable = diffable + @include_stats = diff_options.delete(:include_stats) @diffs = diffable.raw_diffs(diff_options) @project = project @diff_options = diff_options @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs + @repository = project.repository end def diff_files @@ -43,10 +47,27 @@ module Gitlab private + def diff_stats_collection + strong_memoize(:diff_stats) do + # There are scenarios where we don't need to request Diff Stats, + # when caching for instance. + next unless @include_stats + next unless diff_refs + + @repository.diff_stats(diff_refs.base_sha, diff_refs.head_sha) + end + end + def decorate_diff!(diff) return diff if diff.is_a?(File) - Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs) + stats = diff_stats_collection&.find_by_path(diff.new_path) + + Gitlab::Diff::File.new(diff, + repository: project.repository, + diff_refs: diff_refs, + fallback_diff_refs: fallback_diff_refs, + stats: stats) end end end diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 978962ab2eb..fc280f96ec1 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -101,21 +101,21 @@ module Gitlab return @diff_file if defined?(@diff_file) @diff_file = begin - if RequestStore.active? - key = { - project_id: repository.project.id, - start_sha: start_sha, - head_sha: head_sha, - path: file_path - } - - RequestStore.fetch(key) { find_diff_file(repository) } - else - find_diff_file(repository) - end + key = { + project_id: repository.project.id, + start_sha: start_sha, + head_sha: head_sha, + path: file_path + } + + Gitlab::SafeRequestStore.fetch(key) { find_diff_file(repository) } end end + def diff_options + { paths: paths, expanded: true, include_stats: false } + end + def diff_line(repository) @diff_line ||= diff_file(repository)&.line_for_position(self) end @@ -130,7 +130,7 @@ module Gitlab return unless diff_refs.complete? return unless comparison = diff_refs.compare_in(repository.project) - comparison.diffs(paths: paths, expanded: true).diff_files.first + comparison.diffs(diff_options).diff_files.first end def get_formatter_class(type) diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index 4850a6c0430..050a1ad3a0b 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -47,7 +47,7 @@ module Gitlab end def appearance - RequestStore.store[:appearance] ||= (Appearance.current || Appearance.new) + Gitlab::SafeRequestStore[:appearance] ||= (Appearance.current || Appearance.new) end def appearance_favicon diff --git a/lib/gitlab/git/diff_stats_collection.rb b/lib/gitlab/git/diff_stats_collection.rb new file mode 100644 index 00000000000..d4033f56387 --- /dev/null +++ b/lib/gitlab/git/diff_stats_collection.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class DiffStatsCollection + include Gitlab::Utils::StrongMemoize + include Enumerable + + def initialize(diff_stats) + @collection = diff_stats + end + + def each(&block) + @collection.each(&block) + end + + def find_by_path(path) + indexed_by_path[path] + end + + private + + def indexed_by_path + strong_memoize(:indexed_by_path) do + index_by { |stats| stats.path } + end + end + end + end +end diff --git a/lib/gitlab/git/hook_env.rb b/lib/gitlab/git/hook_env.rb index 455e8451c10..620568d8817 100644 --- a/lib/gitlab/git/hook_env.rb +++ b/lib/gitlab/git/hook_env.rb @@ -17,18 +17,18 @@ module Gitlab ].freeze def self.set(gl_repository, env) - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? raise "missing gl_repository" if gl_repository.blank? - RequestStore.store[:gitlab_git_env] ||= {} - RequestStore.store[:gitlab_git_env][gl_repository] = whitelist_git_env(env) + Gitlab::SafeRequestStore[:gitlab_git_env] ||= {} + Gitlab::SafeRequestStore[:gitlab_git_env][gl_repository] = whitelist_git_env(env) end def self.all(gl_repository) - return {} unless RequestStore.active? + return {} unless Gitlab::SafeRequestStore.active? - h = RequestStore.fetch(:gitlab_git_env) { {} } + h = Gitlab::SafeRequestStore.fetch(:gitlab_git_env) { {} } h.fetch(gl_repository, {}) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 1b8d320ff3b..45d42c7078f 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -438,6 +438,16 @@ module Gitlab Gitlab::Git::DiffCollection.new(iterator, options) end + def diff_stats(left_id, right_id) + stats = wrapped_gitaly_errors do + gitaly_commit_client.diff_stats(left_id, right_id) + end + + Gitlab::Git::DiffStatsCollection.new(stats) + rescue CommandError, TypeError + Gitlab::Git::DiffStatsCollection.new([]) + end + # Returns a RefName for a given SHA def ref_name_for_sha(ref_path, sha) raise ArgumentError, "sha can't be empty" unless sha.present? @@ -581,10 +591,6 @@ module Gitlab end end - def user_to_committer(user) - Gitlab::Git.committer_hash(email: user.email, name: user.name) - end - # Delete the specified branch from the repository def delete_branch(branch_name) wrapped_gitaly_errors do @@ -947,6 +953,12 @@ module Gitlab end end + def list_last_commits_for_tree(sha, path, offset: 0, limit: 25) + wrapped_gitaly_errors do + gitaly_commit_client.list_last_commits_for_tree(sha, path, offset: offset, limit: limit) + end + end + def last_commit_for_path(sha, path) wrapped_gitaly_errors do gitaly_commit_client.last_commit_for_path(sha, path) diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 62427ac9cc4..fcee9ae566c 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -11,7 +11,7 @@ module Gitlab to: :failure_info def self.for_storage(storage) - cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do + cached_circuitbreakers = Gitlab::SafeRequestStore.fetch(:circuitbreaker_cache) do Hash.new do |hash, storage_name| hash[storage_name] = build(storage_name) end diff --git a/lib/gitlab/git/storage/failure_info.rb b/lib/gitlab/git/storage/failure_info.rb index 387279c110d..1d28a850049 100644 --- a/lib/gitlab/git/storage/failure_info.rb +++ b/lib/gitlab/git/storage/failure_info.rb @@ -10,7 +10,7 @@ module Gitlab redis.del(*all_storage_keys) unless all_storage_keys.empty? end - RequestStore.delete(:circuitbreaker_cache) + Gitlab::SafeRequestStore.delete(:circuitbreaker_cache) end def self.load(cache_key) diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index ae92a624e05..d2dc4f2e688 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -115,11 +115,7 @@ module Gitlab def version(commit_id) commit_find_proc = -> { Gitlab::Git::Commit.find(@repository, commit_id) } - if RequestStore.active? - RequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call } - else - commit_find_proc.call - end + Gitlab::SafeRequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call } end def assert_type!(object, klass) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 12307338972..500aabcbbb8 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -174,10 +174,29 @@ module Gitlab end private_class_method :current_transaction_labels + # For some time related tasks we can't rely on `Time.now` since it will be + # affected by Timecop in some tests, and the clock of some gitaly-related + # components (grpc's c-core and gitaly server) use system time instead of + # timecop's time, so tests will fail. + # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will circumvent + # timecop. + def self.real_time + Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + end + private_class_method :real_time + + def self.authorization_token(storage) + token = token(storage).to_s + issued_at = real_time.to_i.to_s + hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, token, issued_at) + + "v2.#{hmac}.#{issued_at}" + end + private_class_method :authorization_token + def self.request_kwargs(storage, timeout, remote_storage: nil) - encoded_token = Base64.strict_encode64(token(storage).to_s) metadata = { - 'authorization' => "Bearer #{encoded_token}", + 'authorization' => "Bearer #{authorization_token(storage)}", 'client_name' => CLIENT_NAME } @@ -195,12 +214,7 @@ module Gitlab return result unless timeout > 0 - # Do not use `Time.now` for deadline calculation, since it - # will be affected by Timecop in some tests, but grpc's c-core - # uses system time instead of timecop's time, so tests will fail - # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will - # circumvent timecop - deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout + deadline = real_time + timeout result[:deadline] = deadline result @@ -302,7 +316,7 @@ module Gitlab # Ensures that Gitaly is not being abuse through n+1 misuse etc def self.enforce_gitaly_request_limits(call_site) # Only count limits in request-response environments (not sidekiq for example) - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? # This is this actual number of times this call was made. Used for information purposes only actual_call_count = increment_call_count("gitaly_#{call_site}_actual") @@ -326,7 +340,7 @@ module Gitlab end def self.allow_n_plus_1_calls - return yield unless RequestStore.active? + return yield unless Gitlab::SafeRequestStore.active? begin increment_call_count(:gitaly_call_count_exception_block_depth) @@ -337,25 +351,25 @@ module Gitlab end def self.get_call_count(key) - RequestStore.store[key] || 0 + Gitlab::SafeRequestStore[key] || 0 end private_class_method :get_call_count def self.increment_call_count(key) - RequestStore.store[key] ||= 0 - RequestStore.store[key] += 1 + Gitlab::SafeRequestStore[key] ||= 0 + Gitlab::SafeRequestStore[key] += 1 end private_class_method :increment_call_count def self.decrement_call_count(key) - RequestStore.store[key] -= 1 + Gitlab::SafeRequestStore[key] -= 1 end private_class_method :decrement_call_count # Returns an estimate of the number of Gitaly calls made for this # request def self.get_request_count - return 0 unless RequestStore.active? + return 0 unless Gitlab::SafeRequestStore.active? gitaly_migrate_count = get_call_count("gitaly_migrate_actual") gitaly_call_count = get_call_count("gitaly_call_actual") @@ -372,28 +386,28 @@ module Gitlab end def self.reset_counts - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? %w[migrate call].each do |call_site| - RequestStore.store["gitaly_#{call_site}_actual"] = 0 - RequestStore.store["gitaly_#{call_site}_permitted"] = 0 + Gitlab::SafeRequestStore["gitaly_#{call_site}_actual"] = 0 + Gitlab::SafeRequestStore["gitaly_#{call_site}_permitted"] = 0 end end def self.add_call_details(details) id = details.delete(:id) - return unless id && RequestStore.active? && RequestStore.store[:peek_enabled] + return unless id && Gitlab::SafeRequestStore[:peek_enabled] - RequestStore.store['gitaly_call_details'] ||= {} - RequestStore.store['gitaly_call_details'][id] ||= {} - RequestStore.store['gitaly_call_details'][id].merge!(details) + Gitlab::SafeRequestStore['gitaly_call_details'] ||= {} + Gitlab::SafeRequestStore['gitaly_call_details'][id] ||= {} + Gitlab::SafeRequestStore['gitaly_call_details'][id].merge!(details) end def self.list_call_details - return {} unless RequestStore.active? && RequestStore.store[:peek_enabled] + return {} unless Gitlab::SafeRequestStore[:peek_enabled] - RequestStore.store['gitaly_call_details'] || {} + Gitlab::SafeRequestStore['gitaly_call_details'] || {} end def self.expected_server_version @@ -431,22 +445,22 @@ module Gitlab # Count a stack. Used for n+1 detection def self.count_stack - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? stack_string = Gitlab::Profiler.clean_backtrace(caller).drop(1).join("\n") - RequestStore.store[:stack_counter] ||= Hash.new + Gitlab::SafeRequestStore[:stack_counter] ||= Hash.new - count = RequestStore.store[:stack_counter][stack_string] || 0 - RequestStore.store[:stack_counter][stack_string] = count + 1 + count = Gitlab::SafeRequestStore[:stack_counter][stack_string] || 0 + Gitlab::SafeRequestStore[:stack_counter][stack_string] = count + 1 end private_class_method :count_stack # Returns a count for the stack which called Gitaly the most times. Used for n+1 detection def self.max_call_count - return 0 unless RequestStore.active? + return 0 unless Gitlab::SafeRequestStore.active? - stack_counter = RequestStore.store[:stack_counter] + stack_counter = Gitlab::SafeRequestStore[:stack_counter] return 0 unless stack_counter stack_counter.values.max @@ -455,9 +469,9 @@ module Gitlab # Returns the stacks that calls Gitaly the most times. Used for n+1 detection def self.max_stacks - return nil unless RequestStore.active? + return nil unless Gitlab::SafeRequestStore.active? - stack_counter = RequestStore.store[:stack_counter] + stack_counter = Gitlab::SafeRequestStore[:stack_counter] return nil unless stack_counter max = max_call_count diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index f65d7383dc7..085b2a127a5 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -148,6 +148,24 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end + def list_last_commits_for_tree(revision, path, offset: 0, limit: 25) + request = Gitaly::ListLastCommitsForTreeRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision), + path: encode_binary(path.to_s), + offset: offset, + limit: limit + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout) + + response.each_with_object({}) do |gitaly_response, hsh| + gitaly_response.commits.each do |commit_for_tree| + hsh[commit_for_tree.path] = Gitlab::Git::Commit.new(@repository, commit_for_tree.commit) + end + end + end + def last_commit_for_path(revision, path) request = Gitaly::LastCommitForPathRequest.new( repository: @gitaly_repo, @@ -172,6 +190,17 @@ module Gitlab consume_commits_response(response) end + def diff_stats(left_commit_sha, right_commit_sha) + request = Gitaly::DiffStatsRequest.new( + repository: @gitaly_repo, + left_commit_id: left_commit_sha, + right_commit_id: right_commit_sha + ) + + response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) + response.flat_map(&:stats) + end + def find_all_commits(opts = {}) request = Gitaly::FindAllCommitsRequest.new( repository: @gitaly_repo, @@ -229,22 +258,23 @@ module Gitlab end def find_commit(revision) - if RequestStore.active? - # We don't use RequeStstore.fetch(key) { ... } directly because `revision` - # can be a branch name, so we can't use it as a key as it could point - # to another commit later on (happens a lot in tests). + if Gitlab::SafeRequestStore.active? + # We don't use Gitlab::SafeRequestStore.fetch(key) { ... } directly + # because `revision` can be a branch name, so we can't use it as a key + # as it could point to another commit later on (happens a lot in + # tests). key = { storage: @gitaly_repo.storage_name, relative_path: @gitaly_repo.relative_path, commit_id: revision } - return RequestStore[key] if RequestStore.exist?(key) + return Gitlab::SafeRequestStore[key] if Gitlab::SafeRequestStore.exist?(key) commit = call_find_commit(revision) return unless commit key[:commit_id] = commit.id - RequestStore[key] = commit + Gitlab::SafeRequestStore[key] = commit else call_find_commit(revision) end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 54c78fdb680..0f148614b20 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -333,7 +333,8 @@ module Gitlab action: action[:action].upcase.to_sym, file_path: encode_binary(action[:file_path]), previous_path: encode_binary(action[:previous_path]), - base64_content: action[:encoding] == 'base64' + base64_content: action[:encoding] == 'base64', + execute_filemode: !!action[:execute_filemode] ) rescue RangeError raise ArgumentError, "Unknown action '#{action[:action]}'" diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 8e530de174d..26d1f53f26c 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -13,7 +13,7 @@ module Gitlab Storage is invalid because it has no `path` key. For source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example. - If you're using the Gitlab Development Kit, you can update your configuration running `gdk reconfigure`. + If you're using the GitLab Development Kit, you can update your configuration running `gdk reconfigure`. MSG # This class will give easily recognizable NoMethodErrors diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb index b74e7b10448..8fbfa1a86bf 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/group_hierarchy.rb @@ -89,14 +89,14 @@ module Gitlab ancestors_table = ancestors.alias_to(groups_table) descendants_table = descendants.alias_to(groups_table) - union = SQL::Union.new([model.unscoped.from(ancestors_table), - model.unscoped.from(descendants_table)]) - relation = model .unscoped .with .recursive(ancestors.to_arel, descendants.to_arel) - .from("(#{union.to_sql}) #{model.table_name}") + .from_union([ + model.unscoped.from(ancestors_table), + model.unscoped.from(descendants_table) + ]) read_only(relation) end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 00ea4b833e2..3d693d23c99 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -136,9 +136,18 @@ module Gitlab return if tree_hash[relation_key].blank? tree_array = [tree_hash[relation_key]].flatten + null_iid_pipelines = [] # Avoid keeping a possible heavy object in memory once we are done with it - while relation_item = tree_array.shift + while relation_item = (tree_array.shift || null_iid_pipelines.shift) + if nil_iid_pipeline?(relation_key, relation_item) && tree_array.any? + # Move pipelines with NULL IIDs to the end + # so they don't clash with existing IIDs. + null_iid_pipelines << relation_item + + next + end + # The transaction at this level is less speedy than one single transaction # But we can't have it in the upper level or GC won't get rid of the AR objects # after we save the batch. @@ -201,6 +210,10 @@ module Gitlab def excluded_keys_for_relation(relation) reader.attributes_finder.find_excluded_keys(relation) end + + def nil_iid_pipeline?(relation_key, relation_item) + relation_key == 'pipelines' && relation_item['iid'].nil? + end end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 81807ed659c..2486b1e4921 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -86,7 +86,6 @@ module Gitlab case @relation_name when :merge_request_diff_files then setup_diff when :notes then setup_note - when 'Ci::Pipeline' then setup_pipeline end update_user_references @@ -94,6 +93,8 @@ module Gitlab update_group_references remove_duplicate_assignees + setup_pipeline if @relation_name == 'Ci::Pipeline' + reset_tokens! remove_encrypted_attributes! end diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb index 505810964bc..b5657a36998 100644 --- a/lib/gitlab/issuables_count_for_state.rb +++ b/lib/gitlab/issuables_count_for_state.rb @@ -1,7 +1,7 @@ module Gitlab # Class for counting and caching the number of issuables per state. class IssuablesCountForState - # The name of the RequestStore cache key. + # The name of the Gitlab::SafeRequestStore cache key. CACHE_KEY = :issuables_count_for_state # The state values that can be safely casted to a Symbol. @@ -10,12 +10,7 @@ module Gitlab # finder - The finder class to use for retrieving the issuables. def initialize(finder) @finder = finder - @cache = - if RequestStore.active? - RequestStore[CACHE_KEY] ||= initialize_cache - else - initialize_cache - end + @cache = Gitlab::SafeRequestStore[CACHE_KEY] ||= initialize_cache end def for_state_or_opened(state = nil) diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 8312b901524..588238de608 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -25,12 +25,14 @@ module Gitlab :get_config_map, :get_namespace, :get_pod, + :get_secret, :get_service, :get_service_account, :delete_pod, :create_config_map, :create_namespace, :create_pod, + :create_secret, :create_service_account, :update_config_map, :update_service_account, diff --git a/lib/gitlab/kubernetes/service_account_token.rb b/lib/gitlab/kubernetes/service_account_token.rb new file mode 100644 index 00000000000..2e912b26c09 --- /dev/null +++ b/lib/gitlab/kubernetes/service_account_token.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class ServiceAccountToken + attr_reader :name, :service_account_name, :namespace_name + + def initialize(name, service_account_name, namespace_name) + @name = name + @service_account_name = service_account_name + @namespace_name = namespace_name + end + + def generate + ::Kubeclient::Resource.new(metadata: metadata, type: service_acount_token_type) + end + + private + + # as per https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#to-create-additional-api-tokens + def service_acount_token_type + 'kubernetes.io/service-account-token' + end + + def metadata + { + name: name, + namespace: namespace_name, + annotations: { + "kubernetes.io/service-account.name": service_account_name + } + } + end + end + end +end diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb index e58927a40b9..3d7c049c17f 100644 --- a/lib/gitlab/logger.rb +++ b/lib/gitlab/logger.rb @@ -30,7 +30,7 @@ module Gitlab end def self.build - RequestStore[self.cache_key] ||= new(self.full_log_path) + Gitlab::SafeRequestStore[self.cache_key] ||= new(self.full_log_path) end def self.full_log_path diff --git a/lib/gitlab/null_request_store.rb b/lib/gitlab/null_request_store.rb new file mode 100644 index 00000000000..8db331dcb9f --- /dev/null +++ b/lib/gitlab/null_request_store.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# Used by Gitlab::SafeRequestStore +module Gitlab + # The methods `begin!`, `clear!`, and `end!` are not defined because they + # should only be called directly on `RequestStore`. + class NullRequestStore + def store + {} + end + + def active? + end + + def read(key) + end + + def [](key) + end + + def write(key, value) + value + end + + def []=(key, value) + value + end + + def exist?(key) + false + end + + def fetch(key, &block) + yield + end + + def delete(key, &block) + yield(key) if block_given? + end + end +end diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index f2825db59ae..37ff32b1296 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -23,7 +23,7 @@ module Gitlab end subscribe('sql.active_record') do |_, start, finish, _, data| - if RequestStore.active? && RequestStore.store[:peek_enabled] + if Gitlab::SafeRequestStore.store[:peek_enabled] # data[:cached] is only available starting from Rails 5.1.0 # https://github.com/rails/rails/blob/v5.1.0/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L113 # Before that, data[:name] was set to 'CACHE' diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb index e3da1634fa5..448c3f3a7d8 100644 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ b/lib/gitlab/project_authorizations/with_nested_groups.rb @@ -49,13 +49,11 @@ module Gitlab .where('p_ns.share_with_group_lock IS FALSE') ] - union = Gitlab::SQL::Union.new(relations) - ProjectAuthorization .unscoped .with .recursive(cte.to_arel) - .select_from_union(union) + .select_from_union(relations) end private diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb index 7d0c00c7f36..ed2287dcc7e 100644 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ b/lib/gitlab/project_authorizations/without_nested_groups.rb @@ -24,11 +24,9 @@ module Gitlab user.groups.joins(:shared_projects).select_for_project_authorization ] - union = Gitlab::SQL::Union.new(relations) - ProjectAuthorization .unscoped - .select_from_union(union) + .select_from_union(relations) end end end diff --git a/lib/gitlab/repository_cache.rb b/lib/gitlab/repository_cache.rb index b1bf3ca4143..a03ce07b6a1 100644 --- a/lib/gitlab/repository_cache.rb +++ b/lib/gitlab/repository_cache.rb @@ -29,5 +29,21 @@ module Gitlab def read(key) backend.read(cache_key(key)) end + + def write(key, value) + backend.write(cache_key(key), value) + end + + def fetch_without_caching_false(key, &block) + value = read(key) + return value if value + + value = yield + + # Don't cache false values + write(key, value) if value + + value + end end end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 2ec871f0754..d95024fccf7 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -1,23 +1,80 @@ module Gitlab module RepositoryCacheAdapter extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize class_methods do - # Wraps around the given method and caches its output in Redis and an instance - # variable. + # Caches and strongly memoizes the method. # # This only works for methods that do not take any arguments. - def cache_method(name, fallback: nil, memoize_only: false) - original = :"_uncached_#{name}" + # + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method(name, fallback: nil) + uncached_name = alias_uncached_method(name) + + define_method(name) do + cache_method_output(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + end - alias_method(original, name) + # Caches truthy values from the method. All values are strongly memoized, + # and cached in RequestStore. + # + # Currently only used to cache `exists?` since stale false values are + # particularly troublesome. This can occur, for example, when an NFS mount + # is temporarily down. + # + # This only works for methods that do not take any arguments. + # + # name - The name of the method to be cached. + def cache_method_asymmetrically(name) + uncached_name = alias_uncached_method(name) define_method(name) do - cache_method_output(name, fallback: fallback, memoize_only: memoize_only) do - __send__(original) # rubocop:disable GitlabSecurity/PublicSend + cache_method_output_asymmetrically(name) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend end end end + + # Strongly memoizes the method. + # + # This only works for methods that do not take any arguments. + # + # name - The name of the method to be memoized. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. The fallback value + # is not memoized. + def memoize_method(name, fallback: nil) + uncached_name = alias_uncached_method(name) + + define_method(name) do + memoize_method_output(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + + # Prepends "_uncached_" to the target method name + # + # Returns the uncached method name + def alias_uncached_method(name) + uncached_name = :"_uncached_#{name}" + + alias_method(uncached_name, name) + + uncached_name + end + end + + # RequestStore-backed RepositoryCache to be used. Should be overridden by + # the including class + def request_store_cache + raise NotImplementedError end # RepositoryCache to be used. Should be overridden by the including class @@ -30,65 +87,93 @@ module Gitlab raise NotImplementedError end - # Caches the supplied block both in a cache and in an instance variable. + # Caches and strongly memoizes the supplied block. # - # The cache key and instance variable are named the same way as the value of - # the `key` argument. + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method_output(name, fallback: nil, &block) + memoize_method_output(name, fallback: fallback) do + cache.fetch(name, &block) + end + end + + # Caches truthy values from the supplied block. All values are strongly + # memoized, and cached in RequestStore. # - # This method will return `nil` if the corresponding instance variable is also - # set to `nil`. This ensures we don't keep yielding the block when it returns - # `nil`. + # Currently only used to cache `exists?` since stale false values are + # particularly troublesome. This can occur, for example, when an NFS mount + # is temporarily down. # - # key - The name of the key to cache the data in. - # fallback - A value to fall back to in the event of a Git error. - def cache_method_output(key, fallback: nil, memoize_only: false, &block) - ivar = cache_instance_variable_name(key) - - if instance_variable_defined?(ivar) - instance_variable_get(ivar) - else - # If the repository doesn't exist and a fallback was specified we return - # that value inmediately. This saves us Rugged/gRPC invocations. - return fallback unless fallback.nil? || cache.repository.exists? - - begin - value = - if memoize_only - yield - else - cache.fetch(key, &block) - end - - instance_variable_set(ivar, value) - rescue Gitlab::Git::Repository::NoRepository - # Even if the above `#exists?` check passes these errors might still - # occur (for example because of a non-existing HEAD). We want to - # gracefully handle this and not cache anything - fallback + # name - The name of the method to be cached. + def cache_method_output_asymmetrically(name, &block) + memoize_method_output(name) do + request_store_cache.fetch(name) do + cache.fetch_without_caching_false(name, &block) end end end + # Strongly memoizes the supplied block. + # + # name - The name of the method to be memoized. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. The fallback value is + # not memoized. + def memoize_method_output(name, fallback: nil, &block) + no_repository_fallback(name, fallback: fallback) do + strong_memoize(memoizable_name(name), &block) + end + end + + # Returns the fallback value if the repository does not exist + def no_repository_fallback(name, fallback: nil, &block) + # Avoid unnecessary gRPC invocations + return fallback if fallback && fallback_early?(name) + + yield + rescue Gitlab::Git::Repository::NoRepository + # Even if the `#exists?` check in `fallback_early?` passes, these errors + # might still occur (for example because of a non-existing HEAD). We + # want to gracefully handle this and not memoize anything. + fallback + end + # Expires the caches of a specific set of methods def expire_method_caches(methods) - methods.each do |key| - unless cached_methods.include?(key.to_sym) - Rails.logger.error "Requested to expire non-existent method '#{key}' for Repository" + methods.each do |name| + unless cached_methods.include?(name.to_sym) + Rails.logger.error "Requested to expire non-existent method '#{name}' for Repository" next end - cache.expire(key) + cache.expire(name) - ivar = cache_instance_variable_name(key) - - remove_instance_variable(ivar) if instance_variable_defined?(ivar) + clear_memoization(memoizable_name(name)) end + + expire_request_store_method_caches(methods) end private - def cache_instance_variable_name(key) - :"@#{key.to_s.tr('?!', '')}" + def memoizable_name(name) + "#{name.to_s.tr('?!', '')}" + end + + def expire_request_store_method_caches(methods) + methods.each do |name| + request_store_cache.expire(name) + end + end + + # All cached repository methods depend on the existence of a Git repository, + # so if the repository doesn't exist, we already know not to call it. + def fallback_early?(method_name) + # Avoid infinite loop + return false if method_name == :exists? + + !exists? end end end diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb index fef536ecb0b..8562d4515aa 100644 --- a/lib/gitlab/request_context.rb +++ b/lib/gitlab/request_context.rb @@ -2,7 +2,7 @@ module Gitlab class RequestContext class << self def client_ip - RequestStore[:client_ip] + Gitlab::SafeRequestStore[:client_ip] end end @@ -13,7 +13,7 @@ module Gitlab def call(env) req = Rack::Request.new(env) - RequestStore[:client_ip] = req.ip + Gitlab::SafeRequestStore[:client_ip] = req.ip @app.call(env) end diff --git a/lib/gitlab/safe_request_store.rb b/lib/gitlab/safe_request_store.rb new file mode 100644 index 00000000000..4e82353adb6 --- /dev/null +++ b/lib/gitlab/safe_request_store.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module SafeRequestStore + NULL_STORE = Gitlab::NullRequestStore.new + + class << self + # These methods should always run directly against RequestStore + delegate :clear!, :begin!, :end!, :active?, to: :RequestStore + + # These methods will run against NullRequestStore if RequestStore is disabled + delegate :read, :[], :write, :[]=, :exist?, :fetch, :delete, to: :store + end + + def self.store + if RequestStore.active? + RequestStore + else + NULL_STORE + end + end + end +end diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb deleted file mode 100644 index 5512afa45a8..00000000000 --- a/lib/gitlab/sidekiq_throttler.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - class SidekiqThrottler - class << self - def execute! - if Gitlab::CurrentSettings.sidekiq_throttling_enabled? - require 'sidekiq-limit_fetch' - - Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue| - Sidekiq::Queue[queue].limit = queue_limit - end - end - end - - private - - def queue_limit - @queue_limit ||= - begin - factor = Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_factor - (factor * Sidekiq.options[:concurrency]).ceil - end - end - end - end -end diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb index 831da45191f..b08d9a99e99 100644 --- a/lib/gitlab/template/finders/global_template_finder.rb +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -1,4 +1,4 @@ -# Searches and reads file present on Gitlab installation directory +# Searches and reads file present on GitLab installation directory module Gitlab module Template module Finders diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb index 29bc2393ff9..9140ace879f 100644 --- a/lib/gitlab/template/finders/repo_template_finder.rb +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -1,4 +1,4 @@ -# Searches and reads files present on each Gitlab project repository +# Searches and reads files present on each GitLab project repository module Gitlab module Template module Finders diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index fd040148a1e..deae53cc61b 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -20,7 +20,7 @@ module Gitlab end def base_dir - Rails.root.join('vendor/gitlab-ci-yml') + Rails.root.join('lib/gitlab/ci/templates') end def finder(project = nil) diff --git a/lib/gitlab/temporarily_allow.rb b/lib/gitlab/temporarily_allow.rb index 880e55f71df..8d7dacc6c54 100644 --- a/lib/gitlab/temporarily_allow.rb +++ b/lib/gitlab/temporarily_allow.rb @@ -10,7 +10,7 @@ module Gitlab end def temporarily_allowed?(key) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? temporarily_allow_request_store[key] > 0 else TEMPORARILY_ALLOW_MUTEX.synchronize do @@ -26,11 +26,11 @@ module Gitlab end def temporarily_allow_request_store - RequestStore[:temporarily_allow] ||= Hash.new(0) + Gitlab::SafeRequestStore[:temporarily_allow] ||= Hash.new(0) end def temporarily_allow_add(key, value) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? temporarily_allow_request_store[key] += value else TEMPORARILY_ALLOW_MUTEX.synchronize do diff --git a/lib/gitlab/testing/request_inspector_middleware.rb b/lib/gitlab/testing/request_inspector_middleware.rb index e387667480d..c251e78f5c5 100644 --- a/lib/gitlab/testing/request_inspector_middleware.rb +++ b/lib/gitlab/testing/request_inspector_middleware.rb @@ -35,11 +35,15 @@ module Gitlab request_headers = env_http_headers(env) status, headers, body = @app.call(env) + full_body = '' + body.each { |b| full_body << b } + request = OpenStruct.new( url: url, status_code: status, request_headers: request_headers, - response_headers: headers + response_headers: headers, + body: full_body ) log_request request diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index b05d408b1c0..c2955cd374c 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -73,25 +73,29 @@ module Gitlab end def fill_last_commits!(entries) - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37433 - Gitlab::GitalyClient.allow_n_plus_1_calls do - entries.each do |entry| - raw_commit = repository.last_commit_for_path(commit.id, entry_path(entry)) + # Ensure the path is in "path/" format + ensured_path = + if path + File.join(*[path, ""]) + end + + commits_hsh = repository.list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit) - if raw_commit - commit = resolve_commit(raw_commit) + entries.each do |entry| + path_key = entry_path(entry) + commit = cache_commit(commits_hsh[path_key]) - entry[:commit] = commit - entry[:commit_path] = commit_path(commit) - end + if commit + entry[:commit] = commit + entry[:commit_path] = commit_path(commit) end end end - def resolve_commit(raw_commit) - return nil unless raw_commit.present? + def cache_commit(commit) + return nil unless commit.present? - resolved_commits[raw_commit.id] ||= ::Commit.new(raw_commit, project) + resolved_commits[commit.id] ||= commit end def commit_path(commit) diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb index 09652a4fb5e..bd0d24e4369 100644 --- a/lib/gitlab/user_extractor.rb +++ b/lib/gitlab/user_extractor.rb @@ -18,7 +18,7 @@ module Gitlab def users return User.none unless @text.present? - @users ||= User.from("(#{union.to_sql}) users") + @users ||= User.from_union(union_relations) end # rubocop: enable CodeReuse/ActiveRecord @@ -43,13 +43,13 @@ module Gitlab private - def union + def union_relations relations = [] relations << User.by_any_email(emails) if emails.any? relations << User.by_username(usernames) if usernames.any? - Gitlab::SQL::Union.new(relations) + relations end end end diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 36859b4d025..77b6610286f 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -50,7 +50,7 @@ module GoogleApi service.get_zone_cluster(project_id, zone, cluster_id, options: user_agent_header) end - def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:) + def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:, legacy_abac:) service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token @@ -63,7 +63,7 @@ module GoogleApi "machine_type": machine_type }, "legacy_abac": { - "enabled": true + "enabled": legacy_abac } } } diff --git a/lib/quality/helm_client.rb b/lib/quality/helm_client.rb new file mode 100644 index 00000000000..49d953da681 --- /dev/null +++ b/lib/quality/helm_client.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require 'time' +require_relative '../gitlab/popen' unless defined?(Gitlab::Popen) + +module Quality + class HelmClient + attr_reader :namespace + + Release = Struct.new(:name, :revision, :last_update, :status, :chart, :namespace) do + def revision + @revision ||= self[:revision].to_i + end + + def last_update + @last_update ||= Time.parse(self[:last_update]) + end + end + + def initialize(namespace: ENV['KUBE_NAMESPACE']) + @namespace = namespace + end + + def releases(args: []) + command = ['list', %(--namespace "#{namespace}"), *args] + + run_command(command) + .stdout + .lines + .select { |line| line.include?(namespace) } + .map { |line| Release.new(*line.split(/\t/).map(&:strip)) } + end + + def delete(release_name:) + run_command(['delete', '--purge', release_name]) + end + + private + + def run_command(command) + final_command = ['helm', *command].join(' ') + puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output + + Gitlab::Popen.popen_with_detail([final_command]) + end + end +end diff --git a/lib/quality/kubernetes_client.rb b/lib/quality/kubernetes_client.rb new file mode 100644 index 00000000000..e366a688e3e --- /dev/null +++ b/lib/quality/kubernetes_client.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require_relative '../gitlab/popen' unless defined?(Gitlab::Popen) + +module Quality + class KubernetesClient + attr_reader :namespace + + def initialize(namespace: ENV['KUBE_NAMESPACE']) + @namespace = namespace + end + + def cleanup(release_name:) + command = ['kubectl'] + command << %(-n "#{namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1) + command << '|' << %(grep "#{release_name}") + command << '|' << "awk '{print $1}'" + command << '|' << %(xargs kubectl -n "#{namespace}" delete) + command << '||' << 'true' + + run_command(command) + end + + private + + def run_command(command) + puts "Running command: `#{command.join(' ')}`" # rubocop:disable Rails/Output + + Gitlab::Popen.popen_with_detail(command) + end + end +end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 72eb8adcce2..fc984d737d5 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -17,7 +17,7 @@ ## See installation.md#using-https for additional HTTPS configuration details. upstream gitlab-workhorse { - # Gitlab socket file, + # GitLab socket file, # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } @@ -112,7 +112,7 @@ server { error_page 502 /502.html; error_page 503 /503.html; location ~ ^/(404|422|500|502|503)\.html$ { - # Location to the Gitlab's public directory, + # Location to the GitLab's public directory, # for Omnibus this would be: /opt/gitlab/embedded/service/gitlab-rails/public. root /home/git/gitlab/public; internal; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 2e3799d5e1b..ba01e250bbb 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -21,7 +21,7 @@ ## See installation.md#using-https for additional HTTPS configuration details. upstream gitlab-workhorse { - # Gitlab socket file, + # GitLab socket file, # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } @@ -162,7 +162,7 @@ server { error_page 502 /502.html; error_page 503 /503.html; location ~ ^/(404|422|500|502|503)\.html$ { - # Location to the Gitlab's public directory, + # Location to the GitLab's public directory, # for Omnibus this would be: /opt/gitlab/embedded/service/gitlab-rails/public root /home/git/gitlab/public; internal; diff --git a/lib/system_check/incoming_email/imap_authentication_check.rb b/lib/system_check/incoming_email/imap_authentication_check.rb index e55bea86d3f..3550c5796b0 100644 --- a/lib/system_check/incoming_email/imap_authentication_check.rb +++ b/lib/system_check/incoming_email/imap_authentication_check.rb @@ -7,7 +7,7 @@ module SystemCheck if config try_connect_imap else - @error = "#{mail_room_config_path} does not have mailboxes setup" + @error = "#{mail_room_config_path} does not have mailboxes set up" false end end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index c6942d22926..560a52053d8 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -86,7 +86,7 @@ namespace :gemojione do SPRITESHEET_WIDTH = 860 SPRITESHEET_HEIGHT = 840 - # Setup a map to rename image files + # Set up a map to rename image files emoji_unicode_string_to_name_map = {} Gitlab::Emoji.emojis.each do |name, emoji_hash| # Ignore aliases diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake index bfca4bfb3f7..e7634d2ed4f 100644 --- a/lib/tasks/gitlab/artifacts/migrate.rake +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -15,7 +15,7 @@ namespace :gitlab do build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE) build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE) - logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage") + logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage") rescue => e logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 69166851816..74cd70c6e9f 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -51,6 +51,8 @@ namespace :gitlab do if ActiveRecord::Base.connection.tables.count > 1 Rake::Task['db:migrate'].invoke else + # Add post-migrate paths to ensure we mark all migrations as up + Gitlab::Database.add_post_migrate_path_to_rails(force: true) Rake::Task['db:schema:load'].invoke Rake::Task['db:seed_fu'].invoke end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 4fcbbbf8c9d..0ebc6f00793 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -92,9 +92,11 @@ namespace :gitlab do def setup warn_user_is_not_gitlab + ensure_write_to_authorized_keys_is_enabled + unless ENV['force'] == 'yes' - puts "This will rebuild an authorized_keys file." - puts "You will lose any data stored in authorized_keys file." + puts "This task will now rebuild the authorized_keys file." + puts "You will lose any data stored in the authorized_keys file." ask_to_continue puts "" end @@ -118,4 +120,44 @@ namespace :gitlab do puts "Quitting...".color(:red) exit 1 end + + def ensure_write_to_authorized_keys_is_enabled + return if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled + + puts authorized_keys_is_disabled_warning + + unless ENV['force'] == 'yes' + puts 'Do you want to permanently enable the "Write to authorized_keys file" setting now?' + ask_to_continue + end + + puts 'Enabling the "Write to authorized_keys file" setting...' + Gitlab::CurrentSettings.current_application_settings.update!(authorized_keys_enabled: true) + + puts 'Successfully enabled "Write to authorized_keys file"!' + puts '' + end + + def authorized_keys_is_disabled_warning + <<-MSG.strip_heredoc + WARNING + + The "Write to authorized_keys file" setting is disabled, which prevents + the file from being rebuilt! + + It should be enabled for most GitLab installations. Large installations + may wish to disable it as part of speeding up SSH operations. + + See https://docs.gitlab.com/ee/administration/operations/fast_ssh_key_lookup.html + + If you did not intentionally disable this option in Admin Area > Settings, + then you may have been affected by the 9.3.0 bug in which the new setting + was disabled by default. + + https://gitlab.com/gitlab-org/gitlab-ee/issues/2738 + + It was reverted in 9.3.1 and fixed in 9.3.3, however, if Settings were + saved while the setting was unchecked, then it is still disabled. + MSG + end end diff --git a/lib/tasks/gitlab/site_statistics.rake b/lib/tasks/gitlab/site_statistics.rake index 7d24ec72a9d..d97f11b2ed5 100644 --- a/lib/tasks/gitlab/site_statistics.rake +++ b/lib/tasks/gitlab/site_statistics.rake @@ -10,14 +10,6 @@ namespace :gitlab do SiteStatistic.update_all('repositories_count = (SELECT COUNT(*) FROM projects)') end puts 'OK!'.color(:green) - - print '* Wikis... ' - SiteStatistic.transaction do - # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967 - ActiveRecord::Base.connection.execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? - SiteStatistic.update_all('wikis_count = (SELECT COUNT(*) FROM project_features WHERE wiki_access_level != 0)') - end - puts 'OK!'.color(:green) puts end end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index ef6a32d6730..abe10f5580e 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -100,10 +100,6 @@ namespace :gitlab do /(\.{1,2}|LICENSE|Global|\.gitignore)\z/ ), Template.new( - "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", - /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/ - ), - Template.new( "https://gitlab.com/gitlab-org/Dockerfile.git", /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/ ) diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index f548a266b99..1c93609a006 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -1,6 +1,30 @@ namespace :gitlab do namespace :uploads do - desc 'GitLab | Uploads | Migrate the uploaded files to object storage' + namespace :migrate do + desc "GitLab | Uploads | Migrate all uploaded files to object storage" + task all: :environment do + categories = [%w(AvatarUploader Project :avatar), + %w(AvatarUploader Group :avatar), + %w(AvatarUploader User :avatar), + %w(AttachmentUploader Note :attachment), + %w(AttachmentUploader Appearance :logo), + %w(AttachmentUploader Appearance :header_logo), + %w(FaviconUploader Appearance :favicon), + %w(FileUploader Project), + %w(PersonalFileUploader Snippet), + %w(NamespaceFileUploader Snippet), + %w(FileUploader MergeRequest)] + + categories.each do |args| + Rake::Task["gitlab:uploads:migrate"].invoke(*args) + Rake::Task["gitlab:uploads:migrate"].reenable + end + end + end + + # The following is the actual rake task that migrates uploads of specified + # category to object storage + desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage' task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args| batch_size = ENV.fetch('BATCH', 200).to_i @to_store = ObjectStorage::Store::REMOTE |