diff options
author | Sean McGivern <sean@gitlab.com> | 2019-02-07 17:46:17 +0300 |
---|---|---|
committer | Sean McGivern <sean@gitlab.com> | 2019-02-07 17:46:17 +0300 |
commit | c3da8849c364885841fddf592346974ff4148c8d (patch) | |
tree | 32ee05526f2b193dbd92db8d13e84cf909f1777d /lib | |
parent | 1a0bab0ab2896f021a5d3a80e84b5e9d6678f6c0 (diff) | |
parent | c8fe0d6a8a76ee5865c71b5b6627608e5075797d (diff) |
Merge branch 'master' into fabsrc/gitlab-ce-2105-add-setting-for-first-day-of-the-week
Diffstat (limited to 'lib')
25 files changed, 521 insertions, 169 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 9cbfc0e35ff..4dd1b459554 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -109,6 +109,7 @@ module API mount ::API::Features mount ::API::Files mount ::API::GroupBoards + mount ::API::GroupLabels mount ::API::GroupMilestones mount ::API::Groups mount ::API::GroupVariables diff --git a/lib/api/entities.rb b/lib/api/entities.rb index a1f0efa3c68..beb8ce349b4 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1019,12 +1019,17 @@ module API label.open_merge_requests_count(options[:current_user]) end - expose :priority do |label, options| - label.priority(options[:project]) + expose :subscribed do |label, options| + label.subscribed?(options[:current_user], options[:parent]) end + end - expose :subscribed do |label, options| - label.subscribed?(options[:current_user], options[:project]) + class GroupLabel < Label + end + + class ProjectLabel < Label + expose :priority do |label, options| + label.priority(options[:parent]) end end diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb new file mode 100644 index 00000000000..0dbc5f45a68 --- /dev/null +++ b/lib/api/group_labels.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module API + class GroupLabels < Grape::API + include PaginationParams + helpers ::API::Helpers::LabelHelpers + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get all labels of the group' do + detail 'This feature was added in GitLab 11.8' + success Entities::GroupLabel + end + params do + use :pagination + end + get ':id/labels' do + get_labels(user_group, Entities::GroupLabel) + end + + desc 'Create a new label' do + detail 'This feature was added in GitLab 11.8' + success Entities::GroupLabel + end + params do + use :label_create_params + end + post ':id/labels' do + create_label(user_group, Entities::GroupLabel) + end + + desc 'Update an existing label. At least one optional parameter is required.' do + detail 'This feature was added in GitLab 11.8' + success Entities::GroupLabel + end + params do + requires :name, type: String, desc: 'The name of the label to be updated' + optional :new_name, type: String, desc: 'The new name of the label' + optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" + optional :description, type: String, desc: 'The new description of label' + at_least_one_of :new_name, :color, :description + end + put ':id/labels' do + update_label(user_group, Entities::GroupLabel) + end + + desc 'Delete an existing label' do + detail 'This feature was added in GitLab 11.8' + success Entities::GroupLabel + end + params do + requires :name, type: String, desc: 'The name of the label to be deleted' + end + delete ':id/labels' do + delete_label(user_group) + end + end + end +end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index e3d0b981065..2eb7b04711a 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -84,8 +84,8 @@ module API page || not_found!('Wiki Page') end - def available_labels_for(label_parent) - search_params = { include_ancestor_groups: true } + def available_labels_for(label_parent, include_ancestor_groups: true) + search_params = { include_ancestor_groups: include_ancestor_groups } if label_parent.is_a?(Project) search_params[:project_id] = label_parent.id @@ -170,13 +170,6 @@ module API end end - def find_project_label(id) - labels = available_labels_for(user_project) - label = labels.find_by_id(id) || labels.find_by_title(id) - - label || not_found!('Label') - end - # rubocop: disable CodeReuse/ActiveRecord def find_project_issue(iid) IssuesFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid) diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb new file mode 100644 index 00000000000..c11e7d614ab --- /dev/null +++ b/lib/api/helpers/label_helpers.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module API + module Helpers + module LabelHelpers + extend Grape::API::Helpers + + params :label_create_params do + requires :name, type: String, desc: 'The name of the label to be created' + requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" + optional :description, type: String, desc: 'The description of label to be created' + end + + def find_label(parent, id, include_ancestor_groups: true) + labels = available_labels_for(parent, include_ancestor_groups: include_ancestor_groups) + label = labels.find_by_id(id) || labels.find_by_title(id) + + label || not_found!('Label') + end + + def get_labels(parent, entity) + present paginate(available_labels_for(parent)), with: entity, current_user: current_user, parent: parent + end + + def create_label(parent, entity) + authorize! :admin_label, parent + + label = available_labels_for(parent).find_by_title(params[:name]) + conflict!('Label already exists') if label + + priority = params.delete(:priority) + label_params = declared_params(include_missing: false) + + label = + if parent.is_a?(Project) + ::Labels::CreateService.new(label_params).execute(project: parent) + else + ::Labels::CreateService.new(label_params).execute(group: parent) + end + + if label.persisted? + if parent.is_a?(Project) + label.prioritize!(parent, priority) if priority + end + + present label, with: entity, current_user: current_user, parent: parent + else + render_validation_error!(label) + end + end + + def update_label(parent, entity) + authorize! :admin_label, parent + + label = find_label(parent, params[:name], include_ancestor_groups: false) + update_priority = params.key?(:priority) + priority = params.delete(:priority) + + label = ::Labels::UpdateService.new(declared_params(include_missing: false)).execute(label) + render_validation_error!(label) unless label.valid? + + if parent.is_a?(Project) && update_priority + if priority.nil? + label.unprioritize!(parent) + else + label.prioritize!(parent, priority) + end + end + + present label, with: entity, current_user: current_user, parent: parent + end + + def delete_label(parent) + authorize! :admin_label, parent + + label = find_label(parent, params[:name], include_ancestor_groups: false) + + destroy_conditionally!(label) + end + end + end +end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index d5eb2b94669..d729d3ee625 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -3,6 +3,7 @@ module API class Labels < Grape::API include PaginationParams + helpers ::API::Helpers::LabelHelpers before { authenticate! } @@ -11,62 +12,28 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all labels of the project' do - success Entities::Label + success Entities::ProjectLabel end params do use :pagination end get ':id/labels' do - present paginate(available_labels_for(user_project)), with: Entities::Label, current_user: current_user, project: user_project + get_labels(user_project, Entities::ProjectLabel) end desc 'Create a new label' do - success Entities::Label + success Entities::ProjectLabel end params do - requires :name, type: String, desc: 'The name of the label to be created' - requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" - optional :description, type: String, desc: 'The description of label to be created' + use :label_create_params optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true end - # rubocop: disable CodeReuse/ActiveRecord post ':id/labels' do - authorize! :admin_label, user_project - - label = available_labels_for(user_project).find_by(title: params[:name]) - conflict!('Label already exists') if label - - priority = params.delete(:priority) - label = ::Labels::CreateService.new(declared_params(include_missing: false)).execute(project: user_project) - - if label.valid? - label.prioritize!(user_project, priority) if priority - present label, with: Entities::Label, current_user: current_user, project: user_project - else - render_validation_error!(label) - end - end - # rubocop: enable CodeReuse/ActiveRecord - - desc 'Delete an existing label' do - success Entities::Label - end - params do - requires :name, type: String, desc: 'The name of the label to be deleted' - end - # rubocop: disable CodeReuse/ActiveRecord - delete ':id/labels' do - authorize! :admin_label, user_project - - label = user_project.labels.find_by(title: params[:name]) - not_found!('Label') unless label - - destroy_conditionally!(label) + create_label(user_project, Entities::ProjectLabel) end - # rubocop: enable CodeReuse/ActiveRecord desc 'Update an existing label. At least one optional parameter is required.' do - success Entities::Label + success Entities::ProjectLabel end params do requires :name, type: String, desc: 'The name of the label to be updated' @@ -76,33 +43,19 @@ module API optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true at_least_one_of :new_name, :color, :description, :priority end - # rubocop: disable CodeReuse/ActiveRecord put ':id/labels' do - authorize! :admin_label, user_project - - label = user_project.labels.find_by(title: params[:name]) - not_found!('Label not found') unless label - - update_priority = params.key?(:priority) - priority = params.delete(:priority) - label_params = declared_params(include_missing: false) - # Rename new name to the actual label attribute name - label_params[:name] = label_params.delete(:new_name) if label_params.key?(:new_name) - - label = ::Labels::UpdateService.new(label_params).execute(label) - render_validation_error!(label) unless label.valid? - - if update_priority - if priority.nil? - label.unprioritize!(user_project) - else - label.prioritize!(user_project, priority) - end - end + update_label(user_project, Entities::ProjectLabel) + end - present label, with: Entities::Label, current_user: current_user, project: user_project + desc 'Delete an existing label' do + success Entities::ProjectLabel + end + params do + requires :name, type: String, desc: 'The name of the label to be deleted' + end + delete ':id/labels' do + delete_label(user_project) end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 4179aaa93a0..df46b4446ff 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -343,6 +343,7 @@ module API end params do optional :merge_commit_message, type: String, desc: 'Custom merge commit message' + optional :squash_commit_message, type: String, desc: 'Custom squash commit message' optional :should_remove_source_branch, type: Boolean, desc: 'When true, the source branch will be deleted if possible' optional :merge_when_pipeline_succeeds, type: Boolean, @@ -370,6 +371,7 @@ module API merge_params = { commit_message: params[:merge_commit_message], + squash_commit_message: params[:squash_commit_message], should_remove_source_branch: params[:should_remove_source_branch] } diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 95371961398..b16faffe335 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -121,6 +121,7 @@ module API optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins' + optional :local_markdown_version, type: Integer, desc: "Local markdown version, increase this value when any cached markdown should be invalidated" ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index 74ad3c35a61..dfb54446ddf 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -2,51 +2,88 @@ module API class Subscriptions < Grape::API + helpers ::API::Helpers::LabelHelpers + before { authenticate! } - subscribable_types = { - 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, - 'issues' => proc { |id| find_project_issue(id) }, - 'labels' => proc { |id| find_project_label(id) } - } + subscribables = [ + { + type: 'merge_requests', + entity: Entities::MergeRequest, + source: Project, + finder: ->(id) { find_merge_request_with_access(id, :update_merge_request) } + }, + { + type: 'issues', + entity: Entities::Issue, + source: Project, + finder: ->(id) { find_project_issue(id) } + }, + { + type: 'labels', + entity: Entities::ProjectLabel, + source: Project, + finder: ->(id) { find_label(user_project, id) } + }, + { + type: 'labels', + entity: Entities::GroupLabel, + source: Group, + finder: ->(id) { find_label(user_group, id) } + } + ] - params do - requires :id, type: String, desc: 'The ID of a project' - requires :subscribable_id, type: String, desc: 'The ID of a resource' - end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - subscribable_types.each do |type, finder| - type_singularized = type.singularize - entity_class = Entities.const_get(type_singularized.camelcase) + subscribables.each do |subscribable| + source_type = subscribable[:source].name.underscore + params do + requires :id, type: String, desc: "The #{source_type} ID" + requires :subscribable_id, type: String, desc: 'The ID of a resource' + end + resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Subscribe to a resource' do - success entity_class + success subscribable[:entity] end - post ":id/#{type}/:subscribable_id/subscribe" do - resource = instance_exec(params[:subscribable_id], &finder) + post ":id/#{subscribable[:type]}/:subscribable_id/subscribe" do + parent = parent_resource(source_type) + resource = instance_exec(params[:subscribable_id], &subscribable[:finder]) - if resource.subscribed?(current_user, user_project) + if resource.subscribed?(current_user, parent) not_modified! else - resource.subscribe(current_user, user_project) - present resource, with: entity_class, current_user: current_user, project: user_project + resource.subscribe(current_user, parent) + present resource, with: subscribable[:entity], current_user: current_user, project: parent, parent: parent end end desc 'Unsubscribe from a resource' do - success entity_class + success subscribable[:entity] end - post ":id/#{type}/:subscribable_id/unsubscribe" do - resource = instance_exec(params[:subscribable_id], &finder) + post ":id/#{subscribable[:type]}/:subscribable_id/unsubscribe" do + parent = parent_resource(source_type) + resource = instance_exec(params[:subscribable_id], &subscribable[:finder]) - if !resource.subscribed?(current_user, user_project) + if !resource.subscribed?(current_user, parent) not_modified! else - resource.unsubscribe(current_user, user_project) - present resource, with: entity_class, current_user: current_user, project: user_project + resource.unsubscribe(current_user, parent) + present resource, with: subscribable[:entity], current_user: current_user, project: parent, parent: parent end end end end + + private + + helpers do + def parent_resource(source_type) + case source_type + when 'project' + user_project + else + nil + end + end + end end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 184c7418e75..22ed1d8e7b4 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -93,7 +93,7 @@ module Backup progress.puts "Error: #{e}".color(:red) end else - restore_repo_success = gitlab_shell.create_repository(project.repository_storage, project.disk_path) + restore_repo_success = gitlab_shell.create_project_repository(project) end if restore_repo_success diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 7aa02009aa0..b2ef04d23d7 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -12,6 +12,9 @@ module Gitlab # Scopes used for OpenID Connect OPENID_SCOPES = [:openid].freeze + # OpenID Connect profile scopes + PROFILE_SCOPES = [:profile, :email].freeze + # Default scopes for OAuth applications that don't define their own DEFAULT_SCOPES = [:api].freeze @@ -284,7 +287,7 @@ module Gitlab # Other available scopes def optional_scopes - available_scopes + OPENID_SCOPES - DEFAULT_SCOPES + available_scopes + OPENID_SCOPES + PROFILE_SCOPES - DEFAULT_SCOPES end def registry_scopes diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index eaead41a720..75a3f17f549 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -65,9 +65,9 @@ module Gitlab def import_wiki return if project.wiki.repository_exists? - disk_path = project.wiki.disk_path - import_url = project.import_url.sub(/\.git\z/, ".git/wiki") - gitlab_shell.import_repository(project.repository_storage, disk_path, import_url) + wiki = WikiFormatter.new(project) + + gitlab_shell.import_wiki_repository(project, wiki) rescue StandardError => e errors << { type: :wiki, errors: e.message } end diff --git a/lib/gitlab/bitbucket_import/wiki_formatter.rb b/lib/gitlab/bitbucket_import/wiki_formatter.rb new file mode 100644 index 00000000000..b8ff43b777b --- /dev/null +++ b/lib/gitlab/bitbucket_import/wiki_formatter.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + class WikiFormatter + attr_reader :project + + def initialize(project) + @project = project + end + + def disk_path + project.wiki.disk_path + end + + def full_path + project.wiki.full_path + end + + def import_url + project.import_url.sub(/\.git\z/, ".git/wiki") + end + end + end +end diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml new file mode 100644 index 00000000000..9c534b2b8e7 --- /dev/null +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -0,0 +1,121 @@ +# Read more about how to use this script on this blog post https://about.gitlab.com/2019/01/28/android-publishing-with-gitlab-and-fastlane/ +# You will also need to configure your build.gradle, Dockerfile, and fastlane configuration to make this work. +# If you are looking for a simpler template that does not publish, see the Android template. + +stages: + - environment + - build + - test + - internal + - alpha + - beta + - production + + +.updateContainerJob: + image: docker:stable + stage: environment + services: + - docker:dind + script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG || true + - docker build --cache-from $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + +updateContainer: + extends: .updateContainerJob + only: + changes: + - Dockerfile + +ensureContainer: + extends: .updateContainerJob + allow_failure: true + before_script: + - "mkdir -p ~/.docker && echo '{\"experimental\": \"enabled\"}' > ~/.docker/config.json" + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + # Skip update container `script` if the container already exists + # via https://gitlab.com/gitlab-org/gitlab-ce/issues/26866#note_97609397 -> https://stackoverflow.com/a/52077071/796832 + - docker manifest inspect $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG > /dev/null && exit || true + + +.build_job: + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + stage: build + before_script: + # We store this binary file in a variable as hex with this command: `xxd -p android-app.jks` + # Then we convert the hex back to a binary file + - echo "$signing_jks_file_hex" | xxd -r -p - > android-signing-keystore.jks + - "export VERSION_CODE=$CI_PIPELINE_IID && echo $VERSION_CODE" + - "export VERSION_SHA=`echo ${CI_COMMIT_SHA:0:8}` && echo $VERSION_SHA" + after_script: + - rm -f android-signing-keystore.jks || true + artifacts: + paths: + - app/build/outputs + +buildDebug: + extends: .build_job + script: + - bundle exec fastlane buildDebug + +buildRelease: + extends: .build_job + script: + - bundle exec fastlane buildRelease + environment: + name: production + +testDebug: + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + stage: test + dependencies: + - buildDebug + script: + - bundle exec fastlane test + +publishInternal: + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + stage: internal + dependencies: + - buildRelease + when: manual + before_script: + - echo $google_play_service_account_api_key_json > ~/google_play_api_key.json + after_script: + - rm ~/google_play_api_key.json + script: + - bundle exec fastlane internal + +.promote_job: + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + when: manual + dependencies: [] + before_script: + - echo $google_play_service_account_api_key_json > ~/google_play_api_key.json + after_script: + - rm ~/google_play_api_key.json + +promoteAlpha: + extends: .promote_job + stage: alpha + script: + - bundle exec fastlane promote_internal_to_alpha + +promoteBeta: + extends: .promote_job + stage: beta + script: + - bundle exec fastlane promote_alpha_to_beta + +promoteProduction: + extends: .promote_job + stage: production + # We only allow production promotion on `master` because + # it has its own production scoped secret variables + only: + - master + script: + - bundle exec fastlane promote_beta_to_production +
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml index 6e138639b71..c169e3f7686 100644 --- a/lib/gitlab/ci/templates/Android.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml @@ -1,4 +1,6 @@ # Read more about this script on this blog post https://about.gitlab.com/2018/10/24/setting-up-gitlab-ci-for-android-projects/, by Jason Lenny +# If you are interested in using Android with FastLane for publishing take a look at the Android-Fastlane template. + image: openjdk:8-jdk variables: diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 75a5bf142d2..e369d26f22f 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -21,8 +21,8 @@ # # In order to deploy, you must have a Kubernetes cluster configured either # via a project integration, or via group/project variables. -# AUTO_DEVOPS_DOMAIN must also be set as a variable at the group or project -# level, or manually added below. +# KUBE_INGRESS_BASE_DOMAIN must also be set on the cluster settings, +# as a variable at the group or project level, or manually added below. # # Continuous deployment to production is enabled by default. # If you want to deploy to staging first, set STAGING_ENABLED environment variable. @@ -41,8 +41,8 @@ image: alpine:latest variables: - # AUTO_DEVOPS_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. - # AUTO_DEVOPS_DOMAIN: domain.example.com + # KUBE_INGRESS_BASE_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. + # KUBE_INGRESS_BASE_DOMAIN: domain.example.com POSTGRES_USER: user POSTGRES_PASSWORD: testing-password @@ -251,7 +251,7 @@ review: - persist_environment_url environment: name: review/$CI_COMMIT_REF_NAME - url: http://$CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN + url: http://$CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN on_stop: stop_review artifacts: paths: [environment_url.txt] @@ -306,7 +306,7 @@ staging: - deploy environment: name: staging - url: http://$CI_PROJECT_PATH_SLUG-staging.$AUTO_DEVOPS_DOMAIN + url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN only: refs: - master @@ -330,7 +330,7 @@ canary: - deploy canary environment: name: production - url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN when: manual only: refs: @@ -354,7 +354,7 @@ canary: - persist_environment_url environment: name: production - url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN artifacts: paths: [environment_url.txt] @@ -403,7 +403,7 @@ production_manual: - persist_environment_url environment: name: production - url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN artifacts: paths: [environment_url.txt] @@ -689,7 +689,7 @@ rollout 100%: --set application.database_url="$DATABASE_URL" \ --set application.secretName="$APPLICATION_SECRET_NAME" \ --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le.$AUTO_DEVOPS_DOMAIN" \ + --set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \ --set service.url="$CI_ENVIRONMENT_URL" \ --set service.additionalHosts="$additional_hosts" \ --set replicaCount="$replicas" \ @@ -725,7 +725,7 @@ rollout 100%: --set application.database_url="$DATABASE_URL" \ --set application.secretName="$APPLICATION_SECRET_NAME" \ --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le.$AUTO_DEVOPS_DOMAIN" \ + --set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \ --set service.url="$CI_ENVIRONMENT_URL" \ --set service.additionalHosts="$additional_hosts" \ --set replicaCount="$replicas" \ @@ -823,11 +823,24 @@ rollout 100%: kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" } + + # Function to ensure backwards compatibility with AUTO_DEVOPS_DOMAIN + function ensure_kube_ingress_base_domain() { + if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ]; then + export KUBE_INGRESS_BASE_DOMAIN=$AUTO_DEVOPS_DOMAIN + fi + } + function check_kube_domain() { - if [ -z ${AUTO_DEVOPS_DOMAIN+x} ]; then - echo "In order to deploy or use Review Apps, AUTO_DEVOPS_DOMAIN variable must be set" - echo "You can do it in Auto DevOps project settings or defining a variable at group or project level" + ensure_kube_ingress_base_domain + + if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ]; then + echo "In order to deploy or use Review Apps," + echo "AUTO_DEVOPS_DOMAIN or KUBE_INGRESS_BASE_DOMAIN variables must be set" + echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings" + echo "or by defining a variable at group or project level." echo "You can also manually add it in .gitlab-ci.yml" + echo "AUTO_DEVOPS_DOMAIN support will be dropped on 12.0" false else true diff --git a/lib/gitlab/git/object_pool.rb b/lib/gitlab/git/object_pool.rb index 1c6242b444a..e93ca3e11f8 100644 --- a/lib/gitlab/git/object_pool.rb +++ b/lib/gitlab/git/object_pool.rb @@ -10,12 +10,13 @@ module Gitlab delegate :exists?, :size, to: :repository delegate :unlink_repository, :delete, to: :object_pool_service - attr_reader :storage, :relative_path, :source_repository + attr_reader :storage, :relative_path, :source_repository, :gl_project_path - def initialize(storage, relative_path, source_repository) + def initialize(storage, relative_path, source_repository, gl_project_path) @storage = storage @relative_path = relative_path @source_repository = source_repository + @gl_project_path = gl_project_path end def create @@ -31,12 +32,12 @@ module Gitlab end def to_gitaly_repository - Gitlab::GitalyClient::Util.repository(storage, relative_path, GL_REPOSITORY) + Gitlab::GitalyClient::Util.repository(storage, relative_path, GL_REPOSITORY, gl_project_path) end # Allows for reusing other RPCs by 'tricking' Gitaly to think its a repository def repository - @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY) + @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY, gl_project_path) end private diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 786c90f9272..54bbd531398 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -67,7 +67,7 @@ module Gitlab # Relative path of repo attr_reader :relative_path - attr_reader :storage, :gl_repository, :relative_path + attr_reader :storage, :gl_repository, :relative_path, :gl_project_path # This remote name has to be stable for all types of repositories that # can join an object pool. If it's structure ever changes, a migration @@ -78,10 +78,11 @@ module Gitlab # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. - def initialize(storage, relative_path, gl_repository) + def initialize(storage, relative_path, gl_repository, gl_project_path) @storage = storage @relative_path = relative_path @gl_repository = gl_repository + @gl_project_path = gl_project_path @name = @relative_path.split("/").last end @@ -872,7 +873,7 @@ module Gitlab end def gitaly_repository - Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) + Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository, @gl_project_path) end def gitaly_ref_client diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 8a1abfbf874..a7e20d9429e 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -324,13 +324,40 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end - def search_files_by_content(ref, query) - request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) - GitalyClient.call(@storage, :repository_service, :search_files_by_content, request).flat_map(&:matches) + def search_files_by_content(ref, query, chunked_response: true) + request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query, chunked_response: chunked_response) + response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request) + + search_results_from_response(response) end private + def search_results_from_response(gitaly_response) + matches = [] + current_match = +"" + + gitaly_response.each do |message| + next if message.nil? + + # Old client will ignore :chunked_response flag + # and return messages with `matches` key. + # This code path will be removed post 12.0 release + if message.matches.any? + matches += message.matches + else + current_match << message.match_data + + if message.end_of_match + matches << current_match + current_match = +"" + end + end + end + + matches + end + def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout) request = request_class.new(repository: @gitaly_repo) response = GitalyClient.call( diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index dce5d6a8ad0..899921f76e4 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -4,7 +4,7 @@ module Gitlab module GitalyClient module Util class << self - def repository(repository_storage, relative_path, gl_repository) + def repository(repository_storage, relative_path, gl_repository, gl_project_path) git_env = Gitlab::Git::HookEnv.all(gl_repository) git_object_directory = git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence git_alternate_object_directories = Array.wrap(git_env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE']) @@ -14,14 +14,16 @@ module Gitlab relative_path: relative_path, gl_repository: gl_repository.to_s, git_object_directory: git_object_directory.to_s, - git_alternate_object_directories: git_alternate_object_directories + git_alternate_object_directories: git_alternate_object_directories, + gl_project_path: gl_project_path ) end def git_repository(gitaly_repository) Gitlab::Git::Repository.new(gitaly_repository.storage_name, gitaly_repository.relative_path, - gitaly_repository.gl_repository) + gitaly_repository.gl_repository, + gitaly_repository.gl_project_path) end end end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index bc3ea9e9226..e2dfb00dcc5 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -6,11 +6,12 @@ module Gitlab class RepositoryImporter include Gitlab::ShellAdapter - attr_reader :project, :client + attr_reader :project, :client, :wiki_formatter def initialize(project, client) @project = project @client = client + @wiki_formatter = ::Gitlab::LegacyGithubImport::WikiFormatter.new(project) end # Returns true if we should import the wiki for the project. @@ -57,9 +58,7 @@ module Gitlab end def import_wiki_repository - wiki_path = "#{project.disk_path}.wiki" - - gitlab_shell.import_repository(project.repository_storage, wiki_path, wiki_url) + gitlab_shell.import_wiki_repository(project, wiki_formatter) true rescue Gitlab::Shell::Error => e @@ -72,7 +71,7 @@ module Gitlab end def wiki_url - project.import_url.sub(/\.git\z/, '.wiki.git') + wiki_formatter.import_url end def update_clone_time diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index c526d31a591..f3323c98af2 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -267,7 +267,7 @@ module Gitlab def import_wiki unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(project.repository_storage, wiki.disk_path, wiki.import_url) + gitlab_shell.import_wiki_repository(project, wiki) end rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, diff --git a/lib/gitlab/legacy_github_import/wiki_formatter.rb b/lib/gitlab/legacy_github_import/wiki_formatter.rb index ea52be5ee0f..cf1e21ad1e1 100644 --- a/lib/gitlab/legacy_github_import/wiki_formatter.rb +++ b/lib/gitlab/legacy_github_import/wiki_formatter.rb @@ -13,6 +13,10 @@ module Gitlab project.wiki.disk_path end + def full_path + project.wiki.full_path + end + def import_url project.import_url.sub(/\.git\z/, ".wiki.git") end diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index 4c4ec026823..4c5b849cc51 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -23,13 +23,13 @@ module Gitlab def sample Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| - unicorn_active_connections.set({ type: 'tcp', address: addr }, stats.active) - unicorn_queued_connections.set({ type: 'tcp', address: addr }, stats.queued) + unicorn_active_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.active) + unicorn_queued_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.queued) end Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| - unicorn_active_connections.set({ type: 'unix', address: addr }, stats.active) - unicorn_queued_connections.set({ type: 'unix', address: addr }, stats.queued) + unicorn_active_connections.set({ socket_type: 'unix', socket_address: addr }, stats.active) + unicorn_queued_connections.set({ socket_type: 'unix', socket_address: addr }, stats.queued) end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index bdf21cf3134..1153e69d3de 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -64,27 +64,48 @@ module Gitlab end end + # Convenience methods for initializing a new repository with a Project model. + def create_project_repository(project) + create_repository(project.repository_storage, project.disk_path, project.full_path) + end + + def create_wiki_repository(project) + create_repository(project.repository_storage, project.wiki.disk_path, project.wiki.full_path) + end + # Init new repository # # storage - the shard key - # name - project disk path + # disk_path - project disk path + # gl_project_path - project name # # Ex. - # create_repository("default", "gitlab/gitlab-ci") + # create_repository("default", "path/to/gitlab-ci", "gitlab/gitlab-ci") # - def create_repository(storage, name) - relative_path = name.dup + def create_repository(storage, disk_path, gl_project_path) + relative_path = disk_path.dup relative_path << '.git' unless relative_path.end_with?('.git') - repository = Gitlab::Git::Repository.new(storage, relative_path, '') + # During creation of a repository, gl_repository may not be known + # because that depends on a yet-to-be assigned project ID in the + # database (e.g. project-1234), so for now it is blank. + repository = Gitlab::Git::Repository.new(storage, relative_path, '', gl_project_path) wrapped_gitaly_errors { repository.gitaly_repository_client.create_repository } true rescue => err # Once the Rugged codes gets removes this can be improved - Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}") + Rails.logger.error("Failed to add repository #{storage}/#{disk_path}: #{err}") false end + def import_wiki_repository(project, wiki_formatter) + import_repository(project.repository_storage, wiki_formatter.disk_path, wiki_formatter.import_url, project.wiki.full_path) + end + + def import_project_repository(project) + import_repository(project.repository_storage, project.disk_path, project.import_url, project.full_path) + end + # Import repository # # storage - project's storage name @@ -94,13 +115,13 @@ module Gitlab # Ex. # import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # - def import_repository(storage, name, url) + def import_repository(storage, name, url, gl_project_path) if url.start_with?('.', '/') raise Error.new("don't use disk paths with import_repository: #{url.inspect}") end relative_path = "#{name}.git" - cmd = GitalyGitlabProjects.new(storage, relative_path) + cmd = GitalyGitlabProjects.new(storage, relative_path, gl_project_path) success = cmd.import_project(url, git_timeout) raise Error, cmd.output unless success @@ -125,18 +146,13 @@ module Gitlab end # Fork repository to new path - # forked_from_storage - forked-from project's storage name - # forked_from_disk_path - project disk relative path - # forked_to_storage - forked-to project's storage name - # forked_to_disk_path - forked project disk relative path - # - # Ex. - # fork_repository("nfs-file06", "gitlab/gitlab-ci", "nfs-file07", "new-namespace/gitlab-ci") - def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) - forked_from_relative_path = "#{forked_from_disk_path}.git" - fork_args = [forked_to_storage, "#{forked_to_disk_path}.git"] + # source_project - forked-from Project + # target_project - forked-to Project + def fork_repository(source_project, target_project) + forked_from_relative_path = "#{source_project.disk_path}.git" + fork_args = [target_project.repository_storage, "#{target_project.disk_path}.git", target_project.full_path] - GitalyGitlabProjects.new(forked_from_storage, forked_from_relative_path).fork_repository(*fork_args) + GitalyGitlabProjects.new(source_project.repository_storage, forked_from_relative_path, source_project.full_path).fork_repository(*fork_args) end # Removes a repository from file system, using rm_diretory which is an alias @@ -397,16 +413,17 @@ module Gitlab end class GitalyGitlabProjects - attr_reader :shard_name, :repository_relative_path, :output + attr_reader :shard_name, :repository_relative_path, :output, :gl_project_path - def initialize(shard_name, repository_relative_path) + def initialize(shard_name, repository_relative_path, gl_project_path) @shard_name = shard_name @repository_relative_path = repository_relative_path @output = '' + @gl_project_path = gl_project_path end def import_project(source, _timeout) - raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil, gl_project_path) Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) true @@ -415,9 +432,9 @@ module Gitlab false end - def fork_repository(new_shard_name, new_repository_relative_path) - target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil) - raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + def fork_repository(new_shard_name, new_repository_relative_path, new_project_name) + target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil, new_project_name) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil, gl_project_path) Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) rescue GRPC::BadStatus => e |