diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-14 00:09:31 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-14 00:09:31 +0300 |
commit | c19dce027b11e8172105685f2a306be51fdac8d3 (patch) | |
tree | fc613edfe02d94caceb5cf58d933828480172259 /app | |
parent | feb61d56e7ce9ab2cd994486bbad9887c3c023f5 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app')
47 files changed, 566 insertions, 30 deletions
diff --git a/app/assets/javascripts/blob/components/blob_header.vue b/app/assets/javascripts/blob/components/blob_header.vue index fd40c51fec1..a4a43b7a94e 100644 --- a/app/assets/javascripts/blob/components/blob_header.vue +++ b/app/assets/javascripts/blob/components/blob_header.vue @@ -66,7 +66,7 @@ export default { <template> <div class="js-file-title file-title-flex-parent"> <blob-filepath :blob="blob"> - <template #filepathPrepend> + <template #filepath-prepend> <slot name="prepend"></slot> </template> </blob-filepath> diff --git a/app/assets/javascripts/blob/components/blob_header_filepath.vue b/app/assets/javascripts/blob/components/blob_header_filepath.vue index f99ecba2324..eb8068a8ad7 100644 --- a/app/assets/javascripts/blob/components/blob_header_filepath.vue +++ b/app/assets/javascripts/blob/components/blob_header_filepath.vue @@ -26,7 +26,7 @@ export default { </script> <template> <div class="file-header-content d-flex align-items-center lh-100"> - <slot name="filepathPrepend"></slot> + <slot name="filepath-prepend"></slot> <template v-if="blob.path"> <file-icon :file-name="blob.path" :size="18" aria-hidden="true" css-classes="mr-2" /> diff --git a/app/assets/javascripts/clusters/components/application_row.vue b/app/assets/javascripts/clusters/components/application_row.vue index 412260da958..471c1a0b4a2 100644 --- a/app/assets/javascripts/clusters/components/application_row.vue +++ b/app/assets/javascripts/clusters/components/application_row.vue @@ -343,7 +343,7 @@ export default { > <span v-else class="js-cluster-application-title">{{ title }}</span> </strong> - <slot name="installedVia"></slot> + <slot name="installed-via"></slot> <div> <slot name="description"></slot> </div> diff --git a/app/assets/javascripts/clusters/components/applications.vue b/app/assets/javascripts/clusters/components/applications.vue index b03cf6fc31b..912568c8870 100644 --- a/app/assets/javascripts/clusters/components/applications.vue +++ b/app/assets/javascripts/clusters/components/applications.vue @@ -549,8 +549,8 @@ export default { @set="setKnativeDomain" /> </template> - <template v-if="cloudRun" #installedVia> - <span data-testid="installedVia"> + <template v-if="cloudRun" #installed-via> + <span data-testid="installed-via"> <gl-sprintf :message="s__('ClusterIntegration|installed via %{linkStart}Cloud Run%{linkEnd}')" > diff --git a/app/assets/javascripts/dependency_proxy.js b/app/assets/javascripts/dependency_proxy.js new file mode 100644 index 00000000000..ddf5703b28f --- /dev/null +++ b/app/assets/javascripts/dependency_proxy.js @@ -0,0 +1,5 @@ +import setupToggleButtons from '~/toggle_buttons'; + +export default () => { + setupToggleButtons(document.querySelector('.js-dependency-proxy-toggle-area')); +}; diff --git a/app/assets/javascripts/design_management/components/design_notes/design_discussion.vue b/app/assets/javascripts/design_management/components/design_notes/design_discussion.vue index 845f1aec8cf..6aab4bf423e 100644 --- a/app/assets/javascripts/design_management/components/design_notes/design_discussion.vue +++ b/app/assets/javascripts/design_management/components/design_notes/design_discussion.vue @@ -210,7 +210,7 @@ export default { :class="{ 'gl-bg-blue-50': isDiscussionActive }" @error="$emit('update-note-error', $event)" > - <template v-if="discussion.resolvable" #resolveDiscussion> + <template v-if="discussion.resolvable" #resolve-discussion> <button v-gl-tooltip :class="{ 'is-active': discussion.resolved }" @@ -224,7 +224,7 @@ export default { <gl-loading-icon v-else inline /> </button> </template> - <template v-if="discussion.resolved" #resolvedStatus> + <template v-if="discussion.resolved" #resolved-status> <p class="gl-text-gray-500 gl-font-sm gl-m-0 gl-mt-5" data-testid="resolved-message"> {{ __('Resolved by') }} <gl-link @@ -277,7 +277,7 @@ export default { @submit-form="mutate" @cancel-form="hideForm" > - <template v-if="discussion.resolvable" #resolveCheckbox> + <template v-if="discussion.resolvable" #resolve-checkbox> <label data-testid="resolve-checkbox"> <input v-model="shouldChangeResolvedStatus" type="checkbox" /> {{ resolveCheckboxText }} diff --git a/app/assets/javascripts/design_management/components/design_notes/design_note.vue b/app/assets/javascripts/design_management/components/design_notes/design_note.vue index 7f4b3b31024..421a4dc274a 100644 --- a/app/assets/javascripts/design_management/components/design_notes/design_note.vue +++ b/app/assets/javascripts/design_management/components/design_notes/design_note.vue @@ -108,7 +108,7 @@ export default { </span> </div> <div class="gl-display-flex gl-align-items-baseline"> - <slot name="resolveDiscussion"></slot> + <slot name="resolve-discussion"></slot> <button v-if="isEditButtonVisible" v-gl-tooltip @@ -127,7 +127,7 @@ export default { class="note-text js-note-text md" data-qa-selector="note_content" ></div> - <slot name="resolvedStatus"></slot> + <slot name="resolved-status"></slot> </template> <apollo-mutation v-else diff --git a/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue b/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue index 3754e1dbbc1..7aaac58a1ce 100644 --- a/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue +++ b/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue @@ -110,7 +110,7 @@ export default { </textarea> </template> </markdown-field> - <slot name="resolveCheckbox"></slot> + <slot name="resolve-checkbox"></slot> <div class="note-form-actions gl-display-flex gl-justify-content-space-between"> <gl-button ref="submitButton" diff --git a/app/assets/javascripts/design_management/components/design_sidebar.vue b/app/assets/javascripts/design_management/components/design_sidebar.vue index fb8e74c8c4c..41dcec38abe 100644 --- a/app/assets/javascripts/design_management/components/design_sidebar.vue +++ b/app/assets/javascripts/design_management/components/design_sidebar.vue @@ -207,6 +207,6 @@ export default { /> </gl-collapse> </template> - <slot name="replyForm"></slot> + <slot name="reply-form"></slot> </div> </template> diff --git a/app/assets/javascripts/design_management/pages/design/index.vue b/app/assets/javascripts/design_management/pages/design/index.vue index d1bbb5239d0..e07279ba39d 100644 --- a/app/assets/javascripts/design_management/pages/design/index.vue +++ b/app/assets/javascripts/design_management/pages/design/index.vue @@ -383,7 +383,7 @@ export default { @toggleResolvedComments="toggleResolvedComments" @todoError="onTodoError" > - <template #replyForm> + <template #reply-form> <apollo-mutation v-if="isAnnotating" #default="{ mutate, loading }" diff --git a/app/assets/javascripts/environments/components/container.vue b/app/assets/javascripts/environments/components/container.vue index 4c6d233c4d2..e7697f14802 100644 --- a/app/assets/javascripts/environments/components/container.vue +++ b/app/assets/javascripts/environments/components/container.vue @@ -69,7 +69,7 @@ export default { <div class="environments-container"> <gl-loading-icon v-if="isLoading" size="md" class="gl-mt-3" label="Loading environments" /> - <slot name="emptyState"></slot> + <slot name="empty-state"></slot> <div v-if="!isLoading && environments.length > 0" class="table-holder"> <environment-table diff --git a/app/assets/javascripts/environments/components/environments_app.vue b/app/assets/javascripts/environments/components/environments_app.vue index 9bafc7ed153..c1b9ba755a6 100644 --- a/app/assets/javascripts/environments/components/environments_app.vue +++ b/app/assets/javascripts/environments/components/environments_app.vue @@ -228,7 +228,7 @@ export default { :deploy-boards-help-path="deployBoardsHelpPath" @onChangePage="onChangePage" > - <template v-if="!isLoading && state.environments.length === 0" #emptyState> + <template v-if="!isLoading && state.environments.length === 0" #empty-state> <empty-state :help-path="helpPagePath" /> </template> </container> diff --git a/app/assets/javascripts/import_projects/store/actions.js b/app/assets/javascripts/import_projects/store/actions.js index b31b1224344..7b7afd13c55 100644 --- a/app/assets/javascripts/import_projects/store/actions.js +++ b/app/assets/javascripts/import_projects/store/actions.js @@ -7,11 +7,14 @@ import { visitUrl, objectToQuery } from '~/lib/utils/url_utility'; import { deprecatedCreateFlash as createFlash } from '~/flash'; import { s__, sprintf } from '~/locale'; import axios from '~/lib/utils/axios_utils'; +import httpStatusCodes from '~/lib/utils/http_status'; +import { capitalizeFirstCharacter } from '~/lib/utils/text_utility'; let eTagPoll; const hasRedirectInError = e => e?.response?.data?.error?.redirect; const redirectToUrlInError = e => visitUrl(e.response.data.error.redirect); +const tooManyRequests = e => e.response.status === httpStatusCodes.TOO_MANY_REQUESTS; const pathWithParams = ({ path, ...params }) => { const filteredParams = Object.fromEntries( Object.entries(params).filter(([, value]) => value !== ''), @@ -71,6 +74,14 @@ const fetchReposFactory = ({ reposPath = isRequired() }) => ({ state, commit }) if (hasRedirectInError(e)) { redirectToUrlInError(e); + } else if (tooManyRequests(e)) { + createFlash( + sprintf(s__('ImportProjects|%{provider} rate limit exceeded. Try again later'), { + provider: capitalizeFirstCharacter(provider), + }), + ); + + commit(types.RECEIVE_REPOS_ERROR); } else { createFlash( sprintf(s__('ImportProjects|Requesting your %{provider} repositories failed'), { diff --git a/app/assets/javascripts/incidents/components/incidents_list.vue b/app/assets/javascripts/incidents/components/incidents_list.vue index 6f9b05c08ab..0e3839deaf5 100644 --- a/app/assets/javascripts/incidents/components/incidents_list.vue +++ b/app/assets/javascripts/incidents/components/incidents_list.vue @@ -419,7 +419,7 @@ export default { </template> </gl-table> </template> - <template #emtpy-state> + <template #empty-state> <gl-empty-state :title="emptyStateData.title" :svg-path="emptyListSvgPath" diff --git a/app/assets/javascripts/lib/utils/http_status.js b/app/assets/javascripts/lib/utils/http_status.js index 7132986a7e6..06529f06a66 100644 --- a/app/assets/javascripts/lib/utils/http_status.js +++ b/app/assets/javascripts/lib/utils/http_status.js @@ -22,6 +22,7 @@ const httpStatusCodes = { CONFLICT: 409, GONE: 410, UNPROCESSABLE_ENTITY: 422, + TOO_MANY_REQUESTS: 429, INTERNAL_SERVER_ERROR: 500, SERVICE_UNAVAILABLE: 503, }; diff --git a/app/assets/javascripts/monitoring/components/dashboard.vue b/app/assets/javascripts/monitoring/components/dashboard.vue index cbfacd73b5b..16c2c87a4b7 100644 --- a/app/assets/javascripts/monitoring/components/dashboard.vue +++ b/app/assets/javascripts/monitoring/components/dashboard.vue @@ -423,7 +423,7 @@ export default { :prometheus-alerts-available="prometheusAlertsAvailable" @timerangezoom="onTimeRangeZoom" > - <template #topLeft> + <template #top-left> <gl-button ref="goBackBtn" v-gl-tooltip diff --git a/app/assets/javascripts/monitoring/components/dashboard_panel.vue b/app/assets/javascripts/monitoring/components/dashboard_panel.vue index 18310f7c71e..597600bba07 100644 --- a/app/assets/javascripts/monitoring/components/dashboard_panel.vue +++ b/app/assets/javascripts/monitoring/components/dashboard_panel.vue @@ -365,7 +365,7 @@ export default { <template> <div v-gl-resize-observer="onResize" class="prometheus-graph"> <div class="d-flex align-items-center"> - <slot name="topLeft"></slot> + <slot name="top-left"></slot> <h5 ref="graphTitle" class="prometheus-graph-title gl-font-lg font-weight-bold text-truncate gl-mr-3" diff --git a/app/assets/javascripts/pages/groups/dependency_proxies/index.js b/app/assets/javascripts/pages/groups/dependency_proxies/index.js new file mode 100644 index 00000000000..4c0a1abb0f0 --- /dev/null +++ b/app/assets/javascripts/pages/groups/dependency_proxies/index.js @@ -0,0 +1,17 @@ +import $ from 'jquery'; +import initDependencyProxy from '~/dependency_proxy'; + +document.addEventListener('DOMContentLoaded', () => { + initDependencyProxy(); +}); + +document.addEventListener('DOMContentLoaded', () => { + const form = document.querySelector('form.edit_dependency_proxy_group_setting'); + const toggleInput = $('input.js-project-feature-toggle-input'); + + if (form && toggleInput) { + toggleInput.on('trigger-change', () => { + form.submit(); + }); + } +}); diff --git a/app/assets/javascripts/related_issues/components/related_issues_block.vue b/app/assets/javascripts/related_issues/components/related_issues_block.vue index f7a79c62716..c913745a8e1 100644 --- a/app/assets/javascripts/related_issues/components/related_issues_block.vue +++ b/app/assets/javascripts/related_issues/components/related_issues_block.vue @@ -138,7 +138,7 @@ export default { href="#related-issues" aria-hidden="true" /> - <slot name="headerText">{{ __('Linked issues') }}</slot> + <slot name="header-text">{{ __('Linked issues') }}</slot> <gl-link v-if="hasHelpPath" :href="helpPath" @@ -167,7 +167,7 @@ export default { /> </div> </h3> - <slot name="headerActions"></slot> + <slot name="header-actions"></slot> </div> <div class="linked-issues-card-body bg-gray-light" diff --git a/app/assets/javascripts/reports/components/grouped_test_reports_app.vue b/app/assets/javascripts/reports/components/grouped_test_reports_app.vue index d977ec37126..c13df60198b 100644 --- a/app/assets/javascripts/reports/components/grouped_test_reports_app.vue +++ b/app/assets/javascripts/reports/components/grouped_test_reports_app.vue @@ -134,7 +134,7 @@ export default { class="mr-widget-section grouped-security-reports mr-report" @toggleEvent="handleToggleEvent" > - <template v-if="showViewFullReport" #actionButtons> + <template v-if="showViewFullReport" #action-buttons> <gl-button :href="testTabURL" target="_blank" @@ -145,7 +145,7 @@ export default { {{ s__('ciReport|View full report') }} </gl-button> </template> - <template v-if="hasRecentFailures(summary)" #subHeading> + <template v-if="hasRecentFailures(summary)" #sub-heading> {{ recentFailuresText(summary) }} </template> <template #body> diff --git a/app/assets/javascripts/reports/components/report_section.vue b/app/assets/javascripts/reports/components/report_section.vue index cf5c0ceadfe..f245e2bfd2f 100644 --- a/app/assets/javascripts/reports/components/report_section.vue +++ b/app/assets/javascripts/reports/components/report_section.vue @@ -181,10 +181,10 @@ export default { <slot :name="slotName"></slot> <popover v-if="hasPopover" :options="popoverOptions" class="gl-ml-2" /> </div> - <slot name="subHeading"></slot> + <slot name="sub-heading"></slot> </div> - <slot name="actionButtons"></slot> + <slot name="action-buttons"></slot> <button v-if="isCollapsible" diff --git a/app/assets/javascripts/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs.vue b/app/assets/javascripts/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs.vue index 8e85d93e6d1..1fc39c7cb8e 100644 --- a/app/assets/javascripts/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs.vue +++ b/app/assets/javascripts/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs.vue @@ -308,6 +308,6 @@ export default { @input="handlePageChange" /> - <slot v-if="!showItems" name="emtpy-state"></slot> + <slot v-if="!showItems" name="empty-state"></slot> </div> </template> diff --git a/app/assets/javascripts/vue_shared/components/upload_dropzone/upload_dropzone.vue b/app/assets/javascripts/vue_shared/components/upload_dropzone/upload_dropzone.vue index 25d73ed0855..b645758d891 100644 --- a/app/assets/javascripts/vue_shared/components/upload_dropzone/upload_dropzone.vue +++ b/app/assets/javascripts/vue_shared/components/upload_dropzone/upload_dropzone.vue @@ -147,7 +147,7 @@ export default { class="card upload-dropzone-border upload-dropzone-overlay gl-w-full gl-h-full gl-absolute gl-display-flex gl-align-items-center gl-justify-content-center gl-p-3 gl-bg-white" > <div v-show="!isDragDataValid" class="mw-50 gl-text-center"> - <slot name="invalidDragDataSlot"> + <slot name="invalid-drag-data-slot"> <h3 :class="{ 'gl-font-base gl-display-inline': !displayAsCard }"> {{ __('Oh no!') }} </h3> @@ -159,7 +159,7 @@ export default { </slot> </div> <div v-show="isDragDataValid" class="mw-50 gl-text-center"> - <slot name="validDragDataSlot"> + <slot name="valid-drag-data-slot"> <h3 :class="{ 'gl-font-base gl-display-inline': !displayAsCard }"> {{ __('Incoming!') }} </h3> diff --git a/app/controllers/concerns/dependency_proxy_access.rb b/app/controllers/concerns/dependency_proxy_access.rb new file mode 100644 index 00000000000..5036d0cfce4 --- /dev/null +++ b/app/controllers/concerns/dependency_proxy_access.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module DependencyProxyAccess + extend ActiveSupport::Concern + + included do + before_action :verify_dependency_proxy_enabled! + before_action :authorize_read_dependency_proxy! + end + + private + + def verify_dependency_proxy_enabled! + render_404 unless group.dependency_proxy_feature_available? + end + + def authorize_read_dependency_proxy! + access_denied! unless can?(current_user, :read_dependency_proxy, group) + end + + def authorize_admin_dependency_proxy! + access_denied! unless can?(current_user, :admin_dependency_proxy, group) + end +end diff --git a/app/controllers/groups/dependency_proxies_controller.rb b/app/controllers/groups/dependency_proxies_controller.rb new file mode 100644 index 00000000000..367dbafdd59 --- /dev/null +++ b/app/controllers/groups/dependency_proxies_controller.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Groups + class DependencyProxiesController < Groups::ApplicationController + include DependencyProxyAccess + + before_action :authorize_admin_dependency_proxy!, only: :update + before_action :dependency_proxy + + feature_category :package_registry + + def show + @blobs_count = group.dependency_proxy_blobs.count + @blobs_total_size = group.dependency_proxy_blobs.total_size + end + + def update + dependency_proxy.update(dependency_proxy_params) + + redirect_to group_dependency_proxy_path(group) + end + + private + + def dependency_proxy + @dependency_proxy ||= + group.dependency_proxy_setting || group.create_dependency_proxy_setting + end + + def dependency_proxy_params + params.require(:dependency_proxy_group_setting).permit(:enabled) + end + end +end diff --git a/app/controllers/groups/dependency_proxy_for_containers_controller.rb b/app/controllers/groups/dependency_proxy_for_containers_controller.rb new file mode 100644 index 00000000000..f46902ef90f --- /dev/null +++ b/app/controllers/groups/dependency_proxy_for_containers_controller.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +class Groups::DependencyProxyForContainersController < Groups::ApplicationController + include DependencyProxyAccess + include SendFileUpload + + before_action :ensure_token_granted! + before_action :ensure_feature_enabled! + + attr_reader :token + + feature_category :package_registry + + def manifest + result = DependencyProxy::PullManifestService.new(image, tag, token).execute + + if result[:status] == :success + render json: result[:manifest] + else + render status: result[:http_status], json: result[:message] + end + end + + def blob + result = DependencyProxy::FindOrCreateBlobService + .new(group, image, token, params[:sha]).execute + + if result[:status] == :success + send_upload(result[:blob].file) + else + head result[:http_status] + end + end + + private + + def image + params[:image] + end + + def tag + params[:tag] + end + + def dependency_proxy + @dependency_proxy ||= + group.dependency_proxy_setting || group.create_dependency_proxy_setting + end + + def ensure_feature_enabled! + render_404 unless dependency_proxy.enabled + end + + def ensure_token_granted! + result = DependencyProxy::RequestTokenService.new(image).execute + + if result[:status] == :success + @token = result[:token] + else + render status: result[:http_status], json: result[:message] + end + end +end diff --git a/app/controllers/import/github_controller.rb b/app/controllers/import/github_controller.rb index 03f8020ee7b..8ac93aeb9c0 100644 --- a/app/controllers/import/github_controller.rb +++ b/app/controllers/import/github_controller.rb @@ -15,6 +15,7 @@ class Import::GithubController < Import::BaseController rescue_from OAuthConfigMissingError, with: :missing_oauth_config rescue_from Octokit::Unauthorized, with: :provider_unauthorized rescue_from Octokit::TooManyRequests, with: :provider_rate_limit + rescue_from Gitlab::GithubImport::RateLimitError, with: :rate_limit_threshold_exceeded def new if !ci_cd_only? && github_import_configured? && logged_in_with_provider? @@ -114,7 +115,7 @@ class Import::GithubController < Import::BaseController def client_repos @client_repos ||= if Feature.enabled?(:remove_legacy_github_client) - filtered(concatenated_repos) + concatenated_repos else filtered(client.repos) end @@ -122,8 +123,15 @@ class Import::GithubController < Import::BaseController def concatenated_repos return [] unless client.respond_to?(:each_page) + return client.each_page(:repos).flat_map(&:objects) unless sanitized_filter_param - client.each_page(:repos).flat_map(&:objects) + client.search_repos_by_name(sanitized_filter_param).flat_map(&:objects).flat_map(&:items) + end + + def sanitized_filter_param + super + + @filter = @filter&.tr(' ', '')&.tr(':', '') end def oauth_client @@ -245,6 +253,10 @@ class Import::GithubController < Import::BaseController def extra_import_params {} end + + def rate_limit_threshold_exceeded + head :too_many_requests + end end Import::GithubController.prepend_if_ee('EE::Import::GithubController') diff --git a/app/helpers/groups_helper.rb b/app/helpers/groups_helper.rb index d287d430bc4..29ead76a607 100644 --- a/app/helpers/groups_helper.rb +++ b/app/helpers/groups_helper.rb @@ -170,6 +170,10 @@ module GroupsHelper group_container_registry_nav? end + def group_dependency_proxy_nav? + @group.dependency_proxy_feature_available? + end + def group_packages_list_nav? @group.packages_feature_enabled? end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index a009d7c8062..84abd01786d 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -103,6 +103,10 @@ module Ci ) end + scope :in_pipelines, ->(pipelines) do + where(pipeline: pipelines) + end + scope :with_existing_job_artifacts, ->(query) do where('EXISTS (?)', ::Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').merge(query)) end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 3b8c86f2db9..8707d635e03 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -355,6 +355,14 @@ module Ci end end + def self.latest_running_for_ref(ref) + newest_first(ref: ref).running.take + end + + def self.latest_failed_for_ref(ref) + newest_first(ref: ref).failed.take + end + # Returns a Hash containing the latest pipeline for every given # commit. # diff --git a/app/models/dependency_proxy.rb b/app/models/dependency_proxy.rb new file mode 100644 index 00000000000..510a304ff17 --- /dev/null +++ b/app/models/dependency_proxy.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true +module DependencyProxy + def self.table_name_prefix + 'dependency_proxy_' + end +end diff --git a/app/models/dependency_proxy/blob.rb b/app/models/dependency_proxy/blob.rb new file mode 100644 index 00000000000..3a81112340a --- /dev/null +++ b/app/models/dependency_proxy/blob.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class DependencyProxy::Blob < ApplicationRecord + include FileStoreMounter + + belongs_to :group + + validates :group, presence: true + validates :file, presence: true + validates :file_name, presence: true + + mount_file_store_uploader DependencyProxy::FileUploader + + def self.total_size + sum(:size) + end + + def self.find_or_build(file_name) + find_or_initialize_by(file_name: file_name) + end +end diff --git a/app/models/dependency_proxy/group_setting.rb b/app/models/dependency_proxy/group_setting.rb new file mode 100644 index 00000000000..bcf09b27129 --- /dev/null +++ b/app/models/dependency_proxy/group_setting.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class DependencyProxy::GroupSetting < ApplicationRecord + belongs_to :group + + validates :group, presence: true + + default_value_for :enabled, true +end diff --git a/app/models/dependency_proxy/registry.rb b/app/models/dependency_proxy/registry.rb new file mode 100644 index 00000000000..471d5be2600 --- /dev/null +++ b/app/models/dependency_proxy/registry.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class DependencyProxy::Registry + AUTH_URL = 'https://auth.docker.io'.freeze + LIBRARY_URL = 'https://registry-1.docker.io/v2'.freeze + + class << self + def auth_url(image) + "#{AUTH_URL}/token?service=registry.docker.io&scope=repository:#{image_path(image)}:pull" + end + + def manifest_url(image, tag) + "#{LIBRARY_URL}/#{image_path(image)}/manifests/#{tag}" + end + + def blob_url(image, blob_sha) + "#{LIBRARY_URL}/#{image_path(image)}/blobs/#{blob_sha}" + end + + private + + def image_path(image) + if image.include?('/') + image + else + "library/#{image}" + end + end + end +end diff --git a/app/models/group.rb b/app/models/group.rb index 7b701ad41bd..3509299a579 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -71,6 +71,9 @@ class Group < Namespace has_many :group_deploy_tokens has_many :deploy_tokens, through: :group_deploy_tokens + has_one :dependency_proxy_setting, class_name: 'DependencyProxy::GroupSetting' + has_many :dependency_proxy_blobs, class_name: 'DependencyProxy::Blob' + accepts_nested_attributes_for :variables, allow_destroy: true validate :visibility_level_allowed_by_projects @@ -203,6 +206,10 @@ class Group < Namespace ::Gitlab.config.packages.enabled end + def dependency_proxy_feature_available? + ::Gitlab.config.dependency_proxy.enabled + end + def notification_email_for(user) # Finds the closest notification_setting with a `notification_email` notification_settings = notification_settings_for(user, hierarchy_order: :asc) diff --git a/app/policies/group_policy.rb b/app/policies/group_policy.rb index 92a5ced4fc3..6ff6d1359ab 100644 --- a/app/policies/group_policy.rb +++ b/app/policies/group_policy.rb @@ -46,6 +46,10 @@ class GroupPolicy < BasePolicy group_projects_for(user: @user, group: @subject, only_owned: false).any? { |p| p.design_management_enabled? } end + condition(:dependency_proxy_available) do + @subject.dependency_proxy_feature_available? + end + desc "Deploy token with read_package_registry scope" condition(:read_package_registry_deploy_token) do @user.is_a?(DeployToken) && @user.groups.include?(@subject) && @user.read_package_registry @@ -193,6 +197,12 @@ class GroupPolicy < BasePolicy enable :read_group end + rule { can?(:read_group) & dependency_proxy_available } + .enable :read_dependency_proxy + + rule { developer & dependency_proxy_available } + .enable :admin_dependency_proxy + rule { resource_access_token_available & can?(:admin_group) }.policy do enable :admin_resource_access_tokens end diff --git a/app/services/dependency_proxy/base_service.rb b/app/services/dependency_proxy/base_service.rb new file mode 100644 index 00000000000..1b2d4b14a27 --- /dev/null +++ b/app/services/dependency_proxy/base_service.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module DependencyProxy + class BaseService < ::BaseService + private + + def registry + DependencyProxy::Registry + end + + def auth_headers + { + Authorization: "Bearer #{@token}" + } + end + end +end diff --git a/app/services/dependency_proxy/download_blob_service.rb b/app/services/dependency_proxy/download_blob_service.rb new file mode 100644 index 00000000000..3c690683bf6 --- /dev/null +++ b/app/services/dependency_proxy/download_blob_service.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module DependencyProxy + class DownloadBlobService < DependencyProxy::BaseService + class DownloadError < StandardError + attr_reader :http_status + + def initialize(message, http_status) + @http_status = http_status + + super(message) + end + end + + def initialize(image, blob_sha, token) + @image = image + @blob_sha = blob_sha + @token = token + @temp_file = Tempfile.new + end + + def execute + File.open(@temp_file.path, "wb") do |file| + Gitlab::HTTP.get(blob_url, headers: auth_headers, stream_body: true) do |fragment| + if [301, 302, 307].include?(fragment.code) + # do nothing + elsif fragment.code == 200 + file.write(fragment) + else + raise DownloadError.new('Non-success response code on downloading blob fragment', fragment.code) + end + end + end + + success(file: @temp_file) + rescue DownloadError => exception + error(exception.message, exception.http_status) + rescue Timeout::Error => exception + error(exception.message, 599) + end + + private + + def blob_url + registry.blob_url(@image, @blob_sha) + end + end +end diff --git a/app/services/dependency_proxy/find_or_create_blob_service.rb b/app/services/dependency_proxy/find_or_create_blob_service.rb new file mode 100644 index 00000000000..bd06f9d7628 --- /dev/null +++ b/app/services/dependency_proxy/find_or_create_blob_service.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module DependencyProxy + class FindOrCreateBlobService < DependencyProxy::BaseService + def initialize(group, image, token, blob_sha) + @group = group + @image = image + @token = token + @blob_sha = blob_sha + end + + def execute + file_name = @blob_sha.sub('sha256:', '') + '.gz' + blob = @group.dependency_proxy_blobs.find_or_build(file_name) + + unless blob.persisted? + result = DependencyProxy::DownloadBlobService + .new(@image, @blob_sha, @token).execute + + if result[:status] == :error + log_failure(result) + + return error('Failed to download the blob', result[:http_status]) + end + + blob.file = result[:file] + blob.size = result[:file].size + blob.save! + end + + success(blob: blob) + end + + private + + def log_failure(result) + log_error( + "Dependency proxy: Failed to download the blob." \ + "Blob sha: #{@blob_sha}." \ + "Error message: #{result[:message][0, 100]}" \ + "HTTP status: #{result[:http_status]}" + ) + end + end +end diff --git a/app/services/dependency_proxy/pull_manifest_service.rb b/app/services/dependency_proxy/pull_manifest_service.rb new file mode 100644 index 00000000000..fc54ef85c96 --- /dev/null +++ b/app/services/dependency_proxy/pull_manifest_service.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module DependencyProxy + class PullManifestService < DependencyProxy::BaseService + def initialize(image, tag, token) + @image = image + @tag = tag + @token = token + end + + def execute + response = Gitlab::HTTP.get(manifest_url, headers: auth_headers) + + if response.success? + success(manifest: response.body) + else + error(response.body, response.code) + end + rescue Timeout::Error => exception + error(exception.message, 599) + end + + private + + def manifest_url + registry.manifest_url(@image, @tag) + end + end +end diff --git a/app/services/dependency_proxy/request_token_service.rb b/app/services/dependency_proxy/request_token_service.rb new file mode 100644 index 00000000000..4ca7239b9f6 --- /dev/null +++ b/app/services/dependency_proxy/request_token_service.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module DependencyProxy + class RequestTokenService < DependencyProxy::BaseService + def initialize(image) + @image = image + end + + def execute + response = Gitlab::HTTP.get(auth_url) + + if response.success? + success(token: Gitlab::Json.parse(response.body)['token']) + else + error('Expected 200 response code for an access token', response.code) + end + rescue Timeout::Error => exception + error(exception.message, 599) + rescue JSON::ParserError + error('Failed to parse a response body for an access token', 500) + end + + private + + def auth_url + registry.auth_url(@image) + end + end +end diff --git a/app/uploaders/dependency_proxy/file_uploader.rb b/app/uploaders/dependency_proxy/file_uploader.rb new file mode 100644 index 00000000000..b67a22bae4d --- /dev/null +++ b/app/uploaders/dependency_proxy/file_uploader.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class DependencyProxy::FileUploader < GitlabUploader + include ObjectStorage::Concern + + storage_options Gitlab.config.dependency_proxy + + alias_method :upload, :model + + def filename + model.file_name + end + + def store_dir + dynamic_segment + end + + private + + def dynamic_segment + Gitlab::HashedPath.new('dependency_proxy', model.group_id, 'files', model.id, root_hash: model.group_id) + end +end diff --git a/app/views/groups/dependency_proxies/_url.html.haml b/app/views/groups/dependency_proxies/_url.html.haml new file mode 100644 index 00000000000..9242954b684 --- /dev/null +++ b/app/views/groups/dependency_proxies/_url.html.haml @@ -0,0 +1,12 @@ +- proxy_url = "#{group_url(@group)}/dependency_proxy/containers" + +%h5.prepend-top-20= _('Dependency proxy URL') + +.row + .col-lg-8.col-md-12.input-group + = text_field_tag :url, "#{proxy_url}", class: 'js-dependency-proxy-url form-control', readonly: true + = clipboard_button(text: "#{proxy_url}", title: _("Copy %{proxy_url}") % { proxy_url: proxy_url }) + +.row + .col-12.help-block.gl-mt-3 + = _('Contains %{count} blobs of images (%{size})') % { count: @blobs_count, size: number_to_human_size(@blobs_total_size) } diff --git a/app/views/groups/dependency_proxies/show.html.haml b/app/views/groups/dependency_proxies/show.html.haml new file mode 100644 index 00000000000..ff1312eb763 --- /dev/null +++ b/app/views/groups/dependency_proxies/show.html.haml @@ -0,0 +1,28 @@ +- page_title _("Dependency Proxy") + +.settings-header + %h4= _('Dependency proxy') + + %p + - link_start = '<a href="%{url}">'.html_safe % { url: help_page_path('user/packages/dependency_proxy/index') } + = _('Create a local proxy for storing frequently used upstream images. %{link_start}Learn more%{link_end} about dependency proxies.').html_safe % { link_start: link_start, link_end: '</a>'.html_safe } + +- if @group.public? + - if can?(current_user, :admin_dependency_proxy, @group) + = form_for(@dependency_proxy, method: :put, url: group_dependency_proxy_path(@group)) do |f| + .form-group + %h5.prepend-top-20= _('Enable proxy') + .js-dependency-proxy-toggle-area + = render "shared/buttons/project_feature_toggle", is_checked: @dependency_proxy.enabled?, label: s_("DependencyProxy|Toggle Dependency Proxy") do + = f.hidden_field :enabled, { class: 'js-project-feature-toggle-input'} + + - if @dependency_proxy.enabled + = render 'groups/dependency_proxies/url' + + - else + - if @dependency_proxy.enabled + = render 'groups/dependency_proxies/url' +- else + .gl-alert.gl-alert-info + = sprite_icon('information-o', size: 16, css_class: 'gl-icon gl-alert-icon gl-alert-icon-no-title') + = _('Dependency proxy feature is limited to public groups for now.') diff --git a/app/views/groups/sidebar/_packages.html.haml b/app/views/groups/sidebar/_packages.html.haml index 54510d5df0c..7e0ee032aeb 100644 --- a/app/views/groups/sidebar/_packages.html.haml +++ b/app/views/groups/sidebar/_packages.html.haml @@ -1,7 +1,7 @@ - packages_link = group_packages_list_nav? ? group_packages_path(@group) : group_container_registries_path(@group) - if group_packages_nav? - = nav_link(controller: ['groups/packages', 'groups/registry/repositories']) do + = nav_link(controller: ['groups/packages', 'groups/registry/repositories', 'groups/dependency_proxies']) do = link_to packages_link, title: _('Packages') do .nav-icon-container = sprite_icon('package') @@ -21,3 +21,7 @@ = nav_link(controller: 'groups/registry/repositories') do = link_to group_container_registries_path(@group), title: _('Container Registry') do %span= _('Container Registry') + - if group_dependency_proxy_nav? + = nav_link(controller: 'groups/dependency_proxies') do + = link_to group_dependency_proxy_path(@group), title: _('Dependency Proxy') do + %span= _('Dependency Proxy') diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 7b430d32b11..6f080a97f7a 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -451,6 +451,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: dependency_proxy:purge_dependency_proxy_cache + :feature_category: :dependency_proxy + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: deployment:deployments_drop_older_deployments :feature_category: :continuous_delivery :has_external_dependencies: diff --git a/app/workers/purge_dependency_proxy_cache_worker.rb b/app/workers/purge_dependency_proxy_cache_worker.rb new file mode 100644 index 00000000000..594cdd3ed11 --- /dev/null +++ b/app/workers/purge_dependency_proxy_cache_worker.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class PurgeDependencyProxyCacheWorker + include ApplicationWorker + include Gitlab::Allowable + idempotent! + + queue_namespace :dependency_proxy + feature_category :dependency_proxy + + def perform(current_user_id, group_id) + @current_user = User.find_by_id(current_user_id) + @group = Group.find_by_id(group_id) + + return unless valid? + + @group.dependency_proxy_blobs.destroy_all # rubocop:disable Cop/DestroyAll + end + + private + + def valid? + return unless @group + + can?(@current_user, :admin_group, @group) && @group.dependency_proxy_feature_available? + end +end |