diff options
140 files changed, 2085 insertions, 173 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 04af514c827..9f919fc9fd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ documentation](doc/development/changelog.md) for instructions on adding your own entry. +## 13.5.4 (2020-11-13) + +### Fixed (4 changes) + +- Fix Vue Labels Select dropdown keyboard scroll. !43874 +- Hashed Storage: make migration and rollback resilient to exceptions. !46178 +- Fix compliance framework database migration on CE instances. !46761 +- Resolve problem when namespace_settings were not created for groups created via admin panel. !46875 + + ## 13.5.3 (2020-11-03) ### Fixed (3 changes) diff --git a/app/assets/javascripts/blob/components/blob_header.vue b/app/assets/javascripts/blob/components/blob_header.vue index fd40c51fec1..a4a43b7a94e 100644 --- a/app/assets/javascripts/blob/components/blob_header.vue +++ b/app/assets/javascripts/blob/components/blob_header.vue @@ -66,7 +66,7 @@ export default { <template> <div class="js-file-title file-title-flex-parent"> <blob-filepath :blob="blob"> - <template #filepathPrepend> + <template #filepath-prepend> <slot name="prepend"></slot> </template> </blob-filepath> diff --git a/app/assets/javascripts/blob/components/blob_header_filepath.vue b/app/assets/javascripts/blob/components/blob_header_filepath.vue index f99ecba2324..eb8068a8ad7 100644 --- a/app/assets/javascripts/blob/components/blob_header_filepath.vue +++ b/app/assets/javascripts/blob/components/blob_header_filepath.vue @@ -26,7 +26,7 @@ export default { </script> <template> <div class="file-header-content d-flex align-items-center lh-100"> - <slot name="filepathPrepend"></slot> + <slot name="filepath-prepend"></slot> <template v-if="blob.path"> <file-icon :file-name="blob.path" :size="18" aria-hidden="true" css-classes="mr-2" /> diff --git a/app/assets/javascripts/clusters/components/application_row.vue b/app/assets/javascripts/clusters/components/application_row.vue index 412260da958..471c1a0b4a2 100644 --- a/app/assets/javascripts/clusters/components/application_row.vue +++ b/app/assets/javascripts/clusters/components/application_row.vue @@ -343,7 +343,7 @@ export default { > <span v-else class="js-cluster-application-title">{{ title }}</span> </strong> - <slot name="installedVia"></slot> + <slot name="installed-via"></slot> <div> <slot name="description"></slot> </div> diff --git a/app/assets/javascripts/clusters/components/applications.vue b/app/assets/javascripts/clusters/components/applications.vue index b03cf6fc31b..912568c8870 100644 --- a/app/assets/javascripts/clusters/components/applications.vue +++ b/app/assets/javascripts/clusters/components/applications.vue @@ -549,8 +549,8 @@ export default { @set="setKnativeDomain" /> </template> - <template v-if="cloudRun" #installedVia> - <span data-testid="installedVia"> + <template v-if="cloudRun" #installed-via> + <span data-testid="installed-via"> <gl-sprintf :message="s__('ClusterIntegration|installed via %{linkStart}Cloud Run%{linkEnd}')" > diff --git a/app/assets/javascripts/dependency_proxy.js b/app/assets/javascripts/dependency_proxy.js new file mode 100644 index 00000000000..ddf5703b28f --- /dev/null +++ b/app/assets/javascripts/dependency_proxy.js @@ -0,0 +1,5 @@ +import setupToggleButtons from '~/toggle_buttons'; + +export default () => { + setupToggleButtons(document.querySelector('.js-dependency-proxy-toggle-area')); +}; diff --git a/app/assets/javascripts/design_management/components/design_notes/design_discussion.vue b/app/assets/javascripts/design_management/components/design_notes/design_discussion.vue index 845f1aec8cf..6aab4bf423e 100644 --- a/app/assets/javascripts/design_management/components/design_notes/design_discussion.vue +++ b/app/assets/javascripts/design_management/components/design_notes/design_discussion.vue @@ -210,7 +210,7 @@ export default { :class="{ 'gl-bg-blue-50': isDiscussionActive }" @error="$emit('update-note-error', $event)" > - <template v-if="discussion.resolvable" #resolveDiscussion> + <template v-if="discussion.resolvable" #resolve-discussion> <button v-gl-tooltip :class="{ 'is-active': discussion.resolved }" @@ -224,7 +224,7 @@ export default { <gl-loading-icon v-else inline /> </button> </template> - <template v-if="discussion.resolved" #resolvedStatus> + <template v-if="discussion.resolved" #resolved-status> <p class="gl-text-gray-500 gl-font-sm gl-m-0 gl-mt-5" data-testid="resolved-message"> {{ __('Resolved by') }} <gl-link @@ -277,7 +277,7 @@ export default { @submit-form="mutate" @cancel-form="hideForm" > - <template v-if="discussion.resolvable" #resolveCheckbox> + <template v-if="discussion.resolvable" #resolve-checkbox> <label data-testid="resolve-checkbox"> <input v-model="shouldChangeResolvedStatus" type="checkbox" /> {{ resolveCheckboxText }} diff --git a/app/assets/javascripts/design_management/components/design_notes/design_note.vue b/app/assets/javascripts/design_management/components/design_notes/design_note.vue index 7f4b3b31024..421a4dc274a 100644 --- a/app/assets/javascripts/design_management/components/design_notes/design_note.vue +++ b/app/assets/javascripts/design_management/components/design_notes/design_note.vue @@ -108,7 +108,7 @@ export default { </span> </div> <div class="gl-display-flex gl-align-items-baseline"> - <slot name="resolveDiscussion"></slot> + <slot name="resolve-discussion"></slot> <button v-if="isEditButtonVisible" v-gl-tooltip @@ -127,7 +127,7 @@ export default { class="note-text js-note-text md" data-qa-selector="note_content" ></div> - <slot name="resolvedStatus"></slot> + <slot name="resolved-status"></slot> </template> <apollo-mutation v-else diff --git a/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue b/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue index 3754e1dbbc1..7aaac58a1ce 100644 --- a/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue +++ b/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue @@ -110,7 +110,7 @@ export default { </textarea> </template> </markdown-field> - <slot name="resolveCheckbox"></slot> + <slot name="resolve-checkbox"></slot> <div class="note-form-actions gl-display-flex gl-justify-content-space-between"> <gl-button ref="submitButton" diff --git a/app/assets/javascripts/design_management/components/design_sidebar.vue b/app/assets/javascripts/design_management/components/design_sidebar.vue index fb8e74c8c4c..41dcec38abe 100644 --- a/app/assets/javascripts/design_management/components/design_sidebar.vue +++ b/app/assets/javascripts/design_management/components/design_sidebar.vue @@ -207,6 +207,6 @@ export default { /> </gl-collapse> </template> - <slot name="replyForm"></slot> + <slot name="reply-form"></slot> </div> </template> diff --git a/app/assets/javascripts/design_management/pages/design/index.vue b/app/assets/javascripts/design_management/pages/design/index.vue index d1bbb5239d0..e07279ba39d 100644 --- a/app/assets/javascripts/design_management/pages/design/index.vue +++ b/app/assets/javascripts/design_management/pages/design/index.vue @@ -383,7 +383,7 @@ export default { @toggleResolvedComments="toggleResolvedComments" @todoError="onTodoError" > - <template #replyForm> + <template #reply-form> <apollo-mutation v-if="isAnnotating" #default="{ mutate, loading }" diff --git a/app/assets/javascripts/environments/components/container.vue b/app/assets/javascripts/environments/components/container.vue index 4c6d233c4d2..e7697f14802 100644 --- a/app/assets/javascripts/environments/components/container.vue +++ b/app/assets/javascripts/environments/components/container.vue @@ -69,7 +69,7 @@ export default { <div class="environments-container"> <gl-loading-icon v-if="isLoading" size="md" class="gl-mt-3" label="Loading environments" /> - <slot name="emptyState"></slot> + <slot name="empty-state"></slot> <div v-if="!isLoading && environments.length > 0" class="table-holder"> <environment-table diff --git a/app/assets/javascripts/environments/components/environments_app.vue b/app/assets/javascripts/environments/components/environments_app.vue index 9bafc7ed153..c1b9ba755a6 100644 --- a/app/assets/javascripts/environments/components/environments_app.vue +++ b/app/assets/javascripts/environments/components/environments_app.vue @@ -228,7 +228,7 @@ export default { :deploy-boards-help-path="deployBoardsHelpPath" @onChangePage="onChangePage" > - <template v-if="!isLoading && state.environments.length === 0" #emptyState> + <template v-if="!isLoading && state.environments.length === 0" #empty-state> <empty-state :help-path="helpPagePath" /> </template> </container> diff --git a/app/assets/javascripts/import_projects/store/actions.js b/app/assets/javascripts/import_projects/store/actions.js index b31b1224344..7b7afd13c55 100644 --- a/app/assets/javascripts/import_projects/store/actions.js +++ b/app/assets/javascripts/import_projects/store/actions.js @@ -7,11 +7,14 @@ import { visitUrl, objectToQuery } from '~/lib/utils/url_utility'; import { deprecatedCreateFlash as createFlash } from '~/flash'; import { s__, sprintf } from '~/locale'; import axios from '~/lib/utils/axios_utils'; +import httpStatusCodes from '~/lib/utils/http_status'; +import { capitalizeFirstCharacter } from '~/lib/utils/text_utility'; let eTagPoll; const hasRedirectInError = e => e?.response?.data?.error?.redirect; const redirectToUrlInError = e => visitUrl(e.response.data.error.redirect); +const tooManyRequests = e => e.response.status === httpStatusCodes.TOO_MANY_REQUESTS; const pathWithParams = ({ path, ...params }) => { const filteredParams = Object.fromEntries( Object.entries(params).filter(([, value]) => value !== ''), @@ -71,6 +74,14 @@ const fetchReposFactory = ({ reposPath = isRequired() }) => ({ state, commit }) if (hasRedirectInError(e)) { redirectToUrlInError(e); + } else if (tooManyRequests(e)) { + createFlash( + sprintf(s__('ImportProjects|%{provider} rate limit exceeded. Try again later'), { + provider: capitalizeFirstCharacter(provider), + }), + ); + + commit(types.RECEIVE_REPOS_ERROR); } else { createFlash( sprintf(s__('ImportProjects|Requesting your %{provider} repositories failed'), { diff --git a/app/assets/javascripts/incidents/components/incidents_list.vue b/app/assets/javascripts/incidents/components/incidents_list.vue index 6f9b05c08ab..0e3839deaf5 100644 --- a/app/assets/javascripts/incidents/components/incidents_list.vue +++ b/app/assets/javascripts/incidents/components/incidents_list.vue @@ -419,7 +419,7 @@ export default { </template> </gl-table> </template> - <template #emtpy-state> + <template #empty-state> <gl-empty-state :title="emptyStateData.title" :svg-path="emptyListSvgPath" diff --git a/app/assets/javascripts/lib/utils/http_status.js b/app/assets/javascripts/lib/utils/http_status.js index 7132986a7e6..06529f06a66 100644 --- a/app/assets/javascripts/lib/utils/http_status.js +++ b/app/assets/javascripts/lib/utils/http_status.js @@ -22,6 +22,7 @@ const httpStatusCodes = { CONFLICT: 409, GONE: 410, UNPROCESSABLE_ENTITY: 422, + TOO_MANY_REQUESTS: 429, INTERNAL_SERVER_ERROR: 500, SERVICE_UNAVAILABLE: 503, }; diff --git a/app/assets/javascripts/monitoring/components/dashboard.vue b/app/assets/javascripts/monitoring/components/dashboard.vue index cbfacd73b5b..16c2c87a4b7 100644 --- a/app/assets/javascripts/monitoring/components/dashboard.vue +++ b/app/assets/javascripts/monitoring/components/dashboard.vue @@ -423,7 +423,7 @@ export default { :prometheus-alerts-available="prometheusAlertsAvailable" @timerangezoom="onTimeRangeZoom" > - <template #topLeft> + <template #top-left> <gl-button ref="goBackBtn" v-gl-tooltip diff --git a/app/assets/javascripts/monitoring/components/dashboard_panel.vue b/app/assets/javascripts/monitoring/components/dashboard_panel.vue index 18310f7c71e..597600bba07 100644 --- a/app/assets/javascripts/monitoring/components/dashboard_panel.vue +++ b/app/assets/javascripts/monitoring/components/dashboard_panel.vue @@ -365,7 +365,7 @@ export default { <template> <div v-gl-resize-observer="onResize" class="prometheus-graph"> <div class="d-flex align-items-center"> - <slot name="topLeft"></slot> + <slot name="top-left"></slot> <h5 ref="graphTitle" class="prometheus-graph-title gl-font-lg font-weight-bold text-truncate gl-mr-3" diff --git a/app/assets/javascripts/pages/groups/dependency_proxies/index.js b/app/assets/javascripts/pages/groups/dependency_proxies/index.js new file mode 100644 index 00000000000..4c0a1abb0f0 --- /dev/null +++ b/app/assets/javascripts/pages/groups/dependency_proxies/index.js @@ -0,0 +1,17 @@ +import $ from 'jquery'; +import initDependencyProxy from '~/dependency_proxy'; + +document.addEventListener('DOMContentLoaded', () => { + initDependencyProxy(); +}); + +document.addEventListener('DOMContentLoaded', () => { + const form = document.querySelector('form.edit_dependency_proxy_group_setting'); + const toggleInput = $('input.js-project-feature-toggle-input'); + + if (form && toggleInput) { + toggleInput.on('trigger-change', () => { + form.submit(); + }); + } +}); diff --git a/app/assets/javascripts/related_issues/components/related_issues_block.vue b/app/assets/javascripts/related_issues/components/related_issues_block.vue index f7a79c62716..c913745a8e1 100644 --- a/app/assets/javascripts/related_issues/components/related_issues_block.vue +++ b/app/assets/javascripts/related_issues/components/related_issues_block.vue @@ -138,7 +138,7 @@ export default { href="#related-issues" aria-hidden="true" /> - <slot name="headerText">{{ __('Linked issues') }}</slot> + <slot name="header-text">{{ __('Linked issues') }}</slot> <gl-link v-if="hasHelpPath" :href="helpPath" @@ -167,7 +167,7 @@ export default { /> </div> </h3> - <slot name="headerActions"></slot> + <slot name="header-actions"></slot> </div> <div class="linked-issues-card-body bg-gray-light" diff --git a/app/assets/javascripts/reports/components/grouped_test_reports_app.vue b/app/assets/javascripts/reports/components/grouped_test_reports_app.vue index d977ec37126..c13df60198b 100644 --- a/app/assets/javascripts/reports/components/grouped_test_reports_app.vue +++ b/app/assets/javascripts/reports/components/grouped_test_reports_app.vue @@ -134,7 +134,7 @@ export default { class="mr-widget-section grouped-security-reports mr-report" @toggleEvent="handleToggleEvent" > - <template v-if="showViewFullReport" #actionButtons> + <template v-if="showViewFullReport" #action-buttons> <gl-button :href="testTabURL" target="_blank" @@ -145,7 +145,7 @@ export default { {{ s__('ciReport|View full report') }} </gl-button> </template> - <template v-if="hasRecentFailures(summary)" #subHeading> + <template v-if="hasRecentFailures(summary)" #sub-heading> {{ recentFailuresText(summary) }} </template> <template #body> diff --git a/app/assets/javascripts/reports/components/report_section.vue b/app/assets/javascripts/reports/components/report_section.vue index cf5c0ceadfe..f245e2bfd2f 100644 --- a/app/assets/javascripts/reports/components/report_section.vue +++ b/app/assets/javascripts/reports/components/report_section.vue @@ -181,10 +181,10 @@ export default { <slot :name="slotName"></slot> <popover v-if="hasPopover" :options="popoverOptions" class="gl-ml-2" /> </div> - <slot name="subHeading"></slot> + <slot name="sub-heading"></slot> </div> - <slot name="actionButtons"></slot> + <slot name="action-buttons"></slot> <button v-if="isCollapsible" diff --git a/app/assets/javascripts/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs.vue b/app/assets/javascripts/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs.vue index 8e85d93e6d1..1fc39c7cb8e 100644 --- a/app/assets/javascripts/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs.vue +++ b/app/assets/javascripts/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs.vue @@ -308,6 +308,6 @@ export default { @input="handlePageChange" /> - <slot v-if="!showItems" name="emtpy-state"></slot> + <slot v-if="!showItems" name="empty-state"></slot> </div> </template> diff --git a/app/assets/javascripts/vue_shared/components/upload_dropzone/upload_dropzone.vue b/app/assets/javascripts/vue_shared/components/upload_dropzone/upload_dropzone.vue index 25d73ed0855..b645758d891 100644 --- a/app/assets/javascripts/vue_shared/components/upload_dropzone/upload_dropzone.vue +++ b/app/assets/javascripts/vue_shared/components/upload_dropzone/upload_dropzone.vue @@ -147,7 +147,7 @@ export default { class="card upload-dropzone-border upload-dropzone-overlay gl-w-full gl-h-full gl-absolute gl-display-flex gl-align-items-center gl-justify-content-center gl-p-3 gl-bg-white" > <div v-show="!isDragDataValid" class="mw-50 gl-text-center"> - <slot name="invalidDragDataSlot"> + <slot name="invalid-drag-data-slot"> <h3 :class="{ 'gl-font-base gl-display-inline': !displayAsCard }"> {{ __('Oh no!') }} </h3> @@ -159,7 +159,7 @@ export default { </slot> </div> <div v-show="isDragDataValid" class="mw-50 gl-text-center"> - <slot name="validDragDataSlot"> + <slot name="valid-drag-data-slot"> <h3 :class="{ 'gl-font-base gl-display-inline': !displayAsCard }"> {{ __('Incoming!') }} </h3> diff --git a/app/controllers/concerns/dependency_proxy_access.rb b/app/controllers/concerns/dependency_proxy_access.rb new file mode 100644 index 00000000000..5036d0cfce4 --- /dev/null +++ b/app/controllers/concerns/dependency_proxy_access.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module DependencyProxyAccess + extend ActiveSupport::Concern + + included do + before_action :verify_dependency_proxy_enabled! + before_action :authorize_read_dependency_proxy! + end + + private + + def verify_dependency_proxy_enabled! + render_404 unless group.dependency_proxy_feature_available? + end + + def authorize_read_dependency_proxy! + access_denied! unless can?(current_user, :read_dependency_proxy, group) + end + + def authorize_admin_dependency_proxy! + access_denied! unless can?(current_user, :admin_dependency_proxy, group) + end +end diff --git a/app/controllers/groups/dependency_proxies_controller.rb b/app/controllers/groups/dependency_proxies_controller.rb new file mode 100644 index 00000000000..367dbafdd59 --- /dev/null +++ b/app/controllers/groups/dependency_proxies_controller.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Groups + class DependencyProxiesController < Groups::ApplicationController + include DependencyProxyAccess + + before_action :authorize_admin_dependency_proxy!, only: :update + before_action :dependency_proxy + + feature_category :package_registry + + def show + @blobs_count = group.dependency_proxy_blobs.count + @blobs_total_size = group.dependency_proxy_blobs.total_size + end + + def update + dependency_proxy.update(dependency_proxy_params) + + redirect_to group_dependency_proxy_path(group) + end + + private + + def dependency_proxy + @dependency_proxy ||= + group.dependency_proxy_setting || group.create_dependency_proxy_setting + end + + def dependency_proxy_params + params.require(:dependency_proxy_group_setting).permit(:enabled) + end + end +end diff --git a/app/controllers/groups/dependency_proxy_for_containers_controller.rb b/app/controllers/groups/dependency_proxy_for_containers_controller.rb new file mode 100644 index 00000000000..f46902ef90f --- /dev/null +++ b/app/controllers/groups/dependency_proxy_for_containers_controller.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +class Groups::DependencyProxyForContainersController < Groups::ApplicationController + include DependencyProxyAccess + include SendFileUpload + + before_action :ensure_token_granted! + before_action :ensure_feature_enabled! + + attr_reader :token + + feature_category :package_registry + + def manifest + result = DependencyProxy::PullManifestService.new(image, tag, token).execute + + if result[:status] == :success + render json: result[:manifest] + else + render status: result[:http_status], json: result[:message] + end + end + + def blob + result = DependencyProxy::FindOrCreateBlobService + .new(group, image, token, params[:sha]).execute + + if result[:status] == :success + send_upload(result[:blob].file) + else + head result[:http_status] + end + end + + private + + def image + params[:image] + end + + def tag + params[:tag] + end + + def dependency_proxy + @dependency_proxy ||= + group.dependency_proxy_setting || group.create_dependency_proxy_setting + end + + def ensure_feature_enabled! + render_404 unless dependency_proxy.enabled + end + + def ensure_token_granted! + result = DependencyProxy::RequestTokenService.new(image).execute + + if result[:status] == :success + @token = result[:token] + else + render status: result[:http_status], json: result[:message] + end + end +end diff --git a/app/controllers/import/github_controller.rb b/app/controllers/import/github_controller.rb index 03f8020ee7b..8ac93aeb9c0 100644 --- a/app/controllers/import/github_controller.rb +++ b/app/controllers/import/github_controller.rb @@ -15,6 +15,7 @@ class Import::GithubController < Import::BaseController rescue_from OAuthConfigMissingError, with: :missing_oauth_config rescue_from Octokit::Unauthorized, with: :provider_unauthorized rescue_from Octokit::TooManyRequests, with: :provider_rate_limit + rescue_from Gitlab::GithubImport::RateLimitError, with: :rate_limit_threshold_exceeded def new if !ci_cd_only? && github_import_configured? && logged_in_with_provider? @@ -114,7 +115,7 @@ class Import::GithubController < Import::BaseController def client_repos @client_repos ||= if Feature.enabled?(:remove_legacy_github_client) - filtered(concatenated_repos) + concatenated_repos else filtered(client.repos) end @@ -122,8 +123,15 @@ class Import::GithubController < Import::BaseController def concatenated_repos return [] unless client.respond_to?(:each_page) + return client.each_page(:repos).flat_map(&:objects) unless sanitized_filter_param - client.each_page(:repos).flat_map(&:objects) + client.search_repos_by_name(sanitized_filter_param).flat_map(&:objects).flat_map(&:items) + end + + def sanitized_filter_param + super + + @filter = @filter&.tr(' ', '')&.tr(':', '') end def oauth_client @@ -245,6 +253,10 @@ class Import::GithubController < Import::BaseController def extra_import_params {} end + + def rate_limit_threshold_exceeded + head :too_many_requests + end end Import::GithubController.prepend_if_ee('EE::Import::GithubController') diff --git a/app/helpers/groups_helper.rb b/app/helpers/groups_helper.rb index d287d430bc4..29ead76a607 100644 --- a/app/helpers/groups_helper.rb +++ b/app/helpers/groups_helper.rb @@ -170,6 +170,10 @@ module GroupsHelper group_container_registry_nav? end + def group_dependency_proxy_nav? + @group.dependency_proxy_feature_available? + end + def group_packages_list_nav? @group.packages_feature_enabled? end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index a009d7c8062..84abd01786d 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -103,6 +103,10 @@ module Ci ) end + scope :in_pipelines, ->(pipelines) do + where(pipeline: pipelines) + end + scope :with_existing_job_artifacts, ->(query) do where('EXISTS (?)', ::Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').merge(query)) end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 3b8c86f2db9..8707d635e03 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -355,6 +355,14 @@ module Ci end end + def self.latest_running_for_ref(ref) + newest_first(ref: ref).running.take + end + + def self.latest_failed_for_ref(ref) + newest_first(ref: ref).failed.take + end + # Returns a Hash containing the latest pipeline for every given # commit. # diff --git a/app/models/dependency_proxy.rb b/app/models/dependency_proxy.rb new file mode 100644 index 00000000000..510a304ff17 --- /dev/null +++ b/app/models/dependency_proxy.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true +module DependencyProxy + def self.table_name_prefix + 'dependency_proxy_' + end +end diff --git a/app/models/dependency_proxy/blob.rb b/app/models/dependency_proxy/blob.rb new file mode 100644 index 00000000000..3a81112340a --- /dev/null +++ b/app/models/dependency_proxy/blob.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class DependencyProxy::Blob < ApplicationRecord + include FileStoreMounter + + belongs_to :group + + validates :group, presence: true + validates :file, presence: true + validates :file_name, presence: true + + mount_file_store_uploader DependencyProxy::FileUploader + + def self.total_size + sum(:size) + end + + def self.find_or_build(file_name) + find_or_initialize_by(file_name: file_name) + end +end diff --git a/app/models/dependency_proxy/group_setting.rb b/app/models/dependency_proxy/group_setting.rb new file mode 100644 index 00000000000..bcf09b27129 --- /dev/null +++ b/app/models/dependency_proxy/group_setting.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class DependencyProxy::GroupSetting < ApplicationRecord + belongs_to :group + + validates :group, presence: true + + default_value_for :enabled, true +end diff --git a/app/models/dependency_proxy/registry.rb b/app/models/dependency_proxy/registry.rb new file mode 100644 index 00000000000..471d5be2600 --- /dev/null +++ b/app/models/dependency_proxy/registry.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class DependencyProxy::Registry + AUTH_URL = 'https://auth.docker.io'.freeze + LIBRARY_URL = 'https://registry-1.docker.io/v2'.freeze + + class << self + def auth_url(image) + "#{AUTH_URL}/token?service=registry.docker.io&scope=repository:#{image_path(image)}:pull" + end + + def manifest_url(image, tag) + "#{LIBRARY_URL}/#{image_path(image)}/manifests/#{tag}" + end + + def blob_url(image, blob_sha) + "#{LIBRARY_URL}/#{image_path(image)}/blobs/#{blob_sha}" + end + + private + + def image_path(image) + if image.include?('/') + image + else + "library/#{image}" + end + end + end +end diff --git a/app/models/group.rb b/app/models/group.rb index 7b701ad41bd..3509299a579 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -71,6 +71,9 @@ class Group < Namespace has_many :group_deploy_tokens has_many :deploy_tokens, through: :group_deploy_tokens + has_one :dependency_proxy_setting, class_name: 'DependencyProxy::GroupSetting' + has_many :dependency_proxy_blobs, class_name: 'DependencyProxy::Blob' + accepts_nested_attributes_for :variables, allow_destroy: true validate :visibility_level_allowed_by_projects @@ -203,6 +206,10 @@ class Group < Namespace ::Gitlab.config.packages.enabled end + def dependency_proxy_feature_available? + ::Gitlab.config.dependency_proxy.enabled + end + def notification_email_for(user) # Finds the closest notification_setting with a `notification_email` notification_settings = notification_settings_for(user, hierarchy_order: :asc) diff --git a/app/policies/group_policy.rb b/app/policies/group_policy.rb index 92a5ced4fc3..6ff6d1359ab 100644 --- a/app/policies/group_policy.rb +++ b/app/policies/group_policy.rb @@ -46,6 +46,10 @@ class GroupPolicy < BasePolicy group_projects_for(user: @user, group: @subject, only_owned: false).any? { |p| p.design_management_enabled? } end + condition(:dependency_proxy_available) do + @subject.dependency_proxy_feature_available? + end + desc "Deploy token with read_package_registry scope" condition(:read_package_registry_deploy_token) do @user.is_a?(DeployToken) && @user.groups.include?(@subject) && @user.read_package_registry @@ -193,6 +197,12 @@ class GroupPolicy < BasePolicy enable :read_group end + rule { can?(:read_group) & dependency_proxy_available } + .enable :read_dependency_proxy + + rule { developer & dependency_proxy_available } + .enable :admin_dependency_proxy + rule { resource_access_token_available & can?(:admin_group) }.policy do enable :admin_resource_access_tokens end diff --git a/app/services/dependency_proxy/base_service.rb b/app/services/dependency_proxy/base_service.rb new file mode 100644 index 00000000000..1b2d4b14a27 --- /dev/null +++ b/app/services/dependency_proxy/base_service.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module DependencyProxy + class BaseService < ::BaseService + private + + def registry + DependencyProxy::Registry + end + + def auth_headers + { + Authorization: "Bearer #{@token}" + } + end + end +end diff --git a/app/services/dependency_proxy/download_blob_service.rb b/app/services/dependency_proxy/download_blob_service.rb new file mode 100644 index 00000000000..3c690683bf6 --- /dev/null +++ b/app/services/dependency_proxy/download_blob_service.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module DependencyProxy + class DownloadBlobService < DependencyProxy::BaseService + class DownloadError < StandardError + attr_reader :http_status + + def initialize(message, http_status) + @http_status = http_status + + super(message) + end + end + + def initialize(image, blob_sha, token) + @image = image + @blob_sha = blob_sha + @token = token + @temp_file = Tempfile.new + end + + def execute + File.open(@temp_file.path, "wb") do |file| + Gitlab::HTTP.get(blob_url, headers: auth_headers, stream_body: true) do |fragment| + if [301, 302, 307].include?(fragment.code) + # do nothing + elsif fragment.code == 200 + file.write(fragment) + else + raise DownloadError.new('Non-success response code on downloading blob fragment', fragment.code) + end + end + end + + success(file: @temp_file) + rescue DownloadError => exception + error(exception.message, exception.http_status) + rescue Timeout::Error => exception + error(exception.message, 599) + end + + private + + def blob_url + registry.blob_url(@image, @blob_sha) + end + end +end diff --git a/app/services/dependency_proxy/find_or_create_blob_service.rb b/app/services/dependency_proxy/find_or_create_blob_service.rb new file mode 100644 index 00000000000..bd06f9d7628 --- /dev/null +++ b/app/services/dependency_proxy/find_or_create_blob_service.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module DependencyProxy + class FindOrCreateBlobService < DependencyProxy::BaseService + def initialize(group, image, token, blob_sha) + @group = group + @image = image + @token = token + @blob_sha = blob_sha + end + + def execute + file_name = @blob_sha.sub('sha256:', '') + '.gz' + blob = @group.dependency_proxy_blobs.find_or_build(file_name) + + unless blob.persisted? + result = DependencyProxy::DownloadBlobService + .new(@image, @blob_sha, @token).execute + + if result[:status] == :error + log_failure(result) + + return error('Failed to download the blob', result[:http_status]) + end + + blob.file = result[:file] + blob.size = result[:file].size + blob.save! + end + + success(blob: blob) + end + + private + + def log_failure(result) + log_error( + "Dependency proxy: Failed to download the blob." \ + "Blob sha: #{@blob_sha}." \ + "Error message: #{result[:message][0, 100]}" \ + "HTTP status: #{result[:http_status]}" + ) + end + end +end diff --git a/app/services/dependency_proxy/pull_manifest_service.rb b/app/services/dependency_proxy/pull_manifest_service.rb new file mode 100644 index 00000000000..fc54ef85c96 --- /dev/null +++ b/app/services/dependency_proxy/pull_manifest_service.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module DependencyProxy + class PullManifestService < DependencyProxy::BaseService + def initialize(image, tag, token) + @image = image + @tag = tag + @token = token + end + + def execute + response = Gitlab::HTTP.get(manifest_url, headers: auth_headers) + + if response.success? + success(manifest: response.body) + else + error(response.body, response.code) + end + rescue Timeout::Error => exception + error(exception.message, 599) + end + + private + + def manifest_url + registry.manifest_url(@image, @tag) + end + end +end diff --git a/app/services/dependency_proxy/request_token_service.rb b/app/services/dependency_proxy/request_token_service.rb new file mode 100644 index 00000000000..4ca7239b9f6 --- /dev/null +++ b/app/services/dependency_proxy/request_token_service.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module DependencyProxy + class RequestTokenService < DependencyProxy::BaseService + def initialize(image) + @image = image + end + + def execute + response = Gitlab::HTTP.get(auth_url) + + if response.success? + success(token: Gitlab::Json.parse(response.body)['token']) + else + error('Expected 200 response code for an access token', response.code) + end + rescue Timeout::Error => exception + error(exception.message, 599) + rescue JSON::ParserError + error('Failed to parse a response body for an access token', 500) + end + + private + + def auth_url + registry.auth_url(@image) + end + end +end diff --git a/app/uploaders/dependency_proxy/file_uploader.rb b/app/uploaders/dependency_proxy/file_uploader.rb new file mode 100644 index 00000000000..b67a22bae4d --- /dev/null +++ b/app/uploaders/dependency_proxy/file_uploader.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class DependencyProxy::FileUploader < GitlabUploader + include ObjectStorage::Concern + + storage_options Gitlab.config.dependency_proxy + + alias_method :upload, :model + + def filename + model.file_name + end + + def store_dir + dynamic_segment + end + + private + + def dynamic_segment + Gitlab::HashedPath.new('dependency_proxy', model.group_id, 'files', model.id, root_hash: model.group_id) + end +end diff --git a/app/views/groups/dependency_proxies/_url.html.haml b/app/views/groups/dependency_proxies/_url.html.haml new file mode 100644 index 00000000000..9242954b684 --- /dev/null +++ b/app/views/groups/dependency_proxies/_url.html.haml @@ -0,0 +1,12 @@ +- proxy_url = "#{group_url(@group)}/dependency_proxy/containers" + +%h5.prepend-top-20= _('Dependency proxy URL') + +.row + .col-lg-8.col-md-12.input-group + = text_field_tag :url, "#{proxy_url}", class: 'js-dependency-proxy-url form-control', readonly: true + = clipboard_button(text: "#{proxy_url}", title: _("Copy %{proxy_url}") % { proxy_url: proxy_url }) + +.row + .col-12.help-block.gl-mt-3 + = _('Contains %{count} blobs of images (%{size})') % { count: @blobs_count, size: number_to_human_size(@blobs_total_size) } diff --git a/app/views/groups/dependency_proxies/show.html.haml b/app/views/groups/dependency_proxies/show.html.haml new file mode 100644 index 00000000000..ff1312eb763 --- /dev/null +++ b/app/views/groups/dependency_proxies/show.html.haml @@ -0,0 +1,28 @@ +- page_title _("Dependency Proxy") + +.settings-header + %h4= _('Dependency proxy') + + %p + - link_start = '<a href="%{url}">'.html_safe % { url: help_page_path('user/packages/dependency_proxy/index') } + = _('Create a local proxy for storing frequently used upstream images. %{link_start}Learn more%{link_end} about dependency proxies.').html_safe % { link_start: link_start, link_end: '</a>'.html_safe } + +- if @group.public? + - if can?(current_user, :admin_dependency_proxy, @group) + = form_for(@dependency_proxy, method: :put, url: group_dependency_proxy_path(@group)) do |f| + .form-group + %h5.prepend-top-20= _('Enable proxy') + .js-dependency-proxy-toggle-area + = render "shared/buttons/project_feature_toggle", is_checked: @dependency_proxy.enabled?, label: s_("DependencyProxy|Toggle Dependency Proxy") do + = f.hidden_field :enabled, { class: 'js-project-feature-toggle-input'} + + - if @dependency_proxy.enabled + = render 'groups/dependency_proxies/url' + + - else + - if @dependency_proxy.enabled + = render 'groups/dependency_proxies/url' +- else + .gl-alert.gl-alert-info + = sprite_icon('information-o', size: 16, css_class: 'gl-icon gl-alert-icon gl-alert-icon-no-title') + = _('Dependency proxy feature is limited to public groups for now.') diff --git a/app/views/groups/sidebar/_packages.html.haml b/app/views/groups/sidebar/_packages.html.haml index 54510d5df0c..7e0ee032aeb 100644 --- a/app/views/groups/sidebar/_packages.html.haml +++ b/app/views/groups/sidebar/_packages.html.haml @@ -1,7 +1,7 @@ - packages_link = group_packages_list_nav? ? group_packages_path(@group) : group_container_registries_path(@group) - if group_packages_nav? - = nav_link(controller: ['groups/packages', 'groups/registry/repositories']) do + = nav_link(controller: ['groups/packages', 'groups/registry/repositories', 'groups/dependency_proxies']) do = link_to packages_link, title: _('Packages') do .nav-icon-container = sprite_icon('package') @@ -21,3 +21,7 @@ = nav_link(controller: 'groups/registry/repositories') do = link_to group_container_registries_path(@group), title: _('Container Registry') do %span= _('Container Registry') + - if group_dependency_proxy_nav? + = nav_link(controller: 'groups/dependency_proxies') do + = link_to group_dependency_proxy_path(@group), title: _('Dependency Proxy') do + %span= _('Dependency Proxy') diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 7b430d32b11..6f080a97f7a 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -451,6 +451,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: dependency_proxy:purge_dependency_proxy_cache + :feature_category: :dependency_proxy + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: deployment:deployments_drop_older_deployments :feature_category: :continuous_delivery :has_external_dependencies: diff --git a/app/workers/purge_dependency_proxy_cache_worker.rb b/app/workers/purge_dependency_proxy_cache_worker.rb new file mode 100644 index 00000000000..594cdd3ed11 --- /dev/null +++ b/app/workers/purge_dependency_proxy_cache_worker.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class PurgeDependencyProxyCacheWorker + include ApplicationWorker + include Gitlab::Allowable + idempotent! + + queue_namespace :dependency_proxy + feature_category :dependency_proxy + + def perform(current_user_id, group_id) + @current_user = User.find_by_id(current_user_id) + @group = Group.find_by_id(group_id) + + return unless valid? + + @group.dependency_proxy_blobs.destroy_all # rubocop:disable Cop/DestroyAll + end + + private + + def valid? + return unless @group + + can?(@current_user, :admin_group, @group) && @group.dependency_proxy_feature_available? + end +end diff --git a/changelogs/unreleased/254997-fix-labels-search-scroll.yml b/changelogs/unreleased/254997-fix-labels-search-scroll.yml deleted file mode 100644 index da5a024393b..00000000000 --- a/changelogs/unreleased/254997-fix-labels-search-scroll.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix Vue Labels Select dropdown keyboard scroll -merge_request: 43874 -author: -type: fixed diff --git a/changelogs/unreleased/259605-repositories-gone-after-update-broken-migration-to-hashed-storage.yml b/changelogs/unreleased/259605-repositories-gone-after-update-broken-migration-to-hashed-storage.yml deleted file mode 100644 index d1052e678f0..00000000000 --- a/changelogs/unreleased/259605-repositories-gone-after-update-broken-migration-to-hashed-storage.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: "Hashed Storage: make migration and rollback resilient to exceptions" -merge_request: 46178 -author: -type: fixed diff --git a/changelogs/unreleased/273380-undefined-method-allow_mfa_for_subgroups-for-nil-nilclass-after-up.yml b/changelogs/unreleased/273380-undefined-method-allow_mfa_for_subgroups-for-nil-nilclass-after-up.yml deleted file mode 100644 index 9a6fd31ad7e..00000000000 --- a/changelogs/unreleased/273380-undefined-method-allow_mfa_for_subgroups-for-nil-nilclass-after-up.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: Resolve problem when namespace_settings were not created for groups created - via admin panel -merge_request: 46875 -author: -type: fixed diff --git a/changelogs/unreleased/273655-dependency-proxy-to-core.yml b/changelogs/unreleased/273655-dependency-proxy-to-core.yml new file mode 100644 index 00000000000..ef29e62e344 --- /dev/null +++ b/changelogs/unreleased/273655-dependency-proxy-to-core.yml @@ -0,0 +1,5 @@ +--- +title: Dependency proxy feature is moved to GitLab core +merge_request: 47471 +author: +type: changed diff --git a/changelogs/unreleased/fix-ee-compliance-inline-data-migration-on-ce.yml b/changelogs/unreleased/fix-ee-compliance-inline-data-migration-on-ce.yml deleted file mode 100644 index 2ed186df2b8..00000000000 --- a/changelogs/unreleased/fix-ee-compliance-inline-data-migration-on-ce.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix compliance framework database migration on CE instances -merge_request: 46761 -author: -type: fixed diff --git a/changelogs/unreleased/georgekoltsov-use-github-api-for-filtering.yml b/changelogs/unreleased/georgekoltsov-use-github-api-for-filtering.yml new file mode 100644 index 00000000000..6145846c6fb --- /dev/null +++ b/changelogs/unreleased/georgekoltsov-use-github-api-for-filtering.yml @@ -0,0 +1,5 @@ +--- +title: Filter GitHub projects to import using GitHub Search API +merge_request: 47002 +author: +type: changed diff --git a/changelogs/unreleased/more-coverage-badges.yml b/changelogs/unreleased/more-coverage-badges.yml new file mode 100644 index 00000000000..1a7a212d204 --- /dev/null +++ b/changelogs/unreleased/more-coverage-badges.yml @@ -0,0 +1,5 @@ +--- +title: Expand scope of coverage badge query to all successful builds +merge_request: 45321 +author: +type: changed diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 9b8eef9f117..022f372a608 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -362,18 +362,16 @@ Settings.packages['object_store'] = ObjectStoreSettings.legacy_parse(Settings.p # # Dependency Proxy # -Gitlab.ee do - Settings['dependency_proxy'] ||= Settingslogic.new({}) - Settings.dependency_proxy['enabled'] = true if Settings.dependency_proxy['enabled'].nil? - Settings.dependency_proxy['storage_path'] = Settings.absolute(Settings.dependency_proxy['storage_path'] || File.join(Settings.shared['path'], "dependency_proxy")) - Settings.dependency_proxy['object_store'] = ObjectStoreSettings.legacy_parse(Settings.dependency_proxy['object_store']) - - # For first iteration dependency proxy uses Rails server to download blobs. - # To ensure acceptable performance we only allow feature to be used with - # multithreaded web-server Puma. This will be removed once download logic is moved - # to GitLab workhorse - Settings.dependency_proxy['enabled'] = false unless Gitlab::Runtime.puma? -end +Settings['dependency_proxy'] ||= Settingslogic.new({}) +Settings.dependency_proxy['enabled'] = true if Settings.dependency_proxy['enabled'].nil? +Settings.dependency_proxy['storage_path'] = Settings.absolute(Settings.dependency_proxy['storage_path'] || File.join(Settings.shared['path'], "dependency_proxy")) +Settings.dependency_proxy['object_store'] = ObjectStoreSettings.legacy_parse(Settings.dependency_proxy['object_store']) + +# For first iteration dependency proxy uses Rails server to download blobs. +# To ensure acceptable performance we only allow feature to be used with +# multithreaded web-server Puma. This will be removed once download logic is moved +# to GitLab workhorse +Settings.dependency_proxy['enabled'] = false unless Gitlab::Runtime.puma? # # Terraform state diff --git a/config/routes/group.rb b/config/routes/group.rb index 33464cf3b55..e90be482bbd 100644 --- a/config/routes/group.rb +++ b/config/routes/group.rb @@ -107,6 +107,7 @@ constraints(::Constraints::GroupUrlConstrainer.new) do end resources :container_registries, only: [:index, :show], controller: 'registry/repositories' + resource :dependency_proxy, only: [:show, :update] end scope(path: '*id', @@ -119,3 +120,14 @@ constraints(::Constraints::GroupUrlConstrainer.new) do delete '/', action: :destroy end end + +# Dependency proxy for containers +# Because docker adds v2 prefix to URI this need to be outside of usual group routes +scope format: false do + get 'v2', to: proc { [200, {}, ['']] } # rubocop:disable Cop/PutGroupRoutesUnderScope + + constraints image: Gitlab::PathRegex.container_image_regex, sha: Gitlab::PathRegex.container_image_blob_sha_regex do + get 'v2/*group_id/dependency_proxy/containers/*image/manifests/*tag' => 'groups/dependency_proxy_for_containers#manifest' # rubocop:todo Cop/PutGroupRoutesUnderScope + get 'v2/*group_id/dependency_proxy/containers/*image/blobs/:sha' => 'groups/dependency_proxy_for_containers#blob' # rubocop:todo Cop/PutGroupRoutesUnderScope + end +end diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index 6d65d402af8..bbb5c892298 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -232,6 +232,7 @@ Laravel LDAP ldapsearch Leiningen +Lefthook Libravatar liveness Lograge diff --git a/doc/README.md b/doc/README.md index 08b76e58fc6..17ac5d578b8 100644 --- a/doc/README.md +++ b/doc/README.md @@ -269,7 +269,7 @@ The following documentation relates to the DevOps **Package** stage: | Package topics | Description | |:----------------------------------------------------------------|:-------------------------------------------------------| | [Container Registry](user/packages/container_registry/index.md) | The GitLab Container Registry enables every project in GitLab to have its own space to store [Docker](https://www.docker.com/) images. | -| [Dependency Proxy](user/packages/dependency_proxy/index.md) **(PREMIUM)** | The GitLab Dependency Proxy sets up a local proxy for frequently used upstream images/packages. | +| [Dependency Proxy](user/packages/dependency_proxy/index.md) | The GitLab Dependency Proxy sets up a local proxy for frequently used upstream images/packages. | | [Package Registry](user/packages/package_registry/index.md) | Use GitLab as a private or public registry for a variety of common package managers, including [NPM](user/packages/npm_registry/index.md), [Maven](user/packages/maven_repository/index.md), [PyPI](user/packages/pypi_repository/index.md), and others. You can also store generic files. | <div align="right"> diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index ce1fa328b69..676f402f7cf 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -180,6 +180,12 @@ the steps bellow. Feature.enable(:repository_push_audit_event) ``` +## Retention policy + +On GitLab.com, Audit Event records become subject to deletion after 400 days, or when your license is downgraded to a tier that does not include access to Audit Events. Data that is subject to deletion will be deleted at GitLab's discretion, possibly without additional notice. + +If you require a longer retention period, you should independently archive your Audit Event data, which you can retrieve through the [Audit Events API](../api/audit_events.md). + ## Export to CSV **(PREMIUM ONLY)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1449) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.4. diff --git a/doc/administration/index.md b/doc/administration/index.md index 3b0beafa35a..0aa94b86371 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -143,7 +143,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Container Registry](packages/container_registry.md): Configure Container Registry with GitLab. - [Package Registry](packages/index.md): Enable GitLab to act as an NPM Registry and a Maven Repository. -- [Dependency Proxy](packages/dependency_proxy.md): Configure the Dependency Proxy, a local proxy for frequently used upstream images/packages. **(PREMIUM ONLY)** +- [Dependency Proxy](packages/dependency_proxy.md): Configure the Dependency Proxy, a local proxy for frequently used upstream images/packages. ### Repository settings diff --git a/doc/administration/packages/dependency_proxy.md b/doc/administration/packages/dependency_proxy.md index fba3d51f741..7a37bf4dcea 100644 --- a/doc/administration/packages/dependency_proxy.md +++ b/doc/administration/packages/dependency_proxy.md @@ -4,9 +4,10 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# GitLab Dependency Proxy administration **(PREMIUM ONLY)** +# GitLab Dependency Proxy administration -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.11. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.11. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/273655) to [GitLab Core](https://about.gitlab.com/pricing/) in GitLab 13.6. GitLab can be utilized as a dependency proxy for a variety of common package managers. diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 26dab5218e2..9f522e0d599 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -2007,7 +2007,7 @@ on what features you intend to use: | [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | | [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | -| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) | Yes | | [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | | [Terraform state files](../terraform_state.md#using-object-storage) | Yes | diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 21206907153..b106f7bced1 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -2007,7 +2007,7 @@ on what features you intend to use: | [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | | [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | -| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) | Yes | | [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | | [Terraform state files](../terraform_state.md#using-object-storage) | Yes | diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index 9bcc0281cde..f4842a8568b 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -866,7 +866,7 @@ on what features you intend to use: | [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | | [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | -| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) | Yes | | [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | | [Terraform state files](../terraform_state.md#using-object-storage) | Yes | diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 9b6bbe19e0e..b5b3e4e0300 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -1742,7 +1742,7 @@ on what features you intend to use: | [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | | [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | -| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) | Yes | | [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | | [Terraform state files](../terraform_state.md#using-object-storage) | Yes | diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 5f813c95544..152eb9cb90d 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -2007,7 +2007,7 @@ on what features you intend to use: | [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | | [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | -| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) | Yes | | [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | | [Terraform state files](../terraform_state.md#using-object-storage) | Yes | diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 0e122f564c5..f023971bdc0 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -1741,7 +1741,7 @@ on what features you intend to use: | [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | | [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | -| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) | Yes | | [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | | [Terraform state files](../terraform_state.md#using-object-storage) | Yes | diff --git a/doc/administration/reply_by_email.md b/doc/administration/reply_by_email.md index de19c025cb4..4108635ba2c 100644 --- a/doc/administration/reply_by_email.md +++ b/doc/administration/reply_by_email.md @@ -1,6 +1,6 @@ --- stage: Plan -group: Product Planning +group: Certify info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/api/dependency_proxy.md b/doc/api/dependency_proxy.md index 7590583bdb6..426d2381858 100644 --- a/doc/api/dependency_proxy.md +++ b/doc/api/dependency_proxy.md @@ -4,11 +4,12 @@ group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Dependency Proxy API **(PREMIUM)** +# Dependency Proxy API ## Purge the dependency proxy for a group -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11631) in GitLab 12.10. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11631) in GitLab 12.10. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/273655) to [GitLab Core](https://about.gitlab.com/pricing/) in GitLab 13.6. Deletes the cached blobs for a group. This endpoint requires group admin access. diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index 459cce39ffc..5bb1dc0eff6 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -14967,8 +14967,7 @@ type Project { ): ClusterAgentConnection """ - Code coverages summary associated with the project. Available only when - feature flag `group_coverage_data_report` is enabled + Code coverage summary associated with the project """ codeCoverageSummary: CodeCoverageSummary diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index 3965100cea8..6614396da72 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -44149,7 +44149,7 @@ }, { "name": "codeCoverageSummary", - "description": "Code coverages summary associated with the project. Available only when feature flag `group_coverage_data_report` is enabled", + "description": "Code coverage summary associated with the project", "args": [ ], diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 2480deb197b..34d30e8c7f1 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -2257,7 +2257,7 @@ Autogenerated return type of PipelineRetry. | `boards` | BoardConnection | Boards of the project | | `clusterAgent` | ClusterAgent | Find a single cluster agent by name | | `clusterAgents` | ClusterAgentConnection | Cluster agents associated with the project | -| `codeCoverageSummary` | CodeCoverageSummary | Code coverages summary associated with the project. Available only when feature flag `group_coverage_data_report` is enabled | +| `codeCoverageSummary` | CodeCoverageSummary | Code coverage summary associated with the project | | `complianceFrameworks` | ComplianceFrameworkConnection | Compliance frameworks associated with the project | | `containerExpirationPolicy` | ContainerExpirationPolicy | The container expiration policy of the project | | `containerRegistryEnabled` | Boolean | Indicates if the project stores Docker container images in a container registry | diff --git a/doc/ci/interactive_web_terminal/img/interactive_web_terminal_running_job.png b/doc/ci/interactive_web_terminal/img/interactive_web_terminal_running_job.png Binary files differindex 3ee5e39afc0..8082d17ae6a 100644 --- a/doc/ci/interactive_web_terminal/img/interactive_web_terminal_running_job.png +++ b/doc/ci/interactive_web_terminal/img/interactive_web_terminal_running_job.png diff --git a/doc/ci/introduction/img/gitlab_workflow_example_11_9.png b/doc/ci/introduction/img/gitlab_workflow_example_11_9.png Binary files differindex f3fb9444b55..1f11db55f81 100644 --- a/doc/ci/introduction/img/gitlab_workflow_example_11_9.png +++ b/doc/ci/introduction/img/gitlab_workflow_example_11_9.png diff --git a/doc/ci/pipelines/img/manual_job_variables.png b/doc/ci/pipelines/img/manual_job_variables.png Binary files differindex a5ed351fdcd..63801ade21f 100644 --- a/doc/ci/pipelines/img/manual_job_variables.png +++ b/doc/ci/pipelines/img/manual_job_variables.png diff --git a/doc/development/contributing/style_guides.md b/doc/development/contributing/style_guides.md index 32db9a261f5..fd3fe239110 100644 --- a/doc/development/contributing/style_guides.md +++ b/doc/development/contributing/style_guides.md @@ -48,14 +48,16 @@ for changed files. This saves you time as you don't have to wait for the same er by CI/CD. Lefthook relies on a pre-push hook to prevent commits that violate its ruleset. -If you wish to override this behavior, pass the environment variable `LEFTHOOK=0`. -That is, `LEFTHOOK=0 git push`. +To override this behavior, pass the environment variable `LEFTHOOK=0`. That is, +`LEFTHOOK=0 git push`. You can also: - Define [local configuration](https://github.com/Arkweid/lefthook/blob/master/docs/full_guide.md#local-config). -- Skip [checks per tag on the fly](https://github.com/Arkweid/lefthook/blob/master/docs/full_guide.md#skip-some-tags-on-the-fly), e.g. `LEFTHOOK_EXCLUDE=frontend git push origin`. -- Run [hooks manually](https://github.com/Arkweid/lefthook/blob/master/docs/full_guide.md#run-githook-group-directly), e.g. `lefthook run pre-push`. +- Skip [checks per tag on the fly](https://github.com/Arkweid/lefthook/blob/master/docs/full_guide.md#skip-some-tags-on-the-fly). + For example, `LEFTHOOK_EXCLUDE=frontend git push origin`. +- Run [hooks manually](https://github.com/Arkweid/lefthook/blob/master/docs/full_guide.md#run-githook-group-directly). + For example, `lefthook run pre-push`. ## Ruby, Rails, RSpec diff --git a/doc/development/img/memory_ruby_heap_fragmentation.png b/doc/development/img/memory_ruby_heap_fragmentation.png Binary files differindex 4703da7491d..204130f8d87 100644 --- a/doc/development/img/memory_ruby_heap_fragmentation.png +++ b/doc/development/img/memory_ruby_heap_fragmentation.png diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index b05c0e351a9..44c69acbd87 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -181,7 +181,9 @@ SAST and Dependency Scanning scanners must scan the files in the project directo In order to be consistent with the official Container Scanning for GitLab, scanners must scan the Docker image whose name and tag are given by -`CI_APPLICATION_REPOSITORY` and `CI_APPLICATION_TAG`, respectively. +`CI_APPLICATION_REPOSITORY` and `CI_APPLICATION_TAG`, respectively. If the `DOCKER_IMAGE` +variable is provided, then the `CI_APPLICATION_REPOSITORY` and `CI_APPLICATION_TAG` variables +are ignored, and the image specified in the `DOCKER_IMAGE` variable is scanned instead. If not provided, `CI_APPLICATION_REPOSITORY` should default to `$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG`, which is a combination of predefined CI variables. diff --git a/doc/development/testing_guide/end_to_end/best_practices.md b/doc/development/testing_guide/end_to_end/best_practices.md index 792e5ef1d6f..3ee4a3d1b87 100644 --- a/doc/development/testing_guide/end_to_end/best_practices.md +++ b/doc/development/testing_guide/end_to_end/best_practices.md @@ -325,39 +325,69 @@ In general, we use an `expect` statement to check that something _is_ as we expe ```ruby Page::Project::Pipeline::Show.perform do |pipeline| - expect(pipeline).to have_job("a_job") + expect(pipeline).to have_job('a_job') end ``` -### Ensure `expect` checks for negation efficiently +### Create negatable matchers to speed `expect` checks However, sometimes we want to check that something is _not_ as we _don't_ want it to be. In other -words, we want to make sure something is absent. In such a case we should use an appropriate -predicate method that returns quickly, rather than waiting for a state that won't appear. +words, we want to make sure something is absent. For unit tests and feature specs, +we commonly use `not_to` +because RSpec's built-in matchers are negatable, as are Capybara's, which means the following two statements are +equivalent. -It's most efficient to use a predicate method that returns immediately when there is no job, or waits -until it disappears: +```ruby +except(page).not_to have_text('hidden') +except(page).to have_no_text('hidden') +``` + +Unfortunately, that's not automatically the case for the predicate methods that we add to our +[page objects](page_objects.md). We need to [create our own negatable matchers](https://relishapp.com/rspec/rspec-expectations/v/3-9/docs/custom-matchers/define-a-custom-matcher#matcher-with-separate-logic-for-expect().to-and-expect().not-to). + +The initial example uses the `have_job` matcher which is derived from the [`has_job?` predicate +method of the `Page::Project::Pipeline::Show` page object](https://gitlab.com/gitlab-org/gitlab/-/blob/87864b3047c23b4308f59c27a3757045944af447/qa/qa/page/project/pipeline/show.rb#L53). +To create a negatable matcher, we use `has_no_job?` for the negative case: + +```ruby +RSpec::Matchers.define :have_job do |job_name| + match do |page_object| + page_object.has_job?(job_name) + end + + match_when_negated do |page_object| + page_object.has_no_job?(job_name) + end +end +``` + +And then the two `expect` statements in the following example are equivalent: ```ruby -# Good Page::Project::Pipeline::Show.perform do |pipeline| - expect(pipeline).to have_no_job("a_job") + expect(pipeline).not_to have_job('a_job') + expect(pipeline).to have_no_job('a_job') end ``` -### Problematic alternatives +[See this merge request for a real example of adding a custom matcher](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46302). -Alternatively, if we want to check that a job doesn't exist it might be tempting to use `not_to`: +NOTE: **Note:** +We need to create custom negatable matchers only for the predicate methods we've added to the test framework, and only if we're using `not_to`. If we use `to have_no_*` a negatable matcher is not necessary. + +### Why we need negatable matchers + +Consider the following code, but assume that we _don't_ have a custom negatable matcher for `have_job`. ```ruby # Bad Page::Project::Pipeline::Show.perform do |pipeline| - expect(pipeline).not_to have_job("a_job") + expect(pipeline).not_to have_job('a_job') end ``` -For this statement to pass, `have_job("a_job")` has to return `false` so that `not_to` can negate it. -The problem is that `have_job("a_job")` waits up to ten seconds for `"a job"` to appear before +For this statement to pass, `have_job('a_job')` has to return `false` so that `not_to` can negate it. +The problem is that `have_job('a_job')` waits up to ten seconds for `'a job'` to appear before returning `false`. Under the expected condition this test will take ten seconds longer than it needs to. Instead, we could force no wait: @@ -365,9 +395,13 @@ Instead, we could force no wait: ```ruby # Not as bad but potentially flaky Page::Project::Pipeline::Show.perform do |pipeline| - expect(pipeline).not_to have_job("a_job", wait: 0) + expect(pipeline).not_to have_job('a_job', wait: 0) end ``` -The problem is that if `"a_job"` is present and we're waiting for it to disappear, this statement -will fail. +The problem is that if `'a_job'` is present and we're waiting for it to disappear, this statement will fail. + +Neither problem is present if we create a custom negatable matcher because the `has_no_job?` predicate method +would be used, which would wait only as long as necessary for the job to disappear. + +Lastly, negatable matchers are preferred over using matchers of the form `have_no_*` because it's a common and familiar practice to negate matchers using `not_to`. If we facilitate that practice by adding negatable matchers, we make it easier for subsequent test authors to write efficient tests. diff --git a/doc/integration/img/spam_log.png b/doc/integration/img/spam_log.png Binary files differindex 43e267daff4..693ea2a55cd 100644 --- a/doc/integration/img/spam_log.png +++ b/doc/integration/img/spam_log.png diff --git a/doc/integration/img/submit_issue.png b/doc/integration/img/submit_issue.png Binary files differindex e794eac189e..c1bb725cc03 100644 --- a/doc/integration/img/submit_issue.png +++ b/doc/integration/img/submit_issue.png diff --git a/doc/user/admin_area/analytics/img/instance_activity_pipelines_chart_v13_6.png b/doc/user/admin_area/analytics/img/instance_activity_pipelines_chart_v13_6.png Binary files differindex dfe5847c9b8..da9e4c64e12 100644 --- a/doc/user/admin_area/analytics/img/instance_activity_pipelines_chart_v13_6.png +++ b/doc/user/admin_area/analytics/img/instance_activity_pipelines_chart_v13_6.png diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index ea3a666d1b1..eef15a9c424 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -186,6 +186,7 @@ scanning by using the following environment variables: | `CI_APPLICATION_REPOSITORY` | `$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG` | Docker repository URL for the image to be scanned. | | `CI_APPLICATION_TAG` | `$CI_COMMIT_SHA` | Docker repository tag for the image to be scanned. | | `CS_MAJOR_VERSION` | `3` | The major version of the Docker image tag. | +| `DOCKER_IMAGE` | `$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG` | The Docker image to be scanned. If set, this variable overrides the `$CI_APPLICATION_REPOSITORY` and `$CI_APPLICATION_TAG` variables. | | `DOCKER_INSECURE` | `"false"` | Allow [Klar](https://github.com/optiopay/klar) to access secure Docker registries using HTTPS with bad (or self-signed) SSL certificates. | | `DOCKER_PASSWORD` | `$CI_REGISTRY_PASSWORD` | Password for accessing a Docker registry requiring authentication. | | `DOCKER_USER` | `$CI_REGISTRY_USER` | Username for accessing a Docker registry requiring authentication. | diff --git a/doc/user/clusters/agent/index.md b/doc/user/clusters/agent/index.md index 8fa85b91d73..8b1edc609da 100644 --- a/doc/user/clusters/agent/index.md +++ b/doc/user/clusters/agent/index.md @@ -412,11 +412,56 @@ spec: ## Example projects +The following example projects can help you get started with the Kubernetes Agent. + +### Simple NGINX deployment + This basic GitOps example deploys NGINX: - [Configuration repository](https://gitlab.com/gitlab-org/configure/examples/kubernetes-agent) - [Manifest repository](https://gitlab.com/gitlab-org/configure/examples/gitops-project) -- [Install GitLab Runner](https://gitlab.com/gitlab-examples/install-runner-via-k8s-agent) + +### Deploying GitLab Runner with the Agent + +These instructions assume that the Agent is already set up as described in the +[Get started with GitOps](#get-started-with-gitops-and-the-gitlab-agent): + +1. Check the possible + [Runner chart YAML values](https://gitlab.com/gitlab-org/charts/gitlab-runner/blob/master/values.yaml) + on the Runner chart documentation, and create a `runner-chart-values.yaml` file + with the configuration that fits your needs, such as: + + ```yaml + ## The GitLab Server URL (with protocol) that want to register the runner against + ## ref: https://docs.gitlab.com/runner/commands/README.html#gitlab-runner-register + ## + gitlabUrl: https://gitlab.my.domain.com/ + + ## The Registration Token for adding new Runners to the GitLab Server. This must + ## be retrieved from your GitLab Instance. + ## ref: https://docs.gitlab.com/ce/ci/runners/README.html + ## + runnerRegistrationToken: "XXXXXXYYYYYYZZZZZZ" + + ## For RBAC support: + rbac: + create: true + + ## Run all containers with the privileged flag enabled + ## This will allow the docker:dind image to run if you need to run Docker + ## commands. Please read the docs before turning this on: + ## ref: https://docs.gitlab.com/runner/executors/kubernetes.html#using-dockerdind + runners: + privileged: true + ``` + +1. Create a single manifest file to install the Runner chart with your cluster agent: + + ```shell + helm template --namespace gitlab gitlab-runner -f runner-chart-values.yaml gitlab/gitlab-runner > manifest.yaml + ``` + +1. Push your `manifest.yaml` to your manifest repository. ## Troubleshooting @@ -479,7 +524,7 @@ but KAS on the server side is not available via `wss`. To fix it, make sure the same schemes are configured on both sides. It's not possible to set the `grpc` scheme due to the issue -[It is not possible to configure KAS to work with grpc without directly editing GitLab KAS deployment](https://gitlab.com/gitlab-org/gitlab/-/issues/276888). To use `grpc` while the +[It is not possible to configure KAS to work with `grpc` without directly editing GitLab KAS deployment](https://gitlab.com/gitlab-org/gitlab/-/issues/276888). To use `grpc` while the issue is in progress, directly edit the deployment with the `kubectl edit deployment gitlab-kas` command, and change `--listen-websocket=true` to `--listen-websocket=false`. After running that command, you should be able to use `grpc://gitlab-kas.<YOUR-NAMESPACE>:5005`. diff --git a/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_3_1.png b/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_3_1.png Binary files differdeleted file mode 100644 index 89f4e917567..00000000000 --- a/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_3_1.png +++ /dev/null diff --git a/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_6.png b/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_6.png Binary files differnew file mode 100644 index 00000000000..b2ac4f95e0d --- /dev/null +++ b/doc/user/compliance/compliance_dashboard/img/compliance_dashboard_v13_6.png diff --git a/doc/user/compliance/compliance_dashboard/index.md b/doc/user/compliance/compliance_dashboard/index.md index d2fb8dd318a..151c61b50d8 100644 --- a/doc/user/compliance/compliance_dashboard/index.md +++ b/doc/user/compliance/compliance_dashboard/index.md @@ -17,7 +17,7 @@ for merging into production. To access the Compliance Dashboard for a group, navigate to **{shield}** **Security & Compliance > Compliance** on the group's menu. - + NOTE: **Note:** The Compliance Dashboard shows only the latest MR on each project. @@ -63,7 +63,9 @@ This column has four states: If you do not see the success icon in your Compliance dashboard; please review the above criteria for the Merge Requests project to make sure it complies with the separation of duties described above. -## Chain of Custody report +## Chain of Custody report **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213364) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.3. The Chain of Custody report allows customers to export a list of merge commits within the group. The data provides a comprehensive view with respect to merge commits. It includes the merge commit SHA, @@ -72,6 +74,13 @@ Depending on the merge strategy, the merge commit SHA can either be a merge comm To download the Chain of Custody report, navigate to **{shield}** **Security & Compliance > Compliance** on the group's menu and click **List of all merge commits** +### Commit-specific Chain of Custody Report **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267629) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.6. + +You can generate a commit-specific Chain of Custody report for a given commit SHA. To do so, select +the dropdown next to the **List of all merge commits** button at the top of the Compliance Dashboard. + NOTE: **Note:** The Chain of Custody report download is a CSV file, with a maximum size of 15 MB. The remaining records are truncated when this limit is reached. diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 0127e4435f1..e09c685147a 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -798,7 +798,7 @@ With [GitLab Issue Analytics](issues_analytics/index.md), you can see a bar char With [GitLab Repositories Analytics](repositories_analytics/index.md), you can download a CSV of the latest coverage data for all the projects in your group. -## Dependency Proxy **(PREMIUM)** +## Dependency Proxy Use GitLab as a [dependency proxy](../packages/dependency_proxy/index.md) for upstream Docker images. diff --git a/doc/user/group/repositories_analytics/index.md b/doc/user/group/repositories_analytics/index.md index 9c3a4deb2b6..fe5e7979592 100644 --- a/doc/user/group/repositories_analytics/index.md +++ b/doc/user/group/repositories_analytics/index.md @@ -15,22 +15,12 @@ This feature might not be available to you. Check the **version history** note a ## Latest project test coverage list > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267624) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.6. -> - It's [deployed behind a feature flag](../../../user/feature_flags.md), disabled by default. -> - It's disabled on GitLab.com -> - It can be enabled or disabled per-group. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-latest-project-test-coverage). To see the latest code coverage for each project in your group: 1. Go to **Analytics > Repositories** in the group (not from a project). 1. In the **Latest test coverage results** section, use the **Select projects** dropdown to choose the projects you want to check. -### Enable or disable latest project test coverage - -This feature comes with the `:group_coverage_data_report` feature flag disabled by default. It is disabled on GitLab.com. -[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) can enable it for your instance. -The group test coverage table can be enabled or disabled per-group. - ## Download historic test coverage data > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215104) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.4. diff --git a/doc/user/packages/dependency_proxy/index.md b/doc/user/packages/dependency_proxy/index.md index f1b49699483..e1fae770a5d 100644 --- a/doc/user/packages/dependency_proxy/index.md +++ b/doc/user/packages/dependency_proxy/index.md @@ -4,9 +4,10 @@ group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Dependency Proxy **(PREMIUM)** +# Dependency Proxy -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.11. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.11. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/273655) to [GitLab Core](https://about.gitlab.com/pricing/) in GitLab 13.6. The GitLab Dependency Proxy is a local proxy you can use for your frequently-accessed upstream images. diff --git a/doc/user/permissions.md b/doc/user/permissions.md index 22822268e78..f1365ee1cab 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -256,7 +256,7 @@ group. | Share (invite) groups with groups | | | | | ✓ | | Create/edit/delete group milestones | | | ✓ | ✓ | ✓ | | Create/edit/delete iterations | | | ✓ | ✓ | ✓ | -| Enable/disable a dependency proxy **(PREMIUM)** | | | ✓ | ✓ | ✓ | +| Enable/disable a dependency proxy | | | ✓ | ✓ | ✓ | | Create and edit group wiki pages **(PREMIUM)** | | | ✓ | ✓ | ✓ | | Use security dashboard **(ULTIMATE)** | | | ✓ | ✓ | ✓ | | Create/edit/delete metrics dashboard annotations | | | ✓ | ✓ | ✓ | diff --git a/doc/user/profile/img/busy_status_indicator_v13_6.png b/doc/user/profile/img/busy_status_indicator_v13_6.png Binary files differnew file mode 100644 index 00000000000..fa945264b8e --- /dev/null +++ b/doc/user/profile/img/busy_status_indicator_v13_6.png diff --git a/doc/user/profile/index.md b/doc/user/profile/index.md index f897cae0b0d..8ae92a42581 100644 --- a/doc/user/profile/index.md +++ b/doc/user/profile/index.md @@ -188,17 +188,22 @@ To set your current status: 1. Set the desired emoji and/or status message. 1. Click **Set status**. Alternatively, you can click **Remove status** to remove your user status entirely. + + or 1. Click your avatar. 1. Select **Profile**. 1. Click **Edit profile** (pencil icon). 1. Enter your status message in the **Your status** text field. + 1. Alternatively, select the **Busy** checkbox ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259649) in GitLab 13.6}. 1. Click **Add status emoji** (smiley face), and select the desired emoji. 1. Click **Update profile settings**. You can also set your current status [using the API](../../api/users.md#user-status). +If you previously selected the "Busy" checkbox, remember to deselect it when you become available again. + ## Commit email > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/21598) in GitLab 11.4. diff --git a/doc/user/project/requirements/index.md b/doc/user/project/requirements/index.md index 09e457fb37f..f533f8807d2 100644 --- a/doc/user/project/requirements/index.md +++ b/doc/user/project/requirements/index.md @@ -1,7 +1,7 @@ --- type: reference, howto stage: Plan -group: Product Planning +group: Certify info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/user/project/service_desk.md b/doc/user/project/service_desk.md index 456466a747d..34a075df990 100644 --- a/doc/user/project/service_desk.md +++ b/doc/user/project/service_desk.md @@ -1,6 +1,6 @@ --- stage: Plan -group: Product Planning +group: Certify info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/lib/api/api.rb b/lib/api/api.rb index 0dafbe492b5..ea149f25584 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -162,6 +162,7 @@ module API mount ::API::CommitStatuses mount ::API::ContainerRegistryEvent mount ::API::ContainerRepositories + mount ::API::DependencyProxy mount ::API::DeployKeys mount ::API::DeployTokens mount ::API::Deployments diff --git a/lib/api/dependency_proxy.rb b/lib/api/dependency_proxy.rb new file mode 100644 index 00000000000..3379bb2f029 --- /dev/null +++ b/lib/api/dependency_proxy.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module API + class DependencyProxy < ::API::Base + helpers ::API::Helpers::PackagesHelpers + + feature_category :dependency_proxy + + helpers do + def obtain_new_purge_cache_lease + Gitlab::ExclusiveLease + .new("dependency_proxy:delete_group_blobs:#{user_group.id}", + timeout: 1.hour) + .try_obtain + end + end + + before do + authorize! :admin_group, user_group + end + + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Deletes all dependency_proxy_blobs for a group' do + detail 'This feature was introduced in GitLab 12.10' + end + delete ':id/dependency_proxy/cache' do + not_found! unless user_group.dependency_proxy_feature_available? + + message = 'This request has already been made. It may take some time to purge the cache. You can run this at most once an hour for a given group' + render_api_error!(message, 409) unless obtain_new_purge_cache_lease + + # rubocop:disable CodeReuse/Worker + PurgeDependencyProxyCacheWorker.perform_async(current_user.id, user_group.id) + # rubocop:enable CodeReuse/Worker + end + end + end +end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 40fcd2f89f2..390da014a5a 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -40,23 +40,35 @@ module Gitlab private - def pipeline - @pipeline ||= @project.ci_pipelines.latest_successful_for_ref(@ref) + def successful_pipeline + @successful_pipeline ||= @project.ci_pipelines.latest_successful_for_ref(@ref) + end + + def failed_pipeline + @failed_pipeline ||= @project.ci_pipelines.latest_failed_for_ref(@ref) + end + + def running_pipeline + @running_pipeline ||= @project.ci_pipelines.latest_running_for_ref(@ref) end - # rubocop: disable CodeReuse/ActiveRecord def raw_coverage - return unless pipeline + latest = + if @job.present? + builds = ::Ci::Build + .in_pipelines([successful_pipeline, running_pipeline, failed_pipeline]) + .latest + .success + .for_ref(@ref) + .by_name(@job) + + builds.max_by(&:created_at) + else + successful_pipeline + end - if @job.blank? - pipeline.coverage - else - pipeline.builds - .find_by(name: @job) - .try(:coverage) - end + latest&.coverage end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 29a61f7c20a..dfe60fb5a03 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -18,6 +18,8 @@ module Gitlab attr_reader :octokit + SEARCH_MAX_REQUESTS_PER_MINUTE = 30 + # A single page of data and the corresponding page number. Page = Struct.new(:objects, :number) @@ -28,6 +30,7 @@ module Gitlab # rate limit at once. The threshold is put in place to not hit the limit # in most cases. RATE_LIMIT_THRESHOLD = 50 + SEARCH_RATE_LIMIT_THRESHOLD = 3 # token - The GitHub API token to use. # @@ -152,8 +155,26 @@ module Gitlab end end + def search_repos_by_name(name) + each_page(:search_repositories, search_query(str: name, type: :name)) + end + + def search_query(str:, type:, include_collaborations: true, include_orgs: true) + query = "#{str} in:#{type} is:public,private user:#{octokit.user.login}" + + query = [query, collaborations_subquery].join(' ') if include_collaborations + query = [query, organizations_subquery].join(' ') if include_orgs + + query + end + # Returns `true` if we're still allowed to perform API calls. + # Search API has rate limit of 30, use lowered threshold when search is used. def requests_remaining? + if requests_limit == SEARCH_MAX_REQUESTS_PER_MINUTE + return remaining_requests > SEARCH_RATE_LIMIT_THRESHOLD + end + remaining_requests > RATE_LIMIT_THRESHOLD end @@ -161,6 +182,10 @@ module Gitlab octokit.rate_limit.remaining end + def requests_limit + octokit.rate_limit.limit + end + def raise_or_wait_for_rate_limit rate_limit_counter.increment @@ -221,6 +246,20 @@ module Gitlab 'The number of GitHub API calls performed when importing projects' ) end + + private + + def collaborations_subquery + each_object(:repos, nil, { affiliation: 'collaborator' }) + .map { |repo| "repo:#{repo.full_name}" } + .join(' ') + end + + def organizations_subquery + each_object(:organizations) + .map { |org| "org:#{org.login}" } + .join(' ') + end end end end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 893a3eed689..ad0a5c80604 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -254,6 +254,14 @@ module Gitlab %r{#{personal_snippet_path_regex}|#{project_snippet_path_regex}} end + def container_image_regex + @container_image_regex ||= %r{([\w\.-]+\/){0,1}[\w\.-]+}.freeze + end + + def container_image_blob_sha_regex + @container_image_blob_sha_regex ||= %r{[\w+.-]+:?\w+}.freeze + end + private def personal_snippet_path_regex diff --git a/locale/gitlab.pot b/locale/gitlab.pot index e725c6166c7..4854cccfb3c 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -14157,6 +14157,9 @@ msgstr "" msgid "ImportButtons|Connect repositories from" msgstr "" +msgid "ImportProjects|%{provider} rate limit exceeded. Try again later" +msgstr "" + msgid "ImportProjects|Blocked import URL: %{message}" msgstr "" diff --git a/package.json b/package.json index c8ade903791..67c9c05e042 100644 --- a/package.json +++ b/package.json @@ -162,7 +162,7 @@ }, "devDependencies": { "@babel/plugin-transform-modules-commonjs": "^7.10.1", - "@gitlab/eslint-plugin": "4.1.0", + "@gitlab/eslint-plugin": "5.0.0", "@testing-library/dom": "^7.16.2", "@vue/test-utils": "1.0.0-beta.30", "acorn": "^6.3.0", diff --git a/spec/controllers/groups/dependency_proxies_controller_spec.rb b/spec/controllers/groups/dependency_proxies_controller_spec.rb new file mode 100644 index 00000000000..35bd7d47aed --- /dev/null +++ b/spec/controllers/groups/dependency_proxies_controller_spec.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Groups::DependencyProxiesController do + let(:group) { create(:group) } + let(:user) { create(:user) } + + before do + group.add_owner(user) + sign_in(user) + end + + describe 'GET #show' do + context 'feature enabled' do + before do + enable_dependency_proxy + end + + it 'returns 200 and renders the view' do + get :show, params: { group_id: group.to_param } + + expect(response).to have_gitlab_http_status(:ok) + expect(response).to render_template('groups/dependency_proxies/show') + end + end + + it 'returns 404 when feature is disabled' do + disable_dependency_proxy + + get :show, params: { group_id: group.to_param } + + expect(response).to have_gitlab_http_status(:not_found) + end + end + + describe 'PUT #update' do + context 'feature enabled' do + before do + enable_dependency_proxy + end + + it 'redirects back to show page' do + put :update, params: update_params + + expect(response).to have_gitlab_http_status(:found) + end + end + + it 'returns 404 when feature is disabled' do + put :update, params: update_params + + expect(response).to have_gitlab_http_status(:not_found) + end + + def update_params + { + group_id: group.to_param, + dependency_proxy_group_setting: { enabled: true } + } + end + end + + def enable_dependency_proxy + stub_config(dependency_proxy: { enabled: true }) + + group.create_dependency_proxy_setting!(enabled: true) + end + + def disable_dependency_proxy + group.create_dependency_proxy_setting!(enabled: false) + end +end diff --git a/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb b/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb new file mode 100644 index 00000000000..615b56ff22f --- /dev/null +++ b/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Groups::DependencyProxyForContainersController do + let(:group) { create(:group) } + let(:token_response) { { status: :success, token: 'abcd1234' } } + + shared_examples 'not found when disabled' do + context 'feature disabled' do + before do + disable_dependency_proxy + end + + it 'returns 404' do + subject + + expect(response).to have_gitlab_http_status(:not_found) + end + end + end + + before do + allow(Gitlab.config.dependency_proxy) + .to receive(:enabled).and_return(true) + + allow_next_instance_of(DependencyProxy::RequestTokenService) do |instance| + allow(instance).to receive(:execute).and_return(token_response) + end + end + + describe 'GET #manifest' do + let(:manifest) { { foo: 'bar' }.to_json } + let(:pull_response) { { status: :success, manifest: manifest } } + + before do + allow_next_instance_of(DependencyProxy::PullManifestService) do |instance| + allow(instance).to receive(:execute).and_return(pull_response) + end + end + + subject { get_manifest } + + context 'feature enabled' do + before do + enable_dependency_proxy + end + + context 'remote token request fails' do + let(:token_response) do + { + status: :error, + http_status: 503, + message: 'Service Unavailable' + } + end + + it 'proxies status from the remote token request' do + subject + + expect(response).to have_gitlab_http_status(:service_unavailable) + expect(response.body).to eq('Service Unavailable') + end + end + + context 'remote manifest request fails' do + let(:pull_response) do + { + status: :error, + http_status: 400, + message: '' + } + end + + it 'proxies status from the remote manifest request' do + subject + + expect(response).to have_gitlab_http_status(:bad_request) + expect(response.body).to be_empty + end + end + + it 'returns 200 with manifest file' do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.body).to eq(manifest) + end + end + + it_behaves_like 'not found when disabled' + + def get_manifest + get :manifest, params: { group_id: group.to_param, image: 'alpine', tag: '3.9.2' } + end + end + + describe 'GET #blob' do + let(:blob) { create(:dependency_proxy_blob) } + let(:blob_sha) { blob.file_name.sub('.gz', '') } + let(:blob_response) { { status: :success, blob: blob } } + + before do + allow_next_instance_of(DependencyProxy::FindOrCreateBlobService) do |instance| + allow(instance).to receive(:execute).and_return(blob_response) + end + end + + subject { get_blob } + + context 'feature enabled' do + before do + enable_dependency_proxy + end + + context 'remote blob request fails' do + let(:blob_response) do + { + status: :error, + http_status: 400, + message: '' + } + end + + it 'proxies status from the remote blob request' do + subject + + expect(response).to have_gitlab_http_status(:bad_request) + expect(response.body).to be_empty + end + end + + it 'sends a file' do + expect(controller).to receive(:send_file).with(blob.file.path, {}) + + subject + end + + it 'returns Content-Disposition: attachment' do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.headers['Content-Disposition']).to match(/^attachment/) + end + end + + it_behaves_like 'not found when disabled' + + def get_blob + get :blob, params: { group_id: group.to_param, image: 'alpine', sha: blob_sha } + end + end + + def enable_dependency_proxy + group.create_dependency_proxy_setting!(enabled: true) + end + + def disable_dependency_proxy + group.create_dependency_proxy_setting!(enabled: false) + end +end diff --git a/spec/controllers/import/github_controller_spec.rb b/spec/controllers/import/github_controller_spec.rb index e19b6caca5b..a408d821833 100644 --- a/spec/controllers/import/github_controller_spec.rb +++ b/spec/controllers/import/github_controller_spec.rb @@ -144,6 +144,58 @@ RSpec.describe Import::GithubController do expect(json_response.dig('provider_repos', 0, 'id')).to eq(repo_1.id) expect(json_response.dig('provider_repos', 1, 'id')).to eq(repo_2.id) end + + context 'when filtering' do + let(:filter) { 'test' } + let(:user_login) { 'user' } + let(:collaborations_subquery) { 'repo:repo1 repo:repo2' } + let(:organizations_subquery) { 'org:org1 org:org2' } + + before do + allow_next_instance_of(Octokit::Client) do |client| + allow(client).to receive(:user).and_return(double(login: user_login)) + end + end + + it 'makes request to github search api' do + expected_query = "test in:name is:public,private user:#{user_login} #{collaborations_subquery} #{organizations_subquery}" + + expect_next_instance_of(Gitlab::GithubImport::Client) do |client| + expect(client).to receive(:collaborations_subquery).and_return(collaborations_subquery) + expect(client).to receive(:organizations_subquery).and_return(organizations_subquery) + expect(client).to receive(:each_page).with(:search_repositories, expected_query).and_return([].to_enum) + end + + get :status, params: { filter: filter }, format: :json + end + + context 'when user input contains colons and spaces' do + before do + stub_client(search_repos_by_name: []) + end + + it 'sanitizes user input' do + filter = ' test1:test2 test3 : test4 ' + expected_filter = 'test1test2test3test4' + + get :status, params: { filter: filter }, format: :json + + expect(assigns(:filter)).to eq(expected_filter) + end + end + + context 'when rate limit threshold is exceeded' do + before do + allow(controller).to receive(:status).and_raise(Gitlab::GithubImport::RateLimitError) + end + + it 'returns 429' do + get :status, params: { filter: 'test' }, format: :json + + expect(response).to have_gitlab_http_status(:too_many_requests) + end + end + end end end diff --git a/spec/factories/dependency_proxy.rb b/spec/factories/dependency_proxy.rb new file mode 100644 index 00000000000..5d763392a99 --- /dev/null +++ b/spec/factories/dependency_proxy.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :dependency_proxy_blob, class: 'DependencyProxy::Blob' do + group + file { fixture_file_upload('spec/fixtures/dependency_proxy/a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4.gz') } + file_name { 'a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4.gz' } + end +end diff --git a/spec/features/groups/dependency_proxy_spec.rb b/spec/features/groups/dependency_proxy_spec.rb new file mode 100644 index 00000000000..9bbfdc488fb --- /dev/null +++ b/spec/features/groups/dependency_proxy_spec.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'Group Dependency Proxy' do + let(:developer) { create(:user) } + let(:reporter) { create(:user) } + let(:group) { create(:group) } + let(:path) { group_dependency_proxy_path(group) } + + before do + group.add_developer(developer) + group.add_reporter(reporter) + + enable_feature + end + + describe 'feature settings' do + context 'when not logged in and feature disabled' do + it 'does not show the feature settings' do + group.create_dependency_proxy_setting(enabled: false) + + visit path + + expect(page).not_to have_css('.js-dependency-proxy-toggle-area') + expect(page).not_to have_css('.js-dependency-proxy-url') + end + end + + context 'feature is available', :js do + context 'when logged in as group developer' do + before do + sign_in(developer) + visit path + end + + it 'sidebar menu is open' do + sidebar = find('.nav-sidebar') + expect(sidebar).to have_link _('Dependency Proxy') + end + + it 'toggles defaults to enabled' do + page.within('.js-dependency-proxy-toggle-area') do + expect(find('.js-project-feature-toggle-input', visible: false).value).to eq('true') + end + end + + it 'shows the proxy URL' do + page.within('.edit_dependency_proxy_group_setting') do + expect(find('.js-dependency-proxy-url').value).to have_content('/dependency_proxy/containers') + end + end + + it 'hides the proxy URL when feature is disabled' do + page.within('.edit_dependency_proxy_group_setting') do + find('.js-project-feature-toggle').click + end + + expect(page).not_to have_css('.js-dependency-proxy-url') + expect(find('.js-project-feature-toggle-input', visible: false).value).to eq('false') + end + end + + context 'when logged in as group reporter' do + before do + sign_in(reporter) + visit path + end + + it 'does not show the feature toggle but shows the proxy URL' do + expect(page).not_to have_css('.js-dependency-proxy-toggle-area') + expect(find('.js-dependency-proxy-url').value).to have_content('/dependency_proxy/containers') + end + end + end + + context 'feature is not avaible' do + before do + sign_in(developer) + end + + context 'group is private' do + let(:group) { create(:group, :private) } + + it 'informs user that feature is only available for public groups' do + visit path + + expect(page).to have_content('Dependency proxy feature is limited to public groups for now.') + end + end + + context 'feature is disabled globally' do + it 'renders 404 page' do + disable_feature + + visit path + + expect(page).to have_gitlab_http_status(:not_found) + end + end + end + end + + def enable_feature + stub_config(dependency_proxy: { enabled: true }) + end + + def disable_feature + stub_config(dependency_proxy: { enabled: false }) + end +end diff --git a/spec/features/groups/navbar_spec.rb b/spec/features/groups/navbar_spec.rb index e81f2370d10..dec07eb3783 100644 --- a/spec/features/groups/navbar_spec.rb +++ b/spec/features/groups/navbar_spec.rb @@ -50,6 +50,8 @@ RSpec.describe 'Group navbar' do insert_package_nav(_('Kubernetes')) stub_feature_flags(group_iterations: false) + stub_config(dependency_proxy: { enabled: false }) + stub_config(registry: { enabled: false }) stub_group_wikis(false) group.add_maintainer(user) sign_in(user) @@ -73,6 +75,18 @@ RSpec.describe 'Group navbar' do it_behaves_like 'verified navigation bar' end + context 'when dependency proxy is available' do + before do + stub_config(dependency_proxy: { enabled: true }) + + insert_dependency_proxy_nav(_('Dependency Proxy')) + + visit group_path(group) + end + + it_behaves_like 'verified navigation bar' + end + context 'when invite team members is not available' do it 'does not display the js-invite-members-trigger' do visit group_path(group) diff --git a/spec/fixtures/dependency_proxy/a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4.gz b/spec/fixtures/dependency_proxy/a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4.gz Binary files differnew file mode 100644 index 00000000000..14043720eaf --- /dev/null +++ b/spec/fixtures/dependency_proxy/a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4.gz diff --git a/spec/frontend/blob/components/blob_header_filepath_spec.js b/spec/frontend/blob/components/blob_header_filepath_spec.js index 43057353051..067a4ae61a0 100644 --- a/spec/frontend/blob/components/blob_header_filepath_spec.js +++ b/spec/frontend/blob/components/blob_header_filepath_spec.js @@ -65,7 +65,7 @@ describe('Blob Header Filepath', () => { {}, { scopedSlots: { - filepathPrepend: `<span>${slotContent}</span>`, + 'filepath-prepend': `<span>${slotContent}</span>`, }, }, ); diff --git a/spec/frontend/clusters/components/__snapshots__/applications_spec.js.snap b/spec/frontend/clusters/components/__snapshots__/applications_spec.js.snap index b6e89281fef..744ef318260 100644 --- a/spec/frontend/clusters/components/__snapshots__/applications_spec.js.snap +++ b/spec/frontend/clusters/components/__snapshots__/applications_spec.js.snap @@ -73,7 +73,7 @@ exports[`Applications Ingress application shows the correct warning message 1`] exports[`Applications Knative application shows the correct description 1`] = ` <span - data-testid="installedVia" + data-testid="installed-via" > installed via <a diff --git a/spec/frontend/clusters/components/applications_spec.js b/spec/frontend/clusters/components/applications_spec.js index e0ccf36e868..5438f3053a8 100644 --- a/spec/frontend/clusters/components/applications_spec.js +++ b/spec/frontend/clusters/components/applications_spec.js @@ -429,7 +429,7 @@ describe('Applications', () => { await wrapper.vm.$nextTick(); - expect(findByTestId('installedVia').element).toMatchSnapshot(); + expect(findByTestId('installed-via').element).toMatchSnapshot(); }); it('emits saveKnativeDomain event when knative domain editor emits save event', () => { diff --git a/spec/frontend/import_projects/store/actions_spec.js b/spec/frontend/import_projects/store/actions_spec.js index f8af9f40f4c..06afb20c6a2 100644 --- a/spec/frontend/import_projects/store/actions_spec.js +++ b/spec/frontend/import_projects/store/actions_spec.js @@ -69,6 +69,7 @@ describe('import_projects store actions', () => { importStatus: STATUSES.NONE, }, ], + provider: 'provider', }; localState.getImportTarget = getImportTarget(localState); @@ -150,7 +151,28 @@ describe('import_projects store actions', () => { ); }); - describe('when /home/xanf/projects/gdk/gitlab/spec/frontend/import_projects/store/actions_spec.jsfiltered', () => { + describe('when rate limited', () => { + it('commits RECEIVE_REPOS_ERROR and shows rate limited error message', async () => { + mock.onGet(`${TEST_HOST}/endpoint.json?filter=filter`).reply(429); + + await testAction( + fetchRepos, + null, + { ...localState, filter: 'filter' }, + [ + { type: SET_PAGE, payload: 1 }, + { type: REQUEST_REPOS }, + { type: SET_PAGE, payload: 0 }, + { type: RECEIVE_REPOS_ERROR }, + ], + [], + ); + + expect(createFlash).toHaveBeenCalledWith('Provider rate limit exceeded. Try again later'); + }); + }); + + describe('when filtered', () => { it('fetches repos with filter applied', () => { mock.onGet(`${TEST_HOST}/endpoint.json?filter=filter`).reply(200, payload); diff --git a/spec/frontend/issuable/related_issues/components/related_issues_block_spec.js b/spec/frontend/issuable/related_issues/components/related_issues_block_spec.js index b758b85beef..dd05f49b458 100644 --- a/spec/frontend/issuable/related_issues/components/related_issues_block_spec.js +++ b/spec/frontend/issuable/related_issues/components/related_issues_block_spec.js @@ -56,7 +56,7 @@ describe('RelatedIssuesBlock', () => { pathIdSeparator: PathIdSeparator.Issue, issuableType: 'issue', }, - slots: { headerText }, + slots: { 'header-text': headerText }, }); expect(wrapper.find('.card-title').html()).toContain(headerText); @@ -72,7 +72,7 @@ describe('RelatedIssuesBlock', () => { pathIdSeparator: PathIdSeparator.Issue, issuableType: 'issue', }, - slots: { headerActions }, + slots: { 'header-actions': headerActions }, }); expect(wrapper.find('[data-testid="custom-button"]').html()).toBe(headerActions); diff --git a/spec/frontend/monitoring/components/dashboard_panel_spec.js b/spec/frontend/monitoring/components/dashboard_panel_spec.js index ee0e1fd3176..1808faf8f0e 100644 --- a/spec/frontend/monitoring/components/dashboard_panel_spec.js +++ b/spec/frontend/monitoring/components/dashboard_panel_spec.js @@ -106,7 +106,7 @@ describe('Dashboard Panel', () => { {}, { slots: { - topLeft: `<div class="top-left-content">OK</div>`, + 'top-left': `<div class="top-left-content">OK</div>`, }, }, ); diff --git a/spec/frontend/monitoring/components/dashboard_spec.js b/spec/frontend/monitoring/components/dashboard_spec.js index b7a0ea46b61..27e479ba498 100644 --- a/spec/frontend/monitoring/components/dashboard_spec.js +++ b/spec/frontend/monitoring/components/dashboard_spec.js @@ -508,7 +508,7 @@ describe('Dashboard', () => { const mockKeyup = key => window.dispatchEvent(new KeyboardEvent('keyup', { key })); const MockPanel = { - template: `<div><slot name="topLeft"/></div>`, + template: `<div><slot name="top-left"/></div>`, }; beforeEach(() => { diff --git a/spec/frontend/reports/components/report_section_spec.js b/spec/frontend/reports/components/report_section_spec.js index a620b5d9afc..2d228313a9b 100644 --- a/spec/frontend/reports/components/report_section_spec.js +++ b/spec/frontend/reports/components/report_section_spec.js @@ -244,7 +244,7 @@ describe('Report section', () => { hasIssues: true, }, slots: { - actionButtons: ['Action!'], + 'action-buttons': ['Action!'], }, }); }); diff --git a/spec/frontend/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs_spec.js b/spec/frontend/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs_spec.js index d943aaf3e5f..0f7c8e97635 100644 --- a/spec/frontend/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs_spec.js +++ b/spec/frontend/vue_shared/components/paginated_table_with_search_and_tabs/paginated_table_with_search_and_tabs_spec.js @@ -70,7 +70,7 @@ describe('AlertManagementEmptyState', () => { ...props, }, slots: { - 'emtpy-state': EmptyStateSlot, + 'empty-state': EmptyStateSlot, 'header-actions': HeaderActionsSlot, title: TitleSlot, table: TableSlot, diff --git a/spec/lib/gitlab/badge/coverage/report_spec.rb b/spec/lib/gitlab/badge/coverage/report_spec.rb index ebe6b8603b1..3b5ea3291e4 100644 --- a/spec/lib/gitlab/badge/coverage/report_spec.rb +++ b/spec/lib/gitlab/badge/coverage/report_spec.rb @@ -3,13 +3,15 @@ require 'spec_helper' RSpec.describe Gitlab::Badge::Coverage::Report do - let_it_be(:project) { create(:project) } - let_it_be(:pipeline) { create(:ci_pipeline, :success, project: project) } + let_it_be(:project) { create(:project) } + let_it_be(:success_pipeline) { create(:ci_pipeline, :success, project: project) } + let_it_be(:running_pipeline) { create(:ci_pipeline, :running, project: project) } + let_it_be(:failure_pipeline) { create(:ci_pipeline, :failed, project: project) } let_it_be(:builds) do [ - create(:ci_build, :success, pipeline: pipeline, coverage: 40, name: 'first'), - create(:ci_build, :success, pipeline: pipeline, coverage: 60) + create(:ci_build, :success, pipeline: success_pipeline, coverage: 40, created_at: 9.seconds.ago, name: 'coverage'), + create(:ci_build, :success, pipeline: success_pipeline, coverage: 60, created_at: 8.seconds.ago) ] end @@ -38,28 +40,26 @@ RSpec.describe Gitlab::Badge::Coverage::Report do end describe '#status' do - before do - allow(badge).to receive(:pipeline).and_return(pipeline) - end - - context 'with no pipeline' do - let(:pipeline) { nil } - - it 'returns nil' do - expect(badge.status).to be_nil - end - end - context 'with no job specified' do - it 'returns the pipeline coverage value' do + it 'returns the most recent successful pipeline coverage value' do expect(badge.status).to eq(50.00) end + + context 'and no successful pipelines' do + before do + allow(badge).to receive(:successful_pipeline).and_return(nil) + end + + it 'returns nil' do + expect(badge.status).to eq(nil) + end + end end context 'with a blank job name' do let(:job_name) { ' ' } - it 'returns the pipeline coverage value' do + it 'returns the latest successful pipeline coverage value' do expect(badge.status).to eq(50.00) end end @@ -73,11 +73,27 @@ RSpec.describe Gitlab::Badge::Coverage::Report do end context 'with a matching job name specified' do - let(:job_name) { 'first' } + let(:job_name) { 'coverage' } it 'returns the pipeline coverage value' do expect(badge.status).to eq(40.00) end + + context 'with a more recent running pipeline' do + let!(:another_build) { create(:ci_build, :success, pipeline: running_pipeline, coverage: 20, created_at: 7.seconds.ago, name: 'coverage') } + + it 'returns the running pipeline coverage value' do + expect(badge.status).to eq(20.00) + end + end + + context 'with a more recent failed pipeline' do + let!(:another_build) { create(:ci_build, :success, pipeline: failure_pipeline, coverage: 10, created_at: 6.seconds.ago, name: 'coverage') } + + it 'returns the failed pipeline coverage value' do + expect(badge.status).to eq(10.00) + end + end end end end diff --git a/spec/lib/gitlab/github_import/client_spec.rb b/spec/lib/gitlab/github_import/client_spec.rb index 20e84bacd55..bc734644d29 100644 --- a/spec/lib/gitlab/github_import/client_spec.rb +++ b/spec/lib/gitlab/github_import/client_spec.rb @@ -203,16 +203,40 @@ RSpec.describe Gitlab::GithubImport::Client do describe '#requests_remaining?' do let(:client) { described_class.new('foo') } - it 'returns true if enough requests remain' do - expect(client).to receive(:remaining_requests).and_return(9000) + context 'when default requests limit is set' do + before do + allow(client).to receive(:requests_limit).and_return(5000) + end + + it 'returns true if enough requests remain' do + expect(client).to receive(:remaining_requests).and_return(9000) + + expect(client.requests_remaining?).to eq(true) + end + + it 'returns false if not enough requests remain' do + expect(client).to receive(:remaining_requests).and_return(1) - expect(client.requests_remaining?).to eq(true) + expect(client.requests_remaining?).to eq(false) + end end - it 'returns false if not enough requests remain' do - expect(client).to receive(:remaining_requests).and_return(1) + context 'when search requests limit is set' do + before do + allow(client).to receive(:requests_limit).and_return(described_class::SEARCH_MAX_REQUESTS_PER_MINUTE) + end + + it 'returns true if enough requests remain' do + expect(client).to receive(:remaining_requests).and_return(described_class::SEARCH_RATE_LIMIT_THRESHOLD + 1) + + expect(client.requests_remaining?).to eq(true) + end + + it 'returns false if not enough requests remain' do + expect(client).to receive(:remaining_requests).and_return(described_class::SEARCH_RATE_LIMIT_THRESHOLD - 1) - expect(client.requests_remaining?).to eq(false) + expect(client.requests_remaining?).to eq(false) + end end end @@ -262,6 +286,16 @@ RSpec.describe Gitlab::GithubImport::Client do end end + describe '#requests_limit' do + it 'returns requests limit' do + client = described_class.new('foo') + rate_limit = double(limit: 1) + + expect(client.octokit).to receive(:rate_limit).and_return(rate_limit) + expect(client.requests_limit).to eq(1) + end + end + describe '#rate_limit_resets_in' do it 'returns the number of seconds after which the rate limit is reset' do client = described_class.new('foo') @@ -417,4 +451,61 @@ RSpec.describe Gitlab::GithubImport::Client do expect(client.rate_limiting_enabled?).to eq(false) end end + + describe 'search' do + let(:client) { described_class.new('foo') } + let(:user) { double(:user, login: 'user') } + let(:org1) { double(:org, login: 'org1') } + let(:org2) { double(:org, login: 'org2') } + let(:repo1) { double(:repo, full_name: 'repo1') } + let(:repo2) { double(:repo, full_name: 'repo2') } + + before do + allow(client) + .to receive(:each_object) + .with(:repos, nil, { affiliation: 'collaborator' }) + .and_return([repo1, repo2].to_enum) + + allow(client) + .to receive(:each_object) + .with(:organizations) + .and_return([org1, org2].to_enum) + + allow(client.octokit).to receive(:user).and_return(user) + end + + describe '#search_repos_by_name' do + it 'searches for repositories based on name' do + expected_search_query = 'test in:name is:public,private user:user repo:repo1 repo:repo2 org:org1 org:org2' + + expect(client).to receive(:each_page).with(:search_repositories, expected_search_query) + + client.search_repos_by_name('test') + end + end + + describe '#search_query' do + it 'returns base search query' do + result = client.search_query(str: 'test', type: :test, include_collaborations: false, include_orgs: false) + + expect(result).to eq('test in:test is:public,private user:user') + end + + context 'when include_collaborations is true' do + it 'returns search query including users' do + result = client.search_query(str: 'test', type: :test, include_collaborations: true, include_orgs: false) + + expect(result).to eq('test in:test is:public,private user:user repo:repo1 repo:repo2') + end + end + + context 'when include_orgs is true' do + it 'returns search query including users' do + result = client.search_query(str: 'test', type: :test, include_collaborations: false, include_orgs: true) + + expect(result).to eq('test in:test is:public,private user:user org:org1 org:org2') + end + end + end + end end diff --git a/spec/lib/gitlab/path_regex_spec.rb b/spec/lib/gitlab/path_regex_spec.rb index 9427fdfc0fe..f320b8a66e8 100644 --- a/spec/lib/gitlab/path_regex_spec.rb +++ b/spec/lib/gitlab/path_regex_spec.rb @@ -465,4 +465,34 @@ RSpec.describe Gitlab::PathRegex do it_behaves_like 'invalid snippet routes' end + + describe '.container_image_regex' do + subject { described_class.container_image_regex } + + it { is_expected.to match('gitlab-foss') } + it { is_expected.to match('gitlab_foss') } + it { is_expected.to match('gitlab-org/gitlab-foss') } + it { is_expected.to match('100px.com/100px.ruby') } + + it 'only matches at most one slash' do + expect(subject.match('foo/bar/baz')[0]).to eq('foo/bar') + end + + it 'does not match other non-word characters' do + expect(subject.match('ruby:2.7.0')[0]).to eq('ruby') + end + end + + describe '.container_image_blob_sha_regex' do + subject { described_class.container_image_blob_sha_regex } + + it { is_expected.to match('sha256:asdf1234567890ASDF') } + it { is_expected.to match('foo:123') } + it { is_expected.to match('a12bc3f590szp') } + it { is_expected.not_to match('') } + + it 'does not match malicious characters' do + expect(subject.match('sha256:asdf1234%2f')[0]).to eq('sha256:asdf1234') + end + end end diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index 065e756ea28..1ca370dc950 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -1972,6 +1972,32 @@ RSpec.describe Ci::Pipeline, :mailer, factory_default: :keep do end end + describe '.latest_running_for_ref' do + include_context 'with some outdated pipelines' + + let!(:latest_running_pipeline) do + create_pipeline(:running, 'ref', 'D', project) + end + + it 'returns the latest running pipeline' do + expect(described_class.latest_running_for_ref('ref')) + .to eq(latest_running_pipeline) + end + end + + describe '.latest_failed_for_ref' do + include_context 'with some outdated pipelines' + + let!(:latest_failed_pipeline) do + create_pipeline(:failed, 'ref', 'D', project) + end + + it 'returns the latest failed pipeline' do + expect(described_class.latest_failed_for_ref('ref')) + .to eq(latest_failed_pipeline) + end + end + describe '.latest_successful_for_sha' do include_context 'with some outdated pipelines' diff --git a/spec/models/dependency_proxy/blob_spec.rb b/spec/models/dependency_proxy/blob_spec.rb new file mode 100644 index 00000000000..7c8a1eb95e8 --- /dev/null +++ b/spec/models/dependency_proxy/blob_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe DependencyProxy::Blob, type: :model do + describe 'relationships' do + it { is_expected.to belong_to(:group) } + end + + describe 'validations' do + it { is_expected.to validate_presence_of(:group) } + it { is_expected.to validate_presence_of(:file) } + it { is_expected.to validate_presence_of(:file_name) } + end + + describe '.total_size' do + it 'returns 0 if no files' do + expect(described_class.total_size).to eq(0) + end + + it 'returns a correct sum of all files sizes' do + create(:dependency_proxy_blob, size: 10) + create(:dependency_proxy_blob, size: 20) + + expect(described_class.total_size).to eq(30) + end + end + + describe '.find_or_build' do + let!(:blob) { create(:dependency_proxy_blob) } + + it 'builds new instance if not found' do + expect(described_class.find_or_build('foo.gz')).not_to be_persisted + end + + it 'finds an existing blob' do + expect(described_class.find_or_build(blob.file_name)).to eq(blob) + end + end + + describe 'file is being stored' do + subject { create(:dependency_proxy_blob) } + + context 'when existing object has local store' do + it_behaves_like 'mounted file in local store' + end + + context 'when direct upload is enabled' do + before do + stub_dependency_proxy_object_storage(direct_upload: true) + end + + it_behaves_like 'mounted file in object store' + end + end +end diff --git a/spec/models/dependency_proxy/group_setting_spec.rb b/spec/models/dependency_proxy/group_setting_spec.rb new file mode 100644 index 00000000000..c4c4a877d50 --- /dev/null +++ b/spec/models/dependency_proxy/group_setting_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe DependencyProxy::GroupSetting, type: :model do + describe 'relationships' do + it { is_expected.to belong_to(:group) } + end + + describe 'validations' do + it { is_expected.to validate_presence_of(:group) } + end +end diff --git a/spec/models/dependency_proxy/registry_spec.rb b/spec/models/dependency_proxy/registry_spec.rb new file mode 100644 index 00000000000..5bfa75a2eed --- /dev/null +++ b/spec/models/dependency_proxy/registry_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe DependencyProxy::Registry, type: :model do + let(:tag) { '2.3.5-alpine' } + let(:blob_sha) { '40bd001563085fc35165329ea1ff5c5ecbdbbeef' } + + context 'image name without namespace' do + let(:image) { 'ruby' } + + describe '#auth_url' do + it 'returns a correct auth url' do + expect(described_class.auth_url(image)) + .to eq('https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/ruby:pull') + end + end + + describe '#manifest_url' do + it 'returns a correct manifest url' do + expect(described_class.manifest_url(image, tag)) + .to eq('https://registry-1.docker.io/v2/library/ruby/manifests/2.3.5-alpine') + end + end + + describe '#blob_url' do + it 'returns a correct blob url' do + expect(described_class.blob_url(image, blob_sha)) + .to eq('https://registry-1.docker.io/v2/library/ruby/blobs/40bd001563085fc35165329ea1ff5c5ecbdbbeef') + end + end + end + + context 'image name with namespace' do + let(:image) { 'foo/ruby' } + + describe '#auth_url' do + it 'returns a correct auth url' do + expect(described_class.auth_url(image)) + .to eq('https://auth.docker.io/token?service=registry.docker.io&scope=repository:foo/ruby:pull') + end + end + + describe '#manifest_url' do + it 'returns a correct manifest url' do + expect(described_class.manifest_url(image, tag)) + .to eq('https://registry-1.docker.io/v2/foo/ruby/manifests/2.3.5-alpine') + end + end + + describe '#blob_url' do + it 'returns a correct blob url' do + expect(described_class.blob_url(image, blob_sha)) + .to eq('https://registry-1.docker.io/v2/foo/ruby/blobs/40bd001563085fc35165329ea1ff5c5ecbdbbeef') + end + end + end +end diff --git a/spec/models/group_spec.rb b/spec/models/group_spec.rb index 357238bb8b2..dd1faf999b3 100644 --- a/spec/models/group_spec.rb +++ b/spec/models/group_spec.rb @@ -28,6 +28,8 @@ RSpec.describe Group do it { is_expected.to have_many(:iterations) } it { is_expected.to have_many(:group_deploy_keys) } it { is_expected.to have_many(:services) } + it { is_expected.to have_one(:dependency_proxy_setting) } + it { is_expected.to have_many(:dependency_proxy_blobs) } describe '#members & #requesters' do let(:requester) { create(:user) } diff --git a/spec/requests/api/dependency_proxy_spec.rb b/spec/requests/api/dependency_proxy_spec.rb new file mode 100644 index 00000000000..d59f2bf06e3 --- /dev/null +++ b/spec/requests/api/dependency_proxy_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe API::DependencyProxy, api: true do + include ExclusiveLeaseHelpers + + let_it_be(:user) { create(:user) } + let_it_be(:blob) { create(:dependency_proxy_blob )} + let_it_be(:group, reload: true) { blob.group } + + before do + group.add_owner(user) + stub_config(dependency_proxy: { enabled: true }) + stub_last_activity_update + group.create_dependency_proxy_setting!(enabled: true) + end + + describe 'DELETE /groups/:id/dependency_proxy/cache' do + subject { delete api("/groups/#{group.id}/dependency_proxy/cache", user) } + + context 'with feature available and enabled' do + let_it_be(:lease_key) { "dependency_proxy:delete_group_blobs:#{group.id}" } + + context 'an admin user' do + it 'deletes the blobs and returns no content' do + stub_exclusive_lease(lease_key, timeout: 1.hour) + expect(PurgeDependencyProxyCacheWorker).to receive(:perform_async) + + subject + + expect(response).to have_gitlab_http_status(:no_content) + end + + context 'called multiple times in one hour', :clean_gitlab_redis_shared_state do + it 'returns 409 with an error message' do + stub_exclusive_lease_taken(lease_key, timeout: 1.hour) + + subject + + expect(response).to have_gitlab_http_status(:conflict) + expect(response.body).to include('This request has already been made.') + end + + it 'executes service only for the first time' do + expect(PurgeDependencyProxyCacheWorker).to receive(:perform_async).once + + 2.times { subject } + end + end + end + + context 'a non-admin' do + let(:user) { create(:user) } + + before do + group.add_maintainer(user) + end + + it_behaves_like 'returning response status', :forbidden + end + end + + context 'depencency proxy is not enabled' do + before do + stub_config(dependency_proxy: { enabled: false }) + end + + it_behaves_like 'returning response status', :not_found + end + end +end diff --git a/spec/routing/group_routing_spec.rb b/spec/routing/group_routing_spec.rb index 9de99b73d23..f4d5ccc81b6 100644 --- a/spec/routing/group_routing_spec.rb +++ b/spec/routing/group_routing_spec.rb @@ -79,4 +79,40 @@ RSpec.describe "Groups", "routing" do let(:group_path) { 'projects.abc123' } end end + + describe 'dependency proxy for containers' do + context 'image name without namespace' do + it 'routes to #manifest' do + expect(get('/v2/gitlabhq/dependency_proxy/containers/ruby/manifests/2.3.6')) + .to route_to('groups/dependency_proxy_for_containers#manifest', group_id: 'gitlabhq', image: 'ruby', tag: '2.3.6') + end + + it 'routes to #blob' do + expect(get('/v2/gitlabhq/dependency_proxy/containers/ruby/blobs/abc12345')) + .to route_to('groups/dependency_proxy_for_containers#blob', group_id: 'gitlabhq', image: 'ruby', sha: 'abc12345') + end + + it 'does not route to #blob with an invalid sha' do + expect(get('/v2/gitlabhq/dependency_proxy/containers/ruby/blobs/sha256:asdf1234%2f%2e%2e')) + .not_to route_to(group_id: 'gitlabhq', image: 'ruby', sha: 'sha256:asdf1234%2f%2e%2e') + end + + it 'does not route to #blob with an invalid image' do + expect(get('/v2/gitlabhq/dependency_proxy/containers/ru*by/blobs/abc12345')) + .not_to route_to('groups/dependency_proxy_for_containers#blob', group_id: 'gitlabhq', image: 'ru*by', sha: 'abc12345') + end + end + + context 'image name with namespace' do + it 'routes to #manifest' do + expect(get('/v2/gitlabhq/dependency_proxy/containers/foo/bar/manifests/2.3.6')) + .to route_to('groups/dependency_proxy_for_containers#manifest', group_id: 'gitlabhq', image: 'foo/bar', tag: '2.3.6') + end + + it 'routes to #blob' do + expect(get('/v2/gitlabhq/dependency_proxy/containers/foo/bar/blobs/abc12345')) + .to route_to('groups/dependency_proxy_for_containers#blob', group_id: 'gitlabhq', image: 'foo/bar', sha: 'abc12345') + end + end + end end diff --git a/spec/services/dependency_proxy/download_blob_service_spec.rb b/spec/services/dependency_proxy/download_blob_service_spec.rb new file mode 100644 index 00000000000..4b5c6b5bd6a --- /dev/null +++ b/spec/services/dependency_proxy/download_blob_service_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe DependencyProxy::DownloadBlobService do + include DependencyProxyHelpers + + let(:image) { 'alpine' } + let(:token) { Digest::SHA256.hexdigest('123') } + let(:blob_sha) { Digest::SHA256.hexdigest('ruby:2.7.0') } + + subject { described_class.new(image, blob_sha, token).execute } + + context 'remote request is successful' do + before do + stub_blob_download(image, blob_sha) + end + + it { expect(subject[:status]).to eq(:success) } + it { expect(subject[:file]).to be_a(Tempfile) } + it { expect(subject[:file].size).to eq(6) } + end + + context 'remote request is not found' do + before do + stub_blob_download(image, blob_sha, 404) + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(404) } + it { expect(subject[:message]).to eq('Non-success response code on downloading blob fragment') } + end + + context 'net timeout exception' do + before do + blob_url = DependencyProxy::Registry.blob_url(image, blob_sha) + + stub_full_request(blob_url).to_timeout + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(599) } + it { expect(subject[:message]).to eq('execution expired') } + end +end diff --git a/spec/services/dependency_proxy/find_or_create_blob_service_spec.rb b/spec/services/dependency_proxy/find_or_create_blob_service_spec.rb new file mode 100644 index 00000000000..4ba53d49d38 --- /dev/null +++ b/spec/services/dependency_proxy/find_or_create_blob_service_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe DependencyProxy::FindOrCreateBlobService do + include DependencyProxyHelpers + + let(:blob) { create(:dependency_proxy_blob) } + let(:group) { blob.group } + let(:image) { 'alpine' } + let(:tag) { '3.9' } + let(:token) { Digest::SHA256.hexdigest('123') } + let(:blob_sha) { '40bd001563085fc35165329ea1ff5c5ecbdbbeef' } + + subject { described_class.new(group, image, token, blob_sha).execute } + + before do + stub_registry_auth(image, token) + end + + context 'no cache' do + before do + stub_blob_download(image, blob_sha) + end + + it 'downloads blob from remote registry if there is no cached one' do + expect(subject[:status]).to eq(:success) + expect(subject[:blob]).to be_a(DependencyProxy::Blob) + expect(subject[:blob]).to be_persisted + end + end + + context 'cached blob' do + let(:blob_sha) { blob.file_name.sub('.gz', '') } + + it 'uses cached blob instead of downloading one' do + expect(subject[:status]).to eq(:success) + expect(subject[:blob]).to be_a(DependencyProxy::Blob) + expect(subject[:blob]).to eq(blob) + end + end + + context 'no such blob exists remotely' do + before do + stub_blob_download(image, blob_sha, 404) + end + + it 'returns error message and http status' do + expect(subject[:status]).to eq(:error) + expect(subject[:message]).to eq('Failed to download the blob') + expect(subject[:http_status]).to eq(404) + end + end +end diff --git a/spec/services/dependency_proxy/pull_manifest_service_spec.rb b/spec/services/dependency_proxy/pull_manifest_service_spec.rb new file mode 100644 index 00000000000..030ed9c001b --- /dev/null +++ b/spec/services/dependency_proxy/pull_manifest_service_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe DependencyProxy::PullManifestService do + include DependencyProxyHelpers + + let(:image) { 'alpine' } + let(:tag) { '3.9' } + let(:token) { Digest::SHA256.hexdigest('123') } + let(:manifest) { { foo: 'bar' }.to_json } + + subject { described_class.new(image, tag, token).execute } + + context 'remote request is successful' do + before do + stub_manifest_download(image, tag) + end + + it { expect(subject[:status]).to eq(:success) } + it { expect(subject[:manifest]).to eq(manifest) } + end + + context 'remote request is not found' do + before do + stub_manifest_download(image, tag, 404, 'Not found') + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(404) } + it { expect(subject[:message]).to eq('Not found') } + end + + context 'net timeout exception' do + before do + manifest_link = DependencyProxy::Registry.manifest_url(image, tag) + + stub_full_request(manifest_link).to_timeout + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(599) } + it { expect(subject[:message]).to eq('execution expired') } + end +end diff --git a/spec/services/dependency_proxy/request_token_service_spec.rb b/spec/services/dependency_proxy/request_token_service_spec.rb new file mode 100644 index 00000000000..8b3ba783b8d --- /dev/null +++ b/spec/services/dependency_proxy/request_token_service_spec.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe DependencyProxy::RequestTokenService do + include DependencyProxyHelpers + + let(:image) { 'alpine:3.9' } + let(:token) { Digest::SHA256.hexdigest('123') } + + subject { described_class.new(image).execute } + + context 'remote request is successful' do + before do + stub_registry_auth(image, token) + end + + it { expect(subject[:status]).to eq(:success) } + it { expect(subject[:token]).to eq(token) } + end + + context 'remote request is not found' do + before do + stub_registry_auth(image, token, 404) + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(404) } + it { expect(subject[:message]).to eq('Expected 200 response code for an access token') } + end + + context 'failed to parse response body' do + before do + stub_registry_auth(image, token, 200, 'dasd1321: wow') + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(500) } + it { expect(subject[:message]).to eq('Failed to parse a response body for an access token') } + end + + context 'net timeout exception' do + before do + auth_link = DependencyProxy::Registry.auth_url(image) + + stub_full_request(auth_link, method: :any).to_timeout + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(599) } + it { expect(subject[:message]).to eq('execution expired') } + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 65ab071f179..38e3f851116 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -56,6 +56,7 @@ require_relative('../ee/spec/spec_helper') if Gitlab.ee? # Load these first since they may be required by other helpers require Rails.root.join("spec/support/helpers/git_helpers.rb") +require Rails.root.join("spec/support/helpers/stub_requests.rb") # Then the rest Dir[Rails.root.join("spec/support/helpers/*.rb")].sort.each { |f| require f } diff --git a/spec/support/helpers/dependency_proxy_helpers.rb b/spec/support/helpers/dependency_proxy_helpers.rb new file mode 100644 index 00000000000..545b9d1f4d0 --- /dev/null +++ b/spec/support/helpers/dependency_proxy_helpers.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module DependencyProxyHelpers + include StubRequests + + def stub_registry_auth(image, token, status = 200, body = nil) + auth_body = { 'token' => token }.to_json + auth_link = registry.auth_url(image) + + stub_full_request(auth_link) + .to_return(status: status, body: body || auth_body) + end + + def stub_manifest_download(image, tag, status = 200, body = nil) + manifest_url = registry.manifest_url(image, tag) + + stub_full_request(manifest_url) + .to_return(status: status, body: body || manifest) + end + + def stub_blob_download(image, blob_sha, status = 200, body = '123456') + download_link = registry.blob_url(image, blob_sha) + + stub_full_request(download_link) + .to_return(status: status, body: body) + end + + private + + def registry + @registry ||= DependencyProxy::Registry + end +end diff --git a/spec/support/helpers/navbar_structure_helper.rb b/spec/support/helpers/navbar_structure_helper.rb index 11e67ba394c..e18a708e41c 100644 --- a/spec/support/helpers/navbar_structure_helper.rb +++ b/spec/support/helpers/navbar_structure_helper.rb @@ -36,4 +36,12 @@ module NavbarStructureHelper new_sub_nav_item_name: _('Container Registry') ) end + + def insert_dependency_proxy_nav(within) + insert_after_sub_nav_item( + _('Package Registry'), + within: _('Packages & Registries'), + new_sub_nav_item_name: _('Dependency Proxy') + ) + end end diff --git a/spec/support/shared_examples/controllers/githubish_import_controller_shared_examples.rb b/spec/support/shared_examples/controllers/githubish_import_controller_shared_examples.rb index 93f58a6ca0c..5a4322f73b6 100644 --- a/spec/support/shared_examples/controllers/githubish_import_controller_shared_examples.rb +++ b/spec/support/shared_examples/controllers/githubish_import_controller_shared_examples.rb @@ -145,6 +145,8 @@ RSpec.shared_examples 'a GitHub-ish import controller: GET status' do group.add_owner(user) client = stub_client(repos: repos, orgs: [org], org_repos: [org_repo]) allow(client).to receive(:each_page).and_return([OpenStruct.new(objects: repos)].to_enum) + # GitHub controller has filtering done using GitHub Search API + stub_feature_flags(remove_legacy_github_client: false) end it 'filters list of repositories by name' do diff --git a/spec/uploaders/dependency_proxy/file_uploader_spec.rb b/spec/uploaders/dependency_proxy/file_uploader_spec.rb new file mode 100644 index 00000000000..724a9c42f47 --- /dev/null +++ b/spec/uploaders/dependency_proxy/file_uploader_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe DependencyProxy::FileUploader do + let(:blob) { create(:dependency_proxy_blob) } + let(:uploader) { described_class.new(blob, :file) } + let(:path) { Gitlab.config.dependency_proxy.storage_path } + + subject { uploader } + + it_behaves_like "builds correct paths", + store_dir: %r[\h{2}/\h{2}], + cache_dir: %r[/dependency_proxy/tmp/cache], + work_dir: %r[/dependency_proxy/tmp/work] + + context 'object store is remote' do + before do + stub_dependency_proxy_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like "builds correct paths", + store_dir: %r[\h{2}/\h{2}] + end +end diff --git a/spec/workers/purge_dependency_proxy_cache_worker_spec.rb b/spec/workers/purge_dependency_proxy_cache_worker_spec.rb new file mode 100644 index 00000000000..9cd3b6636f5 --- /dev/null +++ b/spec/workers/purge_dependency_proxy_cache_worker_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe PurgeDependencyProxyCacheWorker do + let_it_be(:user) { create(:admin) } + let_it_be(:blob) { create(:dependency_proxy_blob )} + let_it_be(:group, reload: true) { blob.group } + let_it_be(:group_id) { group.id } + + subject { described_class.new.perform(user.id, group_id) } + + before do + stub_config(dependency_proxy: { enabled: true }) + group.create_dependency_proxy_setting!(enabled: true) + end + + describe '#perform' do + shared_examples 'returns nil' do + it 'returns nil' do + expect { subject }.not_to change { group.dependency_proxy_blobs.size } + expect(subject).to be_nil + end + end + + context 'an admin user' do + include_examples 'an idempotent worker' do + let(:job_args) { [user.id, group_id] } + + it 'deletes the blobs and returns ok' do + expect(group.dependency_proxy_blobs.size).to eq(1) + + subject + + expect(group.dependency_proxy_blobs.size).to eq(0) + end + end + end + + context 'a non-admin user' do + let(:user) { create(:user) } + + it_behaves_like 'returns nil' + end + + context 'an invalid user id' do + let(:user) { double('User', id: 99999 ) } + + it_behaves_like 'returns nil' + end + + context 'an invalid group' do + let(:group_id) { 99999 } + + it_behaves_like 'returns nil' + end + end +end diff --git a/yarn.lock b/yarn.lock index a18b1f9beaf..91f08a59712 100644 --- a/yarn.lock +++ b/yarn.lock @@ -845,10 +845,10 @@ resolved "https://registry.yarnpkg.com/@gitlab/at.js/-/at.js-1.5.5.tgz#5f6bfe6baaef360daa9b038fa78798d7a6a916b4" integrity sha512-282Dn3SPVsUHVDhMsXgfnv+Rzog0uxecjttxGRQvxh25es1+xvkGQFsvJfkSKJ3X1kHVkSjKf+Tt5Rra+Jhp9g== -"@gitlab/eslint-plugin@4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@gitlab/eslint-plugin/-/eslint-plugin-4.1.0.tgz#b59b53b9cd984dc692cd094cca4fbe45e466c8b7" - integrity sha512-NjY8XqT9lPKxBiKeaIXAGPGuxLMmbns/W5YF/cqxfWMnMOCCGBPZjwwnPY5wzAeKocT014LCnA7k0ztIUN7DoQ== +"@gitlab/eslint-plugin@5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@gitlab/eslint-plugin/-/eslint-plugin-5.0.0.tgz#502eb2bccb55d65d6310ce9ef2da76035b6fc319" + integrity sha512-zd4pa6D2OUuhPUD2QmyhfpZh7vuXKsNHaCHJdbTb2ld+mHel5IDqdidAzAshCI9On3e6o9XieD6l7rBTpN6H/g== dependencies: babel-eslint "^10.0.3" eslint-config-airbnb-base "^14.0.0" |
