diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-19 14:50:12 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-19 14:50:12 +0300 |
commit | 6cd5b7dbfaa4ff630ecbbfe351a1faac5fc71a8d (patch) | |
tree | d3563b9f60936c18a02185e2e53b424bb1b83539 /lib | |
parent | b3e0658cb1fbc7c8e7dd381467c656f2e675ee46 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/project_container_repositories.rb | 12 | ||||
-rw-r--r-- | lib/container_registry/client.rb | 51 | ||||
-rw-r--r-- | lib/container_registry/tag.rb | 9 | ||||
-rw-r--r-- | lib/gitlab/ee_compat_check.rb | 444 | ||||
-rw-r--r-- | lib/tasks/ee_compat_check.rake | 4 | ||||
-rw-r--r-- | lib/tasks/gitlab/dev.rake | 31 | ||||
-rw-r--r-- | lib/tasks/lint.rake | 15 |
7 files changed, 80 insertions, 486 deletions
diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index c10ef96922c..2a05974509a 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -106,9 +106,15 @@ module API authorize_destroy_container_image! validate_tag! - tag.delete - - status :ok + result = ::Projects::ContainerRepository::DeleteTagsService + .new(repository.project, current_user, tags: [declared_params[:tag_name]]) + .execute(repository) + + if result[:status] == :success + status :ok + else + status :bad_request + end end end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 15f40993ea3..2bd8eb65306 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -2,6 +2,7 @@ require 'faraday' require 'faraday_middleware' +require 'digest' module ContainerRegistry class Client @@ -9,6 +10,8 @@ module ContainerRegistry DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json' OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json' + CONTAINER_IMAGE_V1_TYPE = 'application/vnd.docker.container.image.v1+json' + ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze # Taken from: FaradayMiddleware::FollowRedirects @@ -36,6 +39,45 @@ module ContainerRegistry faraday.delete("/v2/#{name}/manifests/#{reference}").success? end + def upload_raw_blob(path, blob) + digest = "sha256:#{Digest::SHA256.hexdigest(blob)}" + + if upload_blob(path, blob, digest).success? + [blob, digest] + end + end + + def upload_blob(name, content, digest) + upload = faraday.post("/v2/#{name}/blobs/uploads/") + return unless upload.success? + + location = URI(upload.headers['location']) + + faraday.put("#{location.path}?#{location.query}") do |req| + req.params['digest'] = digest + req.headers['Content-Type'] = 'application/octet-stream' + req.body = content + end + end + + def generate_empty_manifest(path) + image = { + config: {} + } + image, image_digest = upload_raw_blob(path, JSON.pretty_generate(image)) + return unless image + + { + schemaVersion: 2, + mediaType: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, + config: { + mediaType: CONTAINER_IMAGE_V1_TYPE, + size: image.size, + digest: image_digest + } + } + end + def blob(name, digest, type = nil) type ||= 'application/octet-stream' response_body faraday_blob.get("/v2/#{name}/blobs/#{digest}", nil, 'Accept' => type), allow_redirect: true @@ -45,6 +87,15 @@ module ContainerRegistry faraday.delete("/v2/#{name}/blobs/#{digest}").success? end + def put_tag(name, reference, manifest) + response = faraday.put("/v2/#{name}/manifests/#{reference}") do |req| + req.headers['Content-Type'] = DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE + req.body = JSON.pretty_generate(manifest) + end + + response.headers['docker-content-digest'] if response.success? + end + private def initialize_connection(conn, options) diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index ebea84fa1ca..2cc4c8d8b1c 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -98,6 +98,10 @@ module ContainerRegistry end end + def put(digests) + repository.client.put_tag(repository.path, name, digests) + end + # rubocop: disable CodeReuse/ActiveRecord def total_size return unless layers @@ -106,7 +110,10 @@ module ContainerRegistry end # rubocop: enable CodeReuse/ActiveRecord - def delete + # Deletes the image associated with this tag + # Note this will delete the image and all tags associated with it. + # Consider using DeleteTagsService instead. + def unsafe_delete return unless digest client.delete_repository_tag(repository.path, digest) diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb deleted file mode 100644 index 707e45dfb71..00000000000 --- a/lib/gitlab/ee_compat_check.rb +++ /dev/null @@ -1,444 +0,0 @@ -# frozen_string_literal: true - -# rubocop: disable Rails/Output -module Gitlab - # Checks if a set of migrations requires downtime or not. - class EeCompatCheck - CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-foss' - CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab.git' - CHECK_DIR = Rails.root.join('ee_compat_check') - IGNORED_FILES_REGEX = /VERSION|CHANGELOG\.md|doc\/.+/i.freeze - PLEASE_READ_THIS_BANNER = %Q{ - ============================================================ - ===================== PLEASE READ THIS ===================== - ============================================================ - }.freeze - STAY_STRONG_LINK_TO_DOCS = %Q{ - Stay 💪! For more information, see - https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html - }.freeze - THANKS_FOR_READING_BANNER = %Q{ - ============================================================ - ==================== THANKS FOR READING ==================== - ============================================================\n - }.freeze - - attr_reader :ee_repo_dir, :patches_dir - attr_reader :ce_project_url, :ee_repo_url - attr_reader :ce_branch, :ee_remote_with_branch, :ee_branch_found - attr_reader :job_id, :failed_files - - def initialize(branch:, ce_project_url: CANONICAL_CE_PROJECT_URL, job_id: nil) - @ee_repo_dir = CHECK_DIR.join('ee-repo') - @patches_dir = CHECK_DIR.join('patches') - @ce_branch = branch - @ce_project_url = ce_project_url - @ee_repo_url = ce_public_repo_url.sub('gitlab-ce', 'gitlab-ee') - @job_id = job_id - end - - def check - ensure_patches_dir - # We're generating the patch against the canonical-ce remote since forks' - # master branch are not necessarily up-to-date. - add_remote('canonical-ce', "#{CANONICAL_CE_PROJECT_URL}.git") - generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, branch_remote: 'origin', master_remote: 'canonical-ce') - - ensure_ee_repo - Dir.chdir(ee_repo_dir) do - step("In the #{ee_repo_dir} directory") - - ee_remotes.each do |key, url| - add_remote(key, url) - end - fetch(branch: 'master', depth: 20, remote: 'canonical-ee') - - status = catch(:halt_check) do - ce_branch_compat_check! - delete_ee_branches_locally! - ee_branch_presence_check! - - step("Checking out #{ee_remote_with_branch}/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} #{ee_remote_with_branch}/#{ee_branch_found}]) - generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, branch_remote: ee_remote_with_branch, master_remote: 'canonical-ee') - ee_branch_compat_check! - end - - delete_ee_branches_locally! - - status.nil? - end - end - - private - - def fork? - ce_project_url != CANONICAL_CE_PROJECT_URL - end - - def ee_remotes - return @ee_remotes if defined?(@ee_remotes) - - remotes = - { - 'ee' => ee_repo_url, - 'canonical-ee' => CANONICAL_EE_REPO_URL - } - remotes.delete('ee') unless fork? - - @ee_remotes = remotes - end - - def add_remote(name, url) - step( - "Adding the #{name} remote (#{url})", - %W[git remote add #{name} #{url}] - ) - end - - def ensure_ee_repo - unless clone_repo(ee_repo_url, ee_repo_dir) - # Fallback to using the canonical EE if there is no forked EE - clone_repo(CANONICAL_EE_REPO_URL, ee_repo_dir) - end - end - - def clone_repo(url, dir) - _, status = step( - "Cloning #{url} into #{dir}", - %W[git clone --branch master --single-branch --depth=200 #{url} #{dir}] - ) - status.zero? - end - - def ensure_patches_dir - FileUtils.mkdir_p(patches_dir) - end - - def generate_patch(branch:, patch_path:, branch_remote:, master_remote:) - FileUtils.rm(patch_path, force: true) - - find_merge_base_with_master(branch: branch, branch_remote: branch_remote, master_remote: master_remote) - - step( - "Generating the patch against #{master_remote}/master in #{patch_path}", - %W[git diff --binary #{master_remote}/master...#{branch_remote}/#{branch}] - ) do |output, status| - throw(:halt_check, :ko) unless status.zero? - - File.write(patch_path, output) - - throw(:halt_check, :ko) unless File.exist?(patch_path) - end - end - - def ce_branch_compat_check! - if check_patch(ce_patch_full_path).zero? - puts applies_cleanly_msg(ce_branch) - throw(:halt_check) - end - end - - def ee_branch_presence_check! - ee_remotes.keys.each do |remote| - output, _ = step( - "Searching #{remote}", - %W[git ls-remote #{remote} *#{minimal_ee_branch_name}*]) - - branches = - output.scan(%r{(?<=refs/heads/|refs/tags/).+}).sort_by(&:size) - - next if branches.empty? - - branch = branches.first - - step("Fetching #{remote}/#{branch}", %W[git fetch #{remote} #{branch}]) - - @ee_remote_with_branch = remote - @ee_branch_found = branch - - return true - end - - puts - puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg - - throw(:halt_check, :ko) - end - - def ee_branch_compat_check! - unless check_patch(ee_patch_full_path).zero? - puts - puts ee_branch_doesnt_apply_cleanly_msg - - throw(:halt_check, :ko) - end - - puts - puts applies_cleanly_msg(ee_branch_found) - end - - def check_patch(patch_path) - step("Checking out master", %w[git checkout master]) - step("Resetting to latest master", %w[git reset --hard canonical-ee/master]) - step( - "Checking if #{patch_path} applies cleanly to EE/master", - # Don't use --check here because it can result in a 0-exit status even - # though the patch doesn't apply cleanly, e.g.: - # > git apply --check --3way foo.patch - # error: patch failed: lib/gitlab/ee_compat_check.rb:74 - # Falling back to three-way merge... - # Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts. - # > echo $? - # 0 - %W[git apply --3way #{patch_path}] - ) do |output, status| - puts output - - unless status.zero? - @failed_files = output.lines.reduce([]) do |memo, line| - if line.start_with?('error: patch failed:') - file = line.sub(/\Aerror: patch failed: /, '') - memo << file unless file =~ IGNORED_FILES_REGEX - end - - memo - end - - status = 0 if failed_files.empty? - end - - command(%w[git reset --hard]) - status - end - end - - def delete_ee_branches_locally! - command(%w[git checkout master]) - command(%W[git branch --delete --force #{ee_branch_prefix}]) - command(%W[git branch --delete --force #{ee_branch_suffix}]) - end - - def merge_base_found?(branch:, branch_remote:, master_remote:) - step( - "Finding merge base with #{master_remote}/master", - %W[git merge-base #{master_remote}/master #{branch_remote}/#{branch}] - ) do |output, status| - if status.zero? - puts "Merge base was found: #{output}" - true - end - end - end - - def find_merge_base_with_master(branch:, branch_remote:, master_remote:) - # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403) - # In total we go (20 + 54 + 148 + 403 = 625) commits deeper - depth = 20 - success = - (3..6).any? do |factor| - depth += Math.exp(factor).to_i - # Repository is initially cloned with a depth of 20 so we need to fetch - # deeper in the case the branch has more than 20 commits on top of master - fetch(branch: branch, depth: depth, remote: branch_remote) - fetch(branch: 'master', depth: depth, remote: master_remote) - - merge_base_found?(branch: branch, branch_remote: branch_remote, master_remote: master_remote) - end - - raise "\n#{branch} is too far behind #{master_remote}/master, please rebase it!\n" unless success - end - - def fetch(branch:, depth:, remote: 'origin') - step( - "Fetching deeper...", - %W[git fetch --depth=#{depth} --prune #{remote} +refs/heads/#{branch}:refs/remotes/#{remote}/#{branch}] - ) do |output, status| - raise "Fetch failed: #{output}" unless status.zero? - end - end - - def ce_patch_name - @ce_patch_name ||= patch_name_from_branch(ce_branch) - end - - def ce_patch_full_path - @ce_patch_full_path ||= patches_dir.join(ce_patch_name) - end - - def ee_branch_suffix - @ee_branch_suffix ||= "#{ce_branch}-ee" - end - - def ee_branch_prefix - @ee_branch_prefix ||= "ee-#{ce_branch}" - end - - def ee_patch_name - @ee_patch_name ||= patch_name_from_branch(ee_branch_found) - end - - def ee_patch_full_path - @ee_patch_full_path ||= patches_dir.join(ee_patch_name) - end - - def minimal_ee_branch_name - @minimal_ee_branch_name ||= ce_branch.sub(/(\Ace\-|\-ce\z)/, '') - end - - def patch_name_from_branch(branch_name) - "#{branch_name.parameterize}.patch" - end - - def patch_url - "#{ce_project_url}/-/jobs/#{job_id}/artifacts/raw/ee_compat_check/patches/#{ce_patch_name}" - end - - def step(desc, cmd = nil) - puts "\n=> #{desc}\n" - - if cmd - start = Time.now - puts "\n$ #{cmd.join(' ')}" - - output, status = command(cmd) - puts "\n==> Finished in #{Time.now - start} seconds" - - if block_given? - yield(output, status) - else - [output, status] - end - end - end - - def command(cmd) - Gitlab::Popen.popen(cmd) - end - - # We're "re-creating" the repo URL because ENV['CI_REPOSITORY_URL'] contains - # redacted credentials (e.g. "***:****") which are useless in instructions - # the job gives. - def ce_public_repo_url - "#{ce_project_url}.git" - end - - def applies_cleanly_msg(branch) - %Q{ - #{PLEASE_READ_THIS_BANNER} - 🎉 Congratulations!! 🎉 - - The `#{branch}` branch applies cleanly to EE/master! - - Much ❤️! For more information, see - https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html - #{THANKS_FOR_READING_BANNER} - } - end - - def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg - ee_repos = ee_remotes.values.uniq - - %Q{ - #{PLEASE_READ_THIS_BANNER} - 💥 Oh no! 💥 - - The `#{ce_branch}` branch does not apply cleanly to the current - EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch - was found in #{ee_repos.join(' nor in ')}. - - If you're a community contributor, don't worry, someone from - GitLab Inc. will take care of this, and you don't have to do anything. - If you're willing to help, and are ok to contribute to EE as well, - you're welcome to help. You could follow the instructions below. - - #{conflicting_files_msg} - - We advise you to create a `#{ee_branch_prefix}` or `#{ee_branch_suffix}` - branch that includes changes from `#{ce_branch}` but also specific changes - than can be applied cleanly to EE/master. In some cases, the conflicts - are trivial and you can ignore the warning from this job. As always, - use your best judgement! - - There are different ways to create such branch: - - 1. Create a new branch from master and cherry-pick your CE commits - - # In the EE repo - $ git fetch #{CANONICAL_EE_REPO_URL} master - $ git checkout -b #{ee_branch_prefix} FETCH_HEAD - $ git fetch #{ce_public_repo_url} #{ce_branch} - $ git cherry-pick SHA # Repeat for all the commits you want to pick - - Note: You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit. - - 2. Apply your branch's patch to EE - - # In the EE repo - $ git fetch #{CANONICAL_EE_REPO_URL} master - $ git checkout -b #{ee_branch_prefix} FETCH_HEAD - $ wget #{patch_url} && git apply --3way #{ce_patch_name} - - At this point you might have conflicts such as: - - error: patch failed: lib/gitlab/ee_compat_check.rb:5 - Falling back to three-way merge... - Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts. - U lib/gitlab/ee_compat_check.rb - - Resolve them, stage the changes and commit them. - - If the patch couldn't be applied cleanly, use the following command: - - # In the EE repo - $ git apply --reject #{ce_patch_name} - - This option makes git apply the parts of the patch that are applicable, - and leave the rejected hunks in corresponding `.rej` files. - You can then resolve the conflicts highlighted in `.rej` by - manually applying the correct diff from the `.rej` file to the file with conflicts. - When finished, you can delete the `.rej` files and commit your changes. - - ⚠️ Don't forget to push your branch to gitlab-ee: - - # In the EE repo - $ git push origin #{ee_branch_prefix} - - ⚠️ Also, don't forget to create a new merge request on gitlab-ee and - cross-link it with the CE merge request. - - Once this is done, you can retry this failed job, and it should pass. - - #{STAY_STRONG_LINK_TO_DOCS} - #{THANKS_FOR_READING_BANNER} - } - end - - def ee_branch_doesnt_apply_cleanly_msg - %Q{ - #{PLEASE_READ_THIS_BANNER} - 💥 Oh no! 💥 - - The `#{ce_branch}` does not apply cleanly to the current EE/master, and - even though a `#{ee_branch_found}` branch - exists in #{ee_repo_url}, it does not apply cleanly either to - EE/master! - - #{conflicting_files_msg} - - Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and - retry this job. - - #{STAY_STRONG_LINK_TO_DOCS} - #{THANKS_FOR_READING_BANNER} - } - end - - def conflicting_files_msg - header = "The conflicts detected were as follows:\n" - separator = "\n - " - failed_items = failed_files.join(separator) - - "#{header}#{separator}#{failed_items}" - end - end -end diff --git a/lib/tasks/ee_compat_check.rake b/lib/tasks/ee_compat_check.rake deleted file mode 100644 index f494fa5c5c2..00000000000 --- a/lib/tasks/ee_compat_check.rake +++ /dev/null @@ -1,4 +0,0 @@ -desc 'Checks if the branch would apply cleanly to EE' -task ee_compat_check: :environment do - Rake::Task['gitlab:dev:ee_compat_check'].invoke -end diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake deleted file mode 100644 index 17d3ac74375..00000000000 --- a/lib/tasks/gitlab/dev.rake +++ /dev/null @@ -1,31 +0,0 @@ -namespace :gitlab do - namespace :dev do - desc 'Checks if the branch would apply cleanly to EE' - task :ee_compat_check, [:branch] => :environment do |_, args| - opts = - if ENV['CI'] - { - ce_project_url: ENV['CI_PROJECT_URL'], - branch: ENV['CI_COMMIT_REF_NAME'], - job_id: ENV['CI_JOB_ID'] - } - else - unless args[:branch] - puts "Must specify a branch as an argument".color(:red) - exit 1 - end - - args - end - - if File.basename(Rails.root) == 'gitlab' - puts "Skipping EE projects" - exit 0 - elsif Gitlab::EeCompatCheck.new(opts || {}).check - exit 0 - else - exit 1 - end - end - end -end diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 2353b2dc659..9a5693e78a2 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -28,14 +28,23 @@ unless Rails.env.production? task :all do status = 0 - %w[ + tasks = %w[ config_lint lint:haml scss_lint gettext:lint - gettext:updated_check lint:static_verification - ].each do |task| + ] + + if Gitlab.ee? + # This task will fail on CE installations (e.g. gitlab-org/gitlab-foss) + # since it will detect strings in the locale files that do not exist in + # the source files. To work around this we will only enable this task on + # EE installations. + tasks << 'gettext:updated_check' + end + + tasks.each do |task| pid = Process.fork do puts "*** Running rake task: #{task} ***" |