Add latest changes from gitlab-org/gitlab@master

7 files changed, 80 insertions, 486 deletions

diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb
index c10ef96922c..2a05974509a 100644
--- a/lib/api/project_container_repositories.rb
+++ b/lib/api/project_container_repositories.rb
@@ -106,9 +106,15 @@ module API
         authorize_destroy_container_image!
         validate_tag!
 
-        tag.delete
-
-        status :ok
+        result = ::Projects::ContainerRepository::DeleteTagsService
+          .new(repository.project, current_user, tags: [declared_params[:tag_name]])
+          .execute(repository)
+
+        if result[:status] == :success
+          status :ok
+        else
+          status :bad_request
+        end
       end
     end
 
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
index 15f40993ea3..2bd8eb65306 100644
--- a/lib/container_registry/client.rb
+++ b/lib/container_registry/client.rb
@@ -2,6 +2,7 @@
 
 require 'faraday'
 require 'faraday_middleware'
+require 'digest'
 
 module ContainerRegistry
   class Client
@@ -9,6 +10,8 @@ module ContainerRegistry
 
     DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json'
     OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json'
+    CONTAINER_IMAGE_V1_TYPE = 'application/vnd.docker.container.image.v1+json'
+
     ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze
 
     # Taken from: FaradayMiddleware::FollowRedirects
@@ -36,6 +39,45 @@ module ContainerRegistry
       faraday.delete("/v2/#{name}/manifests/#{reference}").success?
     end
 
+    def upload_raw_blob(path, blob)
+      digest = "sha256:#{Digest::SHA256.hexdigest(blob)}"
+
+      if upload_blob(path, blob, digest).success?
+        [blob, digest]
+      end
+    end
+
+    def upload_blob(name, content, digest)
+      upload = faraday.post("/v2/#{name}/blobs/uploads/")
+      return unless upload.success?
+
+      location = URI(upload.headers['location'])
+
+      faraday.put("#{location.path}?#{location.query}") do |req|
+        req.params['digest'] = digest
+        req.headers['Content-Type'] = 'application/octet-stream'
+        req.body = content
+      end
+    end
+
+    def generate_empty_manifest(path)
+      image = {
+        config: {}
+      }
+      image, image_digest = upload_raw_blob(path, JSON.pretty_generate(image))
+      return unless image
+
+      {
+        schemaVersion: 2,
+        mediaType: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE,
+        config: {
+          mediaType: CONTAINER_IMAGE_V1_TYPE,
+          size: image.size,
+          digest: image_digest
+        }
+      }
+    end
+
     def blob(name, digest, type = nil)
       type ||= 'application/octet-stream'
       response_body faraday_blob.get("/v2/#{name}/blobs/#{digest}", nil, 'Accept' => type), allow_redirect: true
@@ -45,6 +87,15 @@ module ContainerRegistry
       faraday.delete("/v2/#{name}/blobs/#{digest}").success?
     end
 
+    def put_tag(name, reference, manifest)
+      response = faraday.put("/v2/#{name}/manifests/#{reference}") do |req|
+        req.headers['Content-Type'] = DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE
+        req.body = JSON.pretty_generate(manifest)
+      end
+
+      response.headers['docker-content-digest'] if response.success?
+    end
+
     private
 
     def initialize_connection(conn, options)
diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb
index ebea84fa1ca..2cc4c8d8b1c 100644
--- a/lib/container_registry/tag.rb
+++ b/lib/container_registry/tag.rb
@@ -98,6 +98,10 @@ module ContainerRegistry
       end
     end
 
+    def put(digests)
+      repository.client.put_tag(repository.path, name, digests)
+    end
+
     # rubocop: disable CodeReuse/ActiveRecord
     def total_size
       return unless layers
@@ -106,7 +110,10 @@ module ContainerRegistry
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
-    def delete
+    # Deletes the image associated with this tag
+    # Note this will delete the image and all tags associated with it.
+    # Consider using DeleteTagsService instead.
+    def unsafe_delete
       return unless digest
 
       client.delete_repository_tag(repository.path, digest)
diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb
deleted file mode 100644
index 707e45dfb71..00000000000
--- a/lib/gitlab/ee_compat_check.rb
+++ /dev/null
@@ -1,444 +0,0 @@
-# frozen_string_literal: true
-
-# rubocop: disable Rails/Output
-module Gitlab
-  # Checks if a set of migrations requires downtime or not.
-  class EeCompatCheck
-    CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-foss'
-    CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab.git'
-    CHECK_DIR = Rails.root.join('ee_compat_check')
-    IGNORED_FILES_REGEX = /VERSION|CHANGELOG\.md|doc\/.+/i.freeze
-    PLEASE_READ_THIS_BANNER = %Q{
-      ============================================================
-      ===================== PLEASE READ THIS =====================
-      ============================================================
-    }.freeze
-    STAY_STRONG_LINK_TO_DOCS = %Q{
-      Stay 💪! For more information, see
-      https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html
-    }.freeze
-    THANKS_FOR_READING_BANNER = %Q{
-      ============================================================
-      ==================== THANKS FOR READING ====================
-      ============================================================\n
-    }.freeze
-
-    attr_reader :ee_repo_dir, :patches_dir
-    attr_reader :ce_project_url, :ee_repo_url
-    attr_reader :ce_branch, :ee_remote_with_branch, :ee_branch_found
-    attr_reader :job_id, :failed_files
-
-    def initialize(branch:, ce_project_url: CANONICAL_CE_PROJECT_URL, job_id: nil)
-      @ee_repo_dir = CHECK_DIR.join('ee-repo')
-      @patches_dir = CHECK_DIR.join('patches')
-      @ce_branch = branch
-      @ce_project_url = ce_project_url
-      @ee_repo_url = ce_public_repo_url.sub('gitlab-ce', 'gitlab-ee')
-      @job_id = job_id
-    end
-
-    def check
-      ensure_patches_dir
-      # We're generating the patch against the canonical-ce remote since forks'
-      # master branch are not necessarily up-to-date.
-      add_remote('canonical-ce', "#{CANONICAL_CE_PROJECT_URL}.git")
-      generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, branch_remote: 'origin', master_remote: 'canonical-ce')
-
-      ensure_ee_repo
-      Dir.chdir(ee_repo_dir) do
-        step("In the #{ee_repo_dir} directory")
-
-        ee_remotes.each do |key, url|
-          add_remote(key, url)
-        end
-        fetch(branch: 'master', depth: 20, remote: 'canonical-ee')
-
-        status = catch(:halt_check) do
-          ce_branch_compat_check!
-          delete_ee_branches_locally!
-          ee_branch_presence_check!
-
-          step("Checking out #{ee_remote_with_branch}/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} #{ee_remote_with_branch}/#{ee_branch_found}])
-          generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, branch_remote: ee_remote_with_branch, master_remote: 'canonical-ee')
-          ee_branch_compat_check!
-        end
-
-        delete_ee_branches_locally!
-
-        status.nil?
-      end
-    end
-
-    private
-
-    def fork?
-      ce_project_url != CANONICAL_CE_PROJECT_URL
-    end
-
-    def ee_remotes
-      return @ee_remotes if defined?(@ee_remotes)
-
-      remotes =
-        {
-          'ee' => ee_repo_url,
-          'canonical-ee' => CANONICAL_EE_REPO_URL
-        }
-      remotes.delete('ee') unless fork?
-
-      @ee_remotes = remotes
-    end
-
-    def add_remote(name, url)
-      step(
-        "Adding the #{name} remote (#{url})",
-        %W[git remote add #{name} #{url}]
-      )
-    end
-
-    def ensure_ee_repo
-      unless clone_repo(ee_repo_url, ee_repo_dir)
-        # Fallback to using the canonical EE if there is no forked EE
-        clone_repo(CANONICAL_EE_REPO_URL, ee_repo_dir)
-      end
-    end
-
-    def clone_repo(url, dir)
-      _, status = step(
-        "Cloning #{url} into #{dir}",
-        %W[git clone --branch master --single-branch --depth=200 #{url} #{dir}]
-      )
-      status.zero?
-    end
-
-    def ensure_patches_dir
-      FileUtils.mkdir_p(patches_dir)
-    end
-
-    def generate_patch(branch:, patch_path:, branch_remote:, master_remote:)
-      FileUtils.rm(patch_path, force: true)
-
-      find_merge_base_with_master(branch: branch, branch_remote: branch_remote, master_remote: master_remote)
-
-      step(
-        "Generating the patch against #{master_remote}/master in #{patch_path}",
-        %W[git diff --binary #{master_remote}/master...#{branch_remote}/#{branch}]
-      ) do |output, status|
-        throw(:halt_check, :ko) unless status.zero?
-
-        File.write(patch_path, output)
-
-        throw(:halt_check, :ko) unless File.exist?(patch_path)
-      end
-    end
-
-    def ce_branch_compat_check!
-      if check_patch(ce_patch_full_path).zero?
-        puts applies_cleanly_msg(ce_branch)
-        throw(:halt_check)
-      end
-    end
-
-    def ee_branch_presence_check!
-      ee_remotes.keys.each do |remote|
-        output, _ = step(
-          "Searching #{remote}",
-          %W[git ls-remote #{remote} *#{minimal_ee_branch_name}*])
-
-        branches =
-          output.scan(%r{(?<=refs/heads/|refs/tags/).+}).sort_by(&:size)
-
-        next if branches.empty?
-
-        branch = branches.first
-
-        step("Fetching #{remote}/#{branch}", %W[git fetch #{remote} #{branch}])
-
-        @ee_remote_with_branch = remote
-        @ee_branch_found = branch
-
-        return true
-      end
-
-      puts
-      puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
-
-      throw(:halt_check, :ko)
-    end
-
-    def ee_branch_compat_check!
-      unless check_patch(ee_patch_full_path).zero?
-        puts
-        puts ee_branch_doesnt_apply_cleanly_msg
-
-        throw(:halt_check, :ko)
-      end
-
-      puts
-      puts applies_cleanly_msg(ee_branch_found)
-    end
-
-    def check_patch(patch_path)
-      step("Checking out master", %w[git checkout master])
-      step("Resetting to latest master", %w[git reset --hard canonical-ee/master])
-      step(
-        "Checking if #{patch_path} applies cleanly to EE/master",
-        # Don't use --check here because it can result in a 0-exit status even
-        # though the patch doesn't apply cleanly, e.g.:
-        #   > git apply --check --3way foo.patch
-        #   error: patch failed: lib/gitlab/ee_compat_check.rb:74
-        #   Falling back to three-way merge...
-        #   Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts.
-        #   > echo $?
-        #   0
-        %W[git apply --3way #{patch_path}]
-      ) do |output, status|
-        puts output
-
-        unless status.zero?
-          @failed_files = output.lines.reduce([]) do |memo, line|
-            if line.start_with?('error: patch failed:')
-              file = line.sub(/\Aerror: patch failed: /, '')
-              memo << file unless file =~ IGNORED_FILES_REGEX
-            end
-
-            memo
-          end
-
-          status = 0 if failed_files.empty?
-        end
-
-        command(%w[git reset --hard])
-        status
-      end
-    end
-
-    def delete_ee_branches_locally!
-      command(%w[git checkout master])
-      command(%W[git branch --delete --force #{ee_branch_prefix}])
-      command(%W[git branch --delete --force #{ee_branch_suffix}])
-    end
-
-    def merge_base_found?(branch:, branch_remote:, master_remote:)
-      step(
-        "Finding merge base with #{master_remote}/master",
-        %W[git merge-base #{master_remote}/master #{branch_remote}/#{branch}]
-      ) do |output, status|
-        if status.zero?
-          puts "Merge base was found: #{output}"
-          true
-        end
-      end
-    end
-
-    def find_merge_base_with_master(branch:, branch_remote:, master_remote:)
-      # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403)
-      # In total we go (20 + 54 + 148 + 403 = 625) commits deeper
-      depth = 20
-      success =
-        (3..6).any? do |factor|
-          depth += Math.exp(factor).to_i
-          # Repository is initially cloned with a depth of 20 so we need to fetch
-          # deeper in the case the branch has more than 20 commits on top of master
-          fetch(branch: branch, depth: depth, remote: branch_remote)
-          fetch(branch: 'master', depth: depth, remote: master_remote)
-
-          merge_base_found?(branch: branch, branch_remote: branch_remote, master_remote: master_remote)
-        end
-
-      raise "\n#{branch} is too far behind #{master_remote}/master, please rebase it!\n" unless success
-    end
-
-    def fetch(branch:, depth:, remote: 'origin')
-      step(
-        "Fetching deeper...",
-        %W[git fetch --depth=#{depth} --prune #{remote} +refs/heads/#{branch}:refs/remotes/#{remote}/#{branch}]
-      ) do |output, status|
-        raise "Fetch failed: #{output}" unless status.zero?
-      end
-    end
-
-    def ce_patch_name
-      @ce_patch_name ||= patch_name_from_branch(ce_branch)
-    end
-
-    def ce_patch_full_path
-      @ce_patch_full_path ||= patches_dir.join(ce_patch_name)
-    end
-
-    def ee_branch_suffix
-      @ee_branch_suffix ||= "#{ce_branch}-ee"
-    end
-
-    def ee_branch_prefix
-      @ee_branch_prefix ||= "ee-#{ce_branch}"
-    end
-
-    def ee_patch_name
-      @ee_patch_name ||= patch_name_from_branch(ee_branch_found)
-    end
-
-    def ee_patch_full_path
-      @ee_patch_full_path ||= patches_dir.join(ee_patch_name)
-    end
-
-    def minimal_ee_branch_name
-      @minimal_ee_branch_name ||= ce_branch.sub(/(\Ace\-|\-ce\z)/, '')
-    end
-
-    def patch_name_from_branch(branch_name)
-      "#{branch_name.parameterize}.patch"
-    end
-
-    def patch_url
-      "#{ce_project_url}/-/jobs/#{job_id}/artifacts/raw/ee_compat_check/patches/#{ce_patch_name}"
-    end
-
-    def step(desc, cmd = nil)
-      puts "\n=> #{desc}\n"
-
-      if cmd
-        start = Time.now
-        puts "\n$ #{cmd.join(' ')}"
-
-        output, status = command(cmd)
-        puts "\n==> Finished in #{Time.now - start} seconds"
-
-        if block_given?
-          yield(output, status)
-        else
-          [output, status]
-        end
-      end
-    end
-
-    def command(cmd)
-      Gitlab::Popen.popen(cmd)
-    end
-
-    # We're "re-creating" the repo URL because ENV['CI_REPOSITORY_URL'] contains
-    # redacted credentials (e.g. "***:****") which are useless in instructions
-    # the job gives.
-    def ce_public_repo_url
-      "#{ce_project_url}.git"
-    end
-
-    def applies_cleanly_msg(branch)
-      %Q{
-        #{PLEASE_READ_THIS_BANNER}
-        🎉 Congratulations!! 🎉
-
-        The `#{branch}` branch applies cleanly to EE/master!
-
-        Much ❤️! For more information, see
-        https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html
-        #{THANKS_FOR_READING_BANNER}
-      }
-    end
-
-    def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
-      ee_repos = ee_remotes.values.uniq
-
-      %Q{
-        #{PLEASE_READ_THIS_BANNER}
-        💥 Oh no! 💥
-
-        The `#{ce_branch}` branch does not apply cleanly to the current
-        EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch
-        was found in #{ee_repos.join(' nor in ')}.
-
-        If you're a community contributor, don't worry, someone from
-        GitLab Inc. will take care of this, and you don't have to do anything.
-        If you're willing to help, and are ok to contribute to EE as well,
-        you're welcome to help. You could follow the instructions below.
-
-        #{conflicting_files_msg}
-
-        We advise you to create a `#{ee_branch_prefix}` or `#{ee_branch_suffix}`
-        branch that includes changes from `#{ce_branch}` but also specific changes
-        than can be applied cleanly to EE/master. In some cases, the conflicts
-        are trivial and you can ignore the warning from this job. As always,
-        use your best judgement!
-
-        There are different ways to create such branch:
-
-        1. Create a new branch from master and cherry-pick your CE commits
-
-          # In the EE repo
-          $ git fetch #{CANONICAL_EE_REPO_URL} master
-          $ git checkout -b #{ee_branch_prefix} FETCH_HEAD
-          $ git fetch #{ce_public_repo_url} #{ce_branch}
-          $ git cherry-pick SHA # Repeat for all the commits you want to pick
-
-          Note: You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit.
-
-        2. Apply your branch's patch to EE
-
-          # In the EE repo
-          $ git fetch #{CANONICAL_EE_REPO_URL} master
-          $ git checkout -b #{ee_branch_prefix} FETCH_HEAD
-          $ wget #{patch_url} && git apply --3way #{ce_patch_name}
-
-          At this point you might have conflicts such as:
-
-            error: patch failed: lib/gitlab/ee_compat_check.rb:5
-            Falling back to three-way merge...
-            Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts.
-            U lib/gitlab/ee_compat_check.rb
-
-          Resolve them, stage the changes and commit them.
-
-          If the patch couldn't be applied cleanly, use the following command:
-
-          # In the EE repo
-          $ git apply --reject #{ce_patch_name}
-
-          This option makes git apply the parts of the patch that are applicable,
-          and leave the rejected hunks in corresponding `.rej` files.
-          You can then resolve the conflicts highlighted in `.rej` by
-          manually applying the correct diff from the `.rej` file to the file with conflicts.
-          When finished, you can delete the `.rej` files and commit your changes.
-
-        ⚠️ Don't forget to push your branch to gitlab-ee:
-
-          # In the EE repo
-          $ git push origin #{ee_branch_prefix}
-
-        ⚠️ Also, don't forget to create a new merge request on gitlab-ee and
-        cross-link it with the CE merge request.
-
-        Once this is done, you can retry this failed job, and it should pass.
-
-        #{STAY_STRONG_LINK_TO_DOCS}
-        #{THANKS_FOR_READING_BANNER}
-      }
-    end
-
-    def ee_branch_doesnt_apply_cleanly_msg
-      %Q{
-        #{PLEASE_READ_THIS_BANNER}
-        💥 Oh no! 💥
-
-        The `#{ce_branch}` does not apply cleanly to the current EE/master, and
-        even though a `#{ee_branch_found}` branch
-        exists in #{ee_repo_url}, it does not apply cleanly either to
-        EE/master!
-
-        #{conflicting_files_msg}
-
-        Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and
-        retry this job.
-
-        #{STAY_STRONG_LINK_TO_DOCS}
-        #{THANKS_FOR_READING_BANNER}
-      }
-    end
-
-    def conflicting_files_msg
-      header = "The conflicts detected were as follows:\n"
-      separator = "\n        - "
-      failed_items = failed_files.join(separator)
-
-      "#{header}#{separator}#{failed_items}"
-    end
-  end
-end
diff --git a/lib/tasks/ee_compat_check.rake b/lib/tasks/ee_compat_check.rake
deleted file mode 100644
index f494fa5c5c2..00000000000
--- a/lib/tasks/ee_compat_check.rake
+++ /dev/null
@@ -1,4 +0,0 @@
-desc 'Checks if the branch would apply cleanly to EE'
-task ee_compat_check: :environment do
-  Rake::Task['gitlab:dev:ee_compat_check'].invoke
-end
diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake
deleted file mode 100644
index 17d3ac74375..00000000000
--- a/lib/tasks/gitlab/dev.rake
+++ /dev/null
@@ -1,31 +0,0 @@
-namespace :gitlab do
-  namespace :dev do
-    desc 'Checks if the branch would apply cleanly to EE'
-    task :ee_compat_check, [:branch] => :environment do |_, args|
-      opts =
-        if ENV['CI']
-          {
-            ce_project_url: ENV['CI_PROJECT_URL'],
-            branch: ENV['CI_COMMIT_REF_NAME'],
-            job_id: ENV['CI_JOB_ID']
-          }
-        else
-          unless args[:branch]
-            puts "Must specify a branch as an argument".color(:red)
-            exit 1
-          end
-
-          args
-        end
-
-      if File.basename(Rails.root) == 'gitlab'
-        puts "Skipping EE projects"
-        exit 0
-      elsif Gitlab::EeCompatCheck.new(opts || {}).check
-        exit 0
-      else
-        exit 1
-      end
-    end
-  end
-end
diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake
index 2353b2dc659..9a5693e78a2 100644
--- a/lib/tasks/lint.rake
+++ b/lib/tasks/lint.rake
@@ -28,14 +28,23 @@ unless Rails.env.production?
     task :all do
       status = 0
 
-      %w[
+      tasks = %w[
         config_lint
         lint:haml
         scss_lint
         gettext:lint
-        gettext:updated_check
         lint:static_verification
-      ].each do |task|
+      ]
+
+      if Gitlab.ee?
+        # This task will fail on CE installations (e.g. gitlab-org/gitlab-foss)
+        # since it will detect strings in the locale files that do not exist in
+        # the source files. To work around this we will only enable this task on
+        # EE installations.
+        tasks << 'gettext:updated_check'
+      end
+
+      tasks.each do |task|
         pid = Process.fork do
           puts "*** Running rake task: #{task} ***"