Add latest changes from gitlab-org/gitlab@master

7 files changed, 140 insertions, 338 deletions

diff --git a/scripts/api/cancel_pipeline b/scripts/api/cancel_pipeline
deleted file mode 100755
index 0965877a69a..00000000000
--- a/scripts/api/cancel_pipeline
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'rubygems'
-require 'gitlab'
-require 'optparse'
-require_relative 'get_job_id'
-
-class CancelPipeline
-  DEFAULT_OPTIONS = {
-    project: ENV['CI_PROJECT_ID'],
-    pipeline_id: ENV['CI_PIPELINE_ID'],
-    api_token: ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
-  }.freeze
-
-  def initialize(options)
-    @project = options.delete(:project)
-    @pipeline_id = options.delete(:pipeline_id)
-
-    Gitlab.configure do |config|
-      config.endpoint = 'https://gitlab.com/api/v4'
-      config.private_token = options.delete(:api_token)
-    end
-  end
-
-  def execute
-    Gitlab.cancel_pipeline(project, pipeline_id)
-  end
-
-  private
-
-  attr_reader :project, :pipeline_id
-end
-
-if $0 == __FILE__
-  options = CancelPipeline::DEFAULT_OPTIONS.dup
-
-  OptionParser.new do |opts|
-    opts.on("-p", "--project PROJECT", String, "Project where to find the job (defaults to $CI_PROJECT_ID)") do |value|
-      options[:project] = value
-    end
-
-    opts.on("-i", "--pipeline-id PIPELINE_ID", String, "A pipeline ID (defaults to $CI_PIPELINE_ID)") do |value|
-      options[:pipeline_id] = value
-    end
-
-    opts.on("-t", "--api-token API_TOKEN", String, "A value API token with the `read_api` scope") do |value|
-      options[:api_token] = value
-    end
-
-    opts.on("-h", "--help", "Prints this help") do
-      puts opts
-      exit
-    end
-  end.parse!
-
-  CancelPipeline.new(options).execute
-end
diff --git a/scripts/api/download_job_artifact b/scripts/api/download_job_artifact
deleted file mode 100755
index 9ac24ff624d..00000000000
--- a/scripts/api/download_job_artifact
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'rubygems'
-require 'optparse'
-require 'fileutils'
-require 'uri'
-require 'cgi'
-require 'net/http'
-
-class ArtifactFinder
-  DEFAULT_OPTIONS = {
-    project: ENV['CI_PROJECT_ID'],
-    api_token: ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
-  }.freeze
-
-  def initialize(options)
-    @project = options.delete(:project)
-    @job_id = options.delete(:job_id)
-    @api_token = options.delete(:api_token)
-    @artifact_path = options.delete(:artifact_path)
-  end
-
-  def execute
-    url = "https://gitlab.com/api/v4/projects/#{CGI.escape(project)}/jobs/#{job_id}/artifacts"
-
-    if artifact_path
-      FileUtils.mkdir_p(File.dirname(artifact_path))
-      url += "/#{artifact_path}"
-    end
-
-    fetch(url)
-  end
-
-  private
-
-  attr_reader :project, :job_id, :api_token, :artifact_path
-
-  def fetch(uri_str, limit = 10)
-    raise 'Too many HTTP redirects' if limit == 0
-
-    uri = URI(uri_str)
-    request = Net::HTTP::Get.new(uri)
-    request['Private-Token'] = api_token
-
-    Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
-      http.request(request) do |response|
-        case response
-        when Net::HTTPSuccess then
-          File.open(artifact_path || 'artifacts.zip', 'w') do |file|
-            response.read_body(&file.method(:write))
-          end
-        when Net::HTTPRedirection then
-          location = response['location']
-          warn "Redirected (#{limit - 1} redirections remaining)."
-          fetch(location, limit - 1)
-        else
-          raise "Unexpected response: #{response.value}"
-        end
-      end
-    end
-  end
-end
-
-if $0 == __FILE__
-  options = ArtifactFinder::DEFAULT_OPTIONS.dup
-
-  OptionParser.new do |opts|
-    opts.on("-p", "--project PROJECT", String, "Project where to find the job (defaults to $CI_PROJECT_ID)") do |value|
-      options[:project] = value
-    end
-
-    opts.on("-j", "--job-id JOB_ID", String, "A job ID") do |value|
-      options[:job_id] = value
-    end
-
-    opts.on("-a", "--artifact-path ARTIFACT_PATH", String, "A valid artifact path") do |value|
-      options[:artifact_path] = value
-    end
-
-    opts.on("-t", "--api-token API_TOKEN", String, "A value API token with the `read_api` scope") do |value|
-      options[:api_token] = value
-    end
-
-    opts.on("-h", "--help", "Prints this help") do
-      puts opts
-      exit
-    end
-  end.parse!
-
-  ArtifactFinder.new(options).execute
-end
diff --git a/scripts/api/get_job_id b/scripts/api/get_job_id
deleted file mode 100755
index 2324f6ca9d3..00000000000
--- a/scripts/api/get_job_id
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'rubygems'
-require 'gitlab'
-require 'optparse'
-
-class JobFinder
-  DEFAULT_OPTIONS = {
-    project: ENV['CI_PROJECT_ID'],
-    pipeline_id: ENV['CI_PIPELINE_ID'],
-    pipeline_query: {},
-    job_query: {},
-    api_token: ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
-  }.freeze
-
-  def initialize(options)
-    @project = options.delete(:project)
-    @pipeline_query = options.delete(:pipeline_query)
-    @job_query = options.delete(:job_query)
-    @pipeline_id = options.delete(:pipeline_id)
-    @job_name = options.delete(:job_name)
-
-    Gitlab.configure do |config|
-      config.endpoint = 'https://gitlab.com/api/v4'
-      config.private_token = options.delete(:api_token)
-    end
-  end
-
-  def execute
-    find_job_with_filtered_pipelines || find_job_in_pipeline
-  end
-
-  private
-
-  attr_reader :project, :pipeline_query, :job_query, :pipeline_id, :job_name
-
-  def find_job_with_filtered_pipelines
-    return if pipeline_query.empty?
-
-    Gitlab.pipelines(project, pipeline_query_params).auto_paginate do |pipeline|
-      Gitlab.pipeline_jobs(project, pipeline.id, job_query_params).auto_paginate do |job|
-        return job if job.name == job_name # rubocop:disable Cop/AvoidReturnFromBlocks
-      end
-    end
-
-    raise 'Job not found!'
-  end
-
-  def find_job_in_pipeline
-    return unless pipeline_id
-
-    Gitlab.pipeline_jobs(project, pipeline_id, job_query_params).auto_paginate do |job|
-      return job if job.name == job_name # rubocop:disable Cop/AvoidReturnFromBlocks
-    end
-
-    raise 'Job not found!'
-  end
-
-  def pipeline_query_params
-    @pipeline_query_params ||= { per_page: 100, **pipeline_query }
-  end
-
-  def job_query_params
-    @job_query_params ||= { per_page: 100, **job_query }
-  end
-end
-
-if $0 == __FILE__
-  options = JobFinder::DEFAULT_OPTIONS.dup
-
-  OptionParser.new do |opts|
-    opts.on("-p", "--project PROJECT", String, "Project where to find the job (defaults to $CI_PROJECT_ID)") do |value|
-      options[:project] = value
-    end
-
-    opts.on("-i", "--pipeline-id pipeline_id", String, "A pipeline ID (defaults to $CI_PIPELINE_ID)") do |value|
-      options[:pipeline_id] = value
-    end
-
-    opts.on("-q", "--pipeline-query pipeline_query", String, "Query to pass to the Pipeline API request") do |value|
-      options[:pipeline_query].merge!(Hash[*value.split('=')])
-    end
-
-    opts.on("-Q", "--job-query job_query", String, "Query to pass to the Job API request") do |value|
-      options[:job_query].merge!(Hash[*value.split('=')])
-    end
-
-    opts.on("-j", "--job-name job_name", String, "A job name that needs to exist in the found pipeline") do |value|
-      options[:job_name] = value
-    end
-
-    opts.on("-t", "--api-token API_TOKEN", String, "A value API token with the `read_api` scope") do |value|
-      options[:api_token] = value
-    end
-
-    opts.on("-h", "--help", "Prints this help") do
-      puts opts
-      exit
-    end
-  end.parse!
-
-  job = JobFinder.new(options).execute
-
-  return if job.nil?
-
-  puts job.id
-end
diff --git a/scripts/api/play_job b/scripts/api/play_job
deleted file mode 100755
index 199f7e65633..00000000000
--- a/scripts/api/play_job
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'rubygems'
-require 'gitlab'
-require 'optparse'
-require_relative 'get_job_id'
-
-class PlayJob
-  DEFAULT_OPTIONS = {
-    project: ENV['CI_PROJECT_ID'],
-    pipeline_id: ENV['CI_PIPELINE_ID'],
-    api_token: ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
-  }.freeze
-
-  def initialize(options)
-    @project = options.delete(:project)
-    @options = options
-
-    Gitlab.configure do |config|
-      config.endpoint = 'https://gitlab.com/api/v4'
-      config.private_token = options.fetch(:api_token)
-    end
-  end
-
-  def execute
-    job = JobFinder.new(project, options.slice(:api_token, :pipeline_id, :job_name).merge(scope: 'manual')).execute
-
-    Gitlab.job_play(project, job.id)
-  end
-
-  private
-
-  attr_reader :project, :options
-end
-
-if $0 == __FILE__
-  options = PlayJob::DEFAULT_OPTIONS.dup
-
-  OptionParser.new do |opts|
-    opts.on("-p", "--project PROJECT", String, "Project where to find the job (defaults to $CI_PROJECT_ID)") do |value|
-      options[:project] = value
-    end
-
-    opts.on("-j", "--job-name JOB_NAME", String, "A job name that needs to exist in the found pipeline") do |value|
-      options[:job_name] = value
-    end
-
-    opts.on("-t", "--api-token API_TOKEN", String, "A value API token with the `read_api` scope") do |value|
-      options[:api_token] = value
-    end
-
-    opts.on("-h", "--help", "Prints this help") do
-      puts opts
-      exit
-    end
-  end.parse!
-
-  PlayJob.new(options).execute
-end
diff --git a/scripts/get-job-id b/scripts/get-job-id
new file mode 100755
index 00000000000..a5d34dc545b
--- /dev/null
+++ b/scripts/get-job-id
@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'gitlab'
+require 'optparse'
+
+#
+# Configure credentials to be used with gitlab gem
+#
+Gitlab.configure do |config|
+  config.endpoint = 'https://gitlab.com/api/v4'
+  config.private_token = ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
+end
+
+options = {}
+OptionParser.new do |opts|
+  opts.on("-s", "--scope=SCOPE", "Find job with matching scope") do |scope|
+    options[:scope] = scope
+  end
+end.parse!
+
+class PipelineJobFinder
+  def initialize(project_id, pipeline_id, job_name, options)
+    @project_id = project_id
+    @pipeline_id = pipeline_id
+    @job_name = job_name
+    @options = options
+  end
+
+  def execute
+    Gitlab.pipeline_jobs(@project_id, @pipeline_id, @options).auto_paginate do |job|
+      break job if job.name == @job_name
+    end
+  end
+end
+
+project_id, pipeline_id, job_name = ARGV
+
+job = PipelineJobFinder.new(project_id, pipeline_id, job_name, options).execute
+
+return if job.nil?
+
+puts job.id
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index 0f14b702de2..5f003d032b7 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -1,39 +1,44 @@
 #!/usr/bin/env bash
 
 function retrieve_tests_metadata() {
-  mkdir -p crystalball/ knapsack/ rspec_flaky/ rspec_profiling/
-
-  local project_path="gitlab-org/gitlab"
-  local test_metadata_job_id
-
-  # Ruby
-  test_metadata_job_id=$(scripts/api/get_job_id --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata")
+  mkdir -p knapsack/ rspec_flaky/ rspec_profiling/
 
   if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
-    scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+    wget -O "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
   fi
 
   if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
-    scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+    wget -O "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
   fi
-
-  # FIXME: We will need to find a pipeline where the $RSPEC_PACKED_TESTS_MAPPING_PATH.gz actually exists (Crystalball only runs every two-hours, but the `update-tests-metadata` runs for all `master` pipelines...).
-  # if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
-  #   (scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
-  # fi
-  #
-  # scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}"
 }
 
 function update_tests_metadata() {
   echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
 
   scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" knapsack/rspec*.json
+  if [[ -n "${TESTS_METADATA_S3_BUCKET}" ]]; then
+    if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
+      scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+    else
+      echo "Not uplaoding report to S3 as the pipeline is not a scheduled one."
+    fi
+  fi
+
   rm -f knapsack/rspec*.json
 
-  export FLAKY_RSPEC_GENERATE_REPORT="true"
   scripts/merge-reports "${FLAKY_RSPEC_SUITE_REPORT_PATH}" rspec_flaky/all_*.json
+
+  export FLAKY_RSPEC_GENERATE_REPORT="true"
   scripts/flaky_examples/prune-old-flaky-examples "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+
+  if [[ -n ${TESTS_METADATA_S3_BUCKET} ]]; then
+    if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
+      scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+    else
+      echo "Not uploading report to S3 as the pipeline is not a scheduled one."
+    fi
+  fi
+
   rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
 
   if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
@@ -43,6 +48,16 @@ function update_tests_metadata() {
   fi
 }
 
+function retrieve_tests_mapping() {
+  mkdir -p crystalball/
+
+  if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
+    (wget -O "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+  fi
+
+  scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}"
+}
+
 function update_tests_mapping() {
   if ! crystalball_rspec_data_exists; then
     echo "No crystalball rspec data found."
@@ -50,9 +65,20 @@ function update_tests_mapping() {
   fi
 
   scripts/generate-test-mapping "${RSPEC_TESTS_MAPPING_PATH}" crystalball/rspec*.yml
+
   scripts/pack-test-mapping "${RSPEC_TESTS_MAPPING_PATH}" "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+
   gzip "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
-  rm -f crystalball/rspec*.yml "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+
+  if [[ -n "${TESTS_METADATA_S3_BUCKET}" ]]; then
+    if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
+      scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz"
+    else
+      echo "Not uploading report to S3 as the pipeline is not a scheduled one."
+    fi
+  fi
+
+  rm -f crystalball/rspec*.yml
 }
 
 function crystalball_rspec_data_exists() {
diff --git a/scripts/utils.sh b/scripts/utils.sh
index 4d6088e94a8..3829bcdf24e 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -87,14 +87,65 @@ function echosuccess() {
   fi
 }
 
+function get_job_id() {
+  local job_name="${1}"
+  local query_string="${2:+&${2}}"
+  local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}"
+  if [ -z "${api_token}" ]; then
+    echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN."
+    return
+  fi
+
+  local max_page=3
+  local page=1
+
+  while true; do
+    local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}"
+    echoinfo "GET ${url}"
+
+    local job_id
+    job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq "map(select(.name == \"${job_name}\")) | map(.id) | last")
+    [[ "${job_id}" == "null" && "${page}" -lt "$max_page" ]] || break
+
+    let "page++"
+  done
+
+  if [[ "${job_id}" == "null" ]]; then # jq prints "null" for non-existent attribute
+    echoerr "The '${job_name}' job ID couldn't be retrieved!"
+  else
+    echoinfo "The '${job_name}' job ID is ${job_id}"
+    echo "${job_id}"
+  fi
+}
+
+function play_job() {
+  local job_name="${1}"
+  local job_id
+  job_id=$(get_job_id "${job_name}" "scope=manual");
+  if [ -z "${job_id}" ]; then return; fi
+
+  local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}"
+  if [ -z "${api_token}" ]; then
+    echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN."
+    return
+  fi
+
+  local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}/play"
+  echoinfo "POST ${url}"
+
+  local job_url
+  job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".web_url")
+  echoinfo "Manual job '${job_name}' started at: ${job_url}"
+}
+
 function fail_pipeline_early() {
   local dont_interrupt_me_job_id
-  dont_interrupt_me_job_id=$(scripts/api/get_job_id --job-query "scope=success" --job-name "dont-interrupt-me")
+  dont_interrupt_me_job_id=$(get_job_id 'dont-interrupt-me' 'scope=success')
 
   if [[ -n "${dont_interrupt_me_job_id}" ]]; then
     echoinfo "This pipeline cannot be interrupted due to \`dont-interrupt-me\` job ${dont_interrupt_me_job_id}"
   else
     echoinfo "Failing pipeline early for fast feedback due to test failures in rspec fail-fast."
-    scripts/api/cancel_pipeline
+    curl --request POST --header "PRIVATE-TOKEN: ${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}" "https://${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/cancel"
   fi
 }