Save Knapsack reports in S3 instead of in shared cache

Signed-off-by: Rémy Coutable <remy@rymai.me>

1 files changed, 36 insertions, 31 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2da8207a3cf..96a9e6687df 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,13 +14,15 @@ variables:
   GIT_DEPTH: "20"
   PHANTOMJS_VERSION: "2.1.1"
   GET_SOURCES_ATTEMPTS: "3"
+  KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-${CI_COMMIT_REF_SLUG}.json
+  KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-${CI_COMMIT_REF_SLUG}.json
 
 before_script:
   - source ./scripts/prepare_build.sh
   - cp config/gitlab.yml.example config/gitlab.yml
   - bundle --version
   - '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) --clean $FLAGS'
-  - retry gem install knapsack
+  - retry gem install knapsack fog-aws mime-types
   - '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql'
 
 stages:
@@ -39,14 +41,15 @@ stages:
   variables:
     SETUP_DB: "false"
     USE_BUNDLE_INSTALL: "false"
+    KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
   cache:
     key: "knapsack"
     paths:
-    - knapsack/
+      - knapsack/
   artifacts:
     expire_in: 31d
     paths:
-    - knapsack/
+      - knapsack/
 
 .use-db: &use-db
   services:
@@ -61,17 +64,17 @@ stages:
     - JOB_NAME=( $CI_JOB_NAME )
     - export CI_NODE_INDEX=${JOB_NAME[1]}
     - export CI_NODE_TOTAL=${JOB_NAME[2]}
-    - export KNAPSACK_REPORT_PATH=knapsack/rspec_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
     - export KNAPSACK_GENERATE_REPORT=true
-    - cp knapsack/rspec_report.json ${KNAPSACK_REPORT_PATH}
+    - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
     - knapsack rspec "--color --format documentation"
   artifacts:
     expire_in: 31d
     when: always
     paths:
-    - coverage/
-    - knapsack/
-    - tmp/capybara/
+      - coverage/
+      - knapsack/
+      - tmp/capybara/
 
 .spinach-knapsack: &spinach-knapsack
   stage: test
@@ -81,28 +84,44 @@ stages:
     - JOB_NAME=( $CI_JOB_NAME )
     - export CI_NODE_INDEX=${JOB_NAME[1]}
     - export CI_NODE_TOTAL=${JOB_NAME[2]}
-    - export KNAPSACK_REPORT_PATH=knapsack/spinach_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
     - export KNAPSACK_GENERATE_REPORT=true
-    - cp knapsack/spinach_report.json ${KNAPSACK_REPORT_PATH}
+    - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
     - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
   artifacts:
     expire_in: 31d
     when: always
     paths:
-    - coverage/
-    - knapsack/
-    - tmp/capybara/
+      - coverage/
+      - knapsack/
+      - tmp/capybara/
 
 # Prepare and merge knapsack tests
-
 knapsack:
   <<: *knapsack-state
   <<: *dedicated-runner
   stage: prepare
   script:
-    - mkdir -p knapsack/
-    - '[[ -f knapsack/rspec_report.json ]] || echo "{}" > knapsack/rspec_report.json'
-    - '[[ -f knapsack/spinach_report.json ]] || echo "{}" > knapsack/spinach_report.json'
+    - mkdir -p knapsack/${CI_PROJECT_NAME}/
+    - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || true
+    - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || true
+    - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
+    - '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}'
+
+update-knapsack:
+  <<: *knapsack-state
+  <<: *dedicated-runner
+  stage: post-test
+  script:
+    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_node_*.json
+    - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_node_*.json
+    - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
+    - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
+  only:
+    - master@gitlab-org/gitlab-ce
+    - master@gitlab-org/gitlab-ee
+    - master@gitlab/gitlabhq
+    - master@gitlab/gitlab-ee
 
 setup-test-env:
   <<: *use-db
@@ -122,20 +141,6 @@ setup-test-env:
       - public/assets
       - tmp/tests
 
-update-knapsack:
-  <<: *knapsack-state
-  <<: *dedicated-runner
-  stage: post-test
-  script:
-    - scripts/merge-reports knapsack/rspec_report.json knapsack/rspec_node_*.json
-    - scripts/merge-reports knapsack/spinach_report.json knapsack/spinach_node_*.json
-    - rm -f knapsack/*_node_*.json
-  only:
-    - master@gitlab-org/gitlab-ce
-    - master@gitlab-org/gitlab-ee
-    - master@gitlab/gitlabhq
-    - master@gitlab/gitlab-ee
-
 rspec 0 20: *rspec-knapsack
 rspec 1 20: *rspec-knapsack
 rspec 2 20: *rspec-knapsack