diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-27 18:06:16 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-27 18:06:16 +0300 |
commit | 8320f7956d72986f5a7c850874fce4f8b5a8e015 (patch) | |
tree | c761b309cfff422609d47a17ac4d6a732c142f49 /.gitlab/ci | |
parent | 45482d5a2704da7fabe4ccf07f85d9be6e0a791a (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to '.gitlab/ci')
-rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 4 | ||||
-rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 21 | ||||
-rw-r--r-- | .gitlab/ci/memory.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 299 | ||||
-rw-r--r-- | .gitlab/ci/test-metadata.gitlab-ci.yml | 50 |
5 files changed, 166 insertions, 210 deletions
diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 92149d5a93e..1feda7ed4d4 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -77,7 +77,7 @@ gitlab:assets:compile pull-cache: - .default-before_script - .assets-compile-cache - .only-code-qa-changes - - .use-pg + - .use-pg9 stage: prepare script: - node --version @@ -120,7 +120,7 @@ compile-assets pull-cache: - .default-only - .default-before_script - .only-code-changes - - .use-pg + - .use-pg9 dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"] karma: diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 49dd778f4fb..e0ce6e2fde2 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -110,19 +110,34 @@ - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" kubernetes: active -.use-pg: +.use-pg9: services: - - name: postgres:9.6.14 + - name: postgres:9.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:alpine -.use-pg-10: +.use-pg10: image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33" services: - name: postgres:10.9 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:alpine +.use-pg9-ee: + services: + - name: postgres:9.6 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:alpine + - name: docker.elastic.co/elasticsearch/elasticsearch:5.6.12 + +.use-pg10-ee: + image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33" + services: + - name: postgres:10.9 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:alpine + - name: docker.elastic.co/elasticsearch/elasticsearch:5.6.12 + .only-ee: only: variables: diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml index d990c7eefa2..93bf87b24b2 100644 --- a/.gitlab/ci/memory.gitlab-ci.yml +++ b/.gitlab/ci/memory.gitlab-ci.yml @@ -36,7 +36,7 @@ memory-static: memory-on-boot: extends: - .only-code-memory-job-base - - .use-pg-10 + - .use-pg10 variables: NODE_ENV: "production" RAILS_ENV: "production" diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 8851f2ba211..c315501b0ba 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -33,31 +33,29 @@ - .default-before_script - .only-code-qa-changes +setup-test-env: + extends: + - .only-code-qa-rails-job-base + - .use-pg9 + stage: prepare + script: + - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init' + - scripts/gitaly-test-build # Do not use 'bundle exec' here + artifacts: + expire_in: 7d + paths: + - tmp/tests + - config/secrets.yml + - vendor/gitaly-ruby + cache: + policy: pull-push + .rspec-base: extends: .only-code-rails-job-base stage: test script: - - JOB_NAME=( $CI_JOB_NAME ) - - TEST_TOOL=${JOB_NAME[0]} - - TEST_LEVEL=${JOB_NAME[1]} - - DATABASE=${JOB_NAME[2]} - - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - - export KNAPSACK_GENERATE_REPORT=true KNAPSACK_LOG_LEVEL=debug KNAPSACK_TEST_DIR=spec - - export SUITE_FLAKY_RSPEC_REPORT_PATH=${FLAKY_RSPEC_SUITE_REPORT_PATH} - - export FLAKY_RSPEC_REPORT_PATH=rspec_flaky/all_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - - export NEW_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/new_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - - export FLAKY_RSPEC_GENERATE_REPORT=true - - export CACHE_CLASSES=true - - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - - '[[ -f $FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_REPORT_PATH}' - - '[[ -f $NEW_FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${NEW_FLAKY_RSPEC_REPORT_PATH}' - - scripts/gitaly-test-spawn - - date - - 'export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new.pattern(:${TEST_LEVEL})")' - - mkdir -p tmp/memory_test - - export MEMORY_TEST_PATH="tmp/memory_test/${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv" - - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag level:${TEST_LEVEL} --tag ~geo" - - date + - source scripts/rspec_helpers.sh + - rspec_paralellized_job "--tag ~quarantine --tag ~geo" artifacts: expire_in: 31d when: always @@ -71,78 +69,145 @@ reports: junit: junit_rspec.xml -.rspec-base-pg: +.rspec-base-pg9: extends: - .rspec-base - - .use-pg + - .use-pg9 -.rspec-base-pg-10: +.rspec-base-pg10: extends: - .rspec-base - - .use-pg-10 + - .use-pg10 + - .only-master -setup-test-env: - extends: - - .only-code-qa-rails-job-base - - .use-pg - stage: prepare - script: - - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init' - - scripts/gitaly-test-build # Do not use 'bundle exec' here - artifacts: - expire_in: 7d - paths: - - tmp/tests - - config/secrets.yml - - vendor/gitaly-ruby - cache: - policy: pull-push +rspec unit pg9: + extends: .rspec-base-pg9 + parallel: 20 + +rspec integration pg9: + extends: .rspec-base-pg9 + parallel: 6 -rspec unit pg: - extends: .rspec-base-pg +rspec system pg9: + extends: .rspec-base-pg9 + parallel: 24 + +rspec unit pg10: + extends: .rspec-base-pg10 parallel: 20 -rspec integration pg: - extends: .rspec-base-pg +rspec integration pg10: + extends: .rspec-base-pg10 parallel: 6 -rspec system pg: - extends: .rspec-base-pg +rspec system pg10: + extends: .rspec-base-pg10 parallel: 24 -rspec unit pg-10: +.rspec-ee-base-pg9: + extends: + - .rspec-base + - .only-ee + - .use-pg9-ee + +.rspec-ee-base-pg10: + extends: + - .rspec-base + - .only-ee + - .use-pg10-ee + +rspec-ee unit pg9: + extends: .rspec-ee-base-pg9 + parallel: 7 + +rspec-ee integration pg9: + extends: .rspec-ee-base-pg9 + parallel: 3 + +rspec-ee system pg9: + extends: .rspec-ee-base-pg9 + parallel: 5 + +rspec-ee unit pg10: extends: - - .rspec-base-pg-10 + - .rspec-ee-base-pg10 - .only-master - parallel: 20 + parallel: 7 -rspec integration pg-10: +rspec-ee integration pg10: extends: - - .rspec-base-pg-10 + - .rspec-ee-base-pg10 - .only-master - parallel: 6 + parallel: 3 -rspec system pg-10: +rspec-ee system pg10: extends: - - .rspec-base-pg-10 + - .rspec-ee-base-pg10 - .only-master - parallel: 24 + parallel: 5 -rspec-fast-spec-helper: - extends: .rspec-base-pg +.rspec-ee-base-geo: + extends: + - .rspec-base + - .only-ee script: - - bundle exec rspec spec/fast_spec_helper.rb + - source scripts/rspec_helpers.sh + - scripts/prepare_postgres_fdw.sh + - rspec_paralellized_job "--tag ~quarantine --tag geo" + +.rspec-ee-base-geo-pg9: + extends: + - .rspec-ee-base-geo + - .use-pg9-ee + +.rspec-ee-base-geo-pg10: + extends: + - .rspec-ee-base-geo + - .use-pg10-ee + +rspec-ee unit pg9 geo: + extends: .rspec-ee-base-geo-pg9 + parallel: 2 + +rspec-ee integration pg9 geo: + extends: .rspec-ee-base-geo-pg9 -rspec quarantine pg: +rspec-ee system pg9 geo: + extends: .rspec-ee-base-geo-pg9 + +rspec-ee unit pg10 geo: + extends: .rspec-ee-base-geo-pg10 + parallel: 2 + +rspec-ee integration pg10 geo: + extends: .rspec-ee-base-geo-pg10 + +rspec-ee system pg10 geo: + extends: .rspec-ee-base-geo-pg10 + +rspec quarantine pg9: extends: - - .rspec-base-pg + - .rspec-base-pg9 - .only-master + variables: + RSPEC_OPTS: "--tag quarantine -- spec/" script: - - export NO_KNAPSACK=1 CACHE_CLASSES=true - - scripts/gitaly-test-spawn - - bin/rspec --color --format documentation --tag quarantine -- spec/ + - source scripts/rspec_helpers.sh + - rspec_simple_job "${RSPEC_OPTS}" allow_failure: true +rspec-ee quarantine pg9: + extends: + - rspec quarantine pg9 + - .only-ee + variables: + RSPEC_OPTS: "--tag quarantine -- ee/spec/" + +rspec fast_spec_helper: + extends: .rspec-base-pg9 + script: + - bin/rspec spec/fast_spec_helper.rb + static-analysis: extends: .only-code-qa-rails-job-base dependencies: ["setup-test-env", "compile-assets", "compile-assets pull-cache"] @@ -174,7 +239,7 @@ downtime_check: .db-job-base: extends: - .only-code-rails-job-base - - .use-pg + - .use-pg9 stage: test dependencies: ["setup-test-env"] needs: ["setup-test-env"] @@ -258,108 +323,6 @@ coverage: - coverage/assets/ - tmp/memory_test/ -## EE-specific content -.rspec-base-ee: - extends: - - .rspec-base - - .only-ee - script: - - JOB_NAME=( $CI_JOB_NAME ) - - TEST_TOOL=${JOB_NAME[0]} - - TEST_LEVEL=${JOB_NAME[1]} - - DATABASE=${JOB_NAME[2]} - - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_ee_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - - export KNAPSACK_GENERATE_REPORT=true KNAPSACK_LOG_LEVEL=debug KNAPSACK_TEST_DIR=spec - - export CACHE_CLASSES=true - - cp ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - - scripts/gitaly-test-spawn - - date - - 'export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new(%(ee/)).pattern(:${TEST_LEVEL})")' - - mkdir -p tmp/memory_test - - export MEMORY_TEST_PATH="tmp/memory_test/ee_${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv" - - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag level:${TEST_LEVEL} --tag ~geo" - - date - -.rspec-base-pg-ee: - extends: .rspec-base-ee - services: - - name: postgres:9.6 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - - name: docker.elastic.co/elasticsearch/elasticsearch:5.6.12 - -rspec unit pg ee: - extends: .rspec-base-pg-ee - parallel: 7 - -rspec integration pg ee: - extends: .rspec-base-pg-ee - parallel: 3 - -rspec system pg ee: - extends: .rspec-base-pg-ee - parallel: 5 - -.rspec-base-geo: - extends: .rspec-base-ee - parallel: 3 - script: - - JOB_NAME=( $CI_JOB_NAME ) - - TEST_TOOL=${JOB_NAME[0]} - - TEST_LEVEL=${JOB_NAME[1]} - - DATABASE=${JOB_NAME[2]} - - export KNAPSACK_TEST_FILE_PATTERN="ee/spec/**{,/*/**}/*_spec.rb" KNAPSACK_GENERATE_REPORT=true CACHE_CLASSES=true - - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - - cp ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - - source scripts/prepare_postgres_fdw.sh - - scripts/gitaly-test-spawn - - mkdir -p tmp/memory_test - - export MEMORY_TEST_PATH="tmp/memory_test/ee_${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv" - - knapsack rspec "-Ispec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag geo" - -rspec geo pg ee: - extends: - - .rspec-base-geo - - .use-pg - except: - variables: - - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/ - -rspec geo pg-10 ee: - extends: - - .rspec-base-geo - - .use-pg-10 - except: - variables: - - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/ - -quick-rspec geo pg ee: - extends: - - .rspec-base-geo - - .use-pg - stage: quick-test - only: - variables: - - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/ - -quick-rspec geo pg-10 ee: - extends: - - .rspec-base-geo - - .use-pg-10 - stage: quick-test - only: - variables: - - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/ - -rspec quarantine pg ee: - extends: - - rspec quarantine pg - - .only-ee - script: - - export NO_KNAPSACK=1 CACHE_CLASSES=true - - scripts/gitaly-test-spawn - - bin/rspec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag quarantine -- ee/spec/ - db:rollback geo: extends: - db:rollback @@ -367,5 +330,3 @@ db:rollback geo: script: - bundle exec rake geo:db:migrate VERSION=20170627195211 - bundle exec rake geo:db:migrate - -## END of EE-specific content diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index 5e2523e8a9c..6a7f3157d59 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -1,10 +1,16 @@ .tests-metadata-state: extends: - .default-only + - .only-code-changes variables: TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache" before_script: - source scripts/utils.sh + cache: + key: tests_metadata + paths: + - knapsack/ + - rspec_flaky/ artifacts: expire_in: 31d paths: @@ -13,55 +19,29 @@ - rspec_profiling/ retrieve-tests-metadata: - extends: - - .tests-metadata-state - - .only-code-changes + extends: .tests-metadata-state stage: prepare cache: - key: tests_metadata policy: pull script: - - mkdir -p knapsack/${CI_PROJECT_NAME}/ - - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH - - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}' - - mkdir -p rspec_flaky/ - - mkdir -p rspec_profiling/ - - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH - - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}' - - '[[ ! -d "ee/" ]] || wget -O $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH' - - '[[ ! -d "ee/" ]] || [[ -f $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH}' + - source scripts/rspec_helpers.sh + - retrieve_tests_metadata update-tests-metadata: - extends: - - .tests-metadata-state - - .only-code-changes + extends: .tests-metadata-state stage: post-test cache: - key: tests_metadata - paths: - - knapsack/ - - rspec_flaky/ policy: push script: - retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document - - echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} - - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_*_pg_node_*.json - - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH' - - '[[ ! -d "ee/" ]] || echo "{}" > ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH}' - - '[[ ! -d "ee/" ]] || scripts/merge-reports ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_*_pg_ee_*node_*.json' - - '[[ ! -d "ee/" ]] || [[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH' - - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json - - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json - - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH} - - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH' - - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json - - scripts/insert-rspec-profiling-data + - source scripts/rspec_helpers.sh + - update_tests_metadata only: refs: - - master + - schedules variables: - - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" - - $CI_SERVER_HOST == "dev.gitlab.org" + # Only update the Knapsack metadata on GitLab.com/gitlab-org/gitlab + - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" flaky-examples-check: extends: |