diff options
203 files changed, 2960 insertions, 2024 deletions
diff --git a/.gitlab/ci/dev-fixtures.gitlab-ci.yml b/.gitlab/ci/dev-fixtures.gitlab-ci.yml index fc3678a7d17..4141cc7f071 100644 --- a/.gitlab/ci/dev-fixtures.gitlab-ci.yml +++ b/.gitlab/ci/dev-fixtures.gitlab-ci.yml @@ -14,14 +14,17 @@ SIZE: 0 # number of external projects to fork, requires network connection # SEED_NESTED_GROUPS: "false" # requires network connection +.run-dev-fixtures-script: &run-dev-fixtures-script + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" + - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu" + run-dev-fixtures: extends: - .run-dev-fixtures - .dev-fixtures:rules:ee-and-foss script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu" + - *run-dev-fixtures-script run-dev-fixtures-ee: extends: @@ -29,7 +32,5 @@ run-dev-fixtures-ee: - .dev-fixtures:rules:ee-only - .use-pg11-ee script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - cp ee/db/fixtures/development/* $FIXTURE_PATH - - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu" + - *run-dev-fixtures-script diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 35555ed0a2e..e4c9f85cf62 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -7,6 +7,10 @@ # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 +.yarn-install: &yarn-install + - source scripts/utils.sh + - run_timed_command "retry yarn install --frozen-lockfile" + .compile-assets-base: extends: - .frontend-base @@ -16,9 +20,7 @@ WEBPACK_VENDOR_DLL: "true" stage: prepare script: - - node --version - - run_timed_command "retry yarn install --frozen-lockfile" - - free -m + - *yarn-install - run_timed_command "bin/rake gitlab:assets:compile" - run_timed_command "scripts/clean-old-cached-assets" @@ -85,8 +87,7 @@ update-yarn-cache: - .shared:rules:update-cache stage: prepare script: - - source scripts/utils.sh - - run_timed_command "retry yarn install --frozen-lockfile" + - *yarn-install cache: policy: push @@ -137,14 +138,14 @@ eslint-as-if-foss: - .as-if-foss needs: [] script: - - run_timed_command "retry yarn install --frozen-lockfile" - - yarn run eslint + - *yarn-install + - run_timed_command "yarn run eslint" .karma-base: extends: .frontend-test-base script: - export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log - - run_timed_command "retry yarn install --frozen-lockfile" + - *yarn-install - run_timed_command "yarn karma" karma: @@ -175,7 +176,7 @@ karma-as-if-foss: .jest-base: extends: .frontend-test-base script: - - run_timed_command "retry yarn install --frozen-lockfile" + - *yarn-install - run_timed_command "yarn jest --ci --coverage --testSequencer ./scripts/frontend/parallel_ci_sequencer.js" jest: @@ -200,7 +201,7 @@ jest-integration: - .frontend-test-base - .frontend:rules:default-frontend-jobs script: - - run_timed_command "retry yarn install --frozen-lockfile" + - *yarn-install - run_timed_command "yarn jest:integration --ci" needs: ["frontend-fixtures"] @@ -220,8 +221,7 @@ coverage-frontend: needs: ["jest"] stage: post-test before_script: - - source scripts/utils.sh - - run_timed_command "retry yarn install --frozen-lockfile" + - *yarn-install script: - run_timed_command "yarn node scripts/frontend/merge_coverage_frontend.js" coverage: '/^Statements\s*:\s*?(\d+(?:\.\d+)?)%/' @@ -241,9 +241,8 @@ coverage-frontend: stage: test dependencies: [] script: - - source scripts/utils.sh - - run_timed_command "yarn install --frozen-lockfile" - - run_timed_command "yarn run webpack-prod" + - *yarn-install + - run_timed_command "retry yarn run webpack-prod" qa-frontend-node:10: extends: .qa-frontend-node @@ -266,8 +265,7 @@ webpack-dev-server: WEBPACK_MEMORY_TEST: "true" WEBPACK_VENDOR_DLL: "true" script: - - source scripts/utils.sh - - run_timed_command "retry yarn install --frozen-lockfile" + - *yarn-install - run_timed_command "retry yarn webpack-vendor" - run_timed_command "node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js" artifacts: diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 42e883369a3..aee815bc3ff 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -6,6 +6,14 @@ - .default-before_script - .rails-cache +.base-script: &base-script + # Only install knapsack after bundle install! Otherwise oddly some native + # gems could not be found under some circumstance. No idea why, hours wasted. + - run_timed_command "gem install knapsack --no-document" + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" + - source ./scripts/rspec_helpers.sh + .rspec-base: extends: .rails-job-base stage: test @@ -14,12 +22,7 @@ RUBY_GC_MALLOC_LIMIT_MAX: 134217728 needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"] script: - # Only install knapsack after bundle install! Otherwise oddly some native - # gems could not be found under some circumstance. No idea why, hours wasted. - - run_timed_command "gem install knapsack --no-document" - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - source scripts/rspec_helpers.sh + - *base-script - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration" artifacts: expire_in: 31d @@ -38,12 +41,7 @@ .rspec-base-migration: extends: .rails:rules:ee-and-foss-migration script: - # Only install knapsack after bundle install! Otherwise oddly some native - # gems could not be found under some circumstance. No idea why, hours wasted. - - run_timed_command "gem install knapsack --no-document" - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - source scripts/rspec_helpers.sh + - *base-script - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration" .rspec-base-pg11: @@ -76,12 +74,7 @@ .rspec-ee-base-geo: extends: .rspec-base script: - # Only install knapsack after bundle install! Otherwise oddly some native - # gems could not be found under some circumstance. No idea why, hours wasted. - - run_timed_command "gem install knapsack --no-document" - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - source scripts/rspec_helpers.sh + - *base-script - rspec_paralellized_job "--tag ~quarantine --tag geo" .rspec-ee-base-geo-pg11: @@ -318,8 +311,7 @@ gitlab:setup: # db/fixtures/development/04_project.rb thanks to SIZE=1 below - git clone https://gitlab.com/gitlab-org/gitlab-test.git /home/git/repositories/gitlab-org/gitlab-test.git - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" + - *base-script - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup artifacts: when: on_failure @@ -542,9 +534,7 @@ rspec fail-fast: stage: test needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"] script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - source scripts/rspec_helpers.sh + - *base-script - rspec_fail_fast tmp/matching_tests.txt "--tag ~quarantine" artifacts: expire_in: 7d @@ -557,9 +547,7 @@ rspec foss-impact: - .rails:rules:rspec-foss-impact needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests as-if-foss"] script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - source scripts/rspec_helpers.sh + - *base-script - rspec_matched_foss_tests tmp/matching_foss_tests.txt "--tag ~quarantine" artifacts: expire_in: 7d diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index c8624e812e0..46a1a957692 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -16,6 +16,11 @@ review-cleanup: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb - gcp_cleanup +.base-before_script: &base-before_script + - source ./scripts/utils.sh + - source ./scripts/review_apps/review-apps.sh + - install_api_client_dependencies_with_apk + review-build-cng: extends: - .default-retry @@ -23,7 +28,7 @@ review-build-cng: image: ruby:2.6-alpine stage: review-prepare before_script: - - source scripts/utils.sh + - source ./scripts/utils.sh - install_api_client_dependencies_with_apk - install_gitlab_gem needs: @@ -62,9 +67,7 @@ review-deploy: - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) - echo "${CI_ENVIRONMENT_URL}" > environment_url.txt - - source ./scripts/utils.sh - - install_api_client_dependencies_with_apk - - source scripts/review_apps/review-apps.sh + - *base-before_script script: - check_kube_domain - ensure_namespace @@ -97,9 +100,7 @@ review-deploy: # See https://gitlab.com/gitlab-org/gitlab/issues/191273 GIT_DEPTH: 1 before_script: - - apk add --update openssl - - source ./scripts/utils.sh - - source ./scripts/review_apps/review-apps.sh + - *base-before_script review-stop-failed-deployment: extends: @@ -142,8 +143,7 @@ review-stop: - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - echo "${CI_ENVIRONMENT_URL}" - echo "${QA_IMAGE}" - - source scripts/utils.sh - - install_api_client_dependencies_with_apk + - *base-before_script - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}} artifacts: paths: @@ -231,6 +231,6 @@ danger-review: stage: test needs: [] script: - - source scripts/utils.sh - - retry yarn install --frozen-lockfile + - source ./scripts/utils.sh + - run_timed_command "retry yarn install --frozen-lockfile" - danger --fail-on-errors=true --verbose diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index 1764e9136a1..2d83531e1db 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -38,6 +38,6 @@ update-tests-metadata: - rspec-ee integration pg11 geo - rspec-ee system pg11 geo script: - - retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document - - source scripts/rspec_helpers.sh + - run_timed_command "retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document" + - source ./scripts/rspec_helpers.sh - update_tests_metadata diff --git a/app/assets/javascripts/pages/search/show/index.js b/app/assets/javascripts/pages/search/show/index.js index 8dcc5aee00e..721219874cf 100644 --- a/app/assets/javascripts/pages/search/show/index.js +++ b/app/assets/javascripts/pages/search/show/index.js @@ -1,10 +1,7 @@ import Search from './search'; -import initStateFilter from '~/search/state_filter'; -import initConfidentialFilter from '~/search/confidential_filter'; +import initSearchApp from '~/search'; document.addEventListener('DOMContentLoaded', () => { - initStateFilter(); - initConfidentialFilter(); - + initSearchApp(); return new Search(); }); diff --git a/app/assets/javascripts/registry/explorer/components/details_page/tags_list_row.vue b/app/assets/javascripts/registry/explorer/components/details_page/tags_list_row.vue index aae3bf8161d..0f6297ca406 100644 --- a/app/assets/javascripts/registry/explorer/components/details_page/tags_list_row.vue +++ b/app/assets/javascripts/registry/explorer/components/details_page/tags_list_row.vue @@ -123,7 +123,7 @@ export default { v-if="tag.location" :title="tag.location" :text="tag.location" - css-class="btn-default btn-transparent btn-clipboard" + category="tertiary" /> <gl-icon diff --git a/app/assets/javascripts/repository/components/preview/index.vue b/app/assets/javascripts/repository/components/preview/index.vue index eca53f73a7f..4e2c8332f37 100644 --- a/app/assets/javascripts/repository/components/preview/index.vue +++ b/app/assets/javascripts/repository/components/preview/index.vue @@ -2,7 +2,7 @@ /* eslint-disable vue/no-v-html */ import $ from 'jquery'; import '~/behaviors/markdown/render_gfm'; -import { GlLink, GlLoadingIcon } from '@gitlab/ui'; +import { GlIcon, GlLink, GlLoadingIcon } from '@gitlab/ui'; import { handleLocationHash } from '~/lib/utils/common_utils'; import readmeQuery from '../../queries/readme.query.graphql'; @@ -19,6 +19,7 @@ export default { }, }, components: { + GlIcon, GlLink, GlLoadingIcon, }, @@ -51,7 +52,7 @@ export default { <article class="file-holder limited-width-container readme-holder"> <div class="js-file-title file-title-flex-parent"> <div class="file-header-content"> - <i aria-hidden="true" class="fa fa-file-text-o fa-fw"></i> + <gl-icon name="doc-text" aria-hidden="true" /> <gl-link :href="blob.webPath"> <strong>{{ blob.name }}</strong> </gl-link> diff --git a/app/assets/javascripts/search/confidential_filter/constants.js b/app/assets/javascripts/search/confidential_filter/constants.js deleted file mode 100644 index 4665ce6a5d1..00000000000 --- a/app/assets/javascripts/search/confidential_filter/constants.js +++ /dev/null @@ -1,28 +0,0 @@ -import { __ } from '~/locale'; - -export const FILTER_HEADER = __('Confidentiality'); - -export const FILTER_STATES = { - ANY: { - label: __('Any'), - value: null, - }, - CONFIDENTIAL: { - label: __('Confidential'), - value: 'yes', - }, - NOT_CONFIDENTIAL: { - label: __('Not confidential'), - value: 'no', - }, -}; - -export const SCOPES = { - ISSUES: 'issues', -}; - -export const FILTER_STATES_BY_SCOPE = { - [SCOPES.ISSUES]: [FILTER_STATES.ANY, FILTER_STATES.CONFIDENTIAL, FILTER_STATES.NOT_CONFIDENTIAL], -}; - -export const FILTER_PARAM = 'confidential'; diff --git a/app/assets/javascripts/search/confidential_filter/index.js b/app/assets/javascripts/search/confidential_filter/index.js deleted file mode 100644 index bec772be0dd..00000000000 --- a/app/assets/javascripts/search/confidential_filter/index.js +++ /dev/null @@ -1,39 +0,0 @@ -import Vue from 'vue'; -import Translate from '~/vue_shared/translate'; -import DropdownFilter from '../components/dropdown_filter.vue'; -import { - FILTER_HEADER, - FILTER_PARAM, - FILTER_STATES_BY_SCOPE, - FILTER_STATES, - SCOPES, -} from './constants'; - -Vue.use(Translate); - -export default () => { - const el = document.getElementById('js-search-filter-by-confidential'); - - if (!el) return false; - - return new Vue({ - el, - data() { - return { ...el.dataset }; - }, - - render(createElement) { - return createElement(DropdownFilter, { - props: { - initialFilter: this.filter, - filtersArray: FILTER_STATES_BY_SCOPE[this.scope], - filters: FILTER_STATES, - header: FILTER_HEADER, - param: FILTER_PARAM, - scope: this.scope, - supportedScopes: Object.values(SCOPES), - }, - }); - }, - }); -}; diff --git a/app/assets/javascripts/search/components/dropdown_filter.vue b/app/assets/javascripts/search/dropdown_filter/components/dropdown_filter.vue index cd9237026f2..b6e2dd46358 100644 --- a/app/assets/javascripts/search/components/dropdown_filter.vue +++ b/app/assets/javascripts/search/dropdown_filter/components/dropdown_filter.vue @@ -1,4 +1,5 @@ <script> +import { mapState } from 'vuex'; import { GlDropdown, GlDropdownItem, GlDropdownDivider } from '@gitlab/ui'; import { setUrlParams, visitUrl } from '~/lib/utils/url_utility'; import { sprintf, s__ } from '~/locale'; @@ -11,50 +12,27 @@ export default { GlDropdownDivider, }, props: { - initialFilter: { - type: String, - required: false, - default: null, - }, - filters: { + filterData: { type: Object, required: true, }, - filtersArray: { - type: Array, - required: true, - }, - header: { - type: String, - required: true, + }, + computed: { + ...mapState(['query']), + scope() { + return this.query.scope; }, - param: { - type: String, - required: true, + supportedScopes() { + return Object.values(this.filterData.scopes); }, - scope: { - type: String, - required: true, + initialFilter() { + return this.query[this.filterData.filterParam]; }, - supportedScopes: { - type: Array, - required: true, - }, - }, - computed: { filter() { - return this.initialFilter || this.filters.ANY.value; + return this.initialFilter || this.filterData.filters.ANY.value; }, - selectedFilterText() { - const f = this.filtersArray.find(({ value }) => value === this.selectedFilter); - if (!f || f === this.filters.ANY) { - return sprintf(s__('Any %{header}'), { header: this.header }); - } - - return f.label; - }, - showDropdown() { - return this.supportedScopes.includes(this.scope); + filtersArray() { + return this.filterData.filterByScope[this.scope]; }, selectedFilter: { get() { @@ -62,18 +40,29 @@ export default { return this.filter; } - return this.filters.ANY.value; + return this.filterData.filters.ANY.value; }, set(filter) { - visitUrl(setUrlParams({ [this.param]: filter })); + visitUrl(setUrlParams({ [this.filterData.filterParam]: filter })); }, }, + selectedFilterText() { + const f = this.filtersArray.find(({ value }) => value === this.selectedFilter); + if (!f || f === this.filterData.filters.ANY) { + return sprintf(s__('Any %{header}'), { header: this.filterData.header }); + } + + return f.label; + }, + showDropdown() { + return this.supportedScopes.includes(this.scope); + }, }, methods: { dropDownItemClass(filter) { return { 'gl-border-b-solid gl-border-b-gray-100 gl-border-b-1 gl-pb-2! gl-mb-2': - filter === this.filters.ANY, + filter === this.filterData.filters.ANY, }; }, isFilterSelected(filter) { @@ -94,7 +83,7 @@ export default { menu-class="gl-w-full! gl-pl-0" > <header class="gl-text-center gl-font-weight-bold gl-font-lg"> - {{ header }} + {{ filterData.header }} </header> <gl-dropdown-divider /> <gl-dropdown-item diff --git a/app/assets/javascripts/search/dropdown_filter/constants/confidential_filter_data.js b/app/assets/javascripts/search/dropdown_filter/constants/confidential_filter_data.js new file mode 100644 index 00000000000..b29daca89cb --- /dev/null +++ b/app/assets/javascripts/search/dropdown_filter/constants/confidential_filter_data.js @@ -0,0 +1,36 @@ +import { __ } from '~/locale'; + +const header = __('Confidentiality'); + +const filters = { + ANY: { + label: __('Any'), + value: null, + }, + CONFIDENTIAL: { + label: __('Confidential'), + value: 'yes', + }, + NOT_CONFIDENTIAL: { + label: __('Not confidential'), + value: 'no', + }, +}; + +const scopes = { + ISSUES: 'issues', +}; + +const filterByScope = { + [scopes.ISSUES]: [filters.ANY, filters.CONFIDENTIAL, filters.NOT_CONFIDENTIAL], +}; + +const filterParam = 'confidential'; + +export default { + header, + filters, + scopes, + filterByScope, + filterParam, +}; diff --git a/app/assets/javascripts/search/dropdown_filter/constants/state_filter_data.js b/app/assets/javascripts/search/dropdown_filter/constants/state_filter_data.js new file mode 100644 index 00000000000..0b93aa0be29 --- /dev/null +++ b/app/assets/javascripts/search/dropdown_filter/constants/state_filter_data.js @@ -0,0 +1,42 @@ +import { __ } from '~/locale'; + +const header = __('Status'); + +const filters = { + ANY: { + label: __('Any'), + value: 'all', + }, + OPEN: { + label: __('Open'), + value: 'opened', + }, + CLOSED: { + label: __('Closed'), + value: 'closed', + }, + MERGED: { + label: __('Merged'), + value: 'merged', + }, +}; + +const scopes = { + ISSUES: 'issues', + MERGE_REQUESTS: 'merge_requests', +}; + +const filterByScope = { + [scopes.ISSUES]: [filters.ANY, filters.OPEN, filters.CLOSED], + [scopes.MERGE_REQUESTS]: [filters.ANY, filters.OPEN, filters.MERGED, filters.CLOSED], +}; + +const filterParam = 'state'; + +export default { + header, + filters, + scopes, + filterByScope, + filterParam, +}; diff --git a/app/assets/javascripts/search/dropdown_filter/index.js b/app/assets/javascripts/search/dropdown_filter/index.js new file mode 100644 index 00000000000..e5e0745d990 --- /dev/null +++ b/app/assets/javascripts/search/dropdown_filter/index.js @@ -0,0 +1,38 @@ +import Vue from 'vue'; +import Translate from '~/vue_shared/translate'; +import DropdownFilter from './components/dropdown_filter.vue'; +import stateFilterData from './constants/state_filter_data'; +import confidentialFilterData from './constants/confidential_filter_data'; + +Vue.use(Translate); + +const mountDropdownFilter = (store, { id, filterData }) => { + const el = document.getElementById(id); + + if (!el) return false; + + return new Vue({ + el, + store, + render(createElement) { + return createElement(DropdownFilter, { + props: { + filterData, + }, + }); + }, + }); +}; + +const dropdownFilters = [ + { + id: 'js-search-filter-by-state', + filterData: stateFilterData, + }, + { + id: 'js-search-filter-by-confidential', + filterData: confidentialFilterData, + }, +]; + +export default store => [...dropdownFilters].map(filter => mountDropdownFilter(store, filter)); diff --git a/app/assets/javascripts/search/index.js b/app/assets/javascripts/search/index.js new file mode 100644 index 00000000000..780d3ff0d25 --- /dev/null +++ b/app/assets/javascripts/search/index.js @@ -0,0 +1,9 @@ +import { queryToObject } from '~/lib/utils/url_utility'; +import createStore from './store'; +import initDropdownFilters from './dropdown_filter'; + +export default () => { + const store = createStore({ query: queryToObject(window.location.search) }); + + initDropdownFilters(store); +}; diff --git a/app/assets/javascripts/search/state_filter/constants.js b/app/assets/javascripts/search/state_filter/constants.js deleted file mode 100644 index 00ae1bd9750..00000000000 --- a/app/assets/javascripts/search/state_filter/constants.js +++ /dev/null @@ -1,39 +0,0 @@ -import { __ } from '~/locale'; - -export const FILTER_HEADER = __('Status'); - -export const FILTER_STATES = { - ANY: { - label: __('Any'), - value: 'all', - }, - OPEN: { - label: __('Open'), - value: 'opened', - }, - CLOSED: { - label: __('Closed'), - value: 'closed', - }, - MERGED: { - label: __('Merged'), - value: 'merged', - }, -}; - -export const SCOPES = { - ISSUES: 'issues', - MERGE_REQUESTS: 'merge_requests', -}; - -export const FILTER_STATES_BY_SCOPE = { - [SCOPES.ISSUES]: [FILTER_STATES.ANY, FILTER_STATES.OPEN, FILTER_STATES.CLOSED], - [SCOPES.MERGE_REQUESTS]: [ - FILTER_STATES.ANY, - FILTER_STATES.OPEN, - FILTER_STATES.MERGED, - FILTER_STATES.CLOSED, - ], -}; - -export const FILTER_PARAM = 'state'; diff --git a/app/assets/javascripts/search/state_filter/index.js b/app/assets/javascripts/search/state_filter/index.js deleted file mode 100644 index 2c12885c40b..00000000000 --- a/app/assets/javascripts/search/state_filter/index.js +++ /dev/null @@ -1,39 +0,0 @@ -import Vue from 'vue'; -import Translate from '~/vue_shared/translate'; -import DropdownFilter from '../components/dropdown_filter.vue'; -import { - FILTER_HEADER, - FILTER_PARAM, - FILTER_STATES_BY_SCOPE, - FILTER_STATES, - SCOPES, -} from './constants'; - -Vue.use(Translate); - -export default () => { - const el = document.getElementById('js-search-filter-by-state'); - - if (!el) return false; - - return new Vue({ - el, - data() { - return { ...el.dataset }; - }, - - render(createElement) { - return createElement(DropdownFilter, { - props: { - initialFilter: this.filter, - filtersArray: FILTER_STATES_BY_SCOPE[this.scope], - filters: FILTER_STATES, - header: FILTER_HEADER, - param: FILTER_PARAM, - scope: this.scope, - supportedScopes: Object.values(SCOPES), - }, - }); - }, - }); -}; diff --git a/app/assets/javascripts/search/store/index.js b/app/assets/javascripts/search/store/index.js new file mode 100644 index 00000000000..10cfb647a92 --- /dev/null +++ b/app/assets/javascripts/search/store/index.js @@ -0,0 +1,12 @@ +import Vue from 'vue'; +import Vuex from 'vuex'; +import createState from './state'; + +Vue.use(Vuex); + +export const getStoreConfig = ({ query }) => ({ + state: createState({ query }), +}); + +const createStore = config => new Vuex.Store(getStoreConfig(config)); +export default createStore; diff --git a/app/assets/javascripts/search/store/state.js b/app/assets/javascripts/search/store/state.js new file mode 100644 index 00000000000..9115a613767 --- /dev/null +++ b/app/assets/javascripts/search/store/state.js @@ -0,0 +1,4 @@ +const createState = ({ query }) => ({ + query, +}); +export default createState; diff --git a/app/controllers/passwords_controller.rb b/app/controllers/passwords_controller.rb index c27226c3f3f..bc6975f8953 100644 --- a/app/controllers/passwords_controller.rb +++ b/app/controllers/passwords_controller.rb @@ -7,6 +7,8 @@ class PasswordsController < Devise::PasswordsController before_action :check_password_authentication_available, only: [:create] before_action :throttle_reset, only: [:create] + feature_category :authentication_and_authorization + # rubocop: disable CodeReuse/ActiveRecord def edit super diff --git a/app/controllers/profiles/accounts_controller.rb b/app/controllers/profiles/accounts_controller.rb index b19285e98bb..d8419be9f23 100644 --- a/app/controllers/profiles/accounts_controller.rb +++ b/app/controllers/profiles/accounts_controller.rb @@ -3,6 +3,8 @@ class Profiles::AccountsController < Profiles::ApplicationController include AuthHelper + feature_category :users + def show render(locals: show_view_variables) end diff --git a/app/controllers/profiles/active_sessions_controller.rb b/app/controllers/profiles/active_sessions_controller.rb index e4cd5d65e1a..1233c906406 100644 --- a/app/controllers/profiles/active_sessions_controller.rb +++ b/app/controllers/profiles/active_sessions_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Profiles::ActiveSessionsController < Profiles::ApplicationController + feature_category :users + def index @sessions = ActiveSession.list(current_user).reject(&:is_impersonated) end diff --git a/app/controllers/profiles/avatars_controller.rb b/app/controllers/profiles/avatars_controller.rb index 3378a09628c..d9e4b9a149d 100644 --- a/app/controllers/profiles/avatars_controller.rb +++ b/app/controllers/profiles/avatars_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Profiles::AvatarsController < Profiles::ApplicationController + feature_category :users + def destroy @user = current_user diff --git a/app/controllers/profiles/chat_names_controller.rb b/app/controllers/profiles/chat_names_controller.rb index 80b8279e91e..8cfec247b7a 100644 --- a/app/controllers/profiles/chat_names_controller.rb +++ b/app/controllers/profiles/chat_names_controller.rb @@ -4,6 +4,8 @@ class Profiles::ChatNamesController < Profiles::ApplicationController before_action :chat_name_token, only: [:new] before_action :chat_name_params, only: [:new, :create, :deny] + feature_category :users + def index @chat_names = current_user.chat_names end diff --git a/app/controllers/profiles/emails_controller.rb b/app/controllers/profiles/emails_controller.rb index da553e34ef6..6e5b18cb885 100644 --- a/app/controllers/profiles/emails_controller.rb +++ b/app/controllers/profiles/emails_controller.rb @@ -5,6 +5,8 @@ class Profiles::EmailsController < Profiles::ApplicationController before_action -> { rate_limit!(:profile_add_new_email) }, only: [:create] before_action -> { rate_limit!(:profile_resend_email_confirmation) }, only: [:resend_confirmation_instructions] + feature_category :users + def index @primary_email = current_user.email @emails = current_user.emails.order_id_desc diff --git a/app/controllers/profiles/gpg_keys_controller.rb b/app/controllers/profiles/gpg_keys_controller.rb index 8c34a66c374..7f04927f517 100644 --- a/app/controllers/profiles/gpg_keys_controller.rb +++ b/app/controllers/profiles/gpg_keys_controller.rb @@ -3,6 +3,8 @@ class Profiles::GpgKeysController < Profiles::ApplicationController before_action :set_gpg_key, only: [:destroy, :revoke] + feature_category :users + def index @gpg_keys = current_user.gpg_keys.with_subkeys @gpg_key = GpgKey.new diff --git a/app/controllers/profiles/groups_controller.rb b/app/controllers/profiles/groups_controller.rb index 04b5ee270dc..e76ee0a6cea 100644 --- a/app/controllers/profiles/groups_controller.rb +++ b/app/controllers/profiles/groups_controller.rb @@ -3,6 +3,8 @@ class Profiles::GroupsController < Profiles::ApplicationController include RoutableActions + feature_category :users + def update group = find_routable!(Group, params[:id]) notification_setting = current_user.notification_settings_for(group) diff --git a/app/controllers/profiles/keys_controller.rb b/app/controllers/profiles/keys_controller.rb index 965493955ac..1e6340f285e 100644 --- a/app/controllers/profiles/keys_controller.rb +++ b/app/controllers/profiles/keys_controller.rb @@ -3,6 +3,8 @@ class Profiles::KeysController < Profiles::ApplicationController skip_before_action :authenticate_user!, only: [:get_keys] + feature_category :users + def index @keys = current_user.keys.order_id_desc @key = Key.new diff --git a/app/controllers/profiles/notifications_controller.rb b/app/controllers/profiles/notifications_controller.rb index bc51830c119..a3e7638cdbc 100644 --- a/app/controllers/profiles/notifications_controller.rb +++ b/app/controllers/profiles/notifications_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Profiles::NotificationsController < Profiles::ApplicationController + feature_category :users + # rubocop: disable CodeReuse/ActiveRecord def show @user = current_user diff --git a/app/controllers/profiles/passwords_controller.rb b/app/controllers/profiles/passwords_controller.rb index fccbc29f598..85e901eb3eb 100644 --- a/app/controllers/profiles/passwords_controller.rb +++ b/app/controllers/profiles/passwords_controller.rb @@ -9,6 +9,8 @@ class Profiles::PasswordsController < Profiles::ApplicationController layout :determine_layout + feature_category :authentication_and_authorization + def new end diff --git a/app/controllers/profiles/personal_access_tokens_controller.rb b/app/controllers/profiles/personal_access_tokens_controller.rb index 21adc032940..b005347c43a 100644 --- a/app/controllers/profiles/personal_access_tokens_controller.rb +++ b/app/controllers/profiles/personal_access_tokens_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Profiles::PersonalAccessTokensController < Profiles::ApplicationController + feature_category :authentication_and_authorization + def index set_index_vars @personal_access_token = finder.build diff --git a/app/controllers/profiles/preferences_controller.rb b/app/controllers/profiles/preferences_controller.rb index ea4d3e861be..4d88491e9a8 100644 --- a/app/controllers/profiles/preferences_controller.rb +++ b/app/controllers/profiles/preferences_controller.rb @@ -3,6 +3,8 @@ class Profiles::PreferencesController < Profiles::ApplicationController before_action :user + feature_category :users + def show end diff --git a/app/controllers/profiles/two_factor_auths_controller.rb b/app/controllers/profiles/two_factor_auths_controller.rb index 5de6d84fdd9..7d82d6a3bcb 100644 --- a/app/controllers/profiles/two_factor_auths_controller.rb +++ b/app/controllers/profiles/two_factor_auths_controller.rb @@ -6,6 +6,8 @@ class Profiles::TwoFactorAuthsController < Profiles::ApplicationController push_frontend_feature_flag(:webauthn) end + feature_category :authentication_and_authorization + def show unless current_user.two_factor_enabled? current_user.otp_secret = User.generate_otp_secret(32) diff --git a/app/controllers/profiles/u2f_registrations_controller.rb b/app/controllers/profiles/u2f_registrations_controller.rb index 84ce4a56e64..32ca303e722 100644 --- a/app/controllers/profiles/u2f_registrations_controller.rb +++ b/app/controllers/profiles/u2f_registrations_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Profiles::U2fRegistrationsController < Profiles::ApplicationController + feature_category :authentication_and_authorization + def destroy u2f_registration = current_user.u2f_registrations.find(params[:id]) u2f_registration.destroy diff --git a/app/controllers/profiles/webauthn_registrations_controller.rb b/app/controllers/profiles/webauthn_registrations_controller.rb index 81b1dd6f710..a4a6d84f1ae 100644 --- a/app/controllers/profiles/webauthn_registrations_controller.rb +++ b/app/controllers/profiles/webauthn_registrations_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Profiles::WebauthnRegistrationsController < Profiles::ApplicationController + feature_category :authentication_and_authorization + def destroy webauthn_registration = current_user.webauthn_registrations.find(params[:id]) webauthn_registration.destroy diff --git a/app/controllers/profiles_controller.rb b/app/controllers/profiles_controller.rb index 248d5755d92..0687c057d47 100644 --- a/app/controllers/profiles_controller.rb +++ b/app/controllers/profiles_controller.rb @@ -10,6 +10,8 @@ class ProfilesController < Profiles::ApplicationController push_frontend_feature_flag(:webauthn) end + feature_category :users + def show end diff --git a/app/controllers/projects/alert_management_controller.rb b/app/controllers/projects/alert_management_controller.rb index 054dc8e6a35..8ecf8fadefd 100644 --- a/app/controllers/projects/alert_management_controller.rb +++ b/app/controllers/projects/alert_management_controller.rb @@ -3,6 +3,8 @@ class Projects::AlertManagementController < Projects::ApplicationController before_action :authorize_read_alert_management_alert! + feature_category :alert_management + def index end diff --git a/app/controllers/projects/alerting/notifications_controller.rb b/app/controllers/projects/alerting/notifications_controller.rb index fef8235628d..2241ded2db8 100644 --- a/app/controllers/projects/alerting/notifications_controller.rb +++ b/app/controllers/projects/alerting/notifications_controller.rb @@ -10,6 +10,8 @@ module Projects prepend_before_action :repository, :project_without_auth + feature_category :alert_management + def create token = extract_alert_manager_token(request) result = notify_service.execute(token) diff --git a/app/controllers/projects/artifacts_controller.rb b/app/controllers/projects/artifacts_controller.rb index 652687932fd..f6a92b07295 100644 --- a/app/controllers/projects/artifacts_controller.rb +++ b/app/controllers/projects/artifacts_controller.rb @@ -15,6 +15,8 @@ class Projects::ArtifactsController < Projects::ApplicationController MAX_PER_PAGE = 20 + feature_category :continuous_integration + def index # Loading artifacts is very expensive in projects with a lot of artifacts. # This feature flag prevents a DOS attack vector. diff --git a/app/controllers/projects/autocomplete_sources_controller.rb b/app/controllers/projects/autocomplete_sources_controller.rb index 26f2fb39972..001967b8bb4 100644 --- a/app/controllers/projects/autocomplete_sources_controller.rb +++ b/app/controllers/projects/autocomplete_sources_controller.rb @@ -3,6 +3,11 @@ class Projects::AutocompleteSourcesController < Projects::ApplicationController before_action :authorize_read_milestone!, only: :milestones + feature_category :issue_tracking, [:issues, :labels, :milestones, :commands] + feature_category :code_review, [:merge_requests] + feature_category :users, [:members] + feature_category :snippets, [:snippets] + def members render json: ::Projects::ParticipantsService.new(@project, current_user).execute(target) end diff --git a/app/controllers/projects/avatars_controller.rb b/app/controllers/projects/avatars_controller.rb index 6e6bf09a32a..f228206032d 100644 --- a/app/controllers/projects/avatars_controller.rb +++ b/app/controllers/projects/avatars_controller.rb @@ -5,6 +5,8 @@ class Projects::AvatarsController < Projects::ApplicationController before_action :authorize_admin_project!, only: [:destroy] + feature_category :projects + def show @blob = @repository.blob_at_branch(@repository.root_ref, @project.avatar_in_git) diff --git a/app/controllers/projects/badges_controller.rb b/app/controllers/projects/badges_controller.rb index eb47fec2b7e..855965ca6e1 100644 --- a/app/controllers/projects/badges_controller.rb +++ b/app/controllers/projects/badges_controller.rb @@ -6,6 +6,8 @@ class Projects::BadgesController < Projects::ApplicationController before_action :no_cache_headers, only: [:pipeline, :coverage] before_action :authorize_read_build!, only: [:pipeline, :coverage] + feature_category :continuous_integration + def pipeline pipeline_status = Gitlab::Badge::Pipeline::Status .new(project, params[:ref], opts: { diff --git a/app/controllers/projects/blame_controller.rb b/app/controllers/projects/blame_controller.rb index 374b4921dbc..d5de0d38152 100644 --- a/app/controllers/projects/blame_controller.rb +++ b/app/controllers/projects/blame_controller.rb @@ -9,6 +9,8 @@ class Projects::BlameController < Projects::ApplicationController before_action :assign_ref_vars before_action :authorize_download_code! + feature_category :source_code_management + def show @blob = @repository.blob_at(@commit.id, @path) diff --git a/app/controllers/projects/blob_controller.rb b/app/controllers/projects/blob_controller.rb index f0ac86d1581..c6251d27b05 100644 --- a/app/controllers/projects/blob_controller.rb +++ b/app/controllers/projects/blob_controller.rb @@ -39,6 +39,8 @@ class Projects::BlobController < Projects::ApplicationController track_redis_hll_event :create, :update, name: 'g_edit_by_sfe', feature: :track_editor_edit_actions, feature_default_enabled: true + feature_category :source_code_management + def new commit unless @repository.empty? end diff --git a/app/controllers/projects/boards_controller.rb b/app/controllers/projects/boards_controller.rb index 5ed35094476..193352ffa70 100644 --- a/app/controllers/projects/boards_controller.rb +++ b/app/controllers/projects/boards_controller.rb @@ -12,6 +12,8 @@ class Projects::BoardsController < Projects::ApplicationController push_frontend_feature_flag(:boards_with_swimlanes, project, default_enabled: false) end + feature_category :boards + private def assign_endpoint_vars diff --git a/app/controllers/projects/branches_controller.rb b/app/controllers/projects/branches_controller.rb index 7cfb4a508da..9124728ee25 100644 --- a/app/controllers/projects/branches_controller.rb +++ b/app/controllers/projects/branches_controller.rb @@ -13,6 +13,8 @@ class Projects::BranchesController < Projects::ApplicationController before_action :redirect_for_legacy_index_sort_or_search, only: [:index] before_action :limit_diverging_commit_counts!, only: [:diverging_commit_counts] + feature_category :source_code_management + def index respond_to do |format| format.html do diff --git a/app/controllers/projects/build_artifacts_controller.rb b/app/controllers/projects/build_artifacts_controller.rb index 99f4524eec5..148080a71f4 100644 --- a/app/controllers/projects/build_artifacts_controller.rb +++ b/app/controllers/projects/build_artifacts_controller.rb @@ -8,6 +8,8 @@ class Projects::BuildArtifactsController < Projects::ApplicationController before_action :extract_ref_name_and_path before_action :validate_artifacts!, except: [:download] + feature_category :continuous_integration + def download redirect_to download_project_job_artifacts_path(project, job, params: request.query_parameters) end diff --git a/app/controllers/projects/builds_controller.rb b/app/controllers/projects/builds_controller.rb index 6b3d70cb720..c5f6ed1c105 100644 --- a/app/controllers/projects/builds_controller.rb +++ b/app/controllers/projects/builds_controller.rb @@ -3,6 +3,8 @@ class Projects::BuildsController < Projects::ApplicationController before_action :authorize_read_build! + feature_category :continuous_integration + def index redirect_to project_jobs_path(project) end diff --git a/app/controllers/projects/ci/daily_build_group_report_results_controller.rb b/app/controllers/projects/ci/daily_build_group_report_results_controller.rb index 3d3b62fa797..d05ab1b4977 100644 --- a/app/controllers/projects/ci/daily_build_group_report_results_controller.rb +++ b/app/controllers/projects/ci/daily_build_group_report_results_controller.rb @@ -9,6 +9,8 @@ class Projects::Ci::DailyBuildGroupReportResultsController < Projects::Applicati before_action :authorize_read_build_report_results! before_action :validate_param_type! + feature_category :continuous_integration + def index respond_to do |format| format.csv { send_data(render_csv(report_results), type: 'text/csv; charset=utf-8') } diff --git a/app/controllers/projects/ci/lints_controller.rb b/app/controllers/projects/ci/lints_controller.rb index 813a0a9ddd5..7e900fc6051 100644 --- a/app/controllers/projects/ci/lints_controller.rb +++ b/app/controllers/projects/ci/lints_controller.rb @@ -6,6 +6,8 @@ class Projects::Ci::LintsController < Projects::ApplicationController push_frontend_feature_flag(:ci_lint_vue, project) end + feature_category :pipeline_authoring + def show end diff --git a/app/controllers/projects/commit_controller.rb b/app/controllers/projects/commit_controller.rb index b0c6f3cc6a1..2e48f2f0e45 100644 --- a/app/controllers/projects/commit_controller.rb +++ b/app/controllers/projects/commit_controller.rb @@ -21,6 +21,8 @@ class Projects::CommitController < Projects::ApplicationController BRANCH_SEARCH_LIMIT = 1000 + feature_category :source_code_management + def show apply_diff_view_cookie! diff --git a/app/controllers/projects/commits_controller.rb b/app/controllers/projects/commits_controller.rb index b161e44660e..d267ab732f9 100644 --- a/app/controllers/projects/commits_controller.rb +++ b/app/controllers/projects/commits_controller.rb @@ -15,6 +15,8 @@ class Projects::CommitsController < Projects::ApplicationController before_action :validate_ref!, except: :commits_root before_action :set_commits, except: :commits_root + feature_category :source_code_management + def commits_root redirect_to project_commits_path(@project, @project.default_branch) end diff --git a/app/controllers/projects/compare_controller.rb b/app/controllers/projects/compare_controller.rb index 943277afe95..6be0b465402 100644 --- a/app/controllers/projects/compare_controller.rb +++ b/app/controllers/projects/compare_controller.rb @@ -19,6 +19,8 @@ class Projects::CompareController < Projects::ApplicationController # Validation before_action :validate_refs! + feature_category :source_code_management + def index end diff --git a/app/controllers/projects/confluences_controller.rb b/app/controllers/projects/confluences_controller.rb index d563b34a362..fccbdf0bf91 100644 --- a/app/controllers/projects/confluences_controller.rb +++ b/app/controllers/projects/confluences_controller.rb @@ -3,6 +3,8 @@ class Projects::ConfluencesController < Projects::ApplicationController before_action :ensure_confluence + feature_category :integrations + def show end diff --git a/app/controllers/projects/cycle_analytics/events_controller.rb b/app/controllers/projects/cycle_analytics/events_controller.rb index c69bf029c73..3a5dd23047c 100644 --- a/app/controllers/projects/cycle_analytics/events_controller.rb +++ b/app/controllers/projects/cycle_analytics/events_controller.rb @@ -11,6 +11,8 @@ module Projects before_action :authorize_read_issue!, only: [:issue, :production] before_action :authorize_read_merge_request!, only: [:code, :review] + feature_category :planning_analytics + def issue render_events(cycle_analytics[:issue].events) end diff --git a/app/controllers/projects/cycle_analytics_controller.rb b/app/controllers/projects/cycle_analytics_controller.rb index ef97bc795f9..1ddc9d567e0 100644 --- a/app/controllers/projects/cycle_analytics_controller.rb +++ b/app/controllers/projects/cycle_analytics_controller.rb @@ -12,6 +12,8 @@ class Projects::CycleAnalyticsController < Projects::ApplicationController track_unique_visits :show, target_id: 'p_analytics_valuestream' + feature_category :planning_analytics + def show @cycle_analytics = ::CycleAnalytics::ProjectLevel.new(@project, options: options(cycle_analytics_project_params)) diff --git a/app/controllers/projects/deploy_keys_controller.rb b/app/controllers/projects/deploy_keys_controller.rb index 4f4adaea56e..ce25f86d692 100644 --- a/app/controllers/projects/deploy_keys_controller.rb +++ b/app/controllers/projects/deploy_keys_controller.rb @@ -10,6 +10,8 @@ class Projects::DeployKeysController < Projects::ApplicationController layout 'project_settings' + feature_category :continuous_delivery + def index respond_to do |format| format.html { redirect_to_repository } diff --git a/app/controllers/projects/deploy_tokens_controller.rb b/app/controllers/projects/deploy_tokens_controller.rb index 830b1f4fe4a..3c890bbafdf 100644 --- a/app/controllers/projects/deploy_tokens_controller.rb +++ b/app/controllers/projects/deploy_tokens_controller.rb @@ -3,6 +3,8 @@ class Projects::DeployTokensController < Projects::ApplicationController before_action :authorize_admin_project! + feature_category :continuous_delivery + def revoke @token = @project.deploy_tokens.find(params[:id]) @token.revoke! diff --git a/app/controllers/projects/deployments_controller.rb b/app/controllers/projects/deployments_controller.rb index 1344cf775e4..231684427fb 100644 --- a/app/controllers/projects/deployments_controller.rb +++ b/app/controllers/projects/deployments_controller.rb @@ -3,6 +3,8 @@ class Projects::DeploymentsController < Projects::ApplicationController before_action :authorize_read_deployment! + feature_category :continuous_delivery + # rubocop: disable CodeReuse/ActiveRecord def index deployments = environment.deployments.reorder(created_at: :desc) diff --git a/app/controllers/projects/design_management/designs_controller.rb b/app/controllers/projects/design_management/designs_controller.rb index fec09fa9515..550d8578396 100644 --- a/app/controllers/projects/design_management/designs_controller.rb +++ b/app/controllers/projects/design_management/designs_controller.rb @@ -3,6 +3,8 @@ class Projects::DesignManagement::DesignsController < Projects::ApplicationController before_action :authorize_read_design! + feature_category :design_management + private def authorize_read_design! diff --git a/app/controllers/projects/discussions_controller.rb b/app/controllers/projects/discussions_controller.rb index 06231607f73..b9ab1076999 100644 --- a/app/controllers/projects/discussions_controller.rb +++ b/app/controllers/projects/discussions_controller.rb @@ -9,6 +9,8 @@ class Projects::DiscussionsController < Projects::ApplicationController before_action :discussion, only: [:resolve, :unresolve] before_action :authorize_resolve_discussion!, only: [:resolve, :unresolve] + feature_category :issue_tracking + def resolve Discussions::ResolveService.new(project, current_user, one_or_more_discussions: discussion).execute diff --git a/app/controllers/projects/environments/prometheus_api_controller.rb b/app/controllers/projects/environments/prometheus_api_controller.rb index f0bb5360f84..97810d7d439 100644 --- a/app/controllers/projects/environments/prometheus_api_controller.rb +++ b/app/controllers/projects/environments/prometheus_api_controller.rb @@ -5,6 +5,8 @@ class Projects::Environments::PrometheusApiController < Projects::ApplicationCon before_action :proxyable + feature_category :metrics + private def proxyable diff --git a/app/controllers/projects/environments/sample_metrics_controller.rb b/app/controllers/projects/environments/sample_metrics_controller.rb index 9176c7cbd56..3df20810cb3 100644 --- a/app/controllers/projects/environments/sample_metrics_controller.rb +++ b/app/controllers/projects/environments/sample_metrics_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Projects::Environments::SampleMetricsController < Projects::ApplicationController + feature_category :metrics + def query result = Metrics::SampleMetricsService.new(params[:identifier], range_start: params[:start], range_end: params[:end]).query diff --git a/app/controllers/projects/environments_controller.rb b/app/controllers/projects/environments_controller.rb index 71195fdb892..c37abf82fe9 100644 --- a/app/controllers/projects/environments_controller.rb +++ b/app/controllers/projects/environments_controller.rb @@ -25,6 +25,8 @@ class Projects::EnvironmentsController < Projects::ApplicationController before_action :expire_etag_cache, only: [:index], unless: -> { request.format.json? } after_action :expire_etag_cache, only: [:cancel_auto_stop] + feature_category :continuous_delivery + def index @environments = project.environments .with_state(params[:scope] || :available) diff --git a/app/controllers/projects/error_tracking/base_controller.rb b/app/controllers/projects/error_tracking/base_controller.rb index 6efc6d00702..ffbe487d8a1 100644 --- a/app/controllers/projects/error_tracking/base_controller.rb +++ b/app/controllers/projects/error_tracking/base_controller.rb @@ -3,6 +3,8 @@ class Projects::ErrorTracking::BaseController < Projects::ApplicationController POLLING_INTERVAL = 1_000 + feature_category :error_tracking + def set_polling_interval Gitlab::PollingInterval.set_header(response, interval: POLLING_INTERVAL) end diff --git a/app/controllers/projects/error_tracking/projects_controller.rb b/app/controllers/projects/error_tracking/projects_controller.rb index 75a2c976d8b..d59cbc25d25 100644 --- a/app/controllers/projects/error_tracking/projects_controller.rb +++ b/app/controllers/projects/error_tracking/projects_controller.rb @@ -7,6 +7,8 @@ module Projects before_action :authorize_read_sentry_issue! + feature_category :error_tracking + def index service = ::ErrorTracking::ListProjectsService.new( project, diff --git a/app/controllers/projects/feature_flags_clients_controller.rb b/app/controllers/projects/feature_flags_clients_controller.rb index 02c9d9ab8fb..9a1f8932a27 100644 --- a/app/controllers/projects/feature_flags_clients_controller.rb +++ b/app/controllers/projects/feature_flags_clients_controller.rb @@ -4,6 +4,8 @@ class Projects::FeatureFlagsClientsController < Projects::ApplicationController before_action :authorize_admin_feature_flags_client! before_action :feature_flags_client + feature_category :feature_flags + def reset_token feature_flags_client.reset_token! diff --git a/app/controllers/projects/feature_flags_controller.rb b/app/controllers/projects/feature_flags_controller.rb index 4452b61508b..e9d450a6ce3 100644 --- a/app/controllers/projects/feature_flags_controller.rb +++ b/app/controllers/projects/feature_flags_controller.rb @@ -19,6 +19,8 @@ class Projects::FeatureFlagsController < Projects::ApplicationController push_frontend_feature_flag(:feature_flags_legacy_read_only_override, project) end + feature_category :feature_flags + def index @feature_flags = FeatureFlagsFinder .new(project, current_user, scope: params[:scope]) diff --git a/app/controllers/projects/feature_flags_user_lists_controller.rb b/app/controllers/projects/feature_flags_user_lists_controller.rb index 5427a892bff..7be3254e966 100644 --- a/app/controllers/projects/feature_flags_user_lists_controller.rb +++ b/app/controllers/projects/feature_flags_user_lists_controller.rb @@ -4,6 +4,8 @@ class Projects::FeatureFlagsUserListsController < Projects::ApplicationControlle before_action :authorize_admin_feature_flags_user_lists! before_action :user_list, only: [:edit, :show] + feature_category :feature_flags + def new end diff --git a/app/controllers/projects/find_file_controller.rb b/app/controllers/projects/find_file_controller.rb index c026e9ff332..89e72d98a33 100644 --- a/app/controllers/projects/find_file_controller.rb +++ b/app/controllers/projects/find_file_controller.rb @@ -10,6 +10,8 @@ class Projects::FindFileController < Projects::ApplicationController before_action :assign_ref_vars before_action :authorize_download_code! + feature_category :source_code_management + def show return render_404 unless @repository.commit(@ref) diff --git a/app/controllers/projects/forks_controller.rb b/app/controllers/projects/forks_controller.rb index c6b6b825bb7..1c2930f6e9b 100644 --- a/app/controllers/projects/forks_controller.rb +++ b/app/controllers/projects/forks_controller.rb @@ -14,6 +14,8 @@ class Projects::ForksController < Projects::ApplicationController before_action :authorize_fork_project!, only: [:new, :create] before_action :authorize_fork_namespace!, only: [:create] + feature_category :source_code_management + def index @total_forks_count = project.forks.size @public_forks_count = project.forks.public_only.size diff --git a/app/controllers/projects/grafana_api_controller.rb b/app/controllers/projects/grafana_api_controller.rb index c9870f1be2b..9c5d6c8ebc3 100644 --- a/app/controllers/projects/grafana_api_controller.rb +++ b/app/controllers/projects/grafana_api_controller.rb @@ -4,6 +4,8 @@ class Projects::GrafanaApiController < Projects::ApplicationController include RenderServiceResults include MetricsDashboard + feature_category :metrics + def proxy result = ::Grafana::ProxyService.new( project, diff --git a/app/controllers/projects/graphs_controller.rb b/app/controllers/projects/graphs_controller.rb index 91c07c37d2a..2b030793c58 100644 --- a/app/controllers/projects/graphs_controller.rb +++ b/app/controllers/projects/graphs_controller.rb @@ -11,6 +11,8 @@ class Projects::GraphsController < Projects::ApplicationController track_unique_visits :charts, target_id: 'p_analytics_repo' + feature_category :source_code_management + def show respond_to do |format| format.html diff --git a/app/controllers/projects/group_links_controller.rb b/app/controllers/projects/group_links_controller.rb index f8ea6f834a3..c6c90ffaba2 100644 --- a/app/controllers/projects/group_links_controller.rb +++ b/app/controllers/projects/group_links_controller.rb @@ -5,6 +5,8 @@ class Projects::GroupLinksController < Projects::ApplicationController before_action :authorize_admin_project! before_action :authorize_admin_project_member!, only: [:update] + feature_category :subgroups + def create group = Group.find(params[:link_group_id]) if params[:link_group_id].present? diff --git a/app/controllers/projects/hook_logs_controller.rb b/app/controllers/projects/hook_logs_controller.rb index ed7e7b68acb..99ebe3335c0 100644 --- a/app/controllers/projects/hook_logs_controller.rb +++ b/app/controllers/projects/hook_logs_controller.rb @@ -12,6 +12,8 @@ class Projects::HookLogsController < Projects::ApplicationController layout 'project_settings' + feature_category :integrations + def show end diff --git a/app/controllers/projects/hooks_controller.rb b/app/controllers/projects/hooks_controller.rb index 47f2e4323f2..8dabf3e640b 100644 --- a/app/controllers/projects/hooks_controller.rb +++ b/app/controllers/projects/hooks_controller.rb @@ -12,6 +12,8 @@ class Projects::HooksController < Projects::ApplicationController layout "project_settings" + feature_category :integrations + def index @hooks = @project.hooks @hook = ProjectHook.new diff --git a/app/controllers/projects/import/jira_controller.rb b/app/controllers/projects/import/jira_controller.rb index 976ac7df976..8418a607659 100644 --- a/app/controllers/projects/import/jira_controller.rb +++ b/app/controllers/projects/import/jira_controller.rb @@ -7,6 +7,8 @@ module Projects before_action :authorize_read_project! before_action :validate_jira_import_settings! + feature_category :integrations + def show end diff --git a/app/controllers/projects/imports_controller.rb b/app/controllers/projects/imports_controller.rb index 9bed12fd151..6cdd1c0bc8c 100644 --- a/app/controllers/projects/imports_controller.rb +++ b/app/controllers/projects/imports_controller.rb @@ -11,6 +11,8 @@ class Projects::ImportsController < Projects::ApplicationController before_action :redirect_if_progress, except: :show before_action :redirect_if_no_import, only: :show + feature_category :importers + def new end diff --git a/app/controllers/projects/incident_management/pager_duty_incidents_controller.rb b/app/controllers/projects/incident_management/pager_duty_incidents_controller.rb index dac1640dd08..f1264ca4a45 100644 --- a/app/controllers/projects/incident_management/pager_duty_incidents_controller.rb +++ b/app/controllers/projects/incident_management/pager_duty_incidents_controller.rb @@ -10,6 +10,8 @@ module Projects prepend_before_action :project_without_auth + feature_category :incident_management + def create result = webhook_processor.execute(params[:token]) diff --git a/app/controllers/projects/incidents_controller.rb b/app/controllers/projects/incidents_controller.rb index 5fef19362da..3395e75666e 100644 --- a/app/controllers/projects/incidents_controller.rb +++ b/app/controllers/projects/incidents_controller.rb @@ -7,6 +7,8 @@ class Projects::IncidentsController < Projects::ApplicationController before_action :authorize_read_issue! before_action :load_incident, only: [:show] + feature_category :incident_management + def index end diff --git a/app/controllers/projects/issue_links_controller.rb b/app/controllers/projects/issue_links_controller.rb index 2f7489373ed..35f3e00fae7 100644 --- a/app/controllers/projects/issue_links_controller.rb +++ b/app/controllers/projects/issue_links_controller.rb @@ -7,6 +7,8 @@ module Projects before_action :authorize_admin_issue_link!, only: [:create, :destroy] before_action :authorize_issue_link_association!, only: :destroy + feature_category :issue_tracking + private def authorize_admin_issue_link! diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index 319a5183429..1722dfa55bb 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -65,6 +65,17 @@ class Projects::IssuesController < Projects::ApplicationController alias_method :designs, :show + feature_category :issue_tracking, [ + :index, :calendar, :show, :new, :create, :edit, :update, + :destroy, :move, :reorder, :designs, :toggle_subscription, + :discussions, :bulk_update, :realtime_changes, + :toggle_award_emoji, :mark_as_spam, :related_branches, + :can_create_branch, :create_merge_request + ] + + feature_category :service_desk, [:service_desk] + feature_category :importers, [:import_csv, :export_csv] + def index @issues = @issuables diff --git a/app/controllers/projects/jobs_controller.rb b/app/controllers/projects/jobs_controller.rb index a8ac20cf96b..2a7dbbb43d7 100644 --- a/app/controllers/projects/jobs_controller.rb +++ b/app/controllers/projects/jobs_controller.rb @@ -16,6 +16,8 @@ class Projects::JobsController < Projects::ApplicationController layout 'project' + feature_category :continuous_integration + def index # We need all builds for tabs counters @all_builds = Ci::JobsFinder.new(current_user: current_user, project: @project).execute diff --git a/app/controllers/projects/labels_controller.rb b/app/controllers/projects/labels_controller.rb index ca2fad35451..ba8e6b90971 100644 --- a/app/controllers/projects/labels_controller.rb +++ b/app/controllers/projects/labels_controller.rb @@ -15,6 +15,8 @@ class Projects::LabelsController < Projects::ApplicationController respond_to :js, :html + feature_category :issue_tracking + def index @prioritized_labels = @available_labels.prioritized(@project) @labels = @available_labels.unprioritized(@project).page(params[:page]) diff --git a/app/controllers/projects/logs_controller.rb b/app/controllers/projects/logs_controller.rb index b9027b3a2cb..b9aa9bfc947 100644 --- a/app/controllers/projects/logs_controller.rb +++ b/app/controllers/projects/logs_controller.rb @@ -7,6 +7,8 @@ module Projects before_action :authorize_read_pod_logs! before_action :ensure_deployments, only: %i(k8s elasticsearch) + feature_category :logging + def index if environment || cluster render :index diff --git a/app/controllers/projects/mattermosts_controller.rb b/app/controllers/projects/mattermosts_controller.rb index cfaeddf711a..63a36732d87 100644 --- a/app/controllers/projects/mattermosts_controller.rb +++ b/app/controllers/projects/mattermosts_controller.rb @@ -10,6 +10,8 @@ class Projects::MattermostsController < Projects::ApplicationController before_action :service before_action :teams, only: [:new] + feature_category :integrations + def new end diff --git a/app/controllers/projects/merge_requests/application_controller.rb b/app/controllers/projects/merge_requests/application_controller.rb index 7b4024ed79c..9cac9f37eb7 100644 --- a/app/controllers/projects/merge_requests/application_controller.rb +++ b/app/controllers/projects/merge_requests/application_controller.rb @@ -5,6 +5,8 @@ class Projects::MergeRequests::ApplicationController < Projects::ApplicationCont before_action :merge_request before_action :authorize_read_merge_request! + feature_category :code_review + private def merge_request diff --git a/app/controllers/projects/metrics/dashboards/builder_controller.rb b/app/controllers/projects/metrics/dashboards/builder_controller.rb index 2ab574d7d10..96ca6d89111 100644 --- a/app/controllers/projects/metrics/dashboards/builder_controller.rb +++ b/app/controllers/projects/metrics/dashboards/builder_controller.rb @@ -6,6 +6,8 @@ module Projects class BuilderController < Projects::ApplicationController before_action :authorize_metrics_dashboard! + feature_category :metrics + def panel_preview respond_to do |format| format.json do diff --git a/app/controllers/projects/metrics_dashboard_controller.rb b/app/controllers/projects/metrics_dashboard_controller.rb index bc0a701b9fd..3f10749602e 100644 --- a/app/controllers/projects/metrics_dashboard_controller.rb +++ b/app/controllers/projects/metrics_dashboard_controller.rb @@ -14,6 +14,8 @@ module Projects push_frontend_feature_flag(:disable_metric_dashboard_refresh_rate) end + feature_category :metrics + def show if environment render 'projects/environments/metrics' diff --git a/app/controllers/projects/milestones_controller.rb b/app/controllers/projects/milestones_controller.rb index 8049a17068b..e6c4af00b29 100644 --- a/app/controllers/projects/milestones_controller.rb +++ b/app/controllers/projects/milestones_controller.rb @@ -21,6 +21,8 @@ class Projects::MilestonesController < Projects::ApplicationController respond_to :html + feature_category :issue_tracking + def index @sort = params[:sort] || 'due_date_asc' @milestones = milestones.sort_by_attribute(@sort) diff --git a/app/controllers/projects/mirrors_controller.rb b/app/controllers/projects/mirrors_controller.rb index 518e6a92afa..01abb72fc86 100644 --- a/app/controllers/projects/mirrors_controller.rb +++ b/app/controllers/projects/mirrors_controller.rb @@ -10,6 +10,8 @@ class Projects::MirrorsController < Projects::ApplicationController layout "project_settings" + feature_category :source_code_management + def show redirect_to_repository_settings(project, anchor: 'js-push-remote-settings') end diff --git a/app/controllers/projects/network_controller.rb b/app/controllers/projects/network_controller.rb index 47788438da9..89b679fc033 100644 --- a/app/controllers/projects/network_controller.rb +++ b/app/controllers/projects/network_controller.rb @@ -11,6 +11,8 @@ class Projects::NetworkController < Projects::ApplicationController before_action :assign_options before_action :assign_commit + feature_category :source_code_management + def show @url = project_network_path(@project, @ref, @options.merge(format: :json)) @commit_url = project_commit_path(@project, 'ae45ca32').gsub("ae45ca32", "%s") diff --git a/app/controllers/projects/notes_controller.rb b/app/controllers/projects/notes_controller.rb index c0b8c9e550d..e50e293a103 100644 --- a/app/controllers/projects/notes_controller.rb +++ b/app/controllers/projects/notes_controller.rb @@ -11,6 +11,8 @@ class Projects::NotesController < Projects::ApplicationController before_action :authorize_create_note!, only: [:create] before_action :authorize_resolve_note!, only: [:resolve, :unresolve] + feature_category :issue_tracking + def delete_attachment note.remove_attachment! note.update_attribute(:attachment, nil) diff --git a/app/controllers/projects/packages/package_files_controller.rb b/app/controllers/projects/packages/package_files_controller.rb index dd6d875cd1e..32aadb4fcf4 100644 --- a/app/controllers/projects/packages/package_files_controller.rb +++ b/app/controllers/projects/packages/package_files_controller.rb @@ -6,6 +6,8 @@ module Projects include PackagesAccess include SendFileUpload + feature_category :package_registry + def download package_file = project.package_files.find(params[:id]) diff --git a/app/controllers/projects/packages/packages_controller.rb b/app/controllers/projects/packages/packages_controller.rb index 302847eeaf5..15dc11f5df8 100644 --- a/app/controllers/projects/packages/packages_controller.rb +++ b/app/controllers/projects/packages/packages_controller.rb @@ -5,6 +5,8 @@ module Projects class PackagesController < Projects::ApplicationController include PackagesAccess + feature_category :package_registry + def show @package = project.packages.find(params[:id]) @package_files = @package.package_files.recent diff --git a/app/controllers/projects/pages_controller.rb b/app/controllers/projects/pages_controller.rb index 9ad6bf4095a..0aac517e3e3 100644 --- a/app/controllers/projects/pages_controller.rb +++ b/app/controllers/projects/pages_controller.rb @@ -8,6 +8,8 @@ class Projects::PagesController < Projects::ApplicationController before_action :authorize_update_pages!, except: [:show, :destroy] before_action :authorize_remove_pages!, only: [:destroy] + feature_category :pages + # rubocop: disable CodeReuse/ActiveRecord def show @domains = @project.pages_domains.order(:domain).present(current_user: current_user) diff --git a/app/controllers/projects/pages_domains_controller.rb b/app/controllers/projects/pages_domains_controller.rb index cccf8fe4358..a6b22a28b17 100644 --- a/app/controllers/projects/pages_domains_controller.rb +++ b/app/controllers/projects/pages_domains_controller.rb @@ -9,6 +9,8 @@ class Projects::PagesDomainsController < Projects::ApplicationController helper_method :domain_presenter + feature_category :pages + def show end diff --git a/app/controllers/projects/performance_monitoring/dashboards_controller.rb b/app/controllers/projects/performance_monitoring/dashboards_controller.rb index ec5a33f5dd6..51a07c1b7a5 100644 --- a/app/controllers/projects/performance_monitoring/dashboards_controller.rb +++ b/app/controllers/projects/performance_monitoring/dashboards_controller.rb @@ -12,6 +12,8 @@ module Projects respond_error(http_status: :bad_request, message: _('Request parameter %{param} is missing.') % { param: exception.param }) end + feature_category :metrics + def create result = ::Metrics::Dashboard::CloneDashboardService.new(project, current_user, dashboard_params).execute diff --git a/app/controllers/projects/pipeline_schedules_controller.rb b/app/controllers/projects/pipeline_schedules_controller.rb index e7e8a900060..5655d3b4c0d 100644 --- a/app/controllers/projects/pipeline_schedules_controller.rb +++ b/app/controllers/projects/pipeline_schedules_controller.rb @@ -10,6 +10,8 @@ class Projects::PipelineSchedulesController < Projects::ApplicationController before_action :authorize_update_pipeline_schedule!, except: [:index, :new, :create, :play] before_action :authorize_admin_pipeline_schedule!, only: [:destroy] + feature_category :continuous_integration + # rubocop: disable CodeReuse/ActiveRecord def index @scope = params[:scope] diff --git a/app/controllers/projects/pipelines/application_controller.rb b/app/controllers/projects/pipelines/application_controller.rb index 92887750813..c147d697888 100644 --- a/app/controllers/projects/pipelines/application_controller.rb +++ b/app/controllers/projects/pipelines/application_controller.rb @@ -10,6 +10,8 @@ module Projects before_action :pipeline before_action :authorize_read_pipeline! + feature_category :continuous_integration + private def pipeline diff --git a/app/controllers/projects/pipelines_controller.rb b/app/controllers/projects/pipelines_controller.rb index 8676c90ca86..954b55a821a 100644 --- a/app/controllers/projects/pipelines_controller.rb +++ b/app/controllers/projects/pipelines_controller.rb @@ -31,6 +31,8 @@ class Projects::PipelinesController < Projects::ApplicationController POLLING_INTERVAL = 10_000 + feature_category :continuous_integration + def index @pipelines = Ci::PipelinesFinder .new(project, current_user, index_params) diff --git a/app/controllers/projects/pipelines_settings_controller.rb b/app/controllers/projects/pipelines_settings_controller.rb index 7f988110977..6e08a889520 100644 --- a/app/controllers/projects/pipelines_settings_controller.rb +++ b/app/controllers/projects/pipelines_settings_controller.rb @@ -3,6 +3,8 @@ class Projects::PipelinesSettingsController < Projects::ApplicationController before_action :authorize_admin_pipeline! + feature_category :continuous_integration + def show redirect_to project_settings_ci_cd_path(@project, params: params.to_unsafe_h) end diff --git a/app/controllers/projects/product_analytics_controller.rb b/app/controllers/projects/product_analytics_controller.rb index c019dc191d6..5db7585d8e0 100644 --- a/app/controllers/projects/product_analytics_controller.rb +++ b/app/controllers/projects/product_analytics_controller.rb @@ -5,6 +5,8 @@ class Projects::ProductAnalyticsController < Projects::ApplicationController before_action :authorize_read_product_analytics! before_action :tracker_variables, only: [:setup, :test] + feature_category :product_analytics + def index @events = product_analytics_events.order_by_time.page(params[:page]) end diff --git a/app/controllers/projects/project_members_controller.rb b/app/controllers/projects/project_members_controller.rb index 3e52248f292..19d12e77217 100644 --- a/app/controllers/projects/project_members_controller.rb +++ b/app/controllers/projects/project_members_controller.rb @@ -8,6 +8,8 @@ class Projects::ProjectMembersController < Projects::ApplicationController # Authorize before_action :authorize_admin_project_member!, except: [:index, :leave, :request_access] + feature_category :authentication_and_authorization + def index @sort = params[:sort].presence || sort_value_name diff --git a/app/controllers/projects/prometheus/alerts_controller.rb b/app/controllers/projects/prometheus/alerts_controller.rb index c6ae65f7832..2892542e63c 100644 --- a/app/controllers/projects/prometheus/alerts_controller.rb +++ b/app/controllers/projects/prometheus/alerts_controller.rb @@ -16,6 +16,8 @@ module Projects before_action :authorize_read_prometheus_alerts!, except: [:notify] before_action :alert, only: [:update, :show, :destroy, :metrics_dashboard] + feature_category :alert_management + def index render json: serialize_as_json(alerts) end diff --git a/app/controllers/projects/prometheus/metrics_controller.rb b/app/controllers/projects/prometheus/metrics_controller.rb index 0340cb5beb0..d70d29a341f 100644 --- a/app/controllers/projects/prometheus/metrics_controller.rb +++ b/app/controllers/projects/prometheus/metrics_controller.rb @@ -6,6 +6,8 @@ module Projects before_action :authorize_admin_project! before_action :require_prometheus_metrics! + feature_category :metrics + def active_common respond_to do |format| format.json do diff --git a/app/controllers/projects/protected_refs_controller.rb b/app/controllers/projects/protected_refs_controller.rb index f1a7ac36138..4cba1a75330 100644 --- a/app/controllers/projects/protected_refs_controller.rb +++ b/app/controllers/projects/protected_refs_controller.rb @@ -10,6 +10,8 @@ class Projects::ProtectedRefsController < Projects::ApplicationController layout "project_settings" + feature_category :source_code_management + def index redirect_to_repository_settings(@project) end diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb index 29f1e4bfd44..a9490c106d4 100644 --- a/app/controllers/projects/raw_controller.rb +++ b/app/controllers/projects/raw_controller.rb @@ -15,6 +15,8 @@ class Projects::RawController < Projects::ApplicationController before_action :no_cache_headers, only: [:show] before_action :redirect_to_external_storage, only: :show, if: :static_objects_external_storage_enabled? + feature_category :source_code_management + def show @blob = @repository.blob_at(@commit.id, @path) diff --git a/app/controllers/projects/refs_controller.rb b/app/controllers/projects/refs_controller.rb index db770d3e438..4d23c853334 100644 --- a/app/controllers/projects/refs_controller.rb +++ b/app/controllers/projects/refs_controller.rb @@ -11,6 +11,8 @@ class Projects::RefsController < Projects::ApplicationController before_action :assign_ref_vars before_action :authorize_download_code! + feature_category :source_code_management + def switch respond_to do |format| format.html do diff --git a/app/controllers/projects/registry/application_controller.rb b/app/controllers/projects/registry/application_controller.rb index 2f891d78c91..e7bf8c8e757 100644 --- a/app/controllers/projects/registry/application_controller.rb +++ b/app/controllers/projects/registry/application_controller.rb @@ -8,6 +8,8 @@ module Projects before_action :verify_registry_enabled! before_action :authorize_read_container_image! + feature_category :container_registry + private def verify_registry_enabled! diff --git a/app/controllers/projects/releases/evidences_controller.rb b/app/controllers/projects/releases/evidences_controller.rb index 34e450d903f..1e2dbf8047c 100644 --- a/app/controllers/projects/releases/evidences_controller.rb +++ b/app/controllers/projects/releases/evidences_controller.rb @@ -7,6 +7,8 @@ module Projects before_action :release before_action :authorize_read_release_evidence! + feature_category :release_evidence + def show respond_to do |format| format.json do diff --git a/app/controllers/projects/releases_controller.rb b/app/controllers/projects/releases_controller.rb index f48afce98fa..b8c6d1d6c9e 100644 --- a/app/controllers/projects/releases_controller.rb +++ b/app/controllers/projects/releases_controller.rb @@ -13,6 +13,8 @@ class Projects::ReleasesController < Projects::ApplicationController before_action :authorize_update_release!, only: %i[edit update] before_action :authorize_create_release!, only: :new + feature_category :release_orchestration + def index respond_to do |format| format.html do diff --git a/app/controllers/projects/repositories_controller.rb b/app/controllers/projects/repositories_controller.rb index 303326bbe23..ba3ab52e3af 100644 --- a/app/controllers/projects/repositories_controller.rb +++ b/app/controllers/projects/repositories_controller.rb @@ -18,6 +18,8 @@ class Projects::RepositoriesController < Projects::ApplicationController before_action :authorize_admin_project!, only: :create before_action :redirect_to_external_storage, only: :archive, if: :static_objects_external_storage_enabled? + feature_category :source_code_management + def create @project.create_repository diff --git a/app/controllers/projects/runner_projects_controller.rb b/app/controllers/projects/runner_projects_controller.rb index cbeb32fd610..d225d5e104c 100644 --- a/app/controllers/projects/runner_projects_controller.rb +++ b/app/controllers/projects/runner_projects_controller.rb @@ -5,6 +5,8 @@ class Projects::RunnerProjectsController < Projects::ApplicationController layout 'project_settings' + feature_category :continuous_integration + def create @runner = Ci::Runner.find(params[:runner_project][:runner_id]) diff --git a/app/controllers/projects/runners_controller.rb b/app/controllers/projects/runners_controller.rb index fb00156d320..544074f9840 100644 --- a/app/controllers/projects/runners_controller.rb +++ b/app/controllers/projects/runners_controller.rb @@ -6,6 +6,8 @@ class Projects::RunnersController < Projects::ApplicationController layout 'project_settings' + feature_category :continuous_integration + def index redirect_to project_settings_ci_cd_path(@project, anchor: 'js-runners-settings') end diff --git a/app/controllers/projects/serverless/functions_controller.rb b/app/controllers/projects/serverless/functions_controller.rb index 0b55414d390..4168880001c 100644 --- a/app/controllers/projects/serverless/functions_controller.rb +++ b/app/controllers/projects/serverless/functions_controller.rb @@ -5,6 +5,8 @@ module Projects class FunctionsController < Projects::ApplicationController before_action :authorize_read_cluster! + feature_category :serverless + def index respond_to do |format| format.json do diff --git a/app/controllers/projects/service_desk_controller.rb b/app/controllers/projects/service_desk_controller.rb index bcd190bbc2c..f7c0a54fb9e 100644 --- a/app/controllers/projects/service_desk_controller.rb +++ b/app/controllers/projects/service_desk_controller.rb @@ -3,6 +3,8 @@ class Projects::ServiceDeskController < Projects::ApplicationController before_action :authorize_admin_project! + feature_category :service_desk + def show json_response end diff --git a/app/controllers/projects/services_controller.rb b/app/controllers/projects/services_controller.rb index 41ce834c658..93ad549bc50 100644 --- a/app/controllers/projects/services_controller.rb +++ b/app/controllers/projects/services_controller.rb @@ -19,6 +19,8 @@ class Projects::ServicesController < Projects::ApplicationController layout "project_settings" + feature_category :integrations + def edit @default_integration = Service.default_integration(service.type, project) end diff --git a/app/controllers/projects/settings/access_tokens_controller.rb b/app/controllers/projects/settings/access_tokens_controller.rb index d6b4c4dd5dc..cbd6716fdf7 100644 --- a/app/controllers/projects/settings/access_tokens_controller.rb +++ b/app/controllers/projects/settings/access_tokens_controller.rb @@ -7,6 +7,8 @@ module Projects before_action :check_feature_availability + feature_category :authentication_and_authorization + def index @project_access_token = PersonalAccessToken.new set_index_vars diff --git a/app/controllers/projects/settings/ci_cd_controller.rb b/app/controllers/projects/settings/ci_cd_controller.rb index b1c9a412cc7..0920a27fcfb 100644 --- a/app/controllers/projects/settings/ci_cd_controller.rb +++ b/app/controllers/projects/settings/ci_cd_controller.rb @@ -14,6 +14,8 @@ module Projects helper_method :highlight_badge + feature_category :continuous_integration + def show if Feature.enabled?(:ci_pipeline_triggers_settings_vue_ui, @project) @triggers_json = ::Ci::TriggerSerializer.new.represent( diff --git a/app/controllers/projects/settings/integrations_controller.rb b/app/controllers/projects/settings/integrations_controller.rb index 96bf7f474d1..fba11ff1497 100644 --- a/app/controllers/projects/settings/integrations_controller.rb +++ b/app/controllers/projects/settings/integrations_controller.rb @@ -6,6 +6,8 @@ module Projects before_action :authorize_admin_project! layout "project_settings" + feature_category :integrations + def show @services = @project.find_or_initialize_services end diff --git a/app/controllers/projects/settings/operations_controller.rb b/app/controllers/projects/settings/operations_controller.rb index 60f61fd7a33..c407b15e29f 100644 --- a/app/controllers/projects/settings/operations_controller.rb +++ b/app/controllers/projects/settings/operations_controller.rb @@ -11,6 +11,8 @@ module Projects helper_method :error_tracking_setting helper_method :tracing_setting + feature_category :incident_management + def update result = ::Projects::Operations::UpdateService.new(project, current_user, update_params).execute diff --git a/app/controllers/projects/settings/repository_controller.rb b/app/controllers/projects/settings/repository_controller.rb index 182968b9a41..0994bebb1d0 100644 --- a/app/controllers/projects/settings/repository_controller.rb +++ b/app/controllers/projects/settings/repository_controller.rb @@ -10,6 +10,9 @@ module Projects push_frontend_feature_flag(:deploy_keys_on_protected_branches, @project) end + feature_category :source_code_management, [:show, :cleanup] + feature_category :continuous_delivery, [:create_deploy_token] + def show render_show end diff --git a/app/controllers/projects/snippets/application_controller.rb b/app/controllers/projects/snippets/application_controller.rb index 3f488b07e96..8ee12bf3795 100644 --- a/app/controllers/projects/snippets/application_controller.rb +++ b/app/controllers/projects/snippets/application_controller.rb @@ -4,6 +4,8 @@ class Projects::Snippets::ApplicationController < Projects::ApplicationControlle include FindSnippet include SnippetAuthorizations + feature_category :snippets + private # This overrides the default snippet create authorization diff --git a/app/controllers/projects/starrers_controller.rb b/app/controllers/projects/starrers_controller.rb index d9654f4f72a..91f49fc4d66 100644 --- a/app/controllers/projects/starrers_controller.rb +++ b/app/controllers/projects/starrers_controller.rb @@ -3,6 +3,8 @@ class Projects::StarrersController < Projects::ApplicationController include SortingHelper + feature_category :projects + def index @starrers = UsersStarProjectsFinder.new(@project, params, current_user: @current_user).execute @sort = params[:sort].presence || sort_value_name diff --git a/app/controllers/projects/static_site_editor_controller.rb b/app/controllers/projects/static_site_editor_controller.rb index f99ffe170b0..036c731f480 100644 --- a/app/controllers/projects/static_site_editor_controller.rb +++ b/app/controllers/projects/static_site_editor_controller.rb @@ -13,6 +13,8 @@ class Projects::StaticSiteEditorController < Projects::ApplicationController push_frontend_feature_flag(:sse_image_uploads) end + feature_category :static_site_editor + def show service_response = ::StaticSiteEditor::ConfigService.new( container: project, diff --git a/app/controllers/projects/tags/releases_controller.rb b/app/controllers/projects/tags/releases_controller.rb index c1f4cbce054..8e5539f546b 100644 --- a/app/controllers/projects/tags/releases_controller.rb +++ b/app/controllers/projects/tags/releases_controller.rb @@ -8,6 +8,8 @@ class Projects::Tags::ReleasesController < Projects::ApplicationController before_action :tag before_action :release + feature_category :release_evidence + def edit end diff --git a/app/controllers/projects/tags_controller.rb b/app/controllers/projects/tags_controller.rb index 71effebf1c0..1d783241196 100644 --- a/app/controllers/projects/tags_controller.rb +++ b/app/controllers/projects/tags_controller.rb @@ -10,6 +10,9 @@ class Projects::TagsController < Projects::ApplicationController before_action :authorize_download_code! before_action :authorize_admin_tag!, only: [:new, :create, :destroy] + feature_category :source_code_management, [:index, :show, :new, :destroy] + feature_category :release_evidence, [:create] + # rubocop: disable CodeReuse/ActiveRecord def index params[:sort] = params[:sort].presence || sort_value_recently_updated diff --git a/app/controllers/projects/templates_controller.rb b/app/controllers/projects/templates_controller.rb index 95739f96d39..7ab23e39cf0 100644 --- a/app/controllers/projects/templates_controller.rb +++ b/app/controllers/projects/templates_controller.rb @@ -5,6 +5,8 @@ class Projects::TemplatesController < Projects::ApplicationController before_action :authorize_can_read_issuable! before_action :get_template_class + feature_category :templates + def show template = @template_type.find(params[:key], project) diff --git a/app/controllers/projects/todos_controller.rb b/app/controllers/projects/todos_controller.rb index 33205b93317..6ba89ab34f8 100644 --- a/app/controllers/projects/todos_controller.rb +++ b/app/controllers/projects/todos_controller.rb @@ -6,6 +6,8 @@ class Projects::TodosController < Projects::ApplicationController before_action :authenticate_user!, only: [:create] + feature_category :issue_tracking + private def issuable diff --git a/app/controllers/projects/tracings_controller.rb b/app/controllers/projects/tracings_controller.rb index 85098b66997..2bc0c590e8d 100644 --- a/app/controllers/projects/tracings_controller.rb +++ b/app/controllers/projects/tracings_controller.rb @@ -12,6 +12,8 @@ module Projects before_action :authorize_update_environment! + feature_category :tracing + def show end diff --git a/app/controllers/projects/tree_controller.rb b/app/controllers/projects/tree_controller.rb index 638e1a05c18..b5cfc3990b2 100644 --- a/app/controllers/projects/tree_controller.rb +++ b/app/controllers/projects/tree_controller.rb @@ -15,6 +15,8 @@ class Projects::TreeController < Projects::ApplicationController before_action :authorize_download_code! before_action :authorize_edit_tree!, only: [:create_dir] + feature_category :source_code_management + def show return render_404 unless @commit diff --git a/app/controllers/projects/triggers_controller.rb b/app/controllers/projects/triggers_controller.rb index 7159d0243a3..eec35fcec8d 100644 --- a/app/controllers/projects/triggers_controller.rb +++ b/app/controllers/projects/triggers_controller.rb @@ -8,6 +8,8 @@ class Projects::TriggersController < Projects::ApplicationController layout 'project_settings' + feature_category :continuous_integration + def index redirect_to project_settings_ci_cd_path(@project, anchor: 'js-pipeline-triggers') end diff --git a/app/controllers/projects/uploads_controller.rb b/app/controllers/projects/uploads_controller.rb index 72251988b5e..c15768e7bbb 100644 --- a/app/controllers/projects/uploads_controller.rb +++ b/app/controllers/projects/uploads_controller.rb @@ -11,6 +11,8 @@ class Projects::UploadsController < Projects::ApplicationController before_action :authorize_upload_file!, only: [:create, :authorize] before_action :verify_workhorse_api!, only: [:authorize] + feature_category :not_owned + private def upload_model_class diff --git a/app/controllers/projects/usage_ping_controller.rb b/app/controllers/projects/usage_ping_controller.rb index 0e2c8f879b1..9b4ddb326c1 100644 --- a/app/controllers/projects/usage_ping_controller.rb +++ b/app/controllers/projects/usage_ping_controller.rb @@ -3,6 +3,8 @@ class Projects::UsagePingController < Projects::ApplicationController before_action :authenticate_user! + feature_category :collection + def web_ide_clientside_preview return render_404 unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? diff --git a/app/controllers/projects/variables_controller.rb b/app/controllers/projects/variables_controller.rb index 0fd047f90cf..d8efc1b7b54 100644 --- a/app/controllers/projects/variables_controller.rb +++ b/app/controllers/projects/variables_controller.rb @@ -3,6 +3,8 @@ class Projects::VariablesController < Projects::ApplicationController before_action :authorize_admin_build! + feature_category :continuous_integration + def show respond_to do |format| format.json do diff --git a/app/controllers/projects/web_ide_schemas_controller.rb b/app/controllers/projects/web_ide_schemas_controller.rb index 3d16a6fafd4..84a191815f4 100644 --- a/app/controllers/projects/web_ide_schemas_controller.rb +++ b/app/controllers/projects/web_ide_schemas_controller.rb @@ -3,6 +3,8 @@ class Projects::WebIdeSchemasController < Projects::ApplicationController before_action :authenticate_user! + feature_category :web_ide + def show return respond_422 unless branch_sha diff --git a/app/controllers/projects/web_ide_terminals_controller.rb b/app/controllers/projects/web_ide_terminals_controller.rb index 76bcaa9e80c..1d179765ad9 100644 --- a/app/controllers/projects/web_ide_terminals_controller.rb +++ b/app/controllers/projects/web_ide_terminals_controller.rb @@ -8,6 +8,8 @@ class Projects::WebIdeTerminalsController < Projects::ApplicationController before_action :authorize_read_web_ide_terminal!, except: [:check_config, :create] before_action :authorize_update_web_ide_terminal!, only: [:cancel, :retry] + feature_category :web_ide + def check_config return respond_422 unless branch_sha diff --git a/app/controllers/projects/wikis_controller.rb b/app/controllers/projects/wikis_controller.rb index d0aa733cadb..8f794512486 100644 --- a/app/controllers/projects/wikis_controller.rb +++ b/app/controllers/projects/wikis_controller.rb @@ -5,6 +5,8 @@ class Projects::WikisController < Projects::ApplicationController alias_method :container, :project + feature_category :wiki + def git_access end end diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 88a4a276750..3bbfc8e20f1 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -47,6 +47,16 @@ class ProjectsController < Projects::ApplicationController layout :determine_layout + feature_category :projects, [ + :index, :show, :new, :create, :edit, :update, :transfer, + :destroy, :resolve, :archive, :unarchive, :toggle_star + ] + + feature_category :source_code_management, [:remove_fork, :housekeeping, :refs] + feature_category :issue_tracking, [:preview_markdown, :new_issuable_address] + feature_category :importers, [:export, :remove_export, :generate_new_export, :download_export] + feature_category :audit_events, [:activity] + def index redirect_to(current_user ? root_path : explore_root_path) end diff --git a/app/graphql/mutations/boards/create.rb b/app/graphql/mutations/boards/create.rb new file mode 100644 index 00000000000..e381205242e --- /dev/null +++ b/app/graphql/mutations/boards/create.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Mutations + module Boards + class Create < ::Mutations::BaseMutation + include Mutations::ResolvesGroup + include ResolvesProject + + graphql_name 'CreateBoard' + + field :board, + Types::BoardType, + null: true, + description: 'The board after mutation.' + + argument :project_path, GraphQL::ID_TYPE, + required: false, + description: 'The project full path the board is associated with.' + argument :group_path, GraphQL::ID_TYPE, + required: false, + description: 'The group full path the board is associated with.' + argument :name, + GraphQL::STRING_TYPE, + required: false, + description: 'The board name.' + argument :assignee_id, + GraphQL::STRING_TYPE, + required: false, + description: 'The ID of the user to be assigned to the board.' + argument :milestone_id, + GraphQL::ID_TYPE, + required: false, + description: 'The ID of the milestone to be assigned to the board.' + argument :weight, + GraphQL::BOOLEAN_TYPE, + required: false, + description: 'The weight of the board.' + argument :label_ids, + [GraphQL::ID_TYPE], + required: false, + description: 'The IDs of labels to be added to the board.' + + authorize :admin_board + + def resolve(args) + group_path = args.delete(:group_path) + project_path = args.delete(:project_path) + + board_parent = authorized_find!(group_path: group_path, project_path: project_path) + response = ::Boards::CreateService.new(board_parent, current_user, args).execute + + { + board: response.payload, + errors: response.errors + } + end + + def ready?(**args) + if args.values_at(:project_path, :group_path).compact.blank? + raise Gitlab::Graphql::Errors::ArgumentError, + 'group_path or project_path arguments are required' + end + + super + end + + private + + def find_object(group_path: nil, project_path: nil) + if group_path + resolve_group(full_path: group_path) + else + resolve_project(full_path: project_path) + end + end + end + end +end diff --git a/app/graphql/types/mutation_type.rb b/app/graphql/types/mutation_type.rb index 3a9d5529266..64325333e08 100644 --- a/app/graphql/types/mutation_type.rb +++ b/app/graphql/types/mutation_type.rb @@ -14,6 +14,7 @@ module Types mount_mutation Mutations::AwardEmojis::Add mount_mutation Mutations::AwardEmojis::Remove mount_mutation Mutations::AwardEmojis::Toggle + mount_mutation Mutations::Boards::Create mount_mutation Mutations::Boards::Destroy mount_mutation Mutations::Boards::Issues::IssueMoveList mount_mutation Mutations::Boards::Lists::Create diff --git a/app/models/design_management/design_at_version.rb b/app/models/design_management/design_at_version.rb index 0cf653ce696..211211144f4 100644 --- a/app/models/design_management/design_at_version.rb +++ b/app/models/design_management/design_at_version.rb @@ -21,10 +21,6 @@ module DesignManagement @design, @version = design, version end - def self.instantiate(attrs) - new(**attrs).tap { |obj| obj.validate! } - end - # The ID, needed by GraphQL types and as part of the Lazy-fetch # protocol, includes information about both the design and the version. # diff --git a/app/views/admin/application_settings/_usage.html.haml b/app/views/admin/application_settings/_usage.html.haml index 4e45db1e10a..88c88b96a91 100644 --- a/app/views/admin/application_settings/_usage.html.haml +++ b/app/views/admin/application_settings/_usage.html.haml @@ -33,7 +33,7 @@ %pre.usage-data.js-syntax-highlight.code.highlight.mt-2.d-none{ class: payload_class, data: { endpoint: usage_data_admin_application_settings_path(format: :html) } } - else = _('The usage ping is disabled, and cannot be configured through this form.') - - deactivating_usage_ping_path = help_page_path('development/telemetry/usage_ping', anchor: 'disable-usage-ping') + - deactivating_usage_ping_path = help_page_path('development/product_analytics/usage_ping', anchor: 'disable-usage-ping') - deactivating_usage_ping_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: deactivating_usage_ping_path } = s_('For more information, see the documentation on %{deactivating_usage_ping_link_start}deactivating the usage ping%{deactivating_usage_ping_link_end}.').html_safe % { deactivating_usage_ping_link_start: deactivating_usage_ping_link_start, deactivating_usage_ping_link_end: '</a>'.html_safe } diff --git a/app/views/admin/dev_ops_report/show.html.haml b/app/views/admin/dev_ops_report/show.html.haml index 1892557d0d6..93ed862d733 100644 --- a/app/views/admin/dev_ops_report/show.html.haml +++ b/app/views/admin/dev_ops_report/show.html.haml @@ -7,7 +7,7 @@ .gl-mt-3 - if !usage_ping_enabled - #js-devops-empty-state{ data: { is_admin: current_user&.admin.to_s, empty_state_svg_path: image_path('illustrations/convdev/convdev_no_index.svg'), enable_usage_ping_link: metrics_and_profiling_admin_application_settings_path(anchor: 'js-usage-settings'), docs_link: help_page_path('development/telemetry/usage_ping') } } + #js-devops-empty-state{ data: { is_admin: current_user&.admin.to_s, empty_state_svg_path: image_path('illustrations/convdev/convdev_no_index.svg'), enable_usage_ping_link: metrics_and_profiling_admin_application_settings_path(anchor: 'js-usage-settings'), docs_link: help_page_path('development/product_analytics/usage_ping') } } - elsif @metric.blank? = render 'no_data' - else diff --git a/app/views/search/results/_filters.html.haml b/app/views/search/results/_filters.html.haml index 8c402ddb3d1..632d3dfd58c 100644 --- a/app/views/search/results/_filters.html.haml +++ b/app/views/search/results/_filters.html.haml @@ -1,7 +1,7 @@ .d-lg-flex.align-items-end - #js-search-filter-by-state{ 'v-cloak': true, data: { scope: @scope, filter: params[:state]} } + #js-search-filter-by-state{ 'v-cloak': true } - if Feature.enabled?(:search_filter_by_confidential, @group) - #js-search-filter-by-confidential{ 'v-cloak': true, data: { scope: @scope, filter: params[:confidential] } } + #js-search-filter-by-confidential{ 'v-cloak': true } - if %w(issues merge_requests).include?(@scope) %hr.gl-mt-4.gl-mb-4 diff --git a/changelogs/unreleased/233433-create-board-graphql.yml b/changelogs/unreleased/233433-create-board-graphql.yml new file mode 100644 index 00000000000..f8db2d62d6f --- /dev/null +++ b/changelogs/unreleased/233433-create-board-graphql.yml @@ -0,0 +1,5 @@ +--- +title: Create an issue board via GraphQL mutation +merge_request: 44298 +author: +type: added diff --git a/changelogs/unreleased/mw-replace-fa-icon-in-repo-preview.yml b/changelogs/unreleased/mw-replace-fa-icon-in-repo-preview.yml new file mode 100644 index 00000000000..e9fd0c66f89 --- /dev/null +++ b/changelogs/unreleased/mw-replace-fa-icon-in-repo-preview.yml @@ -0,0 +1,5 @@ +--- +title: Replace fa icon with GitLab SVG in repository preview +merge_request: 44696 +author: +type: changed diff --git a/danger/product_analytics/Dangerfile b/danger/product_analytics/Dangerfile index 01f2a4fee81..fb441d6e467 100644 --- a/danger/product_analytics/Dangerfile +++ b/danger/product_analytics/Dangerfile @@ -2,7 +2,7 @@ PRODUCT_ANALYTICS_CHANGED_FILES_MESSAGE = <<~MSG For the following files, a review from the [Data team and Product Analytics team](https://gitlab.com/groups/gitlab-org/growth/product_analytics/engineers/-/group_members?with_inherited_permissions=exclude) is recommended -Please check the ~"product analytics(telemetry)" [guide](https://docs.gitlab.com/ee/development/telemetry/usage_ping.html) and reach out to %<product_analytics_engineers_group>s group for a review. +Please check the ~"product analytics(telemetry)" [guide](https://docs.gitlab.com/ee/development/product_analytics/usage_ping.html) and reach out to %<product_analytics_engineers_group>s group for a review. %<changed_files>s @@ -10,7 +10,7 @@ Please check the ~"product analytics(telemetry)" [guide](https://docs.gitlab.com MSG UPDATE_METRICS_DEFINITIONS_MESSAGE = <<~MSG - When adding, changing, or updating metrics, please update the [Event dictionary Usage Ping table](https://docs.gitlab.com/ee/development/telemetry/event_dictionary.html#usage-ping). + When adding, changing, or updating metrics, please update the [Event dictionary Usage Ping table](https://docs.gitlab.com/ee/development/product_analytics/event_dictionary.html#usage-ping). MSG diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index 31e67f617ff..b461f5fe1be 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -3062,6 +3062,71 @@ type CreateAnnotationPayload { } """ +Autogenerated input type of CreateBoard +""" +input CreateBoardInput { + """ + The ID of the user to be assigned to the board. + """ + assigneeId: String + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The group full path the board is associated with. + """ + groupPath: ID + + """ + The IDs of labels to be added to the board. + """ + labelIds: [ID!] + + """ + The ID of the milestone to be assigned to the board. + """ + milestoneId: ID + + """ + The board name. + """ + name: String + + """ + The project full path the board is associated with. + """ + projectPath: ID + + """ + The weight of the board. + """ + weight: Boolean +} + +""" +Autogenerated return type of CreateBoard +""" +type CreateBoardPayload { + """ + The board after mutation. + """ + board: Board + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +""" Autogenerated input type of CreateBranch """ input CreateBranchInput { @@ -11766,6 +11831,7 @@ type Mutation { configureSast(input: ConfigureSastInput!): ConfigureSastPayload createAlertIssue(input: CreateAlertIssueInput!): CreateAlertIssuePayload createAnnotation(input: CreateAnnotationInput!): CreateAnnotationPayload + createBoard(input: CreateBoardInput!): CreateBoardPayload createBranch(input: CreateBranchInput!): CreateBranchPayload createClusterAgent(input: CreateClusterAgentInput!): CreateClusterAgentPayload createDiffNote(input: CreateDiffNoteInput!): CreateDiffNotePayload diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index e3446f22e01..7a484ee5ef4 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -8210,6 +8210,172 @@ }, { "kind": "INPUT_OBJECT", + "name": "CreateBoardInput", + "description": "Autogenerated input type of CreateBoard", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project full path the board is associated with.", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "groupPath", + "description": "The group full path the board is associated with.", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "name", + "description": "The board name.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "assigneeId", + "description": "The ID of the user to be assigned to the board.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "milestoneId", + "description": "The ID of the milestone to be assigned to the board.", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "weight", + "description": "The weight of the board.", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "labelIds", + "description": "The IDs of labels to be added to the board.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CreateBoardPayload", + "description": "Autogenerated return type of CreateBoard", + "fields": [ + { + "name": "board", + "description": "The board after mutation.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Board", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "CreateBranchInput", "description": "Autogenerated input type of CreateBranch", "fields": null, @@ -32821,6 +32987,33 @@ "deprecationReason": null }, { + "name": "createBoard", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "CreateBoardInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CreateBoardPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "createBranch", "description": null, "args": [ diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 9498aeab3ae..c4c0fd08f2c 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -472,6 +472,16 @@ Autogenerated return type of CreateAnnotation. | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `errors` | String! => Array | Errors encountered during execution of the mutation. | +### CreateBoardPayload + +Autogenerated return type of CreateBoard. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `board` | Board | The board after mutation. | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + ### CreateBranchPayload Autogenerated return type of CreateBranch. diff --git a/doc/ci/cloud_deployment/index.md b/doc/ci/cloud_deployment/index.md index 6fa0e6d9475..919d1439acb 100644 --- a/doc/ci/cloud_deployment/index.md +++ b/doc/ci/cloud_deployment/index.md @@ -200,3 +200,81 @@ deploy: script: - aws ecs register-task-definition ... ``` + +### Provision and deploy to your AWS Elastic Compute Cloud (EC2) + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201742) in GitLab 13.5. + +You can use the `AWS/CF-Provision-and-Deploy-EC2` CI template to perform the +following actions within the same pipeline: + +1. **Create stack**: Provision your own infrastructure by leveraging the [AWS CloudFormation](https://aws.amazon.com/cloudformation/) API. +1. **Push to S3**: Push your previously-built artifact to an [AWS S3](https://aws.amazon.com/s3/) bucket. +1. **Deploy to EC2**: Deploy this pushed content onto an [AWS EC2](https://aws.amazon.com/ec2/) instance. + + + +#### Run the `AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml` template + +To run the `AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml` template, you must +pass three JSON input objects, based on existing templates: + +1. The AWS documentation provides templates for the _Create stack_ and _Deploy to EC2_ steps (links + below). We provide the template for the remaining step, _Push to S3_: + + - [Template for the _Create stack_ step on AWS](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-anatomy.html). + - Template for the _Push to S3_ step. Note that `source` is where a preceding `build` job built + your application, exporting the build through [`artifacts:paths`](../yaml/README.md#artifactspaths): + + ```json + { + "applicationName": "string", + "source": "string", + "s3Location": "s3://your/bucket/project_built_file...]" + } + ``` + + - [Template for the _Deploy to EC2_ step on AWS](https://docs.aws.amazon.com/codedeploy/latest/APIReference/API_CreateDeployment.html). + +1. Once you have completed these three templates based on your requirements, you + have two ways to pass in these JSON objects: + + - They can be three actual files located in your project. You must specify their path relative to + your project root in your `.gitlab-ci.yml` file, using the following variables. For example, if + your files are in a `<project_root>/aws` folder: + + ```yaml + variables: + CI_AWS_CF_CREATE_STACK_FILE: 'aws/cf_create_stack.json' + CI_AWS_S3_PUSH_FILE: 'aws/s3_push.json' + CI_AWS_EC2_DEPLOYMENT_FILE: 'aws/create_deployment.json' + ``` + + - Alternatively, you can provide these JSON objects as [file-typed environment variables](../variables/README.md#custom-environment-variables-of-type-file). + In your project, go to **Settings > CI / CD > Variables** and add + the three variables listed above as file-typed environment variables. + For each variable, set the value to its corresponding JSON object. + +1. Provide the name of the stack you're creating and/or targeting, as an environment variable: + + ```yaml + variables: + CI_AWS_CF_STACK_NAME: 'YourStackName' + ``` + +1. Add this CI template to your `.gitlab-ci.yml`: + + ```yaml + include: + - template: AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml + ``` + +When running your project pipeline at this point: + +- Your AWS CloudFormation stack is created based on the content of your + `CI_AWS_CF_CREATE_STACK_FILE` file/variable. + If your stack already exists, this step is skipped, but the `provision` job it belongs to still + runs. +- Your built application is pushed to your S3 bucket then and deployed to your EC2 instance, based + on the related JSON object's content. The deployment job finishes whenever the deployment to EC2 + is done or has failed. diff --git a/doc/ci/img/cf_ec2_diagram_v13_5.png b/doc/ci/img/cf_ec2_diagram_v13_5.png Binary files differnew file mode 100644 index 00000000000..c4e079bc8d9 --- /dev/null +++ b/doc/ci/img/cf_ec2_diagram_v13_5.png diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index a1a4c91e74c..18696ce0993 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -3969,7 +3969,11 @@ The title of each milestone the release is associated with. #### `release:released_at` The date and time when the release is ready. Defaults to the current date and time if not -defined. Expected in ISO 8601 format (2019-03-15T08:00:00Z). +defined. Should be enclosed in quotes and expressed in ISO 8601 format. + +```json +released_at: '2021-03-15T08:00:00Z' +``` #### Complete example for `release` diff --git a/doc/development/README.md b/doc/development/README.md index ee4643ed5a5..74f484a5c7d 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -164,11 +164,11 @@ See [database guidelines](database/index.md). - [Externalization](i18n/externalization.md) - [Translation](i18n/translation.md) -## Telemetry guides +## Product Analytics guides -- [Telemetry guide](telemetry/index.md) -- [Usage Ping guide](telemetry/usage_ping.md) -- [Snowplow guide](telemetry/snowplow.md) +- [Product Analytics guide](product_analytics/index.md) +- [Usage Ping guide](product_analytics/usage_ping.md) +- [Snowplow guide](product_analytics/snowplow.md) ## Experiment guide diff --git a/doc/development/api_graphql_styleguide.md b/doc/development/api_graphql_styleguide.md index 9d8cdf6fa91..184909e0099 100644 --- a/doc/development/api_graphql_styleguide.md +++ b/doc/development/api_graphql_styleguide.md @@ -486,6 +486,28 @@ end Enum values can be deprecated using the [`deprecated` keyword](#deprecating-fields-and-enum-values). +### Defining GraphQL enums dynamically from Rails enums + +If your GraphQL enum is backed by a [Rails enum](creating_enums.md), then consider +using the Rails enum to dynamically define the GraphQL enum values. Doing so +binds the GraphQL enum values to the Rails enum definition, so if values are +ever added to the Rails enum then the GraphQL enum automatically reflects the change. + +Example: + +```ruby +module Types + class IssuableSeverityEnum < BaseEnum + graphql_name 'IssuableSeverity' + description 'Incident severity' + + ::IssuableSeverity.severities.keys.each do |severity| + value severity.upcase, value: severity, description: "#{severity.titleize} severity" + end + end +end +``` + ## JSON When data to be returned by GraphQL is stored as diff --git a/doc/development/changelog.md b/doc/development/changelog.md index fddce629cd7..6586f082219 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -40,7 +40,7 @@ the `author` field. GitLab team members **should not**. and with `type` set to `security`. - Any user-facing change **should** have a changelog entry. This includes both visual changes (regardless of how minor), and changes to the rendered DOM which impact how a screen reader may announce the content. - Performance improvements **should** have a changelog entry. -- Changes that need to be documented in the Telemetry [Event Dictionary](telemetry/event_dictionary.md) +- Changes that need to be documented in the Product Analytics [Event Dictionary](product_analytics/event_dictionary.md) also require a changelog entry. - _Any_ contribution from a community member, no matter how small, **may** have a changelog entry regardless of these guidelines if the contributor wants one. diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 436acb1e4d7..3ff802d3b23 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -104,7 +104,7 @@ with [domain expertise](#domain-experts). by a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors)**. 1. If your merge request only includes end-to-end changes (*3*) **or** if the MR author is a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors), it must be **approved by a [Quality maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_qa)** 1. If your merge request includes a new or updated [application limit](https://about.gitlab.com/handbook/product/product-processes/#introducing-application-limits), it must be **approved by a [product manager](https://about.gitlab.com/company/team/)**. -1. If your merge request includes Product Analytics (telemetry) changes, it should be reviewed and approved by a [Product analytics engineer](https://gitlab.com/gitlab-org/growth/telemetry/engineers). +1. If your merge request includes Product Analytics (telemetry) changes, it should be reviewed and approved by a [Product analytics engineer](https://gitlab.com/gitlab-org/growth/product-analytics/engineers). - (*1*): Please note that specs other than JavaScript specs are considered backend code. - (*2*): We encourage you to seek guidance from a database maintainer if your merge diff --git a/doc/development/database_review.md b/doc/development/database_review.md index 532afb52632..3f5f36b0b6e 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -27,7 +27,7 @@ A database review is required for: database review. - Changes in usage data metrics that use `count` and `distinct_count`. These metrics could have complex queries over large tables. - See the [Telemetry Guide](telemetry/usage_ping.md#implementing-usage-ping) + See the [Product Analytics Guide](product_analytics/usage_ping.md#implementing-usage-ping) for implementation details. A database reviewer is expected to look out for obviously complex diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index 3b01752c373..ed84eab48db 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -584,6 +584,7 @@ tenses, words, and phrases: - Avoid the use of [racially-insensitive terminology or phrases](https://www.marketplace.org/2020/06/17/tech-companies-update-language-to-avoid-offensive-terms/). For example: - Use *primary* and *secondary* for database and server relationships. - Use *allowlist* and *denylist* to describe access control lists. +- Avoid the word *please*. For details, see [the Microsoft style guide](https://docs.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/p/please). ### Word usage clarifications diff --git a/doc/development/event_tracking/backend.md b/doc/development/event_tracking/backend.md index 93e135772ef..79ea80a52ea 100644 --- a/doc/development/event_tracking/backend.md +++ b/doc/development/event_tracking/backend.md @@ -1,5 +1,5 @@ --- -redirect_to: '../telemetry/index.md' +redirect_to: '../product_analytics/index.md' --- -This document was moved to [another location](../telemetry/index.md). +This document was moved to [another location](../product_analytics/index.md). diff --git a/doc/development/event_tracking/frontend.md b/doc/development/event_tracking/frontend.md index 93e135772ef..79ea80a52ea 100644 --- a/doc/development/event_tracking/frontend.md +++ b/doc/development/event_tracking/frontend.md @@ -1,5 +1,5 @@ --- -redirect_to: '../telemetry/index.md' +redirect_to: '../product_analytics/index.md' --- -This document was moved to [another location](../telemetry/index.md). +This document was moved to [another location](../product_analytics/index.md). diff --git a/doc/development/event_tracking/index.md b/doc/development/event_tracking/index.md index 93e135772ef..79ea80a52ea 100644 --- a/doc/development/event_tracking/index.md +++ b/doc/development/event_tracking/index.md @@ -1,5 +1,5 @@ --- -redirect_to: '../telemetry/index.md' +redirect_to: '../product_analytics/index.md' --- -This document was moved to [another location](../telemetry/index.md). +This document was moved to [another location](../product_analytics/index.md). diff --git a/doc/development/experiment_guide/index.md b/doc/development/experiment_guide/index.md index 00a39ac0fc6..18b606450c2 100644 --- a/doc/development/experiment_guide/index.md +++ b/doc/development/experiment_guide/index.md @@ -105,7 +105,7 @@ addressed. end ``` -1. Track necessary events. See the [telemetry guide](../telemetry/index.md) for details. +1. Track necessary events. See the [product analytics guide](../product_analytics/index.md) for details. 1. After the merge request is merged, use [`chatops`](../../ci/chatops/README.md) in the [appropriate channel](../feature_flags/controls.md#communicate-the-change) to start the experiment for 10% of the users. The feature flag should have the name of the experiment with the `_experiment_percentage` suffix appended. diff --git a/doc/development/fe_guide/event_tracking.md b/doc/development/fe_guide/event_tracking.md index 93e135772ef..79ea80a52ea 100644 --- a/doc/development/fe_guide/event_tracking.md +++ b/doc/development/fe_guide/event_tracking.md @@ -1,5 +1,5 @@ --- -redirect_to: '../telemetry/index.md' +redirect_to: '../product_analytics/index.md' --- -This document was moved to [another location](../telemetry/index.md). +This document was moved to [another location](../product_analytics/index.md). diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index cc711589916..59399e54c3e 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -205,7 +205,7 @@ For the static text strings we suggest two patterns for using these translations When possible, you should opt for this pattern, as this allows you to import these strings directly into your component specs for re-use during testing. -- Internal component `$options` object `: +- Internal component `$options` object: ```javascript <script> diff --git a/doc/development/instrumentation.md b/doc/development/instrumentation.md index e420ae0c54f..027a19f91b5 100644 --- a/doc/development/instrumentation.md +++ b/doc/development/instrumentation.md @@ -11,7 +11,7 @@ blocks of Ruby code. Method instrumentation is the primary form of instrumentation with block-based instrumentation only being used when we want to drill down to specific regions of code within a method. -Please refer to [Telemetry](telemetry/index.md) if you are tracking product usage patterns. +Please refer to [Product Analytics](product_analytics/index.md) if you are tracking product usage patterns. ## Instrumenting Methods diff --git a/doc/development/product_analytics/event_dictionary.md b/doc/development/product_analytics/event_dictionary.md new file mode 100644 index 00000000000..b049db21c30 --- /dev/null +++ b/doc/development/product_analytics/event_dictionary.md @@ -0,0 +1,32 @@ +--- +stage: Growth +group: Product Analytics +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Event Dictionary + +**Note: We've temporarily moved the Event Dictionary to a [Google Sheet](https://docs.google.com/spreadsheets/d/1VzE8R72Px_Y_LlE3Z05LxUlG_dumWe3vl-HeUo70TPw/edit?usp=sharing)**. The previous Markdown table exceeded 600 rows making it difficult to manage. In the future, our intention is to move this back into our docs using a [YAML file](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/823). + +The event dictionary is a single source of truth for the metrics and events we collect for product usage data. The Event Dictionary lists all the metrics and events we track, why we're tracking them, and where they are tracked. + +This is a living document that is updated any time a new event is planned or implemented. It includes the following information. + +- Section, stage, or group +- Description +- Implementation status +- Availability by plan type +- Code path + +We're currently focusing our Event Dictionary on [Usage Ping](usage_ping.md). In the future, we will also include [Snowplow](snowplow.md). We currently have an initiative across the entire product organization to complete the [Event Dictionary for Usage Ping](https://gitlab.com/groups/gitlab-org/-/epics/4174). + +## Instructions + +1. Open the Event Dictionary and fill in all the **PM to edit** columns highlighted in yellow. +1. Check that all the metrics and events are assigned to the correct section, stage, or group. If a metric is used across many groups, assign it to the stage. If a metric is used across many stages, assign it to the section. If a metric is incorrectly assigned to another section, stage, or group, let the PM know you have reassigned it. If your group has no assigned metrics and events, check that your metrics and events are not incorrectly assigned to another PM. +1. Add descriptions of what your metrics and events are tracking. Work with your Engineering team or the Product Analytics team if you need help understanding this. +1. Add what plans this metric is available on. Work with your Engineering team or the Product Analytics team if you need help understanding this. + +## Planned metrics and events + +For future metrics and events you plan to track, please add them to the Event Dictionary and note the status as `Planned`, `In Progress`, or `Implemented`. Once you have confirmed the metric has been implemented and have confirmed the metric data is in our data warehouse, change the status to **Data Available**. diff --git a/doc/development/product_analytics/index.md b/doc/development/product_analytics/index.md new file mode 100644 index 00000000000..ab76d6f0561 --- /dev/null +++ b/doc/development/product_analytics/index.md @@ -0,0 +1,182 @@ +--- +stage: Growth +group: Product Analytics +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Product Analytics Guide + +At GitLab, we collect product usage data for the purpose of helping us build a better product. Data helps GitLab understand which parts of the product need improvement and which features we should build next. Product usage data also helps our team better understand the reasons why people use GitLab. With this knowledge we are able to make better product decisions. + +We encourage users to enable tracking, and we embrace full transparency with our tracking approach so it can be easily understood and trusted. + +By enabling tracking, users can: + +- Contribute back to the wider community. +- Help GitLab improve on the product. + +## Our tracking tools + +We use three methods to gather product usage data: + +- [Snowplow](#snowplow) +- [Usage Ping](#usage-ping) +- [Database import](#database-import) + +### Snowplow + +Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way +users engage with our website and application. + +Snowplow consists of two components: + +- [Snowplow JS](https://github.com/snowplow/snowplow/wiki/javascript-tracker) tracks client-side + events. +- [Snowplow Ruby](https://github.com/snowplow/snowplow/wiki/ruby-tracker) tracks server-side events. + +For more details, read the [Snowplow](snowplow.md) guide. + +### Usage Ping + +Usage Ping is a method for GitLab Inc to collect usage data on a GitLab instance. Usage Ping is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. This high-level data is used to help our product, support, and sales teams. + +For more details, read the [Usage Ping](usage_ping.md) guide. + +### Database import + +Database imports are full imports of data into GitLab's data warehouse. For GitLab.com, the PostgreSQL database is loaded into Snowflake data warehouse every 6 hours. For more details, see the [data team handbook](https://about.gitlab.com/handbook/business-ops/data-team/platform/#extract-and-load). + +## What data can be tracked + +Our different tracking tools allows us to track different types of events. The event types and examples of what data can be tracked are outlined below. + +The availability of event types and their tracking tools varies by segment. For example, on Self-Managed Users, we only have reporting using Database records via Usage Ping. + +| Event Types | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | +|----------------------------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| +| Snowplow (JS Pageview events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | +| Snowplow (JS UI events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | +| Snowplow (Ruby Pageview events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | +| Snowplow (Ruby CRUD / API events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | +| Usage Ping (Redis UI counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | +| Usage Ping (Redis Pageview counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | +| Usage Ping (Redis CRUD / API counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | +| Usage Ping (Database counters) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | +| Usage Ping (Instance settings) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | +| Usage Ping (Integration settings) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | +| Database import (Database records) | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | + +[Source file](https://docs.google.com/spreadsheets/d/1e8Afo41Ar8x3JxAXJF3nL83UxVZ3hPIyXdt243VnNuE/edit?usp=sharing) + +**Legend** + +✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible + +SaaS = GitLab.com. SM = Self-Managed instance + +### Pageview events + +- Number of sessions that visited the /dashboard/groups page + +### UI events + +- Number of sessions that clicked on a button or link +- Number of sessions that closed a modal + +UI events are any interface-driven actions from the browser including click data. + +### CRUD or API events + +- Number of Git pushes +- Number of GraphQL queries +- Number of requests to a Rails action or controller + +These are backend events that include the creation, read, update, deletion of records, and other events that might be triggered from layers other than those available in the interface. + +### Database records + +These are raw database records which can be explored using business intelligence tools like Sisense. The full list of available tables can be found in [structure.sql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/structure.sql). + +### Instance settings + +These are settings of your instance such as the instance's Git version and if certain features are enabled such as `container_registry_enabled`. + +### Integration settings + +These are integrations your GitLab instance interacts with such as an [external storage provider](../../administration/static_objects_external_storage.md) or an [external container registry](../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint). These services must be able to send data back into a GitLab instance for data to be tracked. + +## Reporting level + +Our reporting levels of aggregate or individual reporting varies by segment. For example, on Self-Managed Users, we can report at an aggregate user level using Usage Ping but not on an Individual user level. + +| Aggregated Reporting | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | +|----------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| +| Snowplow | ✅ | 📅 | 📅 | ✅ | 📅 | ✅ | 📅 | 📅 | ✅ | 📅 | +| Usage Ping | ✅ | 🔄 | 📅 | 📅 | ✅ | ✅ | ✅ | ✅ | 📅 | ✅ | +| Database import | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | + +| Identifiable Reporting | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | +|------------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| +| Snowplow | ✅ | 📅 | 📅 | ✅ | 📅 | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | +| Usage Ping | ✅ | 🔄 | 📅 | ✖️ | ✖️ | ✅ | ✅ | ✖️ | ✖️ | ✖️ | +| Database import | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | + +**Legend** + +✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible + +SaaS = GitLab.com. SM = Self-Managed instance + +## Reporting time period + +Our reporting time periods varies by segment. For example, on Self-Managed Users, we can report all time counts and 28 day counts in Usage Ping. + +| Reporting Time Period | All Time | 28 Days | 7 Days | Daily | +|-----------------------|----------|---------|--------|-------| +| Snowplow | ✅ | ✅ | ✅ | ✅ | +| Usage Ping | ✅ | ✅ | 📅 | ✖️ | +| Database import | ✅ | ✅ | ✅ | ✅ | + +**Legend** + +✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible + +## Systems overview + +The systems overview is a simplified diagram showing the interactions between GitLab Inc and self-managed instances. + + + +[Source file](https://app.diagrams.net/#G13DVpN-XnhWGz9tqReIj8pp1UE4ehk_EC) + +### GitLab Inc + +For Product Analytics purposes, GitLab Inc has three major components: + +1. [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/): This contains everything managed by our data team including Sisense Dashboards for visualization, Snowflake for Data Warehousing, incoming data sources such as PostgreSQL Pipeline and S3 Bucket, and lastly our data collectors [GitLab.com's Snowplow Collector](https://gitlab.com/gitlab-com/gl-infra/readiness/-/tree/master/library/snowplow/) and GitLab's Versions Application. +1. GitLab.com: This is the production GitLab application which is made up of a Client and Server. On the Client or browser side, a Snowplow JS Tracker (Frontend) is used to track client-side events. On the Server or application side, a Snowplow Ruby Tracker (Backend) is used to track server-side events. The server also contains Usage Ping which leverages a PostgreSQL database and a Redis in-memory data store to report on usage data. Lastly, the server also contains System Logs which are generated from running the GitLab application. +1. [Monitoring infrastructure](https://about.gitlab.com/handbook/engineering/monitoring/): This is the infrastructure used to ensure GitLab.com is operating smoothly. System Logs are sent from GitLab.com to our monitoring infrastructure and collected by a FluentD collector. From FluentD, logs are either sent to long term Google Cloud Services cold storage via Stackdriver, or, they are sent to our Elastic Cluster via Cloud Pub/Sub which can be explored in real-time using Kibana. + +### Self-managed + +For Product Analytics purposes, self-managed instances have two major components: + +1. Data infrastructure: Having a data infrastructure setup is optional on self-managed instances. If you'd like to collect Snowplow tracking events for your self-managed instance, you can setup your own self-managed Snowplow collector and configure your Snowplow events to point to your own collector. +1. GitLab: A self-managed GitLab instance contains all of the same components as GitLab.com mentioned above. + +### Differences between GitLab Inc and Self-managed + +As shown by the orange lines, on GitLab.com Snowplow JS, Snowplow Ruby, Usage Ping, and PostgreSQL database imports all flow into GitLab Inc's data infrastructure. However, on self-managed, only Usage Ping flows into GitLab Inc's data infrastructure. + +As shown by the green lines, on GitLab.com system logs flow into GitLab Inc's monitoring infrastructure. On self-managed, there are no logs sent to GitLab Inc's monitoring infrastructure. + +Note (1): Snowplow JS and Snowplow Ruby are available on self-managed, however, the Snowplow Collector endpoint is set to a self-managed Snowplow Collector which GitLab Inc does not have access to. + +## Additional information + +More useful links: + +- [Product Analytics Direction](https://about.gitlab.com/direction/product-analytics/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) diff --git a/doc/development/product_analytics/snowplow.md b/doc/development/product_analytics/snowplow.md new file mode 100644 index 00000000000..8f3d6961f9e --- /dev/null +++ b/doc/development/product_analytics/snowplow.md @@ -0,0 +1,429 @@ +--- +stage: Growth +group: Product Analytics +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Snowplow Guide + +This guide provides an overview of how Snowplow works, and implementation details. + +For more information about Product Analytics, see: + +- [Product Analytics Guide](index.md) +- [Usage Ping Guide](usage_ping.md) + +More useful links: + +- [Product Analytics Direction](https://about.gitlab.com/direction/product-analytics/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) + +## What is Snowplow + +Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way users engage with our website and application. + +[Snowplow](https://github.com/snowplow/snowplow) consists of the following loosely-coupled sub-systems: + +- **Trackers** fire Snowplow events. Snowplow has 12 trackers, covering web, mobile, desktop, server, and IoT. +- **Collectors** receive Snowplow events from trackers. We have three different event collectors, synchronizing events either to Amazon S3, Apache Kafka, or Amazon Kinesis. +- **Enrich** cleans up the raw Snowplow events, enriches them and puts them into storage. We have an Hadoop-based enrichment process, and a Kinesis-based or Kafka-based process. +- **Storage** is where the Snowplow events live. We store the Snowplow events in a flat file structure on S3, and in the Redshift and PostgreSQL databases. +- **Data modeling** is where event-level data is joined with other data sets and aggregated into smaller data sets, and business logic is applied. This produces a clean set of tables which make it easier to perform analysis on the data. We have data models for Redshift and Looker. +- **Analytics** are performed on the Snowplow events or on the aggregate tables. + + + +## Snowplow schema + +We have many definitions of Snowplow's schema. We have an active issue to [standardize this schema](https://gitlab.com/gitlab-org/gitlab/-/issues/207930) including the following definitions: + +- Frontend and backend taxonomy as listed below +- [Structured event taxonomy](#structured-event-taxonomy) +- [Self describing events](https://github.com/snowplow/snowplow/wiki/Custom-events#self-describing-events) +- [Iglu schema](https://gitlab.com/gitlab-org/iglu/) +- [Snowplow authored events](https://github.com/snowplow/snowplow/wiki/Snowplow-authored-events) + +## Enabling Snowplow + +Tracking can be enabled at: + +- The instance level, which enables tracking on both the frontend and backend layers. +- User level, though user tracking can be disabled on a per-user basis. GitLab tracking respects the [Do Not Track](https://www.eff.org/issues/do-not-track) standard, so any user who has enabled the Do Not Track option in their browser is not tracked at a user level. + +We utilize Snowplow for the majority of our tracking strategy and it is enabled on GitLab.com. On a self-managed instance, Snowplow can be enabled by navigating to: + +- **Admin Area > Settings > Integrations** in the UI. +- `admin/application_settings/integrations` in your browser. + +The following configuration is required: + +| Name | Value | +|---------------|---------------------------| +| Collector | `snowplow.trx.gitlab.net` | +| Site ID | `gitlab` | +| Cookie domain | `.gitlab.com` | + +## Snowplow request flow + +The following example shows a basic request/response flow between the following components: + +- Snowplow JS / Ruby Trackers on GitLab.com +- [GitLab.com Snowplow Collector](https://gitlab.com/gitlab-com/gl-infra/readiness/-/blob/master/library/snowplow/index.md) +- GitLab's S3 Bucket +- GitLab's Snowflake Data Warehouse +- Sisense: + +```mermaid +sequenceDiagram + participant Snowplow JS (Frontend) + participant Snowplow Ruby (Backend) + participant GitLab.com Snowplow Collector + participant S3 Bucket + participant Snowflake DW + participant Sisense Dashboards + Snowplow JS (Frontend) ->> GitLab.com Snowplow Collector: FE Tracking event + Snowplow Ruby (Backend) ->> GitLab.com Snowplow Collector: BE Tracking event + loop Process using Kinesis Stream + GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Log raw events + GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Enrich events + GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Write to disk + end + GitLab.com Snowplow Collector ->> S3 Bucket: Kinesis Firehose + S3 Bucket->>Snowflake DW: Import data + Snowflake DW->>Snowflake DW: Transform data using dbt + Snowflake DW->>Sisense Dashboards: Data available for querying +``` + +## Structured event taxonomy + +When adding new click events, we should add them in a way that's internally consistent. If we don't, it'll be very painful to perform analysis across features since each feature will be capturing events differently. + +The current method provides several attributes that are sent on each click event. Please try to follow these guidelines when specifying events to capture: + +| attribute | type | required | description | +| --------- | ------- | -------- | ----------- | +| category | text | true | The page or backend area of the application. Unless infeasible, please use the Rails page attribute by default in the frontend, and namespace + classname on the backend. | +| action | text | true | The action the user is taking, or aspect that's being instrumented. The first word should always describe the action or aspect: clicks should be `click`, activations should be `activate`, creations should be `create`, etc. Use underscores to describe what was acted on; for example, activating a form field would be `activate_form_input`. An interface action like clicking on a dropdown would be `click_dropdown`, while a behavior like creating a project record from the backend would be `create_project` | +| label | text | false | The specific element, or object that's being acted on. This is either the label of the element (e.g. a tab labeled 'Create from template' may be `create_from_template`) or a unique identifier if no text is available (e.g. closing the Groups dropdown in the top navbar might be `groups_dropdown_close`), or it could be the name or title attribute of a record being created. | +| property | text | false | Any additional property of the element, or object being acted on. | +| value | decimal | false | Describes a numeric value or something directly related to the event. This could be the value of an input (e.g. `10` when clicking `internal` visibility). | + +## Implementing Snowplow JS (Frontend) tracking + +GitLab provides `Tracking`, an interface that wraps the [Snowplow JavaScript Tracker](https://github.com/snowplow/snowplow/wiki/javascript-tracker) for tracking custom events. There are a few ways to utilize tracking, but each generally requires at minimum, a `category` and an `action`. Additional data can be provided that adheres to our [Structured event taxonomy](#structured-event-taxonomy). + +| field | type | default value | description | +|:-----------|:-------|:---------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `category` | string | document.body.dataset.page | Page or subsection of a page that events are being captured within. | +| `action` | string | 'generic' | Action the user is taking. Clicks should be `click` and activations should be `activate`, so for example, focusing a form field would be `activate_form_input`, and clicking a button would be `click_button`. | +| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described in our [Structured event taxonomy](#structured-event-taxonomy). | + +### Tracking in HAML (or Vue Templates) + +When working within HAML (or Vue templates) we can add `data-track-*` attributes to elements of interest. All elements that have a `data-track-event` attribute automatically have event tracking bound on clicks. + +Below is an example of `data-track-*` attributes assigned to a button: + +```haml +%button.btn{ data: { track: { event: "click_button", label: "template_preview", property: "my-template" } } } +``` + +```html +<button class="btn" + data-track-event="click_button" + data-track-label="template_preview" + data-track-property="my-template" +/> +``` + +Event listeners are bound at the document level to handle click events on or within elements with these data attributes. This allows them to be properly handled on re-rendering and changes to the DOM. Note that because of the way these events are bound, click events should not be stopped from propagating up the DOM tree. If for any reason click events are being stopped from propagating, you need to implement your own listeners and follow the instructions in [Tracking in raw JavaScript](#tracking-in-raw-javascript). + +Below is a list of supported `data-track-*` attributes: + +| attribute | required | description | +|:----------------------|:---------|:------------| +| `data-track-event` | true | Action the user is taking. Clicks must be prepended with `click` and activations must be prepended with `activate`. For example, focusing a form field would be `activate_form_input` and clicking a button would be `click_button`. | +| `data-track-label` | false | The `label` as described in our [Structured event taxonomy](#structured-event-taxonomy). | +| `data-track-property` | false | The `property` as described in our [Structured event taxonomy](#structured-event-taxonomy). | +| `data-track-value` | false | The `value` as described in our [Structured event taxonomy](#structured-event-taxonomy). If omitted, this is the element's `value` property or an empty string. For checkboxes, the default value is the element's checked attribute or `false` when unchecked. | +| `data-track-context` | false | The `context` as described in our [Structured event taxonomy](#structured-event-taxonomy). | + +### Tracking within Vue components + +There's a tracking Vue mixin that can be used in components if more complex tracking is required. To use it, first import the `Tracking` library and request a mixin. + +```javascript +import Tracking from '~/tracking'; +const trackingMixin = Tracking.mixin({ label: 'right_sidebar' }); +``` + +You can provide default options that are passed along whenever an event is tracked from within your component. For instance, if all events within a component should be tracked with a given `label`, you can provide one at this time. Available defaults are `category`, `label`, `property`, and `value`. If no category is specified, `document.body.dataset.page` is used as the default. + +You can then use the mixin normally in your component with the `mixin` Vue declaration. The mixin also provides the ability to specify tracking options in `data` or `computed`. These override any defaults and allow the values to be dynamic from props, or based on state. + +```javascript +export default { + mixins: [trackingMixin], + // ...[component implementation]... + data() { + return { + expanded: false, + tracking: { + label: 'left_sidebar' + } + }; + }, +} +``` + +The mixin provides a `track` method that can be called within the template, or from component methods. An example of the whole implementation might look like the following. + +```javascript +export default { + mixins: [Tracking.mixin({ label: 'right_sidebar' })], + data() { + return { + expanded: false, + }; + }, + methods: { + toggle() { + this.expanded = !this.expanded; + this.track('click_toggle', { value: this.expanded }) + } + } +}; +``` + +And if needed within the template, you can use the `track` method directly as well. + +```html +<template> + <div> + <a class="toggle" @click.prevent="toggle">Toggle</a> + <div v-if="expanded"> + <p>Hello world!</p> + <a @click.prevent="track('click_action')">Track an event</a> + </div> + </div> +</template> +``` + +### Tracking in raw JavaScript + +Custom event tracking and instrumentation can be added by directly calling the `Tracking.event` static function. The following example demonstrates tracking a click on a button by calling `Tracking.event` manually. + +```javascript +import Tracking from '~/tracking'; + +const button = document.getElementById('create_from_template_button'); +button.addEventListener('click', () => { + Tracking.event('dashboard:projects:index', 'click_button', { + label: 'create_from_template', + property: 'template_preview', + value: 'rails', + }); +}) +``` + +### Tests and test helpers + +In Jest particularly in Vue tests, you can use the following: + +```javascript +import { mockTracking } from 'helpers/tracking_helper'; + +describe('MyTracking', () => { + let spy; + + beforeEach(() => { + spy = mockTracking('_category_', wrapper.element, jest.spyOn); + }); + + it('tracks an event when clicked on feedback', () => { + wrapper.find('.discover-feedback-icon').trigger('click'); + + expect(spy).toHaveBeenCalledWith('_category_', 'click_button', { + label: 'security-discover-feedback-cta', + property: '0', + }); + }); +}); +``` + +In obsolete Karma tests it's used as below: + +```javascript +import { mockTracking, triggerEvent } from 'spec/helpers/tracking_helper'; + +describe('my component', () => { + let trackingSpy; + + beforeEach(() => { + trackingSpy = mockTracking('_category_', vm.$el, spyOn); + }); + + const triggerEvent = () => { + // action which should trigger a event + }; + + it('tracks an event when toggled', () => { + expect(trackingSpy).not.toHaveBeenCalled(); + + triggerEvent('a.toggle'); + + expect(trackingSpy).toHaveBeenCalledWith('_category_', 'click_edit_button', { + label: 'right_sidebar', + property: 'confidentiality', + }); + }); +}); +``` + +## Implementing Snowplow Ruby (Backend) tracking + +GitLab provides `Gitlab::Tracking`, an interface that wraps the [Snowplow Ruby Tracker](https://github.com/snowplow/snowplow/wiki/ruby-tracker) for tracking custom events. + +Custom event tracking and instrumentation can be added by directly calling the `GitLab::Tracking.event` class method, which accepts the following arguments: + +| argument | type | default value | description | +|:-----------|:-------|:--------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `category` | string | 'application' | Area or aspect of the application. This could be `HealthCheckController` or `Lfs::FileTransformer` for instance. | +| `action` | string | 'generic' | The action being taken, which can be anything from a controller action like `create` to something like an Active Record callback. | +| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described in [Structured event taxonomy](#structured-event-taxonomy). These are set as empty strings if you don't provide them. | + +Tracking can be viewed as either tracking user behavior, or can be utilized for instrumentation to monitor and visualize performance over time in an area or aspect of code. + +For example: + +```ruby +class Projects::CreateService < BaseService + def execute + project = Project.create(params) + + Gitlab::Tracking.event('Projects::CreateService', 'create_project', + label: project.errors.full_messages.to_sentence, + value: project.valid? + ) + end +end +``` + +### Unit testing + +Use the `expect_snowplow_event` helper when testing backend Snowplow events. See [testing best practices]( +https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-snowplow-events) for details. + +### Performance + +We use the [AsyncEmitter](https://github.com/snowplow/snowplow/wiki/Ruby-Tracker#52-the-asyncemitter-class) when tracking events, which allows for instrumentation calls to be run in a background thread. This is still an active area of development. + +## Developing and testing Snowplow + +There are several tools for developing and testing Snowplow Event + +| Testing Tool | Frontend Tracking | Backend Tracking | Local Development Environment | Production Environment | Production Environment | +|----------------------------------------------|--------------------|---------------------|-------------------------------|------------------------|------------------------| +| Snowplow Analytics Debugger Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| Snowplow Inspector Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| Snowplow Micro | **{check-circle}** | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | +| Snowplow Mini | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{status_preparing}** | **{status_preparing}** | + +**Legend** + +**{check-circle}** Available, **{status_preparing}** In progress, **{dotted-circle}** Not Planned + +### Preparing your MR for Review + +1. For frontend events, in the MR description section, add a screenshot of the event's relevant section using the [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) Chrome browser extension. +1. For backend events, please use Snowplow Micro and add the output of the Snowplow Micro good events `GET http://localhost:9090/micro/good`. + +### Snowplow Analytics Debugger Chrome Extension + +Snowplow Analytics Debugger is a browser extension for testing frontend events. This works on production, staging and local development environments. + +1. Install the [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) Chrome browser extension. +1. Open Chrome DevTools to the Snowplow Analytics Debugger tab. +1. Learn more at [Igloo Analytics](https://www.iglooanalytics.com/blog/snowplow-analytics-debugger-chrome-extension.html). + +### Snowplow Inspector Chrome Extension + +Snowplow Inspector Chrome Extension is a browser extension for testing frontend events. This works on production, staging and local development environments. + +1. Install [Snowplow Inspector](https://chrome.google.com/webstore/detail/snowplow-inspector/maplkdomeamdlngconidoefjpogkmljm?hl=en). +1. Open the Chrome extension by pressing the Snowplow Inspector icon beside the address bar. +1. Click around on a webpage with Snowplow and you should see JavaScript events firing in the inspector window. + +### Snowplow Micro + +Snowplow Micro is a very small version of a full Snowplow data collection pipeline: small enough that it can be launched by a test suite. Events can be recorded into Snowplow Micro just as they can a full Snowplow pipeline. Micro then exposes an API that can be queried. + +Snowplow Micro is a Docker-based solution for testing frontend and backend events in a local development environment. You need to modify GDK using the instructions below to set this up. + +- Read [Introducing Snowplow Micro](https://snowplowanalytics.com/blog/2019/07/17/introducing-snowplow-micro/) +- Look at the [Snowplow Micro repository](https://github.com/snowplow-incubator/snowplow-micro) +- Watch our [installation guide recording](https://www.youtube.com/watch?v=OX46fo_A0Ag) + +1. Install [Snowplow Micro](https://github.com/snowplow-incubator/snowplow-micro): + + ```shell + docker run --mount type=bind,source=$(pwd)/example,destination=/config -p 9090:9090 snowplow/snowplow-micro:latest --collector-config /config/micro.conf --iglu /config/iglu.json + ``` + +1. Install snowplow micro by cloning the settings in [this project](https://gitlab.com/a_akgun/snowplow-micro): + + ```shell + git clone git@gitlab.com:a_akgun/snowplow-micro.git + ./snowplow-micro.sh + ``` + +1. Update port in SQL to set `9090`: + + ```shell + gdk psql -d gitlabhq_development + update application_settings set snowplow_collector_hostname='localhost:9090', snowplow_enabled=true, snowplow_cookie_domain='.gitlab.com'; + ``` + +1. Update `app/assets/javascripts/tracking.js` to [remove this line](https://gitlab.com/snippets/1918635): + + ```javascript + forceSecureTracker: true + ``` + +1. Update `lib/gitlab/tracking.rb` to [add these lines](https://gitlab.com/snippets/1918635): + + ```ruby + protocol: 'http', + port: 9090, + ``` + +1. Update `lib/gitlab/tracking.rb` to [change async emitter from https to http](https://gitlab.com/snippets/1918635): + + ```ruby + SnowplowTracker::AsyncEmitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname, protocol: 'http'), + ``` + +1. Enable Snowplow in the admin area, Settings::Integrations::Snowplow to point to: + `http://localhost:3000/admin/application_settings/integrations#js-snowplow-settings`. + +1. Restart GDK: + + ```shell + `gdk restart` + ``` + +1. Send a test Snowplow event from the Rails console: + + ```ruby + Gitlab::Tracking.self_describing_event('iglu:com.gitlab/pageview_context/jsonschema/1-0-0', { page_type: 'MY_TYPE' }, context: nil ) + ``` + +### Snowplow Mini + +[Snowplow Mini](https://github.com/snowplow/snowplow-mini) is an easily-deployable, single-instance version of Snowplow. + +Snowplow Mini can be used for testing frontend and backend events on a production, staging and local development environment. + +For GitLab.com, we're setting up a [QA and Testing environment](https://gitlab.com/gitlab-org/telemetry/-/issues/266) using Snowplow Mini. diff --git a/doc/development/product_analytics/usage_ping.md b/doc/development/product_analytics/usage_ping.md new file mode 100644 index 00000000000..63378762038 --- /dev/null +++ b/doc/development/product_analytics/usage_ping.md @@ -0,0 +1,932 @@ +--- +stage: Growth +group: Product Analytics +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Usage Ping Guide + +> - Introduced in GitLab Enterprise Edition 8.10. +> - More statistics were added in GitLab Enterprise Edition 8.12. +> - Moved to GitLab Core in 9.1. +> - More statistics were added in GitLab Ultimate 11.2. + +This guide describes Usage Ping's purpose and how it's implemented. + +For more information about Product Analytics, see: + +- [Product Analytics Guide](index.md) +- [Snowplow Guide](snowplow.md) + +More useful links: + +- [Product Analytics Direction](https://about.gitlab.com/direction/product-analytics/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) + +## What is Usage Ping? + +- GitLab sends a weekly payload containing usage data to GitLab Inc. Usage Ping provides high-level data to help our product, support, and sales teams. It does not send any project names, usernames, or any other specific data. The information from the usage ping is not anonymous, it is linked to the hostname of the instance. Sending usage ping is optional, and any instance can disable analytics. +- The usage data is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. In addition to counts, other facts + that help us classify and understand GitLab installations are collected. +- Usage ping is important to GitLab as we use it to calculate our Stage Monthly Active Users (SMAU) which helps us measure the success of our stages and features. +- While usage ping is enabled, GitLab will gather data from the other instances and will be able to show usage statistics of your instance to your users. + +### Why should we enable Usage Ping? + +- The main purpose of Usage Ping is to build a better GitLab. Data about how GitLab is used is collected to better understand feature/stage adoption and usage, which helps us understand how GitLab is adding value and helps our team better understand the reasons why people use GitLab and with this knowledge we're able to make better product decisions. +- As a benefit of having the usage ping active, GitLab lets you analyze the users’ activities over time of your GitLab installation. +- As a benefit of having the usage ping active, GitLab provides you with The DevOps Report,which gives you an overview of your entire instance’s adoption of Concurrent DevOps from planning to monitoring. +- You will get better, more proactive support. (assuming that our TAMs and support organization used the data to deliver more value) +- You will get insight and advice into how to get the most value out of your investment in GitLab. Wouldn't you want to know that a number of features or values are not being adopted in your organization? +- You get a report that illustrates how you compare against other similar organizations (anonymized), with specific advice and recommendations on how to improve your DevOps processes. +- Usage Ping is enabled by default. To disable it, see [Disable Usage Ping](#disable-usage-ping). + +### Limitations + +- Usage Ping does not track frontend events things like page views, link clicks, or user sessions, and only focuses on aggregated backend events. +- Because of these limitations we recommend instrumenting your products with Snowplow for more detailed analytics on GitLab.com and use Usage Ping to track aggregated backend events on self-managed. + +## Usage Ping payload + +You can view the exact JSON payload sent to GitLab Inc. in the administration panel. To view the payload: + +1. Navigate to **Admin Area > Settings > Metrics and profiling**. +1. Expand the **Usage statistics** section. +1. Click the **Preview payload** button. + +For an example payload, see [Example Usage Ping payload](#example-usage-ping-payload). + +## Disable Usage Ping + +To disable Usage Ping in the GitLab UI, go to the **Settings** page of your administration panel and uncheck the **Usage Ping** checkbox. + +To disable Usage Ping and prevent it from being configured in the future through the administration panel, Omnibus installs can set the following in [`gitlab.rb`](https://docs.gitlab.com/omnibus/settings/configuration.html#configuration-options): + +```ruby +gitlab_rails['usage_ping_enabled'] = false +``` + +Source installations can set the following in `gitlab.yml`: + +```yaml +production: &base + # ... + gitlab: + # ... + usage_ping_enabled: false +``` + +## Usage Ping request flow + +The following example shows a basic request/response flow between a GitLab instance, the Versions Application, the License Application, Salesforce, GitLab's S3 Bucket, GitLab's Snowflake Data Warehouse, and Sisense: + +```mermaid +sequenceDiagram + participant GitLab Instance + participant Versions Application + participant Licenses Application + participant Salesforce + participant S3 Bucket + participant Snowflake DW + participant Sisense Dashboards + GitLab Instance->>Versions Application: Send Usage Ping + loop Process usage data + Versions Application->>Versions Application: Parse usage data + Versions Application->>Versions Application: Write to database + Versions Application->>Versions Application: Update license ping time + end + loop Process data for Salesforce + Versions Application-xLicenses Application: Request Zuora subscription id + Licenses Application-xVersions Application: Zuora subscription id + Versions Application-xSalesforce: Request Zuora account id by Zuora subscription id + Salesforce-xVersions Application: Zuora account id + Versions Application-xSalesforce: Usage data for the Zuora account + end + Versions Application->>S3 Bucket: Export Versions database + S3 Bucket->>Snowflake DW: Import data + Snowflake DW->>Snowflake DW: Transform data using dbt + Snowflake DW->>Sisense Dashboards: Data available for querying + Versions Application->>GitLab Instance: DevOps Report (Conversational Development Index) +``` + +## How Usage Ping works + +1. The Usage Ping [cron job](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/gitlab_usage_ping_worker.rb#L30) is set in Sidekiq to run weekly. +1. When the cron job runs, it calls [`GitLab::UsageData.to_json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L22). +1. `GitLab::UsageData.to_json` [cascades down](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L22) to ~400+ other counter method calls. +1. The response of all methods calls are [merged together](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L14) into a single JSON payload in `GitLab::UsageData.to_json`. +1. The JSON payload is then [posted to the Versions application]( https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L20). + +## Implementing Usage Ping + +Usage Ping consists of two kinds of data, counters and observations. Counters track how often a certain event +happened over time, such as how many CI pipelines have run. They are monotonic and always trend up. +Observations are facts collected from one or more GitLab instances and can carry arbitrary data. There are no +general guidelines around how to collect those, due to the individual nature of that data. + +There are several types of counters which are all found in `usage_data.rb`: + +- **Ordinary Batch Counters:** Simple count of a given ActiveRecord_Relation +- **Distinct Batch Counters:** Distinct count of a given ActiveRecord_Relation on given column +- **Sum Batch Counters:** Sum the values of a given ActiveRecord_Relation on given column +- **Alternative Counters:** Used for settings and configurations +- **Redis Counters:** Used for in-memory counts. + +NOTE: **Note:** +Only use the provided counter methods. Each counter method contains a built in fail safe to isolate each counter to avoid breaking the entire Usage Ping. + +### Why batch counting + +For large tables, PostgreSQL can take a long time to count rows due to MVCC [(Multi-version Concurrency Control)](https://en.wikipedia.org/wiki/Multiversion_concurrency_control). Batch counting is a counting method where a single large query is broken into multiple smaller queries. For example, instead of a single query querying 1,000,000 records, with batch counting, you can execute 100 queries of 10,000 records each. Batch counting is useful for avoiding database timeouts as each batch query is significantly shorter than one single long running query. + +For GitLab.com, there are extremely large tables with 15 second query timeouts, so we use batch counting to avoid encountering timeouts. Here are the sizes of some GitLab.com tables: + +| Table | Row counts in millions | +|------------------------------|------------------------| +| `merge_request_diff_commits` | 2280 | +| `ci_build_trace_sections` | 1764 | +| `merge_request_diff_files` | 1082 | +| `events` | 514 | + +There are two batch counting methods provided, `Ordinary Batch Counters` and `Distinct Batch Counters`. Batch counting requires indexes on columns to calculate max, min, and range queries. In some cases, a specialized index may need to be added on the columns involved in a counter. + +### Ordinary Batch Counters + +Handles `ActiveRecord::StatementInvalid` error + +Simple count of a given ActiveRecord_Relation, does a non-distinct batch count, smartly reduces batch_size and handles errors. + +Method: `count(relation, column = nil, batch: true, start: nil, finish: nil)` + +Arguments: + +- `relation` the ActiveRecord_Relation to perform the count +- `column` the column to perform the count on, by default is the primary key +- `batch`: default `true` in order to use batch counting +- `start`: custom start of the batch counting in order to avoid complex min calculations +- `end`: custom end of the batch counting in order to avoid complex min calculations + +Examples: + +```ruby +count(User.active) +count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) +count(::Clusters::Cluster.aws_installed.enabled, :cluster_id, start: ::Clusters::Cluster.minimum(:id), finish: ::Clusters::Cluster.maximum(:id)) +``` + +### Distinct Batch Counters + +Handles `ActiveRecord::StatementInvalid` error + +Distinct count of a given ActiveRecord_Relation on given column, a distinct batch count, smartly reduces batch_size and handles errors. + +Method: `distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)` + +Arguments: + +- `relation` the ActiveRecord_Relation to perform the count +- `column` the column to perform the distinct count, by default is the primary key +- `batch`: default `true` in order to use batch counting +- `batch_size`: if none set it will use default value 10000 from `Gitlab::Database::BatchCounter` +- `start`: custom start of the batch counting in order to avoid complex min calculations +- `end`: custom end of the batch counting in order to avoid complex min calculations + +Examples: + +```ruby +distinct_count(::Project, :creator_id) +distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) +distinct_count(::Clusters::Applications::CertManager.where(time_period).available.joins(:cluster), 'clusters.user_id') +``` + +### Sum Batch Counters + +Handles `ActiveRecord::StatementInvalid` error + +Sum the values of a given ActiveRecord_Relation on given column and handles errors. + +Method: `sum(relation, column, batch_size: nil, start: nil, finish: nil)` + +Arguments: + +- `relation` the ActiveRecord_Relation to perform the operation +- `column` the column to sum on +- `batch_size`: if none set it will use default value 1000 from `Gitlab::Database::BatchCounter` +- `start`: custom start of the batch counting in order to avoid complex min calculations +- `end`: custom end of the batch counting in order to avoid complex min calculations + +Examples: + +```ruby +sum(JiraImportState.finished, :imported_issues_count) +``` + +### Grouping & Batch Operations + +The `count`, `distinct_count`, and `sum` batch counters can accept an `ActiveRecord::Relation` +object, which groups by a specified column. With a grouped relation, the methods do batch counting, +handle errors, and returns a hash table of key-value pairs. + +Examples: + +```ruby +count(Namespace.group(:type)) +# returns => {nil=>179, "Group"=>54} + +distinct_count(Project.group(:visibility_level), :creator_id) +# returns => {0=>1, 10=>1, 20=>11} + +sum(Issue.group(:state_id), :weight)) +# returns => {1=>3542, 2=>6820} +``` + +### Redis Counters + +Handles `::Redis::CommandError` and `Gitlab::UsageDataCounters::BaseCounter::UnknownEvent` +returns -1 when a block is sent or hash with all values -1 when a `counter(Gitlab::UsageDataCounters)` is sent +different behavior due to 2 different implementations of Redis counter + +Method: `redis_usage_data(counter, &block)` + +Arguments: + +- `counter`: a counter from `Gitlab::UsageDataCounters`, that has `fallback_totals` method implemented +- or a `block`: which is evaluated + +#### Ordinary Redis Counters + +Examples of implementation: + +- Using Redis methods [`INCR`](https://redis.io/commands/incr), [`GET`](https://redis.io/commands/get), and [`Gitlab::UsageDataCounters::WikiPageCounter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/wiki_page_counter.rb) +- Using Redis methods [`HINCRBY`](https://redis.io/commands/hincrby), [`HGETALL`](https://redis.io/commands/hgetall), and [`Gitlab::UsageCounters::PodLogs`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_counters/pod_logs.rb) + +#### Redis HLL Counters + +With `Gitlab::UsageDataCounters::HLLRedisCounter` we have available data structures used to count unique values. + +Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd) and [PFCOUNT](https://redis.io/commands/pfcount). + +##### Adding new events + +1. Define events in [`known_events.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml). + + Example event: + + ```yaml + - name: i_compliance_credential_inventory + category: compliance + redis_slot: compliance + expiry: 42 # 6 weeks + aggregation: weekly + ``` + + Keys: + + - `name`: unique event name. + + Name format `<prefix>_<redis_slot>_name`. + + Use one of the following prefixes for the event's name: + + - `g_` for group, as an event which is tracked for group. + - `p_` for project, as an event which is tracked for project. + - `i_` for instance, as an event which is tracked for instance. + - `a_` for events encompassing all `g_`, `p_`, `i_`. + - `o_` for other. + + Consider including in the event's name the Redis slot in order to be able to count totals for a specific category. + + Example names: `i_compliance_credential_inventory`, `g_analytics_contribution`. + + - `category`: event category. Used for getting total counts for events in a category, for easier + access to a group of events. + - `redis_slot`: optional Redis slot; default value: event name. Used if needed to calculate totals + for a group of metrics. Ensure keys are in the same slot. For example: + `i_compliance_credential_inventory` with `redis_slot: 'compliance'` will build Redis key + `i_{compliance}_credential_inventory-2020-34`. If `redis_slot` is not defined the Redis key will + be `{i_compliance_credential_inventory}-2020-34`. + - `expiry`: expiry time in days. Default: 29 days for daily aggregation and 6 weeks for weekly + aggregation. + - `aggregation`: aggregation `:daily` or `:weekly`. The argument defines how we build the Redis + keys for data storage. For `daily` we keep a key for metric per day of the year, for `weekly` we + keep a key for metric per week of the year. + +1. Track event in controller using `RedisTracking` module with `track_redis_hll_event(*controller_actions, name:, feature:, feature_default_enabled: false)`. + + Arguments: + + - `controller_actions`: controller actions we want to track. + - `name`: event name. + - `feature`: feature name, all metrics we track should be under feature flag. + - `feature_default_enabled`: feature flag is disabled by default, set to `true` for it to be enabled by default. + + Example usage: + + ```ruby + # controller + class ProjectsController < Projects::ApplicationController + include RedisTracking + + skip_before_action :authenticate_user!, only: :show + track_redis_hll_event :index, :show, name: 'i_analytics_dev_ops_score', feature: :g_compliance_dashboard_feature, feature_default_enabled: true + + def index + render html: 'index' + end + + def new + render html: 'new' + end + + def show + render html: 'show' + end + end + ``` + +1. Track event in API using `increment_unique_values(event_name, values)` helper method. + + In order to be able to track the event, Usage Ping must be enabled and the event feature `usage_data_<event_name>` must be enabled. + + Arguments: + + - `event_name`: event name. + - `values`: values counted, one value or array of values. + + Example usage: + + ```ruby + get ':id/registry/repositories' do + repositories = ContainerRepositoriesFinder.new( + user: current_user, subject: user_group + ).execute + + increment_unique_values('i_list_repositories', current_user.id) + + present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] + end + ``` + +1. Track event using `track_usage_event(event_name, values) in services and graphql + + Increment unique values count using Redis HLL, for given event name. + + Example: + + [Track usage event for incident created in service](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/issues/update_service.rb) + + [Track usage event for incident created in graphql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/mutations/alert_management/update_alert_status.rb) + + ```ruby + track_usage_event(:incident_management_incident_created, current_user.id) + ``` + +1. Track event using `UsageData` API + + Increment unique users count using Redis HLL, for given event name. + + Tracking events using the `UsageData` API requires the `usage_data_api` feature flag to be enabled, which is disabled by default. + + API requests are protected by checking for a valid CSRF token. + + In order to be able to increment the values the related feature `usage_data<event_name>` should be enabled. + + ```plaintext + POST /usage_data/increment_unique_users + ``` + + | Attribute | Type | Required | Description | + | :-------- | :--- | :------- | :---------- | + | `event` | string | yes | The event name it should be tracked | + + Response +w + Return 200 if tracking failed for any reason. + + - `200` if event was tracked or any errors + - `400 Bad request` if event parameter is missing + - `401 Unauthorized` if user is not authenticated + - `403 Forbidden` for invalid CSRF token provided + +1. Track events using JavaScript/Vue API helper which calls the API above + + Example usage for an existing event already defined in [known events](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml): + + Note that `usage_data_api` and `usage_data_#{event_name}` should be enabled in order to be able to track events + + ```javascript + import api from '~/api'; + + api.trackRedisHllUserEvent('my_already_defined_event_name'), + ``` + +1. Track event using base module `Gitlab::UsageDataCounters::HLLRedisCounter.track_event(entity_id, event_name)`. + + Arguments: + + - `entity_id`: value we count. For example: user_id, visitor_id. + - `event_name`: event name. + +1. Get event data using `Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names:, start_date:, end_date)`. + + Arguments: + + - `event_names`: the list of event names. + - `start_date`: start date of the period for which we want to get event data. + - `end_date`: end date of the period for which we want to get event data. + +Recommendations: + +- Key should expire in 29 days for daily and 42 days for weekly. +- If possible, data granularity should be a week. For example a key could be composed from the + metric's name and week of the year, `2020-33-{metric_name}`. +- Use a [feature flag](../../operations/feature_flags.md) to have a control over the impact when + adding new metrics. + +##### Known events in usage data payload + +All events added in [`known_events.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml) are automatically added to usage data generation under the `redis_hll_counters` key. This column is stored in [version-app as a JSON](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L209). +For each event we add metrics for the weekly and monthly time frames, and totals for each where applicable: + +- `#{event_name}_weekly`: Data for 7 days for daily [aggregation](#adding-new-events) events and data for the last complete week for weekly [aggregation](#adding-new-events) events. +- `#{event_name}_monthly`: Data for 28 days for daily [aggregation](#adding-new-events) events and data for the last 4 complete weeks for weekly [aggregation](#adding-new-events) events. +- `#{category}_total_unique_counts_weekly`: Total unique counts for events in the same category for the last 7 days or the last complete week, if events are in the same Redis slot and we have more than one metric. +- `#{category}_total_unique_counts_monthly`: Total unique counts for events in same category for the last 28 days or the last 4 complete weeks, if events are in the same Redis slot and we have more than one metric. + +Example of `redis_hll_counters` data: + +```ruby +{:redis_hll_counters=> + {"compliance"=> + {"g_compliance_dashboard_weekly"=>0, + "g_compliance_dashboard_monthly"=>0, + "g_compliance_audit_events_weekly"=>0, + "g_compliance_audit_events_monthly"=>0, + "compliance_total_unique_counts_weekly"=>0, + "compliance_total_unique_counts_monthly"=>0}, + "analytics"=> + {"g_analytics_contribution_weekly"=>0, + "g_analytics_contribution_monthly"=>0, + "g_analytics_insights_weekly"=>0, + "g_analytics_insights_monthly"=>0, + "analytics_total_unique_counts_weekly"=>0, + "analytics_total_unique_counts_monthly"=>0}, + "ide_edit"=> + {"g_edit_by_web_ide_weekly"=>0, + "g_edit_by_web_ide_monthly"=>0, + "g_edit_by_sfe_weekly"=>0, + "g_edit_by_sfe_monthly"=>0, + "ide_edit_total_unique_counts_weekly"=>0, + "ide_edit_total_unique_counts_monthly"=>0}, + "search"=> + {"i_search_total_weekly"=>0, "i_search_total_monthly"=>0, "i_search_advanced_weekly"=>0, "i_search_advanced_monthly"=>0, "i_search_paid_weekly"=>0, "i_search_paid_monthly"=>0, "search_total_unique_counts_weekly"=>0, "search_total_unique_counts_monthly"=>0}, + "source_code"=>{"wiki_action_weekly"=>0, "wiki_action_monthly"=>0} + } +``` + +Example usage: + +```ruby +# Redis Counters +redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) +redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } + +# Define events in known_events.yml https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml + +# Tracking events +Gitlab::UsageDataCounters::HLLRedisCounter.track_event(visitor_id, 'expand_vulnerabilities') + +# Get unique events for metric +redis_usage_data { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'expand_vulnerabilities', start_date: 28.days.ago, end_date: Date.current) } +``` + +### Alternative Counters + +Handles `StandardError` and fallbacks into -1 this way not all measures fail if we encounter one exception. +Mainly used for settings and configurations. + +Method: `alt_usage_data(value = nil, fallback: -1, &block)` + +Arguments: + +- `value`: a simple static value in which case the value is simply returned. +- or a `block`: which is evaluated +- `fallback: -1`: the common value used for any metrics that are failing. + +Example of usage: + +```ruby +alt_usage_data { Gitlab::VERSION } +alt_usage_data { Gitlab::CurrentSettings.uuid } +alt_usage_data(999) +``` + +### Prometheus Queries + +In those cases where operational metrics should be part of Usage Ping, a database or Redis query is unlikely +to provide useful data. Instead, Prometheus might be more appropriate, since most of GitLab's architectural +components publish metrics to it that can be queried back, aggregated, and included as usage data. + +NOTE: **Note:** +Prometheus as a data source for Usage Ping is currently only available for single-node Omnibus installations +that are running the [bundled Prometheus](../../administration/monitoring/prometheus/index.md) instance. + +In order to query Prometheus for metrics, a helper method is available that will `yield` a fully configured +`PrometheusClient`, given it is available as per the note above: + +```ruby +with_prometheus_client do |client| + response = client.query('<your query>') + ... +end +``` + +Please refer to [the `PrometheusClient` definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/prometheus_client.rb) +for how to use its API to query for data. + +## Developing and testing Usage Ping + +### 1. Use your Rails console to manually test counters + +```ruby +# count +Gitlab::UsageData.count(User.active) +Gitlab::UsageData.count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) + +# count distinct +Gitlab::UsageData.distinct_count(::Project, :creator_id) +Gitlab::UsageData.distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) +``` + +### 2. Generate the SQL query + +Your Rails console will return the generated SQL queries. + +Example: + +```ruby +pry(main)> Gitlab::UsageData.count(User.active) + (2.6ms) SELECT "features"."key" FROM "features" + (15.3ms) SELECT MIN("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) + (2.4ms) SELECT MAX("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) + (1.9ms) SELECT COUNT("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) AND "users"."id" BETWEEN 1 AND 100000 +``` + +### 3. Optimize queries with #database-lab + +Paste the SQL query into `#database-lab` to see how the query performs at scale. + +- `#database-lab` is a Slack channel which uses a production-sized environment to test your queries. +- GitLab.com’s production database has a 15 second timeout. +- Any single query must stay below 1 second execution time with cold caches. +- Add a specialized index on columns involved to reduce the execution time. + +In order to have an understanding of the query's execution we add in the MR description the following information: + +- For counters that have a `time_period` test we add information for both cases: + - `time_period = {}` for all time periods + - `time_period = { created_at: 28.days.ago..Time.current }` for last 28 days period +- Execution plan and query time before and after optimization +- Query generated for the index and time +- Migration output for up and down execution + +We also use `#database-lab` and [explain.depesz.com](https://explain.depesz.com/). For more details, see the [database review guide](../database_review.md#preparation-when-adding-or-modifying-queries). + +#### Optimization recommendations and examples + +- Use specialized indexes [example 1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26871), [example 2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26445). +- Use defined `start` and `finish`, and simple queries, because these values can be memoized and reused, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37155). +- Avoid joins and write the queries as simply as possible, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36316). +- Set a custom `batch_size` for `distinct_count`, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38000). + +### 4. Add the metric definition + +When adding, changing, or updating metrics, please update the [Event Dictionary's **Usage Ping** table](event_dictionary.md). + +### 5. Add new metric to Versions Application + +Check if new metrics need to be added to the Versions Application. See `usage_data` [schema](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L147) and usage data [parameters accepted](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/app/services/usage_ping.rb). Any metrics added under the `counts` key are saved in the `counts` column. + +### 6. Add the feature label + +Add the `feature` label to the Merge Request for new Usage Ping metrics. These are user-facing changes and are part of expanding the Usage Ping feature. + +### 7. Add a changelog file + +Ensure you comply with the [Changelog entries guide](../changelog.md). + +### 8. Ask for a Product Analytics Review + +On GitLab.com, we have DangerBot setup to monitor Product Analytics related files and DangerBot will recommend a Product Analytics review. Mention `@gitlab-org/growth/product-analytics/engineers` in your MR for a review. + +### 9. Verify your metric + +On GitLab.com, the Product Analytics team regularly monitors Usage Ping. They may alert you that your metrics need further optimization to run quicker and with greater success. You may also use the [Usage Ping QA dashboard](https://app.periscopedata.com/app/gitlab/632033/Usage-Ping-QA) to check how well your metric performs. The dashboard allows filtering by GitLab version, by "Self-managed" & "Saas" and shows you how many failures have occurred for each metric. Whenever you notice a high failure rate, you may re-optimize your metric. + +### Optional: Test Prometheus based Usage Ping + +If the data submitted includes metrics [queried from Prometheus](#prometheus-queries) that you would like to inspect and verify, +then you need to ensure that a Prometheus server is running locally, and that furthermore the respective GitLab components +are exporting metrics to it. If you do not need to test data coming from Prometheus, no further action +is necessary, since Usage Ping should degrade gracefully in the absence of a running Prometheus server. + +There are currently three kinds of components that may export data to Prometheus, and which are included in Usage Ping: + +- [`node_exporter`](https://github.com/prometheus/node_exporter) - Exports node metrics from the host machine +- [`gitlab-exporter`](https://gitlab.com/gitlab-org/gitlab-exporter) - Exports process metrics from various GitLab components +- various GitLab services such as Sidekiq and the Rails server that export their own metrics + +#### Test with an Omnibus container + +This is the recommended approach to test Prometheus based Usage Ping. + +The easiest way to verify your changes is to build a new Omnibus image from your code branch via CI, then download the image +and run a local container instance: + +1. From your merge request, click on the `qa` stage, then trigger the `package-and-qa` job. This job will trigger an Omnibus +build in a [downstream pipeline of the `omnibus-gitlab-mirror` project](https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/pipelines). +1. In the downstream pipeline, wait for the `gitlab-docker` job to finish. +1. Open the job logs and locate the full container name including the version. It will take the following form: `registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>`. +1. On your local machine, make sure you are logged in to the GitLab Docker registry. You can find the instructions for this in +[Authenticate to the GitLab Container Registry](../../user/packages/container_registry/index.md#authenticate-with-the-container-registry). +1. Once logged in, download the new image via `docker pull registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>` +1. For more information about working with and running Omnibus GitLab containers in Docker, please refer to [GitLab Docker images](https://docs.gitlab.com/omnibus/docker/README.html) in the Omnibus documentation. + +#### Test with GitLab development toolkits + +This is the less recommended approach, since it comes with a number of difficulties when emulating a real GitLab deployment. + +The [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) is not currently set up to run a Prometheus server or `node_exporter` alongside other GitLab components. If you would +like to do so, [Monitoring the GDK with Prometheus](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/prometheus/index.md#monitoring-the-gdk-with-prometheus) is a good start. + +The [GCK](https://gitlab.com/gitlab-org/gitlab-compose-kit) has limited support for testing Prometheus based Usage Ping. +By default, it already comes with a fully configured Prometheus service that is set up to scrape a number of components, +but with the following limitations: + +- It does not currently run a `gitlab-exporter` instance, so several `process_*` metrics from services such as Gitaly may be missing. +- While it runs a `node_exporter`, `docker-compose` services emulate hosts, meaning that it would normally report itself to not be associated +with any of the other services that are running. That is not how node metrics are reported in a production setup, where `node_exporter` +always runs as a process alongside other GitLab components on any given node. From Usage Ping's perspective none of the node data would therefore +appear to be associated to any of the services running, since they all appear to be running on different hosts. To alleviate this problem, the `node_exporter` in GCK was arbitrarily "assigned" to the `web` service, meaning only for this service `node_*` metrics will appear in Usage Ping. + +## Example Usage Ping payload + +The following is example content of the Usage Ping payload. + +```json +{ + "uuid": "0000000-0000-0000-0000-000000000000", + "hostname": "example.com", + "version": "12.10.0-pre", + "installation_type": "omnibus-gitlab", + "active_user_count": 999, + "recorded_at": "2020-04-17T07:43:54.162+00:00", + "edition": "EEU", + "license_md5": "00000000000000000000000000000000", + "license_id": null, + "historical_max_users": 999, + "licensee": { + "Name": "ABC, Inc.", + "Email": "email@example.com", + "Company": "ABC, Inc." + }, + "license_user_count": 999, + "license_starts_at": "2020-01-01", + "license_expires_at": "2021-01-01", + "license_plan": "ultimate", + "license_add_ons": { + }, + "license_trial": false, + "counts": { + "assignee_lists": 999, + "boards": 999, + "ci_builds": 999, + ... + }, + "container_registry_enabled": true, + "dependency_proxy_enabled": false, + "gitlab_shared_runners_enabled": true, + "gravatar_enabled": true, + "influxdb_metrics_enabled": true, + "ldap_enabled": false, + "mattermost_enabled": false, + "omniauth_enabled": true, + "prometheus_enabled": false, + "prometheus_metrics_enabled": false, + "reply_by_email_enabled": "incoming+%{key}@incoming.gitlab.com", + "signup_enabled": true, + "web_ide_clientside_preview_enabled": true, + "ingress_modsecurity_enabled": true, + "projects_with_expiration_policy_disabled": 999, + "projects_with_expiration_policy_enabled": 999, + ... + "elasticsearch_enabled": true, + "license_trial_ends_on": null, + "geo_enabled": false, + "git": { + "version": { + "major": 2, + "minor": 26, + "patch": 1 + } + }, + "gitaly": { + "version": "12.10.0-rc1-93-g40980d40", + "servers": 56, + "clusters": 14, + "filesystems": [ + "EXT_2_3_4" + ] + }, + "gitlab_pages": { + "enabled": true, + "version": "1.17.0" + }, + "container_registry_server": { + "vendor": "gitlab", + "version": "2.9.1-gitlab" + }, + "database": { + "adapter": "postgresql", + "version": "9.6.15" + }, + "avg_cycle_analytics": { + "issue": { + "average": 999, + "sd": 999, + "missing": 999 + }, + "plan": { + "average": null, + "sd": 999, + "missing": 999 + }, + "code": { + "average": null, + "sd": 999, + "missing": 999 + }, + "test": { + "average": null, + "sd": 999, + "missing": 999 + }, + "review": { + "average": null, + "sd": 999, + "missing": 999 + }, + "staging": { + "average": null, + "sd": 999, + "missing": 999 + }, + "production": { + "average": null, + "sd": 999, + "missing": 999 + }, + "total": 999 + }, + "analytics_unique_visits": { + "g_analytics_contribution": 999, + ... + }, + "usage_activity_by_stage": { + "configure": { + "project_clusters_enabled": 999, + ... + }, + "create": { + "merge_requests": 999, + ... + }, + "manage": { + "events": 999, + ... + }, + "monitor": { + "clusters": 999, + ... + }, + "package": { + "projects_with_packages": 999 + }, + "plan": { + "issues": 999, + ... + }, + "release": { + "deployments": 999, + ... + }, + "secure": { + "user_container_scanning_jobs": 999, + ... + }, + "verify": { + "ci_builds": 999, + ... + } + }, + "usage_activity_by_stage_monthly": { + "configure": { + "project_clusters_enabled": 999, + ... + }, + "create": { + "merge_requests": 999, + ... + }, + "manage": { + "events": 999, + ... + }, + "monitor": { + "clusters": 999, + ... + }, + "package": { + "projects_with_packages": 999 + }, + "plan": { + "issues": 999, + ... + }, + "release": { + "deployments": 999, + ... + }, + "secure": { + "user_container_scanning_jobs": 999, + ... + }, + "verify": { + "ci_builds": 999, + ... + } + }, + "topology": { + "duration_s": 0.013836685999194742, + "application_requests_per_hour": 4224, + "query_apdex_weekly_average": 0.996, + "failures": [], + "nodes": [ + { + "node_memory_total_bytes": 33269903360, + "node_memory_utilization": 0.35, + "node_cpus": 16, + "node_cpu_utilization": 0.2, + "node_uname_info": { + "machine": "x86_64", + "sysname": "Linux", + "release": "4.19.76-linuxkit" + }, + "node_services": [ + { + "name": "web", + "process_count": 16, + "process_memory_pss": 233349888, + "process_memory_rss": 788220927, + "process_memory_uss": 195295487, + "server": "puma" + }, + { + "name": "sidekiq", + "process_count": 1, + "process_memory_pss": 734080000, + "process_memory_rss": 750051328, + "process_memory_uss": 731533312 + }, + ... + ], + ... + }, + ... + ] + } +} +``` + +## Exporting Usage Ping SQL queries and definitions + +Two Rake tasks exist to export Usage Ping definitions. + +- The Rake tasks export the raw SQL queries for `count`, `distinct_count`, `sum`. +- The Rake tasks export the Redis counter class or the line of the Redis block for `redis_usage_data`. +- The Rake tasks calculate the `alt_usage_data` metrics. + +In the home directory of your local GitLab installation run the following Rake tasks for the YAML and JSON versions respectively: + +```shell +# for YAML export +bin/rake gitlab:usage_data:dump_sql_in_yaml + +# for JSON export +bin/rake gitlab:usage_data:dump_sql_in_json + +# You may pipe the output into a file +bin/rake gitlab:usage_data:dump_sql_in_yaml > ~/Desktop/usage-metrics-2020-09-02.yaml +``` diff --git a/doc/development/secure_coding_guidelines.md b/doc/development/secure_coding_guidelines.md index 8e7cc1ec9f2..e35bda82aaa 100644 --- a/doc/development/secure_coding_guidelines.md +++ b/doc/development/secure_coding_guidelines.md @@ -1,3 +1,10 @@ +--- +type: reference, dev +stage: none +group: Development +info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" +--- + # Secure Coding Guidelines This document contains descriptions and guidelines for addressing security @@ -393,5 +400,34 @@ In order to prevent Path Traversal vulnerabilities, user-controlled filenames or #### GitLab specific validations -- [`Gitlab::Utils.check_path_traversal`](https://gitlab.com/gitlab-org/security/gitlab/-/blob/master/lib/gitlab/utils.rb#L12-24) can be used to validate user input against Path Traversal vulnerabilities. Remember to add further validation when setting the `allowed_absolute` option to `true`. -- [`file_path` API validator](https://gitlab.com/gitlab-org/security/gitlab/-/blob/master/lib/api/validations/validators/file_path.rb) to validate user input when working with the Grape gem. +The methods `Gitlab::Utils.check_path_traversal!()` and `Gitlab::Utils.check_allowed_absolute_path!()` +can be used to validate user-supplied paths and prevent vulnerabilities. +`check_path_traversal!()` will detect their Path Traversal payloads and accepts URL-encoded paths. +`check_allowed_absolute_path!()` will check if a path is absolute and whether it is inside the allowed path list. By default, absolute +paths are not allowed, so you need to pass a list of allowed absolute paths to the `path_allowlist` +parameter when using `check_allowed_absolute_path!()`. + +To use a combination of both checks, follow the example below: + +```ruby +path = Gitlab::Utils.check_path_traversal!(path) +Gitlab::Utils.check_allowed_absolute_path!(path, path_allowlist) +``` + +In the REST API, we have the [`FilePath`](https://gitlab.com/gitlab-org/security/gitlab/-/blob/master/lib/api/validations/validators/file_path.rb) +validator that can be used to perform the checking on any file path argument the endpoints have. +It can be used as follows: + +```ruby +requires :file_path, type: String, file_path: { allowlist: ['/foo/bar/', '/home/foo/', '/app/home'] } +``` + +The Path Traversal check can also be used to forbid any absolute path: + +```ruby +requires :file_path, type: String, file_path: true +``` + +NOTE: **Note:** +Absolute paths are not allowed by default. If allowing an absolute path is required, you +need to provide an array of paths to the parameter `allowlist`. diff --git a/doc/development/telemetry/event_dictionary.md b/doc/development/telemetry/event_dictionary.md index d8cc32ea8d0..53b7ddea095 100644 --- a/doc/development/telemetry/event_dictionary.md +++ b/doc/development/telemetry/event_dictionary.md @@ -1,32 +1,5 @@ --- -stage: Growth -group: Telemetry -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../product_analytics/event_dictionary.md' --- -# Event Dictionary - -**Note: We've temporarily moved the Event Dictionary to a [Google Sheet](https://docs.google.com/spreadsheets/d/1VzE8R72Px_Y_LlE3Z05LxUlG_dumWe3vl-HeUo70TPw/edit?usp=sharing)**. The previous Markdown table exceeded 600 rows making it difficult to manage. In the future, our intention is to move this back into our docs using a [YAML file](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/823). - -The event dictionary is a single source of truth for the metrics and events we collect for product usage data. The Event Dictionary lists all the metrics and events we track, why we're tracking them, and where they are tracked. - -This is a living document that is updated any time a new event is planned or implemented. It includes the following information. - -- Section, stage, or group -- Description -- Implementation status -- Availability by plan type -- Code path - -We're currently focusing our Event Dictionary on [Usage Ping](usage_ping.md). In the future, we will also include [Snowplow](snowplow.md). We currently have an initiative across the entire product organization to complete the [Event Dictionary for Usage Ping](https://gitlab.com/groups/gitlab-org/-/epics/4174). - -## Instructions - -1. Open the Event Dictionary and fill in all the **PM to edit** columns highlighted in yellow. -1. Check that all the metrics and events are assigned to the correct section, stage, or group. If a metric is used across many groups, assign it to the stage. If a metric is used across many stages, assign it to the section. If a metric is incorrectly assigned to another section, stage, or group, let the PM know you have reassigned it. If your group has no assigned metrics and events, check that your metrics and events are not incorrectly assigned to another PM. -1. Add descriptions of what your metrics and events are tracking. Work with your Engineering team or the Telemetry team if you need help understanding this. -1. Add what plans this metric is available on. Work with your Engineering team or the Telemetry team if you need help understanding this. - -## Planned metrics and events - -For future metrics and events you plan to track, please add them to the Event Dictionary and note the status as `Planned`, `In Progress`, or `Implemented`. Once you have confirmed the metric has been implemented and have confirmed the metric data is in our data warehouse, change the status to **Data Available**. +This document was moved to [another location](../product_analytics/event_dictionary.md). diff --git a/doc/development/telemetry/index.md b/doc/development/telemetry/index.md index b5032ce3730..79ea80a52ea 100644 --- a/doc/development/telemetry/index.md +++ b/doc/development/telemetry/index.md @@ -1,182 +1,5 @@ --- -stage: Growth -group: Telemetry -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../product_analytics/index.md' --- -# Telemetry Guide - -At GitLab, we collect product usage data for the purpose of helping us build a better product. Data helps GitLab understand which parts of the product need improvement and which features we should build next. Product usage data also helps our team better understand the reasons why people use GitLab. With this knowledge we are able to make better product decisions. - -We encourage users to enable tracking, and we embrace full transparency with our tracking approach so it can be easily understood and trusted. - -By enabling tracking, users can: - -- Contribute back to the wider community. -- Help GitLab improve on the product. - -## Our tracking tools - -We use three methods to gather product usage data: - -- [Snowplow](#snowplow) -- [Usage Ping](#usage-ping) -- [Database import](#database-import) - -### Snowplow - -Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way -users engage with our website and application. - -Snowplow consists of two components: - -- [Snowplow JS](https://github.com/snowplow/snowplow/wiki/javascript-tracker) tracks client-side - events. -- [Snowplow Ruby](https://github.com/snowplow/snowplow/wiki/ruby-tracker) tracks server-side events. - -For more details, read the [Snowplow](snowplow.md) guide. - -### Usage Ping - -Usage Ping is a method for GitLab Inc to collect usage data on a GitLab instance. Usage Ping is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. This high-level data is used to help our product, support, and sales teams. - -For more details, read the [Usage Ping](usage_ping.md) guide. - -### Database import - -Database imports are full imports of data into GitLab's data warehouse. For GitLab.com, the PostgreSQL database is loaded into Snowflake data warehouse every 6 hours. For more details, see the [data team handbook](https://about.gitlab.com/handbook/business-ops/data-team/platform/#extract-and-load). - -## What data can be tracked - -Our different tracking tools allows us to track different types of events. The event types and examples of what data can be tracked are outlined below. - -The availability of event types and their tracking tools varies by segment. For example, on Self-Managed Users, we only have reporting using Database records via Usage Ping. - -| Event Types | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | -|----------------------------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| -| Snowplow (JS Pageview events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | -| Snowplow (JS UI events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | -| Snowplow (Ruby Pageview events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | -| Snowplow (Ruby CRUD / API events) | ✅ | 📅 | 📅 | ✅ | 📅 | 📅 | 📅 | 📅 | 📅 | 📅 | -| Usage Ping (Redis UI counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | -| Usage Ping (Redis Pageview counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | -| Usage Ping (Redis CRUD / API counters) | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | 🔄 | 🔄 | 🔄 | ✖️ | 🔄 | -| Usage Ping (Database counters) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | -| Usage Ping (Instance settings) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | -| Usage Ping (Integration settings) | ✅ | 🔄 | 📅 | ✖️ | ✅ | ✅ | ✅ | ✅ | ✖️ | ✅ | -| Database import (Database records) | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | - -[Source file](https://docs.google.com/spreadsheets/d/1e8Afo41Ar8x3JxAXJF3nL83UxVZ3hPIyXdt243VnNuE/edit?usp=sharing) - -**Legend** - -✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible - -SaaS = GitLab.com. SM = Self-Managed instance - -### Pageview events - -- Number of sessions that visited the /dashboard/groups page - -### UI events - -- Number of sessions that clicked on a button or link -- Number of sessions that closed a modal - -UI events are any interface-driven actions from the browser including click data. - -### CRUD or API events - -- Number of Git pushes -- Number of GraphQL queries -- Number of requests to a Rails action or controller - -These are backend events that include the creation, read, update, deletion of records, and other events that might be triggered from layers other than those available in the interface. - -### Database records - -These are raw database records which can be explored using business intelligence tools like Sisense. The full list of available tables can be found in [structure.sql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/structure.sql). - -### Instance settings - -These are settings of your instance such as the instance's Git version and if certain features are enabled such as `container_registry_enabled`. - -### Integration settings - -These are integrations your GitLab instance interacts with such as an [external storage provider](../../administration/static_objects_external_storage.md) or an [external container registry](../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint). These services must be able to send data back into a GitLab instance for data to be tracked. - -## Reporting level - -Our reporting levels of aggregate or individual reporting varies by segment. For example, on Self-Managed Users, we can report at an aggregate user level using Usage Ping but not on an Individual user level. - -| Aggregated Reporting | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | -|----------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| -| Snowplow | ✅ | 📅 | 📅 | ✅ | 📅 | ✅ | 📅 | 📅 | ✅ | 📅 | -| Usage Ping | ✅ | 🔄 | 📅 | 📅 | ✅ | ✅ | ✅ | ✅ | 📅 | ✅ | -| Database import | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | - -| Identifiable Reporting | SaaS Instance | SaaS Plan | SaaS Group | SaaS Session | SaaS User | SM Instance | SM Plan | SM Group | SM Session | SM User | -|------------------------|---------------|-----------|------------|--------------|-----------|-------------|---------|----------|------------|---------| -| Snowplow | ✅ | 📅 | 📅 | ✅ | 📅 | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | -| Usage Ping | ✅ | 🔄 | 📅 | ✖️ | ✖️ | ✅ | ✅ | ✖️ | ✖️ | ✖️ | -| Database import | ✅ | ✅ | ✅ | ✖️ | ✅ | ✖️ | ✖️ | ✖️ | ✖️ | ✖️ | - -**Legend** - -✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible - -SaaS = GitLab.com. SM = Self-Managed instance - -## Reporting time period - -Our reporting time periods varies by segment. For example, on Self-Managed Users, we can report all time counts and 28 day counts in Usage Ping. - -| Reporting Time Period | All Time | 28 Days | 7 Days | Daily | -|-----------------------|----------|---------|--------|-------| -| Snowplow | ✅ | ✅ | ✅ | ✅ | -| Usage Ping | ✅ | ✅ | 📅 | ✖️ | -| Database import | ✅ | ✅ | ✅ | ✅ | - -**Legend** - -✅ Available, 🔄 In Progress, 📅 Planned, ✖️ Not Possible - -## Systems overview - -The systems overview is a simplified diagram showing the interactions between GitLab Inc and self-managed instances. - - - -[Source file](https://app.diagrams.net/#G13DVpN-XnhWGz9tqReIj8pp1UE4ehk_EC) - -### GitLab Inc - -For Telemetry purposes, GitLab Inc has three major components: - -1. [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/): This contains everything managed by our data team including Sisense Dashboards for visualization, Snowflake for Data Warehousing, incoming data sources such as PostgreSQL Pipeline and S3 Bucket, and lastly our data collectors [GitLab.com's Snowplow Collector](https://gitlab.com/gitlab-com/gl-infra/readiness/-/tree/master/library/snowplow/) and GitLab's Versions Application. -1. GitLab.com: This is the production GitLab application which is made up of a Client and Server. On the Client or browser side, a Snowplow JS Tracker (Frontend) is used to track client-side events. On the Server or application side, a Snowplow Ruby Tracker (Backend) is used to track server-side events. The server also contains Usage Ping which leverages a PostgreSQL database and a Redis in-memory data store to report on usage data. Lastly, the server also contains System Logs which are generated from running the GitLab application. -1. [Monitoring infrastructure](https://about.gitlab.com/handbook/engineering/monitoring/): This is the infrastructure used to ensure GitLab.com is operating smoothly. System Logs are sent from GitLab.com to our monitoring infrastructure and collected by a FluentD collector. From FluentD, logs are either sent to long term Google Cloud Services cold storage via Stackdriver, or, they are sent to our Elastic Cluster via Cloud Pub/Sub which can be explored in real-time using Kibana. - -### Self-managed - -For Telemetry purposes, self-managed instances have two major components: - -1. Data infrastructure: Having a data infrastructure setup is optional on self-managed instances. If you'd like to collect Snowplow tracking events for your self-managed instance, you can setup your own self-managed Snowplow collector and configure your Snowplow events to point to your own collector. -1. GitLab: A self-managed GitLab instance contains all of the same components as GitLab.com mentioned above. - -### Differences between GitLab Inc and Self-managed - -As shown by the orange lines, on GitLab.com Snowplow JS, Snowplow Ruby, Usage Ping, and PostgreSQL database imports all flow into GitLab Inc's data infrastructure. However, on self-managed, only Usage Ping flows into GitLab Inc's data infrastructure. - -As shown by the green lines, on GitLab.com system logs flow into GitLab Inc's monitoring infrastructure. On self-managed, there are no logs sent to GitLab Inc's monitoring infrastructure. - -Note (1): Snowplow JS and Snowplow Ruby are available on self-managed, however, the Snowplow Collector endpoint is set to a self-managed Snowplow Collector which GitLab Inc does not have access to. - -## Additional information - -More useful links: - -- [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) -- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) -- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) -- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) +This document was moved to [another location](../product_analytics/index.md). diff --git a/doc/development/telemetry/snowplow.md b/doc/development/telemetry/snowplow.md index f84cea1124d..fd75d29286b 100644 --- a/doc/development/telemetry/snowplow.md +++ b/doc/development/telemetry/snowplow.md @@ -1,429 +1,5 @@ --- -stage: Growth -group: Telemetry -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../product_analytics/snowplow.md' --- -# Snowplow Guide - -This guide provides an overview of how Snowplow works, and implementation details. - -For more information about Telemetry, see: - -- [Telemetry Guide](index.md) -- [Usage Ping Guide](usage_ping.md) - -More useful links: - -- [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) -- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) -- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) -- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) - -## What is Snowplow - -Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way users engage with our website and application. - -[Snowplow](https://github.com/snowplow/snowplow) consists of the following loosely-coupled sub-systems: - -- **Trackers** fire Snowplow events. Snowplow has 12 trackers, covering web, mobile, desktop, server, and IoT. -- **Collectors** receive Snowplow events from trackers. We have three different event collectors, synchronizing events either to Amazon S3, Apache Kafka, or Amazon Kinesis. -- **Enrich** cleans up the raw Snowplow events, enriches them and puts them into storage. We have an Hadoop-based enrichment process, and a Kinesis-based or Kafka-based process. -- **Storage** is where the Snowplow events live. We store the Snowplow events in a flat file structure on S3, and in the Redshift and PostgreSQL databases. -- **Data modeling** is where event-level data is joined with other data sets and aggregated into smaller data sets, and business logic is applied. This produces a clean set of tables which make it easier to perform analysis on the data. We have data models for Redshift and Looker. -- **Analytics** are performed on the Snowplow events or on the aggregate tables. - - - -## Snowplow schema - -We have many definitions of Snowplow's schema. We have an active issue to [standardize this schema](https://gitlab.com/gitlab-org/gitlab/-/issues/207930) including the following definitions: - -- Frontend and backend taxonomy as listed below -- [Structured event taxonomy](#structured-event-taxonomy) -- [Self describing events](https://github.com/snowplow/snowplow/wiki/Custom-events#self-describing-events) -- [Iglu schema](https://gitlab.com/gitlab-org/iglu/) -- [Snowplow authored events](https://github.com/snowplow/snowplow/wiki/Snowplow-authored-events) - -## Enabling Snowplow - -Tracking can be enabled at: - -- The instance level, which enables tracking on both the frontend and backend layers. -- User level, though user tracking can be disabled on a per-user basis. GitLab tracking respects the [Do Not Track](https://www.eff.org/issues/do-not-track) standard, so any user who has enabled the Do Not Track option in their browser is not tracked at a user level. - -We utilize Snowplow for the majority of our tracking strategy and it is enabled on GitLab.com. On a self-managed instance, Snowplow can be enabled by navigating to: - -- **Admin Area > Settings > Integrations** in the UI. -- `admin/application_settings/integrations` in your browser. - -The following configuration is required: - -| Name | Value | -|---------------|---------------------------| -| Collector | `snowplow.trx.gitlab.net` | -| Site ID | `gitlab` | -| Cookie domain | `.gitlab.com` | - -## Snowplow request flow - -The following example shows a basic request/response flow between the following components: - -- Snowplow JS / Ruby Trackers on GitLab.com -- [GitLab.com Snowplow Collector](https://gitlab.com/gitlab-com/gl-infra/readiness/-/blob/master/library/snowplow/index.md) -- GitLab's S3 Bucket -- GitLab's Snowflake Data Warehouse -- Sisense: - -```mermaid -sequenceDiagram - participant Snowplow JS (Frontend) - participant Snowplow Ruby (Backend) - participant GitLab.com Snowplow Collector - participant S3 Bucket - participant Snowflake DW - participant Sisense Dashboards - Snowplow JS (Frontend) ->> GitLab.com Snowplow Collector: FE Tracking event - Snowplow Ruby (Backend) ->> GitLab.com Snowplow Collector: BE Tracking event - loop Process using Kinesis Stream - GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Log raw events - GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Enrich events - GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Write to disk - end - GitLab.com Snowplow Collector ->> S3 Bucket: Kinesis Firehose - S3 Bucket->>Snowflake DW: Import data - Snowflake DW->>Snowflake DW: Transform data using dbt - Snowflake DW->>Sisense Dashboards: Data available for querying -``` - -## Structured event taxonomy - -When adding new click events, we should add them in a way that's internally consistent. If we don't, it'll be very painful to perform analysis across features since each feature will be capturing events differently. - -The current method provides several attributes that are sent on each click event. Please try to follow these guidelines when specifying events to capture: - -| attribute | type | required | description | -| --------- | ------- | -------- | ----------- | -| category | text | true | The page or backend area of the application. Unless infeasible, please use the Rails page attribute by default in the frontend, and namespace + classname on the backend. | -| action | text | true | The action the user is taking, or aspect that's being instrumented. The first word should always describe the action or aspect: clicks should be `click`, activations should be `activate`, creations should be `create`, etc. Use underscores to describe what was acted on; for example, activating a form field would be `activate_form_input`. An interface action like clicking on a dropdown would be `click_dropdown`, while a behavior like creating a project record from the backend would be `create_project` | -| label | text | false | The specific element, or object that's being acted on. This is either the label of the element (e.g. a tab labeled 'Create from template' may be `create_from_template`) or a unique identifier if no text is available (e.g. closing the Groups dropdown in the top navbar might be `groups_dropdown_close`), or it could be the name or title attribute of a record being created. | -| property | text | false | Any additional property of the element, or object being acted on. | -| value | decimal | false | Describes a numeric value or something directly related to the event. This could be the value of an input (e.g. `10` when clicking `internal` visibility). | - -## Implementing Snowplow JS (Frontend) tracking - -GitLab provides `Tracking`, an interface that wraps the [Snowplow JavaScript Tracker](https://github.com/snowplow/snowplow/wiki/javascript-tracker) for tracking custom events. There are a few ways to utilize tracking, but each generally requires at minimum, a `category` and an `action`. Additional data can be provided that adheres to our [Structured event taxonomy](#structured-event-taxonomy). - -| field | type | default value | description | -|:-----------|:-------|:---------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `category` | string | document.body.dataset.page | Page or subsection of a page that events are being captured within. | -| `action` | string | 'generic' | Action the user is taking. Clicks should be `click` and activations should be `activate`, so for example, focusing a form field would be `activate_form_input`, and clicking a button would be `click_button`. | -| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described in our [Structured event taxonomy](#structured-event-taxonomy). | - -### Tracking in HAML (or Vue Templates) - -When working within HAML (or Vue templates) we can add `data-track-*` attributes to elements of interest. All elements that have a `data-track-event` attribute automatically have event tracking bound on clicks. - -Below is an example of `data-track-*` attributes assigned to a button: - -```haml -%button.btn{ data: { track: { event: "click_button", label: "template_preview", property: "my-template" } } } -``` - -```html -<button class="btn" - data-track-event="click_button" - data-track-label="template_preview" - data-track-property="my-template" -/> -``` - -Event listeners are bound at the document level to handle click events on or within elements with these data attributes. This allows them to be properly handled on re-rendering and changes to the DOM. Note that because of the way these events are bound, click events should not be stopped from propagating up the DOM tree. If for any reason click events are being stopped from propagating, you need to implement your own listeners and follow the instructions in [Tracking in raw JavaScript](#tracking-in-raw-javascript). - -Below is a list of supported `data-track-*` attributes: - -| attribute | required | description | -|:----------------------|:---------|:------------| -| `data-track-event` | true | Action the user is taking. Clicks must be prepended with `click` and activations must be prepended with `activate`. For example, focusing a form field would be `activate_form_input` and clicking a button would be `click_button`. | -| `data-track-label` | false | The `label` as described in our [Structured event taxonomy](#structured-event-taxonomy). | -| `data-track-property` | false | The `property` as described in our [Structured event taxonomy](#structured-event-taxonomy). | -| `data-track-value` | false | The `value` as described in our [Structured event taxonomy](#structured-event-taxonomy). If omitted, this is the element's `value` property or an empty string. For checkboxes, the default value is the element's checked attribute or `false` when unchecked. | -| `data-track-context` | false | The `context` as described in our [Structured event taxonomy](#structured-event-taxonomy). | - -### Tracking within Vue components - -There's a tracking Vue mixin that can be used in components if more complex tracking is required. To use it, first import the `Tracking` library and request a mixin. - -```javascript -import Tracking from '~/tracking'; -const trackingMixin = Tracking.mixin({ label: 'right_sidebar' }); -``` - -You can provide default options that are passed along whenever an event is tracked from within your component. For instance, if all events within a component should be tracked with a given `label`, you can provide one at this time. Available defaults are `category`, `label`, `property`, and `value`. If no category is specified, `document.body.dataset.page` is used as the default. - -You can then use the mixin normally in your component with the `mixin` Vue declaration. The mixin also provides the ability to specify tracking options in `data` or `computed`. These override any defaults and allow the values to be dynamic from props, or based on state. - -```javascript -export default { - mixins: [trackingMixin], - // ...[component implementation]... - data() { - return { - expanded: false, - tracking: { - label: 'left_sidebar' - } - }; - }, -} -``` - -The mixin provides a `track` method that can be called within the template, or from component methods. An example of the whole implementation might look like the following. - -```javascript -export default { - mixins: [Tracking.mixin({ label: 'right_sidebar' })], - data() { - return { - expanded: false, - }; - }, - methods: { - toggle() { - this.expanded = !this.expanded; - this.track('click_toggle', { value: this.expanded }) - } - } -}; -``` - -And if needed within the template, you can use the `track` method directly as well. - -```html -<template> - <div> - <a class="toggle" @click.prevent="toggle">Toggle</a> - <div v-if="expanded"> - <p>Hello world!</p> - <a @click.prevent="track('click_action')">Track an event</a> - </div> - </div> -</template> -``` - -### Tracking in raw JavaScript - -Custom event tracking and instrumentation can be added by directly calling the `Tracking.event` static function. The following example demonstrates tracking a click on a button by calling `Tracking.event` manually. - -```javascript -import Tracking from '~/tracking'; - -const button = document.getElementById('create_from_template_button'); -button.addEventListener('click', () => { - Tracking.event('dashboard:projects:index', 'click_button', { - label: 'create_from_template', - property: 'template_preview', - value: 'rails', - }); -}) -``` - -### Tests and test helpers - -In Jest particularly in Vue tests, you can use the following: - -```javascript -import { mockTracking } from 'helpers/tracking_helper'; - -describe('MyTracking', () => { - let spy; - - beforeEach(() => { - spy = mockTracking('_category_', wrapper.element, jest.spyOn); - }); - - it('tracks an event when clicked on feedback', () => { - wrapper.find('.discover-feedback-icon').trigger('click'); - - expect(spy).toHaveBeenCalledWith('_category_', 'click_button', { - label: 'security-discover-feedback-cta', - property: '0', - }); - }); -}); -``` - -In obsolete Karma tests it's used as below: - -```javascript -import { mockTracking, triggerEvent } from 'spec/helpers/tracking_helper'; - -describe('my component', () => { - let trackingSpy; - - beforeEach(() => { - trackingSpy = mockTracking('_category_', vm.$el, spyOn); - }); - - const triggerEvent = () => { - // action which should trigger a event - }; - - it('tracks an event when toggled', () => { - expect(trackingSpy).not.toHaveBeenCalled(); - - triggerEvent('a.toggle'); - - expect(trackingSpy).toHaveBeenCalledWith('_category_', 'click_edit_button', { - label: 'right_sidebar', - property: 'confidentiality', - }); - }); -}); -``` - -## Implementing Snowplow Ruby (Backend) tracking - -GitLab provides `Gitlab::Tracking`, an interface that wraps the [Snowplow Ruby Tracker](https://github.com/snowplow/snowplow/wiki/ruby-tracker) for tracking custom events. - -Custom event tracking and instrumentation can be added by directly calling the `GitLab::Tracking.event` class method, which accepts the following arguments: - -| argument | type | default value | description | -|:-----------|:-------|:--------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `category` | string | 'application' | Area or aspect of the application. This could be `HealthCheckController` or `Lfs::FileTransformer` for instance. | -| `action` | string | 'generic' | The action being taken, which can be anything from a controller action like `create` to something like an Active Record callback. | -| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described in [Structured event taxonomy](#structured-event-taxonomy). These are set as empty strings if you don't provide them. | - -Tracking can be viewed as either tracking user behavior, or can be utilized for instrumentation to monitor and visualize performance over time in an area or aspect of code. - -For example: - -```ruby -class Projects::CreateService < BaseService - def execute - project = Project.create(params) - - Gitlab::Tracking.event('Projects::CreateService', 'create_project', - label: project.errors.full_messages.to_sentence, - value: project.valid? - ) - end -end -``` - -### Unit testing - -Use the `expect_snowplow_event` helper when testing backend Snowplow events. See [testing best practices]( -https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-snowplow-events) for details. - -### Performance - -We use the [AsyncEmitter](https://github.com/snowplow/snowplow/wiki/Ruby-Tracker#52-the-asyncemitter-class) when tracking events, which allows for instrumentation calls to be run in a background thread. This is still an active area of development. - -## Developing and testing Snowplow - -There are several tools for developing and testing Snowplow Event - -| Testing Tool | Frontend Tracking | Backend Tracking | Local Development Environment | Production Environment | Production Environment | -|----------------------------------------------|--------------------|---------------------|-------------------------------|------------------------|------------------------| -| Snowplow Analytics Debugger Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | -| Snowplow Inspector Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | -| Snowplow Micro | **{check-circle}** | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | -| Snowplow Mini | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{status_preparing}** | **{status_preparing}** | - -**Legend** - -**{check-circle}** Available, **{status_preparing}** In progress, **{dotted-circle}** Not Planned - -### Preparing your MR for Review - -1. For frontend events, in the MR description section, add a screenshot of the event's relevant section using the [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) Chrome browser extension. -1. For backend events, please use Snowplow Micro and add the output of the Snowplow Micro good events `GET http://localhost:9090/micro/good`. - -### Snowplow Analytics Debugger Chrome Extension - -Snowplow Analytics Debugger is a browser extension for testing frontend events. This works on production, staging and local development environments. - -1. Install the [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) Chrome browser extension. -1. Open Chrome DevTools to the Snowplow Analytics Debugger tab. -1. Learn more at [Igloo Analytics](https://www.iglooanalytics.com/blog/snowplow-analytics-debugger-chrome-extension.html). - -### Snowplow Inspector Chrome Extension - -Snowplow Inspector Chrome Extension is a browser extension for testing frontend events. This works on production, staging and local development environments. - -1. Install [Snowplow Inspector](https://chrome.google.com/webstore/detail/snowplow-inspector/maplkdomeamdlngconidoefjpogkmljm?hl=en). -1. Open the Chrome extension by pressing the Snowplow Inspector icon beside the address bar. -1. Click around on a webpage with Snowplow and you should see JavaScript events firing in the inspector window. - -### Snowplow Micro - -Snowplow Micro is a very small version of a full Snowplow data collection pipeline: small enough that it can be launched by a test suite. Events can be recorded into Snowplow Micro just as they can a full Snowplow pipeline. Micro then exposes an API that can be queried. - -Snowplow Micro is a Docker-based solution for testing frontend and backend events in a local development environment. You need to modify GDK using the instructions below to set this up. - -- Read [Introducing Snowplow Micro](https://snowplowanalytics.com/blog/2019/07/17/introducing-snowplow-micro/) -- Look at the [Snowplow Micro repository](https://github.com/snowplow-incubator/snowplow-micro) -- Watch our [installation guide recording](https://www.youtube.com/watch?v=OX46fo_A0Ag) - -1. Install [Snowplow Micro](https://github.com/snowplow-incubator/snowplow-micro): - - ```shell - docker run --mount type=bind,source=$(pwd)/example,destination=/config -p 9090:9090 snowplow/snowplow-micro:latest --collector-config /config/micro.conf --iglu /config/iglu.json - ``` - -1. Install snowplow micro by cloning the settings in [this project](https://gitlab.com/a_akgun/snowplow-micro): - - ```shell - git clone git@gitlab.com:a_akgun/snowplow-micro.git - ./snowplow-micro.sh - ``` - -1. Update port in SQL to set `9090`: - - ```shell - gdk psql -d gitlabhq_development - update application_settings set snowplow_collector_hostname='localhost:9090', snowplow_enabled=true, snowplow_cookie_domain='.gitlab.com'; - ``` - -1. Update `app/assets/javascripts/tracking.js` to [remove this line](https://gitlab.com/snippets/1918635): - - ```javascript - forceSecureTracker: true - ``` - -1. Update `lib/gitlab/tracking.rb` to [add these lines](https://gitlab.com/snippets/1918635): - - ```ruby - protocol: 'http', - port: 9090, - ``` - -1. Update `lib/gitlab/tracking.rb` to [change async emitter from https to http](https://gitlab.com/snippets/1918635): - - ```ruby - SnowplowTracker::AsyncEmitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname, protocol: 'http'), - ``` - -1. Enable Snowplow in the admin area, Settings::Integrations::Snowplow to point to: - `http://localhost:3000/admin/application_settings/integrations#js-snowplow-settings`. - -1. Restart GDK: - - ```shell - `gdk restart` - ``` - -1. Send a test Snowplow event from the Rails console: - - ```ruby - Gitlab::Tracking.self_describing_event('iglu:com.gitlab/pageview_context/jsonschema/1-0-0', { page_type: 'MY_TYPE' }, context: nil ) - ``` - -### Snowplow Mini - -[Snowplow Mini](https://github.com/snowplow/snowplow-mini) is an easily-deployable, single-instance version of Snowplow. - -Snowplow Mini can be used for testing frontend and backend events on a production, staging and local development environment. - -For GitLab.com, we're setting up a [QA and Testing environment](https://gitlab.com/gitlab-org/telemetry/-/issues/266) using Snowplow Mini. +This document was moved to [another location](../product_analytics/snowplow.md). diff --git a/doc/development/telemetry/usage_ping.md b/doc/development/telemetry/usage_ping.md index 71bfb6dca8f..1026e7a5a66 100644 --- a/doc/development/telemetry/usage_ping.md +++ b/doc/development/telemetry/usage_ping.md @@ -1,934 +1,5 @@ --- -stage: Growth -group: Telemetry -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: '../product_analytics/usage_ping.md' --- -# Usage Ping Guide - -> - Introduced in GitLab Enterprise Edition 8.10. -> - More statistics were added in GitLab Enterprise Edition 8.12. -> - Moved to GitLab Core in 9.1. -> - More statistics were added in GitLab Ultimate 11.2. - -This guide describes Usage Ping's purpose and how it's implemented. - -For more information about Telemetry, see: - -- [Telemetry Guide](index.md) -- [Snowplow Guide](snowplow.md) - -More useful links: - -- [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) -- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) -- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) -- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) - -## What is Usage Ping? - -- GitLab sends a weekly payload containing usage data to GitLab Inc. Usage Ping provides high-level data to help our product, support, and sales teams. It does not send any project names, usernames, or any other specific data. The information from the usage ping is not anonymous, it is linked to the hostname of the instance. Sending usage ping is optional, and any instance can disable analytics. -- The usage data is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. In addition to counts, other facts - that help us classify and understand GitLab installations are collected. -- Usage ping is important to GitLab as we use it to calculate our Stage Monthly Active Users (SMAU) which helps us measure the success of our stages and features. -- While usage ping is enabled, GitLab will gather data from the other instances and will be able to show usage statistics of your instance to your users. - -### Why should we enable Usage Ping? - -- The main purpose of Usage Ping is to build a better GitLab. Data about how GitLab is used is collected to better understand feature/stage adoption and usage, which helps us understand how GitLab is adding value and helps our team better understand the reasons why people use GitLab and with this knowledge we're able to make better product decisions. -- As a benefit of having the usage ping active, GitLab lets you analyze the users’ activities over time of your GitLab installation. -- As a benefit of having the usage ping active, GitLab provides you with The DevOps Report,which gives you an overview of your entire instance’s adoption of Concurrent DevOps from planning to monitoring. -- You will get better, more proactive support. (assuming that our TAMs and support organization used the data to deliver more value) -- You will get insight and advice into how to get the most value out of your investment in GitLab. Wouldn't you want to know that a number of features or values are not being adopted in your organization? -- You get a report that illustrates how you compare against other similar organizations (anonymized), with specific advice and recommendations on how to improve your DevOps processes. -- Usage Ping is enabled by default. To disable it, see [Disable Usage Ping](#disable-usage-ping). - -### Limitations - -- Usage Ping does not track frontend events things like page views, link clicks, or user sessions, and only focuses on aggregated backend events. -- Because of these limitations we recommend instrumenting your products with Snowplow for more detailed analytics on GitLab.com and use Usage Ping to track aggregated backend events on self-managed. - -## Usage Ping payload - -You can view the exact JSON payload sent to GitLab Inc. in the administration panel. To view the payload: - -1. Navigate to **Admin Area > Settings > Metrics and profiling**. -1. Expand the **Usage statistics** section. -1. Click the **Preview payload** button. - -For an example payload, see [Example Usage Ping payload](#example-usage-ping-payload). - -## Disable Usage Ping - -To disable Usage Ping in the GitLab UI, go to the **Settings** page of your administration panel and uncheck the **Usage Ping** checkbox. - -To disable Usage Ping and prevent it from being configured in the future through the administration panel, Omnibus installs can set the following in [`gitlab.rb`](https://docs.gitlab.com/omnibus/settings/configuration.html#configuration-options): - -```ruby -gitlab_rails['usage_ping_enabled'] = false -``` - -Source installations can set the following in `gitlab.yml`: - -```yaml -production: &base - # ... - gitlab: - # ... - usage_ping_enabled: false -``` - -## Usage Ping request flow - -The following example shows a basic request/response flow between a GitLab instance, the Versions Application, the License Application, Salesforce, GitLab's S3 Bucket, GitLab's Snowflake Data Warehouse, and Sisense: - -```mermaid -sequenceDiagram - participant GitLab Instance - participant Versions Application - participant Licenses Application - participant Salesforce - participant S3 Bucket - participant Snowflake DW - participant Sisense Dashboards - GitLab Instance->>Versions Application: Send Usage Ping - loop Process usage data - Versions Application->>Versions Application: Parse usage data - Versions Application->>Versions Application: Write to database - Versions Application->>Versions Application: Update license ping time - end - loop Process data for Salesforce - Versions Application-xLicenses Application: Request Zuora subscription id - Licenses Application-xVersions Application: Zuora subscription id - Versions Application-xSalesforce: Request Zuora account id by Zuora subscription id - Salesforce-xVersions Application: Zuora account id - Versions Application-xSalesforce: Usage data for the Zuora account - end - Versions Application->>S3 Bucket: Export Versions database - S3 Bucket->>Snowflake DW: Import data - Snowflake DW->>Snowflake DW: Transform data using dbt - Snowflake DW->>Sisense Dashboards: Data available for querying - Versions Application->>GitLab Instance: DevOps Report (Conversational Development Index) -``` - -## How Usage Ping works - -1. The Usage Ping [cron job](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/gitlab_usage_ping_worker.rb#L30) is set in Sidekiq to run weekly. -1. When the cron job runs, it calls [`GitLab::UsageData.to_json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L22). -1. `GitLab::UsageData.to_json` [cascades down](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L22) to ~400+ other counter method calls. -1. The response of all methods calls are [merged together](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L14) into a single JSON payload in `GitLab::UsageData.to_json`. -1. The JSON payload is then [posted to the Versions application]( https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L20). - -## Implementing Usage Ping - -Usage Ping consists of two kinds of data, counters and observations. Counters track how often a certain event -happened over time, such as how many CI pipelines have run. They are monotonic and always trend up. -Observations are facts collected from one or more GitLab instances and can carry arbitrary data. There are no -general guidelines around how to collect those, due to the individual nature of that data. - -There are several types of counters which are all found in `usage_data.rb`: - -- **Ordinary Batch Counters:** Simple count of a given ActiveRecord_Relation -- **Distinct Batch Counters:** Distinct count of a given ActiveRecord_Relation on given column -- **Sum Batch Counters:** Sum the values of a given ActiveRecord_Relation on given column -- **Alternative Counters:** Used for settings and configurations -- **Redis Counters:** Used for in-memory counts. - -NOTE: **Note:** -Only use the provided counter methods. Each counter method contains a built in fail safe to isolate each counter to avoid breaking the entire Usage Ping. - -### Why batch counting - -For large tables, PostgreSQL can take a long time to count rows due to MVCC [(Multi-version Concurrency Control)](https://en.wikipedia.org/wiki/Multiversion_concurrency_control). Batch counting is a counting method where a single large query is broken into multiple smaller queries. For example, instead of a single query querying 1,000,000 records, with batch counting, you can execute 100 queries of 10,000 records each. Batch counting is useful for avoiding database timeouts as each batch query is significantly shorter than one single long running query. - -For GitLab.com, there are extremely large tables with 15 second query timeouts, so we use batch counting to avoid encountering timeouts. Here are the sizes of some GitLab.com tables: - -| Table | Row counts in millions | -|------------------------------|------------------------| -| `merge_request_diff_commits` | 2280 | -| `ci_build_trace_sections` | 1764 | -| `merge_request_diff_files` | 1082 | -| `events` | 514 | - -There are two batch counting methods provided, `Ordinary Batch Counters` and `Distinct Batch Counters`. Batch counting requires indexes on columns to calculate max, min, and range queries. In some cases, a specialized index may need to be added on the columns involved in a counter. - -### Ordinary Batch Counters - -Handles `ActiveRecord::StatementInvalid` error - -Simple count of a given ActiveRecord_Relation, does a non-distinct batch count, smartly reduces batch_size and handles errors. - -Method: `count(relation, column = nil, batch: true, start: nil, finish: nil)` - -Arguments: - -- `relation` the ActiveRecord_Relation to perform the count -- `column` the column to perform the count on, by default is the primary key -- `batch`: default `true` in order to use batch counting -- `start`: custom start of the batch counting in order to avoid complex min calculations -- `end`: custom end of the batch counting in order to avoid complex min calculations - -Examples: - -```ruby -count(User.active) -count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) -count(::Clusters::Cluster.aws_installed.enabled, :cluster_id, start: ::Clusters::Cluster.minimum(:id), finish: ::Clusters::Cluster.maximum(:id)) -``` - -### Distinct Batch Counters - -Handles `ActiveRecord::StatementInvalid` error - -Distinct count of a given ActiveRecord_Relation on given column, a distinct batch count, smartly reduces batch_size and handles errors. - -Method: `distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)` - -Arguments: - -- `relation` the ActiveRecord_Relation to perform the count -- `column` the column to perform the distinct count, by default is the primary key -- `batch`: default `true` in order to use batch counting -- `batch_size`: if none set it will use default value 10000 from `Gitlab::Database::BatchCounter` -- `start`: custom start of the batch counting in order to avoid complex min calculations -- `end`: custom end of the batch counting in order to avoid complex min calculations - -Examples: - -```ruby -distinct_count(::Project, :creator_id) -distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) -distinct_count(::Clusters::Applications::CertManager.where(time_period).available.joins(:cluster), 'clusters.user_id') -``` - -### Sum Batch Counters - -Handles `ActiveRecord::StatementInvalid` error - -Sum the values of a given ActiveRecord_Relation on given column and handles errors. - -Method: `sum(relation, column, batch_size: nil, start: nil, finish: nil)` - -Arguments: - -- `relation` the ActiveRecord_Relation to perform the operation -- `column` the column to sum on -- `batch_size`: if none set it will use default value 1000 from `Gitlab::Database::BatchCounter` -- `start`: custom start of the batch counting in order to avoid complex min calculations -- `end`: custom end of the batch counting in order to avoid complex min calculations - -Examples: - -```ruby -sum(JiraImportState.finished, :imported_issues_count) -``` - -### Grouping & Batch Operations - -The `count`, `distinct_count`, and `sum` batch counters can accept an `ActiveRecord::Relation` -object, which groups by a specified column. With a grouped relation, the methods do batch counting, -handle errors, and returns a hash table of key-value pairs. - -Examples: - -```ruby -count(Namespace.group(:type)) -# returns => {nil=>179, "Group"=>54} - -distinct_count(Project.group(:visibility_level), :creator_id) -# returns => {0=>1, 10=>1, 20=>11} - -sum(Issue.group(:state_id), :weight)) -# returns => {1=>3542, 2=>6820} -``` - -### Redis Counters - -Handles `::Redis::CommandError` and `Gitlab::UsageDataCounters::BaseCounter::UnknownEvent` -returns -1 when a block is sent or hash with all values -1 when a `counter(Gitlab::UsageDataCounters)` is sent -different behavior due to 2 different implementations of Redis counter - -Method: `redis_usage_data(counter, &block)` - -Arguments: - -- `counter`: a counter from `Gitlab::UsageDataCounters`, that has `fallback_totals` method implemented -- or a `block`: which is evaluated - -#### Ordinary Redis Counters - -Examples of implementation: - -- Using Redis methods [`INCR`](https://redis.io/commands/incr), [`GET`](https://redis.io/commands/get), and [`Gitlab::UsageDataCounters::WikiPageCounter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/wiki_page_counter.rb) -- Using Redis methods [`HINCRBY`](https://redis.io/commands/hincrby), [`HGETALL`](https://redis.io/commands/hgetall), and [`Gitlab::UsageCounters::PodLogs`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_counters/pod_logs.rb) - -#### Redis HLL Counters - -With `Gitlab::UsageDataCounters::HLLRedisCounter` we have available data structures used to count unique values. - -Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd) and [PFCOUNT](https://redis.io/commands/pfcount). - -##### Adding new events - -1. Define events in [`known_events.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml). - - Example event: - - ```yaml - - name: i_compliance_credential_inventory - category: compliance - redis_slot: compliance - expiry: 42 # 6 weeks - aggregation: weekly - ``` - - Keys: - - - `name`: unique event name. - - Name format `<prefix>_<redis_slot>_name`. - - Use one of the following prefixes for the event's name: - - - `g_` for group, as an event which is tracked for group. - - `p_` for project, as an event which is tracked for project. - - `i_` for instance, as an event which is tracked for instance. - - `a_` for events encompassing all `g_`, `p_`, `i_`. - - `o_` for other. - - Consider including in the event's name the Redis slot in order to be able to count totals for a specific category. - - Example names: `i_compliance_credential_inventory`, `g_analytics_contribution`. - - - `category`: event category. Used for getting total counts for events in a category, for easier - access to a group of events. - - `redis_slot`: optional Redis slot; default value: event name. Used if needed to calculate totals - for a group of metrics. Ensure keys are in the same slot. For example: - `i_compliance_credential_inventory` with `redis_slot: 'compliance'` will build Redis key - `i_{compliance}_credential_inventory-2020-34`. If `redis_slot` is not defined the Redis key will - be `{i_compliance_credential_inventory}-2020-34`. - - `expiry`: expiry time in days. Default: 29 days for daily aggregation and 6 weeks for weekly - aggregation. - - `aggregation`: aggregation `:daily` or `:weekly`. The argument defines how we build the Redis - keys for data storage. For `daily` we keep a key for metric per day of the year, for `weekly` we - keep a key for metric per week of the year. - -1. Track event in controller using `RedisTracking` module with `track_redis_hll_event(*controller_actions, name:, feature:, feature_default_enabled: false)`. - - Arguments: - - - `controller_actions`: controller actions we want to track. - - `name`: event name. - - `feature`: feature name, all metrics we track should be under feature flag. - - `feature_default_enabled`: feature flag is disabled by default, set to `true` for it to be enabled by default. - - Example usage: - - ```ruby - # controller - class ProjectsController < Projects::ApplicationController - include RedisTracking - - skip_before_action :authenticate_user!, only: :show - track_redis_hll_event :index, :show, name: 'i_analytics_dev_ops_score', feature: :g_compliance_dashboard_feature, feature_default_enabled: true - - def index - render html: 'index' - end - - def new - render html: 'new' - end - - def show - render html: 'show' - end - end - ``` - -1. Track event in API using `increment_unique_values(event_name, values)` helper method. - - In order to be able to track the event, Usage Ping must be enabled and the event feature `usage_data_<event_name>` must be enabled. - - Arguments: - - - `event_name`: event name. - - `values`: values counted, one value or array of values. - - Example usage: - - ```ruby - get ':id/registry/repositories' do - repositories = ContainerRepositoriesFinder.new( - user: current_user, subject: user_group - ).execute - - increment_unique_values('i_list_repositories', current_user.id) - - present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] - end - ``` - -1. Track event using `track_usage_event(event_name, values)` in services and graphql - - Increment unique values count using Redis HLL, for given event name. - - Example: - - [Track usage event for incident created in service](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/issues/update_service.rb) - - [Track usage event for incident created in graphql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/mutations/alert_management/update_alert_status.rb) - - ```ruby - track_usage_event(:incident_management_incident_created, current_user.id) - ``` - -1. Track event using `UsageData` API - - Increment unique users count using Redis HLL, for given event name. - - Tracking events using the `UsageData` API requires the `usage_data_api` feature flag to be enabled, which is disabled by default. - - API requests are protected by checking for a valid CSRF token. - - In order to be able to increment the values the related feature `usage_data<event_name>` should be enabled. - - ```plaintext - POST /usage_data/increment_unique_users - ``` - - | Attribute | Type | Required | Description | - | :-------- | :--- | :------- | :---------- | - | `event` | string | yes | The event name it should be tracked | - - Response -w - Return 200 if tracking failed for any reason. - - - `200` if event was tracked or any errors - - `400 Bad request` if event parameter is missing - - `401 Unauthorized` if user is not authenticated - - `403 Forbidden` for invalid CSRF token provided - -1. Track events using JavaScript/Vue API helper which calls the API above - - Example usage for an existing event already defined in [known events](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml): - - Note that `usage_data_api` and `usage_data_#{event_name}` should be enabled in order to be able to track events - - ```javascript - import api from '~/api'; - - api.trackRedisHllUserEvent('my_already_defined_event_name'), - ``` - -1. Track event using base module `Gitlab::UsageDataCounters::HLLRedisCounter.track_event(entity_id, event_name)`. - - Arguments: - - - `entity_id`: value we count. For example: user_id, visitor_id. - - `event_name`: event name. - -1. Get event data using `Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names:, start_date:, end_date)`. - - Arguments: - - - `event_names`: the list of event names. - - `start_date`: start date of the period for which we want to get event data. - - `end_date`: end date of the period for which we want to get event data. - -Recommendations: - -- Key should expire in 29 days for daily and 42 days for weekly. -- If possible, data granularity should be a week. For example a key could be composed from the - metric's name and week of the year, `2020-33-{metric_name}`. -- Use a [feature flag](../../operations/feature_flags.md) to have a control over the impact when - adding new metrics. -- Feature flags should be [default on](../documentation/feature_flags.md#criteria) - before final release to ensure we receive data from self-managed instances. - -##### Known events in usage data payload - -All events added in [`known_events.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml) are automatically added to usage data generation under the `redis_hll_counters` key. This column is stored in [version-app as a JSON](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L209). -For each event we add metrics for the weekly and monthly time frames, and totals for each where applicable: - -- `#{event_name}_weekly`: Data for 7 days for daily [aggregation](#adding-new-events) events and data for the last complete week for weekly [aggregation](#adding-new-events) events. -- `#{event_name}_monthly`: Data for 28 days for daily [aggregation](#adding-new-events) events and data for the last 4 complete weeks for weekly [aggregation](#adding-new-events) events. -- `#{category}_total_unique_counts_weekly`: Total unique counts for events in the same category for the last 7 days or the last complete week, if events are in the same Redis slot and we have more than one metric. -- `#{category}_total_unique_counts_monthly`: Total unique counts for events in same category for the last 28 days or the last 4 complete weeks, if events are in the same Redis slot and we have more than one metric. - -Example of `redis_hll_counters` data: - -```ruby -{:redis_hll_counters=> - {"compliance"=> - {"g_compliance_dashboard_weekly"=>0, - "g_compliance_dashboard_monthly"=>0, - "g_compliance_audit_events_weekly"=>0, - "g_compliance_audit_events_monthly"=>0, - "compliance_total_unique_counts_weekly"=>0, - "compliance_total_unique_counts_monthly"=>0}, - "analytics"=> - {"g_analytics_contribution_weekly"=>0, - "g_analytics_contribution_monthly"=>0, - "g_analytics_insights_weekly"=>0, - "g_analytics_insights_monthly"=>0, - "analytics_total_unique_counts_weekly"=>0, - "analytics_total_unique_counts_monthly"=>0}, - "ide_edit"=> - {"g_edit_by_web_ide_weekly"=>0, - "g_edit_by_web_ide_monthly"=>0, - "g_edit_by_sfe_weekly"=>0, - "g_edit_by_sfe_monthly"=>0, - "ide_edit_total_unique_counts_weekly"=>0, - "ide_edit_total_unique_counts_monthly"=>0}, - "search"=> - {"i_search_total_weekly"=>0, "i_search_total_monthly"=>0, "i_search_advanced_weekly"=>0, "i_search_advanced_monthly"=>0, "i_search_paid_weekly"=>0, "i_search_paid_monthly"=>0, "search_total_unique_counts_weekly"=>0, "search_total_unique_counts_monthly"=>0}, - "source_code"=>{"wiki_action_weekly"=>0, "wiki_action_monthly"=>0} - } -``` - -Example usage: - -```ruby -# Redis Counters -redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) -redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } - -# Define events in known_events.yml https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events.yml - -# Tracking events -Gitlab::UsageDataCounters::HLLRedisCounter.track_event(visitor_id, 'expand_vulnerabilities') - -# Get unique events for metric -redis_usage_data { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'expand_vulnerabilities', start_date: 28.days.ago, end_date: Date.current) } -``` - -### Alternative Counters - -Handles `StandardError` and fallbacks into -1 this way not all measures fail if we encounter one exception. -Mainly used for settings and configurations. - -Method: `alt_usage_data(value = nil, fallback: -1, &block)` - -Arguments: - -- `value`: a simple static value in which case the value is simply returned. -- or a `block`: which is evaluated -- `fallback: -1`: the common value used for any metrics that are failing. - -Example of usage: - -```ruby -alt_usage_data { Gitlab::VERSION } -alt_usage_data { Gitlab::CurrentSettings.uuid } -alt_usage_data(999) -``` - -### Prometheus Queries - -In those cases where operational metrics should be part of Usage Ping, a database or Redis query is unlikely -to provide useful data. Instead, Prometheus might be more appropriate, since most of GitLab's architectural -components publish metrics to it that can be queried back, aggregated, and included as usage data. - -NOTE: **Note:** -Prometheus as a data source for Usage Ping is currently only available for single-node Omnibus installations -that are running the [bundled Prometheus](../../administration/monitoring/prometheus/index.md) instance. - -In order to query Prometheus for metrics, a helper method is available that will `yield` a fully configured -`PrometheusClient`, given it is available as per the note above: - -```ruby -with_prometheus_client do |client| - response = client.query('<your query>') - ... -end -``` - -Please refer to [the `PrometheusClient` definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/prometheus_client.rb) -for how to use its API to query for data. - -## Developing and testing Usage Ping - -### 1. Use your Rails console to manually test counters - -```ruby -# count -Gitlab::UsageData.count(User.active) -Gitlab::UsageData.count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) - -# count distinct -Gitlab::UsageData.distinct_count(::Project, :creator_id) -Gitlab::UsageData.distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) -``` - -### 2. Generate the SQL query - -Your Rails console will return the generated SQL queries. - -Example: - -```ruby -pry(main)> Gitlab::UsageData.count(User.active) - (2.6ms) SELECT "features"."key" FROM "features" - (15.3ms) SELECT MIN("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) - (2.4ms) SELECT MAX("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) - (1.9ms) SELECT COUNT("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) AND "users"."id" BETWEEN 1 AND 100000 -``` - -### 3. Optimize queries with #database-lab - -Paste the SQL query into `#database-lab` to see how the query performs at scale. - -- `#database-lab` is a Slack channel which uses a production-sized environment to test your queries. -- GitLab.com’s production database has a 15 second timeout. -- Any single query must stay below 1 second execution time with cold caches. -- Add a specialized index on columns involved to reduce the execution time. - -In order to have an understanding of the query's execution we add in the MR description the following information: - -- For counters that have a `time_period` test we add information for both cases: - - `time_period = {}` for all time periods - - `time_period = { created_at: 28.days.ago..Time.current }` for last 28 days period -- Execution plan and query time before and after optimization -- Query generated for the index and time -- Migration output for up and down execution - -We also use `#database-lab` and [explain.depesz.com](https://explain.depesz.com/). For more details, see the [database review guide](../database_review.md#preparation-when-adding-or-modifying-queries). - -#### Optimization recommendations and examples - -- Use specialized indexes [example 1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26871), [example 2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26445). -- Use defined `start` and `finish`, and simple queries, because these values can be memoized and reused, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37155). -- Avoid joins and write the queries as simply as possible, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36316). -- Set a custom `batch_size` for `distinct_count`, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38000). - -### 4. Add the metric definition - -When adding, changing, or updating metrics, please update the [Event Dictionary's **Usage Ping** table](event_dictionary.md). - -### 5. Add new metric to Versions Application - -Check if new metrics need to be added to the Versions Application. See `usage_data` [schema](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L147) and usage data [parameters accepted](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/app/services/usage_ping.rb). Any metrics added under the `counts` key are saved in the `counts` column. - -### 6. Add the feature label - -Add the `feature` label to the Merge Request for new Usage Ping metrics. These are user-facing changes and are part of expanding the Usage Ping feature. - -### 7. Add a changelog file - -Ensure you comply with the [Changelog entries guide](../changelog.md). - -### 8. Ask for a Telemetry Review - -On GitLab.com, we have DangerBot setup to monitor Telemetry related files and DangerBot will recommend a Telemetry review. Mention `@gitlab-org/growth/telemetry/engineers` in your MR for a review. - -### 9. Verify your metric - -On GitLab.com, the Product Analytics team regularly monitors Usage Ping. They may alert you that your metrics need further optimization to run quicker and with greater success. You may also use the [Usage Ping QA dashboard](https://app.periscopedata.com/app/gitlab/632033/Usage-Ping-QA) to check how well your metric performs. The dashboard allows filtering by GitLab version, by "Self-managed" & "Saas" and shows you how many failures have occurred for each metric. Whenever you notice a high failure rate, you may re-optimize your metric. - -### Optional: Test Prometheus based Usage Ping - -If the data submitted includes metrics [queried from Prometheus](#prometheus-queries) that you would like to inspect and verify, -then you need to ensure that a Prometheus server is running locally, and that furthermore the respective GitLab components -are exporting metrics to it. If you do not need to test data coming from Prometheus, no further action -is necessary, since Usage Ping should degrade gracefully in the absence of a running Prometheus server. - -There are currently three kinds of components that may export data to Prometheus, and which are included in Usage Ping: - -- [`node_exporter`](https://github.com/prometheus/node_exporter) - Exports node metrics from the host machine -- [`gitlab-exporter`](https://gitlab.com/gitlab-org/gitlab-exporter) - Exports process metrics from various GitLab components -- various GitLab services such as Sidekiq and the Rails server that export their own metrics - -#### Test with an Omnibus container - -This is the recommended approach to test Prometheus based Usage Ping. - -The easiest way to verify your changes is to build a new Omnibus image from your code branch via CI, then download the image -and run a local container instance: - -1. From your merge request, click on the `qa` stage, then trigger the `package-and-qa` job. This job will trigger an Omnibus -build in a [downstream pipeline of the `omnibus-gitlab-mirror` project](https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/pipelines). -1. In the downstream pipeline, wait for the `gitlab-docker` job to finish. -1. Open the job logs and locate the full container name including the version. It will take the following form: `registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>`. -1. On your local machine, make sure you are logged in to the GitLab Docker registry. You can find the instructions for this in -[Authenticate to the GitLab Container Registry](../../user/packages/container_registry/index.md#authenticate-with-the-container-registry). -1. Once logged in, download the new image via `docker pull registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>` -1. For more information about working with and running Omnibus GitLab containers in Docker, please refer to [GitLab Docker images](https://docs.gitlab.com/omnibus/docker/README.html) in the Omnibus documentation. - -#### Test with GitLab development toolkits - -This is the less recommended approach, since it comes with a number of difficulties when emulating a real GitLab deployment. - -The [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) is not currently set up to run a Prometheus server or `node_exporter` alongside other GitLab components. If you would -like to do so, [Monitoring the GDK with Prometheus](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/prometheus/index.md#monitoring-the-gdk-with-prometheus) is a good start. - -The [GCK](https://gitlab.com/gitlab-org/gitlab-compose-kit) has limited support for testing Prometheus based Usage Ping. -By default, it already comes with a fully configured Prometheus service that is set up to scrape a number of components, -but with the following limitations: - -- It does not currently run a `gitlab-exporter` instance, so several `process_*` metrics from services such as Gitaly may be missing. -- While it runs a `node_exporter`, `docker-compose` services emulate hosts, meaning that it would normally report itself to not be associated -with any of the other services that are running. That is not how node metrics are reported in a production setup, where `node_exporter` -always runs as a process alongside other GitLab components on any given node. From Usage Ping's perspective none of the node data would therefore -appear to be associated to any of the services running, since they all appear to be running on different hosts. To alleviate this problem, the `node_exporter` in GCK was arbitrarily "assigned" to the `web` service, meaning only for this service `node_*` metrics will appear in Usage Ping. - -## Example Usage Ping payload - -The following is example content of the Usage Ping payload. - -```json -{ - "uuid": "0000000-0000-0000-0000-000000000000", - "hostname": "example.com", - "version": "12.10.0-pre", - "installation_type": "omnibus-gitlab", - "active_user_count": 999, - "recorded_at": "2020-04-17T07:43:54.162+00:00", - "edition": "EEU", - "license_md5": "00000000000000000000000000000000", - "license_id": null, - "historical_max_users": 999, - "licensee": { - "Name": "ABC, Inc.", - "Email": "email@example.com", - "Company": "ABC, Inc." - }, - "license_user_count": 999, - "license_starts_at": "2020-01-01", - "license_expires_at": "2021-01-01", - "license_plan": "ultimate", - "license_add_ons": { - }, - "license_trial": false, - "counts": { - "assignee_lists": 999, - "boards": 999, - "ci_builds": 999, - ... - }, - "container_registry_enabled": true, - "dependency_proxy_enabled": false, - "gitlab_shared_runners_enabled": true, - "gravatar_enabled": true, - "influxdb_metrics_enabled": true, - "ldap_enabled": false, - "mattermost_enabled": false, - "omniauth_enabled": true, - "prometheus_enabled": false, - "prometheus_metrics_enabled": false, - "reply_by_email_enabled": "incoming+%{key}@incoming.gitlab.com", - "signup_enabled": true, - "web_ide_clientside_preview_enabled": true, - "ingress_modsecurity_enabled": true, - "projects_with_expiration_policy_disabled": 999, - "projects_with_expiration_policy_enabled": 999, - ... - "elasticsearch_enabled": true, - "license_trial_ends_on": null, - "geo_enabled": false, - "git": { - "version": { - "major": 2, - "minor": 26, - "patch": 1 - } - }, - "gitaly": { - "version": "12.10.0-rc1-93-g40980d40", - "servers": 56, - "clusters": 14, - "filesystems": [ - "EXT_2_3_4" - ] - }, - "gitlab_pages": { - "enabled": true, - "version": "1.17.0" - }, - "container_registry_server": { - "vendor": "gitlab", - "version": "2.9.1-gitlab" - }, - "database": { - "adapter": "postgresql", - "version": "9.6.15" - }, - "avg_cycle_analytics": { - "issue": { - "average": 999, - "sd": 999, - "missing": 999 - }, - "plan": { - "average": null, - "sd": 999, - "missing": 999 - }, - "code": { - "average": null, - "sd": 999, - "missing": 999 - }, - "test": { - "average": null, - "sd": 999, - "missing": 999 - }, - "review": { - "average": null, - "sd": 999, - "missing": 999 - }, - "staging": { - "average": null, - "sd": 999, - "missing": 999 - }, - "production": { - "average": null, - "sd": 999, - "missing": 999 - }, - "total": 999 - }, - "analytics_unique_visits": { - "g_analytics_contribution": 999, - ... - }, - "usage_activity_by_stage": { - "configure": { - "project_clusters_enabled": 999, - ... - }, - "create": { - "merge_requests": 999, - ... - }, - "manage": { - "events": 999, - ... - }, - "monitor": { - "clusters": 999, - ... - }, - "package": { - "projects_with_packages": 999 - }, - "plan": { - "issues": 999, - ... - }, - "release": { - "deployments": 999, - ... - }, - "secure": { - "user_container_scanning_jobs": 999, - ... - }, - "verify": { - "ci_builds": 999, - ... - } - }, - "usage_activity_by_stage_monthly": { - "configure": { - "project_clusters_enabled": 999, - ... - }, - "create": { - "merge_requests": 999, - ... - }, - "manage": { - "events": 999, - ... - }, - "monitor": { - "clusters": 999, - ... - }, - "package": { - "projects_with_packages": 999 - }, - "plan": { - "issues": 999, - ... - }, - "release": { - "deployments": 999, - ... - }, - "secure": { - "user_container_scanning_jobs": 999, - ... - }, - "verify": { - "ci_builds": 999, - ... - } - }, - "topology": { - "duration_s": 0.013836685999194742, - "application_requests_per_hour": 4224, - "query_apdex_weekly_average": 0.996, - "failures": [], - "nodes": [ - { - "node_memory_total_bytes": 33269903360, - "node_memory_utilization": 0.35, - "node_cpus": 16, - "node_cpu_utilization": 0.2, - "node_uname_info": { - "machine": "x86_64", - "sysname": "Linux", - "release": "4.19.76-linuxkit" - }, - "node_services": [ - { - "name": "web", - "process_count": 16, - "process_memory_pss": 233349888, - "process_memory_rss": 788220927, - "process_memory_uss": 195295487, - "server": "puma" - }, - { - "name": "sidekiq", - "process_count": 1, - "process_memory_pss": 734080000, - "process_memory_rss": 750051328, - "process_memory_uss": 731533312 - }, - ... - ], - ... - }, - ... - ] - } -} -``` - -## Exporting Usage Ping SQL queries and definitions - -Two Rake tasks exist to export Usage Ping definitions. - -- The Rake tasks export the raw SQL queries for `count`, `distinct_count`, `sum`. -- The Rake tasks export the Redis counter class or the line of the Redis block for `redis_usage_data`. -- The Rake tasks calculate the `alt_usage_data` metrics. - -In the home directory of your local GitLab installation run the following Rake tasks for the YAML and JSON versions respectively: - -```shell -# for YAML export -bin/rake gitlab:usage_data:dump_sql_in_yaml - -# for JSON export -bin/rake gitlab:usage_data:dump_sql_in_json - -# You may pipe the output into a file -bin/rake gitlab:usage_data:dump_sql_in_yaml > ~/Desktop/usage-metrics-2020-09-02.yaml -``` +This document was moved to [another location](../product_analytics/usage_ping.md). diff --git a/doc/gitlab-basics/create-project.md b/doc/gitlab-basics/create-project.md index 8c9f8add695..65e76998673 100644 --- a/doc/gitlab-basics/create-project.md +++ b/doc/gitlab-basics/create-project.md @@ -18,7 +18,7 @@ To create a project in GitLab: icon in the navigation bar. This opens the **New project** page. 1. On the **New project** page, choose if you want to: - Create a [blank project](#blank-projects). - - Create a project using with one of the available [project templates](#project-templates). + - Create a project using one of the available [project templates](#project-templates). - [Import a project](../user/project/import/index.md) from a different repository, if enabled on your GitLab instance. Contact your GitLab administrator if this is unavailable. - Run [CI/CD pipelines for external repositories](../ci/ci_cd_for_external_repos/index.md). **(PREMIUM)** diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index 7d6b383174b..a88f2db5c26 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -66,14 +66,12 @@ source. You must [install it separately](https://www.elastic.co/guide/en/elastic Be sure to select your version. Providing detailed information on installing Elasticsearch is out of the scope of this document. -NOTE: **Note:** Elasticsearch should be installed on a separate server, whether you install it yourself or use a cloud hosted offering like Elastic's [Elasticsearch Service](https://www.elastic.co/elasticsearch/service) (available on AWS, GCP, or Azure) or the [Amazon Elasticsearch](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-gsg.html) service. Running Elasticsearch on the same server as GitLab is not recommended and can cause a degradation in GitLab instance performance. -NOTE: **Note:** **For a single node Elasticsearch cluster the functional cluster health status will be yellow** (will never be green) because the primary shard is allocated but replicas can not be as there is no other node to which Elasticsearch can assign a @@ -164,7 +162,6 @@ may need to set the `production -> elasticsearch -> indexer_path` setting in you ## Enabling Advanced Search -NOTE: **Note:** For GitLab instances with more than 50GB repository data you can follow the instructions for [Indexing large instances](#indexing-large-instances) below. @@ -309,7 +306,6 @@ CAUTION: **Caution:** It is highly recommended that you take a snapshot of your cluster to ensure there is a recovery path if anything goes wrong. -NOTE: **Note:** Due to a technical limitation, there will be a slight downtime because of the fact that we need to reclaim the current `primary` index to be used as the alias. @@ -407,7 +403,6 @@ To trigger the re-index from `primary` index: Under **Admin Area > Settings > General > Advanced Search > Elasticsearch zero-downtime reindexing**, click on **Trigger cluster reindexing**. -NOTE: **Note:** Reindexing can be a lengthy process depending on the size of your Elasticsearch cluster. CAUTION: **Caution:** diff --git a/doc/operations/incident_management/alert_integrations.md b/doc/operations/incident_management/alert_integrations.md index febc1d7eda9..87c0e5456e6 100644 --- a/doc/operations/incident_management/alert_integrations.md +++ b/doc/operations/incident_management/alert_integrations.md @@ -15,6 +15,10 @@ to use this endpoint. ## Configuration +You can either configure alerts to integrate with an [external Prometheus server](#external-prometheus-integration), +or provide a [generic HTTP endpoint](#generic-http-endpoint) to receive alerts +from other services. + ### Generic HTTP Endpoint Enabling the Generic HTTP Endpoint creates a unique HTTP endpoint that can receive alert payloads in JSON format. You can always diff --git a/doc/operations/tracing.md b/doc/operations/tracing.md index 07f60c37f1b..6f7e775775a 100644 --- a/doc/operations/tracing.md +++ b/doc/operations/tracing.md @@ -4,9 +4,10 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Tracing **(ULTIMATE)** +# Tracing -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7903) in GitLab Ultimate 11.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7903) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.5. +> - [Moved to GitLab Core](https://gitlab.com/gitlab-org/gitlab/-/issues/42645) in 13.5. Tracing provides insight into the performance and health of a deployed application, tracking each function or microservice which handles a given request. diff --git a/doc/telemetry/index.md b/doc/telemetry/index.md index 977b93b712e..4583dbe4753 100644 --- a/doc/telemetry/index.md +++ b/doc/telemetry/index.md @@ -1,5 +1,5 @@ --- -redirect_to: '../development/telemetry/index.md' +redirect_to: '../development/product_analytics/index.md' --- -This document was moved to [another location](../development/telemetry/index.md). +This document was moved to [another location](../development/product_analytics/index.md). diff --git a/doc/telemetry/snowplow.md b/doc/telemetry/snowplow.md index 977b93b712e..643893bcc22 100644 --- a/doc/telemetry/snowplow.md +++ b/doc/telemetry/snowplow.md @@ -1,5 +1,5 @@ --- -redirect_to: '../development/telemetry/index.md' +redirect_to: '../development/product_analytics/snowplow.md' --- -This document was moved to [another location](../development/telemetry/index.md). +This document was moved to [another location](../development/product_analytics/snowplow.md). diff --git a/doc/topics/autodevops/stages.md b/doc/topics/autodevops/stages.md index 44eebb748a6..be6800abf75 100644 --- a/doc/topics/autodevops/stages.md +++ b/doc/topics/autodevops/stages.md @@ -195,7 +195,7 @@ see the documentation. > Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0. License Compliance uses the -[License Compliance Docker image](https://gitlab.com/gitlab-org/security-products/license-management) +[License Compliance Docker image](https://gitlab.com/gitlab-org/security-products/analyzers/license-finder) to search the project dependencies for their license. The Auto License Compliance stage is skipped on licenses other than [Ultimate](https://about.gitlab.com/pricing/). diff --git a/doc/user/admin_area/settings/index.md b/doc/user/admin_area/settings/index.md index a107e607d7f..09a70dda60d 100644 --- a/doc/user/admin_area/settings/index.md +++ b/doc/user/admin_area/settings/index.md @@ -34,7 +34,7 @@ Access the default page for admin area settings by navigating to **Admin Area > | [PlantUML](../../../administration/integration/plantuml.md#gitlab) | Allow rendering of PlantUML diagrams in AsciiDoc documents. | | [Slack application](../../../user/project/integrations/gitlab_slack_application.md#configuration) **(FREE ONLY)** | Slack integration allows you to interact with GitLab via slash commands in a chat window. This option is only available on GitLab.com, though it may be [available for self-managed instances in the future](https://gitlab.com/gitlab-org/gitlab/-/issues/28164). | | [Third party offers](third_party_offers.md) | Control the display of third party offers. | -| [Snowplow](../../../development/telemetry/snowplow.md) | Configure the Snowplow integration. | +| [Snowplow](../../../development/product_analytics/snowplow.md) | Configure the Snowplow integration. | | [Google GKE](../../project/clusters/add_gke_clusters.md) | Google GKE integration allows you to provision GKE clusters from GitLab. | | [Amazon EKS](../../project/clusters/add_eks_clusters.md) | Amazon EKS integration allows you to provision EKS clusters from GitLab. | diff --git a/doc/user/admin_area/settings/usage_statistics.md b/doc/user/admin_area/settings/usage_statistics.md index 55bbcfbe1d8..140d149555a 100644 --- a/doc/user/admin_area/settings/usage_statistics.md +++ b/doc/user/admin_area/settings/usage_statistics.md @@ -58,7 +58,7 @@ sequenceDiagram ## Usage Ping **(CORE ONLY)** -See [Usage Ping guide](../../../development/telemetry/usage_ping.md). +See [Usage Ping guide](../../../development/product_analytics/usage_ping.md). ## Instance-level statistics **(CORE ONLY)** diff --git a/doc/user/clusters/agent/index.md b/doc/user/clusters/agent/index.md index 4065ee3337b..8332811a727 100644 --- a/doc/user/clusters/agent/index.md +++ b/doc/user/clusters/agent/index.md @@ -111,7 +111,7 @@ gitops: ### Create an Agent record in GitLab Next, create an GitLab Rails Agent record so the Agent can associate itself with -a GitLab project. Creating this record also creates a Secret needed to configure +the configuration repository project. Creating this record also creates a Secret needed to configure the Agent in subsequent steps. You can create an Agent record either: - Through the Rails console, by running `rails c`: diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index 382d536be74..894c0e14862 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -121,7 +121,7 @@ The results will be saved as a [License Compliance report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportslicense_scanning) that you can later download and analyze. Due to implementation limitations, we always take the latest License Compliance artifact available. Behind the scenes, the -[GitLab License Compliance Docker image](https://gitlab.com/gitlab-org/security-products/license-management) +[GitLab License Compliance Docker image](https://gitlab.com/gitlab-org/security-products/analyzers/license-finder) is used to detect the languages/frameworks and in turn analyzes the licenses. The License Compliance settings can be changed through [environment variables](#available-variables) by using the @@ -634,7 +634,7 @@ import the following default License Compliance analyzer images from `registry.g offline [local Docker container registry](../../packages/container_registry/index.md): ```plaintext -registry.gitlab.com/gitlab-org/security-products/license-management:latest +registry.gitlab.com/gitlab-org/security-products/analyzers/license-finder:latest ``` The process for importing Docker images into a local offline Docker registry depends on diff --git a/doc/user/img/gitlab_snippet_v13_5.png b/doc/user/img/gitlab_snippet_v13_5.png Binary files differnew file mode 100644 index 00000000000..f824aa6bf4f --- /dev/null +++ b/doc/user/img/gitlab_snippet_v13_5.png diff --git a/doc/user/project/settings/project_access_tokens.md b/doc/user/project/settings/project_access_tokens.md index bcc5a1dd028..15a0f3ccc2b 100644 --- a/doc/user/project/settings/project_access_tokens.md +++ b/doc/user/project/settings/project_access_tokens.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Source Code +stage: Manage +group: Access info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" type: reference, howto --- diff --git a/doc/user/snippets.md b/doc/user/snippets.md index 6335460ef96..2d0c9d4f943 100644 --- a/doc/user/snippets.md +++ b/doc/user/snippets.md @@ -92,6 +92,40 @@ be changed to `http-a-weird-filename-me` to be included in the snippet's repository. As snippets are stored by ID, changing their filenames will not break direct or embedded links to the snippet. +### Multiple files by Snippet + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2829) in GitLab 13.5. + +GitLab Snippets support multiple files in one single snippet. It can be very handy +when your code snippet is composed of multiple parts or when they relate +to a certain context. For example: + +- A snippet that includes a script and its output. +- A snippet that includes HTML, CSS, and JS code. +- A snippet with a `docker-compose.yml` file and its associated `.env` file. +- A `gulpfile.js` file coupled with a `package.json` file, which together can be used to bootstrap a project and manage its dependencies. + +Snippets support between 1 and 10 files. They can be managed via Git (since they're [versioned](#versioned-snippets) +by a Git repository), through the [Snippets API](../api/snippets.md), or within the GitLab UI. + + + +To add a new file to your snippet through the GitLab UI: + +1. Go to your snippet in the GitLab UI. +1. Click **Edit** in the top right. +1. Select **Add another file**. +1. Add your content to the file in the form fields provided. +1. Click **Save changes**. + +To delete a file from your snippet through the GitLab UI: + +1. Go to your snippet in the GitLab UI. +1. Click **Edit** in the top right. +1. Select **Delete file** alongside the file name of each file +you wish to delete. +1. Click **Save changes**. + ### Cloning snippets Snippets can be cloned as a regular Git repository using SSH or HTTPS. Click the **Clone** @@ -114,16 +148,16 @@ see the documentation on [reducing repository size](../user/project/repository/r ### Limitations - Binary files are not supported. -- Creating or deleting branches is not supported. Only a default *master*. -branch is used. +- Creating or deleting branches is not supported. Only a default *master* branch is used. - Git tags are not supported in snippet repositories. -- Snippets' repositories are limited to one file. Attempting to push more -than one file will result in an error. +- Snippets' repositories are limited to 10 files. Attempting to push more +than 10 files will result in an error. - Revisions are not *yet* visible to the user on the GitLab UI, but it's planned to be added in future iterations. See the [revisions tab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/39271) for updates. - The [maximum size for a snippet](../administration/snippets/index.md#snippets-content-size-limit) is 50 MB, by default. +- Git LFS is not supported. ## Discover snippets diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index cc34d23decc..63237e41376 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -1,7 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/license_compliance/ # # Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/license-management#settings +# List of the variables: https://gitlab.com/gitlab-org/security-products/analyzers/license-finder#settings # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: diff --git a/spec/controllers/every_controller_spec.rb b/spec/controllers/every_controller_spec.rb index d301c8b84ff..b1519c4ef1e 100644 --- a/spec/controllers/every_controller_spec.rb +++ b/spec/controllers/every_controller_spec.rb @@ -25,13 +25,6 @@ RSpec.describe "Every controller" do controller_actions.map do |controller, action| next if controller.feature_category_for_action(action) - next unless controller.to_s.start_with?('B', 'C', 'D', 'E', 'F', - 'H', 'I', 'J', 'K', 'L', - 'M', 'N', 'O', 'Q', 'R', - 'S', 'T', 'U', 'V', 'W', - 'X', 'Y', 'Z', 'A', 'G', - 'Projects::MergeRequestsController') - "#{controller}##{action}" end.compact end diff --git a/spec/frontend/repository/components/preview/__snapshots__/index_spec.js.snap b/spec/frontend/repository/components/preview/__snapshots__/index_spec.js.snap index 69b7a3931f8..23c06dc5e68 100644 --- a/spec/frontend/repository/components/preview/__snapshots__/index_spec.js.snap +++ b/spec/frontend/repository/components/preview/__snapshots__/index_spec.js.snap @@ -10,9 +10,10 @@ exports[`Repository file preview component renders file HTML 1`] = ` <div class="file-header-content" > - <i + <gl-icon-stub aria-hidden="true" - class="fa fa-file-text-o fa-fw" + name="doc-text" + size="16" /> <gl-link-stub diff --git a/spec/frontend/search/components/dropdown_filter_spec.js b/spec/frontend/search/components/dropdown_filter_spec.js deleted file mode 100644 index ffac038e1c5..00000000000 --- a/spec/frontend/search/components/dropdown_filter_spec.js +++ /dev/null @@ -1,108 +0,0 @@ -import { shallowMount } from '@vue/test-utils'; -import { GlDropdown, GlDropdownItem } from '@gitlab/ui'; -import DropdownFilter from '~/search/components/dropdown_filter.vue'; -import { - FILTER_STATES, - FILTER_STATES_BY_SCOPE, - FILTER_HEADER, - SCOPES, -} from '~/search/state_filter/constants'; -import * as urlUtils from '~/lib/utils/url_utility'; - -jest.mock('~/lib/utils/url_utility', () => ({ - visitUrl: jest.fn(), - setUrlParams: jest.fn(), -})); - -function createComponent(props = { scope: 'issues' }) { - return shallowMount(DropdownFilter, { - propsData: { - filtersArray: FILTER_STATES_BY_SCOPE.issues, - filters: FILTER_STATES, - header: FILTER_HEADER, - param: 'state', - supportedScopes: Object.values(SCOPES), - ...props, - }, - }); -} - -describe('DropdownFilter', () => { - let wrapper; - - beforeEach(() => { - wrapper = createComponent(); - }); - - afterEach(() => { - wrapper.destroy(); - wrapper = null; - }); - - const findGlDropdown = () => wrapper.find(GlDropdown); - const findGlDropdownItems = () => findGlDropdown().findAll(GlDropdownItem); - const findDropdownItemsText = () => findGlDropdownItems().wrappers.map(w => w.text()); - const firstDropDownItem = () => findGlDropdownItems().at(0); - - describe('template', () => { - describe.each` - scope | showDropdown - ${'issues'} | ${true} - ${'merge_requests'} | ${true} - ${'projects'} | ${false} - ${'milestones'} | ${false} - ${'users'} | ${false} - ${'notes'} | ${false} - ${'wiki_blobs'} | ${false} - ${'blobs'} | ${false} - `(`dropdown`, ({ scope, showDropdown }) => { - beforeEach(() => { - wrapper = createComponent({ scope }); - }); - - it(`does${showDropdown ? '' : ' not'} render when scope is ${scope}`, () => { - expect(findGlDropdown().exists()).toBe(showDropdown); - }); - }); - - describe.each` - initialFilter | label - ${FILTER_STATES.ANY.value} | ${`Any ${FILTER_HEADER}`} - ${FILTER_STATES.OPEN.value} | ${FILTER_STATES.OPEN.label} - ${FILTER_STATES.CLOSED.value} | ${FILTER_STATES.CLOSED.label} - `(`filter text`, ({ initialFilter, label }) => { - describe(`when initialFilter is ${initialFilter}`, () => { - beforeEach(() => { - wrapper = createComponent({ scope: 'issues', initialFilter }); - }); - - it(`sets dropdown label to ${label}`, () => { - expect(findGlDropdown().attributes('text')).toBe(label); - }); - }); - }); - - describe('Filter options', () => { - it('renders a dropdown item for each filterOption', () => { - expect(findDropdownItemsText()).toStrictEqual( - FILTER_STATES_BY_SCOPE[SCOPES.ISSUES].map(v => { - return v.label; - }), - ); - }); - - it('clicking a dropdown item calls setUrlParams', () => { - const state = FILTER_STATES[Object.keys(FILTER_STATES)[0]].value; - firstDropDownItem().vm.$emit('click'); - - expect(urlUtils.setUrlParams).toHaveBeenCalledWith({ state }); - }); - - it('clicking a dropdown item calls visitUrl', () => { - firstDropDownItem().vm.$emit('click'); - - expect(urlUtils.visitUrl).toHaveBeenCalled(); - }); - }); - }); -}); diff --git a/spec/frontend/search/dropdown_filter/components/dropdown_filter_spec.js b/spec/frontend/search/dropdown_filter/components/dropdown_filter_spec.js new file mode 100644 index 00000000000..4a6b5cebe1c --- /dev/null +++ b/spec/frontend/search/dropdown_filter/components/dropdown_filter_spec.js @@ -0,0 +1,196 @@ +import Vuex from 'vuex'; +import { createLocalVue, shallowMount } from '@vue/test-utils'; +import { GlDropdown, GlDropdownItem } from '@gitlab/ui'; +import * as urlUtils from '~/lib/utils/url_utility'; +import initStore from '~/search/store'; +import DropdownFilter from '~/search/dropdown_filter/components/dropdown_filter.vue'; +import stateFilterData from '~/search/dropdown_filter/constants/state_filter_data'; +import confidentialFilterData from '~/search/dropdown_filter/constants/confidential_filter_data'; +import { MOCK_QUERY } from '../mock_data'; + +jest.mock('~/lib/utils/url_utility', () => ({ + visitUrl: jest.fn(), + setUrlParams: jest.fn(), +})); + +const localVue = createLocalVue(); +localVue.use(Vuex); + +describe('DropdownFilter', () => { + let wrapper; + let store; + + const createStore = options => { + store = initStore({ query: MOCK_QUERY, ...options }); + }; + + const createComponent = (props = { filterData: stateFilterData }) => { + wrapper = shallowMount(DropdownFilter, { + localVue, + store, + propsData: { + ...props, + }, + }); + }; + + afterEach(() => { + wrapper.destroy(); + wrapper = null; + store = null; + }); + + const findGlDropdown = () => wrapper.find(GlDropdown); + const findGlDropdownItems = () => findGlDropdown().findAll(GlDropdownItem); + const findDropdownItemsText = () => findGlDropdownItems().wrappers.map(w => w.text()); + const firstDropDownItem = () => findGlDropdownItems().at(0); + + describe('StatusFilter', () => { + describe('template', () => { + describe.each` + scope | showDropdown + ${'issues'} | ${true} + ${'merge_requests'} | ${true} + ${'projects'} | ${false} + ${'milestones'} | ${false} + ${'users'} | ${false} + ${'notes'} | ${false} + ${'wiki_blobs'} | ${false} + ${'blobs'} | ${false} + `(`dropdown`, ({ scope, showDropdown }) => { + beforeEach(() => { + createStore({ query: { ...MOCK_QUERY, scope } }); + createComponent(); + }); + + it(`does${showDropdown ? '' : ' not'} render when scope is ${scope}`, () => { + expect(findGlDropdown().exists()).toBe(showDropdown); + }); + }); + + describe.each` + initialFilter | label + ${stateFilterData.filters.ANY.value} | ${`Any ${stateFilterData.header}`} + ${stateFilterData.filters.OPEN.value} | ${stateFilterData.filters.OPEN.label} + ${stateFilterData.filters.CLOSED.value} | ${stateFilterData.filters.CLOSED.label} + `(`filter text`, ({ initialFilter, label }) => { + describe(`when initialFilter is ${initialFilter}`, () => { + beforeEach(() => { + createStore({ query: { ...MOCK_QUERY, [stateFilterData.filterParam]: initialFilter } }); + createComponent(); + }); + + it(`sets dropdown label to ${label}`, () => { + expect(findGlDropdown().attributes('text')).toBe(label); + }); + }); + }); + }); + + describe('Filter options', () => { + beforeEach(() => { + createStore(); + createComponent(); + }); + + it('renders a dropdown item for each filterOption', () => { + expect(findDropdownItemsText()).toStrictEqual( + stateFilterData.filterByScope[stateFilterData.scopes.ISSUES].map(v => { + return v.label; + }), + ); + }); + + it('clicking a dropdown item calls setUrlParams', () => { + const filter = stateFilterData.filters[Object.keys(stateFilterData.filters)[0]].value; + firstDropDownItem().vm.$emit('click'); + + expect(urlUtils.setUrlParams).toHaveBeenCalledWith({ + [stateFilterData.filterParam]: filter, + }); + }); + + it('clicking a dropdown item calls visitUrl', () => { + firstDropDownItem().vm.$emit('click'); + + expect(urlUtils.visitUrl).toHaveBeenCalled(); + }); + }); + }); + + describe('ConfidentialFilter', () => { + describe('template', () => { + describe.each` + scope | showDropdown + ${'issues'} | ${true} + ${'merge_requests'} | ${false} + ${'projects'} | ${false} + ${'milestones'} | ${false} + ${'users'} | ${false} + ${'notes'} | ${false} + ${'wiki_blobs'} | ${false} + ${'blobs'} | ${false} + `(`dropdown`, ({ scope, showDropdown }) => { + beforeEach(() => { + createStore({ query: { ...MOCK_QUERY, scope } }); + createComponent({ filterData: confidentialFilterData }); + }); + + it(`does${showDropdown ? '' : ' not'} render when scope is ${scope}`, () => { + expect(findGlDropdown().exists()).toBe(showDropdown); + }); + }); + + describe.each` + initialFilter | label + ${confidentialFilterData.filters.ANY.value} | ${`Any ${confidentialFilterData.header}`} + ${confidentialFilterData.filters.CONFIDENTIAL.value} | ${confidentialFilterData.filters.CONFIDENTIAL.label} + ${confidentialFilterData.filters.NOT_CONFIDENTIAL.value} | ${confidentialFilterData.filters.NOT_CONFIDENTIAL.label} + `(`filter text`, ({ initialFilter, label }) => { + describe(`when initialFilter is ${initialFilter}`, () => { + beforeEach(() => { + createStore({ + query: { ...MOCK_QUERY, [confidentialFilterData.filterParam]: initialFilter }, + }); + createComponent({ filterData: confidentialFilterData }); + }); + + it(`sets dropdown label to ${label}`, () => { + expect(findGlDropdown().attributes('text')).toBe(label); + }); + }); + }); + }); + }); + + describe('Filter options', () => { + beforeEach(() => { + createStore(); + createComponent({ filterData: confidentialFilterData }); + }); + + it('renders a dropdown item for each filterOption', () => { + expect(findDropdownItemsText()).toStrictEqual( + confidentialFilterData.filterByScope[confidentialFilterData.scopes.ISSUES].map(v => { + return v.label; + }), + ); + }); + + it('clicking a dropdown item calls setUrlParams', () => { + const filter = + confidentialFilterData.filters[Object.keys(confidentialFilterData.filters)[0]].value; + firstDropDownItem().vm.$emit('click'); + + expect(urlUtils.setUrlParams).toHaveBeenCalledWith({ + [confidentialFilterData.filterParam]: filter, + }); + }); + + it('clicking a dropdown item calls visitUrl', () => { + firstDropDownItem().vm.$emit('click'); + + expect(urlUtils.visitUrl).toHaveBeenCalled(); + }); + }); +}); diff --git a/spec/frontend/search/dropdown_filter/mock_data.js b/spec/frontend/search/dropdown_filter/mock_data.js new file mode 100644 index 00000000000..f11ab3d9951 --- /dev/null +++ b/spec/frontend/search/dropdown_filter/mock_data.js @@ -0,0 +1,5 @@ +export const MOCK_QUERY = { + scope: 'issues', + state: 'all', + confidential: null, +}; diff --git a/spec/lib/gitlab/kubernetes/kube_client_spec.rb b/spec/lib/gitlab/kubernetes/kube_client_spec.rb index 90c11f29855..b161832c018 100644 --- a/spec/lib/gitlab/kubernetes/kube_client_spec.rb +++ b/spec/lib/gitlab/kubernetes/kube_client_spec.rb @@ -9,7 +9,7 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do let(:api_url) { 'https://kubernetes.example.com/prefix' } let(:kubeclient_options) { { auth_options: { bearer_token: 'xyz' } } } - let(:client) { described_class.new(api_url, kubeclient_options) } + let(:client) { described_class.new(api_url, **kubeclient_options) } before do stub_kubeclient_discover(api_url) @@ -133,7 +133,7 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do end it 'falls back to default options, but allows overriding' do - client = Gitlab::Kubernetes::KubeClient.new(api_url, {}) + client = described_class.new(api_url) defaults = Gitlab::Kubernetes::KubeClient::DEFAULT_KUBECLIENT_OPTIONS expect(client.kubeclient_options[:timeouts]).to eq(defaults[:timeouts]) diff --git a/spec/lib/gitlab/sidekiq_cluster_spec.rb b/spec/lib/gitlab/sidekiq_cluster_spec.rb index 5dd913aebb0..5517abe1010 100644 --- a/spec/lib/gitlab/sidekiq_cluster_spec.rb +++ b/spec/lib/gitlab/sidekiq_cluster_spec.rb @@ -99,7 +99,7 @@ RSpec.describe Gitlab::SidekiqCluster do allow(Process).to receive(:spawn).and_return(1) expect(described_class).to receive(:wait_async).with(1) - expect(described_class.start_sidekiq(%w(foo), options)).to eq(1) + expect(described_class.start_sidekiq(%w(foo), **options)).to eq(1) end it 'handles duplicate queue names' do @@ -109,7 +109,7 @@ RSpec.describe Gitlab::SidekiqCluster do .and_return(1) expect(described_class).to receive(:wait_async).with(1) - expect(described_class.start_sidekiq(%w(foo foo bar baz), options)).to eq(1) + expect(described_class.start_sidekiq(%w(foo foo bar baz), **options)).to eq(1) end it 'runs the sidekiq process in a new process group' do @@ -119,7 +119,7 @@ RSpec.describe Gitlab::SidekiqCluster do .and_return(1) allow(described_class).to receive(:wait_async) - expect(described_class.start_sidekiq(%w(foo bar baz), options)).to eq(1) + expect(described_class.start_sidekiq(%w(foo bar baz), **options)).to eq(1) end end diff --git a/spec/models/concerns/each_batch_spec.rb b/spec/models/concerns/each_batch_spec.rb index 3c93c8a7a79..8b70753633c 100644 --- a/spec/models/concerns/each_batch_spec.rb +++ b/spec/models/concerns/each_batch_spec.rb @@ -18,13 +18,13 @@ RSpec.describe EachBatch do shared_examples 'each_batch handling' do |kwargs| it 'yields an ActiveRecord::Relation when a block is given' do - model.each_batch(kwargs) do |relation| + model.each_batch(**kwargs) do |relation| expect(relation).to be_a_kind_of(ActiveRecord::Relation) end end it 'yields a batch index as the second argument' do - model.each_batch(kwargs) do |_, index| + model.each_batch(**kwargs) do |_, index| expect(index).to eq(1) end end @@ -32,7 +32,7 @@ RSpec.describe EachBatch do it 'accepts a custom batch size' do amount = 0 - model.each_batch(kwargs.merge({ of: 1 })) { amount += 1 } + model.each_batch(**kwargs.merge({ of: 1 })) { amount += 1 } expect(amount).to eq(5) end diff --git a/spec/models/design_management/design_at_version_spec.rb b/spec/models/design_management/design_at_version_spec.rb index 3c1ff45c53f..220de80a52a 100644 --- a/spec/models/design_management/design_at_version_spec.rb +++ b/spec/models/design_management/design_at_version_spec.rb @@ -274,29 +274,6 @@ RSpec.describe DesignManagement::DesignAtVersion do build(:design_at_version, design: design, version: version).id end - describe '.instantiate' do - context 'when attrs are valid' do - subject do - described_class.instantiate(design: design, version: version) - end - - it { is_expected.to be_a(described_class).and(be_valid) } - end - - context 'when attrs are invalid' do - subject do - described_class.instantiate( - design: create(:design), - version: create(:design_version) - ) - end - - it 'raises a validation error' do - expect { subject }.to raise_error(ActiveModel::ValidationError) - end - end - end - describe '.lazy_find' do let!(:version_a) do create(:design_version, designs: create_list(:design, 3, issue: issue)) diff --git a/spec/requests/api/graphql/mutations/boards/create_spec.rb b/spec/requests/api/graphql/mutations/boards/create_spec.rb new file mode 100644 index 00000000000..c5f981262ea --- /dev/null +++ b/spec/requests/api/graphql/mutations/boards/create_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Mutations::Boards::Create do + let_it_be(:parent) { create(:project) } + let(:project_path) { parent.full_path } + let(:params) do + { + project_path: project_path, + name: name + } + end + + it_behaves_like 'boards create mutation' +end diff --git a/spec/support/shared_examples/graphql/mutations/boards_create_shared_examples.rb b/spec/support/shared_examples/graphql/mutations/boards_create_shared_examples.rb new file mode 100644 index 00000000000..ec64519cd9c --- /dev/null +++ b/spec/support/shared_examples/graphql/mutations/boards_create_shared_examples.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'boards create mutation' do + include GraphqlHelpers + + let_it_be(:current_user, reload: true) { create(:user) } + let(:name) { 'board name' } + let(:mutation) { graphql_mutation(:create_board, params) } + + subject { post_graphql_mutation(mutation, current_user: current_user) } + + def mutation_response + graphql_mutation_response(:create_board) + end + + context 'when the user does not have permission' do + it_behaves_like 'a mutation that returns a top-level access error' + + it 'does not create the board' do + expect { subject }.not_to change { Board.count } + end + end + + context 'when the user has permission' do + before do + parent.add_maintainer(current_user) + end + + context 'when the parent (project_path or group_path) param is given' do + context 'when everything is ok' do + it 'creates the board' do + expect { subject }.to change { Board.count }.from(0).to(1) + end + + it 'returns the created board' do + post_graphql_mutation(mutation, current_user: current_user) + + expect(mutation_response).to have_key('board') + expect(mutation_response['board']['name']).to eq(name) + end + end + + context 'when the Boards::CreateService returns an error response' do + before do + allow_next_instance_of(Boards::CreateService) do |service| + allow(service).to receive(:execute).and_return(ServiceResponse.error(message: 'There was an error.')) + end + end + + it 'does not create a board' do + expect { subject }.not_to change { Board.count } + end + + it 'returns an error' do + post_graphql_mutation(mutation, current_user: current_user) + + expect(mutation_response).to have_key('board') + expect(mutation_response['board']).to be_nil + expect(mutation_response['errors'].first).to eq('There was an error.') + end + end + end + + context 'when neither project_path nor group_path param is given' do + let(:params) { { name: name } } + + it_behaves_like 'a mutation that returns top-level errors', + errors: ['group_path or project_path arguments are required'] + + it 'does not create the board' do + expect { subject }.not_to change { Board.count } + end + end + end +end |
