diff options
23 files changed, 689 insertions, 425 deletions
diff --git a/app/models/concerns/each_batch.rb b/app/models/concerns/each_batch.rb index 443e1ab53b4..dbc0887dc97 100644 --- a/app/models/concerns/each_batch.rb +++ b/app/models/concerns/each_batch.rb @@ -2,6 +2,7 @@ module EachBatch extend ActiveSupport::Concern + include LooseIndexScan class_methods do # Iterates over the rows in a relation in batches, similar to Rails' @@ -100,5 +101,65 @@ module EachBatch break unless stop end end + + # Iterates over the rows in a relation in batches by skipping duplicated values in the column. + # Example: counting the number of distinct authors in `issues` + # + # - Table size: 100_000 + # - Column: author_id + # - Distinct author_ids in the table: 1000 + # + # The query will read maximum 1000 rows if we have index coverage on user_id. + # + # > count = 0 + # > Issue.distinct_each_batch(column: 'author_id', of: 1000) { |r| count += r.count(:author_id) } + def distinct_each_batch(column:, order: :asc, of: 1000) + start = except(:select) + .select(column) + .reorder(column => order) + + start = start.take + + return unless start + + start_id = start[column] + arel_table = self.arel_table + arel_column = arel_table[column.to_s] + + 1.step do |index| + stop = loose_index_scan(column: column, order: order) do |cte_query, inner_query| + if order == :asc + [cte_query.where(arel_column.gteq(start_id)), inner_query] + else + [cte_query.where(arel_column.lteq(start_id)), inner_query] + end + end.offset(of).take + + if stop + stop_id = stop[column] + + relation = loose_index_scan(column: column, order: order) do |cte_query, inner_query| + if order == :asc + [cte_query.where(arel_column.gteq(start_id)), inner_query.where(arel_column.lt(stop_id))] + else + [cte_query.where(arel_column.lteq(start_id)), inner_query.where(arel_column.gt(stop_id))] + end + end + start_id = stop_id + else + relation = loose_index_scan(column: column, order: order) do |cte_query, inner_query| + if order == :asc + [cte_query.where(arel_column.gteq(start_id)), inner_query] + else + [cte_query.where(arel_column.lteq(start_id)), inner_query] + end + end + end + + unscoped { yield relation, index } + + break unless stop + end + end end end diff --git a/app/models/concerns/loose_index_scan.rb b/app/models/concerns/loose_index_scan.rb new file mode 100644 index 00000000000..420ac4d0f62 --- /dev/null +++ b/app/models/concerns/loose_index_scan.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module LooseIndexScan + extend ActiveSupport::Concern + + class_methods do + # Builds a recursive query to read distinct values from a column. + # + # Example 1: collect all distinct author ids for the `issues` table + # + # Bad: The DB reads all issues, sorts and dedups them in memory + # + # > Issue.select(:author_id).distinct.map(&:author_id) + # + # Good: Use loose index scan (skip index scan) + # + # > Issue.loose_index_scan(column: :author_id).map(&:author_id) + # + # Example 2: List of users for the DONE todos selector. Select all users who created a todo. + # + # Bad: Loads all DONE todos for the given user and extracts the author_ids + # + # > User.where(id: Todo.where(user_id: 4156052).done.select(:author_id)) + # + # Good: Loads distinct author_ids from todos and then loads users + # + # > distinct_authors = Todo.where(user_id: 4156052).done.loose_index_scan(column: :author_id).select(:author_id) + # > User.where(id: distinct_authors) + def loose_index_scan(column:, order: :asc) + arel_table = self.arel_table + arel_column = arel_table[column.to_s] + + cte = Gitlab::SQL::RecursiveCTE.new(:loose_index_scan_cte, union_args: { remove_order: false }) + + cte_query = except(:select) + .select(column) + .order(column => order) + .limit(1) + + inner_query = except(:select) + + cte_query, inner_query = yield([cte_query, inner_query]) if block_given? + cte << cte_query + + inner_query = if order == :asc + inner_query.where(arel_column.gt(cte.table[column.to_s])) + else + inner_query.where(arel_column.lt(cte.table[column.to_s])) + end + + inner_query = inner_query.order(column => order) + .select(column) + .limit(1) + + cte << cte.table + .project(Arel::Nodes::Grouping.new(Arel.sql(inner_query.to_sql)).as(column.to_s)) + + unscoped do + with + .recursive(cte.to_arel) + .from(cte.alias_to(arel_table)) + .where(arel_column.not_eq(nil)) # filtering out the last NULL value + end + end + end +end diff --git a/app/models/key.rb b/app/models/key.rb index 5268ce2e040..9f96a93cea1 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -28,7 +28,7 @@ class Key < ApplicationRecord validate :key_meets_restrictions validate :expiration, on: :create - validate :banned_key, if: :should_check_for_banned_key? + validate :banned_key, if: :key_changed? delegate :name, :email, to: :user, prefix: true @@ -143,12 +143,6 @@ class Key < ApplicationRecord end end - def should_check_for_banned_key? - return false unless user - - key_changed? && Feature.enabled?(:ssh_banned_key, user) - end - def banned_key return unless public_key.banned? diff --git a/config/feature_flags/development/ssh_banned_key.yml b/config/feature_flags/development/ssh_banned_key.yml deleted file mode 100644 index e628e440176..00000000000 --- a/config/feature_flags/development/ssh_banned_key.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: ssh_banned_key -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87541 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/363410 -milestone: '15.1' -type: development -group: 'group::authentication and authorization' -default_enabled: true diff --git a/config/webpack.config.js b/config/webpack.config.js index aff906353f6..799e3fe6ead 100644 --- a/config/webpack.config.js +++ b/config/webpack.config.js @@ -158,7 +158,6 @@ const alias = { images: path.join(ROOT_PATH, 'app/assets/images'), vendor: path.join(ROOT_PATH, 'vendor/assets/javascripts'), jquery$: 'jquery/dist/jquery.slim.js', - jest: path.join(ROOT_PATH, 'spec/frontend'), shared_queries: path.join(ROOT_PATH, 'app/graphql/queries'), // the following resolves files which are different between CE and EE @@ -175,6 +174,15 @@ const alias = { ROOT_PATH, 'app/assets/javascripts/lib/utils/icons_path.js', ), + + // test-environment-only aliases duplicated from Jest config + 'spec/test_constants$': path.join(ROOT_PATH, 'spec/frontend/__helpers__/test_constants'), + ee_else_ce_jest: path.join(ROOT_PATH, 'spec/frontend'), + helpers: path.join(ROOT_PATH, 'spec/frontend/__helpers__'), + jest: path.join(ROOT_PATH, 'spec/frontend'), + test_fixtures: path.join(ROOT_PATH, 'tmp/tests/frontend/fixtures'), + test_fixtures_static: path.join(ROOT_PATH, 'spec/frontend/fixtures/static'), + test_helpers: path.join(ROOT_PATH, 'spec/frontend_integration/test_helpers'), }; if (IS_EE) { @@ -184,10 +192,14 @@ if (IS_EE) { ee_empty_states: path.join(ROOT_PATH, 'ee/app/views/shared/empty_states'), ee_icons: path.join(ROOT_PATH, 'ee/app/views/shared/icons'), ee_images: path.join(ROOT_PATH, 'ee/app/assets/images'), - ee_jest: path.join(ROOT_PATH, 'ee/spec/frontend'), ee_else_ce: path.join(ROOT_PATH, 'ee/app/assets/javascripts'), jh_else_ee: path.join(ROOT_PATH, 'ee/app/assets/javascripts'), any_else_ce: path.join(ROOT_PATH, 'ee/app/assets/javascripts'), + + // test-environment-only aliases duplicated from Jest config + ee_else_ce_jest: path.join(ROOT_PATH, 'ee/spec/frontend'), + ee_jest: path.join(ROOT_PATH, 'ee/spec/frontend'), + test_fixtures: path.join(ROOT_PATH, 'tmp/tests/frontend/fixtures-ee'), }); } @@ -198,21 +210,13 @@ if (IS_JH) { jh_empty_states: path.join(ROOT_PATH, 'jh/app/views/shared/empty_states'), jh_icons: path.join(ROOT_PATH, 'jh/app/views/shared/icons'), jh_images: path.join(ROOT_PATH, 'jh/app/assets/images'), - jh_jest: path.join(ROOT_PATH, 'jh/spec/frontend'), // jh path alias https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74305#note_732793956 jh_else_ce: path.join(ROOT_PATH, 'jh/app/assets/javascripts'), jh_else_ee: path.join(ROOT_PATH, 'jh/app/assets/javascripts'), any_else_ce: path.join(ROOT_PATH, 'jh/app/assets/javascripts'), - }); -} -if (!IS_PRODUCTION) { - const fixtureDir = IS_EE ? 'fixtures-ee' : 'fixtures'; - - Object.assign(alias, { - test_fixtures: path.join(ROOT_PATH, `tmp/tests/frontend/${fixtureDir}`), - test_fixtures_static: path.join(ROOT_PATH, 'spec/frontend/fixtures/static'), - test_helpers: path.join(ROOT_PATH, 'spec/frontend_integration/test_helpers'), + // test-environment-only aliases duplicated from Jest config + jh_jest: path.join(ROOT_PATH, 'jh/spec/frontend'), }); } diff --git a/doc/administration/troubleshooting/postgresql.md b/doc/administration/troubleshooting/postgresql.md index cdbf786bdb2..61b661d45f8 100644 --- a/doc/administration/troubleshooting/postgresql.md +++ b/doc/administration/troubleshooting/postgresql.md @@ -54,7 +54,7 @@ This section is for links to information elsewhere in the GitLab documentation. - Including [troubleshooting](../postgresql/replication_and_failover.md#troubleshooting) `gitlab-ctl patroni check-leader` and PgBouncer errors. -- [Developer database documentation](../../development/index.md#database-guides), +- [Developer database documentation](../../development/feature_development.md#database-guides), some of which is absolutely not for production use. Including: - Understanding EXPLAIN plans. diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 40daf154a5f..e953aafabcf 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -288,6 +288,9 @@ regular expression displayed in the settings. Available in GitLab 14.10 and earl The regular expression you need is in the **Test coverage parsing** field. +If you need to retrieve the project coverage setting from many projects, you can +[use the API to programmatically retrieve the setting](https://gitlab.com/gitlab-org/gitlab/-/issues/17633#note_945941397). + <!-- end_remove --> ### Test coverage examples diff --git a/doc/development/code_review.md b/doc/development/code_review.md index d864f369945..8fcbd09c8da 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -129,7 +129,7 @@ with [domain expertise](#domain-experts). 1. If your merge request includes documentation changes, it must be **approved by a [Technical writer](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments)**, based on assignments in the appropriate [DevOps stage group](https://about.gitlab.com/handbook/product/categories/#devops-stages). -1. If your merge request includes changes to development guidelines, follow the [review process](index.md#development-guidelines-review) and get the approvals accordingly. +1. If your merge request includes changes to development guidelines, follow the [review process](development_processes.md#development-guidelines-review) and get the approvals accordingly. 1. If your merge request includes end-to-end **and** non-end-to-end changes (*4*), it must be **approved by a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors)**. 1. If your merge request only includes end-to-end changes (*4*) **or** if the MR author is a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors), it must be **approved by a [Quality maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_qa)** diff --git a/doc/development/dangerbot.md b/doc/development/dangerbot.md index 34b61858995..d2b231ebc7c 100644 --- a/doc/development/dangerbot.md +++ b/doc/development/dangerbot.md @@ -58,7 +58,7 @@ itself, increasing visibility. ## Development guidelines -Danger code is Ruby code, so all our [usual backend guidelines](index.md#backend-guides) +Danger code is Ruby code, so all our [usual backend guidelines](feature_development.md#backend-guides) continue to apply. However, there are a few things that deserve special emphasis. ### When to use Danger diff --git a/doc/development/development_processes.md b/doc/development/development_processes.md new file mode 100644 index 00000000000..e199aedd3f5 --- /dev/null +++ b/doc/development/development_processes.md @@ -0,0 +1,124 @@ +--- +stage: none +group: Development +info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" +--- + +# Development processes + +Consult these topics for information on development processes for contributing to GitLab. + +## Processes + +Must-reads: + +- [Guide on adapting existing and introducing new components](architecture.md#adapting-existing-and-introducing-new-components) +- [Code review guidelines](code_review.md) for reviewing code and having code + reviewed +- [Database review guidelines](database_review.md) for reviewing + database-related changes and complex SQL queries, and having them reviewed +- [Secure coding guidelines](secure_coding_guidelines.md) +- [Pipelines for the GitLab project](pipelines.md) + +Complementary reads: + +- [GitLab core team & GitLab Inc. contribution process](https://gitlab.com/gitlab-org/gitlab/-/blob/master/PROCESS.md) +- [Security process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/developer.md#security-releases-critical-non-critical-as-a-developer) +- [Patch release process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/patch/process.md#process-for-developers) +- [Guidelines for implementing Enterprise Edition features](ee_features.md) +- [Adding a new service component to GitLab](adding_service_component.md) +- [Guidelines for changelogs](changelog.md) +- [Dependencies](dependencies.md) +- [Danger bot](dangerbot.md) +- [Requesting access to ChatOps on GitLab.com](chatops_on_gitlabcom.md#requesting-access) (for GitLab team members) + +### Development guidelines review + +When you submit a change to the GitLab development guidelines, who +you ask for reviews depends on the level of change. + +#### Wording, style, or link changes + +Not all changes require extensive review. For example, MRs that don't change the +content's meaning or function can be reviewed, approved, and merged by any +maintainer or Technical Writer. These can include: + +- Typo fixes. +- Clarifying links, such as to external programming language documentation. +- Changes to comply with the [Documentation Style Guide](documentation/index.md) + that don't change the intent of the documentation page. + +#### Specific changes + +If the MR proposes changes that are limited to a particular stage, group, or team, +request a review and approval from an experienced GitLab Team Member in that +group. For example, if you're documenting a new internal API used exclusively by +a given group, request an engineering review from one of the group's members. + +After the engineering review is complete, assign the MR to the +[Technical Writer associated with the stage and group](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments) +in the modified documentation page's metadata. +If the page is not assigned to a specific group, follow the +[Technical Writing review process for development guidelines](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines). + +#### Broader changes + +Some changes affect more than one group. For example: + +- Changes to [code review guidelines](code_review.md). +- Changes to [commit message guidelines](contributing/merge_request_workflow.md#commit-messages-guidelines). +- Changes to guidelines in [feature flags in development of GitLab](feature_flags/). +- Changes to [feature flags documentation guidelines](documentation/feature_flags.md). + +In these cases, use the following workflow: + +1. Request a peer review from a member of your team. +1. Request a review and approval of an Engineering Manager (EM) + or Staff Engineer who's responsible for the area in question: + + - [Frontend](https://about.gitlab.com/handbook/engineering/frontend/) + - [Backend](https://about.gitlab.com/handbook/engineering/) + - [Database](https://about.gitlab.com/handbook/engineering/development/database/) + - [User Experience (UX)](https://about.gitlab.com/handbook/engineering/ux/) + - [Security](https://about.gitlab.com/handbook/engineering/security/) + - [Quality](https://about.gitlab.com/handbook/engineering/quality/) + - [Engineering Productivity](https://about.gitlab.com/handbook/engineering/quality/engineering-productivity/) + - [Infrastructure](https://about.gitlab.com/handbook/engineering/infrastructure/) + - [Technical Writing](https://about.gitlab.com/handbook/engineering/ux/technical-writing/) + + You can skip this step for MRs authored by EMs or Staff Engineers responsible + for their area. + + If there are several affected groups, you may need approvals at the + EM/Staff Engineer level from each affected area. + +1. After completing the reviews, consult with the EM/Staff Engineer + author / approver of the MR. + + If this is a significant change across multiple areas, request final review + and approval from the VP of Development, the DRI for Development Guidelines, + @clefelhocz1. + +1. After all approvals are complete, assign the MR to the + [Technical Writer associated with the stage and group](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments) + in the modified documentation page's metadata. + If the page is not assigned to a specific group, follow the + [Technical Writing review process for development guidelines](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines). + The Technical Writer may ask for additional approvals as previously suggested before merging the MR. + +### Reviewer values + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57293) in GitLab 14.1. + +As a reviewer or as a reviewee, make sure to familiarize yourself with +the [reviewer values](https://about.gitlab.com/handbook/engineering/workflow/reviewer-values/) we strive for at GitLab. + +## Language-specific guides + +### Go guides + +- [Go Guidelines](go_guide/index.md) + +### Shell Scripting guides + +- [Shell scripting standards and style guidelines](shell_scripting_guide/index.md) diff --git a/doc/development/feature_development.md b/doc/development/feature_development.md new file mode 100644 index 00000000000..539746f3e4e --- /dev/null +++ b/doc/development/feature_development.md @@ -0,0 +1,198 @@ +--- +stage: none +group: Development +info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" +--- + +# Feature development + +Consult these topics for information on contributing to specific GitLab features. + +## UX and Frontend guides + +- [GitLab Design System](https://design.gitlab.com/), for building GitLab with + existing CSS styles and elements +- [Frontend guidelines](fe_guide/index.md) +- [Emoji guide](fe_guide/emojis.md) + +## Backend guides + +### General + +- [Directory structure](directory_structure.md) +- [GitLab EventStore](event_store.md) to publish/subscribe to domain events +- [GitLab utilities](utilities.md) +- [Newlines style guide](newlines_styleguide.md) +- [Logging](logging.md) +- [Dealing with email/mailers](emails.md) +- [Kubernetes integration guidelines](kubernetes.md) +- [Permissions](permissions.md) +- [Code comments](code_comments.md) +- [Windows Development on GCP](windows.md) +- [FIPS compliance](fips_compliance.md) +- [`Gemfile` guidelines](gemfile.md) +- [Ruby upgrade guidelines](ruby_upgrade.md) + +### Things to be aware of + +- [Gotchas](gotchas.md) to avoid +- [Avoid modules with instance variables](module_with_instance_variables.md), if + possible +- [Guidelines for reusing abstractions](reusing_abstractions.md) +- [Ruby 3 gotchas](ruby3_gotchas.md) + +### Rails Framework related + +- [Routing](routing.md) +- [Rails initializers](rails_initializers.md) +- [Mass Inserting Models](mass_insert.md) +- [Issuable-like Rails models](issuable-like-models.md) +- [Issue types vs first-class types](issue_types.md) +- [DeclarativePolicy framework](policies.md) +- [Rails update guidelines](rails_update.md) + +### Debugging + +- [Pry debugging](pry_debugging.md) +- [Sidekiq debugging](../administration/troubleshooting/sidekiq.md) + +### Git specifics + +- [How Git object deduplication works in GitLab](git_object_deduplication.md) +- [Git LFS](lfs.md) + +### API + +- [API style guide](api_styleguide.md) for contributing to the API +- [GraphQL API style guide](api_graphql_styleguide.md) for contributing to the + [GraphQL API](../api/graphql/index.md) + +### GitLab components and features + +- [Developing against interacting components or features](interacting_components.md) +- [Manage feature flags](feature_flags/index.md) +- [Licensed feature availability](licensed_feature_availability.md) +- [Accessing session data](session.md) +- [How to dump production data to staging](db_dump.md) +- [Geo development](geo.md) +- [Redis guidelines](redis.md) + - [Adding a new Redis instance](redis/new_redis_instance.md) +- [Sidekiq guidelines](sidekiq/index.md) for working with Sidekiq workers +- [Working with Gitaly](gitaly.md) +- [Elasticsearch integration docs](elasticsearch.md) +- [Working with merge request diffs](diffs.md) +- [Approval Rules](approval_rules.md) +- [Repository mirroring](repository_mirroring.md) +- [Uploads development guide](uploads/index.md) +- [Auto DevOps development guide](auto_devops.md) +- [Renaming features](renaming_features.md) +- [Code Intelligence](code_intelligence/index.md) +- [Feature categorization](feature_categorization/index.md) +- [Wikis development guide](wikis.md) +- [Image scaling guide](image_scaling.md) +- [Cascading Settings](cascading_settings.md) +- [Shell commands](shell_commands.md) in the GitLab codebase +- [Value Stream Analytics development guide](value_stream_analytics.md) +- [Application limits](application_limits.md) + +### Import and Export + +- [Working with the GitHub importer](github_importer.md) +- [Import/Export development documentation](import_export.md) +- [Test Import Project](import_project.md) +- [Group migration](bulk_import.md) +- [Export to CSV](export_csv.md) + +## Performance guides + +- [Performance guidelines](performance.md) for writing code, benchmarks, and + certain patterns to avoid. +- [Caching guidelines](caching.md) for using caching in Rails under a GitLab environment. +- [Merge request performance guidelines](merge_request_performance_guidelines.md) + for ensuring merge requests do not negatively impact GitLab performance +- [Profiling](profiling.md) a URL or tracking down N+1 queries using Bullet. +- [Cached queries guidelines](cached_queries.md), for tracking down N+1 queries + masked by query caching, memory profiling and why should we avoid cached + queries. + +## Database guides + +See [database guidelines](database/index.md). + +## Integration guides + +- [Integrations development guide](integrations/index.md) +- [Jira Connect app](integrations/jira_connect.md) +- [Security Scanners](integrations/secure.md) +- [Secure Partner Integration](integrations/secure_partner_integration.md) +- [How to run Jenkins in development environment](integrations/jenkins.md) +- [How to run local `Codesandbox` integration for Web IDE Live Preview](integrations/codesandbox.md) + +## Testing guides + +- [Testing standards and style guidelines](testing_guide/index.md) +- [Frontend testing standards and style guidelines](testing_guide/frontend_testing.md) + +## Refactoring guides + +- [Refactoring guidelines](refactoring_guide/index.md) + +## Deprecation guides + +- [Deprecation guidelines](deprecation_guidelines/index.md) + +## Documentation guides + +- [Writing documentation](documentation/index.md) +- [Documentation style guide](documentation/styleguide/index.md) +- [Markdown](../user/markdown.md) + +## Internationalization (i18n) guides + +- [Introduction](i18n/index.md) +- [Externalization](i18n/externalization.md) +- [Translation](i18n/translation.md) + +## Product Intelligence guides + +- [Product Intelligence guide](https://about.gitlab.com/handbook/product/product-intelligence-guide/) +- [Service Ping guide](service_ping/index.md) +- [Snowplow guide](snowplow/index.md) + +## Experiment guide + +- [Introduction](experiment_guide/index.md) + +## Build guides + +- [Building a package for testing purposes](build_test_package.md) + +## Compliance + +- [Licensing](licensing.md) for ensuring license compliance + +## Domain-specific guides + +- [CI/CD development documentation](cicd/index.md) +- [AppSec development documentation](appsec/index.md) + +## Technical Reference by Group + +- [Create: Source Code BE](backend/create_source_code_be/index.md) + +## Other development guides + +- [Defining relations between files using projections](projections.md) +- [Reference processing](reference_processing.md) +- [Compatibility with multiple versions of the application running at the same time](multi_version_compatibility.md) +- [Features inside `.gitlab/`](features_inside_dot_gitlab.md) +- [Dashboards for stage groups](stage_group_dashboards.md) +- [Preventing transient bugs](transient/prevention-patterns.md) +- [GitLab Application SLIs](application_slis/index.md) +- [Spam protection and CAPTCHA development guide](spam_protection_and_captcha/index.md) + +## Other GitLab Development Kit (GDK) guides + +- [Run full Auto DevOps cycle in a GDK instance](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/auto_devops.md) +- [Using GitLab Runner with the GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/runner.md) +- [Using the Web IDE terminal with the GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/web_ide_terminal_gdk_setup.md) diff --git a/doc/development/index.md b/doc/development/index.md index 3d5ec24d3e2..1b897db5097 100644 --- a/doc/development/index.md +++ b/doc/development/index.md @@ -46,307 +46,3 @@ GitLab instance, see the [Administrator documentation](../administration/index.m - [Implement design & UI elements](contributing/design.md) - [GitLab Architecture Overview](architecture.md) - [Rake tasks](rake_tasks.md) for development - -## Processes - -**Must-reads:** - -- [Guide on adapting existing and introducing new components](architecture.md#adapting-existing-and-introducing-new-components) -- [Code review guidelines](code_review.md) for reviewing code and having code - reviewed -- [Database review guidelines](database_review.md) for reviewing - database-related changes and complex SQL queries, and having them reviewed -- [Secure coding guidelines](secure_coding_guidelines.md) -- [Pipelines for the GitLab project](pipelines.md) - -Complementary reads: - -- [GitLab core team & GitLab Inc. contribution process](https://gitlab.com/gitlab-org/gitlab/-/blob/master/PROCESS.md) -- [Security process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/developer.md#security-releases-critical-non-critical-as-a-developer) -- [Patch release process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/patch/process.md#process-for-developers) -- [Guidelines for implementing Enterprise Edition features](ee_features.md) -- [Adding a new service component to GitLab](adding_service_component.md) -- [Guidelines for changelogs](changelog.md) -- [Dependencies](dependencies.md) -- [Danger bot](dangerbot.md) -- [Requesting access to ChatOps on GitLab.com](chatops_on_gitlabcom.md#requesting-access) (for GitLab team members) - -### Development guidelines review - -When you submit a change to the GitLab development guidelines, who -you ask for reviews depends on the level of change. - -#### Wording, style, or link changes - -Not all changes require extensive review. For example, MRs that don't change the -content's meaning or function can be reviewed, approved, and merged by any -maintainer or Technical Writer. These can include: - -- Typo fixes. -- Clarifying links, such as to external programming language documentation. -- Changes to comply with the [Documentation Style Guide](documentation/index.md) - that don't change the intent of the documentation page. - -#### Specific changes - -If the MR proposes changes that are limited to a particular stage, group, or team, -request a review and approval from an experienced GitLab Team Member in that -group. For example, if you're documenting a new internal API used exclusively by -a given group, request an engineering review from one of the group's members. - -After the engineering review is complete, assign the MR to the -[Technical Writer associated with the stage and group](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments) -in the modified documentation page's metadata. -If the page is not assigned to a specific group, follow the -[Technical Writing review process for development guidelines](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines). - -#### Broader changes - -Some changes affect more than one group. For example: - -- Changes to [code review guidelines](code_review.md). -- Changes to [commit message guidelines](contributing/merge_request_workflow.md#commit-messages-guidelines). -- Changes to guidelines in [feature flags in development of GitLab](feature_flags/). -- Changes to [feature flags documentation guidelines](documentation/feature_flags.md). - -In these cases, use the following workflow: - -1. Request a peer review from a member of your team. -1. Request a review and approval of an Engineering Manager (EM) - or Staff Engineer who's responsible for the area in question: - - - [Frontend](https://about.gitlab.com/handbook/engineering/frontend/) - - [Backend](https://about.gitlab.com/handbook/engineering/) - - [Database](https://about.gitlab.com/handbook/engineering/development/database/) - - [User Experience (UX)](https://about.gitlab.com/handbook/engineering/ux/) - - [Security](https://about.gitlab.com/handbook/engineering/security/) - - [Quality](https://about.gitlab.com/handbook/engineering/quality/) - - [Engineering Productivity](https://about.gitlab.com/handbook/engineering/quality/engineering-productivity/) - - [Infrastructure](https://about.gitlab.com/handbook/engineering/infrastructure/) - - [Technical Writing](https://about.gitlab.com/handbook/engineering/ux/technical-writing/) - - You can skip this step for MRs authored by EMs or Staff Engineers responsible - for their area. - - If there are several affected groups, you may need approvals at the - EM/Staff Engineer level from each affected area. - -1. After completing the reviews, consult with the EM/Staff Engineer - author / approver of the MR. - - If this is a significant change across multiple areas, request final review - and approval from the VP of Development, the DRI for Development Guidelines, - @clefelhocz1. - -1. After all approvals are complete, assign the MR to the - [Technical Writer associated with the stage and group](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments) - in the modified documentation page's metadata. - If the page is not assigned to a specific group, follow the - [Technical Writing review process for development guidelines](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines). - The Technical Writer may ask for additional approvals as previously suggested before merging the MR. - -### Reviewer values - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57293) in GitLab 14.1. - -As a reviewer or as a reviewee, make sure to familiarize yourself with -the [reviewer values](https://about.gitlab.com/handbook/engineering/workflow/reviewer-values/) we strive for at GitLab. - -## UX and Frontend guides - -- [GitLab Design System](https://design.gitlab.com/), for building GitLab with - existing CSS styles and elements -- [Frontend guidelines](fe_guide/index.md) -- [Emoji guide](fe_guide/emojis.md) - -## Backend guides - -### General - -- [Directory structure](directory_structure.md) -- [GitLab EventStore](event_store.md) to publish/subscribe to domain events -- [GitLab utilities](utilities.md) -- [Newlines style guide](newlines_styleguide.md) -- [Logging](logging.md) -- [Dealing with email/mailers](emails.md) -- [Kubernetes integration guidelines](kubernetes.md) -- [Permissions](permissions.md) -- [Code comments](code_comments.md) -- [Windows Development on GCP](windows.md) -- [FIPS compliance](fips_compliance.md) -- [`Gemfile` guidelines](gemfile.md) -- [Ruby upgrade guidelines](ruby_upgrade.md) - -### Things to be aware of - -- [Gotchas](gotchas.md) to avoid -- [Avoid modules with instance variables](module_with_instance_variables.md), if - possible -- [Guidelines for reusing abstractions](reusing_abstractions.md) -- [Ruby 3 gotchas](ruby3_gotchas.md) - -### Rails Framework related - -- [Routing](routing.md) -- [Rails initializers](rails_initializers.md) -- [Mass Inserting Models](mass_insert.md) -- [Issuable-like Rails models](issuable-like-models.md) -- [Issue types vs first-class types](issue_types.md) -- [DeclarativePolicy framework](policies.md) -- [Rails update guidelines](rails_update.md) - -### Debugging - -- [Pry debugging](pry_debugging.md) -- [Sidekiq debugging](../administration/troubleshooting/sidekiq.md) - -### Git specifics - -- [How Git object deduplication works in GitLab](git_object_deduplication.md) -- [Git LFS](lfs.md) - -### API - -- [API style guide](api_styleguide.md) for contributing to the API -- [GraphQL API style guide](api_graphql_styleguide.md) for contributing to the - [GraphQL API](../api/graphql/index.md) - -### GitLab components and features - -- [Developing against interacting components or features](interacting_components.md) -- [Manage feature flags](feature_flags/index.md) -- [Licensed feature availability](licensed_feature_availability.md) -- [Accessing session data](session.md) -- [How to dump production data to staging](db_dump.md) -- [Geo development](geo.md) -- [Redis guidelines](redis.md) - - [Adding a new Redis instance](redis/new_redis_instance.md) -- [Sidekiq guidelines](sidekiq/index.md) for working with Sidekiq workers -- [Working with Gitaly](gitaly.md) -- [Elasticsearch integration docs](elasticsearch.md) -- [Working with merge request diffs](diffs.md) -- [Approval Rules](approval_rules.md) -- [Repository mirroring](repository_mirroring.md) -- [Uploads development guide](uploads/index.md) -- [Auto DevOps development guide](auto_devops.md) -- [Renaming features](renaming_features.md) -- [Code Intelligence](code_intelligence/index.md) -- [Feature categorization](feature_categorization/index.md) -- [Wikis development guide](wikis.md) -- [Image scaling guide](image_scaling.md) -- [Cascading Settings](cascading_settings.md) -- [Shell commands](shell_commands.md) in the GitLab codebase -- [Value Stream Analytics development guide](value_stream_analytics.md) -- [Application limits](application_limits.md) - -### Import/Export - -- [Working with the GitHub importer](github_importer.md) -- [Import/Export development documentation](import_export.md) -- [Test Import Project](import_project.md) -- [Group migration](bulk_import.md) -- [Export to CSV](export_csv.md) - -## Language-specific guides - -### Go guides - -- [Go Guidelines](go_guide/index.md) - -### Shell Scripting guides - -- [Shell scripting standards and style guidelines](shell_scripting_guide/index.md) - -## Performance guides - -- [Performance guidelines](performance.md) for writing code, benchmarks, and - certain patterns to avoid. -- [Caching guidelines](caching.md) for using caching in Rails under a GitLab environment. -- [Merge request performance guidelines](merge_request_performance_guidelines.md) - for ensuring merge requests do not negatively impact GitLab performance -- [Profiling](profiling.md) a URL or tracking down N+1 queries using Bullet. -- [Cached queries guidelines](cached_queries.md), for tracking down N+1 queries - masked by query caching, memory profiling and why should we avoid cached - queries. - -## Database guides - -See [database guidelines](database/index.md). - -## Integration guides - -- [Integrations development guide](integrations/index.md) -- [Jira Connect app](integrations/jira_connect.md) -- [Security Scanners](integrations/secure.md) -- [Secure Partner Integration](integrations/secure_partner_integration.md) -- [How to run Jenkins in development environment](integrations/jenkins.md) -- [How to run local `Codesandbox` integration for Web IDE Live Preview](integrations/codesandbox.md) - -## Testing guides - -- [Testing standards and style guidelines](testing_guide/index.md) -- [Frontend testing standards and style guidelines](testing_guide/frontend_testing.md) - -## Refactoring guides - -- [Refactoring guidelines](refactoring_guide/index.md) - -## Deprecation guides - -- [Deprecation guidelines](deprecation_guidelines/index.md) - -## Documentation guides - -- [Writing documentation](documentation/index.md) -- [Documentation style guide](documentation/styleguide/index.md) -- [Markdown](../user/markdown.md) - -## Internationalization (i18n) guides - -- [Introduction](i18n/index.md) -- [Externalization](i18n/externalization.md) -- [Translation](i18n/translation.md) - -## Product Intelligence guides - -- [Product Intelligence guide](https://about.gitlab.com/handbook/product/product-intelligence-guide/) -- [Service Ping guide](service_ping/index.md) -- [Snowplow guide](snowplow/index.md) - -## Experiment guide - -- [Introduction](experiment_guide/index.md) - -## Build guides - -- [Building a package for testing purposes](build_test_package.md) - -## Compliance - -- [Licensing](licensing.md) for ensuring license compliance - -## Domain-specific guides - -- [CI/CD development documentation](cicd/index.md) -- [AppSec development documentation](appsec/index.md) - -## Technical Reference by Group - -- [Create: Source Code BE](backend/create_source_code_be/index.md) - -## Other Development guides - -- [Defining relations between files using projections](projections.md) -- [Reference processing](reference_processing.md) -- [Compatibility with multiple versions of the application running at the same time](multi_version_compatibility.md) -- [Features inside `.gitlab/`](features_inside_dot_gitlab.md) -- [Dashboards for stage groups](stage_group_dashboards.md) -- [Preventing transient bugs](transient/prevention-patterns.md) -- [GitLab Application SLIs](application_slis/index.md) -- [Spam protection and CAPTCHA development guide](spam_protection_and_captcha/index.md) - -## Other GitLab Development Kit (GDK) guides - -- [Run full Auto DevOps cycle in a GDK instance](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/auto_devops.md) -- [Using GitLab Runner with the GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/runner.md) -- [Using the Web IDE terminal with the GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/web_ide_terminal_gdk_setup.md) diff --git a/doc/development/iterating_tables_in_batches.md b/doc/development/iterating_tables_in_batches.md index b4459b53efa..1159e3755e5 100644 --- a/doc/development/iterating_tables_in_batches.md +++ b/doc/development/iterating_tables_in_batches.md @@ -42,20 +42,20 @@ The API of this method is similar to `in_batches`, though it doesn't support all of the arguments that `in_batches` supports. You should always use `each_batch` _unless_ you have a specific need for `in_batches`. -## Avoid iterating over non-unique columns +## Iterating over non-unique columns -One should proceed with extra caution, and possibly avoid iterating over a column that can contain -duplicate values. When you iterate over an attribute that is not unique, even with the applied max -batch size, there is no guarantee that the resulting batches do not surpass it. The following -snippet demonstrates this situation when one attempt to select `Ci::Build` entries for users with -`id` between `1` and `10,000`, the database returns `1 215 178` matching rows. +One should proceed with extra caution. When you iterate over an attribute that is not unique, +even with the applied max batch size, there is no guarantee that the resulting batches do not +surpass it. The following snippet demonstrates this situation when one attempt to select +`Ci::Build` entries for users with `id` between `1` and `10,000`, the database returns +`1 215 178` matching rows. ```ruby [ gstg ] production> Ci::Build.where(user_id: (1..10_000)).size => 1215178 ``` -This happens because built relation is translated into the following query +This happens because the built relation is translated into the following query: ```ruby [ gstg ] production> puts Ci::Build.where(user_id: (1..10_000)).to_sql @@ -69,6 +69,27 @@ threshold does not translate to the size of the returned dataset. That happens b `n` possible values of attributes, one can't tell for sure that the number of records that contains them is less than `n`. +### Loose-index scan with `distinct_each_batch` + +When iterating over a non-unique column is necessary, use the `distinct_each_batch` helper +method. The helper uses the [loose-index scan technique](https://wiki.postgresql.org/wiki/Loose_indexscan) +(skip-index scan) to skip duplicated values within a database index. + +Example: iterating over distinct `author_id` in the Issue model + +```ruby +Issue.distinct_each_batch(column: :author_id, of: 1000) do |relation| + users = User.where(id: relation.select(:author_id)).to_a +end +``` + +The technique provides stable performance between the batches regardless of the data distribution. +The `relation` object returns an ActiveRecord scope where only the given `column` is available. +Other columns are not loaded. + +The underlying database queries use recursive CTEs, which adds extra overhead. We therefore advise to use +smaller batch sizes than those used for a standard `each_batch` iteration. + ## Column definition `EachBatch` uses the primary key of the model by default for the iteration. This works most of the diff --git a/doc/install/docker.md b/doc/install/docker.md index 058233520ca..356c025e168 100644 --- a/doc/install/docker.md +++ b/doc/install/docker.md @@ -272,6 +272,10 @@ Here's an example that deploys GitLab with four runners as a [stack](https://doc docker stack deploy --compose-file docker-compose.yml mystack ``` +### Install the product documentation + +This is an optional step. See how to [self-host the product documentation](../administration/docs_self_host.md#self-host-the-product-documentation-with-docker). + ## Configuration This container uses the official Omnibus GitLab package, so all configuration @@ -529,6 +533,11 @@ The following steps assume that you are upgrading the same version. replace `ce` with `ee` in your `docker run` command or `docker-compose.yml` file. However, reuse the CE container name, port and file mappings, and version. +### Upgrade the product documentation + +This is an optional step. If you [installed the documentation site](#install-the-product-documentation), +see how to [upgrade to another version](../administration/docs_self_host.md#upgrade-using-docker). + ## Back up GitLab You can create a GitLab backup with: diff --git a/doc/install/installation.md b/doc/install/installation.md index cc2e57aac96..163c70e1ecc 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -1013,6 +1013,10 @@ To start and stop GitLab when using: - systemd units: use `sudo systemctl start gitlab.target` or `sudo systemctl stop gitlab.target`. - The SysV init script: use `sudo service gitlab start` or `sudo service gitlab stop`. +### Install the product documentation + +This is an optional step. See how to [self-host the product documentation](../administration/docs_self_host.md). + ## Advanced Setup Tips ### Relative URL support diff --git a/doc/security/ssh_keys_restrictions.md b/doc/security/ssh_keys_restrictions.md index eb92694d236..eca52c41e4f 100644 --- a/doc/security/ssh_keys_restrictions.md +++ b/doc/security/ssh_keys_restrictions.md @@ -48,14 +48,10 @@ By default, the GitLab.com and self-managed settings for the - ECDSA_SK SSH keys are allowed (GitLab 14.8 and later). - ED25519_SK SSH keys are allowed (GitLab 14.8 and later). -### Block banned or compromised keys **(FREE)** +## Block banned or compromised keys **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24614) in GitLab 15.1 [with a flag](../administration/feature_flags.md) named `ssh_banned_key`. Enabled by default. - -FLAG: -On self-managed GitLab, by default this feature is available. To hide the feature per user, -ask an administrator to [disable the feature flag](../administration/feature_flags.md) named `ssh_banned_key`. -On GitLab.com, this feature is available. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24614) in GitLab 15.1 [with a flag](../administration/feature_flags.md) named `ssh_banned_key`. Enabled by default. +> - Generally available in GitLab 15.2. [Feature flag `ssh_banned_key`](https://gitlab.com/gitlab-org/gitlab/-/issues/363410) removed. When users attempt to [add a new SSH key](../user/ssh.md#add-an-ssh-key-to-your-gitlab-account) to GitLab accounts, the key is checked against a list of SSH keys which are known diff --git a/doc/update/index.md b/doc/update/index.md index 05a903a8385..0bb9c3aa419 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -63,8 +63,8 @@ can still be found in the Git repository: ### Installation using Docker GitLab provides official Docker images for both Community and Enterprise -editions. They are based on the Omnibus package and instructions on how to -update them are in [a separate document](https://docs.gitlab.com/omnibus/docker/README.html). +editions, and they are based on the Omnibus package. See how to +[install GitLab using Docker](../install/docker.md). ### Installation using Helm diff --git a/doc/update/package/index.md b/doc/update/package/index.md index 15f43f59425..02dd5811bf7 100644 --- a/doc/update/package/index.md +++ b/doc/update/package/index.md @@ -189,6 +189,11 @@ NOTE: For the GitLab Community Edition, replace `gitlab-ee` with `gitlab-ce`. +## Upgrade the product documentation + +This is an optional step. If you [installed the product documentation](../../administration/docs_self_host.md), +see how to [upgrade to a later version](../../administration/docs_self_host.md#upgrade-using-docker). + ## Troubleshooting ### GitLab 13.7 and later unavailable on Amazon Linux 2 diff --git a/doc/update/upgrading_from_source.md b/doc/update/upgrading_from_source.md index 29bb956cb54..4280a761dd5 100644 --- a/doc/update/upgrading_from_source.md +++ b/doc/update/upgrading_from_source.md @@ -406,6 +406,11 @@ sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production If all items are green, then congratulations, the upgrade is complete! +### 17. Upgrade the product documentation + +This is an optional step. If you [installed the product documentation](../install/installation.md#install-the-product-documentation), +see how to [upgrade to a later version](../administration/docs_self_host.md#upgrade-the-product-documentation-to-a-later-version). + ## Version specific upgrading instructions This section contains upgrading instructions for specific versions. When diff --git a/doc/user/application_security/policies/scan-result-policies.md b/doc/user/application_security/policies/scan-result-policies.md index 3da884aca6a..aad425b4104 100644 --- a/doc/user/application_security/policies/scan-result-policies.md +++ b/doc/user/application_security/policies/scan-result-policies.md @@ -84,10 +84,10 @@ the defined policy. |-------|------|-----------------|-------------| | `type` | `string` | `require_approval` | The action's type. | | `approvals_required` | `integer` | Greater than or equal to zero | The number of MR approvals required. | -| `user_approvers` | `array` of `string` | Username of one of more users | The users to consider as approvers. | -| `user_approvers_ids` | `array` of `integer` | ID of one of more users | The IDs of users to consider as approvers. | -| `group_approvers` | `array` of `string` | Path of one of more groups | The groups to consider as approvers. | -| `group_approvers_ids` | `array` of `integer` | ID of one of more groups | The IDs of groups to consider as approvers. | +| `user_approvers` | `array` of `string` | Username of one of more users | The users to consider as approvers. Users must have access to the project to be eligible to approve. | +| `user_approvers_ids` | `array` of `integer` | ID of one of more users | The IDs of users to consider as approvers. Users must have access to the project to be eligible to approve. | +| `group_approvers` | `array` of `string` | Path of one of more groups | The groups to consider as approvers. Users with [direct membership in the group](../../project/merge_requests/approvals/rules.md#group-approvers) are eligible to approve. | +| `group_approvers_ids` | `array` of `integer` | ID of one of more groups | The IDs of groups to consider as approvers. Users with [direct membership in the group](../../project/merge_requests/approvals/rules.md#group-approvers) are eligible to approve. | Requirements and limitations: diff --git a/spec/models/concerns/each_batch_spec.rb b/spec/models/concerns/each_batch_spec.rb index f1fb4fcbd03..2c75d4d5c41 100644 --- a/spec/models/concerns/each_batch_spec.rb +++ b/spec/models/concerns/each_batch_spec.rb @@ -3,17 +3,17 @@ require 'spec_helper' RSpec.describe EachBatch do - describe '.each_batch' do - let(:model) do - Class.new(ActiveRecord::Base) do - include EachBatch + let(:model) do + Class.new(ActiveRecord::Base) do + include EachBatch - self.table_name = 'users' + self.table_name = 'users' - scope :never_signed_in, -> { where(sign_in_count: 0) } - end + scope :never_signed_in, -> { where(sign_in_count: 0) } end + end + describe '.each_batch' do before do create_list(:user, 5, updated_at: 1.day.ago) end @@ -86,4 +86,89 @@ RSpec.describe EachBatch do end end end + + describe '.distinct_each_batch' do + let_it_be(:users) { create_list(:user, 5, sign_in_count: 0) } + + let(:params) { {} } + + subject(:values) do + values = [] + + model.distinct_each_batch(**params) { |rel| values.concat(rel.pluck(params[:column])) } + values + end + + context 'when iterating over a unique column' do + context 'when using ascending order' do + let(:expected_values) { users.pluck(:id).sort } + let(:params) { { column: :id, of: 1, order: :asc } } + + it { is_expected.to eq(expected_values) } + + context 'when using larger batch size' do + before do + params[:of] = 3 + end + + it { is_expected.to eq(expected_values) } + end + + context 'when using larger batch size than the result size' do + before do + params[:of] = 100 + end + + it { is_expected.to eq(expected_values) } + end + end + + context 'when using descending order' do + let(:expected_values) { users.pluck(:id).sort.reverse } + let(:params) { { column: :id, of: 1, order: :desc } } + + it { is_expected.to eq(expected_values) } + + context 'when using larger batch size' do + before do + params[:of] = 3 + end + + it { is_expected.to eq(expected_values) } + end + end + end + + context 'when iterating over a non-unique column' do + let(:params) { { column: :sign_in_count, of: 2, order: :asc } } + + context 'when only one value is present' do + it { is_expected.to eq([0]) } + end + + context 'when duplicated values present' do + let(:expected_values) { [2, 5] } + + before do + users[0].reload.update!(sign_in_count: 5) + users[1].reload.update!(sign_in_count: 2) + users[2].reload.update!(sign_in_count: 5) + users[3].reload.update!(sign_in_count: 2) + users[4].reload.update!(sign_in_count: 5) + end + + it { is_expected.to eq(expected_values) } + + context 'when using descending order' do + let(:expected_values) { [5, 2] } + + before do + params[:order] = :desc + end + + it { is_expected.to eq(expected_values) } + end + end + end + end end diff --git a/spec/models/concerns/loose_index_scan_spec.rb b/spec/models/concerns/loose_index_scan_spec.rb new file mode 100644 index 00000000000..685819bfb86 --- /dev/null +++ b/spec/models/concerns/loose_index_scan_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true +# frozen_string_literal + +require 'spec_helper' + +RSpec.describe LooseIndexScan, type: :model do + let(:issue_model) do + Class.new(ApplicationRecord) do + include LooseIndexScan + + self.table_name = 'issues' + end + end + + let_it_be(:user_1) { create(:user) } + let_it_be(:user_2) { create(:user) } + let_it_be(:user_3) { create(:user) } + + let_it_be(:issue_1) { create(:issue, author: user_2) } + let_it_be(:issue_2) { create(:issue, author: user_1) } + let_it_be(:issue_3) { create(:issue, author: user_1) } + let_it_be(:issue_4) { create(:issue, author: user_2) } + let_it_be(:issue_5) { create(:issue, author: user_3) } + + context 'loading distinct author_ids' do + subject(:author_ids) { issue_model.loose_index_scan(column: :author_id, order: order).pluck(:author_id) } + + shared_examples 'assert distinct values example' do + it 'loads the distinct values in the correct order' do + expect(author_ids).to eq(expected_order) + end + end + + context 'when using ascending order' do + let(:order) { :asc } + let(:expected_order) { [user_1.id, user_2.id, user_3.id] } + + it_behaves_like 'assert distinct values example' + + context 'when null values are present' do + before do + issue_1.author_id = nil + issue_1.save!(validate: false) + end + + it_behaves_like 'assert distinct values example' + end + + context 'when using descending order' do + let(:order) { :desc } + let(:expected_order) { [user_3.id, user_2.id, user_1.id] } + + it_behaves_like 'assert distinct values example' + end + end + end +end diff --git a/spec/models/key_spec.rb b/spec/models/key_spec.rb index a9d1a8a5ef2..f584057be5d 100644 --- a/spec/models/key_spec.rb +++ b/spec/models/key_spec.rb @@ -47,10 +47,9 @@ RSpec.describe Key, :mailer do end describe 'validation of banned keys' do - let_it_be(:user) { create(:user) } - let(:key) { build(:key) } - let(:banned_keys) do + + where(:key_content) do [ 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAwRIdDlHaIqZXND/l1vFT7ue3rc/DvXh2y' \ 'x5EFtuxGQRHVxGMazDhV4vj5ANGXDQwUYI0iZh6aOVrDy8I/y9/y+YDGCvsnqrDbuPDjW' \ @@ -131,68 +130,13 @@ RSpec.describe Key, :mailer do ] end - context 'when ssh_banned_key feature flag is enabled with a user' do - before do - stub_feature_flags(ssh_banned_key: user) - end - - where(:key_content) { banned_keys } - - with_them do - it 'does not allow banned keys' do - key.key = key_content - key.user = user - - expect(key).to be_invalid - expect(key.errors[:key]).to include( - _('cannot be used because it belongs to a compromised private key. Stop using this key and generate a new one.')) - end - - it 'allows when the user is a ghost user' do - key.key = key_content - key.user = User.ghost - - expect(key).to be_valid - end - - it 'allows when the user is nil' do - key.key = key_content - key.user = nil - - expect(key).to be_valid - end - end - - it 'allows other keys' do - key.user = user - - expect(key).to be_valid - end - - it 'allows other users' do - key.user = User.ghost - - expect(key).to be_valid - end - end - - context 'when ssh_banned_key feature flag is disabled' do - before do - stub_feature_flags(ssh_banned_key: false) - end - - where(:key_content) { banned_keys } - - with_them do - it 'allows banned keys' do - key.key = key_content - - expect(key).to be_valid - end - end + with_them do + it 'does not allow banned keys' do + key.key = key_content - it 'allows other keys' do - expect(key).to be_valid + expect(key).to be_invalid + expect(key.errors[:key]).to include( + _('cannot be used because it belongs to a compromised private key. Stop using this key and generate a new one.')) end end end |