diff options
15 files changed, 219 insertions, 119 deletions
diff --git a/app/assets/javascripts/notes/components/discussion_filter.vue b/app/assets/javascripts/notes/components/discussion_filter.vue index 1ce23c941e6..08c22f0b4c6 100644 --- a/app/assets/javascripts/notes/components/discussion_filter.vue +++ b/app/assets/javascripts/notes/components/discussion_filter.vue @@ -116,7 +116,7 @@ export default { <gl-dropdown v-if="displayFilters" id="discussion-filter-dropdown" - class="gl-mr-3 full-width-mobile discussion-filter-container js-discussion-filter-container qa-discussion-filter" + class="gl-mr-3 full-width-mobile discussion-filter-container js-discussion-filter-container" data-qa-selector="discussion_filter_dropdown" :text="currentFilter.title" > diff --git a/changelogs/unreleased/238156_remove_old_security_findings_from_database.yml b/changelogs/unreleased/238156_remove_old_security_findings_from_database.yml new file mode 100644 index 00000000000..c104e8c8e6b --- /dev/null +++ b/changelogs/unreleased/238156_remove_old_security_findings_from_database.yml @@ -0,0 +1,5 @@ +--- +title: Remove all records from `security_findings` table +merge_request: 44312 +author: +type: fixed diff --git a/config/initializers/rack_attack_logging.rb b/config/initializers/rack_attack_logging.rb index a95cb09755b..790f9567409 100644 --- a/config/initializers/rack_attack_logging.rb +++ b/config/initializers/rack_attack_logging.rb @@ -11,7 +11,8 @@ ActiveSupport::Notifications.subscribe(/rack_attack/) do |name, start, finish, r env: req.env['rack.attack.match_type'], remote_ip: req.ip, request_method: req.request_method, - path: req.fullpath + path: req.fullpath, + matched: req.env['rack.attack.matched'] } throttles_with_user_information = [ @@ -25,9 +26,8 @@ ActiveSupport::Notifications.subscribe(/rack_attack/) do |name, start, finish, r user_id = req.env['rack.attack.match_discriminator'] user = User.find_by(id: user_id) - rack_attack_info[:throttle_type] = req.env['rack.attack.matched'] rack_attack_info[:user_id] = user_id - rack_attack_info[:username] = user.username unless user.nil? + rack_attack_info['meta.user'] = user.username unless user.nil? end Gitlab::AuthLogger.error(rack_attack_info) diff --git a/db/post_migrate/20201102152945_truncate_security_findings_table.rb b/db/post_migrate/20201102152945_truncate_security_findings_table.rb new file mode 100644 index 00000000000..8bfaa31f0de --- /dev/null +++ b/db/post_migrate/20201102152945_truncate_security_findings_table.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class TruncateSecurityFindingsTable < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + return unless Gitlab.dev_env_or_com? + + with_lock_retries do + connection.execute('TRUNCATE security_findings RESTART IDENTITY') + end + end + + def down + # no-op + end +end diff --git a/db/schema_migrations/20201102152945 b/db/schema_migrations/20201102152945 new file mode 100644 index 00000000000..cb4ae9c2598 --- /dev/null +++ b/db/schema_migrations/20201102152945 @@ -0,0 +1 @@ +55ffd18d5f55ee0fd51a31d50cf2d51595740c72ca23d5134d93e2da3fc186ff
\ No newline at end of file diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index 1a37752a0ba..3cce32ea8fd 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -1440,6 +1440,11 @@ type BoardEpic implements CurrentUserTodos & Noteable { iids: [ID!] """ + Include epics from descendant groups + """ + includeDescendantGroups: Boolean = true + + """ Filter epics by labels """ labelName: [String!] @@ -6593,6 +6598,11 @@ type Epic implements CurrentUserTodos & Noteable { iids: [ID!] """ + Include epics from descendant groups + """ + includeDescendantGroups: Boolean = true + + """ Filter epics by labels """ labelName: [String!] @@ -8167,6 +8177,11 @@ type Group { iids: [ID!] """ + Include epics from descendant groups + """ + includeDescendantGroups: Boolean = true + + """ Filter epics by labels """ labelName: [String!] @@ -8250,6 +8265,11 @@ type Group { iids: [ID!] """ + Include epics from descendant groups + """ + includeDescendantGroups: Boolean = true + + """ Filter epics by labels """ labelName: [String!] diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index 3939d10b1a7..3b5a30c0bd0 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -3868,6 +3868,16 @@ "defaultValue": null }, { + "name": "includeDescendantGroups", + "description": "Include epics from descendant groups", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": "true" + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -18289,6 +18299,16 @@ "defaultValue": null }, { + "name": "includeDescendantGroups", + "description": "Include epics from descendant groups", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": "true" + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -22575,6 +22595,16 @@ "ofType": null }, "defaultValue": null + }, + { + "name": "includeDescendantGroups", + "description": "Include epics from descendant groups", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": "true" } ], "type": { @@ -22726,6 +22756,16 @@ "defaultValue": null }, { + "name": "includeDescendantGroups", + "description": "Include epics from descendant groups", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": "true" + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { diff --git a/doc/install/installation.md b/doc/install/installation.md index 9ae59207752..feebe601fe0 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -90,6 +90,8 @@ The GitLab installation consists of setting up the following components: ## 1. Packages and dependencies +### sudo + `sudo` is not installed on Debian by default. Make sure your system is up-to-date and install it. @@ -110,6 +112,8 @@ sudo apt-get install -y vim sudo update-alternatives --set editor /usr/bin/vim.basic ``` +### Build dependencies + Install the required packages (needed to compile Ruby and native extensions to Ruby gems): ```shell @@ -129,6 +133,8 @@ If you want to use Kerberos for user authentication, install `libkrb5-dev` sudo apt-get install libkrb5-dev ``` +### Git + Make sure you have the right version of Git installed: ```shell @@ -168,18 +174,9 @@ On Debian, use the following compilation instructions: ```shell # Install dependencies -sudo apt-get install -y libcurl4-openssl-dev libexpat1-dev gettext libz-dev libssl-dev build-essential - -# Download and compile pcre2 from source -curl --silent --show-error --location https://ftp.pcre.org/pub/pcre/pcre2-10.33.tar.gz --output pcre2.tar.gz -tar -xzf pcre2.tar.gz -cd pcre2-10.33 -chmod +x configure -./configure --prefix=/usr --enable-jit -make -sudo make install +sudo apt-get install -y libcurl4-openssl-dev libexpat1-dev gettext libz-dev libssl-dev libpcre2-dev build-essential -# Download and compile from source +# Download and compile Git from source cd /tmp curl --remote-name --location --progress https://www.kernel.org/pub/software/scm/git/git-2.29.0.tar.gz echo 'fa08dc8424ef80c0f9bf307877f9e2e49f1a6049e873530d6747c2be770742ff git-2.29.0.tar.gz' | shasum -a256 -c - && tar -xzf git-2.29.0.tar.gz @@ -193,6 +190,8 @@ sudo make prefix=/usr/local install # When editing config/gitlab.yml later, change the git -> bin_path to /usr/local/bin/git ``` +### GraphicsMagick + For the [Custom Favicon](../user/admin_area/appearance.md#favicon) to work, GraphicsMagick needs to be installed. @@ -200,6 +199,8 @@ needs to be installed. sudo apt-get install -y graphicsmagick ``` +### Mail server + In order to receive mail notifications, make sure to install a mail server. By default, Debian is shipped with `exim4` but this [has problems](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/12754) while @@ -212,6 +213,8 @@ sudo apt-get install -y postfix Then select 'Internet Site' and press enter to confirm the hostname. +### Exiftool + [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse#dependencies) requires `exiftool` to remove EXIF data from uploaded images. diff --git a/qa/qa/page/component/issuable/sidebar.rb b/qa/qa/page/component/issuable/sidebar.rb index 383847e8529..e41eb12a99d 100644 --- a/qa/qa/page/component/issuable/sidebar.rb +++ b/qa/qa/page/component/issuable/sidebar.rb @@ -73,7 +73,7 @@ module QA def has_no_assignee_named?(username) within_element(:assignee_block) do - has_no_text?(username) + has_no_text?(username, wait: QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME) end end diff --git a/qa/qa/page/component/note.rb b/qa/qa/page/component/note.rb index e6defd2ec0c..5ac72d73c78 100644 --- a/qa/qa/page/component/note.rb +++ b/qa/qa/page/component/note.rb @@ -13,20 +13,35 @@ module QA element :toggle_comments_button end + base.view 'app/assets/javascripts/notes/components/comment_form.vue' do + element :comment_button + element :comment_field + element :discussion_menu_item + element :note_dropdown + end + base.view 'app/assets/javascripts/notes/components/discussion_actions.vue' do element :discussion_reply_tab element :resolve_discussion_button end - base.view 'app/assets/javascripts/notes/components/comment_form.vue' do - element :note_dropdown - element :discussion_menu_item + base.view 'app/assets/javascripts/notes/components/discussion_filter.vue' do + element :discussion_filter_dropdown, required: true + element :filter_menu_item + end + + base.view 'app/assets/javascripts/notes/components/discussion_filter_note.vue' do + element :discussion_filter_container end base.view 'app/assets/javascripts/notes/components/noteable_discussion.vue' do element :discussion_content end + base.view 'app/assets/javascripts/notes/components/noteable_note.vue' do + element :noteable_note_container + end + base.view 'app/assets/javascripts/notes/components/note_actions.vue' do element :note_edit_button end @@ -36,6 +51,10 @@ module QA element :reply_comment_button end + base.view 'app/assets/javascripts/notes/components/note_header.vue' do + element :system_note_content + end + base.view 'app/assets/javascripts/notes/components/toggle_replies_widget.vue' do element :expand_replies_button element :collapse_replies_button @@ -44,12 +63,30 @@ module QA base.view 'app/assets/javascripts/vue_shared/components/notes/skeleton_note.vue' do element :skeleton_note_placeholder end + + base.view 'app/views/shared/notes/_form.html.haml' do + element :new_note_form, 'new-note' # rubocop:disable QA/ElementWithPattern + element :new_note_form, 'attr: :note' # rubocop:disable QA/ElementWithPattern + end end def collapse_replies click_element :collapse_replies_button end + # Attachment option should be an absolute path + def comment(text, attachment: nil, filter: :all_activities) + method("select_#{filter}_filter").call + fill_element :comment_field, "#{text}\n" + + unless attachment.nil? + QA::Page::Component::Dropzone.new(self, '.new-note') + .attach_file(attachment) + end + + click_element :comment_button + end + def edit_comment(text) click_element :note_edit_button fill_element :reply_field, text @@ -60,6 +97,18 @@ module QA click_element :expand_replies_button end + def has_comment?(comment_text) + has_element?(:noteable_note_container, text: comment_text, wait: QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME) + end + + def has_system_note?(note_text) + has_element?(:system_note_content, text: note_text, wait: QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME) + end + + def noteable_note_item + find_element(:noteable_note_container) + end + def reply_to_discussion(position, reply_text) type_reply_to_discussion(position, reply_text) click_element :reply_comment_button @@ -71,6 +120,26 @@ module QA end end + def select_all_activities_filter + select_filter_with_text('Show all activity') + end + + def select_comments_only_filter + select_filter_with_text('Show comments only') + + wait_until do + has_no_element?(:system_note_content) + end + end + + def select_history_only_filter + select_filter_with_text('Show history only') + + wait_until do + has_element?(:discussion_filter_container) && has_no_element?(:noteable_note_container) + end + end + def start_discussion(text) fill_element :comment_field, text click_element :note_dropdown @@ -90,6 +159,18 @@ module QA def wait_for_loading has_no_element?(:skeleton_note_placeholer) end + + private + + def select_filter_with_text(text) + retry_on_exception do + click_element(:title) + click_element :discussion_filter_dropdown + find_element(:filter_menu_item, text: text).click + + wait_for_loading + end + end end end end diff --git a/qa/qa/page/project/issue/show.rb b/qa/qa/page/project/issue/show.rb index 2040257d6b0..f7bd74d1882 100644 --- a/qa/qa/page/project/issue/show.rb +++ b/qa/qa/page/project/issue/show.rb @@ -10,28 +10,6 @@ module QA include Page::Component::DesignManagement include Page::Component::Issuable::Sidebar - view 'app/assets/javascripts/notes/components/comment_form.vue' do - element :comment_button - element :comment_field - end - - view 'app/assets/javascripts/notes/components/discussion_filter.vue' do - element :discussion_filter_dropdown, required: true - element :filter_menu_item - end - - view 'app/assets/javascripts/notes/components/discussion_filter_note.vue' do - element :discussion_filter_container - end - - view 'app/assets/javascripts/notes/components/noteable_note.vue' do - element :noteable_note_container - end - - view 'app/assets/javascripts/notes/components/note_header.vue' do - element :system_note_content - end - view 'app/assets/javascripts/vue_shared/components/issue/related_issuable_item.vue' do element :remove_related_issue_button end @@ -41,11 +19,6 @@ module QA element :reopen_issue_button end - view 'app/views/shared/notes/_form.html.haml' do - element :new_note_form, 'new-note' # rubocop:disable QA/ElementWithPattern - element :new_note_form, 'attr: :note' # rubocop:disable QA/ElementWithPattern - end - view 'app/assets/javascripts/related_issues/components/add_issuable_form.vue' do element :add_issue_button end @@ -88,67 +61,9 @@ module QA click_element :close_issue_button end - # Adds a comment to an issue - # attachment option should be an absolute path - def comment(text, attachment: nil, filter: :all_activities) - method("select_#{filter}_filter").call - fill_element :comment_field, "#{text}\n" - - unless attachment.nil? - QA::Page::Component::Dropzone.new(self, '.new-note') - .attach_file(attachment) - end - - click_element :comment_button - end - - def has_comment?(comment_text) - has_element?(:noteable_note_container, text: comment_text, wait: QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME) - end - - def has_system_note?(note_text) - has_element?(:system_note_content, text: note_text, wait: QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME) - end - - def noteable_note_item - find_element(:noteable_note_container) - end - - def select_all_activities_filter - select_filter_with_text('Show all activity') - end - - def select_comments_only_filter - select_filter_with_text('Show comments only') - - wait_until do - has_no_element?(:system_note_content) - end - end - - def select_history_only_filter - select_filter_with_text('Show history only') - - wait_until do - has_element?(:discussion_filter_container) && has_no_element?(:noteable_note_container) - end - end - def has_metrics_unfurled? has_element?(:prometheus_graph_widgets, wait: 30) end - - private - - def select_filter_with_text(text) - retry_on_exception do - click_element(:title) - click_element :discussion_filter_dropdown - find_element(:filter_menu_item, text: text).click - - wait_for_loading - end - end end end end diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/collapse_comments_in_discussions_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/collapse_comments_in_discussions_spec.rb index 6b309716f55..22157d648ca 100644 --- a/qa/qa/specs/features/browser_ui/2_plan/issue/collapse_comments_in_discussions_spec.rb +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/collapse_comments_in_discussions_spec.rb @@ -26,7 +26,7 @@ module QA expect(show).not_to have_content(my_first_reply) show.expand_replies - expect(show).to have_content(my_first_reply) + expect(show).to have_comment(my_first_reply) expect(show).not_to have_content(one_reply) end end diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb index c9ae47459c4..d3780186f36 100644 --- a/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb @@ -16,11 +16,11 @@ module QA show.comment(first_version_of_comment) - expect(show).to have_content(first_version_of_comment) + expect(show).to have_comment(first_version_of_comment) show.edit_comment(second_version_of_comment) - expect(show).to have_content(second_version_of_comment) + expect(show).to have_comment(second_version_of_comment) expect(show).not_to have_content(first_version_of_comment) end end diff --git a/spec/requests/rack_attack_global_spec.rb b/spec/requests/rack_attack_global_spec.rb index 9fdafc06695..d4a8e64eb2e 100644 --- a/spec/requests/rack_attack_global_spec.rb +++ b/spec/requests/rack_attack_global_spec.rb @@ -125,7 +125,8 @@ RSpec.describe 'Rack Attack global throttles' do env: :throttle, remote_ip: '127.0.0.1', request_method: 'GET', - path: '/users/sign_in' + path: '/users/sign_in', + matched: 'throttle_unauthenticated' } expect(Gitlab::AuthLogger).to receive(:error).with(arguments) diff --git a/spec/support/shared_examples/requests/rack_attack_shared_examples.rb b/spec/support/shared_examples/requests/rack_attack_shared_examples.rb index 730df4dc5ab..d4ee68309ff 100644 --- a/spec/support/shared_examples/requests/rack_attack_shared_examples.rb +++ b/spec/support/shared_examples/requests/rack_attack_shared_examples.rb @@ -81,8 +81,15 @@ RSpec.shared_examples 'rate-limited token-authenticated requests' do end it 'logs RackAttack info into structured logs' do - requests_per_period.times do - make_request(request_args) + control_count = 0 + + requests_per_period.times do |i| + if i == 0 + control_count = ActiveRecord::QueryRecorder.new { make_request(request_args) }.count + else + make_request(request_args) + end + expect(response).not_to have_gitlab_http_status(:too_many_requests) end @@ -93,13 +100,15 @@ RSpec.shared_examples 'rate-limited token-authenticated requests' do request_method: request_method, path: request_args.first, user_id: user.id, - username: user.username, - throttle_type: throttle_types[throttle_setting_prefix] + 'meta.user' => user.username, + matched: throttle_types[throttle_setting_prefix] } expect(Gitlab::AuthLogger).to receive(:error).with(arguments).once - expect_rejection { make_request(request_args) } + expect_rejection do + expect { make_request(request_args) }.not_to exceed_query_limit(control_count) + end end end @@ -210,8 +219,15 @@ RSpec.shared_examples 'rate-limited web authenticated requests' do end it 'logs RackAttack info into structured logs' do - requests_per_period.times do - request_authenticated_web_url + control_count = 0 + + requests_per_period.times do |i| + if i == 0 + control_count = ActiveRecord::QueryRecorder.new { request_authenticated_web_url }.count + else + request_authenticated_web_url + end + expect(response).not_to have_gitlab_http_status(:too_many_requests) end @@ -222,13 +238,12 @@ RSpec.shared_examples 'rate-limited web authenticated requests' do request_method: request_method, path: url_that_requires_authentication, user_id: user.id, - username: user.username, - throttle_type: throttle_types[throttle_setting_prefix] + 'meta.user' => user.username, + matched: throttle_types[throttle_setting_prefix] } expect(Gitlab::AuthLogger).to receive(:error).with(arguments).once - - request_authenticated_web_url + expect { request_authenticated_web_url }.not_to exceed_query_limit(control_count) end end |