diff options
24 files changed, 332 insertions, 26 deletions
diff --git a/Gemfile.lock b/Gemfile.lock index 41356d997c9..e6b3673c6e9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -336,7 +336,7 @@ GEM ethon (0.15.0) ffi (>= 1.15.0) eventmachine (1.2.7) - excon (0.89.0) + excon (0.90.0) execjs (2.8.1) expression_parser (0.9.0) extended-markdown-filter (0.6.0) diff --git a/app/assets/javascripts/boards/constants.js b/app/assets/javascripts/boards/constants.js index e0eea761b31..0f290f566ba 100644 --- a/app/assets/javascripts/boards/constants.js +++ b/app/assets/javascripts/boards/constants.js @@ -50,6 +50,8 @@ export const toggleFormEventPrefix = { issue: 'toggle-issue-form-', }; +export const active = 'active'; + export const inactiveId = 0; export const ISSUABLE = 'issuable'; diff --git a/app/assets/javascripts/boards/graphql/group_board_milestones.query.graphql b/app/assets/javascripts/boards/graphql/group_board_milestones.query.graphql index 0963b3fbfaa..6fe8bb799d6 100644 --- a/app/assets/javascripts/boards/graphql/group_board_milestones.query.graphql +++ b/app/assets/javascripts/boards/graphql/group_board_milestones.query.graphql @@ -1,7 +1,7 @@ -query GroupBoardMilestones($fullPath: ID!, $searchTerm: String) { +query GroupBoardMilestones($fullPath: ID!, $searchTerm: String, $state: MilestoneStateEnum) { group(fullPath: $fullPath) { id - milestones(includeAncestors: true, searchTitle: $searchTerm) { + milestones(includeAncestors: true, searchTitle: $searchTerm, state: $state) { nodes { id title diff --git a/app/assets/javascripts/boards/graphql/project_board_milestones.query.graphql b/app/assets/javascripts/boards/graphql/project_board_milestones.query.graphql index e456823d78a..d917c7e809d 100644 --- a/app/assets/javascripts/boards/graphql/project_board_milestones.query.graphql +++ b/app/assets/javascripts/boards/graphql/project_board_milestones.query.graphql @@ -1,7 +1,7 @@ -query ProjectBoardMilestones($fullPath: ID!, $searchTerm: String) { +query ProjectBoardMilestones($fullPath: ID!, $searchTerm: String, $state: MilestoneStateEnum) { project(fullPath: $fullPath) { id - milestones(searchTitle: $searchTerm, includeAncestors: true) { + milestones(searchTitle: $searchTerm, includeAncestors: true, state: $state) { nodes { id title diff --git a/app/assets/javascripts/boards/stores/actions.js b/app/assets/javascripts/boards/stores/actions.js index 1ebfcfc331b..48ca3239cfd 100644 --- a/app/assets/javascripts/boards/stores/actions.js +++ b/app/assets/javascripts/boards/stores/actions.js @@ -15,6 +15,7 @@ import { FilterFields, ListTypeTitles, DraggableItemTypes, + active, } from 'ee_else_ce/boards/constants'; import { formatIssueInput, @@ -209,6 +210,7 @@ export default { const variables = { fullPath, searchTerm, + state: active, }; let query; diff --git a/app/models/clusters/agent.rb b/app/models/clusters/agent.rb index 8705c0fbec4..79fc2b58237 100644 --- a/app/models/clusters/agent.rb +++ b/app/models/clusters/agent.rb @@ -5,6 +5,7 @@ module Clusters self.table_name = 'cluster_agents' INACTIVE_AFTER = 1.hour.freeze + ACTIVITY_EVENT_LIMIT = 200 belongs_to :created_by_user, class_name: 'User', optional: true belongs_to :project, class_name: '::Project' # Otherwise, it will load ::Clusters::Project @@ -39,5 +40,12 @@ module Clusters def connected? agent_tokens.active.where("last_used_at > ?", INACTIVE_AFTER.ago).exists? end + + def activity_event_deletion_cutoff + # Order is defined by the association + activity_events + .offset(ACTIVITY_EVENT_LIMIT - 1) + .pick(:recorded_at) + end end end diff --git a/app/models/clusters/agent_token.rb b/app/models/clusters/agent_token.rb index bcef6b6ecc8..acf13c55ba3 100644 --- a/app/models/clusters/agent_token.rb +++ b/app/models/clusters/agent_token.rb @@ -56,12 +56,13 @@ module Clusters end def log_activity_event!(recorded_at) - agent.activity_events.create!( + Clusters::Agents::CreateActivityEventService.new( # rubocop: disable CodeReuse/ServiceClass + agent, kind: :agent_connected, level: :info, recorded_at: recorded_at, agent_token: self - ) + ).execute end end end diff --git a/app/models/clusters/agents/activity_event.rb b/app/models/clusters/agents/activity_event.rb index 5d9c885c923..ec2bbfde339 100644 --- a/app/models/clusters/agents/activity_event.rb +++ b/app/models/clusters/agents/activity_event.rb @@ -3,6 +3,7 @@ module Clusters module Agents class ActivityEvent < ApplicationRecord + include EachBatch include NullifyIfBlank self.table_name = 'agent_activity_events' @@ -12,6 +13,7 @@ module Clusters belongs_to :agent_token, class_name: 'Clusters::AgentToken' scope :in_timeline_order, -> { order(recorded_at: :desc, id: :desc) } + scope :recorded_before, -> (cutoff) { where('recorded_at < ?', cutoff) } validates :recorded_at, :kind, :level, presence: true diff --git a/app/services/clusters/agent_tokens/create_service.rb b/app/services/clusters/agent_tokens/create_service.rb index 5b8a0e46a6c..2539ffdc5ba 100644 --- a/app/services/clusters/agent_tokens/create_service.rb +++ b/app/services/clusters/agent_tokens/create_service.rb @@ -30,13 +30,14 @@ module Clusters end def log_activity_event!(token) - token.agent.activity_events.create!( + Clusters::Agents::CreateActivityEventService.new( + token.agent, kind: :token_created, level: :info, recorded_at: token.created_at, user: current_user, agent_token: token - ) + ).execute end end end diff --git a/app/services/clusters/agents/create_activity_event_service.rb b/app/services/clusters/agents/create_activity_event_service.rb new file mode 100644 index 00000000000..886dddf1a52 --- /dev/null +++ b/app/services/clusters/agents/create_activity_event_service.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Clusters + module Agents + class CreateActivityEventService + def initialize(agent, **params) + @agent = agent + @params = params + end + + def execute + agent.activity_events.create!(params) + + DeleteExpiredEventsWorker.perform_at(schedule_cleanup_at, agent.id) + + ServiceResponse.success + end + + private + + attr_reader :agent, :params + + def schedule_cleanup_at + 1.hour.from_now.change(min: agent.id % 60) + end + end + end +end diff --git a/app/services/clusters/agents/delete_expired_events_service.rb b/app/services/clusters/agents/delete_expired_events_service.rb new file mode 100644 index 00000000000..a0c0291c1fb --- /dev/null +++ b/app/services/clusters/agents/delete_expired_events_service.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Clusters + module Agents + class DeleteExpiredEventsService + def initialize(agent) + @agent = agent + end + + def execute + agent.activity_events + .recorded_before(remove_events_before) + .each_batch { |batch| batch.delete_all } + end + + private + + attr_reader :agent + + def remove_events_before + agent.activity_event_deletion_cutoff + end + end + end +end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 712191df243..8ae06030d88 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -129,6 +129,15 @@ :weight: 2 :idempotent: :tags: [] +- :name: cluster_agent:clusters_agents_delete_expired_events + :worker_name: Clusters::Agents::DeleteExpiredEventsWorker + :feature_category: :kubernetes_management + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: container_repository:cleanup_container_repository :worker_name: CleanupContainerRepositoryWorker :feature_category: :container_registry diff --git a/app/workers/clusters/agents/delete_expired_events_worker.rb b/app/workers/clusters/agents/delete_expired_events_worker.rb new file mode 100644 index 00000000000..3414365a243 --- /dev/null +++ b/app/workers/clusters/agents/delete_expired_events_worker.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Clusters + module Agents + class DeleteExpiredEventsWorker + include ApplicationWorker + include ClusterAgentQueue + + deduplicate :until_executed, including_scheduled: true + idempotent! + + data_consistency :always + + def perform(agent_id) + if agent = Clusters::Agent.find_by_id(agent_id) + Clusters::Agents::DeleteExpiredEventsService.new(agent).execute + end + end + end + end +end diff --git a/app/workers/concerns/cluster_agent_queue.rb b/app/workers/concerns/cluster_agent_queue.rb new file mode 100644 index 00000000000..68de7cca135 --- /dev/null +++ b/app/workers/concerns/cluster_agent_queue.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module ClusterAgentQueue + extend ActiveSupport::Concern + + included do + queue_namespace :cluster_agent + feature_category :kubernetes_management + end +end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index a234c35c1a6..710bde34fe7 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -77,6 +77,8 @@ - 1 - - ci_upstream_projects_subscriptions_cleanup - 1 +- - cluster_agent + - 1 - - container_repository - 1 - - create_commit_signature diff --git a/doc/administration/auth/ldap/index.md b/doc/administration/auth/ldap/index.md index 36211acc4f3..b773281b216 100644 --- a/doc/administration/auth/ldap/index.md +++ b/doc/administration/auth/ldap/index.md @@ -153,6 +153,14 @@ production: ### Basic configuration settings +> `hosts` configuration setting [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/139) in GitLab 14.7. + +You can configure either: + +- A single LDAP server using `host` and `port`. +- Many LDAP servers using `hosts`. This setting takes precedence over `host` and `port`. GitLab attempts to use the + LDAP servers in the order specified, and the first reachable LDAP server is used. + These configuration settings are available: | Setting | Description | Required | Examples | @@ -160,7 +168,7 @@ These configuration settings are available: | `label` | A human-friendly name for your LDAP server. It is displayed on your sign-in page. | **{check-circle}** Yes | `'Paris'` or `'Acme, Ltd.'` | | `host` | IP address or domain name of your LDAP server. Ignored when `hosts` is defined. | **{check-circle}** Yes | `'ldap.mydomain.com'` | | `port` | The port to connect with on your LDAP server. Always an integer, not a string. Ignored when `hosts` is defined. | **{check-circle}** Yes | `389` or `636` (for SSL) | -| `hosts` | An array of host and port pairs to open connections. This setting takes precedence over `host` and `port`. | **{dotted-circle}** No | `[['ldap1.mydomain.com', 636], ['ldap2.mydomain.com', 636]]` | +| `hosts` (GitLab 14.7 and later) | An array of host and port pairs to open connections. | **{dotted-circle}** No | `[['ldap1.mydomain.com', 636], ['ldap2.mydomain.com', 636]]` | | `uid` | LDAP attribute for username. Should be the attribute, not the value that maps to the `uid`. | **{check-circle}** Yes | `'sAMAccountName'` or `'uid'` or `'userPrincipalName'` | | `bind_dn` | The full DN of the user you bind with. | **{dotted-circle}** No | `'america\momo'` or `'CN=Gitlab,OU=Users,DC=domain,DC=com'` | | `password` | The password of the bind user. | **{dotted-circle}** No | `'your_great_password'` | diff --git a/doc/update/index.md b/doc/update/index.md index 1cc560d5156..35f320841c7 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -98,7 +98,7 @@ that can process jobs in the `background_migration` queue. ```shell sudo gitlab-rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' -sudo gitlab-rails runner -e production 'puts Gitlab::BackgroundMigration.pending' +sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigrationJob.pending' ``` **For installations from source:** @@ -106,7 +106,7 @@ sudo gitlab-rails runner -e production 'puts Gitlab::BackgroundMigration.pending ```shell cd /home/git/gitlab sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' -sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::BackgroundMigration.pending' +sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::BackgroundMigrationJob.pending' ``` ### Batched background migrations @@ -193,9 +193,9 @@ To address the above two scenario's, it is advised to do the following prior to 1. Wait until all jobs are finished. 1. Upgrade GitLab. -## Checking for pending Advanced Search migrations +## Checking for pending Advanced Search migrations **(PREMIUM SELF)** -This section is only applicable if you have enabled the [Elasticsearch integration](../integration/elasticsearch.md). +This section is only applicable if you have enabled the [Elasticsearch integration](../integration/elasticsearch.md) **(PREMIUM SELF)**. Major releases require all [Advanced Search migrations](../integration/elasticsearch.md#advanced-search-migrations) to be finished from the most recent minor release in your current version @@ -239,14 +239,12 @@ It is required to follow the following upgrade steps to ensure a successful *maj Identify a [supported upgrade path](#upgrade-paths). -It's also important to ensure that any background migrations have been fully completed -before upgrading to a new major version. To see the current size of the `background_migration` queue, -[Check for background migrations before upgrading](#checking-for-background-migrations-before-upgrading). +It's also important to ensure that any [background migrations have been fully completed](#checking-for-background-migrations-before-upgrading) +before upgrading to a new major version. -If you have enabled the [Elasticsearch integration](../integration/elasticsearch.md), then ensure -all Advanced Search migrations are completed in the last minor version within -your current version. Be sure to -[check for pending Advanced Search migrations](#checking-for-pending-advanced-search-migrations) +If you have enabled the [Elasticsearch integration](../integration/elasticsearch.md) **(PREMIUM SELF)**, then +[ensure all Advanced Search migrations are completed](#checking-for-pending-advanced-search-migrations) in the last minor version within +your current version before proceeding with the major version upgrade. If your GitLab instance has any runners associated with it, it is very diff --git a/spec/frontend/boards/stores/actions_spec.js b/spec/frontend/boards/stores/actions_spec.js index 51340a3ea4f..7c842d71688 100644 --- a/spec/frontend/boards/stores/actions_spec.js +++ b/spec/frontend/boards/stores/actions_spec.js @@ -29,6 +29,8 @@ import * as types from '~/boards/stores/mutation_types'; import mutations from '~/boards/stores/mutations'; import { getIdFromGraphQLId } from '~/graphql_shared/utils'; +import projectBoardMilestones from '~/boards/graphql/project_board_milestones.query.graphql'; +import groupBoardMilestones from '~/boards/graphql/group_board_milestones.query.graphql'; import { mockLists, mockListsById, @@ -308,6 +310,36 @@ describe('fetchMilestones', () => { expect(() => actions.fetchMilestones(store)).toThrow(new Error('Unknown board type')); }); + it.each([ + [ + 'project', + { + query: projectBoardMilestones, + variables: { fullPath: 'gitlab-org/gitlab', state: 'active' }, + }, + ], + [ + 'group', + { + query: groupBoardMilestones, + variables: { fullPath: 'gitlab-org/gitlab', state: 'active' }, + }, + ], + ])( + 'when boardType is %s it calls fetchMilestones with the correct query and variables', + (boardType, variables) => { + jest.spyOn(gqlClient, 'query').mockResolvedValue(queryResponse); + + const store = createStore(); + + store.state.boardType = boardType; + + actions.fetchMilestones(store); + + expect(gqlClient.query).toHaveBeenCalledWith(variables); + }, + ); + it('sets milestonesLoading to true', async () => { jest.spyOn(gqlClient, 'query').mockResolvedValue(queryResponse); diff --git a/spec/models/clusters/agent_spec.rb b/spec/models/clusters/agent_spec.rb index 692831f6cca..f279e779de5 100644 --- a/spec/models/clusters/agent_spec.rb +++ b/spec/models/clusters/agent_spec.rb @@ -116,4 +116,19 @@ RSpec.describe Clusters::Agent do it { is_expected.to be_truthy } end end + + describe '#activity_event_deletion_cutoff' do + let_it_be(:agent) { create(:cluster_agent) } + let_it_be(:event1) { create(:agent_activity_event, agent: agent, recorded_at: 1.hour.ago) } + let_it_be(:event2) { create(:agent_activity_event, agent: agent, recorded_at: 2.hours.ago) } + let_it_be(:event3) { create(:agent_activity_event, agent: agent, recorded_at: 3.hours.ago) } + + subject { agent.activity_event_deletion_cutoff } + + before do + stub_const("#{described_class}::ACTIVITY_EVENT_LIMIT", 2) + end + + it { is_expected.to be_like_time(event2.recorded_at) } + end end diff --git a/spec/models/clusters/agents/activity_event_spec.rb b/spec/models/clusters/agents/activity_event_spec.rb index 18b9c82fa6a..2e3833898fd 100644 --- a/spec/models/clusters/agents/activity_event_spec.rb +++ b/spec/models/clusters/agents/activity_event_spec.rb @@ -16,11 +16,10 @@ RSpec.describe Clusters::Agents::ActivityEvent do let_it_be(:agent) { create(:cluster_agent) } describe '.in_timeline_order' do - let(:recorded_at) { 1.hour.ago } - - let!(:event1) { create(:agent_activity_event, agent: agent, recorded_at: recorded_at) } - let!(:event2) { create(:agent_activity_event, agent: agent, recorded_at: Time.current) } - let!(:event3) { create(:agent_activity_event, agent: agent, recorded_at: recorded_at) } + let_it_be(:recorded_at) { 1.hour.ago } + let_it_be(:event1) { create(:agent_activity_event, agent: agent, recorded_at: recorded_at) } + let_it_be(:event2) { create(:agent_activity_event, agent: agent, recorded_at: Time.current) } + let_it_be(:event3) { create(:agent_activity_event, agent: agent, recorded_at: recorded_at) } subject { described_class.in_timeline_order } @@ -28,5 +27,19 @@ RSpec.describe Clusters::Agents::ActivityEvent do is_expected.to eq([event2, event3, event1]) end end + + describe '.recorded_before' do + let_it_be(:event1) { create(:agent_activity_event, agent: agent, recorded_at: 1.hour.ago) } + let_it_be(:event2) { create(:agent_activity_event, agent: agent, recorded_at: 2.hours.ago) } + let_it_be(:event3) { create(:agent_activity_event, agent: agent, recorded_at: 3.hours.ago) } + + let(:cutoff) { event2.recorded_at } + + subject { described_class.recorded_before(cutoff) } + + it 'returns only events recorded before the cutoff' do + is_expected.to contain_exactly(event3) + end + end end end diff --git a/spec/services/clusters/agents/create_activity_event_service_spec.rb b/spec/services/clusters/agents/create_activity_event_service_spec.rb new file mode 100644 index 00000000000..7a8f0e16d60 --- /dev/null +++ b/spec/services/clusters/agents/create_activity_event_service_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Clusters::Agents::CreateActivityEventService do + let_it_be(:agent) { create(:cluster_agent) } + let_it_be(:token) { create(:cluster_agent_token, agent: agent) } + let_it_be(:user) { create(:user) } + + describe '#execute' do + let(:params) do + { + kind: :token_created, + level: :info, + recorded_at: token.created_at, + user: user, + agent_token: token + } + end + + subject { described_class.new(agent, **params).execute } + + it 'creates an activity event record' do + expect { subject }.to change(agent.activity_events, :count).from(0).to(1) + + event = agent.activity_events.last + + expect(event).to have_attributes( + kind: 'token_created', + level: 'info', + recorded_at: token.reload.created_at, + user: user, + agent_token_id: token.id + ) + end + + it 'schedules the cleanup worker' do + expect(Clusters::Agents::DeleteExpiredEventsWorker).to receive(:perform_at) + .with(1.hour.from_now.change(min: agent.id % 60), agent.id) + + subject + end + end +end diff --git a/spec/services/clusters/agents/delete_expired_events_service_spec.rb b/spec/services/clusters/agents/delete_expired_events_service_spec.rb new file mode 100644 index 00000000000..3dc166f54eb --- /dev/null +++ b/spec/services/clusters/agents/delete_expired_events_service_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Clusters::Agents::DeleteExpiredEventsService do + let_it_be(:agent) { create(:cluster_agent) } + + describe '#execute' do + let_it_be(:event1) { create(:agent_activity_event, agent: agent, recorded_at: 1.hour.ago) } + let_it_be(:event2) { create(:agent_activity_event, agent: agent, recorded_at: 2.hours.ago) } + let_it_be(:event3) { create(:agent_activity_event, agent: agent, recorded_at: 3.hours.ago) } + let_it_be(:event4) { create(:agent_activity_event, agent: agent, recorded_at: 4.hours.ago) } + let_it_be(:event5) { create(:agent_activity_event, agent: agent, recorded_at: 5.hours.ago) } + + let(:deletion_cutoff) { 1.day.ago } + + subject { described_class.new(agent).execute } + + before do + allow(agent).to receive(:activity_event_deletion_cutoff).and_return(deletion_cutoff) + end + + it 'does not delete events if the limit has not been reached' do + expect { subject }.not_to change(agent.activity_events, :count) + end + + context 'there are more events than the limit' do + let(:deletion_cutoff) { event3.recorded_at } + + it 'removes events to remain at the limit, keeping the most recent' do + expect { subject }.to change(agent.activity_events, :count).from(5).to(3) + expect(agent.activity_events).to contain_exactly(event1, event2, event3) + end + end + end +end diff --git a/spec/workers/clusters/agents/delete_expired_events_worker_spec.rb b/spec/workers/clusters/agents/delete_expired_events_worker_spec.rb new file mode 100644 index 00000000000..1a5ca744091 --- /dev/null +++ b/spec/workers/clusters/agents/delete_expired_events_worker_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Clusters::Agents::DeleteExpiredEventsWorker do + let(:agent) { create(:cluster_agent) } + + describe '#perform' do + let(:agent_id) { agent.id } + let(:deletion_service) { double(execute: true) } + + subject { described_class.new.perform(agent_id) } + + it 'calls the deletion service' do + expect(deletion_service).to receive(:execute).once + expect(Clusters::Agents::DeleteExpiredEventsService).to receive(:new) + .with(agent).and_return(deletion_service) + + subject + end + + context 'agent no longer exists' do + let(:agent_id) { -1 } + + it 'completes without raising an error' do + expect { subject }.not_to raise_error + end + end + end +end diff --git a/spec/workers/concerns/cluster_agent_queue_spec.rb b/spec/workers/concerns/cluster_agent_queue_spec.rb new file mode 100644 index 00000000000..b5189cbd8c8 --- /dev/null +++ b/spec/workers/concerns/cluster_agent_queue_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ClusterAgentQueue do + let(:worker) do + Class.new do + def self.name + 'ExampleWorker' + end + + include ApplicationWorker + include ClusterAgentQueue + end + end + + it { expect(worker.queue).to eq('cluster_agent:example') } + it { expect(worker.get_feature_category).to eq(:kubernetes_management) } +end |