# frozen_string_literal: true module Backup # Backup options provided by the command line interface class Options # SkippableTasks store which tasks can be skipped # Setting any one to true, will create or restore a backup without that data # @example Skipping database content and CI job artifacts # SkippableTasks.new(db: true, artifacts: true) SkippableTasks = Struct.new( :db, # Database content (PostgreSQL) :uploads, # Attachments :builds, # CI job output logs :artifacts, # CI job artifacts :lfs, # LFS objects :terraform_state, # Terraform states :registry, # Container registry images :pages, # GitLab Pages content :repositories, # Repositories :packages, # Packages :ci_secure_files, # Project-level Secure Files keyword_init: true ) # What operations can be skipped SkippableOperations = Struct.new( :archive, # whether to skip .tar step :remote_storage, # whether to skip uploading to remote storage keyword_init: true ) CompressionOptions = Struct.new( :compression_cmd, # custom compression command :decompression_cmd, # custom decompression command keyword_init: true ) # Backup ID is the backup filename portion without extensions # When this option is not provided, the backup name will be based on date, timestamp and gitlab version # # @return [String|Nil] backup id that is used as part of filename attr_accessor :backup_id # Reference to previous backup full path # # @return [String|Nil] previous backup full path attr_accessor :previous_backup # Run incremental backup? # # @return [Boolean] whether to run an incremental backup attr_accessor :incremental alias_method :incremental?, :incremental # Whether to bypass warnings when performing dangerous operations # This is currently being used for the database restore task only # # @return [Boolean] whether to bypass warnings and perform dangerous operations attr_accessor :force alias_method :force?, :force # A list of all tasks and whether they can be skipped or not # # @return [SkippableTasks] attr_accessor :skippable_tasks # A list of all operations and whether they can be skipped or not # # @return [SkippableOperations] attr_accessor :skippable_operations # When using multiple repository storages, repositories can be backed up and restored in parallel # This option allows to customize the overall limit. # # This is only used by repository backup and restore steps (GitalyBackup) # # @return [Integer|Nil] limit of backup or restore operations to happen in parallel overall attr_accessor :max_parallelism # When using multiple repository storages, repositories can be backed up and restored in parallel # This option allows to customize the limit per storage. # # This is only used by repository backup and restore steps (GitalyBackup) # # @return [Integer|Nil] limit of backup or restore operations to happen in parallel per storage attr_accessor :max_storage_parallelism # When using multiple repository storages, repositories from specific storages can be backed up # separately by running the backup operation while setting this option # # @return [Array] a list of repository storages to be backed up attr_accessor :repositories_storages # In order to backup specific repositories, multiple paths containing the # selected namespaces will be used to find which repositories to backup # # Ex: ['group-a', 'group-b/project-c'] will select all projects in group-a and project-c in group-b # This can be combined with #skip_repositories_paths # # @return [Array] a list of paths to select which repositories to backup attr_accessor :repositories_paths # In order to backup specific repositories, multiple paths containing the # selected namespaces can be specified using #repositories_paths. To further # refine the list, a new list of paths can be provided to be skipped among # the previous pre-selected ones. # # Ex: for a repository_paths containing ['group-a'] and skip_repository_paths # containing ['group-a/project-d'], all projects in `group-a` except `project-d` # will be backed up # # @return [Array] a list of paths to skip backup attr_accessor :skip_repositories_paths # Specify GitalyBackup to handle and perform backups server-side and stream it to object storage # # When this is defined, repositories will not be part of the backup archive # @return [Boolean] whether to perform server-side backups for repositories attr_accessor :repositories_server_side_backup # A custom directory to send your remote backups to # It can be used to group different types of backups (ex: daily, weekly) # # @return [String|Nil] attr_accessor :remote_directory # Custom compression and decompression options # # When compression is customized, it will ignore other related options like `:gzip_rsyncable` # @return [CompressionOptions] custom compression and decompression commands attr_accessor :compression_options # Whether to run gzip with `--rsyncable` flag # # This is ignored if custom :compression_options are provided # @return [Boolean] whether to use `--rsyncable` flag with gzip attr_accessor :gzip_rsyncable # rubocop:disable Metrics/ParameterLists -- This is a data object with all possible CMD options def initialize( backup_id: nil, previous_backup: nil, incremental: false, force: false, skippable_tasks: SkippableTasks.new, skippable_operations: SkippableOperations.new, max_parallelism: nil, max_storage_parallelism: nil, repository_storages: [], repository_paths: [], skip_repository_paths: [], repositories_server_side_backup: false, remote_directory: nil, compression_options: CompressionOptions.new, gzip_rsyncable: false) @backup_id = backup_id @previous_backup = previous_backup @incremental = incremental @force = force @skippable_tasks = skippable_tasks @skippable_operations = skippable_operations @max_parallelism = max_parallelism @max_storage_parallelism = max_storage_parallelism @remote_directory = remote_directory @repositories_server_side_backup = repositories_server_side_backup @repositories_storages = repository_storages @repositories_paths = repository_paths @skip_repositories_paths = skip_repository_paths @compression_options = compression_options @gzip_rsyncable = gzip_rsyncable end # rubocop:enable Metrics/ParameterLists # rubocop:disable Metrics/CyclomaticComplexity, Metrics/AbcSize -- TODO: Complexity will be solved in the Unified Backup implementation (https://gitlab.com/groups/gitlab-org/-/epics/11635) # Extract supported options from defined ENV variables def extract_from_env! # We've used lowercase `force` as the key while ENV normally is defined using UPPERCASE letters # This provides a fallback when the user defines using expected standards, while not breaking compatibility force_value = ENV.key?('FORCE') ? ENV['FORCE'] : ENV['force'] self.backup_id = ENV['BACKUP'] self.previous_backup = ENV['PREVIOUS_BACKUP'] self.incremental = Gitlab::Utils.to_boolean(ENV['INCREMENTAL'], default: incremental) self.force = Gitlab::Utils.to_boolean(force_value, default: force) self.max_parallelism = ENV['GITLAB_BACKUP_MAX_CONCURRENCY']&.to_i self.max_storage_parallelism = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY']&.to_i self.remote_directory = ENV['DIRECTORY'] self.repositories_server_side_backup = Gitlab::Utils.to_boolean(ENV['REPOSITORIES_SERVER_SIDE'], default: repositories_server_side_backup) self.repositories_storages = ENV.fetch('REPOSITORIES_STORAGES', '').split(',').uniq self.repositories_paths = ENV.fetch('REPOSITORIES_PATHS', '').split(',').uniq self.skip_repositories_paths = ENV.fetch('SKIP_REPOSITORIES_PATHS', '').split(',').uniq compression_options.compression_cmd = ENV['COMPRESS_CMD'] compression_options.decompression_cmd = ENV['DECOMPRESS_CMD'] self.gzip_rsyncable = Gitlab::Utils.to_boolean(ENV['GZIP_RSYNCABLE'], default: gzip_rsyncable) extract_skippables!(ENV['SKIP']) if ENV['SKIP'].present? end # rubocop:enable Metrics/CyclomaticComplexity, Metrics/AbcSize def update_from_backup_information!(backup_information) self.repositories_storages += backup_information[:repositories_storages]&.split(',') || [] self.repositories_storages.uniq! self.repositories_storages.compact! self.repositories_paths += backup_information[:repositories_paths]&.split(',') || [] self.repositories_paths.uniq! self.repositories_paths.compact! self.skip_repositories_paths += backup_information[:skip_repositories_paths]&.split(',') || [] self.skip_repositories_paths.uniq! self.skip_repositories_paths.compact! extract_skippables!(backup_information[:skipped]) if backup_information[:skipped] end # rubocop:disable Metrics/CyclomaticComplexity -- TODO: Complexity will be solved in the Unified Backup implementation (https://gitlab.com/groups/gitlab-org/-/epics/11635) # Return a String with a list of skippables, separated by commas # # @return [String] a list of skippables def serialize_skippables list = [] list << 'tar' if skippable_operations.archive list << 'remote' if skippable_operations.remote_storage list << 'db' if skippable_tasks.db list << 'uploads' if skippable_tasks.uploads list << 'builds' if skippable_tasks.builds list << 'artifacts' if skippable_tasks.artifacts list << 'lfs' if skippable_tasks.lfs list << 'terraform_state' if skippable_tasks.terraform_state list << 'registry' if skippable_tasks.registry list << 'pages' if skippable_tasks.pages list << 'repositories' if skippable_tasks.repositories list << 'packages' if skippable_tasks.packages list << 'ci_secure_files' if skippable_tasks.ci_secure_files list.join(',') end # rubocop:enable Metrics/CyclomaticComplexity # Extract skippables from provided data field # Current callers will provide either ENV['SKIP'] or backup_information[:skipped] content # # The first time the method is executed it will setup `true` or `false` to each field # subsequent executions will preserve `true` values and evaluate again only when previously set to `false` # # @param [String] field contains a list separated by comma without surrounding spaces def extract_skippables!(field) list = field.split(',').uniq extract_skippable_operations!(list) extract_skippable_tasks(list) end def skip_task?(task_name) !!skippable_tasks[task_name] end private def extract_skippable_operations!(list) skippable_operations.archive ||= list.include?('tar') # SKIP=tar skippable_operations.remote_storage ||= list.include?('remote') # SKIP=remote end def extract_skippable_tasks(list) skippable_tasks.db ||= list.include?('db') # SKIP=db skippable_tasks.uploads ||= list.include?('uploads') # SKIP=uploads skippable_tasks.builds ||= list.include?('builds') # SKIP=builds skippable_tasks.artifacts ||= list.include?('artifacts') # SKIP=artifacts skippable_tasks.lfs ||= list.include?('lfs') # SKIP=lfs skippable_tasks.terraform_state ||= list.include?('terraform_state') # SKIP=terraform_state skippable_tasks.registry ||= list.include?('registry') # SKIP=registry skippable_tasks.pages ||= list.include?('pages') # SKIP=pages skippable_tasks.repositories ||= list.include?('repositories') # SKIP=repositories skippable_tasks.packages ||= list.include?('packages') # SKIP=packages skippable_tasks.ci_secure_files ||= list.include?('ci_secure_files') # SKIP=ci_secure_files end end end