diff options
Diffstat (limited to 'lib/gitlab')
| -rw-r--r-- | lib/gitlab/daemon.rb | 62 | ||||
| -rw-r--r-- | lib/gitlab/git/blob.rb | 136 | ||||
| -rw-r--r-- | lib/gitlab/git/repository.rb | 46 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client/commit_service.rb | 14 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client/repository_service.rb | 5 | ||||
| -rw-r--r-- | lib/gitlab/import_export/import_export.yml | 1 | ||||
| -rw-r--r-- | lib/gitlab/import_sources.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/key_fingerprint.rb | 71 | ||||
| -rw-r--r-- | lib/gitlab/metrics/base_sampler.rb | 75 | ||||
| -rw-r--r-- | lib/gitlab/metrics/sidekiq_metrics_exporter.rb | 39 | ||||
| -rw-r--r-- | lib/gitlab/project_template.rb | 45 | ||||
| -rw-r--r-- | lib/gitlab/shell.rb | 24 |
12 files changed, 346 insertions, 174 deletions
diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb new file mode 100644 index 00000000000..dfd17e35707 --- /dev/null +++ b/lib/gitlab/daemon.rb @@ -0,0 +1,62 @@ +module Gitlab + class Daemon + def self.initialize_instance(*args) + raise "#{name} singleton instance already initialized" if @instance + @instance = new(*args) + Kernel.at_exit(&@instance.method(:stop)) + @instance + end + + def self.instance + @instance ||= initialize_instance + end + + attr_reader :thread + + def thread? + !thread.nil? + end + + def initialize + @mutex = Mutex.new + end + + def enabled? + true + end + + def start + return unless enabled? + + @mutex.synchronize do + return thread if thread? + + @thread = Thread.new { start_working } + end + end + + def stop + @mutex.synchronize do + return unless thread? + + stop_working + + if thread + thread.wakeup if thread.alive? + thread.join + @thread = nil + end + end + end + + private + + def start_working + raise NotImplementedError + end + + def stop_working + # no-ops + end + end +end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index db6cfc9671f..77b81d2d437 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -20,66 +20,7 @@ module Gitlab if is_enabled find_by_gitaly(repository, sha, path) else - find_by_rugged(repository, sha, path) - end - end - end - - def find_by_gitaly(repository, sha, path) - path = path.sub(/\A\/*/, '') - path = '/' if path.empty? - name = File.basename(path) - entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) - return unless entry - - case entry.type - when :COMMIT - new( - id: entry.oid, - name: name, - size: 0, - data: '', - path: path, - commit_id: sha - ) - when :BLOB - new( - id: entry.oid, - name: name, - size: entry.size, - data: entry.data.dup, - mode: entry.mode.to_s(8), - path: path, - commit_id: sha, - binary: binary?(entry.data) - ) - end - end - - def find_by_rugged(repository, sha, path) - commit = repository.lookup(sha) - root_tree = commit.tree - - blob_entry = find_entry_by_path(repository, root_tree.oid, path) - - return nil unless blob_entry - - if blob_entry[:type] == :commit - submodule_blob(blob_entry, path, sha) - else - blob = repository.lookup(blob_entry[:oid]) - - if blob - new( - id: blob.oid, - name: blob_entry[:name], - size: blob.size, - data: blob.content(MAX_DATA_DISPLAY_SIZE), - mode: blob_entry[:filemode].to_s(8), - path: path, - commit_id: sha, - binary: blob.binary? - ) + find_by_rugged(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) end end end @@ -109,6 +50,21 @@ module Gitlab detect && detect[:type] == :binary end + # Returns an array of Blob instances, specified in blob_references as + # [[commit_sha, path], [commit_sha, path], ...]. If blob_size_limit < 0 then the + # full blob contents are returned. If blob_size_limit >= 0 then each blob will + # contain no more than limit bytes in its data attribute. + # + # Keep in mind that this method may allocate a lot of memory. It is up + # to the caller to limit the number of blobs and blob_size_limit. + # + def batch(repository, blob_references, blob_size_limit: nil) + blob_size_limit ||= MAX_DATA_DISPLAY_SIZE + blob_references.map do |sha, path| + find_by_rugged(repository, sha, path, limit: blob_size_limit) + end + end + private # Recursive search of blob id by path @@ -153,6 +109,66 @@ module Gitlab commit_id: sha ) end + + def find_by_gitaly(repository, sha, path) + path = path.sub(/\A\/*/, '') + path = '/' if path.empty? + name = File.basename(path) + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) + return unless entry + + case entry.type + when :COMMIT + new( + id: entry.oid, + name: name, + size: 0, + data: '', + path: path, + commit_id: sha + ) + when :BLOB + new( + id: entry.oid, + name: name, + size: entry.size, + data: entry.data.dup, + mode: entry.mode.to_s(8), + path: path, + commit_id: sha, + binary: binary?(entry.data) + ) + end + end + + def find_by_rugged(repository, sha, path, limit:) + commit = repository.lookup(sha) + root_tree = commit.tree + + blob_entry = find_entry_by_path(repository, root_tree.oid, path) + + return nil unless blob_entry + + if blob_entry[:type] == :commit + submodule_blob(blob_entry, path, sha) + else + blob = repository.lookup(blob_entry[:oid]) + + if blob + new( + id: blob.oid, + name: blob_entry[:name], + size: blob.size, + # Rugged::Blob#content is expensive; don't call it if we don't have to. + data: limit.zero? ? '' : blob.content(limit), + mode: blob_entry[:filemode].to_s(8), + path: path, + commit_id: sha, + binary: blob.binary? + ) + end + end + end end def initialize(options) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 1005a819f95..f246393cfbc 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -282,7 +282,14 @@ module Gitlab # Return repo size in megabytes def size - size = popen(%w(du -sk), path).first.strip.to_i + size = gitaly_migrate(:repository_size) do |is_enabled| + if is_enabled + size_by_gitaly + else + size_by_shelling_out + end + end + (size.to_f / 1024).round(2) end @@ -299,6 +306,21 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/446 def log(options) + default_options = { + limit: 10, + offset: 0, + path: nil, + follow: false, + skip_merges: false, + disable_walk: false, + after: nil, + before: nil + } + + options = default_options.merge(options) + options[:limit] ||= 0 + options[:offset] ||= 0 + raw_log(options).map { |c| Commit.decorate(c) } end @@ -712,20 +734,6 @@ module Gitlab end def raw_log(options) - default_options = { - limit: 10, - offset: 0, - path: nil, - follow: false, - skip_merges: false, - disable_walk: false, - after: nil, - before: nil - } - - options = default_options.merge(options) - options[:limit] ||= 0 - options[:offset] ||= 0 actual_ref = options[:ref] || root_ref begin sha = sha_from_ref(actual_ref) @@ -942,6 +950,14 @@ module Gitlab gitaly_ref_client.tags end + def size_by_shelling_out + popen(%w(du -sk), path).first.strip.to_i + end + + def size_by_gitaly + gitaly_repository_client.repository_size + end + def count_commits_by_gitaly(options) gitaly_commit_client.commit_count(options[:ref], options) end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index ac6817e6d0e..3f577ac8530 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -135,6 +135,20 @@ module Gitlab consume_commits_response(response) end + def commits_by_message(query, revision: '', path: '', limit: 1000, offset: 0) + request = Gitaly::CommitsByMessageRequest.new( + repository: @gitaly_repo, + query: query, + revision: revision.to_s.force_encoding(Encoding::ASCII_8BIT), + path: path.to_s.force_encoding(Encoding::ASCII_8BIT), + limit: limit.to_i, + offset: offset.to_i + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request) + consume_commits_response(response) + end + def languages(ref = nil) request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '') response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 13e75b256a7..79ce784f2f2 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -27,6 +27,11 @@ module Gitlab request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo) GitalyClient.call(@storage, :repository_service, :repack_incremental, request) end + + def repository_size + request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :repository_size, request).size + end end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index c8ad3a7a5e0..c5c05bfe2fb 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -101,6 +101,7 @@ excluded_attributes: merge_requests: - :milestone_id - :ref_fetched + - :merge_jid award_emoji: - :awardable_id statuses: diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 52276cbcd9a..5404dc11a87 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -8,7 +8,7 @@ module Gitlab ImportSource = Struct.new(:name, :title, :importer) ImportTable = [ - ImportSource.new('github', 'GitHub', Gitlab::GithubImport::Importer), + ImportSource.new('github', 'GitHub', Github::Import), ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb index b75ae512d92..d9a79f7c291 100644 --- a/lib/gitlab/key_fingerprint.rb +++ b/lib/gitlab/key_fingerprint.rb @@ -1,55 +1,48 @@ module Gitlab class KeyFingerprint - include Gitlab::Popen + attr_reader :key, :ssh_key - attr_accessor :key + # Unqualified MD5 fingerprint for compatibility + delegate :fingerprint, to: :ssh_key, allow_nil: true def initialize(key) @key = key - end - - def fingerprint - cmd_status = 0 - cmd_output = '' - - Tempfile.open('gitlab_key_file') do |file| - file.puts key - file.rewind - - cmd = [] - cmd.push('ssh-keygen') - cmd.push('-E', 'md5') if explicit_fingerprint_algorithm? - cmd.push('-lf', file.path) - - cmd_output, cmd_status = popen(cmd, '/tmp') - end - - return nil unless cmd_status.zero? - # 16 hex bytes separated by ':', optionally starting with "MD5:" - fingerprint_matches = cmd_output.match(/(MD5:)?(?<fingerprint>(\h{2}:){15}\h{2})/) - return nil unless fingerprint_matches - - fingerprint_matches[:fingerprint] + @ssh_key = + begin + Net::SSH::KeyFactory.load_data_public_key(key) + rescue Net::SSH::Exception, NotImplementedError + end end - private - - def explicit_fingerprint_algorithm? - # OpenSSH 6.8 introduces a new default output format for fingerprints. - # Check the version and decide which command to use. - - version_output, version_status = popen(%w(ssh -V)) - return false unless version_status.zero? + def valid? + ssh_key.present? + end - version_matches = version_output.match(/OpenSSH_(?<major>\d+)\.(?<minor>\d+)/) - return false unless version_matches + def type + return unless valid? - version_info = Gitlab::VersionInfo.new(version_matches[:major].to_i, version_matches[:minor].to_i) + parts = ssh_key.ssh_type.split('-') + parts.shift if parts[0] == 'ssh' - required_version_info = Gitlab::VersionInfo.new(6, 8) + parts[0].upcase + end - version_info >= required_version_info + def bits + return unless valid? + + case type + when 'RSA' + ssh_key.n.num_bits + when 'DSS', 'DSA' + ssh_key.p.num_bits + when 'ECDSA' + ssh_key.group.order.num_bits + when 'ED25519' + 256 + else + raise "Unsupported key type: #{type}" + end end end end diff --git a/lib/gitlab/metrics/base_sampler.rb b/lib/gitlab/metrics/base_sampler.rb index 219accfc029..716d20bb91a 100644 --- a/lib/gitlab/metrics/base_sampler.rb +++ b/lib/gitlab/metrics/base_sampler.rb @@ -1,20 +1,7 @@ require 'logger' module Gitlab module Metrics - class BaseSampler - def self.initialize_instance(*args) - raise "#{name} singleton instance already initialized" if @instance - @instance = new(*args) - at_exit(&@instance.method(:stop)) - @instance - end - - def self.instance - @instance - end - - attr_reader :running - + class BaseSampler < Daemon # interval - The sampling interval in seconds. def initialize(interval) interval_half = interval.to_f / 2 @@ -22,44 +9,7 @@ module Gitlab @interval = interval @interval_steps = (-interval_half..interval_half).step(0.1).to_a - @mutex = Mutex.new - end - - def enabled? - true - end - - def start - return unless enabled? - - @mutex.synchronize do - return if running - @running = true - - @thread = Thread.new do - sleep(sleep_interval) - - while running - safe_sample - - sleep(sleep_interval) - end - end - end - end - - def stop - @mutex.synchronize do - return unless running - - @running = false - - if @thread - @thread.wakeup if @thread.alive? - @thread.join - @thread = nil - end - end + super() end def safe_sample @@ -81,7 +31,7 @@ module Gitlab # potentially missing anything that happens in between samples). # 2. Don't sample data at the same interval two times in a row. def sleep_interval - while step = @interval_steps.sample + while (step = @interval_steps.sample) if step != @last_step @last_step = step @@ -89,6 +39,25 @@ module Gitlab end end end + + private + + attr_reader :running + + def start_working + @running = true + sleep(sleep_interval) + + while running + safe_sample + + sleep(sleep_interval) + end + end + + def stop_working + @running = false + end end end end diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb new file mode 100644 index 00000000000..5980a4ded2b --- /dev/null +++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb @@ -0,0 +1,39 @@ +require 'webrick' +require 'prometheus/client/rack/exporter' + +module Gitlab + module Metrics + class SidekiqMetricsExporter < Daemon + def enabled? + Gitlab::Metrics.metrics_folder_present? && settings.enabled + end + + def settings + Settings.monitoring.sidekiq_exporter + end + + private + + attr_reader :server + + def start_working + @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address) + server.mount "/", Rack::Handler::WEBrick, rack_app + server.start + end + + def stop_working + server.shutdown + @server = nil + end + + def rack_app + Rack::Builder.app do + use Rack::Deflater + use ::Prometheus::Client::Rack::Exporter + run -> (env) { [404, {}, ['']] } + end + end + end + end +end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb new file mode 100644 index 00000000000..cf461adf697 --- /dev/null +++ b/lib/gitlab/project_template.rb @@ -0,0 +1,45 @@ +module Gitlab + class ProjectTemplate + attr_reader :title, :name + + def initialize(name, title) + @name, @title = name, title + end + + alias_method :logo, :name + + def file + archive_path.open + end + + def archive_path + Rails.root.join("vendor/project_templates/#{name}.tar.gz") + end + + def clone_url + "https://gitlab.com/gitlab-org/project-templates/#{name}.git" + end + + def ==(other) + name == other.name && title == other.title + end + + TEMPLATES_TABLE = [ + ProjectTemplate.new('rails', 'Ruby on Rails') + ].freeze + + class << self + def all + TEMPLATES_TABLE + end + + def find(name) + all.find { |template| template.name == name.to_s } + end + + def archive_directory + Rails.root.join("vendor_directory/project_templates") + end + end + end +end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 4366ff336ef..0cb28732402 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -105,12 +105,24 @@ module Gitlab # fetch_remote("gitlab/gitlab-ci", "upstream") # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 - def fetch_remote(storage, name, remote, forced: false, no_tags: false) + def fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false) args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] args << '--force' if forced args << '--no-tags' if no_tags - gitlab_shell_fast_execute_raise_error(args) + vars = {} + + if ssh_auth&.ssh_import? + if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? + vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key + end + + if ssh_auth.ssh_known_hosts.present? + vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts + end + end + + gitlab_shell_fast_execute_raise_error(args, vars) end # Move repository @@ -293,15 +305,15 @@ module Gitlab false end - def gitlab_shell_fast_execute_raise_error(cmd) - output, status = gitlab_shell_fast_execute_helper(cmd) + def gitlab_shell_fast_execute_raise_error(cmd, vars = {}) + output, status = gitlab_shell_fast_execute_helper(cmd, vars) raise Error, output unless status.zero? true end - def gitlab_shell_fast_execute_helper(cmd) - vars = ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS) + def gitlab_shell_fast_execute_helper(cmd, vars = {}) + vars.merge!(ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS)) # Don't pass along the entire parent environment to prevent gitlab-shell # from wasting I/O by searching through GEM_PATH |