Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/banzai/filter/abstract_reference_filter.rb81
-rw-r--r--lib/banzai/filter/milestone_reference_filter.rb34
-rw-r--r--lib/declarative_policy/runner.rb12
-rw-r--r--lib/github/client.rb36
-rw-r--r--lib/github/import.rb34
-rw-r--r--lib/gitlab/daemon.rb62
-rw-r--r--lib/gitlab/git/blob.rb136
-rw-r--r--lib/gitlab/git/repository.rb46
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb14
-rw-r--r--lib/gitlab/gitaly_client/repository_service.rb5
-rw-r--r--lib/gitlab/import_export/import_export.yml1
-rw-r--r--lib/gitlab/import_sources.rb2
-rw-r--r--lib/gitlab/key_fingerprint.rb71
-rw-r--r--lib/gitlab/metrics/base_sampler.rb75
-rw-r--r--lib/gitlab/metrics/sidekiq_metrics_exporter.rb39
-rw-r--r--lib/gitlab/project_template.rb45
-rw-r--r--lib/gitlab/shell.rb24
-rw-r--r--lib/tasks/gitlab/update_templates.rake49
-rw-r--r--lib/tasks/import.rake3
19 files changed, 530 insertions, 239 deletions
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index 685b43605ae..ef4578aabd6 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -54,42 +54,42 @@ module Banzai
self.class.references_in(*args, &block)
end
+ # Implement in child class
+ # Example: project.merge_requests.find
def find_object(project, id)
- # Implement in child class
- # Example: project.merge_requests.find
end
- def find_object_cached(project, id)
- if RequestStore.active?
- cache = find_objects_cache[object_class][project.id]
+ # Override if the link reference pattern produces a different ID (global
+ # ID vs internal ID, for instance) to the regular reference pattern.
+ def find_object_from_link(project, id)
+ find_object(project, id)
+ end
- get_or_set_cache(cache, id) { find_object(project, id) }
- else
+ # Implement in child class
+ # Example: project_merge_request_url
+ def url_for_object(object, project)
+ end
+
+ def find_object_cached(project, id)
+ cached_call(:banzai_find_object, id, path: [object_class, project.id]) do
find_object(project, id)
end
end
- def project_from_ref_cached(ref)
- if RequestStore.active?
- cache = project_refs_cache
-
- get_or_set_cache(cache, ref) { project_from_ref(ref) }
- else
- project_from_ref(ref)
+ def find_object_from_link_cached(project, id)
+ cached_call(:banzai_find_object_from_link, id, path: [object_class, project.id]) do
+ find_object_from_link(project, id)
end
end
- def url_for_object(object, project)
- # Implement in child class
- # Example: project_merge_request_url
+ def project_from_ref_cached(ref)
+ cached_call(:banzai_project_refs, ref) do
+ project_from_ref(ref)
+ end
end
def url_for_object_cached(object, project)
- if RequestStore.active?
- cache = url_for_object_cache[object_class][project.id]
-
- get_or_set_cache(cache, object) { url_for_object(object, project) }
- else
+ cached_call(:banzai_url_for_object, object, path: [object_class, project.id]) do
url_for_object(object, project)
end
end
@@ -120,7 +120,7 @@ module Banzai
if link == inner_html && inner_html =~ /\A#{link_pattern}/
replace_link_node_with_text(node, link) do
- object_link_filter(inner_html, link_pattern)
+ object_link_filter(inner_html, link_pattern, link_reference: true)
end
next
@@ -128,7 +128,7 @@ module Banzai
if link =~ /\A#{link_pattern}\z/
replace_link_node_with_href(node, link) do
- object_link_filter(link, link_pattern, link_content: inner_html)
+ object_link_filter(link, link_pattern, link_content: inner_html, link_reference: true)
end
next
@@ -146,15 +146,26 @@ module Banzai
# text - String text to replace references in.
# pattern - Reference pattern to match against.
# link_content - Original content of the link being replaced.
+ # link_reference - True if this was using the link reference pattern,
+ # false otherwise.
#
# Returns a String with references replaced with links. All links
# have `gfm` and `gfm-OBJECT_NAME` class names attached for styling.
- def object_link_filter(text, pattern, link_content: nil)
+ def object_link_filter(text, pattern, link_content: nil, link_reference: false)
references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches|
project_path = full_project_path(namespace_ref, project_ref)
project = project_from_ref_cached(project_path)
- if project && object = find_object_cached(project, id)
+ if project
+ object =
+ if link_reference
+ find_object_from_link_cached(project, id)
+ else
+ find_object_cached(project, id)
+ end
+ end
+
+ if object
title = object_link_title(object)
klass = reference_class(object_sym)
@@ -297,15 +308,17 @@ module Banzai
RequestStore[:banzai_project_refs] ||= {}
end
- def find_objects_cache
- RequestStore[:banzai_find_objects_cache] ||= Hash.new do |hash, key|
- hash[key] = Hash.new { |h, k| h[k] = {} }
- end
- end
+ def cached_call(request_store_key, cache_key, path: [])
+ if RequestStore.active?
+ cache = RequestStore[request_store_key] ||= Hash.new do |hash, key|
+ hash[key] = Hash.new { |h, k| h[k] = {} }
+ end
- def url_for_object_cache
- RequestStore[:banzai_url_for_object] ||= Hash.new do |hash, key|
- hash[key] = Hash.new { |h, k| h[k] = {} }
+ cache = cache.dig(*path) if path.any?
+
+ get_or_set_cache(cache, cache_key) { yield }
+ else
+ yield
end
end
diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb
index 45c033d32a8..4fc5f211e84 100644
--- a/lib/banzai/filter/milestone_reference_filter.rb
+++ b/lib/banzai/filter/milestone_reference_filter.rb
@@ -8,8 +8,15 @@ module Banzai
Milestone
end
+ # Links to project milestones contain the IID, but when we're handling
+ # 'regular' references, we need to use the global ID to disambiguate
+ # between group and project milestones.
def find_object(project, id)
- project.milestones.find_by(iid: id)
+ find_milestone_with_finder(project, id: id)
+ end
+
+ def find_object_from_link(project, iid)
+ find_milestone_with_finder(project, iid: iid)
end
def references_in(text, pattern = Milestone.reference_pattern)
@@ -22,7 +29,7 @@ module Banzai
milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name])
if milestone
- yield match, milestone.iid, $~[:project], $~[:namespace], $~
+ yield match, milestone.id, $~[:project], $~[:namespace], $~
else
match
end
@@ -36,7 +43,8 @@ module Banzai
return unless project
milestone_params = milestone_params(milestone_id, milestone_name)
- project.milestones.find_by(milestone_params)
+
+ find_milestone_with_finder(project, milestone_params)
end
def milestone_params(iid, name)
@@ -47,15 +55,27 @@ module Banzai
end
end
+ def find_milestone_with_finder(project, params)
+ finder_params = { project_ids: [project.id], order: nil }
+
+ # We don't support IID lookups for group milestones, because IIDs can
+ # clash between group and project milestones.
+ if project.group && !params[:iid]
+ finder_params[:group_ids] = [project.group.id]
+ end
+
+ MilestonesFinder.new(finder_params).execute.find_by(params)
+ end
+
def url_for_object(milestone, project)
- h = Gitlab::Routing.url_helpers
- h.project_milestone_url(project, milestone,
- only_path: context[:only_path])
+ Gitlab::Routing
+ .url_helpers
+ .milestone_url(milestone, only_path: context[:only_path])
end
def object_link_text(object, matches)
milestone_link = escape_once(super)
- reference = object.project.to_reference(project)
+ reference = object.project&.to_reference(project)
if reference.present?
"#{milestone_link} <i>in #{reference}</i>".html_safe
diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb
index b5c615da4e3..56afd1f1392 100644
--- a/lib/declarative_policy/runner.rb
+++ b/lib/declarative_policy/runner.rb
@@ -76,6 +76,8 @@ module DeclarativePolicy
@state = State.new
steps_by_score do |step, score|
+ return if !debug && @state.prevented?
+
passed = nil
case step.action
when :enable then
@@ -93,10 +95,7 @@ module DeclarativePolicy
# been prevented.
unless @state.prevented?
passed = step.pass?
- if passed
- @state.prevent!
- return unless debug
- end
+ @state.prevent! if passed
end
debug << inspect_step(step, score, passed) if debug
@@ -141,13 +140,14 @@ module DeclarativePolicy
end
steps = Set.new(@steps)
+ remaining_enablers = steps.count { |s| s.enable? }
loop do
return if steps.empty?
# if the permission hasn't yet been enabled and we only have
# prevent steps left, we short-circuit the state here
- @state.prevent! if !@state.enabled? && steps.all?(&:prevent?)
+ @state.prevent! if !@state.enabled? && remaining_enablers == 0
lowest_score = Float::INFINITY
next_step = nil
@@ -162,6 +162,8 @@ module DeclarativePolicy
steps.delete(next_step)
+ remaining_enablers -= 1 if next_step.enable?
+
yield next_step, lowest_score
end
end
diff --git a/lib/github/client.rb b/lib/github/client.rb
index e65d908d232..9c476df7d46 100644
--- a/lib/github/client.rb
+++ b/lib/github/client.rb
@@ -1,13 +1,16 @@
module Github
class Client
+ TIMEOUT = 60
+
attr_reader :connection, :rate_limit
def initialize(options)
- @connection = Faraday.new(url: options.fetch(:url)) do |faraday|
- faraday.options.open_timeout = options.fetch(:timeout, 60)
- faraday.options.timeout = options.fetch(:timeout, 60)
+ @connection = Faraday.new(url: options.fetch(:url, root_endpoint)) do |faraday|
+ faraday.options.open_timeout = options.fetch(:timeout, TIMEOUT)
+ faraday.options.timeout = options.fetch(:timeout, TIMEOUT)
faraday.authorization 'token', options.fetch(:token)
faraday.adapter :net_http
+ faraday.ssl.verify = verify_ssl
end
@rate_limit = RateLimit.new(connection)
@@ -19,5 +22,32 @@ module Github
Github::Response.new(connection.get(url, query))
end
+
+ private
+
+ def root_endpoint
+ custom_endpoint || github_endpoint
+ end
+
+ def custom_endpoint
+ github_omniauth_provider.dig('args', 'client_options', 'site')
+ end
+
+ def verify_ssl
+ # If there is no config, we're connecting to github.com
+ # and we should verify ssl.
+ github_omniauth_provider.fetch('verify_ssl', true)
+ end
+
+ def github_endpoint
+ OmniAuth::Strategies::GitHub.default_options[:client_options][:site]
+ end
+
+ def github_omniauth_provider
+ @github_omniauth_provider ||=
+ Gitlab.config.omniauth.providers
+ .find { |provider| provider.name == 'github' }
+ .to_h
+ end
end
end
diff --git a/lib/github/import.rb b/lib/github/import.rb
index cea4be5460b..4cc01593ef4 100644
--- a/lib/github/import.rb
+++ b/lib/github/import.rb
@@ -41,13 +41,16 @@ module Github
self.reset_callbacks :validate
end
- attr_reader :project, :repository, :repo, :options, :errors, :cached, :verbose
+ attr_reader :project, :repository, :repo, :repo_url, :wiki_url,
+ :options, :errors, :cached, :verbose
- def initialize(project, options)
+ def initialize(project, options = {})
@project = project
@repository = project.repository
@repo = project.import_source
- @options = options
+ @repo_url = project.import_url
+ @wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git')
+ @options = options.reverse_merge(token: project.import_data&.credentials&.fetch(:user))
@verbose = options.fetch(:verbose, false)
@cached = Hash.new { |hash, key| hash[key] = Hash.new }
@errors = []
@@ -65,6 +68,8 @@ module Github
fetch_pull_requests
puts 'Fetching issues...'.color(:aqua) if verbose
fetch_issues
+ puts 'Fetching releases...'.color(:aqua) if verbose
+ fetch_releases
puts 'Cloning wiki repository...'.color(:aqua) if verbose
fetch_wiki_repository
puts 'Expiring repository cache...'.color(:aqua) if verbose
@@ -72,6 +77,7 @@ module Github
true
rescue Github::RepositoryFetchError
+ expire_repository_cache
false
ensure
keep_track_of_errors
@@ -81,23 +87,21 @@ module Github
def fetch_repository
begin
- project.create_repository unless project.repository.exists?
- project.repository.add_remote('github', "https://#{options.fetch(:token)}@github.com/#{repo}.git")
+ project.ensure_repository
+ project.repository.add_remote('github', repo_url)
project.repository.set_remote_as_mirror('github')
project.repository.fetch_remote('github', forced: true)
- rescue Gitlab::Shell::Error => e
- error(:project, "https://github.com/#{repo}.git", e.message)
+ rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e
+ error(:project, repo_url, e.message)
raise Github::RepositoryFetchError
end
end
def fetch_wiki_repository
- wiki_url = "https://#{options.fetch(:token)}@github.com/#{repo}.wiki.git"
- wiki_path = "#{project.full_path}.wiki"
+ return if project.wiki.repository_exists?
- unless project.wiki.repository_exists?
- gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url)
- end
+ wiki_path = "#{project.disk_path}.wiki"
+ gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url)
rescue Gitlab::Shell::Error => e
# GitHub error message when the wiki repo has not been created,
# this means that repo has wiki enabled, but have no pages. So,
@@ -309,7 +313,7 @@ module Github
next unless representation.valid?
release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag)
- next unless relese.new_record?
+ next unless release.new_record?
begin
release.description = representation.description
@@ -337,7 +341,7 @@ module Github
def user_id(user, fallback_id = nil)
return unless user.present?
- return cached[:user_ids][user.id] if cached[:user_ids].key?(user.id)
+ return cached[:user_ids][user.id] if cached[:user_ids][user.id].present?
gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email)
@@ -367,7 +371,7 @@ module Github
end
def expire_repository_cache
- repository.expire_content_cache
+ repository.expire_content_cache if project.repository_exists?
end
def keep_track_of_errors
diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb
new file mode 100644
index 00000000000..dfd17e35707
--- /dev/null
+++ b/lib/gitlab/daemon.rb
@@ -0,0 +1,62 @@
+module Gitlab
+ class Daemon
+ def self.initialize_instance(*args)
+ raise "#{name} singleton instance already initialized" if @instance
+ @instance = new(*args)
+ Kernel.at_exit(&@instance.method(:stop))
+ @instance
+ end
+
+ def self.instance
+ @instance ||= initialize_instance
+ end
+
+ attr_reader :thread
+
+ def thread?
+ !thread.nil?
+ end
+
+ def initialize
+ @mutex = Mutex.new
+ end
+
+ def enabled?
+ true
+ end
+
+ def start
+ return unless enabled?
+
+ @mutex.synchronize do
+ return thread if thread?
+
+ @thread = Thread.new { start_working }
+ end
+ end
+
+ def stop
+ @mutex.synchronize do
+ return unless thread?
+
+ stop_working
+
+ if thread
+ thread.wakeup if thread.alive?
+ thread.join
+ @thread = nil
+ end
+ end
+ end
+
+ private
+
+ def start_working
+ raise NotImplementedError
+ end
+
+ def stop_working
+ # no-ops
+ end
+ end
+end
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index db6cfc9671f..77b81d2d437 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -20,66 +20,7 @@ module Gitlab
if is_enabled
find_by_gitaly(repository, sha, path)
else
- find_by_rugged(repository, sha, path)
- end
- end
- end
-
- def find_by_gitaly(repository, sha, path)
- path = path.sub(/\A\/*/, '')
- path = '/' if path.empty?
- name = File.basename(path)
- entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE)
- return unless entry
-
- case entry.type
- when :COMMIT
- new(
- id: entry.oid,
- name: name,
- size: 0,
- data: '',
- path: path,
- commit_id: sha
- )
- when :BLOB
- new(
- id: entry.oid,
- name: name,
- size: entry.size,
- data: entry.data.dup,
- mode: entry.mode.to_s(8),
- path: path,
- commit_id: sha,
- binary: binary?(entry.data)
- )
- end
- end
-
- def find_by_rugged(repository, sha, path)
- commit = repository.lookup(sha)
- root_tree = commit.tree
-
- blob_entry = find_entry_by_path(repository, root_tree.oid, path)
-
- return nil unless blob_entry
-
- if blob_entry[:type] == :commit
- submodule_blob(blob_entry, path, sha)
- else
- blob = repository.lookup(blob_entry[:oid])
-
- if blob
- new(
- id: blob.oid,
- name: blob_entry[:name],
- size: blob.size,
- data: blob.content(MAX_DATA_DISPLAY_SIZE),
- mode: blob_entry[:filemode].to_s(8),
- path: path,
- commit_id: sha,
- binary: blob.binary?
- )
+ find_by_rugged(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE)
end
end
end
@@ -109,6 +50,21 @@ module Gitlab
detect && detect[:type] == :binary
end
+ # Returns an array of Blob instances, specified in blob_references as
+ # [[commit_sha, path], [commit_sha, path], ...]. If blob_size_limit < 0 then the
+ # full blob contents are returned. If blob_size_limit >= 0 then each blob will
+ # contain no more than limit bytes in its data attribute.
+ #
+ # Keep in mind that this method may allocate a lot of memory. It is up
+ # to the caller to limit the number of blobs and blob_size_limit.
+ #
+ def batch(repository, blob_references, blob_size_limit: nil)
+ blob_size_limit ||= MAX_DATA_DISPLAY_SIZE
+ blob_references.map do |sha, path|
+ find_by_rugged(repository, sha, path, limit: blob_size_limit)
+ end
+ end
+
private
# Recursive search of blob id by path
@@ -153,6 +109,66 @@ module Gitlab
commit_id: sha
)
end
+
+ def find_by_gitaly(repository, sha, path)
+ path = path.sub(/\A\/*/, '')
+ path = '/' if path.empty?
+ name = File.basename(path)
+ entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE)
+ return unless entry
+
+ case entry.type
+ when :COMMIT
+ new(
+ id: entry.oid,
+ name: name,
+ size: 0,
+ data: '',
+ path: path,
+ commit_id: sha
+ )
+ when :BLOB
+ new(
+ id: entry.oid,
+ name: name,
+ size: entry.size,
+ data: entry.data.dup,
+ mode: entry.mode.to_s(8),
+ path: path,
+ commit_id: sha,
+ binary: binary?(entry.data)
+ )
+ end
+ end
+
+ def find_by_rugged(repository, sha, path, limit:)
+ commit = repository.lookup(sha)
+ root_tree = commit.tree
+
+ blob_entry = find_entry_by_path(repository, root_tree.oid, path)
+
+ return nil unless blob_entry
+
+ if blob_entry[:type] == :commit
+ submodule_blob(blob_entry, path, sha)
+ else
+ blob = repository.lookup(blob_entry[:oid])
+
+ if blob
+ new(
+ id: blob.oid,
+ name: blob_entry[:name],
+ size: blob.size,
+ # Rugged::Blob#content is expensive; don't call it if we don't have to.
+ data: limit.zero? ? '' : blob.content(limit),
+ mode: blob_entry[:filemode].to_s(8),
+ path: path,
+ commit_id: sha,
+ binary: blob.binary?
+ )
+ end
+ end
+ end
end
def initialize(options)
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 1005a819f95..f246393cfbc 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -282,7 +282,14 @@ module Gitlab
# Return repo size in megabytes
def size
- size = popen(%w(du -sk), path).first.strip.to_i
+ size = gitaly_migrate(:repository_size) do |is_enabled|
+ if is_enabled
+ size_by_gitaly
+ else
+ size_by_shelling_out
+ end
+ end
+
(size.to_f / 1024).round(2)
end
@@ -299,6 +306,21 @@ module Gitlab
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/446
def log(options)
+ default_options = {
+ limit: 10,
+ offset: 0,
+ path: nil,
+ follow: false,
+ skip_merges: false,
+ disable_walk: false,
+ after: nil,
+ before: nil
+ }
+
+ options = default_options.merge(options)
+ options[:limit] ||= 0
+ options[:offset] ||= 0
+
raw_log(options).map { |c| Commit.decorate(c) }
end
@@ -712,20 +734,6 @@ module Gitlab
end
def raw_log(options)
- default_options = {
- limit: 10,
- offset: 0,
- path: nil,
- follow: false,
- skip_merges: false,
- disable_walk: false,
- after: nil,
- before: nil
- }
-
- options = default_options.merge(options)
- options[:limit] ||= 0
- options[:offset] ||= 0
actual_ref = options[:ref] || root_ref
begin
sha = sha_from_ref(actual_ref)
@@ -942,6 +950,14 @@ module Gitlab
gitaly_ref_client.tags
end
+ def size_by_shelling_out
+ popen(%w(du -sk), path).first.strip.to_i
+ end
+
+ def size_by_gitaly
+ gitaly_repository_client.repository_size
+ end
+
def count_commits_by_gitaly(options)
gitaly_commit_client.commit_count(options[:ref], options)
end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index ac6817e6d0e..3f577ac8530 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -135,6 +135,20 @@ module Gitlab
consume_commits_response(response)
end
+ def commits_by_message(query, revision: '', path: '', limit: 1000, offset: 0)
+ request = Gitaly::CommitsByMessageRequest.new(
+ repository: @gitaly_repo,
+ query: query,
+ revision: revision.to_s.force_encoding(Encoding::ASCII_8BIT),
+ path: path.to_s.force_encoding(Encoding::ASCII_8BIT),
+ limit: limit.to_i,
+ offset: offset.to_i
+ )
+
+ response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request)
+ consume_commits_response(response)
+ end
+
def languages(ref = nil)
request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '')
response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request)
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index 13e75b256a7..79ce784f2f2 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -27,6 +27,11 @@ module Gitlab
request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo)
GitalyClient.call(@storage, :repository_service, :repack_incremental, request)
end
+
+ def repository_size
+ request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo)
+ GitalyClient.call(@storage, :repository_service, :repository_size, request).size
+ end
end
end
end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index c8ad3a7a5e0..c5c05bfe2fb 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -101,6 +101,7 @@ excluded_attributes:
merge_requests:
- :milestone_id
- :ref_fetched
+ - :merge_jid
award_emoji:
- :awardable_id
statuses:
diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb
index 52276cbcd9a..5404dc11a87 100644
--- a/lib/gitlab/import_sources.rb
+++ b/lib/gitlab/import_sources.rb
@@ -8,7 +8,7 @@ module Gitlab
ImportSource = Struct.new(:name, :title, :importer)
ImportTable = [
- ImportSource.new('github', 'GitHub', Gitlab::GithubImport::Importer),
+ ImportSource.new('github', 'GitHub', Github::Import),
ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer),
ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer),
ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer),
diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb
index b75ae512d92..d9a79f7c291 100644
--- a/lib/gitlab/key_fingerprint.rb
+++ b/lib/gitlab/key_fingerprint.rb
@@ -1,55 +1,48 @@
module Gitlab
class KeyFingerprint
- include Gitlab::Popen
+ attr_reader :key, :ssh_key
- attr_accessor :key
+ # Unqualified MD5 fingerprint for compatibility
+ delegate :fingerprint, to: :ssh_key, allow_nil: true
def initialize(key)
@key = key
- end
-
- def fingerprint
- cmd_status = 0
- cmd_output = ''
-
- Tempfile.open('gitlab_key_file') do |file|
- file.puts key
- file.rewind
-
- cmd = []
- cmd.push('ssh-keygen')
- cmd.push('-E', 'md5') if explicit_fingerprint_algorithm?
- cmd.push('-lf', file.path)
-
- cmd_output, cmd_status = popen(cmd, '/tmp')
- end
-
- return nil unless cmd_status.zero?
- # 16 hex bytes separated by ':', optionally starting with "MD5:"
- fingerprint_matches = cmd_output.match(/(MD5:)?(?<fingerprint>(\h{2}:){15}\h{2})/)
- return nil unless fingerprint_matches
-
- fingerprint_matches[:fingerprint]
+ @ssh_key =
+ begin
+ Net::SSH::KeyFactory.load_data_public_key(key)
+ rescue Net::SSH::Exception, NotImplementedError
+ end
end
- private
-
- def explicit_fingerprint_algorithm?
- # OpenSSH 6.8 introduces a new default output format for fingerprints.
- # Check the version and decide which command to use.
-
- version_output, version_status = popen(%w(ssh -V))
- return false unless version_status.zero?
+ def valid?
+ ssh_key.present?
+ end
- version_matches = version_output.match(/OpenSSH_(?<major>\d+)\.(?<minor>\d+)/)
- return false unless version_matches
+ def type
+ return unless valid?
- version_info = Gitlab::VersionInfo.new(version_matches[:major].to_i, version_matches[:minor].to_i)
+ parts = ssh_key.ssh_type.split('-')
+ parts.shift if parts[0] == 'ssh'
- required_version_info = Gitlab::VersionInfo.new(6, 8)
+ parts[0].upcase
+ end
- version_info >= required_version_info
+ def bits
+ return unless valid?
+
+ case type
+ when 'RSA'
+ ssh_key.n.num_bits
+ when 'DSS', 'DSA'
+ ssh_key.p.num_bits
+ when 'ECDSA'
+ ssh_key.group.order.num_bits
+ when 'ED25519'
+ 256
+ else
+ raise "Unsupported key type: #{type}"
+ end
end
end
end
diff --git a/lib/gitlab/metrics/base_sampler.rb b/lib/gitlab/metrics/base_sampler.rb
index 219accfc029..716d20bb91a 100644
--- a/lib/gitlab/metrics/base_sampler.rb
+++ b/lib/gitlab/metrics/base_sampler.rb
@@ -1,20 +1,7 @@
require 'logger'
module Gitlab
module Metrics
- class BaseSampler
- def self.initialize_instance(*args)
- raise "#{name} singleton instance already initialized" if @instance
- @instance = new(*args)
- at_exit(&@instance.method(:stop))
- @instance
- end
-
- def self.instance
- @instance
- end
-
- attr_reader :running
-
+ class BaseSampler < Daemon
# interval - The sampling interval in seconds.
def initialize(interval)
interval_half = interval.to_f / 2
@@ -22,44 +9,7 @@ module Gitlab
@interval = interval
@interval_steps = (-interval_half..interval_half).step(0.1).to_a
- @mutex = Mutex.new
- end
-
- def enabled?
- true
- end
-
- def start
- return unless enabled?
-
- @mutex.synchronize do
- return if running
- @running = true
-
- @thread = Thread.new do
- sleep(sleep_interval)
-
- while running
- safe_sample
-
- sleep(sleep_interval)
- end
- end
- end
- end
-
- def stop
- @mutex.synchronize do
- return unless running
-
- @running = false
-
- if @thread
- @thread.wakeup if @thread.alive?
- @thread.join
- @thread = nil
- end
- end
+ super()
end
def safe_sample
@@ -81,7 +31,7 @@ module Gitlab
# potentially missing anything that happens in between samples).
# 2. Don't sample data at the same interval two times in a row.
def sleep_interval
- while step = @interval_steps.sample
+ while (step = @interval_steps.sample)
if step != @last_step
@last_step = step
@@ -89,6 +39,25 @@ module Gitlab
end
end
end
+
+ private
+
+ attr_reader :running
+
+ def start_working
+ @running = true
+ sleep(sleep_interval)
+
+ while running
+ safe_sample
+
+ sleep(sleep_interval)
+ end
+ end
+
+ def stop_working
+ @running = false
+ end
end
end
end
diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb
new file mode 100644
index 00000000000..5980a4ded2b
--- /dev/null
+++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb
@@ -0,0 +1,39 @@
+require 'webrick'
+require 'prometheus/client/rack/exporter'
+
+module Gitlab
+ module Metrics
+ class SidekiqMetricsExporter < Daemon
+ def enabled?
+ Gitlab::Metrics.metrics_folder_present? && settings.enabled
+ end
+
+ def settings
+ Settings.monitoring.sidekiq_exporter
+ end
+
+ private
+
+ attr_reader :server
+
+ def start_working
+ @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address)
+ server.mount "/", Rack::Handler::WEBrick, rack_app
+ server.start
+ end
+
+ def stop_working
+ server.shutdown
+ @server = nil
+ end
+
+ def rack_app
+ Rack::Builder.app do
+ use Rack::Deflater
+ use ::Prometheus::Client::Rack::Exporter
+ run -> (env) { [404, {}, ['']] }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb
new file mode 100644
index 00000000000..cf461adf697
--- /dev/null
+++ b/lib/gitlab/project_template.rb
@@ -0,0 +1,45 @@
+module Gitlab
+ class ProjectTemplate
+ attr_reader :title, :name
+
+ def initialize(name, title)
+ @name, @title = name, title
+ end
+
+ alias_method :logo, :name
+
+ def file
+ archive_path.open
+ end
+
+ def archive_path
+ Rails.root.join("vendor/project_templates/#{name}.tar.gz")
+ end
+
+ def clone_url
+ "https://gitlab.com/gitlab-org/project-templates/#{name}.git"
+ end
+
+ def ==(other)
+ name == other.name && title == other.title
+ end
+
+ TEMPLATES_TABLE = [
+ ProjectTemplate.new('rails', 'Ruby on Rails')
+ ].freeze
+
+ class << self
+ def all
+ TEMPLATES_TABLE
+ end
+
+ def find(name)
+ all.find { |template| template.name == name.to_s }
+ end
+
+ def archive_directory
+ Rails.root.join("vendor_directory/project_templates")
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index 4366ff336ef..0cb28732402 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -105,12 +105,24 @@ module Gitlab
# fetch_remote("gitlab/gitlab-ci", "upstream")
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
- def fetch_remote(storage, name, remote, forced: false, no_tags: false)
+ def fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false)
args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"]
args << '--force' if forced
args << '--no-tags' if no_tags
- gitlab_shell_fast_execute_raise_error(args)
+ vars = {}
+
+ if ssh_auth&.ssh_import?
+ if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
+ vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key
+ end
+
+ if ssh_auth.ssh_known_hosts.present?
+ vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts
+ end
+ end
+
+ gitlab_shell_fast_execute_raise_error(args, vars)
end
# Move repository
@@ -293,15 +305,15 @@ module Gitlab
false
end
- def gitlab_shell_fast_execute_raise_error(cmd)
- output, status = gitlab_shell_fast_execute_helper(cmd)
+ def gitlab_shell_fast_execute_raise_error(cmd, vars = {})
+ output, status = gitlab_shell_fast_execute_helper(cmd, vars)
raise Error, output unless status.zero?
true
end
- def gitlab_shell_fast_execute_helper(cmd)
- vars = ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS)
+ def gitlab_shell_fast_execute_helper(cmd, vars = {})
+ vars.merge!(ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS))
# Don't pass along the entire parent environment to prevent gitlab-shell
# from wasting I/O by searching through GEM_PATH
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index 59c32bbe7a4..a7e30423c7a 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -4,6 +4,55 @@ namespace :gitlab do
TEMPLATE_DATA.each { |template| update(template) }
end
+ desc "GitLab | Update project templates"
+ task :update_project_templates do
+ if Rails.env.production?
+ puts "This rake task is not meant fo production instances".red
+ exit(1)
+ end
+ admin = User.find_by(admin: true)
+
+ unless admin
+ puts "No admin user could be found".red
+ exit(1)
+ end
+
+ Gitlab::ProjectTemplate.all.each do |template|
+ params = {
+ import_url: template.clone_url,
+ namespace_id: admin.namespace.id,
+ path: template.title,
+ skip_wiki: true
+ }
+
+ puts "Creating project for #{template.name}"
+ project = Projects::CreateService.new(admin, params).execute
+
+ loop do
+ if project.finished?
+ puts "Import finished for #{template.name}"
+ break
+ end
+
+ if project.failed?
+ puts "Failed to import from #{project_params[:import_url]}".red
+ exit(1)
+ end
+
+ puts "Waiting for the import to finish"
+
+ sleep(5)
+ project.reload
+ end
+
+ Projects::ImportExport::ExportService.new(project, admin).execute
+ FileUtils.cp(project.export_project_path, template.archive_path)
+ Projects::DestroyService.new(admin, project).execute
+ puts "Exported #{template.name}".green
+ end
+ puts "Done".green
+ end
+
def update(template)
sub_dir = template.repo_url.match(/([A-Za-z-]+)\.git\z/)[1]
dir = File.join(vendor_directory, sub_dir)
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index 50b8e331469..96b8f59242c 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -7,7 +7,7 @@ class GithubImport
end
def initialize(token, gitlab_username, project_path, extras)
- @options = { url: 'https://api.github.com', token: token, verbose: true }
+ @options = { token: token, verbose: true }
@project_path = project_path
@current_user = User.find_by_username(gitlab_username)
@github_repo = extras.empty? ? nil : extras.first
@@ -62,6 +62,7 @@ class GithubImport
visibility_level: visibility_level,
import_type: 'github',
import_source: @repo['full_name'],
+ import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@"),
skip_wiki: @repo['has_wiki']
).execute
end
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-02 01:14:10 +0300