Add repository languages for projects

Our friends at GitHub show the programming languages for a long time,
and inspired by that this commit means to create about the same
functionality.

Language detection is done through Linguist, as before, where the
difference is that we cache the result in the database. Also, Gitaly can
incrementaly scan a repository. This is done through a shell out, which
creates overhead of about 3s each run. For now this won't be improved.

Scans are triggered by pushed to the default branch, usually `master`.
However, one exception to this rule the charts page. If we're requesting
this expensive data anyway, we just cache it in the database.

Edge cases where there is no repository, or its empty are caught in the
Repository model. This makes use of Redis caching, which is probably
already loaded.

The added model is called RepositoryLanguage, which will make it harder
if/when GitLab supports multiple repositories per project. However, for
now I think this shouldn't be a concern. Also, Language could be
confused with the i18n languages and felt like the current name was
suiteable too.

Design of the Project#Show page is done with help from @dimitrieh. This
change is not visible to the end user unless detections are done.

12 files changed, 139 insertions, 2 deletions

diff --git a/app/assets/stylesheets/pages/projects.scss b/app/assets/stylesheets/pages/projects.scss
index aa83e5bdebc..2a41a045e71 100644
--- a/app/assets/stylesheets/pages/projects.scss
+++ b/app/assets/stylesheets/pages/projects.scss
@@ -754,6 +754,11 @@
   }
 }
 
+.repository-languages-bar {
+  height: 6px;
+  margin-bottom: 8px;
+}
+
 pre.light-well {
   border-color: $well-light-border;
 }
diff --git a/app/helpers/repository_languages_helper.rb b/app/helpers/repository_languages_helper.rb
new file mode 100644
index 00000000000..9a842cf5ce0
--- /dev/null
+++ b/app/helpers/repository_languages_helper.rb
@@ -0,0 +1,16 @@
+module RepositoryLanguagesHelper
+  def repository_languages_bar(languages)
+    return if languages.none?
+
+    content_tag :div, class: 'progress repository-languages-bar' do
+      safe_join(languages.map { |lang| language_progress(lang) })
+    end
+  end
+
+  def language_progress(lang)
+    content_tag :div, nil,
+      class: "progress-bar has-tooltip",
+      style: "width: #{lang.share}%; background-color:#{lang.color}",
+      title: lang.name
+  end
+end
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index c1dc2f55346..de06e080a7d 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -122,6 +122,7 @@ class Namespace < ActiveRecord::Base
   def to_param
     full_path
   end
+  alias_method :flipper_id, :to_param
 
   def human_name
     owner_name
diff --git a/app/models/programming_language.rb b/app/models/programming_language.rb
new file mode 100644
index 00000000000..400d6c407a7
--- /dev/null
+++ b/app/models/programming_language.rb
@@ -0,0 +1,4 @@
+class ProgrammingLanguage < ActiveRecord::Base
+  validates :name, presence: true
+  validates :color, allow_blank: false, color: true
+end
diff --git a/app/models/project.rb b/app/models/project.rb
index da30d2fbb4f..af32afc08e2 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -192,6 +192,7 @@ class Project < ActiveRecord::Base
   has_many :hooks, class_name: 'ProjectHook'
   has_many :protected_branches
   has_many :protected_tags
+  has_many :repository_languages, -> { order "share DESC" }
 
   has_many :project_authorizations
   has_many :authorized_users, through: :project_authorizations, source: :user, class_name: 'User'
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 50f42be661b..9a6281bc1f7 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -235,6 +235,12 @@ class Repository
     false
   end
 
+  def languages
+    return [] if empty?
+
+    raw_repository.languages(root_ref)
+  end
+
   # Makes sure a commit is kept around when Git garbage collection runs.
   # Git GC will delete commits from the repository that are no longer in any
   # branches or tags, but we want to keep some of these commits around, for
@@ -432,6 +438,8 @@ class Repository
   # Runs code after a repository has been forked/imported.
   def after_import
     expire_content_cache
+
+    DetectRepositoryLanguagesWorker.perform_async(project.id, project.owner.id)
   end
 
   # Runs code after a new commit has been pushed.
diff --git a/app/models/repository_language.rb b/app/models/repository_language.rb
new file mode 100644
index 00000000000..f467d4eafa3
--- /dev/null
+++ b/app/models/repository_language.rb
@@ -0,0 +1,12 @@
+class RepositoryLanguage < ActiveRecord::Base
+  belongs_to :project
+  belongs_to :programming_language
+
+  default_scope { includes(:programming_language) }
+
+  validates :project, presence: true
+  validates :share, inclusion: { in: 0..100, message: "The share of a lanuage is between 0 and 100" }
+  validates :programming_language, uniqueness: { scope: :project_id }
+
+  delegate :name, :color, to: :programming_language
+end
diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb
index e3640e38bfa..637c1df4ad9 100644
--- a/app/services/git_push_service.rb
+++ b/app/services/git_push_service.rb
@@ -85,6 +85,8 @@ class GitPushService < BaseService
 
         types = Gitlab::FileDetector.types_in_paths(paths.to_a)
       end
+
+      DetectRepositoryLanguagesWorker.perform_async(@project.id, current_user.id)
     else
       types = []
     end
diff --git a/app/services/projects/detect_repository_languages_service.rb b/app/services/projects/detect_repository_languages_service.rb
new file mode 100644
index 00000000000..6669830ac8c
--- /dev/null
+++ b/app/services/projects/detect_repository_languages_service.rb
@@ -0,0 +1,53 @@
+module Projects
+  class DetectRepositoryLanguagesService < BaseService
+    attr_reader :detected_repository_languages, :programming_languages
+
+    def execute
+      repository_languages = project.repository_languages
+      detection = Gitlab::LanguageDetection.new(repository, repository_languages)
+
+      matching_programming_languages = ensure_programming_languages(detection)
+
+      RepositoryLanguage.transaction do
+        project.repository_languages.where(programming_language_id: detection.deletions).delete_all
+
+        detection.updates.each do |update|
+          RepositoryLanguage
+            .arel_table.update_manager
+            .where(project_id: project.id)
+            .where(programming_language_id: update[:programming_language_id])
+            .set(share: update[:share])
+        end
+
+        Gitlab::Database.bulk_insert(
+          RepositoryLanguage.table_name,
+          detection.insertions(matching_programming_languages)
+        )
+      end
+
+      project.repository_languages.reload
+    end
+
+    private
+
+    def ensure_programming_languages(detection)
+      existing_languages = ProgrammingLanguage.where(name: detection.languages)
+      return existing_languages if detection.languages.size == existing_languages.size
+
+      missing_languages = detection.languages - existing_languages.map(&:name)
+      created_languages = missing_languages.map do |name|
+        create_language(name, detection.language_color(name))
+      end
+
+      existing_languages + created_languages
+    end
+
+    def create_language(name, color)
+      ProgrammingLanguage.transaction do
+        ProgrammingLanguage.where(name: name).first_or_create(color: color)
+      end
+    rescue ActiveRecord::RecordNotUnique
+      retry
+    end
+  end
+end
diff --git a/app/views/projects/show.html.haml b/app/views/projects/show.html.haml
index 803ecca48f7..e011851be78 100644
--- a/app/views/projects/show.html.haml
+++ b/app/views/projects/show.html.haml
@@ -18,10 +18,11 @@
 = render "home_panel"
 
 - if can?(current_user, :download_code, @project)
-  %nav.project-stats{ class: container_class }
+  %nav.project-stats{ class: [container_class, ("limit-container-width" unless fluid_layout)] }
     = render 'stat_anchor_list', anchors: @project.statistics_anchors(show_auto_devops_callout: show_auto_devops_callout)
     = render 'stat_anchor_list', anchors: @project.statistics_buttons(show_auto_devops_callout: show_auto_devops_callout)
-
+    - if Feature.enabled?(:repository_languages, @project.namespace.becomes(Namespace))
+      = repository_languages_bar(@project.repository_languages)
 
 %div{ class: [container_class, ("limit-container-width" unless fluid_layout)] }
   - if @project.archived?
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index f2651cb54ec..e8b9999f83b 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -123,3 +123,4 @@
 - repository_update_remote_mirror
 - create_note_diff_file
 - delete_diff_files
+- detect_repository_languages
diff --git a/app/workers/detect_repository_languages_worker.rb b/app/workers/detect_repository_languages_worker.rb
new file mode 100644
index 00000000000..537b8fd5963
--- /dev/null
+++ b/app/workers/detect_repository_languages_worker.rb
@@ -0,0 +1,33 @@
+class DetectRepositoryLanguagesWorker
+  include ApplicationWorker
+  include ExceptionBacktrace
+  include ExclusiveLeaseGuard
+
+  sidekiq_options retry: 1
+
+  LEASE_TIMEOUT = 300
+
+  attr_reader :project
+
+  def perform(project_id, user_id)
+    @project = Project.find_by(id: project_id)
+    user = User.find_by(id: user_id)
+    return unless project && user
+
+    return if Feature.disabled?(:repository_languages, project.namespace)
+
+    try_obtain_lease do
+      ::Projects::DetectRepositoryLanguagesService.new(project, user).execute
+    end
+  end
+
+  private
+
+  def lease_timeout
+    LEASE_TIMEOUT
+  end
+
+  def lease_key
+    "gitlab:detect_repository_languages:#{project.id}"
+  end
+end