1 files changed, 20 insertions, 6 deletions

diff --git a/app/models/namespace/traversal_hierarchy.rb b/app/models/namespace/traversal_hierarchy.rb
index cfb6cfdde74..28cf55f7486 100644
--- a/app/models/namespace/traversal_hierarchy.rb
+++ b/app/models/namespace/traversal_hierarchy.rb
@@ -34,17 +34,20 @@ class Namespace
       sql = """
             UPDATE namespaces
             SET traversal_ids = cte.traversal_ids
-            FROM (#{recursive_traversal_ids}) as cte
+            FROM (#{recursive_traversal_ids(lock: true)}) as cte
             WHERE namespaces.id = cte.id
               AND namespaces.traversal_ids <> cte.traversal_ids
             """
       Namespace.connection.exec_query(sql)
+    rescue ActiveRecord::Deadlocked
+      db_deadlock_counter.increment(source: 'Namespace#sync_traversal_ids!')
+      raise
     end
 
     # Identify all incorrect traversal_ids in the current namespace hierarchy.
-    def incorrect_traversal_ids
+    def incorrect_traversal_ids(lock: false)
       Namespace
-        .joins("INNER JOIN (#{recursive_traversal_ids}) as cte ON namespaces.id = cte.id")
+        .joins("INNER JOIN (#{recursive_traversal_ids(lock: lock)}) as cte ON namespaces.id = cte.id")
         .where('namespaces.traversal_ids <> cte.traversal_ids')
     end
 
@@ -55,10 +58,13 @@ class Namespace
     #
     # Note that the traversal_ids represent a calculated traversal path for the
     # namespace and not the value stored within the traversal_ids attribute.
-    def recursive_traversal_ids
+    #
+    # Optionally locked with FOR UPDATE to ensure isolation between concurrent
+    # updates of the heirarchy.
+    def recursive_traversal_ids(lock: false)
       root_id = Integer(@root.id)
 
-      """
+      sql = <<~SQL
       WITH RECURSIVE cte(id, traversal_ids, cycle) AS (
         VALUES(#{root_id}, ARRAY[#{root_id}], false)
       UNION ALL
@@ -67,7 +73,11 @@ class Namespace
         WHERE n.parent_id = cte.id AND NOT cycle
       )
       SELECT id, traversal_ids FROM cte
-      """
+      SQL
+
+      sql += ' FOR UPDATE' if lock
+
+      sql
     end
 
     # This is essentially Namespace#root_ancestor which will soon be rewritten
@@ -80,5 +90,9 @@ class Namespace
         .reorder(nil)
         .find_by(parent_id: nil)
     end
+
+    def db_deadlock_counter
+      Gitlab::Metrics.counter(:db_deadlock, 'Counts the times we have deadlocked in the database')
+    end
   end
 end