1 files changed, 17 insertions, 0 deletions
diff --git a/lib/gitlab/robots_txt.rb b/lib/gitlab/robots_txt.rb
new file mode 100644
index 00000000000..2f395548770
--- /dev/null
+++ b/lib/gitlab/robots_txt.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module RobotsTxt
+    def self.disallowed?(path)
+      parsed_robots_txt.disallowed?(path)
+    end
+
+    def self.parsed_robots_txt
+      @parsed_robots_txt ||= Parser.new(robots_txt)
+    end
+
+    def self.robots_txt
+      File.read(Rails.root.join('public', 'robots.txt'))
+    end
+  end
+end