Merge branch 'sarahg/1212-pages-no-nav' into 'main'

Add a script to report on pages not in the global nav

Closes #1212

See merge request https://gitlab.com/gitlab-org/gitlab-docs/-/merge_requests/3236

Merged-by: Suzanne Selhorn <sselhorn@gitlab.com>
Co-authored-by: Sarah German <sgerman@gitlab.com>

5 files changed, 110 insertions, 3 deletions

diff --git a/Makefile b/Makefile
index f8069f08..bd4cdcc4 100644
--- a/Makefile
+++ b/Makefile
@@ -90,8 +90,7 @@ install-ruby-dependencies:
 	@bundle install
 
 install-nodejs-dependencies:
-	@printf "\n$(INFO)INFO: Installing Node.js dependencies...$(END)\n"
-	@yarn install --frozen-lockfile
+	@yarn install --frozen-lockfile --silent
 
 setup: brew-bundle setup-asdf install-asdf-dependencies install-ruby-dependencies install-nodejs-dependencies
 
@@ -165,4 +164,7 @@ check-global-navigation:
 	@printf "\n$(INFO)INFO: Checking global navigation...$(END)\n"
 	@scripts/check-navigation.sh
 
+check-pages-not-in-nav: install-nodejs-dependencies
+	@scripts/pages_not_in_nav.js | jq
+
 test: setup rspec-tests jest-tests eslint-tests prettier-tests stylelint-tests hadolint-tests yamllint-tests markdownlint-tests check-global-navigation
diff --git a/doc/raketasks.md b/doc/raketasks.md
index 1ffd0f5c..b30670b6 100644
--- a/doc/raketasks.md
+++ b/doc/raketasks.md
@@ -1,4 +1,4 @@
-# GitLab docs rake tasks
+# GitLab Docs rake tasks and scripts
 
 The GitLab Docs project has some raketasks that automate various things. You
 can see the list of rake tasks with:
@@ -67,3 +67,27 @@ To omit the automatic merge request creation:
 ```shell
 SKIP_MR=true bundle exec rake docs:clean_redirects
 ```
+
+## Find pages that are not included in the global navigation
+
+Run this command to check for pages that are missing from the global navigation:
+
+```shell
+make check-pages-not-in-nav
+```
+
+This invokes a [Node.js script](../scripts/pages_not_in_nav.js) that outputs JSON containing matching page URL paths and the associated Section and Group to the console.
+
+Before running the script, you may want to run `make update-all-docs-projects` to pull down the latest content from each source project.
+
+You can use `jq` on the command line to extract specific fields. For example, to return a list of only URLs:
+
+```shell
+make check-pages-not-in-nav | jq '.[] | .url'
+```
+
+The script intentionally omits:
+
+- Sections referenced in the ["Pages you don’t need to add"](https://docs.gitlab.com/ee/development/documentation/site_architecture/global_nav.html#pages-you-dont-need-to-add) section of our docs.
+- Redirects.
+- Markdown files that are not compiled to HTML pages (see the `ignore` paths in Nanoc's [Rules](../Rules) file).
diff --git a/package.json b/package.json
index ff6a3e2c..530d8f1d 100644
--- a/package.json
+++ b/package.json
@@ -53,10 +53,12 @@
     "clipboard": "^2.0.11",
     "compare-versions": "^5.0.1",
     "eslint-plugin-filenames": "^1.3.2",
+    "front-matter": "^4.0.2",
     "glob": "^8.0.3",
     "instantsearch.css": "^7.4.5",
     "instantsearch.js": "^4.49.0",
     "jquery": "^3.6.1",
+    "js-yaml": "^4.1.0",
     "lunr": "^2.3.9",
     "mermaid": "^9.1.7",
     "pikaday": "^1.8.2",
diff --git a/scripts/pages_not_in_nav.js b/scripts/pages_not_in_nav.js
new file mode 100755
index 00000000..87e15fe0
--- /dev/null
+++ b/scripts/pages_not_in_nav.js
@@ -0,0 +1,72 @@
+#!/usr/bin/env node
+
+/**
+ * @file pages_not_in_nav.js
+ * Generates a report of pages which are not included in navigation.yaml.
+ */
+
+/* eslint-disable no-console */
+
+const fs = require('fs');
+const glob = require('glob');
+const yaml = require('js-yaml');
+const fm = require('front-matter');
+
+// Load site data sources from nanoc config.
+const nanocConfig = yaml.load(fs.readFileSync('nanoc.yaml', 'utf8'));
+const dataSources = nanocConfig.data_sources.filter((source) => source.items_root !== '/');
+
+// Load the global navigation data file.
+const navYaml = yaml.load(fs.readFileSync('content/_data/navigation.yaml', 'utf8'));
+const nav = JSON.stringify(navYaml);
+
+// Read the markdown file and extract the fields we need.
+const getPageData = (filename) => {
+  const contents = fs.readFileSync(filename, 'utf-8');
+  return {
+    filename,
+    isRedirect: contents.includes('redirect_to'),
+    stage: fm(contents).attributes.stage,
+    group: fm(contents).attributes.group,
+  };
+};
+
+// Loop through each data source's markdown files.
+const lostPages = [];
+dataSources.forEach((source) => {
+  glob.sync(`${source.content_dir}/**/*.md`).forEach((filename) => {
+    const pageData = getPageData(filename);
+    if (pageData.isRedirect) {
+      return;
+    }
+
+    // Convert the markdown filepath into a string that matches the URL path on the website.
+    const path =
+      source.items_root.replaceAll('/', '') +
+      filename
+        .replace(source.content_dir, '')
+        .replace(source, '')
+        .replace('index.md', '')
+        .replace('.md', '.html');
+
+    if (
+      // Include pages that are not in the nav.
+      !nav.includes(path) &&
+      // Exclude sections that are intentionally not in the nav.
+      !path.includes('/architecture/blueprints') &&
+      !path.includes('/user/application_security/dast/checks/') &&
+      !path.includes('/legal/') &&
+      !path.includes('/drawers/') &&
+      !path.includes('/adr/')
+    ) {
+      lostPages.push({
+        url: `https://docs.gitlab.com/${path}`,
+        stage: pageData.stage,
+        group: pageData.group,
+      });
+    }
+  });
+});
+
+// Return results as JSON.
+console.log(JSON.stringify(lostPages));
diff --git a/yarn.lock b/yarn.lock
index 2ec887f3..42d7cd15 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3964,6 +3964,13 @@ form-data@^4.0.0:
     combined-stream "^1.0.8"
     mime-types "^2.1.12"
 
+front-matter@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/front-matter/-/front-matter-4.0.2.tgz#b14e54dc745cfd7293484f3210d15ea4edd7f4d5"
+  integrity sha512-I8ZuJ/qG92NWX8i5x1Y8qyj3vizhXS31OxjKDu3LKP+7/qBgfIKValiZIEwoVoJKUHlhWtYrktkxV1XsX+pPlg==
+  dependencies:
+    js-yaml "^3.13.1"
+
 fs.realpath@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"