website: resolve unlinked content (#11391)

* website: temp support for unlinked content

* website: add GitHub check for unlinked content

* website: add hotfix for brand coloration issue

.github/workflows/check-unlinked-content.js

@@ -0,0 +1,148 @@
+var fs = require("fs");
+var path = require("path");
+
+const COLOR_RESET = "\x1b[0m";
+const COLOR_GREEN = "\x1b[32m";
+const COLOR_RED = "\x1b[31m";
+
+runCheck([
+  {
+    contentDir: "website/content/docs",
+    navDataFiles: [
+      "website/data/docs-nav-data.json",
+      "website/data/docs-nav-data-hidden.json",
+    ],
+  },
+  {
+    contentDir: "website/content/api-docs",
+    navDataFiles: [
+      "website/data/api-docs-nav-data.json",
+      "website/data/api-docs-nav-data-hidden.json",
+    ],
+  },
+  {
+    contentDir: "website/content/guides",
+    navDataFiles: ["website/data/guides-nav-data.json"],
+  },
+  {
+    contentDir: "website/content/intro",
+    navDataFiles: ["website/data/intro-nav-data.json"],
+  },
+]);
+
+async function runCheck(baseRoutes) {
+  const validatedBaseRoutes = await Promise.all(
+    baseRoutes.map(async ({ contentDir, navDataFiles }) => {
+      const missingRoutes = await validateMissingRoutes(
+        contentDir,
+        navDataFiles
+      );
+      return { contentDir, navDataFiles, missingRoutes };
+    })
+  );
+  const allMissingRoutes = validatedBaseRoutes.reduce((acc, baseRoute) => {
+    return acc.concat(baseRoute.missingRoutes);
+  }, []);
+  if (allMissingRoutes.length == 0) {
+    console.log(
+      `\n${COLOR_GREEN}✓ All content files have routes, and are included in navigation data.${COLOR_RESET}\n`
+    );
+  } else {
+    validatedBaseRoutes.forEach(
+      ({ contentDir, navDataFiles, missingRoutes }) => {
+        if (missingRoutes.length == 0) return true;
+        console.log(
+          `\n${COLOR_RED}Error: Missing pages found in the ${contentDir} directory.\n\nPlease add these paths to ${navDataFiles.join(
+            " or "
+          )}, or remove the .mdx files.\n\n${JSON.stringify(
+            missingRoutes,
+            null,
+            2
+          )}${COLOR_RESET}\n\n`
+        );
+      }
+    );
+    process.exit(1);
+  }
+}
+
+async function validateMissingRoutes(contentDir, navDataFiles) {
+  // Read in nav-data.json, and make a flattened array of nodes
+  const navDataFlat = navDataFiles.reduce((acc, navDataFile) => {
+    const navDataPath = path.join(process.cwd(), navDataFile);
+    const navData = JSON.parse(fs.readFileSync(navDataPath));
+    return acc.concat(flattenNodes(navData));
+  }, []);
+  // Read all files in the content directory
+  const files = await walkAsync(contentDir);
+  // Filter out content files that are already
+  // included in nav-data.json
+  const missingPages = files
+    // Ignore non-.mdx files
+    .filter((filePath) => {
+      return path.extname(filePath) == ".mdx";
+    })
+    // Transform the filePath into an expected route
+    .map((filePath) => {
+      // Get the relative filepath, that's what we'll see in the route
+      const contentDirPath = path.join(process.cwd(), contentDir);
+      const relativePath = path.relative(contentDirPath, filePath);
+      // Remove extensions, these will not be in routes
+      const pathNoExt = relativePath.replace(/\.mdx$/, "");
+      // Resolve /index routes, these will not have /index in their path
+      const routePath = pathNoExt.replace(/\/?index$/, "");
+      return routePath;
+    })
+    // Determine if there is a match in nav-data.
+    // If there is no match, then this is an unlinked content file.
+    .filter((pathToMatch) => {
+      // If it's the root path index page, we know
+      // it'll be rendered (hard-coded into docs-page/server.js)
+      const isIndexPage = pathToMatch === "";
+      if (isIndexPage) return false;
+      // Otherwise, needs a path match in nav-data
+      const matches = navDataFlat.filter(({ path }) => path == pathToMatch);
+      return matches.length == 0;
+    });
+  return missingPages;
+}
+
+function flattenNodes(nodes) {
+  return nodes.reduce((acc, n) => {
+    if (!n.routes) return acc.concat(n);
+    return acc.concat(flattenNodes(n.routes));
+  }, []);
+}
+
+function walkAsync(relativeDir) {
+  const dirPath = path.join(process.cwd(), relativeDir);
+  return new Promise((resolve, reject) => {
+    walk(dirPath, function (err, result) {
+      if (err) reject(err);
+      resolve(result);
+    });
+  });
+}
+
+function walk(dir, done) {
+  var results = [];
+  fs.readdir(dir, function (err, list) {
+    if (err) return done(err);
+    var pending = list.length;
+    if (!pending) return done(null, results);
+    list.forEach(function (file) {
+      file = path.resolve(dir, file);
+      fs.stat(file, function (err, stat) {
+        if (stat && stat.isDirectory()) {
+          walk(file, function (err, res) {
+            results = results.concat(res);
+            if (!--pending) done(null, results);
+          });
+        } else {
+          results.push(file);
+          if (!--pending) done(null, results);
+        }
+      });
+    });
+  });
+}

.github/workflows/check-unlinked-content.yml

@@ -0,0 +1,26 @@
+#
+# This GitHub action checks that all .mdx files in the
+# the website/content directory are being published.
+# It fails if any of these files are not included
+# in the expected nav-data.json file.
+#
+# To resolve failed checks, add the listed paths
+# to the corresponding nav-data.json file
+# in website/data.
+
+name: "website: Check unlinked content"
+on:
+  pull_request:
+    paths:
+      - "website/**"
+
+jobs:
+  check-unlinked-content:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Node
+        uses: actions/setup-node@v1
+      - name: Check that all content files are included in navigation
+        run: node .github/workflows/check-unlinked-content.js

website/components/_temp-enable-hidden-pages/index.js

@@ -0,0 +1,85 @@
+// Imports below are used in server-side only
+import fs from 'fs'
+import path from 'path'
+import {
+  generateStaticPaths as docsPageStaticPaths,
+  generateStaticProps as docsPageStaticProps,
+} from '@hashicorp/react-docs-page/server'
+
+/**
+ * DEBT
+ * This is a short term hotfix for "hidden" docs-sidenav items.
+ *
+ * We likely do NOT want to support this in docs-page/server,
+ * instead, a simple "hidden" attribute supported on docs-sidenav
+ * nodes would do the trick, ensuring the "path" is "registered"
+ * in the appropriate nav-data file, and located in the correct spot
+ * in the nav-data tree, while also hiding that item in the sidebar.
+ *
+ * We can remove this hack with once support lands for "hidden" items,
+ * currently this is somewhat blocked by branding rollout:
+ * Asana task that will resolve this debt:
+ * https://app.asana.com/0/1100423001970639/1200197752405255/f
+ * Draft PR to support "hidden" nav items:
+ * https://github.com/hashicorp/react-components/pull/220
+ **/
+
+const DEFAULT_PARAM_ID = 'page'
+
+export async function generateStaticPaths({
+  navDataFile,
+  navDataFileHidden,
+  localContentDir,
+}) {
+  const visiblePaths = await docsPageStaticPaths({
+    navDataFile,
+    localContentDir,
+  })
+  const hiddenPaths = await docsPageStaticPaths({
+    navDataFile: navDataFileHidden,
+    localContentDir,
+  })
+  return visiblePaths.concat(hiddenPaths)
+}
+
+export async function generateStaticProps({
+  navDataFile,
+  navDataFileHidden,
+  localContentDir,
+  product,
+  params,
+  paramId = DEFAULT_PARAM_ID,
+  additionalComponents,
+}) {
+  // Read in the "hidden" nav data, and flatten it
+  const navDataVisible = readNavData(navDataFile)
+  const navDataHidden = readNavData(navDataFileHidden)
+  // Check if this is a "hidden" page, if so, use the navDataHidden
+  // to generate static props.
+  const currentPath = params[paramId] ? params[paramId].join('/') : ''
+  const hiddenPaths = flattenNavData(navDataHidden).map((n) => n.path)
+  const isHiddenPage = hiddenPaths.filter((p) => p == currentPath).length > 0
+  // Return the static props, but always pass the navDataVisible
+  // as the navData to be displayed.
+  const staticProps = await docsPageStaticProps({
+    navDataFile: isHiddenPage ? navDataFileHidden : navDataFile,
+    localContentDir,
+    product,
+    params,
+    paramId,
+    additionalComponents,
+  })
+  return { ...staticProps, navData: navDataVisible }
+}
+
+function readNavData(navDataFile) {
+  const filePath = path.join(process.cwd(), navDataFile)
+  return JSON.parse(fs.readFileSync(filePath))
+}
+
+function flattenNavData(nodes) {
+  return nodes.reduce((acc, n) => {
+    if (!n.routes) return acc.concat(n)
+    return acc.concat(flattenNavData(n.routes))
+  }, [])
+}

website/data/api-docs-nav-data-hidden.json

@@ -0,0 +1,20 @@
+[
+  {
+    "title": "Secrets Engines",
+    "routes": [
+      {
+        "title": "Cassandra",
+        "path": "secret/cassandra"
+      }
+    ]
+  },
+  {
+    "title": "System Backend",
+    "routes": [
+      {
+        "title": "<code>/sys/generate-recovery-token</code>",
+        "path": "system/generate-recovery-token"
+      }
+    ]
+  }
+]

website/data/docs-nav-data-hidden.json

@@ -0,0 +1,56 @@
+[
+  {
+    "title": "Configuration",
+    "routes": [
+      {
+        "title": "<code>storage</code>",
+        "routes": [
+          {
+            "title": "Alicloud OSS",
+            "path": "configuration/storage/alicloudoss"
+          }
+        ]
+      }
+    ]
+  },
+  {
+    "title": "Commands",
+    "routes": [
+      {
+        "title": "<code>help</code>",
+        "path": "commands/help"
+      },
+      {
+        "title": "<code>lease</code>",
+        "routes": [
+          {
+            "title": "<code>lookup</code>",
+            "path": "commands/lease/lookup"
+          }
+        ]
+      }
+    ]
+  },
+  {
+    "title": "Secrets",
+    "routes": [
+      {
+        "title": "Cassandra",
+        "path": "secrets/cassandra"
+      },
+      {
+        "title": "Transform <sup>ENTERPRISE</sup>",
+        "routes": [
+          {
+            "title": "FF3-1 Tweak Usage",
+            "path": "secrets/transform/ff3-tweak-details"
+          }
+        ]
+      }
+    ]
+  },
+  {
+    "title": "Versus Other Software",
+    "path": "vs"
+  }
+]

website/data/guides-nav-data.json

@@ -64,6 +64,14 @@
       {
         "title": "Vault Cluster Monitoring",
         "path": "operations/monitoring"
+      },
+      {
+        "title": "Vault Deployment Guide",
+        "path": "operations/deployment-guide"
+      },
+      {
+        "title": "Performance Standby Nodes",
+        "path": "operations/performance-nodes"
       }
     ]
   },
@@ -82,6 +90,10 @@
         "title": "Policies",
         "path": "identity/policies"
       },
+      {
+        "title": "ACL Policy Path Templating",
+        "path": "identity/policy-templating"
+      },
       {
         "title": "AppRole Pull Authentication",
         "path": "identity/authentication"

website/pages/api-docs/[[...page]].jsx

@@ -1,10 +1,19 @@
 import { productName, productSlug } from 'data/metadata'
 import DocsPage from '@hashicorp/react-docs-page'
+// Imports below are used in server-side only
+/**
+ * DEBT: short term patch for "hidden" docs-sidenav items.
+ * See components/_temp-enable-hidden-pages for details.
+ * Revert to importing from @hashicorp/react-docs-page/server
+ * once https://app.asana.com/0/1100423001970639/1200197752405255/f
+ * is complete.
+ **/
 import {
   generateStaticPaths,
   generateStaticProps,
-} from '@hashicorp/react-docs-page/server'
+} from 'components/_temp-enable-hidden-pages'
 
+const NAV_DATA_FILE_HIDDEN = 'data/api-docs-nav-data-hidden.json'
 const NAV_DATA_FILE = 'data/api-docs-nav-data.json'
 const CONTENT_DIR = 'content/api-docs'
 const basePath = 'api-docs'
@@ -24,6 +33,7 @@ export async function getStaticPaths() {
     fallback: false,
     paths: await generateStaticPaths({
       navDataFile: NAV_DATA_FILE,
+      navDataFileHidden: NAV_DATA_FILE_HIDDEN,
       localContentDir: CONTENT_DIR,
     }),
   }
@@ -33,6 +43,7 @@ export async function getStaticProps({ params }) {
   return {
     props: await generateStaticProps({
       navDataFile: NAV_DATA_FILE,
+      navDataFileHidden: NAV_DATA_FILE_HIDDEN,
       localContentDir: CONTENT_DIR,
       product: { name: productName, slug: productSlug },
       params,

website/pages/docs/[[...page]].jsx

@@ -1,12 +1,21 @@
 import { productName, productSlug } from 'data/metadata'
 import DocsPage from '@hashicorp/react-docs-page'
+import Columns from 'components/columns'
+import Tag from 'components/inline-tag'
+// Imports below are used in server-side only
+/**
+ * DEBT: short term patch for "hidden" docs-sidenav items.
+ * See components/_temp-enable-hidden-pages for details.
+ * Revert to importing from @hashicorp/react-docs-page/server
+ * once https://app.asana.com/0/1100423001970639/1200197752405255/f
+ * is complete.
+ **/
 import {
   generateStaticPaths,
   generateStaticProps,
-} from '@hashicorp/react-docs-page/server'
-import Columns from 'components/columns'
-import Tag from 'components/inline-tag'
+} from 'components/_temp-enable-hidden-pages'
 
+const NAV_DATA_FILE_HIDDEN = 'data/docs-nav-data-hidden.json'
 const NAV_DATA_FILE = 'data/docs-nav-data.json'
 const CONTENT_DIR = 'content/docs'
 const basePath = 'docs'
@@ -28,6 +37,7 @@ export async function getStaticPaths() {
     fallback: false,
     paths: await generateStaticPaths({
       navDataFile: NAV_DATA_FILE,
+      navDataFileHidden: NAV_DATA_FILE_HIDDEN,
       localContentDir: CONTENT_DIR,
     }),
   }
@@ -37,6 +47,7 @@ export async function getStaticProps({ params }) {
   return {
     props: await generateStaticProps({
       navDataFile: NAV_DATA_FILE,
+      navDataFileHidden: NAV_DATA_FILE_HIDDEN,
       localContentDir: CONTENT_DIR,
       product: { name: productName, slug: productSlug },
       params,

website/pages/style.css

@@ -101,3 +101,7 @@
 .g-content {
   --brand-text: var(--vagrant) !important;
 }
+.g-docs-sidenav {
+  --brand: var(--vagrant) !important;
+  --brand-text: var(--vagrant) !important;
+}