tools: produce JSON documentation using unified/remark/rehype

PR-URL: #21697 Reviewed-By: Vse Mozhet Byt <vsemozhetbyt@gmail.com> Reviewed-By: Rich Trott <rtrott@gmail.com> Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com>

Author: rubys

.gitignore

@@ -107,6 +107,9 @@ deps/npm/node_modules/.bin/
 /*.pkg
 /SHASUMS*.txt*
 
+# api docs artifacts
+tools/doc/node_modules
+
 # test artifacts
 tools/remark-cli/node_modules
 tools/remark-preset-lint-node/node_modules

Makefile

@@ -320,7 +320,7 @@ ifeq ($(OSTYPE),aix)
 DOCBUILDSTAMP_PREREQS := $(DOCBUILDSTAMP_PREREQS) out/$(BUILDTYPE)/node.exp
 endif
 
-test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS)
+test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS) tools/doc/node_modules
 $(RM) -r test/addons/??_*/
 [ -x $(NODE) ] && $(NODE) $< || node $<
 touch $@
@@ -633,15 +633,15 @@ available-node = \
 run-npm-install = $(PWD)/$(NPM) install --production --no-package-lock
 run-npm-ci = $(PWD)/$(NPM) ci
 
-gen-json = tools/doc/generate.js --format=json $< > $@
-gen-html = tools/doc/generate.js --node-version=$(FULLVERSION) --format=html \
---analytics=$(DOCS_ANALYTICS) $< > $@
+tools/doc/node_modules/js-yaml/package.json:
+cd tools/doc && $(call available-node,$(run-npm-install))
 
-out/doc/api/%.json: doc/api/%.md tools/doc/generate.js tools/doc/json.js
-$(call available-node, $(gen-json))
+gen-api = tools/doc/generate.js --node-version=$(FULLVERSION) \
+--analytics=$(DOCS_ANALYTICS) $< --output-directory=out/doc/api
 
-out/doc/api/%.html: doc/api/%.md tools/doc/generate.js tools/doc/html.js
-$(call available-node, $(gen-html))
+out/doc/api/%.json out/doc/api/%.html: doc/api/%.md tools/doc/generate.js \
+tools/doc/html.js tools/doc/json.js
+$(call available-node, $(gen-api))
 
 out/doc/api/all.html: $(apidocs_html) tools/doc/allhtml.js
 $(call available-node, tools/doc/allhtml.js)

doc/api/https.md

@@ -123,7 +123,7 @@ changes:
  pr-url: https://github.com/nodejs/node/pull/10638
  description: The `options` parameter can be a WHATWG `URL` object.
 -->
-- `url` {string | URL}
+* `url` {string | URL}
 * `options` {Object | string | URL} Accepts the same `options` as
  [`https.request()`][], with the `method` always set to `GET`.
 * `callback` {Function}
@@ -174,7 +174,7 @@ changes:
  pr-url: https://github.com/nodejs/node/pull/10638
  description: The `options` parameter can be a WHATWG `URL` object.
 -->
-- `url` {string | URL}
+* `url` {string | URL}
 * `options` {Object | string | URL} Accepts all `options` from
  [`http.request()`][], with some differences in default values:
  - `protocol` **Default:** `'https:'`

test/doctool/test-doctool-html.js

@@ -11,7 +11,34 @@ try {
 const assert = require('assert');
 const { readFile } = require('fs');
 const fixtures = require('../common/fixtures');
-const toHTML = require('../../tools/doc/html.js');
+const html = require('../../tools/doc/html.js');
+const path = require('path');
+
+module.paths.unshift(
+ path.join(__dirname, '..', '..', 'tools', 'doc', 'node_modules'));
+const unified = require('unified');
+const markdown = require('remark-parse');
+const remark2rehype = require('remark-rehype');
+const raw = require('rehype-raw');
+const htmlStringify = require('rehype-stringify');
+
+function toHTML({ input, filename, nodeVersion, analytics }, cb) {
+ const content = unified()
+ .use(markdown)
+ .use(html.firstHeader)
+ .use(html.preprocessText)
+ .use(html.preprocessElements, { filename })
+ .use(html.buildToc, { filename })
+ .use(remark2rehype, { allowDangerousHTML: true })
+ .use(raw)
+ .use(htmlStringify)
+ .processSync(input);
+
+ html.toHTML(
+ { input, content, filename, nodeVersion, analytics },
+ cb
+ );
+}
 
 // Test data is a list of objects with two properties.
 // The file property is the file path.
@@ -80,7 +107,6 @@ testData.forEach(({ file, html, analyticsId }) => {
 
  readFile(file, 'utf8', common.mustCall((err, input) => {
  assert.ifError(err);
-
  toHTML(
  {
  input: input,

test/doctool/test-doctool-json.js

@@ -10,9 +10,27 @@ try {
 
 const assert = require('assert');
 const fs = require('fs');
+const path = require('path');
 const fixtures = require('../common/fixtures');
 const json = require('../../tools/doc/json.js');
 
+module.paths.unshift(
+ path.join(__dirname, '..', '..', 'tools', 'doc', 'node_modules'));
+const unified = require('unified');
+const markdown = require('remark-parse');
+
+function toJSON(input, filename, cb) {
+ function nullCompiler() {
+ this.Compiler = (tree) => tree;
+ }
+
+ unified()
+ .use(markdown)
+ .use(json.jsonAPI, { filename })
+ .use(nullCompiler)
+ .process(input, cb);
+}
+
 // Outputs valid json with the expected fields when given simple markdown
 // Test data is a list of objects with two properties.
 // The file property is the file path.
@@ -21,15 +39,16 @@ const testData = [
  {
  file: fixtures.path('sample_document.md'),
  json: {
+ type: 'module',
  source: 'foo',
  modules: [{
  textRaw: 'Sample Markdown',
  name: 'sample_markdown',
  modules: [{
  textRaw: 'Seussian Rhymes',
  name: 'seussian_rhymes',
- desc: '<ol>\n<li>fish</li>\n<li><p>fish</p>\n</li>\n<li>' +
- '<p>Red fish</p>\n</li>\n<li>Blue fish</li>\n</ol>\n',
+ desc: '<ol>\n<li>fish</li>\n<li>fish</li>\n</ol>\n' +
+ '<ul>\n<li>Red fish</li>\n<li>Blue fish</li>\n</ul>',
  type: 'module',
  displayName: 'Seussian Rhymes'
  }],
@@ -41,6 +60,7 @@ const testData = [
  {
  file: fixtures.path('order_of_end_tags_5873.md'),
  json: {
+ type: 'module',
  source: 'foo',
  modules: [{
  textRaw: 'Title',
@@ -55,15 +75,10 @@ const testData = [
  signatures: [
  {
  params: [{
- textRaw: '`array` {Array} ',
+ textRaw: '`array` {Array}',
  name: 'array',
  type: 'Array'
  }]
- },
- {
- params: [{
- name: 'array'
- }]
  }
  ]
  }],
@@ -78,6 +93,7 @@ const testData = [
  {
  file: fixtures.path('doc_with_yaml.md'),
  json: {
+ type: 'module',
  source: 'foo',
  modules: [
  {
@@ -92,7 +108,7 @@ const testData = [
  changes: []
  },
  desc: '<p>Describe <code>Foobar</code> in more detail ' +
- 'here.</p>\n',
+ 'here.</p>',
  type: 'module',
  displayName: 'Foobar'
  },
@@ -110,7 +126,7 @@ const testData = [
  ]
  },
  desc: '<p>Describe <code>Foobar II</code> in more detail ' +
- 'here. fg(1)</p>\n',
+ 'here. fg(1)</p>',
  type: 'module',
  displayName: 'Foobar II'
  },
@@ -123,15 +139,15 @@ const testData = [
  changes: []
  },
  desc: '<p>Describe <code>Deprecated thingy</code> in more ' +
- 'detail here. fg(1p)</p>\n',
+ 'detail here. fg(1p)</p>',
  type: 'module',
  displayName: 'Deprecated thingy'
  },
  {
  textRaw: 'Something',
  name: 'something',
  desc: '<!-- This is not a metadata comment -->\n<p>' +
- 'Describe <code>Something</code> in more detail here.</p>\n',
+ 'Describe <code>Something</code> in more detail here.</p>',
  type: 'module',
  displayName: 'Something'
  }
@@ -147,9 +163,9 @@ const testData = [
 testData.forEach((item) => {
  fs.readFile(item.file, 'utf8', common.mustCall((err, input) => {
  assert.ifError(err);
- json(input, 'foo', common.mustCall((err, output) => {
+ toJSON(input, 'foo', common.mustCall((err, output) => {
  assert.ifError(err);
- assert.deepStrictEqual(output, item.json);
+ assert.deepStrictEqual(output.json, item.json);
  }));
  }));
 });

tools/doc/generate.js

@@ -22,53 +22,72 @@
 'use strict';
 
 const fs = require('fs');
+const path = require('path');
+const unified = require('unified');
+const markdown = require('remark-parse');
+const remark2rehype = require('remark-rehype');
+const raw = require('rehype-raw');
+const htmlStringify = require('rehype-stringify');
+
+const html = require('./html');
+const json = require('./json');
 
 // Parse the args.
 // Don't use nopt or whatever for this. It's simple enough.
 
 const args = process.argv.slice(2);
-let format = 'json';
 let filename = null;
 let nodeVersion = null;
 let analytics = null;
+let outputDir = null;
 
 args.forEach(function(arg) {
  if (!arg.startsWith('--')) {
  filename = arg;
- } else if (arg.startsWith('--format=')) {
- format = arg.replace(/^--format=/, '');
  } else if (arg.startsWith('--node-version=')) {
  nodeVersion = arg.replace(/^--node-version=/, '');
  } else if (arg.startsWith('--analytics=')) {
  analytics = arg.replace(/^--analytics=/, '');
+ } else if (arg.startsWith('--output-directory=')) {
+ outputDir = arg.replace(/^--output-directory=/, '');
  }
 });
 
 nodeVersion = nodeVersion || process.version;
 
 if (!filename) {
  throw new Error('No input file specified');
+} else if (!outputDir) {
+ throw new Error('No output directory specified');
 }
 
+
 fs.readFile(filename, 'utf8', (er, input) => {
  if (er) throw er;
- switch (format) {
- case 'json':
- require('./json.js')(input, filename, (er, obj) => {
- if (er) throw er;
- console.log(JSON.stringify(obj, null, 2));
- });
- break;
 
- case 'html':
- require('./html')({ input, filename, nodeVersion, analytics },
- (err, html) => {
- if (err) throw err;
- console.log(html);
- });
- break;
+ const content = unified()
+ .use(markdown)
+ .use(json.jsonAPI, { filename })
+ .use(html.firstHeader)
+ .use(html.preprocessText)
+ .use(html.preprocessElements, { filename })
+ .use(html.buildToc, { filename })
+ .use(remark2rehype, { allowDangerousHTML: true })
+ .use(raw)
+ .use(htmlStringify)
+ .processSync(input);
 
- default:
- throw new Error(`Invalid format: ${format}`);
- }
+ const basename = path.basename(filename, '.md');
+
+ html.toHTML(
+ { input, content, filename, nodeVersion, analytics },
+ (err, html) => {
+ const target = path.join(outputDir, `${basename}.html`);
+ if (err) throw err;
+ fs.writeFileSync(target, html);
+ }
+ );
+
+ const target = path.join(outputDir, `${basename}.json`);
+ fs.writeFileSync(target, JSON.stringify(content.json, null, 2));
 });

tools/doc/html.js

@@ -29,11 +29,13 @@ const visit = require('unist-util-visit');
 const markdown = require('remark-parse');
 const remark2rehype = require('remark-rehype');
 const raw = require('rehype-raw');
-const html = require('rehype-stringify');
+const htmlStringify = require('rehype-stringify');
 const path = require('path');
 const typeParser = require('./type-parser.js');
 
-module.exports = toHTML;
+module.exports = {
+ toHTML, firstHeader, preprocessText, preprocessElements, buildToc
+};
 
 const docPath = path.resolve(__dirname, '..', '..', 'doc');
 
@@ -54,26 +56,15 @@ const gtocHTML = unified()
  .use(remark2rehype, { allowDangerousHTML: true })
  .use(raw)
  .use(navClasses)
- .use(html)
+ .use(htmlStringify)
  .processSync(gtocMD).toString();
 
 const templatePath = path.join(docPath, 'template.html');
 const template = fs.readFileSync(templatePath, 'utf8');
 
-function toHTML({ input, filename, nodeVersion, analytics }, cb) {
+function toHTML({ input, content, filename, nodeVersion, analytics }, cb) {
  filename = path.basename(filename, '.md');
 
- const content = unified()
- .use(markdown)
- .use(firstHeader)
- .use(preprocessText)
- .use(preprocessElements, { filename })
- .use(buildToc, { filename })
- .use(remark2rehype, { allowDangerousHTML: true })
- .use(raw)
- .use(html)
- .processSync(input);
-
  const id = filename.replace(/\W+/g, '-');
 
  let HTML = template.replace('__ID__', id)
@@ -296,7 +287,7 @@ function parseYAML(text) {
  .use(markdown)
  .use(remark2rehype, { allowDangerousHTML: true })
  .use(raw)
- .use(html)
+ .use(htmlStringify)
  .processSync(change.description).toString();
 
  result += `<tr><td>${change.version}</td>\n` +
@@ -381,7 +372,7 @@ function buildToc({ filename }) {
  .use(markdown)
  .use(remark2rehype, { allowDangerousHTML: true })
  .use(raw)
- .use(html)
+ .use(htmlStringify)
  .processSync(toc).toString();
  };
 }