Attempt to migrate external repo docs to new format

.gitignore

@@ -7,4 +7,6 @@
 /htmltest/
 /tmp/
 node_modules/
+/scripts/repos/
+/scripts/temp/
 *.pyc

scripts/changeLink.js

@@ -1,18 +1,27 @@
 module.exports = function changeLink(relative) {
- // replace segment camelcase
- // /Wurst/WALHasenPengPuff => /Wurst/WALHasen-peng-puff
- relative = relative.replace(/(?<![A-Z\/])([A-Z])/g, function(x, char) {
- return "-" + char.toLowerCase();
+ let origLink = relative;
+
+ // replace segment camel case
+ // /Wurst/WALHasenPengPuff => /Wurst/WALHasen-Peng-Puff
+ relative = relative.replace(/(?<![A-Z\/\\])([A-Z])/g, function(x, char) {
+ return "-" + char;
  });
  if (relative.startsWith("-")) {
  relative = relative.substr(1);
  }
  relative = relative.toLowerCase();
- let fileName = relative.replace(/\/(readme\.md|index\.html)$/, '').replace(/\.(md|html)/, '').replace(/[^a-z0-9]+/g, '-') + '.md';
-
+
+ let fileName = relative
+ .replace(/[\/\\](?:readme\.md|index\.html)$/, '')
+ .replace(/\.(?:md|html)/, '')
+ .replace(/[^a-z0-9]+/g, '-') + '.md';
+
  // root level index
  if (fileName == 'readme.md') {
  fileName = 'index.md'
  }
+
+ //console.log(`${origLink} => ${fileName}`)
+
  return fileName;
 }

scripts/fetchRefs.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# Must not set -e or ssh stuff will abort the script
+#set -e
+
+GITAUTH="$1"
+
+if test "${GITAUTH}" == "migrate"; then
+ # Fast path, migration only
+ SOURCES=()
+else
+ SOURCES=(
+ "https://@github.com/arangodb-helper/arangodb.git;arangodb-starter;master;docs/;Manual/"
+ "https://@github.com/arangodb/arangosync.git;arangosync;master;docs/;Manual/"
+ "https://@github.com/arangodb/kube-arangodb.git;kube-arangodb;master;docs/;Manual/"
+ "https://@github.com/arangodb/arangodb-java-driver.git;arangodb-java-driver;master;docs/;Drivers/"
+ "https://@github.com/arangodb/arangodb-php.git;arangodb-php;devel;docs/;Drivers/"
+ "https://@github.com/arangodb/arangodb-spark-connector.git;arangodb-spark-connector;master;docs/;Drivers/"
+ "https://@github.com/arangodb/arangojs.git;arangojs;master;docs/;Drivers/"
+ "https://@github.com/arangodb/go-driver.git;go-driver;master;docs/;Drivers/"
+ "https://@github.com/arangodb/spring-data.git;spring-data;master;docs/;Drivers/"
+ )
+fi
+
+# Ensure agent is running
+if test "${GITAUTH}" == "git"; then
+ ssh-add -l &> /dev/null
+ if [ "$?" == 2 ]; then
+ # Could not open a connection to your authentication agent.
+
+ # Load stored agent connection info
+ test -r ~/.ssh-agent && eval "$(<~/.ssh-agent)" > /dev/null
+ ssh-add -l &> /dev/null
+ if [ "$?" == 2 ]; then
+ # Start agent and store connection info
+ (umask 066; ssh-agent > ~/.ssh-agent)
+ eval "$(<~/.ssh-agent)" > /dev/null
+ fi
+ fi
+
+ # Load identities
+ ssh-add -l &> /dev/null
+ if [ "$?" == 1 ]; then
+ # The agent has no identities.
+ ssh-add
+ fi
+fi
+
+for source in "${SOURCES[@]}"; do
+
+ REPO=$(echo "$source" |cut -d ';' -f 1)
+ CLONEDIR=$(echo "$source" |cut -d ';' -f 2)
+ BRANCH=$(echo "$source" |cut -d ';' -f 3)
+ SUBDIR=$(echo "$source" |cut -d ';' -f 4)
+ DST=$(echo "$source" |cut -d ';' -f 5)
+
+ CODIR="repos/${CLONEDIR}"
+ AUTHREPO="${REPO/@/${GITAUTH}@}"
+ if test -d "${CODIR}"; then
+ (
+ cd "${CODIR}"
+ git pull --all
+ )
+ else
+ if test "${GITAUTH}" == "git"; then
+ AUTHREPO=$(echo "${AUTHREPO}" | sed -e "s;github.com/;github.com:;" -e "s;https://;;" )
+ fi
+ git clone "${AUTHREPO}" "${CODIR}"
+ fi
+
+ if [ -z "${BRANCH}" ]; then
+ echo "No branch specified for ${CLONEDIR}"
+ exit 1
+ fi
+
+ # checkout branch and pull=merge origin
+ (cd "${CODIR}" && git checkout "${BRANCH}" && git pull)
+
+ for oneMD in $(cd "${CODIR}/${SUBDIR}/${DST}"; find "./" -type f |sed "s;\./;;"); do
+ NAME=$(basename "${oneMD}")
+ MDSUBDIR="${oneMD/${NAME}/}"
+ DSTDIR="temp/${DST}/${MDSUBDIR}"
+ if test ! -d "${DSTDIR}"; then
+ mkdir -p "${DSTDIR}"
+ fi
+ sourcefile="${CODIR}/${SUBDIR}/${DST}/${oneMD}"
+ targetfile="${DSTDIR}/${NAME}"
+ cp "$sourcefile" "$targetfile"
+ done
+done
+
+folder="3.5"
+echo "Migrating to ${folder}"
+node migrate.js temp/Drivers ../${folder}/drivers > /dev/null
+node migrate.js temp/Manual ../${folder} > /dev/null

scripts/migrate.js

@@ -10,7 +10,9 @@ const fetch = require('node-fetch');
 
 async function migrateMds(basePath, targetPath) {
  const absoluteBasePath = await fs.realpath(basePath);
- const paths = (await globby([path.join(basePath, "**/*.md")])).filter(filePath => {
+ // NOTE globby >= 10 does not support backslashes anymore
+ const searchPath = path.join(basePath, "**/*.md").replace(/\\/g, '/')
+ const paths = (await globby([searchPath])).filter(filePath => {
  return path.relative(basePath, filePath).toLowerCase() !== 'summary.md';
  });
 
@@ -23,7 +25,7 @@ async function migrateMds(basePath, targetPath) {
  let book = path.basename(basePath);
  const version = targetPath.split("/")[0];
  let urlVersion = version;
- if (urlVersion == "3.5") {
+ if (urlVersion == "3.6") {
  urlVersion = "devel";
  }
 
@@ -63,7 +65,7 @@ async function migrateMds(basePath, targetPath) {
  }
  } else {
  description = paragraph.textContent
- .replace(/\n/g, '')
+ .replace(/\n/g, ' ')
  .replace(/(.*?)[\.:].*/ms, '\$1')
  .replace(/{%\s*hint[^%]*\s*%}/, '')
  .replace(/{%\s*endhint\s*%}/, '')
@@ -87,6 +89,17 @@ async function migrateMds(basePath, targetPath) {
  }
  content = content.replace("ArangoDB VERSION_NUMBER", "ArangoDB {{ site.data.versions[page.version.name] }}");
 
+ // HACK: Fix cross-links
+ content = content.replace(/\]\(https:\/\/docs\.arangodb\.com\/latest\/(.*?)\)/g, (fullMatch, link) => {
+ link = path.relative(relative, link).replace(/\\/g, '/').replace(/\/index\.html/, '.html').replace(/\/README.md/, '.html');
+ return `](${link})`;
+ });
+ content = content.replace(/\]\(https:\/\/www\.arangodb\.com\/docs\/stable\/(.*?)\)/g, (fullMatch, link) => {
+ link = path.relative(relative, link).replace(/\\/g, '/').replace(/\/index\.html/, '.html').replace(/\/README.md/, '.html');
+ console.dir({fullMatch, link});
+ return `](${link})`;
+ });
+
  // replace all md links with their changed link
  content = content.replace(/\]\((?!https?:)(.*?\.(html|md))(#[^\)]+)?\)/g, (x, link, fileExt, anchor) => {
  if (!link.startsWith('/')) {
@@ -156,7 +169,9 @@ async function migrateMds(basePath, targetPath) {
 
 // verified beforehand that all imagenames are unique over all directories
 async function migrateImages(basePath, targetPath) {
- const paths = await globby([path.join(basePath + "/**/*.png")]);
+ // NOTE globby >= 10 does not support backslashes anymore
+ searchPath = path.join(basePath + "/**/*.png").replace(/\\/g, '/')
+ const paths = await globby([searchPath]);
  let imagePath;
  if (path.basename(basePath) == "Manual" || path.basename(basePath) == "Users") {
  imagePath = path.join(targetPath, "images");

scripts/package.json

@@ -9,11 +9,11 @@
  "author": "",
  "license": "ISC",
  "dependencies": {
- "globby": "^9.1.0",
- "jsdom": "^14.0.0",
- "marked-it-core": "^0.13.1",
+ "globby": "^10.0.1",
+ "jsdom": "^15.2.1",
+ "marked-it-core": "^0.14.0",
  "mkdirp-promise": "^5.0.1",
- "node-fetch": "^2.3.0",
- "showdown": "^1.9.0"
+ "node-fetch": "^2.6.0",
+ "showdown": "^1.9.1"
  }
 }