feat: make the CLI download directly from GitHub (#1540)

Author: vladfrangu

packages/cli/src/commands/CreateProjectCommand.ts

@@ -1,13 +1,15 @@
 import { mkdirSync } from 'node:fs';
-import { join } from 'node:path';
+import { dirname, join } from 'node:path';
 import { execSync } from 'node:child_process';
 import type { ArgumentsCamelCase, Argv, CommandModule } from 'yargs';
 import { prompt } from 'inquirer';
 import colors from 'ansi-colors';
+import type { Template } from '@crawlee/templates';
 import { fetchManifest } from '@crawlee/templates';
-import { copy } from 'fs-extra';
-import { readFileSync, writeFileSync } from 'fs';
 import { resolve } from 'path';
+import { readFile, writeFile } from 'node:fs/promises';
+import { get } from 'node:https';
+import { ensureDir } from 'fs-extra';
 
 interface CreateProjectArgs {
  projectName?: string;
@@ -20,16 +22,52 @@ function validateProjectName(name: string) {
  }
 }
 
-function rewrite(path: string, replacer: (from: string) => string): void {
+async function rewrite(path: string, replacer: (from: string) => string) {
  try {
- const file = readFileSync(path).toString();
+ const file = await readFile(path, 'utf8');
  const replaced = replacer(file);
- writeFileSync(path, replaced);
+ await writeFile(path, replaced);
  } catch {
  // not found
  }
 }
 
+async function downloadTemplateFilesToDisk(template: Template, destinationDirectory: string) {
+ const promises: Promise<void>[] = [];
+
+ for (const file of template.files) {
+ const promise = downloadFile(file.url).then(async (buffer) => {
+ // Make sure the folder for the file exists
+ const fileDirName = dirname(file.path);
+ const fileFolder = resolve(destinationDirectory, fileDirName);
+ await ensureDir(fileFolder);
+
+ // Write the actual file
+ await writeFile(resolve(destinationDirectory, file.path), buffer);
+ });
+
+ promises.push(promise);
+ }
+
+ await Promise.all(promises);
+}
+
+async function downloadFile(url: string) {
+ return new Promise<Buffer>((promiseResolve, reject) => {
+ get(url, async (res) => {
+ const bytes: Buffer[] = [];
+
+ res.on('error', (err) => reject(err));
+
+ for await (const byte of res) {
+ bytes.push(byte);
+ }
+
+ promiseResolve(Buffer.concat(bytes));
+ }).on('error', (err) => reject(err));
+ });
+}
+
 export class CreateProjectCommand<T> implements CommandModule<T, CreateProjectArgs> {
  command = 'create [project-name]';
  describe = 'Creates a new Crawlee project directory from a selected boilerplate template.';
@@ -105,8 +143,10 @@ export class CreateProjectCommand<T> implements CommandModule<T, CreateProjectAr
  throw err;
  }
 
- await copy(require.resolve('@crawlee/templates').replace('index.js', `templates/${template}`), projectDir);
- rewrite(resolve(projectDir, 'package.json'), (pkg) => pkg.replace(/"name": "[\w-]+"/, `"name": "${projectName}"`));
+ const templateData = manifest.templates.find((item) => item.name === template)!;
+
+ await downloadTemplateFilesToDisk(templateData, projectDir);
+ await rewrite(resolve(projectDir, 'package.json'), (pkg) => pkg.replace(/"name": "[\w-]+"/, `"name": "${projectName}"`));
 
  // Run npm install in project dir.
  const npm = /^win/.test(process.platform) ? 'npm.cmd' : 'npm';

packages/templates/manifest.json

@@ -2,35 +2,109 @@
  "templates": [
  {
  "name": "getting-started-ts",
- "description": "Getting started example [TypeScript]"
+ "description": "Getting started example [TypeScript]",
+ "files": [
+ "src/main.ts",
+ ".dockerignore",
+ ".gitignore",
+ "Dockerfile",
+ "package.json",
+ "README.md",
+ "tsconfig.json"
+ ]
  },
  {
  "name": "getting-started-js",
- "description": "Getting started example [JavaScript]"
+ "description": "Getting started example [JavaScript]",
+ "files": [
+ "src/main.js",
+ ".dockerignore",
+ ".gitignore",
+ "Dockerfile",
+ "package.json",
+ "README.md"
+ ]
  },
  {
  "name": "cheerio-ts",
- "description": "CheerioCrawler template project [TypeScript]"
+ "description": "CheerioCrawler template project [TypeScript]",
+ "files": [
+ "src/main.ts",
+ "src/routes.ts",
+ ".dockerignore",
+ ".gitignore",
+ "Dockerfile",
+ "package.json",
+ "README.md",
+ "tsconfig.json"
+ ]
  },
  {
  "name": "playwright-ts",
- "description": "PlaywrightCrawler template project [TypeScript]"
+ "description": "PlaywrightCrawler template project [TypeScript]",
+ "files": [
+ "src/main.ts",
+ "src/routes.ts",
+ ".dockerignore",
+ ".gitignore",
+ "Dockerfile",
+ "package.json",
+ "README.md",
+ "tsconfig.json"
+ ]
  },
  {
  "name": "puppeteer-ts",
- "description": "PuppeteerCrawler template project [TypeScript]"
+ "description": "PuppeteerCrawler template project [TypeScript]",
+ "files": [
+ "src/main.ts",
+ "src/routes.ts",
+ ".dockerignore",
+ ".gitignore",
+ "Dockerfile",
+ "package.json",
+ "README.md",
+ "tsconfig.json"
+ ]
  },
  {
  "name": "cheerio-js",
- "description": "CheerioCrawler template project [JavaScript]"
+ "description": "CheerioCrawler template project [JavaScript]",
+ "files": [
+ "src/main.js",
+ "src/routes.js",
+ ".dockerignore",
+ ".gitignore",
+ "Dockerfile",
+ "package.json",
+ "README.md"
+ ]
  },
  {
  "name": "playwright-js",
- "description": "PlaywrightCrawler template project [JavaScript]"
+ "description": "PlaywrightCrawler template project [JavaScript]",
+ "files": [
+ "src/main.js",
+ "src/routes.js",
+ ".dockerignore",
+ ".gitignore",
+ "Dockerfile",
+ "package.json",
+ "README.md"
+ ]
  },
  {
  "name": "puppeteer-js",
- "description": "PuppeteerCrawler template project [JavaScript]"
+ "description": "PuppeteerCrawler template project [JavaScript]",
+ "files": [
+ "src/main.js",
+ "src/routes.js",
+ ".dockerignore",
+ ".gitignore",
+ "Dockerfile",
+ "package.json",
+ "README.md"
+ ]
  }
  ]
 }

packages/templates/package.json

@@ -39,11 +39,11 @@
  },
  "homepage": "https://crawlee.dev",
  "scripts": {
- "build": "npm run clean && npm run compile && npm run copy-templates && npm run copy",
+ "build": "npm run clean && npm run validate && npm run compile && npm run copy",
  "clean": "rimraf ./dist",
  "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs",
- "copy-templates": "node ./scripts/copy-templates.mjs",
- "copy": "ts-node -T ../../scripts/copy.ts"
+ "copy": "ts-node -T ../../scripts/copy.ts",
+ "validate": "node ./scripts/validate-manifest.mjs"
  },
  "publishConfig": {
  "access": "public"

packages/templates/scripts/copy-templates.mjs

@@ -0,0 +1,49 @@
+import { readFile, readdir, access } from 'node:fs/promises';
+import { URL } from 'node:url';
+
+const colors = {
+ red: (text) => `\x1B[31m${text}\x1B[39m`,
+ green: (text) => `\x1B[32m${text}\x1B[39m`,
+ grey: (text) => `\x1B[90m${text}\x1B[39m`,
+ yellow: (text) => `\x1B[33m${text}\x1B[39m`,
+};
+
+const templatesDirectory = new URL('../templates/', import.meta.url);
+const templateNames = await readdir(templatesDirectory);
+/** @type {{ templates: Array<{ name: string; description: string; files: string[] }>; }} */
+const manifest = JSON.parse(await readFile(new URL('../manifest.json', import.meta.url), 'utf8'));
+
+console.log(`Validating ${colors.green(manifest.templates.length)} templates`);
+
+let hasError = false;
+
+for (const manifestTemplate of manifest.templates) {
+ // Check if the folder it points to actually exists
+ if (!templateNames.includes(manifestTemplate.name)) {
+ console.error(colors.red(`Failed to find folder for template called ${colors.yellow(manifestTemplate.name)}`));
+ hasError = true;
+ // Skipping the rest of the validation as the template is missing
+ continue;
+ }
+
+ console.log(colors.grey(`Validating template ${colors.yellow(manifestTemplate.name)}`));
+
+ // Check that all files it requires exist
+ for (const requiredFile of manifestTemplate.files) {
+ try {
+ await access(new URL(`./${manifestTemplate.name}/${requiredFile}`, templatesDirectory));
+ } catch (err) {
+ if (err.code === 'ENOENT') {
+ hasError = true;
+ console.error(`${colors.grey(`[${colors.yellow(manifestTemplate.name)}]:`)} Failed to find file ${colors.yellow(requiredFile)}`);
+ console.error(err);
+ } else {
+ console.warn(`${colors.grey(`[${colors.yellow(manifestTemplate.name)}]:`)} Failed to read file ${colors.yellow(requiredFile)}`, err);
+ }
+ }
+ }
+
+ console.log(colors.green(`Finished validating ${colors.yellow(manifestTemplate.name)}`));
+}
+
+process.exitCode = hasError ? 1 : 0;

packages/templates/scripts/validate-manifest.mjs

@@ -2,19 +2,47 @@ import https from 'https';
 
 export const MANIFEST_URL = 'https://raw.githubusercontent.com/apify/crawlee/master/packages/templates/manifest.json';
 
+function templateFileUrl(templateName: string, path: string) {
+ return `https://raw.githubusercontent.com/apify/crawlee/master/packages/templates/templates/${templateName}/${path}`;
+}
+
+interface SharedTemplateData {
+ name: string;
+ description: string;
+}
+
+// Data received from the github file
+interface RawTemplate extends SharedTemplateData {
+ files: string[];
+}
+
+interface RawManifest {
+ templates: RawTemplate[];
+}
+
+// Data returned for the CLI or users to consume
 export interface Manifest {
- templates: { name: string; description: string }[];
+ templates: Template[];
+}
+
+export interface Template extends SharedTemplateData {
+ files: TemplateFile[];
+}
+
+export interface TemplateFile {
+ path: string;
+ url: string;
 }
 
 export async function fetchManifest(): Promise<Manifest> {
- return new Promise((resolve, reject) => {
+ const rawManifest = await new Promise<RawManifest>((resolve, reject) => {
  https.get(MANIFEST_URL, (res) => {
  let json = '';
  res
  .on('data', (chunk) => {
  json += chunk;
  })
- .on('end', () => {
+ .once('end', () => {
  if (res.statusCode === 200) {
  try {
  const data = JSON.parse(json);
@@ -30,4 +58,19 @@ export async function fetchManifest(): Promise<Manifest> {
  })
  .on('error', (err) => reject(err));
  });
+
+ const newTemplates: Template[] = rawManifest.templates.map((original) => {
+ return {
+ name: original.name,
+ description: original.description,
+ files: original.files.map((file) => ({
+ path: file,
+ url: templateFileUrl(original.name, file),
+ })),
+ };
+ });
+
+ return {
+ templates: newTemplates,
+ };
 }