fix: publish purely static pages router files to the CDN (#29)

* chore: simplify get/remove prerendered content * chore: pin fastly module * chore: add a couple comments to storing of prerendered content * fix: avoid overawaiting when globbing * feat: add static pages content to CDN * fix: ensure json data is not bundled

Author: orinokai

package-lock.json

@@ -1,53 +1,77 @@
 import { cpus } from 'os'
-import path from 'path'
+import { parse, ParsedPath } from 'path'
 
 import { getDeployStore } from '@netlify/blobs'
 import { NetlifyPluginConstants } from '@netlify/build'
-import { copy, move, remove } from 'fs-extra/esm'
+import { copy, move, mkdirp } from 'fs-extra/esm'
 import { globby } from 'globby'
 import pLimit from 'p-limit'
 
 import { buildCacheValue } from './cache.js'
 import { BUILD_DIR } from './constants.js'
+import { EnhancedNetlifyPluginConstants } from './types.js'
+
+type ContentPath = ParsedPath & {
+ relative: string
+ absolute: string
+ publish: string
+}
 
 /**
  * Move the Next.js build output from the publish dir to a temp dir
  */
-export const stashBuildOutput = async ({ PUBLISH_DIR }: NetlifyPluginConstants) => {
- await move(PUBLISH_DIR, `${BUILD_DIR}/.next`, { overwrite: true })
+export const stashBuildOutput = async ({ PUBLISH_DIR }: NetlifyPluginConstants): Promise<void> => {
+ return move(PUBLISH_DIR, `${BUILD_DIR}/.next`, { overwrite: true })
+}
 
- // remove prerendered content from the standalone build (it's also in the main build dir)
- const prerenderedContent = await getPrerenderedContent(`${BUILD_DIR}/.next/standalone`, false)
- await Promise.all(
- prerenderedContent.map((file: string) => remove(`${BUILD_DIR}/.next/standalone/${file}`)),
- ).catch((error) => console.error(error))
+/**
+ * Glob the build output for static page content we can upload to the CDN
+ */
+const getStaticContent = async (cwd: string): Promise<ContentPath[]> => {
+ const content = await globby([`server/pages/**/*.+(html|json)`], {
+ cwd,
+ extglob: true,
+ })
+ return content
+ .map((path) => parsePath(path, cwd))
+ .filter((path) => filterStatic(path, content, 'keep'))
 }
 
 /**
- * Glob for prerendered content in the build output
+ * Glob the build output for prerendered content we can upload to the blob store
  */
-const getPrerenderedContent = async (cwd: string, get = true): Promise<string[]> => {
- // TODO: test this
- return await globby(
- get
- ? [`cache/fetch-cache/*`, `server/+(app|pages)/**/*.+(html|body)`]
- : [
- `cache/fetch-cache/*`,
- `server/+(app|pages)/**/*.+(html|json|rsc|body|meta)`,
- `!server/**/*.js.nft.{html,json}`,
- ],
- { cwd, extglob: true },
+const getPrerenderedContent = async (cwd: string): Promise<ContentPath[]> => {
+ const content = await globby(
+ [`cache/fetch-cache/*`, `server/+(app|pages)/**/*.+(html|body|json)`],
+ {
+ cwd,
+ extglob: true,
+ },
  )
+ return content
+ .map((path) => parsePath(path, cwd))
+ .filter((path) => filterStatic(path, content, 'omit'))
 }
 
 /**
- * Upload prerendered content from the main build dir to the blob store
+ * Glob the build output for JS content we can bundle with the server handler
+ */
+export const getServerContent = async (cwd: string): Promise<ContentPath[]> => {
+ const content = await globby([`**`, `!server/+(app|pages)/**/*.+(html|body|json|rsc|meta)`], {
+ cwd,
+ extglob: true,
+ })
+ return content.map((path) => parsePath(path, cwd))
+}
+
+/**
+ * Upload prerendered content to the blob store and remove it from the bundle
  */
 export const storePrerenderedContent = async ({
  NETLIFY_API_TOKEN,
  NETLIFY_API_HOST,
  SITE_ID,
-}: NetlifyPluginConstants & { NETLIFY_API_TOKEN: string; NETLIFY_API_HOST: string }) => {
+}: EnhancedNetlifyPluginConstants): Promise<void[]> => {
  if (!process.env.DEPLOY_ID) {
  // TODO: maybe change to logging
  throw new Error(
@@ -61,28 +85,52 @@ export const storePrerenderedContent = async ({
  token: NETLIFY_API_TOKEN,
  apiURL: `https://${NETLIFY_API_HOST}`,
  })
-
- // todo: Check out setFiles within Blobs.js to see how to upload files to blob storage
  const limit = pLimit(Math.max(2, cpus().length))
 
- const prerenderedContent = await getPrerenderedContent(`${BUILD_DIR}/.next`)
+ const content = await getPrerenderedContent(`${BUILD_DIR}/.next/standalone/.next`)
  return await Promise.all(
- prerenderedContent.map(async (rawPath: string) => {
- // TODO: test this with files that have a double extension
- const ext = path.extname(rawPath)
- const key = rawPath.replace(ext, '')
- const value = await buildCacheValue(key, ext)
+ content.map((path: ContentPath) => {
+ const { dir, name, ext } = path
+ const key = `${dir}/${name}`
+ const value = buildCacheValue(key, ext)
  return limit(() => blob.setJSON(key, value))
  }),
  )
 }
 
 /**
- * Move static assets to the publish dir so they are uploaded to the CDN
+ * Move static content to the publish dir so it is uploaded to the CDN
  */
-export const publishStaticAssets = ({ PUBLISH_DIR }: NetlifyPluginConstants) => {
- return Promise.all([
+export const publishStaticContent = async ({
+ PUBLISH_DIR,
+}: NetlifyPluginConstants): Promise<void[]> => {
+ const content = await getStaticContent(`${BUILD_DIR}/.next/standalone/.next`)
+ return await Promise.all([
+ mkdirp(PUBLISH_DIR),
  copy('public', PUBLISH_DIR),
  copy(`${BUILD_DIR}/.next/static/`, `${PUBLISH_DIR}/_next/static`),
+ ...content.map((path: ContentPath) => copy(path.absolute, `${PUBLISH_DIR}/${path.publish}`)),
  ])
 }
+
+/**
+ * Keep or remove static content based on whether it has a corresponding JSON file
+ */
+const filterStatic = (
+ { dir, name, ext }: ContentPath,
+ content: string[],
+ type: 'keep' | 'omit',
+): boolean =>
+ type === 'keep'
+ ? dir.startsWith('server/pages') && !content.includes(`${dir}/${name}.json`)
+ : ext !== '.json' &&
+ (!dir.startsWith('server/pages') || content.includes(`${dir}/${name}.json`))
+/**
+ * Parse a file path into an object with file path variants
+ */
+const parsePath = (path: string, cwd: string): ContentPath => ({
+ ...parse(path),
+ relative: path,
+ absolute: `${cwd}/${path}`,
+ publish: path.replace(/^server\/(app|pages)\//, ''),
+})

src/helpers/files.ts

@@ -10,6 +10,7 @@ import {
  SERVER_HANDLER_DIR,
  SERVER_HANDLER_NAME,
 } from './constants.js'
+import { getServerContent } from './files.js'
 
 const pkg = await readJson(`${PLUGIN_DIR}/package.json`)
 
@@ -32,7 +33,10 @@ export const createServerHandler = async () => {
  )
 
  // copy the next.js standalone build output to the handler directory
- await copy(`${BUILD_DIR}/.next/standalone/.next`, `${SERVER_HANDLER_DIR}/.next`)
+ const content = await getServerContent(`${BUILD_DIR}/.next/standalone/.next`)
+ await Promise.all(
+ content.map((path) => copy(path.absolute, `${SERVER_HANDLER_DIR}/.next/${path.relative}`)),
+ )
  await copy(`${BUILD_DIR}/.next/standalone/node_modules`, `${SERVER_HANDLER_DIR}/node_modules`)
 
  // create the handler metadata file

src/helpers/functions.ts

@@ -3,15 +3,18 @@ import type { OutgoingHttpHeaders } from 'http'
 import type { NetlifyPluginOptions, NetlifyPluginConstants } from '@netlify/build'
 import type { NextConfigComplete } from 'next/dist/server/config-shared.js'
 
+type NetlifyPluginOptionsWithFlags = NetlifyPluginOptions & {
+ featureFlags?: Record<string, unknown>
+}
 
-type NetlifyPluginOptionsWithFlags = NetlifyPluginOptions & { featureFlags?: Record<string, unknown> }
-
-type EnhancedNetlifyPluginConstants = NetlifyPluginConstants & {
+export type EnhancedNetlifyPluginConstants = NetlifyPluginConstants & {
  NETLIFY_API_HOST: string
  NETLIFY_API_TOKEN: string
 }
 
-export type EnhancedNetlifyPluginOptions = NetlifyPluginOptions & { constants: EnhancedNetlifyPluginConstants } & {
+export type EnhancedNetlifyPluginOptions = NetlifyPluginOptions & {
+ constants: EnhancedNetlifyPluginConstants
+} & {
  featureFlags?: Record<string, unknown>
 }
 
@@ -27,4 +30,4 @@ export interface RequiredServerFiles {
 export interface MetaFile {
  status?: number
  headers?: OutgoingHttpHeaders
-}
+}

src/helpers/types.ts

@@ -1,5 +1,5 @@
 import { setBuildConfig } from './helpers/config.js'
-import { publishStaticAssets, stashBuildOutput, storePrerenderedContent } from './helpers/files.js'
+import { publishStaticContent, stashBuildOutput, storePrerenderedContent } from './helpers/files.js'
 import { createEdgeHandler, createServerHandler } from './helpers/functions.js'
 import { EnhancedNetlifyPluginOptions } from './helpers/types.js'
 
@@ -11,7 +11,7 @@ export const onBuild = async ({ constants }: EnhancedNetlifyPluginOptions) => {
  await stashBuildOutput(constants)
 
  return Promise.all([
- publishStaticAssets(constants),
+ publishStaticContent(constants),
  storePrerenderedContent(constants),
  createServerHandler(),
  createEdgeHandler(),