fix: add skipNavigation option to enqueueLinks (#2153)

Co-authored-by: Martin Adámek <banan23@gmail.com>

Author: gustavotr

packages/core/src/enqueue_links/enqueue_links.ts

@@ -35,6 +35,12 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
  /** Sets {@apilink Request.label} for newly enqueued requests. */
  label?: string;
 
+ /**
+ * If set to `true`, tells the crawler to skip navigation and process the request directly.
+ * @default false
+ */
+ skipNavigation?: boolean;
+
  /**
  * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
  * since the relative URL resolution is done inside the browser automatically.
@@ -239,6 +245,7 @@ export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 're
  urls: ow.array.ofType(ow.string),
  requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
  forefront: ow.optional.boolean,
+ skipNavigation: ow.optional.boolean,
  limit: ow.optional.number,
  selector: ow.optional.string,
  baseUrl: ow.optional.string,

packages/core/src/enqueue_links/shared.ts

@@ -208,7 +208,7 @@ export function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatt
  */
 export function createRequestOptions(
  sources: (string | Record<string, unknown>)[],
- options: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl'> = {},
+ options: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl' | 'skipNavigation'> = {},
 ): RequestOptions[] {
  return sources
  .map((src) => (typeof src === 'string' ? { url: src } : src as unknown as RequestOptions))
@@ -230,6 +230,10 @@ export function createRequestOptions(
  };
  }
 
+ if (options.skipNavigation) {
+ requestOptions.skipNavigation = true;
+ }
+
  return requestOptions;
  });
 }

packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts

@@ -166,6 +166,12 @@ export interface EnqueueLinksByClickingElementsOptions {
  * @default false
  */
  forefront?: boolean;
+
+ /**
+ * If set to `true`, tells the crawler to skip navigation and process the request directly.
+ * @default false
+ */
+ skipNavigation?: boolean;
 }
 
 /**
@@ -233,6 +239,7 @@ export async function enqueueLinksByClickingElements(options: EnqueueLinksByClic
  maxWaitForPageIdleSecs: ow.optional.number,
  label: ow.optional.string,
  forefront: ow.optional.boolean,
+ skipNavigation: ow.optional.boolean,
  }));
 
  const {

packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts

@@ -167,6 +167,12 @@ export interface EnqueueLinksByClickingElementsOptions {
  * @default false
  */
  forefront?: boolean;
+
+ /**
+ * If set to `true`, tells the crawler to skip navigation and process the request directly.
+ * @default false
+ */
+ skipNavigation?: boolean;
 }
 
 /**
@@ -234,6 +240,7 @@ export async function enqueueLinksByClickingElements(options: EnqueueLinksByClic
  maxWaitForPageIdleSecs: ow.optional.number,
  label: ow.optional.string,
  forefront: ow.optional.boolean,
+ skipNavigation: ow.optional.boolean,
  }));
 
  const {

test/core/autoscaling/autoscaled_pool.test.ts

@@ -541,5 +541,3 @@ describe('AutoscaledPool', () => {
  expect(Date.now() - now).toBeGreaterThanOrEqual(1e3);
  }, 10e3);
 });
-
-/* eslint-enable no-underscore-dangle */

test/core/enqueue_links/enqueue_links.test.ts

@@ -189,6 +189,26 @@ describe('enqueueLinks()', () => {
  expect(enqueued[2].userData).toEqual({ label: 'COOL' });
  });
 
+ test('works with skipNavigation', async () => {
+ const { enqueued, requestQueue } = createRequestQueueMock();
+
+ await browserCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ skipNavigation: true,
+ },
+ page,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+
+ for (const request of enqueued) {
+ expect(request.skipNavigation).toBe(true);
+ }
+ });
+
  test('works with exclude glob', async () => {
  const { enqueued, requestQueue } = createRequestQueueMock();
  const globs = [