fix(http-crawler): do not hang on POST without payload (#1546)

Author: szmarczak

packages/http-crawler/src/internals/http-crawler.ts

@@ -641,7 +641,7 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H
  };
  }
 
- if (/PATCH|POST|PUT/.test(request.method)) requestOptions.body = request.payload;
+ if (/PATCH|POST|PUT/.test(request.method)) requestOptions.body = request.payload ?? '';
 
  return requestOptions;
  }

test/core/crawlers/http_crawler.test.ts

@@ -41,6 +41,11 @@ router.set('/redirectWithoutCookies', (req, res) => {
  res.end();
 });
 
+router.set('/echo', (req, res) => {
+ res.setHeader('content-type', 'text/html');
+ req.pipe(res);
+});
+
 let server: http.Server;
 let url: string;
 
@@ -231,3 +236,23 @@ test('no empty cookie header', async () => {
 
  expect(results).toStrictEqual([]);
 });
+
+test('POST with undefined (empty) payload', async () => {
+ const results: string[] = [];
+
+ const crawler = new HttpCrawler({
+ handlePageFunction: async ({ body }) => {
+ results.push(body.toString());
+ },
+ });
+
+ await crawler.run([
+ {
+ url: `${url}/echo`,
+ payload: undefined,
+ method: 'POST',
+ },
+ ]);
+
+ expect(results).toStrictEqual(['']);
+});