fix: Allow split page logic to process files concurrently (#121)

### Summary copy Unstructured-IO/unstructured-python-client#175 into JS/TS ### Test added a integration tests that is passed on CI Local integration test: * docker start core product api but with -p 8000:5000 (change the line in make file for make docker-start-api) * `make build` * `npx jest --verbose --detectOpenHandles --config jest.config.js test/integration --forceExit -t "SplitPDF async can be used to send multiple files concurrently"` you can also move the unit test to main and `make build npx jest test...` again and will see the test fail, but not on this branch

Author: yuming-long

package-lock.json

@@ -20,6 +20,7 @@
  "devDependencies": {
  "@types/async": "^3.2.24",
  "@types/jest": "^29.5.12",
+ "@types/uuid": "^10.0.0",
  "@typescript-eslint/eslint-plugin": "^7.7.1",
  "@typescript-eslint/parser": "^7.7.1",
  "eslint": "^8.57.0",
@@ -32,6 +33,7 @@
  },
  "dependencies": {
  "async": "^3.2.5",
- "pdf-lib": "^1.17.1"
+ "pdf-lib": "^1.17.1",
+ "uuid": "^10.0.0"
  }
 }

package.json

@@ -1,4 +1,5 @@
 import async from "async";
+import { v4 as uuidv4 } from 'uuid';
 
 import {
  AfterErrorContext,
@@ -96,7 +97,11 @@ export class SplitPdfHook
  hookCtx: BeforeRequestContext,
  request: Request
  ): Promise<Request> {
- const { operationID } = hookCtx;
+
+ // setting the current operationID to be unique
+ const operationID = "partition-" + uuidv4();
+ hookCtx.operationID = operationID;
+
  const requestClone = request.clone();
  const formData = await requestClone.formData();
  const splitPdfPage = stringToBoolean(

src/hooks/custom/SplitPdfHook.ts

@@ -392,4 +392,78 @@ describe("SplitPDF succeeds for large PDF with high concurrency", () => {
  expect(res.elements?.length).toBeGreaterThan(0);
  },
  300000);
-});
+});
+
+
+describe("SplitPDF async can be used to send multiple files concurrently", () => {
+ const FAKE_API_KEY = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+
+ it.each([
+ `${localServer}/general/v0/general`,
+ ])("succeed", async (serverURL) => {
+ const client = new UnstructuredClient({
+ serverURL: serverURL,
+ security: {
+ apiKeyAuth: FAKE_API_KEY,
+ },
+ });
+
+ const file = {
+ content: readFileSync("test/data/layout-parser-paper.pdf"),
+ fileName: "test/data/layout-parser-paper.pdf"
+ };
+
+ const RequestsParams = [
+ {
+ files: file,
+ splitPdfPage: true,
+ strategy: Strategy.Fast,
+ splitPdfPageRange: [1, 3],
+ languages: ["eng"],
+ },
+ {
+ files: file,
+ splitPdfPage: true,
+ strategy: Strategy.Fast,
+ splitPdfPageRange: [10, 12],
+ languages: ["eng"],
+ }
+ ];
+
+ // Process requests serially
+ const serialElements: any[][] = [];
+ for (const requestParams of RequestsParams) {
+ const res: PartitionResponse = await client.general.partition({
+ partitionParameters: {
+ ...requestParams
+ },
+ });
+ expect(res.statusCode).toEqual(200);
+ expect(res.elements?.length).toBeGreaterThan(0);
+ if (res.elements) {
+ serialElements.push(res.elements);
+ }
+ }
+
+ // Process requests concurrently
+ const concurrentElements: any[][] = [];
+ const concurrentResponses = await Promise.all(RequestsParams.map(req =>
+ client.general.partition({
+ partitionParameters: req
+ })
+ ));
+
+ for (const res of concurrentResponses) {
+ expect(res.statusCode).toEqual(200);
+ expect(res.elements?.length).toBeGreaterThan(0);
+ if (res.elements) {
+ concurrentElements.push(res.elements);
+ }
+ }
+
+ const isEqual = JSON.stringify(serialElements) === JSON.stringify(concurrentElements);
+ expect(isEqual).toBe(true);
+
+ },
+ 300000);
+});