paginator: manual pagination with familiar syntax

packages/convex-helpers/README.md

@@ -764,6 +764,94 @@ const { page, indexKeys, hasMore } = await getPage(ctx, {
 });
 ```
 
+### `getPageOfQuery`: manual pagination with familiar syntax
+
+In addition to `getPage`, convex-helpers provides a function
+`getPageOfQuery`. This function has syntax and interface similar to the
+built-in `.paginate`, to make it easy to switch.
+
+It runs on top of `getPage`, so it provides the benefits of being callable
+multiple times from a query, or within a Convex component. On the other hand,
+it provides less control over the index ranges being queried.
+
+The interface is so similar to `.paginate` that you can use it with
+`usePaginatedQuery`. **However**, doing so will result in non-reactive pages.
+Keeping pages contiguous requires rerunning queries and passing through an
+`endCursor` option. For more info on these edge-cases, see
+https://stack.convex.dev/fully-reactive-pagination.
+
+`getPageOfQuery` is especially useful when you cannot use `.paginate` and you
+are not in a reactive query. For example, if you're running a migration,
+it's running in a mutation and it doesn't need reactivity. You can run
+multiple migrations at once or run a migration within a Convex component using
+`getPageOfQuery`.
+
+As a basic example, suppose you have this query:
+
+```ts
+export const list = query({
+ args: { opts: paginationOptsValidator },
+ handler: async (ctx, { opts }) => {
+ return await ctx.db.query("messages").paginate(opts);
+ },
+});
+```
+
+It has the same behavior as this query, except that in this one the pages might
+not stay contiguous as items are added and removed from the list and the query
+updates reactively:
+
+```ts
+import { getPageOfQuery } from "convex-helpers/server/pagination";
+export const list = query({
+ args: { opts: paginationOptsValidator },
+ handler: async (ctx, { opts }) => {
+ return await getPageOfQuery(
+ ctx,
+ (db) => db.query("messages"),
+ opts,
+ );
+ },
+});
+```
+
+You can order by an index, restrict the pagination to a range of the index,
+and change the order to "desc", same as you would with a regular query.
+
+```ts
+import { getPageOfQuery } from "convex-helpers/server/pagination";
+import schema from "./schema";
+export const list = query({
+ args: { opts: paginationOptsValidator, author: v.id("users") },
+ handler: async (ctx, { opts, author }) => {
+ return await getPageOfQuery(
+ ctx,
+ (db) => db.query("messages").withIndex("by_author", q=>q.eq("author", author)).order("desc"),
+ opts,
+ { schema },
+ );
+ },
+});
+```
+
+And for convenience there's an equivalent of the [filter helper](#filter).
+
+```ts
+import { getPageOfQuery } from "convex-helpers/server/pagination";
+import schema from "./schema";
+export const list = query({
+ args: { opts: paginationOptsValidator, author: v.id("users") },
+ handler: async (ctx, { opts, author }) => {
+ return await getPageOfQuery(
+ ctx,
+ (db) => db.query("messages"),
+ opts,
+ { filter: async (message) => !message.isArchived },
+ );
+ },
+});
+```
+
 ## Query Caching
 
 Utilize a query cache implementation which persists subscriptions to the

packages/convex-helpers/server/filter.ts

@@ -25,7 +25,7 @@ import {
  SearchIndexes,
 } from "convex/server";
 
-async function asyncFilter<T>(
+export async function asyncFilter<T>(
  arr: T[],
  predicate: (d: T) => Promise<boolean> | boolean
 ): Promise<T[]> {

packages/convex-helpers/server/pagination.test.ts

@@ -1,7 +1,7 @@
-import { defineTable, defineSchema, GenericDocument } from "convex/server";
+import { defineTable, defineSchema, GenericDocument, DataModelFromSchemaDefinition } from "convex/server";
 import { convexTest } from "convex-test";
 import { expect, test } from "vitest";
-import { IndexKey, getPage } from "./pagination.js";
+import { IndexKey, getPage, getPageOfQuery } from "./pagination.js";
 import { modules } from "./setup.test.js";
 import { GenericId, v } from "convex/values";
 
@@ -13,6 +13,8 @@ const schema = defineSchema({
  }).index("abc", ["a", "b", "c"]),
 });
 
+type DataModel = DataModelFromSchemaDefinition<typeof schema>;
+
 function stripSystemFields(doc: GenericDocument) {
  const { _id, _creationTime, ...rest } = doc;
  return rest;
@@ -243,3 +245,236 @@ describe("manual pagination", () => {
  });
  });
 });
+
+describe("getPageOfQuery", () => {
+ test("full table scan", async () => {
+ const t = convexTest(schema, modules);
+ await t.run(async (ctx) => {
+ await ctx.db.insert("foo", { a: 1, b: 2, c: 3 });
+ await ctx.db.insert("foo", { a: 1, b: 2, c: 4 });
+ await ctx.db.insert("foo", { a: 1, b: 2, c: 5 });
+ const result1 = await getPageOfQuery(
+ ctx,
+ (db) => db.query("foo"),
+ { numItems: 100, cursor: null },
+ );
+ expect(result1.page.map(stripSystemFields)).toEqual([
+ { a: 1, b: 2, c: 3 },
+ { a: 1, b: 2, c: 4 },
+ { a: 1, b: 2, c: 5 },
+ ]);
+ expect(result1.isDone).toBe(true);
+ expect(result1.continueCursor).toBe("endcursor");
+ });
+ });
+
+ test("paginated table scan", async () => {
+ const t = convexTest(schema, modules);
+ await t.run(async (ctx) => {
+ await ctx.db.insert("foo", { a: 1, b: 2, c: 3 });
+ await ctx.db.insert("foo", { a: 1, b: 2, c: 4 });
+ await ctx.db.insert("foo", { a: 1, b: 2, c: 5 });
+ const result1 = await getPageOfQuery(
+ ctx,
+ (db) => db.query("foo"),
+ { numItems: 2, cursor: null },
+ );
+ expect(result1.page.map(stripSystemFields)).toEqual([
+ { a: 1, b: 2, c: 3 },
+ { a: 1, b: 2, c: 4 },
+ ]);
+ expect(result1.isDone).toBe(false);
+
+ const result2 = await getPageOfQuery(
+ ctx,
+ (db) => db.query("foo"),
+ { numItems: 2, cursor: result1.continueCursor },
+ );
+ expect(result2.page.map(stripSystemFields)).toEqual([
+ { a: 1, b: 2, c: 5 },
+ ]);
+ expect(result2.isDone).toBe(true);
+ });
+ });
+
+ test("paginated table scan with filter", async () => {
+ const t = convexTest(schema, modules);
+ await t.run(async (ctx) => {
+ await ctx.db.insert("foo", { a: 1, b: 2, c: 3 });
+ await ctx.db.insert("foo", { a: 1, b: 2, c: 4 });
+ await ctx.db.insert("foo", { a: 1, b: 2, c: 5 });
+ const result1 = await getPageOfQuery(
+ ctx,
+ (db) => db.query("foo"),
+ { numItems: 2, cursor: null },
+ {
+ filter: async (doc) => doc.c !== 4,
+ },
+ );
+ expect(result1.page.map(stripSystemFields)).toEqual([
+ { a: 1, b: 2, c: 3 },
+ ]);
+ expect(result1.isDone).toBe(false);
+ });
+ });
+
+ test("index range", async () => {
+ const t = convexTest(schema, modules);
+ await t.run(async (ctx) => {
+ await ctx.db.insert("foo", { a: 1, b: 5, c: 1 });
+ await ctx.db.insert("foo", { a: 1, b: 6, c: 1 });
+ await ctx.db.insert("foo", { a: 1, b: 3, c: 1 });
+ await ctx.db.insert("foo", { a: 1, b: 4, c: 1 });
+ await ctx.db.insert("foo", { a: 1, b: 4, c: 2 });
+ const result1 = await getPageOfQuery<DataModel, "foo">(
+ ctx,
+ (db) => db.query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5)),
+ { cursor: null, numItems: 100 },
+ { schema },
+ );
+ expect(result1.page.map(stripSystemFields)).toEqual([
+ { a: 1, b: 4, c: 1 },
+ { a: 1, b: 4, c: 2 },
+ { a: 1, b: 5, c: 1 },
+ ]);
+ expect(result1.isDone).toBe(true);
+
+ // Descending.
+ const result2 = await getPageOfQuery<DataModel, "foo">(
+ ctx,
+ (db) => db.query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5)).order("desc"),
+ { cursor: null, numItems: 100 },
+ { schema },
+ );
+ expect(result2.page.map(stripSystemFields)).toEqual([
+ { a: 1, b: 5, c: 1 },
+ { a: 1, b: 4, c: 2 },
+ { a: 1, b: 4, c: 1 },
+ ]);
+ expect(result2.isDone).toBe(true);
+ });
+ });
+
+ test("paginated index range desc", async () => {
+ const t = convexTest(schema, modules);
+ await t.run(async (ctx) => {
+ await ctx.db.insert("foo", { a: 1, b: 5, c: 1 });
+ await ctx.db.insert("foo", { a: 1, b: 6, c: 1 });
+ await ctx.db.insert("foo", { a: 1, b: 3, c: 1 });
+ await ctx.db.insert("foo", { a: 1, b: 4, c: 1 });
+ await ctx.db.insert("foo", { a: 1, b: 4, c: 2 });
+ const result1 = await getPageOfQuery<DataModel, "foo">(
+ ctx,
+ (db) => db.query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5)).order("desc"),
+ { cursor: null, numItems: 2 },
+ {
+ schema,
+ },
+ );
+ expect(result1.page.map(stripSystemFields)).toEqual([
+ { a: 1, b: 5, c: 1 },
+ { a: 1, b: 4, c: 2 },
+ ]);
+ expect(result1.isDone).toBe(false);
+
+ const result2 = await getPageOfQuery<DataModel, "foo">(
+ ctx,
+ (db) => db.query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5)).order("desc"),
+ { cursor: result1.continueCursor, numItems: 2 },
+ {
+ schema,
+ },
+ );
+ expect(result2.page.map(stripSystemFields)).toEqual([
+ { a: 1, b: 4, c: 1 },
+ ]);
+ expect(result2.isDone).toBe(true);
+ });
+ });
+
+ test("invalid index range", async () => {
+ const t = convexTest(schema, modules);
+ await t.run(async (ctx) => {
+ await expect(getPageOfQuery<DataModel, "foo">(
+ ctx,
+ (db) => db.query("foo").withIndex("abc", q => q.gt("c" as any, 3)),
+ { cursor: null, numItems: 100 },
+ { schema },
+ )).rejects.toThrow("Cannot use gt on field 'c'");
+ await expect(getPageOfQuery<DataModel, "foo">(
+ ctx,
+ (db) => db.query("foo").withIndex("abc", q => q.eq("a", 1).eq("c" as any, 3)),
+ { cursor: null, numItems: 100 },
+ { schema },
+ )).rejects.toThrow("Cannot use eq on field 'c'");
+ await expect(getPageOfQuery<DataModel, "foo">(
+ ctx,
+ (db) => db.query("foo").withIndex("abc", q => (q.gt("a", 1) as any).gt("b", 3)),
+ { cursor: null, numItems: 100 },
+ { schema },
+ )).rejects.toThrow("Cannot use gt on field 'b'");
+ await expect(getPageOfQuery<DataModel, "foo">(
+ ctx,
+ (db) => db.query("foo").withIndex("abc", q => (q.gt("a", 1).lt("a", 3) as any).eq("b", 3)),
+ { cursor: null, numItems: 100 },
+ { schema },
+ )).rejects.toThrow("Cannot use eq on field 'b'");
+ });
+ });
+
+ test("endCursor", async () => {
+ const t = convexTest(schema, modules);
+ await t.run(async (ctx) => {
+ await ctx.db.insert("foo", { a: 1, b: 5, c: 1 });
+ await ctx.db.insert("foo", { a: 1, b: 6, c: 1 });
+ await ctx.db.insert("foo", { a: 1, b: 3, c: 1 });
+ await ctx.db.insert("foo", { a: 1, b: 4, c: 1 });
+ await ctx.db.insert("foo", { a: 1, b: 4, c: 3 });
+ const result1 = await getPageOfQuery<DataModel, "foo">(
+ ctx,
+ (db) => db.query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5)),
+ { cursor: null, numItems: 2 },
+ { schema },
+ );
+ expect(result1.page.map(stripSystemFields)).toEqual([
+ { a: 1, b: 4, c: 1 },
+ { a: 1, b: 4, c: 3 },
+ ]);
+ expect(result1.isDone).toBe(false);
+ await ctx.db.insert("foo", { a: 1, b: 4, c: 2 });
+ const result2 = await getPageOfQuery<DataModel, "foo">(
+ ctx,
+ (db) => db.query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5)),
+ { cursor: null, endCursor: result1.continueCursor, numItems: 2 },
+ { schema },
+ );
+ expect(result2.page.map(stripSystemFields)).toEqual([
+ { a: 1, b: 4, c: 1 },
+ { a: 1, b: 4, c: 2 },
+ { a: 1, b: 4, c: 3 },
+ ]);
+ expect(result2.isDone).toBe(false);
+ expect(result1.continueCursor).toStrictEqual(result2.continueCursor);
+ const result3 = await getPageOfQuery<DataModel, "foo">(
+ ctx,
+ (db) => db.query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5)),
+ { cursor: result2.continueCursor, numItems: 2 },
+ { schema },
+ );
+ expect(result3.page.map(stripSystemFields)).toEqual([
+ { a: 1, b: 5, c: 1 },
+ ]);
+ expect(result3.isDone).toBe(true);
+ const result4 = await getPageOfQuery<DataModel, "foo">(
+ ctx,
+ (db) => db.query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5)),
+ { cursor: result2.continueCursor, endCursor: result3.continueCursor, numItems: 2 },
+ { schema },
+ );
+ expect(result4.page.map(stripSystemFields)).toEqual([
+ { a: 1, b: 5, c: 1 },
+ ]);
+ expect(result4.isDone).toBe(true);
+ });
+ });
+});