Docs work + update internal parsing & tooling (#99)

* test.sh.md * check to ensure that test suite names follow convention * some stargate script updates + docs * check.sh.md * repl.sh.md * logs only to stderr by default instead of split between stderr and stdout * more dev docs updates * set-example-client-deps.sh * scripts/playgound.sh * client?: "default" => client: "fetch-h2" * migrate options parsing to 3rd party ts lib * monoids * tests passing again after more refactors (i love monoids) * update dev deps * majorly improve composability of options handlers * split ctx.continue() into ctx.replace() and ctx.nevermind(); made deser codecs run backwards * BaseSerDesCtx.target

Author: toptobes

.gitignore

@@ -141,5 +141,6 @@ vectorize_test_spec.json
 etc/test-reports/
 etc/playgrounds/
 tmp-lib-check
+!examples/astra-db-ts.tgz
 
 .direnv

DEVGUIDE.md

@@ -36,40 +36,41 @@ const db = client.db(process.env.CLIENT_DB_URL!, { token: process.env.CLIENT_DB_
 
 // The `VectorDoc` interface adds `$vector?: DataAPIVector` as a field to the collection type
 interface Dream extends VectorDoc {
- _id: ObjectId, // Uses an `astra-db-ts` provided type here (NOT the `bson` version)
+ _id: ObjectId,
  summary: string,
- tags?: string[], // No sets/maps available without creating custom ser/des rules
+ tags?: string[],
 }
 
 (async () => {
- // Create the table using our helper function.
- // The _id should be an `ObjectId` type, as specified by `defaultId.type`
+ // Create the collection with a custom default ID type
  const collection = await db.createCollection<Dream>('dreams', {
  defaultId: { type: 'objectId' },
  });
 
- // Batch-insert some rows into the table
+ // Batch-insert some rows into the table. 
  // _id can be optionally provided, or be auto-generated @ the server side
  await collection.insertMany([{
  summary: 'A dinner on the Moon',
- $vector: vector([0.2, -0.3, -0.5]), // Shorthand for `new DataAPIVector([0.2, -0.3, -0.5])`
+ $vector: vector([0.2, -0.3, -0.5]),
  }, {
  summary: 'Riding the waves',
  $vector: vector([0, 0.2, 1]),
  tags: ['sport'],
  }, {
- _id: oid('674f0f5c1c162131319fa09e'), // Shorthand for `new ObjectId('674f0f5c1c162131319fa09e')`
+ _id: oid('674f0f5c1c162131319fa09e'),
  summary: 'Meeting Beethoven at the dentist',
  $vector: vector([0.2, 0.6, 0]),
  }]);
 
  // Hm, changed my mind
- await collection.updateOne({ id: 103 }, { $set: { summary: 'Surfers\' paradise' } });
+ await collection.updateOne({ _id: oid('674f0f5c1c162131319fa09e') }, {
+ $set: { summary: 'Surfers\' paradise' } 
+ });
 
- // Let's see what we've got
+ // Let's see what we've got, by performing a vector search
  const cursor = collection.find({})
- .sort({ vector: vector([0, 0.2, 0.4]) }) // Performing a vector search
- .includeSimilarity(true) // The found doc is inferred to have `$similarity` as a property now
+ .sort({ vector: vector([0, 0.2, 0.4]) })
+ .includeSimilarity(true)
  .limit(2);
 
  // This would print:
@@ -87,44 +88,42 @@ interface Dream extends VectorDoc {
 ### Tables
 
 ```typescript
-import { DataAPIClient, InferTableSchema, vector } from '@datastax/astra-db-ts';
+import { DataAPIClient, InferTableSchema, Table, vector } from '@datastax/astra-db-ts';
 
 // Connect to the db
 const client = new DataAPIClient({ logging: 'all' });
 const db = client.db(process.env.CLIENT_DB_URL!, { token: process.env.CLIENT_DB_TOKEN! });
 
-// Create a table through the Data API if it does not yet exist.
-// Returns the created table through a function so we can use the inferred type of the table ourselves
-// (instead of having to manually define it)
-const mkDreamsTable = async () => await db.createTable('dreams', {
- definition: {
- columns: {
- id: 'int', // Shorthand notation for { type: 'int' }
- summary: 'text',
- tags: { type: 'set', valueType: 'text' }, // Collection types require additional type information
- vector: { type: 'vector', dimension: 3 }, // Auto-embedding-generation can be enabled through a `service` block
- },
- primaryKey: 'id', // Shorthand for { partitionBy: ['id'] }
+// Define the table's schema so we can infer the type of the table automatically (TS v5.0+)
+const DreamsTableSchema = Table.schema({
+ columns: {
+ id: 'int',
+ summary: 'text',
+ tags: { type: 'set', valueType: 'text' },
+ vector: { type: 'vector', dimension: 3 },
  },
- ifNotExists: true, // If any table with the same name exists, do nothing
-}); // (note that this does not check if the tables are the same)
+ primaryKey: 'id',
+});
 
 // Infer the TS-equivalent type from the table definition (like zod or arktype). Equivalent to:
 //
 // interface TableSchema {
-// id: number, -- A primary key component, so it's required
-// summary?: string | null, -- Not a primary key, so it's optional and may return as null when found
-// tags?: Set<string>, -- Sets/maps/lists are optional to insert, but will actually be returned as empty collections instead of null
-// vector?: DataAPIVector | null, -- Vectors, however, may be null.
+// id: number,
+// summary?: string | null,
+// tags?: Set<string>,
+// vector?: DataAPIVector | null,
 // }
-type Dream = InferTableSchema<typeof mkDreamsTable>;
+type Dream = InferTableSchema<typeof DreamsTableSchema>;
 
 (async () => {
- // Create the table using our helper function.
+ // Create the table if it doesn't already exist
  // Table will be typed as `Table<Dream, { id: number }>`, where the former is the schema, and the latter is the primary key
- const table = await mkDreamsTable();
+ const table = await db.createTable('dreams', {
+ definition: DreamsTableSchema,
+ ifNotExists: true,
+ });
 
- // Enables vector search on the table (on the 'vector' column)
+ // Create a vector index on the vector column so we can perform ANN searches on the table
  await table.createVectorIndex('dreams_vector_idx', 'vector', {
  options: { metric: 'cosine' },
  ifNotExists: true,
@@ -134,12 +133,12 @@ type Dream = InferTableSchema<typeof mkDreamsTable>;
  const rows: Dream[] = [{
  id: 102,
  summary: 'A dinner on the Moon',
- vector: vector([0.2, -0.3, -0.5]), // Shorthand for `new DataAPIVector([0.2, -0.3, -0.5])`
+ vector: vector([0.2, -0.3, -0.5]),
  }, {
  id: 103,
  summary: 'Riding the waves',
  vector: vector([0, 0.2, 1]),
- tags: new Set(['sport']), // Collection types use native JS collections
+ tags: new Set(['sport']),
  }, {
  id: 37,
  summary: 'Meeting Beethoven at the dentist',
@@ -148,12 +147,14 @@ type Dream = InferTableSchema<typeof mkDreamsTable>;
  await table.insertMany(rows);
 
  // Hm, changed my mind
- await table.updateOne({ id: 103 }, { $set: { summary: 'Surfers\' paradise' } });
+ await table.updateOne({ id: 103 }, {
+ $set: { summary: 'Surfers\' paradise' } 
+ });
 
- // Let's see what we've got
+ // Let's see what we've got, by performing a vector search
  const cursor = table.find({})
- .sort({ vector: vector([0, 0.2, 0.4]) }) // Performing a vector search
- .includeSimilarity(true) // The found doc is inferred to have `$similarity` as a property now
+ .sort({ vector: vector([0, 0.2, 0.4]) })
+ .includeSimilarity(true)
  .limit(2);
 
  // This would print:
@@ -168,6 +169,56 @@ type Dream = InferTableSchema<typeof mkDreamsTable>;
 })();
 ```
 
+<details>
+ <summary><i>Inferring the table schema pre-TS v5.0</i></summary>
+
+ Before TypeScript 5.0, there was no support for "const type parameters" (e.g. `f<const T>(t: T): T`) which `Table.schema` relies on.
+
+ No worries though—if you're using TypeScript 4.x or below, you can still infer the schema automatically, albeit with less language server support.
+
+ Schema object type errors may be non-local and harder to debug, but the code will still work as expected.
+
+ ```ts
+ const DreamsTableSchema = <const>{
+ columns: {
+ id: 'int',
+ summary: 'text',
+ tags: { type: 'set', valueType: 'text' },
+ vector: { type: 'vector', dimension: 3 },
+ },
+ primaryKey: 'id',
+ };
+
+ // Still works, but you need to ensure DreamsTableSchema is a properly typed const object
+ type Dream = InferTableSchema<typeof DreamsTableSchema>;
+ type DreamPK = InferTablePrimaryKey<typeof DreamsTableSchema>;
+
+ (async () => {
+ // Necessary to explicitly set the type of the table schema and primary key here
+ const table = await db.createTable<Dream, DreamPK>('dreams', {
+ definition: DreamsTableSchema,
+ ifNotExists: true,
+ });
+ })();
+ ```
+
+ If you're using TypeScript 4.9, you can at least use the `satisfies` operator to localize any definition type errors.
+
+ ```ts
+ const DreamsTableSchema = <const>{
+ columns: {
+ id: 'int',
+ summary: 'text',
+ tags: { type: 'set', valueType: 'text' },
+ vector: { type: 'vector', dimension: 3 },
+ },
+ primaryKey: 'id',
+ } satisfies CreateTableDefinition;
+
+ type Dream = InferTableSchema<typeof DreamsTableSchema>;
+ ```
+</details>
+
 ### Next steps
 
 - More info and usage patterns are given in the ts-doc of classes and methods
@@ -250,11 +301,9 @@ const tp = new UsernamePasswordTokenProvider('*USERNAME*', '*PASSWORD*');
 const client = new DataAPIClient(tp, { environment: 'dse' });
 const db = client.db('*ENDPOINT*');
 
-// You'll also need to pass it to db.admin() when not using Astra for typing purposes
-// If the environment does not match, an error will be thrown as a reminder
-// `environment: 'dse'` makes the return type be `DataAPIDbAdmin`
+// A common idiom may be to use `dbAdmin.createKeyspace` with `updateDbKeyspace` to initialize the keyspace when necessary
 const dbAdmin = db.admin({ environment: 'dse' });
-dbAdmin.createNamespace('...');
+dbAdmin.createKeyspace('...', { updateDbKeyspace: true });
 ```
 
 The `TokenProvider` class is an extensible concept to allow you to create or even refresh your tokens
@@ -268,9 +317,9 @@ as necessary, depending on the Data API backend. Tokens may even be omitted if n
 
 ## Non-standard environment support
 
-`astra-db-ts` is designed foremost to work in Node.js environments. 
+`astra-db-ts` is designed first and foremost to work in Node.js environments. 
 
-It will work in edge runtimes and other non-node environments as well, though it'll use the native `fetch` API for HTTP
+However, it will work in edge runtimes and other non-node environments as well, though it may use the native `fetch` API for HTTP
 requests, as opposed to `fetch-h2` which provides extended HTTP/2 and HTTP/1.1 support for performance.
 
 By default, it'll attempt to use `fetch-h2` if available, and fall back to `fetch` if not available in that environment.
@@ -331,12 +380,13 @@ to the native fetch implementation instead if importing fails.
 
 ### Browser support
 
-The Data API itself does not natively support browsers, so `astra-db-ts` isn't technically supported in browsers either.
+`astra-db-ts` is designed to work in server-side environments, but it can technically work in the browser as well.
 
-However, if, for some reason, you really want to use this in a browser, you can probably do so by installing the 
-`events` polyfill and setting up a [CORS proxy](https://github.com/Rob--W/cors-anywhere) to forward requests to the Data API.
+However, if, for some reason, you really want to use this in a browser, you may need to install the `events` polyfill,
+and possibly set up a CORS proxy (such as [CORS Anywhere](https://github.com/Rob--W/cors-anywhere)) to forward requests 
+to the Data API.
 
-But keep in mind that this is not officially supported, and may be very insecure if you're encoding sensitive
-data into the browser client.
+But keep in mind that this may be very insecure, especially if you're hardcoding sensitive data into your client-side
+code, as it's trivial for anyone to inspect the code and extract the token (through XSS attacks or otherwise).
 
-(See `examples/browser` for a full example of this workaround in action.)
+(See `examples/browser` for a full example of browser usage in action.)