Merge pull request dynamodb-toolbox#1004 from dynamodb-toolbox/enable-passing-any-filter-to-query-and-scan

enable passing any filter to queries and scans (without entities)

Author: ThomasAribart

docs/docs/2-tables/2-actions/1-scan/index.md

@@ -149,7 +149,7 @@ Available options (see the [DynamoDB documentation](https://docs.aws.amazon.com/
   </td>
   </tr>
   <tr>
-  <td rowSpan="4" align="center" class="vertical"><b>Filters</b></td>
+  <td rowSpan="5" align="center" class="vertical"><b>Filters</b></td>
   <td><code>select</code></td>
   <td align="center"><code>SelectOption</code></td>
   <td align="center">-</td>
@@ -168,6 +168,16 @@ Available options (see the [DynamoDB documentation](https://docs.aws.amazon.com/
   <br/><br/>See the <a href="../../entities/actions/parse-condition#building-conditions"><code>ConditionParser</code></a> action for more details on how to write conditions.
   </td>
   </tr>
+  <tr>
+  <td><code>filter</code></td>
+  <td align="center"><code>Condition</code></td>
+  <td align="center">-</td>
+  <td>
+  An untyped condition that must be satisfied in order for evaluated items to be returned (improves performances but does not reduce costs).
+  <br/><br/>No effect if <a href="#entities"><code>entities</code></a> are provided (use <code>filters</code> instead).
+  <br/><br/>See the <a href="../../entities/actions/parse-condition#building-conditions"><code>ConditionParser</code></a> action for more details on how to write conditions.
+  </td>
+  </tr>
   <tr>
   <td><code>attributes</code></td>
   <td align="center"><code>string[]</code></td>
@@ -319,7 +329,7 @@ const { Items } = await PokeTable.build(ScanCommand)
 :::note[Filtered]
 
 <Tabs>
-<TabItem value="filtered" label="Filtered">
+<TabItem value="filters" label="Filters">
 
 ```ts
 const { Items } = await PokeTable.build(ScanCommand)
@@ -333,6 +343,17 @@ const { Items } = await PokeTable.build(ScanCommand)
  .send()
 ```
 
+</TabItem>
+<TabItem value="filter" label="Filter">
+
+```ts
+const { Items } = await PokeTable.build(ScanCommand)
+ .options({
+ filter: { attr: 'pokeType', eq: 'fire' }
+ })
+ .send()
+```
+
 </TabItem>
 <TabItem value="attributes" label="Attributes">
 

docs/docs/2-tables/2-actions/2-query/index.md

@@ -206,7 +206,7 @@ Available options (see the [DynamoDB documentation](https://docs.aws.amazon.com/
   </td>
   </tr>
   <tr>
-  <td rowSpan="4" align="center" class="vertical"><b>Filters</b></td>
+  <td rowSpan="5" align="center" class="vertical"><b>Filters</b></td>
   <td><code>select</code></td>
   <td align="center"><code>SelectOption</code></td>
   <td align="center">-</td>
@@ -225,6 +225,16 @@ Available options (see the [DynamoDB documentation](https://docs.aws.amazon.com/
   <br/><br/>See the <a href="../../entities/actions/parse-condition#building-conditions"><code>ConditionParser</code></a> action for more details on how to write conditions.
   </td>
   </tr>
+  <tr>
+  <td><code>filter</code></td>
+  <td align="center"><code>Condition</code></td>
+  <td align="center">-</td>
+  <td>
+  An untyped condition that must be satisfied in order for evaluated items to be returned (improves performances but does not reduce costs).
+  <br/><br/>No effect if <a href="#entities"><code>entities</code></a> are provided (use <code>filters</code> instead).
+  <br/><br/>See the <a href="../../entities/actions/parse-condition#building-conditions"><code>ConditionParser</code></a> action for more details on how to write conditions.
+  </td>
+  </tr>
   <tr>
   <td><code>attributes</code></td>
   <td align="center"><code>string[]</code></td>
@@ -344,7 +354,7 @@ const { Items } = await PokeTable.build(QueryCommand)
 :::note[Filtered]
 
 <Tabs>
-<TabItem value="filtered" label="Filtered">
+<TabItem value="filters" label="Filters">
 
 ```ts
 const { Items } = await PokeTable.build(QueryCommand)
@@ -359,6 +369,18 @@ const { Items } = await PokeTable.build(QueryCommand)
  .send()
 ```
 
+</TabItem>
+<TabItem value="filter" label="Filter">
+
+```ts
+const { Items } = await PokeTable.build(QueryCommand)
+ .query({ partition: 'ashKetchum' })
+ .options({
+ filters: { attr: 'pokeType', eq: 'fire' }
+ })
+ .send()
+```
+
 </TabItem>
 <TabItem value="attributes" label="Attributes">
 

src/schema/actions/utils/appendAttributePath.ts

@@ -69,6 +69,7 @@ export const appendAttributePath = (
  let attributeMatches = [...attributePath.matchAll(pathRegex)]
  let attributePathTail: string | undefined
 
+ let root = true
  while (attributeMatches.length > 0) {
  const attributeMatch = attributeMatches.shift() as RegExpMatchArray
 
@@ -93,7 +94,7 @@ export const appendAttributePath = (
  }
  default: {
  const expressionAttributeNameIndex = parser.expressionAttributeNames.push(matchedKey)
- expressionPath += `.#${expressionAttributePrefix}${expressionAttributeNameIndex}`
+ expressionPath += `${root ? '' : '.'}#${expressionAttributePrefix}${expressionAttributeNameIndex}`
  }
  }
 
@@ -118,7 +119,7 @@ export const appendAttributePath = (
  const parsedKey = new Parser(keyAttribute).parse(matchedKey, { fill: false }) as string
 
  const expressionAttributeNameIndex = parser.expressionAttributeNames.push(parsedKey)
- expressionPath += `.#${expressionAttributePrefix}${expressionAttributeNameIndex}`
+ expressionPath += `${root ? '' : '.'}#${expressionAttributePrefix}${expressionAttributeNameIndex}`
 
  parentAttribute = parentAttribute.elements
  break
@@ -134,10 +135,7 @@ export const appendAttributePath = (
  childAttribute.savedAs ?? matchedKey
  )
 
- expressionPath +=
- parentAttribute.type === 'schema'
- ? `#${expressionAttributePrefix}${expressionAttributeNameIndex}`
- : `.#${expressionAttributePrefix}${expressionAttributeNameIndex}`
+ expressionPath += `${root ? '' : '.'}#${expressionAttributePrefix}${expressionAttributeNameIndex}`
  parentAttribute = childAttribute
  break
  }
@@ -171,13 +169,15 @@ export const appendAttributePath = (
  }
 
  parser.expressionAttributeNames = validElementExpressionParser.expressionAttributeNames
- expressionPath += validElementExpressionParser.expression
+ expressionPath += `${root ? '' : '.'}${validElementExpressionParser.expression}`
  // No need to go over the rest of the path
  attributeMatches = []
 
  break
  }
  }
+
+ root = false
  }
 
  if (

src/table/actions/query/options.ts

@@ -22,6 +22,7 @@ export type QueryOptions<
  maxPages?: number
  reverse?: boolean
  entityAttrFilter?: boolean
+ filter?: Entity[] extends ENTITIES ? Condition : never
  filters?: Entity[] extends ENTITIES
  ? Record<string, Condition>
  : { [ENTITY in ENTITIES[number] as ENTITY['name']]?: Condition<ENTITY> }

src/table/actions/query/queryParams/queryParams.ts

@@ -1,5 +1,6 @@
 import type { QueryCommandInput } from '@aws-sdk/lib-dynamodb'
 
+import { AnyAttribute } from '~/attributes/any/index.js'
 import { EntityConditionParser } from '~/entity/actions/parseCondition/index.js'
 import type { Condition } from '~/entity/actions/parseCondition/index.js'
 import { EntityPathParser } from '~/entity/actions/parsePaths/index.js'
@@ -14,6 +15,7 @@ import { parseMaxPagesOption } from '~/options/maxPages.js'
 import { rejectExtraOptions } from '~/options/rejectExtraOptions.js'
 import { parseSelectOption } from '~/options/select.js'
 import { parseTableNameOption } from '~/options/tableName.js'
+import { ConditionParser } from '~/schema/actions/parseCondition/index.js'
 import type { Table } from '~/table/index.js'
 import { isEmpty } from '~/utils/isEmpty.js'
 import { isBoolean } from '~/utils/validation/isBoolean.js'
@@ -34,6 +36,17 @@ type QueryParamsGetter = <
  options?: OPTIONS
 ) => QueryCommandInput
 
+const defaultAnyAttribute = new AnyAttribute({
+ required: 'never',
+ hidden: false,
+ key: false,
+ savedAs: undefined,
+ defaults: { key: undefined, put: undefined, update: undefined },
+ links: { key: undefined, put: undefined, update: undefined },
+ validators: { key: undefined, put: undefined, update: undefined },
+ castAs: undefined
+})
+
 export const queryParams: QueryParamsGetter = <
  TABLE extends Table,
  ENTITIES extends Entity[],
@@ -55,6 +68,7 @@ export const queryParams: QueryParamsGetter = <
  reverse,
  select,
  entityAttrFilter = true,
+ filter,
  filters: _filters,
  attributes: _attributes,
  tableName,
@@ -134,14 +148,50 @@ export const queryParams: QueryParamsGetter = <
  Object.assign(expressionAttributeNames, keyConditionExpressionAttributeNames)
  Object.assign(expressionAttributeValues, keyConditionExpressionAttributeValues)
 
+ if (entities.length === 0 && filter !== undefined) {
+ const {
+ ExpressionAttributeNames: filterExpressionAttributeNames,
+ ExpressionAttributeValues: filterExpressionAttributeValues,
+ ConditionExpression: filterExpression
+ } = new ConditionParser(defaultAnyAttribute).parse(filter).toCommandOptions()
+
+ Object.assign(expressionAttributeNames, filterExpressionAttributeNames)
+ Object.assign(expressionAttributeValues, filterExpressionAttributeValues)
+ commandOptions.FilterExpression = filterExpression
+ }
+
  if (entities.length > 0) {
  const filterExpressions: string[] = []
  let projectionExpression: string | undefined = undefined
 
- entities.forEach((entity, index) => {
+ let index = 0
+ for (const entity of entities) {
+ /**
+ * @debt feature "For now, we compute the projectionExpression using the first entity. Will probably use Table schemas once they exist."
+ */
+ if (projectionExpression === undefined && attributes !== undefined) {
+ const { entityAttributeName } = entity
+
+ const {
+ ExpressionAttributeNames: projectionExpressionAttributeNames,
+ ProjectionExpression
+ } = entity
+ .build(EntityPathParser)
+ .parse(
+ // entityAttributeName is required at all times for formatting
+ attributes.includes(entityAttributeName)
+ ? attributes
+ : [entityAttributeName, ...attributes]
+ )
+ .toCommandOptions()
+
+ Object.assign(expressionAttributeNames, projectionExpressionAttributeNames)
+ projectionExpression = ProjectionExpression
+ }
+
  const entityFilter = filters[entity.name]
  if (entityFilter === undefined && !entityAttrFilter) {
- return
+ continue
  }
 
  const entityNameFilter = { attr: entity.entityAttributeName, eq: entity.name }
@@ -163,29 +213,8 @@ export const queryParams: QueryParamsGetter = <
  Object.assign(expressionAttributeValues, filterExpressionAttributeValues)
  filterExpressions.push(filterExpression)
 
- /**
- * @debt feature "For now, we compute the projectionExpression using the first entity. Will probably use Table schemas once they exist."
- */
- if (attributes !== undefined) {
- const { entityAttributeName } = entity
-
- const {
- ExpressionAttributeNames: projectionExpressionAttributeNames,
- ProjectionExpression
- } = entity
- .build(EntityPathParser)
- .parse(
- // entityAttributeName is required at all times for formatting
- attributes.includes(entityAttributeName)
- ? attributes
- : [entityAttributeName, ...attributes]
- )
- .toCommandOptions()
-
- Object.assign(expressionAttributeNames, projectionExpressionAttributeNames)
- projectionExpression = ProjectionExpression
- }
- })
+ index++
+ }
 
  if (filterExpressions.length > 0) {
  commandOptions.FilterExpression =

src/table/actions/query/queryParams/queryParams.unit.test.ts

@@ -738,6 +738,32 @@ describe('query', () => {
  expect(invalidCall).toThrow(expect.objectContaining({ code: 'options.invalidTableNameOption' }))
  })
 
+ test('applies any filter if no entity has been provided', () => {
+ const command = TestTable.build(QueryCommand)
+ .query({ partition: 'foo' })
+ .options({ filter: { attr: 'foo', eq: 'bar' } })
+ const { FilterExpression, ExpressionAttributeNames, ExpressionAttributeValues } =
+ command.params()
+
+ expect(FilterExpression).toBe('#c_1 = :c_1')
+ expect(ExpressionAttributeNames).toMatchObject({ '#c_1': 'foo' })
+ expect(ExpressionAttributeValues).toMatchObject({ ':c_1': 'bar' })
+ })
+
+ test('ignores filter if entities have been provided', () => {
+ const command = TestTable.build(QueryCommand)
+ .query({ partition: 'foo' })
+ .entities(Entity1)
+ .options({
+ // @ts-expect-error
+ filter: { attr: 'foo', eq: 'bar' }
+ })
+ const { ExpressionAttributeNames = {}, ExpressionAttributeValues = {} } = command.params()
+
+ expect(Object.values(ExpressionAttributeNames)).not.toContain('foo')
+ expect(Object.values(ExpressionAttributeValues)).not.toContain('bar')
+ })
+
  test('applies entity _et filter', () => {
  const command = TestTable.build(QueryCommand).query({ partition: 'foo' }).entities(Entity1)
  const { FilterExpression, ExpressionAttributeNames, ExpressionAttributeValues } =

src/table/actions/scan/options.ts

@@ -16,6 +16,7 @@ export type ScanOptions<TABLE extends Table = Table, ENTITIES extends Entity[] =
  limit?: number
  maxPages?: number
  entityAttrFilter?: boolean
+ filter?: Entity[] extends ENTITIES ? Condition : never
  filters?: Entity[] extends ENTITIES
  ? Record<string, Condition>
  : { [ENTITY in ENTITIES[number] as ENTITY['name']]?: Condition<ENTITY> }