Make knn search a query

docs/changelog/98916.yaml

@@ -0,0 +1,5 @@
+pr: 98916
+summary: Make knn search a query
+area: Vector Search
+type: feature
+issues: []

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/100_knn_query_multiple_shards.yml

@@ -0,0 +1,192 @@
+# test how knn query interacts with filters
+setup:
+ - skip:
+ version: ' - 8.10.99'
+ reason: 'knn as query added in 8.11'
+ features: close_to
+
+ - do:
+ indices.create:
+ index: my_index
+ body:
+ settings:
+ number_of_shards: 2
+ mappings:
+ dynamic: false
+ properties:
+ my_vector:
+ type: dense_vector
+ dims: 4
+ index : true
+ similarity : l2_norm
+ my_name:
+ type: keyword
+ store: true
+
+ - do:
+ bulk:
+ refresh: true
+ index: my_index
+ body:
+ - '{"index": {"_id": "1"}}'
+ - '{"my_vector": [1, 1, 1, 1], "my_name": "v1"}'
+ - '{"index": {"_id": "2"}}'
+ - '{"my_vector": [1, 1, 1, 2], "my_name": "v2"}'
+ - '{"index": {"_id": "3"}}'
+ - '{"my_vector": [1, 1, 1, 3], "my_name": "v1"}'
+ - '{"index": {"_id": "4"}}'
+ - '{"my_vector": [1, 1, 1, 4], "my_name": "v2"}'
+ - '{"index": {"_id": "5"}}'
+ - '{"my_vector": [1, 1, 1, 5], "my_name": "v1"}'
+ - '{"index": {"_id": "6"}}'
+ - '{"my_vector": [1, 1, 1, 6], "my_name": "v2"}'
+ - '{"index": {"_id": "7"}}'
+ - '{"my_vector": [1, 1, 1, 7], "my_name": "v1"}'
+ - '{"index": {"_id": "8"}}'
+ - '{"my_vector": [1, 1, 1, 8], "my_name": "v2"}'
+ - '{"index": {"_id": "9"}}'
+ - '{"my_vector": [1, 1, 1, 9], "my_name": "v1"}'
+ - '{"index": {"_id": "10"}}'
+ - '{"my_vector": [1, 1, 1, 10], "my_name": "v2"}'
+ - '{"index": {"_id": "11"}}'
+ - '{"my_vector": [1, 1, 1, 11], "my_name": "v1"}'
+ - '{"index": {"_id": "12"}}'
+ - '{"my_vector": [1, 1, 1, 12], "my_name": "v2"}'
+
+
+---
+"Search for 2 knn queries combines scores from them":
+ - do:
+ search:
+ index: my_index
+ body:
+ size: 6
+ query:
+ bool:
+ should:
+ - knn:
+ field: my_vector
+ query_vector: [ 1, 1, 1, 1 ]
+ num_candidates: 100
+ boost: 1.1
+ - knn:
+ field: my_vector
+ query_vector: [ 1, 1, 1, 12 ]
+ num_candidates: 100
+
+ - length: {hits.hits: 6}
+ - match: {hits.total.value: 12}
+ - match: {hits.hits.0._id: '1'}
+ - match: {hits.hits.1._id: '12'}
+ - match: {hits.hits.2._id: '2'}
+ - match: { hits.hits.3._id: '11' }
+ - match: { hits.hits.4._id: '3' }
+ - match: { hits.hits.5._id: '10' }
+
+
+---
+"Hybrid search combines scores from knn and other queries":
+ - do:
+ search:
+ index: my_index
+ body:
+ size: 3
+ query:
+ bool:
+ should:
+ - wildcard:
+ my_name:
+ value: "v*" # produces scores 1.0
+ - knn:
+ field: my_vector
+ query_vector: [ 1, 1, 1, 1 ]
+ num_candidates: 3
+
+ - length: {hits.hits: 3}
+ - match: {hits.total.value: 12}
+ - match: {hits.hits.0._id: '1'}
+ - match: {hits.hits.1._id: '2'}
+ - match: {hits.hits.2._id: '3'}
+ - close_to: {hits.hits.0._score: { value: 2.0, error: 0.00001 } }
+ - close_to: {hits.hits.1._score: { value: 1.5, error: 0.00001 } }
+ - close_to: {hits.hits.2._score: { value: 1.2, error: 0.00001 } }
+
+ # the same query with boosts
+ - do:
+ search:
+ index: my_index
+ body:
+ size: 3
+ query:
+ bool:
+ should:
+ - wildcard:
+ my_name:
+ value: "v*" # produces scores 1.0
+ boost: 100
+ - knn:
+ field: my_vector
+ query_vector: [ 1, 1, 1, 1 ]
+ num_candidates: 3
+ boost: 100
+
+ - length: { hits.hits: 3 }
+ - match: { hits.total.value: 12 }
+ - match: { hits.hits.0._id: '1' }
+ - match: { hits.hits.1._id: '2' }
+ - match: { hits.hits.2._id: '3' }
+ - close_to: { hits.hits.0._score: { value: 200.0, error: 0.00001 } }
+ - close_to: { hits.hits.1._score: { value: 150.0, error: 0.00001 } }
+ - close_to: { hits.hits.2._score: { value: 120, error: 0.00001 } }
+
+---
+"Aggregations with collected number of docs depends on num_candidates":
+ - do:
+ search:
+ index: my_index
+ body:
+ size: 2
+ query:
+ knn:
+ field: my_vector
+ query_vector: [1, 1, 1, 1]
+ num_candidates: 100 # collect up to 100 candidates from each shard
+ aggs:
+ my_agg:
+ terms:
+ field: my_name
+ order:
+ _key: asc
+
+ - length: {hits.hits: 2}
+ - match: {hits.total.value: 12}
+ - match: {aggregations.my_agg.buckets.0.key: 'v1'}
+ - match: {aggregations.my_agg.buckets.1.key: 'v2'}
+ - match: {aggregations.my_agg.buckets.0.doc_count: 6}
+ - match: {aggregations.my_agg.buckets.1.doc_count: 6}
+
+ - do:
+ search:
+ index: my_index
+ body:
+ size: 2
+ query:
+ knn:
+ field: my_vector
+ query_vector: [ 1, 1, 1, 1 ]
+ num_candidates: 3 # collect 3 candidates from each shard
+ aggs:
+ my_agg2:
+ terms:
+ field: my_name
+ order:
+ _key: asc
+ my_sum_buckets:
+ sum_bucket:
+ buckets_path: "my_agg2>_count"
+
+ - length: { hits.hits: 2 }
+ - match: { hits.total.value: 6 }
+ - match: { aggregations.my_agg2.buckets.0.key: 'v1' }
+ - match: { aggregations.my_agg2.buckets.1.key: 'v2' }
+ - match: { aggregations.my_sum_buckets.value: 6.0 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml

@@ -294,23 +294,6 @@ setup:
  - match: { error.root_cause.0.reason: "failed to create query: field [nonexistent] does not exist in the mapping" }
 
 ---
-"Direct kNN queries are disallowed":
- - skip:
- version: ' - 8.3.99'
- reason: 'error message changed in 8.4'
- - do:
- catch: bad_request
- search:
- index: test-index
- body:
- query:
- knn:
- field: vector
- query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
- num_candidates: 1
- - match: { error.root_cause.0.type: "illegal_argument_exception" }
- - match: { error.root_cause.0.reason: "[knn] queries cannot be provided directly, use the [knn] body parameter instead" }
----
 "KNN Vector similarity search only":
  - skip:
  version: ' - 8.7.99'

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml

@@ -163,20 +163,6 @@ setup:
  - match: { error.root_cause.0.reason: "failed to create query: field [nonexistent] does not exist in the mapping" }
 
 ---
-"Direct kNN queries are disallowed":
- - do:
- catch: bad_request
- search:
- index: test
- body:
- query:
- knn:
- field: vector
- query_vector: [ -1, 0, 1, 2, 3 ]
- num_candidates: 1
- - match: { error.root_cause.0.type: "illegal_argument_exception" }
- - match: { error.root_cause.0.reason: "[knn] queries cannot be provided directly, use the [knn] body parameter instead" }
----
 "Vector similarity search only":
  - skip:
  version: ' - 8.7.99'