[8.19] New vector_rescore parameter as a quantized index type option (#124581)

docs/changelog/124581.yaml

@@ -0,0 +1,5 @@
+pr: 124581
+summary: New `vector_rescore` parameter as a quantized index type option
+area: Vector Search
+type: enhancement
+issues: []

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml

@@ -250,3 +250,93 @@ setup:
  index: dynamic_dim_bbq_hnsw
  body:
  vector: [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0]
+---
+"Test index configured rescore vector":
+ - requires:
+ cluster_features: ["mapper.dense_vector.rescore_vector"]
+ reason: Needs rescore_vector feature
+ - skip:
+ features: "headers"
+ - do:
+ indices.create:
+ index: bbq_rescore_hnsw
+ body:
+ settings:
+ index:
+ number_of_shards: 1
+ mappings:
+ properties:
+ vector:
+ type: dense_vector
+ dims: 64
+ index: true
+ similarity: max_inner_product
+ index_options:
+ type: bbq_hnsw
+ rescore_vector:
+ oversample: 1.5
+
+ - do:
+ bulk:
+ index: bbq_rescore_hnsw
+ refresh: true
+ body: |
+ { "index": {"_id": "1"}}
+ { "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] }
+ { "index": {"_id": "2"}}
+ { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
+ { "index": {"_id": "3"}}
+ { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
+
+ - do:
+ headers:
+ Content-Type: application/json
+ search:
+ rest_total_hits_as_int: true
+ index: bbq_rescore_hnsw
+ body:
+ knn:
+ field: vector
+ query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
+ 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
+ 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
+ -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
+ -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
+ -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
+ -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
+ -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
+ k: 3
+ num_candidates: 3
+
+ - match: { hits.total: 3 }
+ - set: { hits.hits.0._score: rescore_score0 }
+ - set: { hits.hits.1._score: rescore_score1 }
+ - set: { hits.hits.2._score: rescore_score2 }
+
+ - do:
+ headers:
+ Content-Type: application/json
+ search:
+ rest_total_hits_as_int: true
+ index: bbq_rescore_hnsw
+ body:
+ query:
+ script_score:
+ query: {match_all: {} }
+ script:
+ source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1"
+ params:
+ query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
+ 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
+ 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
+ -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
+ -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
+ -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
+ -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
+ -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
+
+ # Compare scores as hit IDs may change depending on how things are distributed
+ - match: { hits.total: 3 }
+ - match: { hits.hits.0._score: $rescore_score0 }
+ - match: { hits.hits.1._score: $rescore_score1 }
+ - match: { hits.hits.2._score: $rescore_score2 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml

@@ -611,3 +611,92 @@ setup:
  - match: { hits.hits.0._id: "1"}
  - match: { hits.hits.1._id: "2"}
  - match: { hits.hits.2._id: "3"}
+---
+"Test index configured rescore vector":
+ - requires:
+ cluster_features: ["mapper.dense_vector.rescore_vector"]
+ reason: Needs rescore_vector feature
+ - skip:
+ features: "headers"
+ - do:
+ indices.create:
+ index: int8_rescore_hnsw
+ body:
+ settings:
+ index:
+ number_of_shards: 1
+ mappings:
+ properties:
+ vector:
+ type: dense_vector
+ dims: 64
+ index: true
+ similarity: max_inner_product
+ index_options:
+ type: int8_hnsw
+ rescore_vector:
+ oversample: 1.5
+
+ - do:
+ bulk:
+ index: int8_rescore_hnsw
+ refresh: true
+ body: |
+ { "index": {"_id": "1"}}
+ { "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] }
+ { "index": {"_id": "2"}}
+ { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
+ { "index": {"_id": "3"}}
+ { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
+
+ - do:
+ headers:
+ Content-Type: application/json
+ search:
+ rest_total_hits_as_int: true
+ index: int8_rescore_hnsw
+ body:
+ knn:
+ field: vector
+ query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
+ 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
+ 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
+ -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
+ -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
+ -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
+ -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
+ -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
+ k: 3
+ num_candidates: 3
+
+ - match: { hits.total: 3 }
+ - set: { hits.hits.0._score: rescore_score0 }
+ - set: { hits.hits.1._score: rescore_score1 }
+ - set: { hits.hits.2._score: rescore_score2 }
+
+ - do:
+ headers:
+ Content-Type: application/json
+ search:
+ rest_total_hits_as_int: true
+ body:
+ query:
+ script_score:
+ query: {match_all: {} }
+ script:
+ source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1"
+ params:
+ query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
+ 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
+ 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
+ -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
+ -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
+ -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
+ -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
+ -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
+
+ # Compare scores as hit IDs may change depending on how things are distributed
+ - match: { hits.total: 3 }
+ - match: { hits.hits.0._score: $rescore_score0 }
+ - match: { hits.hits.1._score: $rescore_score1 }
+ - match: { hits.hits.2._score: $rescore_score2 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml

@@ -645,3 +645,92 @@ setup:
  index: dynamic_dim_hnsw_quantized
  body:
  vector: [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]
+---
+"Test index configured rescore vector":
+ - requires:
+ cluster_features: ["mapper.dense_vector.rescore_vector"]
+ reason: Needs rescore_vector feature
+ - skip:
+ features: "headers"
+ - do:
+ indices.create:
+ index: int4_rescore_hnsw
+ body:
+ settings:
+ index:
+ number_of_shards: 1
+ mappings:
+ properties:
+ vector:
+ type: dense_vector
+ dims: 64
+ index: true
+ similarity: max_inner_product
+ index_options:
+ type: int4_hnsw
+ rescore_vector:
+ oversample: 1.5
+
+ - do:
+ bulk:
+ index: int4_rescore_hnsw
+ refresh: true
+ body: |
+ { "index": {"_id": "1"}}
+ { "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] }
+ { "index": {"_id": "2"}}
+ { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
+ { "index": {"_id": "3"}}
+ { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
+
+ - do:
+ headers:
+ Content-Type: application/json
+ search:
+ rest_total_hits_as_int: true
+ index: int4_rescore_hnsw
+ body:
+ knn:
+ field: vector
+ query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
+ 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
+ 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
+ -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
+ -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
+ -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
+ -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
+ -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
+ k: 3
+ num_candidates: 3
+
+ - match: { hits.total: 3 }
+ - set: { hits.hits.0._score: rescore_score0 }
+ - set: { hits.hits.1._score: rescore_score1 }
+ - set: { hits.hits.2._score: rescore_score2 }
+
+ - do:
+ headers:
+ Content-Type: application/json
+ search:
+ rest_total_hits_as_int: true
+ body:
+ query:
+ script_score:
+ query: {match_all: {} }
+ script:
+ source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1"
+ params:
+ query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
+ 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
+ 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
+ -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
+ -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
+ -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
+ -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
+ -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
+
+ # Compare scores as hit IDs may change depending on how things are distributed
+ - match: { hits.total: 3 }
+ - match: { hits.hits.0._score: $rescore_score0 }
+ - match: { hits.hits.1._score: $rescore_score1 }
+ - match: { hits.hits.2._score: $rescore_score2 }