Vector Quantization

Initialize your Go project.

In a terminal window, run the following commands to create a new directory named ingest-binary-vectors and initialize your project:

mkdir ingest-binary-vectors-project
cd ingest-binary-vectors-project
go mod init ingest-binary-vectors-project

Install the required libraries.

Run the following command to install the MongoDB Go Driver. This operation might take a few minutes to complete.

go get go.mongodb.org/mongo-driver/v2/mongo

You must install Go v2.1 or later driver. If necessary, you can also install libraries from your embedding model provider. For examples in this tutorial, we will use the Voyage AI REST API to generate embeddings. Therefore, you don't need to install any additional libraries.

Set the environment variables in your terminal.

1. To access the embedding model provider for generating and converting embeddings, set the environment variable for the embedding model provider's API key, if necessary.

   For using embeddings from Voyage AI, set up the VOYAGE_API_KEY environment variable.

   export VOYAGE_API_KEY="<VOYAGE-API-KEY>"

2. To access Atlas cluster, set the MONGODB_URI environment variable.

   export MONGODB_URI="<CONNECTION-STRING>"

   Your connection string should be in the following format:

   mongodb+srv://<db_username>:<db_password>@<clusterName>.<hostname>.mongodb.net

(Conditional) Generate embeddings from your data.

You can use an embedding model provider to generate float32, int8, and int1 embeddings for your data and then use the MongoDB Go driver to convert your native vector embedding to BSON vectors. The following sample code uses Cohere's embed API to generate full-precision vectors.

1. Create a new file named GenerateAndConvertEmbeddings.go in your Go project.

   touch GenerateAndConvertEmbeddings.go

2. Copy and paste the following code in the GenerateAndConvertEmbeddings.go file.

   This code does the following:

   • Generates the float32, int8, and ubinary vector embeddings by using Cohere's embed API.

   • Converts the embeddings to BSON binData vectors by using MongoDB Go driver.

   • Creates a file named embeddings.json and saves the data with embeddings in the file.

   GenerateAndConvertEmbeddings.go

   1
   package main
   2
   3
   import (
   4
   "bytes"
   5
   "context"
   6
   "encoding/json"
   7
   "fmt"
   8
   "io/ioutil"
   9
   "log"
   10
   "net/http"
   11
   "os"
   12
   13
   "go.mongodb.org/mongo-driver/v2/bson"
   14
   )
   15
   16
   // Sample data for embedding
   17
   var data = []string{
   18
   "The Great Wall of China is visible from space.",
   19
   "The Eiffel Tower was completed in Paris in 1889.",
   20
   "Mount Everest is the highest peak on Earth at 8,848m.",
   21
   "Shakespeare wrote 37 plays and 154 sonnets during his lifetime.",
   22
   "The Mona Lisa was painted by Leonardo da Vinci.",
   23
   }
   24
   25
   func main() {
   26
   apiKey := os.Getenv("VOYAGE_API_KEY")
   27
   if apiKey == "" {
   28
   log.Fatal("Ensure VOYAGE_API_KEY is set.")
   29
   }
   30
   31
   model := "voyage-3-large"
   32
   33
   // Generate embeddings for float, int8, and ubinary
   34
   floatEmbeddings, err := fetchEmbeddingsFromVoyage(data, apiKey, model, "float")
   35
   if err != nil {
   36
   log.Fatalf("Error fetching float embeddings: %v", err)
   37
   }
   38
   39
   int8Embeddings, err := fetchEmbeddingsFromVoyage(data, apiKey, model, "int8")
   40
   if err != nil {
   41
   log.Fatalf("Error fetching int8 embeddings: %v", err)
   42
   }
   43
   44
   ubinaryEmbeddings, err := fetchEmbeddingsFromVoyage(data, apiKey, model, "ubinary")
   45
   if err != nil {
   46
   log.Fatalf("Error fetching ubinary embeddings: %v", err)
   47
   }
   48
   49
   // Convert to BSON and store in JSON file
   50
   documents := convertEmbeddingsToBSON(data, floatEmbeddings, int8Embeddings, ubinaryEmbeddings)
   51
   52
   err = writeJSONToFile("embeddings.json", documents)
   53
   if err != nil {
   54
   log.Fatalf("Error writing embeddings to file: %v", err)
   55
   }
   56
   57
   fmt.Println("Embeddings successfully stored in embeddings.json")
   58
   }
   59
   60
   // Fetch embeddings using Voyage AI REST API
   61
   func fetchEmbeddingsFromVoyage(texts []string, apiKey string, model string, outputDType string) ([]map[string]interface{}, error) {
   62
   url := "https://api.voyageai.com/v1/embeddings"
   63
   64
   // Prepare request body
   65
   requestBody := map[string]interface{}{
   66
   "input": texts,
   67
   "model": model,
   68
   "output_dtype": outputDType,
   69
   "output_dimension": 1024,
   70
   "input_type": "document",
   71
   }
   72
   73
   requestBytes, err := json.Marshal(requestBody)
   74
   if err != nil {
   75
   return nil, fmt.Errorf("failed to marshal request body: %w", err)
   76
   }
   77
   78
   req, err := http.NewRequestWithContext(context.TODO(), "POST", url, bytes.NewBuffer(requestBytes))
   79
   if err != nil {
   80
   return nil, fmt.Errorf("failed to create HTTP request: %w", err)
   81
   }
   82
   83
   req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
   84
   req.Header.Set("Content-Type", "application/json")
   85
   86
   client := &http.Client{}
   87
   resp, err := client.Do(req)
   88
   if err != nil {
   89
   return nil, fmt.Errorf("failed to make API request: %w", err)
   90
   }
   91
   defer resp.Body.Close()
   92
   93
   if resp.StatusCode != http.StatusOK {
   94
   body, _ := ioutil.ReadAll(resp.Body)
   95
   return nil, fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, string(body))
   96
   }
   97
   98
   var response struct {
   99
   Data []map[string]interface{} `json:"data"`
   100
   }
   101
   if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
   102
   return nil, fmt.Errorf("failed to parse API response: %w", err)
   103
   }
   104
   105
   return response.Data, nil
   106
   }
   107
   108
   // Convert embeddings to BSON binary vectors
   109
   func convertEmbeddingsToBSON(sentences []string, floatEmbeddings []map[string]interface{}, int8Embeddings []map[string]interface{}, ubinaryEmbeddings []map[string]interface{}) []bson.M {
   110
   var documents []bson.M
   111
   112
   for i, sentence := range sentences {
   113
   floatEmbedding := convertInterfaceToFloat32(floatEmbeddings[i]["embedding"].([]interface{}))
   114
   int8Embedding := convertInterfaceToInt8(int8Embeddings[i]["embedding"].([]interface{}))
   115
   ubinaryEmbedding := convertInterfaceToBytes(ubinaryEmbeddings[i]["embedding"].([]interface{}))
   116
   117
   floatVector := bson.NewVector(floatEmbedding)
   118
   int8Vector := bson.NewVector(int8Embedding)
   119
   ubinaryVector, err := bson.NewPackedBitVector(ubinaryEmbedding, 0)
   120
   if err != nil {
   121
   log.Fatalf("Error creating PackedBitVector: %v", err)
   122
   }
   123
   124
   document := bson.M{
   125
   "text": sentence,
   126
   "embeddings_float32": floatVector.Binary(),
   127
   "embeddings_int8": int8Vector.Binary(),
   128
   "embeddings_int1": ubinaryVector.Binary(),
   129
   }
   130
   documents = append(documents, document)
   131
   }
   132
   133
   return documents
   134
   }
   135
   136
   // Write JSON file from in-memory BSON documents
   137
   func writeJSONToFile(filename string, documents []bson.M) error {
   138
   file, err := os.Create(filename)
   139
   if err != nil {
   140
   return fmt.Errorf("failed to create file: %w", err)
   141
   }
   142
   defer file.Close()
   143
   144
   var jsonData []json.RawMessage
   145
   for _, document := range documents {
   146
   jsonBytes, err := bson.MarshalExtJSON(document, false, false)
   147
   if err != nil {
   148
   return fmt.Errorf("error marshaling BSON to JSON: %w", err)
   149
   }
   150
   jsonData = append(jsonData, jsonBytes)
   151
   }
   152
   153
   marshaledData, err := json.MarshalIndent(jsonData, "", " ")
   154
   if err != nil {
   155
   return fmt.Errorf("failed to marshal JSON: %w", err)
   156
   }
   157
   158
   _, err = file.Write(marshaledData)
   159
   if err != nil {
   160
   return fmt.Errorf("failed to write JSON to file: %w", err)
   161
   }
   162
   163
   return nil
   164
   }
   165
   166
   // Convert a slice of interfaces to a slice of float32
   167
   func convertInterfaceToFloat32(data []interface{}) []float32 {
   168
   f32s := make([]float32, len(data))
   169
   for i, v := range data {
   170
   f32s[i] = float32(v.(float64))
   171
   }
   172
   return f32s
   173
   }
   174
   175
   // Convert a slice of interfaces to a slice of int8
   176
   func convertInterfaceToInt8(data []interface{}) []int8 {
   177
   ints8 := make([]int8, len(data))
   178
   for i, v := range data {
   179
   switch val := v.(type) {
   180
   case int:
   181
   ints8[i] = int8(val)
   182
   case float64:
   183
   ints8[i] = int8(val)
   184
   default:
   185
   log.Fatalf("Unexpected type %T in int8 embedding at index %d", v, i)
   186
   }
   187
   }
   188
   return ints8
   189
   }
   190
   191
   // Convert a slice of interfaces to a slice of bytes
   192
   func convertInterfaceToBytes(data []interface{}) []byte {
   193
   bytes := make([]byte, len(data))
   194
   for i, v := range data {
   195
   switch val := v.(type) {
   196
   case int:
   197
   bytes[i] = byte(val)
   198
   case float64:
   199
   bytes[i] = byte(val)
   200
   default:
   201
   log.Fatalf("Unexpected type %T in ubinary embedding at index %d", v, i)
   202
   }
   203
   }
   204
   return bytes
   205
   }

3. Replace the following placeholder value in the code and save the file.

   VOYAGE_API_KEY	Your Voyage AI API key only if you didn't set the environment variable.

4. Run the program using the following command.

   If you are using a terminal, run the following commands to compile and execute your program.

   go run GenerateAndConvertEmbeddings.go

   HIDE OUTPUT

   Embeddings successfully stored in embeddings.json

5. Verify the embeddings in the embeddings.json file.

To learn more about generating embeddings and converting the embeddings to binData vectors, see How to Create Vector Embeddings.

Ingest the data and create an Atlas Vector Search index.

You must upload your data and embeddings to a collection in your Atlas cluster and create an Atlas Vector Search index on the data to run $vectorSearch queries against the data.

1. Create a new file named UploadDataAndCreateIndex.go in your Go project.

   touch UploadDataAndCreateIndex.go

2. Copy and paste the following code in the UploadDataAndCreateIndex.go file.

   This code does the following:

   • Uploads the float32, int8, and int1 embeddings in the embeddings.json file to your Atlas cluster.

   • Creates an Atlas Vector Search index on the embeddings.float32, embeddings.int8, and embeddings.int1 fields.

   UploadDataAndCreateIndex.go

   1
   package main
   2
   3
   import (
   4
   "context"
   5
   "fmt"
   6
   "io/ioutil"
   7
   "log"
   8
   "time"
   9
   "os"
   10
   11
   "go.mongodb.org/mongo-driver/v2/bson"
   12
   "go.mongodb.org/mongo-driver/v2/mongo"
   13
   "go.mongodb.org/mongo-driver/v2/mongo/options"
   14
   )
   15
   16
   var (
   17
   mongodbURI = os.Getenv("MONGODB_URI")
   18
   dbName = "<DATABASE-NAME>"
   19
   collectionName = "<COLLECTION-NAME>"
   20
   indexName = "<INDEX-NAME>"
   21
   numberOfDimensions = 1024
   22
   embeddingFields = []string{"embeddings_float32", "embeddings_int8", "embeddings_int1"}
   23
   embeddingSimilarity = []string{"dotProduct", "dotProduct", "euclidean"}
   24
   )
   25
   26
   func main() {
   27
   clientOpts := options.Client().ApplyURI(mongodbURI)
   28
   client, err := mongo.Connect(clientOpts)
   29
   if err != nil {
   30
   log.Fatalf("Failed to connect to MongoDB: %v", err)
   31
   }
   32
   33
   defer func() {
   34
   if err := client.Disconnect(context.TODO()); err != nil {
   35
   log.Fatalf("Failed to disconnect MongoDB client: %v", err)
   36
   }
   37
   }()
   38
   39
   storeEmbeddings(client)
   40
   setupVectorSearchIndex(client)
   41
   }
   42
   43
   // Reads JSON data, stores it in MongoDB
   44
   func storeEmbeddings(client *mongo.Client) {
   45
   database := client.Database(dbName)
   46
   collection := database.Collection(collectionName)
   47
   48
   data, err := ioutil.ReadFile("embeddings.json")
   49
   if err != nil {
   50
   log.Fatalf("Failed to read file: %v", err)
   51
   }
   52
   53
   var documents []bson.M
   54
   if err := bson.UnmarshalExtJSON(data, false, &documents); err != nil {
   55
   log.Fatalf("Failed to unmarshal JSON data: %v", err)
   56
   }
   57
   58
   if _, err := collection.InsertMany(context.TODO(), documents); err != nil {
   59
   log.Fatalf("Failed to insert documents: %v", err)
   60
   }
   61
   62
   fmt.Println("Inserted documents into MongoDB")
   63
   }
   64
   65
   // Sets up vector search index in MongoDB
   66
   func setupVectorSearchIndex(client *mongo.Client) {
   67
   database := client.Database(dbName)
   68
   collection := database.Collection(collectionName)
   69
   70
   ctx := context.TODO()
   71
   72
   type vectorDefinitionField struct {
   73
   Type string `bson:"type"`
   74
   Path string `bson:"path"`
   75
   NumDimensions int `bson:"numDimensions"`
   76
   Similarity string `bson:"similarity"`
   77
   }
   78
   79
   type vectorDefinition struct {
   80
   Fields []vectorDefinitionField `bson:"fields"`
   81
   }
   82
   83
   fields := make([]vectorDefinitionField, len(embeddingFields))
   84
   for i, field := range embeddingFields {
   85
   fields[i] = vectorDefinitionField{
   86
   Type: "vector",
   87
   Path: field,
   88
   NumDimensions: numberOfDimensions,
   89
   Similarity: embeddingSimilarity[i],
   90
   }
   91
   }
   92
   fmt.Println(fields)
   93
   94
   opts := options.SearchIndexes().SetName(indexName).SetType("vectorSearch")
   95
   96
   indexModel := mongo.SearchIndexModel{
   97
   Definition: vectorDefinition{
   98
   Fields: fields,
   99
   },
   100
   Options: opts,
   101
   }
   102
   103
   // Create the index
   104
   log.Println("Creating the index.")
   105
   searchIndexName, err := collection.SearchIndexes().CreateOne(ctx, indexModel)
   106
   if err != nil {
   107
   log.Fatalf("Failed to create the search index: %v", err)
   108
   }
   109
   110
   // Polling to confirm successful index creation
   111
   log.Println("Polling to confirm successful index creation.")
   112
   log.Println("NOTE: This may take up to a minute.")
   113
   searchIndexes := collection.SearchIndexes()
   114
   var doc bson.Raw
   115
   116
   for doc == nil {
   117
   cursor, err := searchIndexes.List(ctx, options.SearchIndexes().SetName(searchIndexName))
   118
   if err != nil {
   119
   log.Fatalf("failed to list search indexes: %v", err)
   120
   }
   121
   122
   if !cursor.Next(ctx) {
   123
   break
   124
   }
   125
   126
   name := cursor.Current.Lookup("name").StringValue()
   127
   queryable := cursor.Current.Lookup("queryable").Boolean()
   128
   if name == searchIndexName && queryable {
   129
   doc = cursor.Current
   130
   } else {
   131
   time.Sleep(5 * time.Second)
   132
   }
   133
   }
   134
   135
   log.Println("Name of Index Created: " + searchIndexName)
   136
   }

3. Replace the following with valid values in the code and save the file.

   MONGODB_URI	Your Atlas cluster connection string if you didn't set the environment variable.
   <DATABASE-NAME>	Name of the Atlas Vector Search index for the collection.
   <COLLECTION-NAME>	Name of the Atlas Vector Search index for the collection.
   <INDEX-NAME>	Name of the Atlas Vector Search index for the collection.

4. Run the program using the following command.

   If you are using a terminal, run the following commands to compile and execute your program.

   go run UploadDataAndCreateIndex.go

   HIDE OUTPUT

   Inserted documents into MongoDB
   Creating the index.
   Polling to confirm successful index creation.
   NOTE: This may take up to a minute.
   Name of Index Created: <INDEX-NAME>

5. Log in to your Atlas cluster and verify the following:

   • Data in the namespace.

   • Atlas Vector Search index for the collection.

Create and run query against the collection.

To test your embeddings, you can run a query against your collection. Use an embedding model provider to generate float32, int8, and int1 embeddings for your query text. The following sample code uses Cohere's embed API to generate full-precision vectors. After generating the embeddings, use the MongoDB Go driver to convert your native vector embedding to BSON binary vectors and run $vectorSearch query against the collection.

1. Create a new file named CreateEmbeddingsAndRunQuery.go in your Go project.

   touch CreateEmbeddingsAndRunQuery.go

2. Copy and paste the following code in the CreateEmbeddingsAndRunQuery.go file.

   This code does the following:

   • Generates the float32, int8, and ubinary vector embeddings by using Cohere's embed API.

   • Converts the embeddings to BSON binData vectors by using MongoDB Go driver.

   • Runs the query against your collection and returns the results.

   CreateEmbeddingsAndRunQuery.go

   1
   package main
   2
   3
   import (
   4
   "bytes"
   5
   "context"
   6
   "encoding/json"
   7
   "fmt"
   8
   "io"
   9
   "log"
   10
   "net/http"
   11
   "os"
   12
   13
   "go.mongodb.org/mongo-driver/v2/bson"
   14
   "go.mongodb.org/mongo-driver/v2/mongo"
   15
   "go.mongodb.org/mongo-driver/v2/mongo/options"
   16
   )
   17
   18
   const (
   19
   dbName = "<DATABASE-NAME>"
   20
   collectionName = "<COLLECTION-NAME>"
   21
   vectorIndexName = "<INDEX-NAME>"
   22
   dataFieldName = "<TEXT-FIELD-NAME>"
   23
   queryText = "<QUERY-TEXT>"
   24
   model = "voyage-3-large"
   25
   outputDimension = 1024
   26
   candidates = <NUMBER-OF-CANDIDATES-TO-CONSIDER>
   27
   numDocs = <NUMBER-OF-DOCUMENTS-TO-RETURN>
   28
   )
   29
   30
   func main() {
   31
   apiKey := os.Getenv("VOYAGE_API_KEY")
   32
   mongodbURI := os.Getenv("MONGODB_URI")
   33
   34
   if apiKey == "" {
   35
   log.Fatal("API key not found. Set VOYAGE_API_KEY in your environment.")
   36
   }
   37
   if mongodbURI == "" {
   38
   log.Fatal("MongoDB URI not found. Set MONGODB_URI in your environment.")
   39
   }
   40
   41
   embeddingsData, err := generateAndConvertEmbeddings(apiKey, queryText)
   42
   if err != nil {
   43
   log.Fatalf("Error generating embeddings: %v", err)
   44
   }
   45
   46
   err = runVectorSearchQuery(mongodbURI, embeddingsData)
   47
   if err != nil {
   48
   log.Fatalf("Error running vector search query: %v", err)
   49
   }
   50
   }
   51
   52
   // Generate embeddings using Voyage AI's embedding API from the query text
   53
   func generateAndConvertEmbeddings(apiKey, text string) (map[string]bson.Binary, error) {
   54
   embeddingFormats := []string{"float", "int8", "ubinary"}
   55
   embeddingsData := make(map[string]bson.Binary)
   56
   57
   for _, outputDType := range embeddingFormats {
   58
   response := fetchEmbeddingsFromVoyageAPI(apiKey, text, outputDType)
   59
   embedding := createBSONVectorEmbeddings(outputDType, response)
   60
   embeddingsData[outputDType] = embedding
   61
   }
   62
   63
   return embeddingsData, nil
   64
   }
   65
   66
   // Fetch embeddings using Voyage AI Embedding REST API
   67
   func fetchEmbeddingsFromVoyageAPI(apiKey, text, outputDType string) map[string]interface{} {
   68
   url := "https://api.voyageai.com/v1/embeddings"
   69
   70
   requestBody := map[string]interface{}{
   71
   "input": []string{text},
   72
   "model": model,
   73
   "output_dtype": outputDType,
   74
   "output_dimension": outputDimension,
   75
   "input_type": "query",
   76
   }
   77
   78
   requestBytes, err := json.Marshal(requestBody)
   79
   if err != nil {
   80
   log.Fatalf("Failed to marshal request body: %v", err)
   81
   }
   82
   83
   req, err := http.NewRequestWithContext(context.TODO(), "POST", url, bytes.NewBuffer(requestBytes))
   84
   if err != nil {
   85
   log.Fatalf("Failed to create HTTP request: %v", err)
   86
   }
   87
   88
   req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
   89
   req.Header.Set("Content-Type", "application/json")
   90
   91
   client := &http.Client{}
   92
   resp, err := client.Do(req)
   93
   if err != nil {
   94
   log.Fatalf("Failed to make API request: %v", err)
   95
   }
   96
   defer resp.Body.Close()
   97
   98
   if resp.StatusCode != http.StatusOK {
   99
   body, _ := io.ReadAll(resp.Body)
   100
   log.Fatalf("Unexpected status code %d: %s", resp.StatusCode, string(body))
   101
   }
   102
   103
   var response struct {
   104
   Data []map[string]interface{} `json:"data"`
   105
   }
   106
   if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
   107
   log.Fatalf("Failed to parse API response: %v", err)
   108
   }
   109
   110
   if len(response.Data) == 0 {
   111
   log.Fatalf("No embeddings found in API response")
   112
   }
   113
   114
   return response.Data[0]
   115
   }
   116
   117
   // Convert embeddings to BSON vectors using MongoDB Go Driver
   118
   func createBSONVectorEmbeddings(dataType string, rawEmbedding map[string]interface{}) bson.Binary {
   119
   embeddingArray := rawEmbedding["embedding"].([]interface{})
   120
   121
   switch dataType {
   122
   case "float":
   123
   floatData := convertInterfaceToFloat32(embeddingArray)
   124
   floatVector := bson.NewVector(floatData)
   125
   return floatVector.Binary()
   126
   case "int8":
   127
   int8Data := convertInterfaceToInt8(embeddingArray)
   128
   int8Vector := bson.NewVector(int8Data)
   129
   return int8Vector.Binary()
   130
   case "ubinary":
   131
   int1Data := convertInterfaceToBytes(embeddingArray)
   132
   ubinaryVector, err := bson.NewPackedBitVector(int1Data, 0)
   133
   if err != nil {
   134
   log.Fatalf("Error creating PackedBitVector: %v", err)
   135
   }
   136
   return ubinaryVector.Binary()
   137
   default:
   138
   log.Fatalf("Unknown data type: %s", dataType)
   139
   return bson.Binary{}
   140
   }
   141
   }
   142
   143
   // Run $vectorSearch query using the embeddings
   144
   func runVectorSearchQuery(mongodbURI string, embeddingsData map[string]bson.Binary) error {
   145
   ctx := context.Background()
   146
   clientOptions := options.Client().ApplyURI(mongodbURI)
   147
   client, err := mongo.Connect(clientOptions)
   148
   if err != nil {
   149
   return fmt.Errorf("failed to connect to MongoDB: %w", err)
   150
   }
   151
   defer func() { _ = client.Disconnect(ctx) }()
   152
   153
   db := client.Database(dbName)
   154
   collection := db.Collection(collectionName)
   155
   156
   pathMap := map[string]string{
   157
   "float": "embeddings_float32",
   158
   "int8": "embeddings_int8",
   159
   "ubinary": "embeddings_int1",
   160
   }
   161
   162
   for pathKey, queryVector := range embeddingsData {
   163
   path, ok := pathMap[pathKey]
   164
   if !ok {
   165
   return fmt.Errorf("invalid path key: %s", pathKey)
   166
   }
   167
   168
   pipeline := mongo.Pipeline{
   169
   {
   170
   {"$vectorSearch", bson.D{
   171
   {"queryVector", queryVector},
   172
   {"index", vectorIndexName},
   173
   {"path", path},
   174
   {"numCandidates", candidates},
   175
   {"limit", numDocs},
   176
   }},
   177
   },
   178
   {
   179
   {"$project", bson.D{
   180
   {"_id", 1},
   181
   {dataFieldName, 1},
   182
   {"score", bson.D{
   183
   {"$meta", "vectorSearchScore"},
   184
   }},
   185
   }},
   186
   },
   187
   }
   188
   189
   cursor, err := collection.Aggregate(context.Background(), pipeline)
   190
   if err != nil {
   191
   return fmt.Errorf("failed to run vector search aggregation query: %w", err)
   192
   }
   193
   defer cursor.Close(ctx)
   194
   195
   var results []bson.M
   196
   if err = cursor.All(context.Background(), &results); err != nil {
   197
   return fmt.Errorf("failed to parse aggregation query results: %w", err)
   198
   }
   199
   200
   fmt.Printf("Results from %v embeddings:\n", path)
   201
   for _, result := range results {
   202
   fmt.Println(result)
   203
   }
   204
   }
   205
   206
   return nil
   207
   }
   208
   209
   // Converts []interface{} to []float32 safely
   210
   func convertInterfaceToFloat32(data []interface{}) []float32 {
   211
   f32s := make([]float32, len(data))
   212
   for i, v := range data {
   213
   switch val := v.(type) {
   214
   case float64:
   215
   f32s[i] = float32(val)
   216
   case int:
   217
   f32s[i] = float32(val)
   218
   default:
   219
   log.Fatalf("Unexpected type %T in float32 conversion at index %d", v, i)
   220
   }
   221
   }
   222
   return f32s
   223
   }
   224
   225
   // Converts []interface{} to []int8 safely
   226
   func convertInterfaceToInt8(data []interface{}) []int8 {
   227
   ints8 := make([]int8, len(data))
   228
   for i, v := range data {
   229
   switch val := v.(type) {
   230
   case float64:
   231
   ints8[i] = int8(val)
   232
   case int:
   233
   ints8[i] = int8(val)
   234
   default:
   235
   log.Fatalf("Unexpected type %T in int8 conversion at index %d", v, i)
   236
   }
   237
   }
   238
   return ints8
   239
   }
   240
   241
   // Converts []interface{} to []byte (uint8) safely
   242
   func convertInterfaceToBytes(data []interface{}) []byte {
   243
   bytesOut := make([]byte, len(data))
   244
   for i, v := range data {
   245
   switch val := v.(type) {
   246
   case float64:
   247
   bytesOut[i] = byte(val)
   248
   case int:
   249
   bytesOut[i] = byte(val)
   250
   default:
   251
   log.Fatalf("Unexpected type %T in byte conversion at index %d", v, i)
   252
   }
   253
   }
   254
   return bytesOut
   255
   }

3. Replace the following placeholder values in the code and save the file.

   MONGODB_URI	Your Atlas cluster connection string if you didn't set the environment variable.
   VOYAGE_API_KEY	Your Voyage AI API key only if you didn't set the environment variable.
   <DATABASE-NAME>	Name of the database in your Atlas cluster.
   <COLLECTION-NAME>	Name of the collection where you ingested the data.
   <INDEX-NAME>	Name of the Atlas Vector Search index for the collection.
   <TEXT-FIELD-NAME>	Name of the field that contain the text from which you generated embeddings. For this example, use text.
   <QUERY-TEXT>	Text for the query. For this example, use science fact.
   <NUMBER-OF-CANDIDATES-TO-CONSIDER>	Number of nearest neighbors to consider during the search. For this example, use 5.
   <NUMBER-OF-DOCUMENTS-TO-RETURN>	Number of documents to return in the results. For this example, use 2.

4. Compile and run the file using your application run configuration.

   If you are using a terminal, run the following commands to compile and execute your program.

   go run CreateEmbeddingsAndRunQuery.go

   HIDE OUTPUT

   Results from embeddings_float32 embeddings:
   {"_id":{"$oid":"68630fc85cb353712a1c521d"},"text":"The Great Wall of China is visible from space.","score":{"$numberDouble":"0.7723928093910217"}}
   {"_id":{"$oid":"68630fc85cb353712a1c521f"},"text":"Mount Everest is the highest peak on Earth at 8,848m.","score":{"$numberDouble":"0.7363046407699585"}}
   Results from embeddings_int8 embeddings:
   {"_id":{"$oid":"68630fc85cb353712a1c521d"},"text":"The Great Wall of China is visible from space.","score":{"$numberDouble":"0.5051995515823364"}}
   {"_id":{"$oid":"68630fc85cb353712a1c521f"},"text":"Mount Everest is the highest peak on Earth at 8,848m.","score":{"$numberDouble":"0.5044659972190857"}}
   Results from embeddings_int1 embeddings:
   {"_id":{"$oid":"68630fc85cb353712a1c521d"},"text":"The Great Wall of China is visible from space.","score":{"$numberDouble":"0.6845703125"}}
   {"_id":{"$oid":"68630fc85cb353712a1c521f"},"text":"Mount Everest is the highest peak on Earth at 8,848m.","score":{"$numberDouble":"0.6650390625"}}

Initialize your Go project.

In a terminal window, run the following commands to create a new directory named ingest-binary-vectors and initialize your project:

mkdir ingest-binary-vectors-project
cd ingest-binary-vectors-project
go mod init ingest-binary-vectors-project

Install the required libraries.

Run the following command to install the MongoDB Go Driver. This operation might take a few minutes to complete.

go get go.mongodb.org/mongo-driver/v2/mongo

You must install Go v2.1 or later driver. If necessary, you can also install libraries from your embedding model provider. For examples in this tutorial, we will use the Voyage AI REST API to generate embeddings. Therefore, you don't need to install any additional libraries.

Set the environment variables in your terminal.

1. To access the embedding model provider for generating and converting embeddings, set the environment variable for the embedding model provider's API key, if necessary.

   For using embeddings from Voyage AI, set up the VOYAGE_API_KEY environment variable.

   export VOYAGE_API_KEY="<VOYAGE-API-KEY>"

2. To access Atlas cluster, set the MONGODB_URI environment variable.

   export MONGODB_URI="<CONNECTION-STRING>"

   Your connection string should be in the following format:

   mongodb+srv://<db_username>:<db_password>@<clusterName>.<hostname>.mongodb.net

(Conditional) Generate embeddings from your data.

You can use an embedding model provider to generate float, int8, and int1 embeddings for your data and then use the MongoDB Go driver to convert your native vector embedding to BSON vectors. The following sample code uses Voyage AI's voyage-3-large embedding model to generate full-precision vectors from the data in the sample_airbnb.listingsAndReviews namespace.

1. Create a new file named GenerateAndConvertEmbeddings.go in your Go project.

   touch GenerateAndConvertEmbeddings.go

2. Copy and paste the following code in the GenerateAndConvertEmbeddings.go file.

   This code does the following:

   • Gets the summary field from 50 documents in the sample_airbnb.listingsAndReviews namespace.

   • Generates the float32, int8, and ubinary vector embeddings by using Voyage AI's API.

   • Converts the embeddings to BSON binData vectors by using MongoDB Go driver.

   • Creates a file named embeddings.json and saves the data with embeddings in the file.

   GenerateAndConvertEmbeddings.go

   1
   package main
   2
   3
   import (
   4
   "bytes"
   5
   "context"
   6
   "encoding/json"
   7
   "fmt"
   8
   "io/ioutil"
   9
   "log"
   10
   "net/http"
   11
   "os"
   12
   13
   "go.mongodb.org/mongo-driver/v2/bson"
   14
   "go.mongodb.org/mongo-driver/v2/mongo"
   15
   "go.mongodb.org/mongo-driver/v2/mongo/options"
   16
   )
   17
   18
   const (
   19
   batchSize = 96
   20
   dbName = "sample_airbnb"
   21
   collName = "listingsAndReviews"
   22
   model = "voyage-3-large"
   23
   outputDimension = 1024
   24
   )
   25
   26
   func main() {
   27
   apiKey := os.Getenv("VOYAGE_API_KEY")
   28
   mongodbURI := os.Getenv("MONGODB_URI")
   29
   30
   if apiKey == "" || mongodbURI == "" {
   31
   log.Fatal("Ensure VOYAGE_API_KEY and MONGODB_URI are set.")
   32
   }
   33
   34
   summaries, err := fetchSummariesFromMongoDB(mongodbURI)
   35
   if err != nil {
   36
   log.Fatalf("Error fetching summaries: %v", err)
   37
   }
   38
   39
   for start := 0; start < len(summaries); start += batchSize {
   40
   end := start + batchSize
   41
   if end > len(summaries) {
   42
   end = len(summaries)
   43
   }
   44
   45
   floatEmbeddings, err := fetchEmbeddingsFromVoyage(apiKey, summaries[start:end], "float")
   46
   if err != nil {
   47
   log.Fatalf("Error fetching float embeddings: %v", err)
   48
   }
   49
   50
   int8Embeddings, err := fetchEmbeddingsFromVoyage(apiKey, summaries[start:end], "int8")
   51
   if err != nil {
   52
   log.Fatalf("Error fetching int8 embeddings: %v", err)
   53
   }
   54
   55
   ubinaryEmbeddings, err := fetchEmbeddingsFromVoyage(apiKey, summaries[start:end], "ubinary")
   56
   if err != nil {
   57
   log.Fatalf("Error fetching ubinary embeddings: %v", err)
   58
   }
   59
   60
   documents := convertEmbeddingsToBSON(summaries[start:end], floatEmbeddings, int8Embeddings, ubinaryEmbeddings)
   61
   62
   err = writeJSONToFile("embeddings.json", documents)
   63
   if err != nil {
   64
   log.Fatalf("Error writing embeddings to JSON: %v", err)
   65
   }
   66
   }
   67
   68
   fmt.Println("Embeddings successfully saved to embeddings.json")
   69
   }
   70
   71
   // Fetch documents with the summary field from the collection
   72
   func fetchSummariesFromMongoDB(uri string) ([]string, error) {
   73
   ctx := context.TODO()
   74
   clientOpts := options.Client().ApplyURI(uri)
   75
   76
   client, err := mongo.Connect(clientOpts)
   77
   if err != nil {
   78
   return nil, fmt.Errorf("failed to connect to MongoDB: %w", err)
   79
   }
   80
   defer func() {
   81
   if err := client.Disconnect(ctx); err != nil {
   82
   log.Fatalf("Failed to disconnect MongoDB client: %v", err)
   83
   }
   84
   }()
   85
   86
   collection := client.Database(dbName).Collection(collName)
   87
   filter := bson.M{"summary": bson.M{"$nin": []interface{}{nil, ""}}}
   88
   89
   cursor, err := collection.Find(ctx, filter, options.Find().SetLimit(50))
   90
   if err != nil {
   91
   return nil, fmt.Errorf("error finding documents: %w", err)
   92
   }
   93
   defer cursor.Close(ctx)
   94
   95
   var summaries []string
   96
   for cursor.Next(ctx) {
   97
   var result struct {
   98
   Summary string `bson:"summary"`
   99
   }
   100
   if err := cursor.Decode(&result); err != nil {
   101
   return nil, fmt.Errorf("error decoding document: %w", err)
   102
   }
   103
   if result.Summary != "" {
   104
   summaries = append(summaries, result.Summary)
   105
   }
   106
   }
   107
   108
   if err := cursor.Err(); err != nil {
   109
   return nil, fmt.Errorf("cursor error: %w", err)
   110
   }
   111
   112
   return summaries, nil
   113
   }
   114
   115
   // Fetch embeddings using Voyage AI REST API
   116
   func fetchEmbeddingsFromVoyage(apiKey string, texts []string, outputDType string) ([]map[string]interface{}, error) {
   117
   url := "https://api.voyageai.com/v1/embeddings"
   118
   119
   requestBody := map[string]interface{}{
   120
   "input": texts,
   121
   "model": model,
   122
   "output_dtype": outputDType,
   123
   "output_dimension": outputDimension,
   124
   "input_type": "document",
   125
   }
   126
   127
   requestBytes, err := json.Marshal(requestBody)
   128
   if err != nil {
   129
   return nil, fmt.Errorf("failed to marshal request body: %w", err)
   130
   }
   131
   132
   req, err := http.NewRequestWithContext(context.TODO(), "POST", url, bytes.NewBuffer(requestBytes))
   133
   if err != nil {
   134
   return nil, fmt.Errorf("failed to create HTTP request: %w", err)
   135
   }
   136
   137
   req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
   138
   req.Header.Set("Content-Type", "application/json")
   139
   140
   client := &http.Client{}
   141
   resp, err := client.Do(req)
   142
   if err != nil {
   143
   return nil, fmt.Errorf("failed to make API request: %w", err)
   144
   }
   145
   defer resp.Body.Close()
   146
   147
   if resp.StatusCode != http.StatusOK {
   148
   body, _ := ioutil.ReadAll(resp.Body)
   149
   return nil, fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, string(body))
   150
   }
   151
   152
   var response struct {
   153
   Data []map[string]interface{} `json:"data"`
   154
   }
   155
   if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
   156
   return nil, fmt.Errorf("failed to parse API response: %w", err)
   157
   }
   158
   159
   return response.Data, nil
   160
   }
   161
   162
   // Convert embeddings to BSON binary vectors
   163
   func convertEmbeddingsToBSON(summaries []string, floatEmbeddings []map[string]interface{}, int8Embeddings []map[string]interface{}, ubinaryEmbeddings []map[string]interface{}) []bson.M {
   164
   var documents []bson.M
   165
   166
   for i, summary := range summaries {
   167
   floatEmbedding := convertInterfaceToFloat32(floatEmbeddings[i]["embedding"].([]interface{}))
   168
   int8Embedding := convertInterfaceToInt8(int8Embeddings[i]["embedding"].([]interface{}))
   169
   ubinaryEmbedding := convertInterfaceToBytes(ubinaryEmbeddings[i]["embedding"].([]interface{}))
   170
   171
   floatVector := bson.NewVector(floatEmbedding)
   172
   int8Vector := bson.NewVector(int8Embedding)
   173
   ubinaryVector, err := bson.NewPackedBitVector(ubinaryEmbedding, 0)
   174
   if err != nil {
   175
   log.Fatalf("Error creating PackedBitVector: %v", err)
   176
   }
   177
   178
   document := bson.M{
   179
   "text": summary,
   180
   "embeddings_float32": floatVector.Binary(),
   181
   "embeddings_int8": int8Vector.Binary(),
   182
   "embeddings_int1": ubinaryVector.Binary(),
   183
   }
   184
   185
   documents = append(documents, document)
   186
   }
   187
   188
   return documents
   189
   }
   190
   191
   // Write JSON file from in-memory BSON documents
   192
   func writeJSONToFile(filename string, docs []bson.M) error {
   193
   file, err := os.Create(filename)
   194
   if err != nil {
   195
   return fmt.Errorf("failed to create file: %w", err)
   196
   }
   197
   defer file.Close()
   198
   199
   var jsonDocuments []json.RawMessage
   200
   for _, document := range docs {
   201
   jsonBytes, err := bson.MarshalExtJSON(document, false, false)
   202
   if err != nil {
   203
   log.Fatalf("Error: %v", err)
   204
   }
   205
   jsonDocuments = append(jsonDocuments, jsonBytes)
   206
   }
   207
   208
   jsonData, err := json.MarshalIndent(jsonDocuments, "", " ")
   209
   if err != nil {
   210
   return fmt.Errorf("failed to marshal JSON: %w", err)
   211
   }
   212
   213
   _, err = file.Write(jsonData)
   214
   if err != nil {
   215
   return fmt.Errorf("failed to write JSON to file: %w", err)
   216
   }
   217
   218
   return nil
   219
   }
   220
   221
   // Converts slice of interface{} to []float32
   222
   func convertInterfaceToFloat32(data []interface{}) []float32 {
   223
   f32s := make([]float32, len(data))
   224
   for i, v := range data {
   225
   f32s[i] = float32(v.(float64))
   226
   }
   227
   return f32s
   228
   }
   229
   230
   // Converts slice of interface{} to []int8 safely
   231
   func convertInterfaceToInt8(data []interface{}) []int8 {
   232
   ints8 := make([]int8, len(data))
   233
   for i, v := range data {
   234
   switch val := v.(type) {
   235
   case float64:
   236
   ints8[i] = int8(val)
   237
   case int:
   238
   ints8[i] = int8(val)
   239
   default:
   240
   log.Fatalf("Unexpected type %T in int8 embedding at index %d", v, i)
   241
   }
   242
   }
   243
   return ints8
   244
   }
   245
   246
   // Converts slice of interface{} to []byte safely
   247
   func convertInterfaceToBytes(data []interface{}) []byte {
   248
   bytes := make([]byte, len(data))
   249
   for i, v := range data {
   250
   switch val := v.(type) {
   251
   case float64:
   252
   bytes[i] = byte(val)
   253
   case int:
   254
   bytes[i] = byte(val)
   255
   default:
   256
   log.Fatalf("Unexpected type %T in ubinary embedding at index %d", v, i)
   257
   }
   258
   }
   259
   return bytes
   260
   }

3. Replace the following placeholder values in the code if you didn't set the environment variables and save the file.

   MONGODB_URI	Your Atlas cluster connection string if you didn't set the environment variable.
   VOYAGE_API_KEY	Your Voyage AI API key if you didn't set the environment variable.

4. Compile and run the file using your application run configuration.

   If you are using a terminal, run the following commands to compile and execute your program.

   go run GenerateAndConvertEmbeddings.go

   HIDE OUTPUT

   Embeddings successfully saved to embeddings.json

5. Verify the embeddings in the embeddings.json file.

To learn more about generating embeddings and converting the embeddings to binData vectors, see How to Create Vector Embeddings.

Ingest the data and create an Atlas Vector Search index.

You must upload your data and embeddings to a collection in your Atlas cluster and create an Atlas Vector Search index on the data to run $vectorSearch queries against the data.

1. Create a new file named UploadDataAndCreateIndex.go in your Go project.

   touch UploadDataAndCreateIndex.go

2. Copy and paste the following code in the UploadDataAndCreateIndex.go file.

   This code does the following:

   • Uploads the float32, int8, and int1 embeddings in the embeddings.json file to your Atlas cluster.

   • Creates an Atlas Vector Search index on the embeddings.float32, embeddings.int8, and embeddings.int1 fields.

   UploadDataAndCreateIndex.go

   1
   package main 
   2
    
   3
   import ( 
   4
   "context" 
   5
   "fmt" 
   6
   "io/ioutil" 
   7
   "log" 
   8
    "time"
   9
   "os" 
   10
    
   11
   "go.mongodb.org/mongo-driver/v2/bson" 
   12
   "go.mongodb.org/mongo-driver/v2/mongo" 
   13
   "go.mongodb.org/mongo-driver/v2/mongo/options" 
   14
   ) 
   15
    
   16
   var ( 
   17
   mongodbURI = os.Getenv("MONGODB_URI") 
   18
   dbName = "sample_airbnb" 
   19
   collectionName = "listingsAndReviews" 
   20
   indexName = "<INDEX-NAME>" 
   21
   numberOfDimensions = 1024 
   22
   embeddingFields = []string{"embeddings_float32", "embeddings_int8", "embeddings_int1"} 
   23
   embeddingSimilarity = []string{"dotProduct", "dotProduct", "euclidean"} 
   24
   ) 
   25
    
   26
   func main() { 
   27
   if mongodbURI == "" { 
   28
   log.Fatal("MONGODB_URI environment variable not set") 
   29
   } 
   30
    
   31
   clientOptions := options.Client().ApplyURI(mongodbURI) 
   32
   client, err := mongo.Connect(clientOptions) 
   33
   if err != nil { 
   34
   log.Fatalf("Error connecting to MongoDB: %v", err) 
   35
   } 
   36
   defer func() { 
   37
   if err = client.Disconnect(context.TODO()); err != nil { 
   38
   log.Fatal(err) 
   39
   } 
   40
   }() 
   41
    
   42
   if err := uploadEmbeddingsData(client); err != nil { 
   43
   log.Fatalf("Error uploading embeddings data: %v", err) 
   44
   } 
   45
    
   46
   setupVectorSearchIndex(client) 
   47
   } 
   48
    
   49
   func uploadEmbeddingsData(client *mongo.Client) error { 
   50
   collection := client.Database(dbName).Collection(collectionName) 
   51
    
   52
   // Load embeddings.json file 
   53
   fileContent, err := ioutil.ReadFile("embeddings.json") 
   54
   if err != nil { 
   55
   return fmt.Errorf("error reading file: %w", err) 
   56
   } 
   57
    
   58
   // Convert JSON file content to BSON compatible format using UnmarshalExtJSON 
   59
   var documents []bson.M 
   60
   if err := bson.UnmarshalExtJSON(fileContent, false, &documents); err != nil { 
   61
   return fmt.Errorf("failed to unmarshal JSON data: %w", err) 
   62
   } 
   63
    
   64
   // Update documents in MongoDB 
   65
   for _, doc := range documents { 
   66
   summary, exists := doc["text"].(string) 
   67
   if !exists { 
   68
   return fmt.Errorf("missing 'text' field in document") 
   69
   } 
   70
    
   71
   // Using bson.Binary ensures binary data is correctly interpreted 
   72
   if float32Bin, ok := doc["embeddings_float32"].(bson.Binary); ok { 
   73
   doc["embeddings_float32"] = float32Bin 
   74
   } 
   75
   if int8Bin, ok := doc["embeddings_int8"].(bson.Binary); ok { 
   76
   doc["embeddings_int8"] = int8Bin 
   77
   } 
   78
   if int1Bin, ok := doc["embeddings_int1"].(bson.Binary); ok { 
   79
   doc["embeddings_int1"] = int1Bin 
   80
   } 
   81
    
   82
   filter := bson.M{"summary": summary} 
   83
   update := bson.M{ 
   84
   "$set": doc, 
   85
   } 
   86
    
   87
   // Set the upsert option 
   88
   opts := options.UpdateMany().SetUpsert(true) 
   89
    
   90
   _, err = collection.UpdateMany(context.TODO(), filter, update, opts) 
   91
   if err != nil { 
   92
   return fmt.Errorf("failed to update documents: %w", err) 
   93
   } 
   94
   } 
   95
    
   96
   return nil 
   97
   } 
   98
    
   99
   // Sets up vector search index in MongoDB 
   100
   func setupVectorSearchIndex(client *mongo.Client) { 
   101
   database := client.Database(dbName) 
   102
   collection := database.Collection(collectionName) 
   103
    
   104
   ctx := context.TODO() 
   105
    
   106
   type vectorDefinitionField struct { 
   107
   Type string `bson:"type"` 
   108
   Path string `bson:"path"` 
   109
   NumDimensions int `bson:"numDimensions"` 
   110
   Similarity string `bson:"similarity"` 
   111
   } 
   112
    
   113
   type vectorDefinition struct { 
   114
   Fields []vectorDefinitionField `bson:"fields"` 
   115
   } 
   116
    
   117
   fields := make([]vectorDefinitionField, len(embeddingFields)) 
   118
   for i, field := range embeddingFields { 
   119
   fields[i] = vectorDefinitionField{ 
   120
   Type: "vector", 
   121
   Path: field, 
   122
   NumDimensions: numberOfDimensions, 
   123
   Similarity: embeddingSimilarity[i], 
   124
   } 
   125
   } 
   126
    
   127
   opts := options.SearchIndexes().SetName(indexName).SetType("vectorSearch") 
   128
    
   129
   indexModel := mongo.SearchIndexModel{ 
   130
   Definition: vectorDefinition{ 
   131
   Fields: fields, 
   132
   }, 
   133
   Options: opts, 
   134
   } 
   135
    
   136
   // Create the index  
   137
   log.Println("Creating the index.") 
   138
   searchIndexName, err := collection.SearchIndexes().CreateOne(ctx, indexModel) 
   139
   if err != nil { 
   140
   log.Fatalf("Failed to create the search index: %v", err) 
   141
   } 
   142
    
   143
   // Polling to confirm successful index creation 
   144
   log.Println("Polling to confirm successful index creation.") 
   145
   log.Println("NOTE: This may take up to a minute.") 
   146
   searchIndexes := collection.SearchIndexes() 
   147
   var doc bson.Raw 
   148
    
   149
   for doc == nil { 
   150
   cursor, err := searchIndexes.List(ctx, options.SearchIndexes().SetName(searchIndexName)) 
   151
   if err != nil { 
   152
   log.Fatalf("failed to list search indexes: %v", err) 
   153
   } 
   154
    
   155
   if !cursor.Next(ctx) { 
   156
   break 
   157
   } 
   158
    
   159
   name := cursor.Current.Lookup("name").StringValue() 
   160
   queryable := cursor.Current.Lookup("queryable").Boolean() 
   161
   if name == searchIndexName && queryable { 
   162
   doc = cursor.Current 
   163
   } else { 
   164
   time.Sleep(5 * time.Second) 
   165
   } 
   166
   } 
   167
    
   168
   log.Println("Name of Index Created: " + searchIndexName) 
   169
   } 

3. Replace the following placeholder values in the code and save the file.

   MONGODB_URI	Your Atlas cluster connection string if you didn't set the environment variable.
   <INDEX-NAME>	Name of the Atlas Vector Search index for the collection.

4. Compile and run the file using your application run configuration.

   If you are using a terminal, run the following commands to compile and execute your program.

   go run UploadDataAndCreateIndex.go

   HIDE OUTPUT

   Creating the index.
   Polling to confirm successful index creation.
   NOTE: This may take up to a minute.
   Name of Index Created: <INDEX-NAME>

5. Log in to your Atlas cluster and verify the following:

   • Data in the namespace.

   • Atlas Vector Search index for the collection.

Create and run query against the collection.

To test your embeddings, you can run a query against your collection. Use an embedding model provider to generate float, int8, and int1 embeddings for your query text. The following sample code uses Voyage AI's API to generate full-precision vectors. After generating the embeddings, use the MongoDB Go driver to convert your native vector embedding to BSON vectors and run $vectorSearch query against the collection.

1. Create a new file named CreateEmbeddingsAndRunQuery.go in your Go project.

   touch CreateEmbeddingsAndRunQuery.go

2. Copy and paste the following code in the CreateEmbeddingsAndRunQuery.go file.

   This code does the following:

   • Generates the float32, int8, and ubinary vector embeddings by using Voyage AI's API.

   • Converts the embeddings to BSON binData vectors by using MongoDB Go driver.

   • Runs the query against your collection and returns the results.

   CreateEmbeddingsAndRunQuery.go

   1
   package main
   2
   3
   import (
   4
   "bytes"
   5
   "context"
   6
   "encoding/json"
   7
   "fmt"
   8
   "io"
   9
   "log"
   10
   "net/http"
   11
   "os"
   12
   13
   "go.mongodb.org/mongo-driver/v2/bson"
   14
   "go.mongodb.org/mongo-driver/v2/mongo"
   15
   "go.mongodb.org/mongo-driver/v2/mongo/options"
   16
   )
   17
   18
   const (
   19
   dbName = "<DATABASE-NAME>"
   20
   collectionName = "<COLLECTION-NAME>"
   21
   vectorIndexName = "<INDEX-NAME>"
   22
   dataFieldName = "<TEXT-FIELD-NAME>"
   23
   queryText = "<QUERY-TEXT>"
   24
   model = "voyage-3-large"
   25
   outputDimension = 1024
   26
   candidates = <NUMBER-OF-CANDIDATES-TO-CONSIDER>
   27
   numDocs = <NUMBER-OF-DOCUMENTS-TO-RETURN>
   28
   )
   29
   30
   func main() {
   31
   apiKey := os.Getenv("VOYAGE_API_KEY")
   32
   mongodbURI := os.Getenv("MONGODB_URI")
   33
   34
   if apiKey == "" {
   35
   log.Fatal("API key not found. Set VOYAGE_API_KEY in your environment.")
   36
   }
   37
   if mongodbURI == "" {
   38
   log.Fatal("MongoDB URI not found. Set MONGODB_URI in your environment.")
   39
   }
   40
   41
   embeddingsData, err := generateAndConvertEmbeddings(apiKey, queryText)
   42
   if err != nil {
   43
   log.Fatalf("Error generating embeddings: %v", err)
   44
   }
   45
   46
   err = runVectorSearchQuery(mongodbURI, embeddingsData)
   47
   if err != nil {
   48
   log.Fatalf("Error running vector search query: %v", err)
   49
   }
   50
   }
   51
   52
   // Generate embeddings using Voyage AI's embedding API from the query text
   53
   func generateAndConvertEmbeddings(apiKey, text string) (map[string]bson.Binary, error) {
   54
   embeddingFormats := []string{"float", "int8", "ubinary"}
   55
   embeddingsData := make(map[string]bson.Binary)
   56
   57
   for _, outputDType := range embeddingFormats {
   58
   response := fetchEmbeddingsFromVoyageAPI(apiKey, text, outputDType)
   59
   embedding := createBSONVectorEmbeddings(outputDType, response)
   60
   embeddingsData[outputDType] = embedding
   61
   }
   62
   63
   return embeddingsData, nil
   64
   }
   65
   66
   // Fetch embeddings using Voyage AI Embedding REST API
   67
   func fetchEmbeddingsFromVoyageAPI(apiKey, text, outputDType string) map[string]interface{} {
   68
   url := "https://api.voyageai.com/v1/embeddings"
   69
   70
   requestBody := map[string]interface{}{
   71
   "input": []string{text},
   72
   "model": model,
   73
   "output_dtype": outputDType,
   74
   "output_dimension": outputDimension,
   75
   "input_type": "query",
   76
   }
   77
   78
   requestBytes, err := json.Marshal(requestBody)
   79
   if err != nil {
   80
   log.Fatalf("Failed to marshal request body: %v", err)
   81
   }
   82
   83
   req, err := http.NewRequestWithContext(context.TODO(), "POST", url, bytes.NewBuffer(requestBytes))
   84
   if err != nil {
   85
   log.Fatalf("Failed to create HTTP request: %v", err)
   86
   }
   87
   88
   req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
   89
   req.Header.Set("Content-Type", "application/json")
   90
   91
   client := &http.Client{}
   92
   resp, err := client.Do(req)
   93
   if err != nil {
   94
   log.Fatalf("Failed to make API request: %v", err)
   95
   }
   96
   defer resp.Body.Close()
   97
   98
   if resp.StatusCode != http.StatusOK {
   99
   body, _ := io.ReadAll(resp.Body)
   100
   log.Fatalf("Unexpected status code %d: %s", resp.StatusCode, string(body))
   101
   }
   102
   103
   var response struct {
   104
   Data []map[string]interface{} `json:"data"`
   105
   }
   106
   if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
   107
   log.Fatalf("Failed to parse API response: %v", err)
   108
   }
   109
   110
   if len(response.Data) == 0 {
   111
   log.Fatalf("No embeddings found in API response")
   112
   }
   113
   114
   return response.Data[0]
   115
   }
   116
   117
   // Convert embeddings to BSON vectors using MongoDB Go Driver
   118
   func createBSONVectorEmbeddings(dataType string, rawEmbedding map[string]interface{}) bson.Binary {
   119
   embeddingArray := rawEmbedding["embedding"].([]interface{})
   120
   121
   switch dataType {
   122
   case "float":
   123
   floatData := convertInterfaceToFloat32(embeddingArray)
   124
   floatVector := bson.NewVector(floatData)
   125
   return floatVector.Binary()
   126
   case "int8":
   127
   int8Data := convertInterfaceToInt8(embeddingArray)
   128
   int8Vector := bson.NewVector(int8Data)
   129
   return int8Vector.Binary()
   130
   case "ubinary":
   131
   int1Data := convertInterfaceToBytes(embeddingArray)
   132
   ubinaryVector, err := bson.NewPackedBitVector(int1Data, 0)
   133
   if err != nil {
   134
   log.Fatalf("Error creating PackedBitVector: %v", err)
   135
   }
   136
   return ubinaryVector.Binary()
   137
   default:
   138
   log.Fatalf("Unknown data type: %s", dataType)
   139
   return bson.Binary{}
   140
   }
   141
   }
   142
   143
   // Run $vectorSearch query using the embeddings
   144
   func runVectorSearchQuery(mongodbURI string, embeddingsData map[string]bson.Binary) error {
   145
   ctx := context.Background()
   146
   clientOptions := options.Client().ApplyURI(mongodbURI)
   147
   client, err := mongo.Connect(clientOptions)
   148
   if err != nil {
   149
   return fmt.Errorf("failed to connect to MongoDB: %w", err)
   150
   }
   151
   defer func() { _ = client.Disconnect(ctx) }()
   152
   153
   db := client.Database(dbName)
   154
   collection := db.Collection(collectionName)
   155
   156
   pathMap := map[string]string{
   157
   "float": "embeddings_float32",
   158
   "int8": "embeddings_int8",
   159
   "ubinary": "embeddings_int1",
   160
   }
   161
   162
   for pathKey, queryVector := range embeddingsData {
   163
   path, ok := pathMap[pathKey]
   164
   if !ok {
   165
   return fmt.Errorf("invalid path key: %s", pathKey)
   166
   }
   167
   168
   pipeline := mongo.Pipeline{
   169
   {
   170
   {"$vectorSearch", bson.D{
   171
   {"queryVector", queryVector},
   172
   {"index", vectorIndexName},
   173
   {"path", path},
   174
   {"numCandidates", candidates},
   175
   {"limit", numDocs},
   176
   }},
   177
   },
   178
   {
   179
   {"$project", bson.D{
   180
   {"_id", 1},
   181
   {dataFieldName, 1},
   182
   {"score", bson.D{
   183
   {"$meta", "vectorSearchScore"},
   184
   }},
   185
   }},
   186
   },
   187
   }
   188
   189
   cursor, err := collection.Aggregate(context.Background(), pipeline)
   190
   if err != nil {
   191
   return fmt.Errorf("failed to run vector search aggregation query: %w", err)
   192
   }
   193
   defer cursor.Close(ctx)
   194
   195
   var results []bson.M
   196
   if err = cursor.All(context.Background(), &results); err != nil {
   197
   return fmt.Errorf("failed to parse aggregation query results: %w", err)
   198
   }
   199
   200
   fmt.Printf("Results from %v embeddings:\n", path)
   201
   for _, result := range results {
   202
   fmt.Println(result)
   203
   }
   204
   }
   205
   206
   return nil
   207
   }
   208
   209
   // Converts []interface{} to []float32 safely
   210
   func convertInterfaceToFloat32(data []interface{}) []float32 {
   211
   f32s := make([]float32, len(data))
   212
   for i, v := range data {
   213
   switch val := v.(type) {
   214
   case float64:
   215
   f32s[i] = float32(val)
   216
   case int:
   217
   f32s[i] = float32(val)
   218
   default:
   219
   log.Fatalf("Unexpected type %T in float32 conversion at index %d", v, i)
   220
   }
   221
   }
   222
   return f32s
   223
   }
   224
   225
   // Converts []interface{} to []int8 safely
   226
   func convertInterfaceToInt8(data []interface{}) []int8 {
   227
   ints8 := make([]int8, len(data))
   228
   for i, v := range data {
   229
   switch val := v.(type) {
   230
   case float64:
   231
   ints8[i] = int8(val)
   232
   case int:
   233
   ints8[i] = int8(val)
   234
   default:
   235
   log.Fatalf("Unexpected type %T in int8 conversion at index %d", v, i)
   236
   }
   237
   }
   238
   return ints8
   239
   }
   240
   241
   // Converts []interface{} to []byte (uint8) safely
   242
   func convertInterfaceToBytes(data []interface{}) []byte {
   243
   bytesOut := make([]byte, len(data))
   244
   for i, v := range data {
   245
   switch val := v.(type) {
   246
   case float64:
   247
   bytesOut[i] = byte(val)
   248
   case int:
   249
   bytesOut[i] = byte(val)
   250
   default:
   251
   log.Fatalf("Unexpected type %T in byte conversion at index %d", v, i)
   252
   }
   253
   }
   254
   return bytesOut
   255
   }

3. Replace the following placeholder values in the code and save the file.

   MONGODB_URI	Your Atlas cluster connection string if you didn't set the environment variable.
   VOYAGE_API_KEY	Your Voyage AI API key if you didn't set the environment variable.
   <DATABASE-NAME>	Name of the database in your Atlas cluster. For this example, use sample_airbnb.
   <COLLECTION-NAME>	Name of the collection where you ingested the data. For this example, use listingsAndReviews.
   <INDEX-NAME>	Name of the Atlas Vector Search index for the collection.
   <TEXT-FIELD-NAME>	Name of the field that contain the text from which you generated embeddings. For this example, use summary.
   <QUERY-TEXT>	Text for the query. For this example, use ocean view.
   <NUMBER-OF-CANDIDATES-TO-CONSIDER>	Number of nearest neighbors to consider during the search. For this example, use 20.
   <NUMBER-OF-DOCUMENTS-TO-RETURN>	Number of documents to return in the results. For this example, use 5.

4. Compile and run the file using your application run configuration.

   If you are using a terminal, run the following commands to compile and execute your program.

   go run CreateEmbeddingsAndRunQuery.go

   HIDE OUTPUT

   Results from embeddings_float32 embeddings:
   {"_id":"10266175","summary":"A beautiful and comfortable 1 Bedroom Air Conditioned Condo in Makaha Valley - stunning Ocean & Mountain views All the amenities of home, suited for longer stays. Full kitchen & large bathroom. Several gas BBQ's for all guests to use & a large heated pool surrounded by reclining chairs to sunbathe. The Ocean you see in the pictures is not even a mile away, known as the famous Makaha Surfing Beach. Golfing, hiking,snorkeling paddle boarding, surfing are all just minutes from the front door.","score":{"$numberDouble":"0.7278661131858826"}}
   {"summary":"A short distance from Honolulu's billion dollar mall, and the same distance to Waikiki. Parking included. A great location that work perfectly for business, education, or simple visit. Experience Yacht Harbor views and 5 Star Hilton Hawaiian Village.","score":{"$numberDouble":"0.688639760017395"},"_id":"1001265"}
   Results from embeddings_int8 embeddings:
   {"_id":"10266175","summary":"A beautiful and comfortable 1 Bedroom Air Conditioned Condo in Makaha Valley - stunning Ocean & Mountain views All the amenities of home, suited for longer stays. Full kitchen & large bathroom. Several gas BBQ's for all guests to use & a large heated pool surrounded by reclining chairs to sunbathe. The Ocean you see in the pictures is not even a mile away, known as the famous Makaha Surfing Beach. Golfing, hiking,snorkeling paddle boarding, surfing are all just minutes from the front door.","score":{"$numberDouble":"0.5215557217597961"}}
   {"_id":"1001265","summary":"A short distance from Honolulu's billion dollar mall, and the same distance to Waikiki. Parking included. A great location that work perfectly for business, education, or simple visit. Experience Yacht Harbor views and 5 Star Hilton Hawaiian Village.","score":{"$numberDouble":"0.5179016590118408"}}
   Results from embeddings_int1 embeddings:
   {"_id":"10266175","summary":"A beautiful and comfortable 1 Bedroom Air Conditioned Condo in Makaha Valley - stunning Ocean & Mountain views All the amenities of home, suited for longer stays. Full kitchen & large bathroom. Several gas BBQ's for all guests to use & a large heated pool surrounded by reclining chairs to sunbathe. The Ocean you see in the pictures is not even a mile away, known as the famous Makaha Surfing Beach. Golfing, hiking,snorkeling paddle boarding, surfing are all just minutes from the front door.","score":{"$numberDouble":"0.6591796875"}}
   {"_id":"1001265","summary":"A short distance from Honolulu's billion dollar mall, and the same distance to Waikiki. Parking included. A great location that work perfectly for business, education, or simple visit. Experience Yacht Harbor views and 5 Star Hilton Hawaiian Village.","score":{"$numberDouble":"0.6337890625"}}

To learn more about generating embeddings and converting the embeddings to binData vectors, see How to Create Vector Embeddings.

Create your Java project and install dependencies.

1. From your IDE, create a Java project using Maven or Gradle.

2. Add the following dependencies, depending on your package manager:

   Maven

   Gradle

   If you are using Maven, add the following dependencies to the dependencies array in your project's pom.xml file:

    <dependencies>
    <dependency>
    <groupId>junit</groupId>
    <artifactId>junit</artifactId>
    <version>4.13.2</version>
    <scope>test</scope>
    </dependency>
    <dependency>
    <groupId>org.mongodb</groupId>
    <artifactId>mongodb-driver-sync</artifactId>
    <version>5.3.1</version>
    </dependency>
    <dependency>
    <groupId>org.slf4j</groupId>
    <artifactId>slf4j-api</artifactId>
    <version>2.0.16</version>
    </dependency>
    <dependency>
    <groupId>org.slf4j</groupId>
    <artifactId>slf4j-simple</artifactId>
    <version>2.0.16</version>
    <scope>test</scope>
    </dependency>
    <dependency>
    <groupId>org.json</groupId>
    <artifactId>json</artifactId>
    <version>20250517</version>
    </dependency>
    <dependency>
    <groupId>com.squareup.okhttp3</groupId>
    <artifactId>okhttp</artifactId>
    <version>4.12.0</version>
    </dependency>
   </dependencies>

   If you are using Gradle, add the following to the dependencies array in your project's build.gradle file:

   build.gradle

   dependencies {
    // JUnit for testing
    testImplementation 'junit:junit:3.8.1'
    // MongoDB synchronous driver
    implementation 'org.mongodb:mongodb-driver-sync:5.3.1'
    // SLF4J API for logging
    implementation 'org.slf4j:slf4j-api:2.0.16'
    // SLF4J Simple Logger (scope: test)
    testImplementation 'org.slf4j:slf4j-simple:2.0.16'
    // JSON library
    implementation 'org.json:json:20210307'
    // HTTP client for Java
    implementation 'com.squareup.okhttp3:okhttp:4.12.0' // Or the latest version
   }

3. Run your package manager to install the dependencies to your project.

Set your environment variables.

In your IDE, create a new configuration template and add the following variables to your project:

• If you are using IntelliJ IDEA, create a new Application run configuration template, then add your variables as semicolon-separated values in the Environment variables field (for example, FOO=123;BAR=456). Apply the changes and click OK.

  To learn more, see the Create a run/debug configuration from a template section of the IntelliJ IDEA documentation.

• If you are using Eclipse, create a new Java Application launch configuration, then add each variable as a new key-value pair in the Environment tab. Apply the changes and click OK.

  To learn more, see the Creating a Java application launch configuration section of the Eclipse IDE documentation.

VOYAGE_API_KEY=<api-key>
MONGODB_URI=<connection-string>

Update the placeholders with the following values:

• Replace the <api-key> placeholder value with your Voyage AI API key.

• Replace the <connection-string> placeholder value with the SRV connection string for your Atlas cluster.

  Your connection string should use the following format:

  mongodb+srv://<db_username>:<db_password>@<clusterName>.<hostname>.mongodb.net

Generate embeddings from your data.

You can use an embedding model provider to generate float, int8, and int1 embeddings for your data and then use the MongoDB Java driver to convert your native vector embedding to BSON vectors. The following sample code uses Voyage AI's voyage-3-large API to generate full-precision vectors.

1. Create a new file named GenerateAndConvertEmbeddings.java in your Java project.

   touch GenerateAndConvertEmbeddings.java

2. Copy and paste the following code in the GenerateAndConvertEmbeddings.java file.

   This code does the following:

   • Generates the float32, int8, and ubinary vector embeddings by using Voyage AI's voyage-3-large embedding model.

   • Converts the embeddings to BSON binData vectors by using MongoDB Java driver.

   • Creates a file named embeddings.json and saves the data with embeddings in the file to upload to Atlas.

   GenerateAndConvertEmbeddings.java

   1
   import okhttp3.*; 
   2
   import org.bson.BinaryVector; 
   3
   import org.bson.Document; 
   4
   import org.json.JSONArray; 
   5
   import org.json.JSONObject; 
   6
    
   7
   import java.io.FileOutputStream; 
   8
   import java.io.IOException; 
   9
   import java.util.ArrayList; 
   10
   import java.util.List; 
   11
   import java.util.Objects; 
   12
   import java.util.concurrent.TimeUnit; 
   13
    
   14
   public class GenerateAndConvertEmbeddings { 
   15
    // Sample Data
   16
    private static final List<String> DATA = List.of( 
   17
    "The Great Wall of China is visible from space.", 
   18
    "The Eiffel Tower was completed in Paris in 1889.", 
   19
    "Mount Everest is the highest peak on Earth at 8,848m.", 
   20
    "Shakespeare wrote 37 plays and 154 sonnets during his lifetime.", 
   21
    "The Mona Lisa was painted by Leonardo da Vinci." 
   22
    ); 
   23
    
   24
    // Configuration settings
   25
    private static final String VOYAGE_API_URL = "https://api.voyageai.com/v1/embeddings"; 
   26
    private static final int CONNECTION_TIMEOUT = 30; 
   27
    private static final int READ_TIMEOUT = 60; 
   28
    
   29
    public static void main(String[] args) { 
   30
    String apiKey = System.getenv("VOYAGE_API_KEY"); // Replace with your actual API key 
   31
    
   32
    if (Objects.isNull(apiKey) || apiKey.isEmpty()) { 
   33
    throw new RuntimeException("API key not found."); 
   34
    } 
   35
    
   36
    Document bsonEmbeddings = fetchEmbeddings(apiKey); 
   37
    writeToFile(bsonEmbeddings, "embeddings.json"); 
   38
    } 
   39
    
   40
    // Fetch embeddings from Voyage AI API using the given API key
   41
    private static Document fetchEmbeddings(String apiKey) { 
   42
    OkHttpClient client = new OkHttpClient.Builder() 
   43
    .connectTimeout(CONNECTION_TIMEOUT, TimeUnit.SECONDS) 
   44
    .readTimeout(READ_TIMEOUT, TimeUnit.SECONDS) 
   45
    .build(); 
   46
    
   47
    List<List<List<Integer>>> embeddingsByOutputType = new ArrayList<>(); 
   48
    List<String> outputDtypes = List.of("float", "int8", "ubinary"); 
   49
    
   50
    try { 
   51
    for (String dtype : outputDtypes) { 
   52
    String responseBody = sendRequest(client, apiKey, dtype); 
   53
    embeddingsByOutputType.add(parseEmbeddings(responseBody, dtype)); 
   54
    } 
   55
    } catch (IOException e) { 
   56
    throw new RuntimeException("Error fetching embeddings: " + e.getMessage(), e); 
   57
    } 
   58
    
   59
    return convertEmbeddingsToBson(embeddingsByOutputType); 
   60
    } 
   61
    
   62
    // Send API request to Voyage AI
   63
    private static String sendRequest(OkHttpClient client, String apiKey, String outputDtype) throws IOException { 
   64
    String requestBody = new JSONObject() 
   65
    .put("input", DATA) 
   66
    .put("model", "voyage-3-large") 
   67
    .put("input_type", "document") 
   68
    .put("output_dtype", outputDtype) 
   69
    .put("output_dimension", 1024) 
   70
    .toString(); 
   71
    
   72
    Request request = new Request.Builder() 
   73
    .url(VOYAGE_API_URL) 
   74
    .post(RequestBody.create(requestBody, MediaType.get("application/json"))) 
   75
    .addHeader("Authorization", "Bearer " + apiKey) 
   76
    .build(); 
   77
    
   78
    try (Response response = client.newCall(request).execute()) { 
   79
    if (!response.isSuccessful()) { 
   80
    throw new IOException("API error: HTTP " + response.code()); 
   81
    } 
   82
    return response.body().string(); 
   83
    } 
   84
    } 
   85
    
   86
    // Parse embeddings from Voyage AI API response
   87
    private static List<List<Integer>> parseEmbeddings(String responseBody, String outputDtype) { 
   88
    JSONObject responseJson = new JSONObject(responseBody); 
   89
    JSONArray dataArray = responseJson.optJSONArray("data"); 
   90
    
   91
    if (dataArray == null) { 
   92
    throw new RuntimeException("Invalid response format: 'data' field missing."); 
   93
    } 
   94
    
   95
    List<List<Integer>> embeddings = new ArrayList<>(); 
   96
    for (int i = 0; i < dataArray.length(); i++) { 
   97
    JSONArray embeddingVector = dataArray.getJSONObject(i).getJSONArray("embedding"); 
   98
    
   99
    List<Integer> vector = new ArrayList<>(); 
   100
    for (int j = 0; j < embeddingVector.length(); j++) { 
   101
    int value = embeddingVector.getInt(j); 
   102
    
   103
    // Handle binary quantization offset 
   104
    if ("binary".equals(outputDtype)) { 
   105
    value = value - 128; // Offset binary method (signed int8 representation) 
   106
    } 
   107
    
   108
    vector.add(value); 
   109
    } 
   110
    embeddings.add(vector); 
   111
    } 
   112
    return embeddings; 
   113
    } 
   114
    
   115
    // Convert fetched embeddings into BSON format
   116
    private static Document convertEmbeddingsToBson(List<List<List<Integer>>> embeddingsByOutputType) { 
   117
    List<Document> bsonEmbeddings = new ArrayList<>(); 
   118
    for (int i = 0; i < DATA.size(); i++) { 
   119
    Document embedding = new Document() 
   120
    .append("text", DATA.get(i)) 
   121
    .append("embeddings_float32", BinaryVector.floatVector(listToFloatArray(embeddingsByOutputType.get(0).get(i)))) 
   122
    .append("embeddings_int8", BinaryVector.int8Vector(listToByteArray(embeddingsByOutputType.get(1).get(i)))) // Binary embeddings 
   123
    .append("embeddings_int1", BinaryVector.packedBitVector(listToByteArray(embeddingsByOutputType.get(2).get(i)), (byte) 0)); // Ubinary embeddings 
   124
    bsonEmbeddings.add(embedding); 
   125
    } 
   126
    return new Document("data", bsonEmbeddings); 
   127
    } 
   128
    
   129
    // Save BSON embeddings to a JSON file
   130
    private static void writeToFile(Document bsonEmbeddings, String fileName) { 
   131
    try (FileOutputStream fos = new FileOutputStream(fileName)) { 
   132
    fos.write(bsonEmbeddings.toJson().getBytes()); 
   133
    System.out.println("Embeddings saved to " + fileName); 
   134
    } catch (IOException e) { 
   135
    throw new RuntimeException("Error saving file: " + e.getMessage(), e); 
   136
    } 
   137
    } 
   138
    
   139
    private static float[] listToFloatArray(List<Integer> list) { 
   140
    float[] array = new float[list.size()]; 
   141
    for (int i = 0; i < list.size(); i++) { 
   142
    array[i] = list.get(i).floatValue(); 
   143
    } 
   144
    return array; 
   145
    } 
   146
    
   147
    private static byte[] listToByteArray(List<Integer> list) { 
   148
    byte[] array = new byte[list.size()]; 
   149
    for (int i = 0; i < list.size(); i++) { 
   150
    array[i] = list.get(i).byteValue(); 
   151
    } 
   152
    return array; 
   153
    } 
   154
   } 

3. Compile and run the file using your application run configuration.

   If you are using a terminal, run the following commands to compile and execute your program.

   javac GenerateAndConvertEmbeddings.java
   java GenerateAndConvertEmbeddings

   HIDE OUTPUT

   Embeddings saved to embeddings.json

4. Verify the embeddings in the embeddings.json file.