Image understanding

 from google.genai import types with open('path/to/small-sample.jpg', 'rb') as f: image_bytes = f.read() response = client.models.generate_content( model='gemini-2.5-flash', contents=[ types.Part.from_bytes( data=image_bytes, mime_type='image/jpeg', ), 'Caption this image.' ] ) print(response.text) 

import { GoogleGenAI } from "@google/genai"; import * as fs from "node:fs"; const ai = new GoogleGenAI({}); const base64ImageFile = fs.readFileSync("path/to/small-sample.jpg", {  encoding: "base64", }); const contents = [  {  inlineData: {  mimeType: "image/jpeg",  data: base64ImageFile,  },  },  { text: "Caption this image." }, ]; const response = await ai.models.generateContent({  model: "gemini-2.5-flash",  contents: contents, }); console.log(response.text); 

bytes, _ := os.ReadFile("path/to/small-sample.jpg") parts := []*genai.Part{  genai.NewPartFromBytes(bytes, "image/jpeg"),  genai.NewPartFromText("Caption this image."), } contents := []*genai.Content{  genai.NewContentFromParts(parts, genai.RoleUser), } result, _ := client.Models.GenerateContent(  ctx,  "gemini-2.5-flash",  contents,  nil, ) fmt.Println(result.Text()) 

IMG_PATH="/path/to/your/image1.jpg" if [[ "$(base64 --version 2>&1)" = *"FreeBSD"* ]]; then B64FLAGS="--input" else B64FLAGS="-w0" fi curl "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent" \ -H "x-goog-api-key: $GEMINI_API_KEY" \ -H 'Content-Type: application/json' \ -X POST \ -d '{  "contents": [{  "parts":[  {  "inline_data": {  "mime_type":"image/jpeg",  "data": "'"$(base64 $B64FLAGS $IMG_PATH)"'"  }  },  {"text": "Caption this image."},  ]  }] }' 2> /dev/null 

from google import genai from google.genai import types import requests image_path = "https://goo.gle/instrument-img" image_bytes = requests.get(image_path).content image = types.Part.from_bytes( data=image_bytes, mime_type="image/jpeg" ) client = genai.Client() response = client.models.generate_content( model="gemini-2.5-flash", contents=["What is this image?", image], ) print(response.text) 

import { GoogleGenAI } from "@google/genai"; async function main() {  const ai = new GoogleGenAI({});  const imageUrl = "https://goo.gle/instrument-img";  const response = await fetch(imageUrl);  const imageArrayBuffer = await response.arrayBuffer();  const base64ImageData = Buffer.from(imageArrayBuffer).toString('base64');  const result = await ai.models.generateContent({  model: "gemini-2.5-flash",  contents: [  {  inlineData: {  mimeType: 'image/jpeg',  data: base64ImageData,  },  },  { text: "Caption this image." }  ],  });  console.log(result.text); } main(); 

package main import (  "context"  "fmt"  "os"  "io"  "net/http"  "google.golang.org/genai" ) func main() {  ctx := context.Background()  client, err := genai.NewClient(ctx, nil)  if err != nil {  log.Fatal(err)  }  // Download the image.  imageResp, _ := http.Get("https://goo.gle/instrument-img")  imageBytes, _ := io.ReadAll(imageResp.Body)  parts := []*genai.Part{  genai.NewPartFromBytes(imageBytes, "image/jpeg"),  genai.NewPartFromText("Caption this image."),  }  contents := []*genai.Content{  genai.NewContentFromParts(parts, genai.RoleUser),  }  result, _ := client.Models.GenerateContent(  ctx,  "gemini-2.5-flash",  contents,  nil,  )  fmt.Println(result.Text()) } 

IMG_URL="https://goo.gle/instrument-img" MIME_TYPE=$(curl -sIL "$IMG_URL" | grep -i '^content-type:' | awk -F ': ' '{print $2}' | sed 's/\r$//' | head -n 1) if [[ -z "$MIME_TYPE" || ! "$MIME_TYPE" == image/* ]]; then  MIME_TYPE="image/jpeg" fi # Check for macOS if [[ "$(uname)" == "Darwin" ]]; then  IMAGE_B64=$(curl -sL "$IMG_URL" | base64 -b 0) elif [[ "$(base64 --version 2>&1)" = *"FreeBSD"* ]]; then  IMAGE_B64=$(curl -sL "$IMG_URL" | base64) else  IMAGE_B64=$(curl -sL "$IMG_URL" | base64 -w0) fi curl "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent" \  -H "x-goog-api-key: $GEMINI_API_KEY" \  -H 'Content-Type: application/json' \  -X POST \  -d '{  "contents": [{  "parts":[  {  "inline_data": {  "mime_type":"'"$MIME_TYPE"'",  "data": "'"$IMAGE_B64"'"  }  },  {"text": "Caption this image."}  ]  }]  }' 2> /dev/null 

from google import genai client = genai.Client() my_file = client.files.upload(file="path/to/sample.jpg") response = client.models.generate_content( model="gemini-2.5-flash", contents=[my_file, "Caption this image."], ) print(response.text) 

import {  GoogleGenAI,  createUserContent,  createPartFromUri, } from "@google/genai"; const ai = new GoogleGenAI({}); async function main() {  const myfile = await ai.files.upload({  file: "path/to/sample.jpg",  config: { mimeType: "image/jpeg" },  });  const response = await ai.models.generateContent({  model: "gemini-2.5-flash",  contents: createUserContent([  createPartFromUri(myfile.uri, myfile.mimeType),  "Caption this image.",  ]),  });  console.log(response.text); } await main(); 

package main import (  "context"  "fmt"  "os"  "google.golang.org/genai" ) func main() {  ctx := context.Background()  client, err := genai.NewClient(ctx, nil)  if err != nil {  log.Fatal(err)  }  uploadedFile, _ := client.Files.UploadFromPath(ctx, "path/to/sample.jpg", nil)  parts := []*genai.Part{  genai.NewPartFromText("Caption this image."),  genai.NewPartFromURI(uploadedFile.URI, uploadedFile.MIMEType),  }  contents := []*genai.Content{  genai.NewContentFromParts(parts, genai.RoleUser),  }  result, _ := client.Models.GenerateContent(  ctx,  "gemini-2.5-flash",  contents,  nil,  )  fmt.Println(result.Text()) } 

IMAGE_PATH="path/to/sample.jpg" MIME_TYPE=$(file -b --mime-type "${IMAGE_PATH}") NUM_BYTES=$(wc -c < "${IMAGE_PATH}") DISPLAY_NAME=IMAGE tmp_header_file=upload-header.tmp # Initial resumable request defining metadata. # The upload url is in the response headers dump them to a file. curl "https://generativelanguage.googleapis.com/upload/v1beta/files" \  -H "x-goog-api-key: $GEMINI_API_KEY" \  -D upload-header.tmp \  -H "X-Goog-Upload-Protocol: resumable" \  -H "X-Goog-Upload-Command: start" \  -H "X-Goog-Upload-Header-Content-Length: ${NUM_BYTES}" \  -H "X-Goog-Upload-Header-Content-Type: ${MIME_TYPE}" \  -H "Content-Type: application/json" \  -d "{'file': {'display_name': '${DISPLAY_NAME}'}}" 2> /dev/null upload_url=$(grep -i "x-goog-upload-url: " "${tmp_header_file}" | cut -d" " -f2 | tr -d "\r") rm "${tmp_header_file}" # Upload the actual bytes. curl "${upload_url}" \  -H "x-goog-api-key: $GEMINI_API_KEY" \  -H "Content-Length: ${NUM_BYTES}" \  -H "X-Goog-Upload-Offset: 0" \  -H "X-Goog-Upload-Command: upload, finalize" \  --data-binary "@${IMAGE_PATH}" 2> /dev/null > file_info.json file_uri=$(jq -r ".file.uri" file_info.json) echo file_uri=$file_uri # Now generate content using that file curl "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent" \  -H "x-goog-api-key: $GEMINI_API_KEY" \  -H 'Content-Type: application/json' \  -X POST \  -d '{  "contents": [{  "parts":[  {"file_data":{"mime_type": "'"${MIME_TYPE}"'", "file_uri": "'"${file_uri}"'"}},  {"text": "Caption this image."}]  }]  }' 2> /dev/null > response.json cat response.json echo jq ".candidates[].content.parts[].text" response.json 

from google import genai from google.genai import types client = genai.Client() # Upload the first image image1_path = "path/to/image1.jpg" uploaded_file = client.files.upload(file=image1_path) # Prepare the second image as inline data image2_path = "path/to/image2.png" with open(image2_path, 'rb') as f: img2_bytes = f.read() # Create the prompt with text and multiple images response = client.models.generate_content( model="gemini-2.5-flash", contents=[ "What is different between these two images?", uploaded_file, # Use the uploaded file reference types.Part.from_bytes( data=img2_bytes, mime_type='image/png' ) ] ) print(response.text) 

import {  GoogleGenAI,  createUserContent,  createPartFromUri, } from "@google/genai"; import * as fs from "node:fs"; const ai = new GoogleGenAI({}); async function main() {  // Upload the first image  const image1_path = "path/to/image1.jpg";  const uploadedFile = await ai.files.upload({  file: image1_path,  config: { mimeType: "image/jpeg" },  });  // Prepare the second image as inline data  const image2_path = "path/to/image2.png";  const base64Image2File = fs.readFileSync(image2_path, {  encoding: "base64",  });  // Create the prompt with text and multiple images  const response = await ai.models.generateContent({  model: "gemini-2.5-flash",  contents: createUserContent([  "What is different between these two images?",  createPartFromUri(uploadedFile.uri, uploadedFile.mimeType),  {  inlineData: {  mimeType: "image/png",  data: base64Image2File,  },  },  ]),  });  console.log(response.text); } await main(); 

// Upload the first image image1Path := "path/to/image1.jpg" uploadedFile, _ := client.Files.UploadFromPath(ctx, image1Path, nil) // Prepare the second image as inline data image2Path := "path/to/image2.jpeg" imgBytes, _ := os.ReadFile(image2Path) parts := []*genai.Part{  genai.NewPartFromText("What is different between these two images?"),  genai.NewPartFromBytes(imgBytes, "image/jpeg"),  genai.NewPartFromURI(uploadedFile.URI, uploadedFile.MIMEType), } contents := []*genai.Content{  genai.NewContentFromParts(parts, genai.RoleUser), } result, _ := client.Models.GenerateContent(  ctx,  "gemini-2.5-flash",  contents,  nil, ) fmt.Println(result.Text()) 

# Upload the first image IMAGE1_PATH="path/to/image1.jpg" MIME1_TYPE=$(file -b --mime-type "${IMAGE1_PATH}") NUM1_BYTES=$(wc -c < "${IMAGE1_PATH}") DISPLAY_NAME1=IMAGE1 tmp_header_file1=upload-header1.tmp curl "https://generativelanguage.googleapis.com/upload/v1beta/files" \  -H "x-goog-api-key: $GEMINI_API_KEY" \  -D upload-header1.tmp \  -H "X-Goog-Upload-Protocol: resumable" \  -H "X-Goog-Upload-Command: start" \  -H "X-Goog-Upload-Header-Content-Length: ${NUM1_BYTES}" \  -H "X-Goog-Upload-Header-Content-Type: ${MIME1_TYPE}" \  -H "Content-Type: application/json" \  -d "{'file': {'display_name': '${DISPLAY_NAME1}'}}" 2> /dev/null upload_url1=$(grep -i "x-goog-upload-url: " "${tmp_header_file1}" | cut -d" " -f2 | tr -d "\r") rm "${tmp_header_file1}" curl "${upload_url1}" \  -H "Content-Length: ${NUM1_BYTES}" \  -H "X-Goog-Upload-Offset: 0" \  -H "X-Goog-Upload-Command: upload, finalize" \  --data-binary "@${IMAGE1_PATH}" 2> /dev/null > file_info1.json file1_uri=$(jq ".file.uri" file_info1.json) echo file1_uri=$file1_uri # Prepare the second image (inline) IMAGE2_PATH="path/to/image2.png" MIME2_TYPE=$(file -b --mime-type "${IMAGE2_PATH}") if [[ "$(base64 --version 2>&1)" = *"FreeBSD"* ]]; then  B64FLAGS="--input" else  B64FLAGS="-w0" fi IMAGE2_BASE64=$(base64 $B64FLAGS $IMAGE2_PATH) # Now generate content using both images curl "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent" \  -H "x-goog-api-key: $GEMINI_API_KEY" \  -H 'Content-Type: application/json' \  -X POST \  -d '{  "contents": [{  "parts":[  {"text": "What is different between these two images?"},  {"file_data":{"mime_type": "'"${MIME1_TYPE}"'", "file_uri": '$file1_uri'}},  {  "inline_data": {  "mime_type":"'"${MIME2_TYPE}"'",  "data": "'"$IMAGE2_BASE64"'"  }  }  ]  }]  }' 2> /dev/null > response.json cat response.json echo jq ".candidates[].content.parts[].text" response.json 

from google import genai from google.genai import types from PIL import Image import json client = genai.Client() prompt = "Detect the all of the prominent items in the image. The box_2d should be [ymin, xmin, ymax, xmax] normalized to 0-1000." image = Image.open("/path/to/image.png") config = types.GenerateContentConfig( response_mime_type="application/json" ) response = client.models.generate_content(model="gemini-2.5-flash", contents=[image, prompt], config=config ) width, height = image.size bounding_boxes = json.loads(response.text) converted_bounding_boxes = [] for bounding_box in bounding_boxes: abs_y1 = int(bounding_box["box_2d"][0]/1000 * height) abs_x1 = int(bounding_box["box_2d"][1]/1000 * width) abs_y2 = int(bounding_box["box_2d"][2]/1000 * height) abs_x2 = int(bounding_box["box_2d"][3]/1000 * width) converted_bounding_boxes.append([abs_x1, abs_y1, abs_x2, abs_y2]) print("Image size: ", width, height) print("Bounding boxes:", converted_bounding_boxes) 

from google import genai from google.genai import types from PIL import Image, ImageDraw import io import base64 import json import numpy as np import os client = genai.Client() def parse_json(json_output: str): # Parsing out the markdown fencing lines = json_output.splitlines() for i, line in enumerate(lines): if line == "```json": json_output = "\n".join(lines[i+1:]) # Remove everything before "```json" json_output = json_output.split("```")[0] # Remove everything after the closing "```" break # Exit the loop once "```json" is found return json_output def extract_segmentation_masks(image_path: str, output_dir: str = "segmentation_outputs"): # Load and resize image im = Image.open(image_path) im.thumbnail([1024, 1024], Image.Resampling.LANCZOS) prompt = """  Give the segmentation masks for the wooden and glass items.  Output a JSON list of segmentation masks where each entry contains the 2D  bounding box in the key "box_2d", the segmentation mask in key "mask", and  the text label in the key "label". Use descriptive labels.  """ config = types.GenerateContentConfig( thinking_config=types.ThinkingConfig(thinking_budget=0) # set thinking_budget to 0 for better results in object detection ) response = client.models.generate_content( model="gemini-2.5-flash", contents=[prompt, im], # Pillow images can be directly passed as inputs (which will be converted by the SDK) config=config ) # Parse JSON response items = json.loads(parse_json(response.text)) # Create output directory os.makedirs(output_dir, exist_ok=True) # Process each mask for i, item in enumerate(items): # Get bounding box coordinates box = item["box_2d"] y0 = int(box[0] / 1000 * im.size[1]) x0 = int(box[1] / 1000 * im.size[0]) y1 = int(box[2] / 1000 * im.size[1]) x1 = int(box[3] / 1000 * im.size[0]) # Skip invalid boxes if y0 >= y1 or x0 >= x1: continue # Process mask png_str = item["mask"] if not png_str.startswith("data:image/png;base64,"): continue # Remove prefix png_str = png_str.removeprefix("data:image/png;base64,") mask_data = base64.b64decode(png_str) mask = Image.open(io.BytesIO(mask_data)) # Resize mask to match bounding box mask = mask.resize((x1 - x0, y1 - y0), Image.Resampling.BILINEAR) # Convert mask to numpy array for processing mask_array = np.array(mask) # Create overlay for this mask overlay = Image.new('RGBA', im.size, (0, 0, 0, 0)) overlay_draw = ImageDraw.Draw(overlay) # Create overlay for the mask color = (255, 255, 255, 200) for y in range(y0, y1): for x in range(x0, x1): if mask_array[y - y0, x - x0] > 128: # Threshold for mask overlay_draw.point((x, y), fill=color) # Save individual mask and its overlay mask_filename = f"{item['label']}_{i}_mask.png" overlay_filename = f"{item['label']}_{i}_overlay.png" mask.save(os.path.join(output_dir, mask_filename)) # Create and save overlay composite = Image.alpha_composite(im.convert('RGBA'), overlay) composite.save(os.path.join(output_dir, overlay_filename)) print(f"Saved mask and overlay for {item['label']} to {output_dir}") # Example usage if __name__ == "__main__": extract_segmentation_masks("path/to/image.png")