wip

Author: khromov

lib/pricing.test.ts

@@ -64,19 +64,20 @@ describe("extractPricingFromGatewayModel", () => {
  expect(pricing).toBeNull();
  });
 
- it("should return null for model with empty pricing object", () => {
+ it("should throw error for model with empty pricing object", () => {
  const model: GatewayModel = {
  id: "local/model",
  name: "Local Model",
  pricing: {},
  modelType: "language",
  };
 
- const pricing = extractPricingFromGatewayModel(model);
- expect(pricing).toBeNull();
+ expect(() => extractPricingFromGatewayModel(model)).toThrowError(
+ /Invalid pricing/,
+ );
  });
 
- it("should handle invalid pricing values gracefully", () => {
+ it("should throw error for invalid pricing values", () => {
  const model: GatewayModel = {
  id: "test/model",
  name: "Test Model",
@@ -87,11 +88,9 @@ describe("extractPricingFromGatewayModel", () => {
  modelType: "language",
  };
 
- const pricing = extractPricingFromGatewayModel(model);
-
- expect(pricing).not.toBeNull();
- expect(pricing!.inputCostPerToken).toBe(0);
- expect(pricing!.outputCostPerToken).toBe(0.000015);
+ expect(() => extractPricingFromGatewayModel(model)).toThrowError(
+ /Invalid pricing/,
+ );
  });
 });
 

lib/pricing.ts

@@ -55,16 +55,18 @@ export function extractPricingFromGatewayModel(
 
  const { pricing } = model;
 
- const inputCost = pricing.input ? parseFloat(pricing.input) : 0;
- const outputCost = pricing.output ? parseFloat(pricing.output) : 0;
+ const inputCost = pricing.input ? parseFloat(pricing.input) : NaN;
+ const outputCost = pricing.output ? parseFloat(pricing.output) : NaN;
 
- if ((inputCost === 0 || isNaN(inputCost)) && (outputCost === 0 || isNaN(outputCost))) {
- return null;
+ if (isNaN(inputCost) || isNaN(outputCost)) {
+ throw new Error(
+ `Invalid pricing for model ${model.id}: input and output pricing must be valid numbers.`,
+ );
  }
 
  const result: ModelPricing = {
- inputCostPerToken: isNaN(inputCost) ? 0 : inputCost,
- outputCostPerToken: isNaN(outputCost) ? 0 : outputCost,
+ inputCostPerToken: inputCost,
+ outputCostPerToken: outputCost,
  };
 
  if (pricing.cachedInputTokens) {

lib/utils.test.ts

@@ -1,5 +1,11 @@
 import { describe, it, expect } from "vitest";
-import { sanitizeModelName, getTimestampedFilename } from "./utils.ts";
+import {
+ sanitizeModelName,
+ getTimestampedFilename,
+ calculateTotalCost,
+} from "./utils.ts";
+import type { ModelPricing } from "./pricing.ts";
+import type { SingleTestResult } from "./report.ts";
 
 describe("sanitizeModelName", () => {
  it("replaces slashes with dashes", () => {
@@ -34,12 +40,22 @@ describe("getTimestampedFilename", () => {
  const fixedDate = new Date("2025-12-12T14:30:45Z");
 
  it("generates filename without model name", () => {
- const result = getTimestampedFilename("result", "json", undefined, fixedDate);
+ const result = getTimestampedFilename(
+ "result",
+ "json",
+ undefined,
+ fixedDate,
+ );
  expect(result).toBe("result-2025-12-12-14-30-45.json");
  });
 
  it("generates filename with simple model name", () => {
- const result = getTimestampedFilename("result", "json", "gpt-4o", fixedDate);
+ const result = getTimestampedFilename(
+ "result",
+ "json",
+ "gpt-4o",
+ fixedDate,
+ );
  expect(result).toBe("result-2025-12-12-14-30-45-gpt-4o.json");
  });
 
@@ -50,7 +66,9 @@ describe("getTimestampedFilename", () => {
  "anthropic/claude-sonnet-4",
  fixedDate,
  );
- expect(result).toBe("result-2025-12-12-14-30-45-anthropic-claude-sonnet-4.json");
+ expect(result).toBe(
+ "result-2025-12-12-14-30-45-anthropic-claude-sonnet-4.json",
+ );
  });
 
  it("generates filename with model name containing special characters", () => {
@@ -64,13 +82,111 @@ describe("getTimestampedFilename", () => {
  });
 
  it("handles different file extensions", () => {
- const result = getTimestampedFilename("output", "txt", "test-model", fixedDate);
+ const result = getTimestampedFilename(
+ "output",
+ "txt",
+ "test-model",
+ fixedDate,
+ );
  expect(result).toBe("output-2025-12-12-14-30-45-test-model.txt");
  });
 
  it("pads single-digit months and days", () => {
  const earlyDate = new Date("2025-01-05T08:09:07Z");
- const result = getTimestampedFilename("result", "json", undefined, earlyDate);
+ const result = getTimestampedFilename(
+ "result",
+ "json",
+ undefined,
+ earlyDate,
+ );
  expect(result).toBe("result-2025-01-05-08-09-07.json");
  });
 });
+
+describe("calculateTotalCost", () => {
+ const pricing: ModelPricing = {
+ inputCostPerToken: 1.0 / 1_000_000,
+ outputCostPerToken: 2.0 / 1_000_000,
+ cacheReadInputTokenCost: 0.1 / 1_000_000,
+ };
+
+ it("calculates zero cost for empty results", () => {
+ const tests: SingleTestResult[] = [];
+ const result = calculateTotalCost(tests, pricing);
+
+ expect(result).toEqual({
+ inputCost: 0,
+ outputCost: 0,
+ cacheReadCost: 0,
+ totalCost: 0,
+ inputTokens: 0,
+ outputTokens: 0,
+ cachedInputTokens: 0,
+ });
+ });
+
+ it("aggregates usage from multiple steps and tests", () => {
+ const tests: SingleTestResult[] = [
+ {
+ testName: "test1",
+ prompt: "p1",
+ resultWriteContent: null,
+ verification: {} as any,
+ steps: [
+ {
+ usage: {
+ inputTokens: 100,
+ outputTokens: 50,
+ cachedInputTokens: 10,
+ },
+ } as any,
+ {
+ usage: {
+ inputTokens: 200,
+ outputTokens: 100,
+ cachedInputTokens: 0,
+ },
+ } as any,
+ ],
+ },
+ {
+ testName: "test2",
+ prompt: "p2",
+ resultWriteContent: null,
+ verification: {} as any,
+ steps: [
+ {
+ usage: {
+ inputTokens: 300,
+ outputTokens: 150,
+ cachedInputTokens: 20,
+ },
+ } as any,
+ ],
+ },
+ ];
+
+ // Total Input: 100 + 200 + 300 = 600
+ // Total Output: 50 + 100 + 150 = 300
+ // Total Cached: 10 + 0 + 20 = 30
+ // Uncached Input: 600 - 30 = 570
+
+ // Costs (per Token):
+ // Input: 570 * (1.0 / 1e6) = 0.00057
+ // Output: 300 * (2.0 / 1e6) = 0.0006
+ // Cache: 30 * (0.1 / 1e6) = 0.000003
+ // Total: 0.00057 + 0.0006 + 0.000003 = 0.001173
+
+ const result = calculateTotalCost(tests, pricing);
+
+ expect(result).toEqual({
+ inputCost: 0.00057,
+ outputCost: 0.0006,
+ cacheReadCost: 0.000003,
+ totalCost: 0.001173,
+ inputTokens: 600,
+ outputTokens: 300,
+ cachedInputTokens: 30,
+ });
+ });
+});