audio evals

pai-audio-evals/assets.json

@@ -0,0 +1,147 @@
+[
+{"file":"common_voice_en_537507.mp3","text":"I was impatient to see it opened."},
+{"file":"common_voice_en_19973905.mp3","text":"Lombardi was born in Frugarolo, a small town in Piedmont."},
+{"file":"common_voice_en_103455.mp3","text":"I’ll join you in a minute after I lock my bike."},
+{"file":"common_voice_en_181324.mp3","text":"I'm just a fool."},
+{"file":"common_voice_en_40702.mp3","text":"Then he turned to the boy."},
+{"file":"common_voice_en_22060076.mp3","text":"two"},
+{"file":"common_voice_en_123785.mp3","text":"About eleven, I walked back to my home in Maybury because nothing seemed to be happening."},
+{"file":"common_voice_en_561741.mp3","text":"You get somebody else to wash the diapers."},
+{"file":"common_voice_en_23345629.mp3","text":"Moretti helped Sinatra get bookings in New Jersey clubs in return for kickbacks."},
+{"file":"common_voice_en_19690063.mp3","text":"Originally named the \"School of Commerce and Domestic Science\"."},
+{"file":"common_voice_en_22155967.mp3","text":"yes"},
+{"file":"common_voice_en_20507311.mp3","text":"Reid was born in Toronto, Ontario."},
+{"file":"common_voice_en_22221938.mp3","text":"one"},
+{"file":"common_voice_en_25205829.mp3","text":"A tender will then be spent for its dismantling."},
+{"file":"common_voice_en_20294044.mp3","text":"Regional squads can be either land or marine based."},
+{"file":"common_voice_en_18800368.mp3","text":"The crater Runge on the Moon is named after him."},
+{"file":"common_voice_en_19674497.mp3","text":"Each of the regional affiliates also holds regular meetings."},
+{"file":"common_voice_en_29485619.mp3","text":"Hareton, if I gave you a book, would you take it now?"},
+{"file":"common_voice_en_17398416.mp3","text":"And what became of him?"},
+{"file":"common_voice_en_23856939.mp3","text":"His new band already featured John McVie."},
+{"file":"common_voice_en_22770994.mp3","text":"Any union would require voter approval."},
+{"file":"common_voice_en_20312221.mp3","text":"Some had approached the size of a rhinoceros, such as \"Inostrancevia\", the largest gorgonopsian."},
+{"file":"common_voice_en_17882.mp3","text":"\"Well, why don't you go to Mecca now?\" asked the boy."},
+{"file":"common_voice_en_17663447.mp3","text":"The life of that bungee jumper dangles on a string, in the truest sense of the word."},
+{"file":"common_voice_en_25109715.mp3","text":"She has been part of the singing group Goddess Family Club."},
+{"file":"common_voice_en_602262.mp3","text":"The article addresses image classification with local image descriptors."},
+{"file":"common_voice_en_11896167.mp3","text":"Well, what of that?"},
+{"file":"common_voice_en_18268160.mp3","text":"Beyond the foliage."},
+{"file":"common_voice_en_16544.mp3","text":"\"Men dream more about coming home than about leaving,\" the boy said."},
+{"file":"common_voice_en_23945951.mp3","text":"From there on all migrant transport was handled by airplanes."},
+{"file":"common_voice_en_19740040.mp3","text":"She later performed on many television programs and commercials in Toronto and Los Angeles."},
+{"file":"common_voice_en_570894.mp3","text":"The medical industry saves lives."},
+{"file":"common_voice_en_1142593.mp3","text":"That's a fair amount of loot."},
+{"file":"common_voice_en_27091137.mp3","text":"Toqua's Mississippian inhabitants constructed a platform mound overlooking a central plaza."},
+{"file":"common_voice_en_18472363.mp3","text":"The actors spent months submerged in freezing cold water."},
+{"file":"common_voice_en_30249659.mp3","text":"The city of Awan still has not been located archaeologically."},
+{"file":"common_voice_en_18851641.mp3","text":"Passenger service was never very profitable."},
+{"file":"common_voice_en_23362296.mp3","text":"Google has also installed replicas of SpaceShipOne and a dinosaur skeleton."},
+{"file":"common_voice_en_663688.mp3","text":"I have the diet of a kid who found twenty dollars."},
+{"file":"common_voice_en_19561022.mp3","text":"He continued to run sporadic events over the years but never again contended."},
+{"file":"common_voice_en_132850.mp3","text":"The immense collection of people and animals began to travel faster."},
+{"file":"common_voice_en_21716199.mp3","text":"LaFortune's grandfather, Joseph LaFortune, was an oil executive and a noted philanthropist in Tulsa."},
+{"file":"common_voice_en_33390468.mp3","text":"Approval requires majority."},
+{"file":"common_voice_en_23954684.mp3","text":"Harrelson worked as an encyclopedia salesman in California and as a professional gambler."},
+{"file":"common_voice_en_17387536.mp3","text":"Also, will numbers be written as digits or as words?"},
+{"file":"common_voice_en_519822.mp3","text":"Every one is king in his own house."},
+{"file":"common_voice_en_31398895.mp3","text":"She attended and became an obstetrician and gynecologist."},
+{"file":"common_voice_en_189778.mp3","text":"Did you get it?"},
+{"file":"common_voice_en_174805.mp3","text":"There was no empathy for her now."},
+{"file":"common_voice_en_191137.mp3","text":"When he reached the top of the dune, his heart leapt."},
+{"file":"common_voice_en_18557861.mp3","text":"It is, then, something belonging to her."},
+{"file":"common_voice_en_23352355.mp3","text":"The system handles inmates who have violated, or are accused of violating, federal law."},
+{"file":"common_voice_en_19794577.mp3","text":"The county seat is Savannah."},
+{"file":"common_voice_en_26762867.mp3","text":"This recording was made at Soissons Cathedral with its Gonzales organ."},
+{"file":"common_voice_en_28045285.mp3","text":"It may be the origin of the popular Hispanic surname Zambrano."},
+{"file":"common_voice_en_102993.mp3","text":"How do you think?"},
+{"file":"common_voice_en_530268.mp3","text":"If you're looking for a room there's no vacancy."},
+{"file":"common_voice_en_18688505.mp3","text":"The Budd, Nukusemanu, and Heemskercq Reefs form part of the group."},
+{"file":"common_voice_en_83749.mp3","text":"The coronal mass ejection from the star masked the signal."},
+{"file":"common_voice_en_285844.mp3","text":"If the cap fits, wear it."},
+{"file":"common_voice_en_17248435.mp3","text":"His dessert smelled musty, not sweet."},
+{"file":"common_voice_en_20489528.mp3","text":"Rachmastrivka is one of the larger Hasidic groups."},
+{"file":"common_voice_en_17279830.mp3","text":"I saw Louis the other day, he was super concerned about you."},
+{"file":"common_voice_en_690700.mp3","text":"It's practically up there now."},
+{"file":"common_voice_en_18388080.mp3","text":"Elena will be late."},
+{"file":"common_voice_en_18592283.mp3","text":"The pay will be commensurate with the duties."},
+{"file":"common_voice_en_27910531.mp3","text":"He had two sons and a daughter by Greenwood."},
+{"file":"common_voice_en_25947210.mp3","text":"British people are one of the biggest immigrant groups in many cities."},
+{"file":"common_voice_en_472729.mp3","text":"The lamp shone with a steady green flame."},
+{"file":"common_voice_en_38861.mp3","text":"Lindsey promised to call you from New York tonight."},
+{"file":"common_voice_en_17739749.mp3","text":"Better to wear out than rust out."},
+{"file":"common_voice_en_541491.mp3","text":"Meanwhile, the old man persisted in his attempt to strike up a conversation."},
+{"file":"common_voice_en_21759856.mp3","text":"He was father to the actor Pierre Lindstedt."},
+{"file":"common_voice_en_15904858.mp3","text":"Natalie gave me a pleasing massage, when we were in Luxembourg last week."},
+{"file":"common_voice_en_27003455.mp3","text":"This allowed it to operate freight services."},
+{"file":"common_voice_en_23944497.mp3","text":"Credits for \"Share My World\" adapted from Allmusic."},
+{"file":"common_voice_en_20026882.mp3","text":"The Coquitlam River originates at Disappointment Lake in the Coast Mountains near Indian Arm."},
+{"file":"common_voice_en_26487.mp3","text":"But it is not a class that was requested by our alumni or prospective students."},
+{"file":"common_voice_en_39598234.mp3","text":"She also became the first female to represent Kenya either in Olympics or Paralympics."},
+{"file":"common_voice_en_36906412.mp3","text":"While they are at work they never open their mouths."},
+{"file":"common_voice_en_18095069.mp3","text":"Well, we shall see."},
+{"file":"common_voice_en_19711569.mp3","text":"Mixes of caramel corn sometimes contain nuts, such as peanuts, pecans, almonds, or cashews."},
+{"file":"common_voice_en_528427.mp3","text":"If I know where she is."},
+{"file":"common_voice_en_19664684.mp3","text":"Caution must be applied when using cross correlation for nonlinear systems."},
+{"file":"common_voice_en_26992169.mp3","text":"Gradually, they became aware of existing and further reports of a similar condition worldwide."},
+{"file":"common_voice_en_18152536.mp3","text":"Moreover, I won't be needed for the next two days."},
+{"file":"common_voice_en_24607138.mp3","text":"It is impossible for a material to have stress states outside its yield surface."},
+{"file":"common_voice_en_2322943.mp3","text":"He was bisexual, I think."},
+{"file":"common_voice_en_17247706.mp3","text":"There was an accident at work, a camel stampede nearly killed the janitor."},
+{"file":"common_voice_en_17293231.mp3","text":"Although searching in binary search trees is efficient, you won't find any fruits in there."},
+{"file":"common_voice_en_27683768.mp3","text":"While there he filmed his first documentary."},
+{"file":"common_voice_en_38003860.mp3","text":"He is remembered as the founder of the London Charterhouse and of Charterhouse School."},
+{"file":"common_voice_en_20726373.mp3","text":"The first season of the Gezer excavations concluded successfully and revealed some interesting details."},
+{"file":"common_voice_en_30665144.mp3","text":"He frequently clashed with the police and ended up being arrested multiple times."},
+{"file":"common_voice_en_29457987.mp3","text":"During Elizabeth's lifetime the couple resided at Woolleigh."},
+{"file":"common_voice_en_644237.mp3","text":"I got a lot on my mind."},
+{"file":"common_voice_en_18882419.mp3","text":"They were later banished into Tartarus by Troia."},
+{"file":"common_voice_en_38032465.mp3","text":"Her bronze bust of Alexandre Vinet is in the Wellcome Collection in London."},
+{"file":"common_voice_en_17884835.mp3","text":"Did you check your laser?"},
+{"file":"common_voice_en_27763688.mp3","text":"Brian Baker, also of Bad Religion, appeared on the album as well."},
+{"file":"common_voice_en_19226640.mp3","text":"Poor maintenance has often exacerbated these problems."},
+{"file":"common_voice_en_23700182.mp3","text":"The church was built by the fourth bishop of the diocese, George Blyth."},
+{"file":"common_voice_en_94235.mp3","text":"The internet is built upon many protocols which compliment each other perfectly."},
+{"file":"common_voice_en_20996081.mp3","text":"Blockhouses were built at each of the four corners."},
+{"file":"common_voice_en_27222437.mp3","text":"She graduated from Oklahoma State University with a fine arts degree in theater."},
+{"file":"common_voice_en_9347265.mp3","text":"I am sorry for it."},
+{"file":"common_voice_en_17903796.mp3","text":"Familiarity breeds contempt."},
+{"file":"common_voice_en_24015767.mp3","text":"The game counts as \"Entrada\"."},
+{"file":"common_voice_en_18829922.mp3","text":"In the end, Eppie marries a local boy, Dolly's son Aaron."},
+{"file":"common_voice_en_26995606.mp3","text":"Go swiftly, silently."},
+{"file":"common_voice_en_18698716.mp3","text":"This is also published in the \"London Gazette\" within a few days of issue."},
+{"file":"common_voice_en_22323685.mp3","text":"He felt overall that it was an interesting take and certainly a little different."},
+{"file":"common_voice_en_18861782.mp3","text":"He played college football at Florida State University."},
+{"file":"common_voice_en_22332950.mp3","text":"The Saint George Leagues Club is located nearby on Princes Highway."},
+{"file":"common_voice_en_17954816.mp3","text":"That is the question."},
+{"file":"common_voice_en_50511.mp3","text":"It didn't change anything."},
+{"file":"common_voice_en_17965413.mp3","text":"How do you know that?"},
+{"file":"common_voice_en_27274671.mp3","text":"It crosses the Missouri River on the Pony Express Bridge."},
+{"file":"common_voice_en_26072185.mp3","text":"Coffins usually are big rectangular boxes, carved or painted."},
+{"file":"common_voice_en_25814884.mp3","text":"Her father went on to become Secretary of the Massachusetts Bay Colony."},
+{"file":"common_voice_en_27430268.mp3","text":"In contrast, Hughes was a supporter, and a great admirer of Roy Jenkins."},
+{"file":"common_voice_en_27461741.mp3","text":"All songs written and composed by Mike Oldfield."},
+{"file":"common_voice_en_22730368.mp3","text":"She trained as an actress at the American Academy of Dramatic Arts."},
+{"file":"common_voice_en_38025119.mp3","text":"The sponsor affiliations change from time to time."},
+{"file":"common_voice_en_29169334.mp3","text":"Dolet has been referred to as an Anti-Trinitarian."},
+{"file":"common_voice_en_29789925.mp3","text":"Ango depended on the selling of his drawings as his only source of income."},
+{"file":"common_voice_en_23870040.mp3","text":"This formation is the same size as Clavius on the near side."},
+{"file":"common_voice_en_30174560.mp3","text":"I consider intuition to be the differentiation and conscious function of instinct."},
+{"file":"common_voice_en_19322354.mp3","text":"Tourneux, he was born in Paris."},
+{"file":"common_voice_en_19206590.mp3","text":"The governor appoints members of the Board, each of whom serve seven years."},
+{"file":"common_voice_en_19528344.mp3","text":"One safety concern was the food crops grown alongside the marijuana crops being contaminated."},
+{"file":"common_voice_en_21783394.mp3","text":"The comics were aimed at an older audience."},
+{"file":"common_voice_en_22540623.mp3","text":"Robi replaced Johnny Lozada after Johnny reached the group's age limit."},
+{"file":"common_voice_en_27255867.mp3","text":"The uniform colors were red, white, and blue."},
+{"file":"common_voice_en_42312533.mp3","text":"The system achieves a greater power-generation efficiency than normal heating."},
+{"file":"common_voice_en_27804236.mp3","text":"Bellegarde is located at the confluence of the Valserine and the Rhone."},
+{"file":"common_voice_en_25722913.mp3","text":"It can be deployed in around a minute."},
+{"file":"common_voice_en_21838641.mp3","text":"Thrown in with these plots tend to be references to current events."},
+{"file":"common_voice_en_20876161.mp3","text":"I did not take this possibility seriously..."},
+{"file":"common_voice_en_20962666.mp3","text":"This would be in addition to such sales through Fidelity Resales."},
+{"file":"common_voice_en_21134605.mp3","text":"She was operated by the Caledonian Steam Packet Company."},
+{"file":"common_voice_en_23903904.mp3","text":"The Olivetti empire had been begun by his father Camillo."},
+{"file":"common_voice_en_24098325.mp3","text":"The event's theme was \"Silver Streak\"."},
+{"file":"common_voice_en_24361738.mp3","text":"The Vice President was the \"vice-pitch\", also an elected position."},
+{"file":"common_voice_en_37203091.mp3","text":"Tozer has enjoyed combining a successful pop career with theatre work and acting."}
+]

pai-audio-evals/main.py

@@ -0,0 +1,66 @@
+import random
+import re
+from dataclasses import dataclass
+from functools import partial
+from pathlib import Path
+
+import logfire
+from nltk import edit_distance
+from pydantic import TypeAdapter
+from pydantic_ai import Agent, BinaryContent, AudioUrl
+from pydantic_evals import Dataset, Case
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext, EvaluatorOutput
+
+
+@dataclass
+class EditSimilarity(Evaluator[object, str, object]):
+ def evaluate(self, ctx: EvaluatorContext[object, str, object]) -> EvaluatorOutput:
+ if ctx.expected_output is None:
+ return {} # no metric
+ actual_tokens = re.sub(r'[^a-z0-9\s]', '', ctx.output.lower()).split()
+ expected_tokens = re.sub(r'[^a-z0-9\s]', '', ctx.expected_output.lower()).split()
+ distance = edit_distance(actual_tokens, expected_tokens)
+ normalized_distance = distance / max(len(actual_tokens), len(expected_tokens))
+ return 1 - normalized_distance
+
+
+logfire.configure(service_name='pai-audio-evals', scrubbing=False, console=False)
+logfire.instrument_pydantic_ai()
+
+this_dir = Path(__file__).parent
+assets = this_dir / 'assets'
+
+
+@dataclass
+class AudioFile:
+ file: str
+ text: str
+
+ def audio_url(self) -> AudioUrl:
+ return AudioUrl(f'https://smokeshow.helpmanual.io/4l1l1s0s6q4741012x1w/{self.file}')
+
+ def binary_content(self) -> BinaryContent:
+ path = assets / self.file
+ return BinaryContent(data=path.read_bytes(), media_type='audio/mpeg')
+
+
+n_files = 10
+files_schema = TypeAdapter(list[AudioFile])
+files = files_schema.validate_json((this_dir / 'assets.json').read_bytes())[:n_files]
+# random.seed(42)
+random.shuffle(files)
+
+audio_agent = Agent(instructions='return the transcription only, no prefix or quotes')
+dataset = Dataset(
+ cases=[Case(name=file.file, inputs=file.audio_url(), expected_output=file.text) for file in files],
+ evaluators=[EditSimilarity()],
+)
+
+
+async def task(audio_url: AudioUrl, model: str) -> str:
+ return (await audio_agent.run(['transcribe', audio_url], model=model)).output
+
+
+with logfire.span('Compare models'):
+ for model in 'gpt-4o-audio-preview', 'gpt-4o-mini-audio-preview', 'google-vertex:gemini-2.0-flash':
+ dataset.evaluate_sync(partial(task, model=model), name=model, max_concurrency=10)

pyproject.toml

@@ -9,9 +9,13 @@ dependencies = [
  "devtools>=0.12.2",
  "fastapi>=0.115.14",
  "logfire[asyncpg,fastapi,httpx]>=3.21.1",
- "pydantic-ai>=0.3.6",
+ "pydantic-ai>=0.4.5",
+ "nltk>=3.9.1",
 ]
 
+[tool.uv.sources]
+pydantic-ai = { git = "https://github.com/pydantic/pydantic-ai.git", rev = "0f46928bd07bc1a9f89c1d72c76cd2a86d52d489" }
+
 [dependency-groups]
 dev = ["ruff>=0.12.2", "asyncpg-stubs>=0.30.2"]