Realtime Audio

View as Markdown

Realtime Audio

client.realtime opens a bidirectional WebSocket session to wss://api.meshapi.ai/v1/realtime. The wire format is identical to OpenAI’s Realtime API.

Requires websockets>=12.0. Install with pip install 'meshapi[realtime]'.

Protocol. Configure the session with the GA event shape: session.type: "realtime", output_modalities, and an audio object (below). Input audio is sent as base64 — send_audio() handles that — and output audio arrives as response.output_audio.delta events, which the SDK decodes into msg.audio. Audio is 24 kHz mono PCM16.

Connect and close (sync)

1from meshapi import MeshAPI
2
3client = MeshAPI(base_url="https://api.meshapi.ai", token="rsk_...")
4
5with client.realtime.connect(model="openai/gpt-realtime-2") as session:
6 pass # clean close on exit

Connect and close (async)

1from meshapi import AsyncMeshAPI
2
3async with AsyncMeshAPI(base_url="https://api.meshapi.ai", token="rsk_...") as client:
4 async with client.realtime.connect(model="openai/gpt-realtime-2") as session:
5 pass

Configure the session

1session.send({
2 "type": "session.update",
3 "session": {
4 "type": "realtime",
5 "output_modalities": ["audio"], # or ["text"]
6 "instructions": "You are a helpful assistant.",
7 "audio": {
8 "input": {"format": {"type": "audio/pcm", "rate": 24000}},
9 "output": {"format": {"type": "audio/pcm", "rate": 24000}, "voice": "alloy"},
10 },
11 },
12})

Send audio

1# pcm_bytes is raw 16-bit PCM at 24 kHz mono
2session.send_audio(pcm_bytes)

Receive frames

1from meshapi.resources.realtime import RealtimeError
2
3msg = session.receive()
4
5if msg.audio:
6 play_audio(msg.audio)
7elif msg.event:
8 print("event type:", msg.event.get("type"))

Iterate over frames

1with client.realtime.connect(model="openai/gpt-realtime-2") as session:
2 session.send({"type": "session.update", "session": {"type": "realtime", "output_modalities": ["text"]}})
3 session.send({"type": "response.create"})
4
5 for msg in session: # iterates until connection closes
6 if msg.event:
7 t = msg.event.get("type")
8 if t == "response.output_text.delta":
9 print(msg.event.get("delta"), end="", flush=True)
10 elif t == "response.done":
11 break

Async iteration

1async with client.realtime.connect(model="openai/gpt-realtime-2") as session:
2 await session.send({"type": "session.update", "session": {"type": "realtime", "output_modalities": ["text"]}})
3 await session.send({"type": "response.create"})
4
5 async for msg in session:
6 if msg.event and msg.event.get("type") == "response.output_text.delta":
7 print(msg.event.get("delta"), end="", flush=True)

Error handling

1from meshapi.resources.realtime import RealtimeError
2
3try:
4 msg = session.receive()
5except RealtimeError as e:
6 print("code:", e.code) # "invalid_api_key", "insufficient_quota", …
7 print("message:", str(e))

Full voice agent example

1import threading
2
3with client.realtime.connect(model="openai/gpt-realtime-2") as session:
4 session.send({
5 "type": "session.update",
6 "session": {"type": "realtime", "output_modalities": ["audio"],
7 "audio": {"output": {"format": {"type": "audio/pcm", "rate": 24000}, "voice": "alloy"}}},
8 })
9
10 def send_audio():
11 for chunk in mic_stream():
12 session.send_audio(chunk)
13 session.send({"type": "input_audio_buffer.commit"})
14 session.send({"type": "response.create"})
15
16 threading.Thread(target=send_audio, daemon=True).start()
17
18 for msg in session:
19 if msg.audio:
20 speaker.write(msg.audio)

Supported models

Model IDMode
openai/gpt-realtime-2Speech-to-speech
openai/gpt-realtime-1.5Speech-to-speech
openai/gpt-realtime-miniSpeech-to-speech
elevenlabs/scribe_v2_realtimeRealtime speech-to-text

Consult GET /v1/models for the current set of realtime-capable models.