Realtime Audio

View as Markdown

Realtime Audio

client.realtime opens a bidirectional WebSocket session to wss://api.meshapi.ai/v1/realtime. The wire format is identical to OpenAI’s Realtime API.

Requires websockets>=12.0. Install with pip install 'meshapi[realtime]'.

Connect and close (sync)

1from meshapi import MeshAPI
2
3client = MeshAPI(base_url="https://api.meshapi.ai", token="rsk_...")
4
5with client.realtime.connect(model="openai/gpt-realtime-mini") as session:
6 pass # clean close on exit

Connect and close (async)

1from meshapi import AsyncMeshAPI
2
3async with AsyncMeshAPI(base_url="https://api.meshapi.ai", token="rsk_...") as client:
4 async with client.realtime.connect(model="openai/gpt-realtime-mini") as session:
5 pass

Configure the session

1session.send({
2 "type": "session.update",
3 "session": {
4 "type": "realtime",
5 "modalities": ["audio", "text"],
6 "voice": "alloy",
7 "instructions": "You are a helpful assistant.",
8 },
9})

Send audio

1# pcm_bytes is raw 16-bit PCM at 24 kHz mono
2session.send_audio(pcm_bytes)

Receive frames

1from meshapi.resources.realtime import RealtimeError
2
3msg = session.receive()
4
5if msg.audio:
6 play_audio(msg.audio)
7elif msg.event:
8 print("event type:", msg.event.get("type"))

Iterate over frames

1with client.realtime.connect(model="openai/gpt-realtime-mini") as session:
2 session.send({"type": "session.update", "session": {"type": "realtime", "modalities": ["text"]}})
3 session.send({"type": "response.create"})
4
5 for msg in session: # iterates until connection closes
6 if msg.event:
7 t = msg.event.get("type")
8 if t == "response.text.delta":
9 print(msg.event.get("delta"), end="", flush=True)
10 elif t == "response.done":
11 break

Async iteration

1async with client.realtime.connect(model="openai/gpt-realtime-mini") as session:
2 await session.send({"type": "session.update", "session": {"type": "realtime", "modalities": ["text"]}})
3 await session.send({"type": "response.create"})
4
5 async for msg in session:
6 if msg.event and msg.event.get("type") == "response.text.delta":
7 print(msg.event.get("delta"), end="", flush=True)

Error handling

1from meshapi.resources.realtime import RealtimeError
2
3try:
4 msg = session.receive()
5except RealtimeError as e:
6 print("code:", e.code) # "invalid_api_key", "insufficient_quota", …
7 print("message:", str(e))

Full voice agent example

1import threading
2
3with client.realtime.connect(model="openai/gpt-realtime-mini") as session:
4 session.send({
5 "type": "session.update",
6 "session": {"type": "realtime", "modalities": ["audio"], "voice": "alloy"},
7 })
8
9 def send_audio():
10 for chunk in mic_stream():
11 session.send_audio(chunk)
12 session.send({"type": "input_audio_buffer.commit"})
13 session.send({"type": "response.create"})
14
15 threading.Thread(target=send_audio, daemon=True).start()
16
17 for msg in session:
18 if msg.audio:
19 speaker.write(msg.audio)

Supported models

Model ID
openai/gpt-realtime-mini
openai/gpt-realtime
openai/gpt-realtime-2