Realtime Audio - Mesh API

The Realtime API provides a bidirectional WebSocket session for low-latency audio streaming. The SDK wraps the raw WebSocket with typed send/receive helpers.

Python requires pip install 'meshapi[realtime]' (adds websockets>=12.0).
Node.js on Node 18–21 requires npm install ws. Node 22+ has WebSocket built in.

Protocol. The realtime API uses OpenAI’s GA event shape: configure the session with session.type: "realtime", output_modalities, and an audio object (see below). Input audio is sent as base64 (send_audio handles this for you), and output audio arrives as response.output_audio.delta events — the SDK decodes these into msg.audio so you can play them directly. Audio is 24 kHz mono PCM16.

Connect and configure

Python
Node.js
Go

from meshapi import MeshAPI

client = MeshAPI(base_url="https://api.meshapi.ai", token="rsk_...")

with client.realtime.connect(model="openai/gpt-realtime-mini") as session:
    session.send({
        "type": "session.update",
        "session": {
            "type": "realtime",
            "output_modalities": ["audio"],          # or ["text"]
            "instructions": "You are a helpful assistant.",
            "audio": {
                "input": {"format": {"type": "audio/pcm", "rate": 24000}},
                "output": {"format": {"type": "audio/pcm", "rate": 24000}, "voice": "alloy"},
            },
        },
    })
    # session closes cleanly on context exit

Async variant:

from meshapi import AsyncMeshAPI

async with AsyncMeshAPI(base_url="https://api.meshapi.ai", token="rsk_...") as client:
    async with client.realtime.connect(model="openai/gpt-realtime-mini") as session:
        await session.send({"type": "session.update", "session": {"type": "realtime", "output_modalities": ["text"]}})

import { MeshAPI } from "meshapi-node-sdk";

const client = new MeshAPI({ baseUrl: "https://api.meshapi.ai", token: "rsk_..." });

const session = await client.realtime.connect({ model: "openai/gpt-realtime-mini" });

await session.send({
  type: "session.update",
  session: {
    type: "realtime",
    output_modalities: ["audio"],
    instructions: "You are a helpful assistant.",
    audio: {
      input: { format: { type: "audio/pcm", rate: 24000 } },
      output: { format: { type: "audio/pcm", rate: 24000 }, voice: "alloy" },
    },
  },
});

session, err := client.Realtime.Connect(ctx, meshapi.RealtimeConnectParams{
    Model: "openai/gpt-realtime-mini",
})
if err != nil {
    log.Fatal(err)
}
defer session.Close()

session.Send(ctx, map[string]any{
    "type": "session.update",
    "session": map[string]any{
        "type":              "realtime",
        "output_modalities": []string{"audio"},
        "instructions":      "You are a helpful assistant.",
        "audio": map[string]any{
            "input":  map[string]any{"format": map[string]any{"type": "audio/pcm", "rate": 24000}},
            "output": map[string]any{"format": map[string]any{"type": "audio/pcm", "rate": 24000}, "voice": "alloy"},
        },
    },
})

Receive frames

The server sends a session.created event immediately after the WebSocket handshake completes.

Python
Node.js
Go

with client.realtime.connect(model="openai/gpt-realtime-mini") as session:
    # Receive a single frame
    msg = session.receive()
    if msg.audio:                     # set for output_audio.delta events
        play_audio(msg.audio)
    elif msg.event:
        print("event type:", msg.event.get("type"))

    # Iterate over frames
    for msg in session:
        if msg.audio:
            play_audio(msg.audio)     # 24kHz mono PCM16
        elif msg.event and msg.event.get("type") == "response.output_text.delta":
            print(msg.event.get("delta"), end="", flush=True)
        elif msg.event and msg.event.get("type") == "response.done":
            break

// Async iterator
for await (const msg of session) {
  if (msg.audio) {
    playAudio(msg.audio);  // Uint8Array
  } else if (msg.event?.type === "response.output_text.delta") {
    process.stdout.write(msg.event.delta);
  } else if (msg.event?.type === "response.done") {
    break;
  }
}
await session.close();

// Event emitter style
session
  .on("message", (msg) => { if (msg.audio) playAudio(msg.audio); })
  .on("error", (err) => console.error(err))
  .on("close", (code) => console.log("closed:", code));

// Single receive (blocking)
msg, err := session.Receive(ctx)
if msg.Audio != nil {
    playAudio(msg.Audio)
} else if msg.Event != nil {
    fmt.Println("event:", msg.Event["type"])
}

// Channel-based concurrent pump
msgCh, errCh := session.Events(ctx)
for msg := range msgCh {
    if msg.Audio != nil {
        playAudio(msg.Audio)
    } else if msg.Event != nil {
        switch msg.Event["type"] {
        case "response.output_text.delta":
            fmt.Print(msg.Event["delta"])
        case "response.done":
            fmt.Println("\n[done]")
        }
    }
}
if err := <-errCh; err != nil {
    log.Fatal(err)
}

Send audio

Pass raw 16-bit PCM audio (24kHz mono) to send_audio / sendAudio / SendAudio.

Python
Node.js
Go

session.send_audio(pcm_bytes)
session.send({"type": "input_audio_buffer.commit"})
session.send({"type": "response.create"})

await session.sendAudio(pcmBytes);  // Uint8Array
await session.send({ type: "input_audio_buffer.commit" });
await session.send({ type: "response.create" });

session.SendAudio(ctx, pcmBytes)
session.Send(ctx, map[string]any{"type": "input_audio_buffer.commit"})
session.Send(ctx, map[string]any{"type": "response.create"})

Error handling

Errors in the WebSocket session are surfaced as RealtimeError (Python/Node.js) or *meshapi.MeshAPIError (Go).

Python
Node.js
Go

from meshapi.resources.realtime import RealtimeError

try:
    msg = session.receive()
except RealtimeError as e:
    print("code:", e.code)
    print("message:", str(e))

import { RealtimeError } from "meshapi-node-sdk";

try {
  for await (const msg of session) { ... }
} catch (err) {
  if (err instanceof RealtimeError) {
    console.error("code:", err.code);
    console.error("message:", err.message);
  }
}

msg, err := session.Receive(ctx)
if err != nil {
    var re *meshapi.RealtimeError
    if errors.As(err, &re) {
        fmt.Println("code:", re.Code)
    }
}

Supported models

Model ID	Mode
`openai/gpt-realtime-2.1`	Speech-to-speech
`openai/gpt-realtime-2.1-mini`	Speech-to-speech
`openai/gpt-realtime-2`	Speech-to-speech
`openai/gpt-realtime-1.5`	Speech-to-speech
`openai/gpt-realtime`	Speech-to-speech
`openai/gpt-realtime-mini`	Speech-to-speech
`openai/gpt-realtime-translate`	Speech-to-speech translation
`elevenlabs/scribe_v2_realtime`	Realtime speech-to-text

Consult GET /v1/models for the current set of realtime-capable models — the catalog is live and this table is a snapshot.

​Connect and configure

​Receive frames

​Send audio

​Error handling

​Supported models

Connect and configure

Receive frames

Send audio

Error handling

Supported models