Resilience - Mesh API

Every Mesh API SDK ships two client-side resilience layers plus observability into both them and the gateway’s own routing:

Transport retry — automatic retries on transient HTTP failures, with a configurable policy.
Model fallback chain — try the next model when the primary fails.
debug / logger — see exactly which requests were retried and which were served by a fallback (client- and gateway-side).

This is the SDK counterpart to the gateway’s Resilient Routing. The two layers are independent and safe to combine — here is how a single create() call flows through both: Read it bottom-up for the guarantees: a terminal error (auth, validation, billing) short-circuits every layer; a transient one is first absorbed by gateway-side retries/fallback (no client involvement), then by SDK transport retries of the same request, and only then does the SDK switch models.

Transport retry

Every non-streaming request retries on 429 / 502 / 503 / 504 with exponential backoff + jitter, honouring Retry-After (default: 3 retries, 500 ms base, 30 s max). Streams never retry. The policy is configurable:

Python
Node.js
Go
Java

from meshapi import MeshAPI, RetryPolicy

client = MeshAPI(
    base_url=base_url,
    token=token,
    retry=RetryPolicy(
        max_retries=5,                 # default 3
        retry_on_status=[429, 503],    # default (429, 502, 503, 504)
        backoff_base_ms=250,           # default 500
        backoff_max_ms=10_000,         # default 30_000
        respect_retry_after=True,      # default True
        retry_on_network_error=True,   # default False — POSTs are non-idempotent
    ),
)

const client = new MeshAPI({
  baseUrl,
  token,
  retry: {
    maxRetries: 5,               // default 3
    retryOnStatus: [429, 503],   // default [429, 502, 503, 504]
    backoffBaseMs: 250,          // default 500
    backoffMaxMs: 10_000,        // default 30_000
    respectRetryAfter: true,     // default true
    retryOnNetworkError: true,   // default false — POSTs are non-idempotent
  },
});

maxRetries := 5
baseMs, maxMs := 250, 10_000
respectRetryAfter, retryOnNetworkError := true, true

client := meshapi.New(meshapi.Config{
    BaseURL: baseURL,
    Token:   token,
    Retry: &meshapi.RetryPolicy{
        MaxRetries:          &maxRetries,          // default 3
        RetryOnStatus:       []int{429, 503},      // default [429, 502, 503, 504]
        BackoffBaseMs:       &baseMs,              // default 500
        BackoffMaxMs:        &maxMs,               // default 30_000
        RespectRetryAfter:   &respectRetryAfter,   // default true
        RetryOnNetworkError: &retryOnNetworkError, // default false — POSTs are non-idempotent
    },
})

import com.meshapi.sdk.resilience.RetryPolicy;

MeshAPI client = MeshAPI.builder()
    .baseUrl(baseUrl)
    .token(token)
    .retry(RetryPolicy.builder()
        .maxRetries(5)              // default 3
        .retryOnStatus(429, 503)    // default 429, 502, 503, 504
        .backoffBaseMs(250)         // default 500
        .backoffMaxMs(10_000)       // default 30_000
        .respectRetryAfter(true)    // default true
        .retryOnNetworkError(true)  // default false — POSTs are non-idempotent
        .build())
    .build();

The legacy top-level maxRetries / max_retries option still works and maps onto retry.maxRetries; an explicit retry value wins when both are set.

Network-error retry is opt-in (retryOnNetworkError) and never applies to timeouts or cancellation — a timed-out POST may already be executing server-side, and completions are not idempotent.

Model fallback chain

Non-streaming chat completions can fall back to other models when the primary fails with a transient error (default 502 / 503 / 504, after transport retries are exhausted). Configure a chain client-wide, or override it per call:

Python
Node.js
Go
Java

from meshapi import MeshAPI, FallbackConfig, ChatCompletionParams

client = MeshAPI(
    base_url=base_url,
    token=token,
    fallback=FallbackConfig(models=["anthropic/claude-sonnet-5", "mistral/mistral-large"]),
)

# Per-call override (never sent to the server):
client.chat.completions.create(
    ChatCompletionParams(model="openai/gpt-4o", messages=messages),
    fallback_models=["anthropic/claude-sonnet-5"],
)

const client = new MeshAPI({
  baseUrl,
  token,
  fallback: { models: ["anthropic/claude-sonnet-5", "mistral/mistral-large"] },
});

// Per-call override (never sent to the server):
await client.chat.completions.create({
  model: "openai/gpt-4o",
  messages,
  fallbackModels: ["anthropic/claude-sonnet-5"],
});

client := meshapi.New(meshapi.Config{
    BaseURL: baseURL,
    Token:   token,
    Fallback: &meshapi.FallbackConfig{
        Models: []string{"anthropic/claude-sonnet-5", "mistral/mistral-large"},
    },
})

// Per-call override (never sent to the server):
model := "openai/gpt-4o"
resp, err := client.Chat.Completions.Create(ctx, meshapi.ChatCompletionParams{
    Model:          &model,
    Messages:       messages,
    FallbackModels: []string{"anthropic/claude-sonnet-5"},
})

import com.meshapi.sdk.resilience.FallbackConfig;

MeshAPI client = MeshAPI.builder()
    .baseUrl(baseUrl)
    .token(token)
    .fallback(FallbackConfig.builder()
        .models("anthropic/claude-sonnet-5", "mistral/mistral-large")
        .build())
    .build();

// Per-call override (never sent to the server):
client.chat().completions().create(ChatCompletionRequest.builder()
    .model("openai/gpt-4o")
    .addMessage(ChatMessage.user("Hello!"))
    .fallbackModels("anthropic/claude-sonnet-5")
    .build());

Terminal errors (auth, validation, billing) never advance the chain.

The client-side fallbackModels chain is distinct from the models request parameter. models is a server-side, provider-handled ordered list sent in the request body; fallbackModels is a client-side directive the SDK acts on locally and never puts on the wire.

Seeing what happened: `debug` and `logger`

Set debug to print a readable line to stderr on every retry and fallback:

[meshapi] retrying POST /v1/chat/completions (attempt 1/4 failed: 503, next in 512ms) [req_abc]
[meshapi] falling back openai/gpt-4o → anthropic/claude-sonnet-5 (1/2: 503 provider_not_available)
[meshapi] gateway served /v1/chat/completions (2 attempts, provider fallback) [req_abc]

For structured logging, pass a logger — it receives every retry, fallback, and gateway-routing event:

Python
Node.js
Go
Java

def on_event(event):
    if event.type == "retry":
        my_log.warning("meshapi retry: %s", event)
    if event.type == "fallback":
        my_log.warning("meshapi fallback: %s", event)
    if event.type == "gateway-routing" and event.fallback:
        my_log.info("gateway used a provider fallback after %d attempts", event.attempts)

client = MeshAPI(base_url=base_url, token=token, debug=True, logger=on_event)

const client = new MeshAPI({
  baseUrl,
  token,
  debug: true,
  logger: (event) => {
    if (event.type === "retry") myLog.warn("meshapi retry", event);
    if (event.type === "fallback") myLog.warn("meshapi fallback", event);
    if (event.type === "gateway-routing" && event.fallback) {
      myLog.info(`gateway used a provider fallback after ${event.attempts} attempts`);
    }
  },
});

client := meshapi.New(meshapi.Config{
    BaseURL: baseURL,
    Token:   token,
    Debug:   true,
    Logger: func(event meshapi.ResilienceEvent) {
        switch e := event.(type) {
        case meshapi.RetryEvent:
            log.Printf("meshapi retry: %+v", e)
        case meshapi.FallbackEvent:
            log.Printf("meshapi fallback: %+v", e)
        case meshapi.GatewayRoutingEvent:
            if e.Fallback {
                log.Printf("gateway used a provider fallback after %d attempts", e.Attempts)
            }
        }
    },
})

import com.meshapi.sdk.resilience.*;

MeshAPI client = MeshAPI.builder()
    .baseUrl(baseUrl)
    .token(token)
    .debug(true)
    .logger(event -> {
        if (event instanceof RetryEvent e) log.warn("meshapi retry: {}", e);
        if (event instanceof FallbackEvent e) log.warn("meshapi fallback: {}", e);
        if (event instanceof GatewayRoutingEvent e && e.fallback) {
            log.info("gateway used a provider fallback after {} attempts", e.attempts);
        }
    })
    .build();

`gateway-routing` events

A gateway-routing event reports the server-side resilience the gateway itself performed for your request — the per-key routing_policy’s same-target retries and cross-provider fallback. The SDK builds it by parsing the X-Mesh-Routing-Attempts and X-Mesh-Routing-Fallback response headers, so it only appears when your API key has an active routing policy. Which upstream provider served the request is internal and is not reported.

Streaming responses carry no routing headers (headers are flushed before the stream body). For per-request routing detail on streamed calls, check your Mesh API dashboard Logs.

​Transport retry

​Model fallback chain

​Seeing what happened: debug and logger

​gateway-routing events

Transport retry

Model fallback chain

Seeing what happened: `debug` and `logger`

`gateway-routing` events