/** * Node.js 18+. No packages. Import never sends a request. * node router.mjs --base-url YOUR_SERVICE_URL // routing only * node router.mjs --base-url YOUR_SERVICE_URL --execute // billable inference * Own provider keys stay client-side. Fallbacks are your policy, not budget guarantees. */ import { pathToFileURL } from "node:url"; export const MODELS = { openai: "gpt-4o", fireworks: "accounts/fireworks/models/gpt-oss-120b", cerebras: "gpt-oss-120b", }; const PROVIDERS = { openai: ["https://api.openai.com/v1/chat/completions", "OPENAI_API_KEY"], fireworks: ["https://api.fireworks.ai/inference/v1/chat/completions", "FIREWORKS_API_KEY"], cerebras: ["https://api.cerebras.ai/v1/chat/completions", "CEREBRAS_API_KEY"], }; const PARAMETERS = new Set([ "task_type", "latency_sensitivity", "cost_sensitivity", "provider", "max_latency_ms", "max_cost_per_1k_tokens_usd", ]); export class Router { constructor(baseUrl, fallback, { models = MODELS, routingApiKey = null } = {}) { const url = new URL(baseUrl); if (!["https:", "http:"].includes(url.protocol) || url.username || url.password || url.search || url.hash) { throw new Error("Use an http(s) service URL without credentials"); } this.baseUrl = baseUrl.replace(/\/$/, ""); this.models = { ...models }; if (this.models[fallback.provider] !== fallback.model) throw new Error("Unsupported fallback model"); this.fallback = { ...fallback }; this.routingApiKey = routingApiKey; this.cache = new Map(); } fallbackChoice(reason) { return { ...this.fallback, source: "caller_fallback", reason }; } async choose(params = {}) { if (!Object.keys(params).every((key) => PARAMETERS.has(key))) return this.fallbackChoice("invalid_parameters"); const key = new URLSearchParams(Object.entries(params).sort(([a], [b]) => a.localeCompare(b))).toString(); const cached = this.cache.get(key); if (cached && cached.until > Date.now() && this.models[cached.choice.provider] === cached.choice.model) { return { ...cached.choice, source: "cache" }; } this.cache.delete(key); try { const headers = { Accept: "application/json" }; if (this.routingApiKey) headers["X-API-Key"] = this.routingApiKey; const response = await fetch(`${this.baseUrl}/v1/should-call?${key}`, { headers, signal: AbortSignal.timeout(2000), }); if (!response.ok) throw new Error("Routing unavailable"); const data = await response.json(); if (data.should_call !== true) return this.fallbackChoice("no_eligible_choice"); const { recommended_provider: provider, recommended_model: model, measurement } = data; if (typeof provider !== "string" || typeof model !== "string" || this.models[provider] !== model) { return this.fallbackChoice("unsupported_model"); } if (!(measurement?.sample_count >= 1) || !/Z$|[+-]\d\d:\d\d$/.test(measurement.expires_at)) { return this.fallbackChoice("invalid_evidence"); } const until = Math.min(Date.now() + 30_000, Date.parse(measurement.expires_at) - 1000); if (!Number.isFinite(until) || until <= Date.now()) return this.fallbackChoice("expired_evidence"); const choice = { provider, model, source: "routing_service" }; if (this.cache.size >= 32) this.cache.clear(); this.cache.set(key, { until, choice }); return choice; } catch { return this.fallbackChoice("routing_unavailable"); } } async run(prompt, callProvider, params = {}) { const choice = await this.choose(params); try { return await callProvider(choice.provider, choice.model, prompt); } catch (error) { if (choice.provider === this.fallback.provider && choice.model === this.fallback.model) throw error; this.cache.clear(); return await callProvider(this.fallback.provider, this.fallback.model, prompt); } } } export async function callProvider(provider, model, prompt) { const [endpoint, keyName] = PROVIDERS[provider]; const apiKey = process.env[keyName]; if (!apiKey) throw new Error(`Configure your own ${keyName}`); const response = await fetch(endpoint, { method: "POST", signal: AbortSignal.timeout(30_000), headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` }, body: JSON.stringify({ model, messages: [{ role: "user", content: prompt }], max_tokens: 256 }), }); if (!response.ok) throw new Error("Provider request failed"); const content = (await response.json()).choices?.[0]?.message?.content; if (typeof content !== "string" || !content.trim()) throw new Error("No usable provider text"); return content; } async function main() { const args = process.argv.slice(2); const baseIndex = args.indexOf("--base-url"); if (baseIndex < 0 || !args[baseIndex + 1]) { throw new Error("Usage: node router.mjs --base-url YOUR_SERVICE_URL [--execute]"); } const execute = args.includes("--execute"); const available = Object.fromEntries(Object.entries(MODELS).filter(([key]) => process.env[PROVIDERS[key][1]])); if (execute && !Object.keys(available).length) throw new Error("Configure your own provider keys before --execute"); const models = execute ? available : MODELS; const provider = Object.keys(models)[0]; const router = new Router(args[baseIndex + 1], { provider, model: models[provider] }, { models, routingApiKey: process.env.INFERENCELATENCY_API_KEY, }); if (execute) console.log(await router.run("Explain what an AI agent is in one sentence.", callProvider)); else console.log(JSON.stringify(await router.choose(), null, 2)); } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { main().catch(() => { console.error("Request failed. Check the service URL and your own credentials; provider bodies are not logged."); process.exitCode = 1; }); }