mohitagw15856
827d7f62ec
The "Run evals" step ran 24 API calls sequentially with no request timeout, so it was slow and could stall indefinitely if one call hung. - bin/lib/anthropic.mjs: per-request timeout (120s) via AbortController + retry (2x, backoff) on 429/5xx/timeout. Fails fast on 4xx (bad key/model). - evals/run-evals.mjs: run (case × model) tasks through a concurrency pool (default 4, --concurrency to tune); preserves result order. - eval-leaderboard.yml: job timeout-minutes: 20 as a safety net. Applies to the next run. The hardening also benefits the Action runner and `generate`, which share the client. Claude-Session: https://claude.ai/code/session_016JWn5jRD5tcEFKrubjQ6Px Co-authored-by: Claude <noreply@anthropic.com>
78 lines
3.2 KiB
JavaScript
78 lines
3.2 KiB
JavaScript
// Minimal, dependency-free Anthropic Messages API client (Node 18+ global fetch).
|
|
// Shared by the GitHub Action runner, the eval harness, and skill generation.
|
|
// No SDK, no install — just a thin POST wrapper.
|
|
|
|
const API_URL = 'https://api.anthropic.com/v1/messages';
|
|
|
|
/**
|
|
* Call the Anthropic Messages API and return the concatenated text output.
|
|
* Adds a per-request timeout and limited retries so a slow/transient failure
|
|
* can't hang a CI job forever.
|
|
* @param {object} o
|
|
* @param {string} o.apiKey - Anthropic API key.
|
|
* @param {string} [o.model] - Model id (default claude-sonnet-4-6).
|
|
* @param {string} [o.system]- System prompt.
|
|
* @param {Array} o.messages- [{role, content}] messages.
|
|
* @param {number} [o.maxTokens]
|
|
* @param {number} [o.timeoutMs] - Per-request timeout (default 120s).
|
|
* @param {number} [o.retries] - Retries on timeout / 429 / 5xx (default 2).
|
|
* @returns {Promise<string>}
|
|
*/
|
|
export async function complete({ apiKey, model = 'claude-sonnet-4-6', system, messages, maxTokens = 4096, timeoutMs = 120000, retries = 2 }) {
|
|
if (!apiKey) throw new Error('Missing Anthropic API key (set ANTHROPIC_API_KEY).');
|
|
let lastErr;
|
|
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
const ctrl = new AbortController();
|
|
const timer = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
try {
|
|
const res = await fetch(API_URL, {
|
|
method: 'POST',
|
|
headers: {
|
|
'content-type': 'application/json',
|
|
'x-api-key': apiKey,
|
|
'anthropic-version': '2023-06-01',
|
|
},
|
|
body: JSON.stringify({ model, max_tokens: maxTokens, ...(system ? { system } : {}), messages }),
|
|
signal: ctrl.signal,
|
|
});
|
|
if (res.ok) {
|
|
const data = await res.json();
|
|
return (data.content || []).map((c) => c.text || '').join('').trim();
|
|
}
|
|
const body = await res.text().catch(() => '');
|
|
// Retry transient server / rate-limit errors; fail fast on 4xx (bad key/model).
|
|
if ((res.status === 429 || res.status >= 500) && attempt < retries) {
|
|
lastErr = new Error(`Anthropic API ${res.status}`);
|
|
} else {
|
|
throw new Error(`Anthropic API ${res.status}: ${body.slice(0, 500)}`);
|
|
}
|
|
} catch (e) {
|
|
if (e.name === 'AbortError') e = new Error(`Anthropic API request timed out after ${timeoutMs}ms`);
|
|
const retryable = /timed out/.test(e.message) || e.name === 'TypeError' || /Anthropic API (429|5\d\d)/.test(e.message);
|
|
if (!retryable || attempt >= retries) throw e;
|
|
lastErr = e;
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt)); // backoff: 1s, 2s, 4s
|
|
}
|
|
throw lastErr || new Error('Anthropic API request failed.');
|
|
}
|
|
|
|
/** Parse "name: value" YAML-ish frontmatter + body from a SKILL.md string. */
|
|
export function parseSkill(text) {
|
|
const m = text.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
|
const meta = {};
|
|
if (m) {
|
|
for (const line of m[1].split('\n')) {
|
|
const kv = line.match(/^(\w[\w-]*):\s*(.*)$/);
|
|
if (kv) {
|
|
let v = kv[2].trim();
|
|
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
|
|
meta[kv[1]] = v;
|
|
}
|
|
}
|
|
}
|
|
return { meta, body: m ? m[2].trim() : text.trim() };
|
|
}
|