Mohit
7b02261a3c
- Add compare-mode demo GIF + its Playwright recorder; embed in README eval section - Expand evals/cases.json (6 → 15 flagship skills) so more skills can be eval-scored and sample-generated - Add --generate-missing mode to build-samples.mjs - Add generate-samples.yml: workflow_dispatch job that generates real sample outputs via the ANTHROPIC_API_KEY secret (key never leaves GitHub) and commits Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
81 lines
3.9 KiB
JavaScript
81 lines
3.9 KiB
JavaScript
#!/usr/bin/env node
|
|
// Builds web/samples.json from examples/samples/*.md for the sample-output gallery.
|
|
// Each sample file has frontmatter (skill, title, input, source) + the output body.
|
|
//
|
|
// Build the gallery data: node scripts/build-samples.mjs
|
|
// Generate a new sample: ANTHROPIC_API_KEY=sk-ant-... node scripts/build-samples.mjs --generate <skill>
|
|
// (runs the skill on its eval case input and writes examples/samples/<skill>.md)
|
|
import { readFileSync, writeFileSync, readdirSync, existsSync, mkdirSync } from 'node:fs';
|
|
import { join, dirname } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
const root = join(dirname(fileURLToPath(import.meta.url)), '..');
|
|
const samplesDir = join(root, 'examples', 'samples');
|
|
|
|
function parseFrontmatter(text) {
|
|
const m = text.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
|
if (!m) return { meta: {}, body: text };
|
|
const meta = {};
|
|
for (const line of m[1].split('\n')) {
|
|
const kv = line.match(/^(\w[\w-]*):\s*(.*)$/);
|
|
if (kv) {
|
|
let v = kv[2].trim();
|
|
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
|
|
meta[kv[1]] = v;
|
|
}
|
|
}
|
|
return { meta, body: m[2].trim() };
|
|
}
|
|
|
|
async function generate(skillName) {
|
|
const { complete, parseSkill } = await import('../bin/lib/anthropic.mjs');
|
|
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
if (!apiKey) throw new Error('Set ANTHROPIC_API_KEY to generate.');
|
|
const skillFile = join(root, 'skills', skillName, 'SKILL.md');
|
|
if (!existsSync(skillFile)) throw new Error(`Unknown skill: ${skillName}`);
|
|
const { body } = parseSkill(readFileSync(skillFile, 'utf8'));
|
|
const cases = JSON.parse(readFileSync(join(root, 'evals', 'cases.json'), 'utf8')).cases;
|
|
const input = (cases.find((c) => c.skill === skillName) || {}).input;
|
|
if (!input) throw new Error(`No eval case input for ${skillName}; add one to evals/cases.json first.`);
|
|
const system = body + '\n\n---\nExecute this skill now on the input below and produce the complete output. Do not ask questions.';
|
|
const output = await complete({ apiKey, model: 'claude-sonnet-4-6', system, messages: [{ role: 'user', content: input }], maxTokens: 4096 });
|
|
const title = skillName.split('-').map((w) => w[0].toUpperCase() + w.slice(1)).join(' ');
|
|
if (!existsSync(samplesDir)) mkdirSync(samplesDir, { recursive: true });
|
|
const fm = `---\nskill: ${skillName}\ntitle: ${title}\ninput: ${JSON.stringify(input)}\nsource: generated by claude-sonnet-4-6\n---\n\n${output.trim()}\n`;
|
|
writeFileSync(join(samplesDir, `${skillName}.md`), fm);
|
|
console.log(`Wrote examples/samples/${skillName}.md`);
|
|
}
|
|
|
|
const genIdx = process.argv.indexOf('--generate');
|
|
if (genIdx !== -1) {
|
|
try { await generate(process.argv[genIdx + 1]); }
|
|
catch (e) { console.error(e.message); process.exit(1); }
|
|
}
|
|
|
|
// --generate-missing: generate a sample for every eval-case skill that doesn't
|
|
// already have one (never overwrites hand-written samples).
|
|
if (process.argv.includes('--generate-missing')) {
|
|
const cases = JSON.parse(readFileSync(join(root, 'evals', 'cases.json'), 'utf8')).cases;
|
|
for (const c of cases) {
|
|
if (existsSync(join(samplesDir, `${c.skill}.md`))) { console.log(`skip ${c.skill} (already has a sample)`); continue; }
|
|
try { await generate(c.skill); } catch (e) { console.error(`failed ${c.skill}: ${e.message}`); }
|
|
}
|
|
}
|
|
|
|
// Build samples.json
|
|
const samples = [];
|
|
if (existsSync(samplesDir)) {
|
|
for (const f of readdirSync(samplesDir).filter((f) => f.endsWith('.md')).sort()) {
|
|
const { meta, body } = parseFrontmatter(readFileSync(join(samplesDir, f), 'utf8'));
|
|
samples.push({
|
|
skill: meta.skill || f.replace(/\.md$/, ''),
|
|
title: meta.title || meta.skill || f,
|
|
input: meta.input || '',
|
|
source: meta.source || '',
|
|
output: body,
|
|
});
|
|
}
|
|
}
|
|
writeFileSync(join(root, 'web', 'samples.json'), JSON.stringify({ count: samples.length, samples }));
|
|
console.log(`Wrote web/samples.json — ${samples.length} sample outputs.`);
|