// In main agent — invoke Workflow tool with this script
export const meta = {
name: 'research-extraction',
description: 'Multi-axis extraction from research corpus with reviewer gating',
phases: [
{ title: 'Acquisition', detail: 'Acquire, extract, and create source.md per source' },
{ title: 'Step 0', detail: 'Reading summary + working memory dump' },
{ title: 'Architecture', detail: 'extract-architecture.md' },
{ title: 'Messages', detail: 'extract-messages.md' },
{ title: 'Algorithms', detail: 'extract-algorithms.md' },
{ title: 'Forms', detail: 'extract-forms.md' },
{ title: 'Formalization', detail: 'Code sketches for all SKETCH NEEDED markers' },
{ title: 'Anti-slop', detail: 'Writing quality pass over all axis output files' },
],
}
// args = {
// topic: string, // absolute path to topic directory root
// rawSources: { // sources to acquire, extract, and register
// slug: string, // directory name under topic/sources/
// path?: string, // local file path (provide path OR url)
// url?: string, // remote URL to download
// type?: string, // llm-wiki type: paper|article|book-chapter|... (default: paper)
// }[],
// kb: string[], // absolute paths to knowledge base files to cross-ref
// outputDir: string, // where to write extract-*.md files (use topic/drafts/)
// axes: { // per-axis config (order = execution order)
// name: string, // e.g. "Architecture"
// outputFile: string, // absolute path for this axis output
// extractionSpec: string, // what to extract (H2/H3 structure, tables, etc.)
// }[],
// continueOnFailure?: boolean, // if true, log failed phases and continue instead of halting
// antislopSkillPath?: string, // absolute path to anti-slop/SKILL.md; if provided, runs a final anti-slop pass over all axis output files
// }
// ── Helpers ───────────────────────────────────────────────────────────────────
const STATUS_FILE = `${args.outputDir}/extraction-status.md`
async function writeStatus(currentPhase, completed, failed, notes) {
const lines = [
`# Extraction status`,
``,
`phase: ${currentPhase}`,
`completed: [${completed.join(', ')}]`,
`failed: [${failed.join(', ')}]`,
notes ? `notes: ${notes}` : '',
].filter(Boolean)
await agent(`Write this exact content to ${STATUS_FILE}:\n\n${lines.join('\n')}`,
{ label: 'write-status', phase: currentPhase })
}
async function writeIssues(phaseName, issues) {
const path = `${args.outputDir}/review-issues-${phaseName.toLowerCase().replace(/ /g, '-')}.md`
const content = `# Review issues — ${phaseName}\n\n${issues.map(i => `- ${i}`).join('\n')}`
await agent(`Write this exact content to ${path}:\n\n${content}`,
{ label: `write-issues:${phaseName}`, phase: phaseName })
}
const completedPhases = []
const failedPhases = []
const CONFIDENCE_RULES = `
Every non-trivial claim carries one of three markers placed after the source citation:
| Marker | Meaning |
|---|---|
| [DIRECT] | Verbatim or near-verbatim — copy exact passage into fenced quote block |
| [INFERRED] | Derived from 2+ passages — name all exact section numbers + one-sentence reasoning |
| [SPECULATIVE] | Not grounded in specific passage — follow with > ⚠ SPECULATIVE: block |
Rules:
- [SPECULATIVE] never in a table cell without > ⚠ SPECULATIVE: block immediately below
- [INFERRED] requires exact section numbers — if unavailable, downgrade to [SPECULATIVE]
- [DIRECT] requires exact quoted passage in fenced quote block
Divergence/absence markers:
- > ⚠ DIVERGES FROM KB: — contradicts or extends the knowledge base
- > ✦ NEW: — not present in knowledge base
- > ∅ NOT FOUND IN SOURCES: — source is silent (do not omit — absence is data)
`
// Shared quality rules — used by both extractor self-check and reviewer checklist.
const QUALITY_RULES = `
- Every section has at least one [DIRECT] or [INFERRED] citation in its body OR in the first H3 sub-section beneath it (citations in sub-sections satisfy the parent requirement — do not fail a section whose content lives in labelled sub-sections that each carry their own citations; sections whose primary content is a structural comparison or data-flow table satisfy this requirement via table column attribution such as Document/Source/paper-ref columns — do not require inline [DIRECT]/[INFERRED] markers in the prose of such sections)
- No [SPECULATIVE] claim without > ⚠ SPECULATIVE: block immediately below
- No [INFERRED] claim without exact source section numbers (§N.N) or exact heading in quotes for wiki files (§"Heading Name") — if unavailable, downgrade to [SPECULATIVE]
- Every silent extraction point marked > ∅ NOT FOUND IN SOURCES: as a standalone blockquote (not only in a table cell)
- Every ∅ in a table cell also has a standalone > ∅ NOT FOUND IN SOURCES: blockquote immediately after the table
- Cross-document data-flow table Document column cites original sources (thesis, paper, wiki/ slug) — NOT the current extraction file itself
- All DIVERGES FROM KB and NEW flags present where needed
`
const hasKB = (args.kb || []).length > 0
const QUALITY_RULES_FOR_REVIEWER = hasKB
? QUALITY_RULES
: QUALITY_RULES.split('\n').filter(l => !l.includes('DIVERGES FROM KB')).join('\n')
const AXIS_REVIEWER_CHECKLIST = `
Reviewer checklist — return JSON {pass: bool, issues: string[]}:
${QUALITY_RULES_FOR_REVIEWER}
- Cross-document data-flow table present and complete at end of document
- File written to disk and non-empty
- Return ALL issues found — be exhaustive. Do not stop at 3–5. A pass means zero issues remain.
`
const FORM_REVIEWER_CHECKLIST = `
Reviewer checklist — return JSON {pass: bool, issues: string[]}:
- Every <!-- SKETCH NEEDED: ... --> placeholder has a corresponding sketch in ## Rust Sketches (formalization pass) at the end of its file
- Each sketch cites the claim it formalizes with section reference and confidence marker [DIRECT], [INFERRED], or [SPECULATIVE]
- No sketches appear outside the ## Rust Sketches (formalization pass) section
- Files are written to disk and non-empty
- Return ALL issues found — be exhaustive. Do not stop at 3–5. A pass means zero issues remain.
`
const STEP0_REVIEWER_CHECKLIST = `
Reviewer checklist — return JSON {pass: bool, issues: string[]}:
- Reading summary has exactly one paragraph per source file
- Each paragraph states: scope, contradictions noticed, axes the source is silent on
- Working memory dump is comprehensive — includes named entities, message types,
equations, agent names, algorithm names, each marked with source section number
- Both files are non-empty and written to disk
- Return ALL issues found — be exhaustive. A pass means zero issues remain.
`
const RESULT_SCHEMA = {
type: 'object',
properties: {
pass: { type: 'boolean' },
issues: { type: 'array', items: { type: 'string' } },
confirmed_facts: { type: 'string' }, // markdown block — facts to carry forward
confirmed_absences: { type: 'string' }, // markdown block — sources silent on this axis
divergences: { type: 'string' }, // markdown block — DIVERGES FROM KB / NEW flags
},
required: ['pass', 'issues'],
}
// ── Acquisition phase: acquire, extract, source.md per source ─────────────────
phase('Acquisition')
const acquiredSources = await parallel(args.rawSources.map(src => async () => {
const sourceDir = `${args.topic}/sources/${src.slug}`
const extractedPath = `${sourceDir}/extracted.md`
const sourceMdPath = `${sourceDir}/source.md`
await agent(`
You are a source acquisition assistant running research-paper Steps 1–3 for one source.
Source slug: ${src.slug}
Source dir: ${sourceDir}
${src.url ? `URL: ${src.url}` : ''}
${src.path ? `Local path: ${src.path}` : ''}
Type: ${src.type || 'paper'}
## Step A — Check if already acquired
If ${extractedPath} already exists, skip to Step C (source.md only).
## Step B — Acquire and extract (only if ${extractedPath} does not exist)
1. Run: mkdir -p ${sourceDir}
2. Acquire:
${src.url ? ` curl -L "${src.url}" -o "${sourceDir}/original.pdf"` : ` cp "${src.path}" "${sourceDir}/original.pdf"`}
3. Detect PDF type:
xberg detect "${sourceDir}/original.pdf"
4. Extract:
- Machine-readable: xberg extract "${sourceDir}/original.pdf" --content-format markdown --output "${extractedPath}"
- Scanned/image:
uvx marker-pdf marker_single "${sourceDir}/original.pdf" --output_dir "${sourceDir}/marker-out/"
then: mv "${sourceDir}/marker-out/original/original.md" "${extractedPath}"
(marker-pdf writes <input-stem>.md inside <output_dir>/<input-stem>/ — move it to extracted.md)
## Step C — Create source.md (skip if ${sourceMdPath} already exists)
Write ${sourceMdPath} with this frontmatter:
\`\`\`yaml
---
title: "<infer from PDF title page, URL, or filename>"
type: ${src.type || 'paper'}
summary: ""
tldr: ""
status: draft
last_updated: "<today YYYY-MM-DD>"
tags: []
read_when: []
sources: []
concepts: []
confidence: 0.5
claims: []
---
\`\`\`
Leave summary, tldr, tags, claims, concepts empty — populated after analysis.
`, { label: `acquire:${src.slug}`, phase: 'Acquisition' })
return extractedPath
}))