zaoqu-liu/scienceclaw/devtu-docs-quality

Read https://aiscientist.tools/setup.md and set up ToolUniverse for me.

python3 - <<'EOF'
import re, sys
from pathlib import Path
from collections import defaultdict
DOCS = Path("docs")
EXCLUDE = {"locale", "old", "_build", "__pycache__", "tools", "archive"}
KNOWN_BAD = {
    "list_tools", "run_batch", "run_async", "execute_tool", "call_tool",
    "list_tools_by_category", "configure_api_keys", "get_tool", "get_exposed_name",
    "list_available_methods", "register_tool_from_config", "register_tool",
}
STATIC = [
    (r"\.load_tools\([^)]*(?:use_cache|cache_dir)\s*=", "load_tools() invalid kwargs"),
    (r"ToolUniverse\([^)]*timeout\s*=", "ToolUniverse(timeout=) invalid"),
    (r'"name":\s*"opentarget_', 'old lowercase "opentarget_*" tool name'),
    (r"tu\.run_batch\(", "tu.run_batch() — use tu.run(list, max_workers=N)"),
    (r'\btu\.[A-Z]\w+\s*\(', "tu.ToolName() shorthand — use tu.run({name:...})"),
]
M = re.compile(r'\b(?:tu|tooluni)\.([\w]+)\s*\(')
issues = defaultdict(list)
for f in sorted(list(DOCS.rglob("*.rst")) + list(DOCS.rglob("*.md"))):
    if any(p in f.parts for p in EXCLUDE): continue
    t = f.read_text(errors="replace")
    code = "\n".join(
        re.findall(r"\.\. code-block:: python\n((?:[ \t]+[^\n]*\n|[ \t]*\n)*)", t, re.MULTILINE) +
        re.findall(r"```python\n(.*?)```", t, re.DOTALL))
    if not code.strip(): continue
    rel = str(f.relative_to(DOCS))
    for m in M.finditer(code):
        if not m.group(1).startswith("_") and m.group(1) in KNOWN_BAD:
            issues[rel].append(f"tu.{m.group(1)}()")
    for pat, label in STATIC:
        if re.search(pat, code): issues[rel].append(label)
if issues:
    [print(f"  {f}: {i}") for f in sorted(issues) for i in sorted(set(issues[f]))]
    sys.exit(1)
else:
    print("Phase D clean")
EOF

# wrong → correct
tu.run_batch(list)              → tu.run(list, max_workers=4)
tu.run_async(query)             → await tu.run(query)
tu.call_tool('X', {...})        → tu.run({"name": "X", "arguments": {...}})
tu.execute_tool('X', {...})     → tu.run({"name": "X", "arguments": {...}})
tu.list_tools()                 → tu.list_built_in_tools(mode='list_name')
tu.get_tool('X')                → tu.get_tool_by_name('X')
tu.register_tool(instance)      → tu.register_custom_tool(tool_instance=instance)
tu.register_tool_from_config(c) → tu.register_custom_tool(tool_config=c)
tu.configure_api_keys({})       → REMOVE — use env vars
tu.get_exposed_name(name)       → REMOVE — shortening is automatic
ToolUniverse(timeout=30)        → ToolUniverse()  # no timeout kwarg
load_tools(use_cache=True)      → load_tools()  # no use_cache kwarg
tu.ToolName(key=val)            → tu.run({"name": "ToolName", "arguments": {"key": val}})
opentarget_get_*                → OpenTargets_get_*  (capital O and T)

python3 - <<'EOF'
import re, sys
from pathlib import Path
 
DOCS = Path("docs")
EXCLUDE = {"_build", "__pycache__", "locale"}
issues = []
 
EMOJI_RE = re.compile(
    r'[\U0001F300-\U0001F9FF\U00002600-\U000027BF\U0001FA00-\U0001FA9F'
    r'\U0001F004\U0001F0CF\U00002702-\U000027B0]'
)
 
def in_code_block(text, pos):
    """Return True if pos is inside a code block."""
    before = text[:pos]
    # RST code blocks
    rst_starts = [m.end() for m in re.finditer(r'\.\. code-block::[^\n]*\n', before)]
    for start in reversed(rst_starts):
        block = text[start:]
        end_match = re.search(r'\n(?![ \t]|\n)', block)
        block_end = start + (end_match.start() if end_match else len(block))
        if start <= pos <= block_end:
            return True
    # Markdown fenced blocks
    fences = list(re.finditer(r'```', before))
    return len(fences) % 2 == 1
 
for f in sorted(DOCS.rglob("*.rst")) + sorted(DOCS.rglob("*.md")):
    if any(p in f.parts for p in EXCLUDE):
        continue
    text = f.read_text(errors="replace")
    rel = str(f.relative_to(DOCS))
 
    # Check emojis in prose
    for m in EMOJI_RE.finditer(text):
        if not in_code_block(text, m.start()):
            line = text[:m.start()].count('\n') + 1
            issues.append(f"EMOJI  {rel}:{line}  {repr(m.group())}")
 
    # Check wrong tool counts
    for bad in ["600+", "750+", "1200+", "10000+"]:
        if bad in text and "tools" in text[text.find(bad):text.find(bad)+20]:
            issues.append(f"COUNT  {rel}  contains '{bad} tools' (use '1000+')")
 
    # Check broken :doc: references
    for m in re.finditer(r':doc:`([^`]+)`', text):
        ref = m.group(1).lstrip('/')
        # Strip any display text (format: display <path>)
        if ' <' in ref:
            ref = re.search(r'<([^>]+)>', ref).group(1)
        base = DOCS / Path(ref.replace('/', '/'))
        candidates = [base.with_suffix('.rst'), base.with_suffix('.md'),
                      base / 'index.rst', base / 'index.md']
        if not any(c.exists() for c in candidates):
            line = text[:m.start()].count('\n') + 1
            issues.append(f"LINK   {rel}:{line}  broken :doc:`{ref}`")
 
# Check navbar in conf.py
conf = Path("docs/conf.py").read_text()
for bad_nav in ['"API"', '"API Keys"', '"Contribution"']:
    if bad_nav in conf:
        issues.append(f"NAV    conf.py  navbar contains {bad_nav}")
if "aiscientist.tools" not in conf:
    issues.append("NAV    conf.py  navbar missing aiscientist.tools link")
 
# Check per-platform page line counts
platform_dir = DOCS / "guide" / "building_ai_scientists"
for rst in platform_dir.glob("*.rst"):
    if rst.name in ("index.rst", "mcp_support.rst", "compact_mode.rst",
                    "mcpb_introduction.rst", "mcp_name_shortening.rst"):
        continue
    lines = rst.read_text().splitlines()
    if len(lines) > 15:
        issues.append(f"SIZE   guide/building_ai_scientists/{rst.name}  {len(lines)} lines (max 15)")
 
if issues:
    for i in sorted(issues):
        print(i)
    sys.exit(1)
else:
    print("Phase E clean")
EOF

python scripts/test_doc_code_blocks.py

python scripts/validate_documentation.py

# scripts/validate_documentation.py checks:
DEPRECATED_PATTERNS = [
    (r"python -m tooluniverse\.server", "tooluniverse-server"),
    (r"600\+?\s+tools", "1000+ tools"),
    (r"750\+?\s+tools", "1000+ tools"),
]