ClawLibrary v1: open-access research vault harvester
Topic-driven harvester that builds a deduplicated, metadata-rich offline PDF vault from legitimate open-access APIs (OpenAlex, Unpaywall, arXiv, Semantic Scholar, CORE) plus an optional, config-gated sanctioned-TDM path for subscription content. Replaces the original PRD's authenticated discovery-layer scraping with channels designed for programmatic research access. - config/topics loaders (Pydantic), PaperCandidate/VaultEntry models - OpenAlex search + multi-source OA PDF resolver chain (tries alternatives) - atomic dedup state, polite rate limiting + backoff, streaming PDF downloader with content-type/magic-byte validation, vault writer + index.json/csv - CLI: run/stats/export; structured JSONL run logs - 47 tests, 87% coverage; verified live against real OA APIs Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Omar Sobh
@@ -0,0 +1,171 @@
|
||||
"""ClawLibrary CLI.
|
||||
|
||||
Commands:
|
||||
run [--topic Q] [--dry-run] [--max N] harvest the topic queue
|
||||
stats show vault statistics
|
||||
export [--format csv|json] [--out P] export the vault index
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from .config import Config, Topic, load_config, load_topics
|
||||
from .pipeline import RunStats, run
|
||||
from .vault import VaultManager
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
def _load(config_path: str, topics_path: str) -> tuple[Config, list[Topic]]:
|
||||
config = load_config(config_path)
|
||||
queue = load_topics(topics_path)
|
||||
return config, queue.active()
|
||||
|
||||
|
||||
def _print_summary(stats: RunStats, *, dry_run: bool) -> None:
|
||||
table = Table(title=f"Run {stats.run_id}", show_header=True, header_style="bold")
|
||||
table.add_column("Metric")
|
||||
table.add_column("Count", justify="right")
|
||||
table.add_row("Topics processed", str(stats.topics_processed))
|
||||
if dry_run:
|
||||
table.add_row("Would download", str(stats.would_download))
|
||||
else:
|
||||
table.add_row("Downloaded", str(stats.downloaded))
|
||||
table.add_row("Skipped (dedup)", str(stats.skipped_dedup))
|
||||
table.add_row("Skipped (no OA PDF)", str(stats.skipped_no_pdf))
|
||||
table.add_row("Failed", str(stats.failed))
|
||||
if stats.suspended:
|
||||
table.add_row("Status", "[red]SUSPENDED (server errors)[/red]")
|
||||
console.print(table)
|
||||
|
||||
|
||||
def cmd_run(args: argparse.Namespace) -> int:
|
||||
config, topics = _load(args.config, args.topics)
|
||||
if args.topic:
|
||||
topics = [t for t in topics if t.query == args.topic]
|
||||
if not topics:
|
||||
console.print(f"[red]No enabled topic matching:[/red] {args.topic}")
|
||||
return 1
|
||||
|
||||
def on_event(kind: str, **data):
|
||||
if kind == "topic_start":
|
||||
console.print(f"[bold cyan]→ {data['topic']}[/bold cyan]")
|
||||
elif kind == "downloaded":
|
||||
t = data["title"][:80]
|
||||
console.print(f" [green]✓[/green] {t} [dim]({data['source']})[/dim]")
|
||||
elif kind == "would_download":
|
||||
t = data["title"][:80]
|
||||
console.print(f" [yellow]·[/yellow] {t} [dim]({data['source']})[/dim]")
|
||||
elif kind == "failed":
|
||||
title = data.get("title", "")[:70]
|
||||
console.print(f" [red]✗[/red] {title} [dim]{data['reason']}[/dim]")
|
||||
elif kind == "run_suspended":
|
||||
console.print("[red]Run suspended: too many consecutive server errors.[/red]")
|
||||
|
||||
stats = run(config, topics, dry_run=args.dry_run, max_override=args.max, on_event=on_event)
|
||||
_print_summary(stats, dry_run=args.dry_run)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_stats(args: argparse.Namespace) -> int:
|
||||
config, _ = _load(args.config, args.topics)
|
||||
vault = VaultManager(config.vault.root)
|
||||
entries = vault.load_index()
|
||||
if not entries:
|
||||
console.print("[yellow]Vault is empty (no index.json).[/yellow]")
|
||||
return 0
|
||||
|
||||
by_topic: dict[str, int] = {}
|
||||
by_source: dict[str, int] = {}
|
||||
total_bytes = 0
|
||||
for e in entries:
|
||||
by_topic[e.get("topic", "?")] = by_topic.get(e.get("topic", "?"), 0) + 1
|
||||
by_source[e.get("source") or "?"] = by_source.get(e.get("source") or "?", 0) + 1
|
||||
total_bytes += int(e.get("file_size_bytes", 0) or 0)
|
||||
|
||||
table = Table(title="Vault statistics", header_style="bold")
|
||||
table.add_column("Metric")
|
||||
table.add_column("Value", justify="right")
|
||||
table.add_row("Total papers", str(len(entries)))
|
||||
table.add_row("Total size", f"{total_bytes / 1_048_576:.1f} MiB")
|
||||
table.add_row("Topics", str(len(by_topic)))
|
||||
console.print(table)
|
||||
|
||||
src_table = Table(title="By source", header_style="bold")
|
||||
src_table.add_column("Source")
|
||||
src_table.add_column("Papers", justify="right")
|
||||
for src, n in sorted(by_source.items(), key=lambda kv: -kv[1]):
|
||||
src_table.add_row(src, str(n))
|
||||
console.print(src_table)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_export(args: argparse.Namespace) -> int:
|
||||
config, _ = _load(args.config, args.topics)
|
||||
vault = VaultManager(config.vault.root)
|
||||
entries = vault.load_index()
|
||||
out = Path(args.out) if args.out else Path(f"export.{args.format}")
|
||||
|
||||
if args.format == "json":
|
||||
out.write_text(json.dumps(entries, indent=2))
|
||||
else:
|
||||
fields = [
|
||||
"id", "doi", "title", "authors", "journal", "year", "topic",
|
||||
"source", "vault_path", "file_size_bytes", "downloaded_at",
|
||||
]
|
||||
with open(out, "w", newline="") as f:
|
||||
w = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore")
|
||||
w.writeheader()
|
||||
for e in entries:
|
||||
row = dict(e)
|
||||
if isinstance(row.get("authors"), list):
|
||||
row["authors"] = "; ".join(row["authors"])
|
||||
w.writerow(row)
|
||||
console.print(f"[green]Exported {len(entries)} entries to {out}[/green]")
|
||||
return 0
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(prog="clawlibrary", description=__doc__)
|
||||
p.add_argument("--config", default="config.yaml", help="path to config.yaml")
|
||||
p.add_argument("--topics", default="topics.yaml", help="path to topics.yaml")
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
|
||||
pr = sub.add_parser("run", help="harvest the topic queue")
|
||||
pr.add_argument("--topic", help="run a single topic by exact query string")
|
||||
pr.add_argument("--dry-run", action="store_true", help="resolve only, no downloads")
|
||||
pr.add_argument("--max", type=int, default=None, help="override max downloads this run")
|
||||
pr.set_defaults(func=cmd_run)
|
||||
|
||||
ps = sub.add_parser("stats", help="show vault statistics")
|
||||
ps.set_defaults(func=cmd_stats)
|
||||
|
||||
pe = sub.add_parser("export", help="export the vault index")
|
||||
pe.add_argument("--format", choices=["csv", "json"], default="csv")
|
||||
pe.add_argument("--out", help="output path")
|
||||
pe.set_defaults(func=cmd_export)
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
logger.remove() # quiet console; pipeline adds a JSONL file sink per run
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
return args.func(args)
|
||||
except FileNotFoundError as e:
|
||||
console.print(f"[red]Error:[/red] {e}")
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user