Add insight layer: extract / summarize / embed / search
Local-model pipeline over the vault (no cloud):
- extract: PyMuPDF text layer -> {doi}.txt + sidecar stats; VLM (gemma4)
fallback only for image-only pages (<300 chars)
- summarize: gemma4:E4B structured digest per paper -> sidecar + insights.md
- embed: nomic-embed-text chunk embeddings -> vault/embeddings.npz
- search: brute-force cosine semantic search, deduped to one hit/paper
- ollama_client: thin httpx wrapper (generate/vision/embed)
Full extraction run: 1,202 papers -> 16.7M words. 13 new tests.
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Omar Sobh
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
d13f3d5ea8
commit
0bec0efbd2
@@ -134,6 +134,154 @@ def cmd_export(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_extract(args: argparse.Namespace) -> int:
|
||||
from .extract import already_extracted, extract_one, iter_pdfs
|
||||
|
||||
config, _ = _load(args.config, args.topics)
|
||||
pdfs = list(iter_pdfs(Path(config.vault.root)))
|
||||
if args.limit:
|
||||
pdfs = pdfs[: args.limit]
|
||||
|
||||
vlm = None
|
||||
if args.vlm_fallback:
|
||||
from .extract import VLM_OCR_PROMPT
|
||||
from .ollama_client import OllamaClient
|
||||
|
||||
oc = OllamaClient()
|
||||
if not oc.is_up():
|
||||
console.print("[red]Ollama not reachable; cannot use --vlm-fallback.[/red]")
|
||||
return 2
|
||||
vlm = lambda png: oc.caption_image(png, VLM_OCR_PROMPT) # noqa: E731
|
||||
|
||||
done = skipped = failed = vlm_pages = 0
|
||||
with console.status("Extracting text...") as status:
|
||||
for i, pdf in enumerate(pdfs, 1):
|
||||
if not args.force and already_extracted(pdf):
|
||||
skipped += 1
|
||||
continue
|
||||
try:
|
||||
stats = extract_one(pdf, vlm=vlm)
|
||||
done += 1
|
||||
vlm_pages += stats.vlm_pages
|
||||
except Exception as e: # noqa: BLE001
|
||||
failed += 1
|
||||
console.print(f"[red]✗[/red] {pdf.name}: {e}")
|
||||
if i % 25 == 0:
|
||||
status.update(f"Extracting... {i}/{len(pdfs)} (done {done}, skipped {skipped})")
|
||||
console.print(
|
||||
f"[green]Extract complete[/green] extracted={done} skipped={skipped} "
|
||||
f"failed={failed} vlm_pages={vlm_pages}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_summarize(args: argparse.Namespace) -> int:
|
||||
import json as _json
|
||||
|
||||
from .extract import iter_pdfs
|
||||
from .ollama_client import OllamaClient
|
||||
from .summarize import append_insight, has_summary, summarize_one
|
||||
|
||||
config, _ = _load(args.config, args.topics)
|
||||
oc = OllamaClient()
|
||||
if not oc.is_up():
|
||||
console.print("[red]Ollama not reachable.[/red]")
|
||||
return 2
|
||||
insights = Path(config.vault.root) / "insights.md"
|
||||
pdfs = [p for p in iter_pdfs(Path(config.vault.root)) if p.with_suffix(".txt").exists()]
|
||||
if args.limit:
|
||||
pdfs = pdfs[: args.limit]
|
||||
|
||||
done = skipped = 0
|
||||
with console.status("Summarizing...") as status:
|
||||
for i, pdf in enumerate(pdfs, 1):
|
||||
side = pdf.with_suffix(".json")
|
||||
if not args.force and has_summary(side):
|
||||
skipped += 1
|
||||
continue
|
||||
summary = summarize_one(pdf, oc)
|
||||
if summary is not None:
|
||||
append_insight(insights, _json.loads(side.read_text()))
|
||||
done += 1
|
||||
status.update(f"Summarizing... {i}/{len(pdfs)} (done {done})")
|
||||
console.print(f"[green]Summaries complete[/green] done={done} skipped={skipped}")
|
||||
console.print(f"Insight digest: {insights}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_embed(args: argparse.Namespace) -> int:
|
||||
from .embedstore import EmbedStore, add_document, merge
|
||||
from .extract import iter_pdfs
|
||||
from .ollama_client import OllamaClient
|
||||
|
||||
config, _ = _load(args.config, args.topics)
|
||||
root = Path(config.vault.root)
|
||||
oc = OllamaClient()
|
||||
if not oc.is_up():
|
||||
console.print("[red]Ollama not reachable.[/red]")
|
||||
return 2
|
||||
|
||||
store = EmbedStore.load(root)
|
||||
seen = store.dois
|
||||
pdfs = [p for p in iter_pdfs(root) if p.with_suffix(".txt").exists()]
|
||||
if args.limit:
|
||||
pdfs = pdfs[: args.limit]
|
||||
|
||||
added = skipped = 0
|
||||
with console.status("Embedding...") as status:
|
||||
for i, pdf in enumerate(pdfs, 1):
|
||||
data = json.loads(pdf.with_suffix(".json").read_text())
|
||||
doi = data.get("doi") or data.get("id") or pdf.stem
|
||||
if doi in seen:
|
||||
skipped += 1
|
||||
continue
|
||||
text = pdf.with_suffix(".txt").read_text()
|
||||
vecs, rows = add_document(
|
||||
oc, text=text, doi=doi, title=data.get("title", ""),
|
||||
topic=data.get("topic", ""), text_path=str(pdf.with_suffix(".txt")),
|
||||
)
|
||||
store = merge(store, vecs, rows)
|
||||
seen.add(doi)
|
||||
added += 1
|
||||
if i % 20 == 0:
|
||||
store.save(root) # periodic checkpoint for resumability
|
||||
status.update(f"Embedding... {i}/{len(pdfs)} (added {added})")
|
||||
store.save(root)
|
||||
console.print(
|
||||
f"[green]Embedding complete[/green] papers_added={added} skipped={skipped} "
|
||||
f"vectors={store.vectors.shape[0] if store.vectors.size else 0}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_search(args: argparse.Namespace) -> int:
|
||||
from .embedstore import EmbedStore, search
|
||||
from .ollama_client import OllamaClient
|
||||
|
||||
config, _ = _load(args.config, args.topics)
|
||||
store = EmbedStore.load(Path(config.vault.root))
|
||||
if store.vectors.size == 0:
|
||||
console.print("[yellow]No embeddings yet. Run `clawlibrary embed` first.[/yellow]")
|
||||
return 0
|
||||
oc = OllamaClient()
|
||||
if not oc.is_up():
|
||||
console.print("[red]Ollama not reachable.[/red]")
|
||||
return 2
|
||||
hits = search(store, oc, args.query, k=args.k)
|
||||
table = Table(title=f'Search: "{args.query}"', header_style="bold")
|
||||
table.add_column("Score", justify="right")
|
||||
table.add_column("Title")
|
||||
table.add_column("Topic", style="dim")
|
||||
for h in hits:
|
||||
table.add_row(f"{h.score:.3f}", h.title[:70], h.topic[:30])
|
||||
console.print(table)
|
||||
if args.snippets:
|
||||
for h in hits:
|
||||
console.print(f"\n[bold]{h.title[:80]}[/bold] [dim]{h.doi}[/dim]")
|
||||
console.print(f" {h.snippet}")
|
||||
return 0
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(prog="clawlibrary", description=__doc__)
|
||||
p.add_argument("--config", default="config.yaml", help="path to config.yaml")
|
||||
@@ -153,6 +301,28 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
pe.add_argument("--format", choices=["csv", "json"], default="csv")
|
||||
pe.add_argument("--out", help="output path")
|
||||
pe.set_defaults(func=cmd_export)
|
||||
|
||||
px = sub.add_parser("extract", help="extract full text from vault PDFs")
|
||||
px.add_argument("--limit", type=int, default=None, help="process at most N PDFs")
|
||||
px.add_argument("--force", action="store_true", help="re-extract even if done")
|
||||
px.add_argument("--vlm-fallback", action="store_true",
|
||||
help="OCR image-only pages via the Ollama vision model")
|
||||
px.set_defaults(func=cmd_extract)
|
||||
|
||||
pm = sub.add_parser("summarize", help="LLM structured summaries (gemma4 via Ollama)")
|
||||
pm.add_argument("--limit", type=int, default=None)
|
||||
pm.add_argument("--force", action="store_true", help="re-summarize even if done")
|
||||
pm.set_defaults(func=cmd_summarize)
|
||||
|
||||
pb = sub.add_parser("embed", help="embed papers for semantic search (nomic via Ollama)")
|
||||
pb.add_argument("--limit", type=int, default=None)
|
||||
pb.set_defaults(func=cmd_embed)
|
||||
|
||||
pq = sub.add_parser("search", help="semantic search across the vault")
|
||||
pq.add_argument("query", help="natural-language query")
|
||||
pq.add_argument("--k", type=int, default=10, help="number of results")
|
||||
pq.add_argument("--snippets", action="store_true", help="show matching snippets")
|
||||
pq.set_defaults(func=cmd_search)
|
||||
return p
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user