clawverse/clawlibrary
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace live spinner with periodic progress prints

Browse Source 
console.status doesn't flush to a redirected log, making long background
extract/embed/summarize runs look silent. Print periodic progress lines that
flush instead, so background jobs are observable via their output file.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Omar Sobh
2026-06-03 19:35:57 -05:00
co-authored by Claude Opus 4.8
parent 0bec0efbd2
commit c1da5ad249
3 changed files with 45 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
clawlibrary/__main__.py
+44 -43
View File
@@ -154,20 +154,20 @@ def cmd_extract(args: argparse.Namespace) -> int:
vlm = lambda png: oc.caption_image(png, VLM_OCR_PROMPT) # noqa: E731
done = skipped = failed = vlm_pages = 0
with console.status("Extracting text...") as status:
for i, pdf in enumerate(pdfs, 1):
if not args.force and already_extracted(pdf):
skipped += 1
continue
try:
stats = extract_one(pdf, vlm=vlm)
done += 1
vlm_pages += stats.vlm_pages
except Exception as e: # noqa: BLE001
failed += 1
console.print(f"[red]✗[/red] {pdf.name}: {e}")
if i % 25 == 0:
status.update(f"Extracting... {i}/{len(pdfs)} (done {done}, skipped {skipped})")
total = len(pdfs)
for i, pdf in enumerate(pdfs, 1):
if not args.force and already_extracted(pdf):
skipped += 1
continue
try:
stats = extract_one(pdf, vlm=vlm)
done += 1
vlm_pages += stats.vlm_pages
except Exception as e: # noqa: BLE001
failed += 1
console.print(f"[red]✗[/red] {pdf.name}: {e}")
if i % 100 == 0:
console.print(f" extract {i}/{total} (done {done}, skipped {skipped})")
console.print(
f"[green]Extract complete[/green] extracted={done} skipped={skipped} "
f"failed={failed} vlm_pages={vlm_pages}"
@@ -193,17 +193,18 @@ def cmd_summarize(args: argparse.Namespace) -> int:
pdfs = pdfs[: args.limit]
done = skipped = 0
with console.status("Summarizing...") as status:
for i, pdf in enumerate(pdfs, 1):
side = pdf.with_suffix(".json")
if not args.force and has_summary(side):
skipped += 1
continue
summary = summarize_one(pdf, oc)
if summary is not None:
append_insight(insights, _json.loads(side.read_text()))
done += 1
status.update(f"Summarizing... {i}/{len(pdfs)} (done {done})")
total = len(pdfs)
for i, pdf in enumerate(pdfs, 1):
side = pdf.with_suffix(".json")
if not args.force and has_summary(side):
skipped += 1
continue
summary = summarize_one(pdf, oc)
if summary is not None:
append_insight(insights, _json.loads(side.read_text()))
done += 1
if i % 10 == 0:
console.print(f" summarize {i}/{total} (done {done}, skipped {skipped})")
console.print(f"[green]Summaries complete[/green] done={done} skipped={skipped}")
console.print(f"Insight digest: {insights}")
return 0
@@ -228,24 +229,24 @@ def cmd_embed(args: argparse.Namespace) -> int:
pdfs = pdfs[: args.limit]
added = skipped = 0
with console.status("Embedding...") as status:
for i, pdf in enumerate(pdfs, 1):
data = json.loads(pdf.with_suffix(".json").read_text())
doi = data.get("doi") or data.get("id") or pdf.stem
if doi in seen:
skipped += 1
continue
text = pdf.with_suffix(".txt").read_text()
vecs, rows = add_document(
oc, text=text, doi=doi, title=data.get("title", ""),
topic=data.get("topic", ""), text_path=str(pdf.with_suffix(".txt")),
)
store = merge(store, vecs, rows)
seen.add(doi)
added += 1
if i % 20 == 0:
store.save(root) # periodic checkpoint for resumability
status.update(f"Embedding... {i}/{len(pdfs)} (added {added})")
total = len(pdfs)
for i, pdf in enumerate(pdfs, 1):
data = json.loads(pdf.with_suffix(".json").read_text())
doi = data.get("doi") or data.get("id") or pdf.stem
if doi in seen:
skipped += 1
continue
text = pdf.with_suffix(".txt").read_text()
vecs, rows = add_document(
oc, text=text, doi=doi, title=data.get("title", ""),
topic=data.get("topic", ""), text_path=str(pdf.with_suffix(".txt")),
)
store = merge(store, vecs, rows)
seen.add(doi)
added += 1
if i % 20 == 0:
store.save(root) # periodic checkpoint for resumability
console.print(f" embed {i}/{total} (added {added}, vectors {store.vectors.shape[0]})")
store.save(root)
console.print(
f"[green]Embedding complete[/green] papers_added={added} skipped={skipped} "