Harden Ollama calls: retry transient disconnects; don't abort batch on one failure
A single "Server disconnected" from Ollama (model reload) raised through and killed the summarize run at 651/1202. Now: _post_with_retry retries transient disconnects (3 attempts, backoff); summarize/embed loops catch per-paper errors and continue. Resumable, so re-running picks up the remaining papers. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Omar Sobh
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
766e8f84dc
commit
ee123af6f8
+14
-3
@@ -192,20 +192,27 @@ def cmd_summarize(args: argparse.Namespace) -> int:
|
|||||||
if args.limit:
|
if args.limit:
|
||||||
pdfs = pdfs[: args.limit]
|
pdfs = pdfs[: args.limit]
|
||||||
|
|
||||||
done = skipped = 0
|
done = skipped = failed = 0
|
||||||
total = len(pdfs)
|
total = len(pdfs)
|
||||||
for i, pdf in enumerate(pdfs, 1):
|
for i, pdf in enumerate(pdfs, 1):
|
||||||
side = pdf.with_suffix(".json")
|
side = pdf.with_suffix(".json")
|
||||||
if not args.force and has_summary(side):
|
if not args.force and has_summary(side):
|
||||||
skipped += 1
|
skipped += 1
|
||||||
continue
|
continue
|
||||||
|
try:
|
||||||
summary = summarize_one(pdf, oc)
|
summary = summarize_one(pdf, oc)
|
||||||
|
except Exception as e: # noqa: BLE001 — one paper must not abort the batch
|
||||||
|
failed += 1
|
||||||
|
console.print(f"[red]✗[/red] {pdf.name}: {e}")
|
||||||
|
continue
|
||||||
if summary is not None:
|
if summary is not None:
|
||||||
append_insight(insights, _json.loads(side.read_text()))
|
append_insight(insights, _json.loads(side.read_text()))
|
||||||
done += 1
|
done += 1
|
||||||
if i % 10 == 0:
|
if i % 10 == 0:
|
||||||
console.print(f" summarize {i}/{total} (done {done}, skipped {skipped})")
|
console.print(f" summarize {i}/{total} (done {done}, skipped {skipped}, failed {failed})")
|
||||||
console.print(f"[green]Summaries complete[/green] done={done} skipped={skipped}")
|
console.print(
|
||||||
|
f"[green]Summaries complete[/green] done={done} skipped={skipped} failed={failed}"
|
||||||
|
)
|
||||||
console.print(f"Insight digest: {insights}")
|
console.print(f"Insight digest: {insights}")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
@@ -237,10 +244,14 @@ def cmd_embed(args: argparse.Namespace) -> int:
|
|||||||
skipped += 1
|
skipped += 1
|
||||||
continue
|
continue
|
||||||
text = pdf.with_suffix(".txt").read_text()
|
text = pdf.with_suffix(".txt").read_text()
|
||||||
|
try:
|
||||||
vecs, rows = add_document(
|
vecs, rows = add_document(
|
||||||
oc, text=text, doi=doi, title=data.get("title", ""),
|
oc, text=text, doi=doi, title=data.get("title", ""),
|
||||||
topic=data.get("topic", ""), text_path=str(pdf.with_suffix(".txt")),
|
topic=data.get("topic", ""), text_path=str(pdf.with_suffix(".txt")),
|
||||||
)
|
)
|
||||||
|
except Exception as e: # noqa: BLE001 — checkpoint already saved; skip & continue
|
||||||
|
console.print(f"[red]✗[/red] {pdf.name}: {e}")
|
||||||
|
continue
|
||||||
store = merge(store, vecs, rows)
|
store = merge(store, vecs, rows)
|
||||||
seen.add(doi)
|
seen.add(doi)
|
||||||
added += 1
|
added += 1
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ for best retrieval quality; embed_documents/embed_query apply them.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
|
import time
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
@@ -90,12 +91,8 @@ class OllamaClient:
|
|||||||
payload["format"] = "json"
|
payload["format"] = "json"
|
||||||
if images:
|
if images:
|
||||||
payload["images"] = [base64.b64encode(img).decode("ascii") for img in images]
|
payload["images"] = [base64.b64encode(img).decode("ascii") for img in images]
|
||||||
try:
|
data = self._post_with_retry("/api/generate", payload)
|
||||||
r = self._client.post(f"{self.host}/api/generate", json=payload)
|
return data.get("response", "")
|
||||||
r.raise_for_status()
|
|
||||||
return r.json().get("response", "")
|
|
||||||
except httpx.HTTPError as e:
|
|
||||||
raise OllamaError(f"generate failed: {e}") from e
|
|
||||||
|
|
||||||
def caption_image(self, image: bytes, prompt: str) -> str:
|
def caption_image(self, image: bytes, prompt: str) -> str:
|
||||||
"""OCR/describe a single page image with the vision model."""
|
"""OCR/describe a single page image with the vision model."""
|
||||||
@@ -104,15 +101,24 @@ class OllamaClient:
|
|||||||
# ---- embeddings ---------------------------------------------------------
|
# ---- embeddings ---------------------------------------------------------
|
||||||
|
|
||||||
def _embed(self, inputs: list[str]) -> list[list[float]]:
|
def _embed(self, inputs: list[str]) -> list[list[float]]:
|
||||||
|
data = self._post_with_retry("/api/embed", {"model": self.embed_model, "input": inputs})
|
||||||
|
return data.get("embeddings", [])
|
||||||
|
|
||||||
|
# ---- transport with retry ----------------------------------------------
|
||||||
|
|
||||||
|
def _post_with_retry(self, path: str, payload: dict, *, attempts: int = 3) -> dict:
|
||||||
|
"""POST with retry on transient disconnects (Ollama reloads models, etc.)."""
|
||||||
|
last: Exception | None = None
|
||||||
|
for attempt in range(attempts):
|
||||||
try:
|
try:
|
||||||
r = self._client.post(
|
r = self._client.post(f"{self.host}{path}", json=payload)
|
||||||
f"{self.host}/api/embed",
|
|
||||||
json={"model": self.embed_model, "input": inputs},
|
|
||||||
)
|
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
return r.json().get("embeddings", [])
|
return r.json()
|
||||||
except httpx.HTTPError as e:
|
except httpx.HTTPError as e:
|
||||||
raise OllamaError(f"embed failed: {e}") from e
|
last = e
|
||||||
|
if attempt < attempts - 1:
|
||||||
|
time.sleep(2.0 * (attempt + 1))
|
||||||
|
raise OllamaError(f"POST {path} failed after {attempts} attempts: {last}") from last
|
||||||
|
|
||||||
def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
||||||
return self._embed([f"search_document: {t}" for t in texts])
|
return self._embed([f"search_document: {t}" for t in texts])
|
||||||
|
|||||||
+136
-21
@@ -1,21 +1,136 @@
|
|||||||
summarize 20/1202 (done 5, skipped 15)
|
summarize 230/1202 (done 4, skipped 226)
|
||||||
summarize 30/1202 (done 15, skipped 15)
|
summarize 240/1202 (done 14, skipped 226)
|
||||||
summarize 40/1202 (done 25, skipped 15)
|
summarize 250/1202 (done 24, skipped 226)
|
||||||
summarize 50/1202 (done 35, skipped 15)
|
summarize 260/1202 (done 34, skipped 226)
|
||||||
summarize 60/1202 (done 45, skipped 15)
|
summarize 270/1202 (done 44, skipped 226)
|
||||||
summarize 70/1202 (done 55, skipped 15)
|
summarize 280/1202 (done 54, skipped 226)
|
||||||
summarize 80/1202 (done 65, skipped 15)
|
summarize 290/1202 (done 64, skipped 226)
|
||||||
summarize 90/1202 (done 75, skipped 15)
|
summarize 300/1202 (done 74, skipped 226)
|
||||||
summarize 100/1202 (done 85, skipped 15)
|
summarize 310/1202 (done 84, skipped 226)
|
||||||
summarize 110/1202 (done 95, skipped 15)
|
summarize 320/1202 (done 94, skipped 226)
|
||||||
summarize 120/1202 (done 105, skipped 15)
|
summarize 330/1202 (done 104, skipped 226)
|
||||||
summarize 130/1202 (done 115, skipped 15)
|
summarize 340/1202 (done 114, skipped 226)
|
||||||
summarize 140/1202 (done 125, skipped 15)
|
summarize 350/1202 (done 124, skipped 226)
|
||||||
summarize 150/1202 (done 135, skipped 15)
|
summarize 360/1202 (done 134, skipped 226)
|
||||||
summarize 160/1202 (done 145, skipped 15)
|
summarize 370/1202 (done 144, skipped 226)
|
||||||
summarize 170/1202 (done 155, skipped 15)
|
summarize 380/1202 (done 154, skipped 226)
|
||||||
summarize 180/1202 (done 165, skipped 15)
|
summarize 390/1202 (done 164, skipped 226)
|
||||||
summarize 190/1202 (done 175, skipped 15)
|
summarize 400/1202 (done 174, skipped 226)
|
||||||
summarize 200/1202 (done 185, skipped 15)
|
summarize 410/1202 (done 184, skipped 226)
|
||||||
summarize 210/1202 (done 195, skipped 15)
|
summarize 420/1202 (done 194, skipped 226)
|
||||||
summarize 220/1202 (done 205, skipped 15)
|
summarize 430/1202 (done 204, skipped 226)
|
||||||
|
summarize 440/1202 (done 214, skipped 226)
|
||||||
|
summarize 450/1202 (done 224, skipped 226)
|
||||||
|
summarize 460/1202 (done 234, skipped 226)
|
||||||
|
summarize 470/1202 (done 244, skipped 226)
|
||||||
|
summarize 480/1202 (done 254, skipped 226)
|
||||||
|
summarize 490/1202 (done 264, skipped 226)
|
||||||
|
summarize 500/1202 (done 274, skipped 226)
|
||||||
|
summarize 510/1202 (done 284, skipped 226)
|
||||||
|
summarize 520/1202 (done 294, skipped 226)
|
||||||
|
summarize 530/1202 (done 304, skipped 226)
|
||||||
|
summarize 540/1202 (done 314, skipped 226)
|
||||||
|
summarize 550/1202 (done 324, skipped 226)
|
||||||
|
summarize 560/1202 (done 334, skipped 226)
|
||||||
|
summarize 570/1202 (done 344, skipped 226)
|
||||||
|
summarize 580/1202 (done 354, skipped 226)
|
||||||
|
summarize 590/1202 (done 364, skipped 226)
|
||||||
|
summarize 600/1202 (done 374, skipped 226)
|
||||||
|
summarize 610/1202 (done 384, skipped 226)
|
||||||
|
summarize 620/1202 (done 394, skipped 226)
|
||||||
|
summarize 630/1202 (done 404, skipped 226)
|
||||||
|
summarize 640/1202 (done 414, skipped 226)
|
||||||
|
summarize 650/1202 (done 424, skipped 226)
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions
|
||||||
|
yield
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_transports/default.py", line 250, in handle_request
|
||||||
|
resp = self._pool.handle_request(req)
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/connection_pool.py", line 256, in handle_request
|
||||||
|
raise exc from None
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/connection_pool.py", line 236, in handle_request
|
||||||
|
response = connection.handle_request(
|
||||||
|
pool_request.request
|
||||||
|
)
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/connection.py", line 103, in handle_request
|
||||||
|
return self._connection.handle_request(request)
|
||||||
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/http11.py", line 136, in handle_request
|
||||||
|
raise exc
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/http11.py", line 106, in handle_request
|
||||||
|
) = self._receive_response_headers(**kwargs)
|
||||||
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/http11.py", line 177, in _receive_response_headers
|
||||||
|
event = self._receive_event(timeout=timeout)
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/http11.py", line 231, in _receive_event
|
||||||
|
raise RemoteProtocolError(msg)
|
||||||
|
httpcore.RemoteProtocolError: Server disconnected without sending a response.
|
||||||
|
|
||||||
|
The above exception was the direct cause of the following exception:
|
||||||
|
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/Users/quantum/projects/StacksScraper/clawlibrary/ollama_client.py", line 94, in generate
|
||||||
|
r = self._client.post(f"{self.host}/api/generate", json=payload)
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_client.py", line 1144, in post
|
||||||
|
return self.request(
|
||||||
|
~~~~~~~~~~~~^
|
||||||
|
"POST",
|
||||||
|
^^^^^^^
|
||||||
|
...<11 lines>...
|
||||||
|
extensions=extensions,
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
)
|
||||||
|
^
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_client.py", line 825, in request
|
||||||
|
return self.send(request, auth=auth, follow_redirects=follow_redirects)
|
||||||
|
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_client.py", line 914, in send
|
||||||
|
response = self._send_handling_auth(
|
||||||
|
request,
|
||||||
|
...<2 lines>...
|
||||||
|
history=[],
|
||||||
|
)
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_client.py", line 942, in _send_handling_auth
|
||||||
|
response = self._send_handling_redirects(
|
||||||
|
request,
|
||||||
|
follow_redirects=follow_redirects,
|
||||||
|
history=history,
|
||||||
|
)
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_client.py", line 979, in _send_handling_redirects
|
||||||
|
response = self._send_single_request(request)
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_client.py", line 1014, in _send_single_request
|
||||||
|
response = transport.handle_request(request)
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_transports/default.py", line 249, in handle_request
|
||||||
|
with map_httpcore_exceptions():
|
||||||
|
~~~~~~~~~~~~~~~~~~~~~~~^^
|
||||||
|
File "/opt/homebrew/Cellar/[email protected]/3.14.5/Frameworks/Python.framework/Versions/3.14/lib/python3.14/contextlib.py", line 162, in __exit__
|
||||||
|
self.gen.throw(value)
|
||||||
|
~~~~~~~~~~~~~~^^^^^^^
|
||||||
|
File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions
|
||||||
|
raise mapped_exc(message) from exc
|
||||||
|
httpx.RemoteProtocolError: Server disconnected without sending a response.
|
||||||
|
|
||||||
|
The above exception was the direct cause of the following exception:
|
||||||
|
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "<frozen runpy>", line 203, in _run_module_as_main
|
||||||
|
File "<frozen runpy>", line 88, in _run_code
|
||||||
|
File "/Users/quantum/projects/StacksScraper/clawlibrary/__main__.py", line 374, in <module>
|
||||||
|
sys.exit(main())
|
||||||
|
~~~~^^
|
||||||
|
File "/Users/quantum/projects/StacksScraper/clawlibrary/__main__.py", line 367, in main
|
||||||
|
return args.func(args)
|
||||||
|
~~~~~~~~~^^^^^^
|
||||||
|
File "/Users/quantum/projects/StacksScraper/clawlibrary/__main__.py", line 202, in cmd_summarize
|
||||||
|
summary = summarize_one(pdf, oc)
|
||||||
|
File "/Users/quantum/projects/StacksScraper/clawlibrary/summarize.py", line 86, in summarize_one
|
||||||
|
summary = summarize_paper(
|
||||||
|
client,
|
||||||
|
...<2 lines>...
|
||||||
|
body=body,
|
||||||
|
)
|
||||||
|
File "/Users/quantum/projects/StacksScraper/clawlibrary/summarize.py", line 56, in summarize_paper
|
||||||
|
raw = client.generate(build_prompt(title, abstract, body), json_mode=True, num_predict=500)
|
||||||
|
File "/Users/quantum/projects/StacksScraper/clawlibrary/ollama_client.py", line 98, in generate
|
||||||
|
raise OllamaError(f"generate failed: {e}") from e
|
||||||
|
clawlibrary.ollama_client.OllamaError: generate failed: Server disconnected without sending a response.
|
||||||
|
|||||||
Reference in New Issue
Block a user