Add arXiv API search source (run --source arxiv|both)

Polite topic harvesting via arXiv's sanctioned API (export.arxiv.org), NOT
website scraping: https endpoint, ~1 req/3s spacing, 429/503 + Retry-After
backoff. Per-topic arxiv_categories; every paper gets its DataCite DOI
(10.48550/arXiv.<id>) for clean cross-source dedup. README documents the S3
bulk bucket for thousands-scale needs. 2 new tests.

Debugging note: a rapid manual test burst hit arXiv's 429 rate limit — exactly
what the 3s spacing prevents; switched http->https and added 429 handling.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>

README.md

@@ -43,6 +43,10 @@ python -m clawlibrary run --dry-run
 # Harvest the topic queue (respects max_downloads_per_session)
 python -m clawlibrary run
 
+# Harvest fresh preprints from the arXiv API (polite: ~1 req/3s, 503 backoff)
+python -m clawlibrary run --source arxiv
+python -m clawlibrary run --source both    # OpenAlex + arXiv, deduped
+
 # Single topic, capped at 3 PDFs
 python -m clawlibrary run --topic "Rust systems programming memory safety" --max 3
 
@@ -76,6 +80,20 @@ vault/
 Deduplication is by DOI (primary) then PDF-URL hash (fallback), tracked in
 `state.json`. Re-running never re-downloads; an interrupted run resumes cleanly.
 
+## arXiv harvesting
+
+`run --source arxiv` queries the **arXiv API** (`export.arxiv.org/api`) — the
+sanctioned channel, not website scraping. It stays within arXiv's request norms:
+one connection, ~1 request / 3 seconds, and `503`/`Retry-After` backoff. Each
+topic searches by keywords restricted to relevant categories (override per topic
+with `arxiv_categories: [...]`), newest-first. Every arXiv paper carries its
+DataCite DOI (`10.48550/arXiv.<id>`), so results dedupe cleanly against the rest
+of the vault — no re-downloads.
+
+**For truly bulk access (thousands of papers), do NOT crawl the site.** arXiv
+provides a [requester-pays S3 bucket](https://info.arxiv.org/help/bulk_data_s3.html)
+(`s3://arxiv/`) and a Kaggle dataset for that — no throttling, by design.
+
 ## Subscription content (TDM)
 
 For papers with no open-access copy, the legitimate path is your institution's

clawlibrary/__main__.py

@@ -71,7 +71,10 @@ def cmd_run(args: argparse.Namespace) -> int:
         elif kind == "run_suspended":
             console.print("[red]Run suspended: too many consecutive server errors.[/red]")
 
-    stats = run(config, topics, dry_run=args.dry_run, max_override=args.max, on_event=on_event)
+    stats = run(
+        config, topics, dry_run=args.dry_run, max_override=args.max,
+        on_event=on_event, source=args.source,
+    )
     _print_summary(stats, dry_run=args.dry_run)
     return 0
 
@@ -209,7 +212,7 @@ def cmd_summarize(args: argparse.Namespace) -> int:
             append_insight(insights, _json.loads(side.read_text()))
             done += 1
         if i % 10 == 0:
-            console.print(f"  summarize {i}/{total}  (done {done}, skipped {skipped}, failed {failed})")
+            console.print(f"  summarize {i}/{total}  (done {done}, skip {skipped}, fail {failed})")
     console.print(
         f"[green]Summaries complete[/green]  done={done} skipped={skipped} failed={failed}"
     )
@@ -329,6 +332,8 @@ def build_parser() -> argparse.ArgumentParser:
     pr.add_argument("--topic", help="run a single topic by exact query string")
     pr.add_argument("--dry-run", action="store_true", help="resolve only, no downloads")
     pr.add_argument("--max", type=int, default=None, help="override max downloads this run")
+    pr.add_argument("--source", choices=["openalex", "arxiv", "both"], default="openalex",
+                    help="discovery source (arxiv = polite arXiv API harvest)")
     pr.set_defaults(func=cmd_run)
 
     ps = sub.add_parser("stats", help="show vault statistics")

clawlibrary/config.py

@@ -31,6 +31,9 @@ class Topic(BaseModel):
     priority: int = 100
     enabled: bool = True
     filters: TopicFilters = Field(default_factory=TopicFilters)
+    # Optional arXiv category restriction for `run --source arxiv`
+    # (e.g. ["quant-ph"]); defaults to a broad AI+quantum set if unset.
+    arxiv_categories: list[str] | None = None
 
     @field_validator("query")
     @classmethod

clawlibrary/pipeline.py

@@ -15,11 +15,19 @@ from .dedup import StateStore
 from .downloader import download
 from .models import VaultEntry
 from .ratelimit import ConsecutiveErrorTracker, RateLimiter
-from .sources import openalex
+from .sources import arxiv_search, openalex
 from .sources.resolver import resolve_pdf_candidates
 from .vault import VaultManager, doi_slug
 
 
+def iter_candidates(client: httpx.Client, topic: Topic, config: Config, source: str):
+    """Yield candidates for a topic from the selected search provider(s)."""
+    if source in ("openalex", "both"):
+        yield from openalex.search(client, topic, config.sources.mailto)
+    if source in ("arxiv", "both"):
+        yield from arxiv_search.search(client, topic)
+
+
 def generate_run_id() -> str:
     """ISO-ish timestamp + 6-char hex suffix, e.g. 20260603T142200-4a7f3b."""
     ts = datetime.now(UTC).strftime("%Y%m%dT%H%M%S")
@@ -63,6 +71,7 @@ def run(
     max_override: int | None = None,
     on_event=None,
     transport: httpx.BaseTransport | None = None,
+    source: str = "openalex",
 ) -> RunStats:
     """Process the topic queue. `on_event(kind, **data)` is an optional UI hook.
 
@@ -84,7 +93,8 @@ def run(
     max_downloads = max_override or config.rate_limit.max_downloads_per_session
 
     headers = {"User-Agent": config.user_agent}
-    emit("run_start", dry_run=dry_run, max_downloads=max_downloads, topics=len(topics))
+    emit("run_start", dry_run=dry_run, max_downloads=max_downloads,
+         topics=len(topics), source=source)
 
     try:
         with httpx.Client(headers=headers, timeout=30.0, transport=transport) as client:
@@ -95,7 +105,7 @@ def run(
                 stats.per_topic.setdefault(topic.query, 0)
                 emit("topic_start", topic=topic.query, max_results=topic.max_results)
 
-                for candidate in openalex.search(client, topic, config.sources.mailto):
+                for candidate in iter_candidates(client, topic, config, source):
                     if stats.downloaded >= max_downloads and not dry_run:
                         emit("session_cap_reached", cap=max_downloads)
                         break

clawlibrary/sources/arxiv_search.py

@@ -0,0 +1,165 @@
+"""arXiv API search source — polite, paginated topic harvesting.
+
+This uses arXiv's sanctioned query API (export.arxiv.org/api), NOT website
+scraping. Per arXiv's guidance we keep a single connection, wait ~3s between
+requests, and honor 503/Retry-After. For *thousands* of papers, arXiv asks you
+to use the S3 bulk bucket (s3://arxiv/) instead — see README.
+
+Every arXiv work has a stable DataCite DOI of the form 10.48550/arXiv.<id>,
+which we use as the dedup key so these papers merge cleanly with the rest of
+the vault.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from collections.abc import Iterator
+from xml.etree import ElementTree as ET
+
+import httpx
+
+from ..config import Topic
+from ..models import PaperCandidate
+
+API_URL = "https://export.arxiv.org/api/query"
+_ATOM = "{http://www.w3.org/2005/Atom}"
+_ARX = "{http://arxiv.org/schemas/atom}"
+
+PAGE_SIZE = 100
+MIN_REQUEST_INTERVAL = 3.0  # arXiv asks for ~1 request / 3 seconds
+
+# Default category sets per domain keyword — keeps topic searches on-target.
+DEFAULT_CATEGORIES = ["cs.AI", "cs.LG", "cs.CL", "cs.MA", "cs.DC", "quant-ph"]
+
+
+def _bare_id(entry_id: str) -> str:
+    """Extract the version-stripped arXiv id from an entry id URL."""
+    m = re.search(r"abs/([^\s]+?)(v\d+)?$", entry_id.strip())
+    return m.group(1) if m else entry_id.rsplit("/", 1)[-1]
+
+
+def _build_query(topic: Topic) -> str:
+    categories = getattr(topic, "arxiv_categories", None) or DEFAULT_CATEGORIES
+    cat_clause = " OR ".join(f"cat:{c}" for c in categories)
+    # Quote the topic phrase for arXiv's "all" field.
+    terms = topic.query.replace('"', "").strip()
+    return f'all:"{terms}" AND ({cat_clause})'
+
+
+def _entry_to_candidate(entry: ET.Element, topic_query: str) -> PaperCandidate | None:
+    id_el = entry.find(f"{_ATOM}id")
+    if id_el is None or not id_el.text:
+        return None
+    arxiv_id = _bare_id(id_el.text)
+
+    title_el = entry.find(f"{_ATOM}title")
+    title = (title_el.text or "").strip().replace("\n", " ") if title_el is not None else ""
+
+    authors = [
+        a.find(f"{_ATOM}name").text
+        for a in entry.findall(f"{_ATOM}author")
+        if a.find(f"{_ATOM}name") is not None
+    ]
+    summary_el = entry.find(f"{_ATOM}summary")
+    abstract = (summary_el.text or "").strip() if summary_el is not None else None
+
+    published = entry.find(f"{_ATOM}published")
+    year = None
+    if published is not None and published.text:
+        try:
+            year = int(published.text[:4])
+        except ValueError:
+            year = None
+
+    # Prefer a published-journal DOI if arXiv has one, else the arXiv DataCite DOI.
+    doi_el = entry.find(f"{_ARX}doi")
+    doi = (doi_el.text.strip() if doi_el is not None and doi_el.text else
+           f"10.48550/arxiv.{arxiv_id}")
+
+    journal_el = entry.find(f"{_ARX}journal_ref")
+    journal = journal_el.text.strip() if journal_el is not None and journal_el.text else "arXiv"
+
+    pdf_url = None
+    for link in entry.findall(f"{_ATOM}link"):
+        if link.get("title") == "pdf" or link.get("type") == "application/pdf":
+            href = link.get("href")
+            pdf_url = href if href.endswith(".pdf") else f"{href}.pdf"
+            break
+    if not pdf_url:
+        pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
+
+    return PaperCandidate(
+        title=title or "(untitled)",
+        authors=[a for a in authors if a],
+        doi=doi.lower(),
+        journal=journal,
+        year=year,
+        abstract=abstract,
+        is_open_access=True,
+        record_url=f"https://arxiv.org/abs/{arxiv_id}",
+        resolved_pdf_url=pdf_url,
+        pdf_source="arxiv",
+        topic=topic_query,
+    )
+
+
+def search(
+    client: httpx.Client,
+    topic: Topic,
+    *,
+    sleeper=time.sleep,
+    min_interval: float = MIN_REQUEST_INTERVAL,
+) -> Iterator[PaperCandidate]:
+    """Yield up to topic.max_results candidates from the arXiv API, politely.
+
+    Pages newest-first; sleeps `min_interval` between page requests and backs
+    off on 503 (honoring Retry-After) so we never trip arXiv's rate limits.
+    """
+    query = _build_query(topic)
+    date_from = topic.filters.date_from
+    yielded = 0
+    start = 0
+    first = True
+
+    while yielded < topic.max_results:
+        if not first:
+            sleeper(min_interval)
+        first = False
+        params = {
+            "search_query": query,
+            "start": start,
+            "max_results": min(PAGE_SIZE, topic.max_results - yielded),
+            "sortBy": "submittedDate",
+            "sortOrder": "descending",
+        }
+        try:
+            resp = client.get(API_URL, params=params)
+        except httpx.HTTPError:
+            return
+        if resp.status_code in (429, 503):
+            # arXiv returns 429 "Rate exceeded" on bursts; back off and retry.
+            retry = float(resp.headers.get("Retry-After", min_interval * 2))
+            sleeper(min(max(retry, min_interval), 60.0))
+            continue
+        if resp.status_code != 200:
+            return
+        try:
+            root = ET.fromstring(resp.text)
+        except ET.ParseError:
+            return
+
+        entries = root.findall(f"{_ATOM}entry")
+        if not entries:
+            return
+        for entry in entries:
+            cand = _entry_to_candidate(entry, topic.query)
+            if cand is None:
+                continue
+            if date_from and cand.year and cand.year < date_from:
+                continue
+            yield cand
+            yielded += 1
+            if yielded >= topic.max_results:
+                return
+        start += len(entries)

summarize.out

@@ -1,136 +1,39 @@
-  summarize 230/1202  (done 4, skipped 226)
+  summarize 840/1202  (done 3, skipped 837, failed 0)
-  summarize 240/1202  (done 14, skipped 226)
+  summarize 850/1202  (done 13, skipped 837, failed 0)
-  summarize 250/1202  (done 24, skipped 226)
+  summarize 860/1202  (done 23, skipped 837, failed 0)
-  summarize 260/1202  (done 34, skipped 226)
+  summarize 870/1202  (done 33, skipped 837, failed 0)
-  summarize 270/1202  (done 44, skipped 226)
+  summarize 880/1202  (done 43, skipped 837, failed 0)
-  summarize 280/1202  (done 54, skipped 226)
+  summarize 890/1202  (done 53, skipped 837, failed 0)
-  summarize 290/1202  (done 64, skipped 226)
+  summarize 900/1202  (done 63, skipped 837, failed 0)
-  summarize 300/1202  (done 74, skipped 226)
+  summarize 910/1202  (done 73, skipped 837, failed 0)
-  summarize 310/1202  (done 84, skipped 226)
+  summarize 920/1202  (done 83, skipped 837, failed 0)
-  summarize 320/1202  (done 94, skipped 226)
+  summarize 930/1202  (done 93, skipped 837, failed 0)
-  summarize 330/1202  (done 104, skipped 226)
+  summarize 940/1202  (done 103, skipped 837, failed 0)
-  summarize 340/1202  (done 114, skipped 226)
+  summarize 950/1202  (done 113, skipped 837, failed 0)
-  summarize 350/1202  (done 124, skipped 226)
+  summarize 960/1202  (done 123, skipped 837, failed 0)
-  summarize 360/1202  (done 134, skipped 226)
+  summarize 970/1202  (done 133, skipped 837, failed 0)
-  summarize 370/1202  (done 144, skipped 226)
+  summarize 980/1202  (done 143, skipped 837, failed 0)
-  summarize 380/1202  (done 154, skipped 226)
+  summarize 990/1202  (done 153, skipped 837, failed 0)
-  summarize 390/1202  (done 164, skipped 226)
+  summarize 1000/1202  (done 163, skipped 837, failed 0)
-  summarize 400/1202  (done 174, skipped 226)
+  summarize 1010/1202  (done 173, skipped 837, failed 0)
-  summarize 410/1202  (done 184, skipped 226)
+  summarize 1020/1202  (done 183, skipped 837, failed 0)
-  summarize 420/1202  (done 194, skipped 226)
+  summarize 1030/1202  (done 193, skipped 837, failed 0)
-  summarize 430/1202  (done 204, skipped 226)
+  summarize 1040/1202  (done 203, skipped 837, failed 0)
-  summarize 440/1202  (done 214, skipped 226)
+  summarize 1050/1202  (done 213, skipped 837, failed 0)
-  summarize 450/1202  (done 224, skipped 226)
+  summarize 1060/1202  (done 223, skipped 837, failed 0)
-  summarize 460/1202  (done 234, skipped 226)
+  summarize 1070/1202  (done 233, skipped 837, failed 0)
-  summarize 470/1202  (done 244, skipped 226)
+  summarize 1080/1202  (done 243, skipped 837, failed 0)
-  summarize 480/1202  (done 254, skipped 226)
+  summarize 1090/1202  (done 253, skipped 837, failed 0)
-  summarize 490/1202  (done 264, skipped 226)
+  summarize 1100/1202  (done 263, skipped 837, failed 0)
-  summarize 500/1202  (done 274, skipped 226)
+  summarize 1110/1202  (done 273, skipped 837, failed 0)
-  summarize 510/1202  (done 284, skipped 226)
+  summarize 1120/1202  (done 283, skipped 837, failed 0)
-  summarize 520/1202  (done 294, skipped 226)
+  summarize 1130/1202  (done 293, skipped 837, failed 0)
-  summarize 530/1202  (done 304, skipped 226)
+  summarize 1140/1202  (done 303, skipped 837, failed 0)
-  summarize 540/1202  (done 314, skipped 226)
+  summarize 1150/1202  (done 313, skipped 837, failed 0)
-  summarize 550/1202  (done 324, skipped 226)
+  summarize 1160/1202  (done 323, skipped 837, failed 0)
-  summarize 560/1202  (done 334, skipped 226)
+  summarize 1170/1202  (done 333, skipped 837, failed 0)
-  summarize 570/1202  (done 344, skipped 226)
+  summarize 1180/1202  (done 343, skipped 837, failed 0)
-  summarize 580/1202  (done 354, skipped 226)
+  summarize 1190/1202  (done 353, skipped 837, failed 0)
-  summarize 590/1202  (done 364, skipped 226)
+  summarize 1200/1202  (done 363, skipped 837, failed 0)
-  summarize 600/1202  (done 374, skipped 226)
+Summaries complete  done=365 skipped=837 failed=0
-  summarize 610/1202  (done 384, skipped 226)
+Insight digest: vault/insights.md
-  summarize 620/1202  (done 394, skipped 226)
-  summarize 630/1202  (done 404, skipped 226)
-  summarize 640/1202  (done 414, skipped 226)
-  summarize 650/1202  (done 424, skipped 226)
-Traceback (most recent call last):
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions
-    yield
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_transports/default.py", line 250, in handle_request
-    resp = self._pool.handle_request(req)
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/connection_pool.py", line 256, in handle_request
-    raise exc from None
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/connection_pool.py", line 236, in handle_request
-    response = connection.handle_request(
-        pool_request.request
-    )
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/connection.py", line 103, in handle_request
-    return self._connection.handle_request(request)
-           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/http11.py", line 136, in handle_request
-    raise exc
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/http11.py", line 106, in handle_request
-    ) = self._receive_response_headers(**kwargs)
-        ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/http11.py", line 177, in _receive_response_headers
-    event = self._receive_event(timeout=timeout)
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpcore/_sync/http11.py", line 231, in _receive_event
-    raise RemoteProtocolError(msg)
-httpcore.RemoteProtocolError: Server disconnected without sending a response.
-
-The above exception was the direct cause of the following exception:
-
-Traceback (most recent call last):
-  File "/Users/quantum/projects/StacksScraper/clawlibrary/ollama_client.py", line 94, in generate
-    r = self._client.post(f"{self.host}/api/generate", json=payload)
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_client.py", line 1144, in post
-    return self.request(
-           ~~~~~~~~~~~~^
-        "POST",
-        ^^^^^^^
-    ...<11 lines>...
-        extensions=extensions,
-        ^^^^^^^^^^^^^^^^^^^^^^
-    )
-    ^
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_client.py", line 825, in request
-    return self.send(request, auth=auth, follow_redirects=follow_redirects)
-           ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_client.py", line 914, in send
-    response = self._send_handling_auth(
-        request,
-    ...<2 lines>...
-        history=[],
-    )
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_client.py", line 942, in _send_handling_auth
-    response = self._send_handling_redirects(
-        request,
-        follow_redirects=follow_redirects,
-        history=history,
-    )
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_client.py", line 979, in _send_handling_redirects
-    response = self._send_single_request(request)
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_client.py", line 1014, in _send_single_request
-    response = transport.handle_request(request)
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_transports/default.py", line 249, in handle_request
-    with map_httpcore_exceptions():
-         ~~~~~~~~~~~~~~~~~~~~~~~^^
-  File "/opt/homebrew/Cellar/[email protected]/3.14.5/Frameworks/Python.framework/Versions/3.14/lib/python3.14/contextlib.py", line 162, in __exit__
-    self.gen.throw(value)
-    ~~~~~~~~~~~~~~^^^^^^^
-  File "/Users/quantum/projects/StacksScraper/.venv/lib/python3.14/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions
-    raise mapped_exc(message) from exc
-httpx.RemoteProtocolError: Server disconnected without sending a response.
-
-The above exception was the direct cause of the following exception:
-
-Traceback (most recent call last):
-  File "<frozen runpy>", line 203, in _run_module_as_main
-  File "<frozen runpy>", line 88, in _run_code
-  File "/Users/quantum/projects/StacksScraper/clawlibrary/__main__.py", line 374, in <module>
-    sys.exit(main())
-             ~~~~^^
-  File "/Users/quantum/projects/StacksScraper/clawlibrary/__main__.py", line 367, in main
-    return args.func(args)
-           ~~~~~~~~~^^^^^^
-  File "/Users/quantum/projects/StacksScraper/clawlibrary/__main__.py", line 202, in cmd_summarize
-    summary = summarize_one(pdf, oc)
-  File "/Users/quantum/projects/StacksScraper/clawlibrary/summarize.py", line 86, in summarize_one
-    summary = summarize_paper(
-        client,
-    ...<2 lines>...
-        body=body,
-    )
-  File "/Users/quantum/projects/StacksScraper/clawlibrary/summarize.py", line 56, in summarize_paper
-    raw = client.generate(build_prompt(title, abstract, body), json_mode=True, num_predict=500)
-  File "/Users/quantum/projects/StacksScraper/clawlibrary/ollama_client.py", line 98, in generate
-    raise OllamaError(f"generate failed: {e}") from e
-clawlibrary.ollama_client.OllamaError: generate failed: Server disconnected without sending a response.

tests/test_extract_insight.py

@@ -10,7 +10,6 @@ from clawlibrary.ollama_client import OllamaClient
 from clawlibrary.rag import ask, retrieve
 from clawlibrary.summarize import build_prompt, summarize_paper
 
-
 # ---- a fake Ollama client for offline tests --------------------------------
 _VOCAB = [
     "rust", "memory", "safety", "ownership", "systems", "programming",

tests/test_sources.py

@@ -1,7 +1,7 @@
 import httpx
 
 from clawlibrary.config import SourcesConfig, Topic
-from clawlibrary.sources import openalex, unpaywall
+from clawlibrary.sources import arxiv_search, openalex, unpaywall
 from clawlibrary.sources.base import normalize_doi, reconstruct_abstract
 from clawlibrary.sources.resolver import resolve_pdf_url
 
@@ -59,6 +59,71 @@ def test_unpaywall_resolve():
     assert url == "https://oa/x.pdf"
 
 
+_ARXIV_ATOM = """<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom" xmlns:arxiv="http://arxiv.org/schemas/atom">
+  <entry>
+    <id>http://arxiv.org/abs/2401.01234v2</id>
+    <published>2024-01-03T00:00:00Z</published>
+    <title>Agentic Planning for Tool Use</title>
+    <summary>We study how LLM agents plan.</summary>
+    <author><name>Ada Lovelace</name></author>
+    <author><name>Alan Turing</name></author>
+    <link href="http://arxiv.org/abs/2401.01234v2" rel="alternate" type="text/html"/>
+    <link title="pdf" href="http://arxiv.org/pdf/2401.01234v2" type="application/pdf"/>
+    <arxiv:primary_category term="cs.AI"/>
+  </entry>
+  <entry>
+    <id>http://arxiv.org/abs/1901.00001v1</id>
+    <published>2019-01-01T00:00:00Z</published>
+    <title>Old Paper</title>
+    <summary>Older work.</summary>
+    <author><name>Someone</name></author>
+    <link title="pdf" href="http://arxiv.org/pdf/1901.00001v1" type="application/pdf"/>
+  </entry>
+</feed>"""
+
+
+def test_arxiv_search_parses_and_filters_by_year():
+    def handler(request):
+        # second page empty so pagination terminates
+        if "start=0" in str(request.url) or request.url.params.get("start") in (None, "0"):
+            return httpx.Response(200, text=_ARXIV_ATOM)
+        return httpx.Response(200, text='<feed xmlns="http://www.w3.org/2005/Atom"></feed>')
+
+    topic = Topic(query="agentic planning", max_results=10)
+    topic.filters.date_from = 2020
+    with httpx.Client(transport=httpx.MockTransport(handler)) as client:
+        results = list(arxiv_search.search(client, topic, sleeper=lambda s: None))
+
+    # Old 2019 paper filtered out by date_from=2020
+    assert len(results) == 1
+    c = results[0]
+    assert c.title == "Agentic Planning for Tool Use"
+    assert c.doi == "10.48550/arxiv.2401.01234"  # version-stripped DataCite DOI
+    assert c.pdf_source == "arxiv"
+    assert c.resolved_pdf_url.endswith(".pdf")
+    assert c.is_open_access is True
+    assert c.authors == ["Ada Lovelace", "Alan Turing"]
+
+
+def test_arxiv_search_handles_503_then_succeeds():
+    calls = {"n": 0}
+
+    def handler(request):
+        calls["n"] += 1
+        if calls["n"] == 1:
+            return httpx.Response(503, headers={"Retry-After": "0"})
+        if calls["n"] == 2:
+            return httpx.Response(200, text=_ARXIV_ATOM)
+        return httpx.Response(200, text='<feed xmlns="http://www.w3.org/2005/Atom"></feed>')
+
+    topic = Topic(query="x", max_results=5)
+    with httpx.Client(transport=httpx.MockTransport(handler)) as client:
+        results = list(arxiv_search.search(client, topic, sleeper=lambda s: None))
+    assert len(results) == 2  # both entries (no date filter)
+    assert calls["n"] >= 2
+
+
 def test_resolver_trusts_openalex_first(candidate):
     # The list-based resolver collects alternatives too, but OpenAlex's own link
     # must come first. Other sources are disabled here so no HTTP is needed.