docs: tools to measure how a range reader would read HDF5
inventory.py counts the functions and call sites that take the whole file as &[u8]; range-trace (standalone crate, x86-64 Linux) records every load clawhdf5 makes from a file by mprotect + single-step, unchanged library code; libhdf5_reads.py counts libhdf5's reads through h5py's fileobj driver and prints a dataset's chunk extents. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inventory of whole-file `&[u8]` parameters in the clawhdf5 workspace.
|
||||
|
||||
Used by docs/design/range-reads.md. Run from the repository root:
|
||||
|
||||
python3 docs/design/tools/inventory.py # per-file table
|
||||
python3 docs/design/tools/inventory.py --list # every signature
|
||||
python3 docs/design/tools/inventory.py --patterns # read patterns per crate
|
||||
|
||||
A function counts as taking "the whole file" when it has a parameter named
|
||||
`file_data: &[u8]` (the repository convention), or a `data` / `file` / `buf` /
|
||||
`bytes` / `mmap` parameter of type `&[u8]` *together with* a parameter whose
|
||||
name says it is a file address (`*address*`, `*addr*`, `*offset*` of an
|
||||
integer type). The second rule is a heuristic; `--list` prints every match so
|
||||
it can be checked by eye. Code after the first `#[cfg(test)] mod ... {` in a
|
||||
file is excluded (the repository keeps unit tests at the end of each file),
|
||||
as are the tests/ and benches/ directories.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
ROOT = os.getcwd()
|
||||
FN_RE = re.compile(r"\bfn\s+([A-Za-z_][A-Za-z0-9_]*)\s*(<[^()]*?>)?\s*\(", re.S)
|
||||
PARAM_RE = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)\s*:\s*&(?:'[a-z_]+\s+)?\[u8\]")
|
||||
ADDR_RE = re.compile(r"\b([a-z_]*(?:address|addr|offset)[a-z_]*)\s*:\s*(?:u64|usize|u32)")
|
||||
WHOLE_NAMES = {"data", "file", "buf", "bytes", "file_bytes", "mmap"}
|
||||
|
||||
|
||||
def signature(src, start):
|
||||
depth, i = 0, start
|
||||
while i < len(src):
|
||||
c = src[i]
|
||||
if c == "(":
|
||||
depth += 1
|
||||
elif c == ")":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return src[start + 1 : i]
|
||||
i += 1
|
||||
return ""
|
||||
|
||||
|
||||
def non_test_source(src):
|
||||
m = re.search(r"#\[cfg\(test\)\]\s*mod\s+\w+\s*\{", src)
|
||||
return src[: m.start()] if m else src
|
||||
|
||||
|
||||
PATTERNS = [
|
||||
("`file_data` passed on (call sites)", re.compile(r"[(,]\s*&?file_data\s*[,)]")),
|
||||
("`file_data[..]` slicing", re.compile(r"\bfile_data\s*\[")),
|
||||
("open-ended `file_data[x..]`", re.compile(r"\bfile_data\s*\[[^\]]*\.\.\s*\]")),
|
||||
("`file_data.get(..)`", re.compile(r"\bfile_data\s*\.\s*get\s*\(")),
|
||||
("`file_data.len()`", re.compile(r"\bfile_data\s*\.\s*len\s*\(\)")),
|
||||
("address/offset `as usize` casts", re.compile(r"\b[a-z_]*(?:addr|address|offset)[a-z_]*\s+as\s+usize")),
|
||||
("`ObjectHeader::parse(` calls", re.compile(r"ObjectHeader::parse\s*\(")),
|
||||
("`.as_bytes()` on a file/reader", re.compile(r"\b(?:file|reader|data|self\.file|self\.data|self\.file\.data|root\.file)\s*\.\s*as_bytes\s*\(\)")),
|
||||
]
|
||||
|
||||
|
||||
def patterns():
|
||||
"""Per-crate counts of the read patterns (non-test code only)."""
|
||||
per_crate = defaultdict(lambda: [0] * len(PATTERNS))
|
||||
for crate in sorted(os.listdir(os.path.join(ROOT, "crates"))):
|
||||
srcdir = os.path.join(ROOT, "crates", crate, "src")
|
||||
for dp, _, fns in os.walk(srcdir):
|
||||
for fn in fns:
|
||||
if not fn.endswith(".rs"):
|
||||
continue
|
||||
with open(os.path.join(dp, fn), encoding="utf-8") as fh:
|
||||
src = non_test_source(fh.read())
|
||||
for i, (_, rx) in enumerate(PATTERNS):
|
||||
per_crate[crate][i] += len(rx.findall(src))
|
||||
print("| crate | " + " | ".join(n for n, _ in PATTERNS) + " |")
|
||||
print("|---|" + "---:|" * len(PATTERNS))
|
||||
tot = [0] * len(PATTERNS)
|
||||
for c, v in sorted(per_crate.items()):
|
||||
if any(v):
|
||||
print("| %s | %s |" % (c, " | ".join(str(x) for x in v)))
|
||||
tot = [a + b for a, b in zip(tot, v)]
|
||||
print("| **total** | %s |" % " | ".join("**%d**" % x for x in tot))
|
||||
|
||||
|
||||
def main():
|
||||
if "--patterns" in sys.argv:
|
||||
patterns()
|
||||
return
|
||||
per_file = defaultdict(lambda: [0, 0, 0]) # [file_data, heuristic, pub]
|
||||
rows = []
|
||||
for crate in sorted(os.listdir(os.path.join(ROOT, "crates"))):
|
||||
srcdir = os.path.join(ROOT, "crates", crate, "src")
|
||||
for dp, _, fns in os.walk(srcdir):
|
||||
for fn in sorted(fns):
|
||||
if not fn.endswith(".rs"):
|
||||
continue
|
||||
path = os.path.join(dp, fn)
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
src = non_test_source(fh.read())
|
||||
for m in FN_RE.finditer(src):
|
||||
sig = signature(src, m.end() - 1)
|
||||
params = PARAM_RE.findall(sig)
|
||||
kind = None
|
||||
if "file_data" in params:
|
||||
kind = "file_data"
|
||||
elif any(p in WHOLE_NAMES for p in params) and ADDR_RE.search(sig):
|
||||
kind = "heuristic"
|
||||
if not kind:
|
||||
continue
|
||||
rel = os.path.relpath(path, ROOT)
|
||||
line_start = src.rfind("\n", 0, m.start()) + 1
|
||||
is_pub = src[line_start : m.start()].strip().startswith("pub")
|
||||
per_file[rel][0 if kind == "file_data" else 1] += 1
|
||||
per_file[rel][2] += int(is_pub)
|
||||
line = src.count("\n", 0, m.start()) + 1
|
||||
rows.append((rel, line, m.group(1), kind, is_pub))
|
||||
if "--list" in sys.argv:
|
||||
for r in rows:
|
||||
print("%s:%d %s [%s%s]" % (r[0], r[1], r[2], r[3], ", pub" if r[4] else ""))
|
||||
return
|
||||
print("| file | `file_data` fns | other whole-slice fns (heuristic) | of which `pub` |")
|
||||
print("|---|---:|---:|---:|")
|
||||
tot = [0, 0, 0]
|
||||
for f, (a, b, p) in sorted(per_file.items(), key=lambda kv: (-(kv[1][0] + kv[1][1]), kv[0])):
|
||||
print("| %s | %d | %d | %d |" % (f, a, b, p))
|
||||
tot = [tot[0] + a, tot[1] + b, tot[2] + p]
|
||||
print("| **total** | **%d** | **%d** | **%d** |" % tuple(tot))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user