quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 1 Wiki Activity

h5rs: URLs as FILE arguments (feature remote)

Browse Source 
With the `remote` feature (`remote-https` for https://), ls, dump, stat
and diff take an http(s):// (or s3://, gs://, az:// with those
clawhdf5-remote features) URL wherever they take a file, and read it by
range requests through clawhdf5-remote's block cache. check validates
every byte, so it downloads a remote file whole and checks it as before.
Without the feature a URL is a clean error naming it.

The tools read the file through File::storage instead of as_bytes: object
headers, shared messages, attributes, v1 and v2 group links, dense
storage (fractal heaps and v2 B-trees), path resolution, chunk listings
and variable-length values go through the format crate's *_in functions,
and the fractal-heap block verifier reads each block through the storage
(a read failure of a remote file is reported as a problem, not as "past
the end of the file"). A local file's storage is its mapped bytes, so its
reads are still slices. stat's file size comes from the opened file, so
it is right for a URL.

Tests: tests/remote.rs serves fixtures (old and new formats, a paged
file, a metadata cache image, a multi-block fractal heap, compounds, v1
groups) with the clawhdf5-remote test server and requires every
subcommand's output and exit status for the URL to equal the local
file's, and diff of the two to be clean; 404s, non-HDF5 bodies and
https without its feature are clean errors. Local output is unchanged:
the old and new h5rs print the same for ls -r -v, dump, stat and check
--data on the 747 conformance and CVE corpus files (tank, 2026-09-26;
the dumps of h5diff_hyper1/2.h5 were too large for the comparison
script, their ls, stat and check agree), except cve-2025-2310.h5, whose
dump error messages differ between runs of the old binary too (which
failing chunk is reported first).

ci-test.sh lints h5rs with remote-https, runs the URL tests and checks
h5rs with remote for C.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 17:23:53 -05:00
co-authored by Claude Opus 5.5
parent a4f586e657
commit c513f7e6d7
13 changed files with 357 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-tools/src/heap_blocks.rs
+68 -14
View File
@@ -4,10 +4,12 @@
//! heap header's flag says so). The library reads only the blocks an object
//! lives in and does not verify block checksums, so `check` does it here.
use std::borrow::Cow;
use std::collections::HashSet;
use clawhdf5_format::checksum::jenkins_lookup3;
use clawhdf5_format::fractal_heap::FractalHeapHeader;
use clawhdf5_format::storage::{Storage, read_exact_at};
use crate::h5::{Error, H5};
@@ -26,7 +28,7 @@ pub struct HeapReport {
}
struct Walk<'a> {
data: &'a [u8],
data: &'a dyn Storage,
heap: u64,
fh: FractalHeapHeader,
checksum_dblocks: bool,
@@ -60,14 +62,14 @@ fn log2(v: u64) -> u32 {
/// parsed (and its checksum verified) by the library; an error there is
/// returned as the only problem.
pub fn verify(h5: &H5, heap: u64) -> HeapReport {
let data = h5.data();
let data = h5.store();
let Ok(off) = usize::try_from(heap) else {
return HeapReport {
problems: vec![Error::at(heap, "fractal heap address out of range")],
..Default::default()
};
};
let fh = match FractalHeapHeader::parse(data, off, h5.os(), h5.ls()) {
let fh = match FractalHeapHeader::parse_in(data, heap, h5.os(), h5.ls()) {
Ok(f) => f,
Err(e) => {
return HeapReport {
@@ -77,7 +79,15 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport {
}
};
// Flags: signature(4) version(1) heap ID length(2) filter length(2) flags(1).
let flags = data.get(off + 9).copied().unwrap_or(0);
let flags = match data.read_at(off as u64 + 9, 1) {
Ok(b) => b.first().copied().unwrap_or(0),
Err(e) => {
return HeapReport {
problems: vec![Error::at(heap, format!("fractal heap header: {e}"))],
..Default::default()
};
}
};
let mut w = Walk {
data,
heap,
@@ -107,11 +117,42 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport {
w.r
}
impl Walk<'_> {
impl<'a> Walk<'a> {
fn problem(&mut self, addr: u64, msg: impl Into<String>) {
self.r.problems.push(Error::at(addr, msg));
}
/// Bytes `[start, end)` of the file: `Ok(None)` when they run past its
/// end (what a slice `get` of the whole file answered), `Err` when the
/// storage fails to read them (a remote file).
fn get(&self, start: usize, end: usize) -> Result<Option<Cow<'a, [u8]>>, String> {
let Some(len) = end.checked_sub(start) else {
return Ok(None);
};
if end as u64 > self.data.len() {
return Ok(None);
}
read_exact_at(self.data, start as u64, len)
.map(Some)
.map_err(|e| e.to_string())
}
/// [`Walk::get`], recording a read failure as a problem at `addr`.
fn get_or_note(
&mut self,
addr: u64,
start: usize,
end: usize,
) -> Option<Option<Cow<'a, [u8]>>> {
match self.get(start, end) {
Ok(b) => Some(b),
Err(e) => {
self.problem(addr, format!("fractal heap block: {e}"));
None
}
}
}
fn row_size(&self, row: usize) -> Option<u64> {
let s = self.fh.starting_block_size;
if row <= 1 {
@@ -155,10 +196,11 @@ impl Walk<'_> {
return None;
};
let hdr_len = 5 + self.os + self.boff_bytes;
let Some(b) = start
.checked_add(hdr_len)
.and_then(|e| self.data.get(start..e))
else {
let b = match start.checked_add(hdr_len) {
Some(e) => self.get_or_note(addr, start, e)?,
None => None,
};
let Some(b) = b else {
self.problem(
addr,
format!("fractal heap {what} block lies past the end of the file"),
@@ -209,7 +251,10 @@ impl Walk<'_> {
self.problem(addr, "fractal heap direct block size out of range");
return;
};
let Some(block) = self.data.get(start..end) else {
let Some(block) = self.get_or_note(addr, start, end) else {
return;
};
let Some(block) = block else {
self.problem(
addr,
"fractal heap direct block extends past the end of the file",
@@ -259,14 +304,17 @@ impl Walk<'_> {
};
let direct = row < direct_rows;
for _ in 0..width {
let Some(b) = self.data.get(pos..pos + self.os) else {
let Some(b) = self.get_or_note(addr, pos, pos + self.os) else {
return;
};
let Some(b) = b else {
self.problem(
addr,
"fractal heap indirect block extends past the end of the file",
);
return;
};
let child = le(b);
let child = le(&b);
pos += self.os;
if direct && filtered {
pos += self.ls + 4;
@@ -277,7 +325,10 @@ impl Walk<'_> {
off = off.saturating_add(rs);
}
}
let Some(stored) = self.data.get(pos..pos + 4) else {
let Some(stored) = self.get_or_note(addr, pos, pos + 4) else {
return;
};
let Some(stored) = stored else {
self.problem(
addr,
"fractal heap indirect block extends past the end of the file",
@@ -285,7 +336,10 @@ impl Walk<'_> {
return;
};
let stored = u32::from_le_bytes([stored[0], stored[1], stored[2], stored[3]]);
let computed = jenkins_lookup3(&self.data[start..pos]);
let Some(Some(body)) = self.get_or_note(addr, start, pos) else {
return;
};
let computed = jenkins_lookup3(&body);
self.r.checksums += 1;
if computed != stored {
self.problem(