quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

h5rs: URLs as FILE arguments (feature remote)

Browse Source 
With the `remote` feature (`remote-https` for https://), ls, dump, stat
and diff take an http(s):// (or s3://, gs://, az:// with those
clawhdf5-remote features) URL wherever they take a file, and read it by
range requests through clawhdf5-remote's block cache. check validates
every byte, so it downloads a remote file whole and checks it as before.
Without the feature a URL is a clean error naming it.

The tools read the file through File::storage instead of as_bytes: object
headers, shared messages, attributes, v1 and v2 group links, dense
storage (fractal heaps and v2 B-trees), path resolution, chunk listings
and variable-length values go through the format crate's *_in functions,
and the fractal-heap block verifier reads each block through the storage
(a read failure of a remote file is reported as a problem, not as "past
the end of the file"). A local file's storage is its mapped bytes, so its
reads are still slices. stat's file size comes from the opened file, so
it is right for a URL.

Tests: tests/remote.rs serves fixtures (old and new formats, a paged
file, a metadata cache image, a multi-block fractal heap, compounds, v1
groups) with the clawhdf5-remote test server and requires every
subcommand's output and exit status for the URL to equal the local
file's, and diff of the two to be clean; 404s, non-HDF5 bodies and
https without its feature are clean errors. Local output is unchanged:
the old and new h5rs print the same for ls -r -v, dump, stat and check
--data on the 747 conformance and CVE corpus files (tank, 2026-09-26;
the dumps of h5diff_hyper1/2.h5 were too large for the comparison
script, their ls, stat and check agree), except cve-2025-2310.h5, whose
dump error messages differ between runs of the old binary too (which
failing chunk is reported first).

ci-test.sh lints h5rs with remote-https, runs the URL tests and checks
h5rs with remote for C.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 17:23:53 -05:00
co-authored by Claude Opus 5.5
parent a4f586e657
commit c513f7e6d7
13 changed files with 357 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-tools/src/check.rs
+18 -7
View File
@@ -148,13 +148,24 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
return args.usage_error(out, "missing FILE", USAGE);
};
let path = std::path::Path::new(&file);
if !path.is_file() {
writeln!(out.e, "h5rs check: {file}: no such file")?;
return Ok(2);
}
let mut h5 = match H5::open(path) {
Ok(h) => h,
Err(_) => return unopenable(path, out),
let mut h5 = if crate::h5::is_url(&file) {
// check validates every byte, so a remote file is downloaded whole.
match H5::open_arg_whole(&file) {
Ok(h) => h,
Err(e) => {
writeln!(out.e, "h5rs check: {e}")?;
return Ok(2);
}
}
} else {
if !path.is_file() {
writeln!(out.e, "h5rs check: {file}: no such file")?;
return Ok(2);
}
match H5::open(path) {
Ok(h) => h,
Err(_) => return unopenable(path, out),
}
};
if let Some(m) = max_bytes {
h5.max_bytes = m;
crates/clawhdf5-tools/src/diff.rs
+1 -1
View File
@@ -174,7 +174,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
}
let mut files = Vec::new();
for f in &pos[..2] {
match H5::open(std::path::Path::new(f)) {
match H5::open_arg(f) {
Ok(mut h) => {
if let Some(m) = max_bytes {
h.max_bytes = m;
crates/clawhdf5-tools/src/dump.rs
+1 -1
View File
@@ -82,7 +82,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
let Some(file) = file else {
return args.usage_error(out, "missing FILE", USAGE);
};
let mut h5 = match H5::open(std::path::Path::new(&file)) {
let mut h5 = match H5::open_arg(&file) {
Ok(h) => h,
Err(e) => {
writeln!(out.e, "h5rs dump: {e}")?;
crates/clawhdf5-tools/src/h5.rs
+115 -24
View File
@@ -10,9 +10,9 @@ use std::collections::HashMap;
use std::path::{Path, PathBuf};
use clawhdf5::File;
use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant};
use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant_in};
use clawhdf5_format::attribute_info::AttributeInfoMessage;
use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records};
use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records_in};
use clawhdf5_format::data_layout::DataLayout;
use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
use clawhdf5_format::datatype::Datatype;
@@ -24,6 +24,7 @@ use clawhdf5_format::link_info::LinkInfoMessage;
use clawhdf5_format::link_message::{LinkMessage, LinkTarget};
use clawhdf5_format::message_type::MessageType;
use clawhdf5_format::object_header::ObjectHeader;
use clawhdf5_format::storage::Storage;
use clawhdf5_format::superblock::Superblock;
use clawhdf5_format::symbol_table::SymbolTableMessage;
@@ -186,8 +187,11 @@ pub struct Link {
/// An open file.
pub struct H5 {
/// The file's path, or its URL for a remote file.
pub path: PathBuf,
pub file: File,
/// Size of the whole file in bytes (user block included).
pub size: u64,
pub max_bytes: u64,
/// Fractal heaps whose blocks were verified: `None` = sound.
verified_heaps: RefCell<HashMap<u64, Option<Error>>>,
@@ -204,19 +208,90 @@ impl H5 {
path.display()
))
})?;
Ok(H5 {
path: path.to_path_buf(),
let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
Ok(H5::new(path.to_path_buf(), file, size))
}
fn new(path: PathBuf, file: File, size: u64) -> H5 {
H5 {
path,
file,
size,
max_bytes: DEFAULT_MAX_BYTES,
verified_heaps: RefCell::new(HashMap::new()),
})
}
}
/// Open a command-line FILE argument: a path, or with the `remote`
/// feature an `http(s)://` (or `s3://`, `gs://`, `az://`) URL, read by
/// range requests through a block cache.
pub fn open_arg(arg: &str) -> Result<H5> {
if !is_url(arg) {
return H5::open(Path::new(arg));
}
#[cfg(feature = "remote")]
{
let storage =
clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default())
.map_err(|e| Error::new(format!("{arg}: {e}")))?;
let size = storage.len();
let file = File::open_storage(storage).map_err(|e| {
Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}"))
})?;
Ok(H5::new(PathBuf::from(arg), file, size))
}
#[cfg(not(feature = "remote"))]
Err(Error::new(format!(
"{arg}: URLs need h5rs built with the `remote` feature"
)))
}
/// [`H5::open_arg`], with a remote file downloaded whole into memory
/// first — for `check`, which validates every byte of the file anyway
/// and parses it as one slice ([`H5::data`]).
pub fn open_arg_whole(arg: &str) -> Result<H5> {
let h5 = H5::open_arg(arg)?;
if h5.file.contiguous_bytes().is_some() {
return Ok(h5);
}
#[cfg(feature = "remote")]
{
let storage =
clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default())
.map_err(|e| Error::new(format!("{arg}: {e}")))?;
let len = usize::try_from(storage.len())
.map_err(|_| Error::new(format!("{arg}: too large to download")))?;
let bytes = storage
.read_at(0, len)
.map_err(|e| Error::new(format!("{arg}: {e}")))?
.into_owned();
let size = bytes.len() as u64;
let file = File::from_bytes(bytes).map_err(|e| {
Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}"))
})?;
Ok(H5::new(PathBuf::from(arg), file, size))
}
#[cfg(not(feature = "remote"))]
unreachable!("open_arg refuses URLs without the remote feature")
}
/// The file's bytes from the superblock on: what every address indexes.
///
/// # Panics
///
/// For a remote file opened by [`H5::open_arg`]; read it through
/// [`H5::store`], or open it with [`H5::open_arg_whole`].
pub fn data(&self) -> &[u8] {
self.file.as_bytes()
}
/// The same bytes as a [`Storage`], for local and remote files alike:
/// in memory a read is a slice of [`H5::data`], remote it is served by
/// the block cache.
pub fn store(&self) -> &dyn Storage {
self.file.storage()
}
pub fn sb(&self) -> &Superblock {
self.file.superblock()
}
@@ -239,8 +314,8 @@ impl H5 {
if let Some(e) = self.file.cache_image_error() {
return Err(Error::at(addr, format!("metadata cache image: {e}")));
}
let off = usize::try_from(addr).map_err(|_| Error::at(addr, "address out of range"))?;
ObjectHeader::parse(self.data(), off, self.os(), self.ls())
to_usize(addr)?;
ObjectHeader::parse_in(self.store(), addr, self.os(), self.ls())
.map_err(|e| Error::at(addr, format!("object header: {e}")))
}
@@ -249,11 +324,14 @@ impl H5 {
pub fn payload(&self, h: &ObjectHeader, t: MessageType) -> Result<Option<Vec<u8>>> {
match h.messages.iter().find(|m| m.msg_type == t) {
None => Ok(None),
Some(m) => {
clawhdf5_format::shared_message::message_data(self.data(), m, self.os(), self.ls())
.map(|c| Some(c.into_owned()))
.map_err(|e| Error::new(format!("{t:?} message: {e}")))
}
Some(m) => clawhdf5_format::shared_message::message_data_in(
self.store(),
m,
self.os(),
self.ls(),
)
.map(|c| Some(c.into_owned()))
.map_err(|e| Error::new(format!("{t:?} message: {e}"))),
}
}
@@ -305,8 +383,9 @@ impl H5 {
self.verified_heap(fh)
.map_err(|e| e.context("dense attribute storage"))?;
}
let (mut attrs, errs) = extract_attributes_tolerant(self.data(), h, self.os(), self.ls())
.map_err(|e| Error::new(format!("attributes: {e}")))?;
let (mut attrs, errs) =
extract_attributes_tolerant_in(self.store(), h, self.os(), self.ls())
.map_err(|e| Error::new(format!("attributes: {e}")))?;
attrs.sort_by(|a, b| a.name.cmp(&b.name));
Ok((attrs, errs.iter().map(|e| e.to_string()).collect()))
}
@@ -316,7 +395,7 @@ impl H5 {
pub fn links(&self, h: &ObjectHeader) -> Result<Vec<Link>> {
let os = self.os();
let ls = self.ls();
let data = self.data();
let data = self.store();
let mut out = Vec::new();
if let Some(m) = h
.messages
@@ -325,7 +404,7 @@ impl H5 {
{
let stm = SymbolTableMessage::parse(&m.data, os)
.map_err(|e| Error::new(format!("symbol table message: {e}")))?;
let entries = group_v1::resolve_v1_group_entries(data, &stm, os, ls)
let entries = group_v1::resolve_v1_group_entries_in(data, &stm, os, ls)
.map_err(|e| Error::at(stm.btree_address, format!("symbol table: {e}")))?;
let has_soft = entries.iter().any(group_v1::is_v1_soft_link);
for e in entries {
@@ -337,7 +416,7 @@ impl H5 {
}
}
if has_soft {
let soft = group_v1::v1_soft_links(data, &stm, os, ls)
let soft = group_v1::v1_soft_links_in(data, &stm, os, ls)
.map_err(|e| Error::at(stm.btree_address, format!("soft links: {e}")))?;
for (name, target) in soft {
out.push(Link {
@@ -387,15 +466,15 @@ impl H5 {
name_type: u8,
) -> Result<Vec<Vec<u8>>> {
self.verified_heap(heap)?;
let data = self.data();
let data = self.store();
let os = self.os();
let ls = self.ls();
let fh = FractalHeapHeader::parse(data, to_usize(heap)?, os, ls)
let fh = FractalHeapHeader::parse_in(data, to_usize(heap).map(|_| heap)?, os, ls)
.map_err(|e| Error::at(heap, format!("fractal heap header: {e}")))?;
let bt = btree.ok_or_else(|| Error::at(heap, "dense storage without a name index"))?;
let hdr = BTreeV2Header::parse(data, to_usize(bt)?, os, ls)
let hdr = BTreeV2Header::parse_in(data, to_usize(bt).map(|_| bt)?, os, ls)
.map_err(|e| Error::at(bt, format!("v2 B-tree header: {e}")))?;
let recs = collect_btree_v2_records(data, &hdr, os, ls)
let recs = collect_btree_v2_records_in(data, &hdr, os, ls)
.map_err(|e| Error::at(bt, format!("v2 B-tree: {e}")))?;
// Name-index records: hash(4) + heap ID; creation-order ones: order(8) + heap ID.
let skip = if hdr.tree_type == name_type { 4 } else { 8 };
@@ -407,7 +486,7 @@ impl H5 {
.get(skip..skip + idlen)
.ok_or_else(|| Error::at(bt, "v2 B-tree record shorter than a heap ID"))?;
let obj = fh
.read_managed_object(data, id, os)
.read_managed_object_in(data, id, os)
.map_err(|e| Error::at(heap, format!("fractal heap object: {e}")))?;
out.push(obj);
}
@@ -561,7 +640,7 @@ impl H5 {
if p.is_empty() {
return Ok(self.root());
}
clawhdf5_format::group_v2::resolve_path_any(self.data(), self.sb(), p)
clawhdf5_format::group_v2::resolve_path_any_in(self.store(), self.sb(), p)
.map_err(|e| Error::new(format!("{path}: {e}")))
}
}
@@ -681,7 +760,9 @@ pub fn byte_len(ds: &Dataspace, dt: &Datatype) -> Result<u64> {
/// Split a `FILE[/object/path]` argument the way h5ls does: the longest
/// prefix that is an existing file is the file.
pub fn split_file_arg(arg: &str) -> (String, Option<String>) {
if Path::new(arg).is_file() {
// A URL names the file only (its path cannot be split against the
// local file system).
if is_url(arg) || Path::new(arg).is_file() {
return (arg.to_string(), None);
}
let mut idx: Vec<usize> = arg.match_indices('/').map(|(i, _)| i).collect();
@@ -694,3 +775,13 @@ pub fn split_file_arg(arg: &str) -> (String, Option<String>) {
}
(arg.to_string(), None)
}
/// Whether a FILE argument is a URL (`scheme://...`) rather than a path.
pub fn is_url(arg: &str) -> bool {
arg.split_once("://").is_some_and(|(scheme, _)| {
!scheme.is_empty()
&& scheme
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
})
}
crates/clawhdf5-tools/src/heap_blocks.rs
+68 -14
View File
@@ -4,10 +4,12 @@
//! heap header's flag says so). The library reads only the blocks an object
//! lives in and does not verify block checksums, so `check` does it here.
use std::borrow::Cow;
use std::collections::HashSet;
use clawhdf5_format::checksum::jenkins_lookup3;
use clawhdf5_format::fractal_heap::FractalHeapHeader;
use clawhdf5_format::storage::{Storage, read_exact_at};
use crate::h5::{Error, H5};
@@ -26,7 +28,7 @@ pub struct HeapReport {
}
struct Walk<'a> {
data: &'a [u8],
data: &'a dyn Storage,
heap: u64,
fh: FractalHeapHeader,
checksum_dblocks: bool,
@@ -60,14 +62,14 @@ fn log2(v: u64) -> u32 {
/// parsed (and its checksum verified) by the library; an error there is
/// returned as the only problem.
pub fn verify(h5: &H5, heap: u64) -> HeapReport {
let data = h5.data();
let data = h5.store();
let Ok(off) = usize::try_from(heap) else {
return HeapReport {
problems: vec![Error::at(heap, "fractal heap address out of range")],
..Default::default()
};
};
let fh = match FractalHeapHeader::parse(data, off, h5.os(), h5.ls()) {
let fh = match FractalHeapHeader::parse_in(data, heap, h5.os(), h5.ls()) {
Ok(f) => f,
Err(e) => {
return HeapReport {
@@ -77,7 +79,15 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport {
}
};
// Flags: signature(4) version(1) heap ID length(2) filter length(2) flags(1).
let flags = data.get(off + 9).copied().unwrap_or(0);
let flags = match data.read_at(off as u64 + 9, 1) {
Ok(b) => b.first().copied().unwrap_or(0),
Err(e) => {
return HeapReport {
problems: vec![Error::at(heap, format!("fractal heap header: {e}"))],
..Default::default()
};
}
};
let mut w = Walk {
data,
heap,
@@ -107,11 +117,42 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport {
w.r
}
impl Walk<'_> {
impl<'a> Walk<'a> {
fn problem(&mut self, addr: u64, msg: impl Into<String>) {
self.r.problems.push(Error::at(addr, msg));
}
/// Bytes `[start, end)` of the file: `Ok(None)` when they run past its
/// end (what a slice `get` of the whole file answered), `Err` when the
/// storage fails to read them (a remote file).
fn get(&self, start: usize, end: usize) -> Result<Option<Cow<'a, [u8]>>, String> {
let Some(len) = end.checked_sub(start) else {
return Ok(None);
};
if end as u64 > self.data.len() {
return Ok(None);
}
read_exact_at(self.data, start as u64, len)
.map(Some)
.map_err(|e| e.to_string())
}
/// [`Walk::get`], recording a read failure as a problem at `addr`.
fn get_or_note(
&mut self,
addr: u64,
start: usize,
end: usize,
) -> Option<Option<Cow<'a, [u8]>>> {
match self.get(start, end) {
Ok(b) => Some(b),
Err(e) => {
self.problem(addr, format!("fractal heap block: {e}"));
None
}
}
}
fn row_size(&self, row: usize) -> Option<u64> {
let s = self.fh.starting_block_size;
if row <= 1 {
@@ -155,10 +196,11 @@ impl Walk<'_> {
return None;
};
let hdr_len = 5 + self.os + self.boff_bytes;
let Some(b) = start
.checked_add(hdr_len)
.and_then(|e| self.data.get(start..e))
else {
let b = match start.checked_add(hdr_len) {
Some(e) => self.get_or_note(addr, start, e)?,
None => None,
};
let Some(b) = b else {
self.problem(
addr,
format!("fractal heap {what} block lies past the end of the file"),
@@ -209,7 +251,10 @@ impl Walk<'_> {
self.problem(addr, "fractal heap direct block size out of range");
return;
};
let Some(block) = self.data.get(start..end) else {
let Some(block) = self.get_or_note(addr, start, end) else {
return;
};
let Some(block) = block else {
self.problem(
addr,
"fractal heap direct block extends past the end of the file",
@@ -259,14 +304,17 @@ impl Walk<'_> {
};
let direct = row < direct_rows;
for _ in 0..width {
let Some(b) = self.data.get(pos..pos + self.os) else {
let Some(b) = self.get_or_note(addr, pos, pos + self.os) else {
return;
};
let Some(b) = b else {
self.problem(
addr,
"fractal heap indirect block extends past the end of the file",
);
return;
};
let child = le(b);
let child = le(&b);
pos += self.os;
if direct && filtered {
pos += self.ls + 4;
@@ -277,7 +325,10 @@ impl Walk<'_> {
off = off.saturating_add(rs);
}
}
let Some(stored) = self.data.get(pos..pos + 4) else {
let Some(stored) = self.get_or_note(addr, pos, pos + 4) else {
return;
};
let Some(stored) = stored else {
self.problem(
addr,
"fractal heap indirect block extends past the end of the file",
@@ -285,7 +336,10 @@ impl Walk<'_> {
return;
};
let stored = u32::from_le_bytes([stored[0], stored[1], stored[2], stored[3]]);
let computed = jenkins_lookup3(&self.data[start..pos]);
let Some(Some(body)) = self.get_or_note(addr, start, pos) else {
return;
};
let computed = jenkins_lookup3(&body);
self.r.checksums += 1;
if computed != stored {
self.problem(
crates/clawhdf5-tools/src/info.rs
+3 -3
View File
@@ -1,7 +1,7 @@
//! Dataset facts shared by `ls`, `dump`, `stat` and `check`: shape text,
//! layout, filters and storage.
use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks};
use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks_in};
use clawhdf5_format::data_layout::DataLayout;
use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
use clawhdf5_format::datatype::Datatype;
@@ -189,8 +189,8 @@ pub fn chunks(
let Some(addr) = *btree_address else {
return Ok(Vec::new());
};
list_chunks(
h5.data(),
list_chunks_in(
h5.store(),
layout,
ds,
dt.type_size() as usize,
crates/clawhdf5-tools/src/ls.rs
+1 -1
View File
@@ -63,7 +63,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
return args.usage_error(out, "missing FILE", USAGE);
};
let (file, obj) = split_file_arg(&target);
let mut h5 = match H5::open(std::path::Path::new(&file)) {
let mut h5 = match H5::open_arg(&file) {
Ok(h) => h,
Err(e) => {
writeln!(out.e, "h5rs ls: {e}")?;
crates/clawhdf5-tools/src/stat.rs
+2 -2
View File
@@ -86,7 +86,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
let Some(file) = file else {
return args.usage_error(out, "missing FILE", USAGE);
};
let h5 = match H5::open(std::path::Path::new(&file)) {
let h5 = match H5::open_arg(&file) {
Ok(h) => h,
Err(e) => {
writeln!(out.e, "h5rs stat: {e}")?;
@@ -291,7 +291,7 @@ fn report(h5: &H5, file: &str, s: &Stats, out: &mut Out) -> std::io::Result<()>
s.attr_objects
)?;
writeln!(o, "\tMax. # of attributes to objects: {}", s.max_attrs)?;
let total = std::fs::metadata(&h5.path).map(|m| m.len()).unwrap_or(0);
let total = h5.size;
let ub = h5.file.user_block_size();
writeln!(o, "Summary of file space information:")?;
writeln!(o, " User block: {ub} bytes")?;
crates/clawhdf5-tools/src/value.rs
+6 -5
View File
@@ -151,14 +151,14 @@ pub struct Decoder<'a> {
pub h5: &'a H5,
/// Variable-length elements are resolved as the library resolves them
/// (so as libhdf5 does), not by a decoder of our own.
vl: RefCell<VlResolver<'a>>,
vl: RefCell<VlResolver<'a, dyn clawhdf5_format::storage::Storage + 'a>>,
}
impl<'a> Decoder<'a> {
pub fn new(h5: &'a H5) -> Self {
Self {
h5,
vl: RefCell::new(VlResolver::new(h5.data(), h5.os(), h5.ls())),
vl: RefCell::new(VlResolver::new_in(h5.store(), h5.os(), h5.ls())),
}
}
@@ -268,7 +268,7 @@ impl<'a> Decoder<'a> {
/// has it.
fn decode_vlen(&self, is_string: bool, base: &Datatype, b: &[u8], depth: u32) -> Value {
if is_string {
return match self.vl.borrow_mut().string_element(b) {
return match self.vl.borrow_mut().string_element_in(b) {
Ok(Some(s)) => Value::Str(String::from_utf8_lossy(s).into_owned()),
Ok(None) => Value::NullStr,
Err(e) => Value::Error(e.to_string()),
@@ -278,8 +278,9 @@ impl<'a> Decoder<'a> {
if bs == 0 {
return Value::Error("VL base type of size 0".into());
}
let obj = match self.vl.borrow_mut().element(b, bs) {
Ok(o) => o.unwrap_or(&[]),
// Copied out: decoding an element may resolve nested ones.
let obj = match self.vl.borrow_mut().element_in(b, bs) {
Ok(o) => o.unwrap_or(&[]).to_vec(),
Err(e) => return Value::Error(e.to_string()),
};
Value::Seq(