From c513f7e6d787db81ce3329e57b59b3daaddde291 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:23:47 -0500 Subject: [PATCH] h5rs: URLs as FILE arguments (feature `remote`) With the `remote` feature (`remote-https` for https://), ls, dump, stat and diff take an http(s):// (or s3://, gs://, az:// with those clawhdf5-remote features) URL wherever they take a file, and read it by range requests through clawhdf5-remote's block cache. check validates every byte, so it downloads a remote file whole and checks it as before. Without the feature a URL is a clean error naming it. The tools read the file through File::storage instead of as_bytes: object headers, shared messages, attributes, v1 and v2 group links, dense storage (fractal heaps and v2 B-trees), path resolution, chunk listings and variable-length values go through the format crate's *_in functions, and the fractal-heap block verifier reads each block through the storage (a read failure of a remote file is reported as a problem, not as "past the end of the file"). A local file's storage is its mapped bytes, so its reads are still slices. stat's file size comes from the opened file, so it is right for a URL. Tests: tests/remote.rs serves fixtures (old and new formats, a paged file, a metadata cache image, a multi-block fractal heap, compounds, v1 groups) with the clawhdf5-remote test server and requires every subcommand's output and exit status for the URL to equal the local file's, and diff of the two to be clean; 404s, non-HDF5 bodies and https without its feature are clean errors. Local output is unchanged: the old and new h5rs print the same for ls -r -v, dump, stat and check --data on the 747 conformance and CVE corpus files (tank, 2026-09-26; the dumps of h5diff_hyper1/2.h5 were too large for the comparison script, their ls, stat and check agree), except cve-2025-2310.h5, whose dump error messages differ between runs of the old binary too (which failing chunk is reported first). ci-test.sh lints h5rs with remote-https, runs the URL tests and checks h5rs with remote for C. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-tools/Cargo.toml | 7 ++ crates/clawhdf5-tools/README.md | 20 ++++ crates/clawhdf5-tools/src/check.rs | 25 ++-- crates/clawhdf5-tools/src/diff.rs | 2 +- crates/clawhdf5-tools/src/dump.rs | 2 +- crates/clawhdf5-tools/src/h5.rs | 139 +++++++++++++++++++---- crates/clawhdf5-tools/src/heap_blocks.rs | 82 ++++++++++--- crates/clawhdf5-tools/src/info.rs | 6 +- crates/clawhdf5-tools/src/ls.rs | 2 +- crates/clawhdf5-tools/src/stat.rs | 4 +- crates/clawhdf5-tools/src/value.rs | 11 +- crates/clawhdf5-tools/tests/remote.rs | 100 ++++++++++++++++ scripts/ci-test.sh | 18 ++- 13 files changed, 357 insertions(+), 61 deletions(-) create mode 100644 crates/clawhdf5-tools/tests/remote.rs diff --git a/crates/clawhdf5-tools/Cargo.toml b/crates/clawhdf5-tools/Cargo.toml index d54f39d..d6cf7d5 100644 --- a/crates/clawhdf5-tools/Cargo.toml +++ b/crates/clawhdf5-tools/Cargo.toml @@ -14,9 +14,16 @@ readme = "README.md" name = "h5rs" path = "src/main.rs" +[features] +# FILE arguments may be URLs: http:// with `remote` (no C), https:// with +# `remote-https` (rustls + ring, which compiles C). +remote = ["dep:clawhdf5-remote"] +remote-https = ["remote", "clawhdf5-remote/https"] + [dependencies] clawhdf5 = { path = "../clawhdf5", version = "2.7.0" } clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" } +clawhdf5-remote = { path = "../clawhdf5-remote", version = "2.7.0", optional = true } serde_json = "1" [dev-dependencies] diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index 8f089c3..42d829d 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -24,6 +24,26 @@ Every command takes `--max-bytes N` where it reads values (default 1 GiB): a dataset whose dataspace claims more than that is reported instead of read, so a corrupt size cannot exhaust memory. +### Remote files + +Built with the `remote` feature, every FILE argument may be an `http://` +URL (`remote-https` adds `https://`, through rustls and ring, which compiles +C; the default build has neither). The file is read by range requests +through [clawhdf5-remote](../clawhdf5-remote/README.md)'s block cache, so +`ls` of a large file fetches its metadata blocks, not the file: + +```bash +cargo install --path crates/clawhdf5-tools --features remote +h5rs ls -r -v http://127.0.0.1:8000/file.h5 +h5rs dump http://127.0.0.1:8000/file.h5 +h5rs diff local.h5 http://127.0.0.1:8000/file.h5 +``` + +A URL names the whole file (`FILE/OBJECT` suffixes are for local paths). +`check` validates every byte, so it downloads a remote file whole first. +The output is the local file's (`tests/remote.rs` compares every +subcommand). + ## `h5rs ls` ```console diff --git a/crates/clawhdf5-tools/src/check.rs b/crates/clawhdf5-tools/src/check.rs index da90247..e7d97e5 100644 --- a/crates/clawhdf5-tools/src/check.rs +++ b/crates/clawhdf5-tools/src/check.rs @@ -148,13 +148,24 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { return args.usage_error(out, "missing FILE", USAGE); }; let path = std::path::Path::new(&file); - if !path.is_file() { - writeln!(out.e, "h5rs check: {file}: no such file")?; - return Ok(2); - } - let mut h5 = match H5::open(path) { - Ok(h) => h, - Err(_) => return unopenable(path, out), + let mut h5 = if crate::h5::is_url(&file) { + // check validates every byte, so a remote file is downloaded whole. + match H5::open_arg_whole(&file) { + Ok(h) => h, + Err(e) => { + writeln!(out.e, "h5rs check: {e}")?; + return Ok(2); + } + } + } else { + if !path.is_file() { + writeln!(out.e, "h5rs check: {file}: no such file")?; + return Ok(2); + } + match H5::open(path) { + Ok(h) => h, + Err(_) => return unopenable(path, out), + } }; if let Some(m) = max_bytes { h5.max_bytes = m; diff --git a/crates/clawhdf5-tools/src/diff.rs b/crates/clawhdf5-tools/src/diff.rs index 98d4ba5..af684c5 100644 --- a/crates/clawhdf5-tools/src/diff.rs +++ b/crates/clawhdf5-tools/src/diff.rs @@ -174,7 +174,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { } let mut files = Vec::new(); for f in &pos[..2] { - match H5::open(std::path::Path::new(f)) { + match H5::open_arg(f) { Ok(mut h) => { if let Some(m) = max_bytes { h.max_bytes = m; diff --git a/crates/clawhdf5-tools/src/dump.rs b/crates/clawhdf5-tools/src/dump.rs index 7b1d8e7..4b59f00 100644 --- a/crates/clawhdf5-tools/src/dump.rs +++ b/crates/clawhdf5-tools/src/dump.rs @@ -82,7 +82,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { let Some(file) = file else { return args.usage_error(out, "missing FILE", USAGE); }; - let mut h5 = match H5::open(std::path::Path::new(&file)) { + let mut h5 = match H5::open_arg(&file) { Ok(h) => h, Err(e) => { writeln!(out.e, "h5rs dump: {e}")?; diff --git a/crates/clawhdf5-tools/src/h5.rs b/crates/clawhdf5-tools/src/h5.rs index 5f71a14..c63ccb3 100644 --- a/crates/clawhdf5-tools/src/h5.rs +++ b/crates/clawhdf5-tools/src/h5.rs @@ -10,9 +10,9 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use clawhdf5::File; -use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant}; +use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant_in}; use clawhdf5_format::attribute_info::AttributeInfoMessage; -use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records_in}; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; use clawhdf5_format::datatype::Datatype; @@ -24,6 +24,7 @@ use clawhdf5_format::link_info::LinkInfoMessage; use clawhdf5_format::link_message::{LinkMessage, LinkTarget}; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; +use clawhdf5_format::storage::Storage; use clawhdf5_format::superblock::Superblock; use clawhdf5_format::symbol_table::SymbolTableMessage; @@ -186,8 +187,11 @@ pub struct Link { /// An open file. pub struct H5 { + /// The file's path, or its URL for a remote file. pub path: PathBuf, pub file: File, + /// Size of the whole file in bytes (user block included). + pub size: u64, pub max_bytes: u64, /// Fractal heaps whose blocks were verified: `None` = sound. verified_heaps: RefCell>>, @@ -204,19 +208,90 @@ impl H5 { path.display() )) })?; - Ok(H5 { - path: path.to_path_buf(), + let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + Ok(H5::new(path.to_path_buf(), file, size)) + } + + fn new(path: PathBuf, file: File, size: u64) -> H5 { + H5 { + path, file, + size, max_bytes: DEFAULT_MAX_BYTES, verified_heaps: RefCell::new(HashMap::new()), - }) + } + } + + /// Open a command-line FILE argument: a path, or with the `remote` + /// feature an `http(s)://` (or `s3://`, `gs://`, `az://`) URL, read by + /// range requests through a block cache. + pub fn open_arg(arg: &str) -> Result
{ + if !is_url(arg) { + return H5::open(Path::new(arg)); + } + #[cfg(feature = "remote")] + { + let storage = + clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) + .map_err(|e| Error::new(format!("{arg}: {e}")))?; + let size = storage.len(); + let file = File::open_storage(storage).map_err(|e| { + Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}")) + })?; + Ok(H5::new(PathBuf::from(arg), file, size)) + } + #[cfg(not(feature = "remote"))] + Err(Error::new(format!( + "{arg}: URLs need h5rs built with the `remote` feature" + ))) + } + + /// [`H5::open_arg`], with a remote file downloaded whole into memory + /// first — for `check`, which validates every byte of the file anyway + /// and parses it as one slice ([`H5::data`]). + pub fn open_arg_whole(arg: &str) -> Result
{ + let h5 = H5::open_arg(arg)?; + if h5.file.contiguous_bytes().is_some() { + return Ok(h5); + } + #[cfg(feature = "remote")] + { + let storage = + clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default()) + .map_err(|e| Error::new(format!("{arg}: {e}")))?; + let len = usize::try_from(storage.len()) + .map_err(|_| Error::new(format!("{arg}: too large to download")))?; + let bytes = storage + .read_at(0, len) + .map_err(|e| Error::new(format!("{arg}: {e}")))? + .into_owned(); + let size = bytes.len() as u64; + let file = File::from_bytes(bytes).map_err(|e| { + Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}")) + })?; + Ok(H5::new(PathBuf::from(arg), file, size)) + } + #[cfg(not(feature = "remote"))] + unreachable!("open_arg refuses URLs without the remote feature") } /// The file's bytes from the superblock on: what every address indexes. + /// + /// # Panics + /// + /// For a remote file opened by [`H5::open_arg`]; read it through + /// [`H5::store`], or open it with [`H5::open_arg_whole`]. pub fn data(&self) -> &[u8] { self.file.as_bytes() } + /// The same bytes as a [`Storage`], for local and remote files alike: + /// in memory a read is a slice of [`H5::data`], remote it is served by + /// the block cache. + pub fn store(&self) -> &dyn Storage { + self.file.storage() + } + pub fn sb(&self) -> &Superblock { self.file.superblock() } @@ -239,8 +314,8 @@ impl H5 { if let Some(e) = self.file.cache_image_error() { return Err(Error::at(addr, format!("metadata cache image: {e}"))); } - let off = usize::try_from(addr).map_err(|_| Error::at(addr, "address out of range"))?; - ObjectHeader::parse(self.data(), off, self.os(), self.ls()) + to_usize(addr)?; + ObjectHeader::parse_in(self.store(), addr, self.os(), self.ls()) .map_err(|e| Error::at(addr, format!("object header: {e}"))) } @@ -249,11 +324,14 @@ impl H5 { pub fn payload(&self, h: &ObjectHeader, t: MessageType) -> Result>> { match h.messages.iter().find(|m| m.msg_type == t) { None => Ok(None), - Some(m) => { - clawhdf5_format::shared_message::message_data(self.data(), m, self.os(), self.ls()) - .map(|c| Some(c.into_owned())) - .map_err(|e| Error::new(format!("{t:?} message: {e}"))) - } + Some(m) => clawhdf5_format::shared_message::message_data_in( + self.store(), + m, + self.os(), + self.ls(), + ) + .map(|c| Some(c.into_owned())) + .map_err(|e| Error::new(format!("{t:?} message: {e}"))), } } @@ -305,8 +383,9 @@ impl H5 { self.verified_heap(fh) .map_err(|e| e.context("dense attribute storage"))?; } - let (mut attrs, errs) = extract_attributes_tolerant(self.data(), h, self.os(), self.ls()) - .map_err(|e| Error::new(format!("attributes: {e}")))?; + let (mut attrs, errs) = + extract_attributes_tolerant_in(self.store(), h, self.os(), self.ls()) + .map_err(|e| Error::new(format!("attributes: {e}")))?; attrs.sort_by(|a, b| a.name.cmp(&b.name)); Ok((attrs, errs.iter().map(|e| e.to_string()).collect())) } @@ -316,7 +395,7 @@ impl H5 { pub fn links(&self, h: &ObjectHeader) -> Result> { let os = self.os(); let ls = self.ls(); - let data = self.data(); + let data = self.store(); let mut out = Vec::new(); if let Some(m) = h .messages @@ -325,7 +404,7 @@ impl H5 { { let stm = SymbolTableMessage::parse(&m.data, os) .map_err(|e| Error::new(format!("symbol table message: {e}")))?; - let entries = group_v1::resolve_v1_group_entries(data, &stm, os, ls) + let entries = group_v1::resolve_v1_group_entries_in(data, &stm, os, ls) .map_err(|e| Error::at(stm.btree_address, format!("symbol table: {e}")))?; let has_soft = entries.iter().any(group_v1::is_v1_soft_link); for e in entries { @@ -337,7 +416,7 @@ impl H5 { } } if has_soft { - let soft = group_v1::v1_soft_links(data, &stm, os, ls) + let soft = group_v1::v1_soft_links_in(data, &stm, os, ls) .map_err(|e| Error::at(stm.btree_address, format!("soft links: {e}")))?; for (name, target) in soft { out.push(Link { @@ -387,15 +466,15 @@ impl H5 { name_type: u8, ) -> Result>> { self.verified_heap(heap)?; - let data = self.data(); + let data = self.store(); let os = self.os(); let ls = self.ls(); - let fh = FractalHeapHeader::parse(data, to_usize(heap)?, os, ls) + let fh = FractalHeapHeader::parse_in(data, to_usize(heap).map(|_| heap)?, os, ls) .map_err(|e| Error::at(heap, format!("fractal heap header: {e}")))?; let bt = btree.ok_or_else(|| Error::at(heap, "dense storage without a name index"))?; - let hdr = BTreeV2Header::parse(data, to_usize(bt)?, os, ls) + let hdr = BTreeV2Header::parse_in(data, to_usize(bt).map(|_| bt)?, os, ls) .map_err(|e| Error::at(bt, format!("v2 B-tree header: {e}")))?; - let recs = collect_btree_v2_records(data, &hdr, os, ls) + let recs = collect_btree_v2_records_in(data, &hdr, os, ls) .map_err(|e| Error::at(bt, format!("v2 B-tree: {e}")))?; // Name-index records: hash(4) + heap ID; creation-order ones: order(8) + heap ID. let skip = if hdr.tree_type == name_type { 4 } else { 8 }; @@ -407,7 +486,7 @@ impl H5 { .get(skip..skip + idlen) .ok_or_else(|| Error::at(bt, "v2 B-tree record shorter than a heap ID"))?; let obj = fh - .read_managed_object(data, id, os) + .read_managed_object_in(data, id, os) .map_err(|e| Error::at(heap, format!("fractal heap object: {e}")))?; out.push(obj); } @@ -561,7 +640,7 @@ impl H5 { if p.is_empty() { return Ok(self.root()); } - clawhdf5_format::group_v2::resolve_path_any(self.data(), self.sb(), p) + clawhdf5_format::group_v2::resolve_path_any_in(self.store(), self.sb(), p) .map_err(|e| Error::new(format!("{path}: {e}"))) } } @@ -681,7 +760,9 @@ pub fn byte_len(ds: &Dataspace, dt: &Datatype) -> Result { /// Split a `FILE[/object/path]` argument the way h5ls does: the longest /// prefix that is an existing file is the file. pub fn split_file_arg(arg: &str) -> (String, Option) { - if Path::new(arg).is_file() { + // A URL names the file only (its path cannot be split against the + // local file system). + if is_url(arg) || Path::new(arg).is_file() { return (arg.to_string(), None); } let mut idx: Vec = arg.match_indices('/').map(|(i, _)| i).collect(); @@ -694,3 +775,13 @@ pub fn split_file_arg(arg: &str) -> (String, Option) { } (arg.to_string(), None) } + +/// Whether a FILE argument is a URL (`scheme://...`) rather than a path. +pub fn is_url(arg: &str) -> bool { + arg.split_once("://").is_some_and(|(scheme, _)| { + !scheme.is_empty() + && scheme + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) + }) +} diff --git a/crates/clawhdf5-tools/src/heap_blocks.rs b/crates/clawhdf5-tools/src/heap_blocks.rs index a1c40cd..59cc6d3 100644 --- a/crates/clawhdf5-tools/src/heap_blocks.rs +++ b/crates/clawhdf5-tools/src/heap_blocks.rs @@ -4,10 +4,12 @@ //! heap header's flag says so). The library reads only the blocks an object //! lives in and does not verify block checksums, so `check` does it here. +use std::borrow::Cow; use std::collections::HashSet; use clawhdf5_format::checksum::jenkins_lookup3; use clawhdf5_format::fractal_heap::FractalHeapHeader; +use clawhdf5_format::storage::{Storage, read_exact_at}; use crate::h5::{Error, H5}; @@ -26,7 +28,7 @@ pub struct HeapReport { } struct Walk<'a> { - data: &'a [u8], + data: &'a dyn Storage, heap: u64, fh: FractalHeapHeader, checksum_dblocks: bool, @@ -60,14 +62,14 @@ fn log2(v: u64) -> u32 { /// parsed (and its checksum verified) by the library; an error there is /// returned as the only problem. pub fn verify(h5: &H5, heap: u64) -> HeapReport { - let data = h5.data(); + let data = h5.store(); let Ok(off) = usize::try_from(heap) else { return HeapReport { problems: vec![Error::at(heap, "fractal heap address out of range")], ..Default::default() }; }; - let fh = match FractalHeapHeader::parse(data, off, h5.os(), h5.ls()) { + let fh = match FractalHeapHeader::parse_in(data, heap, h5.os(), h5.ls()) { Ok(f) => f, Err(e) => { return HeapReport { @@ -77,7 +79,15 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport { } }; // Flags: signature(4) version(1) heap ID length(2) filter length(2) flags(1). - let flags = data.get(off + 9).copied().unwrap_or(0); + let flags = match data.read_at(off as u64 + 9, 1) { + Ok(b) => b.first().copied().unwrap_or(0), + Err(e) => { + return HeapReport { + problems: vec![Error::at(heap, format!("fractal heap header: {e}"))], + ..Default::default() + }; + } + }; let mut w = Walk { data, heap, @@ -107,11 +117,42 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport { w.r } -impl Walk<'_> { +impl<'a> Walk<'a> { fn problem(&mut self, addr: u64, msg: impl Into) { self.r.problems.push(Error::at(addr, msg)); } + /// Bytes `[start, end)` of the file: `Ok(None)` when they run past its + /// end (what a slice `get` of the whole file answered), `Err` when the + /// storage fails to read them (a remote file). + fn get(&self, start: usize, end: usize) -> Result>, String> { + let Some(len) = end.checked_sub(start) else { + return Ok(None); + }; + if end as u64 > self.data.len() { + return Ok(None); + } + read_exact_at(self.data, start as u64, len) + .map(Some) + .map_err(|e| e.to_string()) + } + + /// [`Walk::get`], recording a read failure as a problem at `addr`. + fn get_or_note( + &mut self, + addr: u64, + start: usize, + end: usize, + ) -> Option>> { + match self.get(start, end) { + Ok(b) => Some(b), + Err(e) => { + self.problem(addr, format!("fractal heap block: {e}")); + None + } + } + } + fn row_size(&self, row: usize) -> Option { let s = self.fh.starting_block_size; if row <= 1 { @@ -155,10 +196,11 @@ impl Walk<'_> { return None; }; let hdr_len = 5 + self.os + self.boff_bytes; - let Some(b) = start - .checked_add(hdr_len) - .and_then(|e| self.data.get(start..e)) - else { + let b = match start.checked_add(hdr_len) { + Some(e) => self.get_or_note(addr, start, e)?, + None => None, + }; + let Some(b) = b else { self.problem( addr, format!("fractal heap {what} block lies past the end of the file"), @@ -209,7 +251,10 @@ impl Walk<'_> { self.problem(addr, "fractal heap direct block size out of range"); return; }; - let Some(block) = self.data.get(start..end) else { + let Some(block) = self.get_or_note(addr, start, end) else { + return; + }; + let Some(block) = block else { self.problem( addr, "fractal heap direct block extends past the end of the file", @@ -259,14 +304,17 @@ impl Walk<'_> { }; let direct = row < direct_rows; for _ in 0..width { - let Some(b) = self.data.get(pos..pos + self.os) else { + let Some(b) = self.get_or_note(addr, pos, pos + self.os) else { + return; + }; + let Some(b) = b else { self.problem( addr, "fractal heap indirect block extends past the end of the file", ); return; }; - let child = le(b); + let child = le(&b); pos += self.os; if direct && filtered { pos += self.ls + 4; @@ -277,7 +325,10 @@ impl Walk<'_> { off = off.saturating_add(rs); } } - let Some(stored) = self.data.get(pos..pos + 4) else { + let Some(stored) = self.get_or_note(addr, pos, pos + 4) else { + return; + }; + let Some(stored) = stored else { self.problem( addr, "fractal heap indirect block extends past the end of the file", @@ -285,7 +336,10 @@ impl Walk<'_> { return; }; let stored = u32::from_le_bytes([stored[0], stored[1], stored[2], stored[3]]); - let computed = jenkins_lookup3(&self.data[start..pos]); + let Some(Some(body)) = self.get_or_note(addr, start, pos) else { + return; + }; + let computed = jenkins_lookup3(&body); self.r.checksums += 1; if computed != stored { self.problem( diff --git a/crates/clawhdf5-tools/src/info.rs b/crates/clawhdf5-tools/src/info.rs index 606457b..d169f73 100644 --- a/crates/clawhdf5-tools/src/info.rs +++ b/crates/clawhdf5-tools/src/info.rs @@ -1,7 +1,7 @@ //! Dataset facts shared by `ls`, `dump`, `stat` and `check`: shape text, //! layout, filters and storage. -use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks}; +use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks_in}; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; use clawhdf5_format::datatype::Datatype; @@ -189,8 +189,8 @@ pub fn chunks( let Some(addr) = *btree_address else { return Ok(Vec::new()); }; - list_chunks( - h5.data(), + list_chunks_in( + h5.store(), layout, ds, dt.type_size() as usize, diff --git a/crates/clawhdf5-tools/src/ls.rs b/crates/clawhdf5-tools/src/ls.rs index ef966f6..203cce3 100644 --- a/crates/clawhdf5-tools/src/ls.rs +++ b/crates/clawhdf5-tools/src/ls.rs @@ -63,7 +63,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { return args.usage_error(out, "missing FILE", USAGE); }; let (file, obj) = split_file_arg(&target); - let mut h5 = match H5::open(std::path::Path::new(&file)) { + let mut h5 = match H5::open_arg(&file) { Ok(h) => h, Err(e) => { writeln!(out.e, "h5rs ls: {e}")?; diff --git a/crates/clawhdf5-tools/src/stat.rs b/crates/clawhdf5-tools/src/stat.rs index c7e44da..32e91fe 100644 --- a/crates/clawhdf5-tools/src/stat.rs +++ b/crates/clawhdf5-tools/src/stat.rs @@ -86,7 +86,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { let Some(file) = file else { return args.usage_error(out, "missing FILE", USAGE); }; - let h5 = match H5::open(std::path::Path::new(&file)) { + let h5 = match H5::open_arg(&file) { Ok(h) => h, Err(e) => { writeln!(out.e, "h5rs stat: {e}")?; @@ -291,7 +291,7 @@ fn report(h5: &H5, file: &str, s: &Stats, out: &mut Out) -> std::io::Result<()> s.attr_objects )?; writeln!(o, "\tMax. # of attributes to objects: {}", s.max_attrs)?; - let total = std::fs::metadata(&h5.path).map(|m| m.len()).unwrap_or(0); + let total = h5.size; let ub = h5.file.user_block_size(); writeln!(o, "Summary of file space information:")?; writeln!(o, " User block: {ub} bytes")?; diff --git a/crates/clawhdf5-tools/src/value.rs b/crates/clawhdf5-tools/src/value.rs index d9255f1..c4f4af6 100644 --- a/crates/clawhdf5-tools/src/value.rs +++ b/crates/clawhdf5-tools/src/value.rs @@ -151,14 +151,14 @@ pub struct Decoder<'a> { pub h5: &'a H5, /// Variable-length elements are resolved as the library resolves them /// (so as libhdf5 does), not by a decoder of our own. - vl: RefCell>, + vl: RefCell>, } impl<'a> Decoder<'a> { pub fn new(h5: &'a H5) -> Self { Self { h5, - vl: RefCell::new(VlResolver::new(h5.data(), h5.os(), h5.ls())), + vl: RefCell::new(VlResolver::new_in(h5.store(), h5.os(), h5.ls())), } } @@ -268,7 +268,7 @@ impl<'a> Decoder<'a> { /// has it. fn decode_vlen(&self, is_string: bool, base: &Datatype, b: &[u8], depth: u32) -> Value { if is_string { - return match self.vl.borrow_mut().string_element(b) { + return match self.vl.borrow_mut().string_element_in(b) { Ok(Some(s)) => Value::Str(String::from_utf8_lossy(s).into_owned()), Ok(None) => Value::NullStr, Err(e) => Value::Error(e.to_string()), @@ -278,8 +278,9 @@ impl<'a> Decoder<'a> { if bs == 0 { return Value::Error("VL base type of size 0".into()); } - let obj = match self.vl.borrow_mut().element(b, bs) { - Ok(o) => o.unwrap_or(&[]), + // Copied out: decoding an element may resolve nested ones. + let obj = match self.vl.borrow_mut().element_in(b, bs) { + Ok(o) => o.unwrap_or(&[]).to_vec(), Err(e) => return Value::Error(e.to_string()), }; Value::Seq( diff --git a/crates/clawhdf5-tools/tests/remote.rs b/crates/clawhdf5-tools/tests/remote.rs new file mode 100644 index 0000000..d14bd61 --- /dev/null +++ b/crates/clawhdf5-tools/tests/remote.rs @@ -0,0 +1,100 @@ +//! `h5rs` on URLs (feature `remote`): every subcommand prints for +//! `http://…/file.h5` what it prints for the local file (the name aside). +//! The files are served by the range-request test server of +//! clawhdf5-remote on 127.0.0.1. + +#![cfg(feature = "remote")] + +#[path = "../../clawhdf5-remote/tests/common/server.rs"] +mod server; + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn h5rs(args: &[&str]) -> (String, i32) { + let out = Command::new(env!("CARGO_BIN_EXE_h5rs")) + .args(args) + .output() + .expect("run h5rs"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (text, out.status.code().unwrap_or(-1)) +} + +fn fixtures() -> Vec { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + [ + "../clawhdf5/tests/fixtures/tall.h5", + "../clawhdf5/tests/fixtures/written_by_v2_7_0.h5", + "../clawhdf5/tests/fixtures/written_by_v2_7_0_paged.h5", + "../clawhdf5/tests/fixtures/h5clear_mdc_image.h5", + "../clawhdf5-format/tests/fixtures/fractal_heap_multiblock.h5", + "../clawhdf5-format/tests/fixtures/legacy/tcompound.h5", + "../clawhdf5-format/tests/fixtures/legacy/h5ex_g_iterate.h5", + ] + .iter() + .map(|p| root.join(p)) + .collect() +} + +#[test] +fn every_subcommand_reads_a_url_like_the_local_file() { + let files = fixtures(); + let served: Vec<(String, Vec)> = files + .iter() + .enumerate() + .map(|(i, p)| { + let name = p.file_name().unwrap().to_str().unwrap(); + (format!("/{i}/{name}"), std::fs::read(p).unwrap()) + }) + .collect(); + let server = server::Server::start(served.clone()); + for (p, (url_path, _)) in files.iter().zip(&served) { + let url = server.url(url_path); + let local = p.to_str().unwrap(); + for cmd in [ + &["ls", "-r", "-v"][..], + &["dump"], + &["dump", "--json"], + &["stat"], + &["check", "--data"], + ] { + fn args<'a>(cmd: &[&'a str], f: &'a str) -> Vec<&'a str> { + cmd.iter().copied().chain([f]).collect() + } + let (want, want_rc) = h5rs(&args(cmd, local)); + let (got, got_rc) = h5rs(&args(cmd, &url)); + assert_eq!( + got.replace(&url, local), + want, + "h5rs {} {url}", + cmd.join(" ") + ); + assert_eq!(got_rc, want_rc, "h5rs {} {url}", cmd.join(" ")); + } + let (a, rc) = h5rs(&["diff", local, &url]); + assert_eq!(rc, 0, "h5rs diff {local} {url}: {a}"); + } +} + +#[test] +fn url_errors_are_clean() { + let server = server::Server::start(vec![("/x.h5".into(), vec![1u8; 100])]); + let (out, rc) = h5rs(&["ls", &server.url("/missing.h5")]); + assert_eq!(rc, 2, "{out}"); + assert!(out.contains("404"), "{out}"); + let (out, rc) = h5rs(&["ls", &server.url("/x.h5")]); + assert_eq!(rc, 2, "{out}"); + assert!(out.contains("not an HDF5 file"), "{out}"); + let (out, rc) = h5rs(&["check", &server.url("/missing.h5")]); + assert_eq!(rc, 2, "{out}"); + #[cfg(not(feature = "remote-https"))] + { + let (out, rc) = h5rs(&["ls", "https://example.com/a.h5"]); + assert_eq!(rc, 2, "{out}"); + assert!(out.contains("`https` feature"), "{out}"); + } +} diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index fe7242c..4f678ad 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -117,21 +117,28 @@ run_step "cargo clippy (remote, all backends)" cargo clippy \ --features object-store,https,s3,gcs,azure \ -- -D warnings +# h5rs with URL arguments. +run_step "cargo clippy (h5rs remote)" cargo clippy \ + -p clawhdf5-tools \ + --all-targets \ + --features remote-https \ + -- -D warnings + # The README promises that the core crates build no C by default. Hold it to # that: fail if a crate that compiles C (a *-sys crate, cc or cmake) enters the # default dependency tree of any of them. clawhdf5-migrate (bundled SQLite), # clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt. # js-sys (clawhdf5-wasm's bindings to JavaScript) builds no C. # clawhdf5-remote is checked by default (plain HTTP) and with its -# object-store feature; its https (ring) and s3/gcs/azure (aws-lc-rs) -# features build C and are opt-in. +# object-store feature, and h5rs with URL support (remote); the https +# (ring) and s3/gcs/azure (aws-lc-rs) features build C and are opt-in. no_c_in_default_build() { local entry crate features found=0 for entry in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \ clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli \ clawhdf5-tools \ clawhdf5-wasm \ - clawhdf5-remote clawhdf5-remote:object-store; do + clawhdf5-remote clawhdf5-remote:object-store clawhdf5-tools:remote; do crate=${entry%%:*} features=() [ "$entry" != "$crate" ] && features=(--features "${entry#*:}") @@ -210,6 +217,11 @@ run_step "cargo test (remote, object_store backend, s3 URLs)" cargo test \ -p clawhdf5-remote \ --features object-store,s3 +run_step "cargo test (h5rs on URLs)" cargo test \ + -p clawhdf5-tools \ + --features remote \ + --test remote + run_step "cargo test (ann parallel)" cargo test \ -p clawhdf5-ann \ --features parallel