h5rs: URLs as FILE arguments (feature remote)

With the `remote` feature (`remote-https` for https://), ls, dump, stat
and diff take an http(s):// (or s3://, gs://, az:// with those
clawhdf5-remote features) URL wherever they take a file, and read it by
range requests through clawhdf5-remote's block cache. check validates
every byte, so it downloads a remote file whole and checks it as before.
Without the feature a URL is a clean error naming it.

The tools read the file through File::storage instead of as_bytes: object
headers, shared messages, attributes, v1 and v2 group links, dense
storage (fractal heaps and v2 B-trees), path resolution, chunk listings
and variable-length values go through the format crate's *_in functions,
and the fractal-heap block verifier reads each block through the storage
(a read failure of a remote file is reported as a problem, not as "past
the end of the file"). A local file's storage is its mapped bytes, so its
reads are still slices. stat's file size comes from the opened file, so
it is right for a URL.

Tests: tests/remote.rs serves fixtures (old and new formats, a paged
file, a metadata cache image, a multi-block fractal heap, compounds, v1
groups) with the clawhdf5-remote test server and requires every
subcommand's output and exit status for the URL to equal the local
file's, and diff of the two to be clean; 404s, non-HDF5 bodies and
https without its feature are clean errors. Local output is unchanged:
the old and new h5rs print the same for ls -r -v, dump, stat and check
--data on the 747 conformance and CVE corpus files (tank, 2026-09-26;
the dumps of h5diff_hyper1/2.h5 were too large for the comparison
script, their ls, stat and check agree), except cve-2025-2310.h5, whose
dump error messages differ between runs of the old binary too (which
failing chunk is reported first).

ci-test.sh lints h5rs with remote-https, runs the URL tests and checks
h5rs with remote for C.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-tools/Cargo.toml

@@ -14,9 +14,16 @@ readme = "README.md"
 name = "h5rs"
 path = "src/main.rs"
 
+[features]
+# FILE arguments may be URLs: http:// with `remote` (no C), https:// with
+# `remote-https` (rustls + ring, which compiles C).
+remote = ["dep:clawhdf5-remote"]
+remote-https = ["remote", "clawhdf5-remote/https"]
+
 [dependencies]
 clawhdf5 = { path = "../clawhdf5", version = "2.7.0" }
 clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" }
+clawhdf5-remote = { path = "../clawhdf5-remote", version = "2.7.0", optional = true }
 serde_json = "1"
 
 [dev-dependencies]

crates/clawhdf5-tools/README.md

@@ -24,6 +24,26 @@ Every command takes `--max-bytes N` where it reads values (default 1 GiB): a
 dataset whose dataspace claims more than that is reported instead of read, so
 a corrupt size cannot exhaust memory.
 
+### Remote files
+
+Built with the `remote` feature, every FILE argument may be an `http://`
+URL (`remote-https` adds `https://`, through rustls and ring, which compiles
+C; the default build has neither). The file is read by range requests
+through [clawhdf5-remote](../clawhdf5-remote/README.md)'s block cache, so
+`ls` of a large file fetches its metadata blocks, not the file:
+
+```bash
+cargo install --path crates/clawhdf5-tools --features remote
+h5rs ls -r -v http://127.0.0.1:8000/file.h5
+h5rs dump http://127.0.0.1:8000/file.h5
+h5rs diff local.h5 http://127.0.0.1:8000/file.h5
+```
+
+A URL names the whole file (`FILE/OBJECT` suffixes are for local paths).
+`check` validates every byte, so it downloads a remote file whole first.
+The output is the local file's (`tests/remote.rs` compares every
+subcommand).
+
 ## `h5rs ls`
 
 ```console

crates/clawhdf5-tools/src/check.rs

@@ -148,13 +148,24 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
         return args.usage_error(out, "missing FILE", USAGE);
     };
     let path = std::path::Path::new(&file);
+    let mut h5 = if crate::h5::is_url(&file) {
+        // check validates every byte, so a remote file is downloaded whole.
+        match H5::open_arg_whole(&file) {
+            Ok(h) => h,
+            Err(e) => {
+                writeln!(out.e, "h5rs check: {e}")?;
+                return Ok(2);
+            }
+        }
+    } else {
         if !path.is_file() {
             writeln!(out.e, "h5rs check: {file}: no such file")?;
             return Ok(2);
         }
-    let mut h5 = match H5::open(path) {
+        match H5::open(path) {
             Ok(h) => h,
             Err(_) => return unopenable(path, out),
+        }
     };
     if let Some(m) = max_bytes {
         h5.max_bytes = m;

crates/clawhdf5-tools/src/diff.rs

@@ -174,7 +174,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
     }
     let mut files = Vec::new();
     for f in &pos[..2] {
-        match H5::open(std::path::Path::new(f)) {
+        match H5::open_arg(f) {
             Ok(mut h) => {
                 if let Some(m) = max_bytes {
                     h.max_bytes = m;

crates/clawhdf5-tools/src/dump.rs

@@ -82,7 +82,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
     let Some(file) = file else {
         return args.usage_error(out, "missing FILE", USAGE);
     };
-    let mut h5 = match H5::open(std::path::Path::new(&file)) {
+    let mut h5 = match H5::open_arg(&file) {
         Ok(h) => h,
         Err(e) => {
             writeln!(out.e, "h5rs dump: {e}")?;

crates/clawhdf5-tools/src/h5.rs

@@ -10,9 +10,9 @@ use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 
 use clawhdf5::File;
-use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant};
+use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant_in};
 use clawhdf5_format::attribute_info::AttributeInfoMessage;
-use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records};
+use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records_in};
 use clawhdf5_format::data_layout::DataLayout;
 use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
 use clawhdf5_format::datatype::Datatype;
@@ -24,6 +24,7 @@ use clawhdf5_format::link_info::LinkInfoMessage;
 use clawhdf5_format::link_message::{LinkMessage, LinkTarget};
 use clawhdf5_format::message_type::MessageType;
 use clawhdf5_format::object_header::ObjectHeader;
+use clawhdf5_format::storage::Storage;
 use clawhdf5_format::superblock::Superblock;
 use clawhdf5_format::symbol_table::SymbolTableMessage;
 
@@ -186,8 +187,11 @@ pub struct Link {
 
 /// An open file.
 pub struct H5 {
+    /// The file's path, or its URL for a remote file.
     pub path: PathBuf,
     pub file: File,
+    /// Size of the whole file in bytes (user block included).
+    pub size: u64,
     pub max_bytes: u64,
     /// Fractal heaps whose blocks were verified: `None` = sound.
     verified_heaps: RefCell<HashMap<u64, Option<Error>>>,
@@ -204,19 +208,90 @@ impl H5 {
                 path.display()
             ))
         })?;
-        Ok(H5 {
-            path: path.to_path_buf(),
+        let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
+        Ok(H5::new(path.to_path_buf(), file, size))
+    }
+
+    fn new(path: PathBuf, file: File, size: u64) -> H5 {
+        H5 {
+            path,
             file,
+            size,
             max_bytes: DEFAULT_MAX_BYTES,
             verified_heaps: RefCell::new(HashMap::new()),
-        })
+        }
+    }
+
+    /// Open a command-line FILE argument: a path, or with the `remote`
+    /// feature an `http(s)://` (or `s3://`, `gs://`, `az://`) URL, read by
+    /// range requests through a block cache.
+    pub fn open_arg(arg: &str) -> Result<H5> {
+        if !is_url(arg) {
+            return H5::open(Path::new(arg));
+        }
+        #[cfg(feature = "remote")]
+        {
+            let storage =
+                clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default())
+                    .map_err(|e| Error::new(format!("{arg}: {e}")))?;
+            let size = storage.len();
+            let file = File::open_storage(storage).map_err(|e| {
+                Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}"))
+            })?;
+            Ok(H5::new(PathBuf::from(arg), file, size))
+        }
+        #[cfg(not(feature = "remote"))]
+        Err(Error::new(format!(
+            "{arg}: URLs need h5rs built with the `remote` feature"
+        )))
+    }
+
+    /// [`H5::open_arg`], with a remote file downloaded whole into memory
+    /// first — for `check`, which validates every byte of the file anyway
+    /// and parses it as one slice ([`H5::data`]).
+    pub fn open_arg_whole(arg: &str) -> Result<H5> {
+        let h5 = H5::open_arg(arg)?;
+        if h5.file.contiguous_bytes().is_some() {
+            return Ok(h5);
+        }
+        #[cfg(feature = "remote")]
+        {
+            let storage =
+                clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default())
+                    .map_err(|e| Error::new(format!("{arg}: {e}")))?;
+            let len = usize::try_from(storage.len())
+                .map_err(|_| Error::new(format!("{arg}: too large to download")))?;
+            let bytes = storage
+                .read_at(0, len)
+                .map_err(|e| Error::new(format!("{arg}: {e}")))?
+                .into_owned();
+            let size = bytes.len() as u64;
+            let file = File::from_bytes(bytes).map_err(|e| {
+                Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}"))
+            })?;
+            Ok(H5::new(PathBuf::from(arg), file, size))
+        }
+        #[cfg(not(feature = "remote"))]
+        unreachable!("open_arg refuses URLs without the remote feature")
     }
 
     /// The file's bytes from the superblock on: what every address indexes.
+    ///
+    /// # Panics
+    ///
+    /// For a remote file opened by [`H5::open_arg`]; read it through
+    /// [`H5::store`], or open it with [`H5::open_arg_whole`].
     pub fn data(&self) -> &[u8] {
         self.file.as_bytes()
     }
 
+    /// The same bytes as a [`Storage`], for local and remote files alike:
+    /// in memory a read is a slice of [`H5::data`], remote it is served by
+    /// the block cache.
+    pub fn store(&self) -> &dyn Storage {
+        self.file.storage()
+    }
+
     pub fn sb(&self) -> &Superblock {
         self.file.superblock()
     }
@@ -239,8 +314,8 @@ impl H5 {
         if let Some(e) = self.file.cache_image_error() {
             return Err(Error::at(addr, format!("metadata cache image: {e}")));
         }
-        let off = usize::try_from(addr).map_err(|_| Error::at(addr, "address out of range"))?;
-        ObjectHeader::parse(self.data(), off, self.os(), self.ls())
+        to_usize(addr)?;
+        ObjectHeader::parse_in(self.store(), addr, self.os(), self.ls())
             .map_err(|e| Error::at(addr, format!("object header: {e}")))
     }
 
@@ -249,11 +324,14 @@ impl H5 {
     pub fn payload(&self, h: &ObjectHeader, t: MessageType) -> Result<Option<Vec<u8>>> {
         match h.messages.iter().find(|m| m.msg_type == t) {
             None => Ok(None),
-            Some(m) => {
-                clawhdf5_format::shared_message::message_data(self.data(), m, self.os(), self.ls())
+            Some(m) => clawhdf5_format::shared_message::message_data_in(
+                self.store(),
+                m,
+                self.os(),
+                self.ls(),
+            )
             .map(|c| Some(c.into_owned()))
-                    .map_err(|e| Error::new(format!("{t:?} message: {e}")))
-            }
+            .map_err(|e| Error::new(format!("{t:?} message: {e}"))),
         }
     }
 
@@ -305,7 +383,8 @@ impl H5 {
             self.verified_heap(fh)
                 .map_err(|e| e.context("dense attribute storage"))?;
         }
-        let (mut attrs, errs) = extract_attributes_tolerant(self.data(), h, self.os(), self.ls())
+        let (mut attrs, errs) =
+            extract_attributes_tolerant_in(self.store(), h, self.os(), self.ls())
                 .map_err(|e| Error::new(format!("attributes: {e}")))?;
         attrs.sort_by(|a, b| a.name.cmp(&b.name));
         Ok((attrs, errs.iter().map(|e| e.to_string()).collect()))
@@ -316,7 +395,7 @@ impl H5 {
     pub fn links(&self, h: &ObjectHeader) -> Result<Vec<Link>> {
         let os = self.os();
         let ls = self.ls();
-        let data = self.data();
+        let data = self.store();
         let mut out = Vec::new();
         if let Some(m) = h
             .messages
@@ -325,7 +404,7 @@ impl H5 {
         {
             let stm = SymbolTableMessage::parse(&m.data, os)
                 .map_err(|e| Error::new(format!("symbol table message: {e}")))?;
-            let entries = group_v1::resolve_v1_group_entries(data, &stm, os, ls)
+            let entries = group_v1::resolve_v1_group_entries_in(data, &stm, os, ls)
                 .map_err(|e| Error::at(stm.btree_address, format!("symbol table: {e}")))?;
             let has_soft = entries.iter().any(group_v1::is_v1_soft_link);
             for e in entries {
@@ -337,7 +416,7 @@ impl H5 {
                 }
             }
             if has_soft {
-                let soft = group_v1::v1_soft_links(data, &stm, os, ls)
+                let soft = group_v1::v1_soft_links_in(data, &stm, os, ls)
                     .map_err(|e| Error::at(stm.btree_address, format!("soft links: {e}")))?;
                 for (name, target) in soft {
                     out.push(Link {
@@ -387,15 +466,15 @@ impl H5 {
         name_type: u8,
     ) -> Result<Vec<Vec<u8>>> {
         self.verified_heap(heap)?;
-        let data = self.data();
+        let data = self.store();
         let os = self.os();
         let ls = self.ls();
-        let fh = FractalHeapHeader::parse(data, to_usize(heap)?, os, ls)
+        let fh = FractalHeapHeader::parse_in(data, to_usize(heap).map(|_| heap)?, os, ls)
             .map_err(|e| Error::at(heap, format!("fractal heap header: {e}")))?;
         let bt = btree.ok_or_else(|| Error::at(heap, "dense storage without a name index"))?;
-        let hdr = BTreeV2Header::parse(data, to_usize(bt)?, os, ls)
+        let hdr = BTreeV2Header::parse_in(data, to_usize(bt).map(|_| bt)?, os, ls)
             .map_err(|e| Error::at(bt, format!("v2 B-tree header: {e}")))?;
-        let recs = collect_btree_v2_records(data, &hdr, os, ls)
+        let recs = collect_btree_v2_records_in(data, &hdr, os, ls)
             .map_err(|e| Error::at(bt, format!("v2 B-tree: {e}")))?;
         // Name-index records: hash(4) + heap ID; creation-order ones: order(8) + heap ID.
         let skip = if hdr.tree_type == name_type { 4 } else { 8 };
@@ -407,7 +486,7 @@ impl H5 {
                 .get(skip..skip + idlen)
                 .ok_or_else(|| Error::at(bt, "v2 B-tree record shorter than a heap ID"))?;
             let obj = fh
-                .read_managed_object(data, id, os)
+                .read_managed_object_in(data, id, os)
                 .map_err(|e| Error::at(heap, format!("fractal heap object: {e}")))?;
             out.push(obj);
         }
@@ -561,7 +640,7 @@ impl H5 {
         if p.is_empty() {
             return Ok(self.root());
         }
-        clawhdf5_format::group_v2::resolve_path_any(self.data(), self.sb(), p)
+        clawhdf5_format::group_v2::resolve_path_any_in(self.store(), self.sb(), p)
             .map_err(|e| Error::new(format!("{path}: {e}")))
     }
 }
@@ -681,7 +760,9 @@ pub fn byte_len(ds: &Dataspace, dt: &Datatype) -> Result<u64> {
 /// Split a `FILE[/object/path]` argument the way h5ls does: the longest
 /// prefix that is an existing file is the file.
 pub fn split_file_arg(arg: &str) -> (String, Option<String>) {
-    if Path::new(arg).is_file() {
+    // A URL names the file only (its path cannot be split against the
+    // local file system).
+    if is_url(arg) || Path::new(arg).is_file() {
         return (arg.to_string(), None);
     }
     let mut idx: Vec<usize> = arg.match_indices('/').map(|(i, _)| i).collect();
@@ -694,3 +775,13 @@ pub fn split_file_arg(arg: &str) -> (String, Option<String>) {
     }
     (arg.to_string(), None)
 }
+
+/// Whether a FILE argument is a URL (`scheme://...`) rather than a path.
+pub fn is_url(arg: &str) -> bool {
+    arg.split_once("://").is_some_and(|(scheme, _)| {
+        !scheme.is_empty()
+            && scheme
+                .chars()
+                .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
+    })
+}

crates/clawhdf5-tools/src/heap_blocks.rs

@@ -4,10 +4,12 @@
 //! heap header's flag says so). The library reads only the blocks an object
 //! lives in and does not verify block checksums, so `check` does it here.
 
+use std::borrow::Cow;
 use std::collections::HashSet;
 
 use clawhdf5_format::checksum::jenkins_lookup3;
 use clawhdf5_format::fractal_heap::FractalHeapHeader;
+use clawhdf5_format::storage::{Storage, read_exact_at};
 
 use crate::h5::{Error, H5};
 
@@ -26,7 +28,7 @@ pub struct HeapReport {
 }
 
 struct Walk<'a> {
-    data: &'a [u8],
+    data: &'a dyn Storage,
     heap: u64,
     fh: FractalHeapHeader,
     checksum_dblocks: bool,
@@ -60,14 +62,14 @@ fn log2(v: u64) -> u32 {
 /// parsed (and its checksum verified) by the library; an error there is
 /// returned as the only problem.
 pub fn verify(h5: &H5, heap: u64) -> HeapReport {
-    let data = h5.data();
+    let data = h5.store();
     let Ok(off) = usize::try_from(heap) else {
         return HeapReport {
             problems: vec![Error::at(heap, "fractal heap address out of range")],
             ..Default::default()
         };
     };
-    let fh = match FractalHeapHeader::parse(data, off, h5.os(), h5.ls()) {
+    let fh = match FractalHeapHeader::parse_in(data, heap, h5.os(), h5.ls()) {
         Ok(f) => f,
         Err(e) => {
             return HeapReport {
@@ -77,7 +79,15 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport {
         }
     };
     // Flags: signature(4) version(1) heap ID length(2) filter length(2) flags(1).
-    let flags = data.get(off + 9).copied().unwrap_or(0);
+    let flags = match data.read_at(off as u64 + 9, 1) {
+        Ok(b) => b.first().copied().unwrap_or(0),
+        Err(e) => {
+            return HeapReport {
+                problems: vec![Error::at(heap, format!("fractal heap header: {e}"))],
+                ..Default::default()
+            };
+        }
+    };
     let mut w = Walk {
         data,
         heap,
@@ -107,11 +117,42 @@ pub fn verify(h5: &H5, heap: u64) -> HeapReport {
     w.r
 }
 
-impl Walk<'_> {
+impl<'a> Walk<'a> {
     fn problem(&mut self, addr: u64, msg: impl Into<String>) {
         self.r.problems.push(Error::at(addr, msg));
     }
 
+    /// Bytes `[start, end)` of the file: `Ok(None)` when they run past its
+    /// end (what a slice `get` of the whole file answered), `Err` when the
+    /// storage fails to read them (a remote file).
+    fn get(&self, start: usize, end: usize) -> Result<Option<Cow<'a, [u8]>>, String> {
+        let Some(len) = end.checked_sub(start) else {
+            return Ok(None);
+        };
+        if end as u64 > self.data.len() {
+            return Ok(None);
+        }
+        read_exact_at(self.data, start as u64, len)
+            .map(Some)
+            .map_err(|e| e.to_string())
+    }
+
+    /// [`Walk::get`], recording a read failure as a problem at `addr`.
+    fn get_or_note(
+        &mut self,
+        addr: u64,
+        start: usize,
+        end: usize,
+    ) -> Option<Option<Cow<'a, [u8]>>> {
+        match self.get(start, end) {
+            Ok(b) => Some(b),
+            Err(e) => {
+                self.problem(addr, format!("fractal heap block: {e}"));
+                None
+            }
+        }
+    }
+
     fn row_size(&self, row: usize) -> Option<u64> {
         let s = self.fh.starting_block_size;
         if row <= 1 {
@@ -155,10 +196,11 @@ impl Walk<'_> {
             return None;
         };
         let hdr_len = 5 + self.os + self.boff_bytes;
-        let Some(b) = start
-            .checked_add(hdr_len)
-            .and_then(|e| self.data.get(start..e))
-        else {
+        let b = match start.checked_add(hdr_len) {
+            Some(e) => self.get_or_note(addr, start, e)?,
+            None => None,
+        };
+        let Some(b) = b else {
             self.problem(
                 addr,
                 format!("fractal heap {what} block lies past the end of the file"),
@@ -209,7 +251,10 @@ impl Walk<'_> {
             self.problem(addr, "fractal heap direct block size out of range");
             return;
         };
-        let Some(block) = self.data.get(start..end) else {
+        let Some(block) = self.get_or_note(addr, start, end) else {
+            return;
+        };
+        let Some(block) = block else {
             self.problem(
                 addr,
                 "fractal heap direct block extends past the end of the file",
@@ -259,14 +304,17 @@ impl Walk<'_> {
             };
             let direct = row < direct_rows;
             for _ in 0..width {
-                let Some(b) = self.data.get(pos..pos + self.os) else {
+                let Some(b) = self.get_or_note(addr, pos, pos + self.os) else {
+                    return;
+                };
+                let Some(b) = b else {
                     self.problem(
                         addr,
                         "fractal heap indirect block extends past the end of the file",
                     );
                     return;
                 };
-                let child = le(b);
+                let child = le(&b);
                 pos += self.os;
                 if direct && filtered {
                     pos += self.ls + 4;
@@ -277,7 +325,10 @@ impl Walk<'_> {
                 off = off.saturating_add(rs);
             }
         }
-        let Some(stored) = self.data.get(pos..pos + 4) else {
+        let Some(stored) = self.get_or_note(addr, pos, pos + 4) else {
+            return;
+        };
+        let Some(stored) = stored else {
             self.problem(
                 addr,
                 "fractal heap indirect block extends past the end of the file",
@@ -285,7 +336,10 @@ impl Walk<'_> {
             return;
         };
         let stored = u32::from_le_bytes([stored[0], stored[1], stored[2], stored[3]]);
-        let computed = jenkins_lookup3(&self.data[start..pos]);
+        let Some(Some(body)) = self.get_or_note(addr, start, pos) else {
+            return;
+        };
+        let computed = jenkins_lookup3(&body);
         self.r.checksums += 1;
         if computed != stored {
             self.problem(

crates/clawhdf5-tools/src/info.rs

@@ -1,7 +1,7 @@
 //! Dataset facts shared by `ls`, `dump`, `stat` and `check`: shape text,
 //! layout, filters and storage.
 
-use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks};
+use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks_in};
 use clawhdf5_format::data_layout::DataLayout;
 use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
 use clawhdf5_format::datatype::Datatype;
@@ -189,8 +189,8 @@ pub fn chunks(
     let Some(addr) = *btree_address else {
         return Ok(Vec::new());
     };
-    list_chunks(
-        h5.data(),
+    list_chunks_in(
+        h5.store(),
         layout,
         ds,
         dt.type_size() as usize,

crates/clawhdf5-tools/src/ls.rs

@@ -63,7 +63,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
         return args.usage_error(out, "missing FILE", USAGE);
     };
     let (file, obj) = split_file_arg(&target);
-    let mut h5 = match H5::open(std::path::Path::new(&file)) {
+    let mut h5 = match H5::open_arg(&file) {
         Ok(h) => h,
         Err(e) => {
             writeln!(out.e, "h5rs ls: {e}")?;

crates/clawhdf5-tools/src/stat.rs

@@ -86,7 +86,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
     let Some(file) = file else {
         return args.usage_error(out, "missing FILE", USAGE);
     };
-    let h5 = match H5::open(std::path::Path::new(&file)) {
+    let h5 = match H5::open_arg(&file) {
         Ok(h) => h,
         Err(e) => {
             writeln!(out.e, "h5rs stat: {e}")?;
@@ -291,7 +291,7 @@ fn report(h5: &H5, file: &str, s: &Stats, out: &mut Out) -> std::io::Result<()>
         s.attr_objects
     )?;
     writeln!(o, "\tMax. # of attributes to objects: {}", s.max_attrs)?;
-    let total = std::fs::metadata(&h5.path).map(|m| m.len()).unwrap_or(0);
+    let total = h5.size;
     let ub = h5.file.user_block_size();
     writeln!(o, "Summary of file space information:")?;
     writeln!(o, "  User block: {ub} bytes")?;

crates/clawhdf5-tools/src/value.rs

@@ -151,14 +151,14 @@ pub struct Decoder<'a> {
     pub h5: &'a H5,
     /// Variable-length elements are resolved as the library resolves them
     /// (so as libhdf5 does), not by a decoder of our own.
-    vl: RefCell<VlResolver<'a>>,
+    vl: RefCell<VlResolver<'a, dyn clawhdf5_format::storage::Storage + 'a>>,
 }
 
 impl<'a> Decoder<'a> {
     pub fn new(h5: &'a H5) -> Self {
         Self {
             h5,
-            vl: RefCell::new(VlResolver::new(h5.data(), h5.os(), h5.ls())),
+            vl: RefCell::new(VlResolver::new_in(h5.store(), h5.os(), h5.ls())),
         }
     }
 
@@ -268,7 +268,7 @@ impl<'a> Decoder<'a> {
     /// has it.
     fn decode_vlen(&self, is_string: bool, base: &Datatype, b: &[u8], depth: u32) -> Value {
         if is_string {
-            return match self.vl.borrow_mut().string_element(b) {
+            return match self.vl.borrow_mut().string_element_in(b) {
                 Ok(Some(s)) => Value::Str(String::from_utf8_lossy(s).into_owned()),
                 Ok(None) => Value::NullStr,
                 Err(e) => Value::Error(e.to_string()),
@@ -278,8 +278,9 @@ impl<'a> Decoder<'a> {
         if bs == 0 {
             return Value::Error("VL base type of size 0".into());
         }
-        let obj = match self.vl.borrow_mut().element(b, bs) {
-            Ok(o) => o.unwrap_or(&[]),
+        // Copied out: decoding an element may resolve nested ones.
+        let obj = match self.vl.borrow_mut().element_in(b, bs) {
+            Ok(o) => o.unwrap_or(&[]).to_vec(),
             Err(e) => return Value::Error(e.to_string()),
         };
         Value::Seq(

crates/clawhdf5-tools/tests/remote.rs

@@ -0,0 +1,100 @@
+//! `h5rs` on URLs (feature `remote`): every subcommand prints for
+//! `http://…/file.h5` what it prints for the local file (the name aside).
+//! The files are served by the range-request test server of
+//! clawhdf5-remote on 127.0.0.1.
+
+#![cfg(feature = "remote")]
+
+#[path = "../../clawhdf5-remote/tests/common/server.rs"]
+mod server;
+
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+fn h5rs(args: &[&str]) -> (String, i32) {
+    let out = Command::new(env!("CARGO_BIN_EXE_h5rs"))
+        .args(args)
+        .output()
+        .expect("run h5rs");
+    let text = format!(
+        "{}{}",
+        String::from_utf8_lossy(&out.stdout),
+        String::from_utf8_lossy(&out.stderr)
+    );
+    (text, out.status.code().unwrap_or(-1))
+}
+
+fn fixtures() -> Vec<PathBuf> {
+    let root = Path::new(env!("CARGO_MANIFEST_DIR"));
+    [
+        "../clawhdf5/tests/fixtures/tall.h5",
+        "../clawhdf5/tests/fixtures/written_by_v2_7_0.h5",
+        "../clawhdf5/tests/fixtures/written_by_v2_7_0_paged.h5",
+        "../clawhdf5/tests/fixtures/h5clear_mdc_image.h5",
+        "../clawhdf5-format/tests/fixtures/fractal_heap_multiblock.h5",
+        "../clawhdf5-format/tests/fixtures/legacy/tcompound.h5",
+        "../clawhdf5-format/tests/fixtures/legacy/h5ex_g_iterate.h5",
+    ]
+    .iter()
+    .map(|p| root.join(p))
+    .collect()
+}
+
+#[test]
+fn every_subcommand_reads_a_url_like_the_local_file() {
+    let files = fixtures();
+    let served: Vec<(String, Vec<u8>)> = files
+        .iter()
+        .enumerate()
+        .map(|(i, p)| {
+            let name = p.file_name().unwrap().to_str().unwrap();
+            (format!("/{i}/{name}"), std::fs::read(p).unwrap())
+        })
+        .collect();
+    let server = server::Server::start(served.clone());
+    for (p, (url_path, _)) in files.iter().zip(&served) {
+        let url = server.url(url_path);
+        let local = p.to_str().unwrap();
+        for cmd in [
+            &["ls", "-r", "-v"][..],
+            &["dump"],
+            &["dump", "--json"],
+            &["stat"],
+            &["check", "--data"],
+        ] {
+            fn args<'a>(cmd: &[&'a str], f: &'a str) -> Vec<&'a str> {
+                cmd.iter().copied().chain([f]).collect()
+            }
+            let (want, want_rc) = h5rs(&args(cmd, local));
+            let (got, got_rc) = h5rs(&args(cmd, &url));
+            assert_eq!(
+                got.replace(&url, local),
+                want,
+                "h5rs {} {url}",
+                cmd.join(" ")
+            );
+            assert_eq!(got_rc, want_rc, "h5rs {} {url}", cmd.join(" "));
+        }
+        let (a, rc) = h5rs(&["diff", local, &url]);
+        assert_eq!(rc, 0, "h5rs diff {local} {url}: {a}");
+    }
+}
+
+#[test]
+fn url_errors_are_clean() {
+    let server = server::Server::start(vec![("/x.h5".into(), vec![1u8; 100])]);
+    let (out, rc) = h5rs(&["ls", &server.url("/missing.h5")]);
+    assert_eq!(rc, 2, "{out}");
+    assert!(out.contains("404"), "{out}");
+    let (out, rc) = h5rs(&["ls", &server.url("/x.h5")]);
+    assert_eq!(rc, 2, "{out}");
+    assert!(out.contains("not an HDF5 file"), "{out}");
+    let (out, rc) = h5rs(&["check", &server.url("/missing.h5")]);
+    assert_eq!(rc, 2, "{out}");
+    #[cfg(not(feature = "remote-https"))]
+    {
+        let (out, rc) = h5rs(&["ls", "https://example.com/a.h5"]);
+        assert_eq!(rc, 2, "{out}");
+        assert!(out.contains("`https` feature"), "{out}");
+    }
+}

scripts/ci-test.sh

@@ -117,21 +117,28 @@ run_step "cargo clippy (remote, all backends)" cargo clippy \
     --features object-store,https,s3,gcs,azure \
     -- -D warnings
 
+# h5rs with URL arguments.
+run_step "cargo clippy (h5rs remote)" cargo clippy \
+    -p clawhdf5-tools \
+    --all-targets \
+    --features remote-https \
+    -- -D warnings
+
 # The README promises that the core crates build no C by default. Hold it to
 # that: fail if a crate that compiles C (a *-sys crate, cc or cmake) enters the
 # default dependency tree of any of them. clawhdf5-migrate (bundled SQLite),
 # clawhdf5-napi (Node) and clawhdf5-gpu (graphics drivers) are exempt.
 # js-sys (clawhdf5-wasm's bindings to JavaScript) builds no C.
 # clawhdf5-remote is checked by default (plain HTTP) and with its
-# object-store feature; its https (ring) and s3/gcs/azure (aws-lc-rs)
-# features build C and are opt-in.
+# object-store feature, and h5rs with URL support (remote); the https
+# (ring) and s3/gcs/azure (aws-lc-rs) features build C and are opt-in.
 no_c_in_default_build() {
     local entry crate features found=0
     for entry in clawhdf5-format clawhdf5-io clawhdf5-filters clawhdf5 \
         clawhdf5-agent clawhdf5-ann clawhdf5-accel clawhdf5-netcdf4 clawhdf5-cli \
         clawhdf5-tools \
         clawhdf5-wasm \
-        clawhdf5-remote clawhdf5-remote:object-store; do
+        clawhdf5-remote clawhdf5-remote:object-store clawhdf5-tools:remote; do
         crate=${entry%%:*}
         features=()
         [ "$entry" != "$crate" ] && features=(--features "${entry#*:}")
@@ -210,6 +217,11 @@ run_step "cargo test (remote, object_store backend, s3 URLs)" cargo test \
     -p clawhdf5-remote \
     --features object-store,s3
 
+run_step "cargo test (h5rs on URLs)" cargo test \
+    -p clawhdf5-tools \
+    --features remote \
+    --test remote
+
 run_step "cargo test (ann parallel)" cargo test \
     -p clawhdf5-ann \
     --features parallel