h5rs: URLs as FILE arguments (feature remote)

With the `remote` feature (`remote-https` for https://), ls, dump, stat
and diff take an http(s):// (or s3://, gs://, az:// with those
clawhdf5-remote features) URL wherever they take a file, and read it by
range requests through clawhdf5-remote's block cache. check validates
every byte, so it downloads a remote file whole and checks it as before.
Without the feature a URL is a clean error naming it.

The tools read the file through File::storage instead of as_bytes: object
headers, shared messages, attributes, v1 and v2 group links, dense
storage (fractal heaps and v2 B-trees), path resolution, chunk listings
and variable-length values go through the format crate's *_in functions,
and the fractal-heap block verifier reads each block through the storage
(a read failure of a remote file is reported as a problem, not as "past
the end of the file"). A local file's storage is its mapped bytes, so its
reads are still slices. stat's file size comes from the opened file, so
it is right for a URL.

Tests: tests/remote.rs serves fixtures (old and new formats, a paged
file, a metadata cache image, a multi-block fractal heap, compounds, v1
groups) with the clawhdf5-remote test server and requires every
subcommand's output and exit status for the URL to equal the local
file's, and diff of the two to be clean; 404s, non-HDF5 bodies and
https without its feature are clean errors. Local output is unchanged:
the old and new h5rs print the same for ls -r -v, dump, stat and check
--data on the 747 conformance and CVE corpus files (tank, 2026-09-26;
the dumps of h5diff_hyper1/2.h5 were too large for the comparison
script, their ls, stat and check agree), except cve-2025-2310.h5, whose
dump error messages differ between runs of the old binary too (which
failing chunk is reported first).

ci-test.sh lints h5rs with remote-https, runs the URL tests and checks
h5rs with remote for C.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-tools/src/h5.rs

@@ -10,9 +10,9 @@ use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 
 use clawhdf5::File;
-use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant};
+use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant_in};
 use clawhdf5_format::attribute_info::AttributeInfoMessage;
-use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records};
+use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records_in};
 use clawhdf5_format::data_layout::DataLayout;
 use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
 use clawhdf5_format::datatype::Datatype;
@@ -24,6 +24,7 @@ use clawhdf5_format::link_info::LinkInfoMessage;
 use clawhdf5_format::link_message::{LinkMessage, LinkTarget};
 use clawhdf5_format::message_type::MessageType;
 use clawhdf5_format::object_header::ObjectHeader;
+use clawhdf5_format::storage::Storage;
 use clawhdf5_format::superblock::Superblock;
 use clawhdf5_format::symbol_table::SymbolTableMessage;
 
@@ -186,8 +187,11 @@ pub struct Link {
 
 /// An open file.
 pub struct H5 {
+    /// The file's path, or its URL for a remote file.
     pub path: PathBuf,
     pub file: File,
+    /// Size of the whole file in bytes (user block included).
+    pub size: u64,
     pub max_bytes: u64,
     /// Fractal heaps whose blocks were verified: `None` = sound.
     verified_heaps: RefCell<HashMap<u64, Option<Error>>>,
@@ -204,19 +208,90 @@ impl H5 {
                 path.display()
             ))
         })?;
-        Ok(H5 {
-            path: path.to_path_buf(),
+        let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
+        Ok(H5::new(path.to_path_buf(), file, size))
+    }
+
+    fn new(path: PathBuf, file: File, size: u64) -> H5 {
+        H5 {
+            path,
             file,
+            size,
             max_bytes: DEFAULT_MAX_BYTES,
             verified_heaps: RefCell::new(HashMap::new()),
-        })
+        }
+    }
+
+    /// Open a command-line FILE argument: a path, or with the `remote`
+    /// feature an `http(s)://` (or `s3://`, `gs://`, `az://`) URL, read by
+    /// range requests through a block cache.
+    pub fn open_arg(arg: &str) -> Result<H5> {
+        if !is_url(arg) {
+            return H5::open(Path::new(arg));
+        }
+        #[cfg(feature = "remote")]
+        {
+            let storage =
+                clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default())
+                    .map_err(|e| Error::new(format!("{arg}: {e}")))?;
+            let size = storage.len();
+            let file = File::open_storage(storage).map_err(|e| {
+                Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}"))
+            })?;
+            Ok(H5::new(PathBuf::from(arg), file, size))
+        }
+        #[cfg(not(feature = "remote"))]
+        Err(Error::new(format!(
+            "{arg}: URLs need h5rs built with the `remote` feature"
+        )))
+    }
+
+    /// [`H5::open_arg`], with a remote file downloaded whole into memory
+    /// first — for `check`, which validates every byte of the file anyway
+    /// and parses it as one slice ([`H5::data`]).
+    pub fn open_arg_whole(arg: &str) -> Result<H5> {
+        let h5 = H5::open_arg(arg)?;
+        if h5.file.contiguous_bytes().is_some() {
+            return Ok(h5);
+        }
+        #[cfg(feature = "remote")]
+        {
+            let storage =
+                clawhdf5_remote::storage_for_url(arg, &clawhdf5_remote::Options::default())
+                    .map_err(|e| Error::new(format!("{arg}: {e}")))?;
+            let len = usize::try_from(storage.len())
+                .map_err(|_| Error::new(format!("{arg}: too large to download")))?;
+            let bytes = storage
+                .read_at(0, len)
+                .map_err(|e| Error::new(format!("{arg}: {e}")))?
+                .into_owned();
+            let size = bytes.len() as u64;
+            let file = File::from_bytes(bytes).map_err(|e| {
+                Error::new(format!("{arg}: not an HDF5 file this tool can open: {e}"))
+            })?;
+            Ok(H5::new(PathBuf::from(arg), file, size))
+        }
+        #[cfg(not(feature = "remote"))]
+        unreachable!("open_arg refuses URLs without the remote feature")
     }
 
     /// The file's bytes from the superblock on: what every address indexes.
+    ///
+    /// # Panics
+    ///
+    /// For a remote file opened by [`H5::open_arg`]; read it through
+    /// [`H5::store`], or open it with [`H5::open_arg_whole`].
     pub fn data(&self) -> &[u8] {
         self.file.as_bytes()
     }
 
+    /// The same bytes as a [`Storage`], for local and remote files alike:
+    /// in memory a read is a slice of [`H5::data`], remote it is served by
+    /// the block cache.
+    pub fn store(&self) -> &dyn Storage {
+        self.file.storage()
+    }
+
     pub fn sb(&self) -> &Superblock {
         self.file.superblock()
     }
@@ -239,8 +314,8 @@ impl H5 {
         if let Some(e) = self.file.cache_image_error() {
             return Err(Error::at(addr, format!("metadata cache image: {e}")));
         }
-        let off = usize::try_from(addr).map_err(|_| Error::at(addr, "address out of range"))?;
-        ObjectHeader::parse(self.data(), off, self.os(), self.ls())
+        to_usize(addr)?;
+        ObjectHeader::parse_in(self.store(), addr, self.os(), self.ls())
             .map_err(|e| Error::at(addr, format!("object header: {e}")))
     }
 
@@ -249,11 +324,14 @@ impl H5 {
     pub fn payload(&self, h: &ObjectHeader, t: MessageType) -> Result<Option<Vec<u8>>> {
         match h.messages.iter().find(|m| m.msg_type == t) {
             None => Ok(None),
-            Some(m) => {
-                clawhdf5_format::shared_message::message_data(self.data(), m, self.os(), self.ls())
-                    .map(|c| Some(c.into_owned()))
-                    .map_err(|e| Error::new(format!("{t:?} message: {e}")))
-            }
+            Some(m) => clawhdf5_format::shared_message::message_data_in(
+                self.store(),
+                m,
+                self.os(),
+                self.ls(),
+            )
+            .map(|c| Some(c.into_owned()))
+            .map_err(|e| Error::new(format!("{t:?} message: {e}"))),
         }
     }
 
@@ -305,8 +383,9 @@ impl H5 {
             self.verified_heap(fh)
                 .map_err(|e| e.context("dense attribute storage"))?;
         }
-        let (mut attrs, errs) = extract_attributes_tolerant(self.data(), h, self.os(), self.ls())
-            .map_err(|e| Error::new(format!("attributes: {e}")))?;
+        let (mut attrs, errs) =
+            extract_attributes_tolerant_in(self.store(), h, self.os(), self.ls())
+                .map_err(|e| Error::new(format!("attributes: {e}")))?;
         attrs.sort_by(|a, b| a.name.cmp(&b.name));
         Ok((attrs, errs.iter().map(|e| e.to_string()).collect()))
     }
@@ -316,7 +395,7 @@ impl H5 {
     pub fn links(&self, h: &ObjectHeader) -> Result<Vec<Link>> {
         let os = self.os();
         let ls = self.ls();
-        let data = self.data();
+        let data = self.store();
         let mut out = Vec::new();
         if let Some(m) = h
             .messages
@@ -325,7 +404,7 @@ impl H5 {
         {
             let stm = SymbolTableMessage::parse(&m.data, os)
                 .map_err(|e| Error::new(format!("symbol table message: {e}")))?;
-            let entries = group_v1::resolve_v1_group_entries(data, &stm, os, ls)
+            let entries = group_v1::resolve_v1_group_entries_in(data, &stm, os, ls)
                 .map_err(|e| Error::at(stm.btree_address, format!("symbol table: {e}")))?;
             let has_soft = entries.iter().any(group_v1::is_v1_soft_link);
             for e in entries {
@@ -337,7 +416,7 @@ impl H5 {
                 }
             }
             if has_soft {
-                let soft = group_v1::v1_soft_links(data, &stm, os, ls)
+                let soft = group_v1::v1_soft_links_in(data, &stm, os, ls)
                     .map_err(|e| Error::at(stm.btree_address, format!("soft links: {e}")))?;
                 for (name, target) in soft {
                     out.push(Link {
@@ -387,15 +466,15 @@ impl H5 {
         name_type: u8,
     ) -> Result<Vec<Vec<u8>>> {
         self.verified_heap(heap)?;
-        let data = self.data();
+        let data = self.store();
         let os = self.os();
         let ls = self.ls();
-        let fh = FractalHeapHeader::parse(data, to_usize(heap)?, os, ls)
+        let fh = FractalHeapHeader::parse_in(data, to_usize(heap).map(|_| heap)?, os, ls)
             .map_err(|e| Error::at(heap, format!("fractal heap header: {e}")))?;
         let bt = btree.ok_or_else(|| Error::at(heap, "dense storage without a name index"))?;
-        let hdr = BTreeV2Header::parse(data, to_usize(bt)?, os, ls)
+        let hdr = BTreeV2Header::parse_in(data, to_usize(bt).map(|_| bt)?, os, ls)
             .map_err(|e| Error::at(bt, format!("v2 B-tree header: {e}")))?;
-        let recs = collect_btree_v2_records(data, &hdr, os, ls)
+        let recs = collect_btree_v2_records_in(data, &hdr, os, ls)
             .map_err(|e| Error::at(bt, format!("v2 B-tree: {e}")))?;
         // Name-index records: hash(4) + heap ID; creation-order ones: order(8) + heap ID.
         let skip = if hdr.tree_type == name_type { 4 } else { 8 };
@@ -407,7 +486,7 @@ impl H5 {
                 .get(skip..skip + idlen)
                 .ok_or_else(|| Error::at(bt, "v2 B-tree record shorter than a heap ID"))?;
             let obj = fh
-                .read_managed_object(data, id, os)
+                .read_managed_object_in(data, id, os)
                 .map_err(|e| Error::at(heap, format!("fractal heap object: {e}")))?;
             out.push(obj);
         }
@@ -561,7 +640,7 @@ impl H5 {
         if p.is_empty() {
             return Ok(self.root());
         }
-        clawhdf5_format::group_v2::resolve_path_any(self.data(), self.sb(), p)
+        clawhdf5_format::group_v2::resolve_path_any_in(self.store(), self.sb(), p)
             .map_err(|e| Error::new(format!("{path}: {e}")))
     }
 }
@@ -681,7 +760,9 @@ pub fn byte_len(ds: &Dataspace, dt: &Datatype) -> Result<u64> {
 /// Split a `FILE[/object/path]` argument the way h5ls does: the longest
 /// prefix that is an existing file is the file.
 pub fn split_file_arg(arg: &str) -> (String, Option<String>) {
-    if Path::new(arg).is_file() {
+    // A URL names the file only (its path cannot be split against the
+    // local file system).
+    if is_url(arg) || Path::new(arg).is_file() {
         return (arg.to_string(), None);
     }
     let mut idx: Vec<usize> = arg.match_indices('/').map(|(i, _)| i).collect();
@@ -694,3 +775,13 @@ pub fn split_file_arg(arg: &str) -> (String, Option<String>) {
     }
     (arg.to_string(), None)
 }
+
+/// Whether a FILE argument is a URL (`scheme://...`) rather than a path.
+pub fn is_url(arg: &str) -> bool {
+    arg.split_once("://").is_some_and(|(scheme, _)| {
+        !scheme.is_empty()
+            && scheme
+                .chars()
+                .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
+    })
+}