feat(tools): h5rs, pure-Rust HDF5 tools (ls, dump, stat, diff, check)

New workspace crate clawhdf5-tools with one binary, h5rs, built only on the clawhdf5 facade and clawhdf5-format (no libhdf5, no C): - ls [-r] [-v] FILE[/path]: h5ls's listing (same text in its first two columns) plus the datatype; -v adds address, link count, layout and chunk index, chunk size, storage, filters, datatype and attributes. - dump [--json] [-A] [-p] [-d PATH] FILE: h5dump DDL (byte-identical to h5dump 1.14.6 on the test files) or hdf5-json. - stat FILE: h5stat's object/link/rank/layout/filter/attribute counts, raw data and total size. - diff [-r] [-q] [-d D] [-p R] A B [OBJ1 [OBJ2]]: structural and value differences, exit 0/1/2 like h5diff. - check [--data] FILE: walks every object, parses every message, verifies the checksums of every v2+ structure (including the fractal heap blocks the library never checks), checks chunk indexes against their datasets and raw data for out-of-file or overlapping extents; every problem with its address. Values over --max-bytes are reported, not read; dense-storage heaps are verified before objects are read from them; panics are caught (exit 3). Tests compare with h5ls, h5stat, h5dump and h5diff and with h5py's values, and flip the checksum of every checksummed structure in a v1.14-format file. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-09-26 00:29:28 -05:00
parent bb78d70b99
commit 310448bfcb
18 changed files with 6550 additions and 0 deletions
@@ -0,0 +1,245 @@
+//! Dataset facts shared by `ls`, `dump`, `stat` and `check`: shape text,
+//! layout, filters and storage.
+
+use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks};
+use clawhdf5_format::data_layout::DataLayout;
+use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
+use clawhdf5_format::datatype::Datatype;
+use clawhdf5_format::filter_pipeline::{FilterDescription, FilterPipeline};
+use clawhdf5_format::message_type::MessageType;
+use clawhdf5_format::object_header::ObjectHeader;
+
+use crate::h5::{Error, H5, Result};
+
+/// h5ls's `{10/Inf, 20}` shape text. `always_max` prints `cur/max` for every
+/// dimension (h5ls -v).
+pub fn shape_text(ds: &Dataspace, always_max: bool) -> String {
+    match ds.space_type {
+        DataspaceType::Null => "{NULL}".into(),
+        DataspaceType::Scalar => "{SCALAR}".into(),
+        DataspaceType::Simple => {
+            let dims: Vec<String> = ds
+                .dimensions
+                .iter()
+                .enumerate()
+                .map(|(i, &d)| {
+                    let m = ds.max_dimensions.as_ref().and_then(|m| m.get(i).copied());
+                    match m {
+                        Some(u64::MAX) => format!("{d}/Inf"),
+                        Some(m) if m != d || always_max => format!("{d}/{m}"),
+                        None if always_max => format!("{d}/{d}"),
+                        _ => d.to_string(),
+                    }
+                })
+                .collect();
+            format!("{{{}}}", dims.join(", "))
+        }
+    }
+}
+
+/// h5dump's `SIMPLE { ( 3, 4 ) / ( 3, H5S_UNLIMITED ) }`.
+pub fn dataspace_ddl(ds: &Dataspace) -> String {
+    match ds.space_type {
+        DataspaceType::Null => "NULL".into(),
+        DataspaceType::Scalar => "SCALAR".into(),
+        DataspaceType::Simple => {
+            let cur: Vec<String> = ds.dimensions.iter().map(|d| d.to_string()).collect();
+            let max: Vec<String> = ds
+                .dimensions
+                .iter()
+                .enumerate()
+                .map(
+                    |(i, &d)| match ds.max_dimensions.as_ref().and_then(|m| m.get(i).copied()) {
+                        Some(u64::MAX) => "H5S_UNLIMITED".into(),
+                        Some(m) => m.to_string(),
+                        None => d.to_string(),
+                    },
+                )
+                .collect();
+            format!(
+                "SIMPLE {{ ( {} ) / ( {} ) }}",
+                cur.join(", "),
+                max.join(", ")
+            )
+        }
+    }
+}
+
+pub fn filter_name(f: &FilterDescription) -> String {
+    let known = match f.filter_id {
+        1 => "deflate",
+        2 => "shuffle",
+        3 => "fletcher32",
+        4 => "szip",
+        5 => "nbit",
+        6 => "scaleoffset",
+        307 => "bzip2",
+        32000 => "lzf",
+        32001 => "blosc",
+        32004 => "lz4",
+        32008 => "bitshuffle",
+        32013 => "zfp",
+        32015 => "zstd",
+        32026 => "blosc2",
+        _ => "",
+    };
+    if !known.is_empty() {
+        return known.into();
+    }
+    match &f.name {
+        Some(n) if !n.is_empty() => n.clone(),
+        _ => "user-defined".into(),
+    }
+}
+
+/// `deflate-1 OPT {4}` as h5ls prints a filter.
+pub fn filter_text(f: &FilterDescription) -> String {
+    let opt = if f.flags & 1 != 0 { " OPT" } else { "" };
+    let cd = if f.client_data.is_empty() {
+        String::new()
+    } else {
+        format!(
+            " {{{}}}",
+            f.client_data
+                .iter()
+                .map(|c| c.to_string())
+                .collect::<Vec<_>>()
+                .join(", ")
+        )
+    };
+    format!("{}-{}{opt}{cd}", filter_name(f), f.filter_id)
+}
+
+pub fn layout_name(l: &DataLayout) -> &'static str {
+    match l {
+        DataLayout::Compact { .. } => "compact",
+        DataLayout::Contiguous { .. } => "contiguous",
+        DataLayout::Chunked { .. } => "chunked",
+        DataLayout::Virtual { .. } => "virtual",
+    }
+}
+
+/// Chunk index kind of a chunked layout.
+pub fn chunk_index_name(l: &DataLayout) -> &'static str {
+    match l {
+        DataLayout::Chunked {
+            version,
+            chunk_index_type,
+            ..
+        } => {
+            if *version < 4 {
+                return "v1 B-tree";
+            }
+            match chunk_index_type {
+                Some(1) => "single chunk",
+                Some(2) => "implicit",
+                Some(3) => "fixed array",
+                Some(4) => "extensible array",
+                Some(5) => "v2 B-tree",
+                _ => "unknown",
+            }
+        }
+        _ => "",
+    }
+}
+
+/// Everything about one dataset that can be learned without reading its
+/// values.
+pub struct DsInfo {
+    pub dt: Result<Datatype>,
+    pub ds: Result<Dataspace>,
+    pub layout: Result<DataLayout>,
+    pub filters: Result<Option<FilterPipeline>>,
+    pub external: bool,
+}
+
+impl DsInfo {
+    pub fn read(h5: &H5, path: &str, h: &ObjectHeader) -> DsInfo {
+        DsInfo {
+            dt: h5.datatype(h),
+            ds: h5.resolved_dataspace(path, h),
+            layout: h5.layout(h),
+            filters: h5.filters(h),
+            external: h
+                .messages
+                .iter()
+                .any(|m| m.msg_type == MessageType::ExternalDataFiles),
+        }
+    }
+
+    pub fn logical_bytes(&self) -> Option<u64> {
+        let (Ok(dt), Ok(ds)) = (&self.dt, &self.ds) else {
+            return None;
+        };
+        crate::h5::byte_len(ds, dt).ok()
+    }
+}
+
+/// Every allocated chunk (empty when none are). Errors only for a corrupt
+/// chunk index.
+pub fn chunks(
+    h5: &H5,
+    layout: &DataLayout,
+    ds: &Dataspace,
+    dt: &Datatype,
+) -> Result<Vec<ChunkInfo>> {
+    let DataLayout::Chunked { btree_address, .. } = layout else {
+        return Ok(Vec::new());
+    };
+    let Some(addr) = *btree_address else {
+        return Ok(Vec::new());
+    };
+    list_chunks(
+        h5.data(),
+        layout,
+        ds,
+        dt.type_size() as usize,
+        h5.os(),
+        h5.ls(),
+    )
+    .map(|(c, _)| c)
+    .map_err(|e| {
+        Error::at(
+            addr,
+            format!("chunk index ({}): {e}", chunk_index_name(layout)),
+        )
+    })
+}
+
+/// Bytes of raw data the dataset has allocated in the file (what libhdf5's
+/// `H5Dget_storage_size` reports).
+pub fn allocated_bytes(h5: &H5, info: &DsInfo) -> Result<u64> {
+    let layout = info.layout.as_ref().map_err(Clone::clone)?;
+    Ok(match layout {
+        DataLayout::Compact { data } => data.len() as u64,
+        DataLayout::Contiguous { address, size } => {
+            if address.is_some() {
+                *size
+            } else {
+                0
+            }
+        }
+        DataLayout::Chunked { .. } => {
+            let dt = info.dt.as_ref().map_err(Clone::clone)?;
+            let ds = info.ds.as_ref().map_err(Clone::clone)?;
+            chunks(h5, layout, ds, dt)?
+                .iter()
+                .map(|c| u64::from(c.chunk_size))
+                .sum()
+        }
+        DataLayout::Virtual { .. } => 0,
+    })
+}
+
+/// Number of hard links to an object, as its header records it.
+pub fn link_count(h: &ObjectHeader) -> u64 {
+    if let Some(rc) = h.reference_count {
+        return u64::from(rc);
+    }
+    h.messages
+        .iter()
+        .find(|m| m.msg_type == MessageType::ObjectReferenceCount)
+        .and_then(|m| m.data.get(1..5))
+        .map(|b| u64::from(u32::from_le_bytes([b[0], b[1], b[2], b[3]])))
+        .unwrap_or(1)
+}