From c4d96c13900720df3cc9abe48ddd8d4fc7d0aec9 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 01:23:32 -0500 Subject: [PATCH] fix(tools): h5rs dump shows NUL padding in nested strings, like h5dump A null-padded fixed string inside a compound or an array member printed trimmed ("" for three NULs, "a" for "a\0b"), where h5dump prints every byte ("\000\000\000", "a\000b"); only top-level strings were shown in full. DATA blocks now render elements through one function that keeps the padding at any depth. The README now lists the remaining known differences from h5dump: nested compounds print inline, and long double values are printed as errors (exit 1) with the datatype as an H5T_FLOAT block. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 +- crates/clawhdf5-tools/README.md | 20 ++++- crates/clawhdf5-tools/src/dump.rs | 88 +++++++++++++++++---- crates/clawhdf5-tools/tests/gen_files.py | 9 +++ crates/clawhdf5-tools/tests/h5rs_interop.rs | 22 ++++++ 5 files changed, 124 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8203d6..dbaf30b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,8 +123,10 @@ - `h5rs dump [--json] [-A] [-p] [-d PATH] FILE` prints DDL text that is byte-identical to h5dump 1.14.6's on the test files (all layouts and chunk indexes, v1/v2 groups, compound, enum, strings, links, named - types, attributes), or JSON in the HDF Group's hdf5-json layout (schema - in the crate README). + types, attributes; null-padded strings show their NULs at any depth), + or JSON in the HDF Group's hdf5-json layout (schema in the crate + README). Nested compounds print inline and `long double` values as + errors (exit 1); both are listed in the README. - `h5rs stat FILE` reports h5stat's object, link, rank, layout, filter, attribute, raw-data and file-size figures (equal to h5stat's on the test files); metadata space is one figure, not broken down. diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index 8b94845..e045498 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -72,10 +72,22 @@ groups; integers of both byte orders, floats, compound with an array member, enum, fixed and variable-length strings; soft, external and hard links; a named datatype; compact and dense attributes) `h5rs dump` and `h5rs dump -A` print the same bytes as h5dump 1.14.6 — `dump_matches_h5dump` in -`tests/h5rs_interop.rs` checks this. Not covered by that test: references, -opaque, bitfield, variable-length sequences and virtual datasets. Floats print at their own precision (a `float32` 0.1 prints as -`0.1`), which for some values is more digits than h5dump's `%g`. The `-p` -block is h5rs's own (it names the chunk index), not h5dump's. +`tests/h5rs_interop.rs` checks this, and `dump_shows_nul_padding_in_nested_strings` +that null-padded strings show their NULs (`"a\000b"`) at any depth, as +h5dump's do. Not covered by those tests: references, opaque, bitfield, +variable-length sequences and virtual datasets. Known differences from +h5dump: + +- Floats print at their own precision (a `float32` 0.1 prints as `0.1`), + which for some values is more digits than h5dump's `%g`. +- A compound nested in a compound prints inline (`{ 1, 2.5 }`) where + h5dump prints it as an indented block, one member per line; only the + outer compound is a block. +- `long double` (x87 80-bit) and other floats wider than 64 bits: the + datatype is printed as an `H5T_FLOAT { ... }` block instead of h5dump's + one-line description, and each value as ``; `dump` then + exits 1. The library cannot convert them (see `docs/known-issues.md`). +- The `-p` block is h5rs's own (it names the chunk index), not h5dump's. ### JSON schema diff --git a/crates/clawhdf5-tools/src/dump.rs b/crates/clawhdf5-tools/src/dump.rs index e743d36..7b1d8e7 100644 --- a/crates/clawhdf5-tools/src/dump.rs +++ b/crates/clawhdf5-tools/src/dump.rs @@ -518,7 +518,9 @@ impl Dump<'_> { errors += 1; } let comma = if i + 1 < n { "," } else { "" }; - if let Value::Compound(members) = &v { + let esize = dt.type_size() as usize; + let eb = raw.get(i * esize..(i + 1) * esize).unwrap_or_default(); + if let (Value::Compound(_), Datatype::Compound { members, .. }) = (&v, dt) { // h5dump prints each compound element as a block, one // member per line. if !line.is_empty() { @@ -526,25 +528,22 @@ impl Dump<'_> { line.clear(); } writeln!(out.o, "{pad}({}): {{", index_text(i as u64, &dims))?; - for (k, (_, m)) in members.iter().enumerate() { + for (k, m) in members.iter().enumerate() { let sep = if k + 1 < members.len() { "," } else { "" }; - writeln!(out.o, "{pad} {}{sep}", value::text(m, &paths))?; + let mb = usize::try_from(m.byte_offset) + .ok() + .and_then(|o| eb.get(o..)) + .unwrap_or_default(); + let t = ddl_text(&dec, &m.datatype, mb, &paths, 1); + writeln!(out.o, "{pad} {t}{sep}")?; } writeln!(out.o, "{pad} }}{comma}")?; continue; } - let t = match dt { - // h5dump shows a null-padded string's padding. - Datatype::String { - padding: StringPadding::NullPad, - size, - .. - } if !matches!(v, Value::Error(_)) => { - let sz = *size as usize; - let b = raw.get(i * sz..(i + 1) * sz).unwrap_or_default(); - format!("{}{comma}", value::quote_bytes(b)) - } - _ => format!("{}{comma}", value::text(&v, &paths)), + let t = if matches!(v, Value::Error(_)) { + format!("{}{comma}", value::text(&v, &paths)) + } else { + format!("{}{comma}", ddl_text(&dec, dt, eb, &paths, 0)) }; let row_start = i % last == 0; if row_start || line.len() + 1 + t.len() > 77 { @@ -863,6 +862,65 @@ const MAX_DDL_DEPTH: usize = 256; const JSON_BYTES_PER_ELEMENT: u64 = 64; /// `(i,j,k)` index of flat element `i` of an array with `dims`. +/// One element as h5dump prints it in a DATA block: [`value::text`], except +/// that a null-padded fixed string shows its padding (every byte, NULs as +/// `\000`) at any depth, in compound members and array elements too. +fn ddl_text( + dec: &Decoder, + dt: &Datatype, + b: &[u8], + paths: &dyn Fn(u64) -> Option, + depth: u32, +) -> String { + let err = || value::text(&Value::Error("short element".into()), paths); + if depth > 32 { + return value::text(&dec.decode(dt, b, depth), paths); + } + match dt { + Datatype::String { + padding: StringPadding::NullPad, + size, + .. + } => match b.get(..*size as usize) { + Some(s) => value::quote_bytes(s), + None => err(), + }, + Datatype::Compound { members, .. } => format!( + "{{ {} }}", + members + .iter() + .map( + |m| match usize::try_from(m.byte_offset).ok().and_then(|o| b.get(o..)) { + Some(mb) => ddl_text(dec, &m.datatype, mb, paths, depth + 1), + None => err(), + } + ) + .collect::>() + .join(", ") + ), + Datatype::Array { + base_type, + dimensions, + } => { + let bs = base_type.type_size() as usize; + let n = dimensions + .iter() + .try_fold(1usize, |a, &d| a.checked_mul(d as usize)); + match n.filter(|n| n.checked_mul(bs).is_some_and(|t| t <= b.len())) { + Some(n) => format!( + "[ {} ]", + (0..n) + .map(|k| ddl_text(dec, base_type, &b[k * bs..], paths, depth + 1)) + .collect::>() + .join(", ") + ), + None => value::text(&dec.decode(dt, b, depth), paths), + } + } + _ => value::text(&dec.decode(dt, b, depth), paths), + } +} + fn index_text(mut i: u64, dims: &[u64]) -> String { let mut idx = vec![0u64; dims.len()]; for (k, &d) in dims.iter().enumerate().rev() { diff --git a/crates/clawhdf5-tools/tests/gen_files.py b/crates/clawhdf5-tools/tests/gen_files.py index bc18574..8bc4e1e 100644 --- a/crates/clawhdf5-tools/tests/gen_files.py +++ b/crates/clawhdf5-tools/tests/gen_files.py @@ -149,6 +149,15 @@ for name, t in (("target1.h5", "/a"), ("target2.h5", "/b")): f["s"] = h5py.SoftLink(t) f["rel"] = h5py.SoftLink("a") +# Null-padded fixed strings with NULs, at the top level and nested in a +# compound and in an array member. +with h5py.File(os.path.join(out, "nulstrings.h5"), "w") as f: + f["top"] = np.array([b"", b"ab", b"a\x00b"], dtype="S3") + f["cmp"] = np.array( + [(b"", 1), (b"ab", 2), (b"a\x00b", 3)], dtype=[("s", "S3"), ("i", "i4")] + ) + f["arr"] = np.array([([b"", b"x"],)], dtype=[("v", "(2,)S2")]) + with h5py.File(os.path.join(out, "userblock.h5"), "w", userblock_size=1024) as f: f["d"] = np.arange(10) diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index c325533..1669beb 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -280,6 +280,28 @@ fn dump_matches_h5dump() { ); } +/// A null-padded fixed string prints every byte (NULs as `\000`), as +/// h5dump prints it, also inside a compound and an array member. +#[test] +fn dump_shows_nul_padding_in_nested_strings() { + let Some(f) = generate() else { return }; + let p = f.p("nulstrings.h5"); + let ours = stdout(&h5rs(&["dump", &p])); + for want in [ + r#"(0): "\000\000\000", "ab\000", "a\000b""#, + r#""\000\000\000","#, + r#""a\000b","#, + r#"[ "\000\000", "x\000" ]"#, + ] { + assert!(ours.contains(want), "no {want} in\n{ours}"); + } + if missing(tool_available("h5dump"), "h5dump") { + return; + } + let reference = run("h5dump", &[&p]); + assert_eq!(ours, stdout(&reference).replacen(&p, "nulstrings.h5", 1)); +} + #[test] fn dump_json_values_match_h5py() { let Some(f) = generate() else { return };