diff --git a/CHANGELOG.md b/CHANGELOG.md index a8203d6..dbaf30b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,8 +123,10 @@ - `h5rs dump [--json] [-A] [-p] [-d PATH] FILE` prints DDL text that is byte-identical to h5dump 1.14.6's on the test files (all layouts and chunk indexes, v1/v2 groups, compound, enum, strings, links, named - types, attributes), or JSON in the HDF Group's hdf5-json layout (schema - in the crate README). + types, attributes; null-padded strings show their NULs at any depth), + or JSON in the HDF Group's hdf5-json layout (schema in the crate + README). Nested compounds print inline and `long double` values as + errors (exit 1); both are listed in the README. - `h5rs stat FILE` reports h5stat's object, link, rank, layout, filter, attribute, raw-data and file-size figures (equal to h5stat's on the test files); metadata space is one figure, not broken down. diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index 8b94845..e045498 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -72,10 +72,22 @@ groups; integers of both byte orders, floats, compound with an array member, enum, fixed and variable-length strings; soft, external and hard links; a named datatype; compact and dense attributes) `h5rs dump` and `h5rs dump -A` print the same bytes as h5dump 1.14.6 — `dump_matches_h5dump` in -`tests/h5rs_interop.rs` checks this. Not covered by that test: references, -opaque, bitfield, variable-length sequences and virtual datasets. Floats print at their own precision (a `float32` 0.1 prints as -`0.1`), which for some values is more digits than h5dump's `%g`. The `-p` -block is h5rs's own (it names the chunk index), not h5dump's. +`tests/h5rs_interop.rs` checks this, and `dump_shows_nul_padding_in_nested_strings` +that null-padded strings show their NULs (`"a\000b"`) at any depth, as +h5dump's do. Not covered by those tests: references, opaque, bitfield, +variable-length sequences and virtual datasets. Known differences from +h5dump: + +- Floats print at their own precision (a `float32` 0.1 prints as `0.1`), + which for some values is more digits than h5dump's `%g`. +- A compound nested in a compound prints inline (`{ 1, 2.5 }`) where + h5dump prints it as an indented block, one member per line; only the + outer compound is a block. +- `long double` (x87 80-bit) and other floats wider than 64 bits: the + datatype is printed as an `H5T_FLOAT { ... }` block instead of h5dump's + one-line description, and each value as ``; `dump` then + exits 1. The library cannot convert them (see `docs/known-issues.md`). +- The `-p` block is h5rs's own (it names the chunk index), not h5dump's. ### JSON schema diff --git a/crates/clawhdf5-tools/src/dump.rs b/crates/clawhdf5-tools/src/dump.rs index e743d36..7b1d8e7 100644 --- a/crates/clawhdf5-tools/src/dump.rs +++ b/crates/clawhdf5-tools/src/dump.rs @@ -518,7 +518,9 @@ impl Dump<'_> { errors += 1; } let comma = if i + 1 < n { "," } else { "" }; - if let Value::Compound(members) = &v { + let esize = dt.type_size() as usize; + let eb = raw.get(i * esize..(i + 1) * esize).unwrap_or_default(); + if let (Value::Compound(_), Datatype::Compound { members, .. }) = (&v, dt) { // h5dump prints each compound element as a block, one // member per line. if !line.is_empty() { @@ -526,25 +528,22 @@ impl Dump<'_> { line.clear(); } writeln!(out.o, "{pad}({}): {{", index_text(i as u64, &dims))?; - for (k, (_, m)) in members.iter().enumerate() { + for (k, m) in members.iter().enumerate() { let sep = if k + 1 < members.len() { "," } else { "" }; - writeln!(out.o, "{pad} {}{sep}", value::text(m, &paths))?; + let mb = usize::try_from(m.byte_offset) + .ok() + .and_then(|o| eb.get(o..)) + .unwrap_or_default(); + let t = ddl_text(&dec, &m.datatype, mb, &paths, 1); + writeln!(out.o, "{pad} {t}{sep}")?; } writeln!(out.o, "{pad} }}{comma}")?; continue; } - let t = match dt { - // h5dump shows a null-padded string's padding. - Datatype::String { - padding: StringPadding::NullPad, - size, - .. - } if !matches!(v, Value::Error(_)) => { - let sz = *size as usize; - let b = raw.get(i * sz..(i + 1) * sz).unwrap_or_default(); - format!("{}{comma}", value::quote_bytes(b)) - } - _ => format!("{}{comma}", value::text(&v, &paths)), + let t = if matches!(v, Value::Error(_)) { + format!("{}{comma}", value::text(&v, &paths)) + } else { + format!("{}{comma}", ddl_text(&dec, dt, eb, &paths, 0)) }; let row_start = i % last == 0; if row_start || line.len() + 1 + t.len() > 77 { @@ -863,6 +862,65 @@ const MAX_DDL_DEPTH: usize = 256; const JSON_BYTES_PER_ELEMENT: u64 = 64; /// `(i,j,k)` index of flat element `i` of an array with `dims`. +/// One element as h5dump prints it in a DATA block: [`value::text`], except +/// that a null-padded fixed string shows its padding (every byte, NULs as +/// `\000`) at any depth, in compound members and array elements too. +fn ddl_text( + dec: &Decoder, + dt: &Datatype, + b: &[u8], + paths: &dyn Fn(u64) -> Option, + depth: u32, +) -> String { + let err = || value::text(&Value::Error("short element".into()), paths); + if depth > 32 { + return value::text(&dec.decode(dt, b, depth), paths); + } + match dt { + Datatype::String { + padding: StringPadding::NullPad, + size, + .. + } => match b.get(..*size as usize) { + Some(s) => value::quote_bytes(s), + None => err(), + }, + Datatype::Compound { members, .. } => format!( + "{{ {} }}", + members + .iter() + .map( + |m| match usize::try_from(m.byte_offset).ok().and_then(|o| b.get(o..)) { + Some(mb) => ddl_text(dec, &m.datatype, mb, paths, depth + 1), + None => err(), + } + ) + .collect::>() + .join(", ") + ), + Datatype::Array { + base_type, + dimensions, + } => { + let bs = base_type.type_size() as usize; + let n = dimensions + .iter() + .try_fold(1usize, |a, &d| a.checked_mul(d as usize)); + match n.filter(|n| n.checked_mul(bs).is_some_and(|t| t <= b.len())) { + Some(n) => format!( + "[ {} ]", + (0..n) + .map(|k| ddl_text(dec, base_type, &b[k * bs..], paths, depth + 1)) + .collect::>() + .join(", ") + ), + None => value::text(&dec.decode(dt, b, depth), paths), + } + } + _ => value::text(&dec.decode(dt, b, depth), paths), + } +} + fn index_text(mut i: u64, dims: &[u64]) -> String { let mut idx = vec![0u64; dims.len()]; for (k, &d) in dims.iter().enumerate().rev() { diff --git a/crates/clawhdf5-tools/tests/gen_files.py b/crates/clawhdf5-tools/tests/gen_files.py index bc18574..8bc4e1e 100644 --- a/crates/clawhdf5-tools/tests/gen_files.py +++ b/crates/clawhdf5-tools/tests/gen_files.py @@ -149,6 +149,15 @@ for name, t in (("target1.h5", "/a"), ("target2.h5", "/b")): f["s"] = h5py.SoftLink(t) f["rel"] = h5py.SoftLink("a") +# Null-padded fixed strings with NULs, at the top level and nested in a +# compound and in an array member. +with h5py.File(os.path.join(out, "nulstrings.h5"), "w") as f: + f["top"] = np.array([b"", b"ab", b"a\x00b"], dtype="S3") + f["cmp"] = np.array( + [(b"", 1), (b"ab", 2), (b"a\x00b", 3)], dtype=[("s", "S3"), ("i", "i4")] + ) + f["arr"] = np.array([([b"", b"x"],)], dtype=[("v", "(2,)S2")]) + with h5py.File(os.path.join(out, "userblock.h5"), "w", userblock_size=1024) as f: f["d"] = np.arange(10) diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index c325533..1669beb 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -280,6 +280,28 @@ fn dump_matches_h5dump() { ); } +/// A null-padded fixed string prints every byte (NULs as `\000`), as +/// h5dump prints it, also inside a compound and an array member. +#[test] +fn dump_shows_nul_padding_in_nested_strings() { + let Some(f) = generate() else { return }; + let p = f.p("nulstrings.h5"); + let ours = stdout(&h5rs(&["dump", &p])); + for want in [ + r#"(0): "\000\000\000", "ab\000", "a\000b""#, + r#""\000\000\000","#, + r#""a\000b","#, + r#"[ "\000\000", "x\000" ]"#, + ] { + assert!(ours.contains(want), "no {want} in\n{ours}"); + } + if missing(tool_available("h5dump"), "h5dump") { + return; + } + let reference = run("h5dump", &[&p]); + assert_eq!(ours, stdout(&reference).replacen(&p, "nulstrings.h5", 1)); +} + #[test] fn dump_json_values_match_h5py() { let Some(f) = generate() else { return };