diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index c0bf701..54e56d4 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -385,6 +385,9 @@ enum NbitNode { count: usize, base_size: usize, }, + /// `H5Z_NBIT_NOOPTYPE`: a field N-Bit does not reduce (enum, string, + /// opaque, ...), stored as all `size` bytes, 8 bits each. + Noop { size: usize }, } impl NbitNode { @@ -395,6 +398,7 @@ impl NbitNode { NbitNode::Array { count, base_size, .. } => count * base_size, + NbitNode::Noop { size } => *size, } } } @@ -414,6 +418,7 @@ fn parse_nbit_node(cd: &[u32], idx: &mut usize, depth: u32) -> Result NBIT_MAX_DEPTH { return Err(FormatError::ChunkedReadError( "nbit: type tree nested too deeply".into(), @@ -489,8 +494,17 @@ fn parse_nbit_node(cd: &[u32], idx: &mut usize, depth: u32) -> Result { + // class, size + let size = nbit_cd(cd, *idx + 1)? as usize; + *idx += 2; + if size == 0 { + return Err(FormatError::ChunkedReadError( + "nbit: invalid no-op type size".into(), + )); + } + Ok(NbitNode::Noop { size }) + } _ => Err(FormatError::UnsupportedFilter(FILTER_NBIT)), } } @@ -555,6 +569,11 @@ fn decode_nbit_node( decode_nbit_node(bnode, br, elem, base + i * base_size)?; } } + NbitNode::Noop { size } => { + for slot in &mut elem[base..base + size] { + *slot = br.read(8)? as u8; + } + } } Ok(()) } @@ -567,17 +586,28 @@ fn decode_nbit_node( /// type tree — atomic (`[1, size, order, precision, offset]`), array /// (`[2, total_size, ]`) and compound /// (`[3, total_size, nmembers, (offset, )*]`) — preceded by -/// `[nparms, flag, nelmts]`. Decompression walks the tree once per element, +/// `[nparms, need_not_compress, nelmts]`; when `need_not_compress` is set +/// (every field already uses its full width, e.g. a 32-bit int of precision +/// 32) libhdf5 stores the data unchanged and so do we. Fields N-Bit cannot +/// reduce (enums, strings, ...) are no-op nodes (`[4, size]`) copied whole. +/// Decompression walks the tree once per element, /// placing each field's bits at its byte/bit offset in a zero-filled element /// (HDF5's canonical reduced-precision layout). Sign-extension of reduced -/// precision signed integers is the datatype reader's job. Atomic floats are -/// encoded as full-precision atomics and handled transparently. +/// precision signed integers is the datatype reader's job, and so is +/// converting a reduced-precision float (its own sign/exponent/mantissa +/// layout, e.g. `le_data.h5`'s 20-bit `Nbit_float_data_*`) to IEEE: the +/// filter's output is the file type's bytes, as libhdf5's is before type +/// conversion. fn nbit_decompress(data: &[u8], cd: &[u32], expected_bytes: usize) -> Result, FormatError> { - if cd.len() < 4 { + if cd.len() < 3 { return Err(FormatError::ChunkedReadError( "nbit: missing filter client data".into(), )); } + // H5Z__filter_nbit: `if (cd_values[1]) HGOTO_DONE(*buf_size)`. + if cd[1] != 0 { + return Ok(data.to_vec()); + } let nelmts = cd[2] as usize; let mut idx = 3; let root = parse_nbit_node(cd, &mut idx, 0)?; @@ -1584,6 +1614,157 @@ mod tests { // --- LZ4 tests --- + fn unhex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() + } + + fn one_filter(filter_id: u16, client_data: Vec) -> FilterDescription { + FilterDescription { + filter_id, + name: None, + flags: 1, + client_data, + } + } + + /// `le_data.h5` `/Nbit_float_data_{le,be}` chunk (0,0): a 20-bit float + /// (offset 7) packed by N-Bit. The filter must reproduce libhdf5's + /// decoded bytes in the *file* datatype (h5py `DatasetID.read` with the + /// file type as memory type, so no conversion); converting that custom + /// float layout to IEEE is the datatype reader's job, not the filter's. + #[test] + fn nbit_float_matches_libhdf5_file_type_bytes() { + let file: &[u8] = include_bytes!("../tests/fixtures/filters/le_data.h5"); + let cases = [ + ( + 55952, + 0, + "8055d5018055e5010000f0018055e5010000f0018055f5010000f0018055f50180aafa018055f50180aafa0100000002", + ), + ( + 56076, + 1, + "01d5558001e5558001f0000001e5558001f0000001f5558001f0000001f5558001faaa8001f5558001faaa8002000000", + ), + ]; + for (off, order, want) in cases { + let pipeline = FilterPipeline { + version: 2, + filters: vec![one_filter(FILTER_NBIT, vec![8, 0, 12, 1, 4, order, 20, 7])], + }; + let got = decompress_chunk(&file[off..off + 31], &pipeline, 48, 4).unwrap(); + assert_eq!(got, unhex(want), "byte order {order}"); + } + } + + /// libhdf5 sets `cd_values[1]` ("need not compress") when every field is + /// already full width and then stores the data unchanged; we unpacked it + /// anyway and failed with "nbit: packed data too short". + #[test] + fn nbit_need_not_compress_is_passthrough() { + let data: Vec = (0..200u32).map(|i| (i * 7) as u8).collect(); + let pipeline = FilterPipeline { + version: 2, + filters: vec![one_filter(FILTER_NBIT, vec![8, 1, 50, 1, 4, 0, 32, 0])], + }; + assert_eq!(decompress_chunk(&data, &pipeline, 200, 4).unwrap(), data); + // A top-level type N-Bit has no parameters for (e.g. an enum) carries only + // [nparms, need_not_compress, nelmts]. + let pipeline = FilterPipeline { + version: 2, + filters: vec![one_filter(FILTER_NBIT, vec![3, 1, 50])], + }; + assert_eq!(decompress_chunk(&data, &pipeline, 200, 4).unwrap(), data); + } + + /// `tfilters.h5` `/all` chunk (0,0): shuffle, szip, deflate, fletcher32 + /// and a pass-through N-Bit in one pipeline; values from h5py. + #[test] + #[cfg(feature = "szip")] + fn nbit_in_multi_filter_pipeline_matches_libhdf5() { + let raw = unhex(concat!( + "785e3bc1c0c030cb6517ff039e556de1576c0f300b152c771070641170640b99ba879f8167d5cce82f760ccc4285d71b", + "20c2afc226329f60d60ab5636a3fc1905499c3c0a1d0c4a1d05c6a13d7f88271aaf6725fe7170c86363f1858c0ca0104", + "bf1e95c75f3eeb", + )); + let want = unhex(concat!( + "00000000010000000200000003000000040000000a0000000b0000000c0000000d0000000e0000001400000015000000", + "1600000017000000180000001e0000001f00000020000000210000002200000028000000290000002a0000002b000000", + "2c00000032000000330000003400000035000000360000003c0000003d0000003e0000003f0000004000000046000000", + "4700000048000000490000004a00000050000000510000005200000053000000540000005a0000005b0000005c000000", + "5d0000005e000000", + )); + let pipeline = FilterPipeline { + version: 2, + filters: vec![ + one_filter(FILTER_SHUFFLE, vec![4]), + one_filter(FILTER_SZIP, vec![141, 4, 32, 5]), + one_filter(FILTER_DEFLATE, vec![5]), + one_filter(FILTER_FLETCHER32, vec![]), + one_filter(FILTER_NBIT, vec![8, 1, 50, 1, 4, 0, 32, 0]), + ], + }; + assert_eq!(decompress_chunk(&raw, &pipeline, 200, 4).unwrap(), want); + } + + /// `h5repack_nested_8bit_enum_deflated.h5` `/tracks/1/trace` chunk 0: a + /// 376-byte compound whose `u1` enum member N-Bit stores whole as a + /// no-op type (class 4), then deflate. Was `UnsupportedFilter(5)`. + /// Expected bytes: libhdf5's decode in the file datatype. + #[test] + #[cfg(feature = "deflate")] + fn nbit_compound_with_enum_member_matches_libhdf5() { + #[rustfmt::skip] + let cd: Vec = vec![ + 251, 0, 1, 3, 376, 38, + 0, 1, 4, 0, 32, 0, + 8, 2, 96, 1, 8, 0, 64, 0, + 104, 1, 8, 0, 64, 0, 112, 1, 8, 0, 64, 0, 120, 1, 8, 0, 64, 0, + 128, 1, 8, 0, 64, 0, 136, 1, 8, 0, 64, 0, 144, 1, 8, 0, 64, 0, + 152, 1, 8, 0, 64, 0, + 160, 2, 24, 1, 8, 0, 64, 0, 184, 2, 24, 1, 8, 0, 64, 0, + 208, 2, 16, 1, 8, 0, 64, 0, 224, 2, 32, 1, 8, 0, 64, 0, + 256, 2, 16, 1, 4, 0, 32, 0, 272, 2, 16, 1, 4, 0, 32, 0, + 288, 2, 32, 1, 8, 0, 64, 0, 320, 2, 16, 1, 8, 0, 64, 0, + 336, 2, 8, 1, 4, 0, 32, 0, + 344, 4, 1, + 346, 1, 2, 0, 16, 0, 348, 1, 4, 0, 32, 0, 352, 1, 1, 0, 4, 0, + 354, 1, 2, 0, 16, 0, 356, 1, 1, 0, 4, 0, 357, 1, 1, 0, 4, 0, + 358, 1, 1, 0, 4, 0, 359, 1, 1, 0, 4, 0, 360, 1, 1, 0, 4, 0, + 361, 1, 1, 0, 4, 0, 362, 1, 1, 0, 4, 0, 364, 1, 1, 0, 4, 0, + 363, 1, 1, 0, 4, 0, 365, 1, 1, 0, 4, 0, 366, 1, 1, 0, 4, 0, + 367, 1, 1, 0, 4, 0, 368, 1, 1, 0, 4, 0, 369, 1, 1, 0, 4, 0, + 370, 1, 1, 0, 4, 0, + ]; + assert_eq!(cd.len(), 251); + let raw = unhex(concat!( + "780163606078c930c880c3879573a60b2d7883eeac06a880835c47bda16cda7987a0c59ec9f74c4af6ff677e5df16445", + "adfd8493ce3ba592ddedab2a3bee87dd0faaff00d1808b66606006fa9d999b818125014837fd4703fba1fa71d150e720", + "512caa4073dc59212206500946060600604c37fc", + )); + let want = unhex(concat!( + "e90000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "0000000000000000eca012979ca9f040000000000000000000000000000000000000000000000080cf661d317f881e40", + "7434de6349a352407da8e478eb03ffbf47631ab943c9903f52df56df88797a3f000000000000f07f000000000000f07f", + "000000000000f07f000000000000f07fe90300000b0300006004000082030000ffffffffffffffffffffffffffffffff", + "000000000000f0bf000000000000f0bf000000000000f0bf000000000000f0bf00000000000000000000000000000000", + "25040000470300000500000000000000030000000000000000000000000000000100000000000000", + )); + let pipeline = FilterPipeline { + version: 2, + filters: vec![ + one_filter(FILTER_NBIT, cd), + one_filter(FILTER_DEFLATE, vec![1]), + ], + }; + assert_eq!(decompress_chunk(&raw, &pipeline, 376, 376).unwrap(), want); + } + /// Chunk (0,0) of `/DS1` in the HDF Group's `h5ex_d_lz4.h5` example, /// written by libhdf5's registered LZ4 plugin with a 3-byte block size /// (so it has many blocks, some stored raw). Byte range from h5py's