From aef8e766ae9ea9206613bd4fe8dbad7bffc998e5 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:03:49 -0500 Subject: [PATCH] fix(format): write and read the registered HDF5 LZ4 filter format Filter 32004 chunks were framed as a 4-byte little-endian size plus one LZ4 block. That is not the registered HDF5 LZ4 format (H5Zlz4.c: 8-byte big-endian total size, 4-byte big-endian block size, then per block a 4-byte big-endian compressed length and the block, stored raw when the length equals the block size), so libhdf5 + hdf5plugin could not read our LZ4 datasets and we could not read theirs (h5ex_d_lz4.h5: "lz4: 0 is not a valid match offset"). Write the registered format (cd_values[0] is honoured as the block size, default 1 GiB like the plugin) and read it, multi-block and raw blocks included. Chunks in the old framing stay readable: an HDF5 chunk is under 4 GiB, so a registered chunk always starts with four zero bytes and is at least 12 bytes long, while an old one starts with four zero bytes only when empty (5 bytes). Tests: lz4_reads_registered_hdf5_format (chunk of the HDF Group's h5ex_d_lz4.h5, block size 3), lz4_writes_registered_hdf5_format, lz4_reads_legacy_clawhdf5_format, and hdf5plugin_reads_our_lz4 (ignored interop test; failed before with "filter returned failure during read"). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters.rs | 223 ++++++++++++++++-- .../tests/fixtures/filters/README.md | 10 + .../tests/fixtures/filters/h5ex_d_lz4.h5 | Bin 0 -> 23984 bytes .../tests/writer_h5py_tests.rs | 45 ++++ 4 files changed, 254 insertions(+), 24 deletions(-) create mode 100644 crates/clawhdf5-format/tests/fixtures/filters/README.md create mode 100644 crates/clawhdf5-format/tests/fixtures/filters/h5ex_d_lz4.h5 diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index c10cea9..d12302b 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -69,7 +69,7 @@ pub fn compress_chunk( let level = filter.client_data.first().copied().unwrap_or(6); deflate_compress(&result, level)? } - FILTER_LZ4 => lz4_compress(&result)?, + FILTER_LZ4 => lz4_compress(&result, &filter.client_data)?, FILTER_ZSTD => { let level = filter.client_data.first().copied().unwrap_or(3); zstd_compress(&result, level)? @@ -813,12 +813,32 @@ fn deflate_compress(_data: &[u8], _level: u32) -> Result, FormatError> { Err(FormatError::UnsupportedFilter(FILTER_DEFLATE)) } -/// Decompress LZ4 data. Format: 4 bytes LE original size + LZ4 block data. +/// Default LZ4 block size of the registered HDF5 LZ4 filter (`H5Zlz4.c`, +/// `DEFAULT_BLOCK_SIZE`): 1 GiB, so an HDF5 chunk is normally one block. +#[cfg(feature = "lz4")] +const LZ4_DEFAULT_BLOCK_SIZE: usize = 1 << 30; + +/// Decompress an LZ4 (filter 32004) chunk. /// -/// The 4-byte "original size" header is part of the attacker-controlled -/// compressed payload itself, so it is bounded against `expected_bytes` (the -/// pipeline's declared chunk size) before being used to size the output -/// allocation — otherwise a crafted 4-byte value can request up to ~4 GiB. +/// Two framings are read: +/// +/// * The registered HDF5 LZ4 filter format (`H5Zlz4.c`, what libhdf5 + +/// hdf5plugin write, and what clawhdf5 writes after 2.7.0): an 8-byte +/// big-endian total decompressed size, a 4-byte big-endian block size, then +/// per block a 4-byte big-endian compressed length followed by the block. A +/// block whose compressed length equals its decompressed length is stored +/// raw. +/// * The legacy clawhdf5 framing (up to 2.7.0): a 4-byte little-endian size +/// followed by one raw LZ4 block. libhdf5 cannot read it. +/// +/// They are told apart unambiguously: an HDF5 chunk is smaller than 4 GiB, so +/// the registered format's big-endian `u64` size always starts with four zero +/// bytes and the whole chunk is at least 12 bytes; a legacy chunk starts with +/// four zero bytes only when it is empty, and is then 5 bytes long. +/// +/// Every size read from the payload is bounded against `expected_bytes` (the +/// pipeline's declared chunk size) before it sizes an allocation, so a crafted +/// header cannot request gigabytes. #[cfg(feature = "lz4")] fn lz4_decompress(data: &[u8], expected_bytes: usize) -> Result, FormatError> { if data.len() < 4 { @@ -826,38 +846,112 @@ fn lz4_decompress(data: &[u8], expected_bytes: usize) -> Result, FormatE "lz4: data too short".into(), )); } + let check_size = |size: usize| -> Result<(), FormatError> { + if expected_bytes != 0 && size > expected_bytes { + return Err(FormatError::DecompressionError( + "lz4: declared size exceeds chunk size".into(), + )); + } + if size > MAX_DECOMPRESS_SIZE { + return Err(FormatError::DecompressionError( + "lz4: declared size exceeds limit".into(), + )); + } + Ok(()) + }; + if data.len() >= 12 && data[..4] == [0, 0, 0, 0] { + return lz4_decompress_hdf5(data, check_size); + } + // Legacy clawhdf5 framing: 4-byte LE size + one LZ4 block. let orig_size = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize; - if expected_bytes != 0 && orig_size > expected_bytes { - return Err(FormatError::DecompressionError( - "lz4: declared size exceeds chunk size".into(), - )); - } - if orig_size > MAX_DECOMPRESS_SIZE { - return Err(FormatError::DecompressionError( - "lz4: declared size exceeds limit".into(), - )); - } + check_size(orig_size)?; lz4_flex::block::decompress(&data[4..], orig_size) .map_err(|e| FormatError::DecompressionError(format!("lz4: {e}"))) } +/// Decode the registered HDF5 LZ4 framing (see [`lz4_decompress`]). +#[cfg(feature = "lz4")] +fn lz4_decompress_hdf5( + data: &[u8], + check_size: impl Fn(usize) -> Result<(), FormatError>, +) -> Result, FormatError> { + let err = |m: &str| FormatError::DecompressionError(format!("lz4: {m}")); + let be32 = |b: &[u8]| u32::from_be_bytes([b[0], b[1], b[2], b[3]]) as usize; + // The first four bytes are zero (checked by the caller), so the size is + // the low 32 bits of the big-endian u64. + let orig_size = be32(&data[4..8]); + check_size(orig_size)?; + let block_size = be32(&data[8..12]).min(orig_size); + if block_size == 0 && orig_size != 0 { + return Err(err("zero block size")); + } + let mut out = vec![0u8; orig_size]; + let mut pos = 12usize; + let mut done = 0usize; + while done < orig_size { + let this_block = block_size.min(orig_size - done); + let comp_len = be32( + data.get(pos..pos + 4) + .ok_or_else(|| err("truncated block header"))?, + ); + pos += 4; + let block = data + .get(pos..pos.saturating_add(comp_len)) + .ok_or_else(|| err("truncated block"))?; + let dst = &mut out[done..done + this_block]; + if comp_len == this_block { + dst.copy_from_slice(block); + } else { + let n = lz4_flex::block::decompress_into(block, dst) + .map_err(|e| FormatError::DecompressionError(format!("lz4: {e}")))?; + if n != this_block { + return Err(err("block decompressed to the wrong size")); + } + } + pos += comp_len; + done += this_block; + } + Ok(out) +} + #[cfg(not(feature = "lz4"))] fn lz4_decompress(_data: &[u8], _expected_bytes: usize) -> Result, FormatError> { Err(FormatError::UnsupportedFilter(FILTER_LZ4)) } -/// Compress data with LZ4 block format. Format: 4 bytes LE original size + LZ4 block data. +/// Compress data in the registered HDF5 LZ4 filter format (see +/// [`lz4_decompress`]), so libhdf5 with the LZ4 plugin (e.g. hdf5plugin) can +/// read it. `cd[0]`, when present and non-zero, is the block size in bytes, +/// as in `H5Zlz4.c`; otherwise the 1 GiB default applies. #[cfg(feature = "lz4")] -fn lz4_compress(data: &[u8]) -> Result, FormatError> { - let compressed = lz4_flex::block::compress(data); - let mut result = Vec::with_capacity(4 + compressed.len()); - result.extend_from_slice(&(data.len() as u32).to_le_bytes()); - result.extend_from_slice(&compressed); +fn lz4_compress(data: &[u8], cd: &[u32]) -> Result, FormatError> { + let block_size = match cd.first() { + Some(&b) if b != 0 => b as usize, + _ => LZ4_DEFAULT_BLOCK_SIZE, + } + .min(data.len()); + let mut result = Vec::with_capacity(16 + data.len() / 2); + result.extend_from_slice(&(data.len() as u64).to_be_bytes()); + result.extend_from_slice(&(block_size as u32).to_be_bytes()); + if block_size == 0 { + return Ok(result); + } + for block in data.chunks(block_size) { + let compressed = lz4_flex::block::compress(block); + if compressed.len() >= block.len() { + // Incompressible: stored raw, marked by length == block length. + result.extend_from_slice(&(block.len() as u32).to_be_bytes()); + result.extend_from_slice(block); + } else { + result.extend_from_slice(&(compressed.len() as u32).to_be_bytes()); + result.extend_from_slice(&compressed); + } + } Ok(result) } #[cfg(not(feature = "lz4"))] -fn lz4_compress(_data: &[u8]) -> Result, FormatError> { +fn lz4_compress(_data: &[u8], _cd: &[u32]) -> Result, FormatError> { Err(FormatError::UnsupportedFilter(FILTER_LZ4)) } @@ -1480,11 +1574,92 @@ mod tests { // --- LZ4 tests --- + /// Chunk (0,0) of `/DS1` in the HDF Group's `h5ex_d_lz4.h5` example, + /// written by libhdf5's registered LZ4 plugin with a 3-byte block size + /// (so it has many blocks, some stored raw). Byte range from h5py's + /// `get_chunk_info`; values are `i*j - j` (i32 LE), as h5py reads them. + #[test] + #[cfg(feature = "lz4")] + fn lz4_reads_registered_hdf5_format() { + let file: &[u8] = include_bytes!("../tests/fixtures/filters/h5ex_d_lz4.h5"); + let chunk = &file[4016..4016 + 312]; + let pipeline = FilterPipeline { + version: 2, + filters: vec![FilterDescription { + filter_id: FILTER_LZ4, + name: None, + flags: 1, + client_data: vec![3], + }], + }; + let out = decompress_chunk(chunk, &pipeline, 4 * 8 * 4, 4).unwrap(); + let expected: Vec = (0..4i32) + .flat_map(|i| (0..8i32).map(move |j| i * j - j)) + .flat_map(i32::to_le_bytes) + .collect(); + assert_eq!(out, expected); + } + + #[test] + #[cfg(feature = "lz4")] + fn lz4_writes_registered_hdf5_format() { + // Compressible data, one block: 8-byte BE size, 4-byte BE block size, + // 4-byte BE compressed length, block. + let data = vec![7u8; 1000]; + let c = lz4_compress(&data, &[]).unwrap(); + assert_eq!(&c[0..8], &1000u64.to_be_bytes()); + assert_eq!(&c[8..12], &1000u32.to_be_bytes()); + let len = u32::from_be_bytes(c[12..16].try_into().unwrap()) as usize; + assert_eq!(c.len(), 16 + len); + assert!(len < 1000); + assert_eq!(lz4_decompress(&c, 1000).unwrap(), data); + + // Several blocks, incompressible ones stored raw (length == block). + let data: Vec = (0..10u8).collect(); + let c = lz4_compress(&data, &[3]).unwrap(); + assert_eq!(&c[8..12], &3u32.to_be_bytes()); + assert_eq!(&c[12..16], &3u32.to_be_bytes()); + assert_eq!(&c[16..19], &[0, 1, 2]); + assert_eq!(c.len(), 12 + 3 * (4 + 3) + (4 + 1)); + assert_eq!(lz4_decompress(&c, 10).unwrap(), data); + + let c = lz4_compress(&[], &[]).unwrap(); + assert_eq!(lz4_decompress(&c, 0).unwrap(), Vec::::new()); + } + + /// Chunks written by clawhdf5 up to 2.7.0 (4-byte LE size + one LZ4 + /// block) must stay readable. + #[test] + #[cfg(feature = "lz4")] + fn lz4_reads_legacy_clawhdf5_format() { + for data in [vec![], vec![5u8; 300], (0..=255u8).collect::>()] { + let mut legacy = (data.len() as u32).to_le_bytes().to_vec(); + legacy.extend_from_slice(&lz4_flex::block::compress(&data)); + assert_eq!(lz4_decompress(&legacy, data.len()).unwrap(), data); + } + } + + #[test] + #[cfg(feature = "lz4")] + fn lz4_registered_format_rejects_hostile_sizes() { + // Declared total larger than the chunk. + let mut c = 1000u64.to_be_bytes().to_vec(); + c.extend_from_slice(&1000u32.to_be_bytes()); + c.extend_from_slice(&[0u8; 8]); + assert!(lz4_decompress(&c, 64).is_err()); + // Truncated block. + let mut c = 16u64.to_be_bytes().to_vec(); + c.extend_from_slice(&16u32.to_be_bytes()); + c.extend_from_slice(&16u32.to_be_bytes()); + c.extend_from_slice(&[1u8; 4]); + assert!(lz4_decompress(&c, 16).is_err()); + } + #[test] #[cfg(feature = "lz4")] fn lz4_compress_decompress_roundtrip() { let data: Vec = (0..256).map(|i| (i % 256) as u8).collect(); - let compressed = lz4_compress(&data).unwrap(); + let compressed = lz4_compress(&data, &[]).unwrap(); let decompressed = lz4_decompress(&compressed, data.len()).unwrap(); assert_eq!(decompressed, data); } diff --git a/crates/clawhdf5-format/tests/fixtures/filters/README.md b/crates/clawhdf5-format/tests/fixtures/filters/README.md new file mode 100644 index 0000000..d2095a0 --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/filters/README.md @@ -0,0 +1,10 @@ +# Filter conformance fixtures + +Files written by libhdf5 (and its registered filter plugins), used by the +filter regression tests in `src/filters.rs` to compare our decoders against +the values h5py/libhdf5 read from the same bytes. Chunk byte ranges quoted in +the tests come from h5py's `DatasetID.get_chunk_info`. + +| File | Origin | Licence | +|------|--------|---------| +| `h5ex_d_lz4.h5` | HDF Group `HDF5Examples/C/H5FLT/tfiles/h5ex_d_lz4.h5` (hdf5 repository) | HDF5 licence (BSD-3-Clause style) | diff --git a/crates/clawhdf5-format/tests/fixtures/filters/h5ex_d_lz4.h5 b/crates/clawhdf5-format/tests/fixtures/filters/h5ex_d_lz4.h5 new file mode 100644 index 0000000000000000000000000000000000000000..d2790150378c8e9b603a64dc6fcaf43c4f9dea4b GIT binary patch literal 23984 zcmeI4f0$NPwZ}IcqzMs6BSTYsGea^3R6-Q4jx^9zNRyJ1kxl$^EYi7%LWVx0p_12X z@|ICqO-fH>W|LW$yyl(6OIT!+mkJYYEJ89;GE6S{c+$_nfoW+3#x~ZEBoxz_7!HCH&RZr6H-FZ~f&Z{dK)3d%W$uMQ;Ylu996l zIKLT`@V}Hs$$yaYm*ZxiHGO)rNYC$9vNT~xeply?zuW&;Gte~sjAjgYjz;Zg=|-Jr zJ{gT?pTM_%J~lb{U)n+GWjufL3(nB7Isf2%gqrd;$d=oMo$rXOw-Yy?!Z~8UR7d~d zA}88EwX!`(9UpQ9hC3+P<@v_i;>#!27G1b_>B1#bYL_frSZiInwC&XKd}-Ubi#u>Cy`?zUY#1txG?+*pl(Dkgd}d zSqv_g@0We(9xg9-^I*P34^o=a(K9!Pq;&SIw>RR7Mj-xbo-jB6{`xbRN*$aajZjvHHw+xW`r zxCy1W^s4H($)&i->adSj%S9Zr zJi0orJb$Z>td1+sUw5rKt~`HxUNbPRJRg~+G~ubUa!Z5dM? zSIh3c*LqEqp9kJAx%=XpUZ3BV*Ch6RYFO^SAts1LMl$RQFQX zr0Tfx{OvfoI9p#&^88IcePEn! z!`T^L+Zw9l%JaAOjOw`Z{Pj+&jw{dKsK)BJT6XunHrrDG{`~y&ag^t8)r{)6^89tb zxjL>qe|wq+#+Cb@sonqHS{+xOzt*=^$Cc-=b7pm1`S@GTtd6T?ci(IM+pFWs^Vif| z9ao;e_IFgrmFI8WSp(w^&&|Ji-F0?#TzUS+zOy>6Jbx|cRL7O)Z}qv=akcF3d+nK1 z9ao;ey7Q{z%JbLo?&`Sm{4MXRuycGZ8$E6KvxA%|wc9&X-m1O}t0U2R>7^I5Ve17s zpBMXUV0*>>Q_N6*GBo}VS}0|`gZVM7$Zd0??sAr&KZ-O|hkaRKa3PZVl8*7lcsygVK%NhdckT8MA;lhz|2d8SFL2tX1I)8xyT$$-*fU~(+6VLI<9IoaXa8XG z#!Qj+XDcO)%h>_&S&L&dCms!)3-+AhQ;)5p$x{b(XK&BC!mun73UT z|I-@(Q_yPO{wTIn><@uGDfat)FmD{I4d9?Kj{I0tBtEtni{sCa9!26j7{eomkH2Q` z(K;%dbTnINq#~D~MZP9YfYQ))$K{a3vaMp?c4+)hX#Cru)x7;qY@66`1N)8Gpk%8s=UXe~T9|RKvn*^UZdCxP3Qp$5q6=RuG zF>k#Z|F1Rv$Dq}`Ju0?U?2*8JCHC+>n711GYy-d}laGVvi?v=W7SEgfh*Knwpr{y0((g8 zXZv8@xQ5yQ@aW~^z%_&xAD_&L>&V&hV!WoziANo0&0;J&%sD%-bHwJr8pBfUy1dL5 z>3pz?dF#>mf2#362(9MrCt{n#ejM13#2(lO^Tun-27pI19|x~9Yn^01IOFw}AEk=K z>&_S+g(jrKCvI{gL(7&4Ic4){J8g^H6@#b`;W7KfX`aI7n$?)z}m!?$gQ%TRN8e^3XYH9VUE>-)iCT+3WQUn+)(c|8HWht!LWgf-`=zz!F4 zgSyOh!{kJb0Vnj%@|Ifvy71b-5ROQU3xe1e9|QUTHaZ)&aQ}pp_lJskqsF+eQ&F3@ zvI?%<$Q$pse!tkaKciIEUbL9dcU;{tCZNxgBB7 zc)p#HL(&XJrt~%l=u3qk&#&RAn=3JZ&TfBD7xm*m_r0Z6~XRYgDD$X~K%{V?Ih~k{*N92NQ5EvdcJ+F_0 zU!R(@C9nnd9+9ot4Crf{*itd{*>kQC^Ko1e*wtd!!J6QPz&nO3C1G`0R zEv)6dBe1)~?uMjUc++aR_nu!qF9h&>kAHnAsRE9Q-# zt@Yr;;MS3Nu3iP7IXMr;@L79bdH!1KV>yIlb+!gRWBxhb&Q^AIWy5hJ*}A~)5!(oB z>?eWw7=J0YHL%CwGr{(3(jVADBNe-{Nq=E0rRU+KFX zF>n6)|1+ag!ny8wKMS9=cpjMZOM&@0j&lNn?!fMc&zwH~_!;+zZO$PHpS2$kbAcAe zYR+c^+bi}`4$1o=8sp0~#u2cldj*VGO>B(Vaez}<*r;sSs^Qg~W3x#U*g_)}le6I~bf_Oa^}%9ZFC85i zYaxq7EdW7W_bD^yFtL$2gsstp{e3wxu&H7*U@PX$`-)@Xn27Vfo(Z3|W@nS$%@!J| zn3qlB+F^nQIO$78lUOsXIp<`*CV^;@ugBaK7ig`o7p+ro|8^y5pc+Rb2+hHr_ZLPSk@87kyYrQ|b&ztkdf%*FOF!aW{17knT*(0_k zuwM9#?Z}2}fFEy<7~+xh@od<760hd`b2bCk#r4takiZykIsJZm zRA9#`C-0dR^VW-erhOut^b}iY*gknS>?6j3@p*rSDepnBK5+Gw>$*AX#YP34uWw_; zCIpth6D$&AEa&S3^R;h^bd7SEu@BQx$tU-+HqUFRr$H#EKn6C><0$U+=Eo{ZS zajtEQM}k}4F+qF0v=hbRIyFt~En*Ep*CfWZ%yP~NtVPWG+7{UI68p!%xUN|TKceH| z-r?i_v~*t-`)XhvVxJRR6WASM>%{I0Y?Ih8U@PX$*XeVmJwIp{O1oI>lE8c%SBbR; zwn{qo+2(C^V4Y%XrRxfe{kOU462o(V&&_?({a9>sV2_Gz7kfIeUa=>|b_e#NSRLlY z`gt%+o}=P$TH8a0%|3gR+U=i3+C5y_Pbe?zUiwn;1+g_^xF`EK?iBl;Sa)C>#d^eW zzjB?AV~5zT!1hQ-YrSKL#xYFecm=F6)J8tW*NKf2J2|ikVyBAn`lc@xv&80z%?rB4 zVwb`4z1c8tY*|P4#~bUEq$wI32&DfS6QW~bL7{+!hk*y0@0R~JF=IXlE|hBYVFb7yynwThuGb9S{D z`T!^N&b}swS^&=VfwhTwZO8i}ef_l*`T$+5CGMAwbwkCxb&3D0v>S2?wtgnIRSavc z_w^aE7sR>)+a!kT6M}7l?Gk%ltPYIVXaBQwkJzJNcyGq*kmuOa_w~|oO}EI=Xq(^- zFk-!kHz(=@FdWx&dQC7&y0|8CEwo6ke--mq$NctvuC$|(SI2z4n6IaNei7rkZh7A+ zT`lOm_L(f!7?`g^bH#9O`SJXj&~?)TJB!D)eM{+ZPP^_yVyqc#QdpnUmkK`~YYt;k z=W849@y6o4o%eJbAMfK8^VXtcen9=XEcAzKrD<;v`%GZBNcRn~_P`i#y47N90^?d~ z>>ja=fo)dKPO)zUcDL9M#Wn}_xY$!-e+{f#>_IVHJ3ilBW34ZH#D;?9IVwid+V}jh z74x=EdDlz(V95Kh*fz1azV8*=4>|3#>J7S`V&0!2VENcZJ+18(NH;dHiDF(CU|+#? z?7tmoymV6ody91Eh|Leo*VEI)W(Ibibgg1MXXs1CO0gSZv(Mh7cKc_eO?~5dHJEvL zjpw-H1X^4B#NLb$v9rY9Bi0buOfg^I7X;QOwnDm9fqhDhG1iwW0=r)9Gt%)KrY{xW z61!K7&x2x}Vt0t$6LS7YY>U|Tz;=rb#w_u@*)VUMYn!l3q{X`9bMsMYIUdu0DX`nb zz9U^{V7H5{6T3gK9qUG4{B0?SXaVkkkolx~^>4>c*=% zd$P&*Mf&>pE^zHa7x$644`fcSmp0`PMxtJUz-x_{*vd}t%Pl!%z&_H39KyYN91pdQ z8T=KfSJ2{~NyWVF634M1LS60a%@Ml4v#u~UB`~i;-h~)H7~}1HlddKW#f7Fy9Y(lX9Y-_kQ|w^krf=2b{4F)tMg^`#iC(z`DhH#CikUCFb=Do&$XRUWfQ| zeXJ+`C>@{EZEkSS@q2Wa82f81)-T)#u(A8PKP>3D-W%gHZ^gXzD(};>e zUo)@{&p8S{C!2sZfa&n(eXWMt~(*HY0{l7-Gaa_6}wi9 zbtZkO@Z)j)x4uLjpV{qrtE9uTqmRMYHm=b*m(kRY5d34UU~eywHr6oLFuq8>p56*U z&QaVc_5OcX2Iq@*qi=&^Sr-SUff%KZoEIkmG{+P%6qKxo+5^% z&V22!iSc}su0gtHv6i4~6PqA*Mqp=&%@mUt(U*#Du^zGBpxY(3 zPV9leekJw?F|2pq&jW^7!^SWg*7$fB`fQDsZm`(Mz{W^7S?sNW%@w-{wqoAQYk&Q} z(;mZCcAk<=nh9&ie{VKyEy1ffugONOY|u9_JVQY@H5;~?@oLTm*>JCkLo??MVs7}H z#YQS{4MBH$HUoZd${NxbYb{I0&zJEydKMg#&4AyxP7~u^nnhk@Z5ON*!#%HR-d2l$ z9o9tmX2X{6BW)AI9**brJs8$1=IjjYF2q~p#%vPTKKlA=v>{rDMj?`GQz}^Rm~O3f z-v^V<>%}L;c)p4GKGGQFjP(-Eb!1_!X31h-o)P1okg?H-R9}1_iS<^+`8I)ZMS7Et z$!85|FBIcfxTSr^zQv1a957?|(*_SIqh@fW8mI~jqkz0xR+r$xqneHZyS~O5%rbp0)3XA@kJU5Yk7}> z$vOJ!kTb=4!!dU&XPwyLK{tW=A~lNP8u6UIr|dY-(V>H^FCgW4O*eCu;~BG@rwn!(doXTa?$I>sj~axZ<;7tYs|kzlFRH ziTw`NSbJc8Os^;I3C!2;ZPM)tjO(@~>lFJ=U|xs#y1zTHm)r5t34eCSGpw%}-RkEr zlyhg$aj(O4UWbeex<=(Z50*L1hI!kgy!(UOKVIY1q~ZixJ2%A6MTpq@#jX&m1vXL4 z_Zj8})+Xj_^y7V**p7Dy zCtBaj&}YvJ(ruTHpP_UIOE*@!(~w2F*<$a9t(Z3)5yyO0HvI43aGKa(F+H0!m#u-n z)4pEp%VN=t62o(Xr5^AdX^Yr$v5p+V7V8fP*h=Y8v)(#UPV51~xgap?3&VL+HvA2+ z>xhgUCN`0g^cFZrpli)$!0)*5S%cyFtD3iU@EiYWHvEmOAM*vVky>l{T(7lun$}F; zN9tylEWkO0oV&8&e;4a(e@(i{%Gr!a<@CK7ulpWn79>f<-fa55cRLPF>88`#e@g{x zonIDMk91GM8sljv-Dv5&FJ}eaVljS4R&lI?UwEa&FX7k>X`IlW#wD(Lu~iZOoIllx~h4-;YD*VDl)FYDeMSA3Axeorp;d4!1h ze#ir24T1R{OPkoLz&fcf(mFAm51w;}81>c{{vL_GR9r81t917S-Ot6I5_=h#>0#Ih z8yNNGTLUNKQ^7qVV_T%d^Mf<5iN+vLj<+}c^VUmWk)D%w1hhKl>uBx!1z7ucc3@tI zTqS0xPse)51QW!X0&9WK*mALsz}AY@#PEFVeewN}4~VS{>!kNKz=YZ-gLKiCj^F>@4ckxqd%)(yk> zRBVyX>xnur>1xvX9z=6stzurk@G_Uq>yVSsw*0Z4SSa0K<4BAhNF^94Bf&EJCPtvsq#yY~1vF0!x?{ADxMXz*wq~rZf zYd`COpyNK0bia}AuhNag@uWMRwn#Hz?Rea`5%c5itNpIhGz+Ow<6xdxkBs~OMF>hXL{TbZ)!*xOHyZ;`kkvAHh|6;zc!|%LI z*PG?c@;qKG*(m8Iq9q;oh>Wp5G9AxFG43rH+mYq@IZNy)THCjTHQn5h^Fv}ciG4e; zUC3nueh+H8SoisNnZDn3ZP0mNSpQVa8*41f>))?!4_e7j~SgRQ8 z4!W%xhu3H$gKh%#MVbj~y4Jv0@0hbk%>2UA#oc_Jcb7DM)r8_~oS+K@G5OQ`S-kkhi%HHE1;yJyx z@tWwIp!2XX}_31?@-GAMUBxF}ZI^Un;x~c~-iXpj#mw>k-r46`1c& zaPPxd9hla)qqV;IKFU{u&i6xpt>bb0>dT?hO@g(Ytk3DQ^;CU%P`aHV=P>D5|Cnw@ z=nHEIOLjG4D(0v4Q|u?wHH2f%74v%HQ-OJ1ut~ZdfejgI z4eNy0FCPoc>yRHxw=J+i(v60-eold*ufOiOSGui1w^#l2I%HDN%~sAOuod&>H4~3x z$HaQ=$2>(^<}h|~VAqR%1=d(Cu&H8m#FhruA$Ggi#=y3fbe9Hpqu4j4+Z5RE#fI|1 iRR6sv-doT7cz0Cjo|LXmx-k$)*GOyse=V>!Z~p~iOFr-b literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/writer_h5py_tests.rs b/crates/clawhdf5-format/tests/writer_h5py_tests.rs index 9707d04..44c42f6 100644 --- a/crates/clawhdf5-format/tests/writer_h5py_tests.rs +++ b/crates/clawhdf5-format/tests/writer_h5py_tests.rs @@ -940,3 +940,48 @@ fn provenance_verify_written_file() { .unwrap(); assert_eq!(result, clawhdf5_format::provenance::VerifyResult::Ok); } + +// ---- hdf5plugin interop: registered third-party compression filters ---- + +/// Write `data` (f64, 1-D, chunked) with `configure` applied, then read it +/// back with h5py + hdf5plugin (libhdf5's registered filter plugins) and +/// return the values it decodes. +#[cfg(any(feature = "lz4", feature = "zstd"))] +fn hdf5plugin_roundtrip( + tag: &str, + data: &[f64], + configure: impl FnOnce(&mut clawhdf5_format::type_builders::DatasetBuilder), +) -> Vec { + let mut fw = FileWriter::new(); + let ds = fw.create_dataset("data"); + ds.with_f64_data(data) + .with_shape(&[data.len() as u64]) + .with_chunks(&[250]); + configure(ds); + let bytes = fw.finish().unwrap(); + let path = std::env::temp_dir().join(format!("clawhdf5_hdf5plugin_{tag}.h5")); + std::fs::write(&path, &bytes).unwrap(); + let script = format!( + "import h5py,hdf5plugin,json; f=h5py.File('{}','r'); print(json.dumps(f['data'][:].tolist()))", + path.display() + ); + let stdout = h5py_read(&path, &script); + serde_json::from_str(&stdout).unwrap() +} + +/// libhdf5's LZ4 plugin must decode what we write (it could not while we +/// wrote a private 4-byte-LE-size framing). +#[cfg(feature = "lz4")] +#[test] +#[ignore = "requires Python h5py + hdf5plugin"] +fn hdf5plugin_reads_our_lz4() { + let data: Vec = (0..1000).map(|i| (i % 37) as f64 * 0.5).collect(); + let got = hdf5plugin_roundtrip("lz4", &data, |ds| { + ds.with_lz4(); + }); + assert_eq!(got, data); + let got = hdf5plugin_roundtrip("lz4_noshuffle", &data, |ds| { + ds.with_lz4().without_shuffle(); + }); + assert_eq!(got, data); +}