From aef8e766ae9ea9206613bd4fe8dbad7bffc998e5 Mon Sep 17 00:00:00 2001
From: osobh <om.sobh@gmail.com>
Date: Fri, 25 Sep 2026 21:03:49 -0500
Subject: [PATCH 1/8] fix(format): write and read the registered HDF5 LZ4
 filter format

Filter 32004 chunks were framed as a 4-byte little-endian size plus one
LZ4 block. That is not the registered HDF5 LZ4 format (H5Zlz4.c: 8-byte
big-endian total size, 4-byte big-endian block size, then per block a
4-byte big-endian compressed length and the block, stored raw when the
length equals the block size), so libhdf5 + hdf5plugin could not read
our LZ4 datasets and we could not read theirs (h5ex_d_lz4.h5:
"lz4: 0 is not a valid match offset").

Write the registered format (cd_values[0] is honoured as the block size,
default 1 GiB like the plugin) and read it, multi-block and raw blocks
included. Chunks in the old framing stay readable: an HDF5 chunk is under
4 GiB, so a registered chunk always starts with four zero bytes and is at
least 12 bytes long, while an old one starts with four zero bytes only
when empty (5 bytes).

Tests: lz4_reads_registered_hdf5_format (chunk of the HDF Group's
h5ex_d_lz4.h5, block size 3), lz4_writes_registered_hdf5_format,
lz4_reads_legacy_clawhdf5_format, and hdf5plugin_reads_our_lz4 (ignored
interop test; failed before with "filter returned failure during read").

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 crates/clawhdf5-format/src/filters.rs         | 223 ++++++++++++++++--
 .../tests/fixtures/filters/README.md          |  10 +
 .../tests/fixtures/filters/h5ex_d_lz4.h5      | Bin 0 -> 23984 bytes
 .../tests/writer_h5py_tests.rs                |  45 ++++
 4 files changed, 254 insertions(+), 24 deletions(-)
 create mode 100644 crates/clawhdf5-format/tests/fixtures/filters/README.md
 create mode 100644 crates/clawhdf5-format/tests/fixtures/filters/h5ex_d_lz4.h5

diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs
index c10cea9..d12302b 100644
--- a/crates/clawhdf5-format/src/filters.rs
+++ b/crates/clawhdf5-format/src/filters.rs
@@ -69,7 +69,7 @@ pub fn compress_chunk(
                 let level = filter.client_data.first().copied().unwrap_or(6);
                 deflate_compress(&result, level)?
             }
-            FILTER_LZ4 => lz4_compress(&result)?,
+            FILTER_LZ4 => lz4_compress(&result, &filter.client_data)?,
             FILTER_ZSTD => {
                 let level = filter.client_data.first().copied().unwrap_or(3);
                 zstd_compress(&result, level)?
@@ -813,12 +813,32 @@ fn deflate_compress(_data: &[u8], _level: u32) -> Result<Vec<u8>, FormatError> {
     Err(FormatError::UnsupportedFilter(FILTER_DEFLATE))
 }
 
-/// Decompress LZ4 data. Format: 4 bytes LE original size + LZ4 block data.
+/// Default LZ4 block size of the registered HDF5 LZ4 filter (`H5Zlz4.c`,
+/// `DEFAULT_BLOCK_SIZE`): 1 GiB, so an HDF5 chunk is normally one block.
+#[cfg(feature = "lz4")]
+const LZ4_DEFAULT_BLOCK_SIZE: usize = 1 << 30;
+
+/// Decompress an LZ4 (filter 32004) chunk.
 ///
-/// The 4-byte "original size" header is part of the attacker-controlled
-/// compressed payload itself, so it is bounded against `expected_bytes` (the
-/// pipeline's declared chunk size) before being used to size the output
-/// allocation — otherwise a crafted 4-byte value can request up to ~4 GiB.
+/// Two framings are read:
+///
+/// * The registered HDF5 LZ4 filter format (`H5Zlz4.c`, what libhdf5 +
+///   hdf5plugin write, and what clawhdf5 writes after 2.7.0): an 8-byte
+///   big-endian total decompressed size, a 4-byte big-endian block size, then
+///   per block a 4-byte big-endian compressed length followed by the block. A
+///   block whose compressed length equals its decompressed length is stored
+///   raw.
+/// * The legacy clawhdf5 framing (up to 2.7.0): a 4-byte little-endian size
+///   followed by one raw LZ4 block. libhdf5 cannot read it.
+///
+/// They are told apart unambiguously: an HDF5 chunk is smaller than 4 GiB, so
+/// the registered format's big-endian `u64` size always starts with four zero
+/// bytes and the whole chunk is at least 12 bytes; a legacy chunk starts with
+/// four zero bytes only when it is empty, and is then 5 bytes long.
+///
+/// Every size read from the payload is bounded against `expected_bytes` (the
+/// pipeline's declared chunk size) before it sizes an allocation, so a crafted
+/// header cannot request gigabytes.
 #[cfg(feature = "lz4")]
 fn lz4_decompress(data: &[u8], expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
     if data.len() < 4 {
@@ -826,38 +846,112 @@ fn lz4_decompress(data: &[u8], expected_bytes: usize) -> Result<Vec<u8>, FormatE
             "lz4: data too short".into(),
         ));
     }
+    let check_size = |size: usize| -> Result<(), FormatError> {
+        if expected_bytes != 0 && size > expected_bytes {
+            return Err(FormatError::DecompressionError(
+                "lz4: declared size exceeds chunk size".into(),
+            ));
+        }
+        if size > MAX_DECOMPRESS_SIZE {
+            return Err(FormatError::DecompressionError(
+                "lz4: declared size exceeds limit".into(),
+            ));
+        }
+        Ok(())
+    };
+    if data.len() >= 12 && data[..4] == [0, 0, 0, 0] {
+        return lz4_decompress_hdf5(data, check_size);
+    }
+    // Legacy clawhdf5 framing: 4-byte LE size + one LZ4 block.
     let orig_size = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize;
-    if expected_bytes != 0 && orig_size > expected_bytes {
-        return Err(FormatError::DecompressionError(
-            "lz4: declared size exceeds chunk size".into(),
-        ));
-    }
-    if orig_size > MAX_DECOMPRESS_SIZE {
-        return Err(FormatError::DecompressionError(
-            "lz4: declared size exceeds limit".into(),
-        ));
-    }
+    check_size(orig_size)?;
     lz4_flex::block::decompress(&data[4..], orig_size)
         .map_err(|e| FormatError::DecompressionError(format!("lz4: {e}")))
 }
 
+/// Decode the registered HDF5 LZ4 framing (see [`lz4_decompress`]).
+#[cfg(feature = "lz4")]
+fn lz4_decompress_hdf5(
+    data: &[u8],
+    check_size: impl Fn(usize) -> Result<(), FormatError>,
+) -> Result<Vec<u8>, FormatError> {
+    let err = |m: &str| FormatError::DecompressionError(format!("lz4: {m}"));
+    let be32 = |b: &[u8]| u32::from_be_bytes([b[0], b[1], b[2], b[3]]) as usize;
+    // The first four bytes are zero (checked by the caller), so the size is
+    // the low 32 bits of the big-endian u64.
+    let orig_size = be32(&data[4..8]);
+    check_size(orig_size)?;
+    let block_size = be32(&data[8..12]).min(orig_size);
+    if block_size == 0 && orig_size != 0 {
+        return Err(err("zero block size"));
+    }
+    let mut out = vec![0u8; orig_size];
+    let mut pos = 12usize;
+    let mut done = 0usize;
+    while done < orig_size {
+        let this_block = block_size.min(orig_size - done);
+        let comp_len = be32(
+            data.get(pos..pos + 4)
+                .ok_or_else(|| err("truncated block header"))?,
+        );
+        pos += 4;
+        let block = data
+            .get(pos..pos.saturating_add(comp_len))
+            .ok_or_else(|| err("truncated block"))?;
+        let dst = &mut out[done..done + this_block];
+        if comp_len == this_block {
+            dst.copy_from_slice(block);
+        } else {
+            let n = lz4_flex::block::decompress_into(block, dst)
+                .map_err(|e| FormatError::DecompressionError(format!("lz4: {e}")))?;
+            if n != this_block {
+                return Err(err("block decompressed to the wrong size"));
+            }
+        }
+        pos += comp_len;
+        done += this_block;
+    }
+    Ok(out)
+}
+
 #[cfg(not(feature = "lz4"))]
 fn lz4_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
     Err(FormatError::UnsupportedFilter(FILTER_LZ4))
 }
 
-/// Compress data with LZ4 block format. Format: 4 bytes LE original size + LZ4 block data.
+/// Compress data in the registered HDF5 LZ4 filter format (see
+/// [`lz4_decompress`]), so libhdf5 with the LZ4 plugin (e.g. hdf5plugin) can
+/// read it. `cd[0]`, when present and non-zero, is the block size in bytes,
+/// as in `H5Zlz4.c`; otherwise the 1 GiB default applies.
 #[cfg(feature = "lz4")]
-fn lz4_compress(data: &[u8]) -> Result<Vec<u8>, FormatError> {
-    let compressed = lz4_flex::block::compress(data);
-    let mut result = Vec::with_capacity(4 + compressed.len());
-    result.extend_from_slice(&(data.len() as u32).to_le_bytes());
-    result.extend_from_slice(&compressed);
+fn lz4_compress(data: &[u8], cd: &[u32]) -> Result<Vec<u8>, FormatError> {
+    let block_size = match cd.first() {
+        Some(&b) if b != 0 => b as usize,
+        _ => LZ4_DEFAULT_BLOCK_SIZE,
+    }
+    .min(data.len());
+    let mut result = Vec::with_capacity(16 + data.len() / 2);
+    result.extend_from_slice(&(data.len() as u64).to_be_bytes());
+    result.extend_from_slice(&(block_size as u32).to_be_bytes());
+    if block_size == 0 {
+        return Ok(result);
+    }
+    for block in data.chunks(block_size) {
+        let compressed = lz4_flex::block::compress(block);
+        if compressed.len() >= block.len() {
+            // Incompressible: stored raw, marked by length == block length.
+            result.extend_from_slice(&(block.len() as u32).to_be_bytes());
+            result.extend_from_slice(block);
+        } else {
+            result.extend_from_slice(&(compressed.len() as u32).to_be_bytes());
+            result.extend_from_slice(&compressed);
+        }
+    }
     Ok(result)
 }
 
 #[cfg(not(feature = "lz4"))]
-fn lz4_compress(_data: &[u8]) -> Result<Vec<u8>, FormatError> {
+fn lz4_compress(_data: &[u8], _cd: &[u32]) -> Result<Vec<u8>, FormatError> {
     Err(FormatError::UnsupportedFilter(FILTER_LZ4))
 }
 
@@ -1480,11 +1574,92 @@ mod tests {
 
     // --- LZ4 tests ---
 
+    /// Chunk (0,0) of `/DS1` in the HDF Group's `h5ex_d_lz4.h5` example,
+    /// written by libhdf5's registered LZ4 plugin with a 3-byte block size
+    /// (so it has many blocks, some stored raw). Byte range from h5py's
+    /// `get_chunk_info`; values are `i*j - j` (i32 LE), as h5py reads them.
+    #[test]
+    #[cfg(feature = "lz4")]
+    fn lz4_reads_registered_hdf5_format() {
+        let file: &[u8] = include_bytes!("../tests/fixtures/filters/h5ex_d_lz4.h5");
+        let chunk = &file[4016..4016 + 312];
+        let pipeline = FilterPipeline {
+            version: 2,
+            filters: vec![FilterDescription {
+                filter_id: FILTER_LZ4,
+                name: None,
+                flags: 1,
+                client_data: vec![3],
+            }],
+        };
+        let out = decompress_chunk(chunk, &pipeline, 4 * 8 * 4, 4).unwrap();
+        let expected: Vec<u8> = (0..4i32)
+            .flat_map(|i| (0..8i32).map(move |j| i * j - j))
+            .flat_map(i32::to_le_bytes)
+            .collect();
+        assert_eq!(out, expected);
+    }
+
+    #[test]
+    #[cfg(feature = "lz4")]
+    fn lz4_writes_registered_hdf5_format() {
+        // Compressible data, one block: 8-byte BE size, 4-byte BE block size,
+        // 4-byte BE compressed length, block.
+        let data = vec![7u8; 1000];
+        let c = lz4_compress(&data, &[]).unwrap();
+        assert_eq!(&c[0..8], &1000u64.to_be_bytes());
+        assert_eq!(&c[8..12], &1000u32.to_be_bytes());
+        let len = u32::from_be_bytes(c[12..16].try_into().unwrap()) as usize;
+        assert_eq!(c.len(), 16 + len);
+        assert!(len < 1000);
+        assert_eq!(lz4_decompress(&c, 1000).unwrap(), data);
+
+        // Several blocks, incompressible ones stored raw (length == block).
+        let data: Vec<u8> = (0..10u8).collect();
+        let c = lz4_compress(&data, &[3]).unwrap();
+        assert_eq!(&c[8..12], &3u32.to_be_bytes());
+        assert_eq!(&c[12..16], &3u32.to_be_bytes());
+        assert_eq!(&c[16..19], &[0, 1, 2]);
+        assert_eq!(c.len(), 12 + 3 * (4 + 3) + (4 + 1));
+        assert_eq!(lz4_decompress(&c, 10).unwrap(), data);
+
+        let c = lz4_compress(&[], &[]).unwrap();
+        assert_eq!(lz4_decompress(&c, 0).unwrap(), Vec::<u8>::new());
+    }
+
+    /// Chunks written by clawhdf5 up to 2.7.0 (4-byte LE size + one LZ4
+    /// block) must stay readable.
+    #[test]
+    #[cfg(feature = "lz4")]
+    fn lz4_reads_legacy_clawhdf5_format() {
+        for data in [vec![], vec![5u8; 300], (0..=255u8).collect::<Vec<u8>>()] {
+            let mut legacy = (data.len() as u32).to_le_bytes().to_vec();
+            legacy.extend_from_slice(&lz4_flex::block::compress(&data));
+            assert_eq!(lz4_decompress(&legacy, data.len()).unwrap(), data);
+        }
+    }
+
+    #[test]
+    #[cfg(feature = "lz4")]
+    fn lz4_registered_format_rejects_hostile_sizes() {
+        // Declared total larger than the chunk.
+        let mut c = 1000u64.to_be_bytes().to_vec();
+        c.extend_from_slice(&1000u32.to_be_bytes());
+        c.extend_from_slice(&[0u8; 8]);
+        assert!(lz4_decompress(&c, 64).is_err());
+        // Truncated block.
+        let mut c = 16u64.to_be_bytes().to_vec();
+        c.extend_from_slice(&16u32.to_be_bytes());
+        c.extend_from_slice(&16u32.to_be_bytes());
+        c.extend_from_slice(&[1u8; 4]);
+        assert!(lz4_decompress(&c, 16).is_err());
+    }
+
     #[test]
     #[cfg(feature = "lz4")]
     fn lz4_compress_decompress_roundtrip() {
         let data: Vec<u8> = (0..256).map(|i| (i % 256) as u8).collect();
-        let compressed = lz4_compress(&data).unwrap();
+        let compressed = lz4_compress(&data, &[]).unwrap();
         let decompressed = lz4_decompress(&compressed, data.len()).unwrap();
         assert_eq!(decompressed, data);
     }
diff --git a/crates/clawhdf5-format/tests/fixtures/filters/README.md b/crates/clawhdf5-format/tests/fixtures/filters/README.md
new file mode 100644
index 0000000..d2095a0
--- /dev/null
+++ b/crates/clawhdf5-format/tests/fixtures/filters/README.md
@@ -0,0 +1,10 @@
+# Filter conformance fixtures
+
+Files written by libhdf5 (and its registered filter plugins), used by the
+filter regression tests in `src/filters.rs` to compare our decoders against
+the values h5py/libhdf5 read from the same bytes. Chunk byte ranges quoted in
+the tests come from h5py's `DatasetID.get_chunk_info`.
+
+| File | Origin | Licence |
+|------|--------|---------|
+| `h5ex_d_lz4.h5` | HDF Group `HDF5Examples/C/H5FLT/tfiles/h5ex_d_lz4.h5` (hdf5 repository) | HDF5 licence (BSD-3-Clause style) |
diff --git a/crates/clawhdf5-format/tests/fixtures/filters/h5ex_d_lz4.h5 b/crates/clawhdf5-format/tests/fixtures/filters/h5ex_d_lz4.h5
new file mode 100644
index 0000000000000000000000000000000000000000..d2790150378c8e9b603a64dc6fcaf43c4f9dea4b
GIT binary patch
literal 23984
zcmeI4f0$NPwZ}IcqzMs6BSTYsGea^3R6-Q4jx^9zNRyJ1kxl$^EYi7%LWVx0p_12X
z@|ICqO-fH>W|LW$yyl(6OIT!+mkJYYEJ89;GE6S{<F4;_f7i^K)49)cpZi??x=j!7
zdiVOQ?>c+$_nfoW+3#x~ZEBoxz_7!HCH&RZr6H-FZ~f&Z{dK)3d%W$uMQ;Ylu996l
zIKLT`@V}Hs$$yaYm*ZxiHGO)rNYC$9vNT~xeply?zuW&;Gte~sjAjgYjz;Zg=|-Jr
zJ{gT?pTM_%J~lb{U)n+GWjufL3(nB7Isf2%gqrd;$d=oMo$rXOw-Yy?!Z~8UR7d~d
zA}88EwX!`(9UpQ9hC3+P<@v_i;>#!27G1b_>B1#bYL_frSZiInwC&XK<CiU4Hm-HS
zq6?N>d}-Ubi<ew5{*r}DF1v94!b`@_zxbl1OD>#u>Cy`?zUY#1txG?+*pl(Dkgd}d
zSqv_g@0We(9xg9-^I*P34^o=a(K9!Pq;&SIw>RR7Mj-xbo-jB6{`x<h0o?BfrG~$`
z-%Zx%|KFwQIxv%RoPQp!+o21j9Ou`K$JGw5&gb`QkDK#~>bRN*$aajZjvHHw+xW`r
zxCy1W^s4H($)&i->adSj%S9<FrM5!{=JUAn{H;B#I<7o_y|1p0E6?AkBdX(S+1>Zr
zJi0orJb$Z>td1+sUw5rKt~`HxUNbPRJRg~+G~u<?apm~dqpRb}^Vj*h>bUa!Z5dM?
zSIh3c*LqEqp9kJAx%=XpUZ3BV<I3~ber$DIdH&YDVPM?wz9S8Mv+KC(IPa%Nj~!PX
zSDwF?@zrtV`CF}l`FOp5H9w$iPhLJ#Ij%f^bthFHw>*Ch6RYFO^SAts1LMl$RQFQX
zr0Tfx{OvfoI<B0zc1m?zdH&{{S{+x*?!MQKDb;c1`P(?PI<7o_>9p#&^88IcePEn!
z!`T^L+Zw9l%JaAOjOw`Z{Pj+&jw{dKsK)BJT6XunHrrDG{`~y&ag^t8)r{)6^89tb
zxjL>qe|wq+#+Cb@sonqHS{+xOzt*=^$Cc-=b7pm1`S@GTtd6T?ci(IM+pFWs^Vif|
z9ao;e_IFgrmFI8WSp(w^&&|Ji-F0?#TzUS+zOy>6Jbx|cRL7O)Z}qv=akcF3d+nK1
z9ao;ey7Q{z%JbLo?&`Sm{4MXRuycGZ8$E6KvxA%|wc9&X-m1O}t0U2R>7^I5Ve17s
zpBMXUV0*>>Q_N6*GBo}VS}0|`gZVM7$Zd0??sAr&KZ-O|hkaRKa3PZVl8*7lc<va(
zCWhzC>sygVK%NhdckT8MA;lhz|2d8SFL2tX1I)8xyT$$-*fU~(+6VLI<9IoaXa8XG
z#!Qj+XDcO)%h>_&S&L&dCms!)<?EzKEbq;U`>3-+AhQ;)5p$x{b(XK&BC!mun73UT
z|I-@(Q_yPO{wTIn><@uGDfat)FmD{I4d9?Kj{I0tBtEtni{sCa9!26j7{eomkH2Q`
z(K;%dbTnINq#~D~MZP9YfYQ))$K{a3vaMp?c4+)hX#Cru)x7;qY@66`1N)8G<NILV
z{PV{nk&h!k_7sVaHRi<m;4DAF=-e>pk%8s=UXe~T9|RKvn*^UZdCxP3Qp$5q6=RuG
zF>k#Z|F1Rv$Dq}`Ju0?U?2*8JCHC+>n711GYy-d}laGVvi?v=W7SEgfh*Kn<KPEUP
z=o%1jx@p;@Mz+vM#f)rH6I&^v^zh^5vY|+`z$)f#i^l&;jsF+WYTkY>wpr{y0((g8
zXZv8@xQ5yQ@aW~^z%_&xAD_&L>&V&hV!WoziANo0&0;J&%sD%-bHwJr8pBfUy1dL5
z>3pz?dF#>mf2#362(9MrCt{n#ejM13#2(lO^Tun-27pI19|x~9Yn^01IOFw}AEk=K
z>&_S+g<RKycx%neCM{qKjZ`ek#&#mnO6mRCu(cSk=DY}|V%|1t{6EzAHy~Q`_U~f%
zi>(jrKCvI{gL(7&4Ic4){J8g^H6@#b`;W7KfX`aI7n$?)z}m!?<dC#9u*<}ji(MYr
z3b89;E9Q;Zj>$gQ%TRN8e^3XYH9VUE>-)iCT+3WQUn+)(c|8HWht!LWgf-`=zz!F4
zgSyOh!{kJb0Vnj%@|Ifvy71b-5ROQU3xe1e9|QUTHaZ)&aQ}pp_lJskqsF+eQ&F3@
zvI?%<$Q$pse!t<p#aN9;!Ld@|$MpNuae?s~v}Cx>kaKciIEUbL9dcU;{tCZNxgBB7
zc)p#HL(&XJrt~%l=u3qk&#&RAn<ib8bXWsiH(Lz%ysGof^ZI=Q`CXg0LJIZ~&UpU&
zG5L6ItRd(c?LD%1e`W^OEXIDBZcbox#Ta7@k9=nESFr!qdM{gPz<uh%9Fp%vCb$Fw
z`clzCTcky>=3JZ&TfBD7xm*m_r0Z6~XRYgDD$X~K%{V?Ih~k{*N92NQ5EvdcJ+F_0
zU!R(@C9nnd9+9ot4Crf{*itd{*>kQC^Ko1e*wtd!!J6QPz&<H<li242>nO3C1G`0R
zEv)6dBe1)~?uM<Hw-)jFu%^Z5Vg`SOe~$Q!;Ji2+w*0fYRIEKP|EyMt#WDIgJ|(s~
z=rAT6s56`N6}Hew#ckQ7Z?csV#^wF|p4h!&>jUc++aR_nu!qF9h&>kAHnAsRE9Q-#
zt@Yr;;MS3Nu3iP7IXMr;@L79bdH!1KV>yIlb+!gRWBxhb&Q^AIWy5hJ*}A~)5!(oB
z>?eWw7=J0YHL%CwGr{(3(jVADBNe-{Nq=E0rRU+KFX<amd#)dpog-lI-pokaA{_=>
zF>n6)|1+ag!ny8wKMS9=cpjMZOM&@0j&lNn?!fMc&zwH~_!;+zZO$PHpS2$kbAcAe
zYR+c^+bi}`4$1o=8sp0~#u2cldj*VGO>B(Vae<vE#=TS1ofepVX^U2RD{RHQ@tiRJ
z^&B?f8vho2#(1q;-hW{$J9*t3`+g2dy@B~TzC&zhVE*~<7TXgT&IKHAl*V`jTI_!+
zj?y@e5sTwLL2Qy3YCP99i=87jH?VnPtzs7iwp8qL*ot{Gul*JFD*+nr#n=?F#oEl+
z;cO*78#+5Shp?u%5Vf+YQ>z}<*r;sSs^Qg~W3x#U*g_)}le6I~bf_Oa^}%9ZFC85i
zYaxq7EdW7W_bD^yFtL$2gsstp{e3wxu&H7*U@PX$`-)@Xn27Vfo(Z3|W@nS$%@!J|
zn3qlB+F^nQIO$78lUOsXIp<`<RtsLu*_usiV+)N`EYGH|e!+U~Ieq<}&&YxG1lJ~u
z#Jw4UcpqjT%t^hm1tBNSF9>*CV^;@ugBaK7ig`o7p<R~EfOX*-G3KzmxCffSU-7wY
z(wEul-v|X;CDtLf8czCB(Mj9CW^@I%PRw<<zq;-gF~6VS-r<b*3TtgZj5&WASSNhu
zye*pn>+ro|8^y5pc+Rb2+hHr_ZLPSk@87kyYrQ|b&ztkdf%*FOF!aW{17knT*(0_k
zuwM9#?Z}2}fFEy<7~+xh@od<760hd`b2bCk#r<?I@^#@=h@`K7J>4takiZykIsJZm
zRA9#`C-0dR^VW-erhOut^b}iY*gknS>?6j3@p*rSDepnBK5+Gw>$*AX#YP34uWw_;
zCIpth6D$&AEa&S3^R;h^bd7<X$>SEu@BQx$tU-+HqUFRr$H#EKn6C><0$U+=Eo{ZS
zajtEQM}k}4F+qF0v=hbRIyFt~En*Ep*CfWZ%yP~NtVPWG+7{UI68p!%xUN|TKceH|
z-r?i_v~*t-`)XhvVxJRR6WASM>%{I0Y?Ih8U@PX$*XeVmJwIp{O1oI>lE8c%SBbR;
zwn{qo+2(C^V4Y%XrRxfe{kOU462o(V&&_?({a9>sV2_Gz7kfIeUa=>|b_e#NSRLlY
z`gt%+o}=P$TH8a0%|3gR+U=i3+C5y_Pbe?zUiwn;1+g_^xF`EK?iBl;Sa)C>#d^eW
zzjB?AV~5zT!1hQ-YrSKL#xYFecm=F6)J8tW*NKf2J2|ikVyBAn`lc@xv&80z%?rB4
zVwb`4z1c8tY*|P4#~bU<A=$9?8oZhlH6|GQgZI{&mQ7-<Vhn$7hO3498k|Q5hWi<u
ztdEQ}1%`W1f0mRE2(81!j#0?Oz|IhxB^K)?B0Js)G0wM~UV}~+n_<?+;C-0~RxxjL
z;J4`c*>Eq$wI32&DfS6QW~bL7{+!hk*y0@0R~JF=IXlE|hBYVFb7yynwThuGb9S{D
z`T!^N&b}swS^&=VfwhTwZO8i}ef_l*`T$+5CGMAwbwkCxb&3D0v>S2?wtgnIRSavc
z_w^aE7sR>)+a!kT6M}7l?Gk%ltPYIVXaBQwkJzJNcyGq*kmuOa_w~|oO}EI=Xq(^-
zFk-!kHz(=@FdWx&dQC7&y0|8CEwo6ke--mq$NctvuC$|(SI2z4n6IaNei7rkZh7A+
zT`lOm_L(f!7?`g^bH#9O`SJXj&~?)TJB!D)eM{+ZPP^_yVyqc#QdpnUmkK`~YYt;k
z=W849@y6o4o%eJbAMfK8^VXtcen9=XEcAzKrD<;v`%GZBNcRn~_P`i#y47N90^?d~
z>>ja=fo)dKPO)zUcDL9M#Wn}_xY$!-e+{f#>_IVHJ3ilBW34ZH#D;?9IVwid+V}jh
z74x=EdDlz(V95Kh*fz1azV8*=4>|3#>J7S`V&0!2VENcZJ+18(NH;dHiDF(CU|+#?
z?7tmoymV6ody91Eh|Leo*VEI)W(Ibibgg1MXXs1CO0gSZv(Mh7cKc_eO?~5dHJEvL
zjpw-H1X^4B#NLb$v9rY9Bi0buOfg^I7X;QOwnDm9fqhDhG1iwW0=r)9Gt%)KrY{xW
z61!K7&x2x}Vt0t$6LS7YY>U|Tz;=rb#w_u@*)VUMYn!l3q{X`9bMsMYIUdu0DX`nb
zz9U^{V7H5{6T3gK9<i-rI|B2*`Z#_U*fY|-B;9Z@jLFQQv{s|S+P{-v#OlRHiyar(
zsbWoH=LEJuYzZunZ8ppsTXwJm@y24fF35R2ZGWxxwj2U)-}g(tk3=V4jMZn8YHZnW
zKH;P<6%DjSYKApuOEzq^;k7Sv;O~)+62qJzq%lOkQ*4pgvcQN;fM)^djt*=RVgL7T
z=LLrK0|M4B74x=S{JI=AV4s||nQ2*9*)i8MQx>qUG4{B0?SXaVkkkolx~^>4>c*=%
zd$P&*Mf&>pE^zHa7x$644`fcSmp0`PMxtJUz-x_{*vd}t%Pl!%z&_H39KyYN91pdQ
z8T=KfSJ2{~NyWVF634M1LS60a%@Ml4v#u~UB`~i;-h~)H7~}1HlddK<A?O;!SRa_K
zC9pOz>W#f7Fy9Y(lX9Y-_kQ|w^krf=2b{4F)<NnmC(lzJCl%a3G1e;G<w55)0rv_j
z=8YQDE|d1!p!IzQe?I+oVE&x_GwIp`>tMg^`#iC(z`DhH#CikUCFb=Do&$XRUWfQ|
zeXJ+`C>@{EZEkSS@q2Wa82f81)-T)#u(A8PKP>3D-W%gHZ^gXzD(};<miNWb*8>>e
zUo)@{&p8S{C!2sZfa&n(eXW<S25XGxn{*9g&0;M<$92>Mt~(*HY0{l7-Gaa_6}wi9
zbtZkO@Z)j)x4uLjpV{qrtE9uTqmRMYHm=b*m(kRY5d34UU~eywHr6oLFuq8>p56*U
z&QaVc_5<mdP^?3&Q;g@EShrY@SZ~O=OKhze_mSvJh1U#^i0urz7o;1(5$X4qcyHb1
zacq3Qk#2X;4P}h&_eysxm~>OcX2Iq@*qi=&^Sr-SUff%KZoEIkmG{+P%6qKxo+5^%
z&V22!iSc}su0gtHv6i4~6PqA*Mqp=&%@<o5*fnA|iG4M&7BSWn_I_nxjJG*mE8X3J
zZIbRW*ot{$?PA(<rNvtA<G)nem12DEqc0V=h<!_}J?J{bI>mUt(U*#Du^zGBpxY(3
zPV9leekJw?F|2pq&jW^7!^SWg*7$fB`fQDsZm`(Mz{W^7S?sNW%@w-{wqoAQYk&Q}
z(;mZCcAk<=nh9&ie{VKyEy1ffugONOY|u9_JVQY@H5;~?@oLTm*>JCkLo??MVs7}H
z#YQS{4MBH$HUoZd${NxbYb{I0&zJEydKMg#&4AyxP7~u^nnhk@Z5ON*!#%HR-d2l$
z9o9tmX2X{6BW)AI9**brJs8$1=IjjYF2q~p#%vPTKKlA=v>{rDMj?`GQz}^Rm~O3f
z-v^V<>%}L;c)p4GKGGQFjP(-Eb!1_!X31h-o)P1okg?H-R9}1_iS<^+`8I)ZMS7Et
z$!85|FBIc<JC^qoA+PU;d`G&5z-CM5b<v8zI>fxTSr^zQv1a957?|(*_<sJHz_?d$
zbK`p+zYVNKIlZ>SIqh@fW8mI~jqkz0xR+r$xqneHZyS~O5%rbp0)3XA@kJU5Yk7}>
z$vOJ!kTb=4!!dU&XPwyLK{tW=A~lNP8u6UIr|<O?_sJP=zYme_Wa)74aNYaFSXWp-
z|0%F~F@C>dY-(V>H^FCgW4O*eCu;~BG@rwn!(doXTa?$I>sj~axZ<;7tYs|kzlFRH
ziTw`NSbJc8Os^;I3C!2;ZPM)tjO(@~>lFJ=U|xs#y1zTHm)r5t34eCSGpw%}-RkEr
zlyhg$aj(O4UWbeex<=(Z50*L1hI!kgy!(UOKVIY1q~ZixJ2%A6MTpq@#jX&m1vXL4
z_Zj8})+Xj_^y<KFFX`SB80!e@;B`9Q&4F=EH?~3Sk-+?T*H-B6)bTcoJr;Dk#fA+r
zXT`jAh~FyhcZ2o;v8`fH2gY^X`ua-bHQpWAX6bfF_k3XWh&RE}(oIH-zEsQ<W9?wt
z{Q^5oY`k=hfxSzNHG}Cs5?D=al61Vj=}W~tvE^bP54tsCcf(f9+gR}_(#{In_lYeP
z<MT3o_Fa(JH^f?kj`21(tHgM2NY^FZ{bE~!Zl~C4G47Momx}wueksQ5O>7V**p7Dy
zCtBaj&}YvJ(ruTHpP_UIOE*@!(~w2F*<$a9t(Z3)5yyO0HvI43aGKa(F+H0!m#u-n
z)4pEp%VN=t62o(Xr5^AdX^Yr$v5p+V7V8fP*h=Y8v)(#UPV51~xgap?3&VL+HvA2+
z>xhgUCN`0g^cFZrpli)$!0)*5S%cyFtD3iU@EiYWHvEmOAM*vVky>l{T(7lun$}F;
zN9tylEWkO0oV&8&e;4a(e@(i{%Gr!a<@CK7ulpWn79>f<-fa55cRLPF>88`#e@g{x
zonIDMk91GM8sljv-Dv5&FJ}eaVljS4R&l<yiC+h6dA}I6tj)~Hy2{vZ1M}}?4nj_2
z?SZXU&MxJ|Jop%W|HW&Kda&HDVk~WuSmRmFxR1nrAUlxPOI%NRoK*O}jn^7@rtqAd
z>I?UwEa&FX7k>X`IlW#wD(Lu~iZOoIllx~h4-;YD*VDl)FYDeMSA3Axeorp;d4!1h
ze#ir24T1R{OPkoLz&fcf(mFAm51w;}81>c{{vL_GR9r81t917S-Ot6I5_=h#>0#Ih
z8yNNGTLUNKQ^7qVV_T%d^Mf<5iN+vLj<+}c^VUmWk)D%w1hhKl>uBx!1z7ucc3@tI
zTqS0xPse)51QW!X0&9WK*mALsz}AY@#PEFVeewN}4~VS{><iL;R}A+Y*ENW-4zu@5
z1N*obk8AJm3yd{{_46-?shGFb(tZ=x{^2>!kNKz=YZ-gLKiCj^F>@4ckxqd%)(yk>
zRBVyX>xnur>1xvX9z=6stzurk@G_Uq>yVSsw*0Z4SSa0<VxI{)z3w?ytTC|jrSm%E
z#=veD+W=cJZ_VNtz*>K<4BAhNF^94Bf&EJCPtvsq#yY~1vF0!x?{ADxMXz*wq~rZf
zYd`COpyNK0bia}AuhNag@uWMRwn#Hz?Rea`5%c5it<a5=u1UIiA?FIQn_w&Etxo(%
zX(t8knPLmXJ{;JMVqX($2yBj6n;7qJ^x5yC#ny@O{wB6VjQ_`aHvUhBoVSUsmu_od
z&x(y03a6f<@!ncpVn32@8=Ul|;zhAh(xKKVq?<`=|2-UR#k`r<{`&trO3s^{?+{y@
zLk9f5{T4Co^`NiO9In=6IcJ;okT$U!#MTCOznGgN0y|YsJnJDD?^E=p0`m*!9oY={
z{Wg)Y(PC2>NpIhGz+Ow<6xdxkBs~OMF>hXL{TbZ)!*xOHyZ;`kkvAHh|6;zc!|%LI
z*PG?c@;qKG*(m8Iq9q;oh>Wp5G9AxFG43rH+mYq@IZNy)THCjTHQn5h^Fv}ciG4e;
zUC3nueh+H8SoisNnZDn3ZP0mNSpQVa8*41f>))?!4_e<x;&)<p%#*=#j>7j~SgRQ8
z4!W%xhu3H$gKh%#MVbj~y4Jv0@0hbk<NssGIZV1^q-zX1-$%MqEY?e2FZy?cxE6d2
z{+*oHJ(mUMdpfMQcx;0ywc9`Iwf(+V{kc`gycf<KS3E4XTkJqEsLc5HGSitw`@PcL
zBc0bpyCD#(r`~@1DqRzpaxM^Cr{n!R=$=u|1Eu46Bi&o1TO{_8kkj|AytW%2n2(`J
zx<!F;zs9DB-=pNd8||OB79I1$(()Ro&wl44wn^-XkazG9q$$!7u*TbAq`O(Vb<%AP
zjJ1gghD$dVjJZ<LNL!>%>2UA#oc_Jcb7DM)r8_~oS+K@G5OQ`S-kkhi%HHE1;yJyx
z@tWwIp!2<n8<DeO-l(y>XX}_31?@-GAMUBxF}ZI^Un;x~c~-iXpj#mw>k-r46`1c&
zaPPxd9hla)qqV;IKFU{u&i6xpt>bb0>dT?hO@g(Ytk3DQ^;CU%P`aHV=P>D5|Cnw@
z=nHEIOLjG4D(0<T+Bd+KwC@vREn|%JBz>v4Q|u?wHH2f%74v%HQ-OJ1ut~ZdfejgI
z4eNy0FCPoc>yRHxw=J+i(v60-eold*ufOiOSGui1w^#l2I%HDN%~sAOuod&>H4~3x
z$HaQ=$2>(^<}h|~VAqR%1=d(Cu&H8m#FhruA$Ggi#=y3fbe9Hpqu4j4+Z5RE#fI|1
iRR6sv-doT7cz0Cjo|LXmx-k$)*GOyse=V>!Z~p~iOFr-b

literal 0
HcmV?d00001

diff --git a/crates/clawhdf5-format/tests/writer_h5py_tests.rs b/crates/clawhdf5-format/tests/writer_h5py_tests.rs
index 9707d04..44c42f6 100644
--- a/crates/clawhdf5-format/tests/writer_h5py_tests.rs
+++ b/crates/clawhdf5-format/tests/writer_h5py_tests.rs
@@ -940,3 +940,48 @@ fn provenance_verify_written_file() {
             .unwrap();
     assert_eq!(result, clawhdf5_format::provenance::VerifyResult::Ok);
 }
+
+// ---- hdf5plugin interop: registered third-party compression filters ----
+
+/// Write `data` (f64, 1-D, chunked) with `configure` applied, then read it
+/// back with h5py + hdf5plugin (libhdf5's registered filter plugins) and
+/// return the values it decodes.
+#[cfg(any(feature = "lz4", feature = "zstd"))]
+fn hdf5plugin_roundtrip(
+    tag: &str,
+    data: &[f64],
+    configure: impl FnOnce(&mut clawhdf5_format::type_builders::DatasetBuilder),
+) -> Vec<f64> {
+    let mut fw = FileWriter::new();
+    let ds = fw.create_dataset("data");
+    ds.with_f64_data(data)
+        .with_shape(&[data.len() as u64])
+        .with_chunks(&[250]);
+    configure(ds);
+    let bytes = fw.finish().unwrap();
+    let path = std::env::temp_dir().join(format!("clawhdf5_hdf5plugin_{tag}.h5"));
+    std::fs::write(&path, &bytes).unwrap();
+    let script = format!(
+        "import h5py,hdf5plugin,json; f=h5py.File('{}','r'); print(json.dumps(f['data'][:].tolist()))",
+        path.display()
+    );
+    let stdout = h5py_read(&path, &script);
+    serde_json::from_str(&stdout).unwrap()
+}
+
+/// libhdf5's LZ4 plugin must decode what we write (it could not while we
+/// wrote a private 4-byte-LE-size framing).
+#[cfg(feature = "lz4")]
+#[test]
+#[ignore = "requires Python h5py + hdf5plugin"]
+fn hdf5plugin_reads_our_lz4() {
+    let data: Vec<f64> = (0..1000).map(|i| (i % 37) as f64 * 0.5).collect();
+    let got = hdf5plugin_roundtrip("lz4", &data, |ds| {
+        ds.with_lz4();
+    });
+    assert_eq!(got, data);
+    let got = hdf5plugin_roundtrip("lz4_noshuffle", &data, |ds| {
+        ds.with_lz4().without_shuffle();
+    });
+    assert_eq!(got, data);
+}

From 183d96ee265008b686416fb6ef0f1f78381098de Mon Sep 17 00:00:00 2001
From: osobh <om.sobh@gmail.com>
Date: Fri, 25 Sep 2026 21:04:29 -0500
Subject: [PATCH 2/8] fix(format): record the content size in zstd frames

Filter 32015 chunks were written with the streaming encoder
(zstd::encode_all), whose frames carry no content size. The registered
HDF5 Zstandard filter (H5Zzstd.c, libhdf5 + hdf5plugin) sizes its output
from ZSTD_getFrameContentSize and fails on such frames, so h5py could not
read our zstd datasets ("filter returned failure during read"). Compress
with the one-shot API, which records the size.

Tests: zstd_frames_record_content_size (content size was None before),
hdf5plugin_reads_our_zstd (ignored interop test; failed before).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 crates/clawhdf5-format/src/filters.rs         | 25 +++++++++++++++++--
 .../tests/writer_h5py_tests.rs                | 13 ++++++++++
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs
index d12302b..a28d4be 100644
--- a/crates/clawhdf5-format/src/filters.rs
+++ b/crates/clawhdf5-format/src/filters.rs
@@ -988,10 +988,14 @@ fn zstd_decompress(_data: &[u8], _expected_bytes: usize) -> Result<Vec<u8>, Form
     Err(FormatError::UnsupportedFilter(FILTER_ZSTD))
 }
 
-/// Compress data with zstd.
+/// Compress data with zstd as one frame whose header records the content
+/// size. The registered HDF5 Zstandard filter (`H5Zzstd.c`, used by
+/// libhdf5 + hdf5plugin) sizes its output buffer from
+/// `ZSTD_getFrameContentSize` and fails on a frame without it, which is what
+/// the streaming encoder (`zstd::encode_all`) produced.
 #[cfg(feature = "zstd")]
 fn zstd_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
-    zstd::encode_all(data, level as i32)
+    zstd::bulk::compress(data, level as i32)
         .map_err(|e| FormatError::CompressionError(format!("zstd: {e}")))
 }
 
@@ -1710,6 +1714,23 @@ mod tests {
 
     // --- Zstd tests ---
 
+    /// libhdf5's zstd plugin needs the frame content size to size its
+    /// output; frames without it fail to decode there.
+    #[test]
+    #[cfg(feature = "zstd")]
+    fn zstd_frames_record_content_size() {
+        for n in [0usize, 1, 200, 100_000] {
+            let data: Vec<u8> = (0..n).map(|i| (i % 7) as u8).collect();
+            let c = zstd_compress(&data, 3).unwrap();
+            assert_eq!(
+                zstd::zstd_safe::get_frame_content_size(&c).unwrap(),
+                Some(n as u64),
+                "{n} bytes"
+            );
+            assert_eq!(zstd_decompress(&c, n).unwrap(), data);
+        }
+    }
+
     #[test]
     #[cfg(feature = "zstd")]
     fn zstd_compress_decompress_roundtrip() {
diff --git a/crates/clawhdf5-format/tests/writer_h5py_tests.rs b/crates/clawhdf5-format/tests/writer_h5py_tests.rs
index 44c42f6..cfbaaa8 100644
--- a/crates/clawhdf5-format/tests/writer_h5py_tests.rs
+++ b/crates/clawhdf5-format/tests/writer_h5py_tests.rs
@@ -985,3 +985,16 @@ fn hdf5plugin_reads_our_lz4() {
     });
     assert_eq!(got, data);
 }
+
+/// libhdf5's Zstandard plugin must decode what we write (it could not while
+/// our frames lacked the content size).
+#[cfg(feature = "zstd")]
+#[test]
+#[ignore = "requires Python h5py + hdf5plugin"]
+fn hdf5plugin_reads_our_zstd() {
+    let data: Vec<f64> = (0..1000).map(|i| (i % 37) as f64 * 0.5).collect();
+    let got = hdf5plugin_roundtrip("zstd", &data, |ds| {
+        ds.with_zstd(3);
+    });
+    assert_eq!(got, data);
+}

From 06dda26d856d1af3b915196c48f3cbf7333fa1bf Mon Sep 17 00:00:00 2001
From: osobh <om.sobh@gmail.com>
Date: Fri, 25 Sep 2026 21:06:07 -0500
Subject: [PATCH 3/8] fix(format): stop writing pcodec under Granular
 BitRound's filter ID

Pcodec chunks were written as filter 32023, which the HDF Group registry
assigns to Granular BitRound (GBR). Pcodec has no registered ID (checked
2026-09-25 against hdf5_plugins/docs/RegisteredFilterPlugins.md, which
ends at 32033 with no pcodec entry). GBR's decode is a pass-through, so
libhdf5 with that plugin loaded would have returned the compressed bytes
as the dataset's values.

Write pcodec as 480, from the registry's testing/private range (256-511),
named "pcodec (clawhdf5 private)", and document it as non-interoperable:
only clawhdf5 with the `pcodec` feature reads it. Chunks under 32023 are
still read as pcodec when the filter is named exactly "pcodec" (what
clawhdf5 <= 2.7.0 wrote); any other 32023 is UnsupportedFilter.

Test: pcodec_uses_private_id_and_reads_legacy_32023.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 README.md                                     |  2 +-
 crates/clawhdf5-format/src/chunked_write.rs   |  9 +--
 crates/clawhdf5-format/src/filter_pipeline.rs | 19 +++++-
 crates/clawhdf5-format/src/filters.rs         | 61 ++++++++++++++++++-
 crates/clawhdf5-format/src/type_builders.rs   |  7 ++-
 5 files changed, 88 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index d21462c..5d4c707 100644
--- a/README.md
+++ b/README.md
@@ -696,7 +696,7 @@ stores keep their setting. Opt out with `float16 = false` or
 | `fast-checksum` | no | crc32fast-accelerated checksums |
 | `lz4` | no | LZ4 block compression filter (id 32004) |
 | `zstd` | no | Zstandard compression filter (id 32015) |
-| `pcodec` | no | Pcodec lossless numerical codec (id 32023, via `pco` crate) |
+| `pcodec` | no | Pcodec lossless numerical codec (via `pco` crate). Private, unregistered filter id 480: **only clawhdf5 can read these datasets** (h5py/libhdf5 cannot). Files from clawhdf5 <= 2.7.0 used id 32023, which is registered to Granular BitRound; they still read. |
 | `system-zlib` | no | System zlib backend for deflate (C) |
 | `blake3_hash` | no | BLAKE3 content hashing for provenance |
 | `szip` | no | SZIP filter (id 4) via libaec (C, through the internal `libaec-sys` crate) |
diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs
index 6816546..aa8bb8f 100644
--- a/crates/clawhdf5-format/src/chunked_write.rs
+++ b/crates/clawhdf5-format/src/chunked_write.rs
@@ -11,8 +11,8 @@ use crate::chunk_cache::{CACHE_LINE_SIZE, align_to_cache_line};
 use crate::ea_writer;
 use crate::error::FormatError;
 use crate::filter_pipeline::{
-    FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_SHUFFLE, FILTER_ZSTD,
-    FilterDescription, FilterPipeline,
+    FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME,
+    FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline,
 };
 use crate::filters::compress_chunk;
 /// Round a file offset up to the next cache-line boundary.
@@ -44,7 +44,8 @@ pub struct ChunkOptions {
     pub lz4: bool,
     /// Zstandard compression level (1-22), None = no zstd. Filter ID 32015.
     pub zstd_level: Option<u32>,
-    /// Pcodec lossless numerical compression. Filter ID 32023.
+    /// Pcodec lossless numerical compression. Private, unregistered filter
+    /// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it.
     pub pcodec: bool,
 }
 
@@ -115,7 +116,7 @@ impl ChunkOptions {
         if self.pcodec {
             filters.push(FilterDescription {
                 filter_id: FILTER_PCODEC,
-                name: Some("pcodec".into()),
+                name: Some(FILTER_PCODEC_NAME.into()),
                 flags: 0,
                 client_data: vec![element_size],
             });
diff --git a/crates/clawhdf5-format/src/filter_pipeline.rs b/crates/clawhdf5-format/src/filter_pipeline.rs
index f64219d..74bc727 100644
--- a/crates/clawhdf5-format/src/filter_pipeline.rs
+++ b/crates/clawhdf5-format/src/filter_pipeline.rs
@@ -19,8 +19,23 @@ pub const FILTER_SCALEOFFSET: u16 = 6;
 pub const FILTER_LZ4: u16 = 32004;
 /// Zstandard compression.
 pub const FILTER_ZSTD: u16 = 32015;
-/// Pcodec lossless numerical codec (clawhdf5 internal; not yet HDF5-registered).
-pub const FILTER_PCODEC: u16 = 32023;
+/// Pcodec lossless numerical codec — a **private, unregistered** clawhdf5
+/// filter. Pcodec has no ID in the HDF Group's filter registry (checked
+/// 2026-09-25, `hdf5_plugins/docs/RegisteredFilterPlugins.md`), so it uses an
+/// ID from the registry's testing/private range (256–511). No libhdf5 plugin
+/// decodes it: h5py/libhdf5 report the filter as unavailable. Only clawhdf5
+/// (with the `pcodec` feature) reads these datasets.
+pub const FILTER_PCODEC: u16 = 480;
+/// Filter name written with [`FILTER_PCODEC`].
+pub const FILTER_PCODEC_NAME: &str = "pcodec (clawhdf5 private)";
+/// The ID clawhdf5 up to 2.7.0 wrote pcodec under. It is registered to
+/// Granular BitRound (GBR), whose decode is a pass-through, so libhdf5 with
+/// that plugin would have returned the compressed bytes as data. Read as
+/// pcodec only when the filter is named exactly [`FILTER_PCODEC_LEGACY_NAME`],
+/// the name those versions wrote; never written.
+pub const FILTER_PCODEC_LEGACY: u16 = 32023;
+/// The filter name clawhdf5 up to 2.7.0 wrote with [`FILTER_PCODEC_LEGACY`].
+pub const FILTER_PCODEC_LEGACY_NAME: &str = "pcodec";
 
 /// Description of a single filter in a pipeline.
 #[derive(Debug, Clone, PartialEq)]
diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs
index a28d4be..c0bf701 100644
--- a/crates/clawhdf5-format/src/filters.rs
+++ b/crates/clawhdf5-format/src/filters.rs
@@ -8,8 +8,9 @@ use alloc::{boxed::Box, vec, vec::Vec};
 
 use crate::error::FormatError;
 use crate::filter_pipeline::{
-    FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC, FILTER_SCALEOFFSET,
-    FILTER_SHUFFLE, FILTER_SZIP, FILTER_ZSTD, FilterPipeline,
+    FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC,
+    FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME, FILTER_SCALEOFFSET, FILTER_SHUFFLE,
+    FILTER_SZIP, FILTER_ZSTD, FilterPipeline,
 };
 
 /// Absolute ceiling on a single decompressed chunk's output size, used only
@@ -39,6 +40,11 @@ pub fn decompress_chunk(
             FILTER_ZSTD => zstd_decompress(&data, chunk_size)?,
             FILTER_FLETCHER32 => fletcher32_verify(&data)?,
             FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, chunk_size)?,
+            // Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered
+            // to Granular BitRound; recognised by the name those versions wrote.
+            FILTER_PCODEC_LEGACY if filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) => {
+                pcodec_decompress(&data, element_size as usize, chunk_size)?
+            }
             // `chunk_size` is the expected decompressed size; pass it so these
             // decoders can reject an element count that would over-allocate.
             FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, chunk_size)?,
@@ -2192,6 +2198,57 @@ mod tests {
         assert!(pcodec_decompress(&compressed, 4, 16).is_err());
     }
 
+    /// Pcodec is written under the private ID 480, not 32023 (registered to
+    /// Granular BitRound, whose pass-through decode would hand libhdf5 users
+    /// the compressed bytes as data). Chunks under 32023 are read as pcodec
+    /// only with the name clawhdf5 <= 2.7.0 wrote.
+    #[test]
+    #[cfg(feature = "pcodec")]
+    fn pcodec_uses_private_id_and_reads_legacy_32023() {
+        use crate::chunked_write::ChunkOptions;
+        let opts = ChunkOptions {
+            pcodec: true,
+            ..Default::default()
+        };
+        let pl = opts.build_pipeline(8).unwrap();
+        let f = pl.filters.iter().find(|f| f.filter_id == 480).unwrap();
+        assert_eq!(
+            f.name.as_deref(),
+            Some(crate::filter_pipeline::FILTER_PCODEC_NAME)
+        );
+        assert!(pl.filters.iter().all(|f| f.filter_id != 32023));
+
+        let data: Vec<f64> = (0..100).map(|i| i as f64 * 0.25).collect();
+        let raw: Vec<u8> = data.iter().flat_map(|x| x.to_le_bytes()).collect();
+        let compressed = pcodec_compress(&raw, 8).unwrap();
+        let pipeline = |id: u16, name: Option<&str>| FilterPipeline {
+            version: 2,
+            filters: vec![FilterDescription {
+                filter_id: id,
+                name: name.map(Into::into),
+                flags: 0,
+                client_data: vec![8],
+            }],
+        };
+        let legacy = pipeline(32023, Some("pcodec"));
+        assert_eq!(
+            decompress_chunk(&compressed, &legacy, raw.len(), 8).unwrap(),
+            raw
+        );
+        let current = pipeline(480, None);
+        assert_eq!(
+            decompress_chunk(&compressed, &current, raw.len(), 8).unwrap(),
+            raw
+        );
+        // A real Granular BitRound filter is not pcodec.
+        for name in [None, Some("Granular BitRound")] {
+            assert!(matches!(
+                decompress_chunk(&compressed, &pipeline(32023, name), raw.len(), 8),
+                Err(FormatError::UnsupportedFilter(32023))
+            ));
+        }
+    }
+
     #[test]
     #[cfg(feature = "lz4")]
     fn decompress_chunk_rejects_hostile_lz4_size_via_public_entrypoint() {
diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs
index e698899..7b45d70 100644
--- a/crates/clawhdf5-format/src/type_builders.rs
+++ b/crates/clawhdf5-format/src/type_builders.rs
@@ -671,7 +671,12 @@ impl DatasetBuilder {
         self
     }
 
-    /// Enable Pcodec lossless numerical compression (clawhdf5 filter ID 32023).
+    /// Enable Pcodec lossless numerical compression (private clawhdf5 filter
+    /// ID 480).
+    ///
+    /// **Not interoperable:** pcodec has no registered HDF5 filter ID and no
+    /// libhdf5 plugin, so h5py and other HDF5 readers cannot read the
+    /// dataset — only clawhdf5 built with the `pcodec` feature can.
     ///
     /// Pcodec achieves 30–94% better compression ratio than Zstd for f32/f64
     /// columns at 1–5 GiB/s decompression speed (arXiv:2502.06112). Requires

From 5935e13866e30b9a8c362a7733f0fc90f916711a Mon Sep 17 00:00:00 2001
From: osobh <om.sobh@gmail.com>
Date: Fri, 25 Sep 2026 21:11:24 -0500
Subject: [PATCH 4/8] fix(format): decode SZIP chunks the way libhdf5 does

SZIP-filtered datasets from libhdf5 came back as garbage or zeros with no
error (ref_szip.h5, h5repack_szip.h5, noencoder.h5, le_data/be_data
Szip_float_data_*), and 64-bit ones failed with "invalid bits per
sample" (h5wasm compressed.h5). The decoder called aec_buffer_decode
directly, but libhdf5 goes through szlib's SZ_BufftoBuffDecompress
(H5Zszip.c), which libaec implements with reshaping (sz_compat.c).
Differences, all fixed:

- H5Zszip.c prefixes the stream with the 4-byte LE uncompressed size; it
  was fed to libaec as data.
- 32- and 64-bit samples are coded as byte planes of 8-bit samples and
  must be de-interleaved.
- The reference sample interval is ceil(pixels_per_scanline /
  pixels_per_block), not a fixed 128.
- Scanlines that are not a whole number of blocks are padded and must be
  unpadded.
- Byte order comes from the MSB option bit; LE data was decoded as MSB.

Test: szip_decodes_libhdf5_chunks_exactly compares chunks from HDF Group
test files (noencoder.h5, le_data.h5) and an h5py-written file (64-bit,
16-bit, padded scanlines, NN and EC) byte for byte with h5py's values;
it failed before on the first case.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 crates/clawhdf5-format/src/filters_szip.rs    | 212 ++++++++++++++----
 .../tests/fixtures/filters/README.md          |   3 +
 .../tests/fixtures/filters/le_data.h5         | Bin 0 -> 72368 bytes
 .../tests/fixtures/filters/noencoder.h5       | Bin 0 -> 8088 bytes
 .../tests/fixtures/filters/szip_h5py.h5       | Bin 0 -> 11320 bytes
 5 files changed, 177 insertions(+), 38 deletions(-)
 create mode 100644 crates/clawhdf5-format/tests/fixtures/filters/le_data.h5
 create mode 100644 crates/clawhdf5-format/tests/fixtures/filters/noencoder.h5
 create mode 100644 crates/clawhdf5-format/tests/fixtures/filters/szip_h5py.h5

diff --git a/crates/clawhdf5-format/src/filters_szip.rs b/crates/clawhdf5-format/src/filters_szip.rs
index b05a5e4..4e7fd71 100644
--- a/crates/clawhdf5-format/src/filters_szip.rs
+++ b/crates/clawhdf5-format/src/filters_szip.rs
@@ -1,19 +1,39 @@
 //! SZIP (libaec Adaptive Entropy Coding) decompression.
 //!
 //! Gated by the `szip` feature which links against the system libaec library.
+//!
+//! libhdf5's SZIP filter (`H5Zszip.c`) prefixes each chunk with its
+//! uncompressed size and hands the rest to szlib's `SZ_BufftoBuffDecompress`.
+//! libaec implements that call (`sz_compat.c`) on top of `aec_buffer_decode`
+//! with some reshaping — 32/64-bit samples are coded as byte planes of 8-bit
+//! samples, and scanlines that are not a whole number of blocks are padded —
+//! which [`szip_decompress`] reproduces so its output matches libhdf5's.
 
 #[cfg(not(feature = "std"))]
 use alloc::vec::Vec;
 
 use crate::error::FormatError;
 
-/// Decompress SZIP-compressed data using libaec.
+/// `SZ_MSB_OPTION_MASK`: samples are big-endian.
+#[cfg(feature = "szip")]
+const SZ_MSB_OPTION_MASK: u32 = 16;
+/// `SZ_NN_OPTION_MASK`: nearest-neighbour preprocessing.
+#[cfg(feature = "szip")]
+const SZ_NN_OPTION_MASK: u32 = 32;
+
+/// Decompress one SZIP-filtered chunk.
 ///
-/// `cd` is the HDF5 SZIP filter client data (matches `H5Z_SZIP_PARM_*` indices):
-///   cd[0] = options mask  (`H5_SZIP_NN_OPTION_MASK = 0x20` enables NN preprocessing)
-///   cd[1] = pixels per block (H5Z_SZIP_PARM_PPB; 8, 10, 16, or 32)
-///   cd[2] = bits per sample  (H5Z_SZIP_PARM_BPP; element bit width)
-///   cd[3] = pixels per scan line (H5Z_SZIP_PARM_PPS; informational only)
+/// `cd` is the HDF5 SZIP filter client data (`H5Z_SZIP_PARM_*` indices):
+///   cd[0] = options mask (`SZ_*_OPTION_MASK`: 16 = MSB byte order,
+///           32 = nearest-neighbour preprocessing; K13/EC/LSB/RAW bits carry
+///           no decoding information for libaec)
+///   cd[1] = pixels per block
+///   cd[2] = bits per pixel (sample precision, rounded up to 32 or 64 above
+///           24 by libhdf5)
+///   cd[3] = pixels per scanline
+///
+/// The chunk is a 4-byte little-endian uncompressed size followed by the
+/// szlib stream.
 pub(crate) fn szip_decompress(
     _data: &[u8],
     _cd: &[u32],
@@ -33,62 +53,174 @@ pub(crate) fn szip_decompress(
 
 #[cfg(feature = "szip")]
 fn szip_decode_impl(data: &[u8], cd: &[u32], chunk_size: usize) -> Result<Vec<u8>, FormatError> {
-    if cd.len() < 3 {
-        return Err(FormatError::ChunkedReadError(
-            "szip: missing client data".into(),
-        ));
+    let err = |m: &str| FormatError::ChunkedReadError(format!("szip: {m}"));
+    if cd.len() < 4 {
+        return Err(err("missing client data"));
     }
     let options = cd[0];
-    let pixels_per_block = cd[1];
-    let bits_per_sample = cd[2]; // H5Z_SZIP_PARM_BPP
-    if bits_per_sample == 0 || bits_per_sample > 32 {
-        return Err(FormatError::ChunkedReadError(
-            "szip: invalid bits per sample".into(),
-        ));
+    let pixels_per_block = cd[1] as usize;
+    let bits_per_pixel = cd[2];
+    let pixels_per_scanline = cd[3] as usize;
+    if !(1..=32).contains(&bits_per_pixel) && bits_per_pixel != 64 {
+        return Err(err("invalid bits per sample"));
     }
-    if chunk_size == 0 {
-        return Err(FormatError::ChunkedReadError(
-            "szip: unknown output size".into(),
-        ));
+    if pixels_per_block == 0 || pixels_per_scanline == 0 {
+        return Err(err("invalid block or scanline size"));
     }
-    if data.is_empty() {
-        return Err(FormatError::ChunkedReadError("szip: empty input".into()));
+    if data.len() < 4 {
+        return Err(err("chunk too short"));
     }
+    // H5Zszip.c: UINT32DECODE of the uncompressed size, then the stream.
+    let dest_len = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize;
+    let limit = if chunk_size != 0 {
+        chunk_size
+    } else {
+        crate::filters::MAX_DECOMPRESS_SIZE
+    };
+    if dest_len > limit {
+        return Err(err("declared size exceeds chunk size"));
+    }
+    let stream = &data[4..];
 
-    // Map HDF5 option mask to libaec flags.
-    // HDF5 always stores SZIP data in MSB order, so AEC_DATA_MSB is unconditional.
-    // H5_SZIP_NN_OPTION_MASK (0x20): NN differential preprocessing.
-    let mut flags: u32 = libaec_sys::AEC_DATA_MSB;
-    if options & 0x20 != 0 {
+    // --- libaec sz_compat.c: SZ_BufftoBuffDecompress ---
+    let rsi = pixels_per_scanline.div_ceil(pixels_per_block);
+    let mut flags = 0;
+    if options & SZ_MSB_OPTION_MASK != 0 {
+        flags |= libaec_sys::AEC_DATA_MSB;
+    }
+    if options & SZ_NN_OPTION_MASK != 0 {
         flags |= libaec_sys::AEC_DATA_PREPROCESS;
     }
+    let pad_scanline = !pixels_per_scanline.is_multiple_of(pixels_per_block);
+    let deinterleave = bits_per_pixel == 32 || bits_per_pixel == 64;
+    let bits_per_sample = if deinterleave { 8 } else { bits_per_pixel };
+    let pixel_size = match bits_per_sample {
+        17.. => 4,
+        9.. => 2,
+        _ => 1,
+    };
+    let scanlines = (dest_len / pixel_size).div_ceil(pixels_per_scanline);
+    let buf_size = if pad_scanline {
+        rsi.checked_mul(pixels_per_block)
+            .and_then(|n| n.checked_mul(pixel_size))
+            .and_then(|n| n.checked_mul(scanlines))
+            .filter(|&n| n <= crate::filters::MAX_DECOMPRESS_SIZE.max(limit))
+            .ok_or_else(|| err("scanline padding too large"))?
+    } else {
+        dest_len
+    };
 
-    let mut out = vec![0u8; chunk_size];
+    let mut buf = vec![0u8; buf_size];
     let mut strm = libaec_sys::AecStream::zeroed();
-    strm.next_in = data.as_ptr();
-    strm.avail_in = data.len();
-    strm.next_out = out.as_mut_ptr();
-    strm.avail_out = chunk_size;
+    strm.next_in = stream.as_ptr();
+    strm.avail_in = stream.len();
+    strm.next_out = buf.as_mut_ptr();
+    strm.avail_out = buf_size;
     strm.bits_per_sample = bits_per_sample;
-    strm.block_size = pixels_per_block;
-    strm.rsi = 128; // HDF5 default: 128 blocks per reference sample interval
+    strm.block_size = pixels_per_block as u32;
+    strm.rsi = rsi as u32;
     strm.flags = flags;
-
+    // SAFETY: next_in/avail_in and next_out/avail_out describe live buffers
+    // (`stream` and `buf`) that outlive the call.
     let result = unsafe { libaec_sys::aec_buffer_decode(&mut strm) };
     if result != 0 {
         return Err(FormatError::DecompressionError(format!(
             "szip: libaec error {result}"
         )));
     }
-    let decoded_len = chunk_size - strm.avail_out;
-    out.truncate(decoded_len);
-    Ok(out)
+    let mut total_out = strm.total_out;
+    if pad_scanline {
+        let line = pixels_per_scanline * pixel_size;
+        let padded_line = rsi * pixels_per_block * pixel_size;
+        // remove_padding: compact each padded line down to `line` bytes.
+        let mut i = line;
+        let mut j = padded_line;
+        while j < total_out {
+            let end = (j + line).min(buf.len());
+            buf.copy_within(j..end, i);
+            i += line;
+            j += padded_line;
+        }
+        total_out = scanlines * line;
+    }
+    if total_out < dest_len {
+        return Err(err("stream decoded to fewer bytes than declared"));
+    }
+    buf.truncate(dest_len);
+    if deinterleave {
+        // deinterleave_buffer: byte planes back into words.
+        let w = (bits_per_pixel / 8) as usize;
+        let n = dest_len / w;
+        let mut out = vec![0u8; dest_len];
+        for i in 0..n {
+            for j in 0..w {
+                out[i * w + j] = buf[j * n + i];
+            }
+        }
+        Ok(out)
+    } else {
+        Ok(buf)
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[cfg(feature = "szip")]
+    fn unhex(s: &str) -> Vec<u8> {
+        (0..s.len())
+            .step_by(2)
+            .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
+            .collect()
+    }
+
+    /// SZIP chunks written by libhdf5, decoded exactly as libhdf5 decodes
+    /// them. Each case: fixture, chunk byte offset and size (from h5py's
+    /// `get_chunk_info`), the filter's cd_values, and the chunk's values as
+    /// h5py reads them (file byte order, hex). Before the fix every one of
+    /// these came back as garbage or zeros (or "invalid bits per sample" for
+    /// 64-bit): the 4-byte size prefix was fed to libaec, 32/64-bit samples
+    /// were not de-interleaved from byte planes, the reference sample
+    /// interval was fixed at 128 instead of derived from the scanline, padded
+    /// scanlines were not unpadded, and LE data was decoded as MSB.
+    #[cfg(feature = "szip")]
+    #[test]
+    fn szip_decodes_libhdf5_chunks_exactly() {
+        /// (name, file, chunk offset, chunk size, cd_values, decoded hex)
+        type Case<'a> = (&'a str, &'a [u8], usize, usize, [u32; 4], &'a str);
+        let noencoder: &[u8] = include_bytes!("../tests/fixtures/filters/noencoder.h5");
+        let le_data: &[u8] = include_bytes!("../tests/fixtures/filters/le_data.h5");
+        let h5py: &[u8] = include_bytes!("../tests/fixtures/filters/szip_h5py.h5");
+        #[rustfmt::skip]
+        let cases: &[Case] = &[
+            // <i4, 10 px/scanline over 4 px/block: padded scanlines + byte planes.
+            ("noencoder /noencoder_szip_dset.h5", noencoder, 6040, 16, [168, 4, 32, 10],
+             "00000000010000000200000003000000040000000500000006000000070000000800000009000000"),
+            // <f4, LSB + NN.
+            ("le_data /Szip_float_data_le", le_data, 55224, 48, [169, 4, 32, 12],
+             "abaaaa3eabaa2a3f0000803fabaa2a3f0000803fabaaaa3f0000803fabaaaa3f5555d53fabaaaa3f5555d53f00000040"),
+            // >f4, MSB + NN.
+            ("le_data /Szip_float_data_be", le_data, 55396, 48, [177, 4, 32, 12],
+             "3eaaaaab3f2aaaab3f8000003f2aaaab3f8000003faaaaab3f8000003faaaaab3fd555553faaaaab3fd5555540000000"),
+            // <f8 (64-bit), NN.
+            ("szip_h5py /f8", h5py, 4016, 100, [169, 8, 64, 10],
+             "00000000000008c000000000000008c000000000000008c000000000000008c000000000000004c000000000000004c000000000000004c000000000000004c000000000000000c000000000000000c000000000000000c000000000000000c0000000000000f8bf000000000000f8bf000000000000f8bf000000000000f8bf000000000000f0bf000000000000f0bf000000000000f0bf000000000000f0bf000000000000e0bf000000000000e0bf000000000000e0bf000000000000e0bf0000000000000000000000000000000000000000000000000000000000000000000000000000e03f000000000000e03f000000000000e03f000000000000e03f000000000000f03f000000000000f03f000000000000f03f000000000000f03f000000000000f83f000000000000f83f000000000000f83f000000000000f83f"),
+            // <i8 (64-bit), entropy coding without NN.
+            ("szip_h5py /i8", h5py, 4188, 53, [141, 4, 64, 10],
+             "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000030000000000000003000000000000000300000000000000030000000000000003000000000000000300000000000000030000000000000006000000000000000600000000000000060000000000000006000000000000000600000000000000060000000000000006000000000000000600000000000000090000000000000009000000000000000900000000000000090000000000000009000000000000000900000000000000090000000000000009000000000000000c000000000000000c000000000000000c000000000000000c000000000000000c000000000000000c000000000000000c000000000000000c00000000000000"),
+            // <u2, 35 px/scanline over 8 px/block: padded scanlines, 16-bit samples.
+            ("szip_h5py /u2", h5py, 4308, 43, [169, 8, 16, 35],
+             "00000000000000006100610061006100c200c200c200c20023012301230123018401840184018401e501e501e501e5014602460246024602a702a702a702a702080308030803"),
+        ];
+        for (name, file, off, len, cd, want) in cases {
+            let want = unhex(want);
+            let got = szip_decompress(&file[*off..off + len], cd, want.len())
+                .unwrap_or_else(|e| panic!("{name}: {e:?}"));
+            assert_eq!(got, want, "{name}");
+        }
+    }
+
     #[test]
     fn szip_disabled_returns_unsupported() {
         #[cfg(not(feature = "szip"))]
@@ -132,6 +264,8 @@ mod tests {
         assert_eq!(rc, 0, "aec_buffer_encode failed: {rc}");
         let enc_len = encoded.len() - enc.avail_out;
         encoded.truncate(enc_len);
+        // H5Zszip.c prefixes the stream with the uncompressed size.
+        encoded.splice(0..0, (original.len() as u32).to_le_bytes());
 
         // Decode through our public interface.
         // cd[0]=0 (no NN bit 0x20), cd[1]=8 (ppb), cd[2]=8 (bpp), cd[3]=1024 (pps).
@@ -163,6 +297,8 @@ mod tests {
         assert_eq!(rc, 0, "aec_buffer_encode with NN failed: {rc}");
         let enc_len = encoded.len() - enc.avail_out;
         encoded.truncate(enc_len);
+        // H5Zszip.c prefixes the stream with the uncompressed size.
+        encoded.splice(0..0, (original.len() as u32).to_le_bytes());
 
         // cd[0] = 0x20 (H5_SZIP_NN_OPTION_MASK) → decoder must set AEC_DATA_PREPROCESS.
         let cd = [0x20u32, 8, 8, 1024];
diff --git a/crates/clawhdf5-format/tests/fixtures/filters/README.md b/crates/clawhdf5-format/tests/fixtures/filters/README.md
index d2095a0..24b36c6 100644
--- a/crates/clawhdf5-format/tests/fixtures/filters/README.md
+++ b/crates/clawhdf5-format/tests/fixtures/filters/README.md
@@ -8,3 +8,6 @@ the tests come from h5py's `DatasetID.get_chunk_info`.
 | File | Origin | Licence |
 |------|--------|---------|
 | `h5ex_d_lz4.h5` | HDF Group `HDF5Examples/C/H5FLT/tfiles/h5ex_d_lz4.h5` (hdf5 repository) | HDF5 licence (BSD-3-Clause style) |
+| `noencoder.h5` | HDF Group `test/testfiles/noencoder.h5` (hdf5 repository) | HDF5 licence (BSD-3-Clause style) |
+| `le_data.h5` | HDF Group `test/testfiles/le_data.h5` (hdf5 repository) | HDF5 licence (BSD-3-Clause style) |
+| `szip_h5py.h5` | Written for these tests with h5py 3 / libhdf5 2.0.0 (libaec szip): `f8` (8x10, chunks 4x10, `('nn', 8)`), `i8` (8x10, chunks 4x10, `('ec', 4)`), `u2` (70, chunks 35, `('nn', 8)`) | Same as this repository |
diff --git a/crates/clawhdf5-format/tests/fixtures/filters/le_data.h5 b/crates/clawhdf5-format/tests/fixtures/filters/le_data.h5
new file mode 100644
index 0000000000000000000000000000000000000000..e79ab47b416f1cf42e80c5c62307346deb4ec812
GIT binary patch
literal 72368
zcmeHQe{fXA9pAf<<b(tQlaw^t;pkXN<FsW2EjpTebim1|{20OmiXD^i%aDd24X9De
za5|lePV0|R9EnVuwkqVEG*n1Sr-K~zk1{r(Rj(qBV>%-xy%vXYjAc^Eb-%m&P4+hT
z?j4sW>dbfH-oD-MXTQ7q$-8%-@4k2M9<G~rbLsf1@eEQZ#Ky5Qv5T9OO8ct<q6*7s
zZ&W+d%GeH7-W!xV0S5mVt5o%qm46erbjkerj1@BUkGX2bW{#74CsjxN1#)Mv0-RhX
zcdAvnS#{8?@MZszhL9SUf>HJ*VW4jQ+{G}|xcd7&qjKv$Ie`g03&k9#Ru<K^@Y)4W
zC9WGRf4BdI?q7HG&qP!tD@r#8Sg|;M<%aQ09$|r?Jg|xtxJcz@xqu(}fSeqTu#mu5
zZ1w~;J5?YX6!P~2*n`1fK-yX7u3dpRybqyXuY(pL<*y*ns7#CHK)}e;6x9MMU+VnZ
z0><h@y<eO9g^T6|_*nwq&+UcTJ*z>ou$r$M`kb0Y^-(v3tjG6%Ur%=27v1XHaWy@+
z<0>B5wX2hpq90dS2V9LDSNku6GGmwaxI#X{>Nv>%eKw8H5mJ~<4Y8@)#XfwpsP|bx
zFaY-By(MuljpH+iF0pO`+bqffHI5?DuAFUI-PF2v)4FwA)^3GnBEcbD!e7{4*2*&a
z$2zZknEzlwqxuAfAwd^nI_CQtKCnZ!ok%~Em7m(M;Dw>b)Q2Y%pqNu*aqvhoK;oQe
zi==tWc;=L-fssB<)JP}VUZWZ&&rhqS=OE{0;i}Vd2GOM@>vO{Kc0s`Nl+_>Ncz=Z!
z2fOua;y5$}cyoMNaN_|F$>J|B93Mxiz`^+*@VVrXgl#u+aOAYg&+!732UK~`aXNX~
z;x9<K;;f)-7xD0iIIB?g=hCl<v(yJc#2B9&&KjRS4^LdF91ot;a?Mc@_@Nv8drzu2
zuEW#tTZ1v<jjOtYH>EhP<saMt!3iW}e)blYzxn85VG-vR*-&f?%TuXy!h%8ujUyJU
z!F<FU^}|A+D{mlx`T3roO{0;K+)-sG0%u~55jp?!S<k(5>qfn-&eWD|S6E<&96moV
z=T(;T_N}|!aPX;@>2=S*N0-abA&dL6xo?@roE9q+9_mi?O`2NkII&0#e?|G340EW1
zGUN%{#+*B4?>zU<4!sO^sJAM#ocj@F$hq-tr}9^BxOhqL@l3CQd&=*(RC_TY?nmS;
zPAn51>iSMBoY%hl_QNeZw>9_M_ZJlv6_5K?lKp0R_ZttsX-zL?t5c<Sx9^@%dbs7a
zc)YKyszw^y=G_0&Gwiw&_UNMWmEho(zdrE5(WBF6&n{oseaFtzn?EUeEznxY{&=C{
z+4kKBZtrR7IsBEf@6MbV|Ce3&apLvGm1pDeIAd=-F=Yd{H*os}UR%ruP<b}|$h;r!
zTUom2fd^SrJ9K~GNL5vm<NKs#Tl7rt_qp?jK7hWryuPyM)KT8M=={!oE7APYn-$M;
znlmJb>mgpicpmesGGNoH;Ka<6i8D_zS1H^#BwUImwXbKlx!_EHKT<esizi6LVHxd|
z69dEmF+dCu1H=F^Kn(c9fN$%8({o!71ceSG*VhA~ZmjEpur>%Uh-i40bJErW+pmzV
zKzf1<6%y234+PoD*8?Gqc@@v+)dxBHk*!!S$Tu!H7L<;>;(`wq&MPiBq~diGhk74}
z{hkrmV;q1uEF+$BVt^PR28aP-fEXYKhyi~Xz__4VU7yeYJ^_#WCir~<aLKY`7o_z2
zaj*_%ly7{XE{6(IU@E##QR%VK;)DJf5{vPHaBap1`ReVU8mO1vu2wixN1M}lI>&YR
zKY6Uf>-?F0P?;DY28aP-fEXYKhyh|Ce+GQh+g|r6E4>Z(?eWxM-lq(8FpIJ3ZCsBZ
zgWf(gThf_TZ^OFqD2+;Y%JCbqv)^GT?~r;gX|MdGq@?td(0vuVCzc)_zvru!l~=I8
zUq5Ao<eudl&m6m4I`E2TK2$ibc&1!koH22DT|f<&U9WIh#_I)$8+EK8kaA*x7$63S
z0b+m{AO?tmF)@JgOl98J59Id<%eY{ZE~IkPzh4OTLu!<GKM~gD8o#L9SH$-V!4<TZ
z{mC5tt8P3q?Rr6_CD9na2)DUjFR1e9=?<lHO&sc6Y~H8V_YsGB810x3y5LNI<?uyJ
zEulZeA$J;tc8LLEfEXYKhyh}N7$63GV8Azh&7VRdHQS}t*U%@t#-Oj;Z<RcLCi?o@
zO6Q_(#y=VATrAfaIMl;v$9$Cw&h+<V($`2C<-`CnKnxHA!~iis3|vwMeACxlR?9VJ
z`LsEG{Tk|rQgT|jevNhcu3tlYT!R(*TKUhFJZ=3tyaI<NzYi;>VXj}}b1HYJ4#L-D
z@2Odt_nS9{7|rTb`59Ay8@Os!-ce*2zGOwW`)oX>&z?5o3Y@kXSL`=%ZgI-1E;!TQ
z2Oe>XI9PBnloJEQ05L!e5Cg;jF+dFX#(-~e3cn5@HCyI5C8ZYX^*9CU#u}##UkAuP
zPKmCTY~po*jB!f7`g+LF!}y0Y^>AgWw4h#h)7RIy;7otFDIE5P{t!p{8kwP-7$63S
z0b+m{AO?s5e;M#iUx(BKC0SSuP`?fU+HQzHupJDjudyy)eGTo?uLDT`&_^ywyAIH|
zQ8oaVAU`1=_4`-(>g#oe9tPi#Q0K}LV^ISq27yK`?{LAH{tkNRYd8N|SF}qE5Cg;j
zF+dCu1H=F^kOu?4>1)1zEj3$MjKb8{TwaS6Y+YYN-B|VY$oh5O`kHN%yut8Y^|gdB
zHn**<>E4Fcwdv(m!{vJT{mcgW1DXx%S~oRqZCKN^wQ0D8n;ku*+1G5kYZdpVJ8-ot
z2Wj@z%}s4-tr>H%WB2LyEzO(SvW;J}qk}a6>o*#$jsA!2*xInSZrXTfgV+pHkmld8
zgO{6I*KS?iytd8cK2ts5=dE4W+O&183C~n723ddK`pu)j{9rKaaYpsccdc8;|AywF
zkfB~QS-5Kb)=^;Y->gUf7-QmnSa@8TdA{-CidoVVUROcCW<1ws;N0T3pSj>ne=jH;
z_J{ruhh?-=P7DwO!~iis3=jjv05RYX0~jBMmC(HS-ytXy*FoSpLi|5Zme}g?A=Jb5
z7fAZ^P_Ygg<Qpe~Z`8<;>psD+SJJM-?0;B-F-{cIar-_-o$9pm-t0Z#?>ijYkR7)}
z`Rji&JFZ#b<~)=g_eHnn8?VHT_yXsNaRZi58929i;$s(_>2IQXST6cUe~6REK3XX!
z28aP-fEXYKhyh}N81Rb$-{KX1JxgjfOAO77S7fx38q(txtb+z)jaNX%!)pxjO65)%
zh`cNZGBjhnlCRFLR(jX#y1@4g9O_~*GHvq%7aZzmgx*a$8>yh27$63S0b+m{AO?tm
zOUgii%So}+tk#<^{<=V=Xg@hTZTP;isUbFXyPAcrw?Q)=?=6XgNxy##>L#$T@Dotu
zC}8sY)BK(raUaku^*khP9lx;?9F6E=AxX`3e8#FJD#xvxjyB}#C~qg!_aIE1se6MH
zWmCkt>FDcRaHhW&g~I`%Kg3}f?UWM(!~iis3=jjv05L!e_``s2I(m9;Iyzp`dmKit
z>u9LULr3q8NeVzJf@~C$@$=Hrzc6&K9NOsbF~4Ep+;ncyB<aD_+o+Gx{|yR<W%P$Q
zETf%rVt^PR28aP-fEXYKhyi~X@J&biydEvb6<}-R_e>c+CSO~RKKFY`0Z7HqTt`p$
zp!;zpcugGYS!}*h>E5uaaMQVuDcm#zXZq_V9gWmdP7DwO!~iis3=jjvz$InCHy!Qs
zdUOstdd{CEQ<p?Xhdur(Ck%aza8vhwsBm9X05=^yQQp?Vyma){3WxonKkOIFXs4VQ
zAO?s5Vt^PR28aP-z#j&D)6rvIkFHL;KU&`RF^+BF&oP9VcJUWz!Z3fqPc3Qd(ou1v
zi06IM(9%1(-5)*Eqv@fyCEz;-4)rWn_bc2CRpF*{r=FAm{P|nNnf_`O4*i=rETf%r
zVt^PR28aP-fEXYKhyi~X0DU;nIBj72Q1?)G!cHt5?3^^%ow8GQvc+lHw(P~ihUn7z
zrO8*F_dhuQj~k-`f0teJ^7G4{U!F+VsX?b>XTy@%l33DdIdWmiABMV972m9Q?#cQU
zi%w7e3BZ#*H9a>i+gHYJTK4?1Jx=25tT>r^<f*rQ-V(p^)Bk}mlwP+dC9F1fLBfi5
zPHEdyeeKnaq43@OzK#GmoAH;+&Lmpm_W8p3*Did~*_TR2BgrEdmS0%j<#ctW%|v*e
z@zIV~k~@zKY(L*sx9`f1j@RsZK2*EK-exaYo{BkL_CQTf&C9=ax<3`}&z!7eBjXzx
zvGbPm*1*bxmK^K2iSamNv-du^^ZL16x3g`_2lp+qoq23hEM<2kdzzzxSd58(cFbmu
z!^A(^;rlhwny3?X5>A5EMuwBcVd7uRjxi1s{~Vh)ORxm1U==kLHC7~IMJ$yh__#<C
z+$)j<mxv_6agk!+GA8j3>^l-)W7(G7BJt9-#Oq_z$E?pqpQAoDea!k?^f~Hd)5omO
zMeM6F>toZ$tj|TC;|<P%S7WuTKx~q*myAX1de|)Ghf%F_V8FIm0c^$<s*Q7y?>Ps{
z7!x21kbE;f7N2!v!K{%D6QH^F(lW+3Rin(GXWx8ol!yUhfEdV$0pHfq#(X_G`#Re1
z_-tTJD9m5RI@+;5UdI!dK~0FIqtV}!3UiZp%6A=Yjatw0T1R_I;k?$-UQsx&b+n|y
z;mmN|4A<MRjCRV20b+m{AO?s5Vt^PR2K-^bHyu6Z>(S}Y>k5kJhy~ad`8#=19P=q_
z*P&OOl@xgB=Zkw@SN+08^8)<aSfPM)zG78&p8J(~znvYoI4JFp|2I2sj>55jXUByV
z?$G<$aUq4<+?yTew>95*r0g9z5?IvK6UGHtXBjxRcww^(&h*!*aOfZXAr8xEr<@od
z28aP-fEXYKhyh~29|nAjN5;Iaf6?(s=XptiuklE}I{I}(_u^j%^(>azDY=Jw+fC=r
zbitYa8Wj%xqd&x98SRu41H=F^KnxHA!~iis4EV!<Z#p_$CDz`HSve~&DlWz644A*e
z5#1Hwx4ObjEb{k1K<7c-1U4t_duD~g9%36;t=|fG@#AwmhkIjS$sP4!Io~hlFKxYl
zW}#e@2MEYX0Udq648ILBL&A0axLVhL#>oKEs`e_};$Lc&8Fr&;eZ6^-YLEc~Rm}=_
zGMF8AOtoKpi;E*wy;tSuYO~|~wz|b5cwD&k_qd7+OdQ4q*sj~a;qicW%ul%BOn+x7
z9zn_|CkBWCVt^PR28aP-;F2=nTRbv7w|Jx|uqz4^)Z-DT%O@TI#hxJ^X)lvfKCgHr
zUmacYF3gGzKkrSJWI%-bkY8=!+;sFUE;v(1uU9zi1N|Wm%V?*Z7$63S0b+m{AO?s5
JV!$5;{ts@6(^>!k

literal 0
HcmV?d00001

diff --git a/crates/clawhdf5-format/tests/fixtures/filters/noencoder.h5 b/crates/clawhdf5-format/tests/fixtures/filters/noencoder.h5
new file mode 100644
index 0000000000000000000000000000000000000000..84f87526ffe4943f03299168c5ff17fdbec94706
GIT binary patch
literal 8088
zcmeHKF;Buk6n@uMQy?lD6OD^EFm!Uzn6QZn7+BP(0~;n#QjGy4xHz#mxH2(1@)L}^
zt8sR6)L)_ZdiS815E2px>3hlD>wEY0uJ_&b`t_lhKUhd^BmvEJ!GSc`VW6vTW4+DM
zm>`c)FeM|tSlB-fL!pyJ)7CMbT%+kXs?C~zQ)%5@Tvuu>zr7Xg0{qIig4=ri%CD%>
zt_BKCK+`=uQHTG;pPdv600M|-^7Qy0j=qwVU>_?61XyZ!07x_IW%Yhy+%X&e+~mm#
zaKm<80n8b^WrHwVDTF!(M>*Tz(k`eo08NIqfOYMK9&j~`V6b0P;Dtk(W_2%)?*P8V
z2<@wa{Yh2uka1Oi*pqEp-n{b4FinpED++;zn6J;596%PssqOUeD32WKUIE@FoLAC}
zR9>8V$eFwm&ncl+h0Z7P=`pkIt&KM54@_jCN1Hif2ABb6fEi#0n1LB!;IDNk=?T&y
zDJX}UQ5bzUNr$2$L_K4EC<g)*bub$JVD4<|#fa$EU`eOO>el;r)UBjAHF%tUB0a59
z!@AG<X-EcNF$2s1Gr$Zm12JL1Q}dLGOt+L(f<!{jlO=#eRzgNKbkn)L=hw@aP{Gq=
P2ABb6fEi#0V#B}>9B7j0

literal 0
HcmV?d00001

diff --git a/crates/clawhdf5-format/tests/fixtures/filters/szip_h5py.h5 b/crates/clawhdf5-format/tests/fixtures/filters/szip_h5py.h5
new file mode 100644
index 0000000000000000000000000000000000000000..f197d796871fe81bf8eaea6e29ab3e46b40009fb
GIT binary patch
literal 11320
zcmeHMO=uHA6n^`YW@)sW;!jJ7?N%cQS}0PXhhkTnlz8xm+Rz>f=1_zl1VN;+muy8e
zR)kvhDm{3p<PtoI9=d`Dy?O8^h*xj5r=A+;?YzOTNesb?R^Nn~H}n2x=gZ9S_U*k)
zYJ6{dPdi|-EHEJ+bX8NKGJHaE)L6-L)X|6*iLU9gqXGT^x{2RO{=<5c6X`SvP32!v
zGJ`==_LLvRXZn92kV&7ph%23@<vK*)(LSRgTp5<VIifdC%CXv?#8&D}sRnJq{*?{x
z&|`s(rb~|&d5{N49HG$I5IY|=JDpLIxiA!ip^70DBZ7-^-H;Ar!at5R>0X>il$&sU
zVeT4QJQo$`a7D#AEjn3-Zsa3kEDD|!DGG*}pxvqF()kN14UFI!(5FM|XyjkrCgrY1
z8j{~l;;q9!*ZWtbc;9KD*W7orO%_<TrGolb=~}F@+OL2oiP8)2(;7o&FanGKBftnS
z0*nA7zzF<R2)OuOfG_Ey)134tqRV|-y0MW1!$0)K&x7s3Dil27<g^vN12Co~Vs|fZ
z^n&pP9D7^vN8Q%ua1tY4VYQgg>8Y1D?acvukK^6aEz5Dxk6HXW^?Ci=LR)F_{n6~B
zwe-a7(u=cQ%iy?+x@pC02pTP|R}K`m_dPR{k5_K5JSji@(4YKtHCrr}-a>Zz{;Sgb
z{M2ixYmUruu?iPDpckAr!~X^XtuT#%y(zQ+Rda*hNQWb0@t)|SUymMyKHo!aE{p&p
zzz8q`jKCj4;Men!aljBFE|Q3tN0gaFfL8O64}>M=lw46mc}Smfc_^;RLk8(v!P?o<
zD35lk$uu*R9~&b*eAs&BA?MNR!NNR7fDvE>7y(9r5nu#%9Rjs+w2WFc`tXj4AuNg-
zB8e2l&4Q-~5*Zk_8aJ!&_h@}Y)F0pPbt@V84cKyuV4RcO-`qBeqmMV3_anu{10=__
fRF_I9ZdUTDg;lF_pAldL7y(9r5nu#%0RrCvf<eZO

literal 0
HcmV?d00001


From 3000b40cf3d8187dbaec2d1d9aa664d693eccb8f Mon Sep 17 00:00:00 2001
From: osobh <om.sobh@gmail.com>
Date: Fri, 25 Sep 2026 21:14:14 -0500
Subject: [PATCH 5/8] fix(format): N-Bit pass-through flag and no-op (enum)
 members

- libhdf5 sets cd_values[1] ("need not compress") when every field is
  already full width and then stores the chunk unchanged
  (H5Z__filter_nbit: `if (cd_values[1]) HGOTO_DONE`). We ignored it and
  tried to unpack, so tfilters.h5 / h5stat_filters.h5 `/all` (shuffle +
  szip + deflate + fletcher32 + N-Bit) failed with "nbit: packed data too
  short". A type with no N-Bit parameters (cd = [3, 1, nelmts]) is now
  accepted the same way.
- Class 4 (H5Z_NBIT_NOOPTYPE: enum, string, opaque, ... members) is
  stored whole, 8 bits per byte; it was UnsupportedFilter(5)
  (h5repack_nested_8bit_enum_deflated.h5).

N-Bit on floats was not wrong in the filter: for le_data.h5 /
Nbit_float_data_* our output equals libhdf5's decoded bytes in the file
datatype (a 20-bit float, offset 7, bias 31). h5py's values differ
because libhdf5 then converts that custom float layout to IEEE, which
our datatype reader does not do; nbit_float_matches_libhdf5_file_type_bytes
pins the filter output and the doc comment says where conversion belongs.

Tests: nbit_need_not_compress_is_passthrough,
nbit_in_multi_filter_pipeline_matches_libhdf5 (tfilters.h5 chunk, szip
feature), nbit_compound_with_enum_member_matches_libhdf5 all failed
before; nbit_float_matches_libhdf5_file_type_bytes (guard).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 crates/clawhdf5-format/src/filters.rs | 193 +++++++++++++++++++++++++-
 1 file changed, 187 insertions(+), 6 deletions(-)

diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs
index c0bf701..54e56d4 100644
--- a/crates/clawhdf5-format/src/filters.rs
+++ b/crates/clawhdf5-format/src/filters.rs
@@ -385,6 +385,9 @@ enum NbitNode {
         count: usize,
         base_size: usize,
     },
+    /// `H5Z_NBIT_NOOPTYPE`: a field N-Bit does not reduce (enum, string,
+    /// opaque, ...), stored as all `size` bytes, 8 bits each.
+    Noop { size: usize },
 }
 
 impl NbitNode {
@@ -395,6 +398,7 @@ impl NbitNode {
             NbitNode::Array {
                 count, base_size, ..
             } => count * base_size,
+            NbitNode::Noop { size } => *size,
         }
     }
 }
@@ -414,6 +418,7 @@ fn parse_nbit_node(cd: &[u32], idx: &mut usize, depth: u32) -> Result<NbitNode,
     const ATOMIC: u32 = 1;
     const ARRAY: u32 = 2;
     const COMPOUND: u32 = 3;
+    const NOOPTYPE: u32 = 4;
     if depth > NBIT_MAX_DEPTH {
         return Err(FormatError::ChunkedReadError(
             "nbit: type tree nested too deeply".into(),
@@ -489,8 +494,17 @@ fn parse_nbit_node(cd: &[u32], idx: &mut usize, depth: u32) -> Result<NbitNode,
                 members,
             })
         }
-        // Class 4 is H5Z_NBIT_NOOPTYPE (members copied verbatim) — not seen in
-        // practice for the supported leaf types and left unsupported.
+        NOOPTYPE => {
+            // class, size
+            let size = nbit_cd(cd, *idx + 1)? as usize;
+            *idx += 2;
+            if size == 0 {
+                return Err(FormatError::ChunkedReadError(
+                    "nbit: invalid no-op type size".into(),
+                ));
+            }
+            Ok(NbitNode::Noop { size })
+        }
         _ => Err(FormatError::UnsupportedFilter(FILTER_NBIT)),
     }
 }
@@ -555,6 +569,11 @@ fn decode_nbit_node(
                 decode_nbit_node(bnode, br, elem, base + i * base_size)?;
             }
         }
+        NbitNode::Noop { size } => {
+            for slot in &mut elem[base..base + size] {
+                *slot = br.read(8)? as u8;
+            }
+        }
     }
     Ok(())
 }
@@ -567,17 +586,28 @@ fn decode_nbit_node(
 /// type tree — atomic (`[1, size, order, precision, offset]`), array
 /// (`[2, total_size, <base>]`) and compound
 /// (`[3, total_size, nmembers, (offset, <node>)*]`) — preceded by
-/// `[nparms, flag, nelmts]`. Decompression walks the tree once per element,
+/// `[nparms, need_not_compress, nelmts]`; when `need_not_compress` is set
+/// (every field already uses its full width, e.g. a 32-bit int of precision
+/// 32) libhdf5 stores the data unchanged and so do we. Fields N-Bit cannot
+/// reduce (enums, strings, ...) are no-op nodes (`[4, size]`) copied whole.
+/// Decompression walks the tree once per element,
 /// placing each field's bits at its byte/bit offset in a zero-filled element
 /// (HDF5's canonical reduced-precision layout). Sign-extension of reduced
-/// precision signed integers is the datatype reader's job. Atomic floats are
-/// encoded as full-precision atomics and handled transparently.
+/// precision signed integers is the datatype reader's job, and so is
+/// converting a reduced-precision float (its own sign/exponent/mantissa
+/// layout, e.g. `le_data.h5`'s 20-bit `Nbit_float_data_*`) to IEEE: the
+/// filter's output is the file type's bytes, as libhdf5's is before type
+/// conversion.
 fn nbit_decompress(data: &[u8], cd: &[u32], expected_bytes: usize) -> Result<Vec<u8>, FormatError> {
-    if cd.len() < 4 {
+    if cd.len() < 3 {
         return Err(FormatError::ChunkedReadError(
             "nbit: missing filter client data".into(),
         ));
     }
+    // H5Z__filter_nbit: `if (cd_values[1]) HGOTO_DONE(*buf_size)`.
+    if cd[1] != 0 {
+        return Ok(data.to_vec());
+    }
     let nelmts = cd[2] as usize;
     let mut idx = 3;
     let root = parse_nbit_node(cd, &mut idx, 0)?;
@@ -1584,6 +1614,157 @@ mod tests {
 
     // --- LZ4 tests ---
 
+    fn unhex(s: &str) -> Vec<u8> {
+        (0..s.len())
+            .step_by(2)
+            .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
+            .collect()
+    }
+
+    fn one_filter(filter_id: u16, client_data: Vec<u32>) -> FilterDescription {
+        FilterDescription {
+            filter_id,
+            name: None,
+            flags: 1,
+            client_data,
+        }
+    }
+
+    /// `le_data.h5` `/Nbit_float_data_{le,be}` chunk (0,0): a 20-bit float
+    /// (offset 7) packed by N-Bit. The filter must reproduce libhdf5's
+    /// decoded bytes in the *file* datatype (h5py `DatasetID.read` with the
+    /// file type as memory type, so no conversion); converting that custom
+    /// float layout to IEEE is the datatype reader's job, not the filter's.
+    #[test]
+    fn nbit_float_matches_libhdf5_file_type_bytes() {
+        let file: &[u8] = include_bytes!("../tests/fixtures/filters/le_data.h5");
+        let cases = [
+            (
+                55952,
+                0,
+                "8055d5018055e5010000f0018055e5010000f0018055f5010000f0018055f50180aafa018055f50180aafa0100000002",
+            ),
+            (
+                56076,
+                1,
+                "01d5558001e5558001f0000001e5558001f0000001f5558001f0000001f5558001faaa8001f5558001faaa8002000000",
+            ),
+        ];
+        for (off, order, want) in cases {
+            let pipeline = FilterPipeline {
+                version: 2,
+                filters: vec![one_filter(FILTER_NBIT, vec![8, 0, 12, 1, 4, order, 20, 7])],
+            };
+            let got = decompress_chunk(&file[off..off + 31], &pipeline, 48, 4).unwrap();
+            assert_eq!(got, unhex(want), "byte order {order}");
+        }
+    }
+
+    /// libhdf5 sets `cd_values[1]` ("need not compress") when every field is
+    /// already full width and then stores the data unchanged; we unpacked it
+    /// anyway and failed with "nbit: packed data too short".
+    #[test]
+    fn nbit_need_not_compress_is_passthrough() {
+        let data: Vec<u8> = (0..200u32).map(|i| (i * 7) as u8).collect();
+        let pipeline = FilterPipeline {
+            version: 2,
+            filters: vec![one_filter(FILTER_NBIT, vec![8, 1, 50, 1, 4, 0, 32, 0])],
+        };
+        assert_eq!(decompress_chunk(&data, &pipeline, 200, 4).unwrap(), data);
+        // A top-level type N-Bit has no parameters for (e.g. an enum) carries only
+        // [nparms, need_not_compress, nelmts].
+        let pipeline = FilterPipeline {
+            version: 2,
+            filters: vec![one_filter(FILTER_NBIT, vec![3, 1, 50])],
+        };
+        assert_eq!(decompress_chunk(&data, &pipeline, 200, 4).unwrap(), data);
+    }
+
+    /// `tfilters.h5` `/all` chunk (0,0): shuffle, szip, deflate, fletcher32
+    /// and a pass-through N-Bit in one pipeline; values from h5py.
+    #[test]
+    #[cfg(feature = "szip")]
+    fn nbit_in_multi_filter_pipeline_matches_libhdf5() {
+        let raw = unhex(concat!(
+            "785e3bc1c0c030cb6517ff039e556de1576c0f300b152c771070641170640b99ba879f8167d5cce82f760ccc4285d71b",
+            "20c2afc226329f60d60ab5636a3fc1905499c3c0a1d0c4a1d05c6a13d7f88271aaf6725fe7170c86363f1858c0ca0104",
+            "bf1e95c75f3eeb",
+        ));
+        let want = unhex(concat!(
+            "00000000010000000200000003000000040000000a0000000b0000000c0000000d0000000e0000001400000015000000",
+            "1600000017000000180000001e0000001f00000020000000210000002200000028000000290000002a0000002b000000",
+            "2c00000032000000330000003400000035000000360000003c0000003d0000003e0000003f0000004000000046000000",
+            "4700000048000000490000004a00000050000000510000005200000053000000540000005a0000005b0000005c000000",
+            "5d0000005e000000",
+        ));
+        let pipeline = FilterPipeline {
+            version: 2,
+            filters: vec![
+                one_filter(FILTER_SHUFFLE, vec![4]),
+                one_filter(FILTER_SZIP, vec![141, 4, 32, 5]),
+                one_filter(FILTER_DEFLATE, vec![5]),
+                one_filter(FILTER_FLETCHER32, vec![]),
+                one_filter(FILTER_NBIT, vec![8, 1, 50, 1, 4, 0, 32, 0]),
+            ],
+        };
+        assert_eq!(decompress_chunk(&raw, &pipeline, 200, 4).unwrap(), want);
+    }
+
+    /// `h5repack_nested_8bit_enum_deflated.h5` `/tracks/1/trace` chunk 0: a
+    /// 376-byte compound whose `u1` enum member N-Bit stores whole as a
+    /// no-op type (class 4), then deflate. Was `UnsupportedFilter(5)`.
+    /// Expected bytes: libhdf5's decode in the file datatype.
+    #[test]
+    #[cfg(feature = "deflate")]
+    fn nbit_compound_with_enum_member_matches_libhdf5() {
+        #[rustfmt::skip]
+        let cd: Vec<u32> = vec![
+            251, 0, 1, 3, 376, 38,
+            0, 1, 4, 0, 32, 0,
+            8, 2, 96, 1, 8, 0, 64, 0,
+            104, 1, 8, 0, 64, 0, 112, 1, 8, 0, 64, 0, 120, 1, 8, 0, 64, 0,
+            128, 1, 8, 0, 64, 0, 136, 1, 8, 0, 64, 0, 144, 1, 8, 0, 64, 0,
+            152, 1, 8, 0, 64, 0,
+            160, 2, 24, 1, 8, 0, 64, 0, 184, 2, 24, 1, 8, 0, 64, 0,
+            208, 2, 16, 1, 8, 0, 64, 0, 224, 2, 32, 1, 8, 0, 64, 0,
+            256, 2, 16, 1, 4, 0, 32, 0, 272, 2, 16, 1, 4, 0, 32, 0,
+            288, 2, 32, 1, 8, 0, 64, 0, 320, 2, 16, 1, 8, 0, 64, 0,
+            336, 2, 8, 1, 4, 0, 32, 0,
+            344, 4, 1,
+            346, 1, 2, 0, 16, 0, 348, 1, 4, 0, 32, 0, 352, 1, 1, 0, 4, 0,
+            354, 1, 2, 0, 16, 0, 356, 1, 1, 0, 4, 0, 357, 1, 1, 0, 4, 0,
+            358, 1, 1, 0, 4, 0, 359, 1, 1, 0, 4, 0, 360, 1, 1, 0, 4, 0,
+            361, 1, 1, 0, 4, 0, 362, 1, 1, 0, 4, 0, 364, 1, 1, 0, 4, 0,
+            363, 1, 1, 0, 4, 0, 365, 1, 1, 0, 4, 0, 366, 1, 1, 0, 4, 0,
+            367, 1, 1, 0, 4, 0, 368, 1, 1, 0, 4, 0, 369, 1, 1, 0, 4, 0,
+            370, 1, 1, 0, 4, 0,
+        ];
+        assert_eq!(cd.len(), 251);
+        let raw = unhex(concat!(
+            "780163606078c930c880c3879573a60b2d7883eeac06a880835c47bda16cda7987a0c59ec9f74c4af6ff677e5df16445",
+            "adfd8493ce3ba592ddedab2a3bee87dd0faaff00d1808b66606006fa9d999b818125014837fd4703fba1fa71d150e720",
+            "512caa4073dc59212206500946060600604c37fc",
+        ));
+        let want = unhex(concat!(
+            "e90000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+            "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+            "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+            "0000000000000000eca012979ca9f040000000000000000000000000000000000000000000000080cf661d317f881e40",
+            "7434de6349a352407da8e478eb03ffbf47631ab943c9903f52df56df88797a3f000000000000f07f000000000000f07f",
+            "000000000000f07f000000000000f07fe90300000b0300006004000082030000ffffffffffffffffffffffffffffffff",
+            "000000000000f0bf000000000000f0bf000000000000f0bf000000000000f0bf00000000000000000000000000000000",
+            "25040000470300000500000000000000030000000000000000000000000000000100000000000000",
+        ));
+        let pipeline = FilterPipeline {
+            version: 2,
+            filters: vec![
+                one_filter(FILTER_NBIT, cd),
+                one_filter(FILTER_DEFLATE, vec![1]),
+            ],
+        };
+        assert_eq!(decompress_chunk(&raw, &pipeline, 376, 376).unwrap(), want);
+    }
+
     /// Chunk (0,0) of `/DS1` in the HDF Group's `h5ex_d_lz4.h5` example,
     /// written by libhdf5's registered LZ4 plugin with a 3-byte block size
     /// (so it has many blocks, some stored raw). Byte range from h5py's

From 95dcb0445452dcb936de616945866bacaebbf9f0 Mon Sep 17 00:00:00 2001
From: osobh <om.sobh@gmail.com>
Date: Fri, 25 Sep 2026 21:15:49 -0500
Subject: [PATCH 6/8] fix(format): scale-offset float decode with libhdf5's
 arithmetic

D-scale floats were rebuilt as `minval + code / 10^D` in f64 and then
rounded to f32 once, but libhdf5 (H5Z_scaleoffset_modify_3/4 with
`float`/`powf`) computes `(float)(int)code / powf(10, D) + min` in single
precision. The two differ by 1 ULP for some values: le_data.h5
/Scale_offset_float_data_{le,be} gave 1.6663332 (0x3fd54a69) where
libhdf5 gives 1.6663333 (0x3fd54a6a). Use f32 arithmetic for 4-byte
floats and `(double)(long)code / pow(10, D) + min` for 8-byte ones.

Test: scaleoffset_float_dscale_matches_libhdf5_bits (le_data.h5 float
LE/BE and double chunks, bit-exact against h5py); failed before.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 crates/clawhdf5-format/src/filters.rs | 54 ++++++++++++++++++++++++++-
 1 file changed, 53 insertions(+), 1 deletion(-)

diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs
index 54e56d4..94fa15f 100644
--- a/crates/clawhdf5-format/src/filters.rs
+++ b/crates/clawhdf5-format/src/filters.rs
@@ -246,8 +246,20 @@ fn scaleoffset_decompress(
                     fill_value
                 } else if is_escale {
                     minval + code as f64 * powi_f64(2.0, scale_factor)
+                } else if elem_size == 4 {
+                    // H5Z_scaleoffset_modify_3/4 for `float`: the code is
+                    // read as an `int` and everything is single precision,
+                    // `(float)code / powf(10, D) + min`. Doing it in f64 and
+                    // rounding once at the end is off by 1 ULP at times.
+                    let d = if scale_factor >= 0 {
+                        powi_f64(10.0, scale_factor) as f32
+                    } else {
+                        1.0 / powi_f64(10.0, -scale_factor) as f32
+                    };
+                    ((code as u32 as i32) as f32 / d + minval as f32) as f64
                 } else {
-                    minval + code as f64 / powi_f64(10.0, scale_factor)
+                    // ... and for `double`: `(double)(long)code / pow(10, D) + min`.
+                    (code as i64) as f64 / powi_f64(10.0, scale_factor) + minval
                 }
             })
             .collect();
@@ -1630,6 +1642,46 @@ mod tests {
         }
     }
 
+    /// `le_data.h5` scale-offset (D-scale, D = 3, fill -2.2) chunks, decoded
+    /// bit for bit as libhdf5 does: single-precision arithmetic for `float`
+    /// (we computed in f64 and rounded once, which was 1 ULP off for e.g.
+    /// 1.6663333: `694ad53f` instead of `6a4ad53f`), double for `double`.
+    #[test]
+    fn scaleoffset_float_dscale_matches_libhdf5_bits() {
+        let file: &[u8] = include_bytes!("../tests/fixtures/filters/le_data.h5");
+        let cd = |size: u32, order: u32, fill_lo: u32, fill_hi: u32| {
+            let mut cd = vec![0, 3, 12, 1, size, 0, order, 1, fill_lo, fill_hi];
+            cd.resize(20, 0);
+            cd
+        };
+        let f32_le = cd(4, 0, 0xC00C_CCCD, 0);
+        let f32_be = cd(4, 1, 0xC00C_CCCD, 0);
+        let f64_le = cd(8, 0, 2576980378, 3221330329);
+        #[rustfmt::skip]
+        let cases: [(usize, usize, &[u32], &str); 6] = [
+            (2816, 38, &f32_le, "abaaaa3ed2942a3fec0a803fd2942a3fec0a803fabaaaa3fec0a803fabaaaa3f694ad53fabaaaa3f694ad53f76050040"),
+            (2854, 38, &f32_le, "abaaaa3f6a4ad53f760500406a4ad53f7605004056551540760500405655154034a52a405655154034a52a4076054040"),
+            (712, 38, &f32_be, "3eaaaaab3f2a94d23f800aec3f2a94d23f800aec3faaaaab3f800aec3faaaaab3fd54a693faaaaab3fd54a6940000576"),
+            (750, 38, &f32_be, "3faaaaab3fd54a6a400005763fd54a6a40000576401555564000057640155556402aa53440155556402aa53440400576"),
+            (2050, 38, &f64_le, concat!(
+                "555555555555d53fb9d75c489a52e53fce3e7c865d01f03fb9d75c489a52e53fce3e7c865d01f03f555555555555f53f",
+                "ce3e7c865d01f03f555555555555f53fdc6b2e244da9fa3f555555555555f53fdc6b2e244da9fa3f671f3ec3ae000040")),
+            (2088, 38, &f64_le, concat!(
+                "555555555555f53fdc6b2e244da9fa3f671f3ec3ae000040dc6b2e244da9fa3f671f3ec3ae000040aaaaaaaaaaaa0240",
+                "671f3ec3ae000040aaaaaaaaaaaa0240ee351792a6540540aaaaaaaaaaaa0240ee351792a6540540671f3ec3ae000840")),
+        ];
+        for (off, len, cd, want) in cases {
+            let pipeline = FilterPipeline {
+                version: 2,
+                filters: vec![one_filter(FILTER_SCALEOFFSET, cd.to_vec())],
+            };
+            let want = unhex(want);
+            let got =
+                decompress_chunk(&file[off..off + len], &pipeline, want.len(), cd[4]).unwrap();
+            assert_eq!(got, want, "chunk at {off}");
+        }
+    }
+
     /// `le_data.h5` `/Nbit_float_data_{le,be}` chunk (0,0): a 20-bit float
     /// (offset 7) packed by N-Bit. The filter must reproduce libhdf5's
     /// decoded bytes in the *file* datatype (h5py `DatasetID.read` with the

From d99426be94216456eed2c3627e3c2d5744799b74 Mon Sep 17 00:00:00 2001
From: osobh <om.sobh@gmail.com>
Date: Fri, 25 Sep 2026 21:16:48 -0500
Subject: [PATCH 7/8] fix(format): allow Fletcher32 ahead of a compressor in
 the pipeline

libhdf5 applies filters in pipeline order, so with Fletcher32 before
deflate (h5repack_filters.h5 /dset_all: shuffle, fletcher32, deflate; or
h5py's set_fletcher32() then set_deflate()) the compressor holds the
chunk plus a 4-byte checksum. decompress_chunk bounded every stage by the
chunk size and rejected it: "deflate: output exceeds size limit". Bound
each stage by the chunk size plus 4 bytes per Fletcher32 that precedes
it in the pipeline.

Test: fletcher32_before_deflate_decodes (h5py-written chunk, and our own
shuffle + fletcher32 + deflate round trip); failed before.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 crates/clawhdf5-format/src/filters.rs | 49 ++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs
index 94fa15f..5d9fded 100644
--- a/crates/clawhdf5-format/src/filters.rs
+++ b/crates/clawhdf5-format/src/filters.rs
@@ -28,7 +28,20 @@ pub fn decompress_chunk(
 ) -> Result<Vec<u8>, FormatError> {
     let mut data = compressed.to_vec();
 
-    for filter in pipeline.filters.iter().rev() {
+    for (i, filter) in pipeline.filters.iter().enumerate().rev() {
+        // Fletcher32 appends a 4-byte checksum on write, so a filter that
+        // follows it in the (forward) pipeline decodes to the chunk plus one
+        // checksum per earlier Fletcher32 — e.g. libhdf5's
+        // shuffle + fletcher32 + deflate (h5repack_filters.h5 `/dset_all`).
+        let chunk_size = if chunk_size == 0 {
+            0
+        } else {
+            let checksums = pipeline.filters[..i]
+                .iter()
+                .filter(|f| f.filter_id == FILTER_FLETCHER32)
+                .count();
+            chunk_size.saturating_add(4 * checksums)
+        };
         data = match filter.filter_id {
             FILTER_SHUFFLE => shuffle_decompress(&data, element_size as usize)?,
             // `chunk_size` is the expected decompressed size (shuffle/fletcher32
@@ -1642,6 +1655,40 @@ mod tests {
         }
     }
 
+    /// Fletcher32 before a compressor (libhdf5 applies filters in pipeline
+    /// order, so the compressor sees chunk + checksum): deflate's output is 4
+    /// bytes over the chunk size, which we rejected as "deflate: output
+    /// exceeds size limit". Chunks from h5py/libhdf5, values from h5py.
+    #[test]
+    #[cfg(feature = "deflate")]
+    fn fletcher32_before_deflate_decodes() {
+        // h5py: set_fletcher32(); set_deflate(4); i32 0..100, chunks of 10.
+        let pipeline = FilterPipeline {
+            version: 2,
+            filters: vec![
+                one_filter(FILTER_FLETCHER32, vec![]),
+                one_filter(FILTER_DEFLATE, vec![4]),
+            ],
+        };
+        let raw =
+            unhex("785e936360609007620520560462252056066215205605623520560762c6483e3d00234501f0");
+        let want: Vec<u8> = (30..40i32).flat_map(i32::to_le_bytes).collect();
+        assert_eq!(decompress_chunk(&raw, &pipeline, 40, 4).unwrap(), want);
+
+        // And our own writer's round trip through the same pipeline order.
+        let data: Vec<u8> = (0..400u32).map(|i| (i % 13) as u8).collect();
+        let pipeline = FilterPipeline {
+            version: 2,
+            filters: vec![
+                one_filter(FILTER_SHUFFLE, vec![4]),
+                one_filter(FILTER_FLETCHER32, vec![]),
+                one_filter(FILTER_DEFLATE, vec![9]),
+            ],
+        };
+        let c = compress_chunk(&data, &pipeline, 4).unwrap();
+        assert_eq!(decompress_chunk(&c, &pipeline, 400, 4).unwrap(), data);
+    }
+
     /// `le_data.h5` scale-offset (D-scale, D = 3, fill -2.2) chunks, decoded
     /// bit for bit as libhdf5 does: single-precision arithmetic for `float`
     /// (we computed in f64 and rounded once, which was 1 ULP off for e.g.

From 6db13c60b857be3e236849cc19c996463e19b1d1 Mon Sep 17 00:00:00 2001
From: osobh <om.sobh@gmail.com>
Date: Fri, 25 Sep 2026 21:17:08 -0500
Subject: [PATCH 8/8] docs: changelog for the filter interop fixes

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 CHANGELOG.md | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a1ece78..cbe22bb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -179,6 +179,31 @@
   `float16` rounding matches numpy's bit for bit on 4 020 probe values,
   including ties, subnormals and the overflow boundary), and an agent store —
   `f32` and `float16` — opened by h5py with every dataset decoded.
+- `clawhdf5-format` filters, checked against libhdf5 + hdf5plugin:
+  - **LZ4 (32004) now uses the registered HDF5 LZ4 format** (8-byte BE size,
+    4-byte BE block size, BE-length-prefixed blocks). Our old framing (4-byte
+    LE size + one block) was readable only by clawhdf5, and we could not read
+    libhdf5's (`h5ex_d_lz4.h5`). Old clawhdf5 LZ4 chunks still read; they are
+    told apart unambiguously (a registered chunk starts with four zero bytes).
+  - **Zstd (32015) frames now record the content size**, which libhdf5's zstd
+    plugin needs; h5py could not read our zstd datasets.
+  - **Pcodec moved from filter ID 32023 to 480.** 32023 is registered to
+    Granular BitRound, whose decode is a pass-through — libhdf5 with that
+    plugin would have returned compressed bytes as data. Pcodec has no
+    registered ID; 480 is in the registry's private range (256–511) and only
+    clawhdf5 can read it. Chunks written under 32023 with the filter name
+    `pcodec` (clawhdf5 ≤ 2.7.0) still read.
+  - **SZIP decode matches libhdf5.** It returned garbage or zeros with no
+    error for libhdf5-written files (the 4-byte size prefix, 32/64-bit
+    byte-plane interleaving, reference interval, scanline padding and byte
+    order were all handled wrongly) and rejected 64-bit data.
+  - N-Bit honours libhdf5's "need not compress" flag (multi-filter pipelines
+    such as `tfilters.h5` failed) and reads enum/no-op members.
+  - Scale-offset `float` decode uses libhdf5's single-precision arithmetic
+    (was 1 ULP off for some values).
+  - A pipeline with Fletcher32 ahead of the compressor (h5py
+    `set_fletcher32()` then `set_deflate()`) no longer fails with "deflate:
+    output exceeds size limit".
 
 ### Storage
 - `clawhdf5-format`: **half-precision datasets.**