CHANGELOG.md

@@ -236,6 +236,12 @@
   - A pipeline with Fletcher32 ahead of the compressor (h5py
     `set_fletcher32()` then `set_deflate()`) no longer fails with "deflate:
     output exceeds size limit".
+- `clawhdf5-format`: **HDF5 1.4/1.6-era files are readable.** Data Layout
+  message versions 1 and 2 (compact, contiguous, and chunked through the
+  version-1 B-tree) failed with `InvalidLayoutVersion` — 84 of the 686 files in
+  the 2026-09-25 audit sweep, 205 datasets. They now read as libhdf5 does;
+  checked byte for byte against h5py on HDF5's own test files
+  (`tests/legacy_format_interop.rs`).
 
 ### Storage
 - `clawhdf5-format`: **half-precision datasets.**

crates/clawhdf5-format/src/data_layout.rs

@@ -1,7 +1,7 @@
 //! HDF5 Data Layout message parsing (message type 0x0008).
 
 #[cfg(not(feature = "std"))]
-use alloc::{string::String, vec::Vec};
+use alloc::{format, string::String, vec::Vec};
 
 #[cfg(feature = "std")]
 use std::string::String;
@@ -45,7 +45,9 @@ pub enum DataLayout {
         chunk_dimensions: Vec<u32>,
         /// B-tree address, or `None` if undefined.
         btree_address: Option<u64>,
-        /// Layout version (3 or 4).
+        /// Layout version (3 or 4). Version 1/2 messages (HDF5 1.4/1.6-era)
+        /// use the same version-1 B-tree chunk index as version 3 and are
+        /// reported as 3.
         version: u8,
         /// Chunk index type (v4 only).
         chunk_index_type: Option<u8>,
@@ -261,6 +263,7 @@ impl DataLayout {
         let layout_class = data[1];
 
         match version {
+            1 | 2 => Self::parse_v1_v2(data, offset_size),
             3 => Self::parse_v3(data, layout_class, offset_size, length_size),
             // v5 (emitted by HDF5 1.14+/2.0 with `libver=latest`) uses the same
             // message structure as v4 — only the version number was bumped.
@@ -269,6 +272,87 @@ impl DataLayout {
         }
     }
 
+    /// Layout message versions 1 and 2 (HDF5 before 1.6.3):
+    ///
+    /// ```text
+    /// version(1) · dimensionality(1) · layout class(1) · reserved(5)
+    /// · address(offset_size)      — contiguous and chunked only
+    /// · dimension sizes(4 × dimensionality)
+    /// · compact data size(4) · compact raw data — compact only
+    /// ```
+    ///
+    /// The dimension sizes are the dataset's (contiguous/compact) or the
+    /// chunk's (chunked) extent plus a trailing element-size dimension, as in
+    /// version 3's chunked form. libhdf5 ignores them for contiguous storage
+    /// and sizes the data from the dataspace; the product of the stored
+    /// dimensions is that same size, and a disagreement (a dimension that was
+    /// truncated to 32 bits) is caught by the reader's size check rather than
+    /// returning wrong data.
+    fn parse_v1_v2(data: &[u8], offset_size: u8) -> Result<DataLayout, FormatError> {
+        ensure_len(data, 0, 8)?;
+        let dimensionality = data[1] as usize;
+        let layout_class = data[2];
+        // H5O_LAYOUT_NDIMS: 32 dataspace dimensions + the element-size one.
+        if dimensionality > 33 {
+            return Err(FormatError::Overflow(format!(
+                "data layout dimensionality {dimensionality} exceeds 33"
+            )));
+        }
+        let mut p = 8;
+        let os = offset_size as usize;
+        let address = match layout_class {
+            1 | 2 => {
+                ensure_len(data, p, os)?;
+                let a = if is_undefined(data, p, offset_size) {
+                    None
+                } else {
+                    Some(read_offset(data, p, offset_size)?)
+                };
+                p += os;
+                a
+            }
+            0 => None,
+            _ => return Err(FormatError::InvalidLayoutClass(layout_class)),
+        };
+        ensure_len(data, p, dimensionality * 4)?;
+        let dims: Vec<u32> = data[p..p + dimensionality * 4]
+            .as_chunks::<4>()
+            .0
+            .iter()
+            .map(|c| u32::from_le_bytes(*c))
+            .collect();
+        p += dimensionality * 4;
+        match layout_class {
+            0 => {
+                ensure_len(data, p, 4)?;
+                let size =
+                    u32::from_le_bytes([data[p], data[p + 1], data[p + 2], data[p + 3]]) as usize;
+                ensure_len(data, p + 4, size)?;
+                Ok(DataLayout::Compact {
+                    data: data[p + 4..p + 4 + size].to_vec(),
+                })
+            }
+            1 => {
+                let size = dims
+                    .iter()
+                    .try_fold(1u64, |acc, &d| acc.checked_mul(d as u64))
+                    .ok_or_else(|| {
+                        FormatError::Overflow(format!("contiguous layout size {dims:?}"))
+                    })?;
+                Ok(DataLayout::Contiguous { address, size })
+            }
+            _ => Ok(DataLayout::Chunked {
+                chunk_dimensions: dims,
+                btree_address: address,
+                version: 3,
+                chunk_index_type: None,
+                single_chunk_filtered_size: None,
+                single_chunk_filter_mask: None,
+                dont_filter_partial_edge_chunks: false,
+            }),
+        }
+    }
+
     fn parse_v3(
         data: &[u8],
         layout_class: u8,
@@ -546,6 +630,108 @@ impl DataLayout {
 mod tests {
     use super::*;
 
+    /// Version 1/2 header: version, dimensionality, class, reserved(5).
+    fn v1v2_header(version: u8, ndims: u8, class: u8) -> Vec<u8> {
+        vec![version, ndims, class, 0, 0, 0, 0, 0]
+    }
+
+    #[test]
+    fn v2_compact() {
+        let mut buf = v1v2_header(2, 2, 0);
+        // dims (3 elements of 2 bytes) — no address for compact
+        buf.extend_from_slice(&3u32.to_le_bytes());
+        buf.extend_from_slice(&2u32.to_le_bytes());
+        buf.extend_from_slice(&6u32.to_le_bytes()); // compact size (u32 in v1/v2)
+        buf.extend_from_slice(&[1, 0, 2, 0, 3, 0]);
+        assert_eq!(
+            DataLayout::parse(&buf, 8, 8).unwrap(),
+            DataLayout::Compact {
+                data: vec![1, 0, 2, 0, 3, 0]
+            }
+        );
+    }
+
+    #[test]
+    fn v1_contiguous_size_from_dimensions() {
+        let mut buf = v1v2_header(1, 3, 1);
+        buf.extend_from_slice(&0x800u32.to_le_bytes()); // 4-byte address
+        for d in [10u32, 20, 4] {
+            buf.extend_from_slice(&d.to_le_bytes());
+        }
+        assert_eq!(
+            DataLayout::parse(&buf, 4, 4).unwrap(),
+            DataLayout::Contiguous {
+                address: Some(0x800),
+                size: 800,
+            }
+        );
+    }
+
+    #[test]
+    fn v1_contiguous_undefined_address() {
+        let mut buf = v1v2_header(1, 2, 1);
+        buf.extend_from_slice(&[0xFF; 8]);
+        buf.extend_from_slice(&5u32.to_le_bytes());
+        buf.extend_from_slice(&8u32.to_le_bytes());
+        assert_eq!(
+            DataLayout::parse(&buf, 8, 8).unwrap(),
+            DataLayout::Contiguous {
+                address: None,
+                size: 40,
+            }
+        );
+    }
+
+    #[test]
+    fn v1_chunked_maps_to_btree_v1_index() {
+        let mut buf = v1v2_header(1, 3, 2);
+        buf.extend_from_slice(&0x1234u64.to_le_bytes());
+        for d in [50u32, 50, 4] {
+            buf.extend_from_slice(&d.to_le_bytes());
+        }
+        assert_eq!(
+            DataLayout::parse(&buf, 8, 8).unwrap(),
+            DataLayout::Chunked {
+                chunk_dimensions: vec![50, 50, 4],
+                btree_address: Some(0x1234),
+                version: 3,
+                chunk_index_type: None,
+                single_chunk_filtered_size: None,
+                single_chunk_filter_mask: None,
+                dont_filter_partial_edge_chunks: false,
+            }
+        );
+    }
+
+    #[test]
+    fn v1v2_rejects_bad_class_dimensionality_and_truncation() {
+        assert_eq!(
+            DataLayout::parse(&v1v2_header(1, 1, 3), 8, 8).unwrap_err(),
+            FormatError::InvalidLayoutClass(3)
+        );
+        assert!(matches!(
+            DataLayout::parse(&v1v2_header(2, 34, 1), 8, 8).unwrap_err(),
+            FormatError::Overflow(_)
+        ));
+        // Chunked, dims cut short.
+        let mut buf = v1v2_header(1, 2, 2);
+        buf.extend_from_slice(&0x10u64.to_le_bytes());
+        buf.extend_from_slice(&7u32.to_le_bytes());
+        assert!(matches!(
+            DataLayout::parse(&buf, 8, 8).unwrap_err(),
+            FormatError::UnexpectedEof { .. }
+        ));
+        // Compact, raw data shorter than its declared size.
+        let mut buf = v1v2_header(2, 1, 0);
+        buf.extend_from_slice(&4u32.to_le_bytes());
+        buf.extend_from_slice(&100u32.to_le_bytes());
+        buf.extend_from_slice(&[0; 4]);
+        assert!(matches!(
+            DataLayout::parse(&buf, 8, 8).unwrap_err(),
+            FormatError::UnexpectedEof { .. }
+        ));
+    }
+
     #[test]
     fn v3_compact() {
         let mut buf = vec![3u8, 0]; // version=3, class=0 (compact)

crates/clawhdf5-format/tests/fixtures/legacy/README.md

@@ -0,0 +1,11 @@
+# Legacy (HDF5 1.4/1.6-era) fixtures
+
+Unmodified copies of the HDF Group's own test files from
+https://github.com/HDFGroup/hdf5 at a3cf1ea82cc7a66e50029a688121e1b105a7ce88
+(BSD-style license, see that repository's `LICENSE`). Current libraries cannot
+write these structures, so they are kept as files.
+
+| File | Upstream path | Exercises |
+|---|---|---|
+| `deflate.h5` | `test/testfiles/deflate.h5` | Data Layout message v1, chunked + deflate (v1 B-tree index) |
+| `h5ex_g_iterate.h5` | `HDF5Examples/C/H5G/h5ex_g_iterate.h5` | Data Layout message v2, contiguous; an unallocated dataset |

crates/clawhdf5/tests/legacy_format_interop.rs

@@ -0,0 +1,121 @@
+//! Files written by HDF5 1.4/1.6-era libraries: Data Layout message versions
+//! 1 and 2, compound datatype version 1 array members, and version-1 shared
+//! message references. The fixtures are HDF5's own test files (see
+//! `clawhdf5-format/tests/fixtures/legacy/README.md`).
+//!
+//! The expected values were read with h5py 3.16 / HDF5 2.0; the interop test
+//! re-checks every dataset byte for byte against h5py, and is skipped when
+//! python3 with h5py is unavailable unless `CLAWHDF5_REQUIRE_INTEROP=1`.
+
+use std::process::Command;
+
+use clawhdf5::File;
+use clawhdf5_format::selection::Selection;
+
+const FIXTURES: &str = concat!(
+    env!("CARGO_MANIFEST_DIR"),
+    "/../clawhdf5-format/tests/fixtures/legacy"
+);
+
+fn open(name: &str) -> File {
+    File::open(format!("{FIXTURES}/{name}")).unwrap()
+}
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn interop_required() -> bool {
+    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
+}
+
+fn python_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py, numpy"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+/// Layout v1, chunked (50x50 chunks of a 100x200 dataset), deflate: every
+/// read path goes through the version-1 B-tree chunk index.
+#[test]
+fn layout_v1_chunked_deflate() {
+    let file = open("deflate.h5");
+    let ds = file.dataset("Dataset1").unwrap();
+    assert_eq!(ds.shape().unwrap(), [100, 200]);
+    let expected: Vec<i32> = (0..100).flat_map(|_| (0..200).map(|j| j % 5)).collect();
+    assert_eq!(ds.read_i32().unwrap(), expected);
+
+    // A hyperslab that straddles four chunks.
+    let slab = Selection::Hyperslab {
+        start: vec![48, 48],
+        stride: vec![1, 1],
+        count: vec![4, 4],
+        block: vec![1, 1],
+    };
+    let raw = ds.read_selection(&slab).unwrap();
+    let got: Vec<i32> = raw
+        .as_chunks::<4>()
+        .0
+        .iter()
+        .map(|b| i32::from_le_bytes(*b))
+        .collect();
+    assert_eq!(got, [3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1]);
+}
+
+/// Layout v2, contiguous: one dataset with storage, one never written (reads
+/// as its fill value, 0).
+#[test]
+fn layout_v2_contiguous() {
+    let file = open("h5ex_g_iterate.h5");
+    assert_eq!(file.dataset("G1/DS2").unwrap().read_i32().unwrap(), [1]);
+    assert_eq!(file.dataset("DS1").unwrap().read_i32().unwrap(), [0]);
+}
+
+/// Every dataset in every fixture, byte for byte against h5py.
+#[test]
+fn legacy_fixtures_match_h5py() {
+    if !python_available() {
+        assert!(
+            !interop_required(),
+            "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
+        );
+        eprintln!("SKIP: python3 with h5py not available");
+        return;
+    }
+    for (name, datasets) in [
+        ("deflate.h5", &["Dataset1"][..]),
+        ("h5ex_g_iterate.h5", &["DS1", "G1/DS2"][..]),
+    ] {
+        let path = format!("{FIXTURES}/{name}");
+        let script = format!(
+            r#"
+import h5py, numpy as np
+f = h5py.File({path:?}, "r")
+for n in {datasets:?}:
+    print(n, np.ascontiguousarray(f[n][()]).tobytes().hex())
+"#
+        );
+        let out = Command::new(python())
+            .args(["-c", &script])
+            .output()
+            .unwrap();
+        assert!(
+            out.status.success(),
+            "h5py: {}",
+            String::from_utf8_lossy(&out.stderr)
+        );
+        let file = File::open(&path).unwrap();
+        for line in String::from_utf8(out.stdout).unwrap().lines() {
+            let (ds, hex) = line.split_once(' ').unwrap();
+            let ours = file
+                .dataset(ds)
+                .unwrap()
+                .read_selection(&Selection::All)
+                .unwrap();
+            let ours: String = ours.iter().map(|b| format!("{b:02x}")).collect();
+            assert_eq!(ours, hex, "{name}:{ds}");
+        }
+    }
+}

docs/known-issues.md

@@ -68,6 +68,8 @@ the VDS item, which is marked.
 
 - **Layout message versions 1 and 2** (HDF5 1.6-era files): 84 of the 686
   sweep files, `InvalidLayoutVersion`. This is the largest single gap.
+  **Fixed 2026-09-25:** versions 1 and 2 are parsed (compact, contiguous,
+  chunked via the v1 B-tree).
 - **Virtual datasets:**
   - **Wrong data:** unmapped regions read as 0 instead of the fill value.
   - `%b` printf-style source names are not expanded.