fix(format): index every Extensible Array chunk, not just the first 244

The Extensible Array writer only filled the index block's 4 inline
elements and the 6 data blocks it addresses directly (240 elements); its
super block addresses were always undefined. Chunks from index 244 on were
written to the file but never indexed, so they read back as fill values in
our reader and in libhdf5, without an error.

The writer now lays out data blocks and super blocks for any element
count as H5EA__hdr_init sizes them, pages data blocks larger than 1024
elements (page-init bits in the owning super block), leaves blocks with no
defined element unallocated, and records real header statistics
(max_idx_set is one past the highest defined index).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/src/chunked_write.rs

@@ -503,7 +503,7 @@ fn serialize_v4_fixed_array(
 /// default, `H5D_FARRAY_MAX_DBLK_PAGE_NELMTS_BITS`).
 const FA_PAGE_BITS: u8 = 10;
 
-fn push_addr(buf: &mut Vec<u8>, addr: u64, offset_size: u8) {
+pub(crate) fn push_addr(buf: &mut Vec<u8>, addr: u64, offset_size: u8) {
     match offset_size {
         4 => buf.extend_from_slice(&(addr as u32).to_le_bytes()),
         _ => buf.extend_from_slice(&addr.to_le_bytes()),
@@ -734,8 +734,9 @@ pub fn build_chunked_data_from_precompressed(
 
     let layout_message = if use_extensible {
         let ea_address = base_address + data_buf.len() as u64;
+        let slots: Vec<Option<WrittenChunk>> = written_chunks.iter().cloned().map(Some).collect();
         let ea_bytes = ea_writer::build_extensible_array_at(
-            &written_chunks,
+            &slots,
             offset_size,
             length_size,
             pre.has_filters,
@@ -1463,7 +1464,8 @@ mod tests {
                 filter_mask: 0,
             },
         ];
-        let ea = ea_writer::build_extensible_array_at(&chunks, 8, 8, false, 0x2000);
+        let slots: Vec<_> = chunks.into_iter().map(Some).collect();
+        let ea = ea_writer::build_extensible_array_at(&slots, 8, 8, false, 0x2000);
         assert_eq!(&ea[0..4], b"EAHD");
         // Find EAIB after EAHD: 12 fixed + 6*8 stats + 8 addr + 4 checksum = 72
         let aehd_size = 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * 8 + 8 + 4;

crates/clawhdf5-format/src/ea_writer.rs

@@ -7,7 +7,7 @@ extern crate alloc;
 use alloc::{vec, vec::Vec};
 
 use crate::checksum::jenkins_lookup3;
-use crate::chunked_write::WrittenChunk;
+use crate::chunked_write::{WrittenChunk, filtered_chunk_size_len, push_addr, push_index_element};
 
 /// Serialize a v4 Extensible Array layout message.
 pub(crate) fn serialize_v4_extensible_array(
@@ -58,11 +58,11 @@ pub(crate) fn serialize_v4_extensible_array(
     buf.push(4);
 
     // EA creation parameters (must match AEHD and HDF5 C library defaults)
-    buf.push(32); // max_nelmts_bits
+    buf.push(MAX_NELMTS_BITS);
-    buf.push(4); // idx_blk_elmts
+    buf.push(IDX_BLK_ELMTS);
-    buf.push(4); // super_blk_min_data_ptrs
+    buf.push(SUP_BLK_MIN_DATA_PTRS);
-    buf.push(16); // data_blk_min_elmts
+    buf.push(DATA_BLK_MIN_ELMTS);
-    buf.push(10); // max_dblk_page_nelmts_bits
+    buf.push(MAX_DBLK_PAGE_NELMTS_BITS);
 
     // EA header address
     match offset_size {
@@ -74,304 +74,281 @@ pub(crate) fn serialize_v4_extensible_array(
     buf
 }
 
+// EA creation parameters — the HDF5 library's defaults for chunk indexes
+// (`H5D_EARRAY_*`); the layout message above and the header must agree.
+const MAX_NELMTS_BITS: u8 = 32;
+const IDX_BLK_ELMTS: u8 = 4;
+const SUP_BLK_MIN_DATA_PTRS: u8 = 4;
+const DATA_BLK_MIN_ELMTS: u8 = 16;
+const MAX_DBLK_PAGE_NELMTS_BITS: u8 = 10;
+
+/// One data block of the array: its first element (relative to the end of
+/// the index block's own elements), element count, and address when it is
+/// allocated.
+struct DataBlock {
+    start: usize,
+    nelmts: usize,
+    addr: Option<u64>,
+}
+
 /// Build a complete Extensible Array at a known absolute address.
 ///
-/// For simplicity, we put all elements inline in the index block when the
+/// `slots[i]` is the element at linear index `i` (see `chunk_grid`); `None`
-/// number of chunks is small (up to idx_blk_elmts), otherwise use inline +
+/// marks an unallocated chunk. The first `IDX_BLK_ELMTS` elements live in
-/// direct data blocks.
+/// the index block, the rest in data blocks grouped by super block level
+/// exactly as `H5EA__hdr_init` sizes them: level `u` has `2^(u/2)` data
+/// blocks of `DATA_BLK_MIN_ELMTS * 2^ceil(u/2)` elements. The data blocks of
+/// the first levels are addressed straight from the index block; later
+/// levels go through a super block (EASB). Data blocks larger than a page
+/// (`2^MAX_DBLK_PAGE_NELMTS_BITS` elements) are paged, with their page-init
+/// bits kept in the owning super block. Only blocks holding a defined element
+/// are allocated; the rest keep the undefined address, as in a file the
+/// library wrote.
 pub fn build_extensible_array_at(
-    chunks: &[WrittenChunk],
+    slots: &[Option<WrittenChunk>],
     offset_size: u8,
     length_size: u8,
     has_filters: bool,
     ea_base_address: u64,
 ) -> Vec<u8> {
     let os = offset_size as usize;
-    let num_elements = chunks.len();
+    let chunk_size_bytes = has_filters.then(|| filtered_chunk_size_len(slots));
-
+    let elem_size = os + chunk_size_bytes.map_or(0, |n| n + 4);
-    // Compute element encoding size (same logic as Fixed Array)
-    let chunk_size_bytes: usize = if has_filters {
-        let max_raw = chunks.iter().map(|c| c.raw_size).max().unwrap_or(1);
-        let log2_val = if max_raw <= 1 {
-            0
-        } else {
-            63 - max_raw.leading_zeros()
-        };
-        let len = 1 + ((log2_val + 8) / 8) as usize;
-        len.min(8)
-    } else {
-        0
-    };
-
-    let elem_size = if has_filters {
-        os + chunk_size_bytes + 4
-    } else {
-        os
-    };
-
     let client_id: u8 = if has_filters { 1 } else { 0 };
+    let arr_off_size = (MAX_NELMTS_BITS as usize).div_ceil(8);
+    let page_nelmts = 1usize << MAX_DBLK_PAGE_NELMTS_BITS;
+    let idx_blk = IDX_BLK_ELMTS as usize;
 
-    // EA creation parameters — must match HDF5 C library defaults exactly
+    // Elements past the last defined one are never realised
-    let max_nelmts_bits: u8 = 32;
+    // (`max_idx_set` is one past the highest index ever set).
-    let idx_blk_elmts: u8 = 4;
+    let max_idx_set = slots.iter().rposition(Option::is_some).map_or(0, |i| i + 1);
-    let min_dblk_nelmts: u8 = 16;
+    let slots = &slots[..max_idx_set];
-    let super_blk_min_nelmts: u8 = 4;
+    let defined_in = |start: usize, n: usize| -> bool {
-    let max_dblk_nelmts_bits: u8 = 10;
+        let lo = idx_blk.saturating_add(start).min(slots.len());
+        let hi = idx_blk
+            .saturating_add(start)
+            .saturating_add(n)
+            .min(slots.len());
+        slots[lo..hi].iter().any(Option::is_some)
+    };
 
-    // EAHD size: fixed(12) + 6 stats(6*length_size) + addr(offset_size) + checksum(4)
+    // Super block levels: (ndblks, dblk_nelmts, first element).
+    let log2_dmin = (DATA_BLK_MIN_ELMTS as u32).trailing_zeros() as usize;
+    let nsblks = 1 + MAX_NELMTS_BITS as usize - log2_dmin;
+    let ndblk_addrs = 2 * (SUP_BLK_MIN_DATA_PTRS as usize - 1);
+    let mut levels: Vec<(usize, usize, usize)> = Vec::with_capacity(nsblks);
+    let mut start = 0usize;
+    for u in 0..nsblks {
+        let ndblks = 1usize << (u / 2);
+        let nelmts = (DATA_BLK_MIN_ELMTS as usize) << u.div_ceil(2);
+        levels.push((ndblks, nelmts, start));
+        // Saturate: on 32-bit targets the last levels only need to compare
+        // as "beyond the end".
+        start = start.saturating_add(ndblks.saturating_mul(nelmts));
+    }
+    // Levels whose data blocks the index block addresses directly.
+    let mut direct_levels = 0;
+    let mut n = 0;
+    while n < ndblk_addrs {
+        n += levels[direct_levels].0;
+        direct_levels += 1;
+    }
+    let nsblk_addrs = nsblks - direct_levels;
+
+    let dblk_size = |nelmts: usize| -> usize {
+        let prefix = 4 + 1 + 1 + os + arr_off_size + 4;
+        if nelmts > page_nelmts {
+            prefix + (nelmts / page_nelmts) * (page_nelmts * elem_size + 4)
+        } else {
+            prefix + nelmts * elem_size
+        }
+    };
+    let sblk_bitmap_len = |ndblks: usize, nelmts: usize| -> usize {
+        if nelmts > page_nelmts {
+            ndblks * (nelmts / page_nelmts).div_ceil(8)
+        } else {
+            0
+        }
+    };
+
+    // Plan addresses: header, index block, the direct data blocks, then each
+    // allocated super block followed by its allocated data blocks.
     let aehd_size = 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + os + 4;
     let aeib_address = ea_base_address + aehd_size as u64;
+    let aeib_size = 4 + 1 + 1 + os + idx_blk * elem_size + ndblk_addrs * os + nsblk_addrs * os + 4;
+    let mut cursor = aeib_address + aeib_size as u64;
 
-    // Determine how many elements go inline vs data blocks
+    let mut ndata_blks = 0u64;
-    let n_inline = (idx_blk_elmts as usize).min(num_elements);
+    let mut data_blk_size = 0u64;
-    let remaining_after_inline = num_elements.saturating_sub(n_inline);
+    let mut nsuper_blks = 0u64;
+    let mut super_blk_size = 0u64;
+    let mut realized = idx_blk as u64;
 
-    // Compute super block layout per HDF5 spec
+    let mut plan_dblk = |cursor: &mut u64, start: usize, nelmts: usize| -> DataBlock {
-    let sblk_min = super_blk_min_nelmts as usize;
+        let addr = defined_in(start, nelmts).then(|| {
-    let log2_dblk_min = if min_dblk_nelmts <= 1 {
+            let a = *cursor;
-        0
+            let size = dblk_size(nelmts) as u64;
-    } else {
+            *cursor += size;
-        (min_dblk_nelmts as u32).trailing_zeros() as usize
+            ndata_blks += 1;
+            data_blk_size += size;
+            realized += nelmts as u64;
+            a
+        });
+        DataBlock {
+            start,
+            nelmts,
+            addr,
+        }
     };
-    let nsblks = (max_nelmts_bits as usize).saturating_sub(log2_dblk_min) + 1;
 
-    // Direct data block addresses (from super blocks 0..sblk_min-1)
+    let mut direct: Vec<DataBlock> = Vec::with_capacity(ndblk_addrs);
-    let mut dblk_sizes: Vec<usize> = Vec::new();
+    for &(ndblks, nelmts, first) in &levels[..direct_levels] {
-    for sblk_idx in 0..sblk_min.min(nsblks) {
+        for k in 0..ndblks {
-        let ndblks = 1usize << (sblk_idx / 2);
+            direct.push(plan_dblk(&mut cursor, first + k * nelmts, nelmts));
-        let dblk_nelmts = (min_dblk_nelmts as usize) * (1 << sblk_idx.div_ceil(2));
-        for _ in 0..ndblks {
-            dblk_sizes.push(dblk_nelmts);
         }
     }
-    let n_direct_dblks = dblk_sizes.len();
+    // (super block address, level, its data blocks)
-
+    let mut supers: Vec<(Option<u64>, usize, Vec<DataBlock>)> = Vec::with_capacity(nsblk_addrs);
-    // Super block addresses (for super blocks sblk_min..nsblks-1)
+    for (u, &(ndblks, nelmts, first)) in levels.iter().enumerate().skip(direct_levels) {
-    let n_sblk_addrs = nsblks.saturating_sub(sblk_min);
+        if !defined_in(first, ndblks.saturating_mul(nelmts)) {
-
+            supers.push((None, u, Vec::new()));
-    // EAIB size
+            continue;
-    let aeib_size = 4
-        + 1
-        + 1
-        + os
-        + idx_blk_elmts as usize * elem_size
-        + n_direct_dblks * os
-        + n_sblk_addrs * os
-        + 4;
-
-    // Build AEHD
-    let mut aehd = Vec::with_capacity(aehd_size);
-    aehd.extend_from_slice(b"EAHD");
-    aehd.push(0); // version
-    aehd.push(client_id);
-    aehd.push(elem_size as u8);
-    aehd.push(max_nelmts_bits);
-    aehd.push(idx_blk_elmts);
-    aehd.push(min_dblk_nelmts);
-    aehd.push(super_blk_min_nelmts);
-    aehd.push(max_dblk_nelmts_bits);
-
-    // Count data blocks that will have chunks
-    let n_active_dblks: u64 = if remaining_after_inline > 0 {
-        let mut count = 0u64;
-        let mut ci = n_inline;
-        for &sz in &dblk_sizes {
-            if ci < num_elements {
-                count += 1;
-                ci += sz;
-            }
         }
-        count
+        let sb_size =
-    } else {
+            4 + 1 + 1 + os + arr_off_size + sblk_bitmap_len(ndblks, nelmts) + ndblks * os + 4;
-        0
+        let sb_addr = cursor;
-    };
+        cursor += sb_size as u64;
-    let blk_off_size = (max_nelmts_bits as usize).div_ceil(8);
+        nsuper_blks += 1;
-    let aedb_header_overhead = 4 + 1 + 1 + os + blk_off_size + 4;
+        super_blk_size += sb_size as u64;
-    let data_blk_total_size: u64 = if remaining_after_inline > 0 {
+        let dblks = (0..ndblks)
-        let mut total = 0u64;
+            .map(|k| plan_dblk(&mut cursor, first + k * nelmts, nelmts))
-        let mut ci = n_inline;
+            .collect();
-        for &sz in &dblk_sizes {
+        supers.push((Some(sb_addr), u, dblks));
-            if ci < num_elements {
+    }
-                total += (aedb_header_overhead + sz * elem_size) as u64;
-                ci += sz;
-            }
-        }
-        total
-    } else {
-        0
-    };
-    let max_idx_set: u64 = if remaining_after_inline > 0 {
-        let mut max_set = idx_blk_elmts as u64;
-        let mut ci = n_inline;
-        for &sz in &dblk_sizes {
-            if ci < num_elements {
-                max_set += sz as u64;
-                ci += sz;
-            }
-        }
-        max_set
-    } else {
-        idx_blk_elmts as u64
-    };
 
+    let slot = |i: usize| slots.get(i).and_then(Option::as_ref);
     let write_length = |buf: &mut Vec<u8>, val: u64| match length_size {
         4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
         _ => buf.extend_from_slice(&val.to_le_bytes()),
     };
-    let write_addr = |buf: &mut Vec<u8>, val: u64| match offset_size {
+    let write_addr_opt = |buf: &mut Vec<u8>, addr: Option<u64>| match addr {
-        4 => buf.extend_from_slice(&(val as u32).to_le_bytes()),
+        Some(a) => push_addr(buf, a, offset_size),
-        _ => buf.extend_from_slice(&val.to_le_bytes()),
+        None => buf.extend(core::iter::repeat_n(0xFF, os)),
+    };
+    let block_prefix = |buf: &mut Vec<u8>, sig: &[u8; 4], block_off: usize| {
+        buf.extend_from_slice(sig);
+        buf.push(0); // version
+        buf.push(client_id);
+        push_addr(buf, ea_base_address, offset_size);
+        buf.extend_from_slice(&(block_off as u64).to_le_bytes()[..arr_off_size]);
+    };
+    // Serialise one data block (paged or not) onto `out`.
+    let write_dblk = |out: &mut Vec<u8>, db: &DataBlock| {
+        let at = out.len();
+        block_prefix(out, b"EADB", db.start);
+        let first = idx_blk + db.start;
+        if db.nelmts > page_nelmts {
+            // Paged: the prefix carries only its own checksum; each page
+            // follows with one of its own.
+            let sum = jenkins_lookup3(&out[at..]);
+            out.extend_from_slice(&sum.to_le_bytes());
+            for p in 0..db.nelmts / page_nelmts {
+                let page_at = out.len();
+                for e in 0..page_nelmts {
+                    let i = first + p * page_nelmts + e;
+                    push_index_element(out, slot(i), offset_size, chunk_size_bytes);
+                }
+                let sum = jenkins_lookup3(&out[page_at..]);
+                out.extend_from_slice(&sum.to_le_bytes());
+            }
+        } else {
+            for i in first..first + db.nelmts {
+                push_index_element(out, slot(i), offset_size, chunk_size_bytes);
+            }
+            let sum = jenkins_lookup3(&out[at..]);
+            out.extend_from_slice(&sum.to_le_bytes());
+        }
+        debug_assert_eq!(out.len() - at, dblk_size(db.nelmts));
     };
 
-    write_length(&mut aehd, 0);
+    // Header (EAHD). The six statistics are, in order: super blocks, their
-    write_length(&mut aehd, 0);
+    // bytes, data blocks, their bytes, max index set, elements realised.
-    write_length(&mut aehd, n_active_dblks);
+    let mut out = Vec::with_capacity((cursor - ea_base_address) as usize);
-    write_length(&mut aehd, data_blk_total_size);
+    out.extend_from_slice(b"EAHD");
-    write_length(&mut aehd, num_elements as u64);
+    out.push(0); // version
-    write_length(&mut aehd, max_idx_set);
+    out.push(client_id);
+    out.push(elem_size as u8);
+    out.push(MAX_NELMTS_BITS);
+    out.push(IDX_BLK_ELMTS);
+    out.push(DATA_BLK_MIN_ELMTS);
+    out.push(SUP_BLK_MIN_DATA_PTRS);
+    out.push(MAX_DBLK_PAGE_NELMTS_BITS);
+    write_length(&mut out, nsuper_blks);
+    write_length(&mut out, super_blk_size);
+    write_length(&mut out, ndata_blks);
+    write_length(&mut out, data_blk_size);
+    write_length(&mut out, max_idx_set as u64);
+    write_length(&mut out, realized);
+    push_addr(&mut out, aeib_address, offset_size);
+    let sum = jenkins_lookup3(&out);
+    out.extend_from_slice(&sum.to_le_bytes());
+    debug_assert_eq!(out.len(), aehd_size);
 
-    write_addr(&mut aehd, aeib_address);
+    // Index block (EAIB): inline elements, data block and super block
-
+    // addresses.
-    let aehd_checksum = jenkins_lookup3(&aehd);
+    let ib_start = out.len();
-    aehd.extend_from_slice(&aehd_checksum.to_le_bytes());
+    out.extend_from_slice(b"EAIB");
-    debug_assert_eq!(aehd.len(), aehd_size);
+    out.push(0);
-
+    out.push(client_id);
-    // Build AEIB
+    push_addr(&mut out, ea_base_address, offset_size);
-    let mut aeib = Vec::with_capacity(aeib_size);
+    for i in 0..idx_blk {
-    aeib.extend_from_slice(b"EAIB");
+        push_index_element(&mut out, slot(i), offset_size, chunk_size_bytes);
-    aeib.push(0);
-    aeib.push(client_id);
-
-    match offset_size {
-        4 => aeib.extend_from_slice(&(ea_base_address as u32).to_le_bytes()),
-        8 => aeib.extend_from_slice(&ea_base_address.to_le_bytes()),
-        _ => aeib.extend_from_slice(&ea_base_address.to_le_bytes()),
     }
-
+    for db in &direct {
-    // Inline elements
+        write_addr_opt(&mut out, db.addr);
-    #[allow(clippy::needless_range_loop)]
-    for i in 0..idx_blk_elmts as usize {
-        if i < n_inline {
-            write_chunk_element(
-                &mut aeib,
-                &chunks[i],
-                offset_size,
-                has_filters,
-                chunk_size_bytes,
-            );
-        } else {
-            write_undefined_element(&mut aeib, offset_size, has_filters, chunk_size_bytes);
-        }
     }
+    for (sb_addr, _, _) in &supers {
+        write_addr_opt(&mut out, *sb_addr);
+    }
+    let sum = jenkins_lookup3(&out[ib_start..]);
+    out.extend_from_slice(&sum.to_le_bytes());
+    debug_assert_eq!(out.len() - ib_start, aeib_size);
 
-    // Data block addresses + build data blocks
+    for db in direct.iter().filter(|d| d.addr.is_some()) {
-    let mut data_blocks_buf = Vec::new();
+        write_dblk(&mut out, db);
-    let dblks_base = aeib_address + aeib_size as u64;
+    }
-    let mut dblk_cursor = dblks_base;
+    for (sb_addr, u, dblks) in &supers {
-    let mut chunk_idx = n_inline;
+        if sb_addr.is_none() {
-
-    for &nelmts in &dblk_sizes {
-        if chunk_idx >= num_elements {
-            match offset_size {
-                4 => aeib.extend_from_slice(&u32::MAX.to_le_bytes()),
-                8 => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
-                _ => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
-            }
             continue;
         }
-
+        let (ndblks, nelmts, first) = levels[*u];
-        match offset_size {
+        let sb_start = out.len();
-            4 => aeib.extend_from_slice(&(dblk_cursor as u32).to_le_bytes()),
+        block_prefix(&mut out, b"EASB", first);
-            8 => aeib.extend_from_slice(&dblk_cursor.to_le_bytes()),
+        if nelmts > page_nelmts {
-            _ => aeib.extend_from_slice(&dblk_cursor.to_le_bytes()),
+            // Page-init bits, `npages` per data block, packed MSB-first
-        }
+            // (`H5VM_bit_set`): every page of an allocated data block is
-
+            // written.
-        // Build EADB
+            let npages = nelmts / page_nelmts;
-        let mut aedb = Vec::new();
+            let mut bitmap = vec![0u8; sblk_bitmap_len(ndblks, nelmts)];
-        aedb.extend_from_slice(b"EADB");
+            for (k, db) in dblks.iter().enumerate() {
-        aedb.push(0);
+                if db.addr.is_some() {
-        aedb.push(client_id);
+                    for p in 0..npages {
-        match offset_size {
+                        let bit = k * npages + p;
-            4 => aedb.extend_from_slice(&(ea_base_address as u32).to_le_bytes()),
+                        bitmap[bit / 8] |= 0x80 >> (bit % 8);
-            8 => aedb.extend_from_slice(&ea_base_address.to_le_bytes()),
+                    }
-            _ => aedb.extend_from_slice(&ea_base_address.to_le_bytes()),
+                }
-        }
-
-        let blk_off_size = (max_nelmts_bits as usize).div_ceil(8);
-        let blk_off_val = (chunk_idx - n_inline) as u64;
-        aedb.extend_from_slice(&blk_off_val.to_le_bytes()[..blk_off_size]);
-
-        for slot in 0..nelmts {
-            if chunk_idx + slot < num_elements {
-                write_chunk_element(
-                    &mut aedb,
-                    &chunks[chunk_idx + slot],
-                    offset_size,
-                    has_filters,
-                    chunk_size_bytes,
-                );
-            } else {
-                write_undefined_element(&mut aedb, offset_size, has_filters, chunk_size_bytes);
             }
+            out.extend_from_slice(&bitmap);
         }
-
+        for db in dblks {
-        let aedb_checksum = jenkins_lookup3(&aedb);
+            write_addr_opt(&mut out, db.addr);
-        aedb.extend_from_slice(&aedb_checksum.to_le_bytes());
+        }
-
+        let sum = jenkins_lookup3(&out[sb_start..]);
-        dblk_cursor += aedb.len() as u64;
+        out.extend_from_slice(&sum.to_le_bytes());
-        data_blocks_buf.extend_from_slice(&aedb);
+        for db in dblks.iter().filter(|d| d.addr.is_some()) {
-        chunk_idx += nelmts;
+            write_dblk(&mut out, db);
-    }
-
-    // Super block addresses (all undefined)
-    for _ in 0..n_sblk_addrs {
-        match offset_size {
-            4 => aeib.extend_from_slice(&u32::MAX.to_le_bytes()),
-            8 => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
-            _ => aeib.extend_from_slice(&u64::MAX.to_le_bytes()),
         }
     }
-
+    debug_assert_eq!(out.len() as u64, cursor - ea_base_address);
-    let aeib_checksum = jenkins_lookup3(&aeib);
+    out
-    aeib.extend_from_slice(&aeib_checksum.to_le_bytes());
-    debug_assert_eq!(aeib.len(), aeib_size);
-
-    let mut combined = aehd;
-    combined.extend_from_slice(&aeib);
-    combined.extend_from_slice(&data_blocks_buf);
-    combined
-}
-
-fn write_chunk_element(
-    buf: &mut Vec<u8>,
-    chunk: &WrittenChunk,
-    offset_size: u8,
-    has_filters: bool,
-    chunk_size_bytes: usize,
-) {
-    match offset_size {
-        4 => buf.extend_from_slice(&(chunk.address as u32).to_le_bytes()),
-        8 => buf.extend_from_slice(&chunk.address.to_le_bytes()),
-        _ => buf.extend_from_slice(&chunk.address.to_le_bytes()),
-    }
-    if has_filters {
-        let cs_bytes = chunk.compressed_size.to_le_bytes();
-        buf.extend_from_slice(&cs_bytes[..chunk_size_bytes]);
-        buf.extend_from_slice(&chunk.filter_mask.to_le_bytes());
-    }
-}
-
-fn write_undefined_element(
-    buf: &mut Vec<u8>,
-    offset_size: u8,
-    has_filters: bool,
-    chunk_size_bytes: usize,
-) {
-    let os = offset_size as usize;
-    // Use extend with repeat to avoid heap-allocating a temporary Vec on each call.
-    buf.extend(core::iter::repeat_n(0xFF, os));
-    if has_filters {
-        buf.extend(core::iter::repeat_n(0x00, chunk_size_bytes));
-        buf.extend_from_slice(&0u32.to_le_bytes());
-    }
 }

crates/clawhdf5/tests/chunk_index_interop.rs

@@ -390,3 +390,24 @@ fn we_write_paged_fixed_array() {
     cases.push(wcase("fa_2d_1100", &[110, 40], &[1, 4], None));
     check_we_write(&cases);
 }
+
+/// An Extensible Array holds 4 elements in its index block and 240 in the
+/// data blocks the index block addresses; everything after that lives under
+/// super blocks, and from ~131K elements on in paged data blocks. Chunks past
+/// index 243 used to be written but never indexed (read back as fill by us
+/// and by libhdf5).
+#[test]
+fn we_write_extensible_array_past_index_block() {
+    let unl: &[u64] = &[u64::MAX];
+    let mut cases: Vec<WriteCase> = [1u64, 4, 5, 243, 244, 245, 300, 1000, 5000]
+        .iter()
+        .map(|&n| wcase(&format!("ea_{n}"), &[n * 4], &[4], Some(unl)))
+        .collect();
+    let mut filtered = wcase("ea_300_deflate", &[300 * 4], &[4], Some(unl));
+    filtered.deflate = true;
+    cases.push(filtered);
+    // Several super blocks and paged data blocks (level 13, the first with
+    // data blocks over 1024 elements, starts at element 4 + 131056).
+    cases.push(wcase("ea_140000", &[140_000], &[1], Some(unl)));
+    check_we_write(&cases);
+}