fix(format): write paged files libhdf5 can open

FileWriter::with_page_size wrote a "version 4" superblock with an extra page-size field. HDF5 has no superblock version 4, so libhdf5 refused every such file ("bad superblock version number"). A paged file is now what libhdf5 itself writes for fs_strategy="page": a v3 superblock whose extension object header holds a File Space Info message (strategy PAGE, the page size, free space not persisted; same bytes and flags as HDF5 2.0), with the file padded to a whole page. h5py opens it, reports the strategy and page size, and can modify it in r+ mode. Page sizes outside libhdf5's 512 B..1 GiB are an error. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-09-25 21:09:31 -05:00
parent be88e3fec7
commit 74fdf0582b
3 changed files with 163 additions and 21 deletions
@@ -317,3 +317,59 @@ fn raw_attributes_copied_from_h5py_survive_a_rewrite() {
    assert_eq!(out, r#"[["/", "/"], "abcdefgh", 1, [258, 772]]"#);
    h5dump_ok(&path);
 }
+
+// ---- 3. paged file-space strategy ----
+
+fn paged_file(page_size: u32) -> Vec<u8> {
+    let mut fw = FileWriter::new();
+    fw.with_page_size(page_size);
+    fw.create_dataset("d").with_f64_data(&[1.0, 2.0, 3.0]);
+    fw.create_dataset("c")
+        .with_i32_data(&(0..100).collect::<Vec<_>>())
+        .with_chunks(&[10]);
+    fw.set_root_attr("a", AttrValue::I64(7));
+    let mut g = fw.create_group("g");
+    g.create_dataset("e").with_u8_data(&[9; 5000]);
+    fw.add_group(g.finish());
+    fw.finish().unwrap()
+}
+
+#[test]
+fn paged_file_has_a_real_superblock() {
+    // Measured: `with_page_size` wrote superblock version 4, which does not
+    // exist ("bad superblock version number" in libhdf5).
+    for ps in [512u32, 4096, 65536] {
+        let bytes = paged_file(ps);
+        let (sb, _) = header_at(&bytes, "/");
+        assert_eq!(sb.version, 3);
+        assert_eq!(bytes.len() % ps as usize, 0);
+        let (_, e) = header_at(&bytes, "g/e");
+        assert!(
+            e.messages
+                .iter()
+                .any(|m| m.msg_type == MessageType::Dataspace)
+        );
+    }
+}
+
+#[test]
+#[ignore = "requires Python h5py module and h5dump"]
+fn h5py_opens_paged_files() {
+    for ps in [512u32, 4096, 65536] {
+        let path = write_tmp(&format!("paged_{ps}"), &paged_file(ps));
+        let out = h5py(
+            &path,
+            "f = h5py.File(path, 'r')\n\
+             p = f.id.get_create_plist()\n\
+             print(json.dumps([p.get_file_space_strategy()[0], p.get_file_space_page_size(),\n\
+             \x20   f['d'][()].tolist(), int(f['c'][()].sum()), int(f.attrs['a']),\n\
+             \x20   int(f['g/e'][()].sum())]))",
+        );
+        assert_eq!(
+            out,
+            format!("[1, {ps}, [1.0, 2.0, 3.0], 4950, 7, 45000]"),
+            "page size {ps}"
+        );
+        h5dump_ok(&path);
+    }
+}