writer: v2 B-trees with internal nodes (no 65 535-record limit)
Dense link and attribute indexes and the chunk index for several unlimited dimensions were single leaves, capping them at 65 535 records. btree_v2_write builds trees of any depth, with node capacities and pointer widths from libhdf5's H5B2__hdr_init arithmetic (now shared with the reader as btree_v2::node_info) and libhdf5's node sizes (512 dense, 2048 chunks). Indexes that fit the old one-leaf layout are written byte for byte as before (compared for 10..65 535 links, attrs and chunks, tracked and filtered). Tests: 100 000 links (short names; long names with creation order), 70 000 attributes, 200 000 chunks (and 80 000 deflated), read by h5py, h5dump and clawhdf5 and edited by h5py r+; h5rs check on the same shapes, asserting depths 2-3. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -559,20 +559,6 @@ fn we_write_btree_v2_for_several_unlimited_dims() {
|
||||
check_we_write(&cases);
|
||||
}
|
||||
|
||||
/// A single-leaf B-tree has a 16-bit record count; beyond it the writer
|
||||
/// refuses rather than writing a tree libhdf5 would misread.
|
||||
#[test]
|
||||
fn btree_v2_index_past_one_leaf_is_refused() {
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("d")
|
||||
.with_i32_data(&vec![0i32; 70_000])
|
||||
.with_shape(&[70_000, 1])
|
||||
.with_chunks(&[1, 1])
|
||||
.with_maxshape(&[u64::MAX, u64::MAX]);
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
assert!(b.write(dir.path().join("too_many.h5")).is_err());
|
||||
}
|
||||
|
||||
/// A maxshape equal to the shape cannot grow, so it needs no chunks: the
|
||||
/// dataset stays contiguous (as h5py makes it) unless chunks are requested.
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
//! Version-2 B-trees deeper than one leaf, as `FileBuilder` writes them for
|
||||
//! big dense indexes: a group's links (name index, type 5, and creation
|
||||
//! order index, type 6), an object's attributes (name index, type 8) and
|
||||
//! the chunk index of a dataset with two unlimited dimensions (type 10 and,
|
||||
//! with a filter, 11). Read back by h5py (libhdf5), h5dump and clawhdf5,
|
||||
//! then modified by h5py in "r+" mode, which splits, merges and
|
||||
//! redistributes the nodes the writer built.
|
||||
//!
|
||||
//! Skipped when python3 with h5py is unavailable, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::{AttrValue, File, FileBuilder};
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
macro_rules! skip_if_no_python {
|
||||
() => {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with h5py not available");
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// Run `body` under h5py with `path` bound to the file's path.
|
||||
fn h5py(path: &str, body: &str) -> String {
|
||||
let script = format!("import h5py, numpy as np, json\npath = r'{path}'\n{body}");
|
||||
let output = Command::new(python())
|
||||
.args(["-c", &script])
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
if !output.status.success() {
|
||||
panic!(
|
||||
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
||||
}
|
||||
|
||||
/// h5dump of `args` must succeed; returns its output.
|
||||
fn h5dump(args: &[&str]) -> String {
|
||||
let ok = Command::new("h5dump")
|
||||
.arg("--version")
|
||||
.output()
|
||||
.is_ok_and(|o| o.status.success());
|
||||
if !ok {
|
||||
assert!(!interop_required(), "h5dump is not available");
|
||||
return String::new();
|
||||
}
|
||||
let o = Command::new("h5dump").args(args).output().unwrap();
|
||||
let out = String::from_utf8_lossy(&o.stdout).to_string();
|
||||
assert!(
|
||||
o.status.success(),
|
||||
"h5dump {args:?} failed:\n{out}{}",
|
||||
String::from_utf8_lossy(&o.stderr)
|
||||
);
|
||||
out
|
||||
}
|
||||
|
||||
// ---- 100 000 links in one group ----
|
||||
|
||||
const NLINKS: usize = 100_000;
|
||||
|
||||
/// The compact names: `k0`..`k99998` and `k155448`, whose name hash equals
|
||||
/// that of `k69209` — a collision inside a many-level name index.
|
||||
fn compact_name(i: usize) -> String {
|
||||
if i == NLINKS - 1 {
|
||||
"k155448".into()
|
||||
} else {
|
||||
format!("k{i}")
|
||||
}
|
||||
}
|
||||
|
||||
/// The long names (111 bytes), added in a scrambled order so creation order
|
||||
/// is not name order: the `i`th created is `long_name(scramble(i))`.
|
||||
fn long_name(j: usize) -> String {
|
||||
format!("link_{j:06}_{}", "x".repeat(100))
|
||||
}
|
||||
|
||||
fn scramble(i: usize) -> usize {
|
||||
i * 7919 % NLINKS
|
||||
}
|
||||
|
||||
const PY_NAMES: &str = "\
|
||||
N = 100000\n\
|
||||
compact = ['k%d' % i for i in range(N - 1)] + ['k155448']\n\
|
||||
def long_name(j): return 'link_%06d_' % j + 'x' * 100\n\
|
||||
created = [long_name(i * 7919 % N) for i in range(N)]\n";
|
||||
|
||||
#[test]
|
||||
fn a_hundred_thousand_links_in_one_group() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("links.h5").display().to_string();
|
||||
let mut b = FileBuilder::new();
|
||||
for v in 0..10 {
|
||||
b.create_dataset(&format!("v{v}")).with_i32_data(&[v]);
|
||||
}
|
||||
// Name index only (type 5), 11-byte records: depth 3 in 512-byte nodes.
|
||||
let mut g = b.create_group("compact");
|
||||
for i in 0..NLINKS {
|
||||
g.add_hard_link(&compact_name(i), &format!("/v{}", i % 10));
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
// Creation order tracked and indexed: a type-6 index as well.
|
||||
let mut g = b.create_group("long");
|
||||
g.track_order(true);
|
||||
for i in 0..NLINKS {
|
||||
let j = scramble(i);
|
||||
g.add_hard_link(&long_name(j), &format!("/v{}", j % 10));
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
b.write(&path).unwrap();
|
||||
|
||||
let check = format!(
|
||||
"{PY_NAMES}\
|
||||
with h5py.File(path, 'r') as f:\n\
|
||||
\x20 g, l = f['compact'], f['long']\n\
|
||||
\x20 out = [len(g), list(g) == sorted(compact), len(l), list(l) == created]\n\
|
||||
\x20 out.append(all(int(g[compact[i]][0]) == i % 10 for i in range(0, N, 997)))\n\
|
||||
\x20 out.append([int(g['k69209'][0]), int(g['k155448'][0]), 'k155448' in g, 'k100000' in g])\n\
|
||||
\x20 out.append(all(int(l[long_name(j)][0]) == j % 10 for j in range(0, N, 1009)))\n\
|
||||
\x20 out.append(h5py.h5o.get_info(f['v3'].id).rc)\n\
|
||||
\x20 print(json.dumps(out))"
|
||||
);
|
||||
assert_eq!(
|
||||
h5py(&path, &check),
|
||||
"[100000, true, 100000, true, true, [9, 9, true, false], true, 20001]"
|
||||
);
|
||||
let d = h5dump(&["-d", "/compact/k155448", &path]);
|
||||
assert!(d.is_empty() || d.contains("(0): 9"), "{d}");
|
||||
let d = h5dump(&["-d", &format!("/long/{}", long_name(99_999)), &path]);
|
||||
assert!(d.is_empty() || d.contains("(0): 9"), "{d}");
|
||||
|
||||
// clawhdf5 reads both indexes back.
|
||||
let f = File::open(&path).unwrap();
|
||||
let g = f.group("compact").unwrap();
|
||||
let mut names = g.datasets().unwrap();
|
||||
names.sort();
|
||||
let mut want: Vec<String> = (0..NLINKS).map(compact_name).collect();
|
||||
want.sort();
|
||||
assert_eq!(names, want);
|
||||
assert_eq!(g.dataset("k155448").unwrap().read_i32().unwrap(), [9]);
|
||||
let l = f.group("long").unwrap();
|
||||
assert_eq!(l.datasets().unwrap().len(), NLINKS);
|
||||
assert_eq!(
|
||||
l.dataset(&long_name(12_345)).unwrap().read_i32().unwrap(),
|
||||
[5]
|
||||
);
|
||||
drop(f);
|
||||
|
||||
// libhdf5 inserts into and removes from the trees we wrote.
|
||||
let modify = format!(
|
||||
"{PY_NAMES}\
|
||||
with h5py.File(path, 'r+') as f:\n\
|
||||
\x20 g, l = f['compact'], f['long']\n\
|
||||
\x20 for i in range(3000):\n\
|
||||
\x20 g['new%d' % i] = f['v1']\n\
|
||||
\x20 for i in range(0, N, 7):\n\
|
||||
\x20 del g[compact[i]]\n\
|
||||
\x20 l['zz_new'] = f['v2']\n\
|
||||
\x20 for j in range(0, N, 3):\n\
|
||||
\x20 del l[long_name(j)]\n\
|
||||
with h5py.File(path, 'r') as f:\n\
|
||||
\x20 g, l = f['compact'], f['long']\n\
|
||||
\x20 left = sorted([n for i, n in enumerate(compact) if i % 7] + ['new%d' % i for i in range(3000)])\n\
|
||||
\x20 kept = [n for n in created if int(n[5:11]) % 3] + ['zz_new']\n\
|
||||
\x20 print(json.dumps([len(g), list(g) == left, int(g['new2999'][0]),\n\
|
||||
\x20 int(g['k155448'][0]), len(l), list(l) == kept, int(l['zz_new'][0])]))"
|
||||
);
|
||||
assert_eq!(h5py(&path, &modify), "[88714, true, 1, 9, 66667, true, 2]");
|
||||
h5dump(&["-d", "/compact/new0", &path]);
|
||||
let f = File::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
f.group("compact").unwrap().datasets().unwrap().len(),
|
||||
88_714
|
||||
);
|
||||
assert_eq!(f.group("long").unwrap().datasets().unwrap().len(), 66_667);
|
||||
}
|
||||
|
||||
// ---- 70 000 attributes on one object ----
|
||||
|
||||
#[test]
|
||||
fn seventy_thousand_attributes_on_one_object() {
|
||||
skip_if_no_python!();
|
||||
const N: i64 = 70_000;
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("attrs.h5").display().to_string();
|
||||
let mut b = FileBuilder::new();
|
||||
let x = b.create_dataset("x");
|
||||
x.with_i32_data(&[1]);
|
||||
for i in 0..N {
|
||||
x.set_attr(&format!("attr_{i}"), AttrValue::I64(i * 3));
|
||||
}
|
||||
let mut g = b.create_group("g");
|
||||
for i in 0..N {
|
||||
g.set_attr(&format!("s{i:05}"), AttrValue::String(format!("value {i}")));
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
b.write(&path).unwrap();
|
||||
|
||||
let check = "\
|
||||
N = 70000\n\
|
||||
with h5py.File(path, 'r') as f:\n\
|
||||
\x20 a, s = f['x'].attrs, f['g'].attrs\n\
|
||||
\x20 names = list(a)\n\
|
||||
\x20 out = [len(a), names == sorted('attr_%d' % i for i in range(N))]\n\
|
||||
\x20 out.append(all(int(a['attr_%d' % i]) == 3 * i for i in range(0, N, 331)))\n\
|
||||
\x20 out.append(len(s))\n\
|
||||
\x20 v = dict(s.items())\n\
|
||||
\x20 out.append(all(v['s%05d' % i].decode() == 'value %d' % i for i in range(N)))\n\
|
||||
\x20 print(json.dumps(out))";
|
||||
assert_eq!(h5py(&path, check), "[70000, true, true, 70000, true]");
|
||||
let d = h5dump(&["-a", "/x/attr_69999", &path]);
|
||||
assert!(d.is_empty() || d.contains("(0): 209997"), "{d}");
|
||||
|
||||
let f = File::open(&path).unwrap();
|
||||
let attrs = f.dataset("x").unwrap().attrs().unwrap();
|
||||
assert_eq!(attrs.len(), N as usize);
|
||||
for i in [0, 1, 35_000, N - 1] {
|
||||
assert!(
|
||||
matches!(attrs[&format!("attr_{i}")], AttrValue::I64(v) if v == 3 * i),
|
||||
"attr_{i}"
|
||||
);
|
||||
}
|
||||
let attrs = f.group("g").unwrap().attrs().unwrap();
|
||||
assert_eq!(attrs.len(), N as usize);
|
||||
drop(f);
|
||||
|
||||
let modify = "\
|
||||
N = 70000\n\
|
||||
with h5py.File(path, 'r+') as f:\n\
|
||||
\x20 a = f['x'].attrs\n\
|
||||
\x20 for i in range(2000):\n\
|
||||
\x20 a['new_%d' % i] = i\n\
|
||||
\x20 for i in range(0, N, 5):\n\
|
||||
\x20 del a['attr_%d' % i]\n\
|
||||
\x20 f['g'].attrs['s00000'] = 'changed'\n\
|
||||
with h5py.File(path, 'r') as f:\n\
|
||||
\x20 a = f['x'].attrs\n\
|
||||
\x20 want = sorted(['attr_%d' % i for i in range(N) if i % 5] + ['new_%d' % i for i in range(2000)])\n\
|
||||
\x20 print(json.dumps([len(a), list(a) == want, int(a['attr_69999']), int(a['new_1999']),\n\
|
||||
\x20 'attr_5' in a, f['g'].attrs['s00000'], len(f['g'].attrs)]))";
|
||||
assert_eq!(
|
||||
h5py(&path, modify),
|
||||
r#"[58000, true, 209997, 1999, false, "changed", 70000]"#
|
||||
);
|
||||
let f = File::open(&path).unwrap();
|
||||
assert_eq!(f.dataset("x").unwrap().attrs().unwrap().len(), 58_000);
|
||||
}
|
||||
|
||||
// ---- 200 000 chunks with two unlimited dimensions ----
|
||||
|
||||
#[test]
|
||||
fn two_hundred_thousand_chunks_with_two_unlimited_dims() {
|
||||
skip_if_no_python!();
|
||||
const U: u64 = u64::MAX;
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("chunks.h5").display().to_string();
|
||||
let data: Vec<i32> = (0..200_000).collect();
|
||||
let small: Vec<i32> = (0..80_000).map(|v| v * 2).collect();
|
||||
let mut b = FileBuilder::new();
|
||||
// Type 10 (unfiltered): 24-byte records, depth 2 in 2048-byte nodes.
|
||||
b.create_dataset("d")
|
||||
.with_i32_data(&data)
|
||||
.with_shape(&[400, 500])
|
||||
.with_chunks(&[1, 1])
|
||||
.with_maxshape(&[U, U]);
|
||||
// Type 11 (filtered): each record also holds a size and filter mask.
|
||||
b.create_dataset("z")
|
||||
.with_i32_data(&small)
|
||||
.with_shape(&[200, 400])
|
||||
.with_chunks(&[1, 1])
|
||||
.with_maxshape(&[U, U])
|
||||
.with_deflate(1);
|
||||
b.write(&path).unwrap();
|
||||
|
||||
let check = "\
|
||||
with h5py.File(path, 'r') as f:\n\
|
||||
\x20 d, z = f['d'], f['z']\n\
|
||||
\x20 print(json.dumps([d.shape, d.chunks, d.id.get_num_chunks(),\n\
|
||||
\x20 bool(np.array_equal(d[()], np.arange(200000).reshape(400, 500))),\n\
|
||||
\x20 int(d[399, 499]), z.id.get_num_chunks(),\n\
|
||||
\x20 bool(np.array_equal(z[()], 2 * np.arange(80000).reshape(200, 400)))]))";
|
||||
assert_eq!(
|
||||
h5py(&path, check),
|
||||
"[[400, 500], [1, 1], 200000, true, 199999, 80000, true]"
|
||||
);
|
||||
let d = h5dump(&["-d", "/d", "-s", "399,498", "-c", "1,2", &path]);
|
||||
assert!(d.is_empty() || d.contains("199998, 199999"), "{d}");
|
||||
|
||||
let f = File::open(&path).unwrap();
|
||||
assert_eq!(f.dataset("d").unwrap().read_i32().unwrap(), data);
|
||||
assert_eq!(f.dataset("z").unwrap().read_i32().unwrap(), small);
|
||||
drop(f);
|
||||
|
||||
// libhdf5 adds chunks to both trees.
|
||||
let modify = "\
|
||||
with h5py.File(path, 'r+') as f:\n\
|
||||
\x20 d, z = f['d'], f['z']\n\
|
||||
\x20 d.resize((401, 510))\n\
|
||||
\x20 d[400, :] = -1\n\
|
||||
\x20 d[:, 500:] = -2\n\
|
||||
\x20 z.resize((201, 400))\n\
|
||||
\x20 z[200, :] = 7\n\
|
||||
with h5py.File(path, 'r') as f:\n\
|
||||
\x20 d, z = f['d'][()], f['z'][()]\n\
|
||||
\x20 print(json.dumps([f['d'].id.get_num_chunks(),\n\
|
||||
\x20 bool(np.array_equal(d[:400, :500], np.arange(200000).reshape(400, 500))),\n\
|
||||
\x20 int(d[400, 3]), int(d[5, 505]), f['z'].id.get_num_chunks(),\n\
|
||||
\x20 bool(np.array_equal(z[:200], 2 * np.arange(80000).reshape(200, 400))), int(z[200, 9])]))";
|
||||
assert_eq!(
|
||||
h5py(&path, modify),
|
||||
"[204510, true, -1, -2, 80400, true, 7]"
|
||||
);
|
||||
let f = File::open(&path).unwrap();
|
||||
let d = f.dataset("d").unwrap().read_i32().unwrap();
|
||||
assert_eq!(d.len(), 401 * 510);
|
||||
assert_eq!(d[499], 499);
|
||||
assert_eq!(d[400 * 510 + 3], -1);
|
||||
}
|
||||
@@ -531,32 +531,6 @@ fn ten_thousand_links_in_one_group() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn more_links_than_one_index_leaf_holds_is_an_error() {
|
||||
let mut b = FileBuilder::new();
|
||||
for i in 0..70_000 {
|
||||
b.add_soft_link(&format!("s{i}"), "/x");
|
||||
}
|
||||
let err = b.finish().unwrap_err().to_string();
|
||||
assert!(
|
||||
err.contains("70000 links in one group: at most 65535"),
|
||||
"{err}"
|
||||
);
|
||||
// Dense attributes have the same one-leaf index. Their count used to
|
||||
// be written modulo 65 536.
|
||||
let mut b = FileBuilder::new();
|
||||
let x = b.create_dataset("x");
|
||||
x.with_i32_data(&[1]);
|
||||
for i in 0..70_000 {
|
||||
x.set_attr(&format!("a{i}"), AttrValue::I64(i));
|
||||
}
|
||||
let err = b.finish().unwrap_err().to_string();
|
||||
assert!(
|
||||
err.contains("70000 attributes on one object: at most 65535"),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn track_order_lists_members_in_creation_order() {
|
||||
skip_if_no_python!();
|
||||
@@ -643,7 +617,11 @@ fn names_whose_hashes_collide_are_found_by_name() {
|
||||
.chain((0..10).map(|_| ""))
|
||||
.enumerate()
|
||||
{
|
||||
let name = if n.is_empty() { format!("d{i}") } else { n.into() };
|
||||
let name = if n.is_empty() {
|
||||
format!("d{i}")
|
||||
} else {
|
||||
n.into()
|
||||
};
|
||||
g.create_dataset(&name).with_i32_data(&[i as i32]);
|
||||
}
|
||||
b.add_group(g.finish());
|
||||
|
||||
Reference in New Issue
Block a user