edit: dense attributes, compact-to-dense transition, creation order
FileEditor::set_attr now handles every attribute storage libhdf5 uses for version-2 object headers: - objects that track (and index) attribute creation order: compact attributes carry their creation index in the message header, the Attribute Info message its maximum; - the move to dense storage when an object reaches its compact limit (or an attribute is too large for a header message), as H5O__attr_create does it: a new fractal heap, name index (v2 B-tree type 8) and, when creation order is indexed, creation-order index (type 9); the compact attributes moved over in header message order, their messages freed; - objects already in dense storage (h5py- or clawhdf5-written): new attributes inserted (H5A__dense_insert), an attribute replaced by one of the same encoded size rewritten in its heap object (H5A__dense_write), otherwise removed from both indexes and the heap and inserted anew. edit/fheap.rs follows H5HF: managed objects go to the best-fitting free section of the heap's free-space manager (FSHD/FSSE, kept as libhdf5 keeps it — sorted sections, counts, section info reallocated when its size changes, the manager deleted when empty); otherwise to a new direct block: the root direct block of an empty heap, else the block at the allocation iterator in the root indirect block (created from the root direct block, doubled as needed), with libhdf5's managed-space, allocated space, free space and iterator bookkeeping. Objects above the managed limit are huge objects in their own space, indexed by the huge-object B-tree (type 1). Removed objects return their space merged with adjacent free space. Refused before anything is written: heaps with I/O filters, child indirect blocks, an object larger than the next heap block (libhdf5 skips blocks and records them as free space), free sections other than those inside direct blocks, removing a direct block's last object (libhdf5 frees the block), directly addressed huge objects. Attributes are encoded as libhdf5 does when h5py opens a file r+ (low bound "earliest"): message version 1 (3 for non-ASCII names), simple dataspaces with their maximum dimensions. Header chunks are now visited in libhdf5's order (FIFO), which is also the order attributes move to dense storage in. Tests (edit_coverage_interop): 40 attributes on each of a plain group, a group tracking and indexing creation order, and a dataset (earliest, v110, latest), some above the 4 KiB managed limit, then same-size rewrites: the heap statistics, free-space sections and both index B-trees node for node equal libhdf5's doing the same through h5py; then replacements of other sizes, h5py adds/deletes/rewrites; h5py, h5dump, h5rs check and our reader agree throughout, h5py's attribute count included. clawhdf5-written dense storage (tracked and untracked) is extended the same way; refusals leave the file byte for byte as it was. edit_interop's attribute test now expects dense storage and tracked creation order to work. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -213,7 +213,6 @@ fn tmpdir() -> tempfile::TempDir {
|
||||
tempfile::TempDir::new_in(base).unwrap()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn unsupported<T: std::fmt::Debug>(r: Result<T, Error>) {
|
||||
match r {
|
||||
Err(Error::Unsupported(_)) => {}
|
||||
@@ -830,3 +829,497 @@ fn btree2_random_chunk_order() {
|
||||
verify(&path, "x", &m);
|
||||
check_tools(&path, false);
|
||||
}
|
||||
|
||||
// ---- dense attributes ----
|
||||
|
||||
/// An attribute value both sides can write with the same encoded size: a
|
||||
/// 1-D int64 array or a fixed-length string.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
enum AV {
|
||||
Ints(Vec<i64>),
|
||||
Str(String),
|
||||
}
|
||||
|
||||
impl AV {
|
||||
fn value(&self) -> clawhdf5::AttrValue {
|
||||
match self {
|
||||
AV::Ints(v) => clawhdf5::AttrValue::I64Array(v.clone()),
|
||||
AV::Str(s) => clawhdf5::AttrValue::String(s.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
fn py(&self) -> String {
|
||||
match self {
|
||||
AV::Ints(v) => format!("np.array({v:?}, dtype='<i8')"),
|
||||
AV::Str(s) => format!("np.bytes_({s:?})"),
|
||||
}
|
||||
}
|
||||
|
||||
fn py_expect(&self) -> String {
|
||||
match self {
|
||||
AV::Ints(v) => format!("{v:?}"),
|
||||
AV::Str(s) => format!("{s:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn make(i: u64, salt: u64) -> Self {
|
||||
match i % 11 {
|
||||
5 => AV::Str("h".repeat(5000 + (salt % 50) as usize)),
|
||||
1 | 4 | 8 => AV::Str(
|
||||
(0..10 + (i * 13 + salt) % 300)
|
||||
.map(|k| (b'a' + ((k + salt) % 26) as u8) as char)
|
||||
.collect(),
|
||||
),
|
||||
_ => AV::Ints(
|
||||
(0..1 + (i + salt) % 9)
|
||||
.map(|k| (k * 31 + salt) as i64)
|
||||
.collect(),
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An object's dense attribute storage as libhdf5 would compare it: the
|
||||
/// heap's statistics and root shape, its free sections (heap offsets and
|
||||
/// sizes), and the shapes of the name and creation-order index B-trees.
|
||||
fn dense_info(path: &Path, obj: &str) -> String {
|
||||
use clawhdf5_format::message_type::MessageType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
let f = File::open(path).unwrap();
|
||||
let sb = f.superblock();
|
||||
let (os, ls) = (sb.offset_size, sb.length_size);
|
||||
assert_eq!((os, ls), (8, 8));
|
||||
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, obj).unwrap();
|
||||
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
|
||||
let Some(m) = oh
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == MessageType::AttributeInfo)
|
||||
else {
|
||||
return "no attribute info".into();
|
||||
};
|
||||
let d = &m.data;
|
||||
let mut p = 2 + if d[1] & 1 != 0 { 2 } else { 0 };
|
||||
let heap = le(&d[p..p + 8]);
|
||||
p += 8;
|
||||
let name = le(&d[p..p + 8]);
|
||||
let order = (d[1] & 2 != 0).then(|| le(&d[p + 8..p + 16]));
|
||||
if heap == u64::MAX {
|
||||
return "compact".into();
|
||||
}
|
||||
let b = std::fs::read(path).unwrap();
|
||||
let h = &b[heap as usize..];
|
||||
assert_eq!(&h[0..4], b"FRHP");
|
||||
// next huge id, huge bt2, free, fs, man size, alloc, iter, nobjs, huge
|
||||
// size, huge objs (8-byte fields from offset 14).
|
||||
let stat = |k: usize| le(&h[14 + 8 * k..22 + 8 * k]);
|
||||
let fs = stat(3);
|
||||
// Root rows: after 12 8-byte fields, width, start, max direct, max
|
||||
// index, start rows, root address.
|
||||
let rows_at = 14 + 96 + 2 + 16 + 2 + 2 + 8;
|
||||
let mut out = format!(
|
||||
"heap next_huge={} free={} man={} alloc={} iter={} nobjs={} huge={}/{} rows={}",
|
||||
stat(0),
|
||||
stat(2),
|
||||
stat(4),
|
||||
stat(5),
|
||||
stat(6),
|
||||
stat(7),
|
||||
stat(8),
|
||||
stat(9),
|
||||
le(&h[rows_at..rows_at + 2])
|
||||
);
|
||||
if fs != u64::MAX {
|
||||
let s = &b[fs as usize..];
|
||||
assert_eq!(&s[0..4], b"FSHD");
|
||||
let tot = le(&s[6..14]);
|
||||
let n = le(&s[14..22]);
|
||||
// counts (4 x 8), 4 x u16, max section size, section info address.
|
||||
let at = 6 + 32 + 8 + 8;
|
||||
let sect_addr = le(&s[at..at + 8]);
|
||||
let sect_size = le(&s[at + 8..at + 16]);
|
||||
let ss = &b[sect_addr as usize..(sect_addr + sect_size) as usize];
|
||||
// Sections after the prefix (signature, version, header address);
|
||||
// trailing zero padding and the checksum are left out.
|
||||
let body = &ss[13..ss.len() - 4];
|
||||
let used = body.len() - body.iter().rev().take_while(|&&x| x == 0).count();
|
||||
out += &format!(" fs tot={tot} n={n} sections={}", hex(&body[..used]));
|
||||
} else {
|
||||
out += " no-fs";
|
||||
}
|
||||
out += &format!(" names={:?}", bt2_shape_at(&b, name as usize));
|
||||
if let Some(o) = order {
|
||||
out += &format!(" order={:?}", bt2_shape_at(&b, o as usize));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn hex(b: &[u8]) -> String {
|
||||
b.iter().map(|x| format!("{x:02x}")).collect()
|
||||
}
|
||||
|
||||
/// h5py and our reader both see `want` on each object (other attributes
|
||||
/// may exist; these must have these values), and h5py's attribute count
|
||||
/// agrees with libhdf5's object info.
|
||||
fn check_attr_values(path: &Path, want: &[(String, String, AV)]) {
|
||||
let f = File::open(path).unwrap();
|
||||
for (o, n, v) in want {
|
||||
let attrs = if o == "d" {
|
||||
f.dataset(o).unwrap().attrs().unwrap()
|
||||
} else {
|
||||
f.group(o).unwrap().attrs().unwrap()
|
||||
};
|
||||
let got = attrs
|
||||
.get(n.as_str())
|
||||
.unwrap_or_else(|| panic!("{o}/{n} missing ({} attributes)", attrs.len()));
|
||||
match (got, v) {
|
||||
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
||||
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
||||
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
||||
other => panic!("{o}/{n}: {other:?}"),
|
||||
}
|
||||
}
|
||||
let exp: Vec<String> = want
|
||||
.iter()
|
||||
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
||||
.collect();
|
||||
let script = format!(
|
||||
"import h5py, numpy as np\n\
|
||||
f = h5py.File({p:?}, 'r')\n\
|
||||
want = [{w}]\n\
|
||||
for o, n, v in want:\n\
|
||||
\x20 a = f[o].attrs[n]\n\
|
||||
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
||||
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
||||
\x20 a = [a] if isinstance(a, int) else a\n\
|
||||
\x20 assert a == v, (o, n, a if len(str(a)) < 200 else len(a), v if len(str(v)) < 200 else len(v))\n\
|
||||
for o in set(x[0] for x in want):\n\
|
||||
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n\
|
||||
\x20 assert len(list(f[o].attrs)) == len(f[o].attrs)\n",
|
||||
p = path.to_str().unwrap(),
|
||||
w = exp.join(", ")
|
||||
);
|
||||
let sp = path.with_extension("check.py");
|
||||
std::fs::write(&sp, script).unwrap();
|
||||
let o = Command::new(python()).arg(&sp).output().unwrap();
|
||||
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
||||
}
|
||||
|
||||
/// Run python statements (inside `with h5py.File(path, 'r+') as f:`).
|
||||
fn py_r_plus(path: &Path, lines: &[String]) {
|
||||
let script = format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n{l}",
|
||||
p = path.to_str().unwrap(),
|
||||
l = lines.concat()
|
||||
);
|
||||
let sp = path.with_extension("ops.py");
|
||||
std::fs::write(&sp, script).unwrap();
|
||||
let o = Command::new(python()).arg(&sp).output().unwrap();
|
||||
assert!(o.status.success(), "h5py workload failed:\n{}", text(&o));
|
||||
}
|
||||
|
||||
type AttrOp = (String, String, AV);
|
||||
|
||||
fn set_want(want: &mut Vec<AttrOp>, o: &str, n: &str, v: AV) {
|
||||
want.retain(|(wo, wn, _)| !(wo == o && wn == n));
|
||||
want.push((o.into(), n.into(), v));
|
||||
}
|
||||
|
||||
/// The same attribute workload by libhdf5 and by the editor on objects
|
||||
/// with compact attributes that move to dense storage (a plain group, a
|
||||
/// group tracking and indexing creation order, a dataset): 40 new
|
||||
/// attributes each (some larger than the heap's 4 KiB managed limit), then
|
||||
/// same-size rewrites — the heaps, their free space and both index B-trees
|
||||
/// must come out as libhdf5 makes them; then replacements of another size,
|
||||
/// then h5py adds, deletes and rewrites attributes.
|
||||
fn dense_workload(libver: &str, h5dump: bool, tag: &str) {
|
||||
let dir = tmpdir();
|
||||
let a = dir.path().join(format!("dense_{tag}_h5py.h5"));
|
||||
let b = dir.path().join(format!("dense_{tag}_edit.h5"));
|
||||
for p in [&a, &b] {
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
|
||||
\x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \
|
||||
f.create_dataset('d', data=np.arange(4, dtype='<i4'))]\n\
|
||||
\x20 for o in objs:\n\
|
||||
\x20 for i in range(3): o.attrs.create(f'c{{i}}', np.array([i, i], dtype='<i8'))\n",
|
||||
p = p.to_str().unwrap()
|
||||
));
|
||||
}
|
||||
let objs = ["g", "t", "d"];
|
||||
let mut want: Vec<AttrOp> = Vec::new();
|
||||
for o in objs {
|
||||
for i in 0..3i64 {
|
||||
want.push((o.into(), format!("c{i}"), AV::Ints(vec![i, i])));
|
||||
}
|
||||
}
|
||||
// Phase 1: new attributes.
|
||||
let mut ops: Vec<AttrOp> = Vec::new();
|
||||
for i in 0..40u64 {
|
||||
for (k, o) in objs.iter().enumerate() {
|
||||
ops.push((o.to_string(), format!("n{i}"), AV::make(i, k as u64)));
|
||||
}
|
||||
}
|
||||
let lines: Vec<String> = ops
|
||||
.iter()
|
||||
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.create({n:?}, {})\n", v.py()))
|
||||
.collect();
|
||||
py_r_plus(&a, &lines);
|
||||
let mut ed = FileEditor::open(&b).unwrap();
|
||||
for (o, n, v) in &ops {
|
||||
ed.set_attr(o, n, &v.value())
|
||||
.unwrap_or_else(|e| panic!("{tag}: set {o}/{n}: {e}"));
|
||||
set_want(&mut want, o, n, v.clone());
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&b, h5dump);
|
||||
check_attr_values(&b, &want);
|
||||
check_attr_values(&a, &want);
|
||||
for o in objs {
|
||||
assert_eq!(
|
||||
dense_info(&b, o),
|
||||
dense_info(&a, o),
|
||||
"{tag}: dense storage of {o} differs from libhdf5's after insertions"
|
||||
);
|
||||
}
|
||||
// Phase 2: same-size rewrites (H5Awrite in place).
|
||||
let ops2: Vec<AttrOp> = ops
|
||||
.iter()
|
||||
.step_by(4)
|
||||
.map(|(o, n, v)| {
|
||||
let nv = match v {
|
||||
AV::Ints(x) => AV::Ints(x.iter().map(|y| y * 3 + 1).collect()),
|
||||
AV::Str(s) => AV::Str(s.chars().rev().collect()),
|
||||
};
|
||||
(o.clone(), n.clone(), nv)
|
||||
})
|
||||
.collect();
|
||||
let lines: Vec<String> = ops2
|
||||
.iter()
|
||||
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.modify({n:?}, {})\n", v.py()))
|
||||
.collect();
|
||||
py_r_plus(&a, &lines);
|
||||
let mut ed = FileEditor::open(&b).unwrap();
|
||||
for (o, n, v) in &ops2 {
|
||||
ed.set_attr(o, n, &v.value()).unwrap();
|
||||
set_want(&mut want, o, n, v.clone());
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&b, h5dump);
|
||||
check_attr_values(&b, &want);
|
||||
for o in objs {
|
||||
assert_eq!(
|
||||
dense_info(&b, o),
|
||||
dense_info(&a, o),
|
||||
"{tag}: dense storage of {o} differs from libhdf5's after rewrites"
|
||||
);
|
||||
}
|
||||
// Phase 3: replacements of another size (values only: h5py replaces
|
||||
// through a temporary attribute and a rename).
|
||||
let mut ed = FileEditor::open(&b).unwrap();
|
||||
for (k, (o, n, v)) in ops.iter().enumerate().filter(|(k, _)| k % 5 == 2) {
|
||||
let nv = match v {
|
||||
AV::Ints(x) => AV::Ints((0..x.len() as i64 + 3).collect()),
|
||||
AV::Str(s) => AV::Str(format!("{s}-{k}")),
|
||||
};
|
||||
let before = std::fs::read(&b).unwrap();
|
||||
match ed.set_attr(o, n, &nv.value()) {
|
||||
Ok(()) => set_want(&mut want, o, n, nv),
|
||||
Err(Error::Unsupported(msg)) => {
|
||||
assert!(msg.contains("last object"), "{tag}: {o}/{n}: {msg}");
|
||||
assert!(std::fs::read(&b).unwrap() == before, "refused edit wrote");
|
||||
}
|
||||
Err(e) => panic!("{tag}: replace {o}/{n}: {e}"),
|
||||
}
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&b, h5dump);
|
||||
check_attr_values(&b, &want);
|
||||
// libhdf5 goes on: adds, deletes and rewrites.
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 for o in ['g', 't', 'd']:\n\
|
||||
\x20 for i in range(5): f[o].attrs[f'late{{i}}'] = np.arange(i + 1)\n\
|
||||
\x20 del f[o].attrs['n3']\n\
|
||||
\x20 del f[o].attrs['c1']\n\
|
||||
\x20 f[o].attrs['n7'] = 'rewritten by h5py'\n",
|
||||
p = b.to_str().unwrap()
|
||||
));
|
||||
want.retain(|(_, n, _)| n != "n3" && n != "c1" && n != "n7");
|
||||
check_tools(&b, h5dump);
|
||||
check_attr_values(&b, &want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_attributes_match_libhdf5() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
dense_workload(lv, *dump, &format!("{i}"));
|
||||
}
|
||||
}
|
||||
|
||||
/// Dense attributes in files clawhdf5 writes (its own heap and index
|
||||
/// layout, no free-space manager), with and without tracked creation
|
||||
/// order: attributes added (moving a dataset's attributes to dense storage
|
||||
/// too), rewritten and replaced, then h5py goes on.
|
||||
#[test]
|
||||
fn dense_attributes_on_clawhdf5_files() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
for track in [false, true] {
|
||||
let dir = tmpdir();
|
||||
let path = dir.path().join(format!("ours_dense_{track}.h5"));
|
||||
let mut b = clawhdf5::FileBuilder::new();
|
||||
b.track_order(track);
|
||||
for i in 0..12i64 {
|
||||
b.set_attr(&format!("r{i}"), clawhdf5::AttrValue::I64Array(vec![i; 3]));
|
||||
}
|
||||
b.create_dataset("d")
|
||||
.with_i32_data(&[1, 2, 3])
|
||||
.set_attr("c0", clawhdf5::AttrValue::I64Array(vec![5, 5]));
|
||||
b.write(&path).unwrap();
|
||||
let mut want: Vec<AttrOp> = (0..12i64)
|
||||
.map(|i| ("/".to_string(), format!("r{i}"), AV::Ints(vec![i; 3])))
|
||||
.collect();
|
||||
want.push(("d".into(), "c0".into(), AV::Ints(vec![5, 5])));
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
for i in 0..30u64 {
|
||||
for (k, o) in ["/", "d"].iter().enumerate() {
|
||||
// Small attributes: a larger one needs a heap block bigger than
|
||||
// the next (see dense_attribute_refusals_change_nothing).
|
||||
let v = match AV::make(i, k as u64 + 7) {
|
||||
AV::Str(s) if s.len() > 400 => AV::Str(s[..400].to_string()),
|
||||
v => v,
|
||||
};
|
||||
ed.set_attr(o, &format!("n{i}"), &v.value())
|
||||
.unwrap_or_else(|e| panic!("{track} {o}/n{i}: {e}"));
|
||||
set_want(&mut want, o, &format!("n{i}"), v);
|
||||
}
|
||||
}
|
||||
// Rewrites in place, then replacements of another size.
|
||||
for i in (0..12i64).step_by(3) {
|
||||
let v = AV::Ints(vec![-i; 3]);
|
||||
ed.set_attr("/", &format!("r{i}"), &v.value()).unwrap();
|
||||
set_want(&mut want, "/", &format!("r{i}"), v);
|
||||
}
|
||||
for i in (1..30u64).step_by(7) {
|
||||
let v = AV::Str(format!("replaced {i}"));
|
||||
match ed.set_attr("d", &format!("n{i}"), &v.value()) {
|
||||
Ok(()) => set_want(&mut want, "d", &format!("n{i}"), v),
|
||||
Err(Error::Unsupported(msg)) => assert!(msg.contains("last object"), "{msg}"),
|
||||
Err(e) => panic!("{e}"),
|
||||
}
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&path, true);
|
||||
let want_py: Vec<AttrOp> = want
|
||||
.iter()
|
||||
.map(|(o, n, v)| {
|
||||
let o = if o == "/" { "/".to_string() } else { o.clone() };
|
||||
(o, n.clone(), v.clone())
|
||||
})
|
||||
.collect();
|
||||
check_root_and_attrs(&path, &want_py);
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 for o in ['/', 'd']:\n\
|
||||
\x20 f[o].attrs['from_h5py'] = np.arange(3)\n\
|
||||
\x20 del f[o].attrs['n2']\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
want.retain(|(_, n, _)| n != "n2");
|
||||
check_tools(&path, true);
|
||||
check_root_and_attrs(&path, &want);
|
||||
}
|
||||
}
|
||||
|
||||
/// `check_attr_values`, with the root group as "/".
|
||||
fn check_root_and_attrs(path: &Path, want: &[AttrOp]) {
|
||||
let f = File::open(path).unwrap();
|
||||
for (o, n, v) in want {
|
||||
let attrs = match o.as_str() {
|
||||
"/" => f.root().attrs().unwrap(),
|
||||
"d" => f.dataset(o).unwrap().attrs().unwrap(),
|
||||
_ => f.group(o).unwrap().attrs().unwrap(),
|
||||
};
|
||||
let got = attrs
|
||||
.get(n.as_str())
|
||||
.unwrap_or_else(|| panic!("{o}/{n} missing"));
|
||||
match (got, v) {
|
||||
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
||||
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
||||
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
||||
other => panic!("{o}/{n}: {other:?}"),
|
||||
}
|
||||
}
|
||||
let exp: Vec<String> = want
|
||||
.iter()
|
||||
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
||||
.collect();
|
||||
let script = format!(
|
||||
"import h5py, numpy as np\n\
|
||||
f = h5py.File({p:?}, 'r')\n\
|
||||
want = [{w}]\n\
|
||||
for o, n, v in want:\n\
|
||||
\x20 a = f[o].attrs[n]\n\
|
||||
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
||||
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
||||
\x20 a = [a] if isinstance(a, int) else a\n\
|
||||
\x20 assert a == v, (o, n)\n\
|
||||
for o in set(x[0] for x in want):\n\
|
||||
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n",
|
||||
p = path.to_str().unwrap(),
|
||||
w = exp.join(", ")
|
||||
);
|
||||
let sp = path.with_extension("check.py");
|
||||
std::fs::write(&sp, script).unwrap();
|
||||
let o = Command::new(python()).arg(&sp).output().unwrap();
|
||||
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
||||
}
|
||||
|
||||
/// What the editor refuses in dense storage — an object larger than the
|
||||
/// next heap block (libhdf5 would skip blocks and record their space as
|
||||
/// free, which this editor does not do) — is `Error::Unsupported`, and the
|
||||
/// file is left byte for byte as it was.
|
||||
#[test]
|
||||
fn dense_attribute_refusals_change_nothing() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
let dir = tmpdir();
|
||||
let path = dir.path().join("dense_refuse.h5");
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'w', libver='v110') as f:\n\
|
||||
\x20 g = f.create_group('g')\n\
|
||||
\x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
let before = std::fs::read(&path).unwrap();
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("x".repeat(2000))));
|
||||
drop(ed);
|
||||
assert!(
|
||||
std::fs::read(&path).unwrap() == before,
|
||||
"a refused edit wrote"
|
||||
);
|
||||
// What libhdf5 does with it instead works on the untouched file.
|
||||
py(&format!(
|
||||
"import h5py\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 f['g'].attrs['big'] = 'x' * 2000\n\
|
||||
\x20 assert len(f['g'].attrs) == 13\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
check_tools(&path, true);
|
||||
}
|
||||
|
||||
@@ -836,19 +836,16 @@ fn attributes_in_place() {
|
||||
.unwrap();
|
||||
want.push(("d", "units".into(), AttrValue::String("km".into())));
|
||||
if *lv != "'earliest'" {
|
||||
// Up to the compact limit (8) and no further; attributes in
|
||||
// dense storage and tracked creation order are refused, and a
|
||||
// refused edit writes nothing.
|
||||
// Up to the compact limit (8), then into dense storage; objects
|
||||
// already in dense storage and ones tracking creation order.
|
||||
ed.set_attr("g", "eighth", &AttrValue::I64(8)).unwrap();
|
||||
want.push(("g", "eighth".into(), AttrValue::I64(8)));
|
||||
let before = std::fs::read(&path).unwrap();
|
||||
unsupported(ed.set_attr("g", "ninth", &AttrValue::I64(9)));
|
||||
unsupported(ed.set_attr("dense", "k0", &AttrValue::I64(1)));
|
||||
unsupported(ed.set_attr("tracked", "b", &AttrValue::I64(1)));
|
||||
assert!(
|
||||
std::fs::read(&path).unwrap() == before,
|
||||
"a refused edit changed the file"
|
||||
);
|
||||
ed.set_attr("g", "ninth", &AttrValue::I64(9)).unwrap();
|
||||
want.push(("g", "ninth".into(), AttrValue::I64(9)));
|
||||
ed.set_attr("dense", "k0", &AttrValue::I64(1)).unwrap();
|
||||
want.push(("dense", "k0".into(), AttrValue::I64(1)));
|
||||
ed.set_attr("tracked", "b", &AttrValue::I64(1)).unwrap();
|
||||
want.push(("tracked", "b".into(), AttrValue::I64(1)));
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&path, *dump);
|
||||
|
||||
Reference in New Issue
Block a user