edit: dense attributes, compact-to-dense transition, creation order
FileEditor::set_attr now handles every attribute storage libhdf5 uses for version-2 object headers: - objects that track (and index) attribute creation order: compact attributes carry their creation index in the message header, the Attribute Info message its maximum; - the move to dense storage when an object reaches its compact limit (or an attribute is too large for a header message), as H5O__attr_create does it: a new fractal heap, name index (v2 B-tree type 8) and, when creation order is indexed, creation-order index (type 9); the compact attributes moved over in header message order, their messages freed; - objects already in dense storage (h5py- or clawhdf5-written): new attributes inserted (H5A__dense_insert), an attribute replaced by one of the same encoded size rewritten in its heap object (H5A__dense_write), otherwise removed from both indexes and the heap and inserted anew. edit/fheap.rs follows H5HF: managed objects go to the best-fitting free section of the heap's free-space manager (FSHD/FSSE, kept as libhdf5 keeps it — sorted sections, counts, section info reallocated when its size changes, the manager deleted when empty); otherwise to a new direct block: the root direct block of an empty heap, else the block at the allocation iterator in the root indirect block (created from the root direct block, doubled as needed), with libhdf5's managed-space, allocated space, free space and iterator bookkeeping. Objects above the managed limit are huge objects in their own space, indexed by the huge-object B-tree (type 1). Removed objects return their space merged with adjacent free space. Refused before anything is written: heaps with I/O filters, child indirect blocks, an object larger than the next heap block (libhdf5 skips blocks and records them as free space), free sections other than those inside direct blocks, removing a direct block's last object (libhdf5 frees the block), directly addressed huge objects. Attributes are encoded as libhdf5 does when h5py opens a file r+ (low bound "earliest"): message version 1 (3 for non-ASCII names), simple dataspaces with their maximum dimensions. Header chunks are now visited in libhdf5's order (FIFO), which is also the order attributes move to dense storage in. Tests (edit_coverage_interop): 40 attributes on each of a plain group, a group tracking and indexing creation order, and a dataset (earliest, v110, latest), some above the 4 KiB managed limit, then same-size rewrites: the heap statistics, free-space sections and both index B-trees node for node equal libhdf5's doing the same through h5py; then replacements of other sizes, h5py adds/deletes/rewrites; h5py, h5dump, h5rs check and our reader agree throughout, h5py's attribute count included. clawhdf5-written dense storage (tracked and untracked) is extended the same way; refusals leave the file byte for byte as it was. edit_interop's attribute test now expects dense storage and tracked creation order to work. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -213,7 +213,6 @@ fn tmpdir() -> tempfile::TempDir {
|
||||
tempfile::TempDir::new_in(base).unwrap()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn unsupported<T: std::fmt::Debug>(r: Result<T, Error>) {
|
||||
match r {
|
||||
Err(Error::Unsupported(_)) => {}
|
||||
@@ -830,3 +829,497 @@ fn btree2_random_chunk_order() {
|
||||
verify(&path, "x", &m);
|
||||
check_tools(&path, false);
|
||||
}
|
||||
|
||||
// ---- dense attributes ----
|
||||
|
||||
/// An attribute value both sides can write with the same encoded size: a
|
||||
/// 1-D int64 array or a fixed-length string.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
enum AV {
|
||||
Ints(Vec<i64>),
|
||||
Str(String),
|
||||
}
|
||||
|
||||
impl AV {
|
||||
fn value(&self) -> clawhdf5::AttrValue {
|
||||
match self {
|
||||
AV::Ints(v) => clawhdf5::AttrValue::I64Array(v.clone()),
|
||||
AV::Str(s) => clawhdf5::AttrValue::String(s.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
fn py(&self) -> String {
|
||||
match self {
|
||||
AV::Ints(v) => format!("np.array({v:?}, dtype='<i8')"),
|
||||
AV::Str(s) => format!("np.bytes_({s:?})"),
|
||||
}
|
||||
}
|
||||
|
||||
fn py_expect(&self) -> String {
|
||||
match self {
|
||||
AV::Ints(v) => format!("{v:?}"),
|
||||
AV::Str(s) => format!("{s:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn make(i: u64, salt: u64) -> Self {
|
||||
match i % 11 {
|
||||
5 => AV::Str("h".repeat(5000 + (salt % 50) as usize)),
|
||||
1 | 4 | 8 => AV::Str(
|
||||
(0..10 + (i * 13 + salt) % 300)
|
||||
.map(|k| (b'a' + ((k + salt) % 26) as u8) as char)
|
||||
.collect(),
|
||||
),
|
||||
_ => AV::Ints(
|
||||
(0..1 + (i + salt) % 9)
|
||||
.map(|k| (k * 31 + salt) as i64)
|
||||
.collect(),
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An object's dense attribute storage as libhdf5 would compare it: the
|
||||
/// heap's statistics and root shape, its free sections (heap offsets and
|
||||
/// sizes), and the shapes of the name and creation-order index B-trees.
|
||||
fn dense_info(path: &Path, obj: &str) -> String {
|
||||
use clawhdf5_format::message_type::MessageType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
let f = File::open(path).unwrap();
|
||||
let sb = f.superblock();
|
||||
let (os, ls) = (sb.offset_size, sb.length_size);
|
||||
assert_eq!((os, ls), (8, 8));
|
||||
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, obj).unwrap();
|
||||
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
|
||||
let Some(m) = oh
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == MessageType::AttributeInfo)
|
||||
else {
|
||||
return "no attribute info".into();
|
||||
};
|
||||
let d = &m.data;
|
||||
let mut p = 2 + if d[1] & 1 != 0 { 2 } else { 0 };
|
||||
let heap = le(&d[p..p + 8]);
|
||||
p += 8;
|
||||
let name = le(&d[p..p + 8]);
|
||||
let order = (d[1] & 2 != 0).then(|| le(&d[p + 8..p + 16]));
|
||||
if heap == u64::MAX {
|
||||
return "compact".into();
|
||||
}
|
||||
let b = std::fs::read(path).unwrap();
|
||||
let h = &b[heap as usize..];
|
||||
assert_eq!(&h[0..4], b"FRHP");
|
||||
// next huge id, huge bt2, free, fs, man size, alloc, iter, nobjs, huge
|
||||
// size, huge objs (8-byte fields from offset 14).
|
||||
let stat = |k: usize| le(&h[14 + 8 * k..22 + 8 * k]);
|
||||
let fs = stat(3);
|
||||
// Root rows: after 12 8-byte fields, width, start, max direct, max
|
||||
// index, start rows, root address.
|
||||
let rows_at = 14 + 96 + 2 + 16 + 2 + 2 + 8;
|
||||
let mut out = format!(
|
||||
"heap next_huge={} free={} man={} alloc={} iter={} nobjs={} huge={}/{} rows={}",
|
||||
stat(0),
|
||||
stat(2),
|
||||
stat(4),
|
||||
stat(5),
|
||||
stat(6),
|
||||
stat(7),
|
||||
stat(8),
|
||||
stat(9),
|
||||
le(&h[rows_at..rows_at + 2])
|
||||
);
|
||||
if fs != u64::MAX {
|
||||
let s = &b[fs as usize..];
|
||||
assert_eq!(&s[0..4], b"FSHD");
|
||||
let tot = le(&s[6..14]);
|
||||
let n = le(&s[14..22]);
|
||||
// counts (4 x 8), 4 x u16, max section size, section info address.
|
||||
let at = 6 + 32 + 8 + 8;
|
||||
let sect_addr = le(&s[at..at + 8]);
|
||||
let sect_size = le(&s[at + 8..at + 16]);
|
||||
let ss = &b[sect_addr as usize..(sect_addr + sect_size) as usize];
|
||||
// Sections after the prefix (signature, version, header address);
|
||||
// trailing zero padding and the checksum are left out.
|
||||
let body = &ss[13..ss.len() - 4];
|
||||
let used = body.len() - body.iter().rev().take_while(|&&x| x == 0).count();
|
||||
out += &format!(" fs tot={tot} n={n} sections={}", hex(&body[..used]));
|
||||
} else {
|
||||
out += " no-fs";
|
||||
}
|
||||
out += &format!(" names={:?}", bt2_shape_at(&b, name as usize));
|
||||
if let Some(o) = order {
|
||||
out += &format!(" order={:?}", bt2_shape_at(&b, o as usize));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn hex(b: &[u8]) -> String {
|
||||
b.iter().map(|x| format!("{x:02x}")).collect()
|
||||
}
|
||||
|
||||
/// h5py and our reader both see `want` on each object (other attributes
|
||||
/// may exist; these must have these values), and h5py's attribute count
|
||||
/// agrees with libhdf5's object info.
|
||||
fn check_attr_values(path: &Path, want: &[(String, String, AV)]) {
|
||||
let f = File::open(path).unwrap();
|
||||
for (o, n, v) in want {
|
||||
let attrs = if o == "d" {
|
||||
f.dataset(o).unwrap().attrs().unwrap()
|
||||
} else {
|
||||
f.group(o).unwrap().attrs().unwrap()
|
||||
};
|
||||
let got = attrs
|
||||
.get(n.as_str())
|
||||
.unwrap_or_else(|| panic!("{o}/{n} missing ({} attributes)", attrs.len()));
|
||||
match (got, v) {
|
||||
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
||||
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
||||
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
||||
other => panic!("{o}/{n}: {other:?}"),
|
||||
}
|
||||
}
|
||||
let exp: Vec<String> = want
|
||||
.iter()
|
||||
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
||||
.collect();
|
||||
let script = format!(
|
||||
"import h5py, numpy as np\n\
|
||||
f = h5py.File({p:?}, 'r')\n\
|
||||
want = [{w}]\n\
|
||||
for o, n, v in want:\n\
|
||||
\x20 a = f[o].attrs[n]\n\
|
||||
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
||||
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
||||
\x20 a = [a] if isinstance(a, int) else a\n\
|
||||
\x20 assert a == v, (o, n, a if len(str(a)) < 200 else len(a), v if len(str(v)) < 200 else len(v))\n\
|
||||
for o in set(x[0] for x in want):\n\
|
||||
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n\
|
||||
\x20 assert len(list(f[o].attrs)) == len(f[o].attrs)\n",
|
||||
p = path.to_str().unwrap(),
|
||||
w = exp.join(", ")
|
||||
);
|
||||
let sp = path.with_extension("check.py");
|
||||
std::fs::write(&sp, script).unwrap();
|
||||
let o = Command::new(python()).arg(&sp).output().unwrap();
|
||||
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
||||
}
|
||||
|
||||
/// Run python statements (inside `with h5py.File(path, 'r+') as f:`).
|
||||
fn py_r_plus(path: &Path, lines: &[String]) {
|
||||
let script = format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n{l}",
|
||||
p = path.to_str().unwrap(),
|
||||
l = lines.concat()
|
||||
);
|
||||
let sp = path.with_extension("ops.py");
|
||||
std::fs::write(&sp, script).unwrap();
|
||||
let o = Command::new(python()).arg(&sp).output().unwrap();
|
||||
assert!(o.status.success(), "h5py workload failed:\n{}", text(&o));
|
||||
}
|
||||
|
||||
type AttrOp = (String, String, AV);
|
||||
|
||||
fn set_want(want: &mut Vec<AttrOp>, o: &str, n: &str, v: AV) {
|
||||
want.retain(|(wo, wn, _)| !(wo == o && wn == n));
|
||||
want.push((o.into(), n.into(), v));
|
||||
}
|
||||
|
||||
/// The same attribute workload by libhdf5 and by the editor on objects
|
||||
/// with compact attributes that move to dense storage (a plain group, a
|
||||
/// group tracking and indexing creation order, a dataset): 40 new
|
||||
/// attributes each (some larger than the heap's 4 KiB managed limit), then
|
||||
/// same-size rewrites — the heaps, their free space and both index B-trees
|
||||
/// must come out as libhdf5 makes them; then replacements of another size,
|
||||
/// then h5py adds, deletes and rewrites attributes.
|
||||
fn dense_workload(libver: &str, h5dump: bool, tag: &str) {
|
||||
let dir = tmpdir();
|
||||
let a = dir.path().join(format!("dense_{tag}_h5py.h5"));
|
||||
let b = dir.path().join(format!("dense_{tag}_edit.h5"));
|
||||
for p in [&a, &b] {
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
|
||||
\x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \
|
||||
f.create_dataset('d', data=np.arange(4, dtype='<i4'))]\n\
|
||||
\x20 for o in objs:\n\
|
||||
\x20 for i in range(3): o.attrs.create(f'c{{i}}', np.array([i, i], dtype='<i8'))\n",
|
||||
p = p.to_str().unwrap()
|
||||
));
|
||||
}
|
||||
let objs = ["g", "t", "d"];
|
||||
let mut want: Vec<AttrOp> = Vec::new();
|
||||
for o in objs {
|
||||
for i in 0..3i64 {
|
||||
want.push((o.into(), format!("c{i}"), AV::Ints(vec![i, i])));
|
||||
}
|
||||
}
|
||||
// Phase 1: new attributes.
|
||||
let mut ops: Vec<AttrOp> = Vec::new();
|
||||
for i in 0..40u64 {
|
||||
for (k, o) in objs.iter().enumerate() {
|
||||
ops.push((o.to_string(), format!("n{i}"), AV::make(i, k as u64)));
|
||||
}
|
||||
}
|
||||
let lines: Vec<String> = ops
|
||||
.iter()
|
||||
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.create({n:?}, {})\n", v.py()))
|
||||
.collect();
|
||||
py_r_plus(&a, &lines);
|
||||
let mut ed = FileEditor::open(&b).unwrap();
|
||||
for (o, n, v) in &ops {
|
||||
ed.set_attr(o, n, &v.value())
|
||||
.unwrap_or_else(|e| panic!("{tag}: set {o}/{n}: {e}"));
|
||||
set_want(&mut want, o, n, v.clone());
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&b, h5dump);
|
||||
check_attr_values(&b, &want);
|
||||
check_attr_values(&a, &want);
|
||||
for o in objs {
|
||||
assert_eq!(
|
||||
dense_info(&b, o),
|
||||
dense_info(&a, o),
|
||||
"{tag}: dense storage of {o} differs from libhdf5's after insertions"
|
||||
);
|
||||
}
|
||||
// Phase 2: same-size rewrites (H5Awrite in place).
|
||||
let ops2: Vec<AttrOp> = ops
|
||||
.iter()
|
||||
.step_by(4)
|
||||
.map(|(o, n, v)| {
|
||||
let nv = match v {
|
||||
AV::Ints(x) => AV::Ints(x.iter().map(|y| y * 3 + 1).collect()),
|
||||
AV::Str(s) => AV::Str(s.chars().rev().collect()),
|
||||
};
|
||||
(o.clone(), n.clone(), nv)
|
||||
})
|
||||
.collect();
|
||||
let lines: Vec<String> = ops2
|
||||
.iter()
|
||||
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.modify({n:?}, {})\n", v.py()))
|
||||
.collect();
|
||||
py_r_plus(&a, &lines);
|
||||
let mut ed = FileEditor::open(&b).unwrap();
|
||||
for (o, n, v) in &ops2 {
|
||||
ed.set_attr(o, n, &v.value()).unwrap();
|
||||
set_want(&mut want, o, n, v.clone());
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&b, h5dump);
|
||||
check_attr_values(&b, &want);
|
||||
for o in objs {
|
||||
assert_eq!(
|
||||
dense_info(&b, o),
|
||||
dense_info(&a, o),
|
||||
"{tag}: dense storage of {o} differs from libhdf5's after rewrites"
|
||||
);
|
||||
}
|
||||
// Phase 3: replacements of another size (values only: h5py replaces
|
||||
// through a temporary attribute and a rename).
|
||||
let mut ed = FileEditor::open(&b).unwrap();
|
||||
for (k, (o, n, v)) in ops.iter().enumerate().filter(|(k, _)| k % 5 == 2) {
|
||||
let nv = match v {
|
||||
AV::Ints(x) => AV::Ints((0..x.len() as i64 + 3).collect()),
|
||||
AV::Str(s) => AV::Str(format!("{s}-{k}")),
|
||||
};
|
||||
let before = std::fs::read(&b).unwrap();
|
||||
match ed.set_attr(o, n, &nv.value()) {
|
||||
Ok(()) => set_want(&mut want, o, n, nv),
|
||||
Err(Error::Unsupported(msg)) => {
|
||||
assert!(msg.contains("last object"), "{tag}: {o}/{n}: {msg}");
|
||||
assert!(std::fs::read(&b).unwrap() == before, "refused edit wrote");
|
||||
}
|
||||
Err(e) => panic!("{tag}: replace {o}/{n}: {e}"),
|
||||
}
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&b, h5dump);
|
||||
check_attr_values(&b, &want);
|
||||
// libhdf5 goes on: adds, deletes and rewrites.
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 for o in ['g', 't', 'd']:\n\
|
||||
\x20 for i in range(5): f[o].attrs[f'late{{i}}'] = np.arange(i + 1)\n\
|
||||
\x20 del f[o].attrs['n3']\n\
|
||||
\x20 del f[o].attrs['c1']\n\
|
||||
\x20 f[o].attrs['n7'] = 'rewritten by h5py'\n",
|
||||
p = b.to_str().unwrap()
|
||||
));
|
||||
want.retain(|(_, n, _)| n != "n3" && n != "c1" && n != "n7");
|
||||
check_tools(&b, h5dump);
|
||||
check_attr_values(&b, &want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_attributes_match_libhdf5() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
dense_workload(lv, *dump, &format!("{i}"));
|
||||
}
|
||||
}
|
||||
|
||||
/// Dense attributes in files clawhdf5 writes (its own heap and index
|
||||
/// layout, no free-space manager), with and without tracked creation
|
||||
/// order: attributes added (moving a dataset's attributes to dense storage
|
||||
/// too), rewritten and replaced, then h5py goes on.
|
||||
#[test]
|
||||
fn dense_attributes_on_clawhdf5_files() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
for track in [false, true] {
|
||||
let dir = tmpdir();
|
||||
let path = dir.path().join(format!("ours_dense_{track}.h5"));
|
||||
let mut b = clawhdf5::FileBuilder::new();
|
||||
b.track_order(track);
|
||||
for i in 0..12i64 {
|
||||
b.set_attr(&format!("r{i}"), clawhdf5::AttrValue::I64Array(vec![i; 3]));
|
||||
}
|
||||
b.create_dataset("d")
|
||||
.with_i32_data(&[1, 2, 3])
|
||||
.set_attr("c0", clawhdf5::AttrValue::I64Array(vec![5, 5]));
|
||||
b.write(&path).unwrap();
|
||||
let mut want: Vec<AttrOp> = (0..12i64)
|
||||
.map(|i| ("/".to_string(), format!("r{i}"), AV::Ints(vec![i; 3])))
|
||||
.collect();
|
||||
want.push(("d".into(), "c0".into(), AV::Ints(vec![5, 5])));
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
for i in 0..30u64 {
|
||||
for (k, o) in ["/", "d"].iter().enumerate() {
|
||||
// Small attributes: a larger one needs a heap block bigger than
|
||||
// the next (see dense_attribute_refusals_change_nothing).
|
||||
let v = match AV::make(i, k as u64 + 7) {
|
||||
AV::Str(s) if s.len() > 400 => AV::Str(s[..400].to_string()),
|
||||
v => v,
|
||||
};
|
||||
ed.set_attr(o, &format!("n{i}"), &v.value())
|
||||
.unwrap_or_else(|e| panic!("{track} {o}/n{i}: {e}"));
|
||||
set_want(&mut want, o, &format!("n{i}"), v);
|
||||
}
|
||||
}
|
||||
// Rewrites in place, then replacements of another size.
|
||||
for i in (0..12i64).step_by(3) {
|
||||
let v = AV::Ints(vec![-i; 3]);
|
||||
ed.set_attr("/", &format!("r{i}"), &v.value()).unwrap();
|
||||
set_want(&mut want, "/", &format!("r{i}"), v);
|
||||
}
|
||||
for i in (1..30u64).step_by(7) {
|
||||
let v = AV::Str(format!("replaced {i}"));
|
||||
match ed.set_attr("d", &format!("n{i}"), &v.value()) {
|
||||
Ok(()) => set_want(&mut want, "d", &format!("n{i}"), v),
|
||||
Err(Error::Unsupported(msg)) => assert!(msg.contains("last object"), "{msg}"),
|
||||
Err(e) => panic!("{e}"),
|
||||
}
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&path, true);
|
||||
let want_py: Vec<AttrOp> = want
|
||||
.iter()
|
||||
.map(|(o, n, v)| {
|
||||
let o = if o == "/" { "/".to_string() } else { o.clone() };
|
||||
(o, n.clone(), v.clone())
|
||||
})
|
||||
.collect();
|
||||
check_root_and_attrs(&path, &want_py);
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 for o in ['/', 'd']:\n\
|
||||
\x20 f[o].attrs['from_h5py'] = np.arange(3)\n\
|
||||
\x20 del f[o].attrs['n2']\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
want.retain(|(_, n, _)| n != "n2");
|
||||
check_tools(&path, true);
|
||||
check_root_and_attrs(&path, &want);
|
||||
}
|
||||
}
|
||||
|
||||
/// `check_attr_values`, with the root group as "/".
|
||||
fn check_root_and_attrs(path: &Path, want: &[AttrOp]) {
|
||||
let f = File::open(path).unwrap();
|
||||
for (o, n, v) in want {
|
||||
let attrs = match o.as_str() {
|
||||
"/" => f.root().attrs().unwrap(),
|
||||
"d" => f.dataset(o).unwrap().attrs().unwrap(),
|
||||
_ => f.group(o).unwrap().attrs().unwrap(),
|
||||
};
|
||||
let got = attrs
|
||||
.get(n.as_str())
|
||||
.unwrap_or_else(|| panic!("{o}/{n} missing"));
|
||||
match (got, v) {
|
||||
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
||||
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
||||
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
||||
other => panic!("{o}/{n}: {other:?}"),
|
||||
}
|
||||
}
|
||||
let exp: Vec<String> = want
|
||||
.iter()
|
||||
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
||||
.collect();
|
||||
let script = format!(
|
||||
"import h5py, numpy as np\n\
|
||||
f = h5py.File({p:?}, 'r')\n\
|
||||
want = [{w}]\n\
|
||||
for o, n, v in want:\n\
|
||||
\x20 a = f[o].attrs[n]\n\
|
||||
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
||||
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
||||
\x20 a = [a] if isinstance(a, int) else a\n\
|
||||
\x20 assert a == v, (o, n)\n\
|
||||
for o in set(x[0] for x in want):\n\
|
||||
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n",
|
||||
p = path.to_str().unwrap(),
|
||||
w = exp.join(", ")
|
||||
);
|
||||
let sp = path.with_extension("check.py");
|
||||
std::fs::write(&sp, script).unwrap();
|
||||
let o = Command::new(python()).arg(&sp).output().unwrap();
|
||||
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
||||
}
|
||||
|
||||
/// What the editor refuses in dense storage — an object larger than the
|
||||
/// next heap block (libhdf5 would skip blocks and record their space as
|
||||
/// free, which this editor does not do) — is `Error::Unsupported`, and the
|
||||
/// file is left byte for byte as it was.
|
||||
#[test]
|
||||
fn dense_attribute_refusals_change_nothing() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
let dir = tmpdir();
|
||||
let path = dir.path().join("dense_refuse.h5");
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'w', libver='v110') as f:\n\
|
||||
\x20 g = f.create_group('g')\n\
|
||||
\x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
let before = std::fs::read(&path).unwrap();
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("x".repeat(2000))));
|
||||
drop(ed);
|
||||
assert!(
|
||||
std::fs::read(&path).unwrap() == before,
|
||||
"a refused edit wrote"
|
||||
);
|
||||
// What libhdf5 does with it instead works on the untouched file.
|
||||
py(&format!(
|
||||
"import h5py\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 f['g'].attrs['big'] = 'x' * 2000\n\
|
||||
\x20 assert len(f['g'].attrs) == 13\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
check_tools(&path, true);
|
||||
}
|
||||
|
||||
@@ -836,19 +836,16 @@ fn attributes_in_place() {
|
||||
.unwrap();
|
||||
want.push(("d", "units".into(), AttrValue::String("km".into())));
|
||||
if *lv != "'earliest'" {
|
||||
// Up to the compact limit (8) and no further; attributes in
|
||||
// dense storage and tracked creation order are refused, and a
|
||||
// refused edit writes nothing.
|
||||
// Up to the compact limit (8), then into dense storage; objects
|
||||
// already in dense storage and ones tracking creation order.
|
||||
ed.set_attr("g", "eighth", &AttrValue::I64(8)).unwrap();
|
||||
want.push(("g", "eighth".into(), AttrValue::I64(8)));
|
||||
let before = std::fs::read(&path).unwrap();
|
||||
unsupported(ed.set_attr("g", "ninth", &AttrValue::I64(9)));
|
||||
unsupported(ed.set_attr("dense", "k0", &AttrValue::I64(1)));
|
||||
unsupported(ed.set_attr("tracked", "b", &AttrValue::I64(1)));
|
||||
assert!(
|
||||
std::fs::read(&path).unwrap() == before,
|
||||
"a refused edit changed the file"
|
||||
);
|
||||
ed.set_attr("g", "ninth", &AttrValue::I64(9)).unwrap();
|
||||
want.push(("g", "ninth".into(), AttrValue::I64(9)));
|
||||
ed.set_attr("dense", "k0", &AttrValue::I64(1)).unwrap();
|
||||
want.push(("dense", "k0".into(), AttrValue::I64(1)));
|
||||
ed.set_attr("tracked", "b", &AttrValue::I64(1)).unwrap();
|
||||
want.push(("tracked", "b".into(), AttrValue::I64(1)));
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&path, *dump);
|
||||
|
||||
@@ -0,0 +1,513 @@
|
||||
//! Setting attributes, as `H5O__attr_create` / `H5A__dense_insert` do:
|
||||
//! compact attributes are object header messages (with their creation
|
||||
//! index in the message header when the object tracks creation order);
|
||||
//! when an object reaches its compact limit (or an attribute is too large
|
||||
//! for a header message) its attributes move to dense storage — a fractal
|
||||
//! heap for the encoded messages, a version-2 B-tree indexing them by name
|
||||
//! hash (record type 8) and, when creation order is indexed, a second one
|
||||
//! by creation index (type 9) — and the Attribute Info message points at
|
||||
//! them.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
|
||||
use clawhdf5_format::attribute::AttributeMessage;
|
||||
use clawhdf5_format::dataspace::DataspaceType;
|
||||
|
||||
use crate::edit::btree2::Bt2;
|
||||
use crate::edit::fheap::Heap;
|
||||
use crate::edit::image::{Image, get_uint, put_uint, undef};
|
||||
use crate::edit::ohdr::{Header, MSG_ATTRIBUTE};
|
||||
use crate::edit::{MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, MSG_FLAG_SHARED, check_plain};
|
||||
use crate::error::Error;
|
||||
use crate::reader::File;
|
||||
use crate::types::AttrValue;
|
||||
|
||||
/// `H5O_MESG_MAX_SIZE`: a larger attribute goes to dense storage.
|
||||
const MESG_MAX_SIZE: usize = 65536;
|
||||
/// `H5O_MAX_CRT_ORDER_IDX`: the creation index of an attribute of an object
|
||||
/// that does not track creation order.
|
||||
const NO_CRT_IDX: u16 = u16::MAX;
|
||||
/// Name and creation-order index B-trees (`H5A_NAME_BT2_*`,
|
||||
/// `H5A_CORDER_BT2_*`).
|
||||
const NAME_BT2_TYPE: u8 = 8;
|
||||
const CORDER_BT2_TYPE: u8 = 9;
|
||||
const ATTR_BT2_NODE: u32 = 512;
|
||||
/// Heap IDs in attribute records.
|
||||
const ID_LEN: usize = 8;
|
||||
|
||||
/// An object's Attribute Info message.
|
||||
#[derive(Debug, Clone)]
|
||||
struct AInfo {
|
||||
/// Its message index in the header.
|
||||
idx: usize,
|
||||
track: bool,
|
||||
index: bool,
|
||||
max_crt: u16,
|
||||
fheap: u64,
|
||||
name_bt2: u64,
|
||||
corder_bt2: u64,
|
||||
}
|
||||
|
||||
impl AInfo {
|
||||
fn load(img: &Image<'_>, hdr: &Header) -> Result<Option<Self>, Error> {
|
||||
let Some(idx) = hdr.find(MSG_ATTR_INFO) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if hdr.msgs[idx].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared attribute info message".into()));
|
||||
}
|
||||
let d = hdr.data(img, idx)?;
|
||||
let os = img.os as usize;
|
||||
let short = || Error::Unsupported("short attribute info message".into());
|
||||
if d.first() != Some(&0) {
|
||||
return Err(Error::Unsupported("attribute info message version".into()));
|
||||
}
|
||||
let flags = *d.get(1).ok_or_else(short)?;
|
||||
let track = flags & 0x01 != 0;
|
||||
let index = flags & 0x02 != 0;
|
||||
let mut p = 2;
|
||||
let mut max_crt = 0;
|
||||
if track {
|
||||
let b = d.get(p..p + 2).ok_or_else(short)?;
|
||||
max_crt = u16::from_le_bytes([b[0], b[1]]);
|
||||
p += 2;
|
||||
}
|
||||
let n = if index { 3 } else { 2 };
|
||||
if d.len() < p + n * os {
|
||||
return Err(short());
|
||||
}
|
||||
Ok(Some(Self {
|
||||
idx,
|
||||
track,
|
||||
index,
|
||||
max_crt,
|
||||
fheap: get_uint(&d[p..], img.os),
|
||||
name_bt2: get_uint(&d[p + os..], img.os),
|
||||
corder_bt2: if index {
|
||||
get_uint(&d[p + 2 * os..], img.os)
|
||||
} else {
|
||||
undef(img.os)
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
fn dense(&self, os: u8) -> bool {
|
||||
self.fheap != undef(os)
|
||||
}
|
||||
|
||||
/// Store the changeable fields back into the message.
|
||||
fn store(&self, img: &mut Image<'_>, hdr: &mut Header) -> Result<(), Error> {
|
||||
let os = img.os as usize;
|
||||
let mut p = 2;
|
||||
if self.track {
|
||||
hdr.patch(img, self.idx, p, &self.max_crt.to_le_bytes())?;
|
||||
p += 2;
|
||||
}
|
||||
let mut a = vec![0u8; os];
|
||||
for (k, v) in [self.fheap, self.name_bt2, self.corder_bt2]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.take(if self.index { 3 } else { 2 })
|
||||
{
|
||||
put_uint(&mut a, v, img.os);
|
||||
hdr.patch(img, self.idx, p + k * os, &a)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The next creation index (`H5O__attr_create`), or libhdf5's "none".
|
||||
fn next_crt(&mut self) -> Result<u16, Error> {
|
||||
if !self.track {
|
||||
return Ok(NO_CRT_IDX);
|
||||
}
|
||||
if self.max_crt == NO_CRT_IDX {
|
||||
return Err(Error::Unsupported(
|
||||
"object's attribute creation index is exhausted".into(),
|
||||
));
|
||||
}
|
||||
self.max_crt += 1;
|
||||
Ok(self.max_crt - 1)
|
||||
}
|
||||
}
|
||||
|
||||
/// The name bytes of an attribute message body (without the NUL).
|
||||
pub(super) fn attr_name(d: &[u8]) -> Result<&[u8], Error> {
|
||||
let bad = || Error::Unsupported("malformed attribute message".into());
|
||||
let (len, at) = match d.first() {
|
||||
Some(1) | Some(2) if d.len() >= 8 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8),
|
||||
Some(3) if d.len() >= 9 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9),
|
||||
_ => return Err(bad()),
|
||||
};
|
||||
let name = d.get(at..at + len).ok_or_else(bad)?;
|
||||
Ok(name.split(|&b| b == 0).next().unwrap_or(name))
|
||||
}
|
||||
|
||||
/// A new Attribute Info message for a version-2 header with flags
|
||||
/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order
|
||||
/// tracked / indexed as the header's flags say, the maximum creation index,
|
||||
/// and no dense storage (undefined fractal heap and B-tree addresses).
|
||||
fn attr_info_message(hdr_flags: u8, max_crt: u16, os: u8) -> Vec<u8> {
|
||||
let track = hdr_flags & 0x04 != 0;
|
||||
let index = hdr_flags & 0x08 != 0;
|
||||
let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)];
|
||||
if track {
|
||||
b.extend_from_slice(&max_crt.to_le_bytes());
|
||||
}
|
||||
let undef_addr = vec![0xffu8; os as usize];
|
||||
b.extend_from_slice(&undef_addr);
|
||||
b.extend_from_slice(&undef_addr);
|
||||
if index {
|
||||
b.extend_from_slice(&undef_addr);
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
/// A version-2 header's limit on compact attributes: stored when its flags
|
||||
/// say so, else libhdf5's default of 8.
|
||||
fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result<u16, Error> {
|
||||
if hdr.flags & 0x10 == 0 {
|
||||
return Ok(8);
|
||||
}
|
||||
let mut p = hdr.addr + 6;
|
||||
if hdr.flags & 0x20 != 0 {
|
||||
p += 16;
|
||||
}
|
||||
let b = img.read(p, 2)?;
|
||||
Ok(u16::from_le_bytes([b[0], b[1]]))
|
||||
}
|
||||
|
||||
/// A version-1 attribute message (what libhdf5 writes in a version-1 object
|
||||
/// header): name, datatype and dataspace each padded to 8 bytes, the
|
||||
/// dataspace as a version-1 dataspace message.
|
||||
fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec<u8> {
|
||||
let mut name = a.name.as_bytes().to_vec();
|
||||
name.push(0);
|
||||
let dt = a.datatype.serialize();
|
||||
let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0];
|
||||
if a.dataspace.space_type == DataspaceType::Simple {
|
||||
let mut b = vec![0u8; ls as usize];
|
||||
for &d in &a.dataspace.dimensions {
|
||||
put_uint(&mut b, d, ls);
|
||||
ds.extend_from_slice(&b);
|
||||
}
|
||||
if let Some(max) = &a.dataspace.max_dimensions {
|
||||
ds[2] = 0x01;
|
||||
for &d in max {
|
||||
put_uint(&mut b, d, ls);
|
||||
ds.extend_from_slice(&b);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ds[1] = 0;
|
||||
}
|
||||
let mut out = vec![1u8, 0];
|
||||
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(dt.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(ds.len() as u16).to_le_bytes());
|
||||
for part in [&name, &dt, &ds] {
|
||||
out.extend_from_slice(part);
|
||||
out.resize(out.len().next_multiple_of(8), 0);
|
||||
}
|
||||
out.extend_from_slice(&a.raw_data);
|
||||
out
|
||||
}
|
||||
|
||||
/// Dense storage opened for changes.
|
||||
struct Dense {
|
||||
heap: Heap,
|
||||
names: Bt2,
|
||||
order: Option<Bt2>,
|
||||
}
|
||||
|
||||
/// `H5_checksum_lookup3` of a name, as the name index keys it.
|
||||
fn name_hash(name: &[u8]) -> u32 {
|
||||
clawhdf5_format::checksum::jenkins_lookup3(name)
|
||||
}
|
||||
|
||||
/// Compare attribute `name` (hash `hash`) with a name-index record
|
||||
/// (`H5A__dense_btree2_name_compare`: the hash, then the stored name).
|
||||
fn cmp_name(
|
||||
heap: &Heap,
|
||||
img: &Image<'_>,
|
||||
hash: u32,
|
||||
name: &[u8],
|
||||
rec: &[u8],
|
||||
) -> Result<Ordering, Error> {
|
||||
let theirs = u32::from_le_bytes([rec[13], rec[14], rec[15], rec[16]]);
|
||||
match hash.cmp(&theirs) {
|
||||
Ordering::Equal => {
|
||||
if rec[ID_LEN] & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported(
|
||||
"shared attribute in dense storage".into(),
|
||||
));
|
||||
}
|
||||
let obj = heap.read(img, &rec[..ID_LEN])?;
|
||||
Ok(name.cmp(attr_name(&obj)?))
|
||||
}
|
||||
o => Ok(o),
|
||||
}
|
||||
}
|
||||
|
||||
fn corder_of(rec: &[u8]) -> u32 {
|
||||
u32::from_le_bytes([rec[9], rec[10], rec[11], rec[12]])
|
||||
}
|
||||
|
||||
impl Dense {
|
||||
fn open(img: &Image<'_>, ai: &AInfo) -> Result<Self, Error> {
|
||||
let heap = Heap::open(img, ai.fheap)?;
|
||||
let names = Bt2::open(img, ai.name_bt2)?;
|
||||
if names.tree_type() != NAME_BT2_TYPE || names.record_size() != ID_LEN + 9 {
|
||||
return Err(Error::Unsupported("attribute name index layout".into()));
|
||||
}
|
||||
let order = if ai.index {
|
||||
let t = Bt2::open(img, ai.corder_bt2)?;
|
||||
if t.tree_type() != CORDER_BT2_TYPE || t.record_size() != ID_LEN + 5 {
|
||||
return Err(Error::Unsupported(
|
||||
"attribute creation-order index layout".into(),
|
||||
));
|
||||
}
|
||||
Some(t)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(Self { heap, names, order })
|
||||
}
|
||||
|
||||
/// `H5A__dense_create`: heap, name index, [creation-order index].
|
||||
fn create(img: &mut Image<'_>, index: bool) -> Result<Self, Error> {
|
||||
let heap = Heap::create_attribute_heap(img)?;
|
||||
let names = Bt2::create(img, NAME_BT2_TYPE, ATTR_BT2_NODE, ID_LEN + 9, 100, 40)?;
|
||||
let order = if index {
|
||||
Some(Bt2::create(
|
||||
img,
|
||||
CORDER_BT2_TYPE,
|
||||
ATTR_BT2_NODE,
|
||||
ID_LEN + 5,
|
||||
100,
|
||||
40,
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(Self { heap, names, order })
|
||||
}
|
||||
|
||||
/// `H5A__dense_insert` of an encoded attribute message.
|
||||
fn insert(&mut self, img: &mut Image<'_>, body: &[u8], crt: u16) -> Result<(), Error> {
|
||||
let name = attr_name(body)?.to_vec();
|
||||
let id = self.heap.insert(img, body)?;
|
||||
if id.len() != ID_LEN {
|
||||
return Err(Error::Unsupported("attribute heap ID length".into()));
|
||||
}
|
||||
let hash = name_hash(&name);
|
||||
let mut rec = id.clone();
|
||||
rec.push(0);
|
||||
rec.extend_from_slice(&u32::from(crt).to_le_bytes());
|
||||
rec.extend_from_slice(&hash.to_le_bytes());
|
||||
let heap = &self.heap;
|
||||
self.names
|
||||
.insert(img, &mut |im, r| cmp_name(heap, im, hash, &name, r), &rec)?;
|
||||
if let Some(t) = &mut self.order {
|
||||
let key = u32::from(crt);
|
||||
t.insert(
|
||||
img,
|
||||
&mut |_, r| Ok(key.cmp(&corder_of(r))),
|
||||
&rec[..ID_LEN + 5],
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> {
|
||||
self.heap.finish(img)?;
|
||||
self.names.finish(img)?;
|
||||
if let Some(t) = &mut self.order {
|
||||
t.finish(img)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Set attribute `name` of the object at `path` to `value`.
|
||||
pub(super) fn set_attr(
|
||||
f: &File,
|
||||
img: &mut Image<'_>,
|
||||
path: &str,
|
||||
name: &str,
|
||||
value: &AttrValue,
|
||||
) -> Result<(), Error> {
|
||||
let addr = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?;
|
||||
let mut hdr = Header::load(img, addr)?;
|
||||
let mut msg = clawhdf5_format::type_builders::build_attr_message(name, value);
|
||||
check_plain(&msg.datatype)?;
|
||||
// libhdf5 encodes a simple dataspace with its maximum dimensions (the
|
||||
// current ones when none were given), so an attribute takes the same
|
||||
// space in a header or heap as when libhdf5 writes it.
|
||||
if msg.dataspace.space_type == DataspaceType::Simple && msg.dataspace.max_dimensions.is_none() {
|
||||
msg.dataspace.max_dimensions = Some(msg.dataspace.dimensions.clone());
|
||||
}
|
||||
// H5A__set_version: version 1 unless the name is not ASCII (then 3),
|
||||
// raised to the file's low bound — which is the earliest for a file
|
||||
// libhdf5 opens without a libver setting (h5py's `r+`).
|
||||
let body = if hdr.version == 1 || name.is_ascii() {
|
||||
encode_attr_v1(&msg, img.ls)
|
||||
} else {
|
||||
let mut b = msg.serialize_v3(img.ls);
|
||||
if !name.is_ascii() {
|
||||
b[8] = 1; // UTF-8 name
|
||||
}
|
||||
b
|
||||
};
|
||||
let mut ainfo = if hdr.version == 2 {
|
||||
AInfo::load(img, &hdr)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(ai) = ainfo.as_mut().filter(|a| a.dense(img.os)) {
|
||||
let mut ai = ai.clone();
|
||||
set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?;
|
||||
return hdr.finish(img);
|
||||
}
|
||||
|
||||
let mut existing = None;
|
||||
let mut count = 0usize;
|
||||
for i in 0..hdr.msgs.len() {
|
||||
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
|
||||
continue;
|
||||
}
|
||||
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared attribute message".into()));
|
||||
}
|
||||
count += 1;
|
||||
if attr_name(&hdr.data(img, i)?)? == name.as_bytes() {
|
||||
existing = Some(i);
|
||||
}
|
||||
}
|
||||
if let Some(i) = existing {
|
||||
hdr.delete(img, i)?;
|
||||
count -= 1;
|
||||
}
|
||||
if hdr.version == 1 {
|
||||
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?;
|
||||
return hdr.finish(img);
|
||||
}
|
||||
let tracked = hdr.flags & 0x04 != 0;
|
||||
// H5O__attr_create: a missing Attribute Info message starts from
|
||||
// nothing (and is added below, holding the new maximum creation index).
|
||||
let new_ainfo = ainfo.is_none();
|
||||
let mut ai = ainfo.take().unwrap_or(AInfo {
|
||||
idx: usize::MAX,
|
||||
track: tracked,
|
||||
index: hdr.flags & 0x08 != 0,
|
||||
max_crt: 0,
|
||||
fheap: undef(img.os),
|
||||
name_bt2: undef(img.os),
|
||||
corder_bt2: undef(img.os),
|
||||
});
|
||||
let max_compact = usize::from(max_compact_attrs(img, &hdr)?);
|
||||
if count == max_compact || body.len() >= MESG_MAX_SIZE {
|
||||
if new_ainfo {
|
||||
return Err(Error::Unsupported(
|
||||
"dense attribute storage for an object without an Attribute Info message".into(),
|
||||
));
|
||||
}
|
||||
to_dense(img, &mut hdr, &mut ai)?;
|
||||
set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?;
|
||||
return hdr.finish(img);
|
||||
}
|
||||
let crt = ai.next_crt()?;
|
||||
let corder = tracked.then_some(crt);
|
||||
if new_ainfo {
|
||||
// libhdf5 appends the Attribute Info message before the attribute
|
||||
// when free space holds both, else after it, so that a new
|
||||
// continuation chunk made for the attribute has room for it too.
|
||||
let a = attr_info_message(hdr.flags, ai.max_crt, img.os);
|
||||
let first = hdr.has_free(a.len() + hdr.hsize() + body.len());
|
||||
if first {
|
||||
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?;
|
||||
}
|
||||
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?;
|
||||
if !first {
|
||||
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?;
|
||||
}
|
||||
} else {
|
||||
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?;
|
||||
ai.store(img, &mut hdr)?;
|
||||
}
|
||||
hdr.finish(img)
|
||||
}
|
||||
|
||||
/// Move every compact attribute of the object into new dense storage, in
|
||||
/// header message order (`H5O__attr_to_dense_cb`), leaving free space where
|
||||
/// the messages were.
|
||||
fn to_dense(img: &mut Image<'_>, hdr: &mut Header, ai: &mut AInfo) -> Result<(), Error> {
|
||||
let mut dense = Dense::create(img, ai.index)?;
|
||||
for i in 0..hdr.msgs.len() {
|
||||
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
|
||||
continue;
|
||||
}
|
||||
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared attribute message".into()));
|
||||
}
|
||||
let body = hdr.data(img, i)?;
|
||||
let crt = if ai.track {
|
||||
hdr.msgs[i].corder.unwrap_or(0)
|
||||
} else {
|
||||
NO_CRT_IDX
|
||||
};
|
||||
dense.insert(img, &body, crt)?;
|
||||
hdr.delete(img, i)?;
|
||||
}
|
||||
dense.finish(img)?;
|
||||
ai.fheap = dense.heap.address();
|
||||
ai.name_bt2 = dense.names.address();
|
||||
if let Some(t) = &dense.order {
|
||||
ai.corder_bt2 = t.address();
|
||||
}
|
||||
ai.store(img, hdr)
|
||||
}
|
||||
|
||||
/// Set an attribute of an object whose attributes are in dense storage: an
|
||||
/// attribute of that name whose new encoding has the old one's size is
|
||||
/// rewritten in its heap object (`H5A__dense_write`); otherwise the old one
|
||||
/// is removed (`H5A__dense_remove`: name index, creation-order index, heap
|
||||
/// object) and the new one inserted with the next creation index.
|
||||
fn set_dense(
|
||||
img: &mut Image<'_>,
|
||||
hdr: &mut Header,
|
||||
ai: &mut AInfo,
|
||||
name: &[u8],
|
||||
body: &[u8],
|
||||
) -> Result<(), Error> {
|
||||
let mut dense = Dense::open(img, ai)?;
|
||||
let hash = name_hash(name);
|
||||
let found = {
|
||||
let heap = &dense.heap;
|
||||
dense
|
||||
.names
|
||||
.find(img, &mut |im, r| cmp_name(heap, im, hash, name, r))?
|
||||
};
|
||||
if let Some(rec) = found {
|
||||
if dense.heap.write_in_place(img, &rec[..ID_LEN], body)? {
|
||||
return dense.finish(img);
|
||||
}
|
||||
{
|
||||
let heap = &dense.heap;
|
||||
dense
|
||||
.names
|
||||
.remove(img, &mut |im, r| cmp_name(heap, im, hash, name, r))?;
|
||||
}
|
||||
if let Some(t) = &mut dense.order {
|
||||
let key = corder_of(&rec);
|
||||
t.remove(img, &mut |_, r| Ok(key.cmp(&corder_of(r))))?
|
||||
.ok_or_else(|| {
|
||||
Error::Unsupported("attribute missing from its creation-order index".into())
|
||||
})?;
|
||||
}
|
||||
dense.heap.remove(img, &rec[..ID_LEN])?;
|
||||
}
|
||||
let crt = ai.next_crt()?;
|
||||
dense.insert(img, body, crt)?;
|
||||
dense.finish(img)?;
|
||||
ai.store(img, hdr)
|
||||
}
|
||||
@@ -180,11 +180,6 @@ impl Bt2 {
|
||||
self.tree_type
|
||||
}
|
||||
|
||||
/// Records in the whole tree.
|
||||
pub(crate) fn len(&self) -> u64 {
|
||||
self.root.all
|
||||
}
|
||||
|
||||
/// Node geometry for depths `0..=self.depth` (`H5B2__hdr_init`,
|
||||
/// `H5B2__split_root`).
|
||||
fn compute_levels(&mut self, os: u8) -> Result<(), Error> {
|
||||
@@ -389,35 +384,6 @@ impl Bt2 {
|
||||
}
|
||||
}
|
||||
|
||||
/// Every record, in key order.
|
||||
pub(crate) fn records(&mut self, img: &Image<'_>) -> Result<Vec<Vec<u8>>, Error> {
|
||||
let mut out = Vec::new();
|
||||
if self.root.addr != undef(img.os) && self.root.nrec > 0 {
|
||||
self.walk(img, self.root, self.depth, &mut out)?;
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn walk(
|
||||
&mut self,
|
||||
img: &Image<'_>,
|
||||
p: Ptr,
|
||||
depth: u16,
|
||||
out: &mut Vec<Vec<u8>>,
|
||||
) -> Result<(), Error> {
|
||||
self.load(img, p, depth)?;
|
||||
let n = self.peek(p.addr).clone();
|
||||
for i in 0..=n.recs.len() {
|
||||
if depth > 0 {
|
||||
self.walk(img, n.ptrs[i], depth - 1, out)?;
|
||||
}
|
||||
if i < n.recs.len() {
|
||||
out.push(n.recs[i].clone());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Insert `rec`, or replace the record `cmp` matches (`H5B2_update`).
|
||||
pub(crate) fn update(
|
||||
&mut self,
|
||||
@@ -665,6 +631,7 @@ impl Bt2 {
|
||||
/// parent records between them) out over the children `lo..` of
|
||||
/// `parent`, with `counts[k]` records in the k-th; the records between
|
||||
/// them go back into the parent.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn relayout(
|
||||
&mut self,
|
||||
parent: u64,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+22
-168
@@ -15,10 +15,12 @@
|
||||
//! file) is dropped before the first write, so no `&[u8]` over the mapping
|
||||
//! is alive while the file changes (see `image`).
|
||||
|
||||
mod attrs;
|
||||
mod btree1;
|
||||
mod btree2;
|
||||
mod earray;
|
||||
mod farray;
|
||||
mod fheap;
|
||||
mod image;
|
||||
mod ohdr;
|
||||
mod select;
|
||||
@@ -27,7 +29,6 @@ use std::collections::{BTreeMap, HashMap};
|
||||
use std::fs::{OpenOptions, TryLockError};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use clawhdf5_format::attribute::AttributeMessage;
|
||||
use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks};
|
||||
use clawhdf5_format::data_layout::DataLayout;
|
||||
use clawhdf5_format::data_read::NativeElement;
|
||||
@@ -43,8 +44,8 @@ use btree1::{BTree1, Key};
|
||||
use btree2::Bt2;
|
||||
use earray::{Ea, EaParams, Elem};
|
||||
use farray::Fa;
|
||||
use image::{Image, get_uint, put_uint, undef};
|
||||
use ohdr::{Header, MSG_ATTRIBUTE};
|
||||
use image::{Image, put_uint, undef};
|
||||
use ohdr::Header;
|
||||
|
||||
const MSG_DATASPACE: u16 = 0x01;
|
||||
const MSG_LAYOUT: u16 = 0x08;
|
||||
@@ -918,101 +919,29 @@ impl FileEditor {
|
||||
|
||||
/// Set attribute `name` of the object at `path` (a group or dataset;
|
||||
/// `"/"` is the root group) to `value`, replacing an attribute of that
|
||||
/// name. The attribute goes into free space in the object header, or a
|
||||
/// new header continuation chunk at the end of the file.
|
||||
/// name, as libhdf5 creates attributes (`H5O__attr_create`).
|
||||
///
|
||||
/// [`Error::Unsupported`] for an object whose attributes are in dense
|
||||
/// storage (or would have to move there: more than the object's
|
||||
/// compact-attribute limit), one that tracks attribute creation order,
|
||||
/// or one with shared attribute messages.
|
||||
/// A compact attribute goes into free space in the object header, or a
|
||||
/// new header continuation chunk; an object that tracks creation order
|
||||
/// gives it the next creation index. When the object reaches its
|
||||
/// compact-attribute limit (or the attribute is too large for a header
|
||||
/// message) its attributes move to dense storage, and objects already
|
||||
/// using dense storage get the attribute there: a fractal heap object
|
||||
/// (in free heap space, a new heap block, or its own file space when
|
||||
/// larger than the heap's managed limit) indexed by name and, when the
|
||||
/// object indexes creation order, by creation index. An attribute
|
||||
/// replaced by one of the same encoded size is rewritten where it is.
|
||||
///
|
||||
/// [`Error::Unsupported`] for shared attribute messages, heaps this
|
||||
/// editor cannot extend the way libhdf5 would (child indirect blocks,
|
||||
/// skipped blocks, free space other than within direct blocks, freeing
|
||||
/// a whole block), and version-1 object headers asked for an attribute
|
||||
/// larger than a header message holds.
|
||||
pub fn set_attr(&mut self, path: &str, name: &str, value: &AttrValue) -> Result<(), Error> {
|
||||
if name.is_empty() {
|
||||
return Err(Error::InvalidArgument("empty attribute name".into()));
|
||||
}
|
||||
self.edit(|f, img| {
|
||||
let addr =
|
||||
clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?;
|
||||
let mut hdr = Header::load(img, addr)?;
|
||||
if hdr.version == 2 && hdr.flags & 0x04 != 0 {
|
||||
return Err(Error::Unsupported(
|
||||
"object tracks attribute creation order".into(),
|
||||
));
|
||||
}
|
||||
if let Some(i) = hdr.find(MSG_ATTR_INFO) {
|
||||
let d = hdr.data(img, i)?;
|
||||
// version(1) flags(1) [max creation index(2)] fractal heap
|
||||
// address, name index address, [order index address].
|
||||
let mut p = 2;
|
||||
if d.get(1).is_some_and(|f| f & 0x01 != 0) {
|
||||
p += 2;
|
||||
}
|
||||
let os = img.os as usize;
|
||||
if d.len() < p + os {
|
||||
return Err(Error::Unsupported("short attribute info message".into()));
|
||||
}
|
||||
if get_uint(&d[p..], img.os) != undef(img.os) {
|
||||
return Err(Error::Unsupported(
|
||||
"object with attributes in dense storage".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let mut existing = None;
|
||||
let mut count = 0usize;
|
||||
for i in 0..hdr.msgs.len() {
|
||||
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
|
||||
continue;
|
||||
}
|
||||
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared attribute message".into()));
|
||||
}
|
||||
count += 1;
|
||||
if attr_name(&hdr.data(img, i)?)? == name.as_bytes() {
|
||||
existing = Some(i);
|
||||
}
|
||||
}
|
||||
if existing.is_none() && hdr.version == 2 {
|
||||
let max_compact = max_compact_attrs(img, &hdr)?;
|
||||
if count + 1 > usize::from(max_compact) {
|
||||
return Err(Error::Unsupported(format!(
|
||||
"object already has {count} compact attributes (its limit is \
|
||||
{max_compact}); more need dense storage"
|
||||
)));
|
||||
}
|
||||
}
|
||||
let msg = clawhdf5_format::type_builders::build_attr_message(name, value);
|
||||
check_plain(&msg.datatype)?;
|
||||
let body = if hdr.version == 1 {
|
||||
encode_attr_v1(&msg, img.ls)
|
||||
} else {
|
||||
let mut b = msg.serialize_v3(img.ls);
|
||||
if !name.is_ascii() {
|
||||
b[8] = 1; // UTF-8 name
|
||||
}
|
||||
b
|
||||
};
|
||||
if let Some(i) = existing {
|
||||
hdr.delete(img, i)?;
|
||||
}
|
||||
// libhdf5 counts a version-2 header's attributes through its
|
||||
// Attribute Info message and reports none without one; like
|
||||
// H5O__attr_create, add it when missing: before the attribute
|
||||
// when free space holds both (libhdf5's order), else after it,
|
||||
// so that a new continuation chunk made for the attribute has
|
||||
// room for it too.
|
||||
let ainfo = (hdr.version == 2 && hdr.find(MSG_ATTR_INFO).is_none())
|
||||
.then(|| attr_info_message(hdr.flags, img.os));
|
||||
let ainfo_first = ainfo
|
||||
.as_ref()
|
||||
.is_some_and(|a| hdr.has_free(a.len() + hdr.hsize() + body.len()));
|
||||
if let Some(a) = ainfo.as_ref().filter(|_| ainfo_first) {
|
||||
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?;
|
||||
}
|
||||
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?;
|
||||
if let Some(a) = ainfo.as_ref().filter(|_| !ainfo_first) {
|
||||
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?;
|
||||
}
|
||||
hdr.finish(img)
|
||||
})
|
||||
self.edit(|f, img| attrs::set_attr(f, img, path, name, value))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1097,81 +1026,6 @@ fn set_superblock_eof(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The name bytes of an attribute message body (without the NUL).
|
||||
fn attr_name(d: &[u8]) -> Result<&[u8], Error> {
|
||||
let bad = || Error::Unsupported("malformed attribute message".into());
|
||||
let (len, at) = match d.first() {
|
||||
Some(1) | Some(2) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8),
|
||||
Some(3) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9),
|
||||
_ => return Err(bad()),
|
||||
};
|
||||
let name = d.get(at..at + len).ok_or_else(bad)?;
|
||||
Ok(name.split(|&b| b == 0).next().unwrap_or(name))
|
||||
}
|
||||
|
||||
/// A new Attribute Info message for a version-2 header with flags
|
||||
/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order
|
||||
/// tracked / indexed as the header's flags say, maximum creation index 0,
|
||||
/// and no dense storage (undefined fractal heap and B-tree addresses).
|
||||
fn attr_info_message(hdr_flags: u8, os: u8) -> Vec<u8> {
|
||||
let track = hdr_flags & 0x04 != 0;
|
||||
let index = hdr_flags & 0x08 != 0;
|
||||
let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)];
|
||||
if track {
|
||||
b.extend_from_slice(&0u16.to_le_bytes());
|
||||
}
|
||||
let undef_addr = vec![0xffu8; os as usize];
|
||||
b.extend_from_slice(&undef_addr);
|
||||
b.extend_from_slice(&undef_addr);
|
||||
if index {
|
||||
b.extend_from_slice(&undef_addr);
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
/// A version-2 header's limit on compact attributes: stored when its flags
|
||||
/// say so, else libhdf5's default of 8.
|
||||
fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result<u16, Error> {
|
||||
if hdr.flags & 0x10 == 0 {
|
||||
return Ok(8);
|
||||
}
|
||||
let mut p = hdr.addr + 6;
|
||||
if hdr.flags & 0x20 != 0 {
|
||||
p += 16;
|
||||
}
|
||||
let b = img.read(p, 2)?;
|
||||
Ok(u16::from_le_bytes([b[0], b[1]]))
|
||||
}
|
||||
|
||||
/// A version-1 attribute message (what libhdf5 writes in a version-1 object
|
||||
/// header): name, datatype and dataspace each padded to 8 bytes, the
|
||||
/// dataspace as a version-1 dataspace message.
|
||||
fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec<u8> {
|
||||
let mut name = a.name.as_bytes().to_vec();
|
||||
name.push(0);
|
||||
let dt = a.datatype.serialize();
|
||||
let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0];
|
||||
if a.dataspace.space_type == DataspaceType::Simple {
|
||||
for &d in &a.dataspace.dimensions {
|
||||
let mut b = vec![0u8; ls as usize];
|
||||
put_uint(&mut b, d, ls);
|
||||
ds.extend_from_slice(&b);
|
||||
}
|
||||
} else {
|
||||
ds[1] = 0;
|
||||
}
|
||||
let mut out = vec![1u8, 0];
|
||||
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(dt.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(ds.len() as u16).to_le_bytes());
|
||||
for part in [&name, &dt, &ds] {
|
||||
out.extend_from_slice(part);
|
||||
out.resize(out.len().next_multiple_of(8), 0);
|
||||
}
|
||||
out.extend_from_slice(&a.raw_data);
|
||||
out
|
||||
}
|
||||
|
||||
fn write_selection(
|
||||
f: &File,
|
||||
img: &mut Image<'_>,
|
||||
|
||||
@@ -59,7 +59,7 @@ pub(crate) struct Header {
|
||||
added: usize,
|
||||
}
|
||||
|
||||
const MAX_CHUNKS: usize = 1024;
|
||||
const MAX_CHUNKS: usize = 1 << 16;
|
||||
|
||||
fn corrupt(why: &'static str) -> Error {
|
||||
Error::Format(FormatError::InvalidObjectHeader(why))
|
||||
@@ -118,7 +118,11 @@ impl Header {
|
||||
});
|
||||
h.scan(img, 0, addr + 16, addr + 16 + size, &mut pending)?;
|
||||
}
|
||||
while let Some((caddr, clen)) = pending.pop() {
|
||||
// Continuation chunks in the order their messages are found, as
|
||||
// H5O_protect loads them (so messages keep libhdf5's order).
|
||||
let mut next = 0;
|
||||
while let Some(&(caddr, clen)) = pending.get(next) {
|
||||
next += 1;
|
||||
if h.chunks.len() >= MAX_CHUNKS {
|
||||
return Err(corrupt("too many object header chunks"));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user