diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index f2c27fd..036839f 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -213,7 +213,6 @@ fn tmpdir() -> tempfile::TempDir { tempfile::TempDir::new_in(base).unwrap() } -#[allow(dead_code)] fn unsupported(r: Result) { match r { Err(Error::Unsupported(_)) => {} @@ -830,3 +829,497 @@ fn btree2_random_chunk_order() { verify(&path, "x", &m); check_tools(&path, false); } + +// ---- dense attributes ---- + +/// An attribute value both sides can write with the same encoded size: a +/// 1-D int64 array or a fixed-length string. +#[derive(Clone, Debug, PartialEq)] +enum AV { + Ints(Vec), + Str(String), +} + +impl AV { + fn value(&self) -> clawhdf5::AttrValue { + match self { + AV::Ints(v) => clawhdf5::AttrValue::I64Array(v.clone()), + AV::Str(s) => clawhdf5::AttrValue::String(s.clone()), + } + } + + fn py(&self) -> String { + match self { + AV::Ints(v) => format!("np.array({v:?}, dtype=' format!("np.bytes_({s:?})"), + } + } + + fn py_expect(&self) -> String { + match self { + AV::Ints(v) => format!("{v:?}"), + AV::Str(s) => format!("{s:?}"), + } + } + + fn make(i: u64, salt: u64) -> Self { + match i % 11 { + 5 => AV::Str("h".repeat(5000 + (salt % 50) as usize)), + 1 | 4 | 8 => AV::Str( + (0..10 + (i * 13 + salt) % 300) + .map(|k| (b'a' + ((k + salt) % 26) as u8) as char) + .collect(), + ), + _ => AV::Ints( + (0..1 + (i + salt) % 9) + .map(|k| (k * 31 + salt) as i64) + .collect(), + ), + } + } +} + +/// An object's dense attribute storage as libhdf5 would compare it: the +/// heap's statistics and root shape, its free sections (heap offsets and +/// sizes), and the shapes of the name and creation-order index B-trees. +fn dense_info(path: &Path, obj: &str) -> String { + use clawhdf5_format::message_type::MessageType; + use clawhdf5_format::object_header::ObjectHeader; + let f = File::open(path).unwrap(); + let sb = f.superblock(); + let (os, ls) = (sb.offset_size, sb.length_size); + assert_eq!((os, ls), (8, 8)); + let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, obj).unwrap(); + let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap(); + let Some(m) = oh + .messages + .iter() + .find(|m| m.msg_type == MessageType::AttributeInfo) + else { + return "no attribute info".into(); + }; + let d = &m.data; + let mut p = 2 + if d[1] & 1 != 0 { 2 } else { 0 }; + let heap = le(&d[p..p + 8]); + p += 8; + let name = le(&d[p..p + 8]); + let order = (d[1] & 2 != 0).then(|| le(&d[p + 8..p + 16])); + if heap == u64::MAX { + return "compact".into(); + } + let b = std::fs::read(path).unwrap(); + let h = &b[heap as usize..]; + assert_eq!(&h[0..4], b"FRHP"); + // next huge id, huge bt2, free, fs, man size, alloc, iter, nobjs, huge + // size, huge objs (8-byte fields from offset 14). + let stat = |k: usize| le(&h[14 + 8 * k..22 + 8 * k]); + let fs = stat(3); + // Root rows: after 12 8-byte fields, width, start, max direct, max + // index, start rows, root address. + let rows_at = 14 + 96 + 2 + 16 + 2 + 2 + 8; + let mut out = format!( + "heap next_huge={} free={} man={} alloc={} iter={} nobjs={} huge={}/{} rows={}", + stat(0), + stat(2), + stat(4), + stat(5), + stat(6), + stat(7), + stat(8), + stat(9), + le(&h[rows_at..rows_at + 2]) + ); + if fs != u64::MAX { + let s = &b[fs as usize..]; + assert_eq!(&s[0..4], b"FSHD"); + let tot = le(&s[6..14]); + let n = le(&s[14..22]); + // counts (4 x 8), 4 x u16, max section size, section info address. + let at = 6 + 32 + 8 + 8; + let sect_addr = le(&s[at..at + 8]); + let sect_size = le(&s[at + 8..at + 16]); + let ss = &b[sect_addr as usize..(sect_addr + sect_size) as usize]; + // Sections after the prefix (signature, version, header address); + // trailing zero padding and the checksum are left out. + let body = &ss[13..ss.len() - 4]; + let used = body.len() - body.iter().rev().take_while(|&&x| x == 0).count(); + out += &format!(" fs tot={tot} n={n} sections={}", hex(&body[..used])); + } else { + out += " no-fs"; + } + out += &format!(" names={:?}", bt2_shape_at(&b, name as usize)); + if let Some(o) = order { + out += &format!(" order={:?}", bt2_shape_at(&b, o as usize)); + } + out +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +/// h5py and our reader both see `want` on each object (other attributes +/// may exist; these must have these values), and h5py's attribute count +/// agrees with libhdf5's object info. +fn check_attr_values(path: &Path, want: &[(String, String, AV)]) { + let f = File::open(path).unwrap(); + for (o, n, v) in want { + let attrs = if o == "d" { + f.dataset(o).unwrap().attrs().unwrap() + } else { + f.group(o).unwrap().attrs().unwrap() + }; + let got = attrs + .get(n.as_str()) + .unwrap_or_else(|| panic!("{o}/{n} missing ({} attributes)", attrs.len())); + match (got, v) { + (clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"), + (clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"), + (clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"), + other => panic!("{o}/{n}: {other:?}"), + } + } + let exp: Vec = want + .iter() + .map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect())) + .collect(); + let script = format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + want = [{w}]\n\ + for o, n, v in want:\n\ + \x20 a = f[o].attrs[n]\n\ + \x20 a = a.decode() if isinstance(a, bytes) else a\n\ + \x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\ + \x20 a = [a] if isinstance(a, int) else a\n\ + \x20 assert a == v, (o, n, a if len(str(a)) < 200 else len(a), v if len(str(v)) < 200 else len(v))\n\ + for o in set(x[0] for x in want):\n\ + \x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n\ + \x20 assert len(list(f[o].attrs)) == len(f[o].attrs)\n", + p = path.to_str().unwrap(), + w = exp.join(", ") + ); + let sp = path.with_extension("check.py"); + std::fs::write(&sp, script).unwrap(); + let o = Command::new(python()).arg(&sp).output().unwrap(); + assert!(o.status.success(), "attribute check failed:\n{}", text(&o)); +} + +/// Run python statements (inside `with h5py.File(path, 'r+') as f:`). +fn py_r_plus(path: &Path, lines: &[String]) { + let script = format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n{l}", + p = path.to_str().unwrap(), + l = lines.concat() + ); + let sp = path.with_extension("ops.py"); + std::fs::write(&sp, script).unwrap(); + let o = Command::new(python()).arg(&sp).output().unwrap(); + assert!(o.status.success(), "h5py workload failed:\n{}", text(&o)); +} + +type AttrOp = (String, String, AV); + +fn set_want(want: &mut Vec, o: &str, n: &str, v: AV) { + want.retain(|(wo, wn, _)| !(wo == o && wn == n)); + want.push((o.into(), n.into(), v)); +} + +/// The same attribute workload by libhdf5 and by the editor on objects +/// with compact attributes that move to dense storage (a plain group, a +/// group tracking and indexing creation order, a dataset): 40 new +/// attributes each (some larger than the heap's 4 KiB managed limit), then +/// same-size rewrites — the heaps, their free space and both index B-trees +/// must come out as libhdf5 makes them; then replacements of another size, +/// then h5py adds, deletes and rewrites attributes. +fn dense_workload(libver: &str, h5dump: bool, tag: &str) { + let dir = tmpdir(); + let a = dir.path().join(format!("dense_{tag}_h5py.h5")); + let b = dir.path().join(format!("dense_{tag}_edit.h5")); + for p in [&a, &b] { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ + \x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \ + f.create_dataset('d', data=np.arange(4, dtype=' = Vec::new(); + for o in objs { + for i in 0..3i64 { + want.push((o.into(), format!("c{i}"), AV::Ints(vec![i, i]))); + } + } + // Phase 1: new attributes. + let mut ops: Vec = Vec::new(); + for i in 0..40u64 { + for (k, o) in objs.iter().enumerate() { + ops.push((o.to_string(), format!("n{i}"), AV::make(i, k as u64))); + } + } + let lines: Vec = ops + .iter() + .map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.create({n:?}, {})\n", v.py())) + .collect(); + py_r_plus(&a, &lines); + let mut ed = FileEditor::open(&b).unwrap(); + for (o, n, v) in &ops { + ed.set_attr(o, n, &v.value()) + .unwrap_or_else(|e| panic!("{tag}: set {o}/{n}: {e}")); + set_want(&mut want, o, n, v.clone()); + } + drop(ed); + check_tools(&b, h5dump); + check_attr_values(&b, &want); + check_attr_values(&a, &want); + for o in objs { + assert_eq!( + dense_info(&b, o), + dense_info(&a, o), + "{tag}: dense storage of {o} differs from libhdf5's after insertions" + ); + } + // Phase 2: same-size rewrites (H5Awrite in place). + let ops2: Vec = ops + .iter() + .step_by(4) + .map(|(o, n, v)| { + let nv = match v { + AV::Ints(x) => AV::Ints(x.iter().map(|y| y * 3 + 1).collect()), + AV::Str(s) => AV::Str(s.chars().rev().collect()), + }; + (o.clone(), n.clone(), nv) + }) + .collect(); + let lines: Vec = ops2 + .iter() + .map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.modify({n:?}, {})\n", v.py())) + .collect(); + py_r_plus(&a, &lines); + let mut ed = FileEditor::open(&b).unwrap(); + for (o, n, v) in &ops2 { + ed.set_attr(o, n, &v.value()).unwrap(); + set_want(&mut want, o, n, v.clone()); + } + drop(ed); + check_tools(&b, h5dump); + check_attr_values(&b, &want); + for o in objs { + assert_eq!( + dense_info(&b, o), + dense_info(&a, o), + "{tag}: dense storage of {o} differs from libhdf5's after rewrites" + ); + } + // Phase 3: replacements of another size (values only: h5py replaces + // through a temporary attribute and a rename). + let mut ed = FileEditor::open(&b).unwrap(); + for (k, (o, n, v)) in ops.iter().enumerate().filter(|(k, _)| k % 5 == 2) { + let nv = match v { + AV::Ints(x) => AV::Ints((0..x.len() as i64 + 3).collect()), + AV::Str(s) => AV::Str(format!("{s}-{k}")), + }; + let before = std::fs::read(&b).unwrap(); + match ed.set_attr(o, n, &nv.value()) { + Ok(()) => set_want(&mut want, o, n, nv), + Err(Error::Unsupported(msg)) => { + assert!(msg.contains("last object"), "{tag}: {o}/{n}: {msg}"); + assert!(std::fs::read(&b).unwrap() == before, "refused edit wrote"); + } + Err(e) => panic!("{tag}: replace {o}/{n}: {e}"), + } + } + drop(ed); + check_tools(&b, h5dump); + check_attr_values(&b, &want); + // libhdf5 goes on: adds, deletes and rewrites. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for o in ['g', 't', 'd']:\n\ + \x20 for i in range(5): f[o].attrs[f'late{{i}}'] = np.arange(i + 1)\n\ + \x20 del f[o].attrs['n3']\n\ + \x20 del f[o].attrs['c1']\n\ + \x20 f[o].attrs['n7'] = 'rewritten by h5py'\n", + p = b.to_str().unwrap() + )); + want.retain(|(_, n, _)| n != "n3" && n != "c1" && n != "n7"); + check_tools(&b, h5dump); + check_attr_values(&b, &want); +} + +#[test] +fn dense_attributes_match_libhdf5() { + if !tools_ok() { + return; + } + for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)] + .iter() + .enumerate() + { + dense_workload(lv, *dump, &format!("{i}")); + } +} + +/// Dense attributes in files clawhdf5 writes (its own heap and index +/// layout, no free-space manager), with and without tracked creation +/// order: attributes added (moving a dataset's attributes to dense storage +/// too), rewritten and replaced, then h5py goes on. +#[test] +fn dense_attributes_on_clawhdf5_files() { + if !tools_ok() { + return; + } + for track in [false, true] { + let dir = tmpdir(); + let path = dir.path().join(format!("ours_dense_{track}.h5")); + let mut b = clawhdf5::FileBuilder::new(); + b.track_order(track); + for i in 0..12i64 { + b.set_attr(&format!("r{i}"), clawhdf5::AttrValue::I64Array(vec![i; 3])); + } + b.create_dataset("d") + .with_i32_data(&[1, 2, 3]) + .set_attr("c0", clawhdf5::AttrValue::I64Array(vec![5, 5])); + b.write(&path).unwrap(); + let mut want: Vec = (0..12i64) + .map(|i| ("/".to_string(), format!("r{i}"), AV::Ints(vec![i; 3]))) + .collect(); + want.push(("d".into(), "c0".into(), AV::Ints(vec![5, 5]))); + let mut ed = FileEditor::open(&path).unwrap(); + for i in 0..30u64 { + for (k, o) in ["/", "d"].iter().enumerate() { + // Small attributes: a larger one needs a heap block bigger than + // the next (see dense_attribute_refusals_change_nothing). + let v = match AV::make(i, k as u64 + 7) { + AV::Str(s) if s.len() > 400 => AV::Str(s[..400].to_string()), + v => v, + }; + ed.set_attr(o, &format!("n{i}"), &v.value()) + .unwrap_or_else(|e| panic!("{track} {o}/n{i}: {e}")); + set_want(&mut want, o, &format!("n{i}"), v); + } + } + // Rewrites in place, then replacements of another size. + for i in (0..12i64).step_by(3) { + let v = AV::Ints(vec![-i; 3]); + ed.set_attr("/", &format!("r{i}"), &v.value()).unwrap(); + set_want(&mut want, "/", &format!("r{i}"), v); + } + for i in (1..30u64).step_by(7) { + let v = AV::Str(format!("replaced {i}")); + match ed.set_attr("d", &format!("n{i}"), &v.value()) { + Ok(()) => set_want(&mut want, "d", &format!("n{i}"), v), + Err(Error::Unsupported(msg)) => assert!(msg.contains("last object"), "{msg}"), + Err(e) => panic!("{e}"), + } + } + drop(ed); + check_tools(&path, true); + let want_py: Vec = want + .iter() + .map(|(o, n, v)| { + let o = if o == "/" { "/".to_string() } else { o.clone() }; + (o, n.clone(), v.clone()) + }) + .collect(); + check_root_and_attrs(&path, &want_py); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for o in ['/', 'd']:\n\ + \x20 f[o].attrs['from_h5py'] = np.arange(3)\n\ + \x20 del f[o].attrs['n2']\n", + p = path.to_str().unwrap() + )); + want.retain(|(_, n, _)| n != "n2"); + check_tools(&path, true); + check_root_and_attrs(&path, &want); + } +} + +/// `check_attr_values`, with the root group as "/". +fn check_root_and_attrs(path: &Path, want: &[AttrOp]) { + let f = File::open(path).unwrap(); + for (o, n, v) in want { + let attrs = match o.as_str() { + "/" => f.root().attrs().unwrap(), + "d" => f.dataset(o).unwrap().attrs().unwrap(), + _ => f.group(o).unwrap().attrs().unwrap(), + }; + let got = attrs + .get(n.as_str()) + .unwrap_or_else(|| panic!("{o}/{n} missing")); + match (got, v) { + (clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"), + (clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"), + (clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"), + other => panic!("{o}/{n}: {other:?}"), + } + } + let exp: Vec = want + .iter() + .map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect())) + .collect(); + let script = format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + want = [{w}]\n\ + for o, n, v in want:\n\ + \x20 a = f[o].attrs[n]\n\ + \x20 a = a.decode() if isinstance(a, bytes) else a\n\ + \x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\ + \x20 a = [a] if isinstance(a, int) else a\n\ + \x20 assert a == v, (o, n)\n\ + for o in set(x[0] for x in want):\n\ + \x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n", + p = path.to_str().unwrap(), + w = exp.join(", ") + ); + let sp = path.with_extension("check.py"); + std::fs::write(&sp, script).unwrap(); + let o = Command::new(python()).arg(&sp).output().unwrap(); + assert!(o.status.success(), "attribute check failed:\n{}", text(&o)); +} + +/// What the editor refuses in dense storage — an object larger than the +/// next heap block (libhdf5 would skip blocks and record their space as +/// free, which this editor does not do) — is `Error::Unsupported`, and the +/// file is left byte for byte as it was. +#[test] +fn dense_attribute_refusals_change_nothing() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let path = dir.path().join("dense_refuse.h5"); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver='v110') as f:\n\ + \x20 g = f.create_group('g')\n\ + \x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n", + p = path.to_str().unwrap() + )); + let before = std::fs::read(&path).unwrap(); + let mut ed = FileEditor::open(&path).unwrap(); + unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("x".repeat(2000)))); + drop(ed); + assert!( + std::fs::read(&path).unwrap() == before, + "a refused edit wrote" + ); + // What libhdf5 does with it instead works on the untouched file. + py(&format!( + "import h5py\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 f['g'].attrs['big'] = 'x' * 2000\n\ + \x20 assert len(f['g'].attrs) == 13\n", + p = path.to_str().unwrap() + )); + check_tools(&path, true); +} diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index 6a2dbf6..bc4c9a6 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -836,19 +836,16 @@ fn attributes_in_place() { .unwrap(); want.push(("d", "units".into(), AttrValue::String("km".into()))); if *lv != "'earliest'" { - // Up to the compact limit (8) and no further; attributes in - // dense storage and tracked creation order are refused, and a - // refused edit writes nothing. + // Up to the compact limit (8), then into dense storage; objects + // already in dense storage and ones tracking creation order. ed.set_attr("g", "eighth", &AttrValue::I64(8)).unwrap(); want.push(("g", "eighth".into(), AttrValue::I64(8))); - let before = std::fs::read(&path).unwrap(); - unsupported(ed.set_attr("g", "ninth", &AttrValue::I64(9))); - unsupported(ed.set_attr("dense", "k0", &AttrValue::I64(1))); - unsupported(ed.set_attr("tracked", "b", &AttrValue::I64(1))); - assert!( - std::fs::read(&path).unwrap() == before, - "a refused edit changed the file" - ); + ed.set_attr("g", "ninth", &AttrValue::I64(9)).unwrap(); + want.push(("g", "ninth".into(), AttrValue::I64(9))); + ed.set_attr("dense", "k0", &AttrValue::I64(1)).unwrap(); + want.push(("dense", "k0".into(), AttrValue::I64(1))); + ed.set_attr("tracked", "b", &AttrValue::I64(1)).unwrap(); + want.push(("tracked", "b".into(), AttrValue::I64(1))); } drop(ed); check_tools(&path, *dump); diff --git a/crates/clawhdf5/src/edit/attrs.rs b/crates/clawhdf5/src/edit/attrs.rs new file mode 100644 index 0000000..5772db3 --- /dev/null +++ b/crates/clawhdf5/src/edit/attrs.rs @@ -0,0 +1,513 @@ +//! Setting attributes, as `H5O__attr_create` / `H5A__dense_insert` do: +//! compact attributes are object header messages (with their creation +//! index in the message header when the object tracks creation order); +//! when an object reaches its compact limit (or an attribute is too large +//! for a header message) its attributes move to dense storage — a fractal +//! heap for the encoded messages, a version-2 B-tree indexing them by name +//! hash (record type 8) and, when creation order is indexed, a second one +//! by creation index (type 9) — and the Attribute Info message points at +//! them. + +use std::cmp::Ordering; + +use clawhdf5_format::attribute::AttributeMessage; +use clawhdf5_format::dataspace::DataspaceType; + +use crate::edit::btree2::Bt2; +use crate::edit::fheap::Heap; +use crate::edit::image::{Image, get_uint, put_uint, undef}; +use crate::edit::ohdr::{Header, MSG_ATTRIBUTE}; +use crate::edit::{MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, MSG_FLAG_SHARED, check_plain}; +use crate::error::Error; +use crate::reader::File; +use crate::types::AttrValue; + +/// `H5O_MESG_MAX_SIZE`: a larger attribute goes to dense storage. +const MESG_MAX_SIZE: usize = 65536; +/// `H5O_MAX_CRT_ORDER_IDX`: the creation index of an attribute of an object +/// that does not track creation order. +const NO_CRT_IDX: u16 = u16::MAX; +/// Name and creation-order index B-trees (`H5A_NAME_BT2_*`, +/// `H5A_CORDER_BT2_*`). +const NAME_BT2_TYPE: u8 = 8; +const CORDER_BT2_TYPE: u8 = 9; +const ATTR_BT2_NODE: u32 = 512; +/// Heap IDs in attribute records. +const ID_LEN: usize = 8; + +/// An object's Attribute Info message. +#[derive(Debug, Clone)] +struct AInfo { + /// Its message index in the header. + idx: usize, + track: bool, + index: bool, + max_crt: u16, + fheap: u64, + name_bt2: u64, + corder_bt2: u64, +} + +impl AInfo { + fn load(img: &Image<'_>, hdr: &Header) -> Result, Error> { + let Some(idx) = hdr.find(MSG_ATTR_INFO) else { + return Ok(None); + }; + if hdr.msgs[idx].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared attribute info message".into())); + } + let d = hdr.data(img, idx)?; + let os = img.os as usize; + let short = || Error::Unsupported("short attribute info message".into()); + if d.first() != Some(&0) { + return Err(Error::Unsupported("attribute info message version".into())); + } + let flags = *d.get(1).ok_or_else(short)?; + let track = flags & 0x01 != 0; + let index = flags & 0x02 != 0; + let mut p = 2; + let mut max_crt = 0; + if track { + let b = d.get(p..p + 2).ok_or_else(short)?; + max_crt = u16::from_le_bytes([b[0], b[1]]); + p += 2; + } + let n = if index { 3 } else { 2 }; + if d.len() < p + n * os { + return Err(short()); + } + Ok(Some(Self { + idx, + track, + index, + max_crt, + fheap: get_uint(&d[p..], img.os), + name_bt2: get_uint(&d[p + os..], img.os), + corder_bt2: if index { + get_uint(&d[p + 2 * os..], img.os) + } else { + undef(img.os) + }, + })) + } + + fn dense(&self, os: u8) -> bool { + self.fheap != undef(os) + } + + /// Store the changeable fields back into the message. + fn store(&self, img: &mut Image<'_>, hdr: &mut Header) -> Result<(), Error> { + let os = img.os as usize; + let mut p = 2; + if self.track { + hdr.patch(img, self.idx, p, &self.max_crt.to_le_bytes())?; + p += 2; + } + let mut a = vec![0u8; os]; + for (k, v) in [self.fheap, self.name_bt2, self.corder_bt2] + .into_iter() + .enumerate() + .take(if self.index { 3 } else { 2 }) + { + put_uint(&mut a, v, img.os); + hdr.patch(img, self.idx, p + k * os, &a)?; + } + Ok(()) + } + + /// The next creation index (`H5O__attr_create`), or libhdf5's "none". + fn next_crt(&mut self) -> Result { + if !self.track { + return Ok(NO_CRT_IDX); + } + if self.max_crt == NO_CRT_IDX { + return Err(Error::Unsupported( + "object's attribute creation index is exhausted".into(), + )); + } + self.max_crt += 1; + Ok(self.max_crt - 1) + } +} + +/// The name bytes of an attribute message body (without the NUL). +pub(super) fn attr_name(d: &[u8]) -> Result<&[u8], Error> { + let bad = || Error::Unsupported("malformed attribute message".into()); + let (len, at) = match d.first() { + Some(1) | Some(2) if d.len() >= 8 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8), + Some(3) if d.len() >= 9 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9), + _ => return Err(bad()), + }; + let name = d.get(at..at + len).ok_or_else(bad)?; + Ok(name.split(|&b| b == 0).next().unwrap_or(name)) +} + +/// A new Attribute Info message for a version-2 header with flags +/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order +/// tracked / indexed as the header's flags say, the maximum creation index, +/// and no dense storage (undefined fractal heap and B-tree addresses). +fn attr_info_message(hdr_flags: u8, max_crt: u16, os: u8) -> Vec { + let track = hdr_flags & 0x04 != 0; + let index = hdr_flags & 0x08 != 0; + let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)]; + if track { + b.extend_from_slice(&max_crt.to_le_bytes()); + } + let undef_addr = vec![0xffu8; os as usize]; + b.extend_from_slice(&undef_addr); + b.extend_from_slice(&undef_addr); + if index { + b.extend_from_slice(&undef_addr); + } + b +} + +/// A version-2 header's limit on compact attributes: stored when its flags +/// say so, else libhdf5's default of 8. +fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result { + if hdr.flags & 0x10 == 0 { + return Ok(8); + } + let mut p = hdr.addr + 6; + if hdr.flags & 0x20 != 0 { + p += 16; + } + let b = img.read(p, 2)?; + Ok(u16::from_le_bytes([b[0], b[1]])) +} + +/// A version-1 attribute message (what libhdf5 writes in a version-1 object +/// header): name, datatype and dataspace each padded to 8 bytes, the +/// dataspace as a version-1 dataspace message. +fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec { + let mut name = a.name.as_bytes().to_vec(); + name.push(0); + let dt = a.datatype.serialize(); + let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0]; + if a.dataspace.space_type == DataspaceType::Simple { + let mut b = vec![0u8; ls as usize]; + for &d in &a.dataspace.dimensions { + put_uint(&mut b, d, ls); + ds.extend_from_slice(&b); + } + if let Some(max) = &a.dataspace.max_dimensions { + ds[2] = 0x01; + for &d in max { + put_uint(&mut b, d, ls); + ds.extend_from_slice(&b); + } + } + } else { + ds[1] = 0; + } + let mut out = vec![1u8, 0]; + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&(dt.len() as u16).to_le_bytes()); + out.extend_from_slice(&(ds.len() as u16).to_le_bytes()); + for part in [&name, &dt, &ds] { + out.extend_from_slice(part); + out.resize(out.len().next_multiple_of(8), 0); + } + out.extend_from_slice(&a.raw_data); + out +} + +/// Dense storage opened for changes. +struct Dense { + heap: Heap, + names: Bt2, + order: Option, +} + +/// `H5_checksum_lookup3` of a name, as the name index keys it. +fn name_hash(name: &[u8]) -> u32 { + clawhdf5_format::checksum::jenkins_lookup3(name) +} + +/// Compare attribute `name` (hash `hash`) with a name-index record +/// (`H5A__dense_btree2_name_compare`: the hash, then the stored name). +fn cmp_name( + heap: &Heap, + img: &Image<'_>, + hash: u32, + name: &[u8], + rec: &[u8], +) -> Result { + let theirs = u32::from_le_bytes([rec[13], rec[14], rec[15], rec[16]]); + match hash.cmp(&theirs) { + Ordering::Equal => { + if rec[ID_LEN] & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported( + "shared attribute in dense storage".into(), + )); + } + let obj = heap.read(img, &rec[..ID_LEN])?; + Ok(name.cmp(attr_name(&obj)?)) + } + o => Ok(o), + } +} + +fn corder_of(rec: &[u8]) -> u32 { + u32::from_le_bytes([rec[9], rec[10], rec[11], rec[12]]) +} + +impl Dense { + fn open(img: &Image<'_>, ai: &AInfo) -> Result { + let heap = Heap::open(img, ai.fheap)?; + let names = Bt2::open(img, ai.name_bt2)?; + if names.tree_type() != NAME_BT2_TYPE || names.record_size() != ID_LEN + 9 { + return Err(Error::Unsupported("attribute name index layout".into())); + } + let order = if ai.index { + let t = Bt2::open(img, ai.corder_bt2)?; + if t.tree_type() != CORDER_BT2_TYPE || t.record_size() != ID_LEN + 5 { + return Err(Error::Unsupported( + "attribute creation-order index layout".into(), + )); + } + Some(t) + } else { + None + }; + Ok(Self { heap, names, order }) + } + + /// `H5A__dense_create`: heap, name index, [creation-order index]. + fn create(img: &mut Image<'_>, index: bool) -> Result { + let heap = Heap::create_attribute_heap(img)?; + let names = Bt2::create(img, NAME_BT2_TYPE, ATTR_BT2_NODE, ID_LEN + 9, 100, 40)?; + let order = if index { + Some(Bt2::create( + img, + CORDER_BT2_TYPE, + ATTR_BT2_NODE, + ID_LEN + 5, + 100, + 40, + )?) + } else { + None + }; + Ok(Self { heap, names, order }) + } + + /// `H5A__dense_insert` of an encoded attribute message. + fn insert(&mut self, img: &mut Image<'_>, body: &[u8], crt: u16) -> Result<(), Error> { + let name = attr_name(body)?.to_vec(); + let id = self.heap.insert(img, body)?; + if id.len() != ID_LEN { + return Err(Error::Unsupported("attribute heap ID length".into())); + } + let hash = name_hash(&name); + let mut rec = id.clone(); + rec.push(0); + rec.extend_from_slice(&u32::from(crt).to_le_bytes()); + rec.extend_from_slice(&hash.to_le_bytes()); + let heap = &self.heap; + self.names + .insert(img, &mut |im, r| cmp_name(heap, im, hash, &name, r), &rec)?; + if let Some(t) = &mut self.order { + let key = u32::from(crt); + t.insert( + img, + &mut |_, r| Ok(key.cmp(&corder_of(r))), + &rec[..ID_LEN + 5], + )?; + } + Ok(()) + } + + fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + self.heap.finish(img)?; + self.names.finish(img)?; + if let Some(t) = &mut self.order { + t.finish(img)?; + } + Ok(()) + } +} + +/// Set attribute `name` of the object at `path` to `value`. +pub(super) fn set_attr( + f: &File, + img: &mut Image<'_>, + path: &str, + name: &str, + value: &AttrValue, +) -> Result<(), Error> { + let addr = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?; + let mut hdr = Header::load(img, addr)?; + let mut msg = clawhdf5_format::type_builders::build_attr_message(name, value); + check_plain(&msg.datatype)?; + // libhdf5 encodes a simple dataspace with its maximum dimensions (the + // current ones when none were given), so an attribute takes the same + // space in a header or heap as when libhdf5 writes it. + if msg.dataspace.space_type == DataspaceType::Simple && msg.dataspace.max_dimensions.is_none() { + msg.dataspace.max_dimensions = Some(msg.dataspace.dimensions.clone()); + } + // H5A__set_version: version 1 unless the name is not ASCII (then 3), + // raised to the file's low bound — which is the earliest for a file + // libhdf5 opens without a libver setting (h5py's `r+`). + let body = if hdr.version == 1 || name.is_ascii() { + encode_attr_v1(&msg, img.ls) + } else { + let mut b = msg.serialize_v3(img.ls); + if !name.is_ascii() { + b[8] = 1; // UTF-8 name + } + b + }; + let mut ainfo = if hdr.version == 2 { + AInfo::load(img, &hdr)? + } else { + None + }; + if let Some(ai) = ainfo.as_mut().filter(|a| a.dense(img.os)) { + let mut ai = ai.clone(); + set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?; + return hdr.finish(img); + } + + let mut existing = None; + let mut count = 0usize; + for i in 0..hdr.msgs.len() { + if hdr.msgs[i].mtype != MSG_ATTRIBUTE { + continue; + } + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared attribute message".into())); + } + count += 1; + if attr_name(&hdr.data(img, i)?)? == name.as_bytes() { + existing = Some(i); + } + } + if let Some(i) = existing { + hdr.delete(img, i)?; + count -= 1; + } + if hdr.version == 1 { + hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?; + return hdr.finish(img); + } + let tracked = hdr.flags & 0x04 != 0; + // H5O__attr_create: a missing Attribute Info message starts from + // nothing (and is added below, holding the new maximum creation index). + let new_ainfo = ainfo.is_none(); + let mut ai = ainfo.take().unwrap_or(AInfo { + idx: usize::MAX, + track: tracked, + index: hdr.flags & 0x08 != 0, + max_crt: 0, + fheap: undef(img.os), + name_bt2: undef(img.os), + corder_bt2: undef(img.os), + }); + let max_compact = usize::from(max_compact_attrs(img, &hdr)?); + if count == max_compact || body.len() >= MESG_MAX_SIZE { + if new_ainfo { + return Err(Error::Unsupported( + "dense attribute storage for an object without an Attribute Info message".into(), + )); + } + to_dense(img, &mut hdr, &mut ai)?; + set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?; + return hdr.finish(img); + } + let crt = ai.next_crt()?; + let corder = tracked.then_some(crt); + if new_ainfo { + // libhdf5 appends the Attribute Info message before the attribute + // when free space holds both, else after it, so that a new + // continuation chunk made for the attribute has room for it too. + let a = attr_info_message(hdr.flags, ai.max_crt, img.os); + let first = hdr.has_free(a.len() + hdr.hsize() + body.len()); + if first { + hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?; + } + hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?; + if !first { + hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?; + } + } else { + hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?; + ai.store(img, &mut hdr)?; + } + hdr.finish(img) +} + +/// Move every compact attribute of the object into new dense storage, in +/// header message order (`H5O__attr_to_dense_cb`), leaving free space where +/// the messages were. +fn to_dense(img: &mut Image<'_>, hdr: &mut Header, ai: &mut AInfo) -> Result<(), Error> { + let mut dense = Dense::create(img, ai.index)?; + for i in 0..hdr.msgs.len() { + if hdr.msgs[i].mtype != MSG_ATTRIBUTE { + continue; + } + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared attribute message".into())); + } + let body = hdr.data(img, i)?; + let crt = if ai.track { + hdr.msgs[i].corder.unwrap_or(0) + } else { + NO_CRT_IDX + }; + dense.insert(img, &body, crt)?; + hdr.delete(img, i)?; + } + dense.finish(img)?; + ai.fheap = dense.heap.address(); + ai.name_bt2 = dense.names.address(); + if let Some(t) = &dense.order { + ai.corder_bt2 = t.address(); + } + ai.store(img, hdr) +} + +/// Set an attribute of an object whose attributes are in dense storage: an +/// attribute of that name whose new encoding has the old one's size is +/// rewritten in its heap object (`H5A__dense_write`); otherwise the old one +/// is removed (`H5A__dense_remove`: name index, creation-order index, heap +/// object) and the new one inserted with the next creation index. +fn set_dense( + img: &mut Image<'_>, + hdr: &mut Header, + ai: &mut AInfo, + name: &[u8], + body: &[u8], +) -> Result<(), Error> { + let mut dense = Dense::open(img, ai)?; + let hash = name_hash(name); + let found = { + let heap = &dense.heap; + dense + .names + .find(img, &mut |im, r| cmp_name(heap, im, hash, name, r))? + }; + if let Some(rec) = found { + if dense.heap.write_in_place(img, &rec[..ID_LEN], body)? { + return dense.finish(img); + } + { + let heap = &dense.heap; + dense + .names + .remove(img, &mut |im, r| cmp_name(heap, im, hash, name, r))?; + } + if let Some(t) = &mut dense.order { + let key = corder_of(&rec); + t.remove(img, &mut |_, r| Ok(key.cmp(&corder_of(r))))? + .ok_or_else(|| { + Error::Unsupported("attribute missing from its creation-order index".into()) + })?; + } + dense.heap.remove(img, &rec[..ID_LEN])?; + } + let crt = ai.next_crt()?; + dense.insert(img, body, crt)?; + dense.finish(img)?; + ai.store(img, hdr) +} diff --git a/crates/clawhdf5/src/edit/btree2.rs b/crates/clawhdf5/src/edit/btree2.rs index 8ff7205..80be9f0 100644 --- a/crates/clawhdf5/src/edit/btree2.rs +++ b/crates/clawhdf5/src/edit/btree2.rs @@ -180,11 +180,6 @@ impl Bt2 { self.tree_type } - /// Records in the whole tree. - pub(crate) fn len(&self) -> u64 { - self.root.all - } - /// Node geometry for depths `0..=self.depth` (`H5B2__hdr_init`, /// `H5B2__split_root`). fn compute_levels(&mut self, os: u8) -> Result<(), Error> { @@ -389,35 +384,6 @@ impl Bt2 { } } - /// Every record, in key order. - pub(crate) fn records(&mut self, img: &Image<'_>) -> Result>, Error> { - let mut out = Vec::new(); - if self.root.addr != undef(img.os) && self.root.nrec > 0 { - self.walk(img, self.root, self.depth, &mut out)?; - } - Ok(out) - } - - fn walk( - &mut self, - img: &Image<'_>, - p: Ptr, - depth: u16, - out: &mut Vec>, - ) -> Result<(), Error> { - self.load(img, p, depth)?; - let n = self.peek(p.addr).clone(); - for i in 0..=n.recs.len() { - if depth > 0 { - self.walk(img, n.ptrs[i], depth - 1, out)?; - } - if i < n.recs.len() { - out.push(n.recs[i].clone()); - } - } - Ok(()) - } - /// Insert `rec`, or replace the record `cmp` matches (`H5B2_update`). pub(crate) fn update( &mut self, @@ -665,6 +631,7 @@ impl Bt2 { /// parent records between them) out over the children `lo..` of /// `parent`, with `counts[k]` records in the k-th; the records between /// them go back into the parent. + #[allow(clippy::too_many_arguments)] fn relayout( &mut self, parent: u64, diff --git a/crates/clawhdf5/src/edit/fheap.rs b/crates/clawhdf5/src/edit/fheap.rs new file mode 100644 index 0000000..7c7b72a --- /dev/null +++ b/crates/clawhdf5/src/edit/fheap.rs @@ -0,0 +1,1121 @@ +//! Changing a fractal heap (`FRHP`) in place, as libhdf5's `H5HF` code does +//! for the heaps that hold dense attributes: managed objects go into the +//! best-fitting free section the heap's free-space manager records (a new +//! direct block when none fits: the root direct block of an empty heap, the +//! next block of the root indirect block — created from the root direct +//! block, and doubled, as needed), huge objects (larger than the heap's +//! managed maximum) into their own file space tracked by the huge-object +//! version-2 B-tree; removed objects return their space to the free-space +//! manager, merged with adjacent free space. +//! +//! The heap's free-space manager (`FSHD` header, `FSSE` section info) is +//! kept exactly as libhdf5 keeps it: sections sorted by size then offset, +//! the header's counts and sizes, section info moved when its size changes, +//! the manager deleted when it tracks nothing. Header statistics (managed +//! space, allocated space, free space, allocation iterator, object counts) +//! follow libhdf5's arithmetic. +//! +//! What libhdf5 would do differently is refused ([`Error::Unsupported`], +//! before anything is written): I/O filters on the heap, child indirect +//! blocks, skipped blocks (a first object too large for the next block), +//! free sections other than "single" ones, freeing a whole direct block, +//! directly addressed huge objects, tiny objects. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::edit::btree2::Bt2; +use crate::edit::image::{Image, get_uint, put_uint, undef}; +use crate::error::Error; + +fn unsupported(why: &str) -> Error { + Error::Unsupported(format!("fractal heap: {why}")) +} + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("fractal heap: {why}"), + )) +} + +/// `H5VM_log2_gen`: floor(log2(v)). +fn log2(v: u64) -> u32 { + 63 - v.max(1).leading_zeros() +} + +/// `H5VM_limit_enc_size`. +fn enc_size(v: u64) -> usize { + (log2(v) / 8 + 1) as usize +} + +fn le(b: &[u8]) -> u64 { + b.iter() + .take(8) + .enumerate() + .fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i))) +} + +/// Free-space client ID for fractal heaps. +const FS_CLIENT_FHEAP: u8 = 0; +/// `H5HF_FSPACE_SHRINK` / `H5HF_FSPACE_EXPAND`. +const FS_SHRINK: u16 = 80; +const FS_EXPAND: u16 = 120; +/// Section classes the heap registers (single, first row, normal row, +/// indirect). +const FS_NCLASSES: u16 = 4; +/// Huge-object B-tree (`H5HF_HUGE_BT2_*`): record type 1 (indirectly +/// accessed, unfiltered), node size and split/merge percentages. +const HUGE_BT2_TYPE: u8 = 1; +const HUGE_BT2_NODE: u32 = 512; + +/// The heap's free-space manager: only "single" sections (free space inside +/// a direct block), keyed by heap offset. +struct FreeSpace { + addr: u64, + max_sect_addr: u16, + max_sect_size: u64, + shrink: u16, + expand: u16, + nclasses: u16, + sect_addr: u64, + sect_size: u64, + alloc_sect_size: u64, + sects: BTreeMap, +} + +impl FreeSpace { + fn hdr_len(os: u8, ls: u8) -> usize { + 6 + 4 * ls as usize + 8 + ls as usize + os as usize + 2 * ls as usize + 4 + } + + fn open(img: &Image<'_>, addr: u64) -> Result { + let (os, ls) = (img.os, img.ls); + let len = Self::hdr_len(os, ls); + let d = img.read(addr, len)?; + if &d[0..4] != b"FSHD" || d[4] != 0 { + return Err(bad("bad free-space header")); + } + let stored = u32::from_le_bytes(d[len - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..len - 4]) != stored { + return Err(bad("free-space header checksum mismatch")); + } + if d[5] != FS_CLIENT_FHEAP { + return Err(bad("free-space manager of another client")); + } + let l = ls as usize; + let mut p = 6; + let mut next = |w: usize| { + let v = le(&d[p..p + w]); + p += w; + v + }; + let tot_space = next(l); + let tot_count = next(l); + let serial = next(l); + let ghost = next(l); + let nclasses = next(2) as u16; + let shrink = next(2) as u16; + let expand = next(2) as u16; + let max_sect_addr = next(2) as u16; + let max_sect_size = next(l); + let sect_addr = next(os as usize); + let sect_size = next(l); + let alloc_sect_size = next(l); + if ghost != 0 || tot_count != serial { + return Err(unsupported("free space in row or indirect sections")); + } + let mut fs = Self { + addr, + max_sect_addr, + max_sect_size, + shrink, + expand, + nclasses, + sect_addr, + sect_size, + alloc_sect_size, + sects: BTreeMap::new(), + }; + if serial == 0 { + return Ok(fs); + } + if sect_addr == undef(os) || sect_size < 9 + os as u64 || sect_size > alloc_sect_size { + return Err(bad("bad free-space section info")); + } + let n = usize::try_from(sect_size).map_err(|_| bad("section info too large"))?; + let s = img.read(sect_addr, n)?; + let stored = u32::from_le_bytes(s[n - 4..].try_into().unwrap_or([0; 4])); + if &s[0..4] != b"FSSE" + || s[4] != 0 + || get_uint(&s[5..], os) != addr + || clawhdf5_format::checksum::jenkins_lookup3(&s[..n - 4]) != stored + { + return Err(bad("bad free-space section info")); + } + let cnt_w = enc_size(serial); + let len_w = enc_size(max_sect_size); + let off_w = (usize::from(max_sect_addr)).div_ceil(8); + let mut q = 5 + os as usize; + let mut seen = 0u64; + let mut total = 0u64; + while seen < serial { + if q + cnt_w + len_w > n - 4 { + return Err(bad("section info ends early")); + } + let count = le(&s[q..q + cnt_w]); + q += cnt_w; + let size = le(&s[q..q + len_w]); + q += len_w; + if count == 0 || size == 0 { + return Err(bad("empty section size node")); + } + for _ in 0..count { + if q + off_w + 1 > n - 4 { + return Err(bad("section info ends early")); + } + let off = le(&s[q..q + off_w]); + let class = s[q + off_w]; + q += off_w + 1; + if class != 0 { + return Err(unsupported("free space in row or indirect sections")); + } + if fs.sects.insert(off, size).is_some() { + return Err(bad("duplicate free section")); + } + seen += 1; + total += size; + } + } + if total != tot_space { + return Err(bad("free-space total disagrees with its sections")); + } + Ok(fs) + } + + /// Best fit (`H5FS__sect_find_node`): the smallest section of at least + /// `size` bytes, the lowest offset among equals. + fn find(&self, size: u64) -> Option<(u64, u64)> { + self.sects + .iter() + .filter(|&(_, &s)| s >= size) + .min_by_key(|&(&o, &s)| (s, o)) + .map(|(&o, &s)| (o, s)) + } + + /// The serialized section info's size (`H5FS__sect_serialize_size`). + fn needed(&self, os: u8) -> u64 { + let n = self.sects.len() as u64; + let prefix = 4 + 1 + u64::from(os) + 4; + if n == 0 { + return prefix; + } + let sizes: BTreeSet = self.sects.values().copied().collect(); + prefix + + sizes.len() as u64 * (enc_size(n) + enc_size(self.max_sect_size)) as u64 + + n * (u64::from(self.max_sect_addr).div_ceil(8) + 1) + } + + fn serialize(&self, os: u8) -> Vec { + let n = self.sects.len() as u64; + let mut by_size: BTreeMap> = BTreeMap::new(); + for (&o, &s) in &self.sects { + by_size.entry(s).or_default().push(o); + } + let cnt_w = enc_size(n); + let len_w = enc_size(self.max_sect_size); + let off_w = usize::from(self.max_sect_addr).div_ceil(8); + let mut d = Vec::new(); + d.extend_from_slice(b"FSSE"); + d.push(0); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.addr, os); + d.extend_from_slice(&a); + for (size, offs) in by_size { + d.extend_from_slice(&(offs.len() as u64).to_le_bytes()[..cnt_w]); + d.extend_from_slice(&size.to_le_bytes()[..len_w]); + for o in offs { + d.extend_from_slice(&o.to_le_bytes()[..off_w]); + d.push(0); + } + } + d + } + + fn write_header(&self, img: &mut Image<'_>) -> Result<(), Error> { + let (os, ls) = (img.os, img.ls); + let len = Self::hdr_len(os, ls); + let l = ls as usize; + let n = self.sects.len() as u64; + let tot: u64 = self.sects.values().sum(); + let mut d = Vec::with_capacity(len); + d.extend_from_slice(b"FSHD"); + d.push(0); + d.push(FS_CLIENT_FHEAP); + for v in [tot, n, n, 0] { + d.extend_from_slice(&v.to_le_bytes()[..l]); + } + for v in [self.nclasses, self.shrink, self.expand, self.max_sect_addr] { + d.extend_from_slice(&v.to_le_bytes()); + } + d.extend_from_slice(&self.max_sect_size.to_le_bytes()[..l]); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.sect_addr, os); + d.extend_from_slice(&a); + d.extend_from_slice(&self.sect_size.to_le_bytes()[..l]); + d.extend_from_slice(&self.alloc_sect_size.to_le_bytes()[..l]); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + img.write(self.addr, &d) + } +} + +/// An open fractal heap. +pub(crate) struct Heap { + addr: u64, + id_len: u16, + flags: u8, + max_man_size: u32, + next_huge_id: u64, + huge_bt2: u64, + total_man_free: u64, + fs_addr: u64, + man_size: u64, + man_alloc_size: u64, + man_iter_off: u64, + man_nobjs: u64, + huge_size: u64, + huge_nobjs: u64, + tiny_size: u64, + tiny_nobjs: u64, + width: u16, + start_block_size: u64, + max_direct_size: u64, + max_index: u16, + start_root_rows: u16, + root: u64, + root_rows: u16, + heap_off_size: usize, + heap_len_size: usize, + max_direct_rows: usize, + max_root_rows: usize, + /// Root indirect block children (all rows), when the root is one. + ents: Vec, + iblock_dirty: bool, + fs: Option, + fs_dirty: bool, + /// Direct blocks changed (address -> size), rechecksummed by `finish`. + dirty_dblocks: BTreeMap, +} + +impl Heap { + /// Header size (unfiltered heaps): the fixed fields, ten lengths and + /// two addresses of statistics, the doubling table, the checksum. + fn header_len(os: u8, ls: u8) -> usize { + 26 + 12 * ls as usize + 3 * os as usize + } + + /// Open the heap whose header is at `addr`. + pub(crate) fn open(img: &Image<'_>, addr: u64) -> Result { + let (os, ls) = (img.os, img.ls); + let o = os as usize; + let l = ls as usize; + let len = Self::header_len(os, ls); + let d = img.read(addr, len)?; + if &d[0..4] != b"FRHP" || d[4] != 0 { + return Err(bad("bad header")); + } + let filter_len = u16::from_le_bytes([d[7], d[8]]); + if filter_len != 0 { + return Err(unsupported("heaps with I/O filters")); + } + let stored = u32::from_le_bytes(d[len - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..len - 4]) != stored { + return Err(bad("header checksum mismatch")); + } + let mut p = 14usize; + let mut next = |w: usize| { + let v = le(&d[p..p + w]); + p += w; + v + }; + let next_huge_id = next(l); + let huge_bt2 = next(o); + let total_man_free = next(l); + let fs_addr = next(o); + let man_size = next(l); + let man_alloc_size = next(l); + let man_iter_off = next(l); + let man_nobjs = next(l); + let huge_size = next(l); + let huge_nobjs = next(l); + let tiny_size = next(l); + let tiny_nobjs = next(l); + let width = next(2) as u16; + let start_block_size = next(l); + let max_direct_size = next(l); + let max_index = next(2) as u16; + let start_root_rows = next(2) as u16; + let root = next(o); + let root_rows = next(2) as u16; + let pow2 = |v: u64| v != 0 && v.is_power_of_two(); + if !pow2(u64::from(width)) + || !pow2(start_block_size) + || !pow2(max_direct_size) + || max_direct_size < start_block_size + || !(1..=64).contains(&max_index) + { + return Err(bad("bad doubling table")); + } + let start_bits = log2(start_block_size) as usize; + let first_row_bits = start_bits + log2(u64::from(width)) as usize; + if usize::from(max_index) < first_row_bits { + return Err(bad("bad doubling table")); + } + let max_direct_rows = log2(max_direct_size) as usize - start_bits + 2; + let max_root_rows = usize::from(max_index) - first_row_bits + 1; + let heap_off_size = usize::from(max_index).div_ceil(8); + // H5HF__hdr_finish_init_phase1: object lengths are encoded in the + // bytes a direct block offset, or a managed object's size, needs. + let max_man_size = u32::from_le_bytes([d[10], d[11], d[12], d[13]]); + let len_bits = |v: u64| (log2(v) as usize).div_ceil(8); + let heap_len_size = len_bits(max_direct_size).min(len_bits(u64::from(max_man_size))); + let id_len = u16::from_le_bytes([d[5], d[6]]); + let mut h = Self { + addr, + id_len, + flags: d[9], + max_man_size, + next_huge_id, + huge_bt2, + total_man_free, + fs_addr, + man_size, + man_alloc_size, + man_iter_off, + man_nobjs, + huge_size, + huge_nobjs, + tiny_size, + tiny_nobjs, + width, + start_block_size, + max_direct_size, + max_index, + start_root_rows, + root, + root_rows, + heap_off_size, + heap_len_size, + max_direct_rows, + max_root_rows, + ents: Vec::new(), + iblock_dirty: false, + fs: None, + fs_dirty: false, + dirty_dblocks: BTreeMap::new(), + }; + if usize::from(id_len) != 1 + heap_off_size + heap_len_size && id_len < 8 { + return Err(unsupported("unusual heap ID length")); + } + if root_rows > 0 { + if root == undef(os) || usize::from(root_rows) > max_root_rows { + return Err(bad("bad root indirect block")); + } + let n = usize::from(root_rows) * usize::from(width); + let blen = h.iblock_len(root_rows, os); + let b = img.read(root, blen)?; + let stored = u32::from_le_bytes(b[blen - 4..].try_into().unwrap_or([0; 4])); + if &b[0..4] != b"FHIB" + || b[4] != 0 + || get_uint(&b[5..], os) != addr + || le(&b[5 + o..5 + o + heap_off_size]) != 0 + || clawhdf5_format::checksum::jenkins_lookup3(&b[..blen - 4]) != stored + { + return Err(bad("bad root indirect block")); + } + let at = 5 + o + heap_off_size; + h.ents = (0..n).map(|i| get_uint(&b[at + i * o..], os)).collect(); + let direct = h.max_direct_rows * usize::from(width); + if h.ents.iter().skip(direct).any(|&a| a != undef(os)) { + return Err(unsupported("child indirect blocks")); + } + } + if fs_addr != undef(os) { + let fs = FreeSpace::open(img, fs_addr)?; + if fs.max_sect_addr != max_index { + return Err(bad("free-space manager does not match the heap")); + } + h.fs = Some(fs); + } + Ok(h) + } + + /// Create an empty heap with libhdf5's attribute-heap parameters + /// (`H5A__dense_create`: width 4, 1 KiB starting blocks, 64 KiB direct + /// blocks, 40-bit offsets, 1 starting root row, checksummed direct + /// blocks, 4 KiB managed objects, 8-byte IDs). + pub(crate) fn create_attribute_heap(img: &mut Image<'_>) -> Result { + let (os, ls) = (img.os, img.ls); + let addr = img.alloc_reusing(Self::header_len(os, ls) as u64)?; + let mut h = Self { + addr, + id_len: 8, + flags: 0x02, + max_man_size: 4096, + next_huge_id: 0, + huge_bt2: undef(os), + total_man_free: 0, + fs_addr: undef(os), + man_size: 0, + man_alloc_size: 0, + man_iter_off: 0, + man_nobjs: 0, + huge_size: 0, + huge_nobjs: 0, + tiny_size: 0, + tiny_nobjs: 0, + width: 4, + start_block_size: 1024, + max_direct_size: 65536, + max_index: 40, + start_root_rows: 1, + root: undef(os), + root_rows: 0, + heap_off_size: 5, + heap_len_size: 2, + max_direct_rows: 8, + max_root_rows: 29, + ents: Vec::new(), + iblock_dirty: false, + fs: None, + fs_dirty: false, + dirty_dblocks: BTreeMap::new(), + }; + h.write_header(img)?; + Ok(h) + } + + pub(crate) fn address(&self) -> u64 { + self.addr + } + + fn overhead(&self, os: u8) -> u64 { + 5 + u64::from(os) + self.heap_off_size as u64 + if self.flags & 0x02 != 0 { 4 } else { 0 } + } + + fn row_size(&self, row: usize) -> u64 { + if row == 0 { + self.start_block_size + } else { + self.start_block_size << (row - 1) + } + } + + fn row_off(&self, row: usize) -> u64 { + if row == 0 { + 0 + } else { + (self.start_block_size * u64::from(self.width)) << (row - 1) + } + } + + fn iblock_len(&self, rows: u16, os: u8) -> usize { + 5 + os as usize + + self.heap_off_size + + usize::from(rows) * usize::from(self.width) * os as usize + + 4 + } + + fn first_row_bits(&self) -> u32 { + log2(self.start_block_size) + log2(u64::from(self.width)) + } + + /// Row and column of heap offset `off` in the root indirect block + /// (`H5HF__dtable_lookup`). + fn lookup(&self, off: u64) -> (usize, usize) { + let w = u64::from(self.width); + if off < self.start_block_size * w { + (0, (off / self.start_block_size) as usize) + } else { + let hb = log2(off); + let row = (hb - self.first_row_bits() + 1) as usize; + (row, ((off - (1u64 << hb)) / self.row_size(row)) as usize) + } + } + + /// The direct block holding heap offset `off`: (address, size, heap + /// offset of the block). + fn dblock_of(&self, os: u8, off: u64) -> Result<(u64, u64, u64), Error> { + if self.root == undef(os) { + return Err(bad("empty heap")); + } + if self.root_rows == 0 { + if off >= self.start_block_size { + return Err(bad("offset outside the heap")); + } + return Ok((self.root, self.start_block_size, 0)); + } + let (row, col) = self.lookup(off); + if row >= self.max_direct_rows || row >= usize::from(self.root_rows) { + return Err(unsupported("objects below child indirect blocks")); + } + let a = self.ents[row * usize::from(self.width) + col]; + if a == undef(os) { + return Err(bad("object in an unallocated block")); + } + Ok(( + a, + self.row_size(row), + self.row_off(row) + self.row_size(row) * col as u64, + )) + } + + /// Decode a managed heap ID: (offset, length). + fn man_id(&self, id: &[u8]) -> (u64, u64) { + let o = &id[1..]; + ( + le(&o[..self.heap_off_size]), + le(&o[self.heap_off_size..self.heap_off_size + self.heap_len_size]), + ) + } + + fn huge_id(&self, id: &[u8], os: u8, ls: u8) -> Result { + let w = usize::from(self.id_len - 1).min(8); + if self.huge_ids_direct(os, ls) { + return Err(unsupported("directly addressed huge objects")); + } + Ok(le(&id[1..1 + w])) + } + + fn huge_ids_direct(&self, os: u8, ls: u8) -> bool { + usize::from(os) + usize::from(ls) < usize::from(self.id_len) + } + + /// Where an object lives: (file address, length). + fn locate(&self, img: &Image<'_>, id: &[u8]) -> Result<(u64, u64), Error> { + if id.len() != usize::from(self.id_len) || id[0] >> 6 != 0 { + return Err(bad("bad heap ID")); + } + match (id[0] >> 4) & 0x03 { + 0 => { + let (off, len) = self.man_id(id); + let (a, size, boff) = self.dblock_of(img.os, off)?; + let within = off - boff; + if within < self.overhead(img.os) || within + len > size { + return Err(bad("object outside its block")); + } + Ok((a + within, len)) + } + 1 => { + if self.huge_ids_direct(img.os, img.ls) { + return Err(unsupported("directly addressed huge objects")); + } + let key = self.huge_id(id, img.os, img.ls)?; + let mut t = Bt2::open(img, self.huge_bt2)?; + let rec = t + .find( + img, + &mut |_, r| Ok(key.cmp(&huge_rec_id(r, img.os, img.ls))), + )? + .ok_or_else(|| bad("huge object missing from its B-tree"))?; + Ok(( + get_uint(&rec, img.os), + get_uint(&rec[img.os as usize..], img.ls), + )) + } + _ => Err(unsupported("tiny objects")), + } + } + + /// The object `id` names. + pub(crate) fn read(&self, img: &Image<'_>, id: &[u8]) -> Result, Error> { + let (a, len) = self.locate(img, id)?; + let n = usize::try_from(len).map_err(|_| bad("object too large"))?; + img.read(a, n) + } + + /// Overwrite managed object `id` with `obj` of the same length + /// (`H5HF_write`). + pub(crate) fn write_in_place( + &mut self, + img: &mut Image<'_>, + id: &[u8], + obj: &[u8], + ) -> Result { + if (id[0] >> 4) & 0x03 == 1 { + // H5HF__huge_write: an unfiltered huge object in place. + let (a, len) = self.locate(img, id)?; + if len != obj.len() as u64 { + return Ok(false); + } + img.write(a, obj)?; + return Ok(true); + } + if (id[0] >> 4) & 0x03 != 0 { + return Ok(false); + } + let (off, len) = self.man_id(id); + if len != obj.len() as u64 { + return Ok(false); + } + let (a, size, boff) = self.dblock_of(img.os, off)?; + img.write(a + (off - boff), obj)?; + self.dirty_dblocks.insert(a, size); + Ok(true) + } + + /// Insert `obj`; returns its heap ID (`H5HF_insert`). + pub(crate) fn insert(&mut self, img: &mut Image<'_>, obj: &[u8]) -> Result, Error> { + let os = img.os; + let n = obj.len() as u64; + if n == 0 { + return Err(bad("empty object")); + } + if n > u64::from(self.max_man_size) { + return self.huge_insert(img, obj); + } + let tiny_max = u64::from(self.id_len) - 1; + if n <= tiny_max { + return Err(unsupported("tiny objects")); + } + let (off, size) = match self.fs.as_ref().and_then(|fs| fs.find(n)) { + Some((o, s)) => { + self.fs.as_mut().expect("found above").sects.remove(&o); + (o, s) + } + None => self.dblock_new(img, n)?, + }; + // H5HF__sect_single_reduce: the object goes at the section's start. + if size > n { + self.fs_add(img, off + n, size - n)?; + } + self.fs_dirty = true; + let (a, bsize, boff) = self.dblock_of(os, off)?; + img.write(a + (off - boff), obj)?; + self.dirty_dblocks.insert(a, bsize); + self.man_nobjs += 1; + self.total_man_free = self.total_man_free.saturating_sub(n); + let mut id = vec![0u8; usize::from(self.id_len)]; + id[1..1 + self.heap_off_size].copy_from_slice(&off.to_le_bytes()[..self.heap_off_size]); + id[1 + self.heap_off_size..1 + self.heap_off_size + self.heap_len_size] + .copy_from_slice(&n.to_le_bytes()[..self.heap_len_size]); + Ok(id) + } + + /// Add a single section (no merging: `H5FS_sect_add` without + /// `H5FS_ADD_RETURNED_SPACE`), creating the free-space manager if the + /// heap has none. + fn fs_add(&mut self, img: &mut Image<'_>, off: u64, size: u64) -> Result<(), Error> { + if self.fs.is_none() { + let addr = img.alloc_reusing(FreeSpace::hdr_len(img.os, img.ls) as u64)?; + self.fs = Some(FreeSpace { + addr, + max_sect_addr: self.max_index, + max_sect_size: self.max_direct_size, + shrink: FS_SHRINK, + expand: FS_EXPAND, + nclasses: FS_NCLASSES, + sect_addr: undef(img.os), + sect_size: 0, + alloc_sect_size: 0, + sects: BTreeMap::new(), + }); + self.fs_addr = addr; + } + self.fs + .as_mut() + .expect("created above") + .sects + .insert(off, size); + self.fs_dirty = true; + Ok(()) + } + + /// `H5HF__man_dblock_new`: a direct block for an object of `request` + /// bytes; returns its free section (not in the free-space manager). + fn dblock_new(&mut self, img: &mut Image<'_>, request: u64) -> Result<(u64, u64), Error> { + let os = img.os; + let mut min = if request < self.start_block_size { + self.start_block_size + } else { + 1u64 << (1 + log2(request)) + }; + if min < self.overhead(os) + request { + min *= 2; + } + if min > self.max_direct_size { + return Err(bad("object larger than a direct block")); + } + if self.root == undef(os) && min == self.start_block_size { + let a = self.dblock_create(img, 0, self.start_block_size)?; + self.root = a; + self.root_rows = 0; + self.man_size = self.start_block_size; + self.total_man_free += self.start_block_size - self.overhead(os); + return Ok((self.overhead(os), self.start_block_size - self.overhead(os))); + } + if self.root == undef(os) { + return Err(unsupported( + "a first object too large for the starting block", + )); + } + // H5HF__hdr_update_iter. + if self.root_rows == 0 { + self.root_create(img, min)?; + } + let (mut row, mut col) = self.iter_pos(); + let min_row = self.size_to_row(min); + if min_row > row && row < usize::from(self.root_rows) { + return Err(unsupported("skipping blocks too small for an object")); + } + while row >= usize::from(self.root_rows) { + self.root_double(img, min)?; + (row, col) = self.iter_pos(); + } + if row >= self.max_direct_rows { + return Err(unsupported("child indirect blocks")); + } + let size = self.row_size(row); + if min > size { + return Err(unsupported("skipping blocks too small for an object")); + } + self.man_iter_off += size; + let entry = row * usize::from(self.width) + col; + let block_off = self.row_off(row) + size * col as u64; + let a = self.dblock_create(img, block_off, size)?; + self.ents[entry] = a; + self.iblock_dirty = true; + Ok((block_off + self.overhead(os), size - self.overhead(os))) + } + + /// The allocation iterator's (row, column) in the root indirect block. + fn iter_pos(&self) -> (usize, usize) { + if self.man_iter_off >= self.man_size { + (usize::from(self.root_rows), 0) + } else { + self.lookup(self.man_iter_off) + } + } + + /// `H5HF__dtable_size_to_row`. + fn size_to_row(&self, size: u64) -> usize { + if size == self.start_block_size { + 0 + } else { + (log2(size) - log2(self.start_block_size) + 1) as usize + } + } + + /// Allocate and write an empty direct block at heap offset `block_off`. + fn dblock_create( + &mut self, + img: &mut Image<'_>, + block_off: u64, + size: u64, + ) -> Result { + let os = img.os; + let a = img.alloc_reusing(size)?; + let n = usize::try_from(size).map_err(|_| bad("block too large"))?; + let mut d = vec![0u8; n]; + d[0..4].copy_from_slice(b"FHDB"); + put_uint(&mut d[5..], self.addr, os); + let at = 5 + os as usize; + d[at..at + self.heap_off_size] + .copy_from_slice(&block_off.to_le_bytes()[..self.heap_off_size]); + img.write(a, &d)?; + self.dirty_dblocks.insert(a, size); + self.man_alloc_size += size; + Ok(a) + } + + /// `H5HF__man_iblock_root_create`: the root direct block becomes entry + /// 0 of a new root indirect block. + fn root_create(&mut self, img: &mut Image<'_>, min: u64) -> Result<(), Error> { + let os = img.os; + let mut nrows = if self.start_root_rows == 0 { + self.max_root_rows + } else { + usize::from(self.start_root_rows) + }; + if self.start_root_rows != 0 { + let mut block_row_off = (log2(min) - log2(self.start_block_size)) as usize; + if block_row_off > 0 { + block_row_off += 1; + } + nrows = nrows.max(1 + block_row_off); + } + if nrows > self.max_direct_rows { + return Err(unsupported("child indirect blocks")); + } + if min > self.start_block_size { + return Err(unsupported("skipping blocks too small for an object")); + } + let have_direct = self.root != undef(os); + let rows = u16::try_from(nrows).map_err(|_| bad("too many rows"))?; + let a = img.alloc_reusing(self.iblock_len(rows, os) as u64)?; + self.ents = vec![undef(os); nrows * usize::from(self.width)]; + if have_direct { + self.ents[0] = self.root; + self.man_iter_off = self.start_block_size; + } else { + self.man_iter_off = 0; + } + self.iblock_dirty = true; + self.root_rows = rows; + self.root = a; + let w = u64::from(self.width); + let ov = self.overhead(os); + let mut acc: u64 = (0..nrows).map(|u| (self.row_size(u) - ov) * w).sum(); + if have_direct { + acc -= self.row_size(0) - ov; + } + self.man_size = self.row_off(nrows); + self.total_man_free += acc; + Ok(()) + } + + /// `H5HF__man_iblock_root_double`: the root indirect block gets twice + /// its rows, at a new address. + fn root_double(&mut self, img: &mut Image<'_>, min: u64) -> Result<(), Error> { + let os = img.os; + let old = usize::from(self.root_rows); + let (row, _) = self.iter_pos(); + let next_size = self.row_size(row.min(self.max_root_rows - 1)); + if old < self.max_direct_rows && min > next_size { + return Err(unsupported("skipping blocks too small for an object")); + } + let new = (2 * old).min(self.max_root_rows); + if new > self.max_direct_rows || new == old { + return Err(unsupported("child indirect blocks")); + } + img.free(self.root, self.iblock_len(self.root_rows, os) as u64); + let rows = u16::try_from(new).map_err(|_| bad("too many rows"))?; + let a = img.alloc_reusing(self.iblock_len(rows, os) as u64)?; + let w = usize::from(self.width); + self.ents.resize(new * w, undef(os)); + let ov = self.overhead(os); + let acc: u64 = (old * w..new * w).map(|u| self.row_size(u / w) - ov).sum(); + self.root_rows = rows; + self.root = a; + self.iblock_dirty = true; + self.man_size = 2 * self.row_off(new - 1); + self.total_man_free += acc; + Ok(()) + } + + /// Remove object `id` (`H5HF_remove`): a managed object's space goes + /// back to the free-space manager, merged with free space next to it; a + /// huge object's file space is freed. + pub(crate) fn remove(&mut self, img: &mut Image<'_>, id: &[u8]) -> Result<(), Error> { + match (id[0] >> 4) & 0x03 { + 0 => { + let os = img.os; + let (off, len) = self.man_id(id); + let (_, size, boff) = self.dblock_of(os, off)?; + let (mut lo, mut hi) = (off, off + len); + if let Some(fs) = &self.fs { + if let Some((&o, &s)) = fs.sects.range(..off).next_back() + && o + s == off + { + lo = o; + } + if let Some(&s) = fs.sects.get(&hi) { + hi += s; + } + } + if hi - lo == size - self.overhead(os) && boff <= lo { + return Err(unsupported( + "removing the last object of a direct block (libhdf5 frees the block)", + )); + } + if let Some(fs) = &mut self.fs { + fs.sects.remove(&lo); + fs.sects.remove(&(off + len)); + } + self.fs_add(img, lo, hi - lo)?; + self.total_man_free += len; + self.man_nobjs -= 1; + Ok(()) + } + 1 => { + let key = self.huge_id(id, img.os, img.ls)?; + let (os, ls) = (img.os, img.ls); + let mut t = Bt2::open(img, self.huge_bt2)?; + let rec = t + .remove(img, &mut |_, r| Ok(key.cmp(&huge_rec_id(r, os, ls))))? + .ok_or_else(|| bad("huge object missing from its B-tree"))?; + t.finish(img)?; + let len = get_uint(&rec[os as usize..], ls); + img.free(get_uint(&rec, os), len); + self.huge_size = self.huge_size.saturating_sub(len); + self.huge_nobjs = self.huge_nobjs.saturating_sub(1); + Ok(()) + } + _ => Err(unsupported("tiny objects")), + } + } + + /// `H5HF__huge_insert` (unfiltered heap, IDs that index the huge-object + /// B-tree). + fn huge_insert(&mut self, img: &mut Image<'_>, obj: &[u8]) -> Result, Error> { + let (os, ls) = (img.os, img.ls); + if self.huge_ids_direct(os, ls) { + return Err(unsupported("directly addressed huge objects")); + } + let rs = os as usize + 2 * ls as usize; + let mut t = if self.huge_bt2 == undef(os) { + let t = Bt2::create(img, HUGE_BT2_TYPE, HUGE_BT2_NODE, rs, 100, 40)?; + self.huge_bt2 = t.address(); + t + } else { + Bt2::open(img, self.huge_bt2)? + }; + if t.record_size() != rs || t.tree_type() != HUGE_BT2_TYPE { + return Err(unsupported("huge-object B-tree layout")); + } + let n = obj.len() as u64; + let a = img.alloc_reusing(n)?; + img.write(a, obj)?; + let w = usize::from(self.id_len - 1).min(8); + let max_id = if w >= 8 { + u64::MAX + } else { + (1u64 << (8 * w)) - 1 + }; + if self.flags & 0x01 != 0 || self.next_huge_id >= max_id { + return Err(unsupported("huge object IDs wrapped")); + } + self.next_huge_id += 1; + let key = self.next_huge_id; + if key == max_id { + self.flags |= 0x01; + } + let mut rec = vec![0u8; rs]; + put_uint(&mut rec, a, os); + put_uint(&mut rec[os as usize..], n, ls); + put_uint(&mut rec[os as usize + ls as usize..], key, ls); + t.insert(img, &mut |_, r| Ok(key.cmp(&huge_rec_id(r, os, ls))), &rec)?; + t.finish(img)?; + self.huge_size += n; + self.huge_nobjs += 1; + let mut id = vec![0u8; usize::from(self.id_len)]; + id[0] = 0x10; + id[1..1 + w].copy_from_slice(&key.to_le_bytes()[..w]); + Ok(id) + } + + fn write_header(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let (os, ls) = (img.os, img.ls); + let l = ls as usize; + let mut d = Vec::with_capacity(Self::header_len(os, ls)); + d.extend_from_slice(b"FRHP"); + d.push(0); + d.extend_from_slice(&self.id_len.to_le_bytes()); + d.extend_from_slice(&0u16.to_le_bytes()); + d.push(self.flags); + d.extend_from_slice(&self.max_man_size.to_le_bytes()); + let addr = |d: &mut Vec, v: u64| { + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, v, os); + d.extend_from_slice(&a); + }; + d.extend_from_slice(&self.next_huge_id.to_le_bytes()[..l]); + addr(&mut d, self.huge_bt2); + d.extend_from_slice(&self.total_man_free.to_le_bytes()[..l]); + addr(&mut d, self.fs_addr); + for v in [ + self.man_size, + self.man_alloc_size, + self.man_iter_off, + self.man_nobjs, + self.huge_size, + self.huge_nobjs, + self.tiny_size, + self.tiny_nobjs, + ] { + d.extend_from_slice(&v.to_le_bytes()[..l]); + } + d.extend_from_slice(&self.width.to_le_bytes()); + d.extend_from_slice(&self.start_block_size.to_le_bytes()[..l]); + d.extend_from_slice(&self.max_direct_size.to_le_bytes()[..l]); + d.extend_from_slice(&self.max_index.to_le_bytes()); + d.extend_from_slice(&self.start_root_rows.to_le_bytes()); + addr(&mut d, self.root); + d.extend_from_slice(&self.root_rows.to_le_bytes()); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + img.write(self.addr, &d) + } + + /// Write back everything that changed: direct block checksums, the + /// root indirect block, the free-space manager, the header. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let os = img.os; + if self.flags & 0x02 != 0 { + let at = 5 + os as usize + self.heap_off_size; + for (a, size) in std::mem::take(&mut self.dirty_dblocks) { + let n = usize::try_from(size).map_err(|_| bad("block too large"))?; + let mut b = img.read(a, n)?; + b[at..at + 4].fill(0); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&b); + img.write(a + at as u64, &sum.to_le_bytes())?; + } + } + if self.iblock_dirty && self.root_rows > 0 { + let len = self.iblock_len(self.root_rows, os); + let mut b = Vec::with_capacity(len); + b.extend_from_slice(b"FHIB"); + b.push(0); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.addr, os); + b.extend_from_slice(&a); + b.extend_from_slice(&vec![0u8; self.heap_off_size]); + for &e in &self.ents { + put_uint(&mut a, e, os); + b.extend_from_slice(&a); + } + let sum = clawhdf5_format::checksum::jenkins_lookup3(&b); + b.extend_from_slice(&sum.to_le_bytes()); + img.write(self.root, &b)?; + self.iblock_dirty = false; + } + if self.fs_dirty { + self.fs_dirty = false; + if let Some(mut fs) = self.fs.take() { + let hdr_len = FreeSpace::hdr_len(os, img.ls) as u64; + if fs.sects.is_empty() { + // H5HF__space_close: a manager that tracks nothing is + // deleted. + img.free(fs.addr, hdr_len); + if fs.sect_addr != undef(os) { + img.free(fs.sect_addr, fs.alloc_sect_size); + } + self.fs_addr = undef(os); + } else { + let need = fs.needed(os); + if fs.sect_addr == undef(os) || need != fs.alloc_sect_size { + if fs.sect_addr != undef(os) { + img.free(fs.sect_addr, fs.alloc_sect_size); + } + fs.sect_addr = img.alloc_reusing(need)?; + fs.alloc_sect_size = need; + } + fs.sect_size = fs.alloc_sect_size; + let mut s = fs.serialize(os); + let n = usize::try_from(fs.sect_size).map_err(|_| bad("section info"))?; + s.resize(n - 4, 0); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&s); + s.extend_from_slice(&sum.to_le_bytes()); + img.write(fs.sect_addr, &s)?; + fs.write_header(img)?; + self.fs = Some(fs); + } + } + } + self.write_header(img) + } +} + +/// The ID in a huge-object B-tree record (type 1: address, length, ID). +fn huge_rec_id(r: &[u8], os: u8, ls: u8) -> u64 { + get_uint(&r[os as usize + ls as usize..], ls) +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index b14244a..c32431a 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -15,10 +15,12 @@ //! file) is dropped before the first write, so no `&[u8]` over the mapping //! is alive while the file changes (see `image`). +mod attrs; mod btree1; mod btree2; mod earray; mod farray; +mod fheap; mod image; mod ohdr; mod select; @@ -27,7 +29,6 @@ use std::collections::{BTreeMap, HashMap}; use std::fs::{OpenOptions, TryLockError}; use std::path::{Path, PathBuf}; -use clawhdf5_format::attribute::AttributeMessage; use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks}; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::data_read::NativeElement; @@ -43,8 +44,8 @@ use btree1::{BTree1, Key}; use btree2::Bt2; use earray::{Ea, EaParams, Elem}; use farray::Fa; -use image::{Image, get_uint, put_uint, undef}; -use ohdr::{Header, MSG_ATTRIBUTE}; +use image::{Image, put_uint, undef}; +use ohdr::Header; const MSG_DATASPACE: u16 = 0x01; const MSG_LAYOUT: u16 = 0x08; @@ -918,101 +919,29 @@ impl FileEditor { /// Set attribute `name` of the object at `path` (a group or dataset; /// `"/"` is the root group) to `value`, replacing an attribute of that - /// name. The attribute goes into free space in the object header, or a - /// new header continuation chunk at the end of the file. + /// name, as libhdf5 creates attributes (`H5O__attr_create`). /// - /// [`Error::Unsupported`] for an object whose attributes are in dense - /// storage (or would have to move there: more than the object's - /// compact-attribute limit), one that tracks attribute creation order, - /// or one with shared attribute messages. + /// A compact attribute goes into free space in the object header, or a + /// new header continuation chunk; an object that tracks creation order + /// gives it the next creation index. When the object reaches its + /// compact-attribute limit (or the attribute is too large for a header + /// message) its attributes move to dense storage, and objects already + /// using dense storage get the attribute there: a fractal heap object + /// (in free heap space, a new heap block, or its own file space when + /// larger than the heap's managed limit) indexed by name and, when the + /// object indexes creation order, by creation index. An attribute + /// replaced by one of the same encoded size is rewritten where it is. + /// + /// [`Error::Unsupported`] for shared attribute messages, heaps this + /// editor cannot extend the way libhdf5 would (child indirect blocks, + /// skipped blocks, free space other than within direct blocks, freeing + /// a whole block), and version-1 object headers asked for an attribute + /// larger than a header message holds. pub fn set_attr(&mut self, path: &str, name: &str, value: &AttrValue) -> Result<(), Error> { if name.is_empty() { return Err(Error::InvalidArgument("empty attribute name".into())); } - self.edit(|f, img| { - let addr = - clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?; - let mut hdr = Header::load(img, addr)?; - if hdr.version == 2 && hdr.flags & 0x04 != 0 { - return Err(Error::Unsupported( - "object tracks attribute creation order".into(), - )); - } - if let Some(i) = hdr.find(MSG_ATTR_INFO) { - let d = hdr.data(img, i)?; - // version(1) flags(1) [max creation index(2)] fractal heap - // address, name index address, [order index address]. - let mut p = 2; - if d.get(1).is_some_and(|f| f & 0x01 != 0) { - p += 2; - } - let os = img.os as usize; - if d.len() < p + os { - return Err(Error::Unsupported("short attribute info message".into())); - } - if get_uint(&d[p..], img.os) != undef(img.os) { - return Err(Error::Unsupported( - "object with attributes in dense storage".into(), - )); - } - } - let mut existing = None; - let mut count = 0usize; - for i in 0..hdr.msgs.len() { - if hdr.msgs[i].mtype != MSG_ATTRIBUTE { - continue; - } - if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { - return Err(Error::Unsupported("shared attribute message".into())); - } - count += 1; - if attr_name(&hdr.data(img, i)?)? == name.as_bytes() { - existing = Some(i); - } - } - if existing.is_none() && hdr.version == 2 { - let max_compact = max_compact_attrs(img, &hdr)?; - if count + 1 > usize::from(max_compact) { - return Err(Error::Unsupported(format!( - "object already has {count} compact attributes (its limit is \ - {max_compact}); more need dense storage" - ))); - } - } - let msg = clawhdf5_format::type_builders::build_attr_message(name, value); - check_plain(&msg.datatype)?; - let body = if hdr.version == 1 { - encode_attr_v1(&msg, img.ls) - } else { - let mut b = msg.serialize_v3(img.ls); - if !name.is_ascii() { - b[8] = 1; // UTF-8 name - } - b - }; - if let Some(i) = existing { - hdr.delete(img, i)?; - } - // libhdf5 counts a version-2 header's attributes through its - // Attribute Info message and reports none without one; like - // H5O__attr_create, add it when missing: before the attribute - // when free space holds both (libhdf5's order), else after it, - // so that a new continuation chunk made for the attribute has - // room for it too. - let ainfo = (hdr.version == 2 && hdr.find(MSG_ATTR_INFO).is_none()) - .then(|| attr_info_message(hdr.flags, img.os)); - let ainfo_first = ainfo - .as_ref() - .is_some_and(|a| hdr.has_free(a.len() + hdr.hsize() + body.len())); - if let Some(a) = ainfo.as_ref().filter(|_| ainfo_first) { - hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?; - } - hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?; - if let Some(a) = ainfo.as_ref().filter(|_| !ainfo_first) { - hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?; - } - hdr.finish(img) - }) + self.edit(|f, img| attrs::set_attr(f, img, path, name, value)) } } @@ -1097,81 +1026,6 @@ fn set_superblock_eof( Ok(()) } -/// The name bytes of an attribute message body (without the NUL). -fn attr_name(d: &[u8]) -> Result<&[u8], Error> { - let bad = || Error::Unsupported("malformed attribute message".into()); - let (len, at) = match d.first() { - Some(1) | Some(2) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8), - Some(3) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9), - _ => return Err(bad()), - }; - let name = d.get(at..at + len).ok_or_else(bad)?; - Ok(name.split(|&b| b == 0).next().unwrap_or(name)) -} - -/// A new Attribute Info message for a version-2 header with flags -/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order -/// tracked / indexed as the header's flags say, maximum creation index 0, -/// and no dense storage (undefined fractal heap and B-tree addresses). -fn attr_info_message(hdr_flags: u8, os: u8) -> Vec { - let track = hdr_flags & 0x04 != 0; - let index = hdr_flags & 0x08 != 0; - let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)]; - if track { - b.extend_from_slice(&0u16.to_le_bytes()); - } - let undef_addr = vec![0xffu8; os as usize]; - b.extend_from_slice(&undef_addr); - b.extend_from_slice(&undef_addr); - if index { - b.extend_from_slice(&undef_addr); - } - b -} - -/// A version-2 header's limit on compact attributes: stored when its flags -/// say so, else libhdf5's default of 8. -fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result { - if hdr.flags & 0x10 == 0 { - return Ok(8); - } - let mut p = hdr.addr + 6; - if hdr.flags & 0x20 != 0 { - p += 16; - } - let b = img.read(p, 2)?; - Ok(u16::from_le_bytes([b[0], b[1]])) -} - -/// A version-1 attribute message (what libhdf5 writes in a version-1 object -/// header): name, datatype and dataspace each padded to 8 bytes, the -/// dataspace as a version-1 dataspace message. -fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec { - let mut name = a.name.as_bytes().to_vec(); - name.push(0); - let dt = a.datatype.serialize(); - let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0]; - if a.dataspace.space_type == DataspaceType::Simple { - for &d in &a.dataspace.dimensions { - let mut b = vec![0u8; ls as usize]; - put_uint(&mut b, d, ls); - ds.extend_from_slice(&b); - } - } else { - ds[1] = 0; - } - let mut out = vec![1u8, 0]; - out.extend_from_slice(&(name.len() as u16).to_le_bytes()); - out.extend_from_slice(&(dt.len() as u16).to_le_bytes()); - out.extend_from_slice(&(ds.len() as u16).to_le_bytes()); - for part in [&name, &dt, &ds] { - out.extend_from_slice(part); - out.resize(out.len().next_multiple_of(8), 0); - } - out.extend_from_slice(&a.raw_data); - out -} - fn write_selection( f: &File, img: &mut Image<'_>, diff --git a/crates/clawhdf5/src/edit/ohdr.rs b/crates/clawhdf5/src/edit/ohdr.rs index f3abfe0..8aa2a15 100644 --- a/crates/clawhdf5/src/edit/ohdr.rs +++ b/crates/clawhdf5/src/edit/ohdr.rs @@ -59,7 +59,7 @@ pub(crate) struct Header { added: usize, } -const MAX_CHUNKS: usize = 1024; +const MAX_CHUNKS: usize = 1 << 16; fn corrupt(why: &'static str) -> Error { Error::Format(FormatError::InvalidObjectHeader(why)) @@ -118,7 +118,11 @@ impl Header { }); h.scan(img, 0, addr + 16, addr + 16 + size, &mut pending)?; } - while let Some((caddr, clen)) = pending.pop() { + // Continuation chunks in the order their messages are found, as + // H5O_protect loads them (so messages keep libhdf5's order). + let mut next = 0; + while let Some(&(caddr, clen)) = pending.get(next) { + next += 1; if h.chunks.len() >= MAX_CHUNKS { return Err(corrupt("too many object header chunks")); }