quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Remote reads (range-read M2/M3: open_storage, HTTP/S3) and near-complete in-place editing #18

Merged
osobh merged 52 commits from feat/p3-remote-editor into main 2026-09-27 11:15:23 +00:00
Conversation 0 Commits 52 Files Changed 80
+2165 -216
7 changed files with 2165 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 773f427f16 - Show all commits
crates/clawhdf5-tools/tests/edit_coverage_interop.rs
+494 -1
View File
@@ -213,7 +213,6 @@ fn tmpdir() -> tempfile::TempDir {
tempfile::TempDir::new_in(base).unwrap() tempfile::TempDir::new_in(base).unwrap()
} }
#[allow(dead_code)]
fn unsupported<T: std::fmt::Debug>(r: Result<T, Error>) { fn unsupported<T: std::fmt::Debug>(r: Result<T, Error>) {
match r { match r {
Err(Error::Unsupported(_)) => {} Err(Error::Unsupported(_)) => {}
@@ -830,3 +829,497 @@ fn btree2_random_chunk_order() {
verify(&path, "x", &m); verify(&path, "x", &m);
check_tools(&path, false); check_tools(&path, false);
} }
// ---- dense attributes ----
/// An attribute value both sides can write with the same encoded size: a
/// 1-D int64 array or a fixed-length string.
#[derive(Clone, Debug, PartialEq)]
enum AV {
Ints(Vec<i64>),
Str(String),
}
impl AV {
fn value(&self) -> clawhdf5::AttrValue {
match self {
AV::Ints(v) => clawhdf5::AttrValue::I64Array(v.clone()),
AV::Str(s) => clawhdf5::AttrValue::String(s.clone()),
}
}
fn py(&self) -> String {
match self {
AV::Ints(v) => format!("np.array({v:?}, dtype='<i8')"),
AV::Str(s) => format!("np.bytes_({s:?})"),
}
}
fn py_expect(&self) -> String {
match self {
AV::Ints(v) => format!("{v:?}"),
AV::Str(s) => format!("{s:?}"),
}
}
fn make(i: u64, salt: u64) -> Self {
match i % 11 {
5 => AV::Str("h".repeat(5000 + (salt % 50) as usize)),
1 | 4 | 8 => AV::Str(
(0..10 + (i * 13 + salt) % 300)
.map(|k| (b'a' + ((k + salt) % 26) as u8) as char)
.collect(),
),
_ => AV::Ints(
(0..1 + (i + salt) % 9)
.map(|k| (k * 31 + salt) as i64)
.collect(),
),
}
}
}
/// An object's dense attribute storage as libhdf5 would compare it: the
/// heap's statistics and root shape, its free sections (heap offsets and
/// sizes), and the shapes of the name and creation-order index B-trees.
fn dense_info(path: &Path, obj: &str) -> String {
use clawhdf5_format::message_type::MessageType;
use clawhdf5_format::object_header::ObjectHeader;
let f = File::open(path).unwrap();
let sb = f.superblock();
let (os, ls) = (sb.offset_size, sb.length_size);
assert_eq!((os, ls), (8, 8));
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, obj).unwrap();
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
let Some(m) = oh
.messages
.iter()
.find(|m| m.msg_type == MessageType::AttributeInfo)
else {
return "no attribute info".into();
};
let d = &m.data;
let mut p = 2 + if d[1] & 1 != 0 { 2 } else { 0 };
let heap = le(&d[p..p + 8]);
p += 8;
let name = le(&d[p..p + 8]);
let order = (d[1] & 2 != 0).then(|| le(&d[p + 8..p + 16]));
if heap == u64::MAX {
return "compact".into();
}
let b = std::fs::read(path).unwrap();
let h = &b[heap as usize..];
assert_eq!(&h[0..4], b"FRHP");
// next huge id, huge bt2, free, fs, man size, alloc, iter, nobjs, huge
// size, huge objs (8-byte fields from offset 14).
let stat = |k: usize| le(&h[14 + 8 * k..22 + 8 * k]);
let fs = stat(3);
// Root rows: after 12 8-byte fields, width, start, max direct, max
// index, start rows, root address.
let rows_at = 14 + 96 + 2 + 16 + 2 + 2 + 8;
let mut out = format!(
"heap next_huge={} free={} man={} alloc={} iter={} nobjs={} huge={}/{} rows={}",
stat(0),
stat(2),
stat(4),
stat(5),
stat(6),
stat(7),
stat(8),
stat(9),
le(&h[rows_at..rows_at + 2])
);
if fs != u64::MAX {
let s = &b[fs as usize..];
assert_eq!(&s[0..4], b"FSHD");
let tot = le(&s[6..14]);
let n = le(&s[14..22]);
// counts (4 x 8), 4 x u16, max section size, section info address.
let at = 6 + 32 + 8 + 8;
let sect_addr = le(&s[at..at + 8]);
let sect_size = le(&s[at + 8..at + 16]);
let ss = &b[sect_addr as usize..(sect_addr + sect_size) as usize];
// Sections after the prefix (signature, version, header address);
// trailing zero padding and the checksum are left out.
let body = &ss[13..ss.len() - 4];
let used = body.len() - body.iter().rev().take_while(|&&x| x == 0).count();
out += &format!(" fs tot={tot} n={n} sections={}", hex(&body[..used]));
} else {
out += " no-fs";
}
out += &format!(" names={:?}", bt2_shape_at(&b, name as usize));
if let Some(o) = order {
out += &format!(" order={:?}", bt2_shape_at(&b, o as usize));
}
out
}
fn hex(b: &[u8]) -> String {
b.iter().map(|x| format!("{x:02x}")).collect()
}
/// h5py and our reader both see `want` on each object (other attributes
/// may exist; these must have these values), and h5py's attribute count
/// agrees with libhdf5's object info.
fn check_attr_values(path: &Path, want: &[(String, String, AV)]) {
let f = File::open(path).unwrap();
for (o, n, v) in want {
let attrs = if o == "d" {
f.dataset(o).unwrap().attrs().unwrap()
} else {
f.group(o).unwrap().attrs().unwrap()
};
let got = attrs
.get(n.as_str())
.unwrap_or_else(|| panic!("{o}/{n} missing ({} attributes)", attrs.len()));
match (got, v) {
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
other => panic!("{o}/{n}: {other:?}"),
}
}
let exp: Vec<String> = want
.iter()
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
.collect();
let script = format!(
"import h5py, numpy as np\n\
f = h5py.File({p:?}, 'r')\n\
want = [{w}]\n\
for o, n, v in want:\n\
\x20 a = f[o].attrs[n]\n\
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
\x20 a = [a] if isinstance(a, int) else a\n\
\x20 assert a == v, (o, n, a if len(str(a)) < 200 else len(a), v if len(str(v)) < 200 else len(v))\n\
for o in set(x[0] for x in want):\n\
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n\
\x20 assert len(list(f[o].attrs)) == len(f[o].attrs)\n",
p = path.to_str().unwrap(),
w = exp.join(", ")
);
let sp = path.with_extension("check.py");
std::fs::write(&sp, script).unwrap();
let o = Command::new(python()).arg(&sp).output().unwrap();
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
}
/// Run python statements (inside `with h5py.File(path, 'r+') as f:`).
fn py_r_plus(path: &Path, lines: &[String]) {
let script = format!(
"import h5py, numpy as np\n\
with h5py.File({p:?}, 'r+') as f:\n{l}",
p = path.to_str().unwrap(),
l = lines.concat()
);
let sp = path.with_extension("ops.py");
std::fs::write(&sp, script).unwrap();
let o = Command::new(python()).arg(&sp).output().unwrap();
assert!(o.status.success(), "h5py workload failed:\n{}", text(&o));
}
type AttrOp = (String, String, AV);
fn set_want(want: &mut Vec<AttrOp>, o: &str, n: &str, v: AV) {
want.retain(|(wo, wn, _)| !(wo == o && wn == n));
want.push((o.into(), n.into(), v));
}
/// The same attribute workload by libhdf5 and by the editor on objects
/// with compact attributes that move to dense storage (a plain group, a
/// group tracking and indexing creation order, a dataset): 40 new
/// attributes each (some larger than the heap's 4 KiB managed limit), then
/// same-size rewrites — the heaps, their free space and both index B-trees
/// must come out as libhdf5 makes them; then replacements of another size,
/// then h5py adds, deletes and rewrites attributes.
fn dense_workload(libver: &str, h5dump: bool, tag: &str) {
let dir = tmpdir();
let a = dir.path().join(format!("dense_{tag}_h5py.h5"));
let b = dir.path().join(format!("dense_{tag}_edit.h5"));
for p in [&a, &b] {
py(&format!(
"import h5py, numpy as np\n\
with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
\x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \
f.create_dataset('d', data=np.arange(4, dtype='<i4'))]\n\
\x20 for o in objs:\n\
\x20 for i in range(3): o.attrs.create(f'c{{i}}', np.array([i, i], dtype='<i8'))\n",
p = p.to_str().unwrap()
));
}
let objs = ["g", "t", "d"];
let mut want: Vec<AttrOp> = Vec::new();
for o in objs {
for i in 0..3i64 {
want.push((o.into(), format!("c{i}"), AV::Ints(vec![i, i])));
}
}
// Phase 1: new attributes.
let mut ops: Vec<AttrOp> = Vec::new();
for i in 0..40u64 {
for (k, o) in objs.iter().enumerate() {
ops.push((o.to_string(), format!("n{i}"), AV::make(i, k as u64)));
}
}
let lines: Vec<String> = ops
.iter()
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.create({n:?}, {})\n", v.py()))
.collect();
py_r_plus(&a, &lines);
let mut ed = FileEditor::open(&b).unwrap();
for (o, n, v) in &ops {
ed.set_attr(o, n, &v.value())
.unwrap_or_else(|e| panic!("{tag}: set {o}/{n}: {e}"));
set_want(&mut want, o, n, v.clone());
}
drop(ed);
check_tools(&b, h5dump);
check_attr_values(&b, &want);
check_attr_values(&a, &want);
for o in objs {
assert_eq!(
dense_info(&b, o),
dense_info(&a, o),
"{tag}: dense storage of {o} differs from libhdf5's after insertions"
);
}
// Phase 2: same-size rewrites (H5Awrite in place).
let ops2: Vec<AttrOp> = ops
.iter()
.step_by(4)
.map(|(o, n, v)| {
let nv = match v {
AV::Ints(x) => AV::Ints(x.iter().map(|y| y * 3 + 1).collect()),
AV::Str(s) => AV::Str(s.chars().rev().collect()),
};
(o.clone(), n.clone(), nv)
})
.collect();
let lines: Vec<String> = ops2
.iter()
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.modify({n:?}, {})\n", v.py()))
.collect();
py_r_plus(&a, &lines);
let mut ed = FileEditor::open(&b).unwrap();
for (o, n, v) in &ops2 {
ed.set_attr(o, n, &v.value()).unwrap();
set_want(&mut want, o, n, v.clone());
}
drop(ed);
check_tools(&b, h5dump);
check_attr_values(&b, &want);
for o in objs {
assert_eq!(
dense_info(&b, o),
dense_info(&a, o),
"{tag}: dense storage of {o} differs from libhdf5's after rewrites"
);
}
// Phase 3: replacements of another size (values only: h5py replaces
// through a temporary attribute and a rename).
let mut ed = FileEditor::open(&b).unwrap();
for (k, (o, n, v)) in ops.iter().enumerate().filter(|(k, _)| k % 5 == 2) {
let nv = match v {
AV::Ints(x) => AV::Ints((0..x.len() as i64 + 3).collect()),
AV::Str(s) => AV::Str(format!("{s}-{k}")),
};
let before = std::fs::read(&b).unwrap();
match ed.set_attr(o, n, &nv.value()) {
Ok(()) => set_want(&mut want, o, n, nv),
Err(Error::Unsupported(msg)) => {
assert!(msg.contains("last object"), "{tag}: {o}/{n}: {msg}");
assert!(std::fs::read(&b).unwrap() == before, "refused edit wrote");
}
Err(e) => panic!("{tag}: replace {o}/{n}: {e}"),
}
}
drop(ed);
check_tools(&b, h5dump);
check_attr_values(&b, &want);
// libhdf5 goes on: adds, deletes and rewrites.
py(&format!(
"import h5py, numpy as np\n\
with h5py.File({p:?}, 'r+') as f:\n\
\x20 for o in ['g', 't', 'd']:\n\
\x20 for i in range(5): f[o].attrs[f'late{{i}}'] = np.arange(i + 1)\n\
\x20 del f[o].attrs['n3']\n\
\x20 del f[o].attrs['c1']\n\
\x20 f[o].attrs['n7'] = 'rewritten by h5py'\n",
p = b.to_str().unwrap()
));
want.retain(|(_, n, _)| n != "n3" && n != "c1" && n != "n7");
check_tools(&b, h5dump);
check_attr_values(&b, &want);
}
#[test]
fn dense_attributes_match_libhdf5() {
if !tools_ok() {
return;
}
for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
.iter()
.enumerate()
{
dense_workload(lv, *dump, &format!("{i}"));
}
}
/// Dense attributes in files clawhdf5 writes (its own heap and index
/// layout, no free-space manager), with and without tracked creation
/// order: attributes added (moving a dataset's attributes to dense storage
/// too), rewritten and replaced, then h5py goes on.
#[test]
fn dense_attributes_on_clawhdf5_files() {
if !tools_ok() {
return;
}
for track in [false, true] {
let dir = tmpdir();
let path = dir.path().join(format!("ours_dense_{track}.h5"));
let mut b = clawhdf5::FileBuilder::new();
b.track_order(track);
for i in 0..12i64 {
b.set_attr(&format!("r{i}"), clawhdf5::AttrValue::I64Array(vec![i; 3]));
}
b.create_dataset("d")
.with_i32_data(&[1, 2, 3])
.set_attr("c0", clawhdf5::AttrValue::I64Array(vec![5, 5]));
b.write(&path).unwrap();
let mut want: Vec<AttrOp> = (0..12i64)
.map(|i| ("/".to_string(), format!("r{i}"), AV::Ints(vec![i; 3])))
.collect();
want.push(("d".into(), "c0".into(), AV::Ints(vec![5, 5])));
let mut ed = FileEditor::open(&path).unwrap();
for i in 0..30u64 {
for (k, o) in ["/", "d"].iter().enumerate() {
// Small attributes: a larger one needs a heap block bigger than
// the next (see dense_attribute_refusals_change_nothing).
let v = match AV::make(i, k as u64 + 7) {
AV::Str(s) if s.len() > 400 => AV::Str(s[..400].to_string()),
v => v,
};
ed.set_attr(o, &format!("n{i}"), &v.value())
.unwrap_or_else(|e| panic!("{track} {o}/n{i}: {e}"));
set_want(&mut want, o, &format!("n{i}"), v);
}
}
// Rewrites in place, then replacements of another size.
for i in (0..12i64).step_by(3) {
let v = AV::Ints(vec![-i; 3]);
ed.set_attr("/", &format!("r{i}"), &v.value()).unwrap();
set_want(&mut want, "/", &format!("r{i}"), v);
}
for i in (1..30u64).step_by(7) {
let v = AV::Str(format!("replaced {i}"));
match ed.set_attr("d", &format!("n{i}"), &v.value()) {
Ok(()) => set_want(&mut want, "d", &format!("n{i}"), v),
Err(Error::Unsupported(msg)) => assert!(msg.contains("last object"), "{msg}"),
Err(e) => panic!("{e}"),
}
}
drop(ed);
check_tools(&path, true);
let want_py: Vec<AttrOp> = want
.iter()
.map(|(o, n, v)| {
let o = if o == "/" { "/".to_string() } else { o.clone() };
(o, n.clone(), v.clone())
})
.collect();
check_root_and_attrs(&path, &want_py);
py(&format!(
"import h5py, numpy as np\n\
with h5py.File({p:?}, 'r+') as f:\n\
\x20 for o in ['/', 'd']:\n\
\x20 f[o].attrs['from_h5py'] = np.arange(3)\n\
\x20 del f[o].attrs['n2']\n",
p = path.to_str().unwrap()
));
want.retain(|(_, n, _)| n != "n2");
check_tools(&path, true);
check_root_and_attrs(&path, &want);
}
}
/// `check_attr_values`, with the root group as "/".
fn check_root_and_attrs(path: &Path, want: &[AttrOp]) {
let f = File::open(path).unwrap();
for (o, n, v) in want {
let attrs = match o.as_str() {
"/" => f.root().attrs().unwrap(),
"d" => f.dataset(o).unwrap().attrs().unwrap(),
_ => f.group(o).unwrap().attrs().unwrap(),
};
let got = attrs
.get(n.as_str())
.unwrap_or_else(|| panic!("{o}/{n} missing"));
match (got, v) {
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
other => panic!("{o}/{n}: {other:?}"),
}
}
let exp: Vec<String> = want
.iter()
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
.collect();
let script = format!(
"import h5py, numpy as np\n\
f = h5py.File({p:?}, 'r')\n\
want = [{w}]\n\
for o, n, v in want:\n\
\x20 a = f[o].attrs[n]\n\
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
\x20 a = [a] if isinstance(a, int) else a\n\
\x20 assert a == v, (o, n)\n\
for o in set(x[0] for x in want):\n\
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n",
p = path.to_str().unwrap(),
w = exp.join(", ")
);
let sp = path.with_extension("check.py");
std::fs::write(&sp, script).unwrap();
let o = Command::new(python()).arg(&sp).output().unwrap();
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
}
/// What the editor refuses in dense storage — an object larger than the
/// next heap block (libhdf5 would skip blocks and record their space as
/// free, which this editor does not do) — is `Error::Unsupported`, and the
/// file is left byte for byte as it was.
#[test]
fn dense_attribute_refusals_change_nothing() {
if !tools_ok() {
return;
}
let dir = tmpdir();
let path = dir.path().join("dense_refuse.h5");
py(&format!(
"import h5py, numpy as np\n\
with h5py.File({p:?}, 'w', libver='v110') as f:\n\
\x20 g = f.create_group('g')\n\
\x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n",
p = path.to_str().unwrap()
));
let before = std::fs::read(&path).unwrap();
let mut ed = FileEditor::open(&path).unwrap();
unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("x".repeat(2000))));
drop(ed);
assert!(
std::fs::read(&path).unwrap() == before,
"a refused edit wrote"
);
// What libhdf5 does with it instead works on the untouched file.
py(&format!(
"import h5py\n\
with h5py.File({p:?}, 'r+') as f:\n\
\x20 f['g'].attrs['big'] = 'x' * 2000\n\
\x20 assert len(f['g'].attrs) == 13\n",
p = path.to_str().unwrap()
));
check_tools(&path, true);
}
crates/clawhdf5-tools/tests/edit_interop.rs
+8 -11
View File
@@ -836,19 +836,16 @@ fn attributes_in_place() {
.unwrap(); .unwrap();
want.push(("d", "units".into(), AttrValue::String("km".into()))); want.push(("d", "units".into(), AttrValue::String("km".into())));
if *lv != "'earliest'" { if *lv != "'earliest'" {
// Up to the compact limit (8) and no further; attributes in // Up to the compact limit (8), then into dense storage; objects
// dense storage and tracked creation order are refused, and a // already in dense storage and ones tracking creation order.
// refused edit writes nothing.
ed.set_attr("g", "eighth", &AttrValue::I64(8)).unwrap(); ed.set_attr("g", "eighth", &AttrValue::I64(8)).unwrap();
want.push(("g", "eighth".into(), AttrValue::I64(8))); want.push(("g", "eighth".into(), AttrValue::I64(8)));
let before = std::fs::read(&path).unwrap(); ed.set_attr("g", "ninth", &AttrValue::I64(9)).unwrap();
unsupported(ed.set_attr("g", "ninth", &AttrValue::I64(9))); want.push(("g", "ninth".into(), AttrValue::I64(9)));
unsupported(ed.set_attr("dense", "k0", &AttrValue::I64(1))); ed.set_attr("dense", "k0", &AttrValue::I64(1)).unwrap();
unsupported(ed.set_attr("tracked", "b", &AttrValue::I64(1))); want.push(("dense", "k0".into(), AttrValue::I64(1)));
assert!( ed.set_attr("tracked", "b", &AttrValue::I64(1)).unwrap();
std::fs::read(&path).unwrap() == before, want.push(("tracked", "b".into(), AttrValue::I64(1)));
"a refused edit changed the file"
);
} }
drop(ed); drop(ed);
check_tools(&path, *dump); check_tools(&path, *dump);
crates/clawhdf5/src/edit/attrs.rs
+513
View File
@@ -0,0 +1,513 @@
//! Setting attributes, as `H5O__attr_create` / `H5A__dense_insert` do:
//! compact attributes are object header messages (with their creation
//! index in the message header when the object tracks creation order);
//! when an object reaches its compact limit (or an attribute is too large
//! for a header message) its attributes move to dense storage — a fractal
//! heap for the encoded messages, a version-2 B-tree indexing them by name
//! hash (record type 8) and, when creation order is indexed, a second one
//! by creation index (type 9) — and the Attribute Info message points at
//! them.
use std::cmp::Ordering;
use clawhdf5_format::attribute::AttributeMessage;
use clawhdf5_format::dataspace::DataspaceType;
use crate::edit::btree2::Bt2;
use crate::edit::fheap::Heap;
use crate::edit::image::{Image, get_uint, put_uint, undef};
use crate::edit::ohdr::{Header, MSG_ATTRIBUTE};
use crate::edit::{MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, MSG_FLAG_SHARED, check_plain};
use crate::error::Error;
use crate::reader::File;
use crate::types::AttrValue;
/// `H5O_MESG_MAX_SIZE`: a larger attribute goes to dense storage.
const MESG_MAX_SIZE: usize = 65536;
/// `H5O_MAX_CRT_ORDER_IDX`: the creation index of an attribute of an object
/// that does not track creation order.
const NO_CRT_IDX: u16 = u16::MAX;
/// Name and creation-order index B-trees (`H5A_NAME_BT2_*`,
/// `H5A_CORDER_BT2_*`).
const NAME_BT2_TYPE: u8 = 8;
const CORDER_BT2_TYPE: u8 = 9;
const ATTR_BT2_NODE: u32 = 512;
/// Heap IDs in attribute records.
const ID_LEN: usize = 8;
/// An object's Attribute Info message.
#[derive(Debug, Clone)]
struct AInfo {
/// Its message index in the header.
idx: usize,
track: bool,
index: bool,
max_crt: u16,
fheap: u64,
name_bt2: u64,
corder_bt2: u64,
}
impl AInfo {
fn load(img: &Image<'_>, hdr: &Header) -> Result<Option<Self>, Error> {
let Some(idx) = hdr.find(MSG_ATTR_INFO) else {
return Ok(None);
};
if hdr.msgs[idx].flags & MSG_FLAG_SHARED != 0 {
return Err(Error::Unsupported("shared attribute info message".into()));
}
let d = hdr.data(img, idx)?;
let os = img.os as usize;
let short = || Error::Unsupported("short attribute info message".into());
if d.first() != Some(&0) {
return Err(Error::Unsupported("attribute info message version".into()));
}
let flags = *d.get(1).ok_or_else(short)?;
let track = flags & 0x01 != 0;
let index = flags & 0x02 != 0;
let mut p = 2;
let mut max_crt = 0;
if track {
let b = d.get(p..p + 2).ok_or_else(short)?;
max_crt = u16::from_le_bytes([b[0], b[1]]);
p += 2;
}
let n = if index { 3 } else { 2 };
if d.len() < p + n * os {
return Err(short());
}
Ok(Some(Self {
idx,
track,
index,
max_crt,
fheap: get_uint(&d[p..], img.os),
name_bt2: get_uint(&d[p + os..], img.os),
corder_bt2: if index {
get_uint(&d[p + 2 * os..], img.os)
} else {
undef(img.os)
},
}))
}
fn dense(&self, os: u8) -> bool {
self.fheap != undef(os)
}
/// Store the changeable fields back into the message.
fn store(&self, img: &mut Image<'_>, hdr: &mut Header) -> Result<(), Error> {
let os = img.os as usize;
let mut p = 2;
if self.track {
hdr.patch(img, self.idx, p, &self.max_crt.to_le_bytes())?;
p += 2;
}
let mut a = vec![0u8; os];
for (k, v) in [self.fheap, self.name_bt2, self.corder_bt2]
.into_iter()
.enumerate()
.take(if self.index { 3 } else { 2 })
{
put_uint(&mut a, v, img.os);
hdr.patch(img, self.idx, p + k * os, &a)?;
}
Ok(())
}
/// The next creation index (`H5O__attr_create`), or libhdf5's "none".
fn next_crt(&mut self) -> Result<u16, Error> {
if !self.track {
return Ok(NO_CRT_IDX);
}
if self.max_crt == NO_CRT_IDX {
return Err(Error::Unsupported(
"object's attribute creation index is exhausted".into(),
));
}
self.max_crt += 1;
Ok(self.max_crt - 1)
}
}
/// The name bytes of an attribute message body (without the NUL).
pub(super) fn attr_name(d: &[u8]) -> Result<&[u8], Error> {
let bad = || Error::Unsupported("malformed attribute message".into());
let (len, at) = match d.first() {
Some(1) | Some(2) if d.len() >= 8 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8),
Some(3) if d.len() >= 9 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9),
_ => return Err(bad()),
};
let name = d.get(at..at + len).ok_or_else(bad)?;
Ok(name.split(|&b| b == 0).next().unwrap_or(name))
}
/// A new Attribute Info message for a version-2 header with flags
/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order
/// tracked / indexed as the header's flags say, the maximum creation index,
/// and no dense storage (undefined fractal heap and B-tree addresses).
fn attr_info_message(hdr_flags: u8, max_crt: u16, os: u8) -> Vec<u8> {
let track = hdr_flags & 0x04 != 0;
let index = hdr_flags & 0x08 != 0;
let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)];
if track {
b.extend_from_slice(&max_crt.to_le_bytes());
}
let undef_addr = vec![0xffu8; os as usize];
b.extend_from_slice(&undef_addr);
b.extend_from_slice(&undef_addr);
if index {
b.extend_from_slice(&undef_addr);
}
b
}
/// A version-2 header's limit on compact attributes: stored when its flags
/// say so, else libhdf5's default of 8.
fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result<u16, Error> {
if hdr.flags & 0x10 == 0 {
return Ok(8);
}
let mut p = hdr.addr + 6;
if hdr.flags & 0x20 != 0 {
p += 16;
}
let b = img.read(p, 2)?;
Ok(u16::from_le_bytes([b[0], b[1]]))
}
/// A version-1 attribute message (what libhdf5 writes in a version-1 object
/// header): name, datatype and dataspace each padded to 8 bytes, the
/// dataspace as a version-1 dataspace message.
fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec<u8> {
let mut name = a.name.as_bytes().to_vec();
name.push(0);
let dt = a.datatype.serialize();
let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0];
if a.dataspace.space_type == DataspaceType::Simple {
let mut b = vec![0u8; ls as usize];
for &d in &a.dataspace.dimensions {
put_uint(&mut b, d, ls);
ds.extend_from_slice(&b);
}
if let Some(max) = &a.dataspace.max_dimensions {
ds[2] = 0x01;
for &d in max {
put_uint(&mut b, d, ls);
ds.extend_from_slice(&b);
}
}
} else {
ds[1] = 0;
}
let mut out = vec![1u8, 0];
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
out.extend_from_slice(&(dt.len() as u16).to_le_bytes());
out.extend_from_slice(&(ds.len() as u16).to_le_bytes());
for part in [&name, &dt, &ds] {
out.extend_from_slice(part);
out.resize(out.len().next_multiple_of(8), 0);
}
out.extend_from_slice(&a.raw_data);
out
}
/// Dense storage opened for changes.
struct Dense {
heap: Heap,
names: Bt2,
order: Option<Bt2>,
}
/// `H5_checksum_lookup3` of a name, as the name index keys it.
fn name_hash(name: &[u8]) -> u32 {
clawhdf5_format::checksum::jenkins_lookup3(name)
}
/// Compare attribute `name` (hash `hash`) with a name-index record
/// (`H5A__dense_btree2_name_compare`: the hash, then the stored name).
fn cmp_name(
heap: &Heap,
img: &Image<'_>,
hash: u32,
name: &[u8],
rec: &[u8],
) -> Result<Ordering, Error> {
let theirs = u32::from_le_bytes([rec[13], rec[14], rec[15], rec[16]]);
match hash.cmp(&theirs) {
Ordering::Equal => {
if rec[ID_LEN] & MSG_FLAG_SHARED != 0 {
return Err(Error::Unsupported(
"shared attribute in dense storage".into(),
));
}
let obj = heap.read(img, &rec[..ID_LEN])?;
Ok(name.cmp(attr_name(&obj)?))
}
o => Ok(o),
}
}
fn corder_of(rec: &[u8]) -> u32 {
u32::from_le_bytes([rec[9], rec[10], rec[11], rec[12]])
}
impl Dense {
fn open(img: &Image<'_>, ai: &AInfo) -> Result<Self, Error> {
let heap = Heap::open(img, ai.fheap)?;
let names = Bt2::open(img, ai.name_bt2)?;
if names.tree_type() != NAME_BT2_TYPE || names.record_size() != ID_LEN + 9 {
return Err(Error::Unsupported("attribute name index layout".into()));
}
let order = if ai.index {
let t = Bt2::open(img, ai.corder_bt2)?;
if t.tree_type() != CORDER_BT2_TYPE || t.record_size() != ID_LEN + 5 {
return Err(Error::Unsupported(
"attribute creation-order index layout".into(),
));
}
Some(t)
} else {
None
};
Ok(Self { heap, names, order })
}
/// `H5A__dense_create`: heap, name index, [creation-order index].
fn create(img: &mut Image<'_>, index: bool) -> Result<Self, Error> {
let heap = Heap::create_attribute_heap(img)?;
let names = Bt2::create(img, NAME_BT2_TYPE, ATTR_BT2_NODE, ID_LEN + 9, 100, 40)?;
let order = if index {
Some(Bt2::create(
img,
CORDER_BT2_TYPE,
ATTR_BT2_NODE,
ID_LEN + 5,
100,
40,
)?)
} else {
None
};
Ok(Self { heap, names, order })
}
/// `H5A__dense_insert` of an encoded attribute message.
fn insert(&mut self, img: &mut Image<'_>, body: &[u8], crt: u16) -> Result<(), Error> {
let name = attr_name(body)?.to_vec();
let id = self.heap.insert(img, body)?;
if id.len() != ID_LEN {
return Err(Error::Unsupported("attribute heap ID length".into()));
}
let hash = name_hash(&name);
let mut rec = id.clone();
rec.push(0);
rec.extend_from_slice(&u32::from(crt).to_le_bytes());
rec.extend_from_slice(&hash.to_le_bytes());
let heap = &self.heap;
self.names
.insert(img, &mut |im, r| cmp_name(heap, im, hash, &name, r), &rec)?;
if let Some(t) = &mut self.order {
let key = u32::from(crt);
t.insert(
img,
&mut |_, r| Ok(key.cmp(&corder_of(r))),
&rec[..ID_LEN + 5],
)?;
}
Ok(())
}
fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> {
self.heap.finish(img)?;
self.names.finish(img)?;
if let Some(t) = &mut self.order {
t.finish(img)?;
}
Ok(())
}
}
/// Set attribute `name` of the object at `path` to `value`.
pub(super) fn set_attr(
f: &File,
img: &mut Image<'_>,
path: &str,
name: &str,
value: &AttrValue,
) -> Result<(), Error> {
let addr = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?;
let mut hdr = Header::load(img, addr)?;
let mut msg = clawhdf5_format::type_builders::build_attr_message(name, value);
check_plain(&msg.datatype)?;
// libhdf5 encodes a simple dataspace with its maximum dimensions (the
// current ones when none were given), so an attribute takes the same
// space in a header or heap as when libhdf5 writes it.
if msg.dataspace.space_type == DataspaceType::Simple && msg.dataspace.max_dimensions.is_none() {
msg.dataspace.max_dimensions = Some(msg.dataspace.dimensions.clone());
}
// H5A__set_version: version 1 unless the name is not ASCII (then 3),
// raised to the file's low bound — which is the earliest for a file
// libhdf5 opens without a libver setting (h5py's `r+`).
let body = if hdr.version == 1 || name.is_ascii() {
encode_attr_v1(&msg, img.ls)
} else {
let mut b = msg.serialize_v3(img.ls);
if !name.is_ascii() {
b[8] = 1; // UTF-8 name
}
b
};
let mut ainfo = if hdr.version == 2 {
AInfo::load(img, &hdr)?
} else {
None
};
if let Some(ai) = ainfo.as_mut().filter(|a| a.dense(img.os)) {
let mut ai = ai.clone();
set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?;
return hdr.finish(img);
}
let mut existing = None;
let mut count = 0usize;
for i in 0..hdr.msgs.len() {
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
continue;
}
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
return Err(Error::Unsupported("shared attribute message".into()));
}
count += 1;
if attr_name(&hdr.data(img, i)?)? == name.as_bytes() {
existing = Some(i);
}
}
if let Some(i) = existing {
hdr.delete(img, i)?;
count -= 1;
}
if hdr.version == 1 {
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?;
return hdr.finish(img);
}
let tracked = hdr.flags & 0x04 != 0;
// H5O__attr_create: a missing Attribute Info message starts from
// nothing (and is added below, holding the new maximum creation index).
let new_ainfo = ainfo.is_none();
let mut ai = ainfo.take().unwrap_or(AInfo {
idx: usize::MAX,
track: tracked,
index: hdr.flags & 0x08 != 0,
max_crt: 0,
fheap: undef(img.os),
name_bt2: undef(img.os),
corder_bt2: undef(img.os),
});
let max_compact = usize::from(max_compact_attrs(img, &hdr)?);
if count == max_compact || body.len() >= MESG_MAX_SIZE {
if new_ainfo {
return Err(Error::Unsupported(
"dense attribute storage for an object without an Attribute Info message".into(),
));
}
to_dense(img, &mut hdr, &mut ai)?;
set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?;
return hdr.finish(img);
}
let crt = ai.next_crt()?;
let corder = tracked.then_some(crt);
if new_ainfo {
// libhdf5 appends the Attribute Info message before the attribute
// when free space holds both, else after it, so that a new
// continuation chunk made for the attribute has room for it too.
let a = attr_info_message(hdr.flags, ai.max_crt, img.os);
let first = hdr.has_free(a.len() + hdr.hsize() + body.len());
if first {
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?;
}
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?;
if !first {
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?;
}
} else {
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?;
ai.store(img, &mut hdr)?;
}
hdr.finish(img)
}
/// Move every compact attribute of the object into new dense storage, in
/// header message order (`H5O__attr_to_dense_cb`), leaving free space where
/// the messages were.
fn to_dense(img: &mut Image<'_>, hdr: &mut Header, ai: &mut AInfo) -> Result<(), Error> {
let mut dense = Dense::create(img, ai.index)?;
for i in 0..hdr.msgs.len() {
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
continue;
}
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
return Err(Error::Unsupported("shared attribute message".into()));
}
let body = hdr.data(img, i)?;
let crt = if ai.track {
hdr.msgs[i].corder.unwrap_or(0)
} else {
NO_CRT_IDX
};
dense.insert(img, &body, crt)?;
hdr.delete(img, i)?;
}
dense.finish(img)?;
ai.fheap = dense.heap.address();
ai.name_bt2 = dense.names.address();
if let Some(t) = &dense.order {
ai.corder_bt2 = t.address();
}
ai.store(img, hdr)
}
/// Set an attribute of an object whose attributes are in dense storage: an
/// attribute of that name whose new encoding has the old one's size is
/// rewritten in its heap object (`H5A__dense_write`); otherwise the old one
/// is removed (`H5A__dense_remove`: name index, creation-order index, heap
/// object) and the new one inserted with the next creation index.
fn set_dense(
img: &mut Image<'_>,
hdr: &mut Header,
ai: &mut AInfo,
name: &[u8],
body: &[u8],
) -> Result<(), Error> {
let mut dense = Dense::open(img, ai)?;
let hash = name_hash(name);
let found = {
let heap = &dense.heap;
dense
.names
.find(img, &mut |im, r| cmp_name(heap, im, hash, name, r))?
};
if let Some(rec) = found {
if dense.heap.write_in_place(img, &rec[..ID_LEN], body)? {
return dense.finish(img);
}
{
let heap = &dense.heap;
dense
.names
.remove(img, &mut |im, r| cmp_name(heap, im, hash, name, r))?;
}
if let Some(t) = &mut dense.order {
let key = corder_of(&rec);
t.remove(img, &mut |_, r| Ok(key.cmp(&corder_of(r))))?
.ok_or_else(|| {
Error::Unsupported("attribute missing from its creation-order index".into())
})?;
}
dense.heap.remove(img, &rec[..ID_LEN])?;
}
let crt = ai.next_crt()?;
dense.insert(img, body, crt)?;
dense.finish(img)?;
ai.store(img, hdr)
}
crates/clawhdf5/src/edit/btree2.rs
+1 -34
View File
@@ -180,11 +180,6 @@ impl Bt2 {
self.tree_type self.tree_type
} }
/// Records in the whole tree.
pub(crate) fn len(&self) -> u64 {
self.root.all
}
/// Node geometry for depths `0..=self.depth` (`H5B2__hdr_init`, /// Node geometry for depths `0..=self.depth` (`H5B2__hdr_init`,
/// `H5B2__split_root`). /// `H5B2__split_root`).
fn compute_levels(&mut self, os: u8) -> Result<(), Error> { fn compute_levels(&mut self, os: u8) -> Result<(), Error> {
@@ -389,35 +384,6 @@ impl Bt2 {
} }
} }
/// Every record, in key order.
pub(crate) fn records(&mut self, img: &Image<'_>) -> Result<Vec<Vec<u8>>, Error> {
let mut out = Vec::new();
if self.root.addr != undef(img.os) && self.root.nrec > 0 {
self.walk(img, self.root, self.depth, &mut out)?;
}
Ok(out)
}
fn walk(
&mut self,
img: &Image<'_>,
p: Ptr,
depth: u16,
out: &mut Vec<Vec<u8>>,
) -> Result<(), Error> {
self.load(img, p, depth)?;
let n = self.peek(p.addr).clone();
for i in 0..=n.recs.len() {
if depth > 0 {
self.walk(img, n.ptrs[i], depth - 1, out)?;
}
if i < n.recs.len() {
out.push(n.recs[i].clone());
}
}
Ok(())
}
/// Insert `rec`, or replace the record `cmp` matches (`H5B2_update`). /// Insert `rec`, or replace the record `cmp` matches (`H5B2_update`).
pub(crate) fn update( pub(crate) fn update(
&mut self, &mut self,
@@ -665,6 +631,7 @@ impl Bt2 {
/// parent records between them) out over the children `lo..` of /// parent records between them) out over the children `lo..` of
/// `parent`, with `counts[k]` records in the k-th; the records between /// `parent`, with `counts[k]` records in the k-th; the records between
/// them go back into the parent. /// them go back into the parent.
#[allow(clippy::too_many_arguments)]
fn relayout( fn relayout(
&mut self, &mut self,
parent: u64, parent: u64,
crates/clawhdf5/src/edit/fheap.rs
+1121
View File
File diff suppressed because it is too large Load Diff
crates/clawhdf5/src/edit/mod.rs
+22 -168
View File
@@ -15,10 +15,12 @@
//! file) is dropped before the first write, so no `&[u8]` over the mapping //! file) is dropped before the first write, so no `&[u8]` over the mapping
//! is alive while the file changes (see `image`). //! is alive while the file changes (see `image`).
mod attrs;
mod btree1; mod btree1;
mod btree2; mod btree2;
mod earray; mod earray;
mod farray; mod farray;
mod fheap;
mod image; mod image;
mod ohdr; mod ohdr;
mod select; mod select;
@@ -27,7 +29,6 @@ use std::collections::{BTreeMap, HashMap};
use std::fs::{OpenOptions, TryLockError}; use std::fs::{OpenOptions, TryLockError};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use clawhdf5_format::attribute::AttributeMessage;
use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks}; use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks};
use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::data_layout::DataLayout;
use clawhdf5_format::data_read::NativeElement; use clawhdf5_format::data_read::NativeElement;
@@ -43,8 +44,8 @@ use btree1::{BTree1, Key};
use btree2::Bt2; use btree2::Bt2;
use earray::{Ea, EaParams, Elem}; use earray::{Ea, EaParams, Elem};
use farray::Fa; use farray::Fa;
use image::{Image, get_uint, put_uint, undef}; use image::{Image, put_uint, undef};
use ohdr::{Header, MSG_ATTRIBUTE}; use ohdr::Header;
const MSG_DATASPACE: u16 = 0x01; const MSG_DATASPACE: u16 = 0x01;
const MSG_LAYOUT: u16 = 0x08; const MSG_LAYOUT: u16 = 0x08;
@@ -918,101 +919,29 @@ impl FileEditor {
/// Set attribute `name` of the object at `path` (a group or dataset; /// Set attribute `name` of the object at `path` (a group or dataset;
/// `"/"` is the root group) to `value`, replacing an attribute of that /// `"/"` is the root group) to `value`, replacing an attribute of that
/// name. The attribute goes into free space in the object header, or a /// name, as libhdf5 creates attributes (`H5O__attr_create`).
/// new header continuation chunk at the end of the file.
/// ///
/// [`Error::Unsupported`] for an object whose attributes are in dense /// A compact attribute goes into free space in the object header, or a
/// storage (or would have to move there: more than the object's /// new header continuation chunk; an object that tracks creation order
/// compact-attribute limit), one that tracks attribute creation order, /// gives it the next creation index. When the object reaches its
/// or one with shared attribute messages. /// compact-attribute limit (or the attribute is too large for a header
/// message) its attributes move to dense storage, and objects already
/// using dense storage get the attribute there: a fractal heap object
/// (in free heap space, a new heap block, or its own file space when
/// larger than the heap's managed limit) indexed by name and, when the
/// object indexes creation order, by creation index. An attribute
/// replaced by one of the same encoded size is rewritten where it is.
///
/// [`Error::Unsupported`] for shared attribute messages, heaps this
/// editor cannot extend the way libhdf5 would (child indirect blocks,
/// skipped blocks, free space other than within direct blocks, freeing
/// a whole block), and version-1 object headers asked for an attribute
/// larger than a header message holds.
pub fn set_attr(&mut self, path: &str, name: &str, value: &AttrValue) -> Result<(), Error> { pub fn set_attr(&mut self, path: &str, name: &str, value: &AttrValue) -> Result<(), Error> {
if name.is_empty() { if name.is_empty() {
return Err(Error::InvalidArgument("empty attribute name".into())); return Err(Error::InvalidArgument("empty attribute name".into()));
} }
self.edit(|f, img| { self.edit(|f, img| attrs::set_attr(f, img, path, name, value))
let addr =
clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?;
let mut hdr = Header::load(img, addr)?;
if hdr.version == 2 && hdr.flags & 0x04 != 0 {
return Err(Error::Unsupported(
"object tracks attribute creation order".into(),
));
}
if let Some(i) = hdr.find(MSG_ATTR_INFO) {
let d = hdr.data(img, i)?;
// version(1) flags(1) [max creation index(2)] fractal heap
// address, name index address, [order index address].
let mut p = 2;
if d.get(1).is_some_and(|f| f & 0x01 != 0) {
p += 2;
}
let os = img.os as usize;
if d.len() < p + os {
return Err(Error::Unsupported("short attribute info message".into()));
}
if get_uint(&d[p..], img.os) != undef(img.os) {
return Err(Error::Unsupported(
"object with attributes in dense storage".into(),
));
}
}
let mut existing = None;
let mut count = 0usize;
for i in 0..hdr.msgs.len() {
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
continue;
}
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
return Err(Error::Unsupported("shared attribute message".into()));
}
count += 1;
if attr_name(&hdr.data(img, i)?)? == name.as_bytes() {
existing = Some(i);
}
}
if existing.is_none() && hdr.version == 2 {
let max_compact = max_compact_attrs(img, &hdr)?;
if count + 1 > usize::from(max_compact) {
return Err(Error::Unsupported(format!(
"object already has {count} compact attributes (its limit is \
{max_compact}); more need dense storage"
)));
}
}
let msg = clawhdf5_format::type_builders::build_attr_message(name, value);
check_plain(&msg.datatype)?;
let body = if hdr.version == 1 {
encode_attr_v1(&msg, img.ls)
} else {
let mut b = msg.serialize_v3(img.ls);
if !name.is_ascii() {
b[8] = 1; // UTF-8 name
}
b
};
if let Some(i) = existing {
hdr.delete(img, i)?;
}
// libhdf5 counts a version-2 header's attributes through its
// Attribute Info message and reports none without one; like
// H5O__attr_create, add it when missing: before the attribute
// when free space holds both (libhdf5's order), else after it,
// so that a new continuation chunk made for the attribute has
// room for it too.
let ainfo = (hdr.version == 2 && hdr.find(MSG_ATTR_INFO).is_none())
.then(|| attr_info_message(hdr.flags, img.os));
let ainfo_first = ainfo
.as_ref()
.is_some_and(|a| hdr.has_free(a.len() + hdr.hsize() + body.len()));
if let Some(a) = ainfo.as_ref().filter(|_| ainfo_first) {
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?;
}
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?;
if let Some(a) = ainfo.as_ref().filter(|_| !ainfo_first) {
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?;
}
hdr.finish(img)
})
} }
} }
@@ -1097,81 +1026,6 @@ fn set_superblock_eof(
Ok(()) Ok(())
} }
/// The name bytes of an attribute message body (without the NUL).
fn attr_name(d: &[u8]) -> Result<&[u8], Error> {
let bad = || Error::Unsupported("malformed attribute message".into());
let (len, at) = match d.first() {
Some(1) | Some(2) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8),
Some(3) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9),
_ => return Err(bad()),
};
let name = d.get(at..at + len).ok_or_else(bad)?;
Ok(name.split(|&b| b == 0).next().unwrap_or(name))
}
/// A new Attribute Info message for a version-2 header with flags
/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order
/// tracked / indexed as the header's flags say, maximum creation index 0,
/// and no dense storage (undefined fractal heap and B-tree addresses).
fn attr_info_message(hdr_flags: u8, os: u8) -> Vec<u8> {
let track = hdr_flags & 0x04 != 0;
let index = hdr_flags & 0x08 != 0;
let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)];
if track {
b.extend_from_slice(&0u16.to_le_bytes());
}
let undef_addr = vec![0xffu8; os as usize];
b.extend_from_slice(&undef_addr);
b.extend_from_slice(&undef_addr);
if index {
b.extend_from_slice(&undef_addr);
}
b
}
/// A version-2 header's limit on compact attributes: stored when its flags
/// say so, else libhdf5's default of 8.
fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result<u16, Error> {
if hdr.flags & 0x10 == 0 {
return Ok(8);
}
let mut p = hdr.addr + 6;
if hdr.flags & 0x20 != 0 {
p += 16;
}
let b = img.read(p, 2)?;
Ok(u16::from_le_bytes([b[0], b[1]]))
}
/// A version-1 attribute message (what libhdf5 writes in a version-1 object
/// header): name, datatype and dataspace each padded to 8 bytes, the
/// dataspace as a version-1 dataspace message.
fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec<u8> {
let mut name = a.name.as_bytes().to_vec();
name.push(0);
let dt = a.datatype.serialize();
let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0];
if a.dataspace.space_type == DataspaceType::Simple {
for &d in &a.dataspace.dimensions {
let mut b = vec![0u8; ls as usize];
put_uint(&mut b, d, ls);
ds.extend_from_slice(&b);
}
} else {
ds[1] = 0;
}
let mut out = vec![1u8, 0];
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
out.extend_from_slice(&(dt.len() as u16).to_le_bytes());
out.extend_from_slice(&(ds.len() as u16).to_le_bytes());
for part in [&name, &dt, &ds] {
out.extend_from_slice(part);
out.resize(out.len().next_multiple_of(8), 0);
}
out.extend_from_slice(&a.raw_data);
out
}
fn write_selection( fn write_selection(
f: &File, f: &File,
img: &mut Image<'_>, img: &mut Image<'_>,
crates/clawhdf5/src/edit/ohdr.rs
+6 -2
View File
@@ -59,7 +59,7 @@ pub(crate) struct Header {
added: usize, added: usize,
} }
const MAX_CHUNKS: usize = 1024; const MAX_CHUNKS: usize = 1 << 16;
fn corrupt(why: &'static str) -> Error { fn corrupt(why: &'static str) -> Error {
Error::Format(FormatError::InvalidObjectHeader(why)) Error::Format(FormatError::InvalidObjectHeader(why))
@@ -118,7 +118,11 @@ impl Header {
}); });
h.scan(img, 0, addr + 16, addr + 16 + size, &mut pending)?; h.scan(img, 0, addr + 16, addr + 16 + size, &mut pending)?;
} }
while let Some((caddr, clen)) = pending.pop() { // Continuation chunks in the order their messages are found, as
// H5O_protect loads them (so messages keep libhdf5's order).
let mut next = 0;
while let Some(&(caddr, clen)) = pending.get(next) {
next += 1;
if h.chunks.len() >= MAX_CHUNKS { if h.chunks.len() >= MAX_CHUNKS {
return Err(corrupt("too many object header chunks")); return Err(corrupt("too many object header chunks"));
} }