Remote reads (range-read M2/M3: open_storage, HTTP/S3) and near-complete in-place editing #18
@@ -213,7 +213,6 @@ fn tmpdir() -> tempfile::TempDir {
|
||||
tempfile::TempDir::new_in(base).unwrap()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn unsupported<T: std::fmt::Debug>(r: Result<T, Error>) {
|
||||
match r {
|
||||
Err(Error::Unsupported(_)) => {}
|
||||
@@ -830,3 +829,497 @@ fn btree2_random_chunk_order() {
|
||||
verify(&path, "x", &m);
|
||||
check_tools(&path, false);
|
||||
}
|
||||
|
||||
// ---- dense attributes ----
|
||||
|
||||
/// An attribute value both sides can write with the same encoded size: a
|
||||
/// 1-D int64 array or a fixed-length string.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
enum AV {
|
||||
Ints(Vec<i64>),
|
||||
Str(String),
|
||||
}
|
||||
|
||||
impl AV {
|
||||
fn value(&self) -> clawhdf5::AttrValue {
|
||||
match self {
|
||||
AV::Ints(v) => clawhdf5::AttrValue::I64Array(v.clone()),
|
||||
AV::Str(s) => clawhdf5::AttrValue::String(s.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
fn py(&self) -> String {
|
||||
match self {
|
||||
AV::Ints(v) => format!("np.array({v:?}, dtype='<i8')"),
|
||||
AV::Str(s) => format!("np.bytes_({s:?})"),
|
||||
}
|
||||
}
|
||||
|
||||
fn py_expect(&self) -> String {
|
||||
match self {
|
||||
AV::Ints(v) => format!("{v:?}"),
|
||||
AV::Str(s) => format!("{s:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn make(i: u64, salt: u64) -> Self {
|
||||
match i % 11 {
|
||||
5 => AV::Str("h".repeat(5000 + (salt % 50) as usize)),
|
||||
1 | 4 | 8 => AV::Str(
|
||||
(0..10 + (i * 13 + salt) % 300)
|
||||
.map(|k| (b'a' + ((k + salt) % 26) as u8) as char)
|
||||
.collect(),
|
||||
),
|
||||
_ => AV::Ints(
|
||||
(0..1 + (i + salt) % 9)
|
||||
.map(|k| (k * 31 + salt) as i64)
|
||||
.collect(),
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An object's dense attribute storage as libhdf5 would compare it: the
|
||||
/// heap's statistics and root shape, its free sections (heap offsets and
|
||||
/// sizes), and the shapes of the name and creation-order index B-trees.
|
||||
fn dense_info(path: &Path, obj: &str) -> String {
|
||||
use clawhdf5_format::message_type::MessageType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
let f = File::open(path).unwrap();
|
||||
let sb = f.superblock();
|
||||
let (os, ls) = (sb.offset_size, sb.length_size);
|
||||
assert_eq!((os, ls), (8, 8));
|
||||
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, obj).unwrap();
|
||||
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
|
||||
let Some(m) = oh
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == MessageType::AttributeInfo)
|
||||
else {
|
||||
return "no attribute info".into();
|
||||
};
|
||||
let d = &m.data;
|
||||
let mut p = 2 + if d[1] & 1 != 0 { 2 } else { 0 };
|
||||
let heap = le(&d[p..p + 8]);
|
||||
p += 8;
|
||||
let name = le(&d[p..p + 8]);
|
||||
let order = (d[1] & 2 != 0).then(|| le(&d[p + 8..p + 16]));
|
||||
if heap == u64::MAX {
|
||||
return "compact".into();
|
||||
}
|
||||
let b = std::fs::read(path).unwrap();
|
||||
let h = &b[heap as usize..];
|
||||
assert_eq!(&h[0..4], b"FRHP");
|
||||
// next huge id, huge bt2, free, fs, man size, alloc, iter, nobjs, huge
|
||||
// size, huge objs (8-byte fields from offset 14).
|
||||
let stat = |k: usize| le(&h[14 + 8 * k..22 + 8 * k]);
|
||||
let fs = stat(3);
|
||||
// Root rows: after 12 8-byte fields, width, start, max direct, max
|
||||
// index, start rows, root address.
|
||||
let rows_at = 14 + 96 + 2 + 16 + 2 + 2 + 8;
|
||||
let mut out = format!(
|
||||
"heap next_huge={} free={} man={} alloc={} iter={} nobjs={} huge={}/{} rows={}",
|
||||
stat(0),
|
||||
stat(2),
|
||||
stat(4),
|
||||
stat(5),
|
||||
stat(6),
|
||||
stat(7),
|
||||
stat(8),
|
||||
stat(9),
|
||||
le(&h[rows_at..rows_at + 2])
|
||||
);
|
||||
if fs != u64::MAX {
|
||||
let s = &b[fs as usize..];
|
||||
assert_eq!(&s[0..4], b"FSHD");
|
||||
let tot = le(&s[6..14]);
|
||||
let n = le(&s[14..22]);
|
||||
// counts (4 x 8), 4 x u16, max section size, section info address.
|
||||
let at = 6 + 32 + 8 + 8;
|
||||
let sect_addr = le(&s[at..at + 8]);
|
||||
let sect_size = le(&s[at + 8..at + 16]);
|
||||
let ss = &b[sect_addr as usize..(sect_addr + sect_size) as usize];
|
||||
// Sections after the prefix (signature, version, header address);
|
||||
// trailing zero padding and the checksum are left out.
|
||||
let body = &ss[13..ss.len() - 4];
|
||||
let used = body.len() - body.iter().rev().take_while(|&&x| x == 0).count();
|
||||
out += &format!(" fs tot={tot} n={n} sections={}", hex(&body[..used]));
|
||||
} else {
|
||||
out += " no-fs";
|
||||
}
|
||||
out += &format!(" names={:?}", bt2_shape_at(&b, name as usize));
|
||||
if let Some(o) = order {
|
||||
out += &format!(" order={:?}", bt2_shape_at(&b, o as usize));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn hex(b: &[u8]) -> String {
|
||||
b.iter().map(|x| format!("{x:02x}")).collect()
|
||||
}
|
||||
|
||||
/// h5py and our reader both see `want` on each object (other attributes
|
||||
/// may exist; these must have these values), and h5py's attribute count
|
||||
/// agrees with libhdf5's object info.
|
||||
fn check_attr_values(path: &Path, want: &[(String, String, AV)]) {
|
||||
let f = File::open(path).unwrap();
|
||||
for (o, n, v) in want {
|
||||
let attrs = if o == "d" {
|
||||
f.dataset(o).unwrap().attrs().unwrap()
|
||||
} else {
|
||||
f.group(o).unwrap().attrs().unwrap()
|
||||
};
|
||||
let got = attrs
|
||||
.get(n.as_str())
|
||||
.unwrap_or_else(|| panic!("{o}/{n} missing ({} attributes)", attrs.len()));
|
||||
match (got, v) {
|
||||
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
||||
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
||||
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
||||
other => panic!("{o}/{n}: {other:?}"),
|
||||
}
|
||||
}
|
||||
let exp: Vec<String> = want
|
||||
.iter()
|
||||
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
||||
.collect();
|
||||
let script = format!(
|
||||
"import h5py, numpy as np\n\
|
||||
f = h5py.File({p:?}, 'r')\n\
|
||||
want = [{w}]\n\
|
||||
for o, n, v in want:\n\
|
||||
\x20 a = f[o].attrs[n]\n\
|
||||
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
||||
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
||||
\x20 a = [a] if isinstance(a, int) else a\n\
|
||||
\x20 assert a == v, (o, n, a if len(str(a)) < 200 else len(a), v if len(str(v)) < 200 else len(v))\n\
|
||||
for o in set(x[0] for x in want):\n\
|
||||
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n\
|
||||
\x20 assert len(list(f[o].attrs)) == len(f[o].attrs)\n",
|
||||
p = path.to_str().unwrap(),
|
||||
w = exp.join(", ")
|
||||
);
|
||||
let sp = path.with_extension("check.py");
|
||||
std::fs::write(&sp, script).unwrap();
|
||||
let o = Command::new(python()).arg(&sp).output().unwrap();
|
||||
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
||||
}
|
||||
|
||||
/// Run python statements (inside `with h5py.File(path, 'r+') as f:`).
|
||||
fn py_r_plus(path: &Path, lines: &[String]) {
|
||||
let script = format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n{l}",
|
||||
p = path.to_str().unwrap(),
|
||||
l = lines.concat()
|
||||
);
|
||||
let sp = path.with_extension("ops.py");
|
||||
std::fs::write(&sp, script).unwrap();
|
||||
let o = Command::new(python()).arg(&sp).output().unwrap();
|
||||
assert!(o.status.success(), "h5py workload failed:\n{}", text(&o));
|
||||
}
|
||||
|
||||
type AttrOp = (String, String, AV);
|
||||
|
||||
fn set_want(want: &mut Vec<AttrOp>, o: &str, n: &str, v: AV) {
|
||||
want.retain(|(wo, wn, _)| !(wo == o && wn == n));
|
||||
want.push((o.into(), n.into(), v));
|
||||
}
|
||||
|
||||
/// The same attribute workload by libhdf5 and by the editor on objects
|
||||
/// with compact attributes that move to dense storage (a plain group, a
|
||||
/// group tracking and indexing creation order, a dataset): 40 new
|
||||
/// attributes each (some larger than the heap's 4 KiB managed limit), then
|
||||
/// same-size rewrites — the heaps, their free space and both index B-trees
|
||||
/// must come out as libhdf5 makes them; then replacements of another size,
|
||||
/// then h5py adds, deletes and rewrites attributes.
|
||||
fn dense_workload(libver: &str, h5dump: bool, tag: &str) {
|
||||
let dir = tmpdir();
|
||||
let a = dir.path().join(format!("dense_{tag}_h5py.h5"));
|
||||
let b = dir.path().join(format!("dense_{tag}_edit.h5"));
|
||||
for p in [&a, &b] {
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
|
||||
\x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \
|
||||
f.create_dataset('d', data=np.arange(4, dtype='<i4'))]\n\
|
||||
\x20 for o in objs:\n\
|
||||
\x20 for i in range(3): o.attrs.create(f'c{{i}}', np.array([i, i], dtype='<i8'))\n",
|
||||
p = p.to_str().unwrap()
|
||||
));
|
||||
}
|
||||
let objs = ["g", "t", "d"];
|
||||
let mut want: Vec<AttrOp> = Vec::new();
|
||||
for o in objs {
|
||||
for i in 0..3i64 {
|
||||
want.push((o.into(), format!("c{i}"), AV::Ints(vec![i, i])));
|
||||
}
|
||||
}
|
||||
// Phase 1: new attributes.
|
||||
let mut ops: Vec<AttrOp> = Vec::new();
|
||||
for i in 0..40u64 {
|
||||
for (k, o) in objs.iter().enumerate() {
|
||||
ops.push((o.to_string(), format!("n{i}"), AV::make(i, k as u64)));
|
||||
}
|
||||
}
|
||||
let lines: Vec<String> = ops
|
||||
.iter()
|
||||
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.create({n:?}, {})\n", v.py()))
|
||||
.collect();
|
||||
py_r_plus(&a, &lines);
|
||||
let mut ed = FileEditor::open(&b).unwrap();
|
||||
for (o, n, v) in &ops {
|
||||
ed.set_attr(o, n, &v.value())
|
||||
.unwrap_or_else(|e| panic!("{tag}: set {o}/{n}: {e}"));
|
||||
set_want(&mut want, o, n, v.clone());
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&b, h5dump);
|
||||
check_attr_values(&b, &want);
|
||||
check_attr_values(&a, &want);
|
||||
for o in objs {
|
||||
assert_eq!(
|
||||
dense_info(&b, o),
|
||||
dense_info(&a, o),
|
||||
"{tag}: dense storage of {o} differs from libhdf5's after insertions"
|
||||
);
|
||||
}
|
||||
// Phase 2: same-size rewrites (H5Awrite in place).
|
||||
let ops2: Vec<AttrOp> = ops
|
||||
.iter()
|
||||
.step_by(4)
|
||||
.map(|(o, n, v)| {
|
||||
let nv = match v {
|
||||
AV::Ints(x) => AV::Ints(x.iter().map(|y| y * 3 + 1).collect()),
|
||||
AV::Str(s) => AV::Str(s.chars().rev().collect()),
|
||||
};
|
||||
(o.clone(), n.clone(), nv)
|
||||
})
|
||||
.collect();
|
||||
let lines: Vec<String> = ops2
|
||||
.iter()
|
||||
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.modify({n:?}, {})\n", v.py()))
|
||||
.collect();
|
||||
py_r_plus(&a, &lines);
|
||||
let mut ed = FileEditor::open(&b).unwrap();
|
||||
for (o, n, v) in &ops2 {
|
||||
ed.set_attr(o, n, &v.value()).unwrap();
|
||||
set_want(&mut want, o, n, v.clone());
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&b, h5dump);
|
||||
check_attr_values(&b, &want);
|
||||
for o in objs {
|
||||
assert_eq!(
|
||||
dense_info(&b, o),
|
||||
dense_info(&a, o),
|
||||
"{tag}: dense storage of {o} differs from libhdf5's after rewrites"
|
||||
);
|
||||
}
|
||||
// Phase 3: replacements of another size (values only: h5py replaces
|
||||
// through a temporary attribute and a rename).
|
||||
let mut ed = FileEditor::open(&b).unwrap();
|
||||
for (k, (o, n, v)) in ops.iter().enumerate().filter(|(k, _)| k % 5 == 2) {
|
||||
let nv = match v {
|
||||
AV::Ints(x) => AV::Ints((0..x.len() as i64 + 3).collect()),
|
||||
AV::Str(s) => AV::Str(format!("{s}-{k}")),
|
||||
};
|
||||
let before = std::fs::read(&b).unwrap();
|
||||
match ed.set_attr(o, n, &nv.value()) {
|
||||
Ok(()) => set_want(&mut want, o, n, nv),
|
||||
Err(Error::Unsupported(msg)) => {
|
||||
assert!(msg.contains("last object"), "{tag}: {o}/{n}: {msg}");
|
||||
assert!(std::fs::read(&b).unwrap() == before, "refused edit wrote");
|
||||
}
|
||||
Err(e) => panic!("{tag}: replace {o}/{n}: {e}"),
|
||||
}
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&b, h5dump);
|
||||
check_attr_values(&b, &want);
|
||||
// libhdf5 goes on: adds, deletes and rewrites.
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 for o in ['g', 't', 'd']:\n\
|
||||
\x20 for i in range(5): f[o].attrs[f'late{{i}}'] = np.arange(i + 1)\n\
|
||||
\x20 del f[o].attrs['n3']\n\
|
||||
\x20 del f[o].attrs['c1']\n\
|
||||
\x20 f[o].attrs['n7'] = 'rewritten by h5py'\n",
|
||||
p = b.to_str().unwrap()
|
||||
));
|
||||
want.retain(|(_, n, _)| n != "n3" && n != "c1" && n != "n7");
|
||||
check_tools(&b, h5dump);
|
||||
check_attr_values(&b, &want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_attributes_match_libhdf5() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
dense_workload(lv, *dump, &format!("{i}"));
|
||||
}
|
||||
}
|
||||
|
||||
/// Dense attributes in files clawhdf5 writes (its own heap and index
|
||||
/// layout, no free-space manager), with and without tracked creation
|
||||
/// order: attributes added (moving a dataset's attributes to dense storage
|
||||
/// too), rewritten and replaced, then h5py goes on.
|
||||
#[test]
|
||||
fn dense_attributes_on_clawhdf5_files() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
for track in [false, true] {
|
||||
let dir = tmpdir();
|
||||
let path = dir.path().join(format!("ours_dense_{track}.h5"));
|
||||
let mut b = clawhdf5::FileBuilder::new();
|
||||
b.track_order(track);
|
||||
for i in 0..12i64 {
|
||||
b.set_attr(&format!("r{i}"), clawhdf5::AttrValue::I64Array(vec![i; 3]));
|
||||
}
|
||||
b.create_dataset("d")
|
||||
.with_i32_data(&[1, 2, 3])
|
||||
.set_attr("c0", clawhdf5::AttrValue::I64Array(vec![5, 5]));
|
||||
b.write(&path).unwrap();
|
||||
let mut want: Vec<AttrOp> = (0..12i64)
|
||||
.map(|i| ("/".to_string(), format!("r{i}"), AV::Ints(vec![i; 3])))
|
||||
.collect();
|
||||
want.push(("d".into(), "c0".into(), AV::Ints(vec![5, 5])));
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
for i in 0..30u64 {
|
||||
for (k, o) in ["/", "d"].iter().enumerate() {
|
||||
// Small attributes: a larger one needs a heap block bigger than
|
||||
// the next (see dense_attribute_refusals_change_nothing).
|
||||
let v = match AV::make(i, k as u64 + 7) {
|
||||
AV::Str(s) if s.len() > 400 => AV::Str(s[..400].to_string()),
|
||||
v => v,
|
||||
};
|
||||
ed.set_attr(o, &format!("n{i}"), &v.value())
|
||||
.unwrap_or_else(|e| panic!("{track} {o}/n{i}: {e}"));
|
||||
set_want(&mut want, o, &format!("n{i}"), v);
|
||||
}
|
||||
}
|
||||
// Rewrites in place, then replacements of another size.
|
||||
for i in (0..12i64).step_by(3) {
|
||||
let v = AV::Ints(vec![-i; 3]);
|
||||
ed.set_attr("/", &format!("r{i}"), &v.value()).unwrap();
|
||||
set_want(&mut want, "/", &format!("r{i}"), v);
|
||||
}
|
||||
for i in (1..30u64).step_by(7) {
|
||||
let v = AV::Str(format!("replaced {i}"));
|
||||
match ed.set_attr("d", &format!("n{i}"), &v.value()) {
|
||||
Ok(()) => set_want(&mut want, "d", &format!("n{i}"), v),
|
||||
Err(Error::Unsupported(msg)) => assert!(msg.contains("last object"), "{msg}"),
|
||||
Err(e) => panic!("{e}"),
|
||||
}
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&path, true);
|
||||
let want_py: Vec<AttrOp> = want
|
||||
.iter()
|
||||
.map(|(o, n, v)| {
|
||||
let o = if o == "/" { "/".to_string() } else { o.clone() };
|
||||
(o, n.clone(), v.clone())
|
||||
})
|
||||
.collect();
|
||||
check_root_and_attrs(&path, &want_py);
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 for o in ['/', 'd']:\n\
|
||||
\x20 f[o].attrs['from_h5py'] = np.arange(3)\n\
|
||||
\x20 del f[o].attrs['n2']\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
want.retain(|(_, n, _)| n != "n2");
|
||||
check_tools(&path, true);
|
||||
check_root_and_attrs(&path, &want);
|
||||
}
|
||||
}
|
||||
|
||||
/// `check_attr_values`, with the root group as "/".
|
||||
fn check_root_and_attrs(path: &Path, want: &[AttrOp]) {
|
||||
let f = File::open(path).unwrap();
|
||||
for (o, n, v) in want {
|
||||
let attrs = match o.as_str() {
|
||||
"/" => f.root().attrs().unwrap(),
|
||||
"d" => f.dataset(o).unwrap().attrs().unwrap(),
|
||||
_ => f.group(o).unwrap().attrs().unwrap(),
|
||||
};
|
||||
let got = attrs
|
||||
.get(n.as_str())
|
||||
.unwrap_or_else(|| panic!("{o}/{n} missing"));
|
||||
match (got, v) {
|
||||
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
||||
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
||||
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
||||
other => panic!("{o}/{n}: {other:?}"),
|
||||
}
|
||||
}
|
||||
let exp: Vec<String> = want
|
||||
.iter()
|
||||
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
||||
.collect();
|
||||
let script = format!(
|
||||
"import h5py, numpy as np\n\
|
||||
f = h5py.File({p:?}, 'r')\n\
|
||||
want = [{w}]\n\
|
||||
for o, n, v in want:\n\
|
||||
\x20 a = f[o].attrs[n]\n\
|
||||
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
||||
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
||||
\x20 a = [a] if isinstance(a, int) else a\n\
|
||||
\x20 assert a == v, (o, n)\n\
|
||||
for o in set(x[0] for x in want):\n\
|
||||
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n",
|
||||
p = path.to_str().unwrap(),
|
||||
w = exp.join(", ")
|
||||
);
|
||||
let sp = path.with_extension("check.py");
|
||||
std::fs::write(&sp, script).unwrap();
|
||||
let o = Command::new(python()).arg(&sp).output().unwrap();
|
||||
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
||||
}
|
||||
|
||||
/// What the editor refuses in dense storage — an object larger than the
|
||||
/// next heap block (libhdf5 would skip blocks and record their space as
|
||||
/// free, which this editor does not do) — is `Error::Unsupported`, and the
|
||||
/// file is left byte for byte as it was.
|
||||
#[test]
|
||||
fn dense_attribute_refusals_change_nothing() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
let dir = tmpdir();
|
||||
let path = dir.path().join("dense_refuse.h5");
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'w', libver='v110') as f:\n\
|
||||
\x20 g = f.create_group('g')\n\
|
||||
\x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
let before = std::fs::read(&path).unwrap();
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("x".repeat(2000))));
|
||||
drop(ed);
|
||||
assert!(
|
||||
std::fs::read(&path).unwrap() == before,
|
||||
"a refused edit wrote"
|
||||
);
|
||||
// What libhdf5 does with it instead works on the untouched file.
|
||||
py(&format!(
|
||||
"import h5py\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 f['g'].attrs['big'] = 'x' * 2000\n\
|
||||
\x20 assert len(f['g'].attrs) == 13\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
check_tools(&path, true);
|
||||
}
|
||||
|
||||
@@ -836,19 +836,16 @@ fn attributes_in_place() {
|
||||
.unwrap();
|
||||
want.push(("d", "units".into(), AttrValue::String("km".into())));
|
||||
if *lv != "'earliest'" {
|
||||
// Up to the compact limit (8) and no further; attributes in
|
||||
// dense storage and tracked creation order are refused, and a
|
||||
// refused edit writes nothing.
|
||||
// Up to the compact limit (8), then into dense storage; objects
|
||||
// already in dense storage and ones tracking creation order.
|
||||
ed.set_attr("g", "eighth", &AttrValue::I64(8)).unwrap();
|
||||
want.push(("g", "eighth".into(), AttrValue::I64(8)));
|
||||
let before = std::fs::read(&path).unwrap();
|
||||
unsupported(ed.set_attr("g", "ninth", &AttrValue::I64(9)));
|
||||
unsupported(ed.set_attr("dense", "k0", &AttrValue::I64(1)));
|
||||
unsupported(ed.set_attr("tracked", "b", &AttrValue::I64(1)));
|
||||
assert!(
|
||||
std::fs::read(&path).unwrap() == before,
|
||||
"a refused edit changed the file"
|
||||
);
|
||||
ed.set_attr("g", "ninth", &AttrValue::I64(9)).unwrap();
|
||||
want.push(("g", "ninth".into(), AttrValue::I64(9)));
|
||||
ed.set_attr("dense", "k0", &AttrValue::I64(1)).unwrap();
|
||||
want.push(("dense", "k0".into(), AttrValue::I64(1)));
|
||||
ed.set_attr("tracked", "b", &AttrValue::I64(1)).unwrap();
|
||||
want.push(("tracked", "b".into(), AttrValue::I64(1)));
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&path, *dump);
|
||||
|
||||
@@ -0,0 +1,513 @@
|
||||
//! Setting attributes, as `H5O__attr_create` / `H5A__dense_insert` do:
|
||||
//! compact attributes are object header messages (with their creation
|
||||
//! index in the message header when the object tracks creation order);
|
||||
//! when an object reaches its compact limit (or an attribute is too large
|
||||
//! for a header message) its attributes move to dense storage — a fractal
|
||||
//! heap for the encoded messages, a version-2 B-tree indexing them by name
|
||||
//! hash (record type 8) and, when creation order is indexed, a second one
|
||||
//! by creation index (type 9) — and the Attribute Info message points at
|
||||
//! them.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
|
||||
use clawhdf5_format::attribute::AttributeMessage;
|
||||
use clawhdf5_format::dataspace::DataspaceType;
|
||||
|
||||
use crate::edit::btree2::Bt2;
|
||||
use crate::edit::fheap::Heap;
|
||||
use crate::edit::image::{Image, get_uint, put_uint, undef};
|
||||
use crate::edit::ohdr::{Header, MSG_ATTRIBUTE};
|
||||
use crate::edit::{MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, MSG_FLAG_SHARED, check_plain};
|
||||
use crate::error::Error;
|
||||
use crate::reader::File;
|
||||
use crate::types::AttrValue;
|
||||
|
||||
/// `H5O_MESG_MAX_SIZE`: a larger attribute goes to dense storage.
|
||||
const MESG_MAX_SIZE: usize = 65536;
|
||||
/// `H5O_MAX_CRT_ORDER_IDX`: the creation index of an attribute of an object
|
||||
/// that does not track creation order.
|
||||
const NO_CRT_IDX: u16 = u16::MAX;
|
||||
/// Name and creation-order index B-trees (`H5A_NAME_BT2_*`,
|
||||
/// `H5A_CORDER_BT2_*`).
|
||||
const NAME_BT2_TYPE: u8 = 8;
|
||||
const CORDER_BT2_TYPE: u8 = 9;
|
||||
const ATTR_BT2_NODE: u32 = 512;
|
||||
/// Heap IDs in attribute records.
|
||||
const ID_LEN: usize = 8;
|
||||
|
||||
/// An object's Attribute Info message.
|
||||
#[derive(Debug, Clone)]
|
||||
struct AInfo {
|
||||
/// Its message index in the header.
|
||||
idx: usize,
|
||||
track: bool,
|
||||
index: bool,
|
||||
max_crt: u16,
|
||||
fheap: u64,
|
||||
name_bt2: u64,
|
||||
corder_bt2: u64,
|
||||
}
|
||||
|
||||
impl AInfo {
|
||||
fn load(img: &Image<'_>, hdr: &Header) -> Result<Option<Self>, Error> {
|
||||
let Some(idx) = hdr.find(MSG_ATTR_INFO) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if hdr.msgs[idx].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared attribute info message".into()));
|
||||
}
|
||||
let d = hdr.data(img, idx)?;
|
||||
let os = img.os as usize;
|
||||
let short = || Error::Unsupported("short attribute info message".into());
|
||||
if d.first() != Some(&0) {
|
||||
return Err(Error::Unsupported("attribute info message version".into()));
|
||||
}
|
||||
let flags = *d.get(1).ok_or_else(short)?;
|
||||
let track = flags & 0x01 != 0;
|
||||
let index = flags & 0x02 != 0;
|
||||
let mut p = 2;
|
||||
let mut max_crt = 0;
|
||||
if track {
|
||||
let b = d.get(p..p + 2).ok_or_else(short)?;
|
||||
max_crt = u16::from_le_bytes([b[0], b[1]]);
|
||||
p += 2;
|
||||
}
|
||||
let n = if index { 3 } else { 2 };
|
||||
if d.len() < p + n * os {
|
||||
return Err(short());
|
||||
}
|
||||
Ok(Some(Self {
|
||||
idx,
|
||||
track,
|
||||
index,
|
||||
max_crt,
|
||||
fheap: get_uint(&d[p..], img.os),
|
||||
name_bt2: get_uint(&d[p + os..], img.os),
|
||||
corder_bt2: if index {
|
||||
get_uint(&d[p + 2 * os..], img.os)
|
||||
} else {
|
||||
undef(img.os)
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
fn dense(&self, os: u8) -> bool {
|
||||
self.fheap != undef(os)
|
||||
}
|
||||
|
||||
/// Store the changeable fields back into the message.
|
||||
fn store(&self, img: &mut Image<'_>, hdr: &mut Header) -> Result<(), Error> {
|
||||
let os = img.os as usize;
|
||||
let mut p = 2;
|
||||
if self.track {
|
||||
hdr.patch(img, self.idx, p, &self.max_crt.to_le_bytes())?;
|
||||
p += 2;
|
||||
}
|
||||
let mut a = vec![0u8; os];
|
||||
for (k, v) in [self.fheap, self.name_bt2, self.corder_bt2]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.take(if self.index { 3 } else { 2 })
|
||||
{
|
||||
put_uint(&mut a, v, img.os);
|
||||
hdr.patch(img, self.idx, p + k * os, &a)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The next creation index (`H5O__attr_create`), or libhdf5's "none".
|
||||
fn next_crt(&mut self) -> Result<u16, Error> {
|
||||
if !self.track {
|
||||
return Ok(NO_CRT_IDX);
|
||||
}
|
||||
if self.max_crt == NO_CRT_IDX {
|
||||
return Err(Error::Unsupported(
|
||||
"object's attribute creation index is exhausted".into(),
|
||||
));
|
||||
}
|
||||
self.max_crt += 1;
|
||||
Ok(self.max_crt - 1)
|
||||
}
|
||||
}
|
||||
|
||||
/// The name bytes of an attribute message body (without the NUL).
|
||||
pub(super) fn attr_name(d: &[u8]) -> Result<&[u8], Error> {
|
||||
let bad = || Error::Unsupported("malformed attribute message".into());
|
||||
let (len, at) = match d.first() {
|
||||
Some(1) | Some(2) if d.len() >= 8 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8),
|
||||
Some(3) if d.len() >= 9 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9),
|
||||
_ => return Err(bad()),
|
||||
};
|
||||
let name = d.get(at..at + len).ok_or_else(bad)?;
|
||||
Ok(name.split(|&b| b == 0).next().unwrap_or(name))
|
||||
}
|
||||
|
||||
/// A new Attribute Info message for a version-2 header with flags
|
||||
/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order
|
||||
/// tracked / indexed as the header's flags say, the maximum creation index,
|
||||
/// and no dense storage (undefined fractal heap and B-tree addresses).
|
||||
fn attr_info_message(hdr_flags: u8, max_crt: u16, os: u8) -> Vec<u8> {
|
||||
let track = hdr_flags & 0x04 != 0;
|
||||
let index = hdr_flags & 0x08 != 0;
|
||||
let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)];
|
||||
if track {
|
||||
b.extend_from_slice(&max_crt.to_le_bytes());
|
||||
}
|
||||
let undef_addr = vec![0xffu8; os as usize];
|
||||
b.extend_from_slice(&undef_addr);
|
||||
b.extend_from_slice(&undef_addr);
|
||||
if index {
|
||||
b.extend_from_slice(&undef_addr);
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
/// A version-2 header's limit on compact attributes: stored when its flags
|
||||
/// say so, else libhdf5's default of 8.
|
||||
fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result<u16, Error> {
|
||||
if hdr.flags & 0x10 == 0 {
|
||||
return Ok(8);
|
||||
}
|
||||
let mut p = hdr.addr + 6;
|
||||
if hdr.flags & 0x20 != 0 {
|
||||
p += 16;
|
||||
}
|
||||
let b = img.read(p, 2)?;
|
||||
Ok(u16::from_le_bytes([b[0], b[1]]))
|
||||
}
|
||||
|
||||
/// A version-1 attribute message (what libhdf5 writes in a version-1 object
|
||||
/// header): name, datatype and dataspace each padded to 8 bytes, the
|
||||
/// dataspace as a version-1 dataspace message.
|
||||
fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec<u8> {
|
||||
let mut name = a.name.as_bytes().to_vec();
|
||||
name.push(0);
|
||||
let dt = a.datatype.serialize();
|
||||
let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0];
|
||||
if a.dataspace.space_type == DataspaceType::Simple {
|
||||
let mut b = vec![0u8; ls as usize];
|
||||
for &d in &a.dataspace.dimensions {
|
||||
put_uint(&mut b, d, ls);
|
||||
ds.extend_from_slice(&b);
|
||||
}
|
||||
if let Some(max) = &a.dataspace.max_dimensions {
|
||||
ds[2] = 0x01;
|
||||
for &d in max {
|
||||
put_uint(&mut b, d, ls);
|
||||
ds.extend_from_slice(&b);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ds[1] = 0;
|
||||
}
|
||||
let mut out = vec![1u8, 0];
|
||||
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(dt.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(ds.len() as u16).to_le_bytes());
|
||||
for part in [&name, &dt, &ds] {
|
||||
out.extend_from_slice(part);
|
||||
out.resize(out.len().next_multiple_of(8), 0);
|
||||
}
|
||||
out.extend_from_slice(&a.raw_data);
|
||||
out
|
||||
}
|
||||
|
||||
/// Dense storage opened for changes.
|
||||
struct Dense {
|
||||
heap: Heap,
|
||||
names: Bt2,
|
||||
order: Option<Bt2>,
|
||||
}
|
||||
|
||||
/// `H5_checksum_lookup3` of a name, as the name index keys it.
|
||||
fn name_hash(name: &[u8]) -> u32 {
|
||||
clawhdf5_format::checksum::jenkins_lookup3(name)
|
||||
}
|
||||
|
||||
/// Compare attribute `name` (hash `hash`) with a name-index record
|
||||
/// (`H5A__dense_btree2_name_compare`: the hash, then the stored name).
|
||||
fn cmp_name(
|
||||
heap: &Heap,
|
||||
img: &Image<'_>,
|
||||
hash: u32,
|
||||
name: &[u8],
|
||||
rec: &[u8],
|
||||
) -> Result<Ordering, Error> {
|
||||
let theirs = u32::from_le_bytes([rec[13], rec[14], rec[15], rec[16]]);
|
||||
match hash.cmp(&theirs) {
|
||||
Ordering::Equal => {
|
||||
if rec[ID_LEN] & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported(
|
||||
"shared attribute in dense storage".into(),
|
||||
));
|
||||
}
|
||||
let obj = heap.read(img, &rec[..ID_LEN])?;
|
||||
Ok(name.cmp(attr_name(&obj)?))
|
||||
}
|
||||
o => Ok(o),
|
||||
}
|
||||
}
|
||||
|
||||
fn corder_of(rec: &[u8]) -> u32 {
|
||||
u32::from_le_bytes([rec[9], rec[10], rec[11], rec[12]])
|
||||
}
|
||||
|
||||
impl Dense {
|
||||
fn open(img: &Image<'_>, ai: &AInfo) -> Result<Self, Error> {
|
||||
let heap = Heap::open(img, ai.fheap)?;
|
||||
let names = Bt2::open(img, ai.name_bt2)?;
|
||||
if names.tree_type() != NAME_BT2_TYPE || names.record_size() != ID_LEN + 9 {
|
||||
return Err(Error::Unsupported("attribute name index layout".into()));
|
||||
}
|
||||
let order = if ai.index {
|
||||
let t = Bt2::open(img, ai.corder_bt2)?;
|
||||
if t.tree_type() != CORDER_BT2_TYPE || t.record_size() != ID_LEN + 5 {
|
||||
return Err(Error::Unsupported(
|
||||
"attribute creation-order index layout".into(),
|
||||
));
|
||||
}
|
||||
Some(t)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(Self { heap, names, order })
|
||||
}
|
||||
|
||||
/// `H5A__dense_create`: heap, name index, [creation-order index].
|
||||
fn create(img: &mut Image<'_>, index: bool) -> Result<Self, Error> {
|
||||
let heap = Heap::create_attribute_heap(img)?;
|
||||
let names = Bt2::create(img, NAME_BT2_TYPE, ATTR_BT2_NODE, ID_LEN + 9, 100, 40)?;
|
||||
let order = if index {
|
||||
Some(Bt2::create(
|
||||
img,
|
||||
CORDER_BT2_TYPE,
|
||||
ATTR_BT2_NODE,
|
||||
ID_LEN + 5,
|
||||
100,
|
||||
40,
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(Self { heap, names, order })
|
||||
}
|
||||
|
||||
/// `H5A__dense_insert` of an encoded attribute message.
|
||||
fn insert(&mut self, img: &mut Image<'_>, body: &[u8], crt: u16) -> Result<(), Error> {
|
||||
let name = attr_name(body)?.to_vec();
|
||||
let id = self.heap.insert(img, body)?;
|
||||
if id.len() != ID_LEN {
|
||||
return Err(Error::Unsupported("attribute heap ID length".into()));
|
||||
}
|
||||
let hash = name_hash(&name);
|
||||
let mut rec = id.clone();
|
||||
rec.push(0);
|
||||
rec.extend_from_slice(&u32::from(crt).to_le_bytes());
|
||||
rec.extend_from_slice(&hash.to_le_bytes());
|
||||
let heap = &self.heap;
|
||||
self.names
|
||||
.insert(img, &mut |im, r| cmp_name(heap, im, hash, &name, r), &rec)?;
|
||||
if let Some(t) = &mut self.order {
|
||||
let key = u32::from(crt);
|
||||
t.insert(
|
||||
img,
|
||||
&mut |_, r| Ok(key.cmp(&corder_of(r))),
|
||||
&rec[..ID_LEN + 5],
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> {
|
||||
self.heap.finish(img)?;
|
||||
self.names.finish(img)?;
|
||||
if let Some(t) = &mut self.order {
|
||||
t.finish(img)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Set attribute `name` of the object at `path` to `value`.
|
||||
pub(super) fn set_attr(
|
||||
f: &File,
|
||||
img: &mut Image<'_>,
|
||||
path: &str,
|
||||
name: &str,
|
||||
value: &AttrValue,
|
||||
) -> Result<(), Error> {
|
||||
let addr = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?;
|
||||
let mut hdr = Header::load(img, addr)?;
|
||||
let mut msg = clawhdf5_format::type_builders::build_attr_message(name, value);
|
||||
check_plain(&msg.datatype)?;
|
||||
// libhdf5 encodes a simple dataspace with its maximum dimensions (the
|
||||
// current ones when none were given), so an attribute takes the same
|
||||
// space in a header or heap as when libhdf5 writes it.
|
||||
if msg.dataspace.space_type == DataspaceType::Simple && msg.dataspace.max_dimensions.is_none() {
|
||||
msg.dataspace.max_dimensions = Some(msg.dataspace.dimensions.clone());
|
||||
}
|
||||
// H5A__set_version: version 1 unless the name is not ASCII (then 3),
|
||||
// raised to the file's low bound — which is the earliest for a file
|
||||
// libhdf5 opens without a libver setting (h5py's `r+`).
|
||||
let body = if hdr.version == 1 || name.is_ascii() {
|
||||
encode_attr_v1(&msg, img.ls)
|
||||
} else {
|
||||
let mut b = msg.serialize_v3(img.ls);
|
||||
if !name.is_ascii() {
|
||||
b[8] = 1; // UTF-8 name
|
||||
}
|
||||
b
|
||||
};
|
||||
let mut ainfo = if hdr.version == 2 {
|
||||
AInfo::load(img, &hdr)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(ai) = ainfo.as_mut().filter(|a| a.dense(img.os)) {
|
||||
let mut ai = ai.clone();
|
||||
set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?;
|
||||
return hdr.finish(img);
|
||||
}
|
||||
|
||||
let mut existing = None;
|
||||
let mut count = 0usize;
|
||||
for i in 0..hdr.msgs.len() {
|
||||
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
|
||||
continue;
|
||||
}
|
||||
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared attribute message".into()));
|
||||
}
|
||||
count += 1;
|
||||
if attr_name(&hdr.data(img, i)?)? == name.as_bytes() {
|
||||
existing = Some(i);
|
||||
}
|
||||
}
|
||||
if let Some(i) = existing {
|
||||
hdr.delete(img, i)?;
|
||||
count -= 1;
|
||||
}
|
||||
if hdr.version == 1 {
|
||||
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?;
|
||||
return hdr.finish(img);
|
||||
}
|
||||
let tracked = hdr.flags & 0x04 != 0;
|
||||
// H5O__attr_create: a missing Attribute Info message starts from
|
||||
// nothing (and is added below, holding the new maximum creation index).
|
||||
let new_ainfo = ainfo.is_none();
|
||||
let mut ai = ainfo.take().unwrap_or(AInfo {
|
||||
idx: usize::MAX,
|
||||
track: tracked,
|
||||
index: hdr.flags & 0x08 != 0,
|
||||
max_crt: 0,
|
||||
fheap: undef(img.os),
|
||||
name_bt2: undef(img.os),
|
||||
corder_bt2: undef(img.os),
|
||||
});
|
||||
let max_compact = usize::from(max_compact_attrs(img, &hdr)?);
|
||||
if count == max_compact || body.len() >= MESG_MAX_SIZE {
|
||||
if new_ainfo {
|
||||
return Err(Error::Unsupported(
|
||||
"dense attribute storage for an object without an Attribute Info message".into(),
|
||||
));
|
||||
}
|
||||
to_dense(img, &mut hdr, &mut ai)?;
|
||||
set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?;
|
||||
return hdr.finish(img);
|
||||
}
|
||||
let crt = ai.next_crt()?;
|
||||
let corder = tracked.then_some(crt);
|
||||
if new_ainfo {
|
||||
// libhdf5 appends the Attribute Info message before the attribute
|
||||
// when free space holds both, else after it, so that a new
|
||||
// continuation chunk made for the attribute has room for it too.
|
||||
let a = attr_info_message(hdr.flags, ai.max_crt, img.os);
|
||||
let first = hdr.has_free(a.len() + hdr.hsize() + body.len());
|
||||
if first {
|
||||
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?;
|
||||
}
|
||||
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?;
|
||||
if !first {
|
||||
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?;
|
||||
}
|
||||
} else {
|
||||
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?;
|
||||
ai.store(img, &mut hdr)?;
|
||||
}
|
||||
hdr.finish(img)
|
||||
}
|
||||
|
||||
/// Move every compact attribute of the object into new dense storage, in
|
||||
/// header message order (`H5O__attr_to_dense_cb`), leaving free space where
|
||||
/// the messages were.
|
||||
fn to_dense(img: &mut Image<'_>, hdr: &mut Header, ai: &mut AInfo) -> Result<(), Error> {
|
||||
let mut dense = Dense::create(img, ai.index)?;
|
||||
for i in 0..hdr.msgs.len() {
|
||||
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
|
||||
continue;
|
||||
}
|
||||
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared attribute message".into()));
|
||||
}
|
||||
let body = hdr.data(img, i)?;
|
||||
let crt = if ai.track {
|
||||
hdr.msgs[i].corder.unwrap_or(0)
|
||||
} else {
|
||||
NO_CRT_IDX
|
||||
};
|
||||
dense.insert(img, &body, crt)?;
|
||||
hdr.delete(img, i)?;
|
||||
}
|
||||
dense.finish(img)?;
|
||||
ai.fheap = dense.heap.address();
|
||||
ai.name_bt2 = dense.names.address();
|
||||
if let Some(t) = &dense.order {
|
||||
ai.corder_bt2 = t.address();
|
||||
}
|
||||
ai.store(img, hdr)
|
||||
}
|
||||
|
||||
/// Set an attribute of an object whose attributes are in dense storage: an
|
||||
/// attribute of that name whose new encoding has the old one's size is
|
||||
/// rewritten in its heap object (`H5A__dense_write`); otherwise the old one
|
||||
/// is removed (`H5A__dense_remove`: name index, creation-order index, heap
|
||||
/// object) and the new one inserted with the next creation index.
|
||||
fn set_dense(
|
||||
img: &mut Image<'_>,
|
||||
hdr: &mut Header,
|
||||
ai: &mut AInfo,
|
||||
name: &[u8],
|
||||
body: &[u8],
|
||||
) -> Result<(), Error> {
|
||||
let mut dense = Dense::open(img, ai)?;
|
||||
let hash = name_hash(name);
|
||||
let found = {
|
||||
let heap = &dense.heap;
|
||||
dense
|
||||
.names
|
||||
.find(img, &mut |im, r| cmp_name(heap, im, hash, name, r))?
|
||||
};
|
||||
if let Some(rec) = found {
|
||||
if dense.heap.write_in_place(img, &rec[..ID_LEN], body)? {
|
||||
return dense.finish(img);
|
||||
}
|
||||
{
|
||||
let heap = &dense.heap;
|
||||
dense
|
||||
.names
|
||||
.remove(img, &mut |im, r| cmp_name(heap, im, hash, name, r))?;
|
||||
}
|
||||
if let Some(t) = &mut dense.order {
|
||||
let key = corder_of(&rec);
|
||||
t.remove(img, &mut |_, r| Ok(key.cmp(&corder_of(r))))?
|
||||
.ok_or_else(|| {
|
||||
Error::Unsupported("attribute missing from its creation-order index".into())
|
||||
})?;
|
||||
}
|
||||
dense.heap.remove(img, &rec[..ID_LEN])?;
|
||||
}
|
||||
let crt = ai.next_crt()?;
|
||||
dense.insert(img, body, crt)?;
|
||||
dense.finish(img)?;
|
||||
ai.store(img, hdr)
|
||||
}
|
||||
@@ -180,11 +180,6 @@ impl Bt2 {
|
||||
self.tree_type
|
||||
}
|
||||
|
||||
/// Records in the whole tree.
|
||||
pub(crate) fn len(&self) -> u64 {
|
||||
self.root.all
|
||||
}
|
||||
|
||||
/// Node geometry for depths `0..=self.depth` (`H5B2__hdr_init`,
|
||||
/// `H5B2__split_root`).
|
||||
fn compute_levels(&mut self, os: u8) -> Result<(), Error> {
|
||||
@@ -389,35 +384,6 @@ impl Bt2 {
|
||||
}
|
||||
}
|
||||
|
||||
/// Every record, in key order.
|
||||
pub(crate) fn records(&mut self, img: &Image<'_>) -> Result<Vec<Vec<u8>>, Error> {
|
||||
let mut out = Vec::new();
|
||||
if self.root.addr != undef(img.os) && self.root.nrec > 0 {
|
||||
self.walk(img, self.root, self.depth, &mut out)?;
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn walk(
|
||||
&mut self,
|
||||
img: &Image<'_>,
|
||||
p: Ptr,
|
||||
depth: u16,
|
||||
out: &mut Vec<Vec<u8>>,
|
||||
) -> Result<(), Error> {
|
||||
self.load(img, p, depth)?;
|
||||
let n = self.peek(p.addr).clone();
|
||||
for i in 0..=n.recs.len() {
|
||||
if depth > 0 {
|
||||
self.walk(img, n.ptrs[i], depth - 1, out)?;
|
||||
}
|
||||
if i < n.recs.len() {
|
||||
out.push(n.recs[i].clone());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Insert `rec`, or replace the record `cmp` matches (`H5B2_update`).
|
||||
pub(crate) fn update(
|
||||
&mut self,
|
||||
@@ -665,6 +631,7 @@ impl Bt2 {
|
||||
/// parent records between them) out over the children `lo..` of
|
||||
/// `parent`, with `counts[k]` records in the k-th; the records between
|
||||
/// them go back into the parent.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn relayout(
|
||||
&mut self,
|
||||
parent: u64,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+22
-168
@@ -15,10 +15,12 @@
|
||||
//! file) is dropped before the first write, so no `&[u8]` over the mapping
|
||||
//! is alive while the file changes (see `image`).
|
||||
|
||||
mod attrs;
|
||||
mod btree1;
|
||||
mod btree2;
|
||||
mod earray;
|
||||
mod farray;
|
||||
mod fheap;
|
||||
mod image;
|
||||
mod ohdr;
|
||||
mod select;
|
||||
@@ -27,7 +29,6 @@ use std::collections::{BTreeMap, HashMap};
|
||||
use std::fs::{OpenOptions, TryLockError};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use clawhdf5_format::attribute::AttributeMessage;
|
||||
use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks};
|
||||
use clawhdf5_format::data_layout::DataLayout;
|
||||
use clawhdf5_format::data_read::NativeElement;
|
||||
@@ -43,8 +44,8 @@ use btree1::{BTree1, Key};
|
||||
use btree2::Bt2;
|
||||
use earray::{Ea, EaParams, Elem};
|
||||
use farray::Fa;
|
||||
use image::{Image, get_uint, put_uint, undef};
|
||||
use ohdr::{Header, MSG_ATTRIBUTE};
|
||||
use image::{Image, put_uint, undef};
|
||||
use ohdr::Header;
|
||||
|
||||
const MSG_DATASPACE: u16 = 0x01;
|
||||
const MSG_LAYOUT: u16 = 0x08;
|
||||
@@ -918,101 +919,29 @@ impl FileEditor {
|
||||
|
||||
/// Set attribute `name` of the object at `path` (a group or dataset;
|
||||
/// `"/"` is the root group) to `value`, replacing an attribute of that
|
||||
/// name. The attribute goes into free space in the object header, or a
|
||||
/// new header continuation chunk at the end of the file.
|
||||
/// name, as libhdf5 creates attributes (`H5O__attr_create`).
|
||||
///
|
||||
/// [`Error::Unsupported`] for an object whose attributes are in dense
|
||||
/// storage (or would have to move there: more than the object's
|
||||
/// compact-attribute limit), one that tracks attribute creation order,
|
||||
/// or one with shared attribute messages.
|
||||
/// A compact attribute goes into free space in the object header, or a
|
||||
/// new header continuation chunk; an object that tracks creation order
|
||||
/// gives it the next creation index. When the object reaches its
|
||||
/// compact-attribute limit (or the attribute is too large for a header
|
||||
/// message) its attributes move to dense storage, and objects already
|
||||
/// using dense storage get the attribute there: a fractal heap object
|
||||
/// (in free heap space, a new heap block, or its own file space when
|
||||
/// larger than the heap's managed limit) indexed by name and, when the
|
||||
/// object indexes creation order, by creation index. An attribute
|
||||
/// replaced by one of the same encoded size is rewritten where it is.
|
||||
///
|
||||
/// [`Error::Unsupported`] for shared attribute messages, heaps this
|
||||
/// editor cannot extend the way libhdf5 would (child indirect blocks,
|
||||
/// skipped blocks, free space other than within direct blocks, freeing
|
||||
/// a whole block), and version-1 object headers asked for an attribute
|
||||
/// larger than a header message holds.
|
||||
pub fn set_attr(&mut self, path: &str, name: &str, value: &AttrValue) -> Result<(), Error> {
|
||||
if name.is_empty() {
|
||||
return Err(Error::InvalidArgument("empty attribute name".into()));
|
||||
}
|
||||
self.edit(|f, img| {
|
||||
let addr =
|
||||
clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?;
|
||||
let mut hdr = Header::load(img, addr)?;
|
||||
if hdr.version == 2 && hdr.flags & 0x04 != 0 {
|
||||
return Err(Error::Unsupported(
|
||||
"object tracks attribute creation order".into(),
|
||||
));
|
||||
}
|
||||
if let Some(i) = hdr.find(MSG_ATTR_INFO) {
|
||||
let d = hdr.data(img, i)?;
|
||||
// version(1) flags(1) [max creation index(2)] fractal heap
|
||||
// address, name index address, [order index address].
|
||||
let mut p = 2;
|
||||
if d.get(1).is_some_and(|f| f & 0x01 != 0) {
|
||||
p += 2;
|
||||
}
|
||||
let os = img.os as usize;
|
||||
if d.len() < p + os {
|
||||
return Err(Error::Unsupported("short attribute info message".into()));
|
||||
}
|
||||
if get_uint(&d[p..], img.os) != undef(img.os) {
|
||||
return Err(Error::Unsupported(
|
||||
"object with attributes in dense storage".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let mut existing = None;
|
||||
let mut count = 0usize;
|
||||
for i in 0..hdr.msgs.len() {
|
||||
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
|
||||
continue;
|
||||
}
|
||||
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared attribute message".into()));
|
||||
}
|
||||
count += 1;
|
||||
if attr_name(&hdr.data(img, i)?)? == name.as_bytes() {
|
||||
existing = Some(i);
|
||||
}
|
||||
}
|
||||
if existing.is_none() && hdr.version == 2 {
|
||||
let max_compact = max_compact_attrs(img, &hdr)?;
|
||||
if count + 1 > usize::from(max_compact) {
|
||||
return Err(Error::Unsupported(format!(
|
||||
"object already has {count} compact attributes (its limit is \
|
||||
{max_compact}); more need dense storage"
|
||||
)));
|
||||
}
|
||||
}
|
||||
let msg = clawhdf5_format::type_builders::build_attr_message(name, value);
|
||||
check_plain(&msg.datatype)?;
|
||||
let body = if hdr.version == 1 {
|
||||
encode_attr_v1(&msg, img.ls)
|
||||
} else {
|
||||
let mut b = msg.serialize_v3(img.ls);
|
||||
if !name.is_ascii() {
|
||||
b[8] = 1; // UTF-8 name
|
||||
}
|
||||
b
|
||||
};
|
||||
if let Some(i) = existing {
|
||||
hdr.delete(img, i)?;
|
||||
}
|
||||
// libhdf5 counts a version-2 header's attributes through its
|
||||
// Attribute Info message and reports none without one; like
|
||||
// H5O__attr_create, add it when missing: before the attribute
|
||||
// when free space holds both (libhdf5's order), else after it,
|
||||
// so that a new continuation chunk made for the attribute has
|
||||
// room for it too.
|
||||
let ainfo = (hdr.version == 2 && hdr.find(MSG_ATTR_INFO).is_none())
|
||||
.then(|| attr_info_message(hdr.flags, img.os));
|
||||
let ainfo_first = ainfo
|
||||
.as_ref()
|
||||
.is_some_and(|a| hdr.has_free(a.len() + hdr.hsize() + body.len()));
|
||||
if let Some(a) = ainfo.as_ref().filter(|_| ainfo_first) {
|
||||
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?;
|
||||
}
|
||||
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?;
|
||||
if let Some(a) = ainfo.as_ref().filter(|_| !ainfo_first) {
|
||||
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?;
|
||||
}
|
||||
hdr.finish(img)
|
||||
})
|
||||
self.edit(|f, img| attrs::set_attr(f, img, path, name, value))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1097,81 +1026,6 @@ fn set_superblock_eof(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The name bytes of an attribute message body (without the NUL).
|
||||
fn attr_name(d: &[u8]) -> Result<&[u8], Error> {
|
||||
let bad = || Error::Unsupported("malformed attribute message".into());
|
||||
let (len, at) = match d.first() {
|
||||
Some(1) | Some(2) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8),
|
||||
Some(3) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9),
|
||||
_ => return Err(bad()),
|
||||
};
|
||||
let name = d.get(at..at + len).ok_or_else(bad)?;
|
||||
Ok(name.split(|&b| b == 0).next().unwrap_or(name))
|
||||
}
|
||||
|
||||
/// A new Attribute Info message for a version-2 header with flags
|
||||
/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order
|
||||
/// tracked / indexed as the header's flags say, maximum creation index 0,
|
||||
/// and no dense storage (undefined fractal heap and B-tree addresses).
|
||||
fn attr_info_message(hdr_flags: u8, os: u8) -> Vec<u8> {
|
||||
let track = hdr_flags & 0x04 != 0;
|
||||
let index = hdr_flags & 0x08 != 0;
|
||||
let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)];
|
||||
if track {
|
||||
b.extend_from_slice(&0u16.to_le_bytes());
|
||||
}
|
||||
let undef_addr = vec![0xffu8; os as usize];
|
||||
b.extend_from_slice(&undef_addr);
|
||||
b.extend_from_slice(&undef_addr);
|
||||
if index {
|
||||
b.extend_from_slice(&undef_addr);
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
/// A version-2 header's limit on compact attributes: stored when its flags
|
||||
/// say so, else libhdf5's default of 8.
|
||||
fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result<u16, Error> {
|
||||
if hdr.flags & 0x10 == 0 {
|
||||
return Ok(8);
|
||||
}
|
||||
let mut p = hdr.addr + 6;
|
||||
if hdr.flags & 0x20 != 0 {
|
||||
p += 16;
|
||||
}
|
||||
let b = img.read(p, 2)?;
|
||||
Ok(u16::from_le_bytes([b[0], b[1]]))
|
||||
}
|
||||
|
||||
/// A version-1 attribute message (what libhdf5 writes in a version-1 object
|
||||
/// header): name, datatype and dataspace each padded to 8 bytes, the
|
||||
/// dataspace as a version-1 dataspace message.
|
||||
fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec<u8> {
|
||||
let mut name = a.name.as_bytes().to_vec();
|
||||
name.push(0);
|
||||
let dt = a.datatype.serialize();
|
||||
let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0];
|
||||
if a.dataspace.space_type == DataspaceType::Simple {
|
||||
for &d in &a.dataspace.dimensions {
|
||||
let mut b = vec![0u8; ls as usize];
|
||||
put_uint(&mut b, d, ls);
|
||||
ds.extend_from_slice(&b);
|
||||
}
|
||||
} else {
|
||||
ds[1] = 0;
|
||||
}
|
||||
let mut out = vec![1u8, 0];
|
||||
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(dt.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(ds.len() as u16).to_le_bytes());
|
||||
for part in [&name, &dt, &ds] {
|
||||
out.extend_from_slice(part);
|
||||
out.resize(out.len().next_multiple_of(8), 0);
|
||||
}
|
||||
out.extend_from_slice(&a.raw_data);
|
||||
out
|
||||
}
|
||||
|
||||
fn write_selection(
|
||||
f: &File,
|
||||
img: &mut Image<'_>,
|
||||
|
||||
@@ -59,7 +59,7 @@ pub(crate) struct Header {
|
||||
added: usize,
|
||||
}
|
||||
|
||||
const MAX_CHUNKS: usize = 1024;
|
||||
const MAX_CHUNKS: usize = 1 << 16;
|
||||
|
||||
fn corrupt(why: &'static str) -> Error {
|
||||
Error::Format(FormatError::InvalidObjectHeader(why))
|
||||
@@ -118,7 +118,11 @@ impl Header {
|
||||
});
|
||||
h.scan(img, 0, addr + 16, addr + 16 + size, &mut pending)?;
|
||||
}
|
||||
while let Some((caddr, clen)) = pending.pop() {
|
||||
// Continuation chunks in the order their messages are found, as
|
||||
// H5O_protect loads them (so messages keep libhdf5's order).
|
||||
let mut next = 0;
|
||||
while let Some(&(caddr, clen)) = pending.get(next) {
|
||||
next += 1;
|
||||
if h.chunks.len() >= MAX_CHUNKS {
|
||||
return Err(corrupt("too many object header chunks"));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user