edit: dense attributes, compact-to-dense transition, creation order
FileEditor::set_attr now handles every attribute storage libhdf5 uses for version-2 object headers: - objects that track (and index) attribute creation order: compact attributes carry their creation index in the message header, the Attribute Info message its maximum; - the move to dense storage when an object reaches its compact limit (or an attribute is too large for a header message), as H5O__attr_create does it: a new fractal heap, name index (v2 B-tree type 8) and, when creation order is indexed, creation-order index (type 9); the compact attributes moved over in header message order, their messages freed; - objects already in dense storage (h5py- or clawhdf5-written): new attributes inserted (H5A__dense_insert), an attribute replaced by one of the same encoded size rewritten in its heap object (H5A__dense_write), otherwise removed from both indexes and the heap and inserted anew. edit/fheap.rs follows H5HF: managed objects go to the best-fitting free section of the heap's free-space manager (FSHD/FSSE, kept as libhdf5 keeps it — sorted sections, counts, section info reallocated when its size changes, the manager deleted when empty); otherwise to a new direct block: the root direct block of an empty heap, else the block at the allocation iterator in the root indirect block (created from the root direct block, doubled as needed), with libhdf5's managed-space, allocated space, free space and iterator bookkeeping. Objects above the managed limit are huge objects in their own space, indexed by the huge-object B-tree (type 1). Removed objects return their space merged with adjacent free space. Refused before anything is written: heaps with I/O filters, child indirect blocks, an object larger than the next heap block (libhdf5 skips blocks and records them as free space), free sections other than those inside direct blocks, removing a direct block's last object (libhdf5 frees the block), directly addressed huge objects. Attributes are encoded as libhdf5 does when h5py opens a file r+ (low bound "earliest"): message version 1 (3 for non-ASCII names), simple dataspaces with their maximum dimensions. Header chunks are now visited in libhdf5's order (FIFO), which is also the order attributes move to dense storage in. Tests (edit_coverage_interop): 40 attributes on each of a plain group, a group tracking and indexing creation order, and a dataset (earliest, v110, latest), some above the 4 KiB managed limit, then same-size rewrites: the heap statistics, free-space sections and both index B-trees node for node equal libhdf5's doing the same through h5py; then replacements of other sizes, h5py adds/deletes/rewrites; h5py, h5dump, h5rs check and our reader agree throughout, h5py's attribute count included. clawhdf5-written dense storage (tracked and untracked) is extended the same way; refusals leave the file byte for byte as it was. edit_interop's attribute test now expects dense storage and tracked creation order to work. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,513 @@
|
||||
//! Setting attributes, as `H5O__attr_create` / `H5A__dense_insert` do:
|
||||
//! compact attributes are object header messages (with their creation
|
||||
//! index in the message header when the object tracks creation order);
|
||||
//! when an object reaches its compact limit (or an attribute is too large
|
||||
//! for a header message) its attributes move to dense storage — a fractal
|
||||
//! heap for the encoded messages, a version-2 B-tree indexing them by name
|
||||
//! hash (record type 8) and, when creation order is indexed, a second one
|
||||
//! by creation index (type 9) — and the Attribute Info message points at
|
||||
//! them.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
|
||||
use clawhdf5_format::attribute::AttributeMessage;
|
||||
use clawhdf5_format::dataspace::DataspaceType;
|
||||
|
||||
use crate::edit::btree2::Bt2;
|
||||
use crate::edit::fheap::Heap;
|
||||
use crate::edit::image::{Image, get_uint, put_uint, undef};
|
||||
use crate::edit::ohdr::{Header, MSG_ATTRIBUTE};
|
||||
use crate::edit::{MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, MSG_FLAG_SHARED, check_plain};
|
||||
use crate::error::Error;
|
||||
use crate::reader::File;
|
||||
use crate::types::AttrValue;
|
||||
|
||||
/// `H5O_MESG_MAX_SIZE`: a larger attribute goes to dense storage.
|
||||
const MESG_MAX_SIZE: usize = 65536;
|
||||
/// `H5O_MAX_CRT_ORDER_IDX`: the creation index of an attribute of an object
|
||||
/// that does not track creation order.
|
||||
const NO_CRT_IDX: u16 = u16::MAX;
|
||||
/// Name and creation-order index B-trees (`H5A_NAME_BT2_*`,
|
||||
/// `H5A_CORDER_BT2_*`).
|
||||
const NAME_BT2_TYPE: u8 = 8;
|
||||
const CORDER_BT2_TYPE: u8 = 9;
|
||||
const ATTR_BT2_NODE: u32 = 512;
|
||||
/// Heap IDs in attribute records.
|
||||
const ID_LEN: usize = 8;
|
||||
|
||||
/// An object's Attribute Info message.
|
||||
#[derive(Debug, Clone)]
|
||||
struct AInfo {
|
||||
/// Its message index in the header.
|
||||
idx: usize,
|
||||
track: bool,
|
||||
index: bool,
|
||||
max_crt: u16,
|
||||
fheap: u64,
|
||||
name_bt2: u64,
|
||||
corder_bt2: u64,
|
||||
}
|
||||
|
||||
impl AInfo {
|
||||
fn load(img: &Image<'_>, hdr: &Header) -> Result<Option<Self>, Error> {
|
||||
let Some(idx) = hdr.find(MSG_ATTR_INFO) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if hdr.msgs[idx].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared attribute info message".into()));
|
||||
}
|
||||
let d = hdr.data(img, idx)?;
|
||||
let os = img.os as usize;
|
||||
let short = || Error::Unsupported("short attribute info message".into());
|
||||
if d.first() != Some(&0) {
|
||||
return Err(Error::Unsupported("attribute info message version".into()));
|
||||
}
|
||||
let flags = *d.get(1).ok_or_else(short)?;
|
||||
let track = flags & 0x01 != 0;
|
||||
let index = flags & 0x02 != 0;
|
||||
let mut p = 2;
|
||||
let mut max_crt = 0;
|
||||
if track {
|
||||
let b = d.get(p..p + 2).ok_or_else(short)?;
|
||||
max_crt = u16::from_le_bytes([b[0], b[1]]);
|
||||
p += 2;
|
||||
}
|
||||
let n = if index { 3 } else { 2 };
|
||||
if d.len() < p + n * os {
|
||||
return Err(short());
|
||||
}
|
||||
Ok(Some(Self {
|
||||
idx,
|
||||
track,
|
||||
index,
|
||||
max_crt,
|
||||
fheap: get_uint(&d[p..], img.os),
|
||||
name_bt2: get_uint(&d[p + os..], img.os),
|
||||
corder_bt2: if index {
|
||||
get_uint(&d[p + 2 * os..], img.os)
|
||||
} else {
|
||||
undef(img.os)
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
fn dense(&self, os: u8) -> bool {
|
||||
self.fheap != undef(os)
|
||||
}
|
||||
|
||||
/// Store the changeable fields back into the message.
|
||||
fn store(&self, img: &mut Image<'_>, hdr: &mut Header) -> Result<(), Error> {
|
||||
let os = img.os as usize;
|
||||
let mut p = 2;
|
||||
if self.track {
|
||||
hdr.patch(img, self.idx, p, &self.max_crt.to_le_bytes())?;
|
||||
p += 2;
|
||||
}
|
||||
let mut a = vec![0u8; os];
|
||||
for (k, v) in [self.fheap, self.name_bt2, self.corder_bt2]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.take(if self.index { 3 } else { 2 })
|
||||
{
|
||||
put_uint(&mut a, v, img.os);
|
||||
hdr.patch(img, self.idx, p + k * os, &a)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The next creation index (`H5O__attr_create`), or libhdf5's "none".
|
||||
fn next_crt(&mut self) -> Result<u16, Error> {
|
||||
if !self.track {
|
||||
return Ok(NO_CRT_IDX);
|
||||
}
|
||||
if self.max_crt == NO_CRT_IDX {
|
||||
return Err(Error::Unsupported(
|
||||
"object's attribute creation index is exhausted".into(),
|
||||
));
|
||||
}
|
||||
self.max_crt += 1;
|
||||
Ok(self.max_crt - 1)
|
||||
}
|
||||
}
|
||||
|
||||
/// The name bytes of an attribute message body (without the NUL).
|
||||
pub(super) fn attr_name(d: &[u8]) -> Result<&[u8], Error> {
|
||||
let bad = || Error::Unsupported("malformed attribute message".into());
|
||||
let (len, at) = match d.first() {
|
||||
Some(1) | Some(2) if d.len() >= 8 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8),
|
||||
Some(3) if d.len() >= 9 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9),
|
||||
_ => return Err(bad()),
|
||||
};
|
||||
let name = d.get(at..at + len).ok_or_else(bad)?;
|
||||
Ok(name.split(|&b| b == 0).next().unwrap_or(name))
|
||||
}
|
||||
|
||||
/// A new Attribute Info message for a version-2 header with flags
|
||||
/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order
|
||||
/// tracked / indexed as the header's flags say, the maximum creation index,
|
||||
/// and no dense storage (undefined fractal heap and B-tree addresses).
|
||||
fn attr_info_message(hdr_flags: u8, max_crt: u16, os: u8) -> Vec<u8> {
|
||||
let track = hdr_flags & 0x04 != 0;
|
||||
let index = hdr_flags & 0x08 != 0;
|
||||
let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)];
|
||||
if track {
|
||||
b.extend_from_slice(&max_crt.to_le_bytes());
|
||||
}
|
||||
let undef_addr = vec![0xffu8; os as usize];
|
||||
b.extend_from_slice(&undef_addr);
|
||||
b.extend_from_slice(&undef_addr);
|
||||
if index {
|
||||
b.extend_from_slice(&undef_addr);
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
/// A version-2 header's limit on compact attributes: stored when its flags
|
||||
/// say so, else libhdf5's default of 8.
|
||||
fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result<u16, Error> {
|
||||
if hdr.flags & 0x10 == 0 {
|
||||
return Ok(8);
|
||||
}
|
||||
let mut p = hdr.addr + 6;
|
||||
if hdr.flags & 0x20 != 0 {
|
||||
p += 16;
|
||||
}
|
||||
let b = img.read(p, 2)?;
|
||||
Ok(u16::from_le_bytes([b[0], b[1]]))
|
||||
}
|
||||
|
||||
/// A version-1 attribute message (what libhdf5 writes in a version-1 object
|
||||
/// header): name, datatype and dataspace each padded to 8 bytes, the
|
||||
/// dataspace as a version-1 dataspace message.
|
||||
fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec<u8> {
|
||||
let mut name = a.name.as_bytes().to_vec();
|
||||
name.push(0);
|
||||
let dt = a.datatype.serialize();
|
||||
let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0];
|
||||
if a.dataspace.space_type == DataspaceType::Simple {
|
||||
let mut b = vec![0u8; ls as usize];
|
||||
for &d in &a.dataspace.dimensions {
|
||||
put_uint(&mut b, d, ls);
|
||||
ds.extend_from_slice(&b);
|
||||
}
|
||||
if let Some(max) = &a.dataspace.max_dimensions {
|
||||
ds[2] = 0x01;
|
||||
for &d in max {
|
||||
put_uint(&mut b, d, ls);
|
||||
ds.extend_from_slice(&b);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ds[1] = 0;
|
||||
}
|
||||
let mut out = vec![1u8, 0];
|
||||
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(dt.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(ds.len() as u16).to_le_bytes());
|
||||
for part in [&name, &dt, &ds] {
|
||||
out.extend_from_slice(part);
|
||||
out.resize(out.len().next_multiple_of(8), 0);
|
||||
}
|
||||
out.extend_from_slice(&a.raw_data);
|
||||
out
|
||||
}
|
||||
|
||||
/// Dense storage opened for changes.
|
||||
struct Dense {
|
||||
heap: Heap,
|
||||
names: Bt2,
|
||||
order: Option<Bt2>,
|
||||
}
|
||||
|
||||
/// `H5_checksum_lookup3` of a name, as the name index keys it.
|
||||
fn name_hash(name: &[u8]) -> u32 {
|
||||
clawhdf5_format::checksum::jenkins_lookup3(name)
|
||||
}
|
||||
|
||||
/// Compare attribute `name` (hash `hash`) with a name-index record
|
||||
/// (`H5A__dense_btree2_name_compare`: the hash, then the stored name).
|
||||
fn cmp_name(
|
||||
heap: &Heap,
|
||||
img: &Image<'_>,
|
||||
hash: u32,
|
||||
name: &[u8],
|
||||
rec: &[u8],
|
||||
) -> Result<Ordering, Error> {
|
||||
let theirs = u32::from_le_bytes([rec[13], rec[14], rec[15], rec[16]]);
|
||||
match hash.cmp(&theirs) {
|
||||
Ordering::Equal => {
|
||||
if rec[ID_LEN] & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported(
|
||||
"shared attribute in dense storage".into(),
|
||||
));
|
||||
}
|
||||
let obj = heap.read(img, &rec[..ID_LEN])?;
|
||||
Ok(name.cmp(attr_name(&obj)?))
|
||||
}
|
||||
o => Ok(o),
|
||||
}
|
||||
}
|
||||
|
||||
fn corder_of(rec: &[u8]) -> u32 {
|
||||
u32::from_le_bytes([rec[9], rec[10], rec[11], rec[12]])
|
||||
}
|
||||
|
||||
impl Dense {
|
||||
fn open(img: &Image<'_>, ai: &AInfo) -> Result<Self, Error> {
|
||||
let heap = Heap::open(img, ai.fheap)?;
|
||||
let names = Bt2::open(img, ai.name_bt2)?;
|
||||
if names.tree_type() != NAME_BT2_TYPE || names.record_size() != ID_LEN + 9 {
|
||||
return Err(Error::Unsupported("attribute name index layout".into()));
|
||||
}
|
||||
let order = if ai.index {
|
||||
let t = Bt2::open(img, ai.corder_bt2)?;
|
||||
if t.tree_type() != CORDER_BT2_TYPE || t.record_size() != ID_LEN + 5 {
|
||||
return Err(Error::Unsupported(
|
||||
"attribute creation-order index layout".into(),
|
||||
));
|
||||
}
|
||||
Some(t)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(Self { heap, names, order })
|
||||
}
|
||||
|
||||
/// `H5A__dense_create`: heap, name index, [creation-order index].
|
||||
fn create(img: &mut Image<'_>, index: bool) -> Result<Self, Error> {
|
||||
let heap = Heap::create_attribute_heap(img)?;
|
||||
let names = Bt2::create(img, NAME_BT2_TYPE, ATTR_BT2_NODE, ID_LEN + 9, 100, 40)?;
|
||||
let order = if index {
|
||||
Some(Bt2::create(
|
||||
img,
|
||||
CORDER_BT2_TYPE,
|
||||
ATTR_BT2_NODE,
|
||||
ID_LEN + 5,
|
||||
100,
|
||||
40,
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(Self { heap, names, order })
|
||||
}
|
||||
|
||||
/// `H5A__dense_insert` of an encoded attribute message.
|
||||
fn insert(&mut self, img: &mut Image<'_>, body: &[u8], crt: u16) -> Result<(), Error> {
|
||||
let name = attr_name(body)?.to_vec();
|
||||
let id = self.heap.insert(img, body)?;
|
||||
if id.len() != ID_LEN {
|
||||
return Err(Error::Unsupported("attribute heap ID length".into()));
|
||||
}
|
||||
let hash = name_hash(&name);
|
||||
let mut rec = id.clone();
|
||||
rec.push(0);
|
||||
rec.extend_from_slice(&u32::from(crt).to_le_bytes());
|
||||
rec.extend_from_slice(&hash.to_le_bytes());
|
||||
let heap = &self.heap;
|
||||
self.names
|
||||
.insert(img, &mut |im, r| cmp_name(heap, im, hash, &name, r), &rec)?;
|
||||
if let Some(t) = &mut self.order {
|
||||
let key = u32::from(crt);
|
||||
t.insert(
|
||||
img,
|
||||
&mut |_, r| Ok(key.cmp(&corder_of(r))),
|
||||
&rec[..ID_LEN + 5],
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> {
|
||||
self.heap.finish(img)?;
|
||||
self.names.finish(img)?;
|
||||
if let Some(t) = &mut self.order {
|
||||
t.finish(img)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Set attribute `name` of the object at `path` to `value`.
|
||||
pub(super) fn set_attr(
|
||||
f: &File,
|
||||
img: &mut Image<'_>,
|
||||
path: &str,
|
||||
name: &str,
|
||||
value: &AttrValue,
|
||||
) -> Result<(), Error> {
|
||||
let addr = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?;
|
||||
let mut hdr = Header::load(img, addr)?;
|
||||
let mut msg = clawhdf5_format::type_builders::build_attr_message(name, value);
|
||||
check_plain(&msg.datatype)?;
|
||||
// libhdf5 encodes a simple dataspace with its maximum dimensions (the
|
||||
// current ones when none were given), so an attribute takes the same
|
||||
// space in a header or heap as when libhdf5 writes it.
|
||||
if msg.dataspace.space_type == DataspaceType::Simple && msg.dataspace.max_dimensions.is_none() {
|
||||
msg.dataspace.max_dimensions = Some(msg.dataspace.dimensions.clone());
|
||||
}
|
||||
// H5A__set_version: version 1 unless the name is not ASCII (then 3),
|
||||
// raised to the file's low bound — which is the earliest for a file
|
||||
// libhdf5 opens without a libver setting (h5py's `r+`).
|
||||
let body = if hdr.version == 1 || name.is_ascii() {
|
||||
encode_attr_v1(&msg, img.ls)
|
||||
} else {
|
||||
let mut b = msg.serialize_v3(img.ls);
|
||||
if !name.is_ascii() {
|
||||
b[8] = 1; // UTF-8 name
|
||||
}
|
||||
b
|
||||
};
|
||||
let mut ainfo = if hdr.version == 2 {
|
||||
AInfo::load(img, &hdr)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(ai) = ainfo.as_mut().filter(|a| a.dense(img.os)) {
|
||||
let mut ai = ai.clone();
|
||||
set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?;
|
||||
return hdr.finish(img);
|
||||
}
|
||||
|
||||
let mut existing = None;
|
||||
let mut count = 0usize;
|
||||
for i in 0..hdr.msgs.len() {
|
||||
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
|
||||
continue;
|
||||
}
|
||||
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared attribute message".into()));
|
||||
}
|
||||
count += 1;
|
||||
if attr_name(&hdr.data(img, i)?)? == name.as_bytes() {
|
||||
existing = Some(i);
|
||||
}
|
||||
}
|
||||
if let Some(i) = existing {
|
||||
hdr.delete(img, i)?;
|
||||
count -= 1;
|
||||
}
|
||||
if hdr.version == 1 {
|
||||
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?;
|
||||
return hdr.finish(img);
|
||||
}
|
||||
let tracked = hdr.flags & 0x04 != 0;
|
||||
// H5O__attr_create: a missing Attribute Info message starts from
|
||||
// nothing (and is added below, holding the new maximum creation index).
|
||||
let new_ainfo = ainfo.is_none();
|
||||
let mut ai = ainfo.take().unwrap_or(AInfo {
|
||||
idx: usize::MAX,
|
||||
track: tracked,
|
||||
index: hdr.flags & 0x08 != 0,
|
||||
max_crt: 0,
|
||||
fheap: undef(img.os),
|
||||
name_bt2: undef(img.os),
|
||||
corder_bt2: undef(img.os),
|
||||
});
|
||||
let max_compact = usize::from(max_compact_attrs(img, &hdr)?);
|
||||
if count == max_compact || body.len() >= MESG_MAX_SIZE {
|
||||
if new_ainfo {
|
||||
return Err(Error::Unsupported(
|
||||
"dense attribute storage for an object without an Attribute Info message".into(),
|
||||
));
|
||||
}
|
||||
to_dense(img, &mut hdr, &mut ai)?;
|
||||
set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?;
|
||||
return hdr.finish(img);
|
||||
}
|
||||
let crt = ai.next_crt()?;
|
||||
let corder = tracked.then_some(crt);
|
||||
if new_ainfo {
|
||||
// libhdf5 appends the Attribute Info message before the attribute
|
||||
// when free space holds both, else after it, so that a new
|
||||
// continuation chunk made for the attribute has room for it too.
|
||||
let a = attr_info_message(hdr.flags, ai.max_crt, img.os);
|
||||
let first = hdr.has_free(a.len() + hdr.hsize() + body.len());
|
||||
if first {
|
||||
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?;
|
||||
}
|
||||
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?;
|
||||
if !first {
|
||||
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?;
|
||||
}
|
||||
} else {
|
||||
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?;
|
||||
ai.store(img, &mut hdr)?;
|
||||
}
|
||||
hdr.finish(img)
|
||||
}
|
||||
|
||||
/// Move every compact attribute of the object into new dense storage, in
|
||||
/// header message order (`H5O__attr_to_dense_cb`), leaving free space where
|
||||
/// the messages were.
|
||||
fn to_dense(img: &mut Image<'_>, hdr: &mut Header, ai: &mut AInfo) -> Result<(), Error> {
|
||||
let mut dense = Dense::create(img, ai.index)?;
|
||||
for i in 0..hdr.msgs.len() {
|
||||
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
|
||||
continue;
|
||||
}
|
||||
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared attribute message".into()));
|
||||
}
|
||||
let body = hdr.data(img, i)?;
|
||||
let crt = if ai.track {
|
||||
hdr.msgs[i].corder.unwrap_or(0)
|
||||
} else {
|
||||
NO_CRT_IDX
|
||||
};
|
||||
dense.insert(img, &body, crt)?;
|
||||
hdr.delete(img, i)?;
|
||||
}
|
||||
dense.finish(img)?;
|
||||
ai.fheap = dense.heap.address();
|
||||
ai.name_bt2 = dense.names.address();
|
||||
if let Some(t) = &dense.order {
|
||||
ai.corder_bt2 = t.address();
|
||||
}
|
||||
ai.store(img, hdr)
|
||||
}
|
||||
|
||||
/// Set an attribute of an object whose attributes are in dense storage: an
|
||||
/// attribute of that name whose new encoding has the old one's size is
|
||||
/// rewritten in its heap object (`H5A__dense_write`); otherwise the old one
|
||||
/// is removed (`H5A__dense_remove`: name index, creation-order index, heap
|
||||
/// object) and the new one inserted with the next creation index.
|
||||
fn set_dense(
|
||||
img: &mut Image<'_>,
|
||||
hdr: &mut Header,
|
||||
ai: &mut AInfo,
|
||||
name: &[u8],
|
||||
body: &[u8],
|
||||
) -> Result<(), Error> {
|
||||
let mut dense = Dense::open(img, ai)?;
|
||||
let hash = name_hash(name);
|
||||
let found = {
|
||||
let heap = &dense.heap;
|
||||
dense
|
||||
.names
|
||||
.find(img, &mut |im, r| cmp_name(heap, im, hash, name, r))?
|
||||
};
|
||||
if let Some(rec) = found {
|
||||
if dense.heap.write_in_place(img, &rec[..ID_LEN], body)? {
|
||||
return dense.finish(img);
|
||||
}
|
||||
{
|
||||
let heap = &dense.heap;
|
||||
dense
|
||||
.names
|
||||
.remove(img, &mut |im, r| cmp_name(heap, im, hash, name, r))?;
|
||||
}
|
||||
if let Some(t) = &mut dense.order {
|
||||
let key = corder_of(&rec);
|
||||
t.remove(img, &mut |_, r| Ok(key.cmp(&corder_of(r))))?
|
||||
.ok_or_else(|| {
|
||||
Error::Unsupported("attribute missing from its creation-order index".into())
|
||||
})?;
|
||||
}
|
||||
dense.heap.remove(img, &rec[..ID_LEN])?;
|
||||
}
|
||||
let crt = ai.next_crt()?;
|
||||
dense.insert(img, body, crt)?;
|
||||
dense.finish(img)?;
|
||||
ai.store(img, hdr)
|
||||
}
|
||||
@@ -180,11 +180,6 @@ impl Bt2 {
|
||||
self.tree_type
|
||||
}
|
||||
|
||||
/// Records in the whole tree.
|
||||
pub(crate) fn len(&self) -> u64 {
|
||||
self.root.all
|
||||
}
|
||||
|
||||
/// Node geometry for depths `0..=self.depth` (`H5B2__hdr_init`,
|
||||
/// `H5B2__split_root`).
|
||||
fn compute_levels(&mut self, os: u8) -> Result<(), Error> {
|
||||
@@ -389,35 +384,6 @@ impl Bt2 {
|
||||
}
|
||||
}
|
||||
|
||||
/// Every record, in key order.
|
||||
pub(crate) fn records(&mut self, img: &Image<'_>) -> Result<Vec<Vec<u8>>, Error> {
|
||||
let mut out = Vec::new();
|
||||
if self.root.addr != undef(img.os) && self.root.nrec > 0 {
|
||||
self.walk(img, self.root, self.depth, &mut out)?;
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn walk(
|
||||
&mut self,
|
||||
img: &Image<'_>,
|
||||
p: Ptr,
|
||||
depth: u16,
|
||||
out: &mut Vec<Vec<u8>>,
|
||||
) -> Result<(), Error> {
|
||||
self.load(img, p, depth)?;
|
||||
let n = self.peek(p.addr).clone();
|
||||
for i in 0..=n.recs.len() {
|
||||
if depth > 0 {
|
||||
self.walk(img, n.ptrs[i], depth - 1, out)?;
|
||||
}
|
||||
if i < n.recs.len() {
|
||||
out.push(n.recs[i].clone());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Insert `rec`, or replace the record `cmp` matches (`H5B2_update`).
|
||||
pub(crate) fn update(
|
||||
&mut self,
|
||||
@@ -665,6 +631,7 @@ impl Bt2 {
|
||||
/// parent records between them) out over the children `lo..` of
|
||||
/// `parent`, with `counts[k]` records in the k-th; the records between
|
||||
/// them go back into the parent.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn relayout(
|
||||
&mut self,
|
||||
parent: u64,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+22
-168
@@ -15,10 +15,12 @@
|
||||
//! file) is dropped before the first write, so no `&[u8]` over the mapping
|
||||
//! is alive while the file changes (see `image`).
|
||||
|
||||
mod attrs;
|
||||
mod btree1;
|
||||
mod btree2;
|
||||
mod earray;
|
||||
mod farray;
|
||||
mod fheap;
|
||||
mod image;
|
||||
mod ohdr;
|
||||
mod select;
|
||||
@@ -27,7 +29,6 @@ use std::collections::{BTreeMap, HashMap};
|
||||
use std::fs::{OpenOptions, TryLockError};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use clawhdf5_format::attribute::AttributeMessage;
|
||||
use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks};
|
||||
use clawhdf5_format::data_layout::DataLayout;
|
||||
use clawhdf5_format::data_read::NativeElement;
|
||||
@@ -43,8 +44,8 @@ use btree1::{BTree1, Key};
|
||||
use btree2::Bt2;
|
||||
use earray::{Ea, EaParams, Elem};
|
||||
use farray::Fa;
|
||||
use image::{Image, get_uint, put_uint, undef};
|
||||
use ohdr::{Header, MSG_ATTRIBUTE};
|
||||
use image::{Image, put_uint, undef};
|
||||
use ohdr::Header;
|
||||
|
||||
const MSG_DATASPACE: u16 = 0x01;
|
||||
const MSG_LAYOUT: u16 = 0x08;
|
||||
@@ -918,101 +919,29 @@ impl FileEditor {
|
||||
|
||||
/// Set attribute `name` of the object at `path` (a group or dataset;
|
||||
/// `"/"` is the root group) to `value`, replacing an attribute of that
|
||||
/// name. The attribute goes into free space in the object header, or a
|
||||
/// new header continuation chunk at the end of the file.
|
||||
/// name, as libhdf5 creates attributes (`H5O__attr_create`).
|
||||
///
|
||||
/// [`Error::Unsupported`] for an object whose attributes are in dense
|
||||
/// storage (or would have to move there: more than the object's
|
||||
/// compact-attribute limit), one that tracks attribute creation order,
|
||||
/// or one with shared attribute messages.
|
||||
/// A compact attribute goes into free space in the object header, or a
|
||||
/// new header continuation chunk; an object that tracks creation order
|
||||
/// gives it the next creation index. When the object reaches its
|
||||
/// compact-attribute limit (or the attribute is too large for a header
|
||||
/// message) its attributes move to dense storage, and objects already
|
||||
/// using dense storage get the attribute there: a fractal heap object
|
||||
/// (in free heap space, a new heap block, or its own file space when
|
||||
/// larger than the heap's managed limit) indexed by name and, when the
|
||||
/// object indexes creation order, by creation index. An attribute
|
||||
/// replaced by one of the same encoded size is rewritten where it is.
|
||||
///
|
||||
/// [`Error::Unsupported`] for shared attribute messages, heaps this
|
||||
/// editor cannot extend the way libhdf5 would (child indirect blocks,
|
||||
/// skipped blocks, free space other than within direct blocks, freeing
|
||||
/// a whole block), and version-1 object headers asked for an attribute
|
||||
/// larger than a header message holds.
|
||||
pub fn set_attr(&mut self, path: &str, name: &str, value: &AttrValue) -> Result<(), Error> {
|
||||
if name.is_empty() {
|
||||
return Err(Error::InvalidArgument("empty attribute name".into()));
|
||||
}
|
||||
self.edit(|f, img| {
|
||||
let addr =
|
||||
clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?;
|
||||
let mut hdr = Header::load(img, addr)?;
|
||||
if hdr.version == 2 && hdr.flags & 0x04 != 0 {
|
||||
return Err(Error::Unsupported(
|
||||
"object tracks attribute creation order".into(),
|
||||
));
|
||||
}
|
||||
if let Some(i) = hdr.find(MSG_ATTR_INFO) {
|
||||
let d = hdr.data(img, i)?;
|
||||
// version(1) flags(1) [max creation index(2)] fractal heap
|
||||
// address, name index address, [order index address].
|
||||
let mut p = 2;
|
||||
if d.get(1).is_some_and(|f| f & 0x01 != 0) {
|
||||
p += 2;
|
||||
}
|
||||
let os = img.os as usize;
|
||||
if d.len() < p + os {
|
||||
return Err(Error::Unsupported("short attribute info message".into()));
|
||||
}
|
||||
if get_uint(&d[p..], img.os) != undef(img.os) {
|
||||
return Err(Error::Unsupported(
|
||||
"object with attributes in dense storage".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let mut existing = None;
|
||||
let mut count = 0usize;
|
||||
for i in 0..hdr.msgs.len() {
|
||||
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
|
||||
continue;
|
||||
}
|
||||
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared attribute message".into()));
|
||||
}
|
||||
count += 1;
|
||||
if attr_name(&hdr.data(img, i)?)? == name.as_bytes() {
|
||||
existing = Some(i);
|
||||
}
|
||||
}
|
||||
if existing.is_none() && hdr.version == 2 {
|
||||
let max_compact = max_compact_attrs(img, &hdr)?;
|
||||
if count + 1 > usize::from(max_compact) {
|
||||
return Err(Error::Unsupported(format!(
|
||||
"object already has {count} compact attributes (its limit is \
|
||||
{max_compact}); more need dense storage"
|
||||
)));
|
||||
}
|
||||
}
|
||||
let msg = clawhdf5_format::type_builders::build_attr_message(name, value);
|
||||
check_plain(&msg.datatype)?;
|
||||
let body = if hdr.version == 1 {
|
||||
encode_attr_v1(&msg, img.ls)
|
||||
} else {
|
||||
let mut b = msg.serialize_v3(img.ls);
|
||||
if !name.is_ascii() {
|
||||
b[8] = 1; // UTF-8 name
|
||||
}
|
||||
b
|
||||
};
|
||||
if let Some(i) = existing {
|
||||
hdr.delete(img, i)?;
|
||||
}
|
||||
// libhdf5 counts a version-2 header's attributes through its
|
||||
// Attribute Info message and reports none without one; like
|
||||
// H5O__attr_create, add it when missing: before the attribute
|
||||
// when free space holds both (libhdf5's order), else after it,
|
||||
// so that a new continuation chunk made for the attribute has
|
||||
// room for it too.
|
||||
let ainfo = (hdr.version == 2 && hdr.find(MSG_ATTR_INFO).is_none())
|
||||
.then(|| attr_info_message(hdr.flags, img.os));
|
||||
let ainfo_first = ainfo
|
||||
.as_ref()
|
||||
.is_some_and(|a| hdr.has_free(a.len() + hdr.hsize() + body.len()));
|
||||
if let Some(a) = ainfo.as_ref().filter(|_| ainfo_first) {
|
||||
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?;
|
||||
}
|
||||
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?;
|
||||
if let Some(a) = ainfo.as_ref().filter(|_| !ainfo_first) {
|
||||
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?;
|
||||
}
|
||||
hdr.finish(img)
|
||||
})
|
||||
self.edit(|f, img| attrs::set_attr(f, img, path, name, value))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1097,81 +1026,6 @@ fn set_superblock_eof(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The name bytes of an attribute message body (without the NUL).
|
||||
fn attr_name(d: &[u8]) -> Result<&[u8], Error> {
|
||||
let bad = || Error::Unsupported("malformed attribute message".into());
|
||||
let (len, at) = match d.first() {
|
||||
Some(1) | Some(2) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8),
|
||||
Some(3) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9),
|
||||
_ => return Err(bad()),
|
||||
};
|
||||
let name = d.get(at..at + len).ok_or_else(bad)?;
|
||||
Ok(name.split(|&b| b == 0).next().unwrap_or(name))
|
||||
}
|
||||
|
||||
/// A new Attribute Info message for a version-2 header with flags
|
||||
/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order
|
||||
/// tracked / indexed as the header's flags say, maximum creation index 0,
|
||||
/// and no dense storage (undefined fractal heap and B-tree addresses).
|
||||
fn attr_info_message(hdr_flags: u8, os: u8) -> Vec<u8> {
|
||||
let track = hdr_flags & 0x04 != 0;
|
||||
let index = hdr_flags & 0x08 != 0;
|
||||
let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)];
|
||||
if track {
|
||||
b.extend_from_slice(&0u16.to_le_bytes());
|
||||
}
|
||||
let undef_addr = vec![0xffu8; os as usize];
|
||||
b.extend_from_slice(&undef_addr);
|
||||
b.extend_from_slice(&undef_addr);
|
||||
if index {
|
||||
b.extend_from_slice(&undef_addr);
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
/// A version-2 header's limit on compact attributes: stored when its flags
|
||||
/// say so, else libhdf5's default of 8.
|
||||
fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result<u16, Error> {
|
||||
if hdr.flags & 0x10 == 0 {
|
||||
return Ok(8);
|
||||
}
|
||||
let mut p = hdr.addr + 6;
|
||||
if hdr.flags & 0x20 != 0 {
|
||||
p += 16;
|
||||
}
|
||||
let b = img.read(p, 2)?;
|
||||
Ok(u16::from_le_bytes([b[0], b[1]]))
|
||||
}
|
||||
|
||||
/// A version-1 attribute message (what libhdf5 writes in a version-1 object
|
||||
/// header): name, datatype and dataspace each padded to 8 bytes, the
|
||||
/// dataspace as a version-1 dataspace message.
|
||||
fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec<u8> {
|
||||
let mut name = a.name.as_bytes().to_vec();
|
||||
name.push(0);
|
||||
let dt = a.datatype.serialize();
|
||||
let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0];
|
||||
if a.dataspace.space_type == DataspaceType::Simple {
|
||||
for &d in &a.dataspace.dimensions {
|
||||
let mut b = vec![0u8; ls as usize];
|
||||
put_uint(&mut b, d, ls);
|
||||
ds.extend_from_slice(&b);
|
||||
}
|
||||
} else {
|
||||
ds[1] = 0;
|
||||
}
|
||||
let mut out = vec![1u8, 0];
|
||||
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(dt.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(ds.len() as u16).to_le_bytes());
|
||||
for part in [&name, &dt, &ds] {
|
||||
out.extend_from_slice(part);
|
||||
out.resize(out.len().next_multiple_of(8), 0);
|
||||
}
|
||||
out.extend_from_slice(&a.raw_data);
|
||||
out
|
||||
}
|
||||
|
||||
fn write_selection(
|
||||
f: &File,
|
||||
img: &mut Image<'_>,
|
||||
|
||||
@@ -59,7 +59,7 @@ pub(crate) struct Header {
|
||||
added: usize,
|
||||
}
|
||||
|
||||
const MAX_CHUNKS: usize = 1024;
|
||||
const MAX_CHUNKS: usize = 1 << 16;
|
||||
|
||||
fn corrupt(why: &'static str) -> Error {
|
||||
Error::Format(FormatError::InvalidObjectHeader(why))
|
||||
@@ -118,7 +118,11 @@ impl Header {
|
||||
});
|
||||
h.scan(img, 0, addr + 16, addr + 16 + size, &mut pending)?;
|
||||
}
|
||||
while let Some((caddr, clen)) = pending.pop() {
|
||||
// Continuation chunks in the order their messages are found, as
|
||||
// H5O_protect loads them (so messages keep libhdf5's order).
|
||||
let mut next = 0;
|
||||
while let Some(&(caddr, clen)) = pending.get(next) {
|
||||
next += 1;
|
||||
if h.chunks.len() >= MAX_CHUNKS {
|
||||
return Err(corrupt("too many object header chunks"));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user