fix(format): write fill times with libhdf5's codes; add fill values
FillTime::to_byte had the fill-time field rotated against libhdf5 (H5D_FILL_TIME_ALLOC = 0, NEVER = 1, IFSET = 2): Never was written as ALLOC, Alloc as IFSET and IfSet as NEVER, as h5py reported. The flags byte is now late allocation plus the right code, and FillTime::from_byte decodes it. The default becomes IfSet, which is libhdf5's default and exactly the byte (0x0a) every dataset was already written with, so default output does not change; `Alloc` was documented as the C library's default but never was. DatasetCreateProps follows. DatasetBuilder::with_fill_value sets a user-defined fill value (one element's stored bytes, checked against the datatype size), written as a defined value in the fill value message. h5py reports it, and extending the dataset in h5py fills the new elements with it. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -19,7 +19,7 @@ use crate::metadata_index::{DatasetMetadata, MetadataBlock, MetadataIndex};
|
||||
use crate::object_header_writer::ObjectHeaderWriter;
|
||||
use crate::superblock::Superblock;
|
||||
use crate::type_builders::{
|
||||
DatasetBuilder, FillTime, FinishedGroup, GroupBuilder, build_attr_message,
|
||||
DatasetBuilder, FinishedGroup, GroupBuilder, build_attr_message, fill_value_message,
|
||||
};
|
||||
|
||||
// Re-export public types that moved to type_builders for API compatibility.
|
||||
@@ -80,12 +80,12 @@ pub(crate) fn build_chunked_dataset_oh(
|
||||
pipeline_message: Option<&[u8]>,
|
||||
attrs: &[AttributeMessage],
|
||||
dense_blob: Option<&DenseAttrBlob>,
|
||||
fill_time: FillTime,
|
||||
fill_message: &[u8],
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let mut w = ObjectHeaderWriter::new();
|
||||
w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
|
||||
w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
|
||||
w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01);
|
||||
w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
|
||||
w.add_message(MessageType::DataLayout, layout_message.to_vec());
|
||||
if let Some(pm) = pipeline_message {
|
||||
w.add_message(MessageType::FilterPipeline, pm.to_vec());
|
||||
@@ -107,12 +107,12 @@ pub(crate) fn build_dataset_oh(
|
||||
data_size: u64,
|
||||
attrs: &[AttributeMessage],
|
||||
dense_blob: Option<&DenseAttrBlob>,
|
||||
fill_time: FillTime,
|
||||
fill_message: &[u8],
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let mut w = ObjectHeaderWriter::new();
|
||||
w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
|
||||
w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
|
||||
w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01);
|
||||
w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
|
||||
let mut dl = Vec::new();
|
||||
dl.push(4); // version
|
||||
dl.push(1); // class = contiguous
|
||||
@@ -142,12 +142,12 @@ pub(crate) fn build_compact_dataset_oh(
|
||||
data: &[u8],
|
||||
attrs: &[AttributeMessage],
|
||||
dense_blob: Option<&DenseAttrBlob>,
|
||||
fill_time: FillTime,
|
||||
fill_message: &[u8],
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let mut w = ObjectHeaderWriter::new();
|
||||
w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
|
||||
w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
|
||||
w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01);
|
||||
w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
|
||||
// Compact layout message: version=4, class=0, u16 size, inline data
|
||||
let mut dl = Vec::new();
|
||||
dl.push(4); // version
|
||||
@@ -932,12 +932,12 @@ pub(crate) fn build_vds_dataset_oh(
|
||||
global_heap_addr: u64,
|
||||
attrs: &[AttributeMessage],
|
||||
dense_blob: Option<&DenseAttrBlob>,
|
||||
fill_time: FillTime,
|
||||
fill_message: &[u8],
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let mut w = ObjectHeaderWriter::new();
|
||||
w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
|
||||
w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
|
||||
w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01);
|
||||
w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
|
||||
// VDS layout message: version=4, class=3, global_heap_address(8), global_heap_index=1(4)
|
||||
let mut dl = Vec::new();
|
||||
dl.push(4u8); // version
|
||||
@@ -1070,7 +1070,8 @@ impl FileWriter {
|
||||
attrs: Vec<AttributeMessage>,
|
||||
chunk_options: ChunkOptions,
|
||||
maxshape: Option<Vec<u64>>,
|
||||
fill_time: FillTime,
|
||||
/// Serialized Fill Value message.
|
||||
fill_message: Vec<u8>,
|
||||
compact: bool,
|
||||
alignment: usize,
|
||||
/// VDS source mappings (set for Virtual datasets).
|
||||
@@ -1120,6 +1121,7 @@ impl FileWriter {
|
||||
};
|
||||
attrs.extend(p.build_attrs(&raw));
|
||||
}
|
||||
let fill_message = fill_value_message(db.fill_time, db.fill_value.as_deref(), &dt)?;
|
||||
Ok(DsFlat {
|
||||
name: db.name,
|
||||
dt,
|
||||
@@ -1128,7 +1130,7 @@ impl FileWriter {
|
||||
attrs,
|
||||
chunk_options: db.chunk_options,
|
||||
maxshape: db.maxshape,
|
||||
fill_time: db.fill_time,
|
||||
fill_message,
|
||||
compact: db.compact,
|
||||
alignment: db.alignment,
|
||||
virtual_sources: db.virtual_sources,
|
||||
@@ -1274,7 +1276,7 @@ impl FileWriter {
|
||||
0, // dummy address
|
||||
&d.attrs,
|
||||
dense_blob.as_ref(),
|
||||
d.fill_time,
|
||||
&d.fill_message,
|
||||
)?;
|
||||
// Global heap blob size is address-independent; compute it now
|
||||
// so pass 2 can place it correctly.
|
||||
@@ -1318,7 +1320,7 @@ impl FileWriter {
|
||||
result.pipeline_message.as_deref(),
|
||||
&d.attrs,
|
||||
dense_blob.as_ref(),
|
||||
d.fill_time,
|
||||
&d.fill_message,
|
||||
)?;
|
||||
dummy_blobs.push(DataBlob {
|
||||
data: result.data_bytes,
|
||||
@@ -1337,7 +1339,7 @@ impl FileWriter {
|
||||
&d.raw,
|
||||
&d.attrs,
|
||||
dense_blob.as_ref(),
|
||||
d.fill_time,
|
||||
&d.fill_message,
|
||||
)?;
|
||||
dummy_blobs.push(DataBlob {
|
||||
data: vec![],
|
||||
@@ -1357,7 +1359,7 @@ impl FileWriter {
|
||||
d.raw.len() as u64,
|
||||
&d.attrs,
|
||||
dense_blob.as_ref(),
|
||||
d.fill_time,
|
||||
&d.fill_message,
|
||||
)?;
|
||||
dummy_blobs.push(DataBlob {
|
||||
data: d.raw.clone(),
|
||||
@@ -1466,7 +1468,7 @@ impl FileWriter {
|
||||
heap_addr,
|
||||
&d.attrs,
|
||||
ds_dense_blobs[i].as_ref(),
|
||||
d.fill_time,
|
||||
&d.fill_message,
|
||||
)?;
|
||||
ds_blobs2.push(DataBlob {
|
||||
data: gcol_bytes.clone(),
|
||||
@@ -1493,7 +1495,7 @@ impl FileWriter {
|
||||
result.pipeline_message.as_deref(),
|
||||
&d.attrs,
|
||||
ds_dense_blobs[i].as_ref(),
|
||||
d.fill_time,
|
||||
&d.fill_message,
|
||||
)?;
|
||||
ds_blobs2.push(DataBlob {
|
||||
data: result.data_bytes,
|
||||
@@ -1508,7 +1510,7 @@ impl FileWriter {
|
||||
&d.raw,
|
||||
&d.attrs,
|
||||
ds_dense_blobs[i].as_ref(),
|
||||
d.fill_time,
|
||||
&d.fill_message,
|
||||
)?;
|
||||
ds_blobs2.push(DataBlob {
|
||||
data: vec![],
|
||||
@@ -1533,7 +1535,7 @@ impl FileWriter {
|
||||
d.raw.len() as u64,
|
||||
&d.attrs,
|
||||
ds_dense_blobs[i].as_ref(),
|
||||
d.fill_time,
|
||||
&d.fill_message,
|
||||
)?;
|
||||
let mut data = vec![0u8; padding];
|
||||
data.extend_from_slice(&d.raw);
|
||||
|
||||
Reference in New Issue
Block a user