Range reads M0/M1 (indexed lookups, Storage trait), ZFP, in-place editing #17
+4
-2
@@ -40,8 +40,10 @@
|
|||||||
(`Error::Unsupported`, file untouched) when any part is not supported:
|
(`Error::Unsupported`, file untouched) when any part is not supported:
|
||||||
new chunks in a version-2 B-tree index (two or more unlimited
|
new chunks in a version-2 B-tree index (two or more unlimited
|
||||||
dimensions) or an implicit index, shrinking, variable-length and
|
dimensions) or an implicit index, shrinking, variable-length and
|
||||||
reference data, attributes in dense storage, past an object's compact
|
reference data, chunks through a filter this build cannot encode
|
||||||
limit or with tracked creation order, files with a metadata cache
|
(scale-offset, N-Bit, SZIP), attributes in dense storage, past an
|
||||||
|
object's compact limit or with tracked creation order, files with a
|
||||||
|
metadata cache
|
||||||
image, paged or persistent free space, or marked open by another
|
image, paged or persistent free space, or marked open by another
|
||||||
writer. New error variants `Error::Unsupported`,
|
writer. New error variants `Error::Unsupported`,
|
||||||
`Error::InvalidArgument`, `Error::Locked`, and `clawhdf5::Error` is now
|
`Error::InvalidArgument`, `Error::Locked`, and `clawhdf5::Error` is now
|
||||||
|
|||||||
@@ -1402,3 +1402,36 @@ fn attribute_count_in_version_2_headers() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A dataset whose filters this build cannot encode (scale-offset)
|
||||||
|
/// is `Error::Unsupported`, as the editor documents, not a format error,
|
||||||
|
/// and the file is left byte for byte as it was.
|
||||||
|
#[test]
|
||||||
|
fn unencodable_filters_are_unsupported() {
|
||||||
|
if !tools_ok() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let dir = tmpdir();
|
||||||
|
let path = dir.path().join("unencodable.h5");
|
||||||
|
py(&format!(
|
||||||
|
"import h5py, numpy as np\n\
|
||||||
|
with h5py.File({p:?}, 'w') as f:\n\
|
||||||
|
\x20 f.create_dataset('so', data=np.arange(16, dtype='<i4'), chunks=(8,), maxshape=(None,), scaleoffset=0)\n\
|
||||||
|
\x20 f.create_dataset('so_new', shape=(16,), dtype='<i4', chunks=(8,), maxshape=(None,), scaleoffset=0)\n\
|
||||||
|
\x20 f.create_dataset('sof', data=np.arange(16, dtype='<f8'), chunks=(8,), scaleoffset=2)\n",
|
||||||
|
p = path.to_str().unwrap()
|
||||||
|
));
|
||||||
|
let before = std::fs::read(&path).unwrap();
|
||||||
|
let mut ed = FileEditor::open(&path).unwrap();
|
||||||
|
for (ds, es) in [("so", 4), ("so_new", 4), ("sof", 8)] {
|
||||||
|
match ed.write_all(ds, &vec![0u8; 16 * es]) {
|
||||||
|
Err(Error::Unsupported(msg)) => assert!(msg.contains("filter"), "{ds}: {msg}"),
|
||||||
|
other => panic!("{ds}: {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(ed);
|
||||||
|
assert!(
|
||||||
|
std::fs::read(&path).unwrap() == before,
|
||||||
|
"a refused edit changed the file"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -622,7 +622,7 @@ impl FileEditor {
|
|||||||
check_editable(&f)?;
|
check_editable(&f)?;
|
||||||
let sb = f.superblock().clone();
|
let sb = f.superblock().clone();
|
||||||
let mut img = Image::new(f.as_bytes(), sb.offset_size, sb.length_size);
|
let mut img = Image::new(f.as_bytes(), sb.offset_size, sb.length_size);
|
||||||
let r = op(&f, &mut img)?;
|
let r = op(&f, &mut img).map_err(unsupported_filter)?;
|
||||||
if img.is_dirty() {
|
if img.is_dirty() {
|
||||||
if img.eoa() != img.old_eoa() {
|
if img.eoa() != img.old_eoa() {
|
||||||
set_superblock_eof(&mut img, &sb)?;
|
set_superblock_eof(&mut img, &sb)?;
|
||||||
@@ -840,6 +840,20 @@ impl FileEditor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A filter this build cannot run (scale-offset, N-Bit and SZIP have no
|
||||||
|
/// encoder; a plugin filter may be missing) is something the editor does not
|
||||||
|
/// support, not a malformed file.
|
||||||
|
fn unsupported_filter(e: Error) -> Error {
|
||||||
|
match e {
|
||||||
|
Error::Format(clawhdf5_format::error::FormatError::UnsupportedFilter(id)) => {
|
||||||
|
Error::Unsupported(format!(
|
||||||
|
"datasets with filter {id}, which this build cannot run"
|
||||||
|
))
|
||||||
|
}
|
||||||
|
e => e,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// libhdf5's checks before it opens a file for writing, and what this
|
/// libhdf5's checks before it opens a file for writing, and what this
|
||||||
/// editor cannot keep consistent.
|
/// editor cannot keep consistent.
|
||||||
fn check_editable(f: &File) -> Result<(), Error> {
|
fn check_editable(f: &File) -> Result<(), Error> {
|
||||||
|
|||||||
@@ -13,11 +13,16 @@ deleting it.
|
|||||||
with `Error::Unsupported` and without writing anything:
|
with `Error::Unsupported` and without writing anything:
|
||||||
- new, moved or resized chunks in a **version-2 B-tree** chunk index (what
|
- new, moved or resized chunks in a **version-2 B-tree** chunk index (what
|
||||||
libhdf5 uses for two or more unlimited dimensions) — existing unfiltered
|
libhdf5 uses for two or more unlimited dimensions) — existing unfiltered
|
||||||
chunks, and filtered ones that re-encode to the same size, are
|
chunks, and filtered ones that re-encode to the same size and filter
|
||||||
|
mask, are
|
||||||
overwritten in place; `resize` works — and new chunks in an **implicit**
|
overwritten in place; `resize` works — and new chunks in an **implicit**
|
||||||
index (it has all of its chunks from the start);
|
index (it has all of its chunks from the start);
|
||||||
- **shrinking** a dataset;
|
- **shrinking** a dataset;
|
||||||
- variable-length and reference data;
|
- variable-length and reference data;
|
||||||
|
- chunks through a filter this build cannot encode (scale-offset, N-Bit,
|
||||||
|
SZIP, or a plugin filter it lacks), even an optional one: libhdf5 skips
|
||||||
|
an optional filter only when its own build lacks it, which none does for
|
||||||
|
these;
|
||||||
- attributes of an object in **dense storage**, past its compact limit (8
|
- attributes of an object in **dense storage**, past its compact limit (8
|
||||||
by default) or with tracked **creation order**;
|
by default) or with tracked **creation order**;
|
||||||
- partial edge chunks stored unfiltered (`H5Pset_chunk_opts`), external
|
- partial edge chunks stored unfiltered (`H5Pset_chunk_opts`), external
|
||||||
|
|||||||
Reference in New Issue
Block a user