osobhandClaude Opus 5.5
bce07e9cb9
Fixture written by libhdf5 2.2.0 (built from tag 2.2.0) through ctypes: every bit pattern of FP4 E2M1, FP6 E2M3/E3M2, FP8 E4M3/E5M2 and a bfloat16 LE/BE set, as datasets and attributes, with what H5Dread/H5Aread return into double and float and the conversion exceptions libhdf5 raises. clawhdf5 already decoded every value as libhdf5 does, including an all-ones exponent as inf/NaN in the OCP formats that have none (documented as a deliberate match in known-issues). - data_read: NaNs of non-native float layouts get libhdf5's bits (sign kept, every mantissa bit set) in f64 and f32. - h5rs dump/ls name these types as h5dump/h5ls 2.x do (H5T_FLOAT_F4E2M1, "FP4 E2M1 4-bit float", float4-e2m1 ...), checked against h5dump 2.2.0's output of the fixture. - Python bindings read them as h5py 3.16 does (float32 for bfloat16, float16 for the 1-byte formats, file byte order, same bytes as h5py); writing them in 'r+' is refused. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
219 lines
9.3 KiB
Python
219 lines
9.3 KiB
Python
"""Generate mx_floats_hdf5_2_2.h5 and mx_floats_hdf5_2_2.json.
|
|
|
|
The file holds one dataset and one root attribute per small-float datatype
|
|
predefined by libhdf5 2.x:
|
|
|
|
H5T_FLOAT_F4E2M1 all 16 bit patterns
|
|
H5T_FLOAT_F6E2M3 all 64 bit patterns
|
|
H5T_FLOAT_F6E3M2 all 64 bit patterns
|
|
H5T_FLOAT_F8E4M3 all 256 bit patterns
|
|
H5T_FLOAT_F8E5M2 all 256 bit patterns
|
|
H5T_FLOAT_BFLOAT16LE a representative set (zeros, subnormals, normals,
|
|
H5T_FLOAT_BFLOAT16BE max, +-inf, quiet/signalling/negative NaN)
|
|
|
|
plus `<type>_pad` datasets for the 4- and 6-bit types whose bytes also have
|
|
every unused high bit set (libhdf5 ignores bits outside the precision).
|
|
|
|
The raw bit patterns are written with the file type as the memory type (no
|
|
conversion). The JSON is the reference: for every element, the bits of the
|
|
f64 and f32 values that libhdf5 itself returns from H5Dread/H5Aread into
|
|
H5T_NATIVE_DOUBLE/H5T_NATIVE_FLOAT, and each conversion exception libhdf5
|
|
raised through H5Pset_type_conv_cb (the callback returns UNHANDLED, so the
|
|
values are libhdf5's defaults).
|
|
|
|
It talks to libhdf5 2.2.0 through ctypes only (no h5py: h5py 3.16 bundles
|
|
libhdf5 2.0.0, which predates these types). libhdf5 was built from source:
|
|
|
|
git clone --depth 1 --branch 2.2.0 https://github.com/HDFGroup/hdf5 \
|
|
~/.cache/hdf5-2.2.0-src # commit 49df1b4e5ca4108d56e80f06321a749da1535268
|
|
cmake -S ~/.cache/hdf5-2.2.0-src -B ~/.cache/hdf5-2.2.0-build \
|
|
-DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$HOME/.cache/hdf5-2.2.0 \
|
|
-DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=OFF -DHDF5_ENABLE_ZLIB_SUPPORT=ON \
|
|
-DHDF5_BUILD_FORTRAN=OFF -DHDF5_BUILD_JAVA=OFF -DHDF5_BUILD_CPP_LIB=OFF \
|
|
-DBUILD_TESTING=OFF -DHDF5_BUILD_EXAMPLES=OFF -DHDF5_BUILD_TOOLS=ON \
|
|
-DHDF5_BUILD_HL_LIB=OFF
|
|
cmake --build ~/.cache/hdf5-2.2.0-build -j8
|
|
cmake --install ~/.cache/hdf5-2.2.0-build
|
|
|
|
Re-run only if the fixture ever needs regenerating (generated 2026-09-28):
|
|
|
|
python gen_mx_floats.py ~/.cache/hdf5-2.2.0/lib/libhdf5.so mx_floats_hdf5_2_2
|
|
|
|
(h5dump from the same build writes mx_floats_hdf5_2_2.ddl:
|
|
`h5dump mx_floats_hdf5_2_2.h5 > mx_floats_hdf5_2_2.ddl`.)
|
|
"""
|
|
import ctypes
|
|
import json
|
|
import struct
|
|
import sys
|
|
|
|
lib = ctypes.CDLL(sys.argv[1])
|
|
out = sys.argv[2]
|
|
hid = ctypes.c_int64
|
|
herr = ctypes.c_int
|
|
assert lib.H5open() >= 0
|
|
|
|
majnum, minnum, relnum = ctypes.c_uint(), ctypes.c_uint(), ctypes.c_uint()
|
|
lib.H5get_libversion(ctypes.byref(majnum), ctypes.byref(minnum), ctypes.byref(relnum))
|
|
version = f"{majnum.value}.{minnum.value}.{relnum.value}"
|
|
assert version == "2.2.0", version
|
|
|
|
|
|
def g(name):
|
|
return hid.in_dll(lib, name + "_g").value
|
|
|
|
|
|
for fn, res, args in [
|
|
("H5Fcreate", hid, [ctypes.c_char_p, ctypes.c_uint, hid, hid]),
|
|
("H5Fclose", herr, [hid]),
|
|
("H5Fopen", hid, [ctypes.c_char_p, ctypes.c_uint, hid]),
|
|
("H5Screate_simple", hid, [ctypes.c_int, ctypes.POINTER(ctypes.c_uint64), ctypes.c_void_p]),
|
|
("H5Sclose", herr, [hid]),
|
|
("H5Dcreate2", hid, [hid, ctypes.c_char_p, hid, hid, hid, hid, hid]),
|
|
("H5Dopen2", hid, [hid, ctypes.c_char_p, hid]),
|
|
("H5Dwrite", herr, [hid, hid, hid, hid, hid, ctypes.c_void_p]),
|
|
("H5Dread", herr, [hid, hid, hid, hid, hid, ctypes.c_void_p]),
|
|
("H5Dclose", herr, [hid]),
|
|
("H5Acreate2", hid, [hid, ctypes.c_char_p, hid, hid, hid, hid]),
|
|
("H5Aopen", hid, [hid, ctypes.c_char_p, hid]),
|
|
("H5Awrite", herr, [hid, hid, ctypes.c_void_p]),
|
|
("H5Aread", herr, [hid, hid, ctypes.c_void_p]),
|
|
("H5Aclose", herr, [hid]),
|
|
("H5Pcreate", hid, [hid]),
|
|
("H5Pclose", herr, [hid]),
|
|
("H5Pset_type_conv_cb", herr, [hid, ctypes.c_void_p, ctypes.c_void_p]),
|
|
("H5Tget_fields", herr, [hid] + [ctypes.POINTER(ctypes.c_size_t)] * 5),
|
|
("H5Tget_ebias", ctypes.c_size_t, [hid]),
|
|
("H5Tget_precision", ctypes.c_size_t, [hid]),
|
|
("H5Tget_offset", ctypes.c_int, [hid]),
|
|
("H5Tget_size", ctypes.c_size_t, [hid]),
|
|
("H5Tget_order", ctypes.c_int, [hid]),
|
|
]:
|
|
getattr(lib, fn).restype = res
|
|
getattr(lib, fn).argtypes = args
|
|
|
|
H5F_ACC_TRUNC, H5F_ACC_RDONLY = 0x0002, 0x0000
|
|
H5P_DEFAULT, H5S_ALL = 0, 0
|
|
|
|
NATIVE_DOUBLE = g("H5T_NATIVE_DOUBLE")
|
|
NATIVE_FLOAT = g("H5T_NATIVE_FLOAT")
|
|
BF16 = [0x0000, 0x8000, 0x3F80, 0xBF80, 0x3FC0, 0xC010, 0x4049, 0x3F81,
|
|
0x0001, 0x8001, 0x007F, 0x0080, 0x7F7F, 0xFF7F, 0x7F80, 0xFF80,
|
|
0x7FC0, 0x7F81, 0xFFC1, 0x7FFF]
|
|
|
|
# name -> (libhdf5 global, raw element bytes)
|
|
TYPES = {
|
|
"f4e2m1": ("H5T_FLOAT_F4E2M1", [bytes([b]) for b in range(16)]),
|
|
"f4e2m1_pad": ("H5T_FLOAT_F4E2M1", [bytes([0xF0 | b]) for b in range(16)]),
|
|
"f6e2m3": ("H5T_FLOAT_F6E2M3", [bytes([b]) for b in range(64)]),
|
|
"f6e2m3_pad": ("H5T_FLOAT_F6E2M3", [bytes([0xC0 | b]) for b in range(64)]),
|
|
"f6e3m2": ("H5T_FLOAT_F6E3M2", [bytes([b]) for b in range(64)]),
|
|
"f6e3m2_pad": ("H5T_FLOAT_F6E3M2", [bytes([0xC0 | b]) for b in range(64)]),
|
|
"f8e4m3": ("H5T_FLOAT_F8E4M3", [bytes([b]) for b in range(256)]),
|
|
"f8e5m2": ("H5T_FLOAT_F8E5M2", [bytes([b]) for b in range(256)]),
|
|
"bf16le": ("H5T_FLOAT_BFLOAT16LE", [struct.pack("<H", v) for v in BF16]),
|
|
"bf16be": ("H5T_FLOAT_BFLOAT16BE", [struct.pack(">H", v) for v in BF16]),
|
|
}
|
|
EXCEPT_NAMES = ["RANGE_HI", "RANGE_LOW", "PRECISION", "TRUNCATE", "PINF", "NINF", "NAN"]
|
|
|
|
fid = lib.H5Fcreate((out + ".h5").encode(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)
|
|
assert fid >= 0
|
|
for name, (tname, raw) in TYPES.items():
|
|
t = g(tname)
|
|
buf = ctypes.create_string_buffer(b"".join(raw), len(raw) * len(raw[0]))
|
|
sid = lib.H5Screate_simple(1, (ctypes.c_uint64 * 1)(len(raw)), None)
|
|
did = lib.H5Dcreate2(fid, name.encode(), t, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)
|
|
assert did >= 0 and lib.H5Dwrite(did, t, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) >= 0
|
|
lib.H5Dclose(did)
|
|
if not name.endswith("_pad"):
|
|
aid = lib.H5Acreate2(fid, name.encode(), t, sid, H5P_DEFAULT, H5P_DEFAULT)
|
|
assert aid >= 0 and lib.H5Awrite(aid, t, buf) >= 0
|
|
lib.H5Aclose(aid)
|
|
lib.H5Sclose(sid)
|
|
assert lib.H5Fclose(fid) >= 0
|
|
|
|
# --- reference: what libhdf5 converts each element to --------------------
|
|
CB = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, hid, hid, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p)
|
|
events = []
|
|
|
|
|
|
@CB
|
|
def on_except(kind, src_id, dst_id, src_buf, dst_buf, user):
|
|
events.append((EXCEPT_NAMES[kind], ctypes.string_at(src_buf, 1 if lib.H5Tget_size(src_id) == 1 else 2).hex()))
|
|
return 0 # H5T_CONV_UNHANDLED: let libhdf5 apply its default
|
|
|
|
|
|
def f64_text(b):
|
|
"""The f64 as Python's round-tripping repr ("0.125", "inf", "-inf"), or
|
|
"nan:<bits>" for a NaN, whose sign and payload are kept exactly."""
|
|
(bits,) = struct.unpack("<Q", b)
|
|
(v,) = struct.unpack("<d", b)
|
|
return "nan:%016x" % bits if v != v else repr(v)
|
|
|
|
|
|
def read_all(obj, is_dataset, mem, width, n):
|
|
buf = ctypes.create_string_buffer(n * width)
|
|
events.clear()
|
|
if is_dataset:
|
|
assert lib.H5Dread(obj, mem, H5S_ALL, H5S_ALL, dxpl, buf) >= 0
|
|
else:
|
|
# H5Aread takes no transfer property list, so no exception callback.
|
|
assert lib.H5Aread(obj, mem, buf) >= 0
|
|
return [buf.raw[i * width:(i + 1) * width] for i in range(n)], sorted(events)
|
|
|
|
|
|
fid = lib.H5Fopen((out + ".h5").encode(), H5F_ACC_RDONLY, H5P_DEFAULT)
|
|
dxpl = lib.H5Pcreate(g("H5P_CLS_DATASET_XFER_ID"))
|
|
assert lib.H5Pset_type_conv_cb(dxpl, ctypes.cast(on_except, ctypes.c_void_p), None) >= 0
|
|
ref = {
|
|
"libhdf5": version,
|
|
"generator": "gen_mx_floats.py",
|
|
"note": "f64: what H5Dread into H5T_NATIVE_DOUBLE returns per element; "
|
|
"H5Aread (attribute of the same name) and H5T_NATIVE_FLOAT return the same values "
|
|
"(f32 NaN bits in f32_nan); exceptions: H5Pset_type_conv_cb events by source bytes",
|
|
"objects": {},
|
|
}
|
|
for name, (tname, raw) in TYPES.items():
|
|
t = g(tname)
|
|
sizes = [ctypes.c_size_t() for _ in range(5)]
|
|
lib.H5Tget_fields(t, *[ctypes.byref(s) for s in sizes])
|
|
n = len(raw)
|
|
did = lib.H5Dopen2(fid, name.encode(), H5P_DEFAULT)
|
|
d64, exc64 = read_all(did, True, NATIVE_DOUBLE, 8, n)
|
|
d32, exc32 = read_all(did, True, NATIVE_FLOAT, 4, n)
|
|
lib.H5Dclose(did)
|
|
assert exc32 == exc64, name
|
|
for a, b in zip(d64, d32):
|
|
(x,) = struct.unpack("<d", a)
|
|
(y,) = struct.unpack("<f", b)
|
|
assert (x != x and y != y) or x == y, (name, x, y)
|
|
f32_nan = sorted({"%08x" % struct.unpack("<I", b)[0] for b in d32 if struct.unpack("<f", b)[0] != struct.unpack("<f", b)[0]})
|
|
if not name.endswith("_pad"):
|
|
aid = lib.H5Aopen(fid, name.encode(), H5P_DEFAULT)
|
|
assert read_all(aid, False, NATIVE_DOUBLE, 8, n)[0] == d64, name
|
|
assert read_all(aid, False, NATIVE_FLOAT, 4, n)[0] == d32, name
|
|
lib.H5Aclose(aid)
|
|
exceptions = {}
|
|
for kind, src in exc64:
|
|
exceptions.setdefault(kind, []).append(src)
|
|
ref["objects"][name] = {
|
|
"type": tname,
|
|
"size": lib.H5Tget_size(t),
|
|
"precision": lib.H5Tget_precision(t),
|
|
"offset": lib.H5Tget_offset(t),
|
|
"order": ["LE", "BE"][lib.H5Tget_order(t)],
|
|
"fields": [s.value for s in sizes], # spos, epos, esize, mpos, msize
|
|
"ebias": lib.H5Tget_ebias(t),
|
|
"attribute": not name.endswith("_pad"),
|
|
"raw": "".join(r.hex() for r in raw),
|
|
"f64": [f64_text(b) for b in d64],
|
|
"f32_nan": f32_nan,
|
|
"exceptions": exceptions,
|
|
}
|
|
lib.H5Pclose(dxpl)
|
|
lib.H5Fclose(fid)
|
|
|
|
with open(out + ".json", "w") as fh:
|
|
json.dump(ref, fh, separators=(",", ":"))
|
|
fh.write("\n")
|