clawhdf5/crates/clawhdf5-remote/src/lib.rs

//! Read HDF5 files where they are — on an HTTP(S) server or in an object
//! store — without downloading them first.
//!
//! This is milestone M3 of `docs/design/range-reads.md`: remote backends for
//! [`clawhdf5::File::open_storage`], each read through a mandatory
//! [`BlockCache`].
//!
//! ```no_run
//! let file = clawhdf5_remote::open_url("http://127.0.0.1:8000/data.h5")?;
//! let temperature = file.dataset("/grid/temperature")?.read_f64()?;
//! # Ok::<(), Box<dyn std::error::Error>>(())
//! ```
//!
//! - [`open_url`] / [`open_url_with`]: `http://` (default feature `http`),
//!   `https://` (feature `https`), `s3://`, `gs://`, `az://` (features `s3`,
//!   `gcs`, `azure`) → a [`clawhdf5::File`] with the whole read API.
//! - [`storage_for_url`] gives the cached storage itself, to open with
//!   [`clawhdf5::File::open_storage`] and to read its [`CacheStats`].
//! - [`download`] reads a whole remote file into memory, up to a limit.
//! - [`HttpStorage`] (range `GET`s, pinned by ETag/Last-Modified, retried
//!   with backoff), [`ObjectStoreStorage`] (any `object_store` store,
//!   feature `object-store`), and [`BlockCache`] over any
//!   [`Storage`](clawhdf5_format::storage::Storage).
//!
//! What costs what: opening costs one request (it also fetches the first
//! block, 1 MiB by default); listing a file whose metadata sits in its
//! first blocks costs nothing more; reading a chunked dataset costs one
//! parallel batch of requests for the blocks holding its chunk index, then
//! one for its chunks. The zero-copy methods of `clawhdf5`
//! (`read_raw_ref`, `read_*_zerocopy`, `File::as_bytes`) need the file in
//! memory and are errors (`as_bytes` a panic) on a remote file.

#![warn(missing_docs)]

pub mod cache;
pub mod error;
#[cfg(feature = "http")]
pub mod http;
#[cfg(feature = "object-store")]
pub mod object;

use std::sync::Arc;

use clawhdf5::File;
use clawhdf5_format::storage::Storage;

pub use cache::{BlockCache, CacheConfig, CacheStats};
pub use error::{Error, RemoteError, redact_url};
#[cfg(feature = "http")]
pub use http::{HttpOptions, HttpStats, HttpStorage};
#[cfg(feature = "object-store")]
pub use object::ObjectStoreStorage;
#[cfg(feature = "object-store")]
pub use object_store;

/// A backend a [`BlockCache`] can read through.
pub type Backend = Box<dyn Storage + Send + Sync>;

/// The storage [`storage_for_url`] returns: a block cache over the URL's
/// backend.
pub type RemoteStorage = BlockCache<Backend>;

/// Settings of [`open_url_with`] and [`storage_for_url`].
#[derive(Debug, Clone, Default)]
pub struct Options {
    /// The block cache.
    pub cache: CacheConfig,
    /// HTTP(S) requests.
    #[cfg(feature = "http")]
    pub http: HttpOptions,
}

/// Open the HDF5 file at `url` with default [`Options`].
///
/// `http://…` needs the (default) `http` feature, `https://…` the `https`
/// feature, `s3://bucket/key`, `gs://bucket/key` and `az://container/key`
/// the `s3`, `gcs` and `azure` features (credentials and region from the
/// environment, as `object_store`'s `from_env` builders read them).
pub fn open_url(url: &str) -> Result<File, Error> {
    open_url_with(url, &Options::default())
}

/// [`open_url`] with explicit [`Options`].
pub fn open_url_with(url: &str, options: &Options) -> Result<File, Error> {
    let storage = storage_for_url(url, options)?;
    Ok(File::open_storage(storage)?)
}

/// The cached storage for `url`, with the first block already fetched:
/// open it with [`clawhdf5::File::open_storage`] (a clone of the `Arc`),
/// and read its [`BlockCache::stats`] as you go.
pub fn storage_for_url(url: &str, options: &Options) -> Result<Arc<RemoteStorage>, Error> {
    let scheme = url
        .split_once("://")
        .map(|(s, _)| s.to_ascii_lowercase())
        .ok_or_else(|| RemoteError::InvalidUrl(format!("{}: no scheme", redact_url(url))))?;
    match scheme.as_str() {
        "http" | "https" => http_storage(url, options),
        "s3" | "s3a" | "gs" | "az" | "azure" | "abfs" | "abfss" | "adl" => {
            cloud_storage(url, &scheme, options)
        }
        _ => Err(RemoteError::UnsupportedScheme(redact_url(url)).into()),
    }
}

#[cfg(feature = "http")]
fn http_storage(url: &str, options: &Options) -> Result<Arc<RemoteStorage>, Error> {
    let mut http = options.http.clone();
    http.first_request = http.first_request.max(options.cache.block_size.max(1));
    let (storage, first) = HttpStorage::open(url, http)?;
    let cache = BlockCache::new(Box::new(storage) as Backend, options.cache.clone());
    cache.insert(0, &first);
    Ok(Arc::new(cache))
}

#[cfg(not(feature = "http"))]
fn http_storage(url: &str, _options: &Options) -> Result<Arc<RemoteStorage>, Error> {
    Err(RemoteError::UnsupportedScheme(format!(
        "{}: http(s):// needs the `http` feature of clawhdf5-remote",
        redact_url(url)
    ))
    .into())
}

#[cfg(any(feature = "s3", feature = "gcs", feature = "azure"))]
fn cloud_storage(url: &str, _scheme: &str, options: &Options) -> Result<Arc<RemoteStorage>, Error> {
    let (store, path) = object::store_for_url(url)?;
    let storage = ObjectStoreStorage::new(store, path)?;
    Ok(Arc::new(cached(Box::new(storage), options)?))
}

#[cfg(not(any(feature = "s3", feature = "gcs", feature = "azure")))]
fn cloud_storage(url: &str, scheme: &str, _options: &Options) -> Result<Arc<RemoteStorage>, Error> {
    let feature = match scheme {
        "s3" | "s3a" => "s3",
        "gs" => "gcs",
        _ => "azure",
    };
    Err(RemoteError::UnsupportedScheme(format!(
        "{}: {scheme}:// needs the `{feature}` feature of clawhdf5-remote",
        redact_url(url)
    ))
    .into())
}

/// A [`BlockCache`] over `backend` with its first block fetched (readahead
/// of the superblock and the metadata usually written next to it).
pub fn cached(backend: Backend, options: &Options) -> Result<RemoteStorage, Error> {
    let cache = BlockCache::new(backend, options.cache.clone());
    let first = cache.config().block_size;
    cache
        .prefetch(0, first)
        .map_err(|e| Error::Hdf5(clawhdf5::Error::Format(e)))?;
    Ok(cache)
}

/// Default limit of [`download`]: 1 GiB.
pub const DEFAULT_MAX_DOWNLOAD: u64 = 1 << 30;

/// The whole file behind `storage`, read into memory — at most
/// `max_bytes` of it (for example [`DEFAULT_MAX_DOWNLOAD`]).
///
/// The length is only what the server claims, so it is never used to
/// allocate: a file longer than `max_bytes` is refused with
/// [`RemoteError::TooLarge`] before anything is read, and the buffer grows
/// only as bytes arrive (64 MiB per step, fetched as parallel requests by a
/// [`BlockCache`]). A read that comes back short is an error.
pub fn download(storage: &dyn Storage, max_bytes: u64) -> Result<Vec<u8>, Error> {
    let len = storage.len();
    if len > max_bytes {
        return Err(RemoteError::TooLarge {
            len,
            limit: max_bytes,
        }
        .into());
    }
    const STEP: u64 = 64 << 20;
    let mut out = Vec::new();
    let mut pos = 0u64;
    while pos < len {
        let want = (len - pos).min(STEP);
        let got = storage
            .read_at(pos, want as usize)
            .map_err(|e| RemoteError::Backend(e.to_string()))?;
        if got.len() as u64 != want {
            return Err(RemoteError::BadResponse(format!(
                "{} bytes at offset {pos} instead of {want}",
                got.len()
            ))
            .into());
        }
        out.extend_from_slice(&got);
        pos += want;
    }
    Ok(out)
}

/// Open the object at `path` of any `object_store` store (in memory, local
/// files, or a cloud store you configured) through a block cache.
#[cfg(feature = "object-store")]
pub fn open_object(
    store: Arc<dyn object_store::ObjectStore>,
    path: &str,
    options: &Options,
) -> Result<(File, Arc<RemoteStorage>), Error> {
    let path = object_store::path::Path::parse(path)
        .map_err(|e| RemoteError::InvalidUrl(format!("{path}: {e}")))?;
    let storage = Arc::new(cached(
        Box::new(ObjectStoreStorage::new(store, path)?),
        options,
    )?);
    let file = File::open_storage(storage.clone())?;
    Ok((file, storage))
}