clawhdf5/crates/clawhdf5-remote/src/http.rs

//! HTTP(S) range requests: [`HttpStorage`].
//!
//! Every read is a `GET` with a `Range: bytes=a-b` header, answered `206
//! Partial Content`. The file is pinned when it is opened:
//!
//! - its length comes from the `Content-Range` of the first request (which
//!   also fetches the first block, so opening costs one request);
//! - a strong `ETag` is sent back as `If-Match` on every later request, and
//!   compared with the `ETag` of every response; without one, `Last-Modified`
//!   is sent as `If-Unmodified-Since` and compared; the length in every
//!   `Content-Range` must stay the same. A file that changes while it is
//!   open is [`RemoteError::FileChanged`], never a mix of old and new bytes.
//!   (A server that sends neither validator cannot be checked beyond the
//!   length; [`HttpOptions::require_validator`] refuses such servers.)
//! - a `200` answer to the first request whose body is no longer than the
//!   range asked for is the whole file (a server may answer so when the
//!   range covers it): it is kept and read from memory.
//! - a server that ignores `Range` and answers `200` with the whole file is
//!   refused with [`RemoteError::RangeNotSupported`], unless
//!   [`HttpOptions::allow_full_download`] is set: then the file is
//!   downloaded once, at open, and read from memory.
//!
//! Redirects are followed (at most [`HttpOptions::max_redirects`]), but
//! never from `https` to `http`, and the custom
//! [`HttpOptions::headers`] are not sent to another origin.
//!
//! Timeouts scale with the request: [`HttpOptions::timeout`] to connect
//! and to get the response headers, and for the body that plus its size at
//! [`HttpOptions::min_speed`] — a slow link is not cut off mid-block, a
//! stalled connection still is.
//!
//! Transient failures — connection errors, timeouts, `408`/`429`/`5xx`, and
//! a body shorter or longer than its `Content-Range` — are retried with
//! exponential backoff. Responses are requested with
//! `Accept-Encoding: identity`, since a compressed body cannot be a byte
//! range of the file.
//!
//! `HttpStorage` itself does not cache: each `read_at` is one request. Read
//! it through [`BlockCache`](crate::BlockCache) (which [`open_url`](crate::open_url)
//! does); its `read_ranges` fetches the ranges of one call in parallel.

use std::borrow::Cow;
use std::io::Read;
use std::ops::Range;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::time::Duration;

use clawhdf5_format::error::FormatError;
use clawhdf5_format::storage::Storage;

use crate::error::{Redactor, RemoteError, redact_url};

/// Settings of an [`HttpStorage`].
#[derive(Debug, Clone)]
pub struct HttpOptions {
    /// Retries of a request that failed transiently (so up to `retries + 1`
    /// attempts).
    pub retries: u32,
    /// Delay before the first retry; doubled for each further one.
    pub backoff: Duration,
    /// Time allowed to connect, and then to receive the response headers.
    /// The body gets this plus the time it takes at
    /// [`min_speed`](Self::min_speed), so a request's budget grows with its
    /// size: a slow but moving link is not cut off, a stalled one is.
    pub timeout: Duration,
    /// Slowest transfer rate tolerated, in bytes per second: receiving a
    /// body of `n` bytes may take `timeout + n / min_speed` (16 KiB/s by
    /// default: 94 s for a 1 MiB block, 9 min for an 8 MiB request).
    pub min_speed: u64,
    /// Requests of one `read_ranges` call in flight at once.
    pub max_parallel: usize,
    /// Bytes fetched by the first request, from offset 0 (the superblock and
    /// usually the root group's metadata); at least 1.
    pub first_request: u64,
    /// When the server ignores `Range` (answers `200`), download the whole
    /// file once and read it from memory, instead of failing.
    pub allow_full_download: bool,
    /// Largest file [`allow_full_download`](Self::allow_full_download) will
    /// download.
    pub max_full_download: u64,
    /// Refuse a server that sends neither a strong `ETag` nor
    /// `Last-Modified`, since a change of the file could then go unnoticed
    /// (only its length is checked).
    pub require_validator: bool,
    /// Extra headers sent with every request to the URL's own origin (for
    /// example `Authorization` or `X-Api-Key`). They are never sent to
    /// another origin a redirect leads to.
    pub headers: Vec<(String, String)>,
    /// Redirects followed per request (0: none, a redirect is an error).
    /// A redirect from `https` to plain `http` is always refused; once a
    /// redirect leaves the URL's origin (scheme, host and port), the
    /// [`headers`](Self::headers) are no longer sent. Every request of a
    /// file follows the redirects again (the target is not remembered, as
    /// a presigned target may expire).
    pub max_redirects: u32,
}

impl Default for HttpOptions {
    fn default() -> Self {
        HttpOptions {
            retries: 3,
            backoff: Duration::from_millis(200),
            timeout: Duration::from_secs(30),
            min_speed: 16 << 10,
            max_parallel: 8,
            first_request: crate::cache::DEFAULT_BLOCK_SIZE,
            allow_full_download: false,
            max_full_download: 1 << 30,
            require_validator: false,
            headers: Vec::new(),
            max_redirects: 5,
        }
    }
}

/// Requests and bytes an [`HttpStorage`] has used.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct HttpStats {
    /// HTTP requests sent (retries included).
    pub requests: u64,
    /// Requests that were retries.
    pub retries: u64,
    /// Response body bytes received.
    pub bytes: u64,
}

/// How the file is pinned.
#[derive(Debug, Clone, PartialEq, Eq)]
enum Validator {
    ETag(String),
    LastModified(String),
    None,
}

/// An HTTP(S) file read by range requests. See the [module
/// documentation](self).
pub struct HttpStorage {
    agent: ureq::Agent,
    /// The URL as given, credentials and all: only ever sent to the server.
    url: String,
    /// Shows the URL without its credentials, in every message.
    redactor: Redactor,
    len: u64,
    validator: Validator,
    options: HttpOptions,
    /// The whole file, when the server ignores ranges and a full download
    /// was allowed.
    full: Option<Vec<u8>>,
    requests: AtomicU64,
    retries: AtomicU64,
    bytes: AtomicU64,
}

impl std::fmt::Debug for HttpStorage {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("HttpStorage")
            .field("url", &self.redactor.shown())
            .field("len", &self.len)
            .field("validator", &self.validator)
            .field("full_download", &self.full.is_some())
            .finish()
    }
}

/// A parsed `Content-Range: bytes a-b/total`.
fn content_range(v: &str) -> Option<(u64, u64, Option<u64>)> {
    let rest = v.trim().strip_prefix("bytes")?.trim_start();
    let (span, total) = rest.split_once('/')?;
    let (a, b) = span.trim().split_once('-')?;
    let a: u64 = a.trim().parse().ok()?;
    let b: u64 = b.trim().parse().ok()?;
    if b < a {
        return None;
    }
    let total = match total.trim() {
        "*" => None,
        t => Some(t.parse().ok()?),
    };
    Some((a, b, total))
}

fn header<'a>(resp: &'a ureq::http::Response<ureq::Body>, name: &str) -> Option<&'a str> {
    resp.headers().get(name).and_then(|v| v.to_str().ok())
}

/// A body with a `Content-Encoding` is not a byte range of the file.
fn check_identity(url: &str, resp: &ureq::http::Response<ureq::Body>) -> Result<(), RemoteError> {
    match header(resp, "content-encoding") {
        Some(enc) if !enc.trim().eq_ignore_ascii_case("identity") => {
            Err(RemoteError::Usage(format!(
                "{url}: the server sent a {enc}-encoded body despite Accept-Encoding: identity"
            )))
        }
        _ => Ok(()),
    }
}

/// Scheme, host (lowercase, no userinfo) and port of an absolute URL.
fn origin(url: &str) -> Option<(String, String, u16)> {
    let (scheme, rest) = url.split_once("://")?;
    let scheme = scheme.to_ascii_lowercase();
    let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
    let host_port = authority.rsplit_once('@').map_or(authority, |(_, h)| h);
    let default = match scheme.as_str() {
        "http" => 80,
        "https" => 443,
        _ => return None,
    };
    // "[v6]:port", "host:port", or either without a port.
    let (host, port) = match host_port.rsplit_once(':') {
        Some((h, p)) if !p.contains(']') => (h, p.parse().ok()?),
        _ => (host_port, default),
    };
    Some((scheme, host.to_ascii_lowercase(), port))
}

fn same_origin(a: &str, b: &str) -> bool {
    matches!((origin(a), origin(b)), (Some(x), Some(y)) if x == y)
}

/// The URL a redirect from `base` to `location` goes to, if it may be
/// followed: `http`/`https` only, and never from `https` to `http`.
fn redirect_target(base: &str, location: &str) -> Result<String, RemoteError> {
    let location = location.trim();
    let (scheme, rest) = base.split_once("://").unwrap_or(("http", base));
    let authority_end = rest.find(['/', '?', '#']).unwrap_or(rest.len());
    let (authority, path) = rest.split_at(authority_end);
    let target = if location.contains("://") {
        location.to_string()
    } else if let Some(r) = location.strip_prefix("//") {
        format!("{scheme}://{r}")
    } else if location.starts_with('/') {
        format!("{scheme}://{authority}{location}")
    } else {
        let path = path.split(['?', '#']).next().unwrap_or("");
        let dir = path.rsplit_once('/').map_or("", |(d, _)| d);
        format!("{scheme}://{authority}{dir}/{location}")
    };
    let Some((to_scheme, _, _)) = origin(&target) else {
        return Err(RemoteError::Redirect(format!(
            "{} redirects to {}, which is not an http(s) URL",
            redact_url(base),
            redact_url(&target)
        )));
    };
    if scheme.eq_ignore_ascii_case("https") && to_scheme != "https" {
        return Err(RemoteError::Redirect(format!(
            "{} redirects to {}: a downgrade from https is refused",
            redact_url(base),
            redact_url(&target)
        )));
    }
    Ok(target)
}

fn transport(e: ureq::Error) -> RemoteError {
    match e {
        ureq::Error::StatusCode(code) => RemoteError::Status {
            code,
            what: "request".into(),
        },
        ureq::Error::BadUri(s) => RemoteError::InvalidUrl(s),
        other => RemoteError::Transport(other.to_string()),
    }
}

impl HttpStorage {
    /// Open `url` (`http://`, or `https://` with the `https` feature):
    /// one ranged `GET` of the first [`HttpOptions::first_request`] bytes,
    /// which gives the file's length and validators. Returns the storage
    /// and the bytes that request fetched (the file's start), for a
    /// [`BlockCache`](crate::BlockCache) to keep.
    pub fn open(url: &str, options: HttpOptions) -> Result<(HttpStorage, Vec<u8>), RemoteError> {
        let redactor = Redactor::new(url);
        HttpStorage::open_inner(url, options, &redactor).map_err(|e| e.scrubbed(&redactor))
    }

    fn open_inner(
        url: &str,
        options: HttpOptions,
        redactor: &Redactor,
    ) -> Result<(HttpStorage, Vec<u8>), RemoteError> {
        let shown = redactor.shown();
        let lower = url.to_ascii_lowercase();
        if lower.starts_with("https://") {
            if !cfg!(feature = "https") {
                return Err(RemoteError::UnsupportedScheme(format!(
                    "{shown}: https:// needs the `https` feature of clawhdf5-remote"
                )));
            }
        } else if !lower.starts_with("http://") {
            return Err(RemoteError::UnsupportedScheme(shown.to_string()));
        }
        // Redirects are followed by `call`, which applies our rules.
        let config = ureq::Agent::config_builder()
            .http_status_as_error(false)
            .max_redirects(0)
            .timeout_connect(Some(options.timeout))
            .timeout_recv_response(Some(options.timeout))
            .build();
        let mut storage = HttpStorage {
            agent: ureq::Agent::new_with_config(config),
            url: url.to_string(),
            redactor: redactor.clone(),
            len: 0,
            validator: Validator::None,
            options,
            full: None,
            requests: AtomicU64::new(0),
            retries: AtomicU64::new(0),
            bytes: AtomicU64::new(0),
        };
        let first = storage.with_retries(|| storage.probe())?;
        let (len, validator, bytes, full) = first;
        storage.len = len;
        storage.validator = validator;
        if full {
            storage.full = Some(bytes);
            return Ok((storage, Vec::new()));
        }
        if storage.options.require_validator && storage.validator == Validator::None {
            return Err(RemoteError::Usage(format!(
                "{shown}: the server sends neither a strong ETag nor Last-Modified, so a change \
                 of the file could not be detected (HttpOptions::require_validator)"
            )));
        }
        Ok((storage, bytes))
    }

    /// The URL as given — with any credentials it carries, so do not log
    /// it; [`redact_url`](crate::redact_url) gives a form that can be.
    pub fn url(&self) -> &str {
        &self.url
    }

    /// Whether the file was downloaded whole, because the server does not
    /// support ranges and [`HttpOptions::allow_full_download`] was set.
    pub fn is_full_download(&self) -> bool {
        self.full.is_some()
    }

    /// The `ETag` the file is pinned to, if the server sent a strong one.
    pub fn etag(&self) -> Option<&str> {
        match &self.validator {
            Validator::ETag(e) => Some(e),
            _ => None,
        }
    }

    /// Requests and bytes so far.
    pub fn stats(&self) -> HttpStats {
        HttpStats {
            requests: self.requests.load(Ordering::Relaxed),
            retries: self.retries.load(Ordering::Relaxed),
            bytes: self.bytes.load(Ordering::Relaxed),
        }
    }

    fn with_retries<T>(
        &self,
        mut attempt: impl FnMut() -> Result<T, RemoteError>,
    ) -> Result<T, RemoteError> {
        let mut delay = self.options.backoff;
        let mut n = 0;
        loop {
            match attempt() {
                Ok(v) => return Ok(v),
                Err(e) if e.is_transient() && n < self.options.retries => {
                    n += 1;
                    self.retries.fetch_add(1, Ordering::Relaxed);
                    std::thread::sleep(delay);
                    delay = delay.saturating_mul(2);
                }
                Err(e) => return Err(e),
            }
        }
    }

    /// One `GET` of `url`. The options' custom headers are sent only to
    /// the URL's own origin (`trusted`).
    fn request(
        &self,
        url: &str,
        range: Option<(u64, u64)>,
        trusted: bool,
    ) -> ureq::RequestBuilder<ureq::typestate::WithoutBody> {
        let mut req = self.agent.get(url).header("Accept-Encoding", "identity");
        if let Some((a, b)) = range {
            req = req.header("Range", format!("bytes={a}-{b}"));
        }
        // The body's budget scales with what it may carry: the range, or
        // a whole file the server may send instead.
        let mut body = range.map_or(0, |(a, b)| b.saturating_sub(a).saturating_add(1));
        if self.options.allow_full_download {
            body = body.max(self.options.max_full_download);
        }
        let secs = body as f64 / self.options.min_speed.max(1) as f64;
        let body_timeout = self
            .options
            .timeout
            .saturating_add(Duration::try_from_secs_f64(secs).unwrap_or(Duration::MAX));
        let mut req = req.config().timeout_recv_body(Some(body_timeout)).build();
        match &self.validator {
            Validator::ETag(e) => req = req.header("If-Match", e),
            Validator::LastModified(t) => req = req.header("If-Unmodified-Since", t),
            Validator::None => {}
        }
        if trusted {
            for (k, v) in &self.options.headers {
                req = req.header(k, v);
            }
        }
        req
    }

    /// Send a ranged `GET`, following redirects by the rules of
    /// [`HttpOptions::max_redirects`]: at most that many, never from
    /// `https` to anything else, and without the custom headers once the
    /// chain has left the URL's origin. Each hop counts as a request.
    fn call(
        &self,
        range: Option<(u64, u64)>,
    ) -> Result<ureq::http::Response<ureq::Body>, RemoteError> {
        let mut url = self.url.clone();
        let mut trusted = true;
        let mut hops = 0u32;
        loop {
            if hops > 0 {
                self.requests.fetch_add(1, Ordering::Relaxed);
            }
            let resp = self
                .request(&url, range, trusted)
                .call()
                .map_err(|e| transport(e).scrubbed(&Redactor::new(&url)))?;
            if !matches!(resp.status().as_u16(), 301 | 302 | 303 | 307 | 308) {
                return Ok(resp);
            }
            let status = resp.status().as_u16();
            let Some(location) = header(&resp, "location") else {
                return Err(RemoteError::BadResponse(format!(
                    "{}: status {status} without a Location",
                    redact_url(&url)
                )));
            };
            let next = redirect_target(&url, location)?;
            if hops >= self.options.max_redirects {
                return Err(RemoteError::Redirect(format!(
                    "{} redirects to {}: more than HttpOptions::max_redirects ({})",
                    redact_url(&url),
                    redact_url(&next),
                    self.options.max_redirects
                )));
            }
            if !same_origin(&next, &self.url) {
                trusted = false;
            }
            url = next;
            hops += 1;
        }
    }

    /// Read a body of exactly `want` bytes (or up to `limit` when `want` is
    /// unknown).
    fn body(
        &self,
        resp: ureq::http::Response<ureq::Body>,
        want: Option<u64>,
        limit: u64,
    ) -> Result<Vec<u8>, RemoteError> {
        let cap = want.unwrap_or(limit);
        let mut buf = Vec::with_capacity(usize::try_from(cap.min(64 << 20)).unwrap_or(0));
        let reader = resp.into_body().into_reader();
        let got = reader
            .take(cap.saturating_add(1))
            .read_to_end(&mut buf)
            .map_err(|e| {
                RemoteError::Transport(format!("{}: reading the body: {e}", self.redactor.shown()))
            });
        self.bytes.fetch_add(buf.len() as u64, Ordering::Relaxed);
        got?;
        match want {
            Some(n) if buf.len() as u64 != n => Err(RemoteError::BadResponse(format!(
                "{}: body of {} bytes, expected {n}",
                self.redactor.shown(),
                buf.len()
            ))),
            None if buf.len() as u64 > limit => Err(RemoteError::Usage(format!(
                "{}: the file is larger than HttpOptions::max_full_download ({limit} bytes)",
                self.redactor.shown()
            ))),
            _ => Ok(buf),
        }
    }

    /// The first request: length, validators and the file's first bytes.
    /// The last field is true when the server ignored the range and sent
    /// the whole file (only kept when a full download is allowed).
    fn probe(&self) -> Result<(u64, Validator, Vec<u8>, bool), RemoteError> {
        let n = self.options.first_request.max(1);
        self.requests.fetch_add(1, Ordering::Relaxed);
        let resp = self.call(Some((0, n - 1)))?;
        let status = resp.status().as_u16();
        check_identity(self.redactor.shown(), &resp)?;
        let validator = match (header(&resp, "etag"), header(&resp, "last-modified")) {
            (Some(e), _) if !e.starts_with("W/") => Validator::ETag(e.to_string()),
            (_, Some(t)) => Validator::LastModified(t.to_string()),
            _ => Validator::None,
        };
        match status {
            206 => {
                let cr = header(&resp, "content-range").ok_or_else(|| {
                    RemoteError::BadResponse(format!(
                        "{}: 206 without Content-Range",
                        self.redactor.shown()
                    ))
                })?;
                let (a, b, total) = content_range(cr).ok_or_else(|| {
                    RemoteError::BadResponse(format!(
                        "{}: bad Content-Range {cr:?}",
                        self.redactor.shown()
                    ))
                })?;
                let total = total.ok_or_else(|| {
                    RemoteError::BadResponse(format!(
                        "{}: the server does not report the file's length (Content-Range {cr:?})",
                        self.redactor.shown()
                    ))
                })?;
                if a != 0 || b >= total || b > n - 1 {
                    return Err(RemoteError::BadResponse(format!(
                        "{}: asked for bytes 0-{}, got Content-Range {cr:?}",
                        self.redactor.shown(),
                        n - 1
                    )));
                }
                let bytes = self.body(resp, Some(b - a + 1), 0)?;
                Ok((total, validator, bytes, false))
            }
            200 => {
                // RFC 9110 lets a server answer 200 when the range covers
                // the whole file: a body no longer than the range asked
                // for is the whole file, ranges supported or not.
                let want: Option<u64> =
                    header(&resp, "content-length").and_then(|v| v.trim().parse().ok());
                let refused = || {
                    RemoteError::RangeNotSupported(format!(
                        "{} answered a range request with the whole file (status 200); set \
                         HttpOptions::allow_full_download to download it",
                        self.redactor.shown()
                    ))
                };
                if !self.options.allow_full_download {
                    return match want {
                        Some(w) if w <= n => {
                            let bytes = self.body(resp, Some(w), 0)?;
                            Ok((bytes.len() as u64, validator, bytes, true))
                        }
                        Some(_) => Err(refused()),
                        // No length: read at most the range asked for.
                        None => match self.body(resp, None, n) {
                            Ok(bytes) => Ok((bytes.len() as u64, validator, bytes, true)),
                            Err(RemoteError::Usage(_)) => Err(refused()),
                            Err(e) => Err(e),
                        },
                    };
                }
                if want.is_some_and(|w| w > self.options.max_full_download) {
                    return Err(RemoteError::Usage(format!(
                        "{}: the file is larger than HttpOptions::max_full_download ({} bytes)",
                        self.redactor.shown(),
                        self.options.max_full_download
                    )));
                }
                let bytes = self.body(resp, want, self.options.max_full_download)?;
                Ok((bytes.len() as u64, validator, bytes, true))
            }
            416 => Err(RemoteError::Usage(format!(
                "{}: status 416 for the first bytes (an empty file?)",
                self.redactor.shown()
            ))),
            code => Err(RemoteError::Status {
                code,
                what: self.redactor.shown().to_string(),
            }),
        }
    }

    /// One request for `[start, end)` (inside the file, non-empty).
    fn fetch_once(&self, start: u64, end: u64) -> Result<Vec<u8>, RemoteError> {
        self.requests.fetch_add(1, Ordering::Relaxed);
        let resp = self.call(Some((start, end - 1)))?;
        let changed =
            |why: String| RemoteError::FileChanged(format!("{}: {why}", self.redactor.shown()));
        check_identity(self.redactor.shown(), &resp)?;
        match resp.status().as_u16() {
            206 => {}
            200 => {
                return Err(RemoteError::RangeNotSupported(format!(
                    "{} answered a range request with the whole file (status 200)",
                    self.redactor.shown()
                )));
            }
            412 => {
                return Err(changed(
                    "If-Match/If-Unmodified-Since failed (status 412)".into(),
                ));
            }
            416 => return Err(changed("range no longer satisfiable (status 416)".into())),
            code => {
                return Err(RemoteError::Status {
                    code,
                    what: format!("{} bytes {start}-{}", self.redactor.shown(), end - 1),
                });
            }
        }
        match &self.validator {
            Validator::ETag(e) => {
                if let Some(got) = header(&resp, "etag")
                    && got != e
                {
                    return Err(changed(format!("ETag {got} instead of {e}")));
                }
            }
            Validator::LastModified(t) => {
                if let Some(got) = header(&resp, "last-modified")
                    && got != t
                {
                    return Err(changed(format!("Last-Modified {got} instead of {t}")));
                }
            }
            Validator::None => {}
        }
        let cr = header(&resp, "content-range").ok_or_else(|| {
            RemoteError::BadResponse(format!(
                "{}: 206 without Content-Range",
                self.redactor.shown()
            ))
        })?;
        let (a, b, total) = content_range(cr).ok_or_else(|| {
            RemoteError::BadResponse(format!(
                "{}: bad Content-Range {cr:?}",
                self.redactor.shown()
            ))
        })?;
        if let Some(total) = total
            && total != self.len
        {
            return Err(changed(format!("length {total} instead of {}", self.len)));
        }
        if a != start || b != end - 1 {
            return Err(RemoteError::BadResponse(format!(
                "{}: asked for bytes {start}-{}, got Content-Range {cr:?}",
                self.redactor.shown(),
                end - 1
            )));
        }
        self.body(resp, Some(end - start), 0)
    }

    fn fetch(&self, start: u64, end: u64) -> Result<Vec<u8>, RemoteError> {
        self.with_retries(|| self.fetch_once(start, end))
            .map_err(|e| e.scrubbed(&self.redactor))
    }

    /// `[offset, offset + len)` clamped to the file, or `None` if empty.
    fn clamp(&self, offset: u64, len: u64) -> Option<(u64, u64)> {
        let end = offset.saturating_add(len).min(self.len);
        (offset < end).then_some((offset, end))
    }
}

impl Storage for HttpStorage {
    fn read_at(&self, offset: u64, len: usize) -> Result<Cow<'_, [u8]>, FormatError> {
        if let Some(all) = &self.full {
            return all.as_slice().read_at(offset, len);
        }
        match self.clamp(offset, len as u64) {
            None => Ok(Cow::Owned(Vec::new())),
            Some((a, b)) => Ok(Cow::Owned(self.fetch(a, b)?)),
        }
    }

    fn len(&self) -> u64 {
        self.len
    }

    fn read_ranges(&self, ranges: &[Range<u64>]) -> Result<Vec<Cow<'_, [u8]>>, FormatError> {
        if let Some(all) = &self.full {
            return all.as_slice().read_ranges(ranges);
        }
        let parallel = self.options.max_parallel.clamp(1, ranges.len().max(1));
        if parallel <= 1 {
            return ranges
                .iter()
                .map(|r| self.read_at(r.start, (r.end.saturating_sub(r.start)) as usize))
                .collect();
        }
        let next = AtomicUsize::new(0);
        let failed = std::sync::atomic::AtomicBool::new(false);
        type Slot = Option<Result<Vec<u8>, RemoteError>>;
        let results: std::sync::Mutex<Vec<Slot>> =
            std::sync::Mutex::new((0..ranges.len()).map(|_| None).collect());
        std::thread::scope(|s| {
            for _ in 0..parallel {
                s.spawn(|| {
                    loop {
                        let i = next.fetch_add(1, Ordering::Relaxed);
                        if i >= ranges.len() || failed.load(Ordering::Relaxed) {
                            break;
                        }
                        let r = &ranges[i];
                        let got = match self.clamp(r.start, r.end.saturating_sub(r.start)) {
                            None => Ok(Vec::new()),
                            Some((a, b)) => self.fetch(a, b),
                        };
                        if got.is_err() {
                            failed.store(true, Ordering::Relaxed);
                        }
                        results
                            .lock()
                            .unwrap_or_else(std::sync::PoisonError::into_inner)[i] = Some(got);
                    }
                });
            }
        });
        let results = results
            .into_inner()
            .unwrap_or_else(std::sync::PoisonError::into_inner);
        let mut out = Vec::with_capacity(ranges.len());
        for r in results {
            match r {
                Some(Ok(v)) => out.push(Cow::Owned(v)),
                Some(Err(e)) => return Err(e.into()),
                // Not fetched because another range failed first.
                None => continue,
            }
        }
        if out.len() != ranges.len() {
            return Err(FormatError::Storage(format!(
                "{}: a parallel range read failed",
                self.redactor.shown()
            )));
        }
        Ok(out)
    }

    fn as_contiguous(&self) -> Option<&[u8]> {
        self.full.as_deref()
    }
}

#[cfg(test)]
mod tests {
    use super::content_range;

    #[test]
    fn content_range_parses() {
        assert_eq!(content_range("bytes 0-99/1000"), Some((0, 99, Some(1000))));
        assert_eq!(content_range("bytes 5-5/*"), Some((5, 5, None)));
        assert_eq!(content_range("bytes 9-5/10"), None);
        assert_eq!(content_range("items 0-1/2"), None);
        assert_eq!(content_range("bytes */1000"), None);
    }

    #[test]
    fn redirect_targets_and_origins() {
        use super::{redirect_target, same_origin};
        let t = |b: &str, l: &str| redirect_target(b, l).map_err(|e| e.to_string());
        assert_eq!(
            t("http://a:8/d/f.h5?x=1", "g.h5").unwrap(),
            "http://a:8/d/g.h5"
        );
        assert_eq!(t("http://a/d/f.h5", "/g.h5").unwrap(), "http://a/g.h5");
        assert_eq!(t("https://a/d/f.h5", "//b/g.h5").unwrap(), "https://b/g.h5");
        assert_eq!(t("http://a/f", "https://b/g").unwrap(), "https://b/g");
        let e = t("https://a/f.h5", "http://a/f.h5").unwrap_err();
        assert!(e.contains("downgrade"), "{e}");
        assert!(t("HTTPS://a/f.h5", "http://b/f.h5").is_err());
        assert!(t("https://a/f", "//b/g").is_ok());
        assert!(t("http://a/f", "ftp://b/g").is_err());
        let e = t("https://u:pw@a/f?sig=SECRET", "http://b/g?sig=OTHER").unwrap_err();
        assert!(
            !e.contains("SECRET") && !e.contains("OTHER") && !e.contains("pw"),
            "{e}"
        );
        assert!(same_origin("http://a/x", "http://A:80/y"));
        assert!(same_origin("https://u:p@a:443/x", "https://a/y"));
        assert!(!same_origin("http://a/x", "https://a/x"));
        assert!(!same_origin("http://a:1/x", "http://a:2/x"));
        assert!(!same_origin("http://a/x", "http://b/x"));
        assert!(same_origin("http://[::1]:8/x", "http://[::1]:8/y"));
        assert!(!same_origin("http://[::1]/x", "http://[::1]:8/y"));
    }
}