http

openml._api.clients.http

HTTPCache

Filesystem-based cache for HTTP responses.

This class stores HTTP responses on disk using a structured directory layout derived from the request URL and parameters. Each cached response consists of three files: metadata (meta.json), headers (headers.json), and the raw body (body.bin).

Notes

The cache key is derived from the URL (domain and path components) and query parameters, excluding the api_key parameter.

get_key

get_key(url: str, params: dict[str, Any]) -> str

Generate a filesystem-safe cache key for a request.

The key is constructed from URL path segments and URL-encoded query parameters (excluding api_key).

PARAMETER	DESCRIPTION
url	The full request URL. TYPE: str
params	Query parameters associated with the request. TYPE: dict of str to Any

RETURNS	DESCRIPTION
str	A relative path string representing the cache key.

Source code in openml/_api/clients/http.py

def get_key(self, url: str, params: dict[str, Any]) -> str:
    """
    Generate a filesystem-safe cache key for a request.

    The key is constructed from URL path segments and
    URL-encoded query parameters (excluding ``api_key``).

    Parameters
    ----------
    url : str
        The full request URL.
    params : dict of str to Any
        Query parameters associated with the request.

    Returns
    -------
    str
        A relative path string representing the cache key.
    """
    parsed_url = urlparse(url)
    path_parts = parsed_url.path.strip("/").split("/")

    filtered_params = {k: v for k, v in params.items() if k != "api_key"}
    params_part = [urlencode(filtered_params)] if filtered_params else []

    return str(Path(*path_parts, *params_part))

load

load(key: str) -> Response

Load a cached HTTP response from disk.

PARAMETER	DESCRIPTION
key	Cache key identifying the stored response. TYPE: str

RETURNS	DESCRIPTION
Response	Reconstructed response object with status code, headers, body, and metadata.

RAISES	DESCRIPTION
FileNotFoundError	If the cache entry or required files are missing.
ValueError	If required metadata is missing or malformed.

Source code in openml/_api/clients/http.py

def load(self, key: str) -> Response:
    """
    Load a cached HTTP response from disk.

    Parameters
    ----------
    key : str
        Cache key identifying the stored response.

    Returns
    -------
    requests.Response
        Reconstructed response object with status code, headers, body, and metadata.

    Raises
    ------
    FileNotFoundError
        If the cache entry or required files are missing.
    ValueError
        If required metadata is missing or malformed.
    """
    path = self._key_to_path(key)

    meta_path = path / "meta.json"
    meta_raw = meta_path.read_bytes() if meta_path.exists() else "{}"
    meta = json.loads(meta_raw)

    headers_path = path / "headers.json"
    headers_raw = headers_path.read_bytes() if headers_path.exists() else "{}"
    headers = json.loads(headers_raw)

    body_path = path / self._get_body_filename_from_path(path)
    if not body_path.exists():
        raise FileNotFoundError(f"Incomplete cache at {body_path}")
    body = body_path.read_bytes()

    response = Response()
    response.headers = headers
    response._content = body
    response.status_code = meta.get("status_code")
    response.url = meta.get("url")
    response.reason = meta.get("reason")
    response.encoding = meta.get("encoding")

    return response

save

save(key: str, response: Response) -> None

Persist an HTTP response to disk.

PARAMETER	DESCRIPTION
key	Cache key identifying where to store the response. TYPE: str
response	Response object to cache. TYPE: Response

Notes

The response body is stored as binary data. Headers and metadata (status code, URL, reason, encoding, elapsed time, request info, and creation timestamp) are stored as JSON.

Source code in openml/_api/clients/http.py

def save(self, key: str, response: Response) -> None:
    """
    Persist an HTTP response to disk.

    Parameters
    ----------
    key : str
        Cache key identifying where to store the response.
    response : requests.Response
        Response object to cache.

    Notes
    -----
    The response body is stored as binary data. Headers and metadata
    (status code, URL, reason, encoding, elapsed time, request info, and
    creation timestamp) are stored as JSON.
    """
    path = self._key_to_path(key)
    path.mkdir(parents=True, exist_ok=True)

    body_filename = self._get_body_filename_from_response(response)
    with (path / body_filename).open("wb") as f:
        f.write(response.content)

    with (path / "headers.json").open("w", encoding="utf-8") as f:
        json.dump(dict(response.headers), f)

    meta = {
        "status_code": response.status_code,
        "url": response.url,
        "reason": response.reason,
        "encoding": response.encoding,
        "created_at": time.time(),
        "request": {
            "method": response.request.method if response.request else None,
            "url": response.request.url if response.request else None,
            "headers": dict(response.request.headers) if response.request else None,
            "body": response.request.body if response.request else None,
        },
    }

    with (path / "meta.json").open("w", encoding="utf-8") as f:
        json.dump(meta, f)

HTTPClient

HTTPClient(*, api_version: APIVersion)

HTTP client for interacting with the OpenML API.

This client supports configurable retry policies, optional filesystem caching, API key authentication, and response validation including checksum verification.

PARAMETER	DESCRIPTION
api_version	Backend API Version. TYPE: APIVersion

Source code in openml/_api/clients/http.py

def __init__(
    self,
    *,
    api_version: APIVersion,
) -> None:
    self.api_version = api_version

    self.cache = HTTPCache()

__request

__request(session: Session, method: str, url: str, params: Mapping[str, Any], data: Mapping[str, Any], headers: Mapping[str, str], files: Mapping[str, Any] | None, **request_kwargs: Any) -> tuple[Response | None, Exception | None]

Execute a single HTTP request attempt.

PARAMETER	DESCRIPTION
session	Active session used to send the request. TYPE: Session
method	HTTP method (e.g., GET, POST). TYPE: str
url	Full request URL. TYPE: str
params	Query parameters. TYPE: Mapping of str to Any
data	Request body data. TYPE: Mapping of str to Any
headers	HTTP headers. TYPE: Mapping of str to str
files	Files to upload. TYPE: Mapping of str to Any or None
**request_kwargs	Additional arguments forwarded to requests.Session.request. TYPE: Any DEFAULT: {}

RETURNS	DESCRIPTION
tuple of (requests.Response or None, Exception or None)	Response and potential retry exception.

Source code in openml/_api/clients/http.py

def __request(  # noqa: PLR0913
    self,
    session: requests.Session,
    method: str,
    url: str,
    params: Mapping[str, Any],
    data: Mapping[str, Any],
    headers: Mapping[str, str],
    files: Mapping[str, Any] | None,
    **request_kwargs: Any,
) -> tuple[Response | None, Exception | None]:
    """
    Execute a single HTTP request attempt.

    Parameters
    ----------
    session : requests.Session
        Active session used to send the request.
    method : str
        HTTP method (e.g., ``GET``, ``POST``).
    url : str
        Full request URL.
    params : Mapping of str to Any
        Query parameters.
    data : Mapping of str to Any
        Request body data.
    headers : Mapping of str to str
        HTTP headers.
    files : Mapping of str to Any or None
        Files to upload.
    **request_kwargs : Any
        Additional arguments forwarded to ``requests.Session.request``.

    Returns
    -------
    tuple of (requests.Response or None, Exception or None)
        Response and potential retry exception.
    """
    exception: Exception | None = None
    response: Response | None = None

    try:
        response = session.request(
            method=method,
            url=url,
            params=params,
            data=data,
            headers=headers,
            files=files,
            **request_kwargs,
        )
    except (
        requests.exceptions.ChunkedEncodingError,
        requests.exceptions.ConnectionError,
        requests.exceptions.SSLError,
    ) as e:
        exception = e

    if response is not None:
        exception = self._validate_response(
            method=method,
            url=url,
            files=files,
            response=response,
        )

    return response, exception

delete

delete(path: str, **request_kwargs: Any) -> Response

Send a DELETE request.

PARAMETER	DESCRIPTION
path	API path relative to the base URL. TYPE: str
**request_kwargs	Additional request arguments. TYPE: Any DEFAULT: {}

RETURNS	DESCRIPTION
Response	HTTP response.

Source code in openml/_api/clients/http.py

def delete(
    self,
    path: str,
    **request_kwargs: Any,
) -> Response:
    """
    Send a DELETE request.

    Parameters
    ----------
    path : str
        API path relative to the base URL.
    **request_kwargs : Any
        Additional request arguments.

    Returns
    -------
    requests.Response
        HTTP response.
    """
    return self._request(
        method="DELETE",
        path=path,
        enable_cache=False,
        use_api_key=True,
        **request_kwargs,
    )

download

download(url: str, handler: Callable[[Response, Path, str], None] | None = None, encoding: str = 'utf-8', file_name: str = 'response.txt', md5_checksum: str | None = None) -> Path

Download a resource and store it in the cache directory.

PARAMETER	DESCRIPTION
url	Absolute URL of the resource to download. TYPE: str
handler	Custom handler function accepting (response, path, encoding) and returning a pathlib.Path. TYPE: callable or None DEFAULT: None
encoding	Text encoding used when writing the response body. TYPE: str DEFAULT: 'utf-8'
file_name	Name of the saved file. TYPE: str DEFAULT: 'response.txt'
md5_checksum	Expected MD5 checksum for integrity verification. TYPE: str or None DEFAULT: None

RETURNS	DESCRIPTION
Path	Path to the downloaded file.

RAISES	DESCRIPTION
OpenMLHashException	If checksum verification fails.

Source code in openml/_api/clients/http.py

def download(
    self,
    url: str,
    handler: Callable[[Response, Path, str], None] | None = None,
    encoding: str = "utf-8",
    file_name: str = "response.txt",
    md5_checksum: str | None = None,
) -> Path:
    """
    Download a resource and store it in the cache directory.

    Parameters
    ----------
    url : str
        Absolute URL of the resource to download.
    handler : callable or None, optional
        Custom handler function accepting ``(response, path, encoding)``
        and returning a ``pathlib.Path``.
    encoding : str, optional
        Text encoding used when writing the response body.
    file_name : str, optional
        Name of the saved file.
    md5_checksum : str or None, optional
        Expected MD5 checksum for integrity verification.

    Returns
    -------
    pathlib.Path
        Path to the downloaded file.

    Raises
    ------
    OpenMLHashException
        If checksum verification fails.
    """
    base = self.cache.path
    file_path = base / "downloads" / urlparse(url).path.lstrip("/") / file_name
    file_path = file_path.expanduser()
    file_path.parent.mkdir(parents=True, exist_ok=True)
    if file_path.exists():
        return file_path

    response = self.get(url, md5_checksum=md5_checksum)

    def write_to_file(response: Response, path: Path, encoding: str) -> None:
        path.write_text(response.text, encoding)

    handler = handler or write_to_file
    handler(response, file_path, encoding)
    return file_path

get

get(path: str, *, enable_cache: bool = False, refresh_cache: bool = False, use_api_key: bool = False, md5_checksum: str | None = None, **request_kwargs: Any) -> Response

Send a GET request.

PARAMETER	DESCRIPTION
path	API path relative to the base URL. TYPE: str
enable_cache	Whether to use the response cache. TYPE: bool DEFAULT: False
refresh_cache	Whether to ignore existing cached entries. TYPE: bool DEFAULT: False
use_api_key	Whether to include the API key. TYPE: bool DEFAULT: False
md5_checksum	Expected MD5 checksum for response validation. TYPE: str or None DEFAULT: None
**request_kwargs	Additional request arguments. TYPE: Any DEFAULT: {}

RETURNS	DESCRIPTION
Response	HTTP response.

Source code in openml/_api/clients/http.py

def get(
    self,
    path: str,
    *,
    enable_cache: bool = False,
    refresh_cache: bool = False,
    use_api_key: bool = False,
    md5_checksum: str | None = None,
    **request_kwargs: Any,
) -> Response:
    """
    Send a GET request.

    Parameters
    ----------
    path : str
        API path relative to the base URL.
    enable_cache : bool, optional
        Whether to use the response cache.
    refresh_cache : bool, optional
        Whether to ignore existing cached entries.
    use_api_key : bool, optional
        Whether to include the API key.
    md5_checksum : str or None, optional
        Expected MD5 checksum for response validation.
    **request_kwargs : Any
        Additional request arguments.

    Returns
    -------
    requests.Response
        HTTP response.
    """
    return self._request(
        method="GET",
        path=path,
        enable_cache=enable_cache,
        refresh_cache=refresh_cache,
        use_api_key=use_api_key,
        md5_checksum=md5_checksum,
        **request_kwargs,
    )

post

post(path: str, *, use_api_key: bool = True, **request_kwargs: Any) -> Response

Send a POST request.

PARAMETER	DESCRIPTION
path	API path relative to the base URL. TYPE: str
use_api_key	Whether to include the API key. TYPE: bool DEFAULT: True
**request_kwargs	Additional request arguments. TYPE: Any DEFAULT: {}

RETURNS	DESCRIPTION
Response	HTTP response.

Source code in openml/_api/clients/http.py

def post(
    self,
    path: str,
    *,
    use_api_key: bool = True,
    **request_kwargs: Any,
) -> Response:
    """
    Send a POST request.

    Parameters
    ----------
    path : str
        API path relative to the base URL.
    use_api_key : bool, optional
        Whether to include the API key.
    **request_kwargs : Any
        Additional request arguments.

    Returns
    -------
    requests.Response
        HTTP response.
    """
    return self._request(
        method="POST",
        path=path,
        enable_cache=False,
        use_api_key=use_api_key,
        **request_kwargs,
    )