study

openml.study

OpenMLBenchmarkSuite

OpenMLBenchmarkSuite(suite_id: int | None, alias: str | None, name: str, description: str, status: str | None, creation_date: str | None, creator: int | None, tags: list[dict] | None, data: list[int] | None, tasks: list[int] | None)

Bases: BaseStudy

An OpenMLBenchmarkSuite represents the OpenML concept of a suite (a collection of tasks).

It contains the following information: name, id, description, creation date, creator id and the task ids.

According to this list of task ids, the suite object receives a list of OpenML object ids (datasets).

Parameters

suite_id : int the study id alias : str (optional) a string ID, unique on server (url-friendly) main_entity_type : str the entity type (e.g., task, run) that is core in this study. only entities of this type can be added explicitly name : str the name of the study (meta-info) description : str brief description (meta-info) status : str Whether the study is in preparation, active or deactivated creation_date : str date of creation (meta-info) creator : int openml user id of the owner / creator tags : list(dict) The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access. data : list a list of data ids associated with this study tasks : list a list of task ids associated with this study

Source code in openml/study/study.py

def __init__(  # noqa: PLR0913
    self,
    suite_id: int | None,
    alias: str | None,
    name: str,
    description: str,
    status: str | None,
    creation_date: str | None,
    creator: int | None,
    tags: list[dict] | None,
    data: list[int] | None,
    tasks: list[int] | None,
):
    super().__init__(
        study_id=suite_id,
        alias=alias,
        main_entity_type="task",
        benchmark_suite=None,
        name=name,
        description=description,
        status=status,
        creation_date=creation_date,
        creator=creator,
        tags=tags,
        data=data,
        tasks=tasks,
        flows=None,
        runs=None,
        setups=None,
    )

id property

id: int | None

Return the id of the study.

openml_url property

openml_url: str | None

The URL of the object on the server, if it was uploaded, else None.

open_in_browser

open_in_browser() -> None

Opens the OpenML web page corresponding to this object in your default browser.

Source code in openml/base.py

def open_in_browser(self) -> None:
    """Opens the OpenML web page corresponding to this object in your default browser."""
    if self.openml_url is None:
        raise ValueError(
            "Cannot open element on OpenML.org when attribute `openml_url` is `None`",
        )

    webbrowser.open(self.openml_url)

publish

publish() -> OpenMLBase

Publish the object on the OpenML server.

Source code in openml/base.py

def publish(self) -> OpenMLBase:
    """Publish the object on the OpenML server."""
    file_elements = self._get_file_elements()

    if "description" not in file_elements:
        file_elements["description"] = self._to_xml()

    call = f"{_get_rest_api_type_alias(self)}/"
    response_text = openml._api_calls._perform_api_call(
        call,
        "post",
        file_elements=file_elements,
    )
    xml_response = xmltodict.parse(response_text)

    self._parse_publish_response(xml_response)
    return self

push_tag

push_tag(tag: str) -> None

Add a tag to the study.

Source code in openml/study/study.py

def push_tag(self, tag: str) -> None:
    """Add a tag to the study."""
    raise NotImplementedError("Tags for studies is not (yet) supported.")

remove_tag

remove_tag(tag: str) -> None

Remove a tag from the study.

Source code in openml/study/study.py

def remove_tag(self, tag: str) -> None:
    """Remove a tag from the study."""
    raise NotImplementedError("Tags for studies is not (yet) supported.")

url_for_id classmethod

url_for_id(id_: int) -> str

Return the OpenML URL for the object of the class entity with the given id.

Source code in openml/base.py

@classmethod
def url_for_id(cls, id_: int) -> str:
    """Return the OpenML URL for the object of the class entity with the given id."""
    # Sample url for a flow: openml.org/f/123
    return f"{openml.config.get_server_base_url()}/{cls._entity_letter()}/{id_}"

OpenMLStudy

OpenMLStudy(study_id: int | None, alias: str | None, benchmark_suite: int | None, name: str, description: str, status: str | None, creation_date: str | None, creator: int | None, tags: list[dict] | None, data: list[int] | None, tasks: list[int] | None, flows: list[int] | None, runs: list[int] | None, setups: list[int] | None)

Bases: BaseStudy

An OpenMLStudy represents the OpenML concept of a study (a collection of runs).

It contains the following information: name, id, description, creation date, creator id and a list of run ids.

According to this list of run ids, the study object receives a list of OpenML object ids (datasets, flows, tasks and setups).

Parameters

study_id : int the study id alias : str (optional) a string ID, unique on server (url-friendly) benchmark_suite : int (optional) the benchmark suite (another study) upon which this study is ran. can only be active if main entity type is runs. name : str the name of the study (meta-info) description : str brief description (meta-info) status : str Whether the study is in preparation, active or deactivated creation_date : str date of creation (meta-info) creator : int openml user id of the owner / creator tags : list(dict) The list of tags shows which tags are associated with the study. Each tag is a dict of (tag) name, window_start and write_access. data : list a list of data ids associated with this study tasks : list a list of task ids associated with this study flows : list a list of flow ids associated with this study runs : list a list of run ids associated with this study setups : list a list of setup ids associated with this study

Source code in openml/study/study.py

def __init__(  # noqa: PLR0913
    self,
    study_id: int | None,
    alias: str | None,
    benchmark_suite: int | None,
    name: str,
    description: str,
    status: str | None,
    creation_date: str | None,
    creator: int | None,
    tags: list[dict] | None,
    data: list[int] | None,
    tasks: list[int] | None,
    flows: list[int] | None,
    runs: list[int] | None,
    setups: list[int] | None,
):
    super().__init__(
        study_id=study_id,
        alias=alias,
        main_entity_type="run",
        benchmark_suite=benchmark_suite,
        name=name,
        description=description,
        status=status,
        creation_date=creation_date,
        creator=creator,
        tags=tags,
        data=data,
        tasks=tasks,
        flows=flows,
        runs=runs,
        setups=setups,
    )

id property

id: int | None

Return the id of the study.

openml_url property

openml_url: str | None

The URL of the object on the server, if it was uploaded, else None.

open_in_browser

open_in_browser() -> None

Opens the OpenML web page corresponding to this object in your default browser.

Source code in openml/base.py

def open_in_browser(self) -> None:
    """Opens the OpenML web page corresponding to this object in your default browser."""
    if self.openml_url is None:
        raise ValueError(
            "Cannot open element on OpenML.org when attribute `openml_url` is `None`",
        )

    webbrowser.open(self.openml_url)

publish

publish() -> OpenMLBase

Publish the object on the OpenML server.

Source code in openml/base.py

def publish(self) -> OpenMLBase:
    """Publish the object on the OpenML server."""
    file_elements = self._get_file_elements()

    if "description" not in file_elements:
        file_elements["description"] = self._to_xml()

    call = f"{_get_rest_api_type_alias(self)}/"
    response_text = openml._api_calls._perform_api_call(
        call,
        "post",
        file_elements=file_elements,
    )
    xml_response = xmltodict.parse(response_text)

    self._parse_publish_response(xml_response)
    return self

push_tag

push_tag(tag: str) -> None

Add a tag to the study.

Source code in openml/study/study.py

def push_tag(self, tag: str) -> None:
    """Add a tag to the study."""
    raise NotImplementedError("Tags for studies is not (yet) supported.")

remove_tag

remove_tag(tag: str) -> None

Remove a tag from the study.

Source code in openml/study/study.py

def remove_tag(self, tag: str) -> None:
    """Remove a tag from the study."""
    raise NotImplementedError("Tags for studies is not (yet) supported.")

url_for_id classmethod

url_for_id(id_: int) -> str

Return the OpenML URL for the object of the class entity with the given id.

Source code in openml/base.py

@classmethod
def url_for_id(cls, id_: int) -> str:
    """Return the OpenML URL for the object of the class entity with the given id."""
    # Sample url for a flow: openml.org/f/123
    return f"{openml.config.get_server_base_url()}/{cls._entity_letter()}/{id_}"

attach_to_study

attach_to_study(study_id: int, run_ids: list[int]) -> int

Attaches a set of runs to a study.

Parameters

study_id : int OpenML id of the study

list (int)

List of entities to link to the collection

Returns

int new size of the study (in terms of explicitly linked entities)

Source code in openml/study/functions.py

def attach_to_study(study_id: int, run_ids: list[int]) -> int:
    """Attaches a set of runs to a study.

    Parameters
    ----------
    study_id : int
        OpenML id of the study

    run_ids : list (int)
        List of entities to link to the collection

    Returns
    -------
    int
        new size of the study (in terms of explicitly linked entities)
    """
    # Interestingly, there's no need to tell the server about the entity type, it knows by itself
    result_xml = openml._api_calls._perform_api_call(
        call=f"study/{study_id}/attach",
        request_method="post",
        data={"ids": ",".join(str(x) for x in run_ids)},
    )
    result = xmltodict.parse(result_xml)["oml:study_attach"]
    return int(result["oml:linked_entities"])

attach_to_suite

attach_to_suite(suite_id: int, task_ids: list[int]) -> int

Attaches a set of tasks to a benchmarking suite.

Parameters

suite_id : int OpenML id of the study

list (int)

List of entities to link to the collection

Returns

int new size of the suite (in terms of explicitly linked entities)

Source code in openml/study/functions.py

def attach_to_suite(suite_id: int, task_ids: list[int]) -> int:
    """Attaches a set of tasks to a benchmarking suite.

    Parameters
    ----------
    suite_id : int
        OpenML id of the study

    task_ids : list (int)
        List of entities to link to the collection

    Returns
    -------
    int
        new size of the suite (in terms of explicitly linked entities)
    """
    return attach_to_study(suite_id, task_ids)

create_benchmark_suite

create_benchmark_suite(name: str, description: str, task_ids: list[int], alias: str | None = None) -> OpenMLBenchmarkSuite

Creates an OpenML benchmark suite (collection of entity types, where the tasks are the linked entity)

Parameters

name : str the name of the study (meta-info) description : str brief description (meta-info) task_ids : list a list of task ids associated with this study more can be added later with attach_to_suite. alias : str (optional) a string ID, unique on server (url-friendly)

Returns

OpenMLStudy A local OpenML study object (call publish method to upload to server)

Source code in openml/study/functions.py

def create_benchmark_suite(
    name: str,
    description: str,
    task_ids: list[int],
    alias: str | None = None,
) -> OpenMLBenchmarkSuite:
    """
    Creates an OpenML benchmark suite (collection of entity types, where
    the tasks are the linked entity)

    Parameters
    ----------
    name : str
        the name of the study (meta-info)
    description : str
        brief description (meta-info)
    task_ids : list
        a list of task ids associated with this study
        more can be added later with ``attach_to_suite``.
    alias : str (optional)
        a string ID, unique on server (url-friendly)

    Returns
    -------
    OpenMLStudy
        A local OpenML study object (call publish method to upload to server)
    """
    return OpenMLBenchmarkSuite(
        suite_id=None,
        alias=alias,
        name=name,
        description=description,
        status=None,
        creation_date=None,
        creator=None,
        tags=None,
        data=None,
        tasks=task_ids,
    )

create_study

create_study(name: str, description: str, run_ids: list[int] | None = None, alias: str | None = None, benchmark_suite: int | None = None) -> OpenMLStudy

Creates an OpenML study (collection of data, tasks, flows, setups and run), where the runs are the main entity (collection consists of runs and all entities (flows, tasks, etc) that are related to these runs)

Parameters

benchmark_suite : int (optional) the benchmark suite (another study) upon which this study is ran. name : str the name of the study (meta-info) description : str brief description (meta-info) run_ids : list, optional a list of run ids associated with this study, these can also be added later with attach_to_study. alias : str (optional) a string ID, unique on server (url-friendly) benchmark_suite: int (optional) the ID of the suite for which this study contains run results

Returns

OpenMLStudy A local OpenML study object (call publish method to upload to server)

Source code in openml/study/functions.py

def create_study(
    name: str,
    description: str,
    run_ids: list[int] | None = None,
    alias: str | None = None,
    benchmark_suite: int | None = None,
) -> OpenMLStudy:
    """
    Creates an OpenML study (collection of data, tasks, flows, setups and run),
    where the runs are the main entity (collection consists of runs and all
    entities (flows, tasks, etc) that are related to these runs)

    Parameters
    ----------
    benchmark_suite : int (optional)
        the benchmark suite (another study) upon which this study is ran.
    name : str
        the name of the study (meta-info)
    description : str
        brief description (meta-info)
    run_ids : list, optional
        a list of run ids associated with this study,
        these can also be added later with ``attach_to_study``.
    alias : str (optional)
        a string ID, unique on server (url-friendly)
    benchmark_suite: int (optional)
        the ID of the suite for which this study contains run results

    Returns
    -------
    OpenMLStudy
        A local OpenML study object (call publish method to upload to server)
    """
    return OpenMLStudy(
        study_id=None,
        alias=alias,
        benchmark_suite=benchmark_suite,
        name=name,
        description=description,
        status=None,
        creation_date=None,
        creator=None,
        tags=None,
        data=None,
        tasks=None,
        flows=None,
        runs=run_ids if run_ids != [] else None,
        setups=None,
    )

delete_study

delete_study(study_id: int) -> bool

Deletes a study from the OpenML server.

Parameters

study_id : int OpenML id of the study

Returns

bool True iff the deletion was successful. False otherwise

Source code in openml/study/functions.py

def delete_study(study_id: int) -> bool:
    """Deletes a study from the OpenML server.

    Parameters
    ----------
    study_id : int
        OpenML id of the study

    Returns
    -------
    bool
        True iff the deletion was successful. False otherwise
    """
    return openml.utils._delete_entity("study", study_id)

delete_suite

delete_suite(suite_id: int) -> bool

Deletes a study from the OpenML server.

Parameters

suite_id : int OpenML id of the study

Returns

bool True iff the deletion was successful. False otherwise

Source code in openml/study/functions.py

def delete_suite(suite_id: int) -> bool:
    """Deletes a study from the OpenML server.

    Parameters
    ----------
    suite_id : int
        OpenML id of the study

    Returns
    -------
    bool
        True iff the deletion was successful. False otherwise
    """
    return delete_study(suite_id)

detach_from_study

detach_from_study(study_id: int, run_ids: list[int]) -> int

Detaches a set of run ids from a study.

Parameters

study_id : int OpenML id of the study

list (int)

List of entities to unlink from the collection

Returns

int new size of the study (in terms of explicitly linked entities)

Source code in openml/study/functions.py

def detach_from_study(study_id: int, run_ids: list[int]) -> int:
    """Detaches a set of run ids from a study.

    Parameters
    ----------
    study_id : int
        OpenML id of the study

    run_ids : list (int)
        List of entities to unlink from the collection

    Returns
    -------
    int
        new size of the study (in terms of explicitly linked entities)
    """
    # Interestingly, there's no need to tell the server about the entity type, it knows by itself
    uri = "study/%d/detach" % study_id
    post_variables = {"ids": ",".join(str(x) for x in run_ids)}  # type: openml._api_calls.DATA_TYPE
    result_xml = openml._api_calls._perform_api_call(
        call=uri,
        request_method="post",
        data=post_variables,
    )
    result = xmltodict.parse(result_xml)["oml:study_detach"]
    return int(result["oml:linked_entities"])

detach_from_suite

detach_from_suite(suite_id: int, task_ids: list[int]) -> int

Detaches a set of task ids from a suite.

Parameters

suite_id : int OpenML id of the study

list (int)

List of entities to unlink from the collection

Returns

int new size of the study (in terms of explicitly linked entities)

Source code in openml/study/functions.py

def detach_from_suite(suite_id: int, task_ids: list[int]) -> int:
    """Detaches a set of task ids from a suite.

    Parameters
    ----------
    suite_id : int
        OpenML id of the study

    task_ids : list (int)
        List of entities to unlink from the collection

    Returns
    -------
    int
    new size of the study (in terms of explicitly linked entities)
    """
    return detach_from_study(suite_id, task_ids)

get_study

get_study(study_id: int | str, arg_for_backwards_compat: str | None = None) -> OpenMLStudy

Retrieves all relevant information of an OpenML study from the server.

Parameters

study id : int, str study id (numeric or alias)

str, optional

The example given in arxiv.org/pdf/1708.03731.pdf uses an older version of the API which required specifying the type of study, i.e. tasks. We changed the implementation of studies since then and split them up into suites (collections of tasks) and studies (collections of runs) so this argument is no longer needed.

Returns

OpenMLStudy The OpenML study object

Source code in openml/study/functions.py

def get_study(
    study_id: int | str,
    arg_for_backwards_compat: str | None = None,  # noqa: ARG001
) -> OpenMLStudy:  # F401
    """
    Retrieves all relevant information of an OpenML study from the server.

    Parameters
    ----------
    study id : int, str
        study id (numeric or alias)

    arg_for_backwards_compat : str, optional
        The example given in https://arxiv.org/pdf/1708.03731.pdf uses an older version of the
        API which required specifying the type of study, i.e. tasks. We changed the
        implementation of studies since then and split them up into suites (collections of tasks)
        and studies (collections of runs) so this argument is no longer needed.

    Returns
    -------
    OpenMLStudy
        The OpenML study object
    """
    if study_id == "OpenML100":
        message = (
            "It looks like you are running code from the OpenML100 paper. It still works, but lots "
            "of things have changed since then. Please use `get_suite('OpenML100')` instead."
        )
        warnings.warn(message, DeprecationWarning, stacklevel=2)
        openml.config.logger.warning(message)
        study = _get_study(study_id, entity_type="task")
        assert isinstance(study, OpenMLBenchmarkSuite)

        return study  # type: ignore

    study = _get_study(study_id, entity_type="run")
    assert isinstance(study, OpenMLStudy)
    return study

get_suite

get_suite(suite_id: int | str) -> OpenMLBenchmarkSuite

Retrieves all relevant information of an OpenML benchmarking suite from the server.

Parameters

study id : int, str study id (numeric or alias)

Returns

OpenMLSuite The OpenML suite object

Source code in openml/study/functions.py

def get_suite(suite_id: int | str) -> OpenMLBenchmarkSuite:
    """
    Retrieves all relevant information of an OpenML benchmarking suite from the server.

    Parameters
    ----------
    study id : int, str
        study id (numeric or alias)

    Returns
    -------
    OpenMLSuite
        The OpenML suite object
    """
    study = _get_study(suite_id, entity_type="task")
    assert isinstance(study, OpenMLBenchmarkSuite)

    return study

list_studies

list_studies(offset: int | None = None, size: int | None = None, status: str | None = None, uploader: list[str] | None = None, benchmark_suite: int | None = None) -> DataFrame

Return a list of all studies which are on OpenML.

Parameters

offset : int, optional The number of studies to skip, starting from the first. size : int, optional The maximum number of studies to show. status : str, optional Should be {active, in_preparation, deactivated, all}. By default active studies are returned. uploader : list (int), optional Result filter. Will only return studies created by these users. benchmark_suite : int, optional

Returns

datasets : dataframe Every dataset is represented by a dictionary containing the following information: - id - alias (optional) - name - benchmark_suite (optional) - status - creator - creation_date If qualities are calculated for the dataset, some of these are also returned.

Source code in openml/study/functions.py

def list_studies(
    offset: int | None = None,
    size: int | None = None,
    status: str | None = None,
    uploader: list[str] | None = None,
    benchmark_suite: int | None = None,
) -> pd.DataFrame:
    """
    Return a list of all studies which are on OpenML.

    Parameters
    ----------
    offset : int, optional
        The number of studies to skip, starting from the first.
    size : int, optional
        The maximum number of studies to show.
    status : str, optional
        Should be {active, in_preparation, deactivated, all}. By default active
        studies are returned.
    uploader : list (int), optional
        Result filter. Will only return studies created by these users.
    benchmark_suite : int, optional

    Returns
    -------
    datasets : dataframe
        Every dataset is represented by a dictionary containing
        the following information:
        - id
        - alias (optional)
        - name
        - benchmark_suite (optional)
        - status
        - creator
        - creation_date
        If qualities are calculated for the dataset, some of
        these are also returned.
    """
    listing_call = partial(
        _list_studies,
        main_entity_type="run",
        status=status,
        uploader=uploader,
        benchmark_suite=benchmark_suite,
    )
    batches = openml.utils._list_all(listing_call, offset=offset, limit=size)
    if len(batches) == 0:
        return pd.DataFrame()

    return pd.concat(batches)

list_suites

list_suites(offset: int | None = None, size: int | None = None, status: str | None = None, uploader: list[int] | None = None) -> DataFrame

Return a list of all suites which are on OpenML.

Parameters

offset : int, optional The number of suites to skip, starting from the first. size : int, optional The maximum number of suites to show. status : str, optional Should be {active, in_preparation, deactivated, all}. By default active suites are returned. uploader : list (int), optional Result filter. Will only return suites created by these users.

Returns

datasets : dataframe Every row is represented by a dictionary containing the following information: - id - alias (optional) - name - main_entity_type - status - creator - creation_date

Source code in openml/study/functions.py

def list_suites(
    offset: int | None = None,
    size: int | None = None,
    status: str | None = None,
    uploader: list[int] | None = None,
) -> pd.DataFrame:
    """
    Return a list of all suites which are on OpenML.

    Parameters
    ----------
    offset : int, optional
        The number of suites to skip, starting from the first.
    size : int, optional
        The maximum number of suites to show.
    status : str, optional
        Should be {active, in_preparation, deactivated, all}. By default active
        suites are returned.
    uploader : list (int), optional
        Result filter. Will only return suites created by these users.

    Returns
    -------
    datasets : dataframe
        Every row is represented by a dictionary containing the following information:
        - id
        - alias (optional)
        - name
        - main_entity_type
        - status
        - creator
        - creation_date
    """
    listing_call = partial(
        _list_studies,
        main_entity_type="task",
        status=status,
        uploader=uploader,
    )
    batches = openml.utils._list_all(listing_call, limit=size, offset=offset)
    if len(batches) == 0:
        return pd.DataFrame()

    return pd.concat(batches)

update_study_status

update_study_status(study_id: int, status: str) -> None

Updates the status of a study to either 'active' or 'deactivated'.

Parameters

study_id : int The data id of the dataset status : str, 'active' or 'deactivated'

Source code in openml/study/functions.py

def update_study_status(study_id: int, status: str) -> None:
    """
    Updates the status of a study to either 'active' or 'deactivated'.

    Parameters
    ----------
    study_id : int
        The data id of the dataset
    status : str,
        'active' or 'deactivated'
    """
    legal_status = {"active", "deactivated"}
    if status not in legal_status:
        raise ValueError(f"Illegal status value. Legal values: {legal_status}")
    data = {"study_id": study_id, "status": status}  # type: openml._api_calls.DATA_TYPE
    result_xml = openml._api_calls._perform_api_call("study/status/update", "post", data=data)
    result = xmltodict.parse(result_xml)
    server_study_id = result["oml:study_status_update"]["oml:id"]
    server_status = result["oml:study_status_update"]["oml:status"]
    if status != server_status or int(study_id) != int(server_study_id):
        # This should never happen
        raise ValueError("Study id/status does not collide")

update_suite_status

update_suite_status(suite_id: int, status: str) -> None

Updates the status of a study to either 'active' or 'deactivated'.

Parameters

suite_id : int The data id of the dataset status : str, 'active' or 'deactivated'

Source code in openml/study/functions.py

def update_suite_status(suite_id: int, status: str) -> None:
    """
    Updates the status of a study to either 'active' or 'deactivated'.

    Parameters
    ----------
    suite_id : int
        The data id of the dataset
    status : str,
        'active' or 'deactivated'
    """
    return update_study_status(suite_id, status)