Add constants + I/O for new conditions/experiments tables

* constants
* read/write experiment table
* add experiments table to Problem, and populate from yaml

To be complemented by separate pull requests.

Author: dweindl

doc/modules.rst

@@ -30,5 +30,8 @@ API Reference
    petab.v1.yaml
    petab.v2
    petab.v2.C
+   petab.v2.experiments
    petab.v2.lint
+   petab.v2.models
    petab.v2.problem
+   petab.v2.petab1to2

petab/schemas/petab_schema.v2.0.0.yaml

@@ -76,6 +76,10 @@ properties:
           description: List of PEtab condition files.
           $ref: "#/definitions/list_of_files"
 
+        experiment_files:
+          description: List of PEtab condition files.
+          $ref: "#/definitions/list_of_files"
+
         observable_files:
           description: List of PEtab observable files.
           $ref: "#/definitions/list_of_files"
@@ -92,7 +96,6 @@ properties:
         - model_files
         - observable_files
         - measurement_files
-        - condition_files
 
   extensions:
     type: object

petab/v1/C.py

@@ -337,6 +337,8 @@
 MODEL_LANGUAGE = "language"
 #: Condition files key in the YAML file
 CONDITION_FILES = "condition_files"
+#: Experiment files key in the YAML file
+EXPERIMENT_FILES = "experiment_files"
 #: Measurement files key in the YAML file
 MEASUREMENT_FILES = "measurement_files"
 #: Observable files key in the YAML file

petab/v2/C.py

@@ -10,9 +10,14 @@
 #: Observable ID column in the observable and measurement tables
 OBSERVABLE_ID = "observableId"
 
+#: Experiment ID column in the measurement table
+EXPERIMENT_ID = "experimentId"
+
+# TODO: remove
 #: Preequilibration condition ID column in the measurement table
 PREEQUILIBRATION_CONDITION_ID = "preequilibrationConditionId"
 
+# TODO: remove
 #: Simulation condition ID column in the measurement table
 SIMULATION_CONDITION_ID = "simulationConditionId"
 
@@ -40,13 +45,16 @@
 #: Mandatory columns of measurement table
 MEASUREMENT_DF_REQUIRED_COLS = [
     OBSERVABLE_ID,
+    # TODO: add
+    # EXPERIMENT_ID,
     SIMULATION_CONDITION_ID,
     MEASUREMENT,
     TIME,
 ]
 
 #: Optional columns of measurement table
 MEASUREMENT_DF_OPTIONAL_COLS = [
+    # TODO: remove
     PREEQUILIBRATION_CONDITION_ID,
     OBSERVABLE_PARAMETERS,
     NOISE_PARAMETERS,
@@ -125,9 +133,38 @@
 
 #: Condition ID column in the condition table
 CONDITION_ID = "conditionId"
+# TODO: removed?
 #: Condition name column in the condition table
 CONDITION_NAME = "conditionName"
 
+#: Column in the condition table with the ID of an entity that is changed
+TARGET_ID = "targetId"
+#: Column in the condition table with the type of value that is changed
+VALUE_TYPE = "valueType"
+#: Column in the condition table with the new value of the target entity
+TARGET_VALUE = "targetValue"
+# value types:
+VT_CONSTANT = "constant"
+VT_INITIAL = "initial"
+VT_RATE = "rate"
+VT_ASSIGNMENT = "assignment"
+VT_RELATIVE_RATE = "relativeRate"
+VT_RELATIVE_ASSIGNMENT = "relativeAssignment"
+VALUE_TYPES = [
+    VT_CONSTANT,
+    VT_INITIAL,
+    VT_RATE,
+    VT_ASSIGNMENT,
+    VT_RELATIVE_RATE,
+    VT_RELATIVE_ASSIGNMENT,
+]
+
+CONDITION_DF_COLS = [
+    CONDITION_ID,
+    TARGET_ID,
+    VALUE_TYPE,
+    TARGET_VALUE,
+]
 
 # OBSERVABLES
 

petab/v2/__init__.py

@@ -5,6 +5,10 @@
 from warnings import warn
 
 from ..v1 import *  # noqa: F403, F401, E402
+from .experiments import (  # noqa: F401
+    get_experiment_df,
+    write_experiment_df,
+)
 
 # import after v1
 from .problem import Problem  # noqa: F401

petab/v2/experiments.py

@@ -0,0 +1,55 @@
+"""Functions operating on the PEtab experiments table."""
+from pathlib import Path
+
+import pandas as pd
+
+from .C import EXPERIMENT_ID
+
+__all__ = ["get_experiment_df", "write_experiment_df"]
+
+
+def get_experiment_df(
+    experiments_file: str | pd.DataFrame | Path | None,
+) -> pd.DataFrame | None:
+    """
+    Read the provided observable file into a ``pandas.Dataframe``.
+
+    Arguments:
+        observable_file: Name of the file to read from or pandas.Dataframe.
+
+    Returns:
+        Observable DataFrame
+    """
+    if experiments_file is None:
+        return experiments_file
+
+    if isinstance(experiments_file, str | Path):
+        experiments_file = pd.read_csv(
+            experiments_file, sep="\t", float_precision="round_trip"
+        )
+
+    if not isinstance(experiments_file.index, pd.RangeIndex):
+        experiments_file.reset_index(
+            drop=experiments_file.index.name != EXPERIMENT_ID,
+            inplace=True,
+        )
+
+    try:
+        experiments_file.set_index([EXPERIMENT_ID], inplace=True)
+    except KeyError:
+        raise KeyError(
+            f"Experiment table missing mandatory field {EXPERIMENT_ID}."
+        ) from None
+
+    return experiments_file
+
+
+def write_experiment_df(df: pd.DataFrame, filename: str | Path) -> None:
+    """Write PEtab experiments table
+
+    Arguments:
+        df: PEtab experiments table
+        filename: Destination file name
+    """
+    df = get_experiment_df(df)
+    df.to_csv(filename, sep="\t", index=True)

petab/v2/problem.py

@@ -4,6 +4,7 @@
 import logging
 import os
 import tempfile
+import warnings
 from math import nan
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -24,6 +25,7 @@
 from ..v1.C import *  # noqa: F403
 from ..v1.models.model import Model, model_factory
 from ..v1.yaml import get_path_prefix
+from . import experiments
 
 if TYPE_CHECKING:
     from ..v2.lint import ValidationIssue, ValidationResultList, ValidationTask
@@ -38,6 +40,7 @@ class Problem:
 
     - model
     - condition table
+    - experiment table
     - measurement table
     - parameter table
     - observables table
@@ -47,6 +50,7 @@ class Problem:
 
     Parameters:
         condition_df: PEtab condition table
+        experiment_df: PEtab experiment table
         measurement_df: PEtab measurement table
         parameter_df: PEtab parameter table
         observable_df: PEtab observable table
@@ -60,6 +64,7 @@ def __init__(
         self,
         model: Model = None,
         condition_df: pd.DataFrame = None,
+        experiment_df: pd.DataFrame = None,
         measurement_df: pd.DataFrame = None,
         parameter_df: pd.DataFrame = None,
         visualization_df: pd.DataFrame = None,
@@ -70,6 +75,7 @@ def __init__(
         from ..v2.lint import default_validation_tasks
 
         self.condition_df: pd.DataFrame | None = condition_df
+        self.experiment_df: pd.DataFrame | None = experiment_df
         self.measurement_df: pd.DataFrame | None = measurement_df
         self.parameter_df: pd.DataFrame | None = parameter_df
         self.visualization_df: pd.DataFrame | None = visualization_df
@@ -81,8 +87,22 @@ def __init__(
             ValidationTask
         ] = default_validation_tasks.copy()
 
+        if self.experiment_df is not None:
+            warnings.warn(
+                "The experiment table is not yet supported and "
+                "will be ignored.",
+                stacklevel=2,
+            )
+
     def __str__(self):
         model = f"with model ({self.model})" if self.model else "without model"
+
+        experiments = (
+            f"{self.experiment_df.shape[0]} experiments"
+            if self.experiment_df is not None
+            else "without experiments table"
+        )
+
         conditions = (
             f"{self.condition_df.shape[0]} conditions"
             if self.condition_df is not None
@@ -112,8 +132,8 @@ def __str__(self):
             parameters = "without parameter_df table"
 
         return (
-            f"PEtab Problem {model}, {conditions}, {observables}, "
-            f"{measurements}, {parameters}"
+            f"PEtab Problem {model}, {conditions}, {experiments}, "
+            f"{observables}, {measurements}, {parameters}"
         )
 
     @staticmethod
@@ -230,6 +250,16 @@ def get_path(filename):
             else None
         )
 
+        experiment_files = [
+            get_path(f) for f in problem0.get(EXPERIMENT_FILES, [])
+        ]
+        # If there are multiple tables, we will merge them
+        experiment_df = (
+            core.concat_tables(experiment_files, experiments.get_experiment_df)
+            if experiment_files
+            else None
+        )
+
         visualization_files = [
             get_path(f) for f in problem0.get(VISUALIZATION_FILES, [])
         ]
@@ -260,6 +290,7 @@ def get_path(filename):
 
         return Problem(
             condition_df=condition_df,
+            experiment_df=experiment_df,
             measurement_df=measurement_df,
             parameter_df=parameter_df,
             observable_df=observable_df,

pytest.ini

@@ -7,3 +7,5 @@ filterwarnings =
     ignore:Support for PEtab2.0 is experimental:UserWarning
     ignore:.*inspect.getargspec\(\) is deprecated.*:DeprecationWarning
     ignore:.*Passing unrecognized arguments to super\(PyDevIPCompleter6\).*:DeprecationWarning
+    # TODO: until we have proper v2 support
+    ignore:The experiment table is not yet supported and will be ignored:UserWarning

tests/v2/test_experiments.py

@@ -0,0 +1,30 @@
+"""Tests related to ``petab.v2.experiments``."""
+from tempfile import NamedTemporaryFile
+
+import pandas as pd
+
+from petab.v2.C import CONDITION_ID, EXPERIMENT_ID, TIME
+from petab.v2.experiments import get_experiment_df, write_experiment_df
+
+
+def test_experiment_df_io():
+    # Test None
+    assert get_experiment_df(None) is None
+
+    # Test DataFrame
+    df = pd.DataFrame(
+        {
+            EXPERIMENT_ID: ["e1", "e2"],
+            CONDITION_ID: ["c1", "c2"],
+            TIME: [0, 1],
+        }
+    )
+    df = get_experiment_df(df)
+    assert EXPERIMENT_ID not in df.columns
+    assert df.shape == (2, 2)
+
+    # Test writing to file and round trip
+    with NamedTemporaryFile() as tmp:
+        write_experiment_df(df, tmp.name)
+        df2 = get_experiment_df(tmp.name)
+        assert df.equals(df2)

tests/v2/test_problem.py

@@ -7,6 +7,7 @@
 from petab.v2 import Problem
 from petab.v2.C import (
     CONDITION_ID,
+    EXPERIMENT_ID,
     MEASUREMENT,
     NOISE_FORMULA,
     OBSERVABLE_FORMULA,
@@ -54,6 +55,7 @@ def test_problem_from_yaml_multiple_files():
       measurement_files: [measurements1.tsv, measurements2.tsv]
       observable_files: [observables1.tsv, observables2.tsv]
       model_files:
+      experiment_files: [experiments1.tsv, experiments2.tsv]
     """
 
     with tempfile.TemporaryDirectory() as tmpdir:
@@ -72,6 +74,17 @@ def test_problem_from_yaml_multiple_files():
                 condition_df, Path(tmpdir, f"conditions{i}.tsv")
             )
 
+            experiment_df = pd.DataFrame(
+                {
+                    EXPERIMENT_ID: [f"experiment{i}"],
+                    TIME: [0],
+                    CONDITION_ID: [f"condition{i}"],
+                }
+            ).set_index([EXPERIMENT_ID])
+            petab.write_experiment_df(
+                experiment_df, Path(tmpdir, f"experiments{i}.tsv")
+            )
+
             measurement_df = pd.DataFrame(
                 {
                     SIMULATION_CONDITION_ID: [f"condition{i}"],
@@ -105,3 +118,4 @@ def test_problem_from_yaml_multiple_files():
         assert petab_problem.measurement_df.shape[0] == 2
         assert petab_problem.observable_df.shape[0] == 2
         assert petab_problem.condition_df.shape[0] == 2
+        assert petab_problem.experiment_df.shape[0] == 2