Merge pull request #285 from Labelbox/ms/compute-conf-matrix

confusion matrix metrics

Author: msokoloff1

examples/model_diagnostics/custom-metrics.ipynb

@@ -164,7 +164,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from labelbox.data.annotation_types import ScalarMetric, MetricAggregation"
+    "from labelbox.data.annotation_types import ScalarMetric, ScalarMetricAggregation"
    ]
   },
   {
@@ -226,14 +226,14 @@
     "    metric_name = \"true_positives\",\n",
     "    feature_name = \"cat\",\n",
     "    value = 3,\n",
-    "    aggregation = MetricAggregation.SUM\n",
+    "    aggregation = ScalarMetricAggregation.SUM\n",
     ")\n",
     "\n",
     "feature_metric = ScalarMetric(\n",
     "    metric_name = \"true_positives\",\n",
     "    feature_name = \"dog\",\n",
     "    value = 4,\n",
-    "    aggregation = MetricAggregation.SUM\n",
+    "    aggregation = ScalarMetricAggregation.SUM\n",
     ")\n"
    ]
   },

labelbox/data/annotation_types/__init__.py

@@ -29,4 +29,8 @@
 from .collection import LabelGenerator
 
 from .metrics import ScalarMetric
-from .metrics import MetricAggregation
+from .metrics import ScalarMetricAggregation
+from .metrics import ConfusionMatrixMetric
+from .metrics import ConfusionMatrixAggregation
+from .metrics import ScalarMetricValue
+from .metrics import ConfusionMatrixMetricValue

labelbox/data/annotation_types/label.py

@@ -11,7 +11,7 @@
 from .classification import ClassificationAnswer
 from .data import VideoData, TextData, ImageData
 from .geometry import Mask
-from .metrics import ScalarMetric
+from .metrics import ScalarMetric, ConfusionMatrixMetric
 from .types import Cuid
 from .annotation import (ClassificationAnnotation, ObjectAnnotation,
                          VideoClassificationAnnotation, VideoObjectAnnotation)
@@ -23,7 +23,7 @@ class Label(BaseModel):
     annotations: List[Union[ClassificationAnnotation, ObjectAnnotation,
                             VideoObjectAnnotation,
                             VideoClassificationAnnotation, ScalarMetric,
-                            ScalarMetric]] = []
+                            ConfusionMatrixMetric]] = []
     extra: Dict[str, Any] = {}
 
     def object_annotations(self) -> List[ObjectAnnotation]:

labelbox/data/annotation_types/metrics/__init__.py

@@ -1,2 +1,2 @@
-from .scalar import ScalarMetric
-from .aggregations import MetricAggregation
+from .scalar import ScalarMetric, ScalarMetricAggregation, ScalarMetricValue
+from .confusion_matrix import ConfusionMatrixMetric, ConfusionMatrixAggregation, ConfusionMatrixMetricValue

labelbox/data/annotation_types/metrics/aggregations.py

@@ -0,0 +1,35 @@
+from abc import ABC
+from pydantic import ValidationError, confloat, BaseModel, validator
+from pydantic.error_wrappers import ErrorWrapper
+from typing import Dict, Optional, Any, Union
+
+ConfidenceValue = confloat(ge=0, le=1)
+
+MIN_CONFIDENCE_SCORES = 2
+MAX_CONFIDENCE_SCORES = 15
+
+
+class BaseMetric(BaseModel, ABC):
+    value: Union[Any, Dict[float, Any]]
+    feature_name: Optional[str] = None
+    subclass_name: Optional[str] = None
+    extra: Dict[str, Any] = {}
+
+    def dict(self, *args, **kwargs):
+        res = super().dict(*args, **kwargs)
+        return {k: v for k, v in res.items() if v is not None}
+
+    @validator('value')
+    def validate_value(cls, value):
+        if isinstance(value, Dict):
+            if not (MIN_CONFIDENCE_SCORES <= len(value) <=
+                    MAX_CONFIDENCE_SCORES):
+                raise ValidationError([
+                    ErrorWrapper(ValueError(
+                        "Number of confidence scores must be greater"
+                        f" than or equal to {MIN_CONFIDENCE_SCORES} and"
+                        f" less than or equal to {MAX_CONFIDENCE_SCORES}. Found {len(value)}"
+                    ),
+                                 loc='value')
+                ], cls)
+        return value

labelbox/data/annotation_types/metrics/base.py

@@ -0,0 +1,35 @@
+from enum import Enum
+from typing import Tuple, Dict, Union
+
+from pydantic import conint, Field
+from pydantic.main import BaseModel
+
+from .base import ConfidenceValue, BaseMetric
+
+Count = conint(ge=0, le=1e10)
+
+ConfusionMatrixMetricValue = Tuple[Count, Count, Count, Count]
+ConfusionMatrixMetricConfidenceValue = Dict[ConfidenceValue,
+                                            ConfusionMatrixMetricValue]
+
+
+class ConfusionMatrixAggregation(Enum):
+    CONFUSION_MATRIX = "CONFUSION_MATRIX"
+
+
+class ConfusionMatrixMetric(BaseMetric):
+    """ Class representing confusion matrix metrics.
+
+    In the editor, this provides precision, recall, and f-scores.
+    This should be used over multiple scalar metrics so that aggregations are accurate.
+
+    Value should be a tuple representing:
+      [True Positive Count, False Positive Count, True Negative Count, False Negative Count]
+
+    aggregation cannot be adjusted for confusion matrix metrics.
+    """
+    metric_name: str
+    value: Union[ConfusionMatrixMetricValue,
+                 ConfusionMatrixMetricConfidenceValue]
+    aggregation: ConfusionMatrixAggregation = Field(
+        ConfusionMatrixAggregation.CONFUSION_MATRIX, const=True)

labelbox/data/annotation_types/metrics/confusion_matrix.py

@@ -1,26 +1,35 @@
-from labelbox.data.annotation_types.metrics.aggregations import MetricAggregation
-from typing import Any, Dict, Optional
-from pydantic import BaseModel
+from typing import Dict, Optional, Union
+from enum import Enum
 
+from pydantic import confloat
 
-class ScalarMetric(BaseModel):
-    """ Class representing metrics
+from .base import ConfidenceValue, BaseMetric
 
-    # For backwards compatibility, metric_name is optional. This will eventually be deprecated
-    # The metric_name will be set to a default name in the editor if it is not set.
+ScalarMetricValue = confloat(ge=0, le=10_000)
+ScalarMetricConfidenceValue = Dict[ConfidenceValue, ScalarMetricValue]
 
-    # aggregation will be ignored wihtout providing a metric name.
-    # Not providing a metric name is deprecated.
+
+class ScalarMetricAggregation(Enum):
+    ARITHMETIC_MEAN = "ARITHMETIC_MEAN"
+    GEOMETRIC_MEAN = "GEOMETRIC_MEAN"
+    HARMONIC_MEAN = "HARMONIC_MEAN"
+    SUM = "SUM"
+
+
+class ScalarMetric(BaseMetric):
+    """ Class representing scalar metrics
+
+    For backwards compatibility, metric_name is optional.
+    The metric_name will be set to a default name in the editor if it is not set.
+    This is not recommended and support for empty metric_name fields will be removed.
+    aggregation will be ignored wihtout providing a metric name.
     """
-    value: float
     metric_name: Optional[str] = None
-    feature_name: Optional[str] = None
-    subclass_name: Optional[str] = None
-    aggregation: MetricAggregation = MetricAggregation.ARITHMETIC_MEAN
-    extra: Dict[str, Any] = {}
+    value: Union[ScalarMetricValue, ScalarMetricConfidenceValue]
+    aggregation: ScalarMetricAggregation = ScalarMetricAggregation.ARITHMETIC_MEAN
 
     def dict(self, *args, **kwargs):
         res = super().dict(*args, **kwargs)
-        if res['metric_name'] is None:
+        if res.get('metric_name') is None:
             res.pop('aggregation')
-        return {k: v for k, v in res.items() if v is not None}
+        return res

labelbox/data/annotation_types/metrics/scalar.py

@@ -0,0 +1,2 @@
+from .confusion_matrix import confusion_matrix_metric, feature_confusion_matrix_metric
+from .iou import miou_metric, feature_miou_metric

labelbox/data/metrics/__init__.py

@@ -0,0 +1,2 @@
+from .calculation import *
+from .confusion_matrix import *