Merge pull request #770 from Labelbox/sdubinin/al-4149

[AL-4149] Validate metric names against reserved names

Author: dubininsergey

labelbox/data/annotation_types/metrics/scalar.py

@@ -1,7 +1,7 @@
 from typing import Dict, Optional, Union
 from enum import Enum
 
-from pydantic import confloat
+from pydantic import confloat, validator
 
 from .base import ConfidenceValue, BaseMetric
 
@@ -16,6 +16,11 @@ class ScalarMetricAggregation(Enum):
     SUM = "SUM"
 
 
+RESERVED_METRIC_NAMES = ('true_positive_count', 'false_positive_count',
+                         'true_negative_count', 'false_negative_count',
+                         'precision', 'recall', 'f1', 'iou')
+
+
 class ScalarMetric(BaseMetric):
     """ Class representing scalar metrics
 
@@ -28,6 +33,16 @@ class ScalarMetric(BaseMetric):
     value: Union[ScalarMetricValue, ScalarMetricConfidenceValue]
     aggregation: ScalarMetricAggregation = ScalarMetricAggregation.ARITHMETIC_MEAN
 
+    @validator('metric_name')
+    def validate_metric_name(cls, name: Union[str, None]):
+        if name is None:
+            return None
+        clean_name = name.lower().strip()
+        if clean_name in RESERVED_METRIC_NAMES:
+            raise ValueError(f"`{clean_name}` is a reserved metric name. "
+                             "Please provide another value for `metric_name`.")
+        return name
+
     def dict(self, *args, **kwargs):
         res = super().dict(*args, **kwargs)
         if res.get('metric_name') is None:

labelbox/data/metrics/iou/iou.py

@@ -31,7 +31,7 @@ def miou_metric(ground_truths: List[Union[ObjectAnnotation,
     # If both gt and preds are empty there is no metric
     if iou is None:
         return []
-    return [ScalarMetric(metric_name="iou", value=iou)]
+    return [ScalarMetric(metric_name="custom_iou", value=iou)]
 
 
 def feature_miou_metric(ground_truths: List[Union[ObjectAnnotation,
@@ -62,7 +62,9 @@ def feature_miou_metric(ground_truths: List[Union[ObjectAnnotation,
         if value is None:
             continue
         metrics.append(
-            ScalarMetric(metric_name="iou", feature_name=key, value=value))
+            ScalarMetric(metric_name="custom_iou",
+                         feature_name=key,
+                         value=value))
     return metrics
 
 

tests/data/annotation_types/test_metrics.py

@@ -5,6 +5,7 @@
 from labelbox.data.annotation_types.metrics import ConfusionMatrixMetric, ScalarMetric
 from labelbox.data.annotation_types.collection import LabelList
 from labelbox.data.annotation_types import ScalarMetric, Label, ImageData
+from labelbox.data.annotation_types.metrics.scalar import RESERVED_METRIC_NAMES
 
 
 def test_legacy_scalar_metric():
@@ -56,7 +57,7 @@ def test_legacy_scalar_metric():
 ])
 def test_custom_scalar_metric(feature_name, subclass_name, aggregation, value):
     kwargs = {'aggregation': aggregation} if aggregation is not None else {}
-    metric = ScalarMetric(metric_name="iou",
+    metric = ScalarMetric(metric_name="custom_iou",
                           value=value,
                           feature_name=feature_name,
                           subclass_name=subclass_name,
@@ -80,7 +81,7 @@ def test_custom_scalar_metric(feature_name, subclass_name, aggregation, value):
             'value':
                 value,
             'metric_name':
-                'iou',
+                'custom_iou',
             **({
                 'feature_name': feature_name
             } if feature_name else {}),
@@ -192,3 +193,10 @@ def test_invalid_number_of_confidence_scores():
             metric_name="too many scores",
             value={i / 20.: [0, 1, 2, 3] for i in range(20)})
     assert "Number of confidence scores must be greater" in str(exc_info.value)
+
+
+@pytest.mark.parametrize("metric_name", RESERVED_METRIC_NAMES)
+def test_reserved_names(metric_name: str):
+    with pytest.raises(ValidationError) as exc_info:
+        ScalarMetric(metric_name=metric_name, value=0.5)
+    assert 'is a reserved metric name' in exc_info.value.errors()[0]['msg']

tests/data/assets/ndjson/custom_scalar_import.json

@@ -1,3 +1,3 @@
-[{"uuid" : "a22bbf6e-b2da-4abe-9a11-df84759f7672","dataRow" : {"id": "ckrmdnqj4000007msh9p2a27r"}, "metricValue" : 0.1, "metricName" : "iou", "featureName" : "sample_class", "subclassName" : "sample_subclass", "aggregation" : "SUM"},
- {"uuid" : "a22bbf6e-b2da-4abe-9a11-df84759f7672","dataRow" : {"id": "ckrmdnqj4000007msh9p2a27r"}, "metricValue" : 0.1, "metricName" : "iou", "featureName" : "sample_class", "aggregation" : "SUM"},
- {"uuid" : "a22bbf6e-b2da-4abe-9a11-df84759f7672","dataRow" : {"id": "ckrmdnqj4000007msh9p2a27r"}, "metricValue" : { "0.1" : 0.1, "0.2" : 0.5}, "metricName" : "iou", "aggregation" : "SUM"}]
+[{"uuid" : "a22bbf6e-b2da-4abe-9a11-df84759f7672","dataRow" : {"id": "ckrmdnqj4000007msh9p2a27r"}, "metricValue" : 0.1, "metricName" : "custom_iou", "featureName" : "sample_class", "subclassName" : "sample_subclass", "aggregation" : "SUM"},
+ {"uuid" : "a22bbf6e-b2da-4abe-9a11-df84759f7672","dataRow" : {"id": "ckrmdnqj4000007msh9p2a27r"}, "metricValue" : 0.1, "metricName" : "custom_iou", "featureName" : "sample_class", "aggregation" : "SUM"},
+ {"uuid" : "a22bbf6e-b2da-4abe-9a11-df84759f7672","dataRow" : {"id": "ckrmdnqj4000007msh9p2a27r"}, "metricValue" : { "0.1" : 0.1, "0.2" : 0.5}, "metricName" : "custom_iou", "aggregation" : "SUM"}]

tests/data/metrics/iou/feature/test_feature_iou.py

@@ -13,7 +13,7 @@ def check_iou(pair):
         assert math.isclose(result[key], pair.expected[key])
 
     for metric in metrics:
-        assert metric.metric_name == "iou"
+        assert metric.metric_name == "custom_iou"
 
     if len(pair.expected):
         assert len(one_metrics)