fix gt and pred order

Author: Matt Sokoloff

labelbox/data/metrics/__init__.py

@@ -0,0 +1,2 @@
+from .confusion_matrix import confusion_matrix_metric, feature_confusion_matrix_metric
+from .iou import miou_metric, feature_miou_metric

labelbox/data/metrics/confusion_matrix/__init__.py

@@ -0,0 +1,2 @@
+from .calculation import *
+from .confusion_matrix import *

labelbox/data/metrics/confusion_matrix/calculation.py

@@ -31,7 +31,7 @@ def confusion_matrix(ground_truths: List[Union[ObjectAnnotation,
         Returns None if there are no annotations in ground_truth or prediction annotations
     """
 
-    annotation_pairs = get_feature_pairs(predictions, ground_truths)
+    annotation_pairs = get_feature_pairs(ground_truths, predictions)
     conf_matrix = [
         feature_confusion_matrix(annotation_pair[0], annotation_pair[1],
                                  include_subclasses, iou)
@@ -166,7 +166,7 @@ def object_pair_confusion_matrix(pairs: List[Tuple[ObjectAnnotation,
     matched_predictions = set()
     matched_ground_truths = set()
 
-    for prediction, ground_truth, agreement in pairs:
+    for ground_truth, prediction, agreement in pairs:
         prediction_id = id(prediction)
         ground_truth_id = id(ground_truth)
         prediction_ids.add(prediction_id)
@@ -177,14 +177,15 @@ def object_pair_confusion_matrix(pairs: List[Tuple[ObjectAnnotation,
          ground_truth_id not in matched_ground_truths:
             if include_subclasses and (ground_truth.classifications or
                                        prediction.classifications):
-                if miou(prediction.classifications,
-                        ground_truth.classifications,
+                if miou(ground_truth.classifications,
+                        prediction.classifications,
                         include_subclasses=False) < 1.:
                     # Incorrect if the subclasses don't 100% agree then there is no match
                     continue
             matched_predictions.add(prediction_id)
             matched_ground_truths.add(ground_truth_id)
     tps = len(matched_ground_truths)
+
     fps = len(prediction_ids.difference(matched_predictions))
     fns = len(ground_truth_ids.difference(matched_ground_truths))
     # Not defined for object detection.
@@ -210,11 +211,7 @@ def radio_confusion_matrix(ground_truth: Radio,
     key = get_identifying_key([prediction.answer], [ground_truth.answer])
     prediction_id = getattr(prediction.answer, key)
     ground_truth_id = getattr(ground_truth.answer, key)
-
-    if prediction_id == ground_truth_id:
-        return [1, 0, 0, 0]
-    else:
-        return [0, 1, 0, 1]
+    return [1, 0, 0, 0] if prediction_id == ground_truth_id else [0, 1, 0, 1]
 
 
 def checklist_confusion_matrix(

labelbox/data/metrics/confusion_matrix/confusion_matrix.py

@@ -66,7 +66,7 @@ def feature_confusion_matrix_metric(
         There will be one metric for each class in the union of ground truth and prediction classes.
     """
     # Classifications are supported because we just take a naive approach to them..
-    annotation_pairs = get_feature_pairs(predictions, ground_truths)
+    annotation_pairs = get_feature_pairs(ground_truths, predictions)
     metrics = []
     for key in annotation_pairs:
         value = feature_confusion_matrix(annotation_pairs[key][0],

labelbox/data/metrics/iou/calculation.py

@@ -222,7 +222,7 @@ def _get_vector_pairs(
     # Get iou score for all pairs of ground truths and predictions
     """
     pairs = []
-    for prediction, ground_truth in product(predictions, ground_truths):
+    for ground_truth, prediction in product(ground_truths, predictions):
         if isinstance(prediction.value, Geometry) and isinstance(
                 ground_truth.value, Geometry):
             if isinstance(prediction.value, (Line, Point)):
@@ -232,7 +232,7 @@ def _get_vector_pairs(
             else:
                 score = _polygon_iou(prediction.value.shapely,
                                      ground_truth.value.shapely)
-            pairs.append((prediction, ground_truth, score))
+            pairs.append((ground_truth, prediction, score))
     return pairs
 
 
@@ -243,12 +243,12 @@ def _get_mask_pairs(
     # Get iou score for all pairs of ground truths and predictions
     """
     pairs = []
-    for prediction, ground_truth in product(predictions, ground_truths):
+    for ground_truth, prediction in product(ground_truths, predictions):
         if isinstance(prediction.value, Mask) and isinstance(
                 ground_truth.value, Mask):
             score = _mask_iou(prediction.value.draw(color=1),
                               ground_truth.value.draw(color=1))
-            pairs.append((prediction, ground_truth, score))
+            pairs.append((ground_truth, prediction, score))
     return pairs
 
 

tests/data/metrics/confusion_matrix/test_confusion_matrix_data_row.py

@@ -13,8 +13,8 @@
 ])
 def test_overlapping_objects(tool_examples):
     for example in tool_examples:
-        score = confusion_matrix_metric(example.predictions,
-                                        example.ground_truths)
+        score = confusion_matrix_metric(example.ground_truths,
+                                        example.predictions)
 
         if len(example.expected) == 0:
             assert len(score) == 0
@@ -33,8 +33,8 @@ def test_overlapping_objects(tool_examples):
      fixture_ref('radio_pairs')])
 def test_overlapping_classifications(tool_examples):
     for example in tool_examples:
-        score = confusion_matrix_metric(example.predictions,
-                                        example.ground_truths)
+        score = confusion_matrix_metric(example.ground_truths,
+                                        example.predictions)
         if len(example.expected) == 0:
             assert len(score) == 0
         else:

tests/data/metrics/confusion_matrix/test_confusion_matrix_feature.py

@@ -13,8 +13,8 @@
 ])
 def test_overlapping_objects(tool_examples):
     for example in tool_examples:
-        metrics = feature_confusion_matrix_metric(example.predictions,
-                                                  example.ground_truths)
+        metrics = feature_confusion_matrix_metric(example.ground_truths,
+                                                  example.predictions)
 
         metrics = {r.feature_name: list(r.value) for r in metrics}
         if len(example.expected) == 0:
@@ -30,8 +30,8 @@ def test_overlapping_objects(tool_examples):
 def test_overlapping_classifications(tool_examples):
     for example in tool_examples:
 
-        metrics = feature_confusion_matrix_metric(example.predictions,
-                                                  example.ground_truths)
+        metrics = feature_confusion_matrix_metric(example.ground_truths,
+                                                  example.predictions)
 
         metrics = {r.feature_name: list(r.value) for r in metrics}
         if len(example.expected) == 0: