Merge pull request #295 from Labelbox/ms/bulk-metadata-query

bulk metadata export

Author: msokoloff1

CHANGELOG.md

@@ -1,4 +1,18 @@
 # Changelog
+# Version 3.6.0 (2021-04-10)
+## Added
+* Bulk export metadata with `DataRowMetadataOntology.bulk_export()`
+* Add docstring examples of annotation types and a few helper methods
+
+## Updated
+* Update metadata notebook under examples/basics to include bulk_export.
+* Allow color to be a single integer when constructing Mask objects
+* Allow users to pass int arrays to RasterData and attempt coercion to uint8
+
+## Removed
+* data_row.metadata was removed in favor of bulk exports.
+
+
 # Version 3.5.0 (2021-15-09)
 ## Added
 * Diagnostics custom metrics

examples/basics/data_row_metadata.ipynb

@@ -55,9 +55,10 @@
    },
    "outputs": [],
    "source": [
-    "%%capture\n",
-    "!pip install --upgrade tensorflow-hub scikit-learn\n",
-    "!pip install --upgrade \"labelbox[data]\""
+    "!pip install -q --upgrade tensorflow-hub \\\n",
+    "                 scikit-learn \\\n",
+    "                 seaborn \\\n",
+    "                 \"labelbox[data]\""
    ]
   },
   {
@@ -90,12 +91,13 @@
     ")\n",
     "from sklearn.random_projection import GaussianRandomProjection\n",
     "import seaborn as sns\n",
-    "import datetime\n",
+    "from datetime import datetime\n",
     "from pprint import pprint\n",
     "import tensorflow_hub as hub\n",
     "from tqdm.notebook import tqdm\n",
     "import requests\n",
-    "import tensorflow as tf"
+    "import tensorflow as tf\n",
+    "from pprint import pprint"
    ]
   },
   {
@@ -361,7 +363,7 @@
     "        split = \"cko8sbscr0003h2dk04w86hof\"\n",
     "    \n",
     "    embeddings.append(list(model(processor(response.content), training=False)[0].numpy()))\n",
-    "    dt = datetime.datetime.utcnow() \n",
+    "    dt = datetime.utcnow() \n",
     "    message =\"my-new-message\"\n",
     "\n",
     "    uploads.append(\n",
@@ -525,7 +527,7 @@
    },
    "outputs": [],
    "source": [
-    "metadata = mdo.parse_metadata([datarow.metadata])"
+    "metadata = mdo.bulk_export([datarow.uid])[0]"
    ]
   },
   {
@@ -584,7 +586,7 @@
    },
    "outputs": [],
    "source": [
-    "len(client.get_data_row(deletes.data_row_id).metadata[\"fields\"])"
+    "len(mdo.bulk_export(deletes.data_row_id)[0].fields)"
    ]
   },
   {
@@ -608,7 +610,7 @@
    },
    "outputs": [],
    "source": [
-    "len(client.get_data_row(deletes.data_row_id).metadata[\"fields\"])"
+    "len(mdo.bulk_export(deletes.data_row_id)[0].fields)"
    ]
   },
   {

labelbox/__init__.py

@@ -1,5 +1,5 @@
 name = "labelbox"
-__version__ = "3.5.0"
+__version__ = "3.6.0"
 
 from labelbox.schema.project import Project
 from labelbox.client import Client

labelbox/schema/data_row.py

@@ -20,14 +20,12 @@ class DataRow(DbObject, Updateable, BulkDeletable):
         updated_at (datetime)
         created_at (datetime)
         media_attributes (dict): generated media attributes for the datarow
-        metadata (dict): uploaded metadata
 
         dataset (Relationship): `ToOne` relationship to Dataset
         created_by (Relationship): `ToOne` relationship to User
         organization (Relationship): `ToOne` relationship to Organization
         labels (Relationship): `ToMany` relationship to Label
         attachments (Relationship) `ToMany` relationship with AssetAttachment
-        metadata (Relationship): This Relationship is Deprecated. Please use `DataRow.attachments()` instead
     """
     external_id = Field.String("external_id")
     row_data = Field.String("row_data")
@@ -50,33 +48,6 @@ def __init__(self, *args, **kwargs):
         self.attachments.supports_filtering = False
         self.attachments.supports_sorting = False
 
-    @property
-    def metadata(self) -> Dict[str, Union[str, List[Dict]]]:
-        """Get metadata for datarow
-        """
-
-        query = """query GetDataRowMetadataBetaPyApi($dataRowID: ID!) {
-              dataRow(where: {id: $dataRowID}) {
-                customMetadata {
-                    value
-                    schemaId
-                }
-              }
-            }
-        """
-
-        metadata = self.client.execute(
-            query, {"dataRowID": self.uid})["dataRow"]["customMetadata"]
-
-        return {
-            "data_row_id":
-                self.uid,
-            "fields": [{
-                "schema_id": m["schemaId"],
-                "value": m["value"]
-            } for m in metadata]
-        }
-
     @staticmethod
     def bulk_delete(data_rows):
         """ Deletes all the given DataRows.

labelbox/schema/data_row_metadata.py

@@ -64,7 +64,7 @@ class DeleteDataRowMetadata(_CamelCaseMixin):
 
 class DataRowMetadataBatchResponse(_CamelCaseMixin):
     data_row_id: str
-    error: str
+    error: Optional[str] = None
     fields: List[Union[DataRowMetadataField, SchemaId]]
 
 
@@ -200,7 +200,7 @@ def parse_metadata(
         for dr in unparsed:
             fields = []
             for f in dr["fields"]:
-                schema = self.all_fields_id_index[f["schema_id"]]
+                schema = self.all_fields_id_index[f["schemaId"]]
                 if schema.kind == DataRowMetadataKind.enum:
                     continue
                 elif schema.kind == DataRowMetadataKind.option:
@@ -212,7 +212,7 @@ def parse_metadata(
 
                 fields.append(field)
             parsed.append(
-                DataRowMetadata(data_row_id=dr["data_row_id"], fields=fields))
+                DataRowMetadata(data_row_id=dr["dataRowId"], fields=fields))
         return parsed
 
     def bulk_upsert(
@@ -330,6 +330,44 @@ def _batch_delete(
                                  items,
                                  batch_size=self._batch_size)
 
+    def bulk_export(self, data_row_ids: List[str]) -> List[DataRowMetadata]:
+        """ Exports metadata for a list of data rows
+
+        >>> mdo.bulk_export([data_row.uid for data_row in data_rows])
+
+        Args:
+            data_row_ids: List of data data rows to fetch metadata for
+        Returns:
+            A list of DataRowMetadata.
+            There will be one DataRowMetadata for each data_row_id passed in.
+            This is true even if the data row does not have any meta data.
+            Data rows without metadata will have empty `fields`.
+
+        """
+
+        if not len(data_row_ids):
+            raise ValueError("Empty list passed")
+
+        def _bulk_export(_data_row_ids: List[str]) -> List[DataRowMetadata]:
+            query = """query dataRowCustomMetadataPyApi($dataRowIds: [ID!]!) {
+                dataRowCustomMetadata(where: {dataRowIds : $dataRowIds}) {
+                    dataRowId
+                    fields {
+                        value
+                        schemaId
+                    }
+                }
+            }
+            """
+            return self.parse_metadata(
+                self.client.execute(
+                    query,
+                    {"dataRowIds": _data_row_ids})['dataRowCustomMetadata'])
+
+        return _batch_operations(_bulk_export,
+                                 data_row_ids,
+                                 batch_size=self._batch_size)
+
     def _parse_upsert(
             self, metadatum: DataRowMetadataField
     ) -> List[_UpsertDataRowMetadataInput]: