Feat: Add reports for stats from Trello (Issue #19)

.flake8

@@ -112,7 +112,8 @@ hang-closing = True
 ignore =
     E133,
     E203,
-    W503
+    W503,
+    E402
 # Specify the list of error codes you wish Flake8 to report.
 select =
     E,

.gitignore

@@ -14,7 +14,7 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
+#lib/
 lib64/
 parts/
 sdist/

README.md

@@ -19,7 +19,12 @@ Show the command line options by running
 python ./packages/redata_reports/run/main.py -h
 ```
 
-Prior to generating any reports, configure API endpoints and keys. Copy [`./packages/redata_reports/run/secrets.example.py`](packages/redata_reports/run/secrets.example.py) to `./packages/redata_reports/run/secrets.py` and edit the fields with the appropriate values. See the comments in that file for specific instructions. To generate reports locally, only the Figshare API credentials are needed.
+or
+```
+python ./packages/trello_reports/run/main.py -h
+```
+
+Prior to generating any reports, configure API endpoints and keys. Copy [`./lib/secrets.example.py`](lib/secrets.example.py) to `./lib/secrets.py` and edit the fields with the appropriate values. See the comments in that file for specific instructions. To generate reports locally, Google Sheets and DigitalOcean credentials are not needed.
 
 Generate a report. E.g the users report. A CSV will be output in the current working directory in this example.
 ```
@@ -30,7 +35,7 @@ python ./packages/redata_reports/run/main.py -r users -o
 
 The `--sync-to-dashboard` option uploads data to the Google dashboard (data is stored in a Google Sheet and displayed in a Looker Studio dashboard). Setting this flag includes `-u B -r items -r users`. The destination sheet is configured in `secrets.py`. 
 
-Prior to using `--sync-to-dashboard`, the target Google Sheet must be set up. See the [readme](gsheet_webapp/README.md) in `gsheet_webapp`. Once that is done, run:
+Prior to using `--sync-to-dashboard`, the target Google Sheet must be set up. See the [readme](gsheet_webapp/README.md) in `gsheet_webapp`. Once that is done, run (for example):
 ```
 python packages/redata_reports/main.py --sync-to-dashboard
 ```

deploy.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-doctl serverless deploy . --remote-build --env packages/redata_reports/run/secrets.py
+doctl serverless deploy . --remote-build --env lib/secrets.py

gsheet_webapp/Code.gs

@@ -16,7 +16,7 @@
 
 var SCRIPT_PROP = PropertiesService.getScriptProperties(); // new property service
 
-version="1.1.2"; //change when the version changes
+version="2.0.0"; //change when the version changes
 
 //Best practice for using BetterLog and logging to a spreadsheet:
 // You can add and set the property "BetterLogLevel" in File > Project Properties and change it to
@@ -33,8 +33,9 @@ function test(){
     curl -L "script url" -H "Content-Type: application/json"  --data '{"action":"insertupdate"}'
   */
 
-  e=testData_insertExists_users();
+  //e=testData_insertExists_users();
   //e=testData_insertExists_items();
+  e=testData_insertExists_curators();
 
   console.log(doPost(e).getContent());
 }
@@ -123,14 +124,18 @@ function insert_or_update(data, sheet_name, headRow = 1) {
         if (headers[i] == "Timestamp"){ // special case if you include a 'Timestamp' column
           row.push(new Date());
         } else { // else use header name to get data
-          row.push(item[headers[i]]);
+          row.push(item[headers[i].trim()]);
         }
       }
       rows.push(row);
     }
 
+    // clear sheet before adding values
+    if(sheet.getLastRow()>1){
+      sheet.deleteRows(2, sheet.getLastRow()-1);
+    }
+
     // more efficient to set values as [][] array than individually
-    sheet.deleteRows(2, sheet.getLastRow()-1);
     sheet.getRange(sheet.getLastRow()+1, 1, rows.length, rows[0].length).setValues(rows);
 
     SpreadsheetApp.flush();
@@ -153,4 +158,4 @@ function insert_or_update(data, sheet_name, headRow = 1) {
 function setup() {
     var doc = SpreadsheetApp.getActiveSpreadsheet();
     SCRIPT_PROP.setProperty("spreadsheet_id", doc.getId());
-}
+}

gsheet_webapp/README.md

@@ -9,6 +9,9 @@ This Sheet accepts data from the report generator via a POST and stores it for u
 	1. `Readme` (optional)
 	1. `items`. Create columns exactly matching the name and order of the keys in the `articles` dictionary from [`items_report.py`](../packages/redata_reports/run/items_report.py)
 	1. `users`. Create columns exactly matching the name and order of the keys in the `account_info` dictionary from [`userquota_report.py`](../packages/redata_reports/run/userquota_report.py)
+	1. `curators`. Create columns exactly matching the name and order of the `curators` dictionary from [`curators_report.py`](../packages/trello_reports/run/curators_report.py)
+	1. `curators-timeunpivot`. See [Data Transformation Setup](#data-transformation).
+	1. `curators-itemunpivot`. See [Data Transformation Setup](#data-transformation).
 
 ## WebApp
 
@@ -51,4 +54,25 @@ To deploy the update and keep the app URL the same:
 
 1. Deploy -> Manage deployments -> pencil icon -> Version dropdown -> New Version -> Deploy
 
-Note: a new version is always required, you can't update an existing version.
+Note: a new version is always required, you can't update an existing version.
+
+# Data Transformation
+
+Additional data transformation is required for some datasets for use in LookerStudio. Namely, the data from the `curators` sheet.
+
+## Setup
+
+Add the data manipulation script
+1. Go to Extensions -> Apps Script
+1. Create a new file `Unpivot.gs` and copy the contents of the corresponding file in this repo to it.
+
+Create the following formulas
+In `curators-timeunpivot`, in cell A1 enter, the following formula
+```
+=unpivot({curators!A1:A30,curators!D1:D30,curators!F1:F30,curators!H1:H30,curators!J1:J30,curators!L1:L30,curators!N1:N30,curators!P1:P30,curators!R1:R30,curators!T1:T30,curators!V1:V30,curators!X1:X30,curators!Z1:Z30,curators!AB1:AB30,curators!AD1:AD30,curators!AF1:AF30,curators!AH1:AH30,curators!AJ1:AJ30,curators!AL1:AL30,curators!AN1:AN30,curators!AP1:AP30,curators!AR1:AR30,curators!AT1:AT30,curators!AV1:AV30,curators!AX1:AX30,curators!AZ1:AZ30,curators!BB1:BB30,curators!BD1:BD30,curators!BF1:BF30,curators!BH1:BH30,curators!BJ1:BJ30,curators!BL1:BL30,curators!BN1:BN30,curators!BP1:BP30,curators!BR1:BR30,curators!BT1:BT30,curators!BV1:BV30,curators!BX1:BX30,curators!BZ1:BZ30,curators!CB1:CB30,curators!CD1:CD30,curators!CF1:CF30,curators!CH1:CH30,curators!CJ1:CJ30,curators!CL1:CL30,curators!CN1:CN30,curators!CP1:CP30,curators!CR1:CR30,curators!CT1:CT30,curators!CV1:CV30,curators!CX1:CX30,curators!CZ1:CZ30,curators!DB1:DB30,curators!DD1:DD30,curators!DF1:DF30},1,1,"time_category","time")
+```
+
+In `curators-itemunpivot` in cell A1, enter the following formula
+```
+=unpivot({curators!A1:A30,curators!C1:C30,curators!E1:E30,curators!G1:G30,curators!I1:I30,curators!K1:K30,curators!M1:M30,curators!O1:O30,curators!Q1:Q30,curators!S1:S30,curators!U1:U30,curators!W1:W30,curators!Y1:Y30,curators!AA1:AA30,curators!AC1:AC30,curators!AE1:AE30,curators!AG1:AG30,curators!AI1:AI30,curators!AK1:AK30,curators!AM1:AM30,curators!AO1:AO30,curators!AQ1:AQ30,curators!AS1:AS30,curators!AU1:AU30,curators!AW1:AW30,curators!AY1:AY30,curators!BA1:BA30,curators!BC1:BC30,curators!BE1:BE30,curators!BG1:BG30,curators!BI1:BI30,curators!BK1:BK30,curators!BM1:BM30,curators!BO1:BO30,curators!BQ1:BQ30,curators!BS1:BS30,curators!BU1:BU30,curators!BW1:BW30,curators!BY1:BY30,curators!CA1:CA30,curators!CC1:CC30,curators!CE1:CE30,curators!CG1:CG30,curators!CI1:CI30,curators!CK1:CK30,curators!CM1:CM30,curators!CO1:CO30,curators!CQ1:CQ30,curators!CS1:CS30,curators!CU1:CU30,curators!CW1:CW30,curators!CY1:CY30,curators!DA1:DA30,curators!DC1:DC30,curators!DE1:DE30},1,1,"item_category","count")
+```

gsheet_webapp/Tests.gs

@@ -55,7 +55,7 @@ function testData_insertExists_items(){
           }
       )
     }
-  };
+  }
 }
 
 function testData_insertExists_users(){
@@ -93,5 +93,146 @@ function testData_insertExists_users(){
           }
       )
     }
-  };
+  }
 }
+
+function testData_insertExists_curators(){
+  return {
+    "postData": {
+      "contents": JSON.stringify(
+        {
+          "action":"insertupdate",
+          "accesskey": PropertiesService.getScriptProperties().getProperty("accesskey"),
+          "sheet": "curators",
+          "data": 
+              [
+                {
+                  "username": "hadepoju",
+                  "id": "65a860d9a082cedfebebe8de",
+                  "total_items": 3,
+                  "total_time": 14707.716,
+                  "easy_items": 1,
+                  "easy_time": 938.044,
+                  "med_items": 1,
+                  "med_time": 1939.21,
+                  "hard_items": 1,
+                  "hard_time": 11830.462,
+                  "3M_items": 3,
+                  "3M_time": 14707.716,
+                  "6M_items": 3,
+                  "6M_time": 14707.716,
+                  "1Y_items": 3,
+                  "1Y_time": 14707.716,
+                  "2Y_items": 3,
+                  "2Y_time": 14707.716,
+                  "3M_easy_items": 1,
+                  "3M_easy_time": 938.044,
+                  "3M_med_items": 1,
+                  "3M_med_time": 1939.21,
+                  "3M_hard_items": 1,
+                  "3M_hard_time": 11830.462,
+                  "6M_easy_items": 1,
+                  "6M_easy_time": 938.044,
+                  "6M_med_items": 1,
+                  "6M_med_time": 1939.21,
+                  "6M_hard_items": 1,
+                  "6M_hard_time": 11830.462,
+                  "1Y_easy_items": 1,
+                  "1Y_easy_time": 938.044,
+                  "1Y_med_items": 1,
+                  "1Y_med_time": 1939.21,
+                  "1Y_hard_items": 1,
+                  "1Y_hard_time": 11830.462,
+                  "2Y_easy_items": 1,
+                  "2Y_easy_time": 938.044,
+                  "2Y_med_items": 1,
+                  "2Y_med_time": 1939.21,
+                  "2Y_hard_items": 1,
+                  "2Y_hard_time": 11830.462,
+                  "total_reviewer1_items": 1,
+                  "total_reviewer1_time": 11830.462,
+                  "total_reviewer2_items": 2,
+                  "total_reviewer2_time": 2877.254,
+                  "easy_reviewer1_items": 0,
+                  "easy_reviewer1_time": 0,
+                  "easy_reviewer2_items": 1,
+                  "easy_reviewer2_time": 938.044,
+                  "med_reviewer1_items": 0,
+                  "med_reviewer1_time": 0,
+                  "med_reviewer2_items": 1,
+                  "med_reviewer2_time": 1939.21,
+                  "hard_reviewer1_items": 1,
+                  "hard_reviewer1_time": 11830.462,
+                  "hard_reviewer2_items": 0,
+                  "hard_reviewer2_time": 0,
+                  "3M_reviewer1_items": 1,
+                  "3M_reviewer1_time": 11830.462,
+                  "3M_reviewer2_items": 2,
+                  "3M_reviewer2_time": 2877.254,
+                  "6M_reviewer1_items": 1,
+                  "6M_reviewer1_time": 11830.462,
+                  "6M_reviewer2_items": 2,
+                  "6M_reviewer2_time": 2877.254,
+                  "1Y_reviewer1_items": 1,
+                  "1Y_reviewer1_time": 11830.462,
+                  "1Y_reviewer2_items": 2,
+                  "1Y_reviewer2_time": 2877.254,
+                  "2Y_reviewer1_items": 1,
+                  "2Y_reviewer1_time": 11830.462,
+                  "2Y_reviewer2_items": 2,
+                  "2Y_reviewer2_time": 2877.254,
+                  "3M_easy_reviewer1_items": 0,
+                  "3M_easy_reviewer1_time": 0,
+                  "3M_easy_reviewer2_items": 1,
+                  "3M_easy_reviewer2_time": 938.044,
+                  "6M_easy_reviewer1_items": 0,
+                  "6M_easy_reviewer1_time": 0,
+                  "6M_easy_reviewer2_items": 1,
+                  "6M_easy_reviewer2_time": 938.044,
+                  "1Y_easy_reviewer1_items": 0,
+                  "1Y_easy_reviewer1_time": 0,
+                  "1Y_easy_reviewer2_items": 1,
+                  "1Y_easy_reviewer2_time": 938.044,
+                  "2Y_easy_reviewer1_items": 0,
+                  "2Y_easy_reviewer1_time": 0,
+                  "2Y_easy_reviewer2_items": 1,
+                  "2Y_easy_reviewer2_time": 938.044,
+                  "3M_med_reviewer1_items": 0,
+                  "3M_med_reviewer1_time": 0,
+                  "3M_med_reviewer2_items": 1,
+                  "3M_med_reviewer2_time": 1939.21,
+                  "6M_med_reviewer1_items": 0,
+                  "6M_med_reviewer1_time": 0,
+                  "6M_med_reviewer2_items": 1,
+                  "6M_med_reviewer2_time": 1939.21,
+                  "1Y_med_reviewer1_items": 0,
+                  "1Y_med_reviewer1_time": 0,
+                  "1Y_med_reviewer2_items": 1,
+                  "1Y_med_reviewer2_time": 1939.21,
+                  "2Y_med_reviewer1_items": 0,
+                  "2Y_med_reviewer1_time": 0,
+                  "2Y_med_reviewer2_items": 1,
+                  "2Y_med_reviewer2_time": 1939.21,
+                  "3M_hard_reviewer1_items": 1,
+                  "3M_hard_reviewer1_time": 11830.462,
+                  "3M_hard_reviewer2_items": 0,
+                  "3M_hard_reviewer2_time": 0,
+                  "6M_hard_reviewer1_items": 1,
+                  "6M_hard_reviewer1_time": 11830.462,
+                  "6M_hard_reviewer2_items": 0,
+                  "6M_hard_reviewer2_time": 0,
+                  "1Y_hard_reviewer1_items": 1,
+                  "1Y_hard_reviewer1_time": 11830.462,
+                  "1Y_hard_reviewer2_items": 0,
+                  "1Y_hard_reviewer2_time": 0,
+                  "2Y_hard_reviewer1_items": 1,
+                  "2Y_hard_reviewer1_time": 11830.462,
+                  "2Y_hard_reviewer2_items": 0,
+                  "2Y_hard_reviewer2_time": 0
+                }
+              ]
+        }
+      )
+    }
+  }
+}

gsheet_webapp/Unpivot.gs

@@ -0,0 +1,73 @@
+/**
+ * Unpivot a pivot table of any size.
+ *
+ * @param {A1:D30} data The pivot table.
+ * @param {1} fixColumns Number of columns, after which pivoted values begin. Default 1.
+ * @param {1} fixRows Number of rows (1 or 2), after which pivoted values begin. Default 1.
+ * @param {"city"} titlePivot The title of horizontal pivot values. Default "column".
+ * @param {"distance"[,...]} titleValue The title of pivot table values. Default "value".
+ * @return The unpivoted table
+ * @customfunction
+ */
+function unpivot(data,fixColumns,fixRows,titlePivot,titleValue) {
+  // source: https://stackoverflow.com/a/43681525
+
+  var fixColumns = fixColumns || 1; // how many columns are fixed
+  var fixRows = fixRows || 1; // how many rows are fixed
+  var titlePivot = titlePivot || 'column';
+  var titleValue = titleValue || 'value';
+  var ret=[],i,j,row,uniqueCols=1;
+
+  // we handle only 2 dimension arrays
+  if (!Array.isArray(data) || data.length < fixRows || !Array.isArray(data[0]) || data[0].length < fixColumns)
+    throw new Error('no data');
+  // we handle max 2 fixed rows
+  if (fixRows > 2)
+    throw new Error('max 2 fixed rows are allowed');
+
+  // fill empty cells in the first row with value set last in previous columns (for 2 fixed rows)
+  var tmp = '';
+  for (j=0;j<data[0].length;j++)
+    if (data[0][j] != '') 
+      tmp = data[0][j];
+    else
+      data[0][j] = tmp;
+
+  // for 2 fixed rows calculate unique column number
+  if (fixRows == 2)
+  {
+    uniqueCols = 0;
+    tmp = {};
+    for (j=fixColumns;j<data[1].length;j++)
+      if (typeof tmp[ data[1][j] ] == 'undefined')
+      {
+        tmp[ data[1][j] ] = 1;
+        uniqueCols++;
+      }
+  }
+
+  // return first row: fix column titles + pivoted values column title + values column title(s)
+  row = [];
+    for (j=0;j<fixColumns;j++) row.push(fixRows == 2 ? data[0][j]||data[1][j] : data[0][j]); // for 2 fixed rows we try to find the title in row 1 and row 2
+    for (j=3;j<arguments.length;j++) row.push(arguments[j]);
+  ret.push(row);
+
+  // processing rows (skipping the fixed columns, then dedicating a new row for each pivoted value)
+  for (i=fixRows; i<data.length && data[i].length > 0; i++)
+  {
+    // skip totally empty or only whitespace containing rows
+    if (data[i].join('').replace(/\s+/g,'').length == 0 ) continue;
+
+    // unpivot the row
+    row = [];
+    for (j=0;j<fixColumns && j<data[i].length;j++)
+      row.push(data[i][j]);
+    for (j=fixColumns;j<data[i].length;j+=uniqueCols)
+      ret.push( 
+        row.concat([data[0][j]]) // the first row title value
+        .concat(data[i].slice(j,j+uniqueCols)) // pivoted values
+      );
+  }
+
+  return ret;
+}