diff --git a/.flake8 b/.flake8 index c06dcf7..c48025e 100644 --- a/.flake8 +++ b/.flake8 @@ -112,7 +112,8 @@ hang-closing = True ignore = E133, E203, - W503 + W503, + E402 # Specify the list of error codes you wish Flake8 to report. select = E, diff --git a/.gitignore b/.gitignore index 22d81ec..bc33c81 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ dist/ downloads/ eggs/ .eggs/ -lib/ +#lib/ lib64/ parts/ sdist/ diff --git a/README.md b/README.md index 5b7f798..c89cd66 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,12 @@ Show the command line options by running python ./packages/redata_reports/run/main.py -h ``` -Prior to generating any reports, configure API endpoints and keys. Copy [`./packages/redata_reports/run/secrets.example.py`](packages/redata_reports/run/secrets.example.py) to `./packages/redata_reports/run/secrets.py` and edit the fields with the appropriate values. See the comments in that file for specific instructions. To generate reports locally, only the Figshare API credentials are needed. +or +``` +python ./packages/trello_reports/run/main.py -h +``` + +Prior to generating any reports, configure API endpoints and keys. Copy [`./lib/secrets.example.py`](lib/secrets.example.py) to `./lib/secrets.py` and edit the fields with the appropriate values. See the comments in that file for specific instructions. To generate reports locally, Google Sheets and DigitalOcean credentials are not needed. Generate a report. E.g the users report. A CSV will be output in the current working directory in this example. ``` @@ -30,7 +35,7 @@ python ./packages/redata_reports/run/main.py -r users -o The `--sync-to-dashboard` option uploads data to the Google dashboard (data is stored in a Google Sheet and displayed in a Looker Studio dashboard). Setting this flag includes `-u B -r items -r users`. The destination sheet is configured in `secrets.py`. -Prior to using `--sync-to-dashboard`, the target Google Sheet must be set up. See the [readme](gsheet_webapp/README.md) in `gsheet_webapp`. Once that is done, run: +Prior to using `--sync-to-dashboard`, the target Google Sheet must be set up. See the [readme](gsheet_webapp/README.md) in `gsheet_webapp`. Once that is done, run (for example): ``` python packages/redata_reports/main.py --sync-to-dashboard ``` diff --git a/deploy.sh b/deploy.sh index 71b8fb9..b79afcb 100644 --- a/deploy.sh +++ b/deploy.sh @@ -1,3 +1,3 @@ #!/bin/bash -doctl serverless deploy . --remote-build --env packages/redata_reports/run/secrets.py \ No newline at end of file +doctl serverless deploy . --remote-build --env lib/secrets.py \ No newline at end of file diff --git a/gsheet_webapp/Code.gs b/gsheet_webapp/Code.gs index fae4ba5..a2eef73 100644 --- a/gsheet_webapp/Code.gs +++ b/gsheet_webapp/Code.gs @@ -16,7 +16,7 @@ var SCRIPT_PROP = PropertiesService.getScriptProperties(); // new property service -version="1.1.2"; //change when the version changes +version="2.0.0"; //change when the version changes //Best practice for using BetterLog and logging to a spreadsheet: // You can add and set the property "BetterLogLevel" in File > Project Properties and change it to @@ -33,8 +33,9 @@ function test(){ curl -L "script url" -H "Content-Type: application/json" --data '{"action":"insertupdate"}' */ - e=testData_insertExists_users(); + //e=testData_insertExists_users(); //e=testData_insertExists_items(); + e=testData_insertExists_curators(); console.log(doPost(e).getContent()); } @@ -123,14 +124,18 @@ function insert_or_update(data, sheet_name, headRow = 1) { if (headers[i] == "Timestamp"){ // special case if you include a 'Timestamp' column row.push(new Date()); } else { // else use header name to get data - row.push(item[headers[i]]); + row.push(item[headers[i].trim()]); } } rows.push(row); } + // clear sheet before adding values + if(sheet.getLastRow()>1){ + sheet.deleteRows(2, sheet.getLastRow()-1); + } + // more efficient to set values as [][] array than individually - sheet.deleteRows(2, sheet.getLastRow()-1); sheet.getRange(sheet.getLastRow()+1, 1, rows.length, rows[0].length).setValues(rows); SpreadsheetApp.flush(); @@ -153,4 +158,4 @@ function insert_or_update(data, sheet_name, headRow = 1) { function setup() { var doc = SpreadsheetApp.getActiveSpreadsheet(); SCRIPT_PROP.setProperty("spreadsheet_id", doc.getId()); -} \ No newline at end of file +} diff --git a/gsheet_webapp/README.md b/gsheet_webapp/README.md index 8195022..c3b6644 100644 --- a/gsheet_webapp/README.md +++ b/gsheet_webapp/README.md @@ -9,6 +9,9 @@ This Sheet accepts data from the report generator via a POST and stores it for u 1. `Readme` (optional) 1. `items`. Create columns exactly matching the name and order of the keys in the `articles` dictionary from [`items_report.py`](../packages/redata_reports/run/items_report.py) 1. `users`. Create columns exactly matching the name and order of the keys in the `account_info` dictionary from [`userquota_report.py`](../packages/redata_reports/run/userquota_report.py) + 1. `curators`. Create columns exactly matching the name and order of the `curators` dictionary from [`curators_report.py`](../packages/trello_reports/run/curators_report.py) + 1. `curators-timeunpivot`. See [Data Transformation Setup](#data-transformation). + 1. `curators-itemunpivot`. See [Data Transformation Setup](#data-transformation). ## WebApp @@ -51,4 +54,25 @@ To deploy the update and keep the app URL the same: 1. Deploy -> Manage deployments -> pencil icon -> Version dropdown -> New Version -> Deploy -Note: a new version is always required, you can't update an existing version. \ No newline at end of file +Note: a new version is always required, you can't update an existing version. + +# Data Transformation + +Additional data transformation is required for some datasets for use in LookerStudio. Namely, the data from the `curators` sheet. + +## Setup + +Add the data manipulation script +1. Go to Extensions -> Apps Script +1. Create a new file `Unpivot.gs` and copy the contents of the corresponding file in this repo to it. + +Create the following formulas +In `curators-timeunpivot`, in cell A1 enter, the following formula +``` +=unpivot({curators!A1:A30,curators!D1:D30,curators!F1:F30,curators!H1:H30,curators!J1:J30,curators!L1:L30,curators!N1:N30,curators!P1:P30,curators!R1:R30,curators!T1:T30,curators!V1:V30,curators!X1:X30,curators!Z1:Z30,curators!AB1:AB30,curators!AD1:AD30,curators!AF1:AF30,curators!AH1:AH30,curators!AJ1:AJ30,curators!AL1:AL30,curators!AN1:AN30,curators!AP1:AP30,curators!AR1:AR30,curators!AT1:AT30,curators!AV1:AV30,curators!AX1:AX30,curators!AZ1:AZ30,curators!BB1:BB30,curators!BD1:BD30,curators!BF1:BF30,curators!BH1:BH30,curators!BJ1:BJ30,curators!BL1:BL30,curators!BN1:BN30,curators!BP1:BP30,curators!BR1:BR30,curators!BT1:BT30,curators!BV1:BV30,curators!BX1:BX30,curators!BZ1:BZ30,curators!CB1:CB30,curators!CD1:CD30,curators!CF1:CF30,curators!CH1:CH30,curators!CJ1:CJ30,curators!CL1:CL30,curators!CN1:CN30,curators!CP1:CP30,curators!CR1:CR30,curators!CT1:CT30,curators!CV1:CV30,curators!CX1:CX30,curators!CZ1:CZ30,curators!DB1:DB30,curators!DD1:DD30,curators!DF1:DF30},1,1,"time_category","time") +``` + +In `curators-itemunpivot` in cell A1, enter the following formula +``` +=unpivot({curators!A1:A30,curators!C1:C30,curators!E1:E30,curators!G1:G30,curators!I1:I30,curators!K1:K30,curators!M1:M30,curators!O1:O30,curators!Q1:Q30,curators!S1:S30,curators!U1:U30,curators!W1:W30,curators!Y1:Y30,curators!AA1:AA30,curators!AC1:AC30,curators!AE1:AE30,curators!AG1:AG30,curators!AI1:AI30,curators!AK1:AK30,curators!AM1:AM30,curators!AO1:AO30,curators!AQ1:AQ30,curators!AS1:AS30,curators!AU1:AU30,curators!AW1:AW30,curators!AY1:AY30,curators!BA1:BA30,curators!BC1:BC30,curators!BE1:BE30,curators!BG1:BG30,curators!BI1:BI30,curators!BK1:BK30,curators!BM1:BM30,curators!BO1:BO30,curators!BQ1:BQ30,curators!BS1:BS30,curators!BU1:BU30,curators!BW1:BW30,curators!BY1:BY30,curators!CA1:CA30,curators!CC1:CC30,curators!CE1:CE30,curators!CG1:CG30,curators!CI1:CI30,curators!CK1:CK30,curators!CM1:CM30,curators!CO1:CO30,curators!CQ1:CQ30,curators!CS1:CS30,curators!CU1:CU30,curators!CW1:CW30,curators!CY1:CY30,curators!DA1:DA30,curators!DC1:DC30,curators!DE1:DE30},1,1,"item_category","count") +``` \ No newline at end of file diff --git a/gsheet_webapp/Tests.gs b/gsheet_webapp/Tests.gs index b513d88..c758f13 100644 --- a/gsheet_webapp/Tests.gs +++ b/gsheet_webapp/Tests.gs @@ -55,7 +55,7 @@ function testData_insertExists_items(){ } ) } - }; + } } function testData_insertExists_users(){ @@ -93,5 +93,146 @@ function testData_insertExists_users(){ } ) } - }; + } } + +function testData_insertExists_curators(){ + return { + "postData": { + "contents": JSON.stringify( + { + "action":"insertupdate", + "accesskey": PropertiesService.getScriptProperties().getProperty("accesskey"), + "sheet": "curators", + "data": + [ + { + "username": "hadepoju", + "id": "65a860d9a082cedfebebe8de", + "total_items": 3, + "total_time": 14707.716, + "easy_items": 1, + "easy_time": 938.044, + "med_items": 1, + "med_time": 1939.21, + "hard_items": 1, + "hard_time": 11830.462, + "3M_items": 3, + "3M_time": 14707.716, + "6M_items": 3, + "6M_time": 14707.716, + "1Y_items": 3, + "1Y_time": 14707.716, + "2Y_items": 3, + "2Y_time": 14707.716, + "3M_easy_items": 1, + "3M_easy_time": 938.044, + "3M_med_items": 1, + "3M_med_time": 1939.21, + "3M_hard_items": 1, + "3M_hard_time": 11830.462, + "6M_easy_items": 1, + "6M_easy_time": 938.044, + "6M_med_items": 1, + "6M_med_time": 1939.21, + "6M_hard_items": 1, + "6M_hard_time": 11830.462, + "1Y_easy_items": 1, + "1Y_easy_time": 938.044, + "1Y_med_items": 1, + "1Y_med_time": 1939.21, + "1Y_hard_items": 1, + "1Y_hard_time": 11830.462, + "2Y_easy_items": 1, + "2Y_easy_time": 938.044, + "2Y_med_items": 1, + "2Y_med_time": 1939.21, + "2Y_hard_items": 1, + "2Y_hard_time": 11830.462, + "total_reviewer1_items": 1, + "total_reviewer1_time": 11830.462, + "total_reviewer2_items": 2, + "total_reviewer2_time": 2877.254, + "easy_reviewer1_items": 0, + "easy_reviewer1_time": 0, + "easy_reviewer2_items": 1, + "easy_reviewer2_time": 938.044, + "med_reviewer1_items": 0, + "med_reviewer1_time": 0, + "med_reviewer2_items": 1, + "med_reviewer2_time": 1939.21, + "hard_reviewer1_items": 1, + "hard_reviewer1_time": 11830.462, + "hard_reviewer2_items": 0, + "hard_reviewer2_time": 0, + "3M_reviewer1_items": 1, + "3M_reviewer1_time": 11830.462, + "3M_reviewer2_items": 2, + "3M_reviewer2_time": 2877.254, + "6M_reviewer1_items": 1, + "6M_reviewer1_time": 11830.462, + "6M_reviewer2_items": 2, + "6M_reviewer2_time": 2877.254, + "1Y_reviewer1_items": 1, + "1Y_reviewer1_time": 11830.462, + "1Y_reviewer2_items": 2, + "1Y_reviewer2_time": 2877.254, + "2Y_reviewer1_items": 1, + "2Y_reviewer1_time": 11830.462, + "2Y_reviewer2_items": 2, + "2Y_reviewer2_time": 2877.254, + "3M_easy_reviewer1_items": 0, + "3M_easy_reviewer1_time": 0, + "3M_easy_reviewer2_items": 1, + "3M_easy_reviewer2_time": 938.044, + "6M_easy_reviewer1_items": 0, + "6M_easy_reviewer1_time": 0, + "6M_easy_reviewer2_items": 1, + "6M_easy_reviewer2_time": 938.044, + "1Y_easy_reviewer1_items": 0, + "1Y_easy_reviewer1_time": 0, + "1Y_easy_reviewer2_items": 1, + "1Y_easy_reviewer2_time": 938.044, + "2Y_easy_reviewer1_items": 0, + "2Y_easy_reviewer1_time": 0, + "2Y_easy_reviewer2_items": 1, + "2Y_easy_reviewer2_time": 938.044, + "3M_med_reviewer1_items": 0, + "3M_med_reviewer1_time": 0, + "3M_med_reviewer2_items": 1, + "3M_med_reviewer2_time": 1939.21, + "6M_med_reviewer1_items": 0, + "6M_med_reviewer1_time": 0, + "6M_med_reviewer2_items": 1, + "6M_med_reviewer2_time": 1939.21, + "1Y_med_reviewer1_items": 0, + "1Y_med_reviewer1_time": 0, + "1Y_med_reviewer2_items": 1, + "1Y_med_reviewer2_time": 1939.21, + "2Y_med_reviewer1_items": 0, + "2Y_med_reviewer1_time": 0, + "2Y_med_reviewer2_items": 1, + "2Y_med_reviewer2_time": 1939.21, + "3M_hard_reviewer1_items": 1, + "3M_hard_reviewer1_time": 11830.462, + "3M_hard_reviewer2_items": 0, + "3M_hard_reviewer2_time": 0, + "6M_hard_reviewer1_items": 1, + "6M_hard_reviewer1_time": 11830.462, + "6M_hard_reviewer2_items": 0, + "6M_hard_reviewer2_time": 0, + "1Y_hard_reviewer1_items": 1, + "1Y_hard_reviewer1_time": 11830.462, + "1Y_hard_reviewer2_items": 0, + "1Y_hard_reviewer2_time": 0, + "2Y_hard_reviewer1_items": 1, + "2Y_hard_reviewer1_time": 11830.462, + "2Y_hard_reviewer2_items": 0, + "2Y_hard_reviewer2_time": 0 + } + ] + } + ) + } + } +} \ No newline at end of file diff --git a/gsheet_webapp/Unpivot.gs b/gsheet_webapp/Unpivot.gs new file mode 100644 index 0000000..c2fe181 --- /dev/null +++ b/gsheet_webapp/Unpivot.gs @@ -0,0 +1,73 @@ +/** + * Unpivot a pivot table of any size. + * + * @param {A1:D30} data The pivot table. + * @param {1} fixColumns Number of columns, after which pivoted values begin. Default 1. + * @param {1} fixRows Number of rows (1 or 2), after which pivoted values begin. Default 1. + * @param {"city"} titlePivot The title of horizontal pivot values. Default "column". + * @param {"distance"[,...]} titleValue The title of pivot table values. Default "value". + * @return The unpivoted table + * @customfunction + */ +function unpivot(data,fixColumns,fixRows,titlePivot,titleValue) { + // source: https://stackoverflow.com/a/43681525 + + var fixColumns = fixColumns || 1; // how many columns are fixed + var fixRows = fixRows || 1; // how many rows are fixed + var titlePivot = titlePivot || 'column'; + var titleValue = titleValue || 'value'; + var ret=[],i,j,row,uniqueCols=1; + + // we handle only 2 dimension arrays + if (!Array.isArray(data) || data.length < fixRows || !Array.isArray(data[0]) || data[0].length < fixColumns) + throw new Error('no data'); + // we handle max 2 fixed rows + if (fixRows > 2) + throw new Error('max 2 fixed rows are allowed'); + + // fill empty cells in the first row with value set last in previous columns (for 2 fixed rows) + var tmp = ''; + for (j=0;j 0; i++) + { + // skip totally empty or only whitespace containing rows + if (data[i].join('').replace(/\s+/g,'').length == 0 ) continue; + + // unpivot the row + row = []; + for (j=0;j 3)*'s'}" - return f"{value:.2f}" + f' {unit}' if append_unit else f"{value:.2f}" + return f'{value:.2f} {unit}' if append_unit else f'{value:.2f}' + + +def format_duration(duration_string: str, target_unit: str, append_unit: bool=False) -> str: + """ + Converts a duration string from one unit to another. + + Args: + duration_string: A string representing the duration, e.g., "10s", "2m", "3H", "5D", "1W", "0.5M". + Supported units: 's' (seconds), 'm' (minutes), 'H' (hours), + 'D' (days), 'W' (weeks), 'M' (months). + target_unit: The unit to convert to. Supported units are the same as above. + + Returns: + The converted duration as a string. + + Raises: + ValueError: If the input string format is invalid, units are unknown, + or conversion is not possible. + """ + + # Define conversion factors to a base unit (seconds) + # Note: 'M' (months) is approximated as 30 days for simplicity. + # For precise month conversions, a specific date context would be needed. + unit_to_seconds = { + 's': 1, + 'm': 60, + 'H': 60 * 60, + 'D': 24 * 60 * 60, + 'W': 7 * 24 * 60 * 60, + 'M': 30 * 24 * 60 * 60 # Approximation for months + } + + # Validate target unit + if target_unit not in unit_to_seconds: + raise ValueError(f"Unknown target unit: '{target_unit}'. " + "Supported units are 's', 'm', 'H', 'D', 'W', 'M'.") + + # Regular expression to parse the duration string + # It captures a number (integer or float) and then the unit character(s) + match = re.match(r"(\d+(\.\d+)?)([smHDWM])", duration_string) + + if not match: + raise ValueError(f"Invalid duration string format: '{duration_string}'. " + "Expected format: , e.g., '10s', '2.5H'.") + + value_str = match.group(1) + input_unit = match.group(3) + + try: + value = float(value_str) + except ValueError: + # This should ideally be caught by the regex, but as a safeguard + raise ValueError(f"Could not parse numerical value from '{value_str}'.") + + if input_unit not in unit_to_seconds: + raise ValueError(f"Unknown input unit: '{input_unit}'. " + "Supported units are 's', 'm', 'H', 'D', 'W', 'M'.") + + # Convert the input duration to seconds + duration_in_seconds = value * unit_to_seconds[input_unit] + + # Convert from seconds to the target unit + if unit_to_seconds[target_unit] == 0: + # This case should not happen with the current unit_to_seconds, + # but is a good safeguard against division by zero if factors change. + raise ValueError(f"Conversion factor for target unit '{target_unit}' is zero, cannot convert.") + + converted_duration = duration_in_seconds / unit_to_seconds[target_unit] + + return f'{converted_duration:.2f} {target_unit}' if append_unit else f'{converted_duration:.2f}' def get_report_outfile(reportname, prefix=''): diff --git a/packages/redata_reports/run/secrets.example.py b/lib/secrets.example.py similarity index 59% rename from packages/redata_reports/run/secrets.example.py rename to lib/secrets.example.py index 3c13931..4d391d0 100644 --- a/packages/redata_reports/run/secrets.example.py +++ b/lib/secrets.example.py @@ -5,8 +5,14 @@ from os import environ # Figshare API -api_url_base = '' -api_token = '' +api_figshare_url_base = '' +api_figshare_token = '' + +# Trello API +api_trello_url_base = '' +api_trello_key = '' +api_trello_token = '' +trello_board_id = '' # Google sheet gsheets_dashboard_post_url = '' @@ -14,11 +20,3 @@ # DO access token. When deployed as a function, all requests must send this token in the 't' parameter of the GET request do_token = '' - - -# ************************************************************** -environ['API_URL_BASE'] = api_url_base -environ['API_TOKEN'] = api_token -environ['GSHEETS_DASHBOARD_POST_URL'] = gsheets_dashboard_post_url -environ['GSHEETS_DASHBOARD_KEY'] = gsheets_dashboard_key -environ['TOKEN'] = do_token diff --git a/packages/redata_reports/run/build.sh b/packages/redata_reports/run/build.sh index 4da805b..5982300 100644 --- a/packages/redata_reports/run/build.sh +++ b/packages/redata_reports/run/build.sh @@ -7,4 +7,5 @@ set -e virtualenv virtualenv source virtualenv/bin/activate pip install -r requirements.txt +cp ../../../lib/*.py . deactivate \ No newline at end of file diff --git a/packages/redata_reports/run/items_report.py b/packages/redata_reports/run/items_report.py index 90c27ae..a97bf0c 100644 --- a/packages/redata_reports/run/items_report.py +++ b/packages/redata_reports/run/items_report.py @@ -4,11 +4,14 @@ # # Author: Fernando Rios +import sys import requests import simplejson as json from multiprocessing import Pool from datetime import datetime from os import environ + +sys.path.insert(0, 'lib/') import functions as f @@ -16,7 +19,7 @@ def get_institution_articles(): page = 1 article_list = [] while True: - api_url = '{0}/account/institution/articles?page={1}&page_size=1000'.format(environ['API_URL_BASE'], page) + api_url = '{0}/account/institution/articles?page={1}&page_size=1000'.format(environ['API_FIGSHARE_URL_BASE'], page) page += 1 response = requests.get(api_url, headers=f.get_request_headers()) @@ -37,14 +40,14 @@ def get_public_article_info(article_ids): article_ids = [article_ids] for id in article_ids: - api_url = '{0}/articles/{1}/versions'.format(environ['API_URL_BASE'], id) + api_url = '{0}/articles/{1}/versions'.format(environ['API_FIGSHARE_URL_BASE'], id) response = requests.get(api_url, headers=f.get_request_headers()) if response.status_code == 200: article_versions = json.loads(response.text) if len(article_versions) > 0: for version in article_versions: version_num = version['version'] - api_url = '{0}/articles/{1}/versions/{2}'.format(environ['API_URL_BASE'], id, version_num) + api_url = '{0}/articles/{1}/versions/{2}'.format(environ['API_FIGSHARE_URL_BASE'], id, version_num) response = requests.get(api_url, headers=f.get_request_headers()) if response.status_code == 200: @@ -69,7 +72,7 @@ def get_private_article_info(article_ids): article_ids = [article_ids] for id in article_ids: - api_url = '{0}/account/articles/{1}'.format(environ['API_URL_BASE'], id) + api_url = '{0}/account/articles/{1}'.format(environ['API_FIGSHARE_URL_BASE'], id) response = requests.get(api_url, headers=f.get_request_headers()) if response.status_code == 200: article_info.append(json.loads(response.text)) diff --git a/packages/redata_reports/run/main.py b/packages/redata_reports/run/main.py index 417d7c8..536a858 100644 --- a/packages/redata_reports/run/main.py +++ b/packages/redata_reports/run/main.py @@ -5,13 +5,16 @@ # Author: Fernando Rios import argparse +import sys from os import environ from version import __version__, __commit__ -import secrets -import functions as f import items_report import userquota_report +sys.path.insert(0, 'lib/') # Allows loading of shared functions when running locally. No effect when running as DO func. +import functions as f +import secrets + def init_argparse(): parser = argparse.ArgumentParser( @@ -36,7 +39,7 @@ def init_argparse(): ) parser.add_argument( '-u', '--units', choices=['B', 'KB', 'MB', 'GB', 'TB'], - default='GB', help='Set the output units. Default is %(default)s') + default='GB', help='Set the output size units. Default is %(default)s') return parser @@ -71,8 +74,8 @@ def run(args): if __name__ == '__main__': args = init_argparse().parse_args() - environ['API_URL_BASE'] = secrets.api_url_base - environ['API_TOKEN'] = secrets.api_token + environ['API_FIGSHARE_URL_BASE'] = secrets.api_figshare_url_base + environ['API_FIGSHARE_TOKEN'] = secrets.api_figshare_token environ['GSHEETS_DASHBOARD_POST_URL'] = secrets.gsheets_dashboard_post_url environ['GSHEETS_DASHBOARD_KEY'] = secrets.gsheets_dashboard_key environ['TOKEN'] = secrets.do_token diff --git a/packages/redata_reports/run/userquota_report.py b/packages/redata_reports/run/userquota_report.py index c071865..4624263 100644 --- a/packages/redata_reports/run/userquota_report.py +++ b/packages/redata_reports/run/userquota_report.py @@ -8,10 +8,13 @@ # # Author: Fernando Rios +import sys import requests import simplejson as json from multiprocessing import Pool from os import environ + +sys.path.insert(0, 'lib/') import functions as f @@ -19,7 +22,7 @@ def get_institution_accounts(): page = 1 accounts_list = [] while True: - api_url = '{0}/account/institution/accounts?page={1}&page_size=1000'.format(environ['API_URL_BASE'], page) + api_url = '{0}/account/institution/accounts?page={1}&page_size=1000'.format(environ['API_FIGSHARE_URL_BASE'], page) page += 1 response = requests.get(api_url, headers=f.get_request_headers()) @@ -33,14 +36,14 @@ def get_institution_accounts(): def get_account_info(id): - api_url = '{0}/account?impersonate={1}'.format(environ['API_URL_BASE'], id) + api_url = '{0}/account?impersonate={1}'.format(environ['API_FIGSHARE_URL_BASE'], id) response = requests.get(api_url, headers=f.get_request_headers()) if response.status_code == 200: data = json.loads(response.text) elif response.status_code == 403: # can't impersonate ourselves - api_url = '{0}/account'.format(environ['API_URL_BASE']) + api_url = '{0}/account'.format(environ['API_FIGSHARE_URL_BASE']) response = requests.get(api_url, headers=f.get_request_headers()) data = json.loads(response.text) else: diff --git a/packages/redata_reports/run/version.py b/packages/redata_reports/run/version.py index 6237c39..9c86b93 100644 --- a/packages/redata_reports/run/version.py +++ b/packages/redata_reports/run/version.py @@ -14,4 +14,4 @@ def get_commit(repo_path): __commit__ = get_commit('.') -__version__ = '1.1.1' +__version__ = '1.1.2' diff --git a/packages/trello_reports/run/.ignore b/packages/trello_reports/run/.ignore new file mode 100644 index 0000000..585854c --- /dev/null +++ b/packages/trello_reports/run/.ignore @@ -0,0 +1,2 @@ +# These files will not be included in the DO deployed package +secrets* \ No newline at end of file diff --git a/packages/trello_reports/run/__main__.py b/packages/trello_reports/run/__main__.py new file mode 100644 index 0000000..a6cebbb --- /dev/null +++ b/packages/trello_reports/run/__main__.py @@ -0,0 +1,26 @@ +# Entry point for DO functions + +import main as m +from os import environ + + +def main(event, context): + # Only allow authenticated requests + accesstoken = event.get("t", "") + if accesstoken != environ["TOKEN"]: + return { + "headers": {"Content-Type": "text/html"}, + "statusCode": 403, + "body": "Forbidden. Invalid token." + } + + args = m.init_argparse().parse_args() + args.sync_to_dashboard = True + + result = m.run(args) + + return { + "headers": {"Content-Type": "text/html"}, + "statusCode": 200, + "body": f"{result}" + } diff --git a/packages/trello_reports/run/build.sh b/packages/trello_reports/run/build.sh new file mode 100644 index 0000000..5982300 --- /dev/null +++ b/packages/trello_reports/run/build.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Script for remote DO builds + +set -e + +virtualenv virtualenv +source virtualenv/bin/activate +pip install -r requirements.txt +cp ../../../lib/*.py . +deactivate \ No newline at end of file diff --git a/packages/trello_reports/run/curator_report.py b/packages/trello_reports/run/curator_report.py new file mode 100644 index 0000000..7b41d42 --- /dev/null +++ b/packages/trello_reports/run/curator_report.py @@ -0,0 +1,589 @@ +# -*- coding: utf-8 -*- + +# Prints the usage for each public dataset +# +# Author: Fernando Rios + +import sys +import traceback +from ast import literal_eval +from datetime import datetime +from dateutil.relativedelta import relativedelta +from dateutil.parser import parse +from dateutil.tz import tzlocal +from os import environ +from trello import TrelloClient + +sys.path.insert(0, 'lib/') +import functions as f + +debug = 0 # 0 = off, 1 = print card name + output cards to cards.csv, 2 = 1 + print more card info, 3 = 2 + card filter results +tc = None +currdate = datetime.now(tzlocal()) + + +def get_card_curators(card, existing_curators: dict[str, any] = None): + """ + Returns the curators for the given Card object as a list of Trello usernames. + + If the existing_curators dictionary is passed in, if a curator is found that does not exist in the + dictionary, a new entry is created with the key equal to the value of the reviewer_N custom field (N is an integer). + Currently it is the value of the Trello username. + """ + + curators = [] + + if debug > 0: + print(card.name) + + for field in card.custom_fields: + if 'reviewer_' in field.name: + if debug > 1: + print(f' {field.name}: {field.value}') + + curators.append(field.value) + + if type(existing_curators) is dict and field.value not in existing_curators: + # Add a new curator to the list if they're not already there + # The value of the reviewer_x custom field in trello must be a trello username + existing_curators[field.value] = { + 'username': field.value, 'id': '', 'total_items': 0, 'total_time': 0, + 'easy_items': 0, 'easy_time': 0, 'med_items': 0, 'med_time': 0, 'hard_items': 0, 'hard_time': 0, + '3M_items': 0, '3M_time': 0, + '6M_items': 0, '6M_time': 0, + '1Y_items': 0, '1Y_time': 0, + '2Y_items': 0, '2Y_time': 0, + '3M_easy_items': 0, '3M_easy_time': 0, + '3M_med_items': 0, '3M_med_time': 0, + '3M_hard_items': 0, '3M_hard_time': 0, + '6M_easy_items': 0, '6M_easy_time': 0, + '6M_med_items': 0, '6M_med_time': 0, + '6M_hard_items': 0, '6M_hard_time': 0, + '1Y_easy_items': 0, '1Y_easy_time': 0, + '1Y_med_items': 0, '1Y_med_time': 0, + '1Y_hard_items': 0, '1Y_hard_time': 0, + '2Y_easy_items': 0, '2Y_easy_time': 0, + '2Y_med_items': 0, '2Y_med_time': 0, + '2Y_hard_items': 0, '2Y_hard_time': 0, + 'total_reviewer1_items': 0, 'total_reviewer1_time': 0, + 'total_reviewer2_items': 0, 'total_reviewer2_time': 0, + 'easy_reviewer1_items': 0, 'easy_reviewer1_time': 0, + 'easy_reviewer2_items': 0, 'easy_reviewer2_time': 0, + 'med_reviewer1_items': 0, 'med_reviewer1_time': 0, + 'med_reviewer2_items': 0, 'med_reviewer2_time': 0, + 'hard_reviewer1_items': 0, 'hard_reviewer1_time': 0, + 'hard_reviewer2_items': 0, 'hard_reviewer2_time': 0, + '3M_reviewer1_items': 0, '3M_reviewer1_time': 0, + '3M_reviewer2_items': 0, '3M_reviewer2_time': 0, + '6M_reviewer1_items': 0, '6M_reviewer1_time': 0, + '6M_reviewer2_items': 0, '6M_reviewer2_time': 0, + '1Y_reviewer1_items': 0, '1Y_reviewer1_time': 0, + '1Y_reviewer2_items': 0, '1Y_reviewer2_time': 0, + '2Y_reviewer1_items': 0, '2Y_reviewer1_time': 0, + '2Y_reviewer2_items': 0, '2Y_reviewer2_time': 0, + '3M_easy_reviewer1_items': 0, '3M_easy_reviewer1_time': 0, + '3M_easy_reviewer2_items': 0, '3M_easy_reviewer2_time': 0, + '6M_easy_reviewer1_items': 0, '6M_easy_reviewer1_time': 0, + '6M_easy_reviewer2_items': 0, '6M_easy_reviewer2_time': 0, + '1Y_easy_reviewer1_items': 0, '1Y_easy_reviewer1_time': 0, + '1Y_easy_reviewer2_items': 0, '1Y_easy_reviewer2_time': 0, + '2Y_easy_reviewer1_items': 0, '2Y_easy_reviewer1_time': 0, + '2Y_easy_reviewer2_items': 0, '2Y_easy_reviewer2_time': 0, + '3M_med_reviewer1_items': 0, '3M_med_reviewer1_time': 0, + '3M_med_reviewer2_items': 0, '3M_med_reviewer2_time': 0, + '6M_med_reviewer1_items': 0, '6M_med_reviewer1_time': 0, + '6M_med_reviewer2_items': 0, '6M_med_reviewer2_time': 0, + '1Y_med_reviewer1_items': 0, '1Y_med_reviewer1_time': 0, + '1Y_med_reviewer2_items': 0, '1Y_med_reviewer2_time': 0, + '2Y_med_reviewer1_items': 0, '2Y_med_reviewer1_time': 0, + '2Y_med_reviewer2_items': 0, '2Y_med_reviewer2_time': 0, + '3M_hard_reviewer1_items': 0, '3M_hard_reviewer1_time': 0, + '3M_hard_reviewer2_items': 0, '3M_hard_reviewer2_time': 0, + '6M_hard_reviewer1_items': 0, '6M_hard_reviewer1_time': 0, + '6M_hard_reviewer2_items': 0, '6M_hard_reviewer2_time': 0, + '1Y_hard_reviewer1_items': 0, '1Y_hard_reviewer1_time': 0, + '1Y_hard_reviewer2_items': 0, '1Y_hard_reviewer2_time': 0, + '2Y_hard_reviewer1_items': 0, '2Y_hard_reviewer1_time': 0, + '2Y_hard_reviewer2_items': 0, '2Y_hard_reviewer2_time': 0, + 'report_date': f.get_report_date().strftime('%Y-%m-%d %H:%M:%S') + } + + for member in tc.get_board(card.board_id).all_members(): + # Extract the id from the username + if member.username == field.value: + existing_curators[field.value]['id'] = member.id + + if debug > 1: + for plugin in card.plugin_data: + if plugin['idPlugin'] == '5c592ae4d74ac4407f4e3af3': + powerup_timedata = literal_eval(plugin['value'])['users'] + print(f' timedata: {powerup_timedata}') + + return curators + + +def get_card_time(card, userid: str = None): + """ + Returns the total time in seconds from the Activity powerup for this card. + If userid is given, limits the time to only the given userid + """ + + total = 0 + + for plugin in card.plugin_data: + if plugin['idPlugin'] == '5c592ae4d74ac4407f4e3af3': + powerup_timedata = literal_eval(plugin['value'])['users'] + for id, timedata in powerup_timedata.items(): + if userid and userid == id: + total = int(timedata['time']) / 1000 + if not userid: + total += int(timedata['time']) / 1000 + return total + + +def curator_items_time(existing_curators: dict[str, any], curators_field_prefix: str, card, cardfilter: dict[str, str] = None): + """ + For the given curators dictionary as generated by get_card_curators + + - Increments the [curators_field_prefix]_items count for each curator that contributed to the given card. + - Increments the [curators_field_prefix]_time value (in seconds) for each curator that contributed to the given card. + The increment is based on the logged time for that curator. + - Applies the operations only if the card meets ALL the criteria specified in cardfilter. If None, the operation is always applied. + + existing_curators is a dict set up by get_card_curators + """ + + if debug > 2: + print(f'cardfilter: {cardfilter}') + + # Turn the custom fields into a dict for ease of use + customfields = {} + for field in card.custom_fields: + customfields[field.name] = field.value + + include_card = False + filter_result = 'NoFilter' + if not cardfilter: + include_card = True + else: + filter_result = {} + for name, value in cardfilter.items(): + filter_result[name] = False + match name: + case 'difficulty': + if name in customfields.keys() and value == customfields[name]: + filter_result[name] = True + + if debug > 2: + print(f' {name} filter: result={filter_result[name]} | ' + + f'requested_difficulty={value} card_difficulty={customfields[name] if name in customfields.keys() else None}') + case 'published_date': + pubdate = '' + cutoffdate = '' + if name in customfields.keys(): + pubdate = parse(customfields[name]) + cutoffdate = currdate - relativedelta(months=float(value)) + if pubdate and pubdate >= cutoffdate: + filter_result[name] = True + + if debug > 2: + print(f' {name} filter: result={filter_result[name]} | ' + + f'card_pubdate={str(pubdate)} >= cutoff_date={str(cutoffdate)}') + case 'reviewer_1' | 'reviewer_2': + if name in customfields.keys(): + filter_result[name] = True + + include_card = all(filter_result.values()) + + if debug > 2: + print(f' include_card: {include_card} | {filter_result}') + print(' ---') + + if include_card: + for username, curator in existing_curators.items(): + reviewer1_id = '' + reviewer2_id = '' + + # Item counts for each curator, based on card filter criteria + for fieldname, fieldvalue in customfields.items(): + if 'reviewer_' in fieldname and username in fieldvalue: + + # Add to the item count, whether the curator was reviewer 1 or reviewer 2 + curator[curators_field_prefix + '_items'] += 1 + + # Now, add to the specific reviewer 1 or 2 counts for the given curator + # and record whether that curator was reviewer 1 or 2. + if 'reviewer_1' in fieldname and username in fieldvalue: + curator[curators_field_prefix + '_reviewer1_items'] += 1 + reviewer1_id = curator['id'] + if 'reviewer_2' in fieldname and username in fieldvalue: + curator[curators_field_prefix + '_reviewer2_items'] += 1 + reviewer2_id = curator['id'] + + # Time for each curator, based on card filter criteria + curator[curators_field_prefix + '_time'] += get_card_time(card, curator['id']) + + # Record time for the appropriate reviewer1 or reviewer2 categories for this curator (if this curator worked on the item) + if reviewer1_id: + curator[curators_field_prefix + '_reviewer1_time'] += get_card_time(card, reviewer1_id) + if reviewer2_id: + curator[curators_field_prefix + '_reviewer2_time'] += get_card_time(card, reviewer2_id) + + +def curator_total_items_time(existing_curators: dict[str, any], card): + return curator_items_time(existing_curators, 'total', card, None) + + +def curator_easy_items_time(existing_curators: dict[str, any], card): + # All items with difficulty=easy + return curator_items_time(existing_curators, 'easy', card, {'difficulty': 'easy'}) + + +def curator_med_items_time(existing_curators: dict[str, any], card): + # All items with difficulty=medium + return curator_items_time(existing_curators, 'med', card, {'difficulty': 'medium'}) + + +def curator_hard_items_time(existing_curators: dict[str, any], card): + # All items with difficulty=hard + return curator_items_time(existing_curators, 'hard', card, {'difficulty': 'hard'}) + + +def curator_3m_items_time(existing_curators: dict[str, any], card): + # All items published in last 3 months + return curator_items_time(existing_curators, '3M', card, {'published_date': 3}) + + +def curator_6m_items_time(existing_curators: dict[str, any], card): + # All items published in last 6 months + return curator_items_time(existing_curators, '6M', card, {'published_date': 6}) + + +def curator_1y_items_time(existing_curators: dict[str, any], card): + # All items published in last year + return curator_items_time(existing_curators, '1Y', card, {'published_date': 12}) + + +def curator_2y_items_time(existing_curators: dict[str, any], card): + # All items published in last 2 years + return curator_items_time(existing_curators, '2Y', card, {'published_date': 24}) + + +def curator_3m_easy_items_time(existing_curators: dict[str, any], card): + # All easy items published in last 3 months + return curator_items_time(existing_curators, '3M_easy', card, {'difficulty': 'easy', 'published_date': 3}) + + +def curator_6m_easy_items_time(existing_curators: dict[str, any], card): + # All easy items published in last 6 months + return curator_items_time(existing_curators, '6M_easy', card, {'difficulty': 'easy', 'published_date': 6}) + + +def curator_1y_easy_items_time(existing_curators: dict[str, any], card): + # All easy items published in last year + return curator_items_time(existing_curators, '1Y_easy', card, {'difficulty': 'easy', 'published_date': 12}) + + +def curator_2y_easy_items_time(existing_curators: dict[str, any], card): + # All easy items published in last 2 years + return curator_items_time(existing_curators, '2Y_easy', card, {'difficulty': 'easy', 'published_date': 24}) + + +def curator_3m_med_items_time(existing_curators: dict[str, any], card): + # All medium items published in last 3 months + return curator_items_time(existing_curators, '3M_med', card, {'difficulty': 'medium', 'published_date': 3}) + + +def curator_6m_med_items_time(existing_curators: dict[str, any], card): + # All medium items published in last 6 months + return curator_items_time(existing_curators, '6M_med', card, {'difficulty': 'medium', 'published_date': 6}) + + +def curator_1y_med_items_time(existing_curators: dict[str, any], card): + # All medium items published in last year + return curator_items_time(existing_curators, '1Y_med', card, {'difficulty': 'medium', 'published_date': 12}) + + +def curator_2y_med_items_time(existing_curators: dict[str, any], card): + # All medium items published in last 2 years + return curator_items_time(existing_curators, '2Y_med', card, {'difficulty': 'medium', 'published_date': 24}) + + +def curator_3m_hard_items_time(existing_curators: dict[str, any], card): + # All medium items published in last 3 months + return curator_items_time(existing_curators, '3M_hard', card, {'difficulty': 'hard', 'published_date': 3}) + + +def curator_6m_hard_items_time(existing_curators: dict[str, any], card): + # All medium items published in last 6 months + return curator_items_time(existing_curators, '6M_hard', card, {'difficulty': 'hard', 'published_date': 6}) + + +def curator_1y_hard_items_time(existing_curators: dict[str, any], card): + # All medium items published in last year + return curator_items_time(existing_curators, '1Y_hard', card, {'difficulty': 'hard', 'published_date': 12}) + + +def curator_2y_hard_items_time(existing_curators: dict[str, any], card): + # All medium items published in last 2 years + return curator_items_time(existing_curators, '2Y_hard', card, {'difficulty': 'hard', 'published_date': 24}) + + +def run(args): + # Optionally writes a CSV report to file and returns a JSON array of objects with the data that was written. + + global tc + curators = {} + + print('Getting curator stats from Trello') + + tc = TrelloClient(api_key=environ['API_TRELLO_KEY'], api_secret=environ['API_TRELLO_TOKEN']) + + try: + board = tc.get_board(environ['TRELLO_BOARD_ID']) + cards = board.get_cards(filters=f.get_cardlist_filter()['query']) + + if len(cards) >= 1000: + raise Exception("Trello API returned 1000 cards. Pagination may be needed but is not yet implemented") + + # Preprocess board cards to filter out the ones without published_date set. + # Do it this way instead of by returning only cards from the Published list so that + # we grab cards in other lists that may have been published already but have been moved to another list. + publishedcards = [] + fl = None + + if debug > 0: + fl = open('cards.csv', 'w', encoding="utf-8") + + if fl: + fl.write( + 'name,difficulty,total_time_mins,article_version,article_revision,' + + 'reviewer_1,reviewer_2,submission_date,forms_sent_date,reviewed_date,published_date,keep?\n') + for card in cards: + # Turn the custom fields into a dict for ease of use + customfields = {} + for field in card.custom_fields: + customfields[field.name] = field.value + keep = True if not customfields.get('published_date', '') == '' else False + + if keep: + publishedcards.append(card) + + if fl: + fl.write( + '"' + card.name.replace('"', '""') + '",' + + customfields.get('difficulty', '""') + ',' + + str(customfields.get('total_time_mins', '""')) + ',' + + str(customfields.get('article_version', '""')) + ',' + str(customfields.get('article_revision', '""')) + ',' + + str(customfields.get('reviewer_1', '""')) + ',' + str(customfields.get('reviewer_2', '""')) + ',' + + str(customfields.get('submission_date', '""')) + ',' + str(customfields.get('forms_sent_date', '""')) + ',' + + str(customfields.get('reviewed_date', '""')) + ',' + str(customfields.get('published_date', '""')) + ',' + + f'{keep}\n') + if fl: + fl.close() + + for card in publishedcards: + # Preprocess the card to get the reviewers + get_card_curators(card, curators) + + curator_total_items_time(curators, card) + curator_easy_items_time(curators, card) + curator_med_items_time(curators, card) + curator_hard_items_time(curators, card) + curator_3m_items_time(curators, card) + curator_6m_items_time(curators, card) + curator_1y_items_time(curators, card) + curator_2y_items_time(curators, card) + curator_3m_easy_items_time(curators, card) + curator_6m_easy_items_time(curators, card) + curator_1y_easy_items_time(curators, card) + curator_2y_easy_items_time(curators, card) + curator_3m_med_items_time(curators, card) + curator_6m_med_items_time(curators, card) + curator_1y_med_items_time(curators, card) + curator_2y_med_items_time(curators, card) + curator_3m_hard_items_time(curators, card) + curator_6m_hard_items_time(curators, card) + curator_1y_hard_items_time(curators, card) + curator_2y_hard_items_time(curators, card) + print() + + except Exception as e: + tb_list = traceback.extract_tb(sys.exc_info()[2]) + line_number = tb_list[-1][1] + print(f'Error getting board data: {e}, line {line_number}') + return {} + + print(f'Curation stats for items (and versions) published {f.get_cardlist_filter()["description"]}') + print(f'Processed {len(publishedcards)} cards with published_date set, out of {len(cards)} fetched') + + outfile = None + if args.outfile: + outfile = f.get_report_outfile(args.outfile, 'curators') + outfile.write( + f'username,total_items,total_time ({args.units}),' + + f'easy_items,easy_time ({args.units}),med_items,med_time ({args.units}),hard_items,hard_time ({args.units}),' + + f'3M_items,3M_time ({args.units}),6M_items,6M_time ({args.units}),' + + f'1Y_items,1Y_time ({args.units}),2Y_items,2Y_time ({args.units}),' + + f'3M_easy_items,3M_easy_time ({args.units}),3M_med_items,3M_med_time ({args.units}),' + + f'3M_hard_items,3M_hard_time ({args.units}),6M_easy_items,6M_easy_time ({args.units}),' + + f'6M_med_items,6M_med_time ({args.units}),6M_hard_items,6M_hard_time ({args.units}),' + + f'1Y_easy_items,1Y_easy_time ({args.units}),1Y_med_items,1Y_med_time ({args.units}),' + + f'1Y_hard_items,1Y_hard_time ({args.units}),2Y_easy_items,2Y_easy_time ({args.units}),' + + f'2Y_med_items,2Y_med_time ({args.units}),2Y_hard_items,2Y_hard_time ({args.units}),' + + f'total_reviewer1_items,total_reviewer1_time ({args.units}),' + + f'total_reviewer2_items,total_reviewer2_time ({args.units}),' + + f'3M_reviewer1_items,3M_reviewer1_time ({args.units}),' + + f'3M_reviewer2_items,3M_reviewer2_time ({args.units}),' + + f'6M_reviewer1_items,6M_reviewer1_time ({args.units}),' + + f'6M_reviewer2_items,6M_reviewer2_time ({args.units}),' + + f'1Y_reviewer1_items,1Y_reviewer1_time ({args.units}),' + + f'1Y_reviewer2_items,1Y_reviewer2_time ({args.units}),' + + f'2Y_reviewer1_items,2Y_reviewer1_time ({args.units}),' + + f'2Y_reviewer2_items,2Y_reviewer2_time ({args.units}),' + + f'3M_easy_reviewer1_items,3M_easy_reviewer1_time ({args.units}),' + + f'3M_easy_reviewer2_items,3M_easy_reviewer2_time ({args.units}),' + + f'3M_med_reviewer1_items,3M_med_reviewer1_time ({args.units}),' + + f'3M_med_reviewer2_items,3M_med_reviewer2_time ({args.units}),' + + f'3M_hard_reviewer1_items,3M_hard_reviewer1_time ({args.units}),' + + f'3M_hard_reviewer2_items,3M_hard_reviewer2_time ({args.units}),' + + f'6M_easy_reviewer1_items,6M_easy_reviewer1_time ({args.units}),' + + f'6M_easy_reviewer2_items,6M_easy_reviewer2_time ({args.units}),' + + f'6M_med_reviewer1_items,6M_med_reviewer1_time ({args.units}),' + + f'6M_med_reviewer2_items,6M_med_reviewer2_time ({args.units}),' + + f'6M_hard_reviewer1_items,6M_hard_reviewer1_time ({args.units}),' + + f'6M_hard_reviewer2_items,6M_hard_reviewer2_time ({args.units}),' + + f'1Y_easy_reviewer1_items,1Y_easy_reviewer1_time ({args.units}),' + + f'1Y_easy_reviewer2_items,1Y_easy_reviewer2_time ({args.units}),' + + f'1Y_med_reviewer1_items,1Y_med_reviewer1_time ({args.units}),' + + f'1Y_med_reviewer2_items,1Y_med_reviewer2_time ({args.units}),' + + f'1Y_hard_reviewer1_items,1Y_hard_reviewer1_time ({args.units}),' + + f'1Y_hard_reviewer2_items,1Y_hard_reviewer2_time ({args.units}),' + + f'2Y_easy_reviewer1_items,2Y_easy_reviewer1_time ({args.units}),' + + f'2Y_easy_reviewer2_items,2Y_easy_reviewer2_time ({args.units}),' + + f'2Y_med_reviewer1_items,2Y_med_reviewer1_time ({args.units}),' + + f'2Y_med_reviewer2_items,2Y_med_reviewer2_time ({args.units}),' + + f'2Y_hard_reviewer1_items,2Y_hard_reviewer1_time ({args.units}),' + + f'2Y_hard_reviewer2_items,2Y_hard_reviewer2_time ({args.units}),' + + '\n') + + elif not args.sync_to_dashboard: + print( + f'username\ttot_itms\ttot_tm({args.units})\tlast 3m itms\tlast 3m tm({args.units})' + + f' rvwr1_itms rvwr1_tm({args.units})' + + f' rvwr2_itms rvwr2_tm({args.units})') + + total_time = 0 + total_items = 0 + + for username, curator in curators.items(): + if outfile: + s = ( + '{0},{1},{2},' # username, total items, total time + + '{3},{4},{5},{6},{7},{8},' # easy, med, hard items & time + + '{9},{10},{11},{12},' # 3M, 6M items & time + + '{13},{14},{15},{16},' # 1Y, 2Y items & time + + '{17},{18},{19},{20},' # 3M_easy, 3M_med items & time + + '{21},{22},{23},{24},' # 3M_hard, 6M_easy items & time + + '{25},{26},{27},{28},' # 6M_med, 6M_hard items & time + + '{29},{30},{31},{32},' # 1Y_easy, 1Y_med items & time + + '{33},{34},{35},{36},' # 1Y_hard, 2Y_easy items & time + + '{37},{38},{39},{40},' # 2Y_med, 2Y_hard items & time + + '{41},{42},{43},{44},' # total reviewer_1, reviewer_2 items & time + + '{45},{46},{47},{48},' # 3M reviewer_1, reviewer_2 items & time + + '{49},{50},{51},{52},' # 6M reviewer_1, reviewer_2 items & time + + '{53},{54},{55},{56},' # 1Y reviewer_1, reviewer_2 items & time + + '{57},{58},{59},{60},' # 2Y reviewer_1, reviewer_2 items & time + + '{61},{62},{63},{64},' # 3M_easy reviewer_1, reviewer_2 items & time + + '{65},{66},{67},{68},' # 3M_med reviewer_1, reviewer_2 items & time + + '{69},{70},{71},{72},' # 3M_hard reviewer_1, reviewer_2 items & time + + '{73},{74},{75},{76},' # 6M_easy reviewer_1, reviewer_2 items & time + + '{77},{78},{79},{80},' # 6M_med reviewer_1, reviewer_2 items & time + + '{81},{82},{83},{84},' # 6M_hard reviewer_1, reviewer_2 items & time + + '{85},{86},{87},{88},' # 1Y_easy reviewer_1, reviewer_2 items & time + + '{89},{90},{91},{92},' # 1Y_med reviewer_1, reviewer_2 items & time + + '{93},{94},{95},{96},' # 1Y_hard reviewer_1, reviewer_2 items & time + + '{97},{98},{99},{100},' # 2Y_easy reviewer_1, reviewer_2 items & time + + '{101},{102},{103},{104},' # 2Y_med reviewer_1, reviewer_2 items & time + + '{105},{106},{107},{108}' # 2Y_hard reviewer_1, reviewer_2 items & time + ) + else: + s = ('{0} \t' + + '{1} \t\t {2} \t\t' + + '{9} \t\t {10} \t\t' + + '{41}\t {42}\t\t{43}\t{44}') + + s = s.format( + username, # 0 + curator['total_items'], f.format_duration(str(curator['total_time']) + 's', args.units), # 1,2 + curator['easy_items'], f.format_duration(str(curator['easy_time']) + 's', args.units), # 3,4 + curator['med_items'], f.format_duration(str(curator['med_time']) + 's', args.units), # 5,6 + curator['hard_items'], f.format_duration(str(curator['hard_time']) + 's', args.units), # 7,8 + curator['3M_items'], f.format_duration(str(curator['3M_time']) + 's', args.units), # 9,10 + curator['6M_items'], f.format_duration(str(curator['6M_time']) + 's', args.units), # 11,12 + curator['1Y_items'], f.format_duration(str(curator['1Y_time']) + 's', args.units), # 13,14 + curator['2Y_items'], f.format_duration(str(curator['2Y_time']) + 's', args.units), # 15,16 + curator['3M_easy_items'], f.format_duration(str(curator['3M_easy_time']) + 's', args.units), # 17,18 + curator['3M_med_items'], f.format_duration(str(curator['3M_med_time']) + 's', args.units), # 19,20 + curator['3M_hard_items'], f.format_duration(str(curator['3M_hard_time']) + 's', args.units), # 21,22 + curator['6M_easy_items'], f.format_duration(str(curator['6M_easy_time']) + 's', args.units), # 23,24 + curator['6M_med_items'], f.format_duration(str(curator['6M_med_time']) + 's', args.units), # 25,26 + curator['6M_hard_items'], f.format_duration(str(curator['6M_hard_time']) + 's', args.units), # 27,28 + curator['1Y_easy_items'], f.format_duration(str(curator['1Y_easy_time']) + 's', args.units), # 29,30 + curator['1Y_med_items'], f.format_duration(str(curator['1Y_med_time']) + 's', args.units), # 31,32 + curator['1Y_hard_items'], f.format_duration(str(curator['1Y_hard_time']) + 's', args.units), # 33,34 + curator['2Y_easy_items'], f.format_duration(str(curator['2Y_easy_time']) + 's', args.units), # 35,36 + curator['2Y_med_items'], f.format_duration(str(curator['2Y_med_time']) + 's', args.units), # 37,38 + curator['2Y_hard_items'], f.format_duration(str(curator['2Y_hard_time']) + 's', args.units), # 39,40 + curator['total_reviewer1_items'], f.format_duration(str(curator['total_reviewer1_time']) + 's', args.units), # 41,42 + curator['total_reviewer2_items'], f.format_duration(str(curator['total_reviewer2_time']) + 's', args.units), # 43,44 + curator['3M_reviewer1_items'], f.format_duration(str(curator['3M_reviewer1_time']) + 's', args.units), # 45,46 + curator['3M_reviewer2_items'], f.format_duration(str(curator['3M_reviewer2_time']) + 's', args.units), # 47,48 + curator['6M_reviewer1_items'], f.format_duration(str(curator['6M_reviewer1_time']) + 's', args.units), # 49,50 + curator['6M_reviewer2_items'], f.format_duration(str(curator['6M_reviewer2_time']) + 's', args.units), # 51,52 + curator['1Y_reviewer1_items'], f.format_duration(str(curator['1Y_reviewer1_time']) + 's', args.units), # 53,54 + curator['1Y_reviewer2_items'], f.format_duration(str(curator['1Y_reviewer2_time']) + 's', args.units), # 55,56 + curator['2Y_reviewer1_items'], f.format_duration(str(curator['2Y_reviewer1_time']) + 's', args.units), # 57,58 + curator['2Y_reviewer2_items'], f.format_duration(str(curator['2Y_reviewer2_time']) + 's', args.units), # 59,60 + curator['3M_easy_reviewer1_items'], f.format_duration(str(curator['3M_easy_reviewer1_time']) + 's', args.units), # 61,62 + curator['3M_easy_reviewer2_items'], f.format_duration(str(curator['3M_easy_reviewer2_time']) + 's', args.units), # 63,64 + curator['3M_med_reviewer1_items'], f.format_duration(str(curator['3M_med_reviewer1_time']) + 's', args.units), # 65,66 + curator['3M_med_reviewer2_items'], f.format_duration(str(curator['3M_med_reviewer2_time']) + 's', args.units), # 67,68 + curator['3M_hard_reviewer1_items'], f.format_duration(str(curator['3M_hard_reviewer1_time']) + 's', args.units), # 69,70 + curator['3M_hard_reviewer2_items'], f.format_duration(str(curator['3M_hard_reviewer2_time']) + 's', args.units), # 71,72 + curator['6M_easy_reviewer1_items'], f.format_duration(str(curator['6M_easy_reviewer1_time']) + 's', args.units), # 73,74 + curator['6M_easy_reviewer2_items'], f.format_duration(str(curator['6M_easy_reviewer2_time']) + 's', args.units), # 75,76 + curator['6M_med_reviewer1_items'], f.format_duration(str(curator['6M_med_reviewer1_time']) + 's', args.units), # 77,78 + curator['6M_med_reviewer2_items'], f.format_duration(str(curator['6M_med_reviewer2_time']) + 's', args.units), # 79,80 + curator['6M_hard_reviewer1_items'], f.format_duration(str(curator['6M_hard_reviewer1_time']) + 's', args.units), # 81,82 + curator['6M_hard_reviewer2_items'], f.format_duration(str(curator['6M_hard_reviewer2_time']) + 's', args.units), # 83,84 + curator['1Y_easy_reviewer1_items'], f.format_duration(str(curator['1Y_easy_reviewer1_time']) + 's', args.units), # 85,86 + curator['1Y_easy_reviewer2_items'], f.format_duration(str(curator['1Y_easy_reviewer2_time']) + 's', args.units), # 87,88 + curator['1Y_med_reviewer1_items'], f.format_duration(str(curator['1Y_med_reviewer1_time']) + 's', args.units), # 89,90 + curator['1Y_med_reviewer2_items'], f.format_duration(str(curator['1Y_med_reviewer2_time']) + 's', args.units), # 91,92 + curator['1Y_hard_reviewer1_items'], f.format_duration(str(curator['1Y_hard_reviewer1_time']) + 's', args.units), # 93,94 + curator['1Y_hard_reviewer2_items'], f.format_duration(str(curator['1Y_hard_reviewer2_time']) + 's', args.units), # 95,96 + curator['2Y_easy_reviewer1_items'], f.format_duration(str(curator['2Y_easy_reviewer1_time']) + 's', args.units), # 97,98 + curator['2Y_easy_reviewer2_items'], f.format_duration(str(curator['2Y_easy_reviewer2_time']) + 's', args.units), # 99,100 + curator['2Y_med_reviewer1_items'], f.format_duration(str(curator['2Y_med_reviewer1_time']) + 's', args.units), # 101,102 + curator['2Y_med_reviewer2_items'], f.format_duration(str(curator['2Y_med_reviewer2_time']) + 's', args.units), # 103,104 + curator['2Y_hard_reviewer1_items'], f.format_duration(str(curator['2Y_hard_reviewer1_time']) + 's', args.units), # 105,106 + curator['2Y_hard_reviewer2_items'], f.format_duration(str(curator['2Y_hard_reviewer2_time']) + 's', args.units) # 107,108 + ) + + total_time += curator['total_time'] + total_items += curator['total_items'] + + if outfile: + outfile.write(s + '\n') + elif not args.sync_to_dashboard: + print(s) + + print() + print(f'Total curation hours:\t{f.format_duration(str(total_time) + "s", "H")}') + print(f'Avg hours per item:\t{f.format_duration(str(total_time / total_items) + "s", "H")}') + + if outfile: + outfile.close() + + return list(curators.values()) diff --git a/packages/trello_reports/run/main.py b/packages/trello_reports/run/main.py new file mode 100644 index 0000000..2712b42 --- /dev/null +++ b/packages/trello_reports/run/main.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +# Runs various reports for ReDATA +# +# Author: Fernando Rios + +import argparse +import sys +from os import environ +from version import __version__, __commit__ + +sys.path.insert(0, 'lib/') # Allows loading of shared functions when running locally. No effect when running as DO func. +import curator_report +import functions as f +import secrets + + +def init_argparse(): + parser = argparse.ArgumentParser( + description='Runs reports for Trello stats' + ) + parser.add_argument( + '-r', '--report', required=False, action='append', choices=['curators', 'items'], + help="Selects which report to run. This option can appear more than once" + ) + parser.add_argument( + '--sync-to-dashboard', required=False, action='store_true', + help="Uploads report data to the Google dashboard. Additionally, sets these flags: -u B -r items -r users" + ) + parser.add_argument( + '-v', '--version', action='version', + version=f'{parser.prog} v{__version__} {__commit__}' + ) + parser.add_argument( + '-o', '--outfile', metavar='PATH', nargs='?', + const=f"$$*$${f.get_report_date().strftime('%Y-%m-%dT%H%M%S')}.csv", + type=str, help="Write output to a file. If PATH isn't specified, file will default to a timestamped file in the current directory" + ) + parser.add_argument( + '-u', '--units', choices=['M', 'W', 'D', 'H', 'm', 's'], + default='H', help='Set the output time units. Default is %(default)s') + + return parser + + +def run(args): + print(f'This is trello-reports version v{__version__} {__commit__}') + + if args.sync_to_dashboard: + args.units = 's' + args.report = ['curators', 'items'] + + if not args.report: + return 'No report specified. Doing nothing' + + result = '' + if 'curators' in args.report: + print('Running "curators" report') + data = curator_report.run(args) + result = result + f'Running "curators" report completed. {len(data)} curators found.' + if args.sync_to_dashboard: + result = result + f'\nSyncing "curator" to dashboard completed. Result: {f.sync_to_dashboard(data, "curators")}.' + if 'items' in args.report: + print('Running "cards" report') + if args.sync_to_dashboard: + result = result + '\nSyncing "cards" to dashboard completed. Result: Not implemented.' + + return result + + +if __name__ == '__main__': + args = init_argparse().parse_args() + + environ['API_TRELLO_URL_BASE'] = secrets.api_trello_url_base + environ['API_TRELLO_KEY'] = secrets.api_trello_key + environ['API_TRELLO_TOKEN'] = secrets.api_trello_token + environ['TRELLO_BOARD_ID'] = secrets.trello_board_id + environ['GSHEETS_DASHBOARD_POST_URL'] = secrets.gsheets_dashboard_post_url + environ['GSHEETS_DASHBOARD_KEY'] = secrets.gsheets_dashboard_key + environ['TOKEN'] = secrets.do_token + + print(run(args)) diff --git a/packages/trello_reports/run/requirements.txt b/packages/trello_reports/run/requirements.txt new file mode 100644 index 0000000..f5f56a8 --- /dev/null +++ b/packages/trello_reports/run/requirements.txt @@ -0,0 +1,3 @@ +requests +simplejson +py-trello \ No newline at end of file diff --git a/packages/trello_reports/run/version.py b/packages/trello_reports/run/version.py new file mode 100644 index 0000000..cea651d --- /dev/null +++ b/packages/trello_reports/run/version.py @@ -0,0 +1,17 @@ +from pathlib import Path + + +def get_commit(repo_path): + # https://stackoverflow.com/a/68215738 + git_folder = Path(repo_path, '../../../.git') + try: + head_name = Path(git_folder, 'HEAD').read_text().split('\n')[0].split(' ')[-1] + head_ref = Path(git_folder, head_name) + commit = head_ref.read_text().replace('\n', '') + except Exception: + return '' + return commit + + +__commit__ = get_commit('.') +__version__ = '1.0.0' diff --git a/project.yml b/project.yml index 0589014..dd6124b 100644 --- a/project.yml +++ b/project.yml @@ -2,14 +2,30 @@ packages: - name: redata_reports functions: - name: run - runtime: 'python:default' + runtime: 'python:3.11' web: true limits: timeout: 420000 memory: 512 environment: - API_URL_BASE: ${api_url_base} - API_TOKEN: ${api_token} + API_FIGSHARE_URL_BASE: ${api_figshare_url_base} + API_FIGSHARE_TOKEN: ${api_figshare_token} + GSHEETS_DASHBOARD_POST_URL: ${gsheets_dashboard_post_url} + GSHEETS_DASHBOARD_KEY: ${gsheets_dashboard_key} + TOKEN: ${do_token} + - name: trello_reports + functions: + - name: run + runtime: 'python:3.11' + web: true + limits: + timeout: 420000 + memory: 512 + environment: + API_TRELLO_URL_BASE: ${api_trello_url_base} + API_TRELLO_KEY: ${api_trello_key} + API_TRELLO_TOKEN: ${api_trello_token} + TRELLO_BOARD_ID: ${trello_board_id} GSHEETS_DASHBOARD_POST_URL: ${gsheets_dashboard_post_url} GSHEETS_DASHBOARD_KEY: ${gsheets_dashboard_key} TOKEN: ${do_token} \ No newline at end of file