add row-converter, opt in auto-rule actions

dtable_events/automations/actions.py

@@ -30,6 +30,7 @@
 from dtable_events.utils.dtable_server_api import DTableServerAPI
 from dtable_events.utils.dtable_web_api import DTableWebAPI
 from dtable_events.utils.dtable_db_api import DTableDBAPI, RowsQueryError, Request429Error
+from dtable_events.utils.row_converter import convert_row_with_sql_row
 from dtable_events.notification_rules.utils import get_nickname_by_usernames
 from dtable_events.utils.sql_generator import filter2sql, BaseSQLGenerator, ColumnFilterInvalidError, \
     has_user_filter, is_user_filter
@@ -212,12 +213,15 @@ def send_selected_collaborator_notis(self, row_data):
                     if user not in notify_users:
                         notify_users.append(user)
         elif self.action_type == 'update_record':
-            converted_row = self.auto_rule.get_convert_sql_row()
-            row_id = converted_row['_id']
+            sql_row = self.auto_rule.get_sql_row()
+            row_id = sql_row['_id']
             for column_name, value in row_data.items():
                 if column_name not in notify_column_names:
                     continue
-                old_value = converted_row.get(column_name) or []
+                column = next(filter(lambda column: column['name'] == column_name, self.auto_rule.table_info['columns']), None)
+                if not column:
+                    continue
+                old_value = sql_row.get(column['key']) or []
                 for user in (set(value) - set(old_value)):
                     if user not in notify_users:
                         notify_users.append(user)
@@ -1603,7 +1607,8 @@ class LinkRecordsAction(BaseAction):
         ColumnTypes.COLLABORATOR: "is_exactly",
         ColumnTypes.EMAIL: "is",
         ColumnTypes.RATE: "equal",
-        ColumnTypes.AUTO_NUMBER: "is"
+        ColumnTypes.AUTO_NUMBER: "is",
+        ColumnTypes.DEPARTMENT_SINGLE_SELECT: "is"
     }
 
     VALID_COLUMN_TYPES = [
@@ -3381,7 +3386,6 @@ def __init__(self, data, db_session, raw_trigger, raw_actions, options, metadata
         self._sql_query_count = 0
 
         self._convert_sql_row = None
-        self._convert_sql_query_count = 0
 
         self._sql_query_max = 3
 
@@ -3486,22 +3490,9 @@ def related_users_dict(self):
         return self._related_users_dict
 
     def get_convert_sql_row(self):
-        if self._convert_sql_row is not None or self._convert_sql_query_count >= self._sql_query_max:
+        if self._convert_sql_row is not None:
             return self._convert_sql_row
-        if not self.data:
-            return None
-        if 'row_id' not in self.data:
-            return None
-        row_id = self.data['row_id']
-        while not self._convert_sql_row and self._convert_sql_query_count < self._sql_query_max:
-            sql = f"SELECT * FROM `{self.table_info['name']}` WHERE _id='{row_id}'"
-            sql_rows, _ = self.query(sql, convert=True)
-            if not sql_rows:
-                self._convert_sql_query_count += 1
-                logger.warning('auto-rule %s query dtable %s table %s convert row %s not found, query count %s', self.rule_id, self.dtable_uuid, self.table_id, row_id, self._convert_sql_query_count)
-                time.sleep(0.1)
-                continue
-            self._convert_sql_row = sql_rows[0]
+        self._convert_sql_row = convert_row_with_sql_row(self.get_sql_row(), self.table_info['columns'], self.db_session)
         return self._convert_sql_row
 
     def get_sql_row(self):

dtable_events/notification_rules/notification_rules_utils.py

@@ -17,7 +17,7 @@
 from dtable_events.utils.dtable_server_api import DTableServerAPI
 from dtable_events.utils.dtable_web_api import DTableWebAPI
 from dtable_events.utils.dtable_db_api import DTableDBAPI
-from dtable_events.notification_rules.message_formatters import create_formatter_params, formatter_map
+from dtable_events.utils.message_formatters import create_formatter_params, formatter_map
 
 logger = logging.getLogger(__name__)
 
@@ -187,24 +187,29 @@ def convert_zero_in_value(value):
     return value
 
 
+def fill_cell_with_sql_row(value, column, db_session, convert_to_html=False):
+    column_type = column['type']
+    formatter_class = formatter_map.get(column_type)
+    if not formatter_class:
+        return None
+    params = create_formatter_params(column_type, value=value, db_session=db_session, convert_to_html=convert_to_html)
+    if value is None:
+        message = formatter_class(column).format_empty_message()
+        return message
+    try:
+        message = formatter_class(column).format_message(**params)
+        return message
+    except Exception as e:
+        logger.exception('value %s for filling column %s error %s', value, column)
+        return ''
+
+
 def fill_msg_blanks_with_sql_row(msg, column_blanks, col_name_dict, row, db_session, convert_to_html=False):
     for blank in column_blanks:
-        value = row.get(col_name_dict[blank]['key'])
-        column_type = col_name_dict[blank]['type']
-        formatter_class = formatter_map.get(column_type)
-        if not formatter_class:
-            continue
-        params = create_formatter_params(column_type, value=value, db_session=db_session, convert_to_html=convert_to_html)
-        if value is None:
-            message = formatter_class(col_name_dict[blank]).format_empty_message()
-            msg = msg.replace('{' + blank + '}', str(message))
+        cell_message = fill_cell_with_sql_row(row.get(col_name_dict[blank]['key']), col_name_dict[blank], db_session, convert_to_html=False)
+        if cell_message is None:
             continue
-        try:
-            message = formatter_class(col_name_dict[blank]).format_message(**params)
-            msg = msg.replace('{' + blank + '}', str(message))
-        except Exception as e:
-            logger.exception(e)
-            msg = msg.replace('{' + blank + '}', '')
+        msg = msg.replace('{' + blank + '}', str(cell_message))
 
     return msg
 

dtable_events/notification_rules/message_formatters.py → dtable_events/utils/message_formatters.py

@@ -288,7 +288,7 @@ def format_message(self, value):
         except Exception as e:
             logger.warning('parse value: %s to datetime error: %s', value, e)
             return self.format_empty_message()
-        value = datetime_obj.strftime('%Y-%m-%d %H:%M')
+        value = datetime_obj.strftime('%Y-%m-%d %H:%M:%S')
         return value
 
 
@@ -436,11 +436,11 @@ def format_message(self, value):
         minutes = (abs(value) % 3600) // 60
         seconds = abs(value) % 60
         if duration_format == 'h:mm':
-            value = '%s%d:%2d' % (prefix, hours, minutes)
+            value = '%s%d:%02d' % (prefix, hours, minutes)
         elif duration_format == 'h:mm:ss':
             value = '%s%d:%02d:%02d' % (prefix, hours, minutes, seconds)
         else:
-            value = '%s%d:%2d' % (prefix, hours, minutes)
+            value = '%s%d:%02d' % (prefix, hours, minutes)
         return value
 
 

dtable_events/utils/row_converter.py

@@ -0,0 +1,153 @@
+import logging
+import re
+from datetime import datetime
+
+from dateutil import parser
+
+from dtable_events.utils.constants import FormulaResultType, ColumnTypes
+from dtable_events.notification_rules.notification_rules_utils import fill_cell_with_sql_row
+from dtable_events.utils.message_formatters import FormulaMessageFormatter
+
+logger = logging.getLogger(__name__)
+
+
+class BaseCellConverter:
+
+    def __init__(self, column):
+        self.column = column
+
+    def get_column_data(self):
+        return self.column.get('data') or {}
+
+    def convert(self, value):
+        return value
+
+
+class SingleSelectCellConverter(BaseCellConverter):
+
+    def convert(self, value):
+        column_data = self.get_column_data()
+        options = column_data.get('options') or []
+        option = next(filter(lambda option: option.get('id') == value, options), None)
+        return option.get('name') or None
+
+
+class MultiSelectCellConverter(BaseCellConverter):
+
+    def convert(self, value):
+        column_data = self.get_column_data()
+        options = column_data.get('options') or []
+        result = []
+        if not isinstance(value, list):
+            return ''
+        for item in value:
+            item_option = next(filter(lambda option: option.get('id') == item, options), None)
+            if item_option:
+                result.append(item_option['name'])
+        return result
+
+
+class LongTextCellConverter(BaseCellConverter):
+
+    def convert(self, value):
+        if isinstance(value, str):
+            return value
+        elif isinstance(value, dict):
+            return value.get('text') or ''
+        return ''
+
+
+class DateCellConverter(BaseCellConverter):
+
+    def format_text_to_date(self, text, format='YYYY-MM-DD'):
+        if not isinstance(text, str):
+            return None
+        is_all_number = re.match(r'^\d+$', text)
+        if is_all_number:
+            date_obj = datetime.fromtimestamp(int(text) / 1000)
+        else:
+            date_obj = parser.parse(text)
+        if 'HH:mm' not in format:
+            return datetime.strftime(date_obj, '%Y-%m-%d')
+        return datetime.strftime(date_obj, '%Y-%m-%d %H:%M')
+
+    def convert(self, value):
+        value = re.sub(r'([+-]\d{2}:?\d{2})|Z$', '', value)
+        column_data = self.get_column_data()
+        format = column_data.get('format') or 'YYYY-MM-DD'
+        return self.format_text_to_date(value, format)
+
+
+class FormulaCellConverter(BaseCellConverter):
+
+    def convert(self, value, db_session):
+        column_data = self.get_column_data()
+        if not column_data:
+            return None
+        result_type = column_data.get('result_type')
+        if result_type in [
+            FormulaResultType.NUMBER,
+            FormulaResultType.DATE,
+        ]:
+            return FormulaMessageFormatter(self.column).format_message(value, None)
+        if isinstance(value, bool):
+            return 'true' if bool else 'false'
+        if result_type == FormulaResultType.ARRAY:
+            array_type = column_data.get('array_type')
+            array_data = column_data.get('array_data')
+            if not array_type:
+                return ''
+            if array_type in [
+                ColumnTypes.COLLABORATOR,
+                ColumnTypes.CREATOR,
+                ColumnTypes.LAST_MODIFIER
+            ]:
+                return value
+            if array_type not in [
+                ColumnTypes.IMAGE,
+                ColumnTypes.FILE,
+                ColumnTypes.MULTIPLE_SELECT,
+                ColumnTypes.COLLABORATOR
+            ] and isinstance(value, list):
+                return FormulaMessageFormatter(self.column).format_message(value, db_session)
+            # if find bugs in fill_cell_with_sql_row, do not modify message-formatter, fix in this file
+            return fill_cell_with_sql_row(value, {'type': array_type, 'data': array_data}, db_session)
+        return value
+
+
+def convert_row_with_sql_row(row, columns, db_session):
+    result = {}
+    if '_id' in row:
+        result['_id'] = row['_id']
+    if '_ctime' in row:
+        result['_ctime'] = row['_ctime']
+    if '_mtime' in row:
+        result['_mtime'] = row['_mtime']
+    for column in columns:
+        value = row.get(column['key'])
+        if not value:
+            continue
+        column_name = column['name']
+        if not column:
+            continue
+        try:
+            if column['type'] == ColumnTypes.SINGLE_SELECT:
+                result[column_name] = SingleSelectCellConverter(column).convert(value)
+            elif column['type'] == ColumnTypes.MULTIPLE_SELECT:
+                result[column_name] = MultiSelectCellConverter(column).convert(value)
+            elif column['type'] == ColumnTypes.DATE:
+                result[column_name] = DateCellConverter(column).convert(value)
+            elif column['type'] == ColumnTypes.LONG_TEXT:
+                result[column_name] = LongTextCellConverter(column).convert(value)
+            elif column['type'] in [
+                ColumnTypes.FORMULA,
+                ColumnTypes.LINK_FORMULA
+            ]:
+                result[column_name] = FormulaCellConverter(column).convert(value, db_session)
+            else:
+                result[column_name] = BaseCellConverter(column).convert(value)
+        except Exception as e:
+            logger.warning('convert row column %s with value %s', column, value)
+            result[column_name] = value
+
+    return result