diff --git a/README.md b/README.md index aae5685..ffda6ce 100644 --- a/README.md +++ b/README.md @@ -30,13 +30,15 @@ Want to become a sponsor? Reach out to us at [autoidm.com](https://autoidm.com) ## Settings -| Setting | Required | Default | Description | -|:--------------------|:--------:|:-------:|:------------| -| api_token | True | None | Example: 'pk_12345 | -| stream_maps | False | None | Config object for stream maps capability. | -| stream_map_config | False | None | User-defined config values to be used within map expressions. | -| flattening_enabled | False | None | 'True' to enable schema flattening and automatically expand nested properties. | -| flattening_max_depth| False | None | The max depth to flatten schemas. | +| Setting | Required | Default | Description | +|:----------------------|:--------:|:-------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| api_token | True | None | Example: 'pk_12345 | +| time_entry_assignees | False | None | By default, the extractor will get all user ids from your team and use them when fetching time entries. If you want to fetch time entries assigned to specific users, provide a comma-separated list of user IDs here. Ex. '420230,452346,784219' | +| time_entry_start_date | False | None | The start date that determines how far back in time the extractor gets time entries. Without this, only the last thirty days of time entries will be fetched. After the initial run, this value will be ignored in favor of the state, using the replication_key of 'at' to determine the start date. Ex. '2023-01-01T00:00:00Z' to follow singer date format. | +| stream_maps | False | None | Config object for stream maps capability. | +| stream_map_config | False | None | User-defined config values to be used within map expressions. | +| flattening_enabled | False | None | 'True' to enable schema flattening and automatically expand nested properties. | +| flattening_max_depth | False | None | The max depth to flatten schemas. | A full list of supported settings and capabilities is available by running: `tap-clickup --about` @@ -72,8 +74,8 @@ Note that the most up to date information is located in tap_clickup/streams.py. - Table name: time_entries - Description: All time entries are pulled for every team. Currently only pulls the last 30 days of time_entries see https://github.com/AutoIDM/tap-clickup/issues/134 for an issue addressing this! - Primary key column(s): id -- Replicated fully or incrementally: Full -- Bookmark column(s): N/A +- Replicated fully or incrementally: Incremental +- Bookmark column(s): at. _Please note that you must set the start date in the config to get time entries older than 30 days._ - Link to API endpoint documentation: [Time Entries](https://jsapi.apiary.io/apis/clickup20/reference/0/time-tracking-legacy/get-time-entries-within-a-date-range.html) ### Folders diff --git a/meltano.yml b/meltano.yml index 8b6c32c..290976d 100644 --- a/meltano.yml +++ b/meltano.yml @@ -13,6 +13,10 @@ plugins: settings: - name: api_token kind: password + - name: time_entry_assignees + kind: string + - name: time_entry_start_date + kind: string # config: select: - '!shared_hierarchy.*' diff --git a/tap_clickup/schemas/time_entries.json b/tap_clickup/schemas/time_entries.json index b8b9793..2ad29eb 100644 --- a/tap_clickup/schemas/time_entries.json +++ b/tap_clickup/schemas/time_entries.json @@ -5,7 +5,7 @@ "type": "string" }, "task": { - "type": "object", + "type": ["object", "string"], "properties": { "id": { "type": "string" @@ -20,7 +20,10 @@ "type": "string" }, "color": { - "type": "string" + "type": [ + "string", + "null" + ] }, "type": { "type": "string" @@ -28,76 +31,85 @@ "orderindex": { "type": "integer" } - } }, - "custom_type": { - "type": "null" - } - } }, - "wid": { - "type": "string" - }, - "user": { - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "username": { - "type": "string" - }, - "email": { - "type": "string" - }, - "color": { - "type": "string" - }, - "initials": { - "type": "string" - }, - "profilePicture": { - "type": "null" } - } }, - "billable": { - "type": "boolean" }, - "start": { - "type": "string" - }, - "end": { - "type": "string" + "custom_type": { + "type": ["null", "integer"] + } + } + }, + "wid": { + "type": "string" + }, + "user": { + "type": "object", + "properties": { + "id": { + "type": "integer" }, - "duration": { + "username": { "type": "string" }, - "description": { + "email": { "type": "string" }, - "tags": { - "type": "array" + "color": { + "type": [ + "string", + "null" + ] }, - "source": { + "initials": { "type": "string" }, - "at": { - "type": "string" + "profilePicture": { + "type": [ + "string", + "null" + ] + } + } + }, + "billable": { + "type": "boolean" + }, + "start": { + "type": "string" + }, + "end": { + "type": "string" + }, + "duration": { + "type": "string" + }, + "description": { + "type": "string" + }, + "tags": { + "type": "array" + }, + "source": { + "type": "string" + }, + "at": { + "type": "string" + }, + "task_location": { + "type": "object", + "properties": { + "list_id": { + "type": ["string", "null"] }, - "task_location": { - "type": "object", - "properties": { - "list_id": { - "type": "string" - }, - "folder_id": { - "type": "string" - }, - "space_id": { - "type": "string" - } - } + "folder_id": { + "type": ["string", "null"] }, - "task_url": { - "type": "string" + "space_id": { + "type": ["string", "null"] } + } + }, + "task_url": { + "type": "string" + } } -} +} \ No newline at end of file diff --git a/tap_clickup/streams.py b/tap_clickup/streams.py index d4abd88..13e29fd 100644 --- a/tap_clickup/streams.py +++ b/tap_clickup/streams.py @@ -1,4 +1,5 @@ """Stream type classes for tap-clickup.""" +from datetime import datetime from pathlib import Path from typing import Optional, Any, Dict import requests @@ -20,8 +21,12 @@ class TeamsStream(ClickUpStream): def get_child_context(self, record: dict, context: Optional[dict]) -> dict: """Return a context dictionary for child streams.""" + user_ids = [str(member.get("user", {}).get("id")) for member in record.get("members", []) if + isinstance(member, dict)] + return { "team_id": record["id"], + "user_ids": user_ids } @@ -31,13 +36,33 @@ class TimeEntries(ClickUpStream): name = "time_entries" path = "/team/{team_id}/time_entries" primary_keys = ["id"] - replication_key = None + replication_key = "at" schema_filepath = SCHEMAS_DIR / "time_entries.json" records_jsonpath = "$.data[*]" parent_stream_type = TeamsStream # TODO not clear why this is needed partitions = None + def get_url_params( + self, context: Optional[dict], next_page_token: Optional[Any] + ) -> Dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization.""" + params = super().get_url_params(context, next_page_token) + + state_based_date = self.get_starting_replication_key_value(context) + # In the case of the first run, we need to use the start date from the config + if not state_based_date: + start_date = datetime.strptime(self.config["time_entry_start_date"], "%Y-%m-%dT%H:%M:%SZ") + params["start_date"] = int(start_date.timestamp() * 1000) + else: + # Because the state date is already in milliseconds, we can just use it + params["start_date"] = state_based_date + if "time_entry_assignees" in self.config: + params["assignee"] = self.config["time_entry_assignees"] + else: + params["assignee"] = ",".join(context["user_ids"]) + return params + class SpacesStream(ClickUpStream): """Spaces""" diff --git a/tap_clickup/tap.py b/tap_clickup/tap.py index 1c371b3..438c3e2 100644 --- a/tap_clickup/tap.py +++ b/tap_clickup/tap.py @@ -47,6 +47,23 @@ class TapClickUp(Tap): th.Property( "api_token", th.StringType, required=True, description="Example: 'pk_12345" ), + th.Property( + "time_entry_assignees", + th.StringType, + required=False, + description="""By default, the extractor will get all user ids from your + team and use them when fetching time entries. If you want to fetch time entries + assigned to specific users, provide a comma-separated list of user IDs here. Ex. '420230,452346,784219'""" + ), + th.Property( + "time_entry_start_date", + th.StringType, + required=False, + description="""The start date that determines how far back in time the extractor gets time entries. + Without this, only the last thirty days of time entries will be fetched. After the initial run, + this value will be ignored in favor of the state, using the replication_key of 'at' to determine the + start date. Ex. '2023-01-01T00:00:00Z' to follow singer date format.""" + ), # Removing "official" start_date support re https://github.com/AutoIDM/tap-clickup/issues/118 # th.Property( # "start_date",