diff --git a/.gitignore b/.gitignore index 3838572..e6e56c9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__ *.egg-info notebooks/data/* !notebooks/data/harvest-sample.csv +!notebooks/data/justworks-sample.csv !notebooks/data/toggl-project-info-sample.json !notebooks/data/toggl-sample.csv !notebooks/data/toggl-user-info-sample.json diff --git a/LICENSE b/LICENSE index 261eeb9..391a8e5 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2025 Compiler LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 7e76800..4ac78a8 100644 --- a/README.md +++ b/README.md @@ -81,14 +81,17 @@ Use this command to download a time report from Toggl in CSV format: ```bash $ compiler-admin time download -h -usage: compiler-admin time download [-h] [--start YYYY-MM-DD] [--end YYYY-MM-DD] [--output OUTPUT] - [--client CLIENT_ID] [--project PROJECT_ID] [--task TASK_ID] [--user USER_ID] +usage: compiler-admin time download [-h] [--start YYYY-MM-DD] [--end YYYY-MM-DD] + [--output OUTPUT] [--all] [--client CLIENT_ID] + [--project PROJECT_ID] [--task TASK_ID] + [--user USER_ID] options: -h, --help show this help message and exit --start YYYY-MM-DD The start date of the reporting period. Defaults to the beginning of the prior month. --end YYYY-MM-DD The end date of the reporting period. Defaults to the end of the prior month. - --output OUTPUT The path to the file where converted data should be written. Defaults to stdout. + --output OUTPUT The path to the file where downloaded data should be written. Defaults to $TOGGL_DATA or stdout. + --all Download all time entries. The default is to download only billable time entries. --client CLIENT_ID An ID for a Toggl Client to filter for in reports. Can be supplied more than once. --project PROJECT_ID An ID for a Toggl Project to filter for in reports. Can be supplied more than once. --task TASK_ID An ID for a Toggl Project Task to filter for in reports. Can be supplied more than once. @@ -101,13 +104,18 @@ With a CSV exported from either Harvest or Toggl, use this command to convert to ```bash $ compiler-admin time convert -h -usage: compiler-admin time convert [-h] [--input INPUT] [--output OUTPUT] [--client CLIENT] +usage: compiler-admin time convert [-h] [--input INPUT] [--output OUTPUT] [--from {harvest,toggl}] + [--to {harvest,justworks,toggl}] [--client CLIENT] options: - -h, --help show this help message and exit - --input INPUT The path to the source data for conversion. Defaults to stdin. - --output OUTPUT The path to the file where converted data should be written. Defaults to stdout. - --client CLIENT The name of the client to use in converted data. + -h, --help show this help message and exit + --input INPUT The path to the source data for conversion. Defaults to $TOGGL_DATA or stdin. + --output OUTPUT The path to the file where converted data should be written. Defaults to $HARVEST_DATA or stdout. + --from {harvest,toggl} + The format of the source data. Defaults to 'toggl'. + --to {harvest,justworks,toggl} + The format of the converted data. Defaults to 'harvest'. + --client CLIENT The name of the client to use in converted data. ``` ## Working with users diff --git a/compiler_admin/api/toggl.py b/compiler_admin/api/toggl.py index 5044b7b..02c8704 100644 --- a/compiler_admin/api/toggl.py +++ b/compiler_admin/api/toggl.py @@ -60,7 +60,6 @@ def detailed_time_entries(self, start_date: datetime, end_date: datetime, **kwar Extra `kwargs` are passed through as a POST json body. By default, requests a report with the following configuration: - * `billable=True` * `rounding=1` (True, but this is an int param) * `rounding_minutes=15` @@ -82,7 +81,6 @@ def detailed_time_entries(self, start_date: datetime, end_date: datetime, **kwar self.timeout = max(current_timeout, dynamic_timeout) params = dict( - billable=True, start_date=start, end_date=end, rounding=1, diff --git a/compiler_admin/commands/time/convert.py b/compiler_admin/commands/time/convert.py index 4f57a7e..955b1ef 100644 --- a/compiler_admin/commands/time/convert.py +++ b/compiler_admin/commands/time/convert.py @@ -1,26 +1,29 @@ from argparse import Namespace -import pandas as pd - from compiler_admin import RESULT_SUCCESS -from compiler_admin.services.harvest import INPUT_COLUMNS as TOGGL_COLUMNS, convert_to_toggl -from compiler_admin.services.toggl import INPUT_COLUMNS as HARVEST_COLUMNS, convert_to_harvest +from compiler_admin.services.harvest import CONVERTERS as HARVEST_CONVERTERS +from compiler_admin.services.toggl import CONVERTERS as TOGGL_CONVERTERS + + +CONVERTERS = {"harvest": HARVEST_CONVERTERS, "toggl": TOGGL_CONVERTERS} -def _get_source_converter(source): - columns = pd.read_csv(source, nrows=0).columns.tolist() +def _get_source_converter(from_fmt: str, to_fmt: str): + from_fmt = from_fmt.lower().strip() if from_fmt else "" + to_fmt = to_fmt.lower().strip() if to_fmt else "" + converter = CONVERTERS.get(from_fmt, {}).get(to_fmt) - if set(HARVEST_COLUMNS) <= set(columns): - return convert_to_harvest - elif set(TOGGL_COLUMNS) <= set(columns): - return convert_to_toggl + if converter: + return converter else: - raise NotImplementedError("A converter for the given source data does not exist.") + raise NotImplementedError( + f"A converter for the given source and target formats does not exist: {from_fmt} to {to_fmt}" + ) def convert(args: Namespace, *extras): - converter = _get_source_converter(args.input) + converter = _get_source_converter(args.from_fmt, args.to_fmt) - converter(args.input, args.output, args.client) + converter(source_path=args.input, output_path=args.output, client_name=args.client) return RESULT_SUCCESS diff --git a/compiler_admin/commands/time/download.py b/compiler_admin/commands/time/download.py index 6d2e468..4cb3e3b 100644 --- a/compiler_admin/commands/time/download.py +++ b/compiler_admin/commands/time/download.py @@ -1,11 +1,13 @@ from argparse import Namespace from compiler_admin import RESULT_SUCCESS -from compiler_admin.services.toggl import INPUT_COLUMNS as TOGGL_COLUMNS, download_time_entries +from compiler_admin.services.toggl import TOGGL_COLUMNS, download_time_entries def download(args: Namespace, *extras): - params = dict(start_date=args.start, end_date=args.end, output_path=args.output, output_cols=TOGGL_COLUMNS) + params = dict( + start_date=args.start, end_date=args.end, output_path=args.output, output_cols=TOGGL_COLUMNS, billable=args.billable + ) if args.client_ids: params.update(dict(client_ids=args.client_ids)) diff --git a/compiler_admin/main.py b/compiler_admin/main.py index b3632f9..bc16512 100644 --- a/compiler_admin/main.py +++ b/compiler_admin/main.py @@ -9,6 +9,7 @@ from compiler_admin.commands.info import info from compiler_admin.commands.init import init from compiler_admin.commands.time import time +from compiler_admin.commands.time.convert import CONVERTERS from compiler_admin.commands.user import user from compiler_admin.commands.user.convert import ACCOUNT_TYPE_OU @@ -78,6 +79,20 @@ def setup_time_command(cmd_parsers: _SubParsersAction): default=os.environ.get("HARVEST_DATA", sys.stdout), help="The path to the file where converted data should be written. Defaults to $HARVEST_DATA or stdout.", ) + time_convert.add_argument( + "--from", + default="toggl", + choices=sorted(CONVERTERS.keys()), + dest="from_fmt", + help="The format of the source data. Defaults to 'toggl'.", + ) + time_convert.add_argument( + "--to", + default="harvest", + choices=sorted([to_fmt for sub in CONVERTERS.values() for to_fmt in sub.keys()]), + dest="to_fmt", + help="The format of the converted data. Defaults to 'harvest'.", + ) time_convert.add_argument("--client", default=None, help="The name of the client to use in converted data.") time_download = add_sub_cmd(time_subcmds, "download", help="Download a Toggl report in CSV format.") @@ -100,6 +115,13 @@ def setup_time_command(cmd_parsers: _SubParsersAction): default=os.environ.get("TOGGL_DATA", sys.stdout), help="The path to the file where downloaded data should be written. Defaults to $TOGGL_DATA or stdout.", ) + time_download.add_argument( + "--all", + default=True, + action="store_false", + dest="billable", + help="Download all time entries. The default is to download only billable time entries.", + ) time_download.add_argument( "--client", dest="client_ids", diff --git a/compiler_admin/services/harvest.py b/compiler_admin/services/harvest.py index 562fe95..f0cff63 100644 --- a/compiler_admin/services/harvest.py +++ b/compiler_admin/services/harvest.py @@ -8,10 +8,10 @@ import compiler_admin.services.files as files # input CSV columns needed for conversion -INPUT_COLUMNS = ["Date", "Client", "Project", "Notes", "Hours", "First name", "Last name"] +HARVEST_COLUMNS = ["Date", "Client", "Project", "Notes", "Hours", "First name", "Last name"] # default output CSV columns -OUTPUT_COLUMNS = ["Email", "Start date", "Start time", "Duration", "Project", "Task", "Client", "Billable", "Description"] +TOGGL_COLUMNS = ["Email", "Start date", "Start time", "Duration", "Project", "Task", "Client", "Billable", "Description"] def _calc_start_time(group: pd.DataFrame): @@ -33,8 +33,9 @@ def _toggl_client_name(): def convert_to_toggl( source_path: str | TextIO = sys.stdin, output_path: str | TextIO = sys.stdout, + output_cols: list[str] = TOGGL_COLUMNS, client_name: str = None, - output_cols: list[str] = OUTPUT_COLUMNS, + **kwargs, ): """Convert Harvest formatted entries in source_path to equivalent Toggl formatted entries. @@ -52,7 +53,7 @@ def convert_to_toggl( client_name = _toggl_client_name() # read CSV file, parsing dates - source = files.read_csv(source_path, usecols=INPUT_COLUMNS, parse_dates=["Date"], cache_dates=True) + source = files.read_csv(source_path, usecols=HARVEST_COLUMNS, parse_dates=["Date"], cache_dates=True) # rename columns that can be imported as-is source.rename(columns={"Project": "Task", "Notes": "Description", "Date": "Start date"}, inplace=True) @@ -89,3 +90,6 @@ def convert_to_toggl( output_data.sort_values(["Start date", "Start time", "Email"], inplace=True) files.write_csv(output_path, output_data, output_cols) + + +CONVERTERS = {"toggl": convert_to_toggl} diff --git a/compiler_admin/services/toggl.py b/compiler_admin/services/toggl.py index eaedd47..06c4bfa 100644 --- a/compiler_admin/services/toggl.py +++ b/compiler_admin/services/toggl.py @@ -1,4 +1,5 @@ from datetime import datetime +from functools import cache import io import os import sys @@ -10,46 +11,74 @@ from compiler_admin.services.google import user_info as google_user_info import compiler_admin.services.files as files -# cache of previously seen user information, keyed on email -USER_INFO = files.JsonFileCache("TOGGL_USER_INFO") -NOT_FOUND = "NOT FOUND" +# input columns needed for conversion +TOGGL_COLUMNS = ["Email", "Project", "Client", "Start date", "Start time", "Duration", "Description"] -# input CSV columns needed for conversion -INPUT_COLUMNS = ["Email", "Project", "Client", "Start date", "Start time", "Duration", "Description"] +# default output CSV columns for Harvest +HARVEST_COLUMNS = ["Date", "Client", "Project", "Task", "Notes", "Hours", "First name", "Last name"] +# default output CSV columns for Justworks +JUSTWORKS_COLUMNS = ["First Name", "Last Name", "Work Email", "Start Date", "End Date", "Regular Hours"] -# default output CSV columns -OUTPUT_COLUMNS = ["Date", "Client", "Project", "Task", "Notes", "Hours", "First name", "Last name"] + +@cache +def user_info(): + """Cache of previously seen user information, keyed on email.""" + return files.JsonFileCache("TOGGL_USER_INFO") def _get_first_name(email: str) -> str: """Get cached first name or derive from email.""" - user = USER_INFO.get(email) + info = user_info() + user = info.get(email) first_name = user.get("First Name") if user else None if first_name is None: parts = email.split("@") first_name = parts[0].capitalize() data = {"First Name": first_name} - if email in USER_INFO: - USER_INFO[email].update(data) + if email in info: + info[email].update(data) else: - USER_INFO[email] = data + info[email] = data return first_name def _get_last_name(email: str): """Get cached last name or query from Google.""" - user = USER_INFO.get(email) + info = user_info() + user = info.get(email) last_name = user.get("Last Name") if user else None if last_name is None: user = google_user_info(email) last_name = user.get("Last Name") if user else None - if email in USER_INFO: - USER_INFO[email].update(user) + if email in info: + info[email].update(user) else: - USER_INFO[email] = user + info[email] = user return last_name +def _prepare_input(source_path: str | TextIO, column_renames: dict = {}) -> pd.DataFrame: + """Parse and prepare CSV data from `source_path` into an initial `pandas.DataFrame`.""" + df = files.read_csv(source_path, usecols=TOGGL_COLUMNS, parse_dates=["Start date"], cache_dates=True) + + df["Start time"] = df["Start time"].apply(_str_timedelta) + df["Duration"] = df["Duration"].apply(_str_timedelta) + + # assign First and Last name + df["First name"] = df["Email"].apply(_get_first_name) + df["Last name"] = df["Email"].apply(_get_last_name) + + # calculate hours as a decimal from duration timedelta + df["Hours"] = (df["Duration"].dt.total_seconds() / 3600).round(2) + + df.sort_values(["Start date", "Start time", "Email"], inplace=True) + + if column_renames: + df.rename(columns=column_renames, inplace=True) + + return df + + def _str_timedelta(td: str): """Convert a string formatted duration (e.g. 01:30) to a timedelta.""" return pd.to_timedelta(pd.to_datetime(td, format="%H:%M:%S").strftime("%H:%M:%S")) @@ -58,19 +87,20 @@ def _str_timedelta(td: str): def convert_to_harvest( source_path: str | TextIO = sys.stdin, output_path: str | TextIO = sys.stdout, + output_cols: list[str] = HARVEST_COLUMNS, client_name: str = None, - output_cols: list[str] = OUTPUT_COLUMNS, + **kwargs, ): """Convert Toggl formatted entries in source_path to equivalent Harvest formatted entries. Args: source_path: The path to a readable CSV file of Toggl time entries; or a readable buffer of the same. - client_name (str): The value to assign in the output "Client" field + output_path: The path to a CSV file where Harvest time entries will be written; or a writeable buffer for the same. output_cols (list[str]): A list of column names for the output - output_path: The path to a CSV file where Harvest time entries will be written; or a writeable buffer for the same. + client_name (str): The value to assign in the output "Client" field Returns: None. Either prints the resulting CSV data or writes to output_path. @@ -78,14 +108,9 @@ def convert_to_harvest( if client_name is None: client_name = os.environ.get("HARVEST_CLIENT_NAME") - # read CSV file, parsing dates and times - source = files.read_csv(source_path, usecols=INPUT_COLUMNS, parse_dates=["Start date"], cache_dates=True) - source["Start time"] = source["Start time"].apply(_str_timedelta) - source["Duration"] = source["Duration"].apply(_str_timedelta) - source.sort_values(["Start date", "Start time", "Email"], inplace=True) - - # rename columns that can be imported as-is - source.rename(columns={"Project": "Project", "Description": "Notes", "Start date": "Date"}, inplace=True) + source = _prepare_input( + source_path=source_path, column_renames={"Project": "Project", "Description": "Notes", "Start date": "Date"} + ) # update static calculated columns source["Client"] = client_name @@ -95,21 +120,60 @@ def convert_to_harvest( project_info = files.JsonFileCache("TOGGL_PROJECT_INFO") source["Project"] = source["Project"].apply(lambda x: project_info.get(key=x, default=x)) - # assign First and Last name - source["First name"] = source["Email"].apply(_get_first_name) - source["Last name"] = source["Email"].apply(_get_last_name) + files.write_csv(output_path, source, columns=output_cols) - # calculate hours as a decimal from duration timedelta - source["Hours"] = (source["Duration"].dt.total_seconds() / 3600).round(2) - files.write_csv(output_path, source, columns=output_cols) +def convert_to_justworks( + source_path: str | TextIO = sys.stdin, + output_path: str | TextIO = sys.stdout, + output_cols: list[str] = JUSTWORKS_COLUMNS, + **kwargs, +): + """Convert Toggl formatted entries in source_path to equivalent Justworks formatted entries. + + Args: + source_path: The path to a readable CSV file of Toggl time entries; or a readable buffer of the same. + + output_path: The path to a CSV file where Harvest time entries will be written; or a writeable buffer for the same. + + output_cols (list[str]): A list of column names for the output + + Returns: + None. Either prints the resulting CSV data or writes to output_path. + """ + source = _prepare_input( + source_path=source_path, + column_renames={ + "Email": "Work Email", + "First name": "First Name", + "Hours": "Regular Hours", + "Last name": "Last Name", + "Start date": "Start Date", + }, + ) + + # aggregate hours per person per day + cols = ["Work Email", "First Name", "Last Name", "Start Date"] + people = source.sort_values(cols).groupby(cols, observed=False) + people_agg = people.agg({"Regular Hours": "sum"}) + people_agg.reset_index(inplace=True) + + # aggregate hours per person and rollup to the week (starting on Sunday) + cols = ["Work Email", "First Name", "Last Name"] + weekly_agg = people_agg.groupby(cols).resample("W", label="left", on="Start Date") + weekly_agg = weekly_agg["Regular Hours"].sum().reset_index() + + # calculate the week end date (the following Saturday) + weekly_agg["End Date"] = weekly_agg["Start Date"] + pd.Timedelta(days=6) + + files.write_csv(output_path, weekly_agg, columns=output_cols) def download_time_entries( start_date: datetime, end_date: datetime, output_path: str | TextIO = sys.stdout, - output_cols: list[str] | None = INPUT_COLUMNS, + output_cols: list[str] | None = TOGGL_COLUMNS, **kwargs, ): """Download a CSV report from Toggl of detailed time entries for the given date range. @@ -153,3 +217,6 @@ def download_time_entries( df = pd.read_csv(io.StringIO(csv)) files.write_csv(output_path, df, columns=output_cols) + + +CONVERTERS = {"harvest": convert_to_harvest, "justworks": convert_to_justworks} diff --git a/notebooks/data/justworks-sample.csv b/notebooks/data/justworks-sample.csv new file mode 100644 index 0000000..62a6b99 --- /dev/null +++ b/notebooks/data/justworks-sample.csv @@ -0,0 +1,26 @@ +First Name,Last Name,Work Email,Start Date,End Date,Regular Hours +Aymer,Hauck,aymer@compiler.la,2023-01-01,2023-01-07,23.9 +Aymer,Hauck,aymer@compiler.la,2023-01-08,2023-01-14,28.3 +Aymer,Hauck,aymer@compiler.la,2023-01-15,2023-01-21,20.5 +Aymer,Hauck,aymer@compiler.la,2023-01-22,2023-01-28,19.7 +Aymer,Hauck,aymer@compiler.la,2023-01-29,2023-02-04,4.1 +Gusella,Swaile,gusella@compiler.la,2023-01-01,2023-01-07,23.9 +Gusella,Swaile,gusella@compiler.la,2023-01-08,2023-01-14,29.9 +Gusella,Swaile,gusella@compiler.la,2023-01-15,2023-01-21,11.7 +Gusella,Swaile,gusella@compiler.la,2023-01-22,2023-01-28,36.3 +Gusella,Swaile,gusella@compiler.la,2023-01-29,2023-02-04,10.2 +Hetti,Becken,hetti@compiler.la,2023-01-01,2023-01-07,24.1 +Hetti,Becken,hetti@compiler.la,2023-01-08,2023-01-14,26.2 +Hetti,Becken,hetti@compiler.la,2023-01-15,2023-01-21,34.2 +Hetti,Becken,hetti@compiler.la,2023-01-22,2023-01-28,34.0 +Hetti,Becken,hetti@compiler.la,2023-01-29,2023-02-04,4.4 +Sawyer,Berrey,sawyer@compiler.la,2023-01-01,2023-01-07,19.6 +Sawyer,Berrey,sawyer@compiler.la,2023-01-08,2023-01-14,22.8 +Sawyer,Berrey,sawyer@compiler.la,2023-01-15,2023-01-21,17.7 +Sawyer,Berrey,sawyer@compiler.la,2023-01-22,2023-01-28,15.9 +Sawyer,Berrey,sawyer@compiler.la,2023-01-29,2023-02-04,3.7 +Silas,Idenden,silas@compiler.la,2023-01-01,2023-01-07,26.4 +Silas,Idenden,silas@compiler.la,2023-01-08,2023-01-14,19.8 +Silas,Idenden,silas@compiler.la,2023-01-15,2023-01-21,30.6 +Silas,Idenden,silas@compiler.la,2023-01-22,2023-01-28,25.1 +Silas,Idenden,silas@compiler.la,2023-01-29,2023-02-04,5.4 diff --git a/notebooks/toggl-to-justworks.ipynb b/notebooks/toggl-to-justworks.ipynb new file mode 100644 index 0000000..63ecceb --- /dev/null +++ b/notebooks/toggl-to-justworks.ipynb @@ -0,0 +1,111 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Toggl to Justworks conversion\n", + "\n", + "This notebook explores a conversion process for Toggl hours into a format for import into Justworks.\n", + "\n", + "Toggl hours are tracked daily, and we can export a report of each entry in Toggl.\n", + "\n", + "Justworks expects a rollup of total hours per-person per-week." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "from compiler_admin.services.toggl import _prepare_input\n", + "from compiler_admin.services import files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "renames = {\n", + " \"Email\": \"Work Email\",\n", + " \"First name\": \"First Name\",\n", + " \"Hours\": \"Regular Hours\",\n", + " \"Last name\": \"Last Name\",\n", + " \"Start date\": \"Start Date\"\n", + "}\n", + "df = _prepare_input(\"./data/toggl-sample.csv\", column_renames=renames)\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cols = [\"Work Email\", \"First Name\", \"Last Name\", \"Start Date\"]\n", + "df_people = df.sort_values(cols).groupby(cols, observed=False)\n", + "df_people_agg = df_people.agg({\"Regular Hours\": \"sum\"})\n", + "df_people_agg.reset_index(inplace=True)\n", + "df_people_agg.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cols = [\"Work Email\", \"First Name\", \"Last Name\"]\n", + "weekly_agg = df_people_agg.groupby(cols).resample(\"W\", label=\"left\", on=\"Start Date\")\n", + "weekly_agg = weekly_agg[\"Regular Hours\"].sum().round(1).reset_index()\n", + "weekly_agg.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "weekly_agg[\"End Date\"] = weekly_agg[\"Start Date\"] + pd.Timedelta(days=6)\n", + "weekly_agg.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "files.write_csv(\"./data/justworks-sample.csv\", weekly_agg, columns=[\"First Name\", \"Last Name\", \"Work Email\", \"Start Date\", \"End Date\", \"Regular Hours\"])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/api/test_toggl.py b/tests/api/test_toggl.py index 2e5dccd..9635477 100644 --- a/tests/api/test_toggl.py +++ b/tests/api/test_toggl.py @@ -72,7 +72,6 @@ def test_toggl_detailed_time_entries(toggl_mock_post_reports): toggl_mock_post_reports.post_reports.assert_called_once_with( "search/time_entries.csv", - billable=True, start_date="2024-09-25", end_date="2024-09-25", rounding=1, diff --git a/tests/commands/time/test_convert.py b/tests/commands/time/test_convert.py index b3fbc98..68fc580 100644 --- a/tests/commands/time/test_convert.py +++ b/tests/commands/time/test_convert.py @@ -1,15 +1,15 @@ from argparse import Namespace -from io import StringIO import pytest from compiler_admin import RESULT_SUCCESS from compiler_admin.commands.time.convert import ( __name__ as MODULE, + CONVERTERS, _get_source_converter, - convert_to_harvest, - convert_to_toggl, convert, ) +from compiler_admin.services.harvest import CONVERTERS as HARVEST_CONVERTERS +from compiler_admin.services.toggl import CONVERTERS as TOGGL_CONVERTERS @pytest.fixture @@ -17,29 +17,40 @@ def mock_get_source_converter(mocker): return mocker.patch(f"{MODULE}._get_source_converter") -def test_get_source_converter_match_toggl(toggl_file): - result = _get_source_converter(toggl_file) - - assert result == convert_to_harvest +@pytest.fixture +def mock_converters(mocker): + return mocker.patch(f"{MODULE}.CONVERTERS", new={}) -def test_get_source_converter_match_harvest(harvest_file): - result = _get_source_converter(harvest_file) +def test_get_source_converter_match(mock_converters): + mock_converters["toggl"] = {"test_fmt": "converter"} + result = _get_source_converter("toggl", "test_fmt") - assert result == convert_to_toggl + assert result == "converter" def test_get_source_converter_mismatch(): - data = StringIO("one,two,three\n1,2,3") - - with pytest.raises(NotImplementedError, match="A converter for the given source data does not exist."): - _get_source_converter(data) + with pytest.raises( + NotImplementedError, match="A converter for the given source and target formats does not exist: nope to toggl" + ): + _get_source_converter("nope", "toggl") + with pytest.raises( + NotImplementedError, match="A converter for the given source and target formats does not exist: toggl to nope" + ): + _get_source_converter("toggl", "nope") def test_convert(mock_get_source_converter): - args = Namespace(input="input", output="output", client="client") + args = Namespace(input="input", output="output", client="client", from_fmt="from", to_fmt="to") res = convert(args) assert res == RESULT_SUCCESS - mock_get_source_converter.assert_called_once_with(args.input) - mock_get_source_converter.return_value.assert_called_once_with(args.input, args.output, args.client) + mock_get_source_converter.assert_called_once_with(args.from_fmt, args.to_fmt) + mock_get_source_converter.return_value.assert_called_once_with( + source_path=args.input, output_path=args.output, client_name=args.client + ) + + +def test_converters(): + assert CONVERTERS.get("harvest") == HARVEST_CONVERTERS + assert CONVERTERS.get("toggl") == TOGGL_CONVERTERS diff --git a/tests/commands/time/test_download.py b/tests/commands/time/test_download.py index a634fdf..fe9f80b 100644 --- a/tests/commands/time/test_download.py +++ b/tests/commands/time/test_download.py @@ -11,16 +11,18 @@ def mock_download_time_entries(mocker): return mocker.patch(f"{MODULE}.download_time_entries") -def test_download_default(mock_download_time_entries): +@pytest.mark.parametrize("billable", [True, False]) +def test_download(mock_download_time_entries, billable): date = datetime.now() args = Namespace( start=date, end=date, output="output", - client_ids=None, - project_ids=None, - task_ids=None, - user_ids=None, + billable=billable, + client_ids=["c1", "c2"], + project_ids=["p1", "p2"], + task_ids=["t1", "t2"], + user_ids=["u1", "u2"], ) res = download(args) @@ -31,24 +33,9 @@ def test_download_default(mock_download_time_entries): end_date=args.end, output_path=args.output, output_cols=TOGGL_COLUMNS, - ) - - -def test_download_ids(mock_download_time_entries): - date = datetime.now() - ids = [1, 2, 3] - args = Namespace(start=date, end=date, output="output", client_ids=ids, project_ids=ids, task_ids=ids, user_ids=ids) - - res = download(args) - - assert res == RESULT_SUCCESS - mock_download_time_entries.assert_called_once_with( - start_date=args.start, - end_date=args.end, - output_path=args.output, - output_cols=TOGGL_COLUMNS, - client_ids=ids, - project_ids=ids, - task_ids=ids, - user_ids=ids, + billable=args.billable, + client_ids=args.client_ids, + project_ids=args.project_ids, + task_ids=args.task_ids, + user_ids=args.user_ids, ) diff --git a/tests/conftest.py b/tests/conftest.py index a2ea7e5..0625742 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -191,6 +191,11 @@ def harvest_file(): return "notebooks/data/harvest-sample.csv" +@pytest.fixture +def justworks_file(): + return "notebooks/data/justworks-sample.csv" + + @pytest.fixture def toggl_file(): return "notebooks/data/toggl-sample.csv" diff --git a/tests/services/test_harvest.py b/tests/services/test_harvest.py index 29c0f1d..5eee7b3 100644 --- a/tests/services/test_harvest.py +++ b/tests/services/test_harvest.py @@ -10,8 +10,9 @@ from compiler_admin.services.harvest import ( __name__ as MODULE, files, - INPUT_COLUMNS, - OUTPUT_COLUMNS, + HARVEST_COLUMNS, + TOGGL_COLUMNS, + CONVERTERS, _calc_start_time, _duration_str, _toggl_client_name, @@ -81,26 +82,26 @@ def test_convert_to_toggl_mocked(harvest_file, spy_files, mock_toggl_client_name spy_files.read_csv.assert_called_once() call_args = spy_files.read_csv.call_args assert (harvest_file,) in call_args - assert call_args.kwargs["usecols"] == INPUT_COLUMNS + assert call_args.kwargs["usecols"] == HARVEST_COLUMNS assert call_args.kwargs["parse_dates"] == ["Date"] assert call_args.kwargs["cache_dates"] is True spy_files.write_csv.assert_called_once() call_args = spy_files.write_csv.call_args assert call_args[0][0] == sys.stdout - assert call_args[0][2] == OUTPUT_COLUMNS + assert call_args[0][2] == TOGGL_COLUMNS def test_convert_to_toggl_sample(harvest_file, toggl_file): output = None with StringIO() as output_data: - convert_to_toggl(harvest_file, output_data, "Test Client 123") + convert_to_toggl(harvest_file, output_data, client_name="Test Client 123") output = output_data.getvalue() assert output assert isinstance(output, str) - assert ",".join(OUTPUT_COLUMNS) in output + assert ",".join(TOGGL_COLUMNS) in output order = ["Start date", "Start time", "Email"] sample_output_df = pd.read_csv(toggl_file).sort_values(order) @@ -109,3 +110,7 @@ def test_convert_to_toggl_sample(harvest_file, toggl_file): assert set(output_df.columns.to_list()) <= set(sample_output_df.columns.to_list()) assert output_df["Client"].eq("Test Client 123").all() assert output_df["Project"].eq("Test Client 123").all() + + +def test_converters(): + assert CONVERTERS.get("toggl") == convert_to_toggl diff --git a/tests/services/test_toggl.py b/tests/services/test_toggl.py index 994a77c..d6dc80a 100644 --- a/tests/services/test_toggl.py +++ b/tests/services/test_toggl.py @@ -9,15 +9,19 @@ import compiler_admin.services.toggl from compiler_admin.services.toggl import ( + CONVERTERS, __name__ as MODULE, - files, - INPUT_COLUMNS, - OUTPUT_COLUMNS, _get_first_name, _get_last_name, + _prepare_input, _str_timedelta, convert_to_harvest, + convert_to_justworks, download_time_entries, + TOGGL_COLUMNS, + HARVEST_COLUMNS, + JUSTWORKS_COLUMNS, + files, ) @@ -33,9 +37,9 @@ def spy_files(mocker): return mocker.patch.object(compiler_admin.services.toggl, "files", wraps=files) -@pytest.fixture(autouse=True) -def mock_USER_INFO(mocker): - return mocker.patch(f"{MODULE}.USER_INFO", new={}) +@pytest.fixture() +def mock_user_info(mocker): + return mocker.patch(f"{MODULE}.user_info") @pytest.fixture @@ -63,38 +67,38 @@ def mock_toggl_detailed_time_entries(mock_toggl_api, toggl_file): return mock_toggl_api -def test_get_first_name_matching(mock_USER_INFO): - mock_USER_INFO["email"] = {"First Name": "User"} +def test_get_first_name_matching(mock_user_info): + mock_user_info.return_value = {"email": {"First Name": "User"}} result = _get_first_name("email") assert result == "User" -def test_get_first_name_calcuated_with_record(mock_USER_INFO): +def test_get_first_name_calcuated_with_record(mock_user_info): email = "user@email.com" - mock_USER_INFO[email] = {"Data": 1234} + mock_user_info.return_value = {email: {"Data": 1234}} result = _get_first_name(email) assert result == "User" - assert mock_USER_INFO[email]["First Name"] == "User" - assert mock_USER_INFO[email]["Data"] == 1234 + assert mock_user_info.return_value[email]["First Name"] == "User" + assert mock_user_info.return_value[email]["Data"] == 1234 -def test_get_first_name_calcuated_without_record(mock_USER_INFO): +def test_get_first_name_calcuated_without_record(mock_user_info): email = "user@email.com" - mock_USER_INFO[email] = {} + mock_user_info.return_value = {email: {}} result = _get_first_name(email) assert result == "User" - assert mock_USER_INFO[email]["First Name"] == "User" - assert list(mock_USER_INFO[email].keys()) == ["First Name"] + assert mock_user_info.return_value[email]["First Name"] == "User" + assert list(mock_user_info.return_value[email].keys()) == ["First Name"] -def test_get_last_name_matching(mock_USER_INFO, mock_google_user_info): - mock_USER_INFO["email"] = {"Last Name": "User"} +def test_get_last_name_matching(mock_user_info, mock_google_user_info): + mock_user_info.return_value = {"email": {"Last Name": "User"}} result = _get_last_name("email") @@ -102,29 +106,29 @@ def test_get_last_name_matching(mock_USER_INFO, mock_google_user_info): mock_google_user_info.assert_not_called() -def test_get_last_name_lookup_with_record(mock_USER_INFO, mock_google_user_info): +def test_get_last_name_lookup_with_record(mock_user_info, mock_google_user_info): email = "user@email.com" - mock_USER_INFO[email] = {"Data": 1234} + mock_user_info.return_value = {email: {"Data": 1234}} mock_google_user_info.return_value = {"Last Name": "User"} result = _get_last_name(email) assert result == "User" - assert mock_USER_INFO[email]["Last Name"] == "User" - assert mock_USER_INFO[email]["Data"] == 1234 + assert mock_user_info.return_value[email]["Last Name"] == "User" + assert mock_user_info.return_value[email]["Data"] == 1234 mock_google_user_info.assert_called_once_with(email) -def test_get_last_name_lookup_without_record(mock_USER_INFO, mock_google_user_info): +def test_get_last_name_lookup_without_record(mock_user_info, mock_google_user_info): email = "user@email.com" - mock_USER_INFO[email] = {} + mock_user_info.return_value = {email: {}} mock_google_user_info.return_value = {"Last Name": "User"} result = _get_last_name(email) assert result == "User" - assert mock_USER_INFO[email]["Last Name"] == "User" - assert list(mock_USER_INFO[email].keys()) == ["Last Name"] + assert mock_user_info.return_value[email]["Last Name"] == "User" + assert list(mock_user_info.return_value[email].keys()) == ["Last Name"] mock_google_user_info.assert_called_once_with(email) @@ -137,22 +141,46 @@ def test_str_timedelta(): assert result.total_seconds() == (1 * 60 * 60) + (30 * 60) + 15 -def test_convert_to_harvest_mocked(toggl_file, spy_files, mock_google_user_info): - mock_google_user_info.return_value = {} - - convert_to_harvest(toggl_file, client_name=None) +@pytest.mark.usefixtures("mock_google_user_info") +def test_prepare_input(toggl_file, spy_files): + df = _prepare_input(toggl_file) spy_files.read_csv.assert_called_once() call_args = spy_files.read_csv.call_args assert (toggl_file,) in call_args - assert call_args.kwargs["usecols"] == INPUT_COLUMNS + assert call_args.kwargs["usecols"] == TOGGL_COLUMNS assert call_args.kwargs["parse_dates"] == ["Start date"] assert call_args.kwargs["cache_dates"] is True + df_cols = df.columns.to_list() + assert set(df_cols) <= set(TOGGL_COLUMNS) or set(TOGGL_COLUMNS) <= set(df_cols) + + assert "First name" in df_cols + assert "Last name" in df_cols + assert df["Start date"].dtype.name == "datetime64[ns]" + assert df["Start time"].dtype.name == "timedelta64[ns]" + assert df["Duration"].dtype.name == "timedelta64[ns]" + assert df["Hours"].dtype.name == "float64" + + df = _prepare_input(toggl_file, column_renames={"Start date": "SD", "Start time": "ST", "Duration": "D"}) + + assert "Start date" not in df.columns + assert "Start time" not in df.columns + assert "Duration" not in df.columns + assert df["SD"].dtype.name == "datetime64[ns]" + assert df["ST"].dtype.name == "timedelta64[ns]" + assert df["D"].dtype.name == "timedelta64[ns]" + + +def test_convert_to_harvest_mocked(toggl_file, spy_files, mock_google_user_info): + mock_google_user_info.return_value = {} + + convert_to_harvest(toggl_file, client_name=None) + spy_files.write_csv.assert_called_once() call_args = spy_files.write_csv.call_args assert sys.stdout in call_args[0] - assert call_args.kwargs["columns"] == OUTPUT_COLUMNS + assert call_args.kwargs["columns"] == HARVEST_COLUMNS def test_convert_to_harvest_sample(toggl_file, harvest_file, mock_google_user_info): @@ -160,12 +188,12 @@ def test_convert_to_harvest_sample(toggl_file, harvest_file, mock_google_user_in output = None with StringIO() as output_data: - convert_to_harvest(toggl_file, output_data, "Test Client 123") + convert_to_harvest(toggl_file, output_data, client_name="Test Client 123") output = output_data.getvalue() assert output assert isinstance(output, str) - assert ",".join(OUTPUT_COLUMNS) in output + assert ",".join(HARVEST_COLUMNS) in output order = ["Date", "First name", "Hours"] sample_output_df = pd.read_csv(harvest_file).sort_values(order) @@ -175,6 +203,34 @@ def test_convert_to_harvest_sample(toggl_file, harvest_file, mock_google_user_in assert output_df["Client"].eq("Test Client 123").all() +def test_convert_to_justworks_mocked(toggl_file, spy_files): + convert_to_justworks(toggl_file) + + spy_files.write_csv.assert_called_once() + call_args = spy_files.write_csv.call_args + assert sys.stdout in call_args[0] + assert call_args.kwargs["columns"] == JUSTWORKS_COLUMNS + + +def test_convert_to_justworks_sample(toggl_file, justworks_file): + output = None + + with StringIO() as output_data: + convert_to_justworks(toggl_file, output_data) + output = output_data.getvalue() + + assert output + assert isinstance(output, str) + assert ",".join(JUSTWORKS_COLUMNS) in output + + order = ["Start Date", "First Name", "Regular Hours"] + sample_output_df = pd.read_csv(justworks_file).sort_values(order) + output_df = pd.read_csv(StringIO(output)).sort_values(order) + + assert set(output_df.columns.to_list()) <= set(sample_output_df.columns.to_list()) + assert output_df.shape == sample_output_df.shape + + @pytest.mark.usefixtures("mock_toggl_api_env", "mock_toggl_detailed_time_entries") def test_download_time_entries(toggl_file): dt = datetime.now() @@ -196,3 +252,8 @@ def test_download_time_entries(toggl_file): # as corresponding column values from the mock DataFrame for col in response_df.columns: assert response_df[col].equals(mock_df[col]) + + +def test_converters(): + assert CONVERTERS.get("harvest") == convert_to_harvest + assert CONVERTERS.get("justworks") == convert_to_justworks diff --git a/tests/test_main.py b/tests/test_main.py index 5e7acaa..fcec85f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -104,7 +104,7 @@ def test_main_init_gyb(mock_commands_init): def test_main_init_no_username(mock_commands_init): with pytest.raises(SystemExit): main(argv=["init"]) - assert mock_commands_init.call_count == 0 + assert mock_commands_init.call_count == 0 def test_main_time_convert_default(mock_commands_time): @@ -114,7 +114,14 @@ def test_main_time_convert_default(mock_commands_time): call_args = mock_commands_time.call_args.args assert ( Namespace( - func=mock_commands_time, command="time", subcommand="convert", client=None, input=sys.stdin, output=sys.stdout + func=mock_commands_time, + command="time", + subcommand="convert", + client=None, + input=sys.stdin, + output=sys.stdout, + from_fmt="toggl", + to_fmt="harvest", ) in call_args ) @@ -129,7 +136,16 @@ def test_main_time_convert_env(monkeypatch, mock_commands_time): mock_commands_time.assert_called_once() call_args = mock_commands_time.call_args.args assert ( - Namespace(func=mock_commands_time, command="time", subcommand="convert", client=None, input="toggl", output="harvest") + Namespace( + func=mock_commands_time, + command="time", + subcommand="convert", + client=None, + input="toggl", + output="harvest", + from_fmt="toggl", + to_fmt="harvest", + ) in call_args ) @@ -148,6 +164,7 @@ def test_main_time_download_default(mock_commands_time, mock_start, mock_end): start=mock_start, end=mock_end, output=sys.stdout, + billable=True, client_ids=None, project_ids=None, task_ids=None, @@ -168,6 +185,7 @@ def test_main_time_download_args(mock_commands_time): "2024-01-31", "--output", "file.csv", + "--all", "--client", "1", "--client", @@ -209,6 +227,7 @@ def test_main_time_download_args(mock_commands_time): start=expected_start, end=expected_end, output="file.csv", + billable=False, client_ids=ids, project_ids=ids, task_ids=ids, @@ -231,6 +250,8 @@ def test_main_time_convert_client(mock_commands_time): client="client123", input=sys.stdin, output=sys.stdout, + from_fmt="toggl", + to_fmt="harvest", ) in call_args ) @@ -249,6 +270,8 @@ def test_main_time_convert_input(mock_commands_time): client=None, input="file.csv", output=sys.stdout, + from_fmt="toggl", + to_fmt="harvest", ) in call_args ) @@ -267,10 +290,62 @@ def test_main_time_convert_output(mock_commands_time): client=None, input=sys.stdin, output="file.csv", + from_fmt="toggl", + to_fmt="harvest", + ) + in call_args + ) + + +def test_main_time_convert_from(mock_commands_time): + main(argv=["time", "convert", "--from", "harvest"]) + + mock_commands_time.assert_called_once() + call_args = mock_commands_time.call_args.args + assert ( + Namespace( + func=mock_commands_time, + command="time", + subcommand="convert", + client=None, + input=sys.stdin, + output=sys.stdout, + from_fmt="harvest", + to_fmt="harvest", ) in call_args ) + with pytest.raises(SystemExit): + main(argv=["time", "convert", "--from", "nope"]) + # it should not have been called an additional time from the first + mock_commands_time.assert_called_once() + + +def test_main_time_convert_to(mock_commands_time): + main(argv=["time", "convert", "--to", "toggl"]) + + mock_commands_time.assert_called_once() + call_args = mock_commands_time.call_args.args + assert ( + Namespace( + func=mock_commands_time, + command="time", + subcommand="convert", + client=None, + input=sys.stdin, + output=sys.stdout, + from_fmt="toggl", + to_fmt="toggl", + ) + in call_args + ) + + with pytest.raises(SystemExit): + main(argv=["time", "convert", "--to", "nope"]) + # it should not have been called an additional time from the first + mock_commands_time.assert_called_once() + def test_main_user_alumni(mock_commands_user): main(argv=["user", "alumni", "username"]) @@ -355,7 +430,7 @@ def test_main_user_create_extras(mock_commands_user): def test_main_user_create_no_username(mock_commands_user): with pytest.raises(SystemExit): main(argv=["user", "create"]) - assert mock_commands_user.call_count == 0 + assert mock_commands_user.call_count == 0 def test_main_user_convert(mock_commands_user): @@ -380,13 +455,13 @@ def test_main_user_convert(mock_commands_user): def test_main_user_convert_no_username(mock_commands_user): with pytest.raises(SystemExit): main(argv=["user", "convert"]) - assert mock_commands_user.call_count == 0 + assert mock_commands_user.call_count == 0 def test_main_user_convert_bad_account_type(mock_commands_user): with pytest.raises(SystemExit): main(argv=["user", "convert", "username", "account_type"]) - assert mock_commands_user.call_count == 0 + assert mock_commands_user.call_count == 0 def test_main_user_delete(mock_commands_user): @@ -412,7 +487,7 @@ def test_main_user_delete_force(mock_commands_user): def test_main_user_delete_no_username(mock_commands_user): with pytest.raises(SystemExit): main(argv=["user", "delete"]) - assert mock_commands_user.call_count == 0 + assert mock_commands_user.call_count == 0 def test_main_user_offboard(mock_commands_user): @@ -458,7 +533,7 @@ def test_main_user_offboard_with_alias(mock_commands_user): def test_main_user_offboard_no_username(mock_commands_user): with pytest.raises(SystemExit): main(argv=["user", "offboard"]) - assert mock_commands_user.call_count == 0 + assert mock_commands_user.call_count == 0 def test_main_user_reset(mock_commands_user): @@ -504,7 +579,7 @@ def test_main_user_reset_force(mock_commands_user): def test_main_user_reset_no_username(mock_commands_user): with pytest.raises(SystemExit): main(argv=["user", "reset"]) - assert mock_commands_user.call_count == 0 + assert mock_commands_user.call_count == 0 def test_main_user_restore(mock_commands_user): @@ -518,7 +593,7 @@ def test_main_user_restore(mock_commands_user): def test_main_user_restore_no_username(mock_commands_user): with pytest.raises(SystemExit): main(argv=["user", "restore"]) - assert mock_commands_user.call_count == 0 + assert mock_commands_user.call_count == 0 def test_main_user_signout(mock_commands_user): @@ -544,7 +619,7 @@ def test_main_user_signout_force(mock_commands_user): def test_main_user_signout_no_username(mock_commands_user): with pytest.raises(SystemExit): main(argv=["user", "signout"]) - assert mock_commands_user.call_count == 0 + assert mock_commands_user.call_count == 0 @pytest.mark.e2e