From b806f691fe8de18e462c62d39b748c61188903b0 Mon Sep 17 00:00:00 2001 From: Jeremy Dang Date: Mon, 14 Apr 2025 16:14:29 +0100 Subject: [PATCH 01/13] add quickbooks source --- sources/quickbooks/README.md | 56 +++++++++++++++++++++++++ sources/quickbooks/__init__.py | 63 +++++++++++++++++++++++++++++ sources/quickbooks/requirements.txt | 2 + 3 files changed, 121 insertions(+) create mode 100644 sources/quickbooks/README.md create mode 100644 sources/quickbooks/__init__.py create mode 100644 sources/quickbooks/requirements.txt diff --git a/sources/quickbooks/README.md b/sources/quickbooks/README.md new file mode 100644 index 000000000..ce20d7e55 --- /dev/null +++ b/sources/quickbooks/README.md @@ -0,0 +1,56 @@ +# Quickbooks + +QuickBooks is a cloud-based accounting software designed for small to medium-sized businesses. This QuickBooks `dlt` verified source and pipeline example offers the capability to load QuickBooks endpoints such as "Customer" to a destination of your choosing. It enables you to conveniently load the following endpoint as a start: + +### Single loading endpoints (replace mode) + +| Endpoint | Mode | Description | +| --- | --- | --- | +| Customer | replace | A customer is a consumer of the service or product that your business offers. An individual customer can have an underlying nested structure, with a parent customer (the top-level object) having zero or more sub-customers and jobs associated with it. | + + +## Initialize the pipeline with Quickbooks verified source +```bash +dlt init quickbooks duckdb +``` + +Here, we chose DuckDB as the destination. Alternatively, you can also choose redshift, snowflake, or any of the other [destinations.](https://dlthub.com/docs/dlt-ecosystem/destinations/) + +## Setup verified source and pipeline example + +To grab credentials and initialize the verified source, please refer to the [full documentation here.](https://dlthub.com/docs/dlt-ecosystem/verified-sources/salesforce) + +## Add credentials + +1. Open `.dlt/secrets.toml`. +2. Enter the user name, password and security token: + ```toml + # put your secret values and credentials here. do not share this file and do not push it to github + [sources.quickbooks] + client_id="" + client_secret="" +a ccess_token="" + refresh_token= "" + ``` + +3. Enter credentials for your chosen destination as per the [docs.](https://dlthub.com/docs/dlt-ecosystem/destinations/) + +## Run the pipeline example + +1. Install the necessary dependencies by running the following command: + ```bash + pip install -r requirements.txt + ``` + +2. Now the pipeline can be run by using the command: + ```bash + python3 quickbooks_pipeline.py + ``` + +3. To make sure that everything is loaded as expected, use the command: + ```bash + dlt pipeline show + ``` + + For example, the pipeline_name for the above pipeline is `quickbooks`, you may also use any custom name instead. + diff --git a/sources/quickbooks/__init__.py b/sources/quickbooks/__init__.py new file mode 100644 index 000000000..056012aaa --- /dev/null +++ b/sources/quickbooks/__init__.py @@ -0,0 +1,63 @@ +"""Source for Quickbooks depending on the quickbooks-python python package. + +Quickbooks-python docs: https://github.com/ej2/python-quickbooks +Quickbooks api docs: https://developer.salesforce.com/docs/apis +Quickbooks company id: https://quickbooks.intuit.com/learn-support/en-uk/help-article/customer-company-settings/find-quickbooks-online-company-id/L7lp8O9yU_GB_en_GB +To get API credentials: https://developer.intuit.com/app/developer/qbo/docs/get-started/start-developing-your-app +Get oAuth Authorization code from: https://developer.intuit.com/app/developer/playground +""" + +from dlt.sources import DltResource + +from typing import Iterable + +import dlt +from dlt.common.typing import TDataItem +from intuitlib.client import AuthClient +from quickbooks import QuickBooks +from quickbooks.objects.customer import Customer + + +@dlt.source(name="quickbooks") +def quickbooks_source( + environment: str = dlt.secrets.value, + client_id: str = dlt.secrets.value, + client_secret: str = dlt.secrets.value, + access_token: str = dlt.secrets.value, + refresh_token: str = dlt.secrets.value, + company_id: str = dlt.secrets.value # also called realm_id +) -> Iterable[DltResource]: + """ + Retrieves data from Quickbooks using the Quickbooks API. + + Args: + environment (str): The environment used for authentication ('sandbox' or 'production') + client_id (str): The client id provided by quickbooks for authentication. Defaults to the value in the `dlt.secrets` object. + client_secret (str): The client secret provided by quickbooks for authentication. Defaults to the value in the `dlt.secrets` object. + access_token (str): The access token provided by quickbooks oAuth playground for authentication. Defaults to the value in the `dlt.secrets` object. + refresh_token (str): The refresh token provided by quickbooks oAuth playground for authentication. Defaults to the value in the `dlt.secrets` object. + company_id (str): The company id provided by quickbooks. Defaults to the value in the `dlt.secrets` object. + + Yields: + DltResource: Data resources from Quickbooks. + """ + auth_client = AuthClient( + client_id=client_id, + client_secret=client_secret, + environment=environment, + redirect_uri='http://localhost:8000/callback', + ) + + client = QuickBooks( + auth_client=auth_client, + refresh_token=refresh_token, + company_id=company_id, + access_token=access_token, + ) + + # define resources + @dlt.resource(write_disposition="replace") + def customer() -> Iterable[TDataItem]: + customer = Customer.all(qb=client) # returns a list of iterables + for record in customer: + yield record.to_dict() \ No newline at end of file diff --git a/sources/quickbooks/requirements.txt b/sources/quickbooks/requirements.txt new file mode 100644 index 000000000..a9400f4ac --- /dev/null +++ b/sources/quickbooks/requirements.txt @@ -0,0 +1,2 @@ +dlt>=1.0.0 +python-quickbooks \ No newline at end of file From e2f689dbfab2b364bc1f7ec6e66275103f1a1124 Mon Sep 17 00:00:00 2001 From: Jeremy Dang Date: Mon, 14 Apr 2025 16:16:33 +0100 Subject: [PATCH 02/13] fix readme --- sources/quickbooks/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/quickbooks/README.md b/sources/quickbooks/README.md index ce20d7e55..75da5e9e2 100644 --- a/sources/quickbooks/README.md +++ b/sources/quickbooks/README.md @@ -29,7 +29,7 @@ To grab credentials and initialize the verified source, please refer to the [ful [sources.quickbooks] client_id="" client_secret="" -a ccess_token="" + access_token="" refresh_token= "" ``` From 7ea5b240e4d96dfb59fd7100978c84ff55f867f4 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 22 Apr 2025 14:33:18 +0100 Subject: [PATCH 03/13] rename due to circular imports --- .../README.md | 10 +++--- .../__init__.py | 34 +++++++++++-------- .../requirements.txt | 0 sources/quickbooks_pipeline.py | 14 ++++++++ 4 files changed, 38 insertions(+), 20 deletions(-) rename sources/{quickbooks => quickbooks_online}/README.md (97%) rename sources/{quickbooks => quickbooks_online}/__init__.py (63%) rename sources/{quickbooks => quickbooks_online}/requirements.txt (100%) create mode 100644 sources/quickbooks_pipeline.py diff --git a/sources/quickbooks/README.md b/sources/quickbooks_online/README.md similarity index 97% rename from sources/quickbooks/README.md rename to sources/quickbooks_online/README.md index 75da5e9e2..014afa8f9 100644 --- a/sources/quickbooks/README.md +++ b/sources/quickbooks_online/README.md @@ -11,7 +11,7 @@ QuickBooks is a cloud-based accounting software designed for small to medium-siz ## Initialize the pipeline with Quickbooks verified source ```bash -dlt init quickbooks duckdb +dlt init quickbooks_online duckdb ``` Here, we chose DuckDB as the destination. Alternatively, you can also choose redshift, snowflake, or any of the other [destinations.](https://dlthub.com/docs/dlt-ecosystem/destinations/) @@ -32,7 +32,7 @@ To grab credentials and initialize the verified source, please refer to the [ful access_token="" refresh_token= "" ``` - + 3. Enter credentials for your chosen destination as per the [docs.](https://dlthub.com/docs/dlt-ecosystem/destinations/) ## Run the pipeline example @@ -41,16 +41,16 @@ To grab credentials and initialize the verified source, please refer to the [ful ```bash pip install -r requirements.txt ``` - + 2. Now the pipeline can be run by using the command: ```bash python3 quickbooks_pipeline.py ``` - + 3. To make sure that everything is loaded as expected, use the command: ```bash dlt pipeline show ``` - + For example, the pipeline_name for the above pipeline is `quickbooks`, you may also use any custom name instead. diff --git a/sources/quickbooks/__init__.py b/sources/quickbooks_online/__init__.py similarity index 63% rename from sources/quickbooks/__init__.py rename to sources/quickbooks_online/__init__.py index 056012aaa..e2776096d 100644 --- a/sources/quickbooks/__init__.py +++ b/sources/quickbooks_online/__init__.py @@ -1,9 +1,9 @@ -"""Source for Quickbooks depending on the quickbooks-python python package. +"""Source for Quickbooks depending on the quickbooks_online-python python package. Quickbooks-python docs: https://github.com/ej2/python-quickbooks Quickbooks api docs: https://developer.salesforce.com/docs/apis Quickbooks company id: https://quickbooks.intuit.com/learn-support/en-uk/help-article/customer-company-settings/find-quickbooks-online-company-id/L7lp8O9yU_GB_en_GB -To get API credentials: https://developer.intuit.com/app/developer/qbo/docs/get-started/start-developing-your-app +To get API credentials: https://developer.intuit.com/app/developer/qbo/docs/get-started/start-developing-your-app Get oAuth Authorization code from: https://developer.intuit.com/app/developer/playground """ @@ -18,25 +18,26 @@ from quickbooks.objects.customer import Customer -@dlt.source(name="quickbooks") -def quickbooks_source( +@dlt.source(name="quickbooks_online") +def quickbooks_online( environment: str = dlt.secrets.value, client_id: str = dlt.secrets.value, client_secret: str = dlt.secrets.value, access_token: str = dlt.secrets.value, refresh_token: str = dlt.secrets.value, - company_id: str = dlt.secrets.value # also called realm_id + company_id: str = dlt.secrets.value, # also called realm_id + redirect_uri: str = dlt.secrets.value # get from quickbooks_online dev portal ) -> Iterable[DltResource]: """ Retrieves data from Quickbooks using the Quickbooks API. Args: environment (str): The environment used for authentication ('sandbox' or 'production') - client_id (str): The client id provided by quickbooks for authentication. Defaults to the value in the `dlt.secrets` object. - client_secret (str): The client secret provided by quickbooks for authentication. Defaults to the value in the `dlt.secrets` object. - access_token (str): The access token provided by quickbooks oAuth playground for authentication. Defaults to the value in the `dlt.secrets` object. - refresh_token (str): The refresh token provided by quickbooks oAuth playground for authentication. Defaults to the value in the `dlt.secrets` object. - company_id (str): The company id provided by quickbooks. Defaults to the value in the `dlt.secrets` object. + client_id (str): The client id provided by quickbooks_online for authentication. Defaults to the value in the `dlt.secrets` object. + client_secret (str): The client secret provided by quickbooks_online for authentication. Defaults to the value in the `dlt.secrets` object. + access_token (str): The access token provided by quickbooks_online oAuth playground for authentication. Defaults to the value in the `dlt.secrets` object. + refresh_token (str): The refresh token provided by quickbooks_online oAuth playground for authentication. Defaults to the value in the `dlt.secrets` object. + company_id (str): The company id provided by quickbooks_online. Defaults to the value in the `dlt.secrets` object. Yields: DltResource: Data resources from Quickbooks. @@ -45,19 +46,22 @@ def quickbooks_source( client_id=client_id, client_secret=client_secret, environment=environment, - redirect_uri='http://localhost:8000/callback', + redirect_uri=redirect_uri, + access_token=access_token, + ) client = QuickBooks( auth_client=auth_client, refresh_token=refresh_token, - company_id=company_id, - access_token=access_token, + company_id=company_id ) # define resources - @dlt.resource(write_disposition="replace") + @dlt.resource def customer() -> Iterable[TDataItem]: customer = Customer.all(qb=client) # returns a list of iterables for record in customer: - yield record.to_dict() \ No newline at end of file + yield record.to_dict() + + return customer diff --git a/sources/quickbooks/requirements.txt b/sources/quickbooks_online/requirements.txt similarity index 100% rename from sources/quickbooks/requirements.txt rename to sources/quickbooks_online/requirements.txt diff --git a/sources/quickbooks_pipeline.py b/sources/quickbooks_pipeline.py new file mode 100644 index 000000000..82367b91d --- /dev/null +++ b/sources/quickbooks_pipeline.py @@ -0,0 +1,14 @@ +import dlt +from quickbooks_online import quickbooks_online + +def load_customer(): + pipeline = dlt.pipeline( + pipeline_name='quickbooks_customer', + destination='duckdb', + dataset_name='quickbooks_online' + ) + load_info = pipeline.run(quickbooks_online()) + print(load_info) + +if __name__ == "__main__": + load_customer() \ No newline at end of file From 560f69ccb5d4d9afec2958b09b8f3d61cc40e1c5 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 22 Apr 2025 15:21:40 +0100 Subject: [PATCH 04/13] update mypy, add tests, update readme, add versions in requirements, rename pipeline, add tests --- mypy.ini | 6 ++++ sources/quickbooks_online/README.md | 13 +++++---- sources/quickbooks_online/__init__.py | 11 +++----- sources/quickbooks_online/requirements.txt | 5 ++-- ...eline.py => quickbooks_online_pipeline.py} | 12 ++++---- tests/quickbooks_online/__init__.py | 0 .../test_quickbooks_online_source.py | 28 +++++++++++++++++++ 7 files changed, 56 insertions(+), 19 deletions(-) rename sources/{quickbooks_pipeline.py => quickbooks_online_pipeline.py} (53%) create mode 100644 tests/quickbooks_online/__init__.py create mode 100644 tests/quickbooks_online/test_quickbooks_online_source.py diff --git a/mypy.ini b/mypy.ini index dc12fb3ae..23d9672cf 100644 --- a/mypy.ini +++ b/mypy.ini @@ -40,6 +40,12 @@ ignore_missing_imports=true [mypy-pyarrow.*] ignore_missing_imports=true +[mypy-quickbooks.*] +ignore_missing_imports=true + +[mypy-intuitlib.*] +ignore_missing_imports=true + [mypy-tests.*] disallow_untyped_defs=false disallow_any_generics=false diff --git a/sources/quickbooks_online/README.md b/sources/quickbooks_online/README.md index 014afa8f9..2c1b7e33e 100644 --- a/sources/quickbooks_online/README.md +++ b/sources/quickbooks_online/README.md @@ -23,14 +23,17 @@ To grab credentials and initialize the verified source, please refer to the [ful ## Add credentials 1. Open `.dlt/secrets.toml`. -2. Enter the user name, password and security token: +2. Put these credentials in, these can be sourced from quickbooks developer portal and quickbooks oauth playground: ```toml # put your secret values and credentials here. do not share this file and do not push it to github - [sources.quickbooks] + [sources.quickbooks_online] + environment="" + company_id="" client_id="" client_secret="" access_token="" - refresh_token= "" + refresh_token="" + redirect_uri="" ``` 3. Enter credentials for your chosen destination as per the [docs.](https://dlthub.com/docs/dlt-ecosystem/destinations/) @@ -44,7 +47,7 @@ To grab credentials and initialize the verified source, please refer to the [ful 2. Now the pipeline can be run by using the command: ```bash - python3 quickbooks_pipeline.py + python3 quickbooks_online_pipeline.py ``` 3. To make sure that everything is loaded as expected, use the command: @@ -52,5 +55,5 @@ To grab credentials and initialize the verified source, please refer to the [ful dlt pipeline show ``` - For example, the pipeline_name for the above pipeline is `quickbooks`, you may also use any custom name instead. + For example, the pipeline_name for the above pipeline is `quickbooks_online`, you may also use any custom name instead. diff --git a/sources/quickbooks_online/__init__.py b/sources/quickbooks_online/__init__.py index e2776096d..0ee3167b8 100644 --- a/sources/quickbooks_online/__init__.py +++ b/sources/quickbooks_online/__init__.py @@ -25,8 +25,8 @@ def quickbooks_online( client_secret: str = dlt.secrets.value, access_token: str = dlt.secrets.value, refresh_token: str = dlt.secrets.value, - company_id: str = dlt.secrets.value, # also called realm_id - redirect_uri: str = dlt.secrets.value # get from quickbooks_online dev portal + company_id: str = dlt.secrets.value, # also called realm_id + redirect_uri: str = dlt.secrets.value, # get from quickbooks_online dev portal ) -> Iterable[DltResource]: """ Retrieves data from Quickbooks using the Quickbooks API. @@ -48,19 +48,16 @@ def quickbooks_online( environment=environment, redirect_uri=redirect_uri, access_token=access_token, - ) client = QuickBooks( - auth_client=auth_client, - refresh_token=refresh_token, - company_id=company_id + auth_client=auth_client, refresh_token=refresh_token, company_id=company_id ) # define resources @dlt.resource def customer() -> Iterable[TDataItem]: - customer = Customer.all(qb=client) # returns a list of iterables + customer = Customer.all(qb=client) # returns a list of iterables for record in customer: yield record.to_dict() diff --git a/sources/quickbooks_online/requirements.txt b/sources/quickbooks_online/requirements.txt index a9400f4ac..21062dfcf 100644 --- a/sources/quickbooks_online/requirements.txt +++ b/sources/quickbooks_online/requirements.txt @@ -1,2 +1,3 @@ -dlt>=1.0.0 -python-quickbooks \ No newline at end of file +dlt>=1.9.0 +python-quickbooks>=0.9.12 +intuit-oauth==1.2.6 \ No newline at end of file diff --git a/sources/quickbooks_pipeline.py b/sources/quickbooks_online_pipeline.py similarity index 53% rename from sources/quickbooks_pipeline.py rename to sources/quickbooks_online_pipeline.py index 82367b91d..a93fc6be7 100644 --- a/sources/quickbooks_pipeline.py +++ b/sources/quickbooks_online_pipeline.py @@ -1,14 +1,16 @@ import dlt from quickbooks_online import quickbooks_online -def load_customer(): + +def load_customer() -> None: pipeline = dlt.pipeline( - pipeline_name='quickbooks_customer', - destination='duckdb', - dataset_name='quickbooks_online' + pipeline_name="quickbooks_customer", + destination="duckdb", + dataset_name="quickbooks_online", ) load_info = pipeline.run(quickbooks_online()) print(load_info) + if __name__ == "__main__": - load_customer() \ No newline at end of file + load_customer() diff --git a/tests/quickbooks_online/__init__.py b/tests/quickbooks_online/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/quickbooks_online/test_quickbooks_online_source.py b/tests/quickbooks_online/test_quickbooks_online_source.py new file mode 100644 index 000000000..914ce7934 --- /dev/null +++ b/tests/quickbooks_online/test_quickbooks_online_source.py @@ -0,0 +1,28 @@ +from tests.utils import ALL_DESTINATIONS, assert_load_info, load_table_counts +import pytest +import dlt +from sources.quickbooks_online import quickbooks_online + + +@pytest.mark.parametrize("destination_name", ALL_DESTINATIONS) +def test_quickbooks_online(destination_name: str) -> None: + pipeline = dlt.pipeline( + pipeline_name="quickbooks_customer", + destination=destination_name, + dataset_name="duckdb_customer", + dev_mode=True, + ) + data = quickbooks_online() + load_info = pipeline.run(data) + assert_load_info(load_info) + + expected_tables = ["customer"] + # only those tables in the schema + assert set(t["name"] for t in pipeline.default_schema.data_tables()) == set( + expected_tables + ) + # get counts + table_counts = load_table_counts(pipeline, *expected_tables) + # all tables loaded + assert set(table_counts.keys()) == set(expected_tables) + assert all(c > 0 for c in table_counts.values()) From 0f34463ecbea8ad9c11acdc94ecbd9029ea09209 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 22 Apr 2025 15:22:49 +0100 Subject: [PATCH 05/13] fix dlt version used --- sources/quickbooks_online/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/quickbooks_online/requirements.txt b/sources/quickbooks_online/requirements.txt index 21062dfcf..1fe81732f 100644 --- a/sources/quickbooks_online/requirements.txt +++ b/sources/quickbooks_online/requirements.txt @@ -1,3 +1,3 @@ -dlt>=1.9.0 +dlt==1.9.0 python-quickbooks>=0.9.12 intuit-oauth==1.2.6 \ No newline at end of file From 38c626dd39a70472ca2abb2432b658df228f5fa6 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 22 Apr 2025 15:25:16 +0100 Subject: [PATCH 06/13] fix doc string and syntax --- sources/quickbooks_online/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sources/quickbooks_online/__init__.py b/sources/quickbooks_online/__init__.py index 0ee3167b8..0f0f4e894 100644 --- a/sources/quickbooks_online/__init__.py +++ b/sources/quickbooks_online/__init__.py @@ -26,19 +26,19 @@ def quickbooks_online( access_token: str = dlt.secrets.value, refresh_token: str = dlt.secrets.value, company_id: str = dlt.secrets.value, # also called realm_id - redirect_uri: str = dlt.secrets.value, # get from quickbooks_online dev portal + redirect_uri: str = dlt.secrets.value # get from quickbooks_online dev portal ) -> Iterable[DltResource]: """ Retrieves data from Quickbooks using the Quickbooks API. Args: environment (str): The environment used for authentication ('sandbox' or 'production') - client_id (str): The client id provided by quickbooks_online for authentication. Defaults to the value in the `dlt.secrets` object. - client_secret (str): The client secret provided by quickbooks_online for authentication. Defaults to the value in the `dlt.secrets` object. - access_token (str): The access token provided by quickbooks_online oAuth playground for authentication. Defaults to the value in the `dlt.secrets` object. - refresh_token (str): The refresh token provided by quickbooks_online oAuth playground for authentication. Defaults to the value in the `dlt.secrets` object. - company_id (str): The company id provided by quickbooks_online. Defaults to the value in the `dlt.secrets` object. - + client_id (str): The client id provided by quickbooks for authentication. Defaults to the value in the `dlt.secrets` object. + client_secret (str): The client secret provided by quickbooks for authentication. Defaults to the value in the `dlt.secrets` object. + access_token (str): The access token provided by quickbooks oAuth playground for authentication. Defaults to the value in the `dlt.secrets` object. + refresh_token (str): The refresh token provided by quickbooks oAuth playground for authentication. Defaults to the value in the `dlt.secrets` object. + company_id (str): The company id / realm id provided by quickbooks. Defaults to the value in the `dlt.secrets` object. + redirect_uri (str): The redirect uri provided by quickbooks, found in the developer application created. Defaults to the value in the `dlt.secrets` object. Yields: DltResource: Data resources from Quickbooks. """ From 130b4d5f322caf562115da83d48c9bb7099ea9fe Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 22 Apr 2025 15:27:21 +0100 Subject: [PATCH 07/13] remove comments --- sources/quickbooks_online/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sources/quickbooks_online/__init__.py b/sources/quickbooks_online/__init__.py index 0f0f4e894..5b623d7c2 100644 --- a/sources/quickbooks_online/__init__.py +++ b/sources/quickbooks_online/__init__.py @@ -25,8 +25,8 @@ def quickbooks_online( client_secret: str = dlt.secrets.value, access_token: str = dlt.secrets.value, refresh_token: str = dlt.secrets.value, - company_id: str = dlt.secrets.value, # also called realm_id - redirect_uri: str = dlt.secrets.value # get from quickbooks_online dev portal + company_id: str = dlt.secrets.value, + redirect_uri: str = dlt.secrets.value ) -> Iterable[DltResource]: """ Retrieves data from Quickbooks using the Quickbooks API. From 8e6b506646a8aa62fb92881dd666ede5ce5bc90c Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 22 Apr 2025 15:28:36 +0100 Subject: [PATCH 08/13] run linting --- sources/quickbooks_online/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/quickbooks_online/__init__.py b/sources/quickbooks_online/__init__.py index 5b623d7c2..bec9ac521 100644 --- a/sources/quickbooks_online/__init__.py +++ b/sources/quickbooks_online/__init__.py @@ -26,7 +26,7 @@ def quickbooks_online( access_token: str = dlt.secrets.value, refresh_token: str = dlt.secrets.value, company_id: str = dlt.secrets.value, - redirect_uri: str = dlt.secrets.value + redirect_uri: str = dlt.secrets.value, ) -> Iterable[DltResource]: """ Retrieves data from Quickbooks using the Quickbooks API. From 2beb6eff329f83258a13894b43c091bd078ca40a Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 22 Apr 2025 16:10:11 +0100 Subject: [PATCH 09/13] update docstring in init --- sources/quickbooks_online/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/quickbooks_online/__init__.py b/sources/quickbooks_online/__init__.py index bec9ac521..981ae7ee6 100644 --- a/sources/quickbooks_online/__init__.py +++ b/sources/quickbooks_online/__init__.py @@ -1,7 +1,7 @@ """Source for Quickbooks depending on the quickbooks_online-python python package. Quickbooks-python docs: https://github.com/ej2/python-quickbooks -Quickbooks api docs: https://developer.salesforce.com/docs/apis +Quickbooks api docs: https://developer.intuit.com/app/developer/qbo/docs/api/accounting/all-entities/ Quickbooks company id: https://quickbooks.intuit.com/learn-support/en-uk/help-article/customer-company-settings/find-quickbooks-online-company-id/L7lp8O9yU_GB_en_GB To get API credentials: https://developer.intuit.com/app/developer/qbo/docs/get-started/start-developing-your-app Get oAuth Authorization code from: https://developer.intuit.com/app/developer/playground From e7a715038f1bc8803ce291565ebfaf851851fbc7 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Wed, 23 Apr 2025 09:23:00 +0100 Subject: [PATCH 10/13] add invoice as described in issue --- sources/quickbooks_online/__init__.py | 13 ++++++++++--- .../test_quickbooks_online_source.py | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/sources/quickbooks_online/__init__.py b/sources/quickbooks_online/__init__.py index 981ae7ee6..ed6f1880c 100644 --- a/sources/quickbooks_online/__init__.py +++ b/sources/quickbooks_online/__init__.py @@ -9,13 +9,14 @@ from dlt.sources import DltResource -from typing import Iterable +from typing import Iterable, Sequence import dlt from dlt.common.typing import TDataItem from intuitlib.client import AuthClient from quickbooks import QuickBooks from quickbooks.objects.customer import Customer +from quickbooks.objects.invoice import Invoice @dlt.source(name="quickbooks_online") @@ -27,7 +28,7 @@ def quickbooks_online( refresh_token: str = dlt.secrets.value, company_id: str = dlt.secrets.value, redirect_uri: str = dlt.secrets.value, -) -> Iterable[DltResource]: +) -> Sequence[DltResource]: """ Retrieves data from Quickbooks using the Quickbooks API. @@ -61,4 +62,10 @@ def customer() -> Iterable[TDataItem]: for record in customer: yield record.to_dict() - return customer + @dlt.resource + def invoice() -> Iterable[TDataItem]: + invoice = Invoice.all(qb=client) + for record in invoice: + yield record.to_dict() + + return [customer, invoice] diff --git a/tests/quickbooks_online/test_quickbooks_online_source.py b/tests/quickbooks_online/test_quickbooks_online_source.py index 914ce7934..9eae3d56f 100644 --- a/tests/quickbooks_online/test_quickbooks_online_source.py +++ b/tests/quickbooks_online/test_quickbooks_online_source.py @@ -16,7 +16,7 @@ def test_quickbooks_online(destination_name: str) -> None: load_info = pipeline.run(data) assert_load_info(load_info) - expected_tables = ["customer"] + expected_tables = ["customer", "invoice"] # only those tables in the schema assert set(t["name"] for t in pipeline.default_schema.data_tables()) == set( expected_tables From 744b20dc5202f9b24adfa4958732d9eedf5cb6d9 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Wed, 28 May 2025 23:10:32 +0100 Subject: [PATCH 11/13] change version, add in settings and oauth_setup py --- sources/quickbooks_online/__init__.py | 28 +++- sources/quickbooks_online/oauth_setup.py | 185 +++++++++++++++++++++ sources/quickbooks_online/requirements.txt | 2 +- sources/quickbooks_online/settings.py | 11 ++ 4 files changed, 216 insertions(+), 10 deletions(-) create mode 100644 sources/quickbooks_online/oauth_setup.py create mode 100644 sources/quickbooks_online/settings.py diff --git a/sources/quickbooks_online/__init__.py b/sources/quickbooks_online/__init__.py index ed6f1880c..c5dd3081b 100644 --- a/sources/quickbooks_online/__init__.py +++ b/sources/quickbooks_online/__init__.py @@ -12,6 +12,8 @@ from typing import Iterable, Sequence import dlt +from .oauth_setup import QuickBooksAuth +from .settings import sandbox_env, production_env from dlt.common.typing import TDataItem from intuitlib.client import AuthClient from quickbooks import QuickBooks @@ -21,34 +23,42 @@ @dlt.source(name="quickbooks_online") def quickbooks_online( - environment: str = dlt.secrets.value, + environment: str = sandbox_env, client_id: str = dlt.secrets.value, client_secret: str = dlt.secrets.value, - access_token: str = dlt.secrets.value, refresh_token: str = dlt.secrets.value, company_id: str = dlt.secrets.value, - redirect_uri: str = dlt.secrets.value, + redirect_url: str = dlt.secrets.value, ) -> Sequence[DltResource]: """ Retrieves data from Quickbooks using the Quickbooks API. Args: - environment (str): The environment used for authentication ('sandbox' or 'production') + environment (str): The environment used for authentication, choose variable sandbox_env | production_env client_id (str): The client id provided by quickbooks for authentication. Defaults to the value in the `dlt.secrets` object. client_secret (str): The client secret provided by quickbooks for authentication. Defaults to the value in the `dlt.secrets` object. - access_token (str): The access token provided by quickbooks oAuth playground for authentication. Defaults to the value in the `dlt.secrets` object. - refresh_token (str): The refresh token provided by quickbooks oAuth playground for authentication. Defaults to the value in the `dlt.secrets` object. + refresh_token (str): The refresh token given a quickbooks scope. Defaults to the value in the `dlt.secrets` object. company_id (str): The company id / realm id provided by quickbooks. Defaults to the value in the `dlt.secrets` object. - redirect_uri (str): The redirect uri provided by quickbooks, found in the developer application created. Defaults to the value in the `dlt.secrets` object. + redirect_url (str): The redirect uri end user creates in quickbooks, found in the developer application created. Defaults to the value in the `dlt.secrets` object. Yields: DltResource: Data resources from Quickbooks. """ + + bearer_access_token = QuickBooksAuth( + client_id=client_id, + client_secret=client_secret, + company_id=company_id, + redirect_url=redirect_url, + refresh_token=refresh_token, + is_sandbox=False if environment == production_env else True, + ).get_bearer_token_from_refresh_token() + auth_client = AuthClient( client_id=client_id, client_secret=client_secret, environment=environment, - redirect_uri=redirect_uri, - access_token=access_token, + redirect_uri=redirect_url, + access_token=bearer_access_token.accessToken, ) client = QuickBooks( diff --git a/sources/quickbooks_online/oauth_setup.py b/sources/quickbooks_online/oauth_setup.py new file mode 100644 index 000000000..7dc664776 --- /dev/null +++ b/sources/quickbooks_online/oauth_setup.py @@ -0,0 +1,185 @@ +from dlt.sources.helpers import requests +import base64 +import json +import random +from typing import Union +from urllib.parse import urlencode +from .settings import ( + discovery_document_url_sandbox, + discovery_document_url_prod, + Scopes, +) + + +class OAuth2Config: + def __init__( + self, + issuer: str = "", + auth_endpoint: str = "", + token_endpoint: str = "", + userinfo_endpoint: str = "", + revoke_endpoint: str = "", + jwks_uri: str = "", + ): + self.issuer = issuer + self.auth_endpoint = auth_endpoint + self.token_endpoint = token_endpoint + self.userinfo_endpoint = userinfo_endpoint + self.revoke_endpoint = revoke_endpoint + self.jwks_uri = jwks_uri + + +class Bearer: + def __init__( + self, + refresh_expiry: str, + access_token: str, + token_type: str, + refresh_token: str, + access_token_expiry: str, + id_token: Union[str, None] = None, + ): + self.refreshExpiry = refresh_expiry + self.accessToken = access_token + self.tokenType = token_type + self.refreshToken = refresh_token + self.accessTokenExpiry = access_token_expiry + self.idToken = id_token + + +class QuickBooksAuth: + def __init__( + self, + client_id: str, + client_secret: str, + company_id: str, + redirect_url: str, + refresh_token: str = None, + is_sandbox: Union[bool, None] = True, + ): + """ + End user should use this class to generate refresh token once manually and store in secrets.toml + and continually use it to generate access tokens + + Should the user need to change scopes, then this should be generated again and stored safely + + Source code used is from: https://github.com/IntuitDeveloper/OAuth2PythonSampleApp/blob/master/sampleAppOAuth2/services.py + """ + self.is_sandbox = is_sandbox or None + self.client_id = client_id + self.client_secret = client_secret + self.company_id = company_id + self.redirect_url = redirect_url + self.refresh_token = refresh_token + + @staticmethod + def string_to_base64(s: str) -> str: + return base64.b64encode(bytes(s, "utf-8")).decode() + + @staticmethod + def get_random_string( + length: int = 64, + allowed_chars: str = "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", + ) -> str: + return "".join(random.choice(allowed_chars) for i in range(length)) + + def get_discovery_document(self) -> OAuth2Config: + if self.is_sandbox: + discovery_document_url = discovery_document_url_sandbox + else: + discovery_document_url = discovery_document_url_prod + r = requests.get(discovery_document_url) + if r.status_code >= 400: + raise ConnectionError(r.json()) + + discovery_doc_json = r.json() + discovery_doc = OAuth2Config( + issuer=discovery_doc_json["issuer"], + auth_endpoint=discovery_doc_json["authorization_endpoint"], + userinfo_endpoint=discovery_doc_json["userinfo_endpoint"], + revoke_endpoint=discovery_doc_json["revocation_endpoint"], + token_endpoint=discovery_doc_json["token_endpoint"], + jwks_uri=discovery_doc_json["jwks_uri"], + ) + + return discovery_doc + + def get_auth_url(self, scope: Union[str, Scopes]) -> str: + """ + scopes available in settings.py from intuitlib.enums + """ + auth_endpoint = self.get_discovery_document().auth_endpoint + auth_url_params = { + "client_id": self.client_id, + "redirect_uri": self.redirect_url, + "response_type": "code", + "scope": scope, + "state": self.get_random_string(), + } + url = f"{auth_endpoint}?{urlencode(auth_url_params)}" + + return url + + def get_bearer_token( + self, auth_code: str, client_id: str, client_secret: str, redirect_uri: str + ) -> Union[str, Bearer]: + token_endpoint = self.get_discovery_document().token_endpoint + auth_header = "Basic " + self.string_to_base64(client_id + ":" + client_secret) + headers = { + "Accept": "application/json", + "content-type": "application/x-www-form-urlencoded", + "Authorization": auth_header, + } + payload = { + "code": auth_code, + "redirect_uri": redirect_uri, + "grant_type": "authorization_code", + } + r = requests.post(token_endpoint, data=payload, headers=headers) + if r.status_code != 200: + return r.text + bearer_raw = json.loads(r.text) + + if "id_token" in bearer_raw: + id_token = bearer_raw["id_token"] + else: + id_token = None + + return Bearer( + bearer_raw["x_refresh_token_expires_in"], + bearer_raw["access_token"], + bearer_raw["token_type"], + bearer_raw["refresh_token"], + bearer_raw["expires_in"], + id_token=id_token, + ) + + def get_bearer_token_from_refresh_token(self) -> Bearer: + token_endpoint = self.get_discovery_document().token_endpoint + auth_header = "Basic " + self.string_to_base64( + self.client_id + ":" + self.client_secret + ) + headers = { + "Accept": "application/json", + "content-type": "application/x-www-form-urlencoded", + "Authorization": auth_header, + } + + payload = {"refresh_token": self.refresh_token, "grant_type": "refresh_token"} + r = requests.post(token_endpoint, data=payload, headers=headers) + bearer_raw = json.loads(r.text) + + if "id_token" in bearer_raw: + id_token = bearer_raw["id_token"] + else: + id_token = None + + return Bearer( + bearer_raw["x_refresh_token_expires_in"], + bearer_raw["access_token"], + bearer_raw["token_type"], + bearer_raw["refresh_token"], + bearer_raw["expires_in"], + id_token=id_token, + ) diff --git a/sources/quickbooks_online/requirements.txt b/sources/quickbooks_online/requirements.txt index 1fe81732f..d710b33af 100644 --- a/sources/quickbooks_online/requirements.txt +++ b/sources/quickbooks_online/requirements.txt @@ -1,3 +1,3 @@ -dlt==1.9.0 +dlt>=0.5.1 python-quickbooks>=0.9.12 intuit-oauth==1.2.6 \ No newline at end of file diff --git a/sources/quickbooks_online/settings.py b/sources/quickbooks_online/settings.py new file mode 100644 index 000000000..38d37b197 --- /dev/null +++ b/sources/quickbooks_online/settings.py @@ -0,0 +1,11 @@ +from intuitlib.enums import Scopes + +discovery_document_url_sandbox = ( + "https://developer.api.intuit.com/.well-known/openid_sandbox_configuration" +) +discovery_document_url_prod = ( + "https://developer.api.intuit.com/.well-known/openid_configuration" +) + +sandbox_env = "sandbox" +production_env = "production" From 79de24f9b6615c69d70e17586203eaa79e1e2887 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Wed, 28 May 2025 23:27:30 +0100 Subject: [PATCH 12/13] update readme, clarify settings and defined use is_sandbox --- sources/quickbooks_online/README.md | 4 +--- sources/quickbooks_online/__init__.py | 2 +- sources/quickbooks_online/settings.py | 1 + sources/quickbooks_online_pipeline.py | 3 ++- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sources/quickbooks_online/README.md b/sources/quickbooks_online/README.md index 2c1b7e33e..f10d4e37b 100644 --- a/sources/quickbooks_online/README.md +++ b/sources/quickbooks_online/README.md @@ -27,13 +27,11 @@ To grab credentials and initialize the verified source, please refer to the [ful ```toml # put your secret values and credentials here. do not share this file and do not push it to github [sources.quickbooks_online] - environment="" company_id="" client_id="" client_secret="" - access_token="" refresh_token="" - redirect_uri="" + redirect_url="" ``` 3. Enter credentials for your chosen destination as per the [docs.](https://dlthub.com/docs/dlt-ecosystem/destinations/) diff --git a/sources/quickbooks_online/__init__.py b/sources/quickbooks_online/__init__.py index c5dd3081b..0678f198d 100644 --- a/sources/quickbooks_online/__init__.py +++ b/sources/quickbooks_online/__init__.py @@ -23,7 +23,7 @@ @dlt.source(name="quickbooks_online") def quickbooks_online( - environment: str = sandbox_env, + environment: str, client_id: str = dlt.secrets.value, client_secret: str = dlt.secrets.value, refresh_token: str = dlt.secrets.value, diff --git a/sources/quickbooks_online/settings.py b/sources/quickbooks_online/settings.py index 38d37b197..ef4d00a93 100644 --- a/sources/quickbooks_online/settings.py +++ b/sources/quickbooks_online/settings.py @@ -7,5 +7,6 @@ "https://developer.api.intuit.com/.well-known/openid_configuration" ) +# comes directly from quickbooks https://developer.intuit.com/app/developer/qbo/docs/develop/authentication-and-authorization/oauth-2.0#authorization-request sandbox_env = "sandbox" production_env = "production" diff --git a/sources/quickbooks_online_pipeline.py b/sources/quickbooks_online_pipeline.py index a93fc6be7..4217dbb53 100644 --- a/sources/quickbooks_online_pipeline.py +++ b/sources/quickbooks_online_pipeline.py @@ -1,5 +1,6 @@ import dlt from quickbooks_online import quickbooks_online +from quickbooks_online.settings import sandbox_env, production_env def load_customer() -> None: @@ -8,7 +9,7 @@ def load_customer() -> None: destination="duckdb", dataset_name="quickbooks_online", ) - load_info = pipeline.run(quickbooks_online()) + load_info = pipeline.run(quickbooks_online(environment=sandbox_env)) print(load_info) From 5a8fbea9b9300f908a4ee60106a91aea6e90073d Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 24 Jun 2025 18:44:44 +0100 Subject: [PATCH 13/13] action on feedback: update docs & small logical refactor --- sources/quickbooks_online/README.md | 10 +++------- sources/quickbooks_online/__init__.py | 2 +- sources/quickbooks_online/oauth_setup.py | 4 ++-- sources/quickbooks_online/settings.py | 2 -- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/sources/quickbooks_online/README.md b/sources/quickbooks_online/README.md index f10d4e37b..1501715ad 100644 --- a/sources/quickbooks_online/README.md +++ b/sources/quickbooks_online/README.md @@ -18,12 +18,10 @@ Here, we chose DuckDB as the destination. Alternatively, you can also choose red ## Setup verified source and pipeline example -To grab credentials and initialize the verified source, please refer to the [full documentation here.](https://dlthub.com/docs/dlt-ecosystem/verified-sources/salesforce) - -## Add credentials +### Add credentials 1. Open `.dlt/secrets.toml`. -2. Put these credentials in, these can be sourced from quickbooks developer portal and quickbooks oauth playground: +2. Put the credentials in, these can be sourced from [quickbooks developer portal and quickbooks oauth playground](https://developer.intuit.com/app/developer/qbo/docs/develop/authentication-and-authorization/oauth-2.0#authorization-request): ```toml # put your secret values and credentials here. do not share this file and do not push it to github [sources.quickbooks_online] @@ -34,9 +32,7 @@ To grab credentials and initialize the verified source, please refer to the [ful redirect_url="" ``` -3. Enter credentials for your chosen destination as per the [docs.](https://dlthub.com/docs/dlt-ecosystem/destinations/) - -## Run the pipeline example +### Run the pipeline example 1. Install the necessary dependencies by running the following command: ```bash diff --git a/sources/quickbooks_online/__init__.py b/sources/quickbooks_online/__init__.py index 0678f198d..c2108d9bf 100644 --- a/sources/quickbooks_online/__init__.py +++ b/sources/quickbooks_online/__init__.py @@ -34,7 +34,7 @@ def quickbooks_online( Retrieves data from Quickbooks using the Quickbooks API. Args: - environment (str): The environment used for authentication, choose variable sandbox_env | production_env + environment (str): The environment used for authentication, only "sandbox" or "production" values are allowed client_id (str): The client id provided by quickbooks for authentication. Defaults to the value in the `dlt.secrets` object. client_secret (str): The client secret provided by quickbooks for authentication. Defaults to the value in the `dlt.secrets` object. refresh_token (str): The refresh token given a quickbooks scope. Defaults to the value in the `dlt.secrets` object. diff --git a/sources/quickbooks_online/oauth_setup.py b/sources/quickbooks_online/oauth_setup.py index 7dc664776..14b9da06f 100644 --- a/sources/quickbooks_online/oauth_setup.py +++ b/sources/quickbooks_online/oauth_setup.py @@ -2,12 +2,12 @@ import base64 import json import random +from intuitlib.enums import Scopes from typing import Union from urllib.parse import urlencode from .settings import ( discovery_document_url_sandbox, - discovery_document_url_prod, - Scopes, + discovery_document_url_prod ) diff --git a/sources/quickbooks_online/settings.py b/sources/quickbooks_online/settings.py index ef4d00a93..dee07b5cd 100644 --- a/sources/quickbooks_online/settings.py +++ b/sources/quickbooks_online/settings.py @@ -1,5 +1,3 @@ -from intuitlib.enums import Scopes - discovery_document_url_sandbox = ( "https://developer.api.intuit.com/.well-known/openid_sandbox_configuration" )