Skip to content

Commit 7e19d72

Browse files
SalimAbdul-snaplogicSalimAbdul-snaplogic
committed
feat(ingestion/snaplogic): Add snaplogic as a source for metadata ingestion
1 parent 59384df commit 7e19d72

File tree

19 files changed

+1716
-1
lines changed

19 files changed

+1716
-1
lines changed

‎datahub-web-react/src/app/ingest/source/builder/constants.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import qlikLogo from '@images/qliklogo.png';
3636
import redshiftLogo from '@images/redshiftlogo.png';
3737
import sacLogo from '@images/saclogo.svg';
3838
import sigmaLogo from '@images/sigmalogo.png';
39+
import snaplogic from '@images/snaplogic.png';
3940
import snowflakeLogo from '@images/snowflakelogo.png';
4041
import supersetLogo from '@images/supersetlogo.png';
4142
import tableauLogo from '@images/tableaulogo.png';
@@ -149,6 +150,8 @@ export const NEO4J = 'neo4j';
149150
export const NEO4J_URN = `urn:li:dataPlatform:${NEO4J}`;
150151
export const VERTEX_AI = 'vertexai';
151152
export const VERTEXAI_URN = `urn:li:dataPlatform:${VERTEX_AI}`;
153+
export const SNAPLOGIC = 'snaplogic';
154+
export const SNAPLOGIC_URN = `urn:li:dataPlatform:${SNAPLOGIC}`;
152155

153156
export const PLATFORM_URN_TO_LOGO = {
154157
[ATHENA_URN]: athenaLogo,
@@ -196,6 +199,7 @@ export const PLATFORM_URN_TO_LOGO = {
196199
[DATAHUB_URN]: datahubLogo,
197200
[NEO4J_URN]: neo4j,
198201
[VERTEXAI_URN]: vertexAI,
202+
[SNAPLOGIC_URN]: snaplogic,
199203
};
200204

201205
export const SOURCE_TO_PLATFORM_URN = {

‎datahub-web-react/src/app/ingest/source/builder/sources.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,14 @@
1515
"docsUrl": "https://docs.datahub.com/docs/quick-ingestion-guides/redshift/overview",
1616
"recipe": "source: \n type: redshift\n config:\n # Coordinates\n host_port: # Your Redshift host and post, e.g. example.something.us-west-2.redshift.amazonaws.com:5439\n database: # Your Redshift database, e.g. SampleDatabase\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: null # Your Redshift username, e.g. admin\n\n table_lineage_mode: stl_scan_based\n include_table_lineage: true\n include_tables: true\n include_views: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true"
1717
},
18+
{
19+
"urn": "urn:li:dataPlatform:snaplogic",
20+
"name": "snaplogic",
21+
"displayName": "Snaplogic",
22+
"description": "Import lineage from Snaplogic.",
23+
"docsUrl": "https://docs.datahub.com/docs/quick-ingestion-guides/snaplogic/overview",
24+
"recipe": "source:\n type: snaplogic\n config:\n username: # username\n password: # password\n base_url: https://elastic.snaplogic.com\n org_name: # Organization name from Snaplogic instance\n stateful_ingestion:\n enabled: True\n remove_stale_metadata: False\n"
25+
},
1826
{
1927
"urn": "urn:li:dataPlatform:snowflake",
2028
"name": "snowflake",

‎datahub-web-react/src/app/ingestV2/source/builder/sources.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,14 @@
1515
"docsUrl": "https://docs.datahub.com/docs/quick-ingestion-guides/redshift/overview",
1616
"recipe": "source: \n type: redshift\n config:\n # Coordinates\n host_port: # Your Redshift host and post, e.g. example.something.us-west-2.redshift.amazonaws.com:5439\n database: # Your Redshift database, e.g. SampleDatabase\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: null # Your Redshift username, e.g. admin\n\n table_lineage_mode: stl_scan_based\n include_table_lineage: true\n include_tables: true\n include_views: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true"
1717
},
18+
{
19+
"urn": "urn:li:dataPlatform:snaplogic",
20+
"name": "snaplogic",
21+
"displayName": "Snaplogic",
22+
"description": "Import lineage from Snaplogic.",
23+
"docsUrl": "https://docs.datahub.com/docs/quick-ingestion-guides/snaplogic/overview",
24+
"recipe": "source:\n type: snaplogic\n config:\n username: # username\n password: # password\n base_url: https://elastic.snaplogic.com\n org_name: # Organization name from Snaplogic instance\n stateful_ingestion:\n enabled: True\n remove_stale_metadata: False\n"
25+
},
1826
{
1927
"urn": "urn:li:dataPlatform:snowflake",
2028
"name": "snowflake",

‎datahub-web-react/src/images/snaplogic.png

4.84 KB
Loading

‎datahub-web-react/src/images/snaplogic.svg

Lines changed: 74 additions & 0 deletions
Loading

‎metadata-ingestion/docs/sources/snaplogic/snaplogic_pre.md

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
## Integration Details
2+
3+
<!-- Plain-language description of what this integration is meant to do. -->
4+
<!-- Include details about where metadata is extracted from (ie. logs, source API, manifest, etc.) -->
5+
6+
This integration extracts data lineage information from the public SnapLogic Lineage API and ingests it into DataHub. It enables visibility into how data flows through SnapLogic pipelines by capturing metadata directly from the source API. This allows users to track data transformations and dependencies across their data ecosystem, enhancing observability, governance, and impact analysis within DataHub.
7+
8+
### Concept Mapping
9+
10+
<!-- This should be a manual mapping of concepts from the source to the DataHub Metadata Model -->
11+
<!-- Authors should provide as much context as possible about how this mapping was generated, including assumptions made, known shortcuts, & any other caveats -->
12+
13+
This ingestion source maps the following Source System Concepts to DataHub Concepts:
14+
15+
<!-- Remove all unnecessary/irrevant DataHub Concepts -->
16+
17+
| Source Concept | DataHub Concept | Notes |
18+
| -------------- | ------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------- |
19+
| Snap-pack | [Data Platform](docs/generated/metamodel/entities/dataPlatform.md) | Snap-packs are mapped to Data Platforms, either directly (e.g., Snowflake) or dynamically based on connection details (e.g., JDBC URL). |
20+
| Table/Dataset | [Dataset](docs/generated/metamodel/entities/dataset.md) | May be differernt. It depends on a snap type. For sql databases it's table. For kafka it's topic, etc |
21+
| Snap | [Data Job](docs/generated/metamodel/entities/dataJob.md) | |
22+
| Pipeline | [Data Flow](docs/generated/metamodel/entities/dataFlow.md) | |
23+
24+
## Metadata Ingestion Quickstart
25+
26+
### Prerequisites
27+
28+
In order to ingest lineage from snaplogic, you will need valid snaplogic credentials with access to the SnapLogic Lineage API.
29+
30+
### Install the Plugin(s)
31+
32+
Run the following commands to install the relevant plugin(s):
33+
34+
`pip install 'acryl-datahub[snaplogic]'`
35+
36+
### Configure the Ingestion Recipe(s)
37+
38+
Use the following recipe(s) to get started with ingestion.
39+
40+
#### `'acryl-datahub[snaplogic]'`
41+
42+
```yml
43+
pipeline_name: <action-pipeline-name>
44+
source:
45+
type: snaplogic
46+
config:
47+
username: <snaplogic-username>
48+
password: <snaplogic-password>
49+
base_url: https://elastic.snaplogic.com
50+
org_name: <snaplogic-org-name>
51+
stateful_ingestion:
52+
enabled: True
53+
remove_stale_metadata: False
54+
```
55+
56+
<details>
57+
<summary>View All Recipe Configuartion Options</summary>
58+
59+
| Field | Required | Default | Description |
60+
| ----------------------------- | :------: | :---------------------------: | --------------------------------------------------------------- |
61+
| `username` | ✅ | | SnapLogic account login |
62+
| `password` | ✅ | | SnapLogic account password. |
63+
| `base_url` | ✅ | https://elastic.snaplogic.com | Snaplogic url |
64+
| `org_name` | ✅ | | Organisation name in snaplogic platform |
65+
| `namespace_mapping` | ❌ | | Namespace mapping. Used to map namespaces to platform instances |
66+
| `case_insensitive_namespaces` | ❌ | | List of case insensitive namespaces |
67+
68+
</details>
69+
70+
## Troubleshooting
71+
72+
### [Common Issue]

‎metadata-ingestion/docs/sources/snaplogic/snaplogic_recipe.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
pipeline_name: "snaplogic_incremental_ingestion"
2+
source:
3+
type: snaplogic
4+
config:
5+
username: example@snaplogic.com
6+
password: password
7+
base_url: https://elastic.snaplogic.com
8+
org_name: "ExampleOrg"
9+
namespace_mapping:
10+
snowflake://snaplogic: snaplogic
11+
case_insensitive_namespaces:
12+
- snowflake://snaplogic
13+
stateful_ingestion:
14+
enabled: True
15+
remove_stale_metadata: False

‎metadata-ingestion/setup.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -569,6 +569,7 @@
569569
# databricks is alias for unity-catalog and needs to be kept in sync
570570
"databricks": databricks | sql_common,
571571
"fivetran": snowflake_common | bigquery_common | sqlalchemy_lib | sqlglot_lib,
572+
"snaplogic": set(),
572573
"qlik-sense": sqlglot_lib | {"requests", "websocket-client"},
573574
"sigma": sqlglot_lib | {"requests"},
574575
"sac": sac,
@@ -700,6 +701,7 @@
700701
"redshift",
701702
"s3",
702703
"snowflake",
704+
"snaplogic",
703705
"slack",
704706
"tableau",
705707
"teradata",
@@ -842,6 +844,7 @@
842844
"gcs = datahub.ingestion.source.gcs.gcs_source:GCSSource",
843845
"sql-queries = datahub.ingestion.source.sql_queries:SqlQueriesSource",
844846
"fivetran = datahub.ingestion.source.fivetran.fivetran:FivetranSource",
847+
"snaplogic = datahub.ingestion.source.snaplogic.snaplogic:SnaplogicSource",
845848
"qlik-sense = datahub.ingestion.source.qlik_sense.qlik_sense:QlikSenseSource",
846849
"sigma = datahub.ingestion.source.sigma.sigma:SigmaSource",
847850
"sac = datahub.ingestion.source.sac.sac:SACSource",

‎metadata-ingestion/src/datahub/ingestion/autogenerated/capability_summary.json

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2922,6 +2922,38 @@
29222922
"platform_name": "Slack",
29232923
"support_status": "TESTING"
29242924
},
2925+
"snaplogic": {
2926+
"capabilities": [
2927+
{
2928+
"capability": "LINEAGE_FINE",
2929+
"description": "Enabled by default",
2930+
"subtype_modifier": null,
2931+
"supported": true
2932+
},
2933+
{
2934+
"capability": "DELETION_DETECTION",
2935+
"description": "Not supported yet",
2936+
"subtype_modifier": null,
2937+
"supported": false
2938+
},
2939+
{
2940+
"capability": "PLATFORM_INSTANCE",
2941+
"description": "Snaplogic does not support platform instances",
2942+
"subtype_modifier": null,
2943+
"supported": false
2944+
},
2945+
{
2946+
"capability": "LINEAGE_COARSE",
2947+
"description": "Enabled by default",
2948+
"subtype_modifier": null,
2949+
"supported": true
2950+
}
2951+
],
2952+
"classname": "datahub.ingestion.source.snaplogic.snaplogic.SnaplogicSource",
2953+
"platform_id": "snaplogic",
2954+
"platform_name": "Snaplogic",
2955+
"support_status": "TESTING"
2956+
},
29252957
"snowflake": {
29262958
"capabilities": [
29272959
{
@@ -3565,4 +3597,4 @@
35653597
"support_status": "CERTIFIED"
35663598
}
35673599
}
3568-
}
3600+
}

‎metadata-ingestion/src/datahub/ingestion/source/snaplogic/__init__.py

Whitespace-only changes.

0 commit comments

Comments
 (0)