From 218186e07966365677dfd7c5c040a97e677c73a8 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 20 Mar 2025 20:22:35 +0100 Subject: [PATCH 1/3] Add list write agent tool --- .../tool.json | 91 +++++++++++++++++++ .../sharepoint-online-write-list-tool/tool.py | 81 +++++++++++++++++ python-lib/sharepoint_client.py | 59 +++++++++++- 3 files changed, 226 insertions(+), 5 deletions(-) create mode 100644 python-agent-tools/sharepoint-online-write-list-tool/tool.json create mode 100644 python-agent-tools/sharepoint-online-write-list-tool/tool.py diff --git a/python-agent-tools/sharepoint-online-write-list-tool/tool.json b/python-agent-tools/sharepoint-online-write-list-tool/tool.json new file mode 100644 index 0000000..ade30e9 --- /dev/null +++ b/python-agent-tools/sharepoint-online-write-list-tool/tool.json @@ -0,0 +1,91 @@ +{ + "id": "write-sharepoint-list", + "meta": { + "icon": "icon-cloud", + "label": "Write to a SharePoint Online list" + }, + + "params" : [ + { + "name": "auth_type", + "label": "Type of authentication", + "type": "SELECT", + "selectChoices": [ + { + "value": "login", + "label": "User name / password" + }, + { + "value": "oauth", + "label": "Azure Single Sign On" + }, + { + "value": "site-app-permissions", + "label": "Site App Permissions" + }, + { + "value": "app-certificate", + "label": "Certificates" + } + ] + }, + { + "name": "sharepoint_oauth", + "label": "Azure preset", + "type": "PRESET", + "parameterSetId": "oauth-login", + "visibilityCondition": "model.auth_type == 'oauth'" + }, + { + "name": "sharepoint_sharepy", + "label": "SharePoint preset", + "type": "PRESET", + "parameterSetId": "sharepoint-login", + "visibilityCondition": "model.auth_type == 'login'" + }, + { + "name": "site_app_permissions", + "label": "Site App preset", + "type": "PRESET", + "parameterSetId": "site-app-permissions", + "visibilityCondition": "model.auth_type == 'site-app-permissions'" + }, + { + "name": "app_certificate", + "label": "Certificates", + "type": "PRESET", + "parameterSetId": "app-certificate", + "visibilityCondition": "model.auth_type == 'app-certificate'" + }, + { + "name": "sharepoint_list_title", + "label": "List title", + "defaultValue": "DSS_${projectKey}_", + "description": "", + "type": "STRING", + "mandatory": true + }, + { + "name": "advanced_parameters", + "label": "Show advanced parameters", + "description": "", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "name": "sharepoint_site_overwrite", + "label": "Site path preset overwrite", + "type": "STRING", + "description": "sites/site_name/subsite...", + "visibilityCondition": "model.advanced_parameters == true" + }, + { + "name": "sharepoint_list_view_title", + "label": "View name", + "description": "Read data from a specific view", + "type": "STRING", + "defaultValue": "", + "visibilityCondition": "model.advanced_parameters == true" + } + ] +} \ No newline at end of file diff --git a/python-agent-tools/sharepoint-online-write-list-tool/tool.py b/python-agent-tools/sharepoint-online-write-list-tool/tool.py new file mode 100644 index 0000000..6c3def1 --- /dev/null +++ b/python-agent-tools/sharepoint-online-write-list-tool/tool.py @@ -0,0 +1,81 @@ +from dataiku.llm.agent_tools import BaseAgentTool +from sharepoint_client import SharePointClient +from safe_logger import SafeLogger +from dss_constants import DSSConstants + +logger = SafeLogger("sharepoint-online plugin", DSSConstants.SECRET_PARAMETERS_KEYS) + + +class WriteToSharePointListTool(BaseAgentTool): + + def set_config(self, config, plugin_config): + logger.info('SharePoint Online plugin list write tool v{}'.format(DSSConstants.PLUGIN_VERSION)) + self.sharepoint_list_title = config.get("sharepoint_list_title") + self.auth_type = config.get('auth_type') + logger.info('init:sharepoint_list_title={}, auth_type={}'.format(self.sharepoint_list_title, self.auth_type)) + self.expand_lookup = config.get("expand_lookup", False) + self.metadata_to_retrieve = config.get("metadata_to_retrieve", []) + advanced_parameters = config.get("advanced_parameters", False) + self.write_mode = "create" + if not advanced_parameters: + self.max_workers = 1 # no multithread per default + self.batch_size = 100 + self.sharepoint_list_view_title = "" + else: + self.max_workers = config.get("max_workers", 1) + self.batch_size = config.get("batch_size", 100) + self.sharepoint_list_view_title = config.get("sharepoint_list_view_title", "") + logger.info("init:advanced_parameters={}, max_workers={}, batch_size={}".format(advanced_parameters, self.max_workers, self.batch_size)) + self.metadata_to_retrieve.append("Title") + self.display_metadata = len(self.metadata_to_retrieve) > 0 + self.client = SharePointClient(config) + self.sharepoint_list_view_id = None + if self.sharepoint_list_view_title: + self.sharepoint_list_view_id = self.client.get_view_id(self.sharepoint_list_title, self.sharepoint_list_view_title) + self.sharepoint_column_of_interest = config.get("sharepoint_column_of_interest") + self.output_schema = None + + def get_descriptor(self, tool): + schema = self.client.get_read_schema(display_metadata=self.display_metadata, metadata_to_retrieve=self.metadata_to_retrieve, add_description=True) + columns = schema.get("columns", []) + properties = {} + required = [] + output_columns = [] + for column in columns: + column_description = column.get("description") + if column_description: + properties[column.get("name")] = { + "type": column.get("type"), + "name": column.get("name") + } + required.append(column.get("name")) # For now... + output_columns.append({ + "type": column.get("type"), + "name": column.get("name") + }) + self.output_schema = { + "columns": output_columns + } + return { + "description": "This tool can be used to access lists on SharePoint Online. The input to this tool is a dictionary containing the new issue summary and description, e.g. '{'summary':'new issue summary', 'description':'new issue description'}'", + "inputSchema" : { + "$id": "https://dataiku.com/agents/tools/search/input", + "title": "Add an item to a SharePoint Online list tool", + "type": "object", + "properties" : properties + } + } + + def invoke(self, input, trace): + sharepoint_writer = self.client.get_writer( + self.output_schema, + None, None, 1, 1, + "append" + ) + row = input.get("input", {}) + sharepoint_writer.write_row_dict(row) + sharepoint_writer.close() + + return { + "output" : 'The record was added' + } diff --git a/python-lib/sharepoint_client.py b/python-lib/sharepoint_client.py index 656b3e0..ca032da 100644 --- a/python-lib/sharepoint_client.py +++ b/python-lib/sharepoint_client.py @@ -7,6 +7,8 @@ import time import json import re +import dataiku +import dataikuapi from xml.etree.ElementTree import Element, tostring from xml.dom import minidom @@ -135,6 +137,28 @@ def __init__(self, config): max_retries=SharePointConstants.MAX_RETRIES, base_retry_timer_sec=SharePointConstants.WAIT_TIME_BEFORE_RETRY_SEC ) + elif config.get('auth_type') == "dss-connection": + connection_name = config.get("dss_connection") + print("ALX:connection_name={}".format(connection_name)) + client = dataiku.api_client() + print("ALX:client={}".format(client)) + connection = client.get_connection(connection_name) + print("ALX:connection={}".format(connection)) + connection_info = connection.get_info() + print("ALX:connection_info={}".format(connection_info)) + credentials = connection_info.get_oauth2_credential() + print("ALX:credentials={}".format(credentials)) + self.sharepoint_access_token = credentials.get("accessToken") + self.session.update_settings(session=SharePointSession( + None, + None, + self.sharepoint_url, + self.sharepoint_site, + sharepoint_access_token=self.sharepoint_access_token + ), + max_retries=SharePointConstants.MAX_RETRIES, + base_retry_timer_sec=SharePointConstants.WAIT_TIME_BEFORE_RETRY_SEC + ) else: raise SharePointClientError("The type of authentication is not selected") self.sharepoint_list_title = config.get("sharepoint_list_title") @@ -349,6 +373,16 @@ def get_list_fields(self, list_title): return None return self.extract_results(json_response) + def search_list(self, list_title, column_to_query, query): + list_search_url = self.get_list_search_url(list_title, column_to_query, query) + response = self.session.get( + list_search_url + ) + json_response = response.json() + if self.is_response_empty(json_response): + return None + return self.extract_results(json_response) + @staticmethod def is_response_empty(response): return SharePointConstants.RESULTS_CONTAINER_V2 not in response or SharePointConstants.RESULTS not in response[SharePointConstants.RESULTS_CONTAINER_V2] @@ -691,6 +725,13 @@ def get_list_add_item_using_path_url(self, list_title): def get_list_fields_url(self, list_title): return self.get_lists_by_title_url(list_title) + "/fields" + + def get_list_search_url(self, list_title, column_to_query, query): + return self.get_lists_by_title_url(list_title) + "/Items?expand=File&$filter=substringof('{}',{})".format( + query, + column_to_query + ) + # /Items?$expand=File&$filter=substringof('T15', Title) def get_lists_add_field_url(self, list_title): return self.get_base_url() + "/GetList(@a1)/Fields/CreateFieldAsXml?@a1='/{}/Lists/{}'".format( @@ -930,7 +971,7 @@ def get_writer(self, dataset_schema, dataset_partitioning, write_mode=write_mode ) - def get_read_schema(self, display_metadata=False, metadata_to_retrieve=[]): + def get_read_schema(self, display_metadata=False, metadata_to_retrieve=[], add_description=False): logger.info('get_read_schema') sharepoint_columns = self.get_list_fields(self.sharepoint_list_title) dss_columns = [] @@ -951,10 +992,18 @@ def get_read_schema(self, display_metadata=False, metadata_to_retrieve=[]): sharepoint_type = get_dss_type(column[SharePointConstants.TYPE_AS_STRING]) self.column_sharepoint_type[column[SharePointConstants.STATIC_NAME]] = column[SharePointConstants.TYPE_AS_STRING] if sharepoint_type is not None: - dss_columns.append({ - SharePointConstants.NAME_COLUMN: column[SharePointConstants.TITLE_COLUMN], - SharePointConstants.TYPE_COLUMN: sharepoint_type - }) + if add_description: + column_record = { + SharePointConstants.NAME_COLUMN: column["InternalName"], + SharePointConstants.TYPE_COLUMN: sharepoint_type, + "description": column.get("Description") + } + else: + column_record = { + SharePointConstants.NAME_COLUMN: column[SharePointConstants.TITLE_COLUMN], + SharePointConstants.TYPE_COLUMN: sharepoint_type + } + dss_columns.append(column_record) self.column_ids[column[SharePointConstants.STATIC_NAME]] = sharepoint_type self.column_names[column[SharePointConstants.STATIC_NAME]] = column[SharePointConstants.TITLE_COLUMN] self.column_entity_property_name[column[SharePointConstants.STATIC_NAME]] = column[SharePointConstants.ENTITY_PROPERTY_NAME] From 1a9b4d8985dfdee7fb34b242699033edc75bdf49 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 20 Mar 2025 20:24:07 +0100 Subject: [PATCH 2/3] version --- plugin.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.json b/plugin.json index c357c1b..cdb516f 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "id": "sharepoint-online", - "version": "1.1.4", + "version": "1.1.5", "meta": { "label": "SharePoint Online", "description": "Read and write data from/to your SharePoint Online account", From b9e43f8c9c0dd095f3be13b54dd9692d379e7958 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 20 Mar 2025 20:25:54 +0100 Subject: [PATCH 3/3] beta.1 --- python-lib/dss_constants.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python-lib/dss_constants.py b/python-lib/dss_constants.py index 89fedcc..db8540b 100644 --- a/python-lib/dss_constants.py +++ b/python-lib/dss_constants.py @@ -37,7 +37,7 @@ class DSSConstants(object): "sharepoint_oauth": "The access token is missing" } PATH = 'path' - PLUGIN_VERSION = "1.1.4" + PLUGIN_VERSION = "1.1.5-beta.1" SECRET_PARAMETERS_KEYS = ["Authorization", "sharepoint_username", "sharepoint_password", "client_secret", "client_certificate", "passphrase"] SITE_APP_DETAILS = { "sharepoint_tenant": "The tenant name is missing", @@ -58,5 +58,6 @@ class DSSConstants(object): "bigint": "Integer", "smallint": "Integer", "tinyint": "Integer", - "date": "DateTime" + "date": "DateTime", + "datetimenotz": "DateTime" }