From 5b6e1af94036db0f7592587a5bb2d55784e3620a Mon Sep 17 00:00:00 2001 From: shanialbeck Date: Thu, 3 Aug 2023 16:33:38 +0300 Subject: [PATCH 01/13] add connection only queries to workbook --- src/types.ts | 7 +++++++ src/utils/constants.ts | 10 ++++++++++ src/utils/xmlInnerPartsUtils.ts | 23 +++++++++++++++++++++++ src/utils/xmlPartsUtils.ts | 14 ++++++++++++++ src/workbookManager.ts | 30 +++++++++++++++++++++++++++++- 5 files changed, 83 insertions(+), 1 deletion(-) diff --git a/src/types.ts b/src/types.ts index 9c325b9..d6df55b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -7,6 +7,13 @@ export interface QueryInfo { queryName?: string; } +export interface MultipleQueriesInfo { + loadedQueryName: string; + refreshOnOpen: boolean; + connectionOnlyQueryNames: string[]; + mashupDocument: string; +} + export interface DocProps { title?: string | null; subject?: string | null; diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 4694d6f..410b9c7 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import { v4 } from "uuid"; + export const connectionsXmlPath = "xl/connections.xml"; export const sharedStringsXmlPath = "xl/sharedStrings.xml"; export const sheetsXmlPath = "xl/worksheets/sheet1.xml"; @@ -85,6 +87,9 @@ export const element = { dimension: "dimension", selection: "selection", kindCell: "c", + connection: "connection", + connections: "connections", + dbpr: "dbPr", }; export const elementAttributes = { @@ -117,6 +122,10 @@ export const elementAttributes = { spans: "spans", x14acDyDescent: "x14ac:dyDescent", xr3uid: "xr3:uid", + xr16uid: "xr16:uid", + keepAlive: "keepAlive", + refreshedVersion: "refreshedVersion", + background: "background" }; export const dataTypeKind = { @@ -131,6 +140,7 @@ export const elementAttributesValues = { connection: (queryName: string) => `Provider=Microsoft.Mashup.OleDb.1;Data Source=$Workbook$;Location="${queryName}";`, connectionCommand: (queryName: string) => `SELECT * FROM [${queryName}]`, tableResultType: () => "sTable", + randomizedUid: () => "{" + v4().toUpperCase() + "}", }; export const defaults = { diff --git a/src/utils/xmlInnerPartsUtils.ts b/src/utils/xmlInnerPartsUtils.ts index 4621221..bc7defb 100644 --- a/src/utils/xmlInnerPartsUtils.ts +++ b/src/utils/xmlInnerPartsUtils.ts @@ -268,6 +268,28 @@ const updatePivotTable = (tableXmlString: string, connectionId: string, refreshO return { isPivotTableUpdated, newPivotTable }; }; +const addNewConnection = async (connectionsXmlString: string, queryName: string): Promise => { + const parser: DOMParser = new DOMParser(); + const serializer: XMLSerializer = new XMLSerializer(); + const connectionsDoc: Document = parser.parseFromString(connectionsXmlString, xmlTextResultType); + const connections = connectionsDoc.getElementsByTagName(element.connections)[0]; + const newConnection = connectionsDoc.createElementNS(connectionsDoc.documentElement.namespaceURI, element.connection); + connections.append(newConnection); + newConnection.setAttribute(elementAttributes.id, [...connectionsDoc.getElementsByTagName(element.connection)].length.toString()); + newConnection.setAttribute(elementAttributes.xr16uid, elementAttributesValues.randomizedUid()); + newConnection.setAttribute(elementAttributes.keepAlive, trueValue); + newConnection.setAttribute(elementAttributes.name, elementAttributesValues.connectionName(queryName)); + newConnection.setAttribute(elementAttributes.description, elementAttributesValues.connectionDescription(queryName)); + newConnection.setAttribute(elementAttributes.type, "5"); + newConnection.setAttribute(elementAttributes.refreshedVersion, falseValue); + newConnection.setAttribute(elementAttributes.background, trueValue); + const newDbPr = connectionsDoc.createElementNS(connectionsDoc.documentElement.namespaceURI, element.dbpr); + newDbPr.setAttribute(elementAttributes.connection, elementAttributesValues.connection(queryName)); + newDbPr.setAttribute(elementAttributes.command, elementAttributesValues.connectionCommand(queryName)); + newConnection.appendChild(newDbPr); + return serializer.serializeToString(connectionsDoc); + }; + export default { updateDocProps, clearLabelInfo, @@ -277,4 +299,5 @@ export default { updatePivotTablesandQueryTables, updateQueryTable, updatePivotTable, + addNewConnection, }; diff --git a/src/utils/xmlPartsUtils.ts b/src/utils/xmlPartsUtils.ts index 36fd306..65f37b6 100644 --- a/src/utils/xmlPartsUtils.ts +++ b/src/utils/xmlPartsUtils.ts @@ -70,8 +70,22 @@ const updateWorkbookSingleQueryAttributes = async (zip: JSZip, queryName: string await xmlInnerPartsUtils.updatePivotTablesandQueryTables(zip, queryName, refreshOnOpen, connectionId!); }; +const addConnectionOnlyQueriesToWorkbook = async (zip: JSZip, connectionOnlyQueryNames: string[]): Promise => { + // Update connections + let connectionsXmlString: string | undefined = await zip.file(connectionsXmlPath)?.async(textResultType); + if (connectionsXmlString === undefined) { + throw new Error(connectionsNotFoundErr); + } + + connectionOnlyQueryNames.forEach(async (queryName: string) => { + connectionsXmlString = await xmlInnerPartsUtils.addNewConnection(connectionsXmlString!, queryName); + }); + +}; + export default { updateWorkbookDataAndConfigurations, updateWorkbookPowerQueryDocument, updateWorkbookSingleQueryAttributes, + addConnectionOnlyQueriesToWorkbook, }; diff --git a/src/workbookManager.ts b/src/workbookManager.ts index e56a902..8f54a63 100644 --- a/src/workbookManager.ts +++ b/src/workbookManager.ts @@ -13,7 +13,7 @@ import { tableNotFoundErr, templateFileNotSupportedErr, } from "./utils/constants"; -import { QueryInfo, TableData, Grid, FileConfigs } from "./types"; +import { QueryInfo, TableData, Grid, FileConfigs, MultipleQueriesInfo } from "./types"; import { generateSingleQueryMashup } from "./generators"; export const generateSingleQueryWorkbook = async (query: QueryInfo, initialDataGrid?: Grid, fileConfigs?: FileConfigs): Promise => { @@ -40,6 +40,22 @@ export const generateSingleQueryWorkbook = async (query: QueryInfo, initialDataG return await generateSingleQueryWorkbookFromZip(zip, query, fileConfigs, tableData); }; +export const generateMultipleQueryWorkbook = async (queries: MultipleQueriesInfo, initialDataGrid?: Grid, fileConfigs?: FileConfigs): Promise => { + const templateFile: File | undefined = fileConfigs?.templateFile; + if (templateFile !== undefined && initialDataGrid !== undefined) { + throw new Error(templateWithInitialDataErr); + } + + pqUtils.validateQueryName(queries.loadedQueryName); + + const zip: JSZip = + templateFile === undefined ? await JSZip.loadAsync(SIMPLE_QUERY_WORKBOOK_TEMPLATE, { base64: true }) : await JSZip.loadAsync(templateFile); + + const tableData = initialDataGrid ? gridUtils.parseToTableData(initialDataGrid) : undefined; + + return await generateMultipleQueryWorkbookFromZip(zip, queries, fileConfigs, tableData); +}; + export const generateTableWorkbookFromHtml = async (htmlTable: HTMLTableElement, fileConfigs?: FileConfigs): Promise => { if (fileConfigs?.templateFile !== undefined) { throw new Error(templateFileNotSupportedErr); @@ -81,6 +97,18 @@ const generateSingleQueryWorkbookFromZip = async (zip: JSZip, query: QueryInfo, }); }; +const generateMultipleQueryWorkbookFromZip = async (zip: JSZip, queries: MultipleQueriesInfo, fileConfigs?: FileConfigs, tableData?: TableData): Promise => { + await xmlPartsUtils.updateWorkbookPowerQueryDocument(zip, queries.loadedQueryName, queries.mashupDocument); + await xmlPartsUtils.updateWorkbookSingleQueryAttributes(zip, queries.loadedQueryName, queries.refreshOnOpen); + await xmlPartsUtils.updateWorkbookDataAndConfigurations(zip, fileConfigs, tableData, true /*updateQueryTable*/); + await xmlPartsUtils.addConnectionOnlyQueriesToWorkbook(zip, queries.connectionOnlyQueryNames); + + return await zip.generateAsync({ + type: blobFileType, + mimeType: application, + }); +}; + export const downloadWorkbook = (file: Blob, filename: string): void => { const nav = window.navigator as any; if (nav.msSaveOrOpenBlob) From 712a80db1c30215a422d00c9b117bf062f4390ce Mon Sep 17 00:00:00 2001 From: shanialbeck Date: Thu, 10 Aug 2023 17:56:08 +0300 Subject: [PATCH 02/13] support connectiononly queries in mashupdoc --- src/utils/constants.ts | 14 ++- src/utils/mashupDocumentParser.ts | 139 ++++++++++++++++++++++++++++++ src/utils/xmlPartsUtils.ts | 10 ++- src/workbookManager.ts | 2 +- 4 files changed, 160 insertions(+), 5 deletions(-) diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 410b9c7..46b5d59 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -125,7 +125,16 @@ export const elementAttributes = { xr16uid: "xr16:uid", keepAlive: "keepAlive", refreshedVersion: "refreshedVersion", - background: "background" + background: "background", + isPrivate: "IsPrivate", + fillEnabled: "FillEnabled", + fillObjectType: "FillObjectType", + fillToDataModelEnabled: "FillToDataModelEnabled", + filLastUpdated: "FillLastUpdated", + filledCompleteResultToWorksheet: "FilledCompleteResultToWorksheet", + addedToDataModel: "AddedToDataModel", + fillErrorCode: "FillErrorCode", + fillStatus: "FillStatus", }; export const dataTypeKind = { @@ -140,6 +149,9 @@ export const elementAttributesValues = { connection: (queryName: string) => `Provider=Microsoft.Mashup.OleDb.1;Data Source=$Workbook$;Location="${queryName}";`, connectionCommand: (queryName: string) => `SELECT * FROM [${queryName}]`, tableResultType: () => "sTable", + connectionOnlyResultType: () => "sConnectionOnly", + fillStatusComplete: () => "sComplete", + fillErrorCodeUnknown: () => "sUnknown", randomizedUid: () => "{" + v4().toUpperCase() + "}", }; diff --git a/src/utils/mashupDocumentParser.ts b/src/utils/mashupDocumentParser.ts index b7c58ca..0c671e5 100644 --- a/src/utils/mashupDocumentParser.ts +++ b/src/utils/mashupDocumentParser.ts @@ -42,6 +42,17 @@ export const replaceSingleQuery = async (base64Str: string, queryName: string, q return base64.fromByteArray(newMashup); }; +export const addConnectionOnlyQuery = async (base64Str: string, connectionOnlyQueryNames: string[]): Promise => { + var { version, packageOPC, permissionsSize, permissions, metadata, endBuffer } = getPackageComponents(base64Str); + const packageSizeBuffer: Uint8Array = arrayUtils.getInt32Buffer(packageOPC.byteLength); + const permissionsSizeBuffer: Uint8Array = arrayUtils.getInt32Buffer(permissionsSize); + const newMetadataBuffer: Uint8Array = addConnectionOnlyQueryMetadata(metadata, connectionOnlyQueryNames); + const metadataSizeBuffer: Uint8Array = arrayUtils.getInt32Buffer(newMetadataBuffer.byteLength); + const newMashup: Uint8Array = arrayUtils.concatArrays(version, packageSizeBuffer, packageOPC, permissionsSizeBuffer, permissions, metadataSizeBuffer, newMetadataBuffer, endBuffer); + + return base64.fromByteArray(newMashup); + } + type PackageComponents = { version: Uint8Array; packageOPC: Uint8Array; @@ -150,3 +161,131 @@ export const editSingleQueryMetadata = (metadataArray: Uint8Array, metadata: Met return newMetadataArray; }; + +const addConnectionOnlyQueryMetadata = (metadataArray: Uint8Array, connectionOnlyQueryNames: string[]) => { + // extract metadataXml + const mashupArray: ArrayReader = new arrayUtils.ArrayReader(metadataArray.buffer); + const metadataVersion: Uint8Array = mashupArray.getBytes(4); + const metadataXmlSize: number = mashupArray.getInt32(); + const metadataXml: Uint8Array = mashupArray.getBytes(metadataXmlSize); + const endBuffer: Uint8Array = mashupArray.getBytes(); + + // parse metadataXml + const metadataString: string = new TextDecoder("utf-8").decode(metadataXml); + const newMetadataString: string = updateConnectionOnlyMetadataStr(metadataString, connectionOnlyQueryNames); + const encoder: TextEncoder = new TextEncoder(); + const newMetadataXml: Uint8Array = encoder.encode(newMetadataString); + const newMetadataXmlSize: Uint8Array = arrayUtils.getInt32Buffer(newMetadataXml.byteLength); + const newMetadataArray: Uint8Array = arrayUtils.concatArrays( + metadataVersion, + newMetadataXmlSize, + newMetadataXml, + endBuffer + ); + + return newMetadataArray; + }; + + const updateConnectionOnlyMetadataStr = (metadataString: string, connectionOnlyQueryNames: string[]) => { + const parser: DOMParser = new DOMParser(); + let updatedMetdataString: string = metadataString; + connectionOnlyQueryNames.forEach((queryName: string) => { + const metadataDoc: Document = parser.parseFromString(updatedMetdataString, xmlTextResultType); + const items: Element = metadataDoc.getElementsByTagName(element.items)[0]; + const stableEntriesItem: Element = createStableEntriesItem(metadataDoc, queryName); + items.appendChild(stableEntriesItem); + const sourceItem: Element = createSourceItem(metadataDoc, queryName); + items.appendChild(sourceItem); + const serializer: XMLSerializer = new XMLSerializer(); + updatedMetdataString = serializer.serializeToString(metadataDoc); + }); + + return updatedMetdataString; + }; + + const createSourceItem = (metadataDoc: Document, queryName: string) => { + const newItemSource: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.item); + const newItemLocation: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemLocation); + const newItemType: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemType); + newItemType.textContent = "Formula"; + const newItemPath: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemPath); + newItemPath.textContent = `Section1/${queryName}/Source`; + newItemLocation.appendChild(newItemType); + newItemLocation.appendChild(newItemPath); + newItemSource.appendChild(newItemLocation); + + return newItemSource; + }; + + const createStableEntriesItem = (metadataDoc: Document, queryName: string) => { + const newItem: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.item); + const newItemLocation: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemLocation); + const newItemType: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemType); + newItemType.textContent = "Formula"; + const newItemPath: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemPath); + newItemPath.textContent = `Section1/${queryName}`; + newItemLocation.appendChild(newItemType); + newItemLocation.appendChild(newItemPath); + newItem.appendChild(newItemLocation); + const stableEntries: Element = createConnectionOnlyEntries(metadataDoc); + newItem.appendChild(stableEntries); + + return newItem; + }; + + const createConnectionOnlyEntries = (metadataDoc: Document) => { + const stableEntries: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.stableEntries); + + const IsPrivate: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); + IsPrivate.setAttribute(elementAttributes.type, elementAttributes.isPrivate); + IsPrivate.setAttribute(elementAttributes.value, "l0"); + + stableEntries.appendChild(IsPrivate); + const FillEnabled: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); + FillEnabled.setAttribute(elementAttributes.type, elementAttributes.fillEnabled); + FillEnabled.setAttribute(elementAttributes.value, "l0"); + stableEntries.appendChild(FillEnabled); + + const FillObjectType: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); + FillObjectType.setAttribute(elementAttributes.type, elementAttributes.fillObjectType); + FillObjectType.setAttribute(elementAttributes.value, elementAttributesValues.connectionOnlyResultType()); + stableEntries.appendChild(FillObjectType); + + const FillToDataModelEnabled: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); + FillToDataModelEnabled.setAttribute(elementAttributes.type, elementAttributes.fillToDataModelEnabled); + FillToDataModelEnabled.setAttribute(elementAttributes.value, "l0"); + stableEntries.appendChild(FillToDataModelEnabled); + + const FillLastUpdated: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); + FillLastUpdated.setAttribute(elementAttributes.type, elementAttributes.fillLastUpdated); + const nowTime: string = new Date().toISOString(); + FillLastUpdated.setAttribute(elementAttributes.value, (elementAttributes.day + nowTime).replace(/Z/, "0000Z")); + stableEntries.appendChild(FillLastUpdated); + + const ResultType: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); + ResultType.setAttribute(elementAttributes.type, elementAttributes.resultType); + ResultType.setAttribute(elementAttributes.value, elementAttributesValues.tableResultType()); + stableEntries.appendChild(ResultType); + + const FilledCompleteResultToWorksheet: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); + FilledCompleteResultToWorksheet.setAttribute(elementAttributes.type, elementAttributes.filledCompleteResultToWorksheet); + FilledCompleteResultToWorksheet.setAttribute(elementAttributes.value, "l0"); + stableEntries.appendChild(FilledCompleteResultToWorksheet); + + const AddedToDataModel: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); + AddedToDataModel.setAttribute(elementAttributes.type, elementAttributes.addedToDataModel); + AddedToDataModel.setAttribute(elementAttributes.value, "l0"); + stableEntries.appendChild(AddedToDataModel); + + const FillErrorCode: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); + FillErrorCode.setAttribute(elementAttributes.type, elementAttributes.fillErrorCode); + FillErrorCode.setAttribute(elementAttributes.value, elementAttributesValues.fillErrorCodeUnknown()); + stableEntries.appendChild(FillErrorCode); + + const FillStatus: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); + FillStatus.setAttribute(elementAttributes.type, elementAttributes.fillStatus); + FillStatus.setAttribute(elementAttributes.value, elementAttributesValues.fillStatusComplete()); + stableEntries.appendChild(FillStatus); + + return stableEntries; + }; \ No newline at end of file diff --git a/src/utils/xmlPartsUtils.ts b/src/utils/xmlPartsUtils.ts index 65f37b6..351269e 100644 --- a/src/utils/xmlPartsUtils.ts +++ b/src/utils/xmlPartsUtils.ts @@ -12,7 +12,7 @@ import { sheetsXmlPath, sheetsNotFoundErr, } from "./constants"; -import { replaceSingleQuery } from "./mashupDocumentParser"; +import { addConnectionOnlyQuery, replaceSingleQuery } from "./mashupDocumentParser"; import { FileConfigs, TableData } from "../types"; import pqUtils from "./pqUtils"; import xmlInnerPartsUtils from "./xmlInnerPartsUtils"; @@ -27,7 +27,7 @@ const updateWorkbookDataAndConfigurations = async (zip: JSZip, fileConfigs?: Fil await tableUtils.updateTableInitialDataIfNeeded(zip, tableData, updateQueryTable); }; -const updateWorkbookPowerQueryDocument = async (zip: JSZip, queryName: string, queryMashupDoc: string): Promise => { +const updateWorkbookPowerQueryDocument = async (zip: JSZip, queryName: string, queryMashupDoc: string, connectionOnlyQueryNames?: string[]): Promise => { const old_base64: string | undefined = await pqUtils.getBase64(zip); if (!old_base64) { @@ -35,7 +35,11 @@ const updateWorkbookPowerQueryDocument = async (zip: JSZip, queryName: string, q } const new_base64: string = await replaceSingleQuery(old_base64, queryName, queryMashupDoc); - await pqUtils.setBase64(zip, new_base64); + let updated_base64: string = new_base64; + if (connectionOnlyQueryNames) { + updated_base64 = await addConnectionOnlyQuery(new_base64, connectionOnlyQueryNames); + } + await pqUtils.setBase64(zip, updated_base64); }; const updateWorkbookSingleQueryAttributes = async (zip: JSZip, queryName: string, refreshOnOpen: boolean): Promise => { diff --git a/src/workbookManager.ts b/src/workbookManager.ts index 8f54a63..e12c5f2 100644 --- a/src/workbookManager.ts +++ b/src/workbookManager.ts @@ -98,7 +98,7 @@ const generateSingleQueryWorkbookFromZip = async (zip: JSZip, query: QueryInfo, }; const generateMultipleQueryWorkbookFromZip = async (zip: JSZip, queries: MultipleQueriesInfo, fileConfigs?: FileConfigs, tableData?: TableData): Promise => { - await xmlPartsUtils.updateWorkbookPowerQueryDocument(zip, queries.loadedQueryName, queries.mashupDocument); + await xmlPartsUtils.updateWorkbookPowerQueryDocument(zip, queries.loadedQueryName, queries.mashupDocument, queries.connectionOnlyQueryNames); await xmlPartsUtils.updateWorkbookSingleQueryAttributes(zip, queries.loadedQueryName, queries.refreshOnOpen); await xmlPartsUtils.updateWorkbookDataAndConfigurations(zip, fileConfigs, tableData, true /*updateQueryTable*/); await xmlPartsUtils.addConnectionOnlyQueriesToWorkbook(zip, queries.connectionOnlyQueryNames); From a7897b98b3c653c35d84c1e22c3467c9bd47453b Mon Sep 17 00:00:00 2001 From: shanialbeck Date: Tue, 19 Sep 2023 09:47:21 +0300 Subject: [PATCH 03/13] refactor - extract logic to separate funcs, constants --- src/utils/constants.ts | 6 +- src/utils/mashupDocumentParser.ts | 110 +++++++++--------------------- src/utils/xmlInnerPartsUtils.ts | 21 +++--- src/utils/xmlPartsUtils.ts | 13 +++- 4 files changed, 59 insertions(+), 91 deletions(-) diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 46b5d59..cf6165a 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -89,7 +89,7 @@ export const element = { kindCell: "c", connection: "connection", connections: "connections", - dbpr: "dbPr", + databaseProps: "dbPr", }; export const elementAttributes = { @@ -104,6 +104,7 @@ export const elementAttributes = { name: "name", description: "description", id: "id", + typeLowerCase: "type", type: "Type", value: "Value", relationshipInfo: "RelationshipInfoContainer", @@ -143,6 +144,8 @@ export const dataTypeKind = { boolean: "b", }; +export const itemPathTextContext = (queryName: string, isSource: boolean) => isSource ? `Section1/${queryName}/Source` : `Section1/${queryName}`; + export const elementAttributesValues = { connectionName: (queryName: string) => `Query - ${queryName}`, connectionDescription: (queryName: string) => `Connection to the '${queryName}' query in the workbook.`, @@ -153,6 +156,7 @@ export const elementAttributesValues = { fillStatusComplete: () => "sComplete", fillErrorCodeUnknown: () => "sUnknown", randomizedUid: () => "{" + v4().toUpperCase() + "}", + defaultConnectionType: () => "5", }; export const defaults = { diff --git a/src/utils/mashupDocumentParser.ts b/src/utils/mashupDocumentParser.ts index 0c671e5..b2d3966 100644 --- a/src/utils/mashupDocumentParser.ts +++ b/src/utils/mashupDocumentParser.ts @@ -16,6 +16,7 @@ import { divider, elementAttributes, elementAttributesValues, + itemPathTextContext, } from "./constants"; import { arrayUtils } from "."; import { Metadata } from "../types"; @@ -42,16 +43,13 @@ export const replaceSingleQuery = async (base64Str: string, queryName: string, q return base64.fromByteArray(newMashup); }; -export const addConnectionOnlyQuery = async (base64Str: string, connectionOnlyQueryNames: string[]): Promise => { +export const addConnectionOnlyQueries = async (base64Str: string, connectionOnlyQueryNames: string[]): Promise => { var { version, packageOPC, permissionsSize, permissions, metadata, endBuffer } = getPackageComponents(base64Str); - const packageSizeBuffer: Uint8Array = arrayUtils.getInt32Buffer(packageOPC.byteLength); - const permissionsSizeBuffer: Uint8Array = arrayUtils.getInt32Buffer(permissionsSize); const newMetadataBuffer: Uint8Array = addConnectionOnlyQueryMetadata(metadata, connectionOnlyQueryNames); - const metadataSizeBuffer: Uint8Array = arrayUtils.getInt32Buffer(newMetadataBuffer.byteLength); - const newMashup: Uint8Array = arrayUtils.concatArrays(version, packageSizeBuffer, packageOPC, permissionsSizeBuffer, permissions, metadataSizeBuffer, newMetadataBuffer, endBuffer); + const newMashup: Uint8Array = arrayUtils.concatArrays(version, arrayUtils.getInt32Buffer(packageOPC.byteLength), packageOPC, arrayUtils.getInt32Buffer(permissionsSize), permissions, arrayUtils.getInt32Buffer(newMetadataBuffer.byteLength), newMetadataBuffer, endBuffer); return base64.fromByteArray(newMashup); - } +}; type PackageComponents = { version: Uint8Array; @@ -188,104 +186,64 @@ const addConnectionOnlyQueryMetadata = (metadataArray: Uint8Array, connectionOnl const updateConnectionOnlyMetadataStr = (metadataString: string, connectionOnlyQueryNames: string[]) => { const parser: DOMParser = new DOMParser(); - let updatedMetdataString: string = metadataString; + let metadataDoc: Document = parser.parseFromString(metadataString, xmlTextResultType); connectionOnlyQueryNames.forEach((queryName: string) => { - const metadataDoc: Document = parser.parseFromString(updatedMetdataString, xmlTextResultType); const items: Element = metadataDoc.getElementsByTagName(element.items)[0]; const stableEntriesItem: Element = createStableEntriesItem(metadataDoc, queryName); items.appendChild(stableEntriesItem); - const sourceItem: Element = createSourceItem(metadataDoc, queryName); + const sourceItem: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.item); + sourceItem.appendChild(createItemLocation(metadataDoc, queryName, true)); items.appendChild(sourceItem); - const serializer: XMLSerializer = new XMLSerializer(); - updatedMetdataString = serializer.serializeToString(metadataDoc); }); + const updatedMetdataString: string = new XMLSerializer().serializeToString(metadataDoc); + return updatedMetdataString; }; - const createSourceItem = (metadataDoc: Document, queryName: string) => { - const newItemSource: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.item); + const createItemLocation = (metadataDoc: Document, queryName: string, isSource: boolean) => { const newItemLocation: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemLocation); const newItemType: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemType); newItemType.textContent = "Formula"; const newItemPath: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemPath); - newItemPath.textContent = `Section1/${queryName}/Source`; + newItemPath.textContent = itemPathTextContext(queryName, isSource); newItemLocation.appendChild(newItemType); - newItemLocation.appendChild(newItemPath); - newItemSource.appendChild(newItemLocation); + newItemLocation.appendChild(newItemPath); - return newItemSource; + return newItemLocation; }; const createStableEntriesItem = (metadataDoc: Document, queryName: string) => { const newItem: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.item); - const newItemLocation: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemLocation); - const newItemType: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemType); - newItemType.textContent = "Formula"; - const newItemPath: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemPath); - newItemPath.textContent = `Section1/${queryName}`; - newItemLocation.appendChild(newItemType); - newItemLocation.appendChild(newItemPath); - newItem.appendChild(newItemLocation); + newItem.appendChild(createItemLocation(metadataDoc, queryName, false)); const stableEntries: Element = createConnectionOnlyEntries(metadataDoc); newItem.appendChild(stableEntries); return newItem; }; + const createElementObject = (metadataDoc: Document, type: string, value: string) => { + const elementObject: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); + elementObject.setAttribute(elementAttributes.type, type); + elementObject.setAttribute(elementAttributes.value, value); + + return elementObject; + }; + const createConnectionOnlyEntries = (metadataDoc: Document) => { const stableEntries: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.stableEntries); - - const IsPrivate: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); - IsPrivate.setAttribute(elementAttributes.type, elementAttributes.isPrivate); - IsPrivate.setAttribute(elementAttributes.value, "l0"); - - stableEntries.appendChild(IsPrivate); - const FillEnabled: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); - FillEnabled.setAttribute(elementAttributes.type, elementAttributes.fillEnabled); - FillEnabled.setAttribute(elementAttributes.value, "l0"); - stableEntries.appendChild(FillEnabled); - - const FillObjectType: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); - FillObjectType.setAttribute(elementAttributes.type, elementAttributes.fillObjectType); - FillObjectType.setAttribute(elementAttributes.value, elementAttributesValues.connectionOnlyResultType()); - stableEntries.appendChild(FillObjectType); - - const FillToDataModelEnabled: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); - FillToDataModelEnabled.setAttribute(elementAttributes.type, elementAttributes.fillToDataModelEnabled); - FillToDataModelEnabled.setAttribute(elementAttributes.value, "l0"); - stableEntries.appendChild(FillToDataModelEnabled); - - const FillLastUpdated: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); - FillLastUpdated.setAttribute(elementAttributes.type, elementAttributes.fillLastUpdated); const nowTime: string = new Date().toISOString(); - FillLastUpdated.setAttribute(elementAttributes.value, (elementAttributes.day + nowTime).replace(/Z/, "0000Z")); - stableEntries.appendChild(FillLastUpdated); - - const ResultType: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); - ResultType.setAttribute(elementAttributes.type, elementAttributes.resultType); - ResultType.setAttribute(elementAttributes.value, elementAttributesValues.tableResultType()); - stableEntries.appendChild(ResultType); - - const FilledCompleteResultToWorksheet: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); - FilledCompleteResultToWorksheet.setAttribute(elementAttributes.type, elementAttributes.filledCompleteResultToWorksheet); - FilledCompleteResultToWorksheet.setAttribute(elementAttributes.value, "l0"); - stableEntries.appendChild(FilledCompleteResultToWorksheet); - - const AddedToDataModel: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); - AddedToDataModel.setAttribute(elementAttributes.type, elementAttributes.addedToDataModel); - AddedToDataModel.setAttribute(elementAttributes.value, "l0"); - stableEntries.appendChild(AddedToDataModel); - - const FillErrorCode: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); - FillErrorCode.setAttribute(elementAttributes.type, elementAttributes.fillErrorCode); - FillErrorCode.setAttribute(elementAttributes.value, elementAttributesValues.fillErrorCodeUnknown()); - stableEntries.appendChild(FillErrorCode); - - const FillStatus: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry); - FillStatus.setAttribute(elementAttributes.type, elementAttributes.fillStatus); - FillStatus.setAttribute(elementAttributes.value, elementAttributesValues.fillStatusComplete()); - stableEntries.appendChild(FillStatus); - + + stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.isPrivate, "l0")); + stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillEnabled, "l0")); + stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillObjectType, elementAttributesValues.connectionOnlyResultType())); + stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillToDataModelEnabled, "l0")); + stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillLastUpdated, (elementAttributes.day + nowTime).replace(/Z/, "0000Z"))); + stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.resultType, elementAttributesValues.tableResultType())); + stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.filledCompleteResultToWorksheet, "l0")); + stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.addedToDataModel, "l0")); + stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillErrorCode, elementAttributesValues.fillErrorCodeUnknown())); + stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillStatus, elementAttributesValues.fillStatusComplete())); + return stableEntries; }; \ No newline at end of file diff --git a/src/utils/xmlInnerPartsUtils.ts b/src/utils/xmlInnerPartsUtils.ts index bc7defb..ebde0b1 100644 --- a/src/utils/xmlInnerPartsUtils.ts +++ b/src/utils/xmlInnerPartsUtils.ts @@ -268,26 +268,25 @@ const updatePivotTable = (tableXmlString: string, connectionId: string, refreshO return { isPivotTableUpdated, newPivotTable }; }; -const addNewConnection = async (connectionsXmlString: string, queryName: string): Promise => { - const parser: DOMParser = new DOMParser(); - const serializer: XMLSerializer = new XMLSerializer(); - const connectionsDoc: Document = parser.parseFromString(connectionsXmlString, xmlTextResultType); +const addNewConnection = async (connectionsDoc: Document, queryName: string): Promise => { const connections = connectionsDoc.getElementsByTagName(element.connections)[0]; const newConnection = connectionsDoc.createElementNS(connectionsDoc.documentElement.namespaceURI, element.connection); - connections.append(newConnection); newConnection.setAttribute(elementAttributes.id, [...connectionsDoc.getElementsByTagName(element.connection)].length.toString()); newConnection.setAttribute(elementAttributes.xr16uid, elementAttributesValues.randomizedUid()); newConnection.setAttribute(elementAttributes.keepAlive, trueValue); newConnection.setAttribute(elementAttributes.name, elementAttributesValues.connectionName(queryName)); newConnection.setAttribute(elementAttributes.description, elementAttributesValues.connectionDescription(queryName)); - newConnection.setAttribute(elementAttributes.type, "5"); + newConnection.setAttribute(elementAttributes.typeLowerCase, elementAttributesValues.defaultConnectionType()); newConnection.setAttribute(elementAttributes.refreshedVersion, falseValue); newConnection.setAttribute(elementAttributes.background, trueValue); - const newDbPr = connectionsDoc.createElementNS(connectionsDoc.documentElement.namespaceURI, element.dbpr); - newDbPr.setAttribute(elementAttributes.connection, elementAttributesValues.connection(queryName)); - newDbPr.setAttribute(elementAttributes.command, elementAttributesValues.connectionCommand(queryName)); - newConnection.appendChild(newDbPr); - return serializer.serializeToString(connectionsDoc); + connections.append(newConnection); + + const databaseProperties = connectionsDoc.createElementNS(connectionsDoc.documentElement.namespaceURI, element.databaseProps); + databaseProperties.setAttribute(elementAttributes.connection, elementAttributesValues.connection(queryName)); + databaseProperties.setAttribute(elementAttributes.command, elementAttributesValues.connectionCommand(queryName)); + newConnection.appendChild(databaseProperties); + + return connectionsDoc; }; export default { diff --git a/src/utils/xmlPartsUtils.ts b/src/utils/xmlPartsUtils.ts index 351269e..de5b322 100644 --- a/src/utils/xmlPartsUtils.ts +++ b/src/utils/xmlPartsUtils.ts @@ -11,8 +11,9 @@ import { sharedStringsNotFoundErr, sheetsXmlPath, sheetsNotFoundErr, + xmlTextResultType, } from "./constants"; -import { addConnectionOnlyQuery, replaceSingleQuery } from "./mashupDocumentParser"; +import { addConnectionOnlyQueries, replaceSingleQuery } from "./mashupDocumentParser"; import { FileConfigs, TableData } from "../types"; import pqUtils from "./pqUtils"; import xmlInnerPartsUtils from "./xmlInnerPartsUtils"; @@ -37,7 +38,7 @@ const updateWorkbookPowerQueryDocument = async (zip: JSZip, queryName: string, q const new_base64: string = await replaceSingleQuery(old_base64, queryName, queryMashupDoc); let updated_base64: string = new_base64; if (connectionOnlyQueryNames) { - updated_base64 = await addConnectionOnlyQuery(new_base64, connectionOnlyQueryNames); + updated_base64 = await addConnectionOnlyQueries(new_base64, connectionOnlyQueryNames); } await pqUtils.setBase64(zip, updated_base64); }; @@ -81,10 +82,16 @@ const addConnectionOnlyQueriesToWorkbook = async (zip: JSZip, connectionOnlyQuer throw new Error(connectionsNotFoundErr); } + const parser: DOMParser = new DOMParser(); + const serializer: XMLSerializer = new XMLSerializer(); + let connectionsDoc: Document = parser.parseFromString(connectionsXmlString, xmlTextResultType); connectionOnlyQueryNames.forEach(async (queryName: string) => { - connectionsXmlString = await xmlInnerPartsUtils.addNewConnection(connectionsXmlString!, queryName); + connectionsDoc = await xmlInnerPartsUtils.addNewConnection(connectionsDoc, queryName); }); + connectionsXmlString = serializer.serializeToString(connectionsDoc); + zip.file(connectionsXmlPath, connectionsXmlString); + }; export default { From fff99abd9bec511a30db25b6620a69df9d1e880d Mon Sep 17 00:00:00 2001 From: shanialbeck Date: Mon, 8 Jan 2024 00:13:20 -0800 Subject: [PATCH 04/13] change API to contain queryInfo array, not mashupdoc --- src/generators.ts | 15 ++++++++++++++ src/types.ts | 8 ++++---- src/utils/constants.ts | 2 ++ src/utils/pqUtils.ts | 44 +++++++++++++++++++++++++++++++++++++++++- src/workbookManager.ts | 18 ++++++++++++----- 5 files changed, 77 insertions(+), 10 deletions(-) diff --git a/src/generators.ts b/src/generators.ts index ee4ecc2..c7c8490 100644 --- a/src/generators.ts +++ b/src/generators.ts @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import { ConnectionOnlyQueryInfo, QueryInfo } from "./types"; + export const generateMashupXMLTemplate = (base64: string): string => `${base64}`; @@ -10,4 +12,17 @@ export const generateSingleQueryMashup = (queryName: string, query: string): str shared #"${queryName}" = ${query};`; +export const generateMultipleQueryMashup = (loadedQuery: QueryInfo, queries: ConnectionOnlyQueryInfo[]): string => { + let mashup: string = generateSingleQueryMashup(loadedQuery.queryName!, loadedQuery.queryMashup!); + queries.forEach((query: ConnectionOnlyQueryInfo) => { + const queryName = query.queryName ? query.queryName : "Query" + queries.indexOf(query); + mashup += ` + + shared #"${queryName}" = + ${query.queryMashup};`; + }); + + return mashup; +} + export const generateCustomXmlFilePath = (i: number): string => `customXml/item${i}.xml`; diff --git a/src/types.ts b/src/types.ts index d6df55b..7151753 100644 --- a/src/types.ts +++ b/src/types.ts @@ -8,12 +8,12 @@ export interface QueryInfo { } export interface MultipleQueriesInfo { - loadedQueryName: string; - refreshOnOpen: boolean; - connectionOnlyQueryNames: string[]; - mashupDocument: string; + loadedQuery: QueryInfo; + connectionOnlyQueries: ConnectionOnlyQueryInfo[]; } +export type ConnectionOnlyQueryInfo = Omit; + export interface DocProps { title?: string | null; subject?: string | null; diff --git a/src/utils/constants.ts b/src/utils/constants.ts index cf6165a..a04f96d 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -42,6 +42,7 @@ export const unexpectedErr = "Unexpected error"; export const arrayIsntMxNErr = "Array isn't MxN"; export const templateFileNotSupportedErr = "Template file is not supported for this API call"; export const relsNotFoundErr = ".rels were not found in template"; +export const queryNameAlreadyExistsErr = "Queries must have unique names"; export const blobFileType = "blob"; export const uint8ArrayType = "uint8array"; @@ -161,6 +162,7 @@ export const elementAttributesValues = { export const defaults = { queryName: "Query1", + queryNamePrefix: "Query", sheetName: "Sheet1", columnName: "Column", }; diff --git a/src/utils/pqUtils.ts b/src/utils/pqUtils.ts index 4812320..10a6d50 100644 --- a/src/utils/pqUtils.ts +++ b/src/utils/pqUtils.ts @@ -2,9 +2,10 @@ // Licensed under the MIT license. import JSZip from "jszip"; -import { EmptyQueryNameErr, QueryNameMaxLengthErr, maxQueryLength, URLS, BOM, QueryNameInvalidCharsErr } from "./constants"; +import { EmptyQueryNameErr, QueryNameMaxLengthErr, maxQueryLength, URLS, BOM, QueryNameInvalidCharsErr, queryNameAlreadyExistsErr, defaults } from "./constants"; import { generateMashupXMLTemplate, generateCustomXmlFilePath } from "../generators"; import { Buffer } from "buffer"; +import { ConnectionOnlyQueryInfo } from "../types"; type CustomXmlFile = { found: boolean; @@ -110,10 +111,51 @@ const validateQueryName = (queryName: string): void => { throw new Error(EmptyQueryNameErr); } }; + +const validateMultipleQueryNames = (queries: ConnectionOnlyQueryInfo[], loadedQueryName: string): string[] => { + const queryNames: string[] = []; + queries.forEach((query: ConnectionOnlyQueryInfo) => { + if (query.queryName) { + validateQueryName(query.queryName); + if (queryNames.includes(query.queryName) || query.queryName === loadedQueryName) { + throw new Error(queryNameAlreadyExistsErr); + } + + queryNames.push(query.queryName); + } + }); + + return queryNames; +}; + +const assignQueryNames = (queries: ConnectionOnlyQueryInfo[], loadedQueryName: string, queryNames: string[]): ConnectionOnlyQueryInfo[] => { + // Generate unique name for queries without a name + queries.forEach((query: ConnectionOnlyQueryInfo) => { + if (!query.queryName) { + query.queryName = generateUniqueQueryName(queryNames); + queryNames.push(query.queryName); + } + }); + + return queries; +}; + +const generateUniqueQueryName = (queryNames: string[]): string => { + let queryName: string = defaults.queryName; + let index: number = 2; + while (queryNames.includes(queryName)) { + queryName = defaults.queryNamePrefix + index++; + } + + return queryName; +}; + export default { getBase64, setBase64, getCustomXmlFile, getDataMashupFile, validateQueryName, + assignQueryNames, + validateMultipleQueryNames, }; diff --git a/src/workbookManager.ts b/src/workbookManager.ts index e12c5f2..3ac057b 100644 --- a/src/workbookManager.ts +++ b/src/workbookManager.ts @@ -14,7 +14,7 @@ import { templateFileNotSupportedErr, } from "./utils/constants"; import { QueryInfo, TableData, Grid, FileConfigs, MultipleQueriesInfo } from "./types"; -import { generateSingleQueryMashup } from "./generators"; +import { generateMultipleQueryMashup, generateSingleQueryMashup } from "./generators"; export const generateSingleQueryWorkbook = async (query: QueryInfo, initialDataGrid?: Grid, fileConfigs?: FileConfigs): Promise => { if (!query.queryMashup) { @@ -46,13 +46,19 @@ export const generateMultipleQueryWorkbook = async (queries: MultipleQueriesInfo throw new Error(templateWithInitialDataErr); } - pqUtils.validateQueryName(queries.loadedQueryName); + if (!queries.loadedQuery.queryName) { + queries.loadedQuery.queryName = defaults.queryName; + } const zip: JSZip = templateFile === undefined ? await JSZip.loadAsync(SIMPLE_QUERY_WORKBOOK_TEMPLATE, { base64: true }) : await JSZip.loadAsync(templateFile); const tableData = initialDataGrid ? gridUtils.parseToTableData(initialDataGrid) : undefined; + const connectionOnlyQueryNames: string[] = pqUtils.validateMultipleQueryNames(queries.connectionOnlyQueries, queries.loadedQuery.queryName!); + pqUtils.validateQueryName(queries.loadedQuery.queryName!); + pqUtils.assignQueryNames(queries.connectionOnlyQueries, queries.loadedQuery.queryName!, connectionOnlyQueryNames); + return await generateMultipleQueryWorkbookFromZip(zip, queries, fileConfigs, tableData); }; @@ -98,10 +104,12 @@ const generateSingleQueryWorkbookFromZip = async (zip: JSZip, query: QueryInfo, }; const generateMultipleQueryWorkbookFromZip = async (zip: JSZip, queries: MultipleQueriesInfo, fileConfigs?: FileConfigs, tableData?: TableData): Promise => { - await xmlPartsUtils.updateWorkbookPowerQueryDocument(zip, queries.loadedQueryName, queries.mashupDocument, queries.connectionOnlyQueryNames); - await xmlPartsUtils.updateWorkbookSingleQueryAttributes(zip, queries.loadedQueryName, queries.refreshOnOpen); + const connectionOnlyQueryNames: string[] = queries.connectionOnlyQueries.map((query) => query.queryName!); + const mashupDocument: string = generateMultipleQueryMashup(queries.loadedQuery, queries.connectionOnlyQueries); + await xmlPartsUtils.updateWorkbookPowerQueryDocument(zip, queries.loadedQuery.queryName!, mashupDocument, connectionOnlyQueryNames); + await xmlPartsUtils.updateWorkbookSingleQueryAttributes(zip, queries.loadedQuery.queryName!, queries.loadedQuery.refreshOnOpen); await xmlPartsUtils.updateWorkbookDataAndConfigurations(zip, fileConfigs, tableData, true /*updateQueryTable*/); - await xmlPartsUtils.addConnectionOnlyQueriesToWorkbook(zip, queries.connectionOnlyQueryNames); + await xmlPartsUtils.addConnectionOnlyQueriesToWorkbook(zip, connectionOnlyQueryNames); return await zip.generateAsync({ type: blobFileType, From b5cf0a340381511403dc95d6bd5f169deece06d0 Mon Sep 17 00:00:00 2001 From: shanialbeck Date: Mon, 8 Jan 2024 00:50:48 -0800 Subject: [PATCH 05/13] fix no connection-only queries scenario --- src/utils/xmlPartsUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/xmlPartsUtils.ts b/src/utils/xmlPartsUtils.ts index de5b322..2a57884 100644 --- a/src/utils/xmlPartsUtils.ts +++ b/src/utils/xmlPartsUtils.ts @@ -37,7 +37,7 @@ const updateWorkbookPowerQueryDocument = async (zip: JSZip, queryName: string, q const new_base64: string = await replaceSingleQuery(old_base64, queryName, queryMashupDoc); let updated_base64: string = new_base64; - if (connectionOnlyQueryNames) { + if (connectionOnlyQueryNames && connectionOnlyQueryNames.length > 0) { updated_base64 = await addConnectionOnlyQueries(new_base64, connectionOnlyQueryNames); } await pqUtils.setBase64(zip, updated_base64); From da417893284e3f263793e54d9b7c8d12ddcba248 Mon Sep 17 00:00:00 2001 From: shanialbeck Date: Mon, 8 Jan 2024 01:24:29 -0800 Subject: [PATCH 06/13] add missing stablentries from metadata --- src/utils/mashupDocumentParser.ts | 2 ++ src/utils/xmlPartsUtils.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils/mashupDocumentParser.ts b/src/utils/mashupDocumentParser.ts index b2d3966..151919b 100644 --- a/src/utils/mashupDocumentParser.ts +++ b/src/utils/mashupDocumentParser.ts @@ -193,6 +193,8 @@ const addConnectionOnlyQueryMetadata = (metadataArray: Uint8Array, connectionOnl items.appendChild(stableEntriesItem); const sourceItem: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.item); sourceItem.appendChild(createItemLocation(metadataDoc, queryName, true)); + const stableEntries: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.stableEntries); + sourceItem.appendChild(stableEntries); items.appendChild(sourceItem); }); diff --git a/src/utils/xmlPartsUtils.ts b/src/utils/xmlPartsUtils.ts index 2a57884..62e98d0 100644 --- a/src/utils/xmlPartsUtils.ts +++ b/src/utils/xmlPartsUtils.ts @@ -37,7 +37,7 @@ const updateWorkbookPowerQueryDocument = async (zip: JSZip, queryName: string, q const new_base64: string = await replaceSingleQuery(old_base64, queryName, queryMashupDoc); let updated_base64: string = new_base64; - if (connectionOnlyQueryNames && connectionOnlyQueryNames.length > 0) { + if (connectionOnlyQueryNames && (connectionOnlyQueryNames.length > 0)) { updated_base64 = await addConnectionOnlyQueries(new_base64, connectionOnlyQueryNames); } await pqUtils.setBase64(zip, updated_base64); From b844a9c7aecc9e48aea3267a3344804306c24541 Mon Sep 17 00:00:00 2001 From: shanialbeck Date: Mon, 8 Jan 2024 05:04:37 -0800 Subject: [PATCH 07/13] trim whitespace in queryname validation --- src/utils/pqUtils.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/utils/pqUtils.ts b/src/utils/pqUtils.ts index 10a6d50..c47448c 100644 --- a/src/utils/pqUtils.ts +++ b/src/utils/pqUtils.ts @@ -117,11 +117,12 @@ const validateMultipleQueryNames = (queries: ConnectionOnlyQueryInfo[], loadedQ queries.forEach((query: ConnectionOnlyQueryInfo) => { if (query.queryName) { validateQueryName(query.queryName); - if (queryNames.includes(query.queryName) || query.queryName === loadedQueryName) { + const trimmedQueryName: string | undefined = query.queryName.trim(); + if (queryNames.includes(trimmedQueryName) || trimmedQueryName === loadedQueryName.trim()) { throw new Error(queryNameAlreadyExistsErr); } - queryNames.push(query.queryName); + queryNames.push(trimmedQueryName); } }); From c84308754f471c1478283b2a61e6b72a169e69b2 Mon Sep 17 00:00:00 2001 From: shanialbeck Date: Mon, 8 Jan 2024 06:21:25 -0800 Subject: [PATCH 08/13] fix query name not case-sensitive --- src/utils/pqUtils.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/utils/pqUtils.ts b/src/utils/pqUtils.ts index c47448c..dbbfaf5 100644 --- a/src/utils/pqUtils.ts +++ b/src/utils/pqUtils.ts @@ -114,15 +114,16 @@ const validateQueryName = (queryName: string): void => { const validateMultipleQueryNames = (queries: ConnectionOnlyQueryInfo[], loadedQueryName: string): string[] => { const queryNames: string[] = []; + const cleanedLoadedQueryName: string = loadedQueryName.trim().toLowerCase(); queries.forEach((query: ConnectionOnlyQueryInfo) => { if (query.queryName) { validateQueryName(query.queryName); - const trimmedQueryName: string | undefined = query.queryName.trim(); - if (queryNames.includes(trimmedQueryName) || trimmedQueryName === loadedQueryName.trim()) { + const cleanedQueryName: string | undefined = query.queryName.trim().toLowerCase(); + if (queryNames.includes(cleanedQueryName) || cleanedQueryName === cleanedLoadedQueryName) { throw new Error(queryNameAlreadyExistsErr); } - queryNames.push(trimmedQueryName); + queryNames.push(cleanedQueryName); } }); From 5eddb158be77cd763eb778d59ec2ef6c33b64f3b Mon Sep 17 00:00:00 2001 From: shanialbeck Date: Mon, 8 Jan 2024 07:57:05 -0800 Subject: [PATCH 09/13] fixed generated unique name to be case insensitive --- src/utils/pqUtils.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/utils/pqUtils.ts b/src/utils/pqUtils.ts index dbbfaf5..a717b54 100644 --- a/src/utils/pqUtils.ts +++ b/src/utils/pqUtils.ts @@ -134,18 +134,20 @@ const assignQueryNames = (queries: ConnectionOnlyQueryInfo[], loadedQueryName: s // Generate unique name for queries without a name queries.forEach((query: ConnectionOnlyQueryInfo) => { if (!query.queryName) { - query.queryName = generateUniqueQueryName(queryNames); - queryNames.push(query.queryName); + query.queryName = generateUniqueQueryName(queryNames, loadedQueryName); + queryNames.push(query.queryName.toLowerCase()); } }); return queries; }; -const generateUniqueQueryName = (queryNames: string[]): string => { +const generateUniqueQueryName = (queryNames: string[], loadedQueryName: string,): string => { let queryName: string = defaults.queryName; let index: number = 2; - while (queryNames.includes(queryName)) { + const cleanedLoadedQueryName: string = loadedQueryName.trim().toLowerCase(); + // Assumes that query names are lower case + while (queryNames.includes(queryName.toLowerCase()) || queryName.toLowerCase() === cleanedLoadedQueryName) { queryName = defaults.queryNamePrefix + index++; } @@ -160,4 +162,5 @@ export default { validateQueryName, assignQueryNames, validateMultipleQueryNames, + generateUniqueQueryName }; From 2c6382aece83bb2d1dba9bf45455e6ce922bb1dd Mon Sep 17 00:00:00 2001 From: shanialbeck Date: Tue, 9 Jan 2024 03:57:31 -0800 Subject: [PATCH 10/13] name changes + comments --- src/types.ts | 2 +- src/utils/constants.ts | 2 +- src/utils/mashupDocumentParser.ts | 4 ++-- src/utils/pqUtils.ts | 6 +++--- src/utils/xmlPartsUtils.ts | 10 ++++++---- src/workbookManager.ts | 18 ++++++++---------- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/types.ts b/src/types.ts index 7151753..8582433 100644 --- a/src/types.ts +++ b/src/types.ts @@ -7,7 +7,7 @@ export interface QueryInfo { queryName?: string; } -export interface MultipleQueriesInfo { +export interface QueriesInfo { loadedQuery: QueryInfo; connectionOnlyQueries: ConnectionOnlyQueryInfo[]; } diff --git a/src/utils/constants.ts b/src/utils/constants.ts index a04f96d..03831b2 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -162,7 +162,7 @@ export const elementAttributesValues = { export const defaults = { queryName: "Query1", - queryNamePrefix: "Query", + connectionOnlyQueryNamePrefix: "Connection only query-", sheetName: "Sheet1", columnName: "Column", }; diff --git a/src/utils/mashupDocumentParser.ts b/src/utils/mashupDocumentParser.ts index 151919b..50ab205 100644 --- a/src/utils/mashupDocumentParser.ts +++ b/src/utils/mashupDocumentParser.ts @@ -170,7 +170,7 @@ const addConnectionOnlyQueryMetadata = (metadataArray: Uint8Array, connectionOnl // parse metadataXml const metadataString: string = new TextDecoder("utf-8").decode(metadataXml); - const newMetadataString: string = updateConnectionOnlyMetadataStr(metadataString, connectionOnlyQueryNames); + const newMetadataString: string = addConnectionOnlyQueriesToMetadataStr(metadataString, connectionOnlyQueryNames); const encoder: TextEncoder = new TextEncoder(); const newMetadataXml: Uint8Array = encoder.encode(newMetadataString); const newMetadataXmlSize: Uint8Array = arrayUtils.getInt32Buffer(newMetadataXml.byteLength); @@ -184,7 +184,7 @@ const addConnectionOnlyQueryMetadata = (metadataArray: Uint8Array, connectionOnl return newMetadataArray; }; - const updateConnectionOnlyMetadataStr = (metadataString: string, connectionOnlyQueryNames: string[]) => { + const addConnectionOnlyQueriesToMetadataStr = (metadataString: string, connectionOnlyQueryNames: string[]) => { const parser: DOMParser = new DOMParser(); let metadataDoc: Document = parser.parseFromString(metadataString, xmlTextResultType); connectionOnlyQueryNames.forEach((queryName: string) => { diff --git a/src/utils/pqUtils.ts b/src/utils/pqUtils.ts index a717b54..d2634d8 100644 --- a/src/utils/pqUtils.ts +++ b/src/utils/pqUtils.ts @@ -143,12 +143,12 @@ const assignQueryNames = (queries: ConnectionOnlyQueryInfo[], loadedQueryName: s }; const generateUniqueQueryName = (queryNames: string[], loadedQueryName: string,): string => { - let queryName: string = defaults.queryName; - let index: number = 2; + let index: number = 1; + let queryName: string = defaults.connectionOnlyQueryNamePrefix + index++; const cleanedLoadedQueryName: string = loadedQueryName.trim().toLowerCase(); // Assumes that query names are lower case while (queryNames.includes(queryName.toLowerCase()) || queryName.toLowerCase() === cleanedLoadedQueryName) { - queryName = defaults.queryNamePrefix + index++; + queryName = defaults.connectionOnlyQueryNamePrefix + index++; } return queryName; diff --git a/src/utils/xmlPartsUtils.ts b/src/utils/xmlPartsUtils.ts index 62e98d0..9b08290 100644 --- a/src/utils/xmlPartsUtils.ts +++ b/src/utils/xmlPartsUtils.ts @@ -28,18 +28,20 @@ const updateWorkbookDataAndConfigurations = async (zip: JSZip, fileConfigs?: Fil await tableUtils.updateTableInitialDataIfNeeded(zip, tableData, updateQueryTable); }; -const updateWorkbookPowerQueryDocument = async (zip: JSZip, queryName: string, queryMashupDoc: string, connectionOnlyQueryNames?: string[]): Promise => { +const updateWorkbookPowerQueryDocument = async (zip: JSZip, loadedQueryName: string, queryMashupDoc: string, connectionOnlyQueryNames?: string[]): Promise => { const old_base64: string | undefined = await pqUtils.getBase64(zip); - if (!old_base64) { throw new Error(base64NotFoundErr); } - - const new_base64: string = await replaceSingleQuery(old_base64, queryName, queryMashupDoc); + // The mashupDoc contains a default query, we replace that query with the loaded query + const new_base64: string = await replaceSingleQuery(old_base64, loadedQueryName, queryMashupDoc); let updated_base64: string = new_base64; + + // If connection-only queries were given, add them to the mashupDoc if (connectionOnlyQueryNames && (connectionOnlyQueryNames.length > 0)) { updated_base64 = await addConnectionOnlyQueries(new_base64, connectionOnlyQueryNames); } + await pqUtils.setBase64(zip, updated_base64); }; diff --git a/src/workbookManager.ts b/src/workbookManager.ts index 3ac057b..7a2a302 100644 --- a/src/workbookManager.ts +++ b/src/workbookManager.ts @@ -13,7 +13,7 @@ import { tableNotFoundErr, templateFileNotSupportedErr, } from "./utils/constants"; -import { QueryInfo, TableData, Grid, FileConfigs, MultipleQueriesInfo } from "./types"; +import { QueryInfo, TableData, Grid, FileConfigs, QueriesInfo } from "./types"; import { generateMultipleQueryMashup, generateSingleQueryMashup } from "./generators"; export const generateSingleQueryWorkbook = async (query: QueryInfo, initialDataGrid?: Grid, fileConfigs?: FileConfigs): Promise => { @@ -40,7 +40,7 @@ export const generateSingleQueryWorkbook = async (query: QueryInfo, initialDataG return await generateSingleQueryWorkbookFromZip(zip, query, fileConfigs, tableData); }; -export const generateMultipleQueryWorkbook = async (queries: MultipleQueriesInfo, initialDataGrid?: Grid, fileConfigs?: FileConfigs): Promise => { +export const generateMultipleQueryWorkbook = async (queries: QueriesInfo, initialDataGrid?: Grid, fileConfigs?: FileConfigs): Promise => { const templateFile: File | undefined = fileConfigs?.templateFile; if (templateFile !== undefined && initialDataGrid !== undefined) { throw new Error(templateWithInitialDataErr); @@ -49,15 +49,13 @@ export const generateMultipleQueryWorkbook = async (queries: MultipleQueriesInfo if (!queries.loadedQuery.queryName) { queries.loadedQuery.queryName = defaults.queryName; } - - const zip: JSZip = - templateFile === undefined ? await JSZip.loadAsync(SIMPLE_QUERY_WORKBOOK_TEMPLATE, { base64: true }) : await JSZip.loadAsync(templateFile); - - const tableData = initialDataGrid ? gridUtils.parseToTableData(initialDataGrid) : undefined; - - const connectionOnlyQueryNames: string[] = pqUtils.validateMultipleQueryNames(queries.connectionOnlyQueries, queries.loadedQuery.queryName!); + pqUtils.validateQueryName(queries.loadedQuery.queryName!); + const connectionOnlyQueryNames: string[] = pqUtils.validateMultipleQueryNames(queries.connectionOnlyQueries, queries.loadedQuery.queryName!); pqUtils.assignQueryNames(queries.connectionOnlyQueries, queries.loadedQuery.queryName!, connectionOnlyQueryNames); + const zip: JSZip = + templateFile === undefined ? await JSZip.loadAsync(SIMPLE_QUERY_WORKBOOK_TEMPLATE, { base64: true }) : await JSZip.loadAsync(templateFile); + const tableData: TableData | undefined = initialDataGrid ? gridUtils.parseToTableData(initialDataGrid) : undefined; return await generateMultipleQueryWorkbookFromZip(zip, queries, fileConfigs, tableData); }; @@ -103,7 +101,7 @@ const generateSingleQueryWorkbookFromZip = async (zip: JSZip, query: QueryInfo, }); }; -const generateMultipleQueryWorkbookFromZip = async (zip: JSZip, queries: MultipleQueriesInfo, fileConfigs?: FileConfigs, tableData?: TableData): Promise => { +const generateMultipleQueryWorkbookFromZip = async (zip: JSZip, queries: QueriesInfo, fileConfigs?: FileConfigs, tableData?: TableData): Promise => { const connectionOnlyQueryNames: string[] = queries.connectionOnlyQueries.map((query) => query.queryName!); const mashupDocument: string = generateMultipleQueryMashup(queries.loadedQuery, queries.connectionOnlyQueries); await xmlPartsUtils.updateWorkbookPowerQueryDocument(zip, queries.loadedQuery.queryName!, mashupDocument, connectionOnlyQueryNames); From 9684355f35135d310e331d0082c45e4483b52d59 Mon Sep 17 00:00:00 2001 From: shanialbeck Date: Wed, 10 Jan 2024 01:03:35 -0800 Subject: [PATCH 11/13] small logic changes in mashupdoc editing --- src/utils/constants.ts | 9 +++++--- src/utils/mashupDocumentParser.ts | 37 ++++++++++++++++--------------- src/utils/xmlPartsUtils.ts | 20 ++++++++++++----- 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 03831b2..6ebca73 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -145,15 +145,18 @@ export const dataTypeKind = { boolean: "b", }; -export const itemPathTextContext = (queryName: string, isSource: boolean) => isSource ? `Section1/${queryName}/Source` : `Section1/${queryName}`; +export const powerQueryResultType = { + table: "sTable", + connectionOnly: "sConnectionOnly", +}; + +export const itemPathTextContext = (queryName: string, isSource: boolean) => isSource ? `Section1/${encodeURIComponent(queryName)}/Source` : `Section1/${queryName}`; export const elementAttributesValues = { connectionName: (queryName: string) => `Query - ${queryName}`, connectionDescription: (queryName: string) => `Connection to the '${queryName}' query in the workbook.`, connection: (queryName: string) => `Provider=Microsoft.Mashup.OleDb.1;Data Source=$Workbook$;Location="${queryName}";`, connectionCommand: (queryName: string) => `SELECT * FROM [${queryName}]`, - tableResultType: () => "sTable", - connectionOnlyResultType: () => "sConnectionOnly", fillStatusComplete: () => "sComplete", fillErrorCodeUnknown: () => "sUnknown", randomizedUid: () => "{" + v4().toUpperCase() + "}", diff --git a/src/utils/mashupDocumentParser.ts b/src/utils/mashupDocumentParser.ts index 50ab205..5a2dca8 100644 --- a/src/utils/mashupDocumentParser.ts +++ b/src/utils/mashupDocumentParser.ts @@ -17,6 +17,7 @@ import { elementAttributes, elementAttributesValues, itemPathTextContext, + powerQueryResultType, } from "./constants"; import { arrayUtils } from "."; import { Metadata } from "../types"; @@ -140,7 +141,7 @@ export const editSingleQueryMetadata = (metadataArray: Uint8Array, metadata: Met return prop?.name === elementAttributes.type; }); if (entryProp?.nodeValue == elementAttributes.resultType) { - entry.setAttribute(elementAttributes.value, elementAttributesValues.tableResultType()); + entry.setAttribute(elementAttributes.value, powerQueryResultType.table); } if (entryProp?.nodeValue == elementAttributes.fillLastUpdated) { @@ -184,19 +185,19 @@ const addConnectionOnlyQueryMetadata = (metadataArray: Uint8Array, connectionOnl return newMetadataArray; }; - const addConnectionOnlyQueriesToMetadataStr = (metadataString: string, connectionOnlyQueryNames: string[]) => { - const parser: DOMParser = new DOMParser(); - let metadataDoc: Document = parser.parseFromString(metadataString, xmlTextResultType); - connectionOnlyQueryNames.forEach((queryName: string) => { - const items: Element = metadataDoc.getElementsByTagName(element.items)[0]; - const stableEntriesItem: Element = createStableEntriesItem(metadataDoc, queryName); - items.appendChild(stableEntriesItem); - const sourceItem: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.item); - sourceItem.appendChild(createItemLocation(metadataDoc, queryName, true)); - const stableEntries: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.stableEntries); - sourceItem.appendChild(stableEntries); - items.appendChild(sourceItem); - }); +export const addConnectionOnlyQueriesToMetadataStr = (metadataString: string, connectionOnlyQueryNames: string[]) => { + const parser: DOMParser = new DOMParser(); + let metadataDoc: Document = parser.parseFromString(metadataString, xmlTextResultType); + connectionOnlyQueryNames.forEach((queryName: string) => { + const items: Element = metadataDoc.getElementsByTagName(element.items)[0]; + const stableEntriesItem: Element = createStableEntriesItem(metadataDoc, queryName); + items.appendChild(stableEntriesItem); + const sourceItem: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.item); + sourceItem.appendChild(createItemLocation(metadataDoc, queryName, true)); + const stableEntries: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.stableEntries); + sourceItem.appendChild(stableEntries); + items.appendChild(sourceItem); + }); const updatedMetdataString: string = new XMLSerializer().serializeToString(metadataDoc); @@ -218,7 +219,7 @@ const addConnectionOnlyQueryMetadata = (metadataArray: Uint8Array, connectionOnl const createStableEntriesItem = (metadataDoc: Document, queryName: string) => { const newItem: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.item); newItem.appendChild(createItemLocation(metadataDoc, queryName, false)); - const stableEntries: Element = createConnectionOnlyEntries(metadataDoc); + const stableEntries: Element = createEntries(metadataDoc, powerQueryResultType.connectionOnly); newItem.appendChild(stableEntries); return newItem; @@ -232,16 +233,16 @@ const addConnectionOnlyQueryMetadata = (metadataArray: Uint8Array, connectionOnl return elementObject; }; - const createConnectionOnlyEntries = (metadataDoc: Document) => { + const createEntries = (metadataDoc: Document, fillObjectType: string) => { const stableEntries: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.stableEntries); const nowTime: string = new Date().toISOString(); stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.isPrivate, "l0")); stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillEnabled, "l0")); - stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillObjectType, elementAttributesValues.connectionOnlyResultType())); + stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillObjectType, fillObjectType)); stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillToDataModelEnabled, "l0")); stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillLastUpdated, (elementAttributes.day + nowTime).replace(/Z/, "0000Z"))); - stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.resultType, elementAttributesValues.tableResultType())); + stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.resultType, powerQueryResultType.table)); stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.filledCompleteResultToWorksheet, "l0")); stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.addedToDataModel, "l0")); stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillErrorCode, elementAttributesValues.fillErrorCodeUnknown())); diff --git a/src/utils/xmlPartsUtils.ts b/src/utils/xmlPartsUtils.ts index 9b08290..fdf3041 100644 --- a/src/utils/xmlPartsUtils.ts +++ b/src/utils/xmlPartsUtils.ts @@ -34,17 +34,25 @@ const updateWorkbookPowerQueryDocument = async (zip: JSZip, loadedQueryName: str throw new Error(base64NotFoundErr); } // The mashupDoc contains a default query, we replace that query with the loaded query - const new_base64: string = await replaceSingleQuery(old_base64, loadedQueryName, queryMashupDoc); - let updated_base64: string = new_base64; - + let updated_base64: string = await replaceSingleQuery(old_base64, loadedQueryName, queryMashupDoc); + // If connection-only queries were given, add them to the mashupDoc - if (connectionOnlyQueryNames && (connectionOnlyQueryNames.length > 0)) { - updated_base64 = await addConnectionOnlyQueries(new_base64, connectionOnlyQueryNames); - } + updated_base64 = await addConnectionOnlyQueriesIfNeeded(updated_base64, connectionOnlyQueryNames); await pqUtils.setBase64(zip, updated_base64); }; +const addConnectionOnlyQueriesIfNeeded = async(base64: string, connectionOnlyQueryNames?:string[]):Promise => { + if (!connectionOnlyQueryNames || (connectionOnlyQueryNames.length == 0)) + { + return base64; + } + + return await addConnectionOnlyQueries(base64, connectionOnlyQueryNames); +}; + + + const updateWorkbookSingleQueryAttributes = async (zip: JSZip, queryName: string, refreshOnOpen: boolean): Promise => { // Update connections const connectionsXmlString: string | undefined = await zip.file(connectionsXmlPath)?.async(textResultType); From 85808740c2a551e8975f2cd20a6d3456f18d611d Mon Sep 17 00:00:00 2001 From: shanialbeck Date: Mon, 22 Jan 2024 02:28:15 -0800 Subject: [PATCH 12/13] add testing --- tests/mashupDocumentParser.test.ts | 21 +++++++++++++- tests/mocks/xmlMocks.ts | 10 +++++++ tests/pqUtils.test.ts | 44 ++++++++++++++++++++++++++++++ tests/xmlInnerPartsUtils.test.ts | 9 ++++++ 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 tests/pqUtils.test.ts diff --git a/tests/mashupDocumentParser.test.ts b/tests/mashupDocumentParser.test.ts index e483f98..8a1e832 100644 --- a/tests/mashupDocumentParser.test.ts +++ b/tests/mashupDocumentParser.test.ts @@ -2,7 +2,7 @@ // Licensed under the MIT license. import { TextDecoder, TextEncoder } from "util"; -import { replaceSingleQuery, getPackageComponents, editSingleQueryMetadata } from "../src/utils/mashupDocumentParser"; +import { replaceSingleQuery, getPackageComponents, editSingleQueryMetadata, addConnectionOnlyQueriesToMetadataStr } from "../src/utils/mashupDocumentParser"; import { arrayUtils, pqUtils } from "../src/utils"; import { section1mNewQueryNameSimpleMock, pqMetadataXmlMockPart1, pqMetadataXmlMockPart2 } from "./mocks"; import base64 from "base64-js"; @@ -11,6 +11,7 @@ import { SIMPLE_QUERY_WORKBOOK_TEMPLATE } from "../src/workbookTemplate"; import { section1mPath } from "../src/utils/constants"; import util from "util"; +import { pqSingleQueryMetadataXmlMock, pqConnectionOnlyMetadataXmlMockPart1, pqConnectionOnlyMetadataXmlMockPart2 } from "./mocks/xmlMocks"; (global as any).TextDecoder = TextDecoder; (global as any).TextEncoder = TextEncoder; @@ -47,4 +48,22 @@ describe("Mashup Document Parser tests", () => { expect(metadataString.replace(/ /g, "")).toContain(pqMetadataXmlMockPart2.replace(/ /g, "")); } }); + + test("Power Query Multiple Queries MetadataXml test", async () => { + const metadataStr: string = pqSingleQueryMetadataXmlMock; + const newMetadataString: string = addConnectionOnlyQueriesToMetadataStr(metadataStr, ["Query2", "Query3"]); + expect(newMetadataString.replace(/ /g, "")).toContain(pqConnectionOnlyMetadataXmlMockPart1("Query2").replace(/ /g, "")); + expect(newMetadataString.replace(/ /g, "")).toContain(pqConnectionOnlyMetadataXmlMockPart2("Query2").replace(/ /g, "")); + expect(newMetadataString.replace(/ /g, "")).toContain(pqConnectionOnlyMetadataXmlMockPart1("Query3").replace(/ /g, "")); + expect(newMetadataString.replace(/ /g, "")).toContain(pqConnectionOnlyMetadataXmlMockPart2("Query3").replace(/ /g, "")); + // checks that there are exactly 7 tags in the metadata + expect((newMetadataString.replace(/ /g, "").match(//g) || []).length).toEqual(7); + + }); + + test("Power Query Add Empty ConnectionOnly Array test", async () => { + const metadataStr: string = pqSingleQueryMetadataXmlMock; + const newMetadataStr: string = addConnectionOnlyQueriesToMetadataStr(metadataStr, []); + expect(newMetadataStr.replace(/ /g, "")).toEqual(pqSingleQueryMetadataXmlMock.replace(/ /g, "")); + }); }); diff --git a/tests/mocks/xmlMocks.ts b/tests/mocks/xmlMocks.ts index e6aff56..d22e98b 100644 --- a/tests/mocks/xmlMocks.ts +++ b/tests/mocks/xmlMocks.ts @@ -15,3 +15,13 @@ export const pqMetadataXmlMockPart1 = ' AllFormulas Formula Section1/newQueryName '; export const pqMetadataXmlMockPart2 = ' Formula Section1/newQueryName/Source '; + + export const pqConnectionOnlyMetadataXmlMockPart1 = (queryName: string) => { + return 'FormulaSection1/'+ queryName + ''; +}; + +export const pqConnectionOnlyMetadataXmlMockPart2 = (queryName: string) => { + return 'FormulaSection1/'+ queryName + '/Source'; +}; + +export const pqSingleQueryMetadataXmlMock = `AllFormulasFormulaSection1/Query1FormulaSection1/Query1/Source`; diff --git a/tests/pqUtils.test.ts b/tests/pqUtils.test.ts new file mode 100644 index 0000000..e142fd1 --- /dev/null +++ b/tests/pqUtils.test.ts @@ -0,0 +1,44 @@ +import { pqUtils } from "../src/utils/"; + +describe("Pq Utils tests", () => { + test("tests that validation fails when non unique query names are given", () => { + try { + pqUtils.validateMultipleQueryNames([{ queryName: " QuErY2 ", queryMashup: "" }, { queryName: "queRy2 ", queryMashup: "" }], "Query1"); + // If the above line doesn't throw an error, the test fails + expect(true).toEqual(false); + } catch (e) { + expect(e.message).toEqual("Queries must have unique names"); + } + try { + pqUtils.validateMultipleQueryNames([{ queryName: " qUeRy1 ", queryMashup: "" }, { queryName: "Query2", queryMashup: "" }], " QuERy1 "); + // If the above line doesn't throw an error, the test fails + expect(true).toEqual(false); + } catch (e) { + expect(e.message).toEqual("Queries must have unique names"); + } + }); + + test("tests that validation succeeds when valid unique query names are given", () => { + try { + pqUtils.validateMultipleQueryNames([{ queryName: "Query 1", queryMashup: "" }, { queryName: "Query1", queryMashup: "" }], "Query2"); + expect(true).toEqual(true); + } catch (e) { + // If the above line throws an error, the test fails + expect(true).toEqual(false); + } + try { + pqUtils.validateMultipleQueryNames([{ queryName: "Query 1", queryMashup: "" }, { queryName: "Query1", queryMashup: "" }], "Query 1"); + expect(true).toEqual(true); + } catch (e) { + // If the above line throws an error, the test fails + expect(true).toEqual(false); + } + }); + + test("tests generated query name", () => { + expect(pqUtils.generateUniqueQueryName(["connection only query-1", "connection only query-2", "connection only query-4"], " Connection only query-3 ")).toEqual("Connection only query-5"); + expect(pqUtils.generateUniqueQueryName(["connection only query-1", "connection only query-2", "connection only query-3"], "connection only query -4")).toEqual("Connection only query-4"); + expect(pqUtils.generateUniqueQueryName(["Connection only query - 1", "connection only query-2", "connection only query-3"], "connection only query-4")).toEqual("Connection only query-1"); + }); + +}); \ No newline at end of file diff --git a/tests/xmlInnerPartsUtils.test.ts b/tests/xmlInnerPartsUtils.test.ts index 94ecc75..5e64087 100644 --- a/tests/xmlInnerPartsUtils.test.ts +++ b/tests/xmlInnerPartsUtils.test.ts @@ -61,4 +61,13 @@ describe("Workbook Manager tests", () => { expect(sharedStringIndex).toEqual(2); expect(newSharedStrings.replace(/ /g, "")).toContain(sharedStringsXmlMock.replace(/ /g, "")); }); + + test("Connections XML contains new connection", async () => { + const serializer = new XMLSerializer(); + const mockXml = new DOMParser().parseFromString(mockConnectionString, "text/xml"); + const newConnectionsXml: Document = await xmlInnerPartsUtils.addNewConnection(mockXml, "newQueryName"); + const newConnectionsXmlString = serializer.serializeToString(newConnectionsXml); + expect((newConnectionsXmlString.match(/ Date: Tue, 30 Jan 2024 01:25:51 -0800 Subject: [PATCH 13/13] small fixes --- src/generators.ts | 13 +++++++++++-- src/utils/constants.ts | 1 + src/utils/mashupDocumentParser.ts | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/generators.ts b/src/generators.ts index c7c8490..400368b 100644 --- a/src/generators.ts +++ b/src/generators.ts @@ -2,6 +2,7 @@ // Licensed under the MIT license. import { ConnectionOnlyQueryInfo, QueryInfo } from "./types"; +import { missingQueryNameErr } from "./utils/constants"; export const generateMashupXMLTemplate = (base64: string): string => `${base64}`; @@ -13,9 +14,17 @@ export const generateSingleQueryMashup = (queryName: string, query: string): str ${query};`; export const generateMultipleQueryMashup = (loadedQuery: QueryInfo, queries: ConnectionOnlyQueryInfo[]): string => { - let mashup: string = generateSingleQueryMashup(loadedQuery.queryName!, loadedQuery.queryMashup!); + if (!loadedQuery.queryName) { + throw new Error(missingQueryNameErr); + } + + let mashup: string = generateSingleQueryMashup(loadedQuery.queryName, loadedQuery.queryMashup); queries.forEach((query: ConnectionOnlyQueryInfo) => { - const queryName = query.queryName ? query.queryName : "Query" + queries.indexOf(query); + const queryName = query.queryName; + if (!queryName) { + throw new Error(missingQueryNameErr); + } + mashup += ` shared #"${queryName}" = diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 6ebca73..958f3dc 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -43,6 +43,7 @@ export const arrayIsntMxNErr = "Array isn't MxN"; export const templateFileNotSupportedErr = "Template file is not supported for this API call"; export const relsNotFoundErr = ".rels were not found in template"; export const queryNameAlreadyExistsErr = "Queries must have unique names"; +export const missingQueryNameErr = "Query name is missing"; export const blobFileType = "blob"; export const uint8ArrayType = "uint8array"; diff --git a/src/utils/mashupDocumentParser.ts b/src/utils/mashupDocumentParser.ts index 5a2dca8..7d13ca1 100644 --- a/src/utils/mashupDocumentParser.ts +++ b/src/utils/mashupDocumentParser.ts @@ -46,7 +46,7 @@ export const replaceSingleQuery = async (base64Str: string, queryName: string, q export const addConnectionOnlyQueries = async (base64Str: string, connectionOnlyQueryNames: string[]): Promise => { var { version, packageOPC, permissionsSize, permissions, metadata, endBuffer } = getPackageComponents(base64Str); - const newMetadataBuffer: Uint8Array = addConnectionOnlyQueryMetadata(metadata, connectionOnlyQueryNames); + const newMetadataBuffer: Uint8Array = addConnectionOnlyQuerieMetadata(metadata, connectionOnlyQueryNames); const newMashup: Uint8Array = arrayUtils.concatArrays(version, arrayUtils.getInt32Buffer(packageOPC.byteLength), packageOPC, arrayUtils.getInt32Buffer(permissionsSize), permissions, arrayUtils.getInt32Buffer(newMetadataBuffer.byteLength), newMetadataBuffer, endBuffer); return base64.fromByteArray(newMashup); @@ -161,7 +161,7 @@ export const editSingleQueryMetadata = (metadataArray: Uint8Array, metadata: Met return newMetadataArray; }; -const addConnectionOnlyQueryMetadata = (metadataArray: Uint8Array, connectionOnlyQueryNames: string[]) => { +const addConnectionOnlyQuerieMetadata = (metadataArray: Uint8Array, connectionOnlyQueryNames: string[]) => { // extract metadataXml const mashupArray: ArrayReader = new arrayUtils.ArrayReader(metadataArray.buffer); const metadataVersion: Uint8Array = mashupArray.getBytes(4);