From ffa12ea83c7f938ee3e50d16f2be2337de8d5e91 Mon Sep 17 00:00:00 2001 From: Michael Gartner Date: Tue, 19 Nov 2024 21:14:53 -0600 Subject: [PATCH 1/6] async.q and bypassing getDatalogQuery() --- src/utils/fireQuery.ts | 144 +++++++++++++------- src/utils/getDiscourseContextResults.ts | 168 ++++++++++++++++-------- src/utils/getDiscourseRelations.ts | 5 + 3 files changed, 213 insertions(+), 104 deletions(-) diff --git a/src/utils/fireQuery.ts b/src/utils/fireQuery.ts index 5ba742ba..3e0706b8 100644 --- a/src/utils/fireQuery.ts +++ b/src/utils/fireQuery.ts @@ -27,6 +27,12 @@ type RelationInQuery = { text: string; isComplement: boolean; }; +type QuerySelection = { + mapper: PredefinedSelection["mapper"]; + pull: string; + label: string; + key: string; +}; export type FireQueryArgs = QueryArgs & { isSamePageEnabled?: boolean; isCustomEnabled?: boolean; @@ -36,6 +42,7 @@ export type FireQueryArgs = QueryArgs & { customNodes?: DiscourseNode[]; customRelations?: DiscourseRelation[]; }; + definedSelections?: QuerySelection[]; }; type FireQuery = (query: FireQueryArgs) => Promise; @@ -184,6 +191,69 @@ const getConditionTargets = (conditions: Condition[]): string[] => : getConditionTargets(c.conditions.flat()) ); +type FormatResultFn = (result: unknown[]) => Promise; +type FormattedOutput = { + output: string | number | Record | Date; + label: string; +}; + +const createFormatResultFn = ( + definedSelections: QuerySelection[] +): FormatResultFn => { + const formatSingleResult = async ( + pullResult: unknown, + selection: QuerySelection, + prev: QueryResult + ): Promise => { + if (typeof pullResult === "object" && pullResult !== null) { + const output = await selection.mapper( + pullResult as PullBlock, + selection.key, + prev + ); + return { output, label: selection.label }; + } + + if (typeof pullResult === "string" || typeof pullResult === "number") { + return { output: pullResult, label: selection.label }; + } + + return { output: "", label: selection.label }; + }; + + const applyOutputToResult = ( + result: QueryResult, + { output, label }: FormattedOutput + ): void => { + if (typeof output === "object" && !(output instanceof Date)) { + Object.entries(output as Record).forEach(([k, v]) => { + result[label + k] = String(v); + }); + } else if (typeof output === "number") { + result[label] = output.toString(); + } else { + result[label] = String(output); + } + }; + + return async (results: unknown[]): Promise => { + const formatters = definedSelections.map( + (selection, i) => (prev: QueryResult) => + formatSingleResult(results[i], selection, prev) + ); + + return formatters.reduce( + (prev, formatter) => + prev.then(async (p) => { + const output = await formatter(p); + applyOutputToResult(p, output); + return p; + }), + Promise.resolve({} as QueryResult) + ); + }; +}; + export const getDatalogQuery = ({ conditions, selections, @@ -258,39 +328,7 @@ export const getDatalogQuery = ({ ? `[?node :block/uid _]` : "" }${where}\n]`, - formatResult: (result: unknown[]) => - definedSelections - .map((c, i) => (prev: QueryResult) => { - const pullResult = result[i]; - return typeof pullResult === "object" && pullResult !== null - ? Promise.resolve( - c.mapper(pullResult as PullBlock, c.key, prev) - ).then((output) => ({ - output, - label: c.label, - })) - : typeof pullResult === "string" || typeof pullResult === "number" - ? Promise.resolve({ output: pullResult, label: c.label }) - : Promise.resolve({ output: "", label: c.label }); - }) - .reduce( - (prev, c) => - prev.then((p) => - c(p).then(({ output, label }) => { - if (typeof output === "object" && !(output instanceof Date)) { - Object.entries(output).forEach(([k, v]) => { - p[label + k] = v; - }); - } else if (typeof output === "number") { - p[label] = output.toString(); - } else { - p[label] = output; - } - return p; - }) - ), - Promise.resolve({} as QueryResult) - ), + formatResult: createFormatResultFn(definedSelections), inputs: expectedInputs.map((i) => inputs[i]), }; }; @@ -304,7 +342,13 @@ export const fireQuerySync = (args: FireQueryArgs): QueryResult[] => { }; const fireQuery: FireQuery = async (_args) => { - const { isCustomEnabled, customNode, isSamePageEnabled, ...args } = _args; + const { + isCustomEnabled, + customNode, + isSamePageEnabled, + definedSelections, + ...args + } = _args; if (isSamePageEnabled) { return getSamePageAPI() .then((api) => api.postToAppBackend({ path: "query", data: { ...args } })) @@ -318,18 +362,20 @@ const fireQuery: FireQuery = async (_args) => { const { query, formatResult, inputs } = isCustomEnabled ? { query: customNode as string, - formatResult: (r: unknown[]): Promise => - Promise.resolve({ - text: "", - uid: "", - ...Object.fromEntries( - r.flatMap((p, index) => - typeof p === "object" && p !== null - ? Object.entries(p) - : [[index.toString(), p]] - ) - ), - }), + formatResult: definedSelections + ? createFormatResultFn(definedSelections) + : (r: unknown[]): Promise => + Promise.resolve({ + text: "", + uid: "", + ...Object.fromEntries( + r.flatMap((p, index) => + typeof p === "object" && p !== null + ? Object.entries(p) + : [[index.toString(), p]] + ) + ), + }), inputs: [], } : getDatalogQuery(args); @@ -339,9 +385,11 @@ const fireQuery: FireQuery = async (_args) => { console.log(query); if (inputs.length) console.log("Inputs:", ...inputs); } - return Promise.all( - window.roamAlphaAPI.data.fast.q(query, ...inputs).map(formatResult) + const queryResults = await window.roamAlphaAPI.data.async.q( + query, + ...inputs ); + return Promise.all(queryResults.map(formatResult)); } catch (e) { console.error("Error from Roam:"); console.error((e as Error).message); diff --git a/src/utils/getDiscourseContextResults.ts b/src/utils/getDiscourseContextResults.ts index 97679d13..cf8a2bc8 100644 --- a/src/utils/getDiscourseContextResults.ts +++ b/src/utils/getDiscourseContextResults.ts @@ -7,6 +7,7 @@ import getDiscourseRelations, { DiscourseRelation, } from "./getDiscourseRelations"; import { OnloadArgs } from "roamjs-components/types"; +import { Selection } from "./types"; const resultCache: Record>> = {}; const CACHE_TIMEOUT = 1000 * 60 * 5; @@ -46,85 +47,140 @@ const getDiscourseContextResults = async ({ queries.push({ r, complement: false, + query: r.query, }); } if (r.destination === nodeType || r.destination === "*") { queries.push({ r, complement: true, + query: r.complementQuery, }); } return queries; }) - .map(({ r, complement: isComplement }) => { + .map(({ r, complement: isComplement, query }) => { const target = isComplement ? r.source : r.destination; const text = isComplement ? r.complement : r.label; const returnNode = nodeTextByType[target]; const cacheKey = `${uid}~${text}~${target}`; const conditionUid = window.roamAlphaAPI.util.generateUID(); - const selections = []; - if (r.triples.some((t) => t.some((a) => /context/i.test(a)))) { - selections.push({ - uid: window.roamAlphaAPI.util.generateUID(), - label: "context", - text: `node:${conditionUid}-Context`, - }); - } else if (r.triples.some((t) => t.some((a) => /anchor/i.test(a)))) { - selections.push({ - uid: window.roamAlphaAPI.util.generateUID(), - label: "anchor", - text: `node:${conditionUid}-Anchor`, - }); - } + const selections: Selection[] = []; + + // TODO - not currently supported + // we are bypassing definedSelections creation + + // if (r.triples.some((t) => t.some((a) => /context/i.test(a)))) { + // selections.push({ + // uid: window.roamAlphaAPI.util.generateUID(), + // label: "context", + // text: `node:${conditionUid}-Context`, + // }); + // } else if (r.triples.some((t) => t.some((a) => /anchor/i.test(a)))) { + // selections.push({ + // uid: window.roamAlphaAPI.util.generateUID(), + // label: "anchor", + // text: `node:${conditionUid}-Anchor`, + // }); + // } + + const definedSelections = [ + { + key: "", + label: "text", + pull: `(pull ?${returnNode} [:block/string :node/title :block/uid])`, + mapper: (r: any) => { + return { + "": r?.[":node/title"] || r?.[":block/string"] || "", + "-uid": r[":block/uid"] || "", + }; + }, + }, + { + key: "", + label: "uid", + pull: `(pull ?${returnNode} [:block/uid])`, + mapper: (r: any) => { + return r?.[":block/uid"] || ""; + }, + }, + ]; const relation = { id: r.id, text, target, isComplement, }; - const rawResults = - resultCache[cacheKey] && !ignoreCache - ? Promise.resolve(resultCache[cacheKey]) - : fireQuery({ - returnNode, - conditions: [ - { - source: returnNode, - // NOTE! This MUST be the OPPOSITE of `label` - relation: isComplement ? r.label : r.complement, - target: uid, - uid: conditionUid, - type: "clause", - }, - ], - selections, - isSamePageEnabled, - context: { - relationsInQuery: [relation], - customNodes: nodes, - customRelations: relations, + + if (resultCache[cacheKey] && !ignoreCache) { + return { + relation, + queryPromise: Promise.resolve(resultCache[cacheKey]), + }; + } + + if (query) { + return { + relation: { + text, + isComplement, + target, + id: r.id, + }, + queryPromise: fireQuery({ + customNode: query.replaceAll("{{placeholder}}", uid), + isCustomEnabled: true, + conditions: [], + selections, + definedSelections, + isSamePageEnabled, + }), + }; + } else { + return { + relation: { + text, + isComplement, + target, + id: r.id, + }, + queryPromise: fireQuery({ + returnNode, + conditions: [ + { + source: returnNode, + // NOTE! This MUST be the OPPOSITE of `label` + relation: isComplement ? r.label : r.complement, + target: uid, + uid: conditionUid, + type: "clause", }, - }).then((results) => { - resultCache[cacheKey] = results; - setTimeout(() => { - delete resultCache[cacheKey]; - }, CACHE_TIMEOUT); - return results; - }); - return rawResults.then((results) => ({ - relation: { - text, - isComplement, - target, - id: r.id, - }, - results, - })); + ], + selections, + isSamePageEnabled, + definedSelections, + context: { + relationsInQuery: [relation], + customNodes: nodes, + customRelations: relations, + }, + }), + }; + } }) - ).catch((e) => { - console.error(e); - return [] as const; - }); + ) + .then((items) => { + return Promise.all( + items.map(async ({ relation, queryPromise }) => ({ + relation, + results: await queryPromise, + })) + ); + }) + .catch((e) => { + console.error(e); + return [] as const; + }); const groupedResults = Object.fromEntries( resultsWithRelation.map((r) => [ r.relation.text, diff --git a/src/utils/getDiscourseRelations.ts b/src/utils/getDiscourseRelations.ts index 4977ffc9..540ac2f5 100644 --- a/src/utils/getDiscourseRelations.ts +++ b/src/utils/getDiscourseRelations.ts @@ -30,6 +30,11 @@ const getDiscourseRelations = () => { source: getSettingValueFromTree({ tree, key: "Source" }), destination: getSettingValueFromTree({ tree, key: "Destination" }), complement: getSettingValueFromTree({ tree, key: "Complement" }), + query: getSettingValueFromTree({ tree, key: "query" }), + complementQuery: getSettingValueFromTree({ + tree, + key: "queryComplement", + }), }; const ifNode = tree.find(matchNodeText("if"))?.children || []; return ifNode.map((node) => ({ From 0d32c068b7e27edb8b322d90f202ee4923cc583e Mon Sep 17 00:00:00 2001 From: Michael Gartner Date: Thu, 21 Nov 2024 16:23:52 -0600 Subject: [PATCH 2/6] Query Tester --- src/components/QueryTester.tsx | 528 +++++++++++++++++++++++++++++++++ src/index.ts | 11 + src/utils/mockQuery.ts | 89 ++++++ 3 files changed, 628 insertions(+) create mode 100644 src/components/QueryTester.tsx create mode 100644 src/utils/mockQuery.ts diff --git a/src/components/QueryTester.tsx b/src/components/QueryTester.tsx new file mode 100644 index 00000000..7d48a81b --- /dev/null +++ b/src/components/QueryTester.tsx @@ -0,0 +1,528 @@ +import { + Button, + Classes, + Dialog, + Intent, + RadioGroup, + Radio, + FormGroup, + NumericInput, +} from "@blueprintjs/core"; +import React, { + useState, + useEffect, + useMemo, + useCallback, + useRef, +} from "react"; +import getCurrentPageUid from "roamjs-components/dom/getCurrentPageUid"; +import getBasicTreeByParentUid from "roamjs-components/queries/getBasicTreeByParentUid"; +import renderOverlay from "roamjs-components/util/renderOverlay"; +import createBlock from "roamjs-components/writes/createBlock"; + +type QueryTesterProps = { + onClose: () => void; + isOpen: boolean; +}; + +type QueryType = { + label: string; + description: string; + fn: () => Promise; +}; + +const getTimestamp = () => { + const timestamp = new Date(); + const minutes = String(timestamp.getMinutes()).padStart(2, "0"); + const seconds = String(timestamp.getSeconds()).padStart(2, "0"); + const milliseconds = String(timestamp.getMilliseconds()).padStart(3, "0"); + return `${minutes}:${seconds}:${milliseconds}`; +}; +const getRandomTimestamp = () => { + const now = Date.now(); + const thirteenMonthsAgo = now - 13 * 30 * 24 * 60 * 60 * 1000; + const fifteenMonthsAgo = now - 15 * 30 * 24 * 60 * 60 * 1000; + + return Math.floor( + Math.random() * (thirteenMonthsAgo - fifteenMonthsAgo) + fifteenMonthsAgo + ); +}; +const fakeBackendQuery = (id: number, delayTime: number) => { + return new Promise((resolve) => { + setTimeout(() => { + resolve([id]); + }, delayTime); + }); +}; +const artificialGetDatalogQueryTime = (buildTime: number) => { + const startTime = Date.now(); + while (Date.now() - startTime < buildTime) { + // Simulate work by blocking the thread + } +}; +const queryTypes = ["EVD", "CLM", "RES", "HYP", "ISS", "CON"]; +const baseQuery = `[:find + (pull ?node [:block/string :node/title :block/uid]) + (pull ?node [:block/uid]) + :where + [(re-pattern "^\\\\[\\\\[TYPE\\\\]\\\\] - (.*?)$") ?QUE-.*?$-regex] + [?node :node/title ?Question-Title] + [?node :block/children ?Summary] + [?node :block/children ?Workbench] + [?Workbench :block/children ?Notes] + [?Notes :block/children ?childNotes] + [?node :create/time ?node-CreateTime] + [(< ${getRandomTimestamp()} ?node-CreateTime)] + (or [?Summary :block/string ?Summary-String] + [?Summary :node/title ?Summary-String]) + (not + [?Summary :block/children ?childSummary] + ) + (or [?Notes :block/string ?Notes-String] + [?Notes :node/title ?Notes-String]) + [(re-find ?QUE-.*?$-regex ?Question-Title)] + [(clojure.string/includes? ?Summary-String "Summary")] + [(clojure.string/includes? ?Notes-String "Notes")] + ]`; + +const CellEmbed = ({ + selectedQuery, + queryBlockUid, +}: { + selectedQuery: number; + queryBlockUid: string; +}) => { + const contentRef = useRef(null); + + const tree = useMemo( + () => getBasicTreeByParentUid(queryBlockUid), + [queryBlockUid] + ); + useEffect(() => { + const container = contentRef.current; + const uid = tree[selectedQuery].uid; + + if (container && uid) { + const existingBlock = container.querySelector(".roam-block-container"); + if (existingBlock) { + existingBlock.remove(); + } + + const blockEl = document.createElement("div"); + blockEl.className = "roam-block-container"; + container.appendChild(blockEl); + + window.roamAlphaAPI.ui.components.renderBlock({ + uid, + el: blockEl, + }); + } + + return () => { + if (container) { + const existingBlock = container.querySelector(".roam-block-container"); + if (existingBlock) { + existingBlock.remove(); + } + } + }; + }, [tree, selectedQuery, queryBlockUid]); + + return ( +
+
+
+ ); +}; + +const QueryTester = ({ onClose, isOpen }: QueryTesterProps) => { + const [selectedQuery, setSelectedQuery] = useState(0); + const [isRunning, setIsRunning] = useState(false); + const [buildTime, setBuildTime] = useState(3000); + const [delayTime, setDelayTime] = useState(3000); + const [queryBlockUid, setQueryBlockUid] = useState(null); + + // lol couldn't get highlighting to work properly, so creating the blocks and rending them + useEffect(() => { + const createQueryBlock = async () => { + const currentPageUid = getCurrentPageUid(); + const newUid = await createBlock({ + node: { + open: false, + text: "", + children: [ + ...queries.map((query, i) => ({ + text: `\`\`\`const ${query.label.replace(/\s+/g, "")} = ${query.fn.toString()} \`\`\``, + })), + ], + }, + parentUid: currentPageUid, + }); + setQueryBlockUid(newUid); + return () => { + window.roamAlphaAPI.deleteBlock({ + block: { uid: newUid }, + }); + }; + }; + createQueryBlock(); + }, []); + const asyncQWithDelay = useCallback(async () => { + console.log("async.q: Promise.all(map(async) => await fireQuery)"); + console.log("with artificial query delay"); + console.log(`buildTime: no getDatalogQueryTime`); + console.log(`delayTime: ${delayTime}`); + + const fireQueryX = async (type: string, i: number) => { + console.log(`🔎🟢`, getTimestamp(), `Query`, type, i); + + // Artifical Query Delay + await new Promise((resolve) => setTimeout(resolve, delayTime)); + + const queryResults = await window.roamAlphaAPI.data.async.q( + baseQuery.replace("TYPE", type) + ); + console.log(`🔎🛑`, getTimestamp(), `Query`, type, i); + return { type, results: queryResults }; + }; + + // adding async/await results in the same behavior + Promise.all(queryTypes.map((type, i) => fireQueryX(type, i))); + + // Results + // + // These results makes sense ✅ + // queries sent at 32:079 + // delay is 3 seconds + // queries ends between 35:581 - 35:774 + // difference is actual query time, server queues, and network time + // + // 🔎🟢 50:32:078 Query EVD 0 + // 🔎🟢 50:32:079 Query CLM 1 + // 🔎🟢 50:32:079 Query RES 2 + // 🔎🟢 50:32:079 Query HYP 3 + // 🔎🟢 50:32:079 Query ISS 4 + // 🔎🟢 50:32:079 Query CON 5 + // 🔎🛑 50:35:581 Query HYP 3 + // 🔎🛑 50:35:686 Query CON 5 + // 🔎🛑 50:35:760 Query ISS 4 + // 🔎🛑 50:35:770 Query EVD 0 + // 🔎🛑 50:35:771 Query RES 2 + // 🔎🛑 50:35:774 Query CLM 1 + }, [delayTime]); + const asyncQWithDelayAndBuildTime = useCallback(async () => { + console.log("async.q: Promise.all(map(async) => await fireQuery)"); + console.log("with artificial query delay and query build time"); + console.log(`buildTime: ${buildTime}`); + console.log(`delayTime: ${delayTime}`); + + const fireQueryX = async (type: string, i: number) => { + // Artificial getDatalogQueryTime + console.log(`💽💽`, getTimestamp(), `Build`, type, i); + artificialGetDatalogQueryTime(buildTime); + + // Artifical Query Delay + console.log(`🔎🟢`, getTimestamp(), `Query`, type, i); + await new Promise((resolve) => setTimeout(resolve, delayTime)); + + const queryResults = await window.roamAlphaAPI.data.async.q( + baseQuery.replace("TYPE", type) + ); + console.log(`🔎🛑`, getTimestamp(), `Query`, type, i); + return { type, results: queryResults }; + }; + + // adding async/await results in the same behavior + Promise.all(queryTypes.map((type, i) => fireQueryX(type, i))); + + // Results + // + // I would like the query to be sent right after the build time is over + // I'm not sure if that is possible, and that doesn't seem to be the case with these results + // + // The delay is 3 seconds + // + // these results don't make sense + // last query starts at 34:322 + // RES - starts at 31:322, ends at 35:624 + // 1.5 seconds after last query start (The delay is 3 seconds) + // + // 36:535 Query HYP 3 - 1 second after previous query? + // 37:533 Query ISS 4 - 1 second after previous query? + // 38:540 Query CON 5 - 1 second after previous query? + // + // 🔎🟢 55:29:322 Query EVD 0 + // 🔎🟢 55:30:322 Query CLM 1 + // 🔎🟢 55:31:322 Query RES 2 + // 🔎🟢 55:32:322 Query HYP 3 + // 🔎🟢 55:33:322 Query ISS 4 + // 🔎🟢 55:34:322 Query CON 5 + // 🔎🛑 55:35:624 Query RES 2 + // 🔎🛑 55:35:683 Query CLM 1 + // 🔎🛑 55:35:689 Query EVD 0 + // 🔎🛑 55:36:535 Query HYP 3 (this delay doesn't make sense) + // 🔎🛑 55:37:533 Query ISS 4 (this delay doesn't make sense) + // 🔎🛑 55:38:540 Query CON 5 (this delay doesn't make sense) + }, [delayTime, buildTime]); + const fakeBackendQueryNoDelay = useCallback(async () => { + console.log("fakeBackendQuery()"); + console.log(`buildTime: no getDatalogQueryTime`); + console.log(`fakeQueryTime: ${delayTime}`); + + const fireQueryX = async (type: string, i: number) => { + console.log(`🔎🟢`, getTimestamp(), `Query`, type, i); + + const queryResults = await fakeBackendQuery(i, delayTime); + + console.log(`🔎🛑`, getTimestamp(), `Query`, type, i); + return { type, results: queryResults }; + }; + + // adding async/await results in the same behavior + Promise.all(queryTypes.map((type, i) => fireQueryX(type, i))); + + // Results + // + // Queryies returned at the same time ✅ + // + // 🔎🟢 22:19:526 Query EVD 0 + // 🔎🟢 22:19:527 Query CLM 1 + // 🔎🟢 22:19:527 Query RES 2 + // 🔎🟢 22:19:527 Query HYP 3 + // 🔎🟢 22:19:527 Query ISS 4 + // 🔎🟢 22:19:527 Query CON 5 + // 🔎🛑 22:22:541 Query EVD 0 + // 🔎🛑 22:22:541 Query CLM 1 + // 🔎🛑 22:22:541 Query RES 2 + // 🔎🛑 22:22:542 Query HYP 3 + // 🔎🛑 22:22:542 Query ISS 4 + // 🔎🛑 22:22:542 Query CON 5 + }, []); + const fakeBackendQueryWithBuildTime = useCallback(async () => { + console.log("async.q: fakeBackendQuery() with artificial query build time"); + console.log(`buildTime: ${buildTime}`); + console.log(`fakeQueryTime: ${delayTime}`); + + const fireQueryX = async (type: string, i: number) => { + console.log(`💽💽`, getTimestamp(), `Build`, type, i); + artificialGetDatalogQueryTime(buildTime); + + console.log(`🔎🟢`, getTimestamp(), `Query`, type, i); + const queryResults = await fakeBackendQuery(i, delayTime); + console.log(`🔎🛑`, getTimestamp(), `Query`, type, i); + + return { type, results: queryResults }; + }; + + // adding async/await results in the same behavior + Promise.all(queryTypes.map((type, i) => fireQueryX(type, i))); + + // Results + // + // Run 1 + // these results don't make sense + // last query starts at 34:322 + // RES - starts at 31:322, ends at 35:624 + // 1 seconds after last query start (The delay is 3 seconds) + // + // 59:752 Query HYP 3 - 1 second after previous query? + // 00:751 Query ISS 4 - 1 second after previous query? + // 01:754 Query CON 5 - 1 second after previous query? + // + // 🔎🟢 23:52:750 Query EVD 0 + // 🔎🟢 23:53:750 Query CLM 1 + // 🔎🟢 23:54:750 Query RES 2 + // 🔎🟢 23:55:750 Query HYP 3 + // 🔎🟢 23:56:750 Query ISS 4 + // 🔎🟢 23:57:750 Query CON 5 + // 🔎🛑 23:58:757 Query EVD 0 + // 🔎🛑 23:58:758 Query CLM 1 + // 🔎🛑 23:58:758 Query RES 2 + // 🔎🛑 23:59:752 Query HYP 3 + // 🔎🛑 24:00:751 Query ISS 4 + // 🔎🛑 24:01:754 Query CON 5 + + // Run 2 + // 10 second build time + // 3 second delay time + // + // no delay between last query start and first query return + // no delay between query1 - query5 + // 3 seconds between last query and second last query + // + // fakeBackendQuery() with artificial query build time + // buildTime: 10000 + // fakeQueryTime: 3000 + // 💽💽 02:05:873 Build EVD 0 + // 🔎🟢 02:15:873 Query EVD 0 + // 💽💽 02:15:873 Build CLM 1 + // 🔎🟢 02:25:873 Query CLM 1 + // 💽💽 02:25:873 Build RES 2 + // 🔎🟢 02:35:873 Query RES 2 + // 💽💽 02:35:873 Build HYP 3 + // 🔎🟢 02:45:873 Query HYP 3 + // 💽💽 02:45:873 Build ISS 4 + // 🔎🟢 02:55:873 Query ISS 4 + // 💽💽 02:55:873 Build CON 5 + // 🔎🟢 03:05:873 Query CON 5 + // 🔎🛑 03:05:886 Query EVD 0 + // 🔎🛑 03:05:887 Query CLM 1 + // 🔎🛑 03:05:887 Query RES 2 + // 🔎🛑 03:05:887 Query HYP 3 + // 🔎🛑 03:05:888 Query ISS 4 + // 🔎🛑 03:08:881 Query CON 5 + + // Run 3 + // 10 second build time + // 10 second delay time + // + // no delay between last query start and first query return + // no delay between query1 - query5 + // 10 seconds between last query and second last query + // + // fakeBackendQuery() with artificial query build time + // buildTime: 10000 + // fakeQueryTime: 10000 + // 💽💽 05:16:136 Build EVD 0 + // 🔎🟢 05:26:137 Query EVD 0 + // 💽💽 05:26:137 Build CLM 1 + // 🔎🟢 05:36:137 Query CLM 1 + // 💽💽 05:36:137 Build RES 2 + // 🔎🟢 05:46:137 Query RES 2 + // 💽💽 05:46:137 Build HYP 3 + // 🔎🟢 05:56:137 Query HYP 3 + // 💽💽 05:56:137 Build ISS 4 + // 🔎🟢 06:06:137 Query ISS 4 + // 💽💽 06:06:137 Build CON 5 + // 🔎🟢 06:16:137 Query CON 5 + // 🔎🛑 06:16:176 Query EVD 0 + // 🔎🛑 06:16:176 Query CLM 1 + // 🔎🛑 06:16:176 Query RES 2 + // 🔎🛑 06:16:177 Query HYP 3 + // 🔎🛑 06:16:177 Query ISS 4 + // 🔎🛑 06:26:145 Query CON 5 + }, [delayTime, buildTime]); + + const queries: QueryType[] = useMemo( + () => [ + { + label: "async.q with Delay", + description: + "async.q: Promise.all(map(async) => await fireQuery) with artificial query delay", + fn: asyncQWithDelay, + }, + { + label: "async.q with Delay and Build Time", + description: + "async.q: Promise.all(map(async) => await fireQuery) with artificial query delay and query build time", + fn: asyncQWithDelayAndBuildTime, + }, + { + label: "fakeBackendQuery", + description: "fakeBackendQuery() - no delay", + fn: fakeBackendQueryNoDelay, + }, + { + label: "fakeBackendQuery with Delay and Build Time", + description: "fakeBackendQuery() with artificial query build time", + fn: fakeBackendQueryWithBuildTime, + }, + ], + [delayTime, buildTime] + ); + + return ( + { + onClose(); + if (queryBlockUid) { + window.roamAlphaAPI.deleteBlock({ + block: { uid: queryBlockUid }, + }); + } + }} + autoFocus={false} + enforceFocus={false} + canEscapeKeyClose={true} + className="w-full h-full bg-white" + > + +
+
+
+ + setBuildTime(Number(e))} + disabled={isRunning} + /> + + + setDelayTime(Number(e))} + disabled={isRunning} + /> + +
+ + setSelectedQuery(Number(e.currentTarget.value))} + // className="flex flex-col gap-3" + > + {queries.map((query, i) => ( + +
+
{query.label}
+
+ {query.description} +
+
+ + ))} + + +
+ +
+ + {queryBlockUid && ( + <> + + + )} +
+
+ + ); +}; + +export const renderQueryTester = (props: QueryTesterProps) => + renderOverlay({ Overlay: QueryTester, props }); + +export default QueryTester; diff --git a/src/index.ts b/src/index.ts index ef178ecc..5d82f9dd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -56,6 +56,7 @@ import { NodeMenuTriggerComponent, render as renderDiscourseNodeMenu, } from "./components/DiscourseNodeMenu"; +import { renderQueryTester } from "./components/QueryTester"; const loadedElsewhere = document.currentScript ? document.currentScript.getAttribute("data-source") === "discourse-graph" : false; @@ -626,6 +627,16 @@ svg.rs-svg-container { callback: openCanvasDrawer, }); + extensionAPI.ui.commandPalette.addCommand({ + label: "Query Tester", + callback: () => { + renderQueryTester({ + onClose: () => {}, + isOpen: true, + }); + }, + }); + extensionAPI.ui.commandPalette.addCommand({ label: "Open Query Drawer", callback: () => diff --git a/src/utils/mockQuery.ts b/src/utils/mockQuery.ts new file mode 100644 index 00000000..829c442d --- /dev/null +++ b/src/utils/mockQuery.ts @@ -0,0 +1,89 @@ +// Mock types +const getTimestamp = () => { + const timestamp = new Date(); + const minutes = String(timestamp.getMinutes()).padStart(2, "0"); + const seconds = String(timestamp.getSeconds()).padStart(2, "0"); + const milliseconds = String(timestamp.getMilliseconds()).padStart(3, "0"); + return `${minutes}:${seconds}:${milliseconds}`; +}; + +const getDatalogQuery = (id, duration = 1000) => { + console.log(`🏋️‍♀️🟢 - ${id} getDatalogQuery -`, getTimestamp()); + const startTime = Date.now(); + + // Block the thread for 1 second + while (Date.now() - startTime < duration) { + // Do nothing, just wait + } + + console.log(`🏋️‍♀️🔴 - ${id} getDatalogQuery -`, getTimestamp()); +}; + +const fakeBackendQuery = (id, durationMin = 1, durationMax = 4) => { + // Convert seconds to milliseconds + const durationMinMs = durationMin * 1000; + const durationMaxMs = durationMax * 1000; + return new Promise((resolve) => { + const delay = + id === 1 + ? 0 + : Math.floor(Math.random() * ((durationMaxMs - durationMinMs) / 1000)) * + 1000 + + durationMinMs; + + const startTime = getTimestamp(); + console.log(`🔎🟢 - ${id} Query -`, startTime, delay); + + setTimeout(() => { + console.log(`🔎🔴 - ${id} Query`); + console.log(startTime, "Start"); + console.log(getTimestamp(), "End"); + console.log("Delay -", delay); + resolve([id]); + }, delay); + }); +}; + +const fireQuery = async (args) => { + getDatalogQuery(args); + const results = await fakeBackendQuery(args); + return Promise.all( + results.map((r) => ({ + text: r, + })) + ); +}; + +// Mock function that mimics resultsWithRelation behavior +const getDiscourseResults = async () => { + console.log("--- Starting Parallel Queries ---"); + console.time("total"); + + // Mock relations data + const relations = [1, 2, 3, 4]; + + const resultsWithRelation = await Promise.all( + relations.map(async (r) => { + const results = fireQuery(r); + + return results.then((results) => ({ + relation: r, + results, + })); + }) + ).catch((e) => { + console.error(e); + return []; + }); + + console.timeEnd("total"); + return resultsWithRelation; +}; + +// Run the test +// getDiscourseResults().then((results) => { +// console.log("\nFinal Results:"); +// console.log(JSON.stringify(results, null, 2)); +// }); + +export { getTimestamp, getDatalogQuery, fakeBackendQuery, fireQuery }; From 403b9b064b4ab620e2a143534c0a89cc16839e66 Mon Sep 17 00:00:00 2001 From: Michael Gartner Date: Thu, 21 Nov 2024 17:40:26 -0600 Subject: [PATCH 3/6] tester cleanup, add number of queries, add fast.q --- src/components/QueryTester.tsx | 204 ++++++++++++++------------------- 1 file changed, 84 insertions(+), 120 deletions(-) diff --git a/src/components/QueryTester.tsx b/src/components/QueryTester.tsx index 7d48a81b..8d0e0d83 100644 --- a/src/components/QueryTester.tsx +++ b/src/components/QueryTester.tsx @@ -54,13 +54,23 @@ const fakeBackendQuery = (id: number, delayTime: number) => { }, delayTime); }); }; -const artificialGetDatalogQueryTime = (buildTime: number) => { +const fakeGetDatalogQuery = (buildTime: number) => { + console.log(`💽💽`, getTimestamp(), `Build`); const startTime = Date.now(); while (Date.now() - startTime < buildTime) { // Simulate work by blocking the thread } }; -const queryTypes = ["EVD", "CLM", "RES", "HYP", "ISS", "CON"]; +const PREDEFINED_TYPES = ["EVD", "CLM", "RES", "HYP", "ISS", "CON"]; + +const getQueryType = (index: number) => { + // For first 6, use predefined types in order + if (index < PREDEFINED_TYPES.length) { + return PREDEFINED_TYPES[index]; + } + // For additional ones, randomly select from predefined types + return PREDEFINED_TYPES[Math.floor(Math.random() * PREDEFINED_TYPES.length)]; +}; const baseQuery = `[:find (pull ?node [:block/string :node/title :block/uid]) (pull ?node [:block/uid]) @@ -138,12 +148,24 @@ const CellEmbed = ({ const QueryTester = ({ onClose, isOpen }: QueryTesterProps) => { const [selectedQuery, setSelectedQuery] = useState(0); const [isRunning, setIsRunning] = useState(false); - const [buildTime, setBuildTime] = useState(3000); + const [buildTime, setBuildTime] = useState(1000); const [delayTime, setDelayTime] = useState(3000); const [queryBlockUid, setQueryBlockUid] = useState(null); + const [numberOfQueries, setNumberOfQueries] = useState(6); + + const queryTypes = useMemo(() => { + return Array.from({ length: numberOfQueries }, (_, i) => getQueryType(i)); + }, [numberOfQueries]); // lol couldn't get highlighting to work properly, so creating the blocks and rending them useEffect(() => { + const removeConsoleLogLines = (input: string) => { + return input + .split("\n") // Split the input string into lines + .filter((line) => !line.trim().startsWith("console.log(")) // Filter out lines with `console.log` + .join("\n"); // Join the remaining lines back into a string + }; + const createQueryBlock = async () => { const currentPageUid = getCurrentPageUid(); const newUid = await createBlock({ @@ -152,7 +174,7 @@ const QueryTester = ({ onClose, isOpen }: QueryTesterProps) => { text: "", children: [ ...queries.map((query, i) => ({ - text: `\`\`\`const ${query.label.replace(/\s+/g, "")} = ${query.fn.toString()} \`\`\``, + text: `\`\`\`const ${query.label.replace(/\s+/g, "")} = ${removeConsoleLogLines(query.fn.toString())} \`\`\``, })), ], }, @@ -167,73 +189,22 @@ const QueryTester = ({ onClose, isOpen }: QueryTesterProps) => { }; createQueryBlock(); }, []); - const asyncQWithDelay = useCallback(async () => { - console.log("async.q: Promise.all(map(async) => await fireQuery)"); - console.log("with artificial query delay"); - console.log(`buildTime: no getDatalogQueryTime`); - console.log(`delayTime: ${delayTime}`); - - const fireQueryX = async (type: string, i: number) => { - console.log(`🔎🟢`, getTimestamp(), `Query`, type, i); - // Artifical Query Delay - await new Promise((resolve) => setTimeout(resolve, delayTime)); - - const queryResults = await window.roamAlphaAPI.data.async.q( - baseQuery.replace("TYPE", type) - ); - console.log(`🔎🛑`, getTimestamp(), `Query`, type, i); - return { type, results: queryResults }; - }; - - // adding async/await results in the same behavior - Promise.all(queryTypes.map((type, i) => fireQueryX(type, i))); - - // Results - // - // These results makes sense ✅ - // queries sent at 32:079 - // delay is 3 seconds - // queries ends between 35:581 - 35:774 - // difference is actual query time, server queues, and network time - // - // 🔎🟢 50:32:078 Query EVD 0 - // 🔎🟢 50:32:079 Query CLM 1 - // 🔎🟢 50:32:079 Query RES 2 - // 🔎🟢 50:32:079 Query HYP 3 - // 🔎🟢 50:32:079 Query ISS 4 - // 🔎🟢 50:32:079 Query CON 5 - // 🔎🛑 50:35:581 Query HYP 3 - // 🔎🛑 50:35:686 Query CON 5 - // 🔎🛑 50:35:760 Query ISS 4 - // 🔎🛑 50:35:770 Query EVD 0 - // 🔎🛑 50:35:771 Query RES 2 - // 🔎🛑 50:35:774 Query CLM 1 - }, [delayTime]); - const asyncQWithDelayAndBuildTime = useCallback(async () => { + const asyncQ = useCallback(async () => { console.log("async.q: Promise.all(map(async) => await fireQuery)"); console.log("with artificial query delay and query build time"); console.log(`buildTime: ${buildTime}`); console.log(`delayTime: ${delayTime}`); const fireQueryX = async (type: string, i: number) => { - // Artificial getDatalogQueryTime - console.log(`💽💽`, getTimestamp(), `Build`, type, i); - artificialGetDatalogQueryTime(buildTime); - - // Artifical Query Delay + if (buildTime) fakeGetDatalogQuery(buildTime); console.log(`🔎🟢`, getTimestamp(), `Query`, type, i); await new Promise((resolve) => setTimeout(resolve, delayTime)); - - const queryResults = await window.roamAlphaAPI.data.async.q( - baseQuery.replace("TYPE", type) - ); + await window.roamAlphaAPI.data.async.q(baseQuery.replace("TYPE", type)); console.log(`🔎🛑`, getTimestamp(), `Query`, type, i); - return { type, results: queryResults }; }; - // adding async/await results in the same behavior - Promise.all(queryTypes.map((type, i) => fireQueryX(type, i))); + await Promise.all(queryTypes.map((type, i) => fireQueryX(type, i))); // Results // @@ -263,59 +234,21 @@ const QueryTester = ({ onClose, isOpen }: QueryTesterProps) => { // 🔎🛑 55:36:535 Query HYP 3 (this delay doesn't make sense) // 🔎🛑 55:37:533 Query ISS 4 (this delay doesn't make sense) // 🔎🛑 55:38:540 Query CON 5 (this delay doesn't make sense) - }, [delayTime, buildTime]); - const fakeBackendQueryNoDelay = useCallback(async () => { - console.log("fakeBackendQuery()"); - console.log(`buildTime: no getDatalogQueryTime`); - console.log(`fakeQueryTime: ${delayTime}`); - - const fireQueryX = async (type: string, i: number) => { - console.log(`🔎🟢`, getTimestamp(), `Query`, type, i); - - const queryResults = await fakeBackendQuery(i, delayTime); - - console.log(`🔎🛑`, getTimestamp(), `Query`, type, i); - return { type, results: queryResults }; - }; - - // adding async/await results in the same behavior - Promise.all(queryTypes.map((type, i) => fireQueryX(type, i))); - - // Results - // - // Queryies returned at the same time ✅ - // - // 🔎🟢 22:19:526 Query EVD 0 - // 🔎🟢 22:19:527 Query CLM 1 - // 🔎🟢 22:19:527 Query RES 2 - // 🔎🟢 22:19:527 Query HYP 3 - // 🔎🟢 22:19:527 Query ISS 4 - // 🔎🟢 22:19:527 Query CON 5 - // 🔎🛑 22:22:541 Query EVD 0 - // 🔎🛑 22:22:541 Query CLM 1 - // 🔎🛑 22:22:541 Query RES 2 - // 🔎🛑 22:22:542 Query HYP 3 - // 🔎🛑 22:22:542 Query ISS 4 - // 🔎🛑 22:22:542 Query CON 5 - }, []); - const fakeBackendQueryWithBuildTime = useCallback(async () => { + }, [delayTime, buildTime, queryTypes]); + const fakeBackend = useCallback(async () => { console.log("async.q: fakeBackendQuery() with artificial query build time"); console.log(`buildTime: ${buildTime}`); console.log(`fakeQueryTime: ${delayTime}`); const fireQueryX = async (type: string, i: number) => { - console.log(`💽💽`, getTimestamp(), `Build`, type, i); - artificialGetDatalogQueryTime(buildTime); - + if (buildTime) fakeGetDatalogQuery(buildTime); console.log(`🔎🟢`, getTimestamp(), `Query`, type, i); - const queryResults = await fakeBackendQuery(i, delayTime); + await new Promise((resolve) => setTimeout(resolve, delayTime)); + await fakeBackendQuery(i, delayTime); console.log(`🔎🛑`, getTimestamp(), `Query`, type, i); - - return { type, results: queryResults }; }; - // adding async/await results in the same behavior - Promise.all(queryTypes.map((type, i) => fireQueryX(type, i))); + await Promise.all(queryTypes.map((type, i) => fireQueryX(type, i))); // Results // @@ -401,34 +334,41 @@ const QueryTester = ({ onClose, isOpen }: QueryTesterProps) => { // 🔎🛑 06:16:177 Query HYP 3 // 🔎🛑 06:16:177 Query ISS 4 // 🔎🛑 06:26:145 Query CON 5 - }, [delayTime, buildTime]); + }, [delayTime, buildTime, queryTypes]); + const fastQ = useCallback(async () => { + console.log("fast.q: with artificial query delay and query build time"); + console.log(`buildTime: ${buildTime}`); + console.log(`delayTime: ${delayTime}`); + const fireQueryX = async (type: string, i: number) => { + if (buildTime) fakeGetDatalogQuery(buildTime); + console.log(`🔎🟢`, getTimestamp(), `Query`, type, i); + await new Promise((resolve) => setTimeout(resolve, delayTime)); + await window.roamAlphaAPI.data.fast.q(baseQuery.replace("TYPE", type)); + console.log(`🔎🛑`, getTimestamp(), `Query`, type, i); + }; + + await Promise.all(queryTypes.map((type, i) => fireQueryX(type, i))); + }, [delayTime, buildTime, queryTypes]); const queries: QueryType[] = useMemo( () => [ { - label: "async.q with Delay", - description: - "async.q: Promise.all(map(async) => await fireQuery) with artificial query delay", - fn: asyncQWithDelay, - }, - { - label: "async.q with Delay and Build Time", - description: - "async.q: Promise.all(map(async) => await fireQuery) with artificial query delay and query build time", - fn: asyncQWithDelayAndBuildTime, + label: "async q", + description: "async.q: Promise.all(map(async) => await fireQuery)", + fn: asyncQ, }, { label: "fakeBackendQuery", - description: "fakeBackendQuery() - no delay", - fn: fakeBackendQueryNoDelay, + description: "fakeBackendQuery()", + fn: fakeBackend, }, { - label: "fakeBackendQuery with Delay and Build Time", - description: "fakeBackendQuery() with artificial query build time", - fn: fakeBackendQueryWithBuildTime, + label: "fast q", + description: "window.roamAlphaAPI.data.fast.q", + fn: fastQ, }, ], - [delayTime, buildTime] + [delayTime, buildTime, numberOfQueries] ); return ( @@ -459,6 +399,15 @@ const QueryTester = ({ onClose, isOpen }: QueryTesterProps) => {
+ { disabled={isRunning} /> + + setNumberOfQueries(Number(e))} + disabled={isRunning} + /> +
{