diff --git a/apps/webapp/app/assets/icons/LogsIcon.tsx b/apps/webapp/app/assets/icons/LogsIcon.tsx new file mode 100644 index 0000000000..3178da237e --- /dev/null +++ b/apps/webapp/app/assets/icons/LogsIcon.tsx @@ -0,0 +1,66 @@ +export function LogsIcon({ className }: { className?: string }) { + return ( + + + + + + + + + + + + + + + ); +} diff --git a/apps/webapp/app/components/code/CodeBlock.tsx b/apps/webapp/app/components/code/CodeBlock.tsx index efa144138c..431a02ba35 100644 --- a/apps/webapp/app/components/code/CodeBlock.tsx +++ b/apps/webapp/app/components/code/CodeBlock.tsx @@ -5,6 +5,7 @@ import { Highlight, Prism } from "prism-react-renderer"; import { forwardRef, ReactNode, useCallback, useEffect, useState } from "react"; import { TextWrapIcon } from "~/assets/icons/TextWrapIcon"; import { cn } from "~/utils/cn"; +import { highlightSearchText } from "~/utils/logUtils"; import { Button } from "../primitives/Buttons"; import { Dialog, DialogContent, DialogHeader, DialogTitle } from "../primitives/Dialog"; import { Paragraph } from "../primitives/Paragraph"; @@ -64,6 +65,9 @@ type CodeBlockProps = { /** Whether to show the open in modal button */ showOpenInModal?: boolean; + + /** Search term to highlight in the code */ + searchTerm?: string; }; const dimAmount = 0.5; @@ -202,6 +206,7 @@ export const CodeBlock = forwardRef( showChrome = false, fileName, rowTitle, + searchTerm, ...props }: CodeBlockProps, ref @@ -238,7 +243,7 @@ export const CodeBlock = forwardRef( [code] ); - code = code.trim(); + code = code?.trim() ?? ""; const lineCount = code.split("\n").length; const maxLineWidth = lineCount.toString().length; let maxHeight: string | undefined = undefined; @@ -340,6 +345,7 @@ export const CodeBlock = forwardRef( className="px-2 py-3" preClassName="text-xs" isWrapped={isWrapped} + searchTerm={searchTerm} /> ) : (
( )} dir="ltr" > - {code} + {highlightSearchText(code, searchTerm)}
)} @@ -402,7 +408,7 @@ export const CodeBlock = forwardRef( className="overflow-auto px-3 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600" > 
-                  {code}
+                  {highlightSearchText(code, searchTerm)}
)} @@ -451,6 +457,7 @@ type HighlightCodeProps = { className?: string; preClassName?: string; isWrapped: boolean; + searchTerm?: string; }; function HighlightCode({ @@ -463,6 +470,7 @@ function HighlightCode({ className, preClassName, isWrapped, + searchTerm, }: HighlightCodeProps) { const [isLoaded, setIsLoaded] = useState(false); @@ -556,6 +564,10 @@ function HighlightCode({
{line.map((token, key) => { const tokenProps = getTokenProps({ token, key }); + + // Highlight search term matches in token + const content = highlightSearchText(token.content, searchTerm); + return ( + > + {content} + ); })}
diff --git a/apps/webapp/app/components/logs/LogDetailView.tsx b/apps/webapp/app/components/logs/LogDetailView.tsx new file mode 100644 index 0000000000..a367e75495 --- /dev/null +++ b/apps/webapp/app/components/logs/LogDetailView.tsx @@ -0,0 +1,483 @@ +import { XMarkIcon, ArrowTopRightOnSquareIcon, CheckIcon } from "@heroicons/react/20/solid"; +import { Link } from "@remix-run/react"; +import { + type MachinePresetName, + formatDurationMilliseconds, +} from "@trigger.dev/core/v3"; +import { useEffect, useState } from "react"; +import { useTypedFetcher } from "remix-typedjson"; +import { cn } from "~/utils/cn"; +import { Button } from "~/components/primitives/Buttons"; +import { DateTime } from "~/components/primitives/DateTime"; +import { Header2, Header3 } from "~/components/primitives/Headers"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { Spinner } from "~/components/primitives/Spinner"; +import { TabButton, TabContainer } from "~/components/primitives/Tabs"; +import * as Property from "~/components/primitives/PropertyTable"; +import { TextLink } from "~/components/primitives/TextLink"; +import { CopyableText } from "~/components/primitives/CopyableText"; +import { SimpleTooltip, InfoIconTooltip } from "~/components/primitives/Tooltip"; +import { useEnvironment } from "~/hooks/useEnvironment"; +import { useOrganization } from "~/hooks/useOrganizations"; +import { useProject } from "~/hooks/useProject"; +import type { LogEntry } from "~/presenters/v3/LogsListPresenter.server"; +import { getLevelColor, getKindColor, getKindLabel } from "~/utils/logUtils"; +import { v3RunSpanPath, v3RunsPath, v3DeploymentVersionPath } from "~/utils/pathBuilder"; +import type { loader as logDetailLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId"; +import { TaskRunStatusCombo, descriptionForTaskRunStatus } from "~/components/runs/v3/TaskRunStatus"; +import { MachineLabelCombo } from "~/components/MachineLabelCombo"; +import { EnvironmentCombo } from "~/components/environments/EnvironmentLabel"; +import { RunTag } from "~/components/runs/v3/RunTag"; +import { formatCurrencyAccurate } from "~/utils/numberFormatter"; +import type { TaskRunStatus } from "@trigger.dev/database"; +import { PacketDisplay } from "~/components/runs/v3/PacketDisplay"; +import type { RunContext } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.run"; + +type RunContextData = { + run: RunContext | null; +}; + + +type LogDetailViewProps = { + logId: string; + // If we have the log entry from the list, we can display it immediately + initialLog?: LogEntry; + onClose: () => void; + searchTerm?: string; +}; + +type TabType = "details" | "run"; + +type LogAttributes = Record & { + error?: { + message?: string; + }; +}; + +function formatStringJSON(str: string): string { + return str + .replace(/\\n/g, "\n") // Converts literal "\n" to newline + .replace(/\\t/g, "\t"); // Converts literal "\t" to tab +} + + +export function LogDetailView({ logId, initialLog, onClose, searchTerm }: LogDetailViewProps) { + const organization = useOrganization(); + const project = useProject(); + const environment = useEnvironment(); + const fetcher = useTypedFetcher(); + const [activeTab, setActiveTab] = useState("details"); + const [error, setError] = useState(null); + + // Fetch full log details when logId changes + useEffect(() => { + if (!logId) return; + + setError(null); + fetcher.load( + `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/logs/${encodeURIComponent(logId)}` + ); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [organization.slug, project.slug, environment.slug, logId]); + + // Handle fetch errors + useEffect(() => { + if (fetcher.data && typeof fetcher.data === "object" && "error" in fetcher.data) { + setError(fetcher.data.error as string); + } else if (fetcher.state === "idle" && fetcher.data === null && !initialLog) { + setError("Failed to load log details"); + } else { + setError(null); + } + }, [fetcher.data, initialLog, fetcher.state]); + + const isLoading = fetcher.state === "loading"; + const log = fetcher.data ?? initialLog; + + // Handle Escape key to close panel + useEffect(() => { + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === "Escape") { + onClose(); + } + }; + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }, [onClose]); + + if (isLoading && !log) { + return ( +
+ +
+ ); + } + + if (!log) { + return ( +
+
+ Log Details + +
+
+ {error ?? "Log not found"} +
+
+ ); + } + + const runPath = v3RunSpanPath( + organization, + project, + environment, + { friendlyId: log.runId }, + { spanId: log.spanId } + ); + + return ( +
+ {/* Header */} +
+
+ + {getKindLabel(log.kind)} + + + {log.level} + +
+ +
+ + {/* Tabs */} +
+ + setActiveTab("details")} + shortcut={{ key: "d" }} + > + Details + + setActiveTab("run")} + shortcut={{ key: "r" }} + > + Run + + + + + +
+ + {/* Content */} +
+ {activeTab === "details" && ( + + )} + {activeTab === "run" && ( + + )} +
+
+ ); +} + +function DetailsTab({ log, runPath, searchTerm }: { log: LogEntry; runPath: string; searchTerm?: string }) { + const logWithExtras = log as LogEntry & { + attributes?: LogAttributes; + }; + + + let beautifiedAttributes: string | null = null; + + if (logWithExtras.attributes) { + beautifiedAttributes = JSON.stringify(logWithExtras.attributes, null, 2); + beautifiedAttributes = formatStringJSON(beautifiedAttributes); + } + + const showAttributes = beautifiedAttributes && beautifiedAttributes !== "{}"; + + // Determine message to show + let message = log.message ?? ""; + if (log.level === "ERROR") { + const maybeErrorMessage = logWithExtras.attributes?.error?.message; + if (typeof maybeErrorMessage === "string" && maybeErrorMessage.length > 0) { + message = maybeErrorMessage; + } + } + + return ( + <> + {/* Time */} +
+ Timestamp +
+ +
+
+ + {/* Message */} +
+ +
+ + {/* Attributes - only available in full log detail */} + {showAttributes && beautifiedAttributes && ( +
+ +
+ )} + + ); +} + +function RunTab({ log, runPath }: { log: LogEntry; runPath: string }) { + const organization = useOrganization(); + const project = useProject(); + const environment = useEnvironment(); + const fetcher = useTypedFetcher(); + + // Fetch run details when tab is active + useEffect(() => { + if (!log.runId) return; + + fetcher.load( + `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/logs/${encodeURIComponent(log.id)}/run?runId=${encodeURIComponent(log.runId)}` + ); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [organization.slug, project.slug, environment.slug, log.id, log.runId]); + + const isLoading = fetcher.state === "loading"; + const runData = fetcher.data?.run; + + if (isLoading) { + return ( +
+ +
+ ); + } + + if (!runData) { + return ( +
+ Run not found in database. +
+ ); + } + + return ( +
+ + + Run ID + + + + + + + Status + + } + content={descriptionForTaskRunStatus(runData.status as TaskRunStatus)} + disableHoverableContent + /> + + + + + Task + + + + + + {runData.rootRun && ( + + Root and parent run + + + + + )} + + {runData.batch && ( + + Batch + + + + + )} + + + Version + + {runData.version ? ( + environment.type === "DEVELOPMENT" ? ( + + ) : ( + + + + } + content={"Jump to deployment"} + /> + ) + ) : ( + + Never started + + + )} + + + + + Test run + + {runData.isTest ? : "–"} + + + + + Environment + + + + + + + Queue + +
Name: {runData.queue}
+
Concurrency key: {runData.concurrencyKey ? runData.concurrencyKey : "–"}
+ + + + {runData.tags && runData.tags.length > 0 && ( + + Tags + +
+ {runData.tags.map((tag: string) => ( + + ))} +
+ + + )} + + + Machine + + {runData.machinePreset ? ( + + ) : ( + "–" + )} + + + + + Run invocation cost + + {runData.baseCostInCents > 0 + ? formatCurrencyAccurate(runData.baseCostInCents / 100) + : "–"} + + + + + Compute cost + + {runData.costInCents > 0 ? formatCurrencyAccurate(runData.costInCents / 100) : "–"} + + + + + Total cost + + {runData.costInCents > 0 || runData.baseCostInCents > 0 + ? formatCurrencyAccurate((runData.baseCostInCents + runData.costInCents) / 100) + : "–"} + + + + + Usage duration + + {runData.usageDurationMs > 0 + ? formatDurationMilliseconds(runData.usageDurationMs, { style: "short" }) + : "–"} + + + +
+ ); +} + diff --git a/apps/webapp/app/components/logs/LogsLevelFilter.tsx b/apps/webapp/app/components/logs/LogsLevelFilter.tsx new file mode 100644 index 0000000000..4ec7d95730 --- /dev/null +++ b/apps/webapp/app/components/logs/LogsLevelFilter.tsx @@ -0,0 +1,182 @@ +import * as Ariakit from "@ariakit/react"; +import { ExclamationTriangleIcon } from "@heroicons/react/20/solid"; +import { type ReactNode, useMemo } from "react"; +import { AppliedFilter } from "~/components/primitives/AppliedFilter"; +import { + ComboBox, + SelectItem, + SelectList, + SelectPopover, + SelectProvider, + SelectTrigger, + shortcutFromIndex, +} from "~/components/primitives/Select"; +import { useSearchParams } from "~/hooks/useSearchParam"; +import { FilterMenuProvider, appliedSummary } from "~/components/runs/v3/SharedFilters"; +import type { LogLevel } from "~/presenters/v3/LogsListPresenter.server"; +import { cn } from "~/utils/cn"; + +const allLogLevels: { level: LogLevel; label: string; color: string }[] = [ + { level: "ERROR", label: "Error", color: "text-error" }, + { level: "WARN", label: "Warning", color: "text-warning" }, + { level: "INFO", label: "Info", color: "text-blue-400" }, + { level: "CANCELLED", label: "Cancelled", color: "text-charcoal-400" }, + { level: "DEBUG", label: "Debug", color: "text-charcoal-400" }, + { level: "TRACE", label: "Trace", color: "text-charcoal-500" }, +]; + +function getAvailableLevels(showDebug: boolean): typeof allLogLevels { + if (showDebug) { + return allLogLevels; + } + return allLogLevels.filter((level) => level.level !== "DEBUG"); +} + +function getLevelBadgeColor(level: LogLevel): string { + switch (level) { + case "ERROR": + return "text-error bg-error/10 border-error/20"; + case "WARN": + return "text-warning bg-warning/10 border-warning/20"; + case "DEBUG": + return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; + case "INFO": + return "text-blue-400 bg-blue-500/10 border-blue-500/20"; + case "TRACE": + return "text-charcoal-500 bg-charcoal-800 border-charcoal-700"; + case "CANCELLED": + return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; + default: + return "text-text-dimmed bg-charcoal-750 border-charcoal-700"; + } +} + +const shortcut = { key: "l" }; + +export function LogsLevelFilter({ showDebug = false }: { showDebug?: boolean }) { + const { values } = useSearchParams(); + const selectedLevels = values("levels"); + const hasLevels = selectedLevels.length > 0 && selectedLevels.some((v) => v !== ""); + + if (hasLevels) { + return ; + } + + return ( + + {(search, setSearch) => ( + } + variant="secondary/small" + shortcut={shortcut} + tooltipTitle="Filter by level" + > + Level + + } + searchValue={search} + clearSearchValue={() => setSearch("")} + showDebug={showDebug} + /> + )} + + ); +} + +function LevelDropdown({ + trigger, + clearSearchValue, + searchValue, + onClose, + showDebug = false, +}: { + trigger: ReactNode; + clearSearchValue: () => void; + searchValue: string; + onClose?: () => void; + showDebug?: boolean; +}) { + const { values, replace } = useSearchParams(); + + const handleChange = (values: string[]) => { + clearSearchValue(); + replace({ levels: values, cursor: undefined, direction: undefined }); + }; + + const availableLevels = getAvailableLevels(showDebug); + const filtered = useMemo(() => { + return availableLevels.filter((item) => + item.label.toLowerCase().includes(searchValue.toLowerCase()) + ); + }, [searchValue, availableLevels]); + + return ( + + {trigger} + { + if (onClose) { + onClose(); + return false; + } + return true; + }} + > + + + {filtered.map((item, index) => ( + + + {item.level} + + + ))} + + + + ); +} + +function AppliedLevelFilter({ showDebug = false }: { showDebug?: boolean }) { + const { values, del } = useSearchParams(); + const levels = values("levels"); + + if (levels.length === 0 || levels.every((v) => v === "")) { + return null; + } + + return ( + + {(search, setSearch) => ( + }> + } + value={appliedSummary(levels)} + onRemove={() => del(["levels", "cursor", "direction"])} + variant="secondary/small" + /> + + } + searchValue={search} + clearSearchValue={() => setSearch("")} + showDebug={showDebug} + /> + )} + + ); +} diff --git a/apps/webapp/app/components/logs/LogsRunIdFilter.tsx b/apps/webapp/app/components/logs/LogsRunIdFilter.tsx new file mode 100644 index 0000000000..5c23d1a192 --- /dev/null +++ b/apps/webapp/app/components/logs/LogsRunIdFilter.tsx @@ -0,0 +1,161 @@ +import * as Ariakit from "@ariakit/react"; +import { FingerPrintIcon } from "@heroicons/react/20/solid"; +import { useCallback, useState } from "react"; +import { AppliedFilter } from "~/components/primitives/AppliedFilter"; +import { Button } from "~/components/primitives/Buttons"; +import { FormError } from "~/components/primitives/FormError"; +import { Input } from "~/components/primitives/Input"; +import { Label } from "~/components/primitives/Label"; +import { + SelectPopover, + SelectProvider, + SelectTrigger, +} from "~/components/primitives/Select"; +import { useSearchParams } from "~/hooks/useSearchParam"; +import { FilterMenuProvider } from "~/components/runs/v3/SharedFilters"; + +const shortcut = { key: "r" }; + +export function LogsRunIdFilter() { + const { value } = useSearchParams(); + const runIdValue = value("runId"); + + if (runIdValue) { + return ; + } + + return ( + + {(search, setSearch) => ( + } + variant="secondary/small" + shortcut={shortcut} + tooltipTitle="Filter by run ID" + > + Run ID + + } + clearSearchValue={() => setSearch("")} + /> + )} + + ); +} + +function RunIdDropdown({ + trigger, + clearSearchValue, + onClose, +}: { + trigger: React.ReactNode; + clearSearchValue: () => void; + onClose?: () => void; +}) { + const [open, setOpen] = useState(); + const { value, replace } = useSearchParams(); + const runIdValue = value("runId"); + + const [runId, setRunId] = useState(runIdValue); + + const apply = useCallback(() => { + clearSearchValue(); + replace({ + cursor: undefined, + direction: undefined, + runId: runId === "" ? undefined : runId?.toString(), + }); + + setOpen(false); + }, [runId, replace, clearSearchValue]); + + let error: string | undefined = undefined; + if (runId) { + if (!runId.startsWith("run_")) { + error = "Run IDs start with 'run_'"; + } else if (runId.length !== 25 && runId.length !== 29) { + error = "Run IDs are 25 or 29 characters long"; + } + } + + return ( + + {trigger} + { + if (onClose) { + onClose(); + return false; + } + return true; + }} + className="max-w-[min(32ch,var(--popover-available-width))]" + > +
+
+ + setRunId(e.target.value)} + variant="small" + className="w-[27ch] font-mono" + spellCheck={false} + /> + {error ? {error} : null} +
+
+ + +
+
+ + + ); +} + +function AppliedRunIdFilter() { + const { value, del } = useSearchParams(); + + const runId = value("runId"); + if (!runId) { + return null; + } + + return ( + + {(search, setSearch) => ( + }> + } + value={runId} + onRemove={() => del(["runId", "cursor", "direction"])} + variant="secondary/small" + /> + + } + clearSearchValue={() => setSearch("")} + /> + )} + + ); +} diff --git a/apps/webapp/app/components/logs/LogsSearchInput.tsx b/apps/webapp/app/components/logs/LogsSearchInput.tsx new file mode 100644 index 0000000000..41871722b9 --- /dev/null +++ b/apps/webapp/app/components/logs/LogsSearchInput.tsx @@ -0,0 +1,97 @@ +import { MagnifyingGlassIcon, XMarkIcon } from "@heroicons/react/20/solid"; +import { useNavigate } from "@remix-run/react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { Input } from "~/components/primitives/Input"; +import { ShortcutKey } from "~/components/primitives/ShortcutKey"; +import { cn } from "~/utils/cn"; +import { useOptimisticLocation } from "~/hooks/useOptimisticLocation"; + +export function LogsSearchInput() { + const location = useOptimisticLocation(); + const navigate = useNavigate(); + const inputRef = useRef(null); + + // Get initial search value from URL + const searchParams = new URLSearchParams(location.search); + const initialSearch = searchParams.get("search") ?? ""; + + const [text, setText] = useState(initialSearch); + const [isFocused, setIsFocused] = useState(false); + + // Update text when URL search param changes (only when not focused to avoid overwriting user input) + useEffect(() => { + const params = new URLSearchParams(location.search); + const urlSearch = params.get("search") ?? ""; + if (urlSearch !== text && !isFocused) { + setText(urlSearch); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [location.search]); + + const handleSubmit = useCallback(() => { + const params = new URLSearchParams(location.search); + if (text.trim()) { + params.set("search", text.trim()); + } else { + params.delete("search"); + } + // Reset cursor when searching + params.delete("cursor"); + params.delete("direction"); + navigate(`${location.pathname}?${params.toString()}`, { replace: true }); + }, [text, location.pathname, location.search, navigate]); + + const handleClear = useCallback(() => { + setText(""); + const params = new URLSearchParams(location.search); + params.delete("search"); + params.delete("cursor"); + params.delete("direction"); + navigate(`${location.pathname}?${params.toString()}`, { replace: true }); + }, [location.pathname, location.search, navigate]); + + return ( +
+
+ setText(e.target.value)} + fullWidth + className={cn(isFocused && "placeholder:text-text-dimmed/70")} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + handleSubmit(); + } + if (e.key === "Escape") { + e.currentTarget.blur(); + } + }} + onFocus={() => setIsFocused(true)} + onBlur={() => setIsFocused(false)} + icon={} + accessory={ + text.length > 0 ? ( + + ) : undefined + } + /> +
+ + {text.length > 0 && ( + + )} +
+ ); +} diff --git a/apps/webapp/app/components/logs/LogsTable.tsx b/apps/webapp/app/components/logs/LogsTable.tsx new file mode 100644 index 0000000000..169e65515b --- /dev/null +++ b/apps/webapp/app/components/logs/LogsTable.tsx @@ -0,0 +1,248 @@ +import { ArrowPathIcon, ArrowTopRightOnSquareIcon } from "@heroicons/react/20/solid"; +import { useEffect, useRef, useState } from "react"; +import { cn } from "~/utils/cn"; +import { Button } from "~/components/primitives/Buttons"; +import { useEnvironment } from "~/hooks/useEnvironment"; +import { useOrganization } from "~/hooks/useOrganizations"; +import { useProject } from "~/hooks/useProject"; +import type { LogEntry } from "~/presenters/v3/LogsListPresenter.server"; +import { getLevelColor, highlightSearchText } from "~/utils/logUtils"; +import { v3RunSpanPath } from "~/utils/pathBuilder"; +import { DateTime } from "../primitives/DateTime"; +import { Paragraph } from "../primitives/Paragraph"; +import { Spinner } from "../primitives/Spinner"; +import { TruncatedCopyableValue } from "../primitives/TruncatedCopyableValue"; +import { + Table, + TableBlankRow, + TableBody, + TableCell, + TableCellMenu, + TableHeader, + TableHeaderCell, + TableRow, + type TableVariant, +} from "../primitives/Table"; +import { PopoverMenuItem } from "~/components/primitives/Popover"; + +type LogsTableProps = { + logs: LogEntry[]; + hasFilters: boolean; + searchTerm?: string; + isLoading?: boolean; + isLoadingMore?: boolean; + hasMore?: boolean; + onLoadMore?: () => void; + variant?: TableVariant; + selectedLogId?: string; + onLogSelect?: (logId: string) => void; +}; + +// Left border color for error highlighting +function getLevelBorderColor(level: LogEntry["level"]): string { + switch (level) { + case "ERROR": + return "border-l-error"; + case "WARN": + return "border-l-warning"; + case "INFO": + return "border-l-blue-500"; + case "CANCELLED": + return "border-l-charcoal-600"; + case "DEBUG": + case "TRACE": + default: + return "border-l-transparent hover:border-l-charcoal-800"; + } +} + + +export function LogsTable({ + logs, + hasFilters, + searchTerm, + isLoading = false, + isLoadingMore = false, + hasMore = false, + onLoadMore, + selectedLogId, + onLogSelect, +}: LogsTableProps) { + const organization = useOrganization(); + const project = useProject(); + const environment = useEnvironment(); + const loadMoreRef = useRef(null); + const [showLoadMoreSpinner, setShowLoadMoreSpinner] = useState(false); + + // Show load more spinner only after 0.2 seconds of loading time + useEffect(() => { + if (!isLoadingMore) { + setShowLoadMoreSpinner(false); + return; + } + + const timer = setTimeout(() => { + setShowLoadMoreSpinner(true); + }, 200); + + return () => clearTimeout(timer); + }, [isLoadingMore]); + + // Intersection observer for infinite scroll + useEffect(() => { + if (!hasMore || isLoadingMore || !onLoadMore) return; + + const observer = new IntersectionObserver( + (entries) => { + if (entries[0].isIntersecting) { + onLoadMore(); + } + }, + { threshold: 0.1 } + ); + + const currentRef = loadMoreRef.current; + if (currentRef) { + observer.observe(currentRef); + } + + return () => { + if (currentRef) { + observer.unobserve(currentRef); + } + }; + }, [hasMore, isLoadingMore, onLoadMore]); + + return ( +
+ + + + Time + Run + Task + Level + Message + + + + {logs.length === 0 && !hasFilters ? ( + + {!isLoading && } + + ) : logs.length === 0 ? ( + window.location.reload()} /> + ) : ( + logs.map((log) => { + const isSelected = selectedLogId === log.id; + const runPath = v3RunSpanPath( + organization, + project, + environment, + { friendlyId: log.runId }, + { spanId: log.spanId } + ); + + const handleRowClick = () => onLogSelect?.(log.id); + + return ( + + + + + + + + + {log.taskIdentifier} + + + + {log.level} + + + + + {highlightSearchText(log.message, searchTerm)} + + + + } + /> + + ); + }) + )} + +
+ {/* Infinite scroll trigger */} + {hasMore && logs.length > 0 && ( +
+ {showLoadMoreSpinner && ( +
+ Loading more… +
+ )} +
+ )} +
+ ); +} + +function NoLogs({ title }: { title: string }) { + return ( +
+ {title} +
+ ); +} + +function BlankState({ isLoading, onRefresh }: { isLoading?: boolean; onRefresh?: () => void }) { + if (isLoading) return ; + + const handleRefresh = onRefresh ?? (() => window.location.reload()); + + return ( + +
+ + No logs match your filters. Try refreshing or modifying your filters. + +
+ +
+
+ + ); +} diff --git a/apps/webapp/app/components/navigation/SideMenu.tsx b/apps/webapp/app/components/navigation/SideMenu.tsx index 93e6b13420..86678bc1ca 100644 --- a/apps/webapp/app/components/navigation/SideMenu.tsx +++ b/apps/webapp/app/components/navigation/SideMenu.tsx @@ -26,9 +26,10 @@ import { import { Link, useNavigation } from "@remix-run/react"; import { useEffect, useRef, useState, type ReactNode } from "react"; import simplur from "simplur"; -import { BranchEnvironmentIconSmall } from "~/assets/icons/EnvironmentIcons"; import { ConcurrencyIcon } from "~/assets/icons/ConcurrencyIcon"; +import { BranchEnvironmentIconSmall } from "~/assets/icons/EnvironmentIcons"; import { ListCheckedIcon } from "~/assets/icons/ListCheckedIcon"; +import { LogsIcon } from "~/assets/icons/LogsIcon"; import { RunsIconExtraSmall } from "~/assets/icons/RunsIcon"; import { TaskIconSmall } from "~/assets/icons/TaskIcon"; import { WaitpointTokenIcon } from "~/assets/icons/WaitpointTokenIcon"; @@ -64,6 +65,7 @@ import { v3DeploymentsPath, v3EnvironmentPath, v3EnvironmentVariablesPath, + v3LogsPath, v3ProjectAlertsPath, v3ProjectPath, v3ProjectSettingsPath, @@ -267,6 +269,16 @@ export function SideMenu({ to={v3DeploymentsPath(organization, project, environment)} data-action="deployments" /> + {(isAdmin || user.isImpersonating) && ( + } + /> + )} } side="right" + asChild={true} /> ); }; @@ -273,6 +274,7 @@ const DateTimeAccurateInner = ({ button={{formattedDateTime.replace(/\s/g, String.fromCharCode(32))}} content={tooltipContent} side="right" + asChild={true} /> ); }; diff --git a/apps/webapp/app/components/primitives/Popover.tsx b/apps/webapp/app/components/primitives/Popover.tsx index 7bdf9e902e..02454864c4 100644 --- a/apps/webapp/app/components/primitives/Popover.tsx +++ b/apps/webapp/app/components/primitives/Popover.tsx @@ -65,6 +65,7 @@ const PopoverMenuItem = React.forwardRef< className?: string; onClick?: React.MouseEventHandler; disabled?: boolean; + openInNewTab?: boolean; } >( ( @@ -78,6 +79,7 @@ const PopoverMenuItem = React.forwardRef< className, onClick, disabled, + openInNewTab = false, }, ref ) => { @@ -102,6 +104,8 @@ const PopoverMenuItem = React.forwardRef< ref={ref as React.Ref} className={cn("group/button focus-custom", contentProps.fullWidth ? "w-full" : "")} onClick={onClick as any} + target={openInNewTab ? "_blank" : undefined} + rel={openInNewTab ? "noopener noreferrer" : undefined} > {title} diff --git a/apps/webapp/app/components/primitives/Table.tsx b/apps/webapp/app/components/primitives/Table.tsx index faf24958c5..5b28ddbc5e 100644 --- a/apps/webapp/app/components/primitives/Table.tsx +++ b/apps/webapp/app/components/primitives/Table.tsx @@ -10,7 +10,10 @@ import { InfoIconTooltip, SimpleTooltip } from "./Tooltip"; const variants = { bright: { header: "bg-background-bright", + headerCell: "px-3 py-2.5 pb-3 text-sm", cell: "group-hover/table-row:bg-charcoal-750 group-has-[[tabindex='0']:focus]/table-row:bg-charcoal-750", + cellSize: "px-3 py-3", + cellText: "text-xs group-hover/table-row:text-text-bright", stickyCell: "bg-background-bright group-hover/table-row:bg-charcoal-750", menuButton: "bg-background-bright group-hover/table-row:bg-charcoal-750 group-hover/table-row:ring-charcoal-600/70 group-has-[[tabindex='0']:focus]/table-row:bg-charcoal-750", @@ -19,7 +22,10 @@ const variants = { }, "bright/no-hover": { header: "bg-transparent", + headerCell: "px-3 py-2.5 pb-3 text-sm", cell: "group-hover/table-row:bg-transparent", + cellSize: "px-3 py-3", + cellText: "text-xs", stickyCell: "bg-background-bright", menuButton: "bg-background-bright", menuButtonDivider: "", @@ -27,7 +33,22 @@ const variants = { }, dimmed: { header: "bg-background-dimmed", + headerCell: "px-3 py-2.5 pb-3 text-sm", cell: "group-hover/table-row:bg-charcoal-800 group-has-[[tabindex='0']:focus]/table-row:bg-background-bright", + cellSize: "px-3 py-3", + cellText: "text-xs group-hover/table-row:text-text-bright", + stickyCell: "group-hover/table-row:bg-charcoal-800", + menuButton: + "bg-background-dimmed group-hover/table-row:bg-charcoal-800 group-hover/table-row:ring-grid-bright group-has-[[tabindex='0']:focus]/table-row:bg-background-bright", + menuButtonDivider: "group-hover/table-row:border-grid-bright", + rowSelected: "bg-charcoal-750 group-hover/table-row:bg-charcoal-750", + }, + "compact/mono": { + header: "bg-background-dimmed", + headerCell: "px-2 py-1.5 text-sm", + cell: "group-hover/table-row:bg-charcoal-800 group-has-[[tabindex='0']:focus]/table-row:bg-background-bright", + cellSize: "px-2 py-1.5", + cellText: "text-xs font-mono group-hover/table-row:text-text-bright", stickyCell: "group-hover/table-row:bg-charcoal-800", menuButton: "bg-background-dimmed group-hover/table-row:bg-charcoal-800 group-hover/table-row:ring-grid-bright group-has-[[tabindex='0']:focus]/table-row:bg-background-bright", @@ -147,6 +168,7 @@ type TableHeaderCellProps = TableCellBasicProps & { export const TableHeaderCell = forwardRef( ({ className, alignment = "left", children, colSpan, hiddenLabel = false, tooltip }, ref) => { + const { variant } = useContext(TableContext); let alignmentClassName = "text-left"; switch (alignment) { case "center": @@ -164,7 +186,8 @@ export const TableHeaderCell = forwardRef( break; } + const { variant } = useContext(TableContext); const flexClasses = cn( - "flex w-full whitespace-nowrap px-3 py-3 items-center text-xs text-text-dimmed", + "flex w-full whitespace-nowrap items-center text-text-dimmed", + variants[variant].cellSize, + variants[variant].cellText, alignment === "left" ? "justify-start text-left" : alignment === "center" ? "justify-center text-center" : "justify-end text-right" ); - const { variant } = useContext(TableContext); return ( ); } @@ -63,6 +66,7 @@ export function PacketDisplay({ maxLines={20} showLineNumbers={false} showTextWrapping + searchTerm={searchTerm} /> ); } diff --git a/apps/webapp/app/components/runs/v3/RunFilters.tsx b/apps/webapp/app/components/runs/v3/RunFilters.tsx index 297d95be0b..5695081816 100644 --- a/apps/webapp/app/components/runs/v3/RunFilters.tsx +++ b/apps/webapp/app/components/runs/v3/RunFilters.tsx @@ -324,6 +324,10 @@ type RunFiltersProps = { }[]; rootOnlyDefault: boolean; hasFilters: boolean; + /** Hide the AI search input (useful when replacing with a custom search component) */ + hideSearch?: boolean; + /** Custom default period for the time filter (e.g., "1h", "7d") */ + defaultPeriod?: string; }; export function RunsFilters(props: RunFiltersProps) { @@ -344,9 +348,9 @@ export function RunsFilters(props: RunFiltersProps) { return (
- + {!props.hideSearch && } - + {hasFilters && (
diff --git a/apps/webapp/app/components/runs/v3/SharedFilters.tsx b/apps/webapp/app/components/runs/v3/SharedFilters.tsx index 89b7ab5611..bbb8ee8dc0 100644 --- a/apps/webapp/app/components/runs/v3/SharedFilters.tsx +++ b/apps/webapp/app/components/runs/v3/SharedFilters.tsx @@ -260,13 +260,18 @@ export function timeFilterRenderValues({ return { label, valueLabel, rangeType }; } -export function TimeFilter() { +export interface TimeFilterProps { + defaultPeriod?: string; +} + +export function TimeFilter({ defaultPeriod }: TimeFilterProps = {}) { const { value, del } = useSearchParams(); const { period, from, to, label, valueLabel } = timeFilters({ period: value("period"), from: value("from"), to: value("to"), + defaultPeriod, }); return ( @@ -287,6 +292,7 @@ export function TimeFilter() { period={period} from={from} to={to} + defaultPeriod={defaultPeriod} /> )} diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 6c0405bb79..31ef4deffe 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1175,6 +1175,18 @@ const EnvironmentSchema = z CLICKHOUSE_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"), + // Logs List Query Settings (for paginated log views) + CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE: z.coerce.number().int().default(256_000_000), + CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT: z.coerce.number().int().default(256_000_000), + CLICKHOUSE_LOGS_LIST_MAX_THREADS: z.coerce.number().int().default(2), + CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ: z.coerce.number().int().default(10_000_000), + CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME: z.coerce.number().int().default(120), + + // Logs Detail Query Settings (for single log views) + CLICKHOUSE_LOGS_DETAIL_MAX_MEMORY_USAGE: z.coerce.number().int().default(64_000_000), + CLICKHOUSE_LOGS_DETAIL_MAX_THREADS: z.coerce.number().int().default(2), + CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME: z.coerce.number().int().default(60), + EVENTS_CLICKHOUSE_URL: z .string() .optional() diff --git a/apps/webapp/app/presenters/RunFilters.server.ts b/apps/webapp/app/presenters/RunFilters.server.ts index 8d70b4d3bd..ef428f6ca2 100644 --- a/apps/webapp/app/presenters/RunFilters.server.ts +++ b/apps/webapp/app/presenters/RunFilters.server.ts @@ -1,3 +1,4 @@ +import { type TaskRunStatus } from "@trigger.dev/database"; import { getRunFiltersFromSearchParams, TaskRunListSearchFilters, @@ -39,7 +40,7 @@ export async function getRunFiltersFromRequest(request: Request): Promise>; + +export class LogDetailPresenter { + constructor( + private readonly replica: PrismaClientOrTransaction, + private readonly clickhouse: ClickHouse + ) {} + + public async call(options: LogDetailOptions) { + const { environmentId, organizationId, projectId, spanId, traceId, startTime } = options; + + // Build ClickHouse query + const queryBuilder = this.clickhouse.taskEventsV2.logDetailQueryBuilder(); + + // Required filters - spanId, traceId, and startTime uniquely identify the log + // Multiple events can share the same spanId (span, span events, logs), so startTime is needed + queryBuilder.where("environment_id = {environmentId: String}", { + environmentId, + }); + queryBuilder.where("organization_id = {organizationId: String}", { + organizationId, + }); + queryBuilder.where("project_id = {projectId: String}", { projectId }); + queryBuilder.where("span_id = {spanId: String}", { spanId }); + queryBuilder.where("trace_id = {traceId: String}", { traceId }); + queryBuilder.where("start_time = {startTime: String}", { startTime }); + + queryBuilder.limit(1); + + // Execute query + const [queryError, records] = await queryBuilder.execute(); + + if (queryError) { + throw queryError; + } + + if (!records || records.length === 0) { + return null; + } + + const log = records[0]; + + // Parse metadata and attributes + let parsedMetadata: Record = {}; + let parsedAttributes: Record = {}; + let rawAttributesString = ""; + + try { + if (log.metadata) { + parsedMetadata = JSON.parse(log.metadata) as Record; + } + } catch { + // Ignore parse errors + } + + try { + // Handle attributes which could be a JSON object or string + if (log.attributes) { + if (typeof log.attributes === "string") { + parsedAttributes = JSON.parse(log.attributes) as Record; + rawAttributesString = log.attributes; + } else if (typeof log.attributes === "object") { + parsedAttributes = log.attributes as Record; + rawAttributesString = JSON.stringify(log.attributes); + } + } + } catch { + // Ignore parse errors + } + + return { + // Use :: separator to match LogsListPresenter format + id: `${log.trace_id}::${log.span_id}::${log.run_id}::${log.start_time}`, + runId: log.run_id, + taskIdentifier: log.task_identifier, + startTime: convertClickhouseDateTime64ToJsDate(log.start_time).toISOString(), + traceId: log.trace_id, + spanId: log.span_id, + parentSpanId: log.parent_span_id || null, + message: log.message, + kind: log.kind, + status: log.status, + duration: typeof log.duration === "number" ? log.duration : Number(log.duration), + level: kindToLevel(log.kind, log.status), + metadata: parsedMetadata, + attributes: parsedAttributes, + // Raw strings for display + rawMetadata: log.metadata, + rawAttributes: rawAttributesString, + }; + } +} diff --git a/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts b/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts new file mode 100644 index 0000000000..0b5f2c175a --- /dev/null +++ b/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts @@ -0,0 +1,587 @@ +import { z } from "zod"; +import { type ClickHouse, type LogsListResult } from "@internal/clickhouse"; +import { MachinePresetName } from "@trigger.dev/core/v3"; +import { + type PrismaClient, + type PrismaClientOrTransaction, + type TaskRunStatus, + TaskRunStatus as TaskRunStatusEnum, + TaskTriggerSource, +} from "@trigger.dev/database"; + +// Create a schema that validates TaskRunStatus enum values +const TaskRunStatusSchema = z.array(z.nativeEnum(TaskRunStatusEnum)); +import parseDuration from "parse-duration"; +import { type Direction } from "~/components/ListPagination"; +import { timeFilters } from "~/components/runs/v3/SharedFilters"; +import { findDisplayableEnvironment } from "~/models/runtimeEnvironment.server"; +import { getAllTaskIdentifiers } from "~/models/task.server"; +import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; +import { ServiceValidationError } from "~/v3/services/baseService.server"; +import { + convertDateToClickhouseDateTime, + convertClickhouseDateTime64ToJsDate, +} from "~/v3/eventRepository/clickhouseEventRepository.server"; +import { kindToLevel, type LogLevel, LogLevelSchema } from "~/utils/logUtils"; + +export type { LogLevel }; + +type ErrorAttributes = { + error?: { + message?: unknown; + }; + [key: string]: unknown; +}; + +export type LogsListOptions = { + userId?: string; + projectId: string; + // filters + tasks?: string[]; + versions?: string[]; + statuses?: TaskRunStatus[]; + tags?: string[]; + scheduleId?: string; + period?: string; + bulkId?: string; + from?: number; + to?: number; + isTest?: boolean; + rootOnly?: boolean; + batchId?: string; + runId?: string[]; + queues?: string[]; + machines?: MachinePresetName[]; + levels?: LogLevel[]; + defaultPeriod?: string; + // search + search?: string; + includeDebugLogs?: boolean; + // pagination + direction?: Direction; + cursor?: string; + pageSize?: number; +}; + +export const LogsListOptionsSchema = z.object({ + userId: z.string().optional(), + projectId: z.string(), + tasks: z.array(z.string()).optional(), + versions: z.array(z.string()).optional(), + statuses: TaskRunStatusSchema.optional(), + tags: z.array(z.string()).optional(), + scheduleId: z.string().optional(), + period: z.string().optional(), + bulkId: z.string().optional(), + from: z.number().int().nonnegative().optional(), + to: z.number().int().nonnegative().optional(), + isTest: z.boolean().optional(), + rootOnly: z.boolean().optional(), + batchId: z.string().optional(), + runId: z.array(z.string()).optional(), + queues: z.array(z.string()).optional(), + machines: z.array(MachinePresetName).optional(), + levels: z.array(LogLevelSchema).optional(), + defaultPeriod: z.string().optional(), + search: z.string().max(1000).optional(), + includeDebugLogs: z.boolean().optional(), + direction: z.enum(["forward", "backward"]).optional(), + cursor: z.string().optional(), + pageSize: z.number().int().positive().max(1000).optional(), +}); + +const DEFAULT_PAGE_SIZE = 50; +const MAX_RUN_IDS = 5000; + +export type LogsList = Awaited>; +export type LogEntry = LogsList["logs"][0]; +export type LogsListAppliedFilters = LogsList["filters"]; + +// Cursor is a base64 encoded JSON of the pagination keys +type LogCursor = { + startTime: string; + traceId: string; + spanId: string; + runId: string; +}; + +const LogCursorSchema = z.object({ + startTime: z.string(), + traceId: z.string(), + spanId: z.string(), + runId: z.string(), +}); + +function encodeCursor(cursor: LogCursor): string { + return Buffer.from(JSON.stringify(cursor)).toString("base64"); +} + +function decodeCursor(cursor: string): LogCursor | null { + try { + const decoded = Buffer.from(cursor, "base64").toString("utf-8"); + const parsed = JSON.parse(decoded); + const validated = LogCursorSchema.safeParse(parsed); + if (!validated.success) { + return null; + } + return validated.data; + } catch { + return null; + } +} + +// Convert display level to ClickHouse kinds and statuses +function levelToKindsAndStatuses( + level: LogLevel +): { kinds?: string[]; statuses?: string[] } { + switch (level) { + case "DEBUG": + return { kinds: ["DEBUG_EVENT", "LOG_DEBUG"] }; + case "INFO": + return { kinds: ["LOG_INFO", "LOG_LOG"] }; + case "WARN": + return { kinds: ["LOG_WARN"] }; + case "ERROR": + return { kinds: ["LOG_ERROR"], statuses: ["ERROR"] }; + case "CANCELLED": + return { statuses: ["CANCELLED"] }; + case "TRACE": + return { kinds: ["SPAN", "ANCESTOR_OVERRIDE", "SPAN_EVENT"] }; + } +} + + +function convertDateToNanoseconds(date: Date): bigint { + return BigInt(date.getTime()) * 1_000_000n; +} + +function formatNanosecondsForClickhouse(ns: bigint): string { + const nsString = ns.toString(); + // Handle negative numbers (dates before 1970-01-01) + if (nsString.startsWith("-")) { + const absString = nsString.slice(1); + const padded = absString.padStart(19, "0"); + return "-" + padded.slice(0, 10) + "." + padded.slice(10); + } + // Pad positive numbers to 19 digits to ensure correct slicing + const padded = nsString.padStart(19, "0"); + return padded.slice(0, 10) + "." + padded.slice(10); +} + +export class LogsListPresenter { + constructor( + private readonly replica: PrismaClientOrTransaction, + private readonly clickhouse: ClickHouse + ) {} + + public async call( + organizationId: string, + environmentId: string, + { + userId, + projectId, + tasks, + versions, + statuses, + tags, + scheduleId, + period, + bulkId, + isTest, + rootOnly, + batchId, + runId, + queues, + machines, + levels, + search, + from, + to, + cursor, + pageSize = DEFAULT_PAGE_SIZE, + includeDebugLogs = true, + defaultPeriod, + }: LogsListOptions + ) { + const time = timeFilters({ + period, + from, + to, + defaultPeriod, + }); + + let effectiveFrom = time.from; + let effectiveTo = time.to; + + if (!effectiveFrom && !effectiveTo && time.period) { + const periodMs = parseDuration(time.period); + if (periodMs) { + effectiveFrom = new Date(Date.now() - periodMs); + effectiveTo = new Date(); + } + } + + const hasStatusFilters = statuses && statuses.length > 0; + const hasRunLevelFilters = + (versions !== undefined && versions.length > 0) || + hasStatusFilters || + (bulkId !== undefined && bulkId !== "") || + (scheduleId !== undefined && scheduleId !== "") || + (tags !== undefined && tags.length > 0) || + batchId !== undefined || + (runId !== undefined && runId.length > 0) || + (queues !== undefined && queues.length > 0) || + (machines !== undefined && machines.length > 0) || + typeof isTest === "boolean" || + rootOnly === true; + + const hasFilters = + (tasks !== undefined && tasks.length > 0) || + hasRunLevelFilters || + (levels !== undefined && levels.length > 0) || + (search !== undefined && search !== "") || + !time.isDefault; + + const possibleTasksAsync = getAllTaskIdentifiers( + this.replica, + environmentId + ); + + const bulkActionsAsync = this.replica.bulkActionGroup.findMany({ + select: { + friendlyId: true, + type: true, + createdAt: true, + name: true, + }, + where: { + projectId: projectId, + environmentId, + }, + orderBy: { + createdAt: "desc", + }, + take: 20, + }); + + const [possibleTasks, bulkActions, displayableEnvironment] = + await Promise.all([ + possibleTasksAsync, + bulkActionsAsync, + findDisplayableEnvironment(environmentId, userId), + ]); + + if ( + bulkId && + !bulkActions.some((bulkAction) => bulkAction.friendlyId === bulkId) + ) { + const selectedBulkAction = + await this.replica.bulkActionGroup.findFirst({ + select: { + friendlyId: true, + type: true, + createdAt: true, + name: true, + }, + where: { + friendlyId: bulkId, + projectId, + environmentId, + }, + }); + + if (selectedBulkAction) { + bulkActions.push(selectedBulkAction); + } + } + + if (!displayableEnvironment) { + throw new ServiceValidationError("No environment found"); + } + + // If we have run-level filters, we need to first get matching run IDs from Postgres + let runIds: string[] | undefined; + if (hasRunLevelFilters) { + const runsRepository = new RunsRepository({ + clickhouse: this.clickhouse, + prisma: this.replica, + }); + + function clampToNow(date: Date): Date { + const now = new Date(); + return date > now ? now : date; + } + + runIds = await runsRepository.listFriendlyRunIds({ + organizationId, + environmentId, + projectId, + tasks, + versions, + statuses, + tags, + scheduleId, + period, + from: effectiveFrom ? effectiveFrom.getTime() : undefined, + to: effectiveTo ? clampToNow(effectiveTo).getTime() : undefined, + isTest, + rootOnly, + batchId, + runId, + bulkId, + queues, + machines, + page: { + size: MAX_RUN_IDS, + direction: "forward", + }, + }); + + if (runIds.length === 0) { + return { + logs: [], + pagination: { + next: undefined, + previous: undefined, + }, + possibleTasks: possibleTasks + .map((task) => ({ + slug: task.slug, + triggerSource: task.triggerSource, + })) + .sort((a, b) => a.slug.localeCompare(b.slug)), + bulkActions: bulkActions.map((bulkAction) => ({ + id: bulkAction.friendlyId, + type: bulkAction.type, + createdAt: bulkAction.createdAt, + name: bulkAction.name || bulkAction.friendlyId, + })), + filters: { + tasks: tasks || [], + versions: versions || [], + statuses: statuses || [], + levels: levels || [], + from: effectiveFrom, + to: effectiveTo, + }, + hasFilters, + hasAnyLogs: false, + searchTerm: search, + }; + } + } + + const queryBuilder = this.clickhouse.taskEventsV2.logsListQueryBuilder(); + + queryBuilder.prewhere("environment_id = {environmentId: String}", { + environmentId, + }); + + queryBuilder.where("organization_id = {organizationId: String}", { + organizationId, + }); + queryBuilder.where("project_id = {projectId: String}", { projectId }); + + // Time filters - inserted_at in PREWHERE for partition pruning, start_time in WHERE + if (effectiveFrom) { + const fromNs = convertDateToNanoseconds(effectiveFrom); + queryBuilder.prewhere("inserted_at >= {insertedAtStart: DateTime64(3)}", { + insertedAtStart: convertDateToClickhouseDateTime(effectiveFrom), + }); + queryBuilder.where("start_time >= {fromTime: String}", { + fromTime: formatNanosecondsForClickhouse(fromNs), + }); + } + + if (effectiveTo) { + const clampedTo = effectiveTo > new Date() ? new Date() : effectiveTo; + const toNs = convertDateToNanoseconds(clampedTo); + queryBuilder.prewhere("inserted_at <= {insertedAtEnd: DateTime64(3)}", { + insertedAtEnd: convertDateToClickhouseDateTime(clampedTo), + }); + queryBuilder.where("start_time <= {toTime: String}", { + toTime: formatNanosecondsForClickhouse(toNs), + }); + } + + // Task filter (applies directly to ClickHouse) + if (tasks && tasks.length > 0) { + queryBuilder.where("task_identifier IN {tasks: Array(String)}", { + tasks, + }); + } + + // Run IDs filter (from Postgres lookup) + if (runIds && runIds.length > 0) { + queryBuilder.where("run_id IN {runIds: Array(String)}", { runIds }); + } + + // Case-insensitive search in message, attributes, and status fields + if (search && search.trim() !== "") { + const searchTerm = search.trim(); + queryBuilder.where( + "(message ilike {searchPattern: String} OR attributes_text ilike {searchPattern: String} OR status = {statusTerm: String})", + { + searchPattern: `%${searchTerm}%`, + statusTerm: searchTerm.toUpperCase(), + } + ); + } + + + if (levels && levels.length > 0) { + const conditions: string[] = []; + const params: Record = {}; + const hasErrorOrCancelledLevel = levels.includes("ERROR") || levels.includes("CANCELLED"); + + for (const level of levels) { + const filter = levelToKindsAndStatuses(level); + const levelConditions: string[] = []; + + if (filter.kinds && filter.kinds.length > 0) { + const kindsKey = `kinds_${level}`; + let kindCondition = `kind IN {${kindsKey}: Array(String)}`; + + // For TRACE: exclude error/cancelled traces if ERROR/CANCELLED not explicitly selected + if (level === "TRACE" && !hasErrorOrCancelledLevel) { + kindCondition += ` AND status NOT IN {excluded_statuses: Array(String)}`; + params["excluded_statuses"] = ["ERROR", "CANCELLED"]; + } + + levelConditions.push(kindCondition); + params[kindsKey] = filter.kinds; + } + + if (filter.statuses && filter.statuses.length > 0) { + const statusesKey = `statuses_${level}`; + levelConditions.push(`status IN {${statusesKey}: Array(String)}`); + params[statusesKey] = filter.statuses; + } + + if (levelConditions.length > 0) { + conditions.push(`(${levelConditions.join(" OR ")})`); + } + } + + if (conditions.length > 0) { + queryBuilder.where(`(${conditions.join(" OR ")})`, params); + } + } + + // Debug logs are available only to admins + if (includeDebugLogs === false) { + queryBuilder.where("kind NOT IN {debugKinds: Array(String)}", { + debugKinds: ["DEBUG_EVENT", "LOG_DEBUG"], + }); + } + + queryBuilder.where("NOT (kind = 'SPAN' AND status = 'PARTIAL')"); + + + // Cursor pagination + const decodedCursor = cursor ? decodeCursor(cursor) : null; + if (decodedCursor) { + queryBuilder.where( + "(start_time, trace_id, span_id, run_id) < ({cursorStartTime: String}, {cursorTraceId: String}, {cursorSpanId: String}, {cursorRunId: String})", + { + cursorStartTime: decodedCursor.startTime, + cursorTraceId: decodedCursor.traceId, + cursorSpanId: decodedCursor.spanId, + cursorRunId: decodedCursor.runId, + } + ); + } + + queryBuilder.orderBy("start_time DESC, trace_id DESC, span_id DESC, run_id DESC"); + + // Limit + 1 to check if there are more results + queryBuilder.limit(pageSize + 1); + + const [queryError, records] = await queryBuilder.execute(); + + if (queryError) { + throw queryError; + } + + const results = records || []; + const hasMore = results.length > pageSize; + const logs = results.slice(0, pageSize); + + // Build next cursor from the last item + let nextCursor: string | undefined; + if (hasMore && logs.length > 0) { + const lastLog = logs[logs.length - 1]; + nextCursor = encodeCursor({ + startTime: lastLog.start_time, + traceId: lastLog.trace_id, + spanId: lastLog.span_id, + runId: lastLog.run_id, + }); + } + + // Transform results + // Use :: as separator since dash conflicts with date format in start_time + const transformedLogs = logs.map((log) => { + let displayMessage = log.message; + + // For error logs with status ERROR, try to extract error message from attributes + if (log.status === "ERROR" && log.attributes) { + try { + let attributes = log.attributes as ErrorAttributes; + + if (attributes?.error?.message && typeof attributes.error.message === 'string') { + displayMessage = attributes.error.message; + } + } catch { + // If attributes parsing fails, use the regular message + } + } + + return { + id: `${log.trace_id}::${log.span_id}::${log.run_id}::${log.start_time}`, + runId: log.run_id, + taskIdentifier: log.task_identifier, + startTime: convertClickhouseDateTime64ToJsDate(log.start_time).toISOString(), + traceId: log.trace_id, + spanId: log.span_id, + parentSpanId: log.parent_span_id || null, + message: displayMessage, + kind: log.kind, + status: log.status, + duration: typeof log.duration === "number" ? log.duration : Number(log.duration), + level: kindToLevel(log.kind, log.status), + }; + }); + + return { + logs: transformedLogs, + pagination: { + next: nextCursor, + previous: undefined, // For now, only support forward pagination + }, + possibleTasks: possibleTasks + .map((task) => ({ + slug: task.slug, + triggerSource: task.triggerSource, + })) + .sort((a, b) => a.slug.localeCompare(b.slug)), + bulkActions: bulkActions.map((bulkAction) => ({ + id: bulkAction.friendlyId, + type: bulkAction.type, + createdAt: bulkAction.createdAt, + name: bulkAction.name || bulkAction.friendlyId, + })), + filters: { + tasks: tasks || [], + versions: versions || [], + statuses: statuses || [], + levels: levels || [], + from: effectiveFrom, + to: effectiveTo, + }, + hasFilters, + hasAnyLogs: transformedLogs.length > 0, + searchTerm: search, + }; + } +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx new file mode 100644 index 0000000000..aad6a2be53 --- /dev/null +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx @@ -0,0 +1,340 @@ +import { type LoaderFunctionArgs , redirect} from "@remix-run/server-runtime"; +import { type MetaFunction, useFetcher, useNavigation, useLocation } from "@remix-run/react"; +import { + TypedAwait, + typeddefer, + type UseDataFunctionReturn, + useTypedLoaderData, +} from "remix-typedjson"; +import { requireUser } from "~/services/session.server"; + +import { EnvironmentParamSchema } from "~/utils/pathBuilder"; +import { findProjectBySlug } from "~/models/project.server"; +import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { getRunFiltersFromRequest } from "~/presenters/RunFilters.server"; +import { LogsListPresenter } from "~/presenters/v3/LogsListPresenter.server"; +import type { LogLevel } from "~/utils/logUtils"; +import { $replica } from "~/db.server"; +import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { + setRootOnlyFilterPreference, + uiPreferencesStorage, +} from "~/services/preferences/uiPreferences.server"; +import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; +import { PageBody, PageContainer } from "~/components/layout/AppLayout"; +import { Suspense, useCallback, useEffect, useMemo, useState, useTransition } from "react"; +import { Spinner } from "~/components/primitives/Spinner"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { Callout } from "~/components/primitives/Callout"; +import { RunsFilters } from "~/components/runs/v3/RunFilters"; +import { LogsTable } from "~/components/logs/LogsTable"; +import type { LogEntry } from "~/presenters/v3/LogsListPresenter.server"; +import { LogDetailView } from "~/components/logs/LogDetailView"; +import { LogsSearchInput } from "~/components/logs/LogsSearchInput"; +import { LogsLevelFilter } from "~/components/logs/LogsLevelFilter"; +import { + ResizableHandle, + ResizablePanel, + ResizablePanelGroup, +} from "~/components/primitives/Resizable"; +import { Switch } from "~/components/primitives/Switch"; + +// Valid log levels for filtering +const validLevels: LogLevel[] = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "CANCELLED"]; + +function parseLevelsFromUrl(url: URL): LogLevel[] | undefined { + const levelParams = url.searchParams.getAll("levels").filter((v) => v.length > 0); + if (levelParams.length === 0) return undefined; + return levelParams.filter((l): l is LogLevel => validLevels.includes(l as LogLevel)); +} + +export const meta: MetaFunction = () => { + return [ + { + title: `Logs | Trigger.dev`, + }, + ]; +}; + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const user = await requireUser(request); + const userId = user.id; + const isAdmin = user.admin || user.isImpersonating; + + if (!isAdmin) { + throw redirect("/"); + } + + const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params); + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) { + throw new Response("Project not found", { status: 404 }); + } + + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) { + throw new Response("Environment not found", { status: 404 }); + } + + const filters = await getRunFiltersFromRequest(request); + + // Get search term, levels, and showDebug from query params + const url = new URL(request.url); + const search = url.searchParams.get("search") ?? undefined; + const levels = parseLevelsFromUrl(url); + const showDebug = url.searchParams.get("showDebug") === "true"; + + const presenter = new LogsListPresenter($replica, clickhouseClient); + const list = presenter.call(project.organizationId, environment.id, { + userId, + projectId: project.id, + ...filters, + search, + levels, + includeDebugLogs: isAdmin && showDebug, + defaultPeriod: "1h", + }); + + const session = await setRootOnlyFilterPreference(filters.rootOnly, request); + const cookieValue = await uiPreferencesStorage.commitSession(session); + + return typeddefer( + { + data: list, + rootOnlyDefault: filters.rootOnly, + filters, + isAdmin, + showDebug, + defaultPeriod: "1h", + }, + { + headers: { + "Set-Cookie": cookieValue, + }, + } + ); +}; + +export default function Page() { + const { data, rootOnlyDefault, isAdmin, showDebug, defaultPeriod } = useTypedLoaderData(); + + return ( + + + + + + + +
+
+
+ + Loading logs +
+
+
+ } + > + + + Unable to load your logs. Please refresh the page or try again in a moment. + +
+ } + > + {(list) => { + return ( + + ); + }} + + + + + ); +} + +function LogsList({ + list, + rootOnlyDefault, + isAdmin, + showDebug, + defaultPeriod, +}: { + list: Awaited["data"]>; + rootOnlyDefault: boolean; + isAdmin: boolean; + showDebug: boolean; + defaultPeriod?: string; +}) { + const navigation = useNavigation(); + const location = useLocation(); + const fetcher = useFetcher<{ logs: LogEntry[]; pagination: { next?: string } }>(); + const [, startTransition] = useTransition(); + const isLoading = navigation.state !== "idle"; + + // Accumulated logs state + const [accumulatedLogs, setAccumulatedLogs] = useState(list.logs); + const [nextCursor, setNextCursor] = useState(list.pagination.next); + + // Selected log state - managed locally to avoid triggering navigation + const [selectedLogId, setSelectedLogId] = useState(); + + const handleDebugToggle = useCallback( + (checked: boolean) => { + const url = new URL(window.location.href); + if (checked) { + url.searchParams.set("showDebug", "true"); + } else { + url.searchParams.delete("showDebug"); + } + window.location.href = url.toString(); + }, + [] + ); + + + // Reset accumulated logs when the initial list changes (e.g., filters change) + useEffect(() => { + setAccumulatedLogs(list.logs); + setNextCursor(list.pagination.next); + }, [list.logs, list.pagination.next]); + + // Append new logs when fetcher completes (with deduplication) + useEffect(() => { + if (fetcher.data && fetcher.state === "idle") { + const existingIds = new Set(accumulatedLogs.map((log) => log.id)); + const newLogs = fetcher.data.logs.filter((log) => !existingIds.has(log.id)); + if (newLogs.length > 0) { + setAccumulatedLogs((prev) => [...prev, ...newLogs]); + setNextCursor(fetcher.data.pagination.next); + } + } + }, [fetcher.data, fetcher.state, accumulatedLogs]); + + // Build resource URL for loading more + const loadMoreUrl = useMemo(() => { + if (!nextCursor) return null; + const resourcePath = `/resources${location.pathname}`; + const params = new URLSearchParams(location.search); + params.set("cursor", nextCursor); + params.delete("log"); + return `${resourcePath}?${params.toString()}`; + }, [location.pathname, location.search, nextCursor]); + + const handleLoadMore = useCallback(() => { + if (loadMoreUrl && fetcher.state === "idle") { + fetcher.load(loadMoreUrl); + } + }, [loadMoreUrl, fetcher]); + + const selectedLog = useMemo(() => { + if (!selectedLogId) return undefined; + return accumulatedLogs.find((log) => log.id === selectedLogId); + }, [selectedLogId, accumulatedLogs]); + + const updateUrlWithLog = useCallback( + (logId: string | undefined) => { + const url = new URL(window.location.href); + if (logId) { + url.searchParams.set("log", logId); + } else { + url.searchParams.delete("log"); + } + window.history.replaceState(null, "", url.toString()); + }, + [] + ); + + const handleLogSelect = useCallback( + (logId: string) => { + startTransition(() => { + setSelectedLogId(logId); + }); + updateUrlWithLog(logId); + }, + [updateUrlWithLog, startTransition] + ); + + const handleClosePanel = useCallback(() => { + startTransition(() => { + setSelectedLogId(undefined); + }); + updateUrlWithLog(undefined); + }, [updateUrlWithLog, startTransition]); + + return ( + + +
+ {/* Filters */} +
+
+ + + +
+ {isAdmin && ( + + )} +
+ + {/* Table */} + +
+ + + {/* Side panel for log details */} + {selectedLogId && ( + <> + + + }> + + + + + )} + + ); +} diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.run.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.run.tsx new file mode 100644 index 0000000000..15986c305a --- /dev/null +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.run.tsx @@ -0,0 +1,202 @@ +import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { json } from "@remix-run/node"; +import { z } from "zod"; +import { MachinePresetName } from "@trigger.dev/core/v3"; +import { requireUserId } from "~/services/session.server"; +import { EnvironmentParamSchema } from "~/utils/pathBuilder"; +import { findProjectBySlug } from "~/models/project.server"; +import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { $replica } from "~/db.server"; +import type { TaskRunStatus } from "@trigger.dev/database"; + +// Valid TaskRunStatus values +const VALID_TASK_RUN_STATUSES = [ + "PENDING", + "QUEUED", + "EXECUTING", + "WAITING_FOR_EXECUTION", + "WAITING", + "COMPLETED_SUCCESSFULLY", + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "FAILURE", + "CANCELED", +] as const; + +// Schema for validating run context data +export const RunContextSchema = z.object({ + id: z.string(), + friendlyId: z.string(), + taskIdentifier: z.string(), + status: z.enum(VALID_TASK_RUN_STATUSES), + createdAt: z.string().datetime(), + startedAt: z.string().datetime().optional(), + completedAt: z.string().datetime().optional(), + isTest: z.boolean(), + tags: z.array(z.string()), + queue: z.string(), + concurrencyKey: z.string().nullable(), + usageDurationMs: z.number(), + costInCents: z.number(), + baseCostInCents: z.number(), + machinePreset: MachinePresetName.nullable(), + version: z.string().optional(), + rootRun: z + .object({ + friendlyId: z.string(), + taskIdentifier: z.string(), + }) + .nullable(), + parentRun: z + .object({ + friendlyId: z.string(), + taskIdentifier: z.string(), + }) + .nullable(), + batch: z + .object({ + friendlyId: z.string(), + }) + .nullable(), + schedule: z + .object({ + friendlyId: z.string(), + }) + .nullable(), +}); + +export type RunContext = z.infer; + +// Fetch run context for a log entry +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const { projectParam, organizationSlug, envParam, logId } = { + ...EnvironmentParamSchema.parse(params), + logId: params.logId, + }; + + if (!logId) { + throw new Response("Log ID is required", { status: 400 }); + } + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) { + throw new Response("Project not found", { status: 404 }); + } + + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) { + throw new Response("Environment not found", { status: 404 }); + } + + // Parse the logId to extract runId + // Log ID format: traceId::spanId::runId::startTime (base64 encoded or plain) + const url = new URL(request.url); + const runId = url.searchParams.get("runId"); + + if (!runId) { + throw new Response("Run ID is required", { status: 400 }); + } + + // Fetch run details from Postgres + const run = await $replica.taskRun.findFirst({ + select: { + id: true, + friendlyId: true, + taskIdentifier: true, + status: true, + createdAt: true, + startedAt: true, + completedAt: true, + isTest: true, + runTags: true, + queue: true, + concurrencyKey: true, + usageDurationMs: true, + costInCents: true, + baseCostInCents: true, + machinePreset: true, + scheduleId: true, + lockedToVersion: { + select: { + version: true, + }, + }, + rootTaskRun: { + select: { + friendlyId: true, + taskIdentifier: true, + }, + }, + parentTaskRun: { + select: { + friendlyId: true, + taskIdentifier: true, + }, + }, + batch: { + select: { + friendlyId: true, + }, + }, + }, + where: { + friendlyId: runId, + runtimeEnvironmentId: environment.id, + }, + }); + + if (!run) { + return json({ run: null }); + } + + // Fetch schedule if scheduleId exists + let schedule: { friendlyId: string } | null = null; + if (run.scheduleId) { + const scheduleData = await $replica.taskSchedule.findFirst({ + select: { friendlyId: true }, + where: { id: run.scheduleId }, + }); + schedule = scheduleData; + } + + const runData = { + id: run.id, + friendlyId: run.friendlyId, + taskIdentifier: run.taskIdentifier, + status: run.status, + createdAt: run.createdAt.toISOString(), + startedAt: run.startedAt?.toISOString(), + completedAt: run.completedAt?.toISOString(), + isTest: run.isTest, + tags: run.runTags, + queue: run.queue, + concurrencyKey: run.concurrencyKey, + usageDurationMs: run.usageDurationMs, + costInCents: run.costInCents, + baseCostInCents: run.baseCostInCents, + machinePreset: run.machinePreset, + version: run.lockedToVersion?.version, + rootRun: run.rootTaskRun + ? { + friendlyId: run.rootTaskRun.friendlyId, + taskIdentifier: run.rootTaskRun.taskIdentifier, + } + : null, + parentRun: run.parentTaskRun + ? { + friendlyId: run.parentTaskRun.friendlyId, + taskIdentifier: run.parentTaskRun.taskIdentifier, + } + : null, + batch: run.batch ? { friendlyId: run.batch.friendlyId } : null, + schedule: schedule, + }; + + // Validate the run data + const validatedRun = RunContextSchema.parse(runData); + + return json({ + run: validatedRun, + }); +}; diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.spans.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.spans.tsx new file mode 100644 index 0000000000..53bc655a02 --- /dev/null +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.spans.tsx @@ -0,0 +1,100 @@ +import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { json } from "@remix-run/node"; +import { requireUserId } from "~/services/session.server"; +import { EnvironmentParamSchema } from "~/utils/pathBuilder"; +import { findProjectBySlug } from "~/models/project.server"; +import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { clickhouseClient } from "~/services/clickhouseInstance.server"; + +// Convert ClickHouse kind to display level +function kindToLevel( + kind: string +): "TRACE" | "DEBUG" | "INFO" | "WARN" | "ERROR" | "LOG" { + switch (kind) { + case "DEBUG_EVENT": + case "LOG_DEBUG": + return "DEBUG"; + case "LOG_INFO": + return "INFO"; + case "LOG_WARN": + return "WARN"; + case "LOG_ERROR": + return "ERROR"; + case "LOG_LOG": + return "LOG"; + case "SPAN": + case "ANCESTOR_OVERRIDE": + case "SPAN_EVENT": + default: + return "TRACE"; + } +} + +// Fetch related spans for a log entry from the same trace +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const { projectParam, organizationSlug, envParam, logId } = { + ...EnvironmentParamSchema.parse(params), + logId: params.logId, + }; + + if (!logId) { + throw new Response("Log ID is required", { status: 400 }); + } + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) { + throw new Response("Project not found", { status: 404 }); + } + + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) { + throw new Response("Environment not found", { status: 404 }); + } + + // Get trace ID and run ID from query params + const url = new URL(request.url); + const traceId = url.searchParams.get("traceId"); + const runId = url.searchParams.get("runId"); + const currentSpanId = url.searchParams.get("spanId"); + + if (!traceId || !runId) { + throw new Response("Trace ID and Run ID are required", { status: 400 }); + } + + // Query ClickHouse for related spans in the same trace + const queryBuilder = clickhouseClient.taskEventsV2.logsListQueryBuilder(); + + queryBuilder.where("environment_id = {environmentId: String}", { + environmentId: environment.id, + }); + queryBuilder.where("trace_id = {traceId: String}", { traceId }); + queryBuilder.where("run_id = {runId: String}", { runId }); + + // Order by start time to show spans in chronological order + queryBuilder.orderBy("start_time ASC"); + queryBuilder.limit(50); + + const [queryError, records] = await queryBuilder.execute(); + + if (queryError) { + throw queryError; + } + + const results = records || []; + + const spans = results.map((row) => ({ + id: `${row.trace_id}::${row.span_id}::${row.run_id}::${row.start_time}`, + spanId: row.span_id, + parentSpanId: row.parent_span_id || null, + message: row.message.substring(0, 200), // Truncate for list view + kind: row.kind, + level: kindToLevel(row.kind), + status: row.status, + startTime: new Date(Number(row.start_time) / 1_000_000).toISOString(), + duration: Number(row.duration), + isCurrent: row.span_id === currentSpanId, + })); + + return json({ spans }); +}; diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx new file mode 100644 index 0000000000..fe162fb34c --- /dev/null +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx @@ -0,0 +1,60 @@ +import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { typedjson } from "remix-typedjson"; +import { z } from "zod"; +import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { requireUserId } from "~/services/session.server"; +import { LogDetailPresenter } from "~/presenters/v3/LogDetailPresenter.server"; +import { findProjectBySlug } from "~/models/project.server"; +import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { $replica } from "~/db.server"; + +const LogIdParamsSchema = z.object({ + organizationSlug: z.string(), + projectParam: z.string(), + envParam: z.string(), + logId: z.string(), +}); + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const { organizationSlug, projectParam, envParam, logId } = LogIdParamsSchema.parse(params); + + // Validate access to project and environment + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) { + throw new Response("Project not found", { status: 404 }); + } + + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) { + throw new Response("Environment not found", { status: 404 }); + } + + // Parse logId to extract traceId, spanId, runId, and startTime + // Format: {traceId}::{spanId}::{runId}::{startTime} + // All 4 parts are needed to uniquely identify a log entry (multiple events can share the same spanId) + const decodedLogId = decodeURIComponent(logId); + const parts = decodedLogId.split("::"); + if (parts.length !== 4) { + throw new Response("Invalid log ID format", { status: 400 }); + } + + const [traceId, spanId, , startTime] = parts; + + const presenter = new LogDetailPresenter($replica, clickhouseClient); + + const result = await presenter.call({ + environmentId: environment.id, + organizationId: project.organizationId, + projectId: project.id, + spanId, + traceId, + startTime, + }); + + if (!result) { + throw new Response("Log not found", { status: 404 }); + } + + return typedjson(result); +}; diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts new file mode 100644 index 0000000000..fd6f1c1a6a --- /dev/null +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts @@ -0,0 +1,66 @@ +import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { json } from "@remix-run/node"; +import { requireUser, requireUserId } from "~/services/session.server"; +import { EnvironmentParamSchema } from "~/utils/pathBuilder"; +import { findProjectBySlug } from "~/models/project.server"; +import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { getRunFiltersFromRequest } from "~/presenters/RunFilters.server"; +import { LogsListPresenter, type LogLevel, LogsListOptionsSchema } from "~/presenters/v3/LogsListPresenter.server"; +import { $replica } from "~/db.server"; +import { clickhouseClient } from "~/services/clickhouseInstance.server"; + +// Valid log levels for filtering +const validLevels: LogLevel[] = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "CANCELLED"]; + +function parseLevelsFromUrl(url: URL): LogLevel[] | undefined { + const levelParams = url.searchParams.getAll("levels").filter((v) => v.length > 0); + if (levelParams.length === 0) return undefined; + return levelParams.filter((l): l is LogLevel => validLevels.includes(l as LogLevel)); +} + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params); + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) { + throw new Response("Project not found", { status: 404 }); + } + + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) { + throw new Response("Environment not found", { status: 404 }); + } + + const user = await requireUser(request); + const isAdmin = user?.admin || user?.isImpersonating; + + const filters = await getRunFiltersFromRequest(request); + + // Get search term, cursor, levels, and showDebug from query params + const url = new URL(request.url); + const search = url.searchParams.get("search") ?? undefined; + const cursor = url.searchParams.get("cursor") ?? undefined; + const levels = parseLevelsFromUrl(url); + const showDebug = url.searchParams.get("showDebug") === "true"; + + + const options = LogsListOptionsSchema.parse({ + userId, + projectId: project.id, + ...filters, + search, + cursor, + levels, + includeDebugLogs: isAdmin && showDebug, + defaultPeriod: "1h", + }) as any; // Validated by LogsListOptionsSchema at runtime + + const presenter = new LogsListPresenter($replica, clickhouseClient); + const result = await presenter.call(project.organizationId, environment.id, options); + + return json({ + logs: result.logs, + pagination: result.pagination, + }); +}; diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index e8e472bfc7..b16cc97f7f 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -71,6 +71,7 @@ import { docsPath, v3BatchPath, v3DeploymentVersionPath, + v3LogsPath, v3RunDownloadLogsPath, v3RunIdempotencyKeyResetPath, v3RunPath, @@ -572,7 +573,11 @@ function RunBody({
-
{run.idempotencyKey ? run.idempotencyKey : "–"}
+ {run.idempotencyKey ? ( + + ) : ( +
–
+ )} {run.idempotencyKey && (
Expires:{" "} @@ -587,7 +592,9 @@ function RunBody({ {run.idempotencyKey && (
diff --git a/apps/webapp/app/services/clickhouseInstance.server.ts b/apps/webapp/app/services/clickhouseInstance.server.ts index 32fc9bc0d4..156448c005 100644 --- a/apps/webapp/app/services/clickhouseInstance.server.ts +++ b/apps/webapp/app/services/clickhouseInstance.server.ts @@ -12,6 +12,28 @@ function initializeClickhouseClient() { console.log(`🗃️ Clickhouse service enabled to host ${url.host}`); + // Build logs query settings from environment variables + const logsQuerySettings = { + list: { + max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(), + max_bytes_before_external_sort: env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(), + max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS, + ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && { + max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(), + }), + ...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && { + max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME, + }), + }, + detail: { + max_memory_usage: env.CLICKHOUSE_LOGS_DETAIL_MAX_MEMORY_USAGE.toString(), + max_threads: env.CLICKHOUSE_LOGS_DETAIL_MAX_THREADS, + ...(env.CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME && { + max_execution_time: env.CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME, + }), + }, + }; + const clickhouse = new ClickHouse({ url: url.toString(), name: "clickhouse-instance", @@ -24,6 +46,7 @@ function initializeClickhouseClient() { request: true, }, maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, + logsQuerySettings, }); return clickhouse; diff --git a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts index 8b5cee04ca..9d3a92e911 100644 --- a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts @@ -52,6 +52,29 @@ export class ClickHouseRunsRepository implements IRunsRepository { return runIds; } + async listFriendlyRunIds(options: ListRunsOptions) { + // First get internal IDs from ClickHouse + const internalIds = await this.listRunIds(options); + + if (internalIds.length === 0) { + return []; + } + + // Then get friendly IDs from Prisma + const runs = await this.options.prisma.taskRun.findMany({ + where: { + id: { + in: internalIds, + }, + }, + select: { + friendlyId: true, + }, + }); + + return runs.map((run) => run.friendlyId); + } + async listRuns(options: ListRunsOptions) { const runIds = await this.listRunIds(options); diff --git a/apps/webapp/app/services/runsRepository/postgresRunsRepository.server.ts b/apps/webapp/app/services/runsRepository/postgresRunsRepository.server.ts index 93edbd9349..eaf2242090 100644 --- a/apps/webapp/app/services/runsRepository/postgresRunsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/postgresRunsRepository.server.ts @@ -31,6 +31,18 @@ export class PostgresRunsRepository implements IRunsRepository { return runs.map((run) => run.id); } + async listFriendlyRunIds(options: ListRunsOptions) { + const filterOptions = await convertRunListInputOptionsToFilterRunsOptions( + options, + this.options.prisma + ); + + const query = this.#buildFriendlyRunIdsQuery(filterOptions, options.page); + const runs = await this.options.prisma.$queryRaw<{ friendlyId: string }[]>(query); + + return runs.map((run) => run.friendlyId); + } + async listRuns(options: ListRunsOptions) { const filterOptions = await convertRunListInputOptionsToFilterRunsOptions( options, @@ -146,6 +158,21 @@ export class PostgresRunsRepository implements IRunsRepository { `; } + #buildFriendlyRunIdsQuery( + filterOptions: FilterRunsOptions, + page: { size: number; cursor?: string; direction?: "forward" | "backward" } + ) { + const whereConditions = this.#buildWhereConditions(filterOptions, page.cursor, page.direction); + + return Prisma.sql` + SELECT tr."friendlyId" + FROM ${sqlDatabaseSchema}."TaskRun" tr + WHERE ${whereConditions} + ORDER BY ${page.direction === "backward" ? Prisma.sql`tr.id ASC` : Prisma.sql`tr.id DESC`} + LIMIT ${page.size + 1} + `; + } + #buildRunsQuery( filterOptions: FilterRunsOptions, page: { size: number; cursor?: string; direction?: "forward" | "backward" } diff --git a/apps/webapp/app/services/runsRepository/runsRepository.server.ts b/apps/webapp/app/services/runsRepository/runsRepository.server.ts index 7bf81a4aa5..895c8b5fe5 100644 --- a/apps/webapp/app/services/runsRepository/runsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/runsRepository.server.ts @@ -7,7 +7,7 @@ import { type Prisma, TaskRunStatus } from "@trigger.dev/database"; import parseDuration from "parse-duration"; import { z } from "zod"; import { timeFilters } from "~/components/runs/v3/SharedFilters"; -import { type PrismaClient } from "~/db.server"; +import { type PrismaClient, type PrismaClientOrTransaction } from "~/db.server"; import { FEATURE_FLAG, makeFlags } from "~/v3/featureFlags.server"; import { startActiveSpan } from "~/v3/tracer.server"; import { logger } from "../logger.server"; @@ -16,7 +16,7 @@ import { PostgresRunsRepository } from "./postgresRunsRepository.server"; export type RunsRepositoryOptions = { clickhouse: ClickHouse; - prisma: PrismaClient; + prisma: PrismaClientOrTransaction; logger?: Logger; logLevel?: LogLevel; tracer?: Tracer; @@ -127,6 +127,8 @@ export type TagList = { export interface IRunsRepository { name: string; listRunIds(options: ListRunsOptions): Promise; + /** Returns friendly IDs (e.g., run_xxx) instead of internal UUIDs. Used for ClickHouse task_events queries. */ + listFriendlyRunIds(options: ListRunsOptions): Promise; listRuns(options: ListRunsOptions): Promise<{ runs: ListedRun[]; pagination: { @@ -223,6 +225,48 @@ export class RunsRepository implements IRunsRepository { ); } + async listFriendlyRunIds(options: ListRunsOptions): Promise { + const repository = await this.#getRepository(); + return startActiveSpan( + "runsRepository.listFriendlyRunIds", + async () => { + try { + return await repository.listFriendlyRunIds(options); + } catch (error) { + // If ClickHouse fails, retry with Postgres + if (repository.name === "clickhouse") { + this.logger?.warn("ClickHouse failed, retrying with Postgres", { error }); + return startActiveSpan( + "runsRepository.listFriendlyRunIds.fallback", + async () => { + return await this.postgresRunsRepository.listFriendlyRunIds(options); + }, + { + attributes: { + "repository.name": "postgres", + "fallback.reason": "clickhouse_error", + "fallback.error": error instanceof Error ? error.message : String(error), + organizationId: options.organizationId, + projectId: options.projectId, + environmentId: options.environmentId, + }, + } + ); + } + throw error; + } + }, + { + attributes: { + "repository.name": repository.name, + organizationId: options.organizationId, + projectId: options.projectId, + environmentId: options.environmentId, + }, + } + ); + } + async listRuns(options: ListRunsOptions): Promise<{ runs: ListedRun[]; pagination: { diff --git a/apps/webapp/app/utils/logUtils.ts b/apps/webapp/app/utils/logUtils.ts new file mode 100644 index 0000000000..b4a130b8e5 --- /dev/null +++ b/apps/webapp/app/utils/logUtils.ts @@ -0,0 +1,155 @@ +import { createElement, Fragment, type ReactNode } from "react"; +import { z } from "zod"; + +export const LogLevelSchema = z.enum(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "CANCELLED"]); +export type LogLevel = z.infer; + +export const validLogLevels: LogLevel[] = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "CANCELLED"]; + +// Default styles for search highlighting +const DEFAULT_HIGHLIGHT_STYLES: React.CSSProperties = { + backgroundColor: "#facc15", // yellow-400 + color: "#000000", + fontWeight: "500", + borderRadius: "0.25rem", + padding: "0 0.125rem", +} as const; + +/** + * Highlights all occurrences of a search term in text with consistent styling. + * Case-insensitive search with regex special character escaping. + * + * @param text - The text to search within + * @param searchTerm - The term to highlight (optional) + * @param style - Optional custom inline styles for highlights + * @returns React nodes with highlighted matches, or the original text if no matches + */ +export function highlightSearchText( + text: string, + searchTerm?: string, + style: React.CSSProperties = DEFAULT_HIGHLIGHT_STYLES +): ReactNode { + if (!searchTerm || searchTerm.trim() === "") { + return text; + } + + // Defense in depth: limit search term length to prevent ReDoS and performance issues + const MAX_SEARCH_LENGTH = 500; + if (searchTerm.length > MAX_SEARCH_LENGTH) { + return text; + } + + // Escape special regex characters in search term + const escapedSearch = searchTerm.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const regex = new RegExp(escapedSearch, "gi"); + + const parts: ReactNode[] = []; + let lastIndex = 0; + let match; + let matchCount = 0; + + while ((match = regex.exec(text)) !== null) { + // Add text before match + if (match.index > lastIndex) { + parts.push(text.substring(lastIndex, match.index)); + } + // Add highlighted match + parts.push( + createElement("span", { key: `match-${matchCount}`, style }, match[0]) + ); + lastIndex = regex.lastIndex; + matchCount++; + } + + // Add remaining text + if (lastIndex < text.length) { + parts.push(text.substring(lastIndex)); + } + + return parts.length > 0 ? parts : text; +} + +// Convert ClickHouse kind to display level +export function kindToLevel(kind: string, status: string): LogLevel { + if (status === "CANCELLED") { + return "CANCELLED"; + } + + // ERROR can come from either kind or status + if (kind === "LOG_ERROR" || status === "ERROR") { + return "ERROR"; + } + + switch (kind) { + case "DEBUG_EVENT": + case "LOG_DEBUG": + return "DEBUG"; + case "LOG_INFO": + return "INFO"; + case "LOG_WARN": + return "WARN"; + case "LOG_LOG": + return "INFO"; // Changed from "LOG" + case "SPAN": + case "ANCESTOR_OVERRIDE": + case "SPAN_EVENT": + default: + return "TRACE"; + } +} + +// Level badge color styles +export function getLevelColor(level: LogLevel): string { + switch (level) { + case "ERROR": + return "text-error bg-error/10 border-error/20"; + case "WARN": + return "text-warning bg-warning/10 border-warning/20"; + case "DEBUG": + return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; + case "INFO": + return "text-blue-400 bg-blue-500/10 border-blue-500/20"; + case "TRACE": + return "text-charcoal-500 bg-charcoal-800 border-charcoal-700"; + case "CANCELLED": + return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; + default: + return "text-text-dimmed bg-charcoal-750 border-charcoal-700"; + } +} + +// Event kind badge color styles +export function getKindColor(kind: string): string { + if (kind === "SPAN") { + return "text-purple-400 bg-purple-500/10 border-purple-500/20"; + } + if (kind === "SPAN_EVENT") { + return "text-amber-400 bg-amber-500/10 border-amber-500/20"; + } + if (kind.startsWith("LOG_")) { + return "text-blue-400 bg-blue-500/10 border-blue-500/20"; + } + return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; +} + +// Get human readable kind label +export function getKindLabel(kind: string): string { + switch (kind) { + case "SPAN": + return "Span"; + case "SPAN_EVENT": + return "Event"; + case "LOG_DEBUG": + case "LOG_INFO": + case "LOG_WARN": + case "LOG_ERROR": + case "LOG_LOG": + return "Log"; + case "DEBUG_EVENT": + return "Debug"; + case "ANCESTOR_OVERRIDE": + return "Override"; + default: + return kind; + } +} diff --git a/apps/webapp/app/utils/pathBuilder.ts b/apps/webapp/app/utils/pathBuilder.ts index ebdec3401a..a2756f7e5b 100644 --- a/apps/webapp/app/utils/pathBuilder.ts +++ b/apps/webapp/app/utils/pathBuilder.ts @@ -455,6 +455,14 @@ export function v3ProjectSettingsPath( return `${v3EnvironmentPath(organization, project, environment)}/settings`; } +export function v3LogsPath( + organization: OrgForPath, + project: ProjectForPath, + environment: EnvironmentForPath, +) { + return `${v3EnvironmentPath(organization, project, environment)}/logs`; +} + export function v3DeploymentsPath( organization: OrgForPath, project: ProjectForPath, diff --git a/apps/webapp/tailwind.config.js b/apps/webapp/tailwind.config.js index 7ca81fd8ee..9f4e4381b8 100644 --- a/apps/webapp/tailwind.config.js +++ b/apps/webapp/tailwind.config.js @@ -160,6 +160,7 @@ const batches = colors.pink[500]; const schedules = colors.yellow[500]; const queues = colors.purple[500]; const deployments = colors.green[500]; +const logs = colors.blue[500]; const tests = colors.lime[500]; const apiKeys = colors.amber[500]; const environmentVariables = colors.pink[500]; @@ -236,6 +237,7 @@ module.exports = { schedules, queues, deployments, + logs, tests, apiKeys, environmentVariables, diff --git a/internal-packages/clickhouse/src/client/queryBuilder.ts b/internal-packages/clickhouse/src/client/queryBuilder.ts index 78383fd270..30aad98486 100644 --- a/internal-packages/clickhouse/src/client/queryBuilder.ts +++ b/internal-packages/clickhouse/src/client/queryBuilder.ts @@ -98,6 +98,7 @@ export class ClickhouseQueryFastBuilder> { private columns: Array; private reader: ClickhouseReader; private settings: ClickHouseSettings | undefined; + private prewhereClauses: string[] = []; private whereClauses: string[] = []; private params: QueryParams = {}; private orderByClause: string | null = null; @@ -118,6 +119,25 @@ export class ClickhouseQueryFastBuilder> { this.settings = settings; } + /** + * Add a PREWHERE clause - filters applied before reading columns. + * Use for primary key columns (environment_id, start_time) to reduce I/O. + */ + prewhere(clause: string, params?: QueryParams): this { + this.prewhereClauses.push(clause); + if (params) { + Object.assign(this.params, params); + } + return this; + } + + prewhereIf(condition: any, clause: string, params?: QueryParams): this { + if (condition) { + this.prewhere(clause, params); + } + return this; + } + where(clause: string, params?: QueryParams): this { this.whereClauses.push(clause); if (params) { @@ -163,6 +183,9 @@ export class ClickhouseQueryFastBuilder> { build(): { query: string; params: QueryParams } { let query = `SELECT ${this.buildColumns().join(", ")} FROM ${this.table}`; + if (this.prewhereClauses.length > 0) { + query += " PREWHERE " + this.prewhereClauses.join(" AND "); + } if (this.whereClauses.length > 0) { query += " WHERE " + this.whereClauses.join(" AND "); } diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index 03b8b81e13..ca28a0a022 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -22,6 +22,8 @@ import { getTraceSummaryQueryBuilderV2, insertTaskEvents, insertTaskEventsV2, + getLogsListQueryBuilder, + getLogDetailQueryBuilder, } from "./taskEvents.js"; import { Logger, type LogLevel } from "@trigger.dev/core/logger"; import type { Agent as HttpAgent } from "http"; @@ -44,6 +46,11 @@ export { } from "./client/tsql.js"; export type { OutputColumnMetadata } from "@internal/tsql"; +export type LogsQuerySettings = { + list?: ClickHouseSettings; + detail?: ClickHouseSettings; +}; + export type ClickhouseCommonConfig = { keepAlive?: { enabled?: boolean; @@ -58,6 +65,7 @@ export type ClickhouseCommonConfig = { response?: boolean; }; maxOpenConnections?: number; + logsQuerySettings?: LogsQuerySettings; }; export type ClickHouseConfig = @@ -81,9 +89,11 @@ export class ClickHouse { public readonly writer: ClickhouseWriter; private readonly logger: Logger; private _splitClients: boolean; + private readonly logsQuerySettings?: LogsQuerySettings; constructor(config: ClickHouseConfig) { this.logger = config.logger ?? new Logger("ClickHouse", config.logLevel ?? "debug"); + this.logsQuerySettings = config.logsQuerySettings; if (config.url) { const url = new URL(config.url); @@ -195,6 +205,8 @@ export class ClickHouse { traceSummaryQueryBuilder: getTraceSummaryQueryBuilderV2(this.reader), traceDetailedSummaryQueryBuilder: getTraceDetailedSummaryQueryBuilderV2(this.reader), spanDetailsQueryBuilder: getSpanDetailsQueryBuilderV2(this.reader), + logsListQueryBuilder: getLogsListQueryBuilder(this.reader, this.logsQuerySettings?.list), + logDetailQueryBuilder: getLogDetailQueryBuilder(this.reader, this.logsQuerySettings?.detail), }; } } diff --git a/internal-packages/clickhouse/src/taskEvents.ts b/internal-packages/clickhouse/src/taskEvents.ts index d8c1b8b7f6..f526cdf0b6 100644 --- a/internal-packages/clickhouse/src/taskEvents.ts +++ b/internal-packages/clickhouse/src/taskEvents.ts @@ -230,3 +230,98 @@ export function getSpanDetailsQueryBuilderV2( settings, }); } + +// ============================================================================ +// Logs List Query Builders (for aggregated logs page) +// ============================================================================ + +export const LogsListResult = z.object({ + environment_id: z.string(), + organization_id: z.string(), + project_id: z.string(), + task_identifier: z.string(), + run_id: z.string(), + start_time: z.string(), + trace_id: z.string(), + span_id: z.string(), + parent_span_id: z.string(), + message: z.string(), + kind: z.string(), + status: z.string(), + duration: z.number().or(z.string()), + metadata: z.string(), + attributes: z.any(), +}); + +export type LogsListResult = z.output; + +export function getLogsListQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) { + return ch.queryBuilderFast({ + name: "getLogsList", + table: "trigger_dev.task_events_v2", + columns: [ + "environment_id", + "organization_id", + "project_id", + "task_identifier", + "run_id", + "start_time", + "trace_id", + "span_id", + "parent_span_id", + { name: "message", expression: "LEFT(message, 512)" }, + "kind", + "status", + "duration", + "metadata", + "attributes" + ], + settings, + }); +} + +// Single log detail query builder (for side panel) +export const LogDetailV2Result = z.object({ + environment_id: z.string(), + organization_id: z.string(), + project_id: z.string(), + task_identifier: z.string(), + run_id: z.string(), + start_time: z.string(), + trace_id: z.string(), + span_id: z.string(), + parent_span_id: z.string(), + message: z.string(), + kind: z.string(), + status: z.string(), + duration: z.number().or(z.string()), + metadata: z.string(), + attributes: z.any() +}); + +export type LogDetailV2Result = z.output; + +export function getLogDetailQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) { + return ch.queryBuilderFast({ + name: "getLogDetail", + table: "trigger_dev.task_events_v2", + columns: [ + "environment_id", + "organization_id", + "project_id", + "task_identifier", + "run_id", + "start_time", + "trace_id", + "span_id", + "parent_span_id", + "message", + "kind", + "status", + "duration", + "metadata", + "attributes", + ], + settings, + }); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 185ae798c3..1fc310de96 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2715,6 +2715,40 @@ importers: specifier: ^5 version: 5.5.4 + references/seed: + dependencies: + '@sinclair/typebox': + specifier: ^0.34.3 + version: 0.34.38 + '@trigger.dev/build': + specifier: workspace:* + version: link:../../packages/build + '@trigger.dev/sdk': + specifier: workspace:* + version: link:../../packages/trigger-sdk + arktype: + specifier: ^2.0.0 + version: 2.1.20 + openai: + specifier: ^4.97.0 + version: 4.97.0(encoding@0.1.13)(ws@8.18.3(bufferutil@4.0.9))(zod@3.25.76) + puppeteer-core: + specifier: ^24.15.0 + version: 24.15.0(bufferutil@4.0.9) + replicate: + specifier: ^1.0.1 + version: 1.0.1 + yup: + specifier: ^1.6.1 + version: 1.7.0 + zod: + specifier: 3.25.76 + version: 3.25.76 + devDependencies: + trigger.dev: + specifier: workspace:* + version: link:../../packages/cli-v3 + references/telemetry: dependencies: '@opentelemetry/resources': @@ -11587,9 +11621,6 @@ packages: balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} - bare-events@2.5.4: - resolution: {integrity: sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==} - bare-events@2.8.2: resolution: {integrity: sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==} peerDependencies: @@ -23623,7 +23654,7 @@ snapshots: '@hono/node-ws@1.0.4(@hono/node-server@1.12.2(hono@4.5.11))(bufferutil@4.0.9)': dependencies: '@hono/node-server': 1.12.2(hono@4.5.11) - ws: 8.18.0(bufferutil@4.0.9) + ws: 8.18.3(bufferutil@4.0.9) transitivePeerDependencies: - bufferutil - utf-8-validate @@ -32012,17 +32043,14 @@ snapshots: balanced-match@1.0.2: {} - bare-events@2.5.4: - optional: true - bare-events@2.8.2: optional: true bare-fs@4.5.1: dependencies: - bare-events: 2.5.4 + bare-events: 2.8.2 bare-path: 3.0.0 - bare-stream: 2.6.5(bare-events@2.5.4) + bare-stream: 2.6.5(bare-events@2.8.2) bare-url: 2.3.2 fast-fifo: 1.3.2 transitivePeerDependencies: @@ -32037,11 +32065,11 @@ snapshots: bare-os: 3.6.1 optional: true - bare-stream@2.6.5(bare-events@2.5.4): + bare-stream@2.6.5(bare-events@2.8.2): dependencies: streamx: 2.22.0 optionalDependencies: - bare-events: 2.5.4 + bare-events: 2.8.2 transitivePeerDependencies: - bare-abort-controller optional: true @@ -32094,7 +32122,7 @@ snapshots: dependencies: buffer: 5.7.1 inherits: 2.0.4 - readable-stream: 3.6.0 + readable-stream: 3.6.2 body-parser@1.20.3: dependencies: @@ -40724,7 +40752,7 @@ snapshots: end-of-stream: 1.4.4 fs-constants: 1.0.0 inherits: 2.0.4 - readable-stream: 3.6.0 + readable-stream: 3.6.2 tar-stream@3.1.7: dependencies: diff --git a/references/nextjs-realtime/.eslintrc.json b/references/nextjs-realtime/.eslintrc.json new file mode 100644 index 0000000000..6b10a5b739 --- /dev/null +++ b/references/nextjs-realtime/.eslintrc.json @@ -0,0 +1,6 @@ +{ + "extends": [ + "next/core-web-vitals", + "next/typescript" + ] +} diff --git a/references/seed/.gitignore b/references/seed/.gitignore new file mode 100644 index 0000000000..6524f048dc --- /dev/null +++ b/references/seed/.gitignore @@ -0,0 +1 @@ +.trigger \ No newline at end of file diff --git a/references/seed/package.json b/references/seed/package.json new file mode 100644 index 0000000000..aa788c467a --- /dev/null +++ b/references/seed/package.json @@ -0,0 +1,23 @@ +{ + "name": "references-seed", + "private": true, + "type": "module", + "devDependencies": { + "trigger.dev": "workspace:*" + }, + "dependencies": { + "@trigger.dev/build": "workspace:*", + "@trigger.dev/sdk": "workspace:*", + "arktype": "^2.0.0", + "openai": "^4.97.0", + "puppeteer-core": "^24.15.0", + "replicate": "^1.0.1", + "yup": "^1.6.1", + "zod": "3.25.76", + "@sinclair/typebox": "^0.34.3" + }, + "scripts": { + "dev": "trigger dev", + "deploy": "trigger deploy" + } +} diff --git a/references/seed/src/index.ts b/references/seed/src/index.ts new file mode 100644 index 0000000000..cb0ff5c3b5 --- /dev/null +++ b/references/seed/src/index.ts @@ -0,0 +1 @@ +export {}; diff --git a/references/seed/src/trigger/logSpammer.ts b/references/seed/src/trigger/logSpammer.ts new file mode 100644 index 0000000000..7156b55602 --- /dev/null +++ b/references/seed/src/trigger/logSpammer.ts @@ -0,0 +1,109 @@ +import { logger, task, wait } from "@trigger.dev/sdk/v3"; + +const LONG_TEXT = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.`; + +const SEARCHABLE_TERMS = [ + "authentication_failed", + "database_connection_error", + "payment_processed", + "user_registration_complete", + "api_rate_limit_exceeded", + "cache_invalidation", + "webhook_delivery_success", + "session_expired", + "file_upload_complete", + "email_sent_successfully", +]; + +function generateLargeJson(index: number) { + return { + requestId: `req_${Date.now()}_${index}`, + timestamp: new Date().toISOString(), + metadata: { + source: "log-spammer-task", + environment: "development", + version: "1.0.0", + region: ["us-east-1", "eu-west-1", "ap-southeast-1"][index % 3], + }, + user: { + id: `user_${1000 + index}`, + email: `testuser${index}@example.com`, + name: `Test User ${index}`, + preferences: { + theme: index % 2 === 0 ? "dark" : "light", + notifications: { email: true, push: false, sms: index % 3 === 0 }, + language: ["en", "es", "fr", "de"][index % 4], + }, + }, + payload: { + items: Array.from({ length: 5 }, (_, i) => ({ + itemId: `item_${index}_${i}`, + name: `Product ${i}`, + price: Math.random() * 100, + quantity: Math.floor(Math.random() * 10) + 1, + tags: ["electronics", "sale", "featured"].slice(0, (i % 3) + 1), + })), + totals: { + subtotal: Math.random() * 500, + tax: Math.random() * 50, + shipping: Math.random() * 20, + discount: Math.random() * 30, + }, + }, + debugInfo: { + stackTrace: `Error: ${SEARCHABLE_TERMS[index % SEARCHABLE_TERMS.length]}\n at processRequest (/app/src/handlers/main.ts:${100 + index}:15)\n at handleEvent (/app/src/events/processor.ts:${50 + index}:8)\n at async Runtime.handler (/app/src/index.ts:25:3)`, + memoryUsage: { heapUsed: 45000000 + index * 1000, heapTotal: 90000000 }, + cpuTime: Math.random() * 1000, + }, + longDescription: LONG_TEXT.repeat(2), + }; +} + +export const logSpammerTask = task({ + id: "log-spammer", + maxDuration: 300, + run: async () => { + logger.info("Starting log spammer task for search testing"); + + for (let i = 0; i < 50; i++) { + const term = SEARCHABLE_TERMS[i % SEARCHABLE_TERMS.length]; + const jsonPayload = generateLargeJson(i); + + logger.log(`Processing event: ${term}`, { data: jsonPayload }); + + if (i % 5 === 0) { + logger.warn(`Warning triggered for ${term}`, { + warningCode: `WARN_${i}`, + details: jsonPayload, + longMessage: LONG_TEXT, + }); + } + + if (i % 10 === 0) { + logger.error(`Error encountered: ${term}`, { + errorCode: `ERR_${i}`, + stack: jsonPayload.debugInfo.stackTrace, + context: jsonPayload, + }); + } + + logger.debug(`Debug info for iteration ${i}`, { + iteration: i, + searchTerm: term, + fullPayload: jsonPayload, + additionalText: `${LONG_TEXT} --- Iteration ${i} complete with term ${term}`, + }); + + if (i % 10 === 0) { + await wait.for({ seconds: 0.5 }); + } + } + + logger.info("Log spammer task completed", { + totalLogs: 50 * 4, + searchableTerms: SEARCHABLE_TERMS, + }); + + return { success: true, logsGenerated: 200 }; + }, +}); diff --git a/references/seed/src/trigger/seedTask.ts b/references/seed/src/trigger/seedTask.ts new file mode 100644 index 0000000000..2fb4305492 --- /dev/null +++ b/references/seed/src/trigger/seedTask.ts @@ -0,0 +1,32 @@ +import { task, batch } from "@trigger.dev/sdk/v3"; +import { ErrorTask } from "./throwError.js"; +import { SpanSpammerTask } from "./spanSpammer.js"; +import { logSpammerTask } from "./logSpammer.js"; + +export const seedTask = task({ + id: "seed-task", + run: async (payload: any, { ctx }) => { + let tasksToRun = []; + + for (let i = 0; i < 10; i++) { + tasksToRun.push({ + id: "simple-throw-error", + payload: {}, + options: { delay: `${i}s` }, + }); + } + + tasksToRun.push({ + id: "span-spammer", + payload: {}, + }); + + tasksToRun.push({ + id: "log-spammer", + payload: {}, + }); + + await batch.triggerAndWait(tasksToRun); + return; + }, +}); diff --git a/references/seed/src/trigger/spanSpammer.ts b/references/seed/src/trigger/spanSpammer.ts new file mode 100644 index 0000000000..b16f00c4c2 --- /dev/null +++ b/references/seed/src/trigger/spanSpammer.ts @@ -0,0 +1,42 @@ +import { logger, task, wait } from "@trigger.dev/sdk/v3"; + +const CONFIG = { + delayBetweenBatchesSeconds: 0.2, + logsPerBatch: 30, + totalBatches: 100, + initialDelaySeconds: 5, +} as const; + +export const SpanSpammerTask = task({ + id: "span-spammer", + maxDuration: 300, + run: async (payload: any, { ctx }) => { + const context = { payload, ctx }; + let logCount = 0; + + logger.info("Starting span spammer task", context); + logger.warn("This will generate a lot of logs", context); + + + const emitBatch = (prefix: string) => { + logger.debug("Started spam batch emit!", context); + + for (let i = 0; i < CONFIG.logsPerBatch; i++) { + logger.log(`${prefix} ${++logCount}`, context); + } + + logger.debug('Completed spam batch emit!', context); + }; + + emitBatch("Log number"); + await wait.for({ seconds: CONFIG.initialDelaySeconds }); + + for (let batch = 0; batch < CONFIG.totalBatches; batch++) { + await wait.for({ seconds: CONFIG.delayBetweenBatchesSeconds }); + emitBatch("This is a test log!!! Log number: "); + } + + logger.info("Completed span spammer task", context); + return { message: `Created ${logCount} logs` }; + }, +}); diff --git a/references/seed/src/trigger/throwError.ts b/references/seed/src/trigger/throwError.ts new file mode 100644 index 0000000000..5f2d623a01 --- /dev/null +++ b/references/seed/src/trigger/throwError.ts @@ -0,0 +1,16 @@ +import { logger, task, wait } from "@trigger.dev/sdk/v3"; + + +export const ErrorTask = task({ + id: "simple-throw-error", + maxDuration: 60, + run: async (payload: any, { ctx }) => { + logger.log("This task is about to throw an error!", { payload, ctx }); + + await wait.for({ seconds: 9 }); + throw new Error("This is an expected test error from ErrorTask!"); + }, + onFailure: async ({ payload, error, ctx }) => { + logger.warn("ErrorTask failed!", { payload, error, ctx }); + } +}); diff --git a/references/seed/trigger.config.ts b/references/seed/trigger.config.ts new file mode 100644 index 0000000000..f87620cd78 --- /dev/null +++ b/references/seed/trigger.config.ts @@ -0,0 +1,56 @@ +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { syncEnvVars } from "@trigger.dev/build/extensions/core"; +import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; + +export default defineConfig({ + compatibilityFlags: ["run_engine_v2"], + project: process.env.TRIGGER_PROJECT_REF!, + experimental_processKeepAlive: { + enabled: true, + maxExecutionsPerProcess: 20, + }, + logLevel: "debug", + maxDuration: 3600, + retries: { + enabledInDev: true, + default: { + maxAttempts: 3, + minTimeoutInMs: 1000, + maxTimeoutInMs: 10000, + factor: 2, + randomize: true, + }, + }, + machine: "small-2x", + build: { + extensions: [ + lightpanda(), + syncEnvVars(async (ctx) => { + return [ + { name: "SYNC_ENV", value: ctx.environment }, + { name: "BRANCH", value: ctx.branch ?? "NO_BRANCH" }, + { name: "BRANCH", value: "PARENT", isParentEnv: true }, + { name: "SECRET_KEY", value: "secret-value" }, + { name: "ANOTHER_SECRET", value: "another-secret-value" }, + ]; + }), + { + name: "npm-token", + onBuildComplete: async (context, manifest) => { + if (context.target === "dev") { + return; + } + + context.addLayer({ + id: "npm-token", + build: { + env: { + NPM_TOKEN: manifest.deploy.env?.NPM_TOKEN, + }, + }, + }); + }, + }, + ], + }, +}); diff --git a/references/seed/tsconfig.json b/references/seed/tsconfig.json new file mode 100644 index 0000000000..9a5ee0b9d6 --- /dev/null +++ b/references/seed/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2023", + "module": "Node16", + "moduleResolution": "Node16", + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "customConditions": ["@triggerdotdev/source"], + "jsx": "preserve", + "lib": ["DOM", "DOM.Iterable"], + "noEmit": true + }, + "include": ["./src/**/*.ts", "trigger.config.ts"] +}