Skip to content

Add LM Studio provider #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
khromov wants to merge 8 commits into
base: prompt-cache
Choose a base branch
from
Open

Add LM Studio provider #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b4c6766
add lmstudio local provider
khromov Dec 14, 2025
f35742b
wip
khromov Dec 14, 2025
b44968c
wip
khromov Dec 14, 2025
cef065d
cleanup
khromov Dec 14, 2025
9173cee
Update index.ts
khromov Dec 14, 2025
712b985
refactor
khromov Dec 14, 2025
64d93b3
Update report.ts
khromov Dec 14, 2025
fab439f
Update ai-gateway.ts
khromov Dec 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
238 changes: 99 additions & 139 deletions index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,159 +19,90 @@ import {
runTestVerification,
} from "./lib/output-test-runner.ts";
import { resultWriteTool, testComponentTool } from "./lib/tools/index.ts";
import { getModelPricingDisplay, formatCost, formatMTokCost } from "./lib/pricing.ts";
import {
buildPricingMap,
lookupPricingFromMap,
getModelPricingDisplay,
formatCost,
formatMTokCost,
} from "./lib/pricing.ts";
import type { LanguageModel } from "ai";
gateway,
getGatewayModelsAndPricing,
selectModelsFromGateway,
type PricingMap,
type PricingLookup,
type PricingResult,
} from "./lib/providers/ai-gateway.ts";
import {
intro,
multiselect,
isCancel,
cancel,
text,
select,
confirm,
note,
} from "@clack/prompts";
import { gateway } from "ai";

async function validateAndConfirmPricing(
models: string[],
pricingMap: ReturnType<typeof buildPricingMap>,
) {
const lookups = new Map<string, ReturnType<typeof lookupPricingFromMap>>();

for (const modelId of models) {
const lookup = lookupPricingFromMap(modelId, pricingMap);
lookups.set(modelId, lookup);
}

const modelsWithPricing = models.filter((m) => lookups.get(m) !== null);
const modelsWithoutPricing = models.filter((m) => lookups.get(m) === null);

if (modelsWithoutPricing.length === 0) {
const pricingLines = models.map((modelId) => {
const lookup = lookups.get(modelId)!;
const display = getModelPricingDisplay(lookup.pricing);
const cacheReadText =
display.cacheReadCostPerMTok !== undefined
? `, ${formatMTokCost(display.cacheReadCostPerMTok)}/MTok cache read`
: "";
const cacheWriteText =
display.cacheCreationCostPerMTok !== undefined
? `, ${formatMTokCost(display.cacheCreationCostPerMTok)}/MTok cache write`
: "";
return `${modelId}\n → ${formatMTokCost(display.inputCostPerMTok)}/MTok in, ${formatMTokCost(display.outputCostPerMTok)}/MTok out${cacheReadText}${cacheWriteText}`;
});

note(pricingLines.join("\n\n"), "💰 Pricing Found");

const usePricing = await confirm({
message: "Enable cost calculation?",
initialValue: true,
});

if (isCancel(usePricing)) {
cancel("Operation cancelled.");
process.exit(0);
}

return { enabled: usePricing, lookups };
} else {
const lines: string[] = [];

if (modelsWithoutPricing.length > 0) {
lines.push("No pricing found for:");
for (const modelId of modelsWithoutPricing) {
lines.push(` ✗ ${modelId}`);
}
}

if (modelsWithPricing.length > 0) {
lines.push("");
lines.push("Pricing available for:");
for (const modelId of modelsWithPricing) {
const lookup = lookups.get(modelId)!;
const display = getModelPricingDisplay(lookup.pricing);
const cacheReadText =
display.cacheReadCostPerMTok !== undefined
? `, ${formatMTokCost(display.cacheReadCostPerMTok)}/MTok cache read`
: "";
const cacheWriteText =
display.cacheCreationCostPerMTok !== undefined
? `, ${formatMTokCost(display.cacheCreationCostPerMTok)}/MTok cache write`
: "";
lines.push(
` ✓ ${modelId} (${formatMTokCost(display.inputCostPerMTok)}/MTok in, ${formatMTokCost(display.outputCostPerMTok)}/MTok out${cacheReadText}${cacheWriteText})`,
);
}
}
configureLMStudio,
selectModelsFromLMStudio,
getLMStudioModel,
isLMStudioModel,
type LMStudioConfig,
} from "./lib/providers/lmstudio.ts";
import type { LanguageModel } from "ai";
import { intro, isCancel, cancel, select, confirm, text } from "@clack/prompts";
import { buildPricingMap } from "./lib/pricing.ts";

lines.push("");
lines.push("Cost calculation will be disabled.");
type ProviderType = "gateway" | "lmstudio";

note(lines.join("\n"), "⚠️ Pricing Incomplete");
interface ProviderConfig {
type: ProviderType;
lmstudio?: LMStudioConfig;
}

const proceed = await confirm({
message: "Continue without pricing?",
initialValue: true,
});
async function selectProvider(): Promise<ProviderConfig> {
const provider = await select({
message: "Select model provider",
options: [
{
value: "gateway",
label: "Vercel AI Gateway",
hint: "Cloud-hosted models via Vercel",
},
{
value: "lmstudio",
label: "LM Studio",
hint: "Local models via LM Studio",
},
],
initialValue: "gateway",
});

if (isCancel(proceed) || !proceed) {
cancel("Operation cancelled.");
process.exit(0);
}
if (isCancel(provider)) {
cancel("Operation cancelled.");
process.exit(0);
}

return { enabled: false, lookups };
if (provider === "lmstudio") {
const lmstudioConfig = await configureLMStudio();
return { type: "lmstudio", lmstudio: lmstudioConfig };
}

return { type: "gateway" };
}

async function selectOptions() {
intro("🚀 Svelte AI Bench");

const available_models = await gateway.getAvailableModels();

const pricingMap = buildPricingMap(available_models.models);
const providerConfig = await selectProvider();

const models = await multiselect({
message: "Select model(s) to benchmark",
options: [{ value: "custom", label: "Custom" }].concat(
available_models.models.reduce<Array<{ value: string; label: string }>>(
(arr, model) => {
if (model.modelType === "language") {
arr.push({ value: model.id, label: model.name });
}
return arr;
},
[],
),
),
});

if (isCancel(models)) {
cancel("Operation cancelled.");
process.exit(0);
}
let pricingMap: PricingMap;
let selectedModels: string[];
let pricing: PricingResult;

if (models.includes("custom")) {
const custom_model = await text({
message: "Enter custom model id",
});
if (isCancel(custom_model)) {
cancel("Operation cancelled.");
process.exit(0);
}
models.push(custom_model);
if (providerConfig.type === "gateway") {
const gatewayData = await getGatewayModelsAndPricing();
pricingMap = gatewayData.pricingMap;
const result = await selectModelsFromGateway(pricingMap);
selectedModels = result.selectedModels;
pricing = result.pricing;
} else {
pricingMap = buildPricingMap([]);
selectedModels = await selectModelsFromLMStudio(
providerConfig.lmstudio!.baseURL,
);
pricing = {
enabled: false,
lookups: new Map<string, PricingLookup>(),
};
}

const selectedModels = models.filter((model) => model !== "custom");

const pricing = await validateAndConfirmPricing(selectedModels, pricingMap);

const mcp_integration = await select({
message: "Which MCP integration to use?",
options: [
Expand Down Expand Up @@ -233,6 +164,7 @@ async function selectOptions() {
mcp,
testingTool,
pricing,
providerConfig,
};
}

Expand All @@ -246,6 +178,17 @@ function parseCommandString(commandString: string): {
return { command, args };
}

function getModelForId(
modelId: string,
providerConfig: ProviderConfig,
): LanguageModel {
if (isLMStudioModel(modelId)) {
return getLMStudioModel(modelId, providerConfig.lmstudio?.baseURL);
}

return gateway.languageModel(modelId);
}

async function runSingleTest(
test: TestDefinition,
model: LanguageModel,
Expand Down Expand Up @@ -375,7 +318,8 @@ async function runSingleTest(
}

async function main() {
const { models, mcp, testingTool, pricing } = await selectOptions();
const { models, mcp, testingTool, pricing, providerConfig } =
await selectOptions();

const mcpServerUrl = mcp;
const mcpEnabled = !!mcp;
Expand All @@ -389,6 +333,13 @@ async function main() {
console.log("║ SvelteBench 2.0 - Multi-Test ║");
console.log("╚════════════════════════════════════════════════════╝");

console.log(
`\n🔌 Provider: ${providerConfig.type === "gateway" ? "Vercel AI Gateway" : "LM Studio"}`,
);
if (providerConfig.type === "lmstudio" && providerConfig.lmstudio) {
console.log(` URL: ${providerConfig.lmstudio.baseURL}`);
}

console.log("\n📋 Models:");
for (const modelId of models) {
const lookup = pricing.lookups.get(modelId);
Expand All @@ -408,6 +359,9 @@ async function main() {
);
} else {
console.log(` ${modelId}`);
if (isLMStudioModel(modelId)) {
console.log(` 🖥️ Local model (free)`);
}
}
}

Expand Down Expand Up @@ -486,7 +440,7 @@ async function main() {
);
}

const model = gateway.languageModel(modelId);
const model = getModelForId(modelId, providerConfig);

const testResults = [];
const startTime = Date.now();
Expand Down Expand Up @@ -566,7 +520,6 @@ async function main() {
}
console.log(`Total cost: ${formatCost(totalCost.totalCost)}`);

// Simulate cache savings
cacheSimulation = simulateCacheSavings(
testResults,
pricingLookup.pricing,
Expand Down Expand Up @@ -620,10 +573,17 @@ async function main() {
mcpTransportType: mcpEnabled ? mcpTransportType : null,
timestamp: new Date().toISOString(),
model: modelId,
provider: providerConfig.type,
pricingKey: pricingLookup?.matchedKey ?? null,
pricing: pricingInfo,
totalCost,
cacheSimulation,
lmstudio:
providerConfig.type === "lmstudio" && providerConfig.lmstudio
? {
baseURL: providerConfig.lmstudio.baseURL,
}
: null,
},
};

Expand Down
Loading