Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,10 @@ HYPERBOLIC_API_BASE_URL=https://api.hyperbolic.xyz/v1/chat/completions
# USE: http://127.0.0.1:1234
LMSTUDIO_API_BASE_URL=http://127.0.0.1:1234

# Docker Model Runner (DMR)
# Default host TCP endpoint when Model Runner TCP is enabled in Docker Desktop
DMR_API_BASE_URL=http://127.0.0.1:12434

# ======================================
# CLOUD SERVICES CONFIGURATION
# ======================================
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ const CloudProvidersTab = () => {
// Load and filter providers
useEffect(() => {
const newFilteredProviders = Object.entries(settings.providers || {})
.filter(([key]) => !['Ollama', 'LMStudio', 'OpenAILike'].includes(key))
// Exclude local providers from the cloud list
.filter(([key]) => !['Ollama', 'LMStudio', 'OpenAILike', 'DockerModelRunner'].includes(key))
.map(([key, value]) => ({
name: key,
settings: value.settings,
Expand Down
116 changes: 108 additions & 8 deletions app/components/@settings/tabs/providers/local/LocalProvidersTab.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ export default function LocalProvidersTab() {
const [isLoadingLMStudioModels, setIsLoadingLMStudioModels] = useState(false);
const { toast } = useToast();
const { startMonitoring, stopMonitoring } = useLocalModelHealth();
const [dmrModels, setDmrModels] = useState<string[]>([]);
const [isLoadingDmrModels, setIsLoadingDmrModels] = useState(false);

// Memoized filtered providers to prevent unnecessary re-renders
const filteredProviders = useMemo(() => {
Expand All @@ -45,13 +47,15 @@ export default function LocalProvidersTab() {
// Set default base URLs for local providers
let defaultBaseUrl = provider.settings.baseUrl || envUrl;

if (!defaultBaseUrl) {
if (key === 'Ollama') {
defaultBaseUrl = 'http://127.0.0.1:11434';
} else if (key === 'LMStudio') {
defaultBaseUrl = 'http://127.0.0.1:1234';
if (!defaultBaseUrl) {
if (key === 'Ollama') {
defaultBaseUrl = 'http://127.0.0.1:11434';
} else if (key === 'LMStudio') {
defaultBaseUrl = 'http://127.0.0.1:1234';
} else if (key === 'DockerModelRunner') {
defaultBaseUrl = 'http://127.0.0.1:12434';
}
}
}

return {
name: key,
Expand Down Expand Up @@ -84,10 +88,10 @@ export default function LocalProvidersTab() {

if (provider.settings.enabled && baseUrl) {
console.log(`[LocalProvidersTab] Starting monitoring for ${provider.name} at ${baseUrl}`);
startMonitoring(provider.name as 'Ollama' | 'LMStudio' | 'OpenAILike', baseUrl);
startMonitoring(provider.name as 'Ollama' | 'LMStudio' | 'OpenAILike' | 'DockerModelRunner', baseUrl);
} else if (!provider.settings.enabled && baseUrl) {
console.log(`[LocalProvidersTab] Stopping monitoring for ${provider.name} at ${baseUrl}`);
stopMonitoring(provider.name as 'Ollama' | 'LMStudio' | 'OpenAILike', baseUrl);
stopMonitoring(provider.name as 'Ollama' | 'LMStudio' | 'OpenAILike' | 'DockerModelRunner', baseUrl);
}
});
}, [filteredProviders, startMonitoring, stopMonitoring]);
Expand All @@ -110,6 +114,16 @@ export default function LocalProvidersTab() {
}
}, [filteredProviders]);

// Fetch Docker Model Runner models when enabled
useEffect(() => {
const dmrProvider = filteredProviders.find((p) => p.name === 'DockerModelRunner');

if (dmrProvider?.settings.enabled && dmrProvider.settings.baseUrl) {
fetchDMRModels(dmrProvider.settings.baseUrl);
}
}, [filteredProviders]);


const fetchOllamaModels = async () => {
try {
setIsLoadingModels(true);
Expand Down Expand Up @@ -190,6 +204,28 @@ export default function LocalProvidersTab() {
[updateProviderSettings, toast],
);

// Fetch Docker Model Runner models using local proxy to avoid CORS
const fetchDMRModels = async (baseUrl: string) => {
try {
setIsLoadingDmrModels(true);
const normalized = baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl;
const proxyUrl = `/api/local-proxy?url=${encodeURIComponent(`${normalized}/engines/v1/models`)}`;
const response = await fetch(proxyUrl);

if (!response.ok) {
throw new Error('Failed to fetch Docker Model Runner models');
}

const data = (await response.json()) as { data?: Array<{ id: string }> };
setDmrModels((data.data || []).map((m) => m.id));
} catch (e) {
console.error('Error fetching Docker Model Runner models', e);
setDmrModels([]);
} finally {
setIsLoadingDmrModels(false);
}
};

const handleUpdateOllamaModel = async (modelName: string) => {
try {
setOllamaModels((prev) => prev.map((m) => (m.name === modelName ? { ...m, status: 'updating' } : m)));
Expand Down Expand Up @@ -259,6 +295,7 @@ export default function LocalProvidersTab() {
}
};


const handleDeleteOllamaModel = async (modelName: string) => {
if (!window.confirm(`Are you sure you want to delete ${modelName}?`)) {
return;
Expand Down Expand Up @@ -440,6 +477,69 @@ export default function LocalProvidersTab() {
</Card>
)}


{/* Docker Model Runner Models Section */}
{provider.name === 'DockerModelRunner' && provider.settings.enabled && (
<Card className="mt-4 bg-bolt-elements-background-depth-2">
<CardHeader className="pb-3">
<div className="flex items-center justify-between">
<div className="flex items-center gap-2">
<Server className="w-5 h-5 text-cyan-500" />
<h3 className="text-lg font-semibold text-bolt-elements-textPrimary">Available Models</h3>
</div>
<Button
variant="outline"
size="sm"
onClick={() => fetchDMRModels(provider.settings.baseUrl!)}
disabled={isLoadingDmrModels}
className="bg-transparent hover:bg-bolt-elements-background-depth-2"
>
{isLoadingDmrModels ? (
<Loader2 className="w-4 h-4 animate-spin mr-2" />
) : (
<RotateCw className="w-4 h-4 mr-2" />
)}
Refresh
</Button>
</div>
</CardHeader>
<CardContent className="space-y-4">
{isLoadingDmrModels ? (
<div className="space-y-4">
{Array.from({ length: 3 }).map((_, i) => (
<ModelCardSkeleton key={i} />
))}
</div>
) : dmrModels.length === 0 ? (
<div className="text-center py-8">
<Server className="w-16 h-16 mx-auto text-bolt-elements-textTertiary mb-4" />
<h3 className="text-lg font-medium text-bolt-elements-textPrimary mb-2">No Models Available</h3>
<p className="text-sm text-bolt-elements-textSecondary">
Pull a model with Docker (e.g., "docker model pull ai/smollm2") and ensure DMR is enabled.
</p>
</div>
) : (
<div className="grid gap-4">
{dmrModels.map((id) => (
<Card key={id} className="bg-bolt-elements-background-depth-3">
<CardContent className="p-4">
<div className="flex items-center gap-2">
<h4 className="text-sm font-medium text-bolt-elements-textPrimary font-mono truncate">
{id}
</h4>
<span className="px-2 py-0.5 rounded-full text-xs font-medium bg-cyan-500/10 text-cyan-500">
Available
</span>
</div>
</CardContent>
</Card>
))}
</div>
)}
</CardContent>
</Card>
)}

{/* LM Studio Models Section */}
{provider.name === 'LMStudio' && provider.settings.enabled && (
<Card className="mt-4 bg-bolt-elements-background-depth-2">
Expand Down
91 changes: 91 additions & 0 deletions app/components/@settings/tabs/providers/local/SetupGuide.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,97 @@ function SetupGuide({ onBack }: { onBack: () => void }) {
</CardContent>
</Card>

{/* Docker Model Runner Setup Section */}
<Card className="bg-bolt-elements-background-depth-2 shadow-sm">
<CardHeader className="pb-6">
<div className="flex items-center gap-4">
<div className="w-12 h-12 rounded-xl bg-gradient-to-br from-cyan-500/20 to-blue-500/20 flex items-center justify-center ring-1 ring-cyan-500/30">
<Server className="w-6 h-6 text-cyan-500" />
</div>
<div className="flex-1">
<h3 className="text-xl font-semibold text-bolt-elements-textPrimary">Docker Model Runner (DMR) Setup</h3>
<p className="text-sm text-bolt-elements-textSecondary">OpenAI-compatible API via Docker Desktop</p>
</div>
</div>
</CardHeader>
<CardContent className="space-y-6">
{/* Requirements */}
<div className="p-4 rounded-lg bg-bolt-elements-background-depth-3">
<h4 className="font-medium text-bolt-elements-textPrimary mb-2">Requirements</h4>
<ul className="text-sm text-bolt-elements-textSecondary list-disc list-inside space-y-1">
<li>Docker Desktop 4.41+ (Windows) or 4.40+ (macOS) or Docker Engine (Linux)</li>
<li>Enable GPU for Windows (NVIDIA drivers 576.57+) if using GPU models</li>
</ul>
</div>

{/* Enable DMR and TCP */}
<div className="space-y-4">
<h4 className="font-medium text-bolt-elements-textPrimary flex items-center gap-2">
<Settings className="w-4 h-4" />
1. Enable DMR + Host TCP (12434)
</h4>
<div className="text-xs bg-bolt-elements-background-depth-4 p-3 rounded font-mono text-bolt-elements-textPrimary space-y-2">
<div># Docker Desktop CLI</div>
<div>docker desktop enable model-runner --tcp 12434</div>
</div>
<p className="text-xs text-bolt-elements-textSecondary">Alternatively, enable in Docker Desktop: Settings → Features in development → Model Runner → Enable host-side TCP support.</p>
</div>

{/* Pull a model */}
<div className="space-y-4">
<h4 className="font-medium text-bolt-elements-textPrimary flex items-center gap-2">
<Download className="w-4 h-4" />
2. Pull a model
</h4>
<div className="text-xs bg-bolt-elements-background-depth-4 p-3 rounded font-mono text-bolt-elements-textPrimary space-y-1">
<div>docker model pull ai/smollm2</div>
</div>
<p className="text-xs text-bolt-elements-textSecondary">Models are pulled from Docker Hub and cached locally.</p>
</div>

{/* Verify API */}
<div className="space-y-4">
<h4 className="font-medium text-bolt-elements-textPrimary flex items-center gap-2">
<Terminal className="w-4 h-4" />
3. Verify OpenAI-compatible API
</h4>
<div className="text-xs bg-bolt-elements-background-depth-4 p-3 rounded font-mono text-bolt-elements-textPrimary space-y-1">
<div># List models</div>
<div>curl http://localhost:12434/engines/v1/models</div>
<div></div>
<div># Create chat completion</div>
<div>{`curl http://localhost:12434/engines/v1/chat/completions -H "Content-Type: application/json" -d '{"model":"ai/smollm2","messages":[{"role":"user","content":"hello"}]}'`}</div>
</div>
<p className="text-xs text-bolt-elements-textSecondary">From containers, use http://model-runner.docker.internal/engines/v1/…</p>
</div>

{/* Configure in Bolt DIY */}
<div className="space-y-4">
<h4 className="font-medium text-bolt-elements-textPrimary flex items-center gap-2">
<Globe className="w-4 h-4" />
4. Configure in Bolt DIY
</h4>
<ul className="text-sm text-bolt-elements-textSecondary list-disc list-inside space-y-1">
<li>Enable provider: Settings → Providers → Local → Docker Model Runner</li>
<li>Base URL: http://127.0.0.1:12434 (we route to /engines/v1 automatically)</li>
<li>If Bolt runs in Docker, we automatically use host.docker.internal</li>
</ul>
</div>

{/* Known issues */}
<div className="p-4 rounded-lg bg-yellow-500/10 border border-yellow-500/20">
<div className="flex items-center gap-2 mb-2">
<AlertCircle className="w-4 h-4 text-yellow-500" />
<span className="font-medium text-yellow-500">Known issues</span>
</div>
<ul className="text-xs text-bolt-elements-textSecondary space-y-1">
<li>docker: 'model' is not a docker command → ensure the plugin is detected by Docker Desktop.</li>
<li>Linux containers in Compose may need extra_hosts: "model-runner.docker.internal:host-gateway"</li>
</ul>
</div>
</CardContent>
</Card>

{/* LocalAI Setup Section */}
<Card className="bg-bolt-elements-background-depth-2 shadow-sm">
<CardHeader className="pb-6">
Expand Down
15 changes: 10 additions & 5 deletions app/components/@settings/tabs/providers/local/types.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
// Type definitions
export type ProviderName = 'Ollama' | 'LMStudio' | 'OpenAILike';
import type { ComponentType } from 'react';
import { Server, Monitor, Globe } from 'lucide-react';

export type ProviderName = 'Ollama' | 'LMStudio' | 'OpenAILike' | 'DockerModelRunner';

export interface OllamaModel {
name: string;
Expand Down Expand Up @@ -31,14 +34,16 @@ export interface LMStudioModel {
// Constants
export const OLLAMA_API_URL = 'http://127.0.0.1:11434';

export const PROVIDER_ICONS = {
Ollama: 'Server',
LMStudio: 'Monitor',
OpenAILike: 'Globe',
export const PROVIDER_ICONS: Record<ProviderName, ComponentType<any>> = {
Ollama: Server,
LMStudio: Monitor,
OpenAILike: Globe,
DockerModelRunner: Server,
} as const;

export const PROVIDER_DESCRIPTIONS = {
Ollama: 'Run open-source models locally on your machine',
LMStudio: 'Local model inference with LM Studio',
OpenAILike: 'Connect to OpenAI-compatible API endpoints',
DockerModelRunner: 'Docker Desktop Model Runner with OpenAI-compatible API (/engines/v1)',
} as const;
2 changes: 1 addition & 1 deletion app/components/chat/ChatBox.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ export const ChatBox: React.FC<ChatBoxProps> = (props) => {
<div>
<ClientOnly>
{() => (
<div className={props.isModelSettingsCollapsed ? 'hidden' : ''}>
<div className={classNames(props.isModelSettingsCollapsed ? 'hidden' : '', 'mb-2')}>
<ModelSelector
key={props.provider?.name + ':' + props.modelList.length}
model={props.model}
Expand Down
10 changes: 9 additions & 1 deletion app/components/chat/ModelSelector.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -573,7 +573,15 @@ export const ModelSelector = ({
</div>

{/* Model Combobox */}
<div className="relative flex w-full min-w-[70%]" onKeyDown={handleModelKeyDown} ref={modelDropdownRef}>
<div
className={classNames(
'relative flex w-full',
// Prevent overflow when provider name is long (e.g., DockerModelRunner)
provider?.name === 'DockerModelRunner' ? 'min-w-0 sm:min-w-[60%]' : 'min-w-[70%]'
)}
onKeyDown={handleModelKeyDown}
ref={modelDropdownRef}
>
<div
className={classNames(
'w-full p-2 rounded-lg border border-bolt-elements-borderColor',
Expand Down
Loading