feat(connectors): add Ollama and LM Studio as a local auth kind
Both servers expose OpenAI-compatible /v1/chat/completions and /v1/models, so we route them through pi-ai's `openai-completions` API the same way Fireworks does — but with a per-installation base URL the user enters during onboarding instead of a hardcoded one. The shared schemas grow a `local` credential payload carrying just the baseUrl; the credential resolver surfaces it on `ResolvedCredential.baseUrl`, the runtime bridge forwards it through `prepareRun`, and the worker plumbs it into `resolveModel` so the streamed Model<openai-completions> targets the user's server. The local-cost path short-circuits to zero in cost-writer since users bill their own hardware, and the live model list is fetched from the saved baseUrl by `fetchLocalConnectorModels` (Ollama and LM Studio both speak the standard OpenAI catalog shape). Onboarding ships a new LocalProviderCard and the Settings panel a LocalForm; both default to the catalog's per-connector localhost URL and accept any custom URL.
This commit is contained in:
1 parent
07d9c00fab
commit
a532120649
26 files changed
+504
-18
No files matched your search
@@ -1,4 +1,4 @@
|
||||
import { calculateCost, type Usage, type Api, type Model } from "@mariozechner/pi-ai";
|
||||
import { calculateCost, type Usage } from "@mariozechner/pi-ai";
|
||||
import type { AppDatabase } from "@main/database";
|
||||
import { getConnectorDefinition } from "@main/connectors";
|
||||
import { resolveModel, resolveModelCostFallback } from "@main/model-resolver";
|
||||
@@ -17,6 +17,12 @@ interface ObserveInput {
|
||||
usage: Usage;
|
||||
}
|
||||
|
||||
// Local OpenAI-compatible servers run on the user's own machine, so the
|
||||
// LLM call has no third-party billing line. Record a zero cost entry
|
||||
// without invoking pi-ai's calculator (which would require the model's
|
||||
// baseUrl from the credential — unnecessary work for a flat-zero cost).
|
||||
const ZERO_COST_PROVIDERS: ReadonlySet<string> = new Set(["ollama", "lm-studio"]);
|
||||
|
||||
export class CostWriter {
|
||||
constructor(private readonly deps: CostWriterDeps) {}
|
||||
|
||||
@@ -25,6 +31,18 @@ export class CostWriter {
|
||||
if (!definition.provider) {
|
||||
throw new Error(`[cost-writer] connector ${input.connectorId} has no provider mapping`);
|
||||
}
|
||||
if (ZERO_COST_PROVIDERS.has(definition.provider)) {
|
||||
this.deps.db.core.runs.recordCostEntry({
|
||||
runId: input.runId,
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
taskId: input.taskId,
|
||||
connectorId: input.connectorId,
|
||||
amountUsd: 0,
|
||||
unattributed: false,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const model = resolveModel(definition.provider, input.modelId);
|
||||
if (!model) {
|
||||
throw new Error(`[cost-writer] model ${input.modelId} not found for provider ${definition.provider}`);
|
||||
|
||||
@@ -440,6 +440,7 @@ export class RuntimeBridge {
|
||||
provider: definition.provider,
|
||||
apiKey: resolved.apiKey,
|
||||
extraEnv: resolved.extraEnv ?? {},
|
||||
baseUrl: resolved.baseUrl ?? null,
|
||||
priorMessages,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
|
||||
import type { KnownProvider } from "@mariozechner/pi-ai";
|
||||
import type { ConnectorId } from "@shared/types";
|
||||
import { OAUTH_CONNECTORS, API_KEY_CONNECTORS, CLOUD_CONNECTORS } from "./connector-definitions/index";
|
||||
import { OAUTH_CONNECTORS, API_KEY_CONNECTORS, CLOUD_CONNECTORS, LOCAL_CONNECTORS } from "./connector-definitions/index";
|
||||
|
||||
/**
|
||||
* OAuth provider id recognised by the AgentCompany credential layer. These
|
||||
@@ -61,14 +61,21 @@ export interface ConnectorAuthGcpAdc {
|
||||
type: "gcp_adc";
|
||||
}
|
||||
|
||||
export interface ConnectorAuthLocal {
|
||||
type: "local";
|
||||
/** Pre-filled base URL the onboarding card shows on first save. */
|
||||
defaultBaseUrl: string;
|
||||
}
|
||||
|
||||
export type ConnectorAuth =
|
||||
| ConnectorAuthOAuth
|
||||
| ConnectorAuthApiKey
|
||||
| ConnectorAuthAws
|
||||
| ConnectorAuthAzure
|
||||
| ConnectorAuthGcpAdc;
|
||||
| ConnectorAuthGcpAdc
|
||||
| ConnectorAuthLocal;
|
||||
|
||||
export type LocalModelResolverProvider = "fireworks" | "fireworks-firepass";
|
||||
export type LocalModelResolverProvider = "fireworks" | "fireworks-firepass" | "ollama" | "lm-studio";
|
||||
export type ConnectorProvider = KnownProvider | LocalModelResolverProvider;
|
||||
|
||||
export interface ConnectorDefinition {
|
||||
@@ -94,4 +101,5 @@ export const CONNECTORS: ConnectorDefinition[] = [
|
||||
...OAUTH_CONNECTORS,
|
||||
...API_KEY_CONNECTORS,
|
||||
...CLOUD_CONNECTORS,
|
||||
...LOCAL_CONNECTORS,
|
||||
];
|
||||
@@ -9,3 +9,4 @@
|
||||
export { OAUTH_CONNECTORS } from "./oauth";
|
||||
export { API_KEY_CONNECTORS } from "./apikey";
|
||||
export { CLOUD_CONNECTORS } from "./cloud";
|
||||
export { LOCAL_CONNECTORS } from "./local";
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* Local pi-mono connectors.
|
||||
*
|
||||
* Ollama and LM Studio both expose an OpenAI-compatible
|
||||
* `/v1/chat/completions` endpoint, so they are routed through pi-ai's
|
||||
* `openai-completions` API path with a per-installation `baseUrl` that
|
||||
* the user enters during onboarding. The provider ids are deliberately
|
||||
* absent from `@mariozechner/pi-ai`'s `KnownProvider` set — model
|
||||
* dispatch resolves them locally via `src/main/model-resolver.ts` the
|
||||
* same way `fireworks` and `fireworks-firepass` do.
|
||||
*
|
||||
* No environment variable name accompanies the auth descriptor: a local
|
||||
* server's address is a user choice, never read from `process.env`.
|
||||
*/
|
||||
|
||||
import type { ConnectorDefinition } from "../connector-definitions";
|
||||
|
||||
export const LOCAL_CONNECTORS: ConnectorDefinition[] = [
|
||||
{
|
||||
id: "ollama",
|
||||
provider: "ollama",
|
||||
label: "Ollama",
|
||||
description:
|
||||
"Local Ollama server (https://ollama.com) — runs open-weights models on your own machine through its OpenAI-compatible endpoint.",
|
||||
defaultModel: "llama3.2",
|
||||
auth: { type: "local", defaultBaseUrl: "http://localhost:11434/v1" },
|
||||
notes:
|
||||
"Ollama exposes /v1/chat/completions and /v1/models. The default port is 11434; override if you launched `ollama serve` on a different address.",
|
||||
},
|
||||
{
|
||||
id: "lm-studio",
|
||||
provider: "lm-studio",
|
||||
label: "LM Studio",
|
||||
description:
|
||||
"Local LM Studio server (https://lmstudio.ai) — host any loaded model through its OpenAI-compatible endpoint.",
|
||||
defaultModel: "local-model",
|
||||
auth: { type: "local", defaultBaseUrl: "http://localhost:1234/v1" },
|
||||
notes:
|
||||
"Enable LM Studio's local server from the Developer tab; the default port is 1234. Model ids match whatever you have loaded.",
|
||||
},
|
||||
];
|
||||
@@ -109,6 +109,15 @@ export function createConnectorHealthService({
|
||||
}
|
||||
return { status: "ready", authState: "ok", lastError: null };
|
||||
}
|
||||
case "local": {
|
||||
if (stored?.kind !== "local") {
|
||||
return { status: "auth_required", authState: "required", lastError: "Local server URL not configured." };
|
||||
}
|
||||
if (!stored.value.baseUrl) {
|
||||
return { status: "error", authState: "error", lastError: "Local server URL missing." };
|
||||
}
|
||||
return { status: "ready", authState: "ok", lastError: null };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,8 @@ const RESERVED_ENV_BINDING_KEYS = new Set([
|
||||
const LOCAL_MODEL_RESOLVER_PROVIDERS: ReadonlySet<LocalModelResolverProvider> = new Set([
|
||||
"fireworks",
|
||||
"fireworks-firepass",
|
||||
"ollama",
|
||||
"lm-studio",
|
||||
]);
|
||||
const PI_KNOWN_PROVIDER_IDS: ReadonlySet<string> = new Set(getProviders());
|
||||
|
||||
@@ -110,6 +112,11 @@ function renderConfigurationDoc(definition: ConnectorDefinition): string {
|
||||
definition.label,
|
||||
"Uses Google Cloud Application Default Credentials. Run `gcloud auth application-default login` or set GOOGLE_APPLICATION_CREDENTIALS.",
|
||||
);
|
||||
case "local":
|
||||
return buildCloudDoc(
|
||||
definition.label,
|
||||
`OpenAI-compatible local server. Default base URL: ${definition.auth.defaultBaseUrl}. Configure under Onboarding → Providers (Advanced) and pick a model id that your server actually serves.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -34,6 +34,8 @@ export class CredentialError extends UpstreamError {
|
||||
export interface ResolvedCredential {
|
||||
apiKey: string;
|
||||
extraEnv?: Record<string, string>;
|
||||
/** Base URL for OpenAI-compatible local servers (Ollama, LM Studio). */
|
||||
baseUrl?: string;
|
||||
}
|
||||
|
||||
export class CredentialResolver {
|
||||
@@ -144,6 +146,16 @@ export class CredentialResolver {
|
||||
if (cfg.credentialsPath) env.GOOGLE_APPLICATION_CREDENTIALS = cfg.credentialsPath;
|
||||
return { apiKey: "", extraEnv: env };
|
||||
}
|
||||
case "local": {
|
||||
if (stored?.kind !== "local") {
|
||||
throw new CredentialError("CREDENTIAL_MISSING", connectorId);
|
||||
}
|
||||
// Ollama and LM Studio ignore the bearer value but the OpenAI SDK
|
||||
// refuses an empty string, so we hand it the connector id as a
|
||||
// stable non-empty placeholder. The actual transport is keyed off
|
||||
// `baseUrl`, surfaced to the worker through the stream model.
|
||||
return { apiKey: connectorId, baseUrl: stored.value.baseUrl };
|
||||
}
|
||||
default: {
|
||||
const exhaustive: never = auth;
|
||||
throw new CredentialError("CREDENTIAL_INVALID", connectorId, `Unsupported auth descriptor: ${JSON.stringify(exhaustive)}`);
|
||||
|
||||
@@ -39,6 +39,8 @@ export function deriveDisplayHint(payload: StoredCredentialPayload): string {
|
||||
: "no-expiry";
|
||||
return `oauth:expires=${expiresLabel}`;
|
||||
}
|
||||
case "local":
|
||||
return `local:${payload.value.baseUrl}`;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import type {
|
||||
AwsCredential,
|
||||
AzureCredential,
|
||||
GcpAdcCredential,
|
||||
LocalCredential,
|
||||
} from "@shared/contracts/schemas-credentials";
|
||||
|
||||
/**
|
||||
@@ -25,6 +26,7 @@ export type StoredCredentialPayload =
|
||||
| { kind: "api_key"; value: ApiKeyValue }
|
||||
| { kind: "azure"; value: AzureCredential }
|
||||
| { kind: "aws"; value: AwsCredential }
|
||||
| { kind: "gcp_adc"; value: GcpAdcCredential };
|
||||
| { kind: "gcp_adc"; value: GcpAdcCredential }
|
||||
| { kind: "local"; value: LocalCredential };
|
||||
|
||||
export type CredentialKind = StoredCredentialPayload["kind"];
|
||||
@@ -2,7 +2,7 @@ import type { DatabaseSync } from "node:sqlite";
|
||||
import type { ReadAll, Transaction } from "./helpers";
|
||||
import { nowIso } from "./helpers";
|
||||
|
||||
export type CredentialKind = "oauth" | "api_key" | "aws" | "azure" | "gcp_adc";
|
||||
export type CredentialKind = "oauth" | "api_key" | "aws" | "azure" | "gcp_adc" | "local";
|
||||
export type CredentialStatus = "active" | "expired" | "revoked" | "error";
|
||||
export type CredentialBackend = "safe_storage" | "vault";
|
||||
|
||||
|
||||
@@ -177,7 +177,7 @@ export function registerCredentialIpcHandlers(deps: CredentialHandlerDeps): () =
|
||||
}
|
||||
const provider = definition.provider;
|
||||
const defaultModel = definition.defaultModel;
|
||||
const model = resolveModel(provider, defaultModel);
|
||||
const model = resolveModel(provider, defaultModel, { baseUrl: resolved.baseUrl });
|
||||
if (!model) return ok({ ok: false, message: `Unknown model ${defaultModel} for provider ${provider}` });
|
||||
const extraHeaders = resolveExtraStreamHeaders(provider, resolved.apiKey);
|
||||
const options: SimpleStreamOptions = {
|
||||
|
||||
@@ -184,8 +184,18 @@ export function createMainRuntimeSupport(state: MainRuntimeState) {
|
||||
// user has selected a company; the onboarding path uses this
|
||||
// catalog directly because no company exists yet.
|
||||
const { getConnectorDefinition } = await import("./connectors");
|
||||
const { resolveModels } = await import("./model-resolver");
|
||||
const { resolveModels, fetchLocalConnectorModels } = await import("./model-resolver");
|
||||
const definition = getConnectorDefinition(connectorId as ConnectorId);
|
||||
if (definition.auth.type === "local") {
|
||||
// Local servers' catalogs depend on the user's installed/loaded
|
||||
// models. Read the saved baseUrl and ask the server directly.
|
||||
const piRuntime = requireServiceGraph().getPiRuntime();
|
||||
const credentials = piRuntime?.credentialStore.read(connectorId);
|
||||
const baseUrl = credentials?.kind === "local" ? credentials.value.baseUrl : null;
|
||||
if (!baseUrl) return [];
|
||||
const models = await fetchLocalConnectorModels(baseUrl);
|
||||
return models.map((m) => ({ id: m.id, label: m.name }));
|
||||
}
|
||||
return resolveModels(definition.provider).map((m) => ({ id: m.id, label: m.name }));
|
||||
},
|
||||
logger: log,
|
||||
|
||||
@@ -53,6 +53,21 @@ const FIREWORKS_CURATED_MODELS: Array<{
|
||||
const FIREWORKS_FIREPASS_COST = { input: 0, output: 0 };
|
||||
const FIREWORKS_FALLBACK_COST = { input: 0.50, output: 0.50 };
|
||||
|
||||
// Local OpenAI-compatible servers (Ollama, LM Studio) bill the user's own
|
||||
// hardware, so the surfaced cost is zero. Context window varies wildly
|
||||
// across self-hosted models; the configured window is conservative
|
||||
// enough to keep pi's compactor honest without rejecting reasonable
|
||||
// requests on smaller models.
|
||||
const LOCAL_DEFAULT_COST = { input: 0, output: 0 };
|
||||
const LOCAL_DEFAULT_CONTEXT = 32_768;
|
||||
const LOCAL_DEFAULT_MAX_TOKENS = 8192;
|
||||
|
||||
type LocalOpenAiCompatProvider = "ollama" | "lm-studio";
|
||||
|
||||
function isLocalOpenAiCompatProvider(provider: string): provider is LocalOpenAiCompatProvider {
|
||||
return provider === "ollama" || provider === "lm-studio";
|
||||
}
|
||||
|
||||
function buildFireworksModel(
|
||||
modelId: string,
|
||||
name?: string,
|
||||
@@ -74,7 +89,30 @@ function buildFireworksModel(
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveModel(provider: string, modelId: string): Model<Api> | null {
|
||||
function buildLocalModel(
|
||||
provider: LocalOpenAiCompatProvider,
|
||||
modelId: string,
|
||||
baseUrl: string,
|
||||
): Model<"openai-completions"> {
|
||||
return {
|
||||
id: modelId,
|
||||
name: modelId,
|
||||
api: "openai-completions",
|
||||
provider,
|
||||
baseUrl,
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: LOCAL_DEFAULT_COST.input, output: LOCAL_DEFAULT_COST.output, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: LOCAL_DEFAULT_CONTEXT,
|
||||
maxTokens: LOCAL_DEFAULT_MAX_TOKENS,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveModel(
|
||||
provider: string,
|
||||
modelId: string,
|
||||
options?: { baseUrl?: string },
|
||||
): Model<Api> | null {
|
||||
if (provider === "fireworks-firepass") {
|
||||
return buildFireworksModel(FIREWORKS_FIREPASS_MODEL_ID, FIREWORKS_FIREPASS_MODEL_NAME, 262_144, FIREWORKS_FIREPASS_COST);
|
||||
}
|
||||
@@ -82,6 +120,13 @@ export function resolveModel(provider: string, modelId: string): Model<Api> | nu
|
||||
const curated = FIREWORKS_CURATED_MODELS.find((entry) => entry.id === modelId);
|
||||
return buildFireworksModel(modelId, curated?.name, curated?.contextWindow, curated?.cost);
|
||||
}
|
||||
if (isLocalOpenAiCompatProvider(provider)) {
|
||||
if (!options?.baseUrl) {
|
||||
log.warn(`[model-resolver] ${provider} requires a baseUrl from credentials; none provided.`);
|
||||
return null;
|
||||
}
|
||||
return buildLocalModel(provider, modelId, options.baseUrl);
|
||||
}
|
||||
if (!isPiKnownProvider(provider)) {
|
||||
log.warn(`[model-resolver] Unknown pi provider: ${provider}`);
|
||||
return null;
|
||||
@@ -144,6 +189,13 @@ export function resolveModels(provider: string): Array<{ id: string; name: strin
|
||||
if (provider === "fireworks") {
|
||||
return FIREWORKS_CURATED_MODELS.map(({ id, name }) => ({ id, name }));
|
||||
}
|
||||
if (isLocalOpenAiCompatProvider(provider)) {
|
||||
// Local-server catalogs depend on what the user has loaded and must be
|
||||
// fetched against the saved baseUrl. The sync resolver returns an
|
||||
// empty list; the renderer reaches the live catalog through
|
||||
// `listLocalConnectorModels` in `main-runtime-support.ts`.
|
||||
return [];
|
||||
}
|
||||
if (!isPiKnownProvider(provider)) {
|
||||
log.warn(`[model-resolver] Unknown pi provider: ${provider}`);
|
||||
return [];
|
||||
@@ -157,6 +209,37 @@ export function resolveModels(provider: string): Array<{ id: string; name: strin
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the live model list from a local OpenAI-compatible server.
|
||||
*
|
||||
* Both Ollama (`/v1/models`) and LM Studio (`/v1/models`) expose the
|
||||
* standard OpenAI catalog shape — `{ data: [{ id, ...}] }` — so a single
|
||||
* fetch path handles both. Returns an empty list when the server is
|
||||
* unreachable so the renderer can surface a "not running" message
|
||||
* instead of throwing.
|
||||
*/
|
||||
export async function fetchLocalConnectorModels(
|
||||
baseUrl: string,
|
||||
): Promise<Array<{ id: string; name: string }>> {
|
||||
const url = `${baseUrl.replace(/\/$/, "")}/models`;
|
||||
try {
|
||||
const response = await fetch(url, { method: "GET" });
|
||||
if (!response.ok) {
|
||||
log.warn(`[model-resolver] local model list ${url} returned HTTP ${response.status}`);
|
||||
return [];
|
||||
}
|
||||
const body = (await response.json()) as { data?: Array<{ id?: unknown }> };
|
||||
if (!Array.isArray(body.data)) return [];
|
||||
return body.data
|
||||
.map((row) => (typeof row?.id === "string" ? row.id : null))
|
||||
.filter((id): id is string => id !== null)
|
||||
.map((id) => ({ id, name: id }));
|
||||
} catch (err) {
|
||||
log.warn(`[model-resolver] fetch ${url} failed:`, err instanceof Error ? err.message : String(err));
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extra HTTP headers that must be merged into the streamSimple call for the
|
||||
* given provider. Returns undefined when no extra headers are required.
|
||||
|
||||
@@ -19,6 +19,8 @@ interface PrepareRunResult {
|
||||
provider: string;
|
||||
apiKey: string;
|
||||
extraEnv: Record<string, string>;
|
||||
/** OpenAI-compatible server base URL for local connectors (Ollama, LM Studio). */
|
||||
baseUrl: string | null;
|
||||
priorMessages: AgentMessage[];
|
||||
}
|
||||
|
||||
@@ -133,7 +135,9 @@ export class WorkerRunner {
|
||||
const runEnv = mergeRunEnvs(job.env, prepared.extraEnv);
|
||||
redact = buildRedactor(runEnv);
|
||||
|
||||
const model = resolveModel(prepared.provider, job.model);
|
||||
const model = resolveModel(prepared.provider, job.model, {
|
||||
baseUrl: prepared.baseUrl ?? undefined,
|
||||
});
|
||||
if (!model) {
|
||||
throw new Error(`Unknown model ${job.model} for provider ${prepared.provider}.`);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user