gateway.local.example.toml

# Runvil gateway configuration, schema version 0.
# The Config UI writes this file. The ordering below is canonical so equivalent
# edits do not create avoidable merge noise.
config-version = 0

# Gateway API listener and shared bearer credential.
[server]
bind = "127.0.0.1:8081"
# Use a variable reference (api_key = "${RUNVIL_GATEWAY_API_KEY}") to
# keep the secret out of the file, or a literal for single-user setups.
api_key = "change-me-to-a-secret"
# Loopback callers that present no key are trusted (the default). On a shared
# machine any other OS account can then use the gateway, including reading
# upstream API keys from the admin config surface - set trust_loopback = false
# to require the bearer key from every caller.
trust_loopback = true

# Optional speech pipeline tuning. STT model files and roles belong in
# [[stt_model]], not this table.
[stt]
window_seconds = 15
interval_ms = 500
vocabulary = ["Runvil", "WG21", "GGUF"]
# Which whisper runtime build to download on Windows x86-64 and Linux x86-64:
# "auto" (the default) picks CUDA when an NVIDIA GPU is found and CPU
# otherwise; "cpu" and "cuda" force the build. The Linux CUDA build needs
# NVIDIA driver 570 or later: below it "auto" picks CPU, while "cuda" is
# still honored and loading it can end the gateway. It also needs a GCC 12 or
# later C++ runtime: "auto" picks CPU where the machine's libstdc++.so.6 lacks
# GLIBCXX_3.4.30, as on RHEL 9 and Amazon Linux 2023, and "cuda" there fails
# to load. On both platforms a CUDA_VISIBLE_DEVICES that hides every GPU
# counts as no GPU for "auto". On Linux a first entry that is a GPU- or MIG-
# identifier counts as visible without being matched, so one that names no
# GPU, or an abbreviated one that names several, still picks CUDA. On Windows
# "auto" counts a first entry that is not a device index, a GPU- or MIG-
# identifier included, as hiding every GPU and picks CPU; "cuda" restores GPU
# decoding for a valid identifier. On Windows with every GPU hidden, "cuda"
# decodes on the CPU and a graceful stop crashes the gateway. The Windows
# CUDA build needs driver 580 or later and has native code only for compute
# capability 8.6, 8.9, 12.0, and 12.1: on an older driver, an unreadable
# version, or any other GPU "auto" picks CPU. There "cuda" is still honored:
# a GPU without native code under a driver older than CUDA 13.3's ends the
# gateway at the first transcription, and with no usable CUDA device the
# build decodes on the CPU and a graceful stop crashes the gateway; "auto" or
# "cpu" recovers. Every x86-64 build needs SSE4.2, AVX, AVX2, BMI2, FMA, and
# F16C; on a CPU without one, speech fails to load and the gateway serves
# without it.
# whisper_backend = "auto"

# Optional artifact cache root. The default is ~/.runvil, or
# %USERPROFILE%\.runvil on Windows.
[local]
cache_dir = "~/.runvil"

# Optional built-in web search configuration.
# [tools.web_search]
# provider = "brave"
# api_key = "${BRAVE_SEARCH_API_KEY}"
# base_url = "https://api.search.brave.com/res/v1"
# default_count = 10
# max_count = 20
# max_per_host = 2
# default_freshness = ""
# default_safesearch = "moderate"
# strip_tracking = true

# A local dominion shares concurrency and a declared VRAM budget across
# selected local chat and STT models.
[[dominion]]
id = "gpu0"
kind = "local"
max_concurrency = 4
max_queue = 100
policy = "queue"
fair_scheduling = true
vram_gb = 24

# Optional remote dominion for shared provider admission.
# [[dominion]]
# id = "remote-pool"
# kind = "remote"
# max_concurrency = 10
# max_queue = 100
# policy = "queue"
# fair_scheduling = true

# Optional remote endpoint. A remote [[model]] references this id.
# [[endpoint]]
# id = "openai"
# protocol = "openai"
# base_url = "https://api.openai.com/v1"
# api_key = "${OPENAI_API_KEY}"
# dominion = "remote-pool"

# Optional remote model catalog entry.
# [[model]]
# name = "gpt-5"
# kind = "chat"
# description = "A remote general-purpose reasoning and coding model"
# context = 200000
# thinking = "switchable"
# upstream = "gpt-5"
# endpoints = ["openai"]
# default_max_tokens = 8192
# tool_dialect = "openai"
# max_output = 32768
# images = true
# parallel_tool_calls = true
# effort_levels = ["low", "medium", "high"]
# default_effort = "medium"
# adaptive_thinking = false

# Optional remote speech synthesis model, backed by its own endpoint.
# [[endpoint]]
# id = "together"
# protocol = "openai"
# base_url = "https://api.together.xyz/v1"
# api_key = "${TOGETHER_API_KEY}"
#
# [[model]]
# name = "orpheus"
# kind = "speech"
# description = "Orpheus 3B conversational speech synthesis"
# upstream = "canopylabs/orpheus-3b-0.1-ft"
# endpoints = ["together"]
# context = 8192
# voices = ["tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe"]

# Local chat model served by a managed llama-server child.
[[local_model]]
name = "qwen-local"
kind = "chat"
description = "A local analysis model for structured reasoning and long-context review"
source = "https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/main/Qwen3.5-9B-Q4_K_M.gguf"
sha256 = "03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8"
dominion = "gpu0"
parallel = 1
vram_gb = 6
context = 65536
thinking = "never"
gpu_layers = 99
flash_attention = true
cache_type_k = "q8_0"
cache_type_v = "q4_0"
n_predict = 8192
# chat_template_file = "templates/qwen.jinja"
# max_output = 8192
# default_temperature = 0.7
# images = false
# parallel_tool_calls = true

# Optional local-model companion tables follow their owning entry.
# [local_model.speculative]
# type = "draft-mtp"
# source = "/models/qwen-mtp.gguf"
# draft_max = 3
#
# [local_model.multimodal_projector]
# source = "/models/qwen-mmproj.gguf"

# Recommended digest-pinned STT pair. Profile membership enables each role.
[[stt_model]]
name = "whisper-base-en"
role = "interim"
source = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin"
sha256 = "a03779c86df3323075f5e796cb2ce5029f00ec8869eee3fdfb897afe36c6d002"
vram_gb = 1.0
dominion = "gpu0"

[[stt_model]]
name = "whisper-small-en"
role = "final"
source = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin"
sha256 = "c6138d6d58ecc8322097e0f987c32f1be8bb0a18532a3f88f734d1bbf9c41e5d"
vram_gb = 2.0
dominion = "gpu0"

# Profiles are pure catalog checklists. Settings never live under a profile.
[[profile]]
name = "work"
models = ["qwen-local", "whisper-base-en", "whisper-small-en"]

[[profile]]
name = "travel"
models = ["qwen-local"]

# Active selection is not stored in this file. The sibling state file is
# gateway.local.example.state.toml and contains one key:
# active_profile = "work"