diff --git a/.gitignore b/.gitignore index cc9cf1d..595686f 100644 --- a/.gitignore +++ b/.gitignore @@ -69,4 +69,6 @@ PLAN.md # Third Party: tracked as git submodules (see .gitmodules), not ignored. # Built artifacts (reproduce with scripts/build-llama-xcframework.sh) -Frameworks/llama.xcframework/ \ No newline at end of file +Frameworks/llama.xcframework/ +Frameworks/whisper.xcframework/ +Frameworks/sd.xcframework/ \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 5eac229..b441a60 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,11 @@ path = third_party/llama.cpp url = https://github.com/ggml-org/llama.cpp shallow = true +[submodule "third_party/whisper.cpp"] + path = third_party/whisper.cpp + url = https://github.com/ggml-org/whisper.cpp + shallow = true +[submodule "third_party/stable-diffusion.cpp"] + path = third_party/stable-diffusion.cpp + url = https://github.com/leejet/stable-diffusion.cpp + shallow = true diff --git a/README.md b/README.md index 04776fb..709065d 100644 --- a/README.md +++ b/README.md @@ -35,17 +35,25 @@ Baseline device: **iPhone 11 (A13, 4 GB RAM)**. ## Layout - `SynapLink/` — app sources (SwiftUI + Swift service layer + C/C++ engine) -- `scripts/build-llama-xcframework.sh` — builds `Frameworks/llama.xcframework` - (Metal + libmtmd) from a pinned llama.cpp tag -- `scripts/test-engine-macos.sh` — desktop functional test of the engine core +- `scripts/build-frameworks.sh` — builds all three native xcframeworks (run this first) +- `scripts/build-{llama,whisper,sd}-xcframework.sh` — individual framework builds + (llama.cpp + libmtmd, whisper.cpp ASR, stable-diffusion.cpp image-gen) +- `scripts/test-{engine,whisper,sd}-macos.sh` — desktop functional tests of the bridges - `docs/PHASE_0.md` — scaffolding & inference core: build + smoke-test instructions - `docs/PHASE_1.md` — text chat MVP: architecture, model catalog, chat pipeline - `docs/PHASE_2.md` — multimodal input: attachments, capture, media validation +- `docs/PHASE_3.md` — sidecar specialists: whisper ASR, SmolVLM vision, SD image-gen ## Quick start +The `*.xcframework`s under `Frameworks/` are **gitignored build artifacts** — +you must build them before opening Xcode, or the build fails with +"There is no XCFramework found at …". One command does it (≈40–60 min first run; +each is skipped if already present): + ```sh -scripts/build-llama-xcframework.sh +git submodule update --init # llama.cpp, whisper.cpp, stable-diffusion.cpp +scripts/build-frameworks.sh # → Frameworks/{llama,whisper,sd}.xcframework open SynapLink.xcodeproj ``` diff --git a/SynapLink.xcodeproj/project.pbxproj b/SynapLink.xcodeproj/project.pbxproj index 1f269dc..482192a 100644 --- a/SynapLink.xcodeproj/project.pbxproj +++ b/SynapLink.xcodeproj/project.pbxproj @@ -10,6 +10,10 @@ AA0000000000000000000031 /* llama.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000030 /* llama.xcframework */; }; AA0000000000000000000032 /* llama.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000030 /* llama.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; AA0000000000000000000062 /* Hub in Frameworks */ = {isa = PBXBuildFile; productRef = AA0000000000000000000061 /* Hub */; }; + AA0000000000000000000071 /* whisper.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000070 /* whisper.xcframework */; }; + AA0000000000000000000072 /* whisper.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000070 /* whisper.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + AA0000000000000000000081 /* sd.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000080 /* sd.xcframework */; }; + AA0000000000000000000082 /* sd.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000080 /* sd.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; /* End PBXBuildFile section */ /* Begin PBXCopyFilesBuildPhase section */ @@ -20,6 +24,8 @@ dstSubfolderSpec = 10; files = ( AA0000000000000000000032 /* llama.xcframework in Embed Frameworks */, + AA0000000000000000000072 /* whisper.xcframework in Embed Frameworks */, + AA0000000000000000000082 /* sd.xcframework in Embed Frameworks */, ); name = "Embed Frameworks"; runOnlyForDeploymentPostprocessing = 0; @@ -29,6 +35,8 @@ /* Begin PBXFileReference section */ AA0000000000000000000020 /* SynapLink.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = SynapLink.app; sourceTree = BUILT_PRODUCTS_DIR; }; AA0000000000000000000030 /* llama.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = llama.xcframework; path = Frameworks/llama.xcframework; sourceTree = ""; }; + AA0000000000000000000070 /* whisper.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = whisper.xcframework; path = Frameworks/whisper.xcframework; sourceTree = ""; }; + AA0000000000000000000080 /* sd.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = sd.xcframework; path = Frameworks/sd.xcframework; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFileSystemSynchronizedRootGroup section */ @@ -45,6 +53,8 @@ buildActionMask = 2147483647; files = ( AA0000000000000000000031 /* llama.xcframework in Frameworks */, + AA0000000000000000000071 /* whisper.xcframework in Frameworks */, + AA0000000000000000000081 /* sd.xcframework in Frameworks */, AA0000000000000000000062 /* Hub in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -73,6 +83,8 @@ isa = PBXGroup; children = ( AA0000000000000000000030 /* llama.xcframework */, + AA0000000000000000000070 /* whisper.xcframework */, + AA0000000000000000000080 /* sd.xcframework */, ); name = Frameworks; sourceTree = ""; diff --git a/SynapLink/Core/Engine/SynapLink-Bridging-Header.h b/SynapLink/Core/Engine/SynapLink-Bridging-Header.h index 3b1e034..7bd2191 100644 --- a/SynapLink/Core/Engine/SynapLink-Bridging-Header.h +++ b/SynapLink/Core/Engine/SynapLink-Bridging-Header.h @@ -9,6 +9,8 @@ #pragma once #include "synap_engine.h" +#include "synap_whisper.h" +#include "synap_sd.h" #if __has_include() #include // os_proc_available_memory() — jetsam headroom (iOS only) diff --git a/SynapLink/Core/Engine/synap_engine.cpp b/SynapLink/Core/Engine/synap_engine.cpp index d40bcf4..cde4dc2 100644 --- a/SynapLink/Core/Engine/synap_engine.cpp +++ b/SynapLink/Core/Engine/synap_engine.cpp @@ -20,11 +20,25 @@ #include #include +#include #include #include // MARK: - §1 Lifecycle +// Refcount the global llama backend so multiple engines can coexist (main +// text model + an on-demand vision specialist). Without this, freeing one +// engine's backend would pull the rug out from under the other. +static std::atomic g_backend_refcount{0}; + +static void backend_retain() { + if (g_backend_refcount.fetch_add(1) == 0) { llama_backend_init(); } +} + +static void backend_release() { + if (g_backend_refcount.fetch_sub(1) == 1) { llama_backend_free(); } +} + SynapEngineParams synap_engine_params_default(void) { SynapEngineParams p = {}; p.model_path = nullptr; @@ -64,14 +78,14 @@ static llama_sampler* build_sampler(const SynapEngineParams& p) { SynapEngine* synap_engine_create(const SynapEngineParams* params) { if (!params || !params->model_path) { return nullptr; } - llama_backend_init(); + backend_retain(); llama_model_params mp = llama_model_default_params(); mp.n_gpu_layers = params->n_gpu_layers; llama_model* model = llama_model_load_from_file(params->model_path, mp); if (!model) { - llama_backend_free(); + backend_release(); return nullptr; } @@ -87,7 +101,7 @@ SynapEngine* synap_engine_create(const SynapEngineParams* params) { llama_context* ctx = llama_init_from_model(model, cp); if (!ctx) { llama_model_free(model); - llama_backend_free(); + backend_release(); return nullptr; } @@ -95,7 +109,7 @@ SynapEngine* synap_engine_create(const SynapEngineParams* params) { if (!sampler) { llama_free(ctx); llama_model_free(model); - llama_backend_free(); + backend_release(); return nullptr; } @@ -113,7 +127,7 @@ SynapEngine* synap_engine_create(const SynapEngineParams* params) { llama_sampler_free(sampler); llama_free(ctx); llama_model_free(model); - llama_backend_free(); + backend_release(); return nullptr; } } @@ -132,7 +146,7 @@ void synap_engine_free(SynapEngine* engine) { if (engine->sampler) { llama_sampler_free(engine->sampler); } if (engine->ctx) { llama_free(engine->ctx); } if (engine->model) { llama_model_free(engine->model); } - llama_backend_free(); + backend_release(); delete engine; } diff --git a/SynapLink/Core/Engine/synap_sd.cpp b/SynapLink/Core/Engine/synap_sd.cpp new file mode 100644 index 0000000..e004698 --- /dev/null +++ b/SynapLink/Core/Engine/synap_sd.cpp @@ -0,0 +1,114 @@ +// +// synap_sd.cpp +// SynapLink +// +// C++ implementation of the synap_sd bridge over stable-diffusion.cpp. +// + +#include "synap_sd.h" + +#include + +#include +#include + +struct SynapSD { + sd_ctx_t* ctx = nullptr; +}; + +// stable-diffusion.cpp logs verbosely (per-step, per-tensor) to stderr by +// default. Route everything to a no-op so it doesn't flood logs / temp space. +static void sd_silent_log(enum sd_log_level_t level, const char* text, void* data) { + (void)level; + (void)text; + (void)data; +} + +static void sd_silent_progress(int step, int steps, float time, void* data) { + (void)step; + (void)steps; + (void)time; + (void)data; +} + +SynapSD* synap_sd_create(const char* model_path, const char* taesd_path, int32_t n_threads, bool use_gpu) { + if (!model_path) { return nullptr; } + + sd_set_log_callback(sd_silent_log, nullptr); + sd_set_progress_callback(sd_silent_progress, nullptr); + + sd_ctx_params_t params; + sd_ctx_params_init(¶ms); + params.model_path = model_path; + if (taesd_path && taesd_path[0] != '\0') { params.taesd_path = taesd_path; } + params.n_threads = n_threads > 0 ? n_threads : 4; + // mmap avoids loading all weights into anonymous RAM. Flash attention is + // left at the init default (off): forcing it produced blank output on the + // CPU backend, and it is not reliably supported across SD backends. + params.enable_mmap = true; + // Backend selection: default resolves GPU-first, which is correct on an + // Apple-GPU device (Metal). On hosts whose only GPU is non-Apple (the + // Intel dev Mac, the simulator) Metal produces garbage, so pin to CPU. + params.backend = use_gpu ? nullptr : "CPU"; + + sd_ctx_t* ctx = new_sd_ctx(¶ms); + if (!ctx) { return nullptr; } + + auto* sd = new SynapSD(); + sd->ctx = ctx; + return sd; +} + +void synap_sd_free(SynapSD* sd) { + if (!sd) { return; } + if (sd->ctx) { free_sd_ctx(sd->ctx); } + delete sd; +} + +uint8_t* synap_sd_generate(SynapSD* sd, const char* prompt, const char* negative_prompt, int32_t width, int32_t height, + int32_t steps, float cfg_scale, int32_t sample_method, int64_t seed, int32_t* out_width, + int32_t* out_height) { + if (!sd || !sd->ctx || !prompt) { return nullptr; } + + sd_img_gen_params_t g; + sd_img_gen_params_init(&g); + g.prompt = prompt; + g.negative_prompt = negative_prompt ? negative_prompt : ""; + g.width = width; + g.height = height; + g.sample_params.sample_steps = steps; + g.sample_params.guidance.txt_cfg = cfg_scale; + g.sample_params.sample_method = static_cast(sample_method); + g.seed = seed; + g.batch_count = 1; + + sd_image_t* images = generate_image(sd->ctx, &g); + if (!images || !images[0].data) { + if (images) { free_sd_images(images, 1); } + return nullptr; + } + + const sd_image_t img = images[0]; + const size_t bytes = static_cast(img.width) * img.height * img.channel; + // Normalize to 3-channel RGB for the caller. + uint8_t* out = static_cast(malloc(static_cast(img.width) * img.height * 3)); + if (out) { + if (img.channel == 3) { + memcpy(out, img.data, bytes); + } else { + for (uint32_t i = 0; i < img.width * img.height; ++i) { + out[i * 3 + 0] = img.data[i * img.channel + 0]; + out[i * 3 + 1] = img.data[i * img.channel + (img.channel > 1 ? 1 : 0)]; + out[i * 3 + 2] = img.data[i * img.channel + (img.channel > 2 ? 2 : 0)]; + } + } + if (out_width) { *out_width = static_cast(img.width); } + if (out_height) { *out_height = static_cast(img.height); } + } + free_sd_images(images, 1); + return out; +} + +void synap_sd_free_rgb(uint8_t* rgb) { + free(rgb); +} diff --git a/SynapLink/Core/Engine/synap_sd.h b/SynapLink/Core/Engine/synap_sd.h new file mode 100644 index 0000000..0cbc5e8 --- /dev/null +++ b/SynapLink/Core/Engine/synap_sd.h @@ -0,0 +1,43 @@ +// +// synap_sd.h +// SynapLink +// +// Pure-C bridge to stable-diffusion.cpp — the experimental on-device image +// generation specialist. Swift imports this via the bridging header; the +// sd/ggml C++ symbols stay inside sd.xcframework. +// +// Output is raw RGB (width*height*3 bytes); Swift JPEG-encodes it. +// + +#pragma once + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SynapSD SynapSD; + +/// Load a Stable Diffusion GGUF (full checkpoint). `taesd_path` is an optional +/// tiny VAE for memory-light decoding (NULL to use the checkpoint's VAE). +/// Returns NULL on failure. +SynapSD* synap_sd_create(const char* model_path, const char* taesd_path, int32_t n_threads, bool use_gpu); + +void synap_sd_free(SynapSD* sd); + +/// Generate one image. `sample_method` is a sample_method_t raw value +/// (0 = EULER, 1 = EULER_A, 9 = LCM). Returns a malloc'd RGB buffer of +/// (*out_width * *out_height * 3) bytes — free with synap_sd_free_rgb — or +/// NULL on failure. +uint8_t* synap_sd_generate(SynapSD* sd, const char* prompt, const char* negative_prompt, int32_t width, int32_t height, + int32_t steps, float cfg_scale, int32_t sample_method, int64_t seed, int32_t* out_width, + int32_t* out_height); + +void synap_sd_free_rgb(uint8_t* rgb); + +#ifdef __cplusplus +} +#endif diff --git a/SynapLink/Core/Engine/synap_whisper.cpp b/SynapLink/Core/Engine/synap_whisper.cpp new file mode 100644 index 0000000..593262d --- /dev/null +++ b/SynapLink/Core/Engine/synap_whisper.cpp @@ -0,0 +1,85 @@ +// +// synap_whisper.cpp +// SynapLink +// +// C++ implementation of the synap_whisper bridge over whisper.cpp's C API. +// + +#include "synap_whisper.h" + +#include + +#include +#include + +struct SynapWhisper { + whisper_context* ctx = nullptr; + int32_t n_threads = 4; +}; + +SynapWhisper* synap_whisper_create(const char* model_path, bool use_gpu, int32_t n_threads) { + if (!model_path) { return nullptr; } + + whisper_context_params cparams = whisper_context_default_params(); + cparams.use_gpu = use_gpu; + cparams.flash_attn = false; + + whisper_context* ctx = whisper_init_from_file_with_params(model_path, cparams); + if (!ctx) { return nullptr; } + + auto* w = new SynapWhisper(); + w->ctx = ctx; + w->n_threads = n_threads > 0 ? n_threads : 4; + return w; +} + +void synap_whisper_free(SynapWhisper* whisper) { + if (!whisper) { return; } + if (whisper->ctx) { whisper_free(whisper->ctx); } + delete whisper; +} + +int32_t synap_whisper_transcribe(SynapWhisper* whisper, const float* samples, int32_t n_samples, const char* language, + char* out_buf, int32_t out_buf_size) { + if (!whisper || !whisper->ctx || !samples || n_samples <= 0 || !out_buf || out_buf_size <= 0) { return -1; } + + whisper_full_params params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY); + params.n_threads = whisper->n_threads; + params.print_progress = false; + params.print_realtime = false; + params.print_timestamps = false; + params.translate = false; + params.no_timestamps = true; + params.suppress_blank = true; + if (language && language[0] != '\0' && std::string(language) != "auto") { + params.language = language; + params.detect_language = false; + } else { + params.language = "auto"; + params.detect_language = true; + } + + if (whisper_full(whisper->ctx, params, samples, n_samples) != 0) { return -2; } + + std::string text; + const int n_segments = whisper_full_n_segments(whisper->ctx); + for (int i = 0; i < n_segments; ++i) { + const char* seg = whisper_full_get_segment_text(whisper->ctx, i); + if (seg) { text += seg; } + } + + // Trim the leading space whisper segments usually carry. + const auto begin = text.find_first_not_of(" \t\n\r"); + if (begin == std::string::npos) { + out_buf[0] = '\0'; + return 0; + } + const auto end = text.find_last_not_of(" \t\n\r"); + text = text.substr(begin, end - begin + 1); + + const int32_t total = static_cast(text.size()); + const int32_t n = total < out_buf_size - 1 ? total : out_buf_size - 1; + memcpy(out_buf, text.data(), static_cast(n)); + out_buf[n] = '\0'; + return total; +} diff --git a/SynapLink/Core/Engine/synap_whisper.h b/SynapLink/Core/Engine/synap_whisper.h new file mode 100644 index 0000000..fcfe4eb --- /dev/null +++ b/SynapLink/Core/Engine/synap_whisper.h @@ -0,0 +1,46 @@ +// +// synap_whisper.h +// SynapLink +// +// Pure-C bridge to whisper.cpp — on-device speech-to-text for the iPhone 11 +// tier (the omni Gemma 4 E2B audio path can't fit in 4 GB). Swift imports +// this through the bridging header; the whisper/ggml C++ symbols stay inside +// the whisper.xcframework. +// +// Audio contract: mono float32 PCM at 16 kHz (SYNAP_WHISPER_SAMPLE_RATE). +// The caller decodes the recording to that format (AVAudioConverter). +// + +#pragma once + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define SYNAP_WHISPER_SAMPLE_RATE 16000 + +/// Opaque transcriber. Swift holds this as `OpaquePointer`. +typedef struct SynapWhisper SynapWhisper; + +/// Load a whisper GGML model (ggml-tiny.bin / ggml-base.bin ...). `use_gpu` +/// runs the encoder on Metal where available. Returns NULL on failure. +SynapWhisper* synap_whisper_create(const char* model_path, bool use_gpu, int32_t n_threads); + +/// Release the model and all state. +void synap_whisper_free(SynapWhisper* whisper); + +/// Transcribe mono float32 PCM at 16 kHz. `language` is an ISO code +/// ("en", "fr", ...) or NULL/"auto" to auto-detect. The recognized text is +/// written to `out_buf` (null-terminated). Returns the number of bytes +/// written (excluding null), or negative on error: +/// -1 invalid arguments -2 transcription failed +int32_t synap_whisper_transcribe(SynapWhisper* whisper, const float* samples, int32_t n_samples, const char* language, + char* out_buf, int32_t out_buf_size); + +#ifdef __cplusplus +} +#endif diff --git a/SynapLink/Data/DataSources/Inference/ChatSession.swift b/SynapLink/Data/DataSources/Inference/ChatSession.swift index d925183..13f7ad4 100644 --- a/SynapLink/Data/DataSources/Inference/ChatSession.swift +++ b/SynapLink/Data/DataSources/Inference/ChatSession.swift @@ -142,6 +142,30 @@ final class ChatSession { engineState = .unloaded } + // MARK: - Effective input capabilities (main model OR sidecar specialist) + + /// Image input works if the main model sees natively (E2B) or the vision + /// specialist (SmolVLM) is installed for the sidecar pipeline. + var canSendImages: Bool { + capabilities?.hasVision == true || SpecialistManager.shared.isInstalled(.vision) + } + + /// Audio input works if the main model hears natively or whisper is installed. + var canSendAudio: Bool { + capabilities?.hasAudio == true || SpecialistManager.shared.isInstalled(.whisper) + } + + /// Show the attach (＋) button if photos can be understood OR images created. + var canAttachMedia: Bool { + canSendImages || SpecialistManager.shared.isInstalled(.imageGen) + } + + /// Whisper transcribes at 16 kHz; a native-audio main model may want its + /// own rate. + var audioCaptureSampleRate: Int32 { + capabilities?.hasAudio == true ? (capabilities?.audioSampleRate ?? 16000) : 16000 + } + // MARK: - Turns func send(_ text: String, attachments: [PendingAttachment] = []) { @@ -180,17 +204,55 @@ final class ChatSession { engine.cancel() } + // MARK: - Image generation (experimental specialist) + + /// True while the SD specialist is producing an image (no token stream). + private(set) var isCreatingImage = false + + /// Generate an image from `prompt` and post it as an assistant message. + /// Unloads the main chat model first — on the 4 GB tier SD needs nearly + /// the whole memory budget. + func createImage(prompt: String) { + let trimmed = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty, !isGenerating, !isCreatingImage, let chat = activeChat else { return } + + if let saved = store.appendMessage(chatID: chat.id, role: .user, content: trimmed) { + messages.append(saved) + } + autoTitleIfNeeded(chat: chat, firstUserText: trimmed) + + isCreatingImage = true + lastError = nil + generationTask = Task { [weak self] in + guard let self else { return } + // Free the chat model's RAM for the duration of diffusion. + await self.unloadEngine() + do { + let jpeg = try await ImageGenerator.shared.generate(prompt: trimmed) + if self.activeChat?.id == chat.id, + let message = self.store.appendMessage(chatID: chat.id, role: .assistant, content: "") { + var saved = message + if let row = self.store.appendAttachment(messageID: message.id, kind: .image, data: jpeg) { + saved.attachments.append(row) + } + self.messages.append(saved) + } + } catch { + self.lastError = error.localizedDescription + } + self.isCreatingImage = false + } + } + private func generate() { guard let chat = activeChat else { return } isGenerating = true streamingText = "" lastError = nil - // The current turn's media comes from the last user message's STORED - // attachments — the same path serves fresh sends and regenerates. let lastUser = messages.last(where: { $0.role == .user }) - let mediaData = lastUser?.attachments.compactMap { store.attachmentData($0) } ?? [] - isAnalyzingMedia = !mediaData.isEmpty + let attachments = lastUser?.attachments ?? [] + isAnalyzingMedia = !attachments.isEmpty generationTask = Task { [weak self] in guard let self else { return } @@ -201,18 +263,23 @@ final class ChatSession { return } do { + // Resolve attachments to either DIRECT engine media (when the + // main model is itself multimodal, e.g. E2B) or SIDECAR text + // (whisper transcription / SmolVLM description) folded into the + // prompt when the main model is text-only (iPhone 11 tier). + let resolved = try await self.resolveMedia( + attachments, userText: lastUser?.content ?? "") + var history = self.trimmedHistory() - // Media turn: the engine replaces one marker per attachment - // with encoded image/audio chunks (mtmd_tokenize); markers - // exist only in the prompt, never in persisted content. - if !mediaData.isEmpty, let lastIndex = history.lastIndex(where: { $0.role == "user" }) { - let markers = String(repeating: InferenceEngine.mediaMarker, count: mediaData.count) - history[lastIndex].content = markers + (lastUser?.content ?? "") + if let lastIndex = history.lastIndex(where: { $0.role == "user" }) { + let markers = String(repeating: InferenceEngine.mediaMarker, + count: resolved.directMedia.count) + history[lastIndex].content = markers + resolved.promptText } let prompt = try await self.engine.applyChatTemplate(history) for try await piece in self.engine.generate( prompt: prompt, - media: mediaData, + media: resolved.directMedia, maxNewTokens: Int32(self.settings.maxNewTokens)) { self.isAnalyzingMedia = false self.streamingText += piece @@ -235,6 +302,50 @@ final class ChatSession { } } + private struct ResolvedMedia { + var directMedia: [Data] = [] // fed straight to a multimodal main model + var promptText: String // user text, augmented by sidecar output + } + + /// Decide, per attachment, between the main model's own multimodal path + /// and the sidecar specialists, and assemble what the prompt needs. + private func resolveMedia(_ attachments: [Attachment], userText: String) async throws -> ResolvedMedia { + let caps = capabilities + var result = ResolvedMedia(promptText: userText) + var contextBlocks: [String] = [] + var transcripts: [String] = [] + + for attachment in attachments { + switch attachment.kind { + case .image: + if caps?.hasVision == true, let data = store.attachmentData(attachment) { + result.directMedia.append(data) + } else if let data = store.attachmentData(attachment) { + let description = try await VisionDescriber.shared.describe( + imageData: data, userQuestion: userText) + contextBlocks.append("[Photo the user shared — description: \(description)]") + } + case .audio: + if caps?.hasAudio == true, let data = store.attachmentData(attachment) { + result.directMedia.append(data) + } else if let url = store.attachmentURL(attachment) { + let transcript = try await WhisperTranscriber.shared.transcribe(fileURL: url) + if !transcript.isEmpty { transcripts.append(transcript) } + } + } + } + + // Voice notes become (part of) the user's message; image descriptions + // are prepended as grounding context the text model can reason over. + var pieces: [String] = [] + if !contextBlocks.isEmpty { pieces.append(contextBlocks.joined(separator: "\n")) } + let spoken = transcripts.joined(separator: " ") + let combinedUser = [userText, spoken].filter { !$0.isEmpty }.joined(separator: " ") + if !combinedUser.isEmpty { pieces.append(combinedUser) } + result.promptText = pieces.joined(separator: "\n\n") + return result + } + // MARK: - Model-generated chat title /// Once the conversation has ≥5 messages, ask the model itself for a diff --git a/SynapLink/Data/DataSources/Inference/ImageGenerator.swift b/SynapLink/Data/DataSources/Inference/ImageGenerator.swift new file mode 100644 index 0000000..193fa42 --- /dev/null +++ b/SynapLink/Data/DataSources/Inference/ImageGenerator.swift @@ -0,0 +1,108 @@ +// +// ImageGenerator.swift +// SynapLink +// +// On-demand text-to-image via stable-diffusion.cpp (experimental specialist). +// Load → generate one image → unload, like the other specialists, so it only +// occupies RAM while working. The caller unloads the main chat model first; +// on the 4 GB tier SD's ~1.6 GB weights need essentially the whole budget. +// + +import Foundation +import UIKit + +enum ImageGenError: Error, LocalizedError { + case modelNotInstalled + case loadFailed + case generationFailed + case unsupportedDevice + + var errorDescription: String? { + switch self { + case .modelNotInstalled: return "The image model isn't installed. Download it in the Model Library." + case .loadFailed: return "Failed to load the image model." + case .generationFailed: return "Image generation failed." + case .unsupportedDevice: return "This device doesn't have enough memory to generate images." + } + } +} + +/// One generation request's tunables, resolved per device tier. +struct ImageGenSettings { + var width: Int32 + var height: Int32 + var steps: Int32 + var cfgScale: Float + var sampleMethod: Int32 // sample_method_t raw (1 = EULER_A) +} + +final class ImageGenerator: @unchecked Sendable { + + static let shared = ImageGenerator() + + private let queue = DispatchQueue(label: "com.dedicatus.synaplink.imagegen", qos: .userInitiated) + + private init() {} + + /// Generate an image for `prompt`. Returns JPEG data. Heavy and slow on the + /// 4 GB tier (tens of seconds to minutes); callers show progress. + func generate(prompt: String, negativePrompt: String = "") async throws -> Data { + let spec = SpecialistModel.imageGen + guard spec.isSupportedOnThisDevice else { throw ImageGenError.unsupportedDevice } + guard let model = spec.modelURL() else { throw ImageGenError.modelNotInstalled } + let taesd = spec.mmprojURL() // TAESD reuses the catalog's "mmproj" slot + let settings = RuntimeProfile.imageGenSettings + + return try await withCheckedThrowingContinuation { continuation in + queue.async { + guard let handle = synap_sd_create( + model.path, taesd?.path, RuntimeProfile.specialistThreads, + RuntimeProfile.specialistUsesGPU) else { + continuation.resume(throwing: ImageGenError.loadFailed) + return + } + defer { synap_sd_free(handle) } + + var width: Int32 = 0 + var height: Int32 = 0 + let seed = Int64.random(in: 0...Int64(Int32.max)) + let rgb = prompt.withCString { cPrompt in + negativePrompt.withCString { cNeg in + synap_sd_generate( + handle, cPrompt, cNeg, + settings.width, settings.height, settings.steps, + settings.cfgScale, settings.sampleMethod, seed, + &width, &height) + } + } + guard let rgb, width > 0, height > 0 else { + continuation.resume(throwing: ImageGenError.generationFailed) + return + } + defer { synap_sd_free_rgb(rgb) } + + guard let jpeg = Self.jpeg(fromRGB: rgb, width: Int(width), height: Int(height)) else { + continuation.resume(throwing: ImageGenError.generationFailed) + return + } + continuation.resume(returning: jpeg) + } + } + } + + /// Pack tightly-packed RGB888 into a JPEG. + private static func jpeg(fromRGB rgb: UnsafePointer, width: Int, height: Int) -> Data? { + let count = width * height * 3 + let data = Data(bytes: rgb, count: count) + guard let provider = CGDataProvider(data: data as CFData) else { return nil } + let colorSpace = CGColorSpaceCreateDeviceRGB() + guard let cgImage = CGImage( + width: width, height: height, + bitsPerComponent: 8, bitsPerPixel: 24, bytesPerRow: width * 3, + space: colorSpace, + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.none.rawValue), + provider: provider, decode: nil, shouldInterpolate: false, + intent: .defaultIntent) else { return nil } + return UIImage(cgImage: cgImage).jpegData(compressionQuality: 0.9) + } +} diff --git a/SynapLink/Data/DataSources/Inference/InferenceEngine.swift b/SynapLink/Data/DataSources/Inference/InferenceEngine.swift index c49c9f6..fb646ed 100644 --- a/SynapLink/Data/DataSources/Inference/InferenceEngine.swift +++ b/SynapLink/Data/DataSources/Inference/InferenceEngine.swift @@ -53,17 +53,21 @@ struct EngineCapabilities: Sendable { final class InferenceEngine: @unchecked Sendable { + /// The main chat engine (the model the user talks to). On-demand + /// specialists (e.g. the SmolVLM vision describer) create their own + /// transient instances — the C backend is refcounted so they coexist. static let shared = InferenceEngine() - private let queue = DispatchQueue( - label: "com.dedicatus.synaplink.inference", qos: .userInitiated) + private let queue: DispatchQueue private let handleLock = NSLock() private var _handle: OpaquePointer? /// The media marker the prompt must contain once per media attachment. static var mediaMarker: String { String(cString: synap_engine_media_marker()) } - private init() {} + init(label: String = "com.dedicatus.synaplink.inference") { + queue = DispatchQueue(label: label, qos: .userInitiated) + } deinit { if let h = currentHandle() { synap_engine_free(h) } diff --git a/SynapLink/Data/DataSources/Inference/RuntimeProfile.swift b/SynapLink/Data/DataSources/Inference/RuntimeProfile.swift index 83e4c03..bcfc992 100644 --- a/SynapLink/Data/DataSources/Inference/RuntimeProfile.swift +++ b/SynapLink/Data/DataSources/Inference/RuntimeProfile.swift @@ -47,4 +47,58 @@ enum RuntimeProfile { params.nThreads = min(params.nThreads, Int32(ProcessInfo.processInfo.processorCount)) return params } + + // MARK: - Specialists (whisper ASR, SmolVLM vision) + + /// Specialists run while the main chat model is also resident. On the + /// simulator the GPU path is emulated/absent (and CLIP-on-Metal crashes + /// on non-Apple GPUs); device specialists use Metal. + static var specialistUsesGPU: Bool { + #if targetEnvironment(simulator) + return false + #else + return true + #endif + } + + static var specialistThreads: Int32 { + Int32(min(2, ProcessInfo.processInfo.processorCount)) + } + + /// Image-generation (stable-diffusion.cpp) tunables per RAM tier. The + /// 4 GB tier is deliberately experimental: small canvas keeps the + /// diffusion working set under the jetsam ceiling, and EULER_A at a modest + /// step count is the quality/speed compromise on the A13. Bigger devices + /// get a full 512² canvas. + static var imageGenSettings: ImageGenSettings { + let eulerAncestral: Int32 = 1 + switch physicalMemoryGB { + case ..<5.0: + return ImageGenSettings(width: 384, height: 384, steps: 16, + cfgScale: 7.0, sampleMethod: eulerAncestral) + case ..<7.0: + return ImageGenSettings(width: 512, height: 512, steps: 20, + cfgScale: 7.0, sampleMethod: eulerAncestral) + default: + return ImageGenSettings(width: 512, height: 512, steps: 28, + cfgScale: 7.5, sampleMethod: eulerAncestral) + } + } + + /// Vision specialist (SmolVLM) engine params: small context — it captions + /// one image per call, no long history. + static func visionEngineParams(modelPath: String, mmprojPath: String?) -> EngineParams { + var params = EngineParams(modelPath: modelPath) + params.mmprojPath = mmprojPath + params.nCtx = 1024 + params.nThreads = specialistThreads + params.kvTypeK = .f16 + params.kvTypeV = .f16 + params.flashAttention = .disabled // tiny model; avoids the quant-KV+FA coupling + #if targetEnvironment(simulator) + params.nGPULayers = 0 + params.mmprojUseGPU = false + #endif + return params + } } diff --git a/SynapLink/Data/DataSources/Inference/VisionDescriber.swift b/SynapLink/Data/DataSources/Inference/VisionDescriber.swift new file mode 100644 index 0000000..194b83d --- /dev/null +++ b/SynapLink/Data/DataSources/Inference/VisionDescriber.swift @@ -0,0 +1,91 @@ +// +// VisionDescriber.swift +// SynapLink +// +// On-demand image captioning via SmolVLM (the vision specialist in the +// sidecar pipeline). Runs in its OWN transient engine so the main chat +// model stays resident; the C backend is refcounted to allow both. The +// resulting description is fed back to the main model as context. +// + +import Foundation + +enum VisionError: Error, LocalizedError { + case modelNotInstalled + case loadFailed + case describeFailed + + var errorDescription: String? { + switch self { + case .modelNotInstalled: return "The vision model isn't installed. Download it in the Model Library." + case .loadFailed: return "Failed to load the vision model." + case .describeFailed: return "Couldn't analyze the image." + } + } +} + +final class VisionDescriber: @unchecked Sendable { + + static let shared = VisionDescriber() + + /// Its own engine instance — coexists with InferenceEngine.shared. + private let engine = InferenceEngine(label: "com.dedicatus.synaplink.vision") + + private static let describePrompt = """ + Look at this image and describe it thoroughly in 4 to 6 complete sentences. \ + Cover: the main subject and what is happening; people (appearance, clothing, \ + expressions, what they are doing); any visible text read word for word; \ + colors, lighting, and mood; the setting or background; and any small or \ + unusual details worth noting. Be specific and concrete. + """ + + private static func focusedPrompt(_ question: String) -> String { + """ + Look at this image carefully. First describe what you see in detail — \ + the main subject, people, any visible text, colors, and setting. Then, \ + using only what is visible, answer this question: \(question) \ + Write several complete sentences. + """ + } + + private init() {} + + /// Load SmolVLM, describe the image, unload. Returns the description text. + func describe(imageData: Data, + userQuestion: String? = nil) async throws -> String { + let vision = SpecialistModel.vision + guard let model = vision.modelURL(), let mmproj = vision.mmprojURL() else { + throw VisionError.modelNotInstalled + } + + let params = RuntimeProfile.visionEngineParams(modelPath: model.path, mmprojPath: mmproj.path) + do { + _ = try await engine.load(params) + } catch { + throw VisionError.loadFailed + } + defer { Task { await engine.unload() } } + + let question = userQuestion?.trimmingCharacters(in: .whitespacesAndNewlines) + let ask = (question?.isEmpty == false) ? Self.focusedPrompt(question!) : Self.describePrompt + let userContent = InferenceEngine.mediaMarker + ask + + do { + let prompt = try await engine.applyChatTemplate([ + ChatMessage(role: "user", content: userContent) + ]) + var description = "" + for try await piece in engine.generate( + prompt: prompt, media: [imageData], maxNewTokens: 384) { + description += piece + } + let trimmed = description.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { throw VisionError.describeFailed } + return trimmed + } catch let error as VisionError { + throw error + } catch { + throw VisionError.describeFailed + } + } +} diff --git a/SynapLink/Data/DataSources/Inference/WhisperTranscriber.swift b/SynapLink/Data/DataSources/Inference/WhisperTranscriber.swift new file mode 100644 index 0000000..b89c262 --- /dev/null +++ b/SynapLink/Data/DataSources/Inference/WhisperTranscriber.swift @@ -0,0 +1,116 @@ +// +// WhisperTranscriber.swift +// SynapLink +// +// On-demand speech-to-text via the whisper.cpp bridge. A specialist in the +// sidecar pipeline: load model → transcribe one clip → unload, so it only +// occupies RAM while actually working (the main chat model stays resident). +// + +import AVFoundation +import Foundation + +enum WhisperError: Error, LocalizedError { + case modelNotInstalled + case loadFailed + case audioDecodeFailed + case transcriptionFailed + + var errorDescription: String? { + switch self { + case .modelNotInstalled: return "The speech model isn't installed. Download it in the Model Library." + case .loadFailed: return "Failed to load the speech model." + case .audioDecodeFailed: return "Couldn't read the recorded audio." + case .transcriptionFailed: return "Transcription failed." + } + } +} + +final class WhisperTranscriber: @unchecked Sendable { + + static let shared = WhisperTranscriber() + + private let queue = DispatchQueue(label: "com.dedicatus.synaplink.whisper", qos: .userInitiated) + + private init() {} + + /// Transcribe a recorded audio file (any AVFoundation-decodable format). + /// Loads the whisper model, runs, and unloads before returning. + func transcribe(fileURL: URL, languageCode: String? = nil) async throws -> String { + guard let model = SpecialistModel.whisper.modelURL() else { + throw WhisperError.modelNotInstalled + } + let samples = try Self.decodeToMono16kFloat(url: fileURL) + + return try await withCheckedThrowingContinuation { continuation in + queue.async { + guard let handle = synap_whisper_create( + model.path, RuntimeProfile.specialistUsesGPU, RuntimeProfile.specialistThreads) else { + continuation.resume(throwing: WhisperError.loadFailed) + return + } + defer { synap_whisper_free(handle) } + + var buffer = [CChar](repeating: 0, count: 8 * 1024) + let written = samples.withUnsafeBufferPointer { ptr in + synap_whisper_transcribe( + handle, ptr.baseAddress, Int32(ptr.count), + languageCode, &buffer, Int32(buffer.count)) + } + if written < 0 { + continuation.resume(throwing: WhisperError.transcriptionFailed) + } else { + continuation.resume(returning: String(cString: buffer)) + } + } + } + } + + // MARK: - Audio decode + + /// Decode any recorded clip to the mono float32 @ 16 kHz that whisper wants. + private static func decodeToMono16kFloat(url: URL) throws -> [Float] { + let file = try AVAudioFile(forReading: url) + guard let target = AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: Double(SYNAP_WHISPER_SAMPLE_RATE), + channels: 1, interleaved: false), + let converter = AVAudioConverter(from: file.processingFormat, to: target) else { + throw WhisperError.audioDecodeFailed + } + + // Read the whole source file into one buffer. + let frameCount = AVAudioFrameCount(file.length) + guard frameCount > 0, + let inputBuffer = AVAudioPCMBuffer( + pcmFormat: file.processingFormat, frameCapacity: frameCount) else { + throw WhisperError.audioDecodeFailed + } + try file.read(into: inputBuffer) + + // Resample/downmix to 16 kHz mono. + let ratio = target.sampleRate / file.processingFormat.sampleRate + let outCapacity = AVAudioFrameCount(Double(frameCount) * ratio) + 1024 + guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: target, frameCapacity: outCapacity) else { + throw WhisperError.audioDecodeFailed + } + + var fed = false + var conversionError: NSError? + converter.convert(to: outputBuffer, error: &conversionError) { _, status in + if fed { + status.pointee = .noDataNow + return nil + } + fed = true + status.pointee = .haveData + return inputBuffer + } + if conversionError != nil { throw WhisperError.audioDecodeFailed } + + guard let channel = outputBuffer.floatChannelData?[0] else { + throw WhisperError.audioDecodeFailed + } + return Array(UnsafeBufferPointer(start: channel, count: Int(outputBuffer.frameLength))) + } +} diff --git a/SynapLink/Data/DataSources/ModelDownload/SpecialistCatalog.swift b/SynapLink/Data/DataSources/ModelDownload/SpecialistCatalog.swift new file mode 100644 index 0000000..4355f55 --- /dev/null +++ b/SynapLink/Data/DataSources/ModelDownload/SpecialistCatalog.swift @@ -0,0 +1,151 @@ +// +// SpecialistCatalog.swift +// SynapLink +// +// The sidecar specialist models: small, single-purpose models that feed the +// main chat model (PLAN.md §2 split-pipeline fallback). They give the +// 4 GB iPhone 11 tier real audio + image input, which the omni Gemma 4 E2B +// can't provide there. +// +// • whisper — speech → text (whisper.cpp, ggml-base.bin, ~142 MB) +// • vision — image → description (SmolVLM-500M via the mtmd path, ~550 MB) +// +// Both download through the same Hugging Face Hub mechanism as the main +// catalog and resolve out of the shared Application Support/hub cache. +// + +import Foundation + +enum SpecialistModel: String, CaseIterable, Identifiable { + case whisper = "Speech to Text" + case vision = "Image Understanding" + case imageGen = "Image Creation" + + var id: String { rawValue } + + var repoID: String { + switch self { + case .whisper: return "ggerganov/whisper.cpp" + case .vision: return "ggml-org/SmolVLM-500M-Instruct-GGUF" + case .imageGen: return "second-state/stable-diffusion-v1-5-GGUF" + } + } + + /// Primary model file (whisper .bin / VLM .gguf / SD checkpoint .gguf). + var filename: String { + switch self { + case .whisper: return "ggml-base.bin" + case .vision: return "SmolVLM-500M-Instruct-Q8_0.gguf" + case .imageGen: return "stable-diffusion-v1-5-pruned-emaonly-Q4_0.gguf" + } + } + + /// Vision projector (vision) / TAESD tiny VAE slot (image-gen, unused for now). + var mmprojFilename: String? { + switch self { + case .whisper, .imageGen: return nil + case .vision: return "mmproj-SmolVLM-500M-Instruct-Q8_0.gguf" + } + } + + var downloadFilenames: [String] { + [filename] + (mmprojFilename.map { [$0] } ?? []) + } + + var estimatedSizeMB: Double { + switch self { + case .whisper: return 142 + case .vision: return 437 + 109 + case .imageGen: return 1567 + } + } + + var iconName: String { + switch self { + case .whisper: return "waveform" + case .vision: return "photo" + case .imageGen: return "wand.and.stars" + } + } + + var tagline: String { + switch self { + case .whisper: return "Speak instead of type — transcribed on-device." + case .vision: return "Attach a photo and ask about it." + case .imageGen: return "Create images from a text prompt." + } + } + + /// Experimental on the 4 GB tier: heavy and slow, may strain memory. + var isExperimental: Bool { self == .imageGen } + + /// whisper/vision are tiny and fit any supported device. Image generation + /// is ~1.6 GB resident during diffusion — gate it to ≈4 GB+ and warn it's + /// experimental there (the main chat model is unloaded while it runs). + var requiredRAMGB: Double { + switch self { + case .whisper, .vision: return 0 + case .imageGen: return 3.5 + } + } + + var isSupportedOnThisDevice: Bool { + RuntimeProfile.physicalMemoryGB + 0.5 >= requiredRAMGB + } + + // MARK: - Local path resolution (shared hub cache) + + private var hubSlug: String { + "models--" + repoID.replacingOccurrences(of: "/", with: "--") + } + + private var pathKey: String { "SynapLinkSpecialistPath_\(hubSlug)_\(filename)" } + private var mmprojPathKey: String { "SynapLinkSpecialistMMProj_\(hubSlug)" } + + func modelURL() -> URL? { resolve(filename: filename, key: pathKey) } + + func mmprojURL() -> URL? { + guard let mmprojFilename else { return nil } + return resolve(filename: mmprojFilename, key: mmprojPathKey) + } + + var isInstalled: Bool { + modelURL() != nil && (mmprojFilename == nil || mmprojURL() != nil) + } + + private func resolve(filename: String, key: String) -> URL? { + let appSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + if let relative = UserDefaults.standard.string(forKey: key) { + let url = appSupport.appendingPathComponent(relative) + if url.lastPathComponent == filename, FileManager.default.fileExists(atPath: url.path) { + return url + } + } + let snapshots = appSupport.appendingPathComponent("hub/\(hubSlug)/snapshots") + guard let entries = try? FileManager.default.contentsOfDirectory( + at: snapshots, includingPropertiesForKeys: nil) else { return nil } + for snapshot in entries.sorted(by: { $0.path > $1.path }) { + let candidate = snapshot.appendingPathComponent(filename) + if FileManager.default.fileExists(atPath: candidate.path) { + persistPath(candidate, key: key) + return candidate + } + } + return nil + } + + func persistPath(_ url: URL, key: String) { + let appSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + UserDefaults.standard.set(url.path.replacingOccurrences(of: appSupport.path + "/", with: ""), forKey: key) + } + + func persistPaths(modelURL: URL, mmprojURL: URL?) { + persistPath(modelURL, key: pathKey) + if let mmprojURL { persistPath(mmprojURL, key: mmprojPathKey) } + } + + var cacheURL: URL { + FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + .appendingPathComponent("hub/\(hubSlug)") + } +} diff --git a/SynapLink/Data/DataSources/ModelDownload/SpecialistManager.swift b/SynapLink/Data/DataSources/ModelDownload/SpecialistManager.swift new file mode 100644 index 0000000..e2ad48e --- /dev/null +++ b/SynapLink/Data/DataSources/ModelDownload/SpecialistManager.swift @@ -0,0 +1,112 @@ +// +// SpecialistManager.swift +// SynapLink +// +// Download/installed-state tracking for the sidecar specialist models, +// mirroring ModelDownloadManager but for SpecialistModel. Uses the same +// Hugging Face Hub snapshot mechanism (only the needed files) into the +// shared Application Support/hub cache. +// + +import Foundation +import Hub +import Observation + +@MainActor +@Observable +final class SpecialistManager { + + static let shared = SpecialistManager() + + private(set) var states: [SpecialistModel: ModelDownloadState] = [:] + + @ObservationIgnored private var activeTask: Task? + @ObservationIgnored private var activeModel: SpecialistModel? + + private init() { + refreshStates() + } + + func refreshStates() { + for model in SpecialistModel.allCases where model != activeModel { + states[model] = model.isInstalled ? .ready : .notDownloaded + } + } + + func isInstalled(_ model: SpecialistModel) -> Bool { states[model] == .ready } + + var anyActive: Bool { + states.values.contains { if case .downloading = $0 { return true } else { return false } } + } + + func startDownload(_ model: SpecialistModel) { + guard activeTask == nil, states[model] != .ready, model.isSupportedOnThisDevice else { return } + activeModel = model + states[model] = .downloading(progress: 0) + + activeTask = Task { [weak self] in + do { + try await Self.download(model) { progress in + Task { @MainActor [weak self] in + guard let self, self.activeModel == model else { return } + self.states[model] = .downloading(progress: progress) + } + } + guard !Task.isCancelled else { return } + self?.states[model] = model.isInstalled ? .ready : .failed("File missing after download") + } catch is CancellationError { + // state already set by pause/cancel + } catch { + if !Task.isCancelled { self?.states[model] = .failed(error.localizedDescription) } + } + self?.activeTask = nil + self?.activeModel = nil + } + } + + func pauseDownload() { + guard let model = activeModel, case .downloading(let progress) = states[model] else { return } + activeTask?.cancel() + activeTask = nil + activeModel = nil + states[model] = .paused(progress: progress) + } + + func resumeDownload(_ model: SpecialistModel) { + guard case .paused = states[model] else { return } + states[model] = .notDownloaded + startDownload(model) + } + + func deleteModel(_ model: SpecialistModel) { + try? FileManager.default.removeItem(at: model.cacheURL) + refreshStates() + } + + private static func download(_ model: SpecialistModel, + progressHandler: @escaping (Double) -> Void) async throws { + let appSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + let api = HubApi(downloadBase: appSupport) + let repo = Hub.Repo(id: model.repoID) + let snapshotDir = try await api.snapshot(from: repo, matching: model.downloadFilenames) { progress in + progressHandler(progress.fractionCompleted * 0.97) + } + + func locate(_ filename: String) throws -> URL { + let direct = snapshotDir.appendingPathComponent(filename) + if FileManager.default.fileExists(atPath: direct.path) { return direct } + let entries = (try? FileManager.default.contentsOfDirectory( + at: snapshotDir, includingPropertiesForKeys: nil)) ?? [] + for entry in entries { + let candidate = entry.appendingPathComponent(filename) + if FileManager.default.fileExists(atPath: candidate.path) { return candidate } + } + throw ModelDownloadError.fileMissing(filename) + } + + let modelURL = try locate(model.filename) + let mmprojURL = try model.mmprojFilename.map { try locate($0) } + model.persistPaths(modelURL: modelURL, mmprojURL: mmprojURL) + progressHandler(1.0) + } +} diff --git a/SynapLink/Presentation/Components/AttachmentSheet.swift b/SynapLink/Presentation/Components/AttachmentSheet.swift new file mode 100644 index 0000000..2b6d06e --- /dev/null +++ b/SynapLink/Presentation/Components/AttachmentSheet.swift @@ -0,0 +1,106 @@ +// +// AttachmentSheet.swift +// SynapLink +// +// Bottom-sheet attachment picker (Messenger / ChatGPT style): a compact +// card of large tappable rows instead of a system action sheet. +// + +import SwiftUI +import UIKit + +struct AttachmentSheet: View { + var canAttachPhoto: Bool = true + var onCamera: () -> Void + var onLibrary: () -> Void + var canCreateImage: Bool = false + var onCreateImage: () -> Void = {} + + @Environment(\.dismiss) private var dismiss + + private var cameraAvailable: Bool { + canAttachPhoto && UIImagePickerController.isSourceTypeAvailable(.camera) + } + + private var detentHeight: CGFloat { + var rows = 0 + if canAttachPhoto { rows += 1 } // library + if cameraAvailable { rows += 1 } + if canCreateImage { rows += 1 } + return CGFloat(max(1, rows)) * 68 + 70 + } + + var body: some View { + VStack(spacing: 0) { + Capsule() + .fill(.tertiary) + .frame(width: 36, height: 5) + .padding(.top, 8) + .padding(.bottom, 12) + + Text("Add to the conversation") + .font(.headline) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 20) + .padding(.bottom, 8) + + VStack(spacing: 0) { + if cameraAvailable { + row(title: "Take Photo", subtitle: "Use the camera", + icon: "camera.fill", tint: .blue) { + dismiss(); onCamera() + } + Divider().padding(.leading, 72) + } + if canAttachPhoto { + row(title: "Photo Library", subtitle: "Choose an existing photo", + icon: "photo.on.rectangle.angled", tint: .green) { + dismiss(); onLibrary() + } + } + if canCreateImage { + if canAttachPhoto { Divider().padding(.leading, 72) } + row(title: "Create with AI", subtitle: "Generate an image from text", + icon: "wand.and.stars", tint: .purple) { + dismiss(); onCreateImage() + } + } + } + .background(Color(.secondarySystemGroupedBackground)) + .clipShape(RoundedRectangle(cornerRadius: 16, style: .continuous)) + .padding(.horizontal, 16) + + Spacer(minLength: 12) + } + .presentationDetents([.height(detentHeight)]) + .presentationDragIndicator(.hidden) + } + + private func row(title: String, subtitle: String, icon: String, tint: Color, + action: @escaping () -> Void) -> some View { + Button(action: action) { + HStack(spacing: 16) { + Image(systemName: icon) + .font(.title3) + .foregroundStyle(.white) + .frame(width: 40, height: 40) + .background(tint.gradient, in: RoundedRectangle(cornerRadius: 10, style: .continuous)) + VStack(alignment: .leading, spacing: 2) { + Text(title).font(.body.weight(.medium)) + Text(subtitle).font(.caption).foregroundStyle(.secondary) + } + Spacer() + } + .padding(.horizontal, 16) + .padding(.vertical, 12) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + } +} + +#Preview { + Color.black.sheet(isPresented: .constant(true)) { + AttachmentSheet(onCamera: {}, onLibrary: {}) + } +} diff --git a/SynapLink/Presentation/Components/ChatInputBar.swift b/SynapLink/Presentation/Components/ChatInputBar.swift index cd6dfc4..a1ee28d 100644 --- a/SynapLink/Presentation/Components/ChatInputBar.swift +++ b/SynapLink/Presentation/Components/ChatInputBar.swift @@ -19,6 +19,7 @@ struct ChatInputBar: View { let supportsAudio: Bool let isGenerating: Bool let canSend: Bool + var hasAttachment: Bool = false var onSend: () -> Void var onStop: () -> Void @@ -31,6 +32,9 @@ struct ChatInputBar: View { !draft.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } + /// A turn can be sent with text, an attachment, or both. + private var canSubmit: Bool { hasDraft || hasAttachment } + var body: some View { HStack(spacing: 12) { if supportsVision { @@ -72,9 +76,9 @@ struct ChatInputBar: View { if isGenerating { circleButton(systemImage: "stop.fill", background: .red, action: onStop) .accessibilityLabel("Stop generating") - } else if hasDraft || !supportsAudio { + } else if canSubmit || !supportsAudio { circleButton(systemImage: "arrow.up", background: Color.accentColor, action: onSend) - .disabled(!hasDraft || !canSend) + .disabled(!canSubmit || !canSend) .accessibilityLabel("Send") } else { circleButton(systemImage: "waveform", background: .green, action: onVoice) diff --git a/SynapLink/Presentation/Components/MarkdownText.swift b/SynapLink/Presentation/Components/MarkdownText.swift index ad78a65..f81a158 100644 --- a/SynapLink/Presentation/Components/MarkdownText.swift +++ b/SynapLink/Presentation/Components/MarkdownText.swift @@ -2,11 +2,10 @@ // MarkdownText.swift // SynapLink // -// Block-level markdown for assistant replies: fenced ``` code blocks render -// as monospaced cards with a copy button, #/##/### headings get heading -// fonts, and everything else keeps inline markdown (bold, italics, `code`, -// links). An unterminated fence is treated as an open code block so -// streamed code renders live while the model is still typing it. +// Block-level markdown for a FINISHED assistant reply (the streaming path is +// StreamingText). Both share the same typed chunks (StreamChunk) and renderer +// (StreamChunkView): fenced ``` code as monospaced cards with a copy button, +// #/##/### headings, inline styles (bold, italics, `code`, links). // import SwiftUI @@ -16,95 +15,12 @@ struct MarkdownText: View { var body: some View { VStack(alignment: .leading, spacing: 8) { - ForEach(Self.parse(text)) { block in - switch block.kind { - case .code(let language, let code): - CodeBlockView(language: language, code: code) - case .heading(let level, let content): - Text(Self.inline(content)) - .font(headingFont(level)) - case .paragraph(let content): - Text(Self.inline(content)) - } + ForEach(StreamChunkParser.parse(text)) { chunk in + StreamChunkView(chunk: chunk).equatable() } } } - private func headingFont(_ level: Int) -> Font { - switch level { - case 1: return .title2.weight(.bold) - case 2: return .title3.weight(.semibold) - default: return .headline - } - } - - // MARK: - Parsing - - static func parse(_ text: String) -> [MarkdownBlock] { - var blocks: [MarkdownBlock] = [] - var paragraph: [String] = [] - var codeLines: [String] = [] - var codeLanguage: String? - var inCode = false - - func flushParagraph() { - let joined = paragraph.joined(separator: "\n") - if !joined.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { - blocks.append(MarkdownBlock(id: blocks.count, kind: .paragraph(joined))) - } - paragraph = [] - } - - for line in text.split(separator: "\n", omittingEmptySubsequences: false) { - let trimmed = line.trimmingCharacters(in: .whitespaces) - - if trimmed.hasPrefix("```") { - if inCode { - blocks.append(MarkdownBlock(id: blocks.count, - kind: .code(language: codeLanguage, - codeLines.joined(separator: "\n")))) - codeLines = [] - codeLanguage = nil - inCode = false - } else { - flushParagraph() - let lang = trimmed.dropFirst(3).trimmingCharacters(in: .whitespaces) - codeLanguage = lang.isEmpty ? nil : lang - inCode = true - } - continue - } - - if inCode { - codeLines.append(String(line)) - } else if let heading = headingLine(trimmed) { - flushParagraph() - blocks.append(MarkdownBlock(id: blocks.count, - kind: .heading(level: heading.level, heading.text))) - } else { - paragraph.append(String(line)) - } - } - - // Streaming: an open fence renders as a live code block. - if inCode { - blocks.append(MarkdownBlock(id: blocks.count, - kind: .code(language: codeLanguage, - codeLines.joined(separator: "\n")))) - } - flushParagraph() - return blocks - } - - private static func headingLine(_ line: String) -> (level: Int, text: String)? { - guard line.hasPrefix("#") else { return nil } - let hashes = line.prefix(while: { $0 == "#" }).count - guard hashes <= 6 else { return nil } - let rest = line.dropFirst(hashes) - guard rest.first == " " else { return nil } - return (hashes, rest.trimmingCharacters(in: .whitespaces)) - } - static func inline(_ text: String) -> AttributedString { var options = AttributedString.MarkdownParsingOptions() options.interpretedSyntax = .inlineOnlyPreservingWhitespace @@ -113,18 +29,6 @@ struct MarkdownText: View { } } -/// One rendered block of a markdown message. -struct MarkdownBlock: Identifiable { - enum Kind { - case paragraph(String) - case heading(level: Int, String) - case code(language: String?, String) - } - - let id: Int - let kind: Kind -} - // MARK: - Code block card struct CodeBlockView: View { @@ -186,10 +90,7 @@ struct CodeBlockView: View { ```swift func quicksort(_ array: [T]) -> [T] { guard array.count > 1 else { return array } - let pivot = array[array.count / 2] - return quicksort(array.filter { $0 < pivot }) - + array.filter { $0 == pivot } - + quicksort(array.filter { $0 > pivot }) + return array } ``` diff --git a/SynapLink/Presentation/Components/MessageBubble.swift b/SynapLink/Presentation/Components/MessageBubble.swift index 67d8bdb..993681d 100644 --- a/SynapLink/Presentation/Components/MessageBubble.swift +++ b/SynapLink/Presentation/Components/MessageBubble.swift @@ -61,9 +61,16 @@ struct MessageBubble: View { } else if text.isEmpty { EmptyView() } else if role == .assistant { - // Block markdown: fenced code with copy button, headings, inline styles. - MarkdownText(text: isStreaming ? text + " ●" : text) - .textSelection(.enabled) + if isStreaming { + // Plain, smoothly-revealed text while generating — markdown is + // applied once the reply settles (avoids per-token reflow). + StreamingText(fullText: text) + } else { + // Block markdown: fenced code with copy button, headings, inline styles. + MarkdownText(text: text) + .textSelection(.enabled) + .transition(.opacity) + } } else { Text(MarkdownText.inline(text)) .textSelection(.enabled) diff --git a/SynapLink/Presentation/Components/StreamChunk.swift b/SynapLink/Presentation/Components/StreamChunk.swift new file mode 100644 index 0000000..a54d786 --- /dev/null +++ b/SynapLink/Presentation/Components/StreamChunk.swift @@ -0,0 +1,146 @@ +// +// StreamChunk.swift +// SynapLink +// +// Typed units of a streamed reply. A reply is parsed into an ordered list of +// chunks; the LAST chunk is the one still being written. The text model emits +// `.paragraph` / `.heading` / `.code`; `.image` and `.audio` are reserved for +// future producers (streaming TTS, image output) so one renderer handles +// every stream — and markdown renders *as it streams*, not only at the end. +// +// Rendering perf: StreamChunkView is Equatable and used via `.equatable()`, +// so SwiftUI skips re-rendering chunks that haven't changed. During streaming +// only the open (last) chunk is rebuilt each frame; finished chunks above it +// format exactly once. +// + +import SwiftUI + +enum StreamChunk: Identifiable, Equatable { + case paragraph(id: Int, text: String) + case heading(id: Int, level: Int, text: String) + case code(id: Int, language: String?, code: String) + case image(id: Int, url: URL) // reserved for streamed image output + case audio(id: Int, url: URL) // reserved for streamed audio output + + var id: Int { + switch self { + case .paragraph(let id, _), .heading(let id, _, _), .code(let id, _, _), + .image(let id, _), .audio(let id, _): + return id + } + } +} + +/// Splits reply text into typed chunks. Pure text scanning — no AttributedString +/// work (that happens lazily in StreamChunkView for the chunks actually shown). +/// An unterminated final ``` fence is treated as an open code block so code +/// streams inside its card rather than as raw text that pops into one later. +enum StreamChunkParser { + + static func parse(_ text: String) -> [StreamChunk] { + var chunks: [StreamChunk] = [] + var paragraph: [String] = [] + var codeLines: [String] = [] + var codeLanguage: String? + var inCode = false + + func flushParagraph() { + let joined = paragraph.joined(separator: "\n") + if !joined.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + chunks.append(.paragraph(id: chunks.count, text: joined)) + } + paragraph = [] + } + + for line in text.split(separator: "\n", omittingEmptySubsequences: false) { + let trimmed = line.trimmingCharacters(in: .whitespaces) + + if trimmed.hasPrefix("```") { + if inCode { + chunks.append(.code(id: chunks.count, language: codeLanguage, + code: codeLines.joined(separator: "\n"))) + codeLines = [] + codeLanguage = nil + inCode = false + } else { + flushParagraph() + let lang = trimmed.dropFirst(3).trimmingCharacters(in: .whitespaces) + codeLanguage = lang.isEmpty ? nil : lang + inCode = true + } + continue + } + + if inCode { + codeLines.append(String(line)) + } else if let heading = headingLine(trimmed) { + flushParagraph() + chunks.append(.heading(id: chunks.count, level: heading.level, text: heading.text)) + } else { + paragraph.append(String(line)) + } + } + + if inCode { + chunks.append(.code(id: chunks.count, language: codeLanguage, + code: codeLines.joined(separator: "\n"))) + } + flushParagraph() + return chunks + } + + private static func headingLine(_ line: String) -> (level: Int, text: String)? { + guard line.hasPrefix("#") else { return nil } + let hashes = line.prefix(while: { $0 == "#" }).count + guard hashes <= 6 else { return nil } + let rest = line.dropFirst(hashes) + guard rest.first == " " else { return nil } + return (hashes, rest.trimmingCharacters(in: .whitespaces)) + } +} + +/// Renders one chunk. Equatable so SwiftUI skips unchanged chunks while streaming. +struct StreamChunkView: View, Equatable { + let chunk: StreamChunk + var showCursor = false + + static func == (lhs: StreamChunkView, rhs: StreamChunkView) -> Bool { + lhs.chunk == rhs.chunk && lhs.showCursor == rhs.showCursor + } + + var body: some View { + switch chunk { + case .paragraph(_, let text): + (Text(MarkdownText.inline(text)) + cursor) + .textSelection(.enabled) + case .heading(_, let level, let text): + (Text(MarkdownText.inline(text)).font(Self.headingFont(level)) + cursor) + .textSelection(.enabled) + case .code(_, let language, let code): + CodeBlockView(language: language, code: code) + case .image(_, let url): + if let image = UIImage(contentsOfFile: url.path) { + Image(uiImage: image) + .resizable().scaledToFit() + .frame(maxHeight: 220) + .clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous)) + } + case .audio(_, let url): + AudioAttachmentView(url: url) + } + } + + /// Neutral caret — no accent tint (intentionally low-key while streaming). + private var cursor: Text { + showCursor ? Text(" ▌").foregroundColor(.secondary) : Text("") + } + + static func headingFont(_ level: Int) -> Font { + switch level { + case 1: return .title2.weight(.bold) + case 2: return .title3.weight(.semibold) + default: return .headline + } + } +} diff --git a/SynapLink/Presentation/Components/StreamingText.swift b/SynapLink/Presentation/Components/StreamingText.swift new file mode 100644 index 0000000..e042708 --- /dev/null +++ b/SynapLink/Presentation/Components/StreamingText.swift @@ -0,0 +1,79 @@ +// +// StreamingText.swift +// SynapLink +// +// Smooth, easy-on-the-eyes rendering of an in-progress reply. Two ideas: +// +// 1. Decouple the visual reveal from token arrival. Tokens land in bursts; +// a typewriter drains the buffer at a steady cadence (catching up when +// it falls behind) so text flows in evenly instead of lurching. +// 2. Render plain text while streaming — NOT markdown. Re-parsing markdown +// every token made bold/code/headings pop in and reflow (jarring, and +// costly on the A13). Full markdown formatting is applied once, by +// MessageBubble, when the finished reply is persisted. +// + +import SwiftUI + +struct StreamingText: View { + let fullText: String + + @State private var typewriter = Typewriter() + + /// Live typed chunks of the smoothly-revealed prefix: code/headings/markdown + /// render in place as they stream, not only once the reply is done. + private var chunks: [StreamChunk] { StreamChunkParser.parse(typewriter.shown) } + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + let chunks = chunks + ForEach(chunks) { chunk in + StreamChunkView(chunk: chunk, showCursor: chunk.id == chunks.last?.id) + .equatable() + } + } + .onAppear { typewriter.update(fullText) } + .onChange(of: fullText) { typewriter.update(fullText) } + } +} + +/// Reveals text toward a moving target at a steady ~33 fps, accelerating when +/// it falls far behind so it never lags noticeably during fast bursts. +@MainActor +@Observable +final class Typewriter { + private(set) var shown = "" + + @ObservationIgnored private var target = "" + @ObservationIgnored private var shownCount = 0 + @ObservationIgnored private var task: Task? + + func update(_ text: String) { + target = text + // A shorter target means a new turn/regeneration — restart cleanly. + if text.count < shownCount { + shownCount = 0 + shown = "" + } + if task == nil { start() } + } + + private func start() { + task = Task { [weak self] in + while !Task.isCancelled { + guard let self else { return } + let remaining = self.target.count - self.shownCount + if remaining > 0 { + // Reveal ~1/6 of the backlog per tick (min 1): smooth when + // close, brisk when far behind. + let step = max(1, remaining / 6) + self.shownCount += step + self.shown = String(self.target.prefix(self.shownCount)) + } + try? await Task.sleep(for: .milliseconds(30)) + } + } + } + + deinit { task?.cancel() } +} diff --git a/SynapLink/Presentation/Views/Chat/ChatView.swift b/SynapLink/Presentation/Views/Chat/ChatView.swift index 605523b..90b1cef 100644 --- a/SynapLink/Presentation/Views/Chat/ChatView.swift +++ b/SynapLink/Presentation/Views/Chat/ChatView.swift @@ -21,6 +21,8 @@ struct ChatView: View { @State private var photoItem: PhotosPickerItem? @State private var pendingImage: Data? @State private var showVoiceMode = false + @State private var showImageGen = false + @State private var specialists = SpecialistManager.shared var body: some View { VStack(spacing: 0) { @@ -28,10 +30,11 @@ struct ChatView: View { attachmentChip ChatInputBar( draft: $draft, - supportsVision: session.capabilities?.hasVision ?? false, - supportsAudio: session.capabilities?.hasAudio ?? false, + supportsVision: session.canAttachMedia, + supportsAudio: session.canSendAudio, isGenerating: session.isGenerating, canSend: session.engineState != .loading, + hasAttachment: pendingImage != nil, onSend: sendDraft, onStop: { session.stop() }, onCamera: { showAttachDialog = true }, @@ -50,11 +53,16 @@ struct ChatView: View { } } .navigationBarTitleDisplayMode(.inline) - .confirmationDialog("Add a photo", isPresented: $showAttachDialog) { - if UIImagePickerController.isSourceTypeAvailable(.camera) { - Button("Take Photo") { showCamera = true } - } - Button("Choose from Library") { showPhotoPicker = true } + .sheet(isPresented: $showAttachDialog) { + AttachmentSheet( + canAttachPhoto: session.canSendImages, + onCamera: { presentAfterSheet { showCamera = true } }, + onLibrary: { presentAfterSheet { showPhotoPicker = true } }, + canCreateImage: specialists.isInstalled(.imageGen), + onCreateImage: { presentAfterSheet { showImageGen = true } }) + } + .sheet(isPresented: $showImageGen) { + ImageGenSheet { prompt in session.createImage(prompt: prompt) } } .fullScreenCover(isPresented: $showCamera) { CameraPicker { image in @@ -76,7 +84,7 @@ struct ChatView: View { } } .fullScreenCover(isPresented: $showVoiceMode) { - ImmersiveAudioView(sampleRate: session.capabilities?.audioSampleRate ?? 16000) { wav in + ImmersiveAudioView(sampleRate: session.audioCaptureSampleRate) { wav in showVoiceMode = false if let wav { session.send(draft, attachments: [.init(kind: .audio, data: wav)]) @@ -112,6 +120,12 @@ struct ChatView: View { return resized.jpegData(compressionQuality: 0.85) } + /// Presenting a picker straight from the attachment sheet's dismissal + /// races SwiftUI's sheet machinery; let the sheet finish closing first. + private func presentAfterSheet(_ action: @escaping () -> Void) { + DispatchQueue.main.asyncAfter(deadline: .now() + 0.35, execute: action) + } + private func sendDraft() { let text = draft let attachments: [PendingAttachment] = @@ -181,6 +195,9 @@ struct ChatView: View { MessageBubble(role: .assistant, text: session.streamingText, isStreaming: true) } } + if session.isCreatingImage { + creatingImageIndicator + } statusBanner // Stable scroll target: always present, always last. Color.clear @@ -216,6 +233,27 @@ struct ChatView: View { } } + /// Diffusion is slow on-device — a calmer, longer-wait affordance. + private var creatingImageIndicator: some View { + HStack { + HStack(spacing: 10) { + Image(systemName: "wand.and.stars") + .symbolEffect(.variableColor.iterative) + VStack(alignment: .leading, spacing: 2) { + Text("Creating image…").font(.callout) + Text("On-device — this can take a while.") + .font(.caption2).foregroundStyle(.secondary) + } + } + .foregroundStyle(.secondary) + .padding(.horizontal, 14) + .padding(.vertical, 10) + .background(Color(.secondarySystemBackground)) + .clipShape(RoundedRectangle(cornerRadius: 18, style: .continuous)) + Spacer(minLength: 48) + } + } + /// Media prefill (encode) takes seconds before the first token — say so. private var analyzingIndicator: some View { HStack { diff --git a/SynapLink/Presentation/Views/Chat/ImageGenSheet.swift b/SynapLink/Presentation/Views/Chat/ImageGenSheet.swift new file mode 100644 index 0000000..f069e7f --- /dev/null +++ b/SynapLink/Presentation/Views/Chat/ImageGenSheet.swift @@ -0,0 +1,67 @@ +// +// ImageGenSheet.swift +// SynapLink +// +// Prompt entry for the experimental on-device image generator. +// + +import SwiftUI + +struct ImageGenSheet: View { + var onGenerate: (String) -> Void + + @Environment(\.dismiss) private var dismiss + @State private var prompt = "" + @FocusState private var focused: Bool + + private var experimentalOn4GB: Bool { + RuntimeProfile.physicalMemoryGB < 5.0 + } + + var body: some View { + NavigationStack { + Form { + Section { + TextField("Describe the image…", text: $prompt, axis: .vertical) + .lineLimit(2...5) + .focused($focused) + } footer: { + Text("Generated entirely on your device.") + } + + if experimentalOn4GB { + Section { + Label( + "Experimental on this device — generation can take a minute or two and quality is limited.", + systemImage: "exclamationmark.triangle") + .font(.footnote) + .foregroundStyle(.orange) + } + } + } + .navigationTitle("Create Image") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .cancellationAction) { + Button("Cancel") { dismiss() } + } + ToolbarItem(placement: .confirmationAction) { + Button("Generate") { + let text = prompt + dismiss() + onGenerate(text) + } + .disabled(prompt.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + } + } + .onAppear { focused = true } + } + .presentationDetents([.medium]) + } +} + +#Preview { + Color.black.sheet(isPresented: .constant(true)) { + ImageGenSheet { _ in } + } +} diff --git a/SynapLink/Presentation/Views/ModelLibrary/ModelLibraryView.swift b/SynapLink/Presentation/Views/ModelLibrary/ModelLibraryView.swift index 7bc7716..f7cbad3 100644 --- a/SynapLink/Presentation/Views/ModelLibrary/ModelLibraryView.swift +++ b/SynapLink/Presentation/Views/ModelLibrary/ModelLibraryView.swift @@ -11,16 +11,30 @@ import SwiftUI struct ModelLibraryView: View { @State private var manager = ModelDownloadManager.shared + @State private var specialists = SpecialistManager.shared + var body: some View { List { Section { ForEach(ModelConfiguration.allCases) { config in ModelLibraryCard(config: config) } + } header: { + Text("Chat model") } footer: { Text("Models download once from Hugging Face and never leave your device. All chat runs fully offline.") } + Section { + ForEach(SpecialistModel.allCases) { model in + SpecialistCard(model: model) + } + } header: { + Text("Add-on capabilities") + } footer: { + Text("Small helpers that add voice and photo input to any chat model — ideal where the full multimodal model is too large. Speech is transcribed and photos are described on-device, then handed to your chat model.") + } + Section("Storage") { LabeledContent("Total model storage", value: ByteCountFormatter.string( @@ -196,6 +210,84 @@ struct ModelLibraryCard: View { } } +/// Download card for a sidecar specialist (Speech / Image Understanding). +struct SpecialistCard: View { + let model: SpecialistModel + + @State private var manager = SpecialistManager.shared + + private var state: ModelDownloadState { manager.states[model] ?? .notDownloaded } + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + HStack(spacing: 12) { + Image(systemName: model.iconName) + .font(.title2) + .foregroundStyle(.tint) + .frame(width: 34) + VStack(alignment: .leading, spacing: 2) { + Text(model.rawValue).font(.headline) + Text(model.tagline).font(.caption).foregroundStyle(.secondary) + } + Spacer() + if case .ready = state { + Label("Ready", systemImage: "checkmark.circle.fill") + .labelStyle(.iconOnly) + .foregroundStyle(.green) + } + } + stateRow + } + .padding(.vertical, 6) + } + + @ViewBuilder + private var stateRow: some View { + switch state { + case .notDownloaded: + Button { + manager.startDownload(model) + } label: { + Label("Download · \(Int(model.estimatedSizeMB)) MB", systemImage: "arrow.down.circle") + .frame(maxWidth: .infinity) + } + .buttonStyle(.bordered) + .disabled(manager.anyActive) + + case .downloading(let progress): + VStack(alignment: .leading, spacing: 6) { + ProgressView(value: progress) + HStack { + Text("\(Int(progress * 100))%") + .font(.caption.monospacedDigit()).foregroundStyle(.secondary) + Spacer() + Button("Pause") { manager.pauseDownload() }.font(.caption) + } + } + + case .paused(let progress): + HStack { + ProgressView(value: progress).tint(.orange) + Button("Resume") { manager.resumeDownload(model) }.font(.caption) + } + + case .ready: + HStack { + Spacer() + Button("Delete", role: .destructive) { manager.deleteModel(model) } + .font(.caption) + } + + case .failed(let message): + VStack(alignment: .leading, spacing: 4) { + Label(message, systemImage: "exclamationmark.triangle") + .font(.caption).foregroundStyle(.red) + Button("Retry") { manager.startDownload(model) }.buttonStyle(.bordered) + } + } + } +} + #Preview { NavigationStack { ModelLibraryView() } } diff --git a/docs/PHASE_3.md b/docs/PHASE_3.md new file mode 100644 index 0000000..e74f38f --- /dev/null +++ b/docs/PHASE_3.md @@ -0,0 +1,108 @@ +# Phase 3 — Audio & Image for the 4 GB tier (sidecar specialists) + +Builds on [PHASE_2.md](PHASE_2.md). The PLAN's Phase 3 is the realtime voice +loop; the prerequisite decided here is the **split-pipeline fallback** from +PLAN §2: since the omni Gemma 4 E2B can't fit on iPhone 11, dedicated small +models give the 4 GB tier real audio and image input, feeding a normal text +chat model. + +## Architecture: sidecar specialists + +``` + ┌─────────────── main chat model (e.g. Gemma 3 1B, text) ───────────────┐ + photo →│ VisionDescriber (SmolVLM-500M, own transient engine) → description ──→│→ reply + voice →│ WhisperTranscriber (whisper.cpp base) → transcript ──→│ + text →│ ─────────────────────────────────────────────────────────────────────│ + └────────────────────────────────────────────────────────────────────────┘ +``` + +- **Audio = whisper.cpp** (`third_party/whisper.cpp`, pinned v1.8.6). A second + ggml-based xcframework (`scripts/build-whisper-xcframework.sh` → + `Frameworks/whisper.xcframework`). Two ggml frameworks coexist because each + force-loads its **own** ggml and Apple's two-level namespace keeps the symbol + sets private per dylib — verified: app links + runs with both. Bridge: + `synap_whisper.{h,cpp}` → `WhisperTranscriber` (load → transcribe → unload). +- **Image = SmolVLM-500M** through the **existing mtmd vision path** (no new + engine code). `VisionDescriber` runs it in its own transient `InferenceEngine` + instance; the C llama backend is now **refcounted** (`backend_retain/release`) + so the specialist engine and the main engine coexist. +- **Specialists load-use-unload**: only resident while working. Peak during an + image turn ≈ main model + SmolVLM (~1.35 GB weights for the 1B + 500M combo), + well under the iPhone 11 budget. + +## Capability gating + +The input bar's camera/voice buttons light up when the main model is natively +multimodal (E2B) **or** the matching specialist is installed +(`ChatSession.canSendImages / canSendAudio`). A turn's attachments route per +the main model's own capabilities: native (direct mtmd media, E2B) else sidecar +(`ChatSession.resolveMedia`). Image descriptions are injected as grounding +context; voice transcripts become the user's message text. + +## Specialist catalog + +| Specialist | Source | Files | Size | +|---|---|---|---| +| Speech to Text | ggerganov/whisper.cpp | ggml-base.bin | ~142 MB | +| Image Understanding | ggml-org/SmolVLM-500M-Instruct-GGUF | model + mmproj (Q8_0) | ~550 MB | + +Downloaded via the same HF Hub mechanism (`SpecialistManager`), shown in the +Model Library under "Add-on capabilities". + +## E2E validation (desktop, CPU) + +```sh +scripts/test-whisper-macos.sh build/models/whisper/ggml-base.bin speech.wav fox +scripts/test-engine-macos.sh smolvlm.gguf smolvlm-mmproj.gguf photo.jpg +``` + +Both passed: whisper transcribed `say`-generated speech verbatim; SmolVLM-500M +described a screenshot (image encode ~5 s on Intel CPU — vs E2B's ~87 s — and +30 tok/s decode). Device Metal will be markedly faster. + +## Constraints (inherited) + +- whisper wants mono float32 @ 16 kHz; `WhisperTranscriber` decodes any + recording via `AVAudioConverter`. `VoiceRecorder` already captures mono WAV. +- SmolVLM images go through stb_image (JPEG; HEIC re-encoded as in Phase 2). +- The CLIP/mmproj encoder must run CPU on non-Apple GPUs (simulator) — handled + by `RuntimeProfile.visionEngineParams` / `specialistUsesGPU`. + +## Image generation (experimental specialist) + +A third ggml framework — **stable-diffusion.cpp** (`third_party/stable-diffusion.cpp`, +`scripts/build-sd-xcframework.sh` → `Frameworks/sd.xcframework`) — adds on-device +text-to-image. Bridge: `synap_sd.{h,cpp}` → `ImageGenerator` (load → generate → +unload). Three ggml frameworks now coexist in one app (each force-loads its own +ggml; two-level namespace keeps them apart — verified building + running). + +- **Model:** SD 1.5 (`second-state/...-Q4_0.gguf`, ~1.57 GB). Gated to ≈4 GB+ + (`requiredRAMGB 3.5`) and flagged experimental on the 4 GB tier. The main chat + model is **unloaded during diffusion** (`ChatSession.createImage`) — SD needs + nearly the whole budget. +- **Params per tier** (`RuntimeProfile.imageGenSettings`): 4 GB → 384², 16 steps, + EULER_A; ≥6 GB → 512². Tens of seconds to minutes on A13. +- **UI:** "Create with AI" row in the attachment sheet (shown when installed) → + prompt sheet → the generated image posts as an assistant message. +- **Backend gotcha (important):** ggml-Metal on a non-Apple GPU (the Intel dev + Mac, the simulator) produces **all-white (NaN) output**. `synap_sd` pins the + backend to `"CPU"` whenever `use_gpu` is false (`RuntimeProfile.specialistUsesGPU`); + real iPhones use Metal (Apple GPU). This cost a long debug detour — the white + output looked like a quantization failure (Q4 *and* Q8 were both white) until + forcing CPU produced a correct image. **Not** a quant issue. +- **TAESD** (tiny VAE) was tried to shrink memory but the available file + segfaulted in sd.cpp — dropped; the checkpoint's own VAE is used. First lever + if 4 GB memory testing struggles. +- E2E validated on desktop (CPU): `scripts/test-sd-macos.sh` generated a coherent + "red apple on a table" at 256²/18 steps (~35 s on the Intel CPU). + +⚠️ On-device 4 GB testing is still required: whether SD 1.5's ~1.6 GB resident + +diffusion working set survives the iPhone 11 jetsam ceiling is the open question +the experiment exists to answer. + +## Still open (the PLAN's realtime voice loop proper) + +This phase delivered the **input models**; the streaming voice *loop* (VAD → +segment → transcribe → reply → **TTS out** → barge-in, ≤2.5 s budget) is the +next slice. TTS (PLAN §2) is not yet implemented — voice currently returns a +text reply, not spoken audio. diff --git a/scripts/build-frameworks.sh b/scripts/build-frameworks.sh new file mode 100755 index 0000000..ac010cf --- /dev/null +++ b/scripts/build-frameworks.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# Builds ALL native xcframeworks SynapLink needs, in order. Run this once after +# cloning (and after any submodule pin bump) BEFORE opening Xcode — the +# frameworks are gitignored build artifacts, so without them Xcode fails with +# "There is no XCFramework found at .../Frameworks/.xcframework". +# +# llama.xcframework — text + multimodal inference core (llama.cpp + libmtmd) +# whisper.xcframework — speech-to-text specialist (whisper.cpp) +# sd.xcframework — experimental image generation (stable-diffusion.cpp) +# +# Each sub-build is skipped if its framework already exists; pass --force to +# rebuild everything. +# +# Usage: scripts/build-frameworks.sh [--force] + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FORCE="${1:-}" + +build_one() { + local name=$1 script=$2 + if [[ "${FORCE}" != "--force" && -d "${REPO_ROOT}/Frameworks/${name}.xcframework" ]]; then + echo "✓ ${name}.xcframework already built (use --force to rebuild)" + return + fi + echo "== Building ${name}.xcframework ==" + "${REPO_ROOT}/scripts/${script}" +} + +build_one llama build-llama-xcframework.sh +build_one whisper build-whisper-xcframework.sh +build_one sd build-sd-xcframework.sh + +echo "" +echo "All frameworks ready in Frameworks/. You can now build SynapLink in Xcode." diff --git a/scripts/build-sd-xcframework.sh b/scripts/build-sd-xcframework.sh new file mode 100755 index 0000000..8586ec9 --- /dev/null +++ b/scripts/build-sd-xcframework.sh @@ -0,0 +1,239 @@ +#!/usr/bin/env bash +# +# Builds sd.xcframework (stable-diffusion.cpp + ggml, Metal) for SynapLink. +# +# The THIRD ggml-based framework (after llama + whisper) — the experimental +# on-device image-generation specialist. Each framework force-loads its own +# ggml; Apple's two-level namespace keeps the three symbol sets private per +# dylib, so they coexist. webp/webm/examples are disabled: we read raw RGB +# from sd_image_t and JPEG-encode in Swift. +# +# Slices: iOS device (arm64), iOS simulator (arm64 + x86_64), macOS +# (arm64 + x86_64) — same Intel-dev-Mac rationale as the other builds. +# +# Usage: scripts/build-sd-xcframework.sh +# Output: Frameworks/sd.xcframework + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SD_SRC="${REPO_ROOT}/third_party/stable-diffusion.cpp" +BUILD_ROOT="${REPO_ROOT}/build" +OUT_DIR="${REPO_ROOT}/Frameworks" + +SD_TAG="master" +SD_COMMIT="bb90bfa00f858c7df6502e75f31c4440d4d11fde" + +if [[ ! -f "${SD_SRC}/CMakeLists.txt" ]]; then + echo "== Initialising stable-diffusion.cpp submodule ==" + git -C "${REPO_ROOT}" submodule update --init --depth 1 third_party/stable-diffusion.cpp +fi +# stable-diffusion.cpp needs its own ggml fork submodule to build. +if [[ ! -f "${SD_SRC}/ggml/include/ggml.h" ]]; then + git -C "${SD_SRC}" submodule update --init --depth 1 ggml +fi + +CHECKED_OUT="$(git -C "${SD_SRC}" rev-parse HEAD 2>/dev/null || true)" +if [[ "${CHECKED_OUT}" != "${SD_COMMIT}" ]]; then + echo "error: third_party/stable-diffusion.cpp is at '${CHECKED_OUT:-unknown}', expected ${SD_COMMIT} (${SD_TAG})" >&2 + echo " if you bumped the submodule, update SD_COMMIT in this script." >&2 + exit 1 +fi + +IOS_MIN_OS_VERSION=16.4 +MACOS_MIN_OS_VERSION=13.3 + +COMMON_C_FLAGS="-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g" + +COMMON_CMAKE_ARGS=( + -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=NO + -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY="" + -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=NO + -DCMAKE_XCODE_ATTRIBUTE_DEBUG_INFORMATION_FORMAT="dwarf-with-dsym" + -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml + -DSD_METAL=ON + -DSD_BUILD_EXAMPLES=OFF + -DSD_WEBP=OFF + -DSD_WEBM=OFF + -DSD_BUILD_SHARED_LIBS=OFF + -DGGML_METAL_EMBED_LIBRARY=ON + -DGGML_NATIVE=OFF + -DGGML_OPENMP=OFF + -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" + -DCMAKE_CXX_FLAGS="${COMMON_C_FLAGS}" +) + +FRAMEWORK_HEADERS=( + "${SD_SRC}/include/stable-diffusion.h" + "${SD_SRC}/ggml/include/ggml.h" + "${SD_SRC}/ggml/include/ggml-cpu.h" + "${SD_SRC}/ggml/include/ggml-backend.h" + "${SD_SRC}/ggml/include/ggml-alloc.h" + "${SD_SRC}/ggml/include/ggml-metal.h" +) + +configure_and_build() { + local build_dir=$1; shift + cmake -B "${build_dir}" -G Xcode "${COMMON_CMAKE_ARGS[@]}" "$@" -S "${SD_SRC}" + cmake --build "${build_dir}" --config Release --target stable-diffusion -j "$(sysctl -n hw.logicalcpu)" -- -quiet +} + +# $1 build_dir, $2 platform (ios|macos), $3 sdk, $4 min_version_flag, $5 archs +make_framework() { + local build_dir=$1 platform=$2 sdk=$3 min_version_flag=$4 archs=$5 + + local fw="${build_dir}/framework/sd.framework" + local header_path module_path plist_path install_name output_lib + local supported_platform platform_name min_os device_family="" + + if [[ "$platform" == "macos" ]]; then + mkdir -p "${fw}/Versions/A/Headers" "${fw}/Versions/A/Modules" "${fw}/Versions/A/Resources" + ln -sfh A "${fw}/Versions/Current" + ln -sfh Versions/Current/Headers "${fw}/Headers" + ln -sfh Versions/Current/Modules "${fw}/Modules" + ln -sfh Versions/Current/Resources "${fw}/Resources" + ln -sfh Versions/Current/sd "${fw}/sd" + header_path="${fw}/Versions/A/Headers"; module_path="${fw}/Versions/A/Modules" + plist_path="${fw}/Versions/A/Resources/Info.plist"; output_lib="${fw}/Versions/A/sd" + install_name="@rpath/sd.framework/Versions/Current/sd" + supported_platform="MacOSX"; platform_name="macosx"; min_os="${MACOS_MIN_OS_VERSION}" + else + mkdir -p "${fw}/Headers" "${fw}/Modules" + header_path="${fw}/Headers"; module_path="${fw}/Modules" + plist_path="${fw}/Info.plist"; output_lib="${fw}/sd" + install_name="@rpath/sd.framework/sd" + supported_platform="iPhoneOS"; platform_name="iphoneos"; min_os="${IOS_MIN_OS_VERSION}" + device_family=' + UIDeviceFamily + + 1 + 2 + ' + if [[ "$sdk" == "iphonesimulator" ]]; then + supported_platform="iPhoneSimulator"; platform_name="iphonesimulator" + fi + fi + + cp "${FRAMEWORK_HEADERS[@]}" "${header_path}/" + + cat > "${module_path}/module.modulemap" << 'EOF' +framework module sd { + umbrella "Headers" + + link "c++" + link framework "Accelerate" + link framework "Metal" + link framework "Foundation" + + export * +} +EOF + + cat > "${plist_path}" << EOF + + + + + CFBundleDevelopmentRegionen + CFBundleExecutablesd + CFBundleIdentifierorg.ggml.sd + CFBundleInfoDictionaryVersion6.0 + CFBundleNamesd + CFBundlePackageTypeFMWK + CFBundleShortVersionString1.0 + CFBundleVersion1 + MinimumOSVersion${min_os} + CFBundleSupportedPlatforms${supported_platform}${device_family} + DTPlatformName${platform_name} + + +EOF + + # Use the FINAL per-target archives only — NOT Xcode's per-arch + # intermediates under */Objects-normal/ or */*.build/ (combining those + # merges duplicate ggml objects → "duplicate symbols" at link). + local release_dir + case "$sdk" in + iphonesimulator) release_dir="Release-iphonesimulator" ;; + iphoneos) release_dir="Release-iphoneos" ;; + *) release_dir="Release" ;; + esac + local libs=( + "${build_dir}/${release_dir}/libstable-diffusion.a" + "${build_dir}/ggml/src/${release_dir}/libggml.a" + "${build_dir}/ggml/src/${release_dir}/libggml-base.a" + "${build_dir}/ggml/src/${release_dir}/libggml-cpu.a" + "${build_dir}/ggml/src/ggml-blas/${release_dir}/libggml-blas.a" + "${build_dir}/ggml/src/ggml-metal/${release_dir}/libggml-metal.a" + ) + # NOTE: the vendored zip code is already compiled into libstable-diffusion.a, + # so libzip.a is NOT added here (doing so duplicates its symbols). + for lib in "${libs[@]}"; do + if [[ ! -f "${lib}" ]]; then echo "error: missing ${lib}" >&2; exit 1; fi + done + + local temp_dir="${build_dir}/temp" + mkdir -p "${temp_dir}" + xcrun libtool -static -o "${temp_dir}/combined.a" "${libs[@]}" 2> /dev/null + + local arch_flags="" + for arch in ${archs}; do arch_flags+=" -arch ${arch}"; done + + echo "Creating dynamic library for ${sdk} (${archs})..." + xcrun -sdk "$sdk" clang++ -dynamiclib \ + -isysroot "$(xcrun --sdk "$sdk" --show-sdk-path)" \ + ${arch_flags} ${min_version_flag} \ + -Wl,-force_load,"${temp_dir}/combined.a" \ + -framework Foundation -framework Metal -framework Accelerate \ + -install_name "${install_name}" \ + -o "${output_lib}" + + if [[ "$sdk" == "iphoneos" ]]; then + xcrun vtool -set-build-version ios ${IOS_MIN_OS_VERSION} ${IOS_MIN_OS_VERSION} -replace \ + -output "${output_lib}" "${output_lib}" + fi + + mkdir -p "${build_dir}/dSYMs" + xcrun dsymutil "${output_lib}" -o "${build_dir}/dSYMs/sd.dSYM" 2> /dev/null || true + xcrun strip -S "${output_lib}" -o "${temp_dir}/stripped" + mv "${temp_dir}/stripped" "${output_lib}" + rm -rf "${output_lib}.dSYM" "${temp_dir}" +} + +echo "== Configuring & building: iOS simulator (arm64 + x86_64) ==" +configure_and_build "${BUILD_ROOT}/sd-ios-sim" \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} -DIOS=ON \ + -DCMAKE_SYSTEM_NAME=iOS -DCMAKE_OSX_SYSROOT=iphonesimulator \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ + -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphonesimulator + +echo "== Configuring & building: iOS device (arm64) ==" +configure_and_build "${BUILD_ROOT}/sd-ios-device" \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \ + -DCMAKE_SYSTEM_NAME=iOS -DCMAKE_OSX_SYSROOT=iphoneos \ + -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphoneos + +echo "== Configuring & building: macOS (arm64 + x86_64) ==" +configure_and_build "${BUILD_ROOT}/sd-macos" \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOS_MIN_OS_VERSION} \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" + +echo "== Assembling framework slices ==" +make_framework "${BUILD_ROOT}/sd-ios-sim" "ios" "iphonesimulator" "-mios-simulator-version-min=${IOS_MIN_OS_VERSION}" "arm64 x86_64" +make_framework "${BUILD_ROOT}/sd-ios-device" "ios" "iphoneos" "-mios-version-min=${IOS_MIN_OS_VERSION}" "arm64" +make_framework "${BUILD_ROOT}/sd-macos" "macos" "macosx" "-mmacosx-version-min=${MACOS_MIN_OS_VERSION}" "arm64 x86_64" + +echo "== Creating XCFramework ==" +rm -rf "${OUT_DIR}/sd.xcframework" +mkdir -p "${OUT_DIR}" +xcrun xcodebuild -create-xcframework \ + -framework "${BUILD_ROOT}/sd-ios-sim/framework/sd.framework" \ + -debug-symbols "${BUILD_ROOT}/sd-ios-sim/dSYMs/sd.dSYM" \ + -framework "${BUILD_ROOT}/sd-ios-device/framework/sd.framework" \ + -debug-symbols "${BUILD_ROOT}/sd-ios-device/dSYMs/sd.dSYM" \ + -framework "${BUILD_ROOT}/sd-macos/framework/sd.framework" \ + -debug-symbols "${BUILD_ROOT}/sd-macos/dSYMs/sd.dSYM" \ + -output "${OUT_DIR}/sd.xcframework" + +echo "Done: ${OUT_DIR}/sd.xcframework" diff --git a/scripts/build-whisper-xcframework.sh b/scripts/build-whisper-xcframework.sh new file mode 100755 index 0000000..ea0cdf6 --- /dev/null +++ b/scripts/build-whisper-xcframework.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash +# +# Builds whisper.xcframework (libwhisper + ggml, Metal, Accelerate) for SynapLink. +# +# Sibling to build-llama-xcframework.sh. whisper.cpp is the dedicated on-device +# ASR (speech -> text) for the iPhone 11 tier, where the omni Gemma 4 E2B model +# can't fit. Two ggml-based frameworks coexist in one app: each force-loads its +# OWN copy of ggml, and Apple's two-level namespace keeps the symbol sets +# private to each dylib, so there is no cross-framework ggml clash. +# +# Slices: iOS device (arm64), iOS simulator (arm64 + x86_64), macOS +# (arm64 + x86_64) — same Intel-dev-Mac rationale as the llama build. +# +# Usage: scripts/build-whisper-xcframework.sh +# Output: Frameworks/whisper.xcframework + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WHISPER_SRC="${REPO_ROOT}/third_party/whisper.cpp" +BUILD_ROOT="${REPO_ROOT}/build" +OUT_DIR="${REPO_ROOT}/Frameworks" + +# whisper.cpp lives in a git submodule; the gitlink is the authoritative pin. +# WHISPER_TAG/WHISPER_COMMIT mirror it; the check fails the build on drift. +WHISPER_TAG="v1.8.6" +WHISPER_COMMIT="23ee03506a91ac3d3f0071b40e66a430eebdfa1d" + +if [[ ! -f "${WHISPER_SRC}/CMakeLists.txt" ]]; then + echo "== Initialising whisper.cpp submodule ==" + git -C "${REPO_ROOT}" submodule update --init --depth 1 third_party/whisper.cpp +fi + +CHECKED_OUT="$(git -C "${WHISPER_SRC}" rev-parse HEAD 2>/dev/null || true)" +if [[ "${CHECKED_OUT}" != "${WHISPER_COMMIT}" ]]; then + echo "error: third_party/whisper.cpp is at '${CHECKED_OUT:-unknown}', expected ${WHISPER_COMMIT} (${WHISPER_TAG})" >&2 + echo " if you bumped the submodule, update WHISPER_TAG/WHISPER_COMMIT in this script;" >&2 + echo " otherwise run: git submodule update --init third_party/whisper.cpp" >&2 + exit 1 +fi + +IOS_MIN_OS_VERSION=16.4 +MACOS_MIN_OS_VERSION=13.3 + +COMMON_C_FLAGS="-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g" +COMMON_CXX_FLAGS="${COMMON_C_FLAGS}" + +COMMON_CMAKE_ARGS=( + -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=NO + -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY="" + -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=NO + -DCMAKE_XCODE_ATTRIBUTE_DEBUG_INFORMATION_FORMAT="dwarf-with-dsym" + -DCMAKE_XCODE_ATTRIBUTE_GCC_GENERATE_DEBUGGING_SYMBOLS=YES + -DCMAKE_XCODE_ATTRIBUTE_COPY_PHASE_STRIP=NO + -DCMAKE_XCODE_ATTRIBUTE_STRIP_INSTALLED_PRODUCT=NO + -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml + -DBUILD_SHARED_LIBS=OFF + -DWHISPER_BUILD_EXAMPLES=OFF + -DWHISPER_BUILD_TESTS=OFF + -DWHISPER_BUILD_SERVER=OFF + -DGGML_METAL=ON + -DGGML_METAL_EMBED_LIBRARY=ON + -DGGML_METAL_USE_BF16=ON + -DGGML_BLAS_DEFAULT=ON + -DGGML_NATIVE=OFF + -DGGML_OPENMP=OFF + -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" + -DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" +) + +FRAMEWORK_HEADERS=( + "${WHISPER_SRC}/include/whisper.h" + "${WHISPER_SRC}/ggml/include/ggml.h" + "${WHISPER_SRC}/ggml/include/ggml-cpu.h" + "${WHISPER_SRC}/ggml/include/ggml-backend.h" + "${WHISPER_SRC}/ggml/include/ggml-alloc.h" + "${WHISPER_SRC}/ggml/include/ggml-metal.h" +) + +configure_and_build() { + local build_dir=$1; shift + cmake -B "${build_dir}" -G Xcode "${COMMON_CMAKE_ARGS[@]}" "$@" -S "${WHISPER_SRC}" + cmake --build "${build_dir}" --config Release --target whisper -j "$(sysctl -n hw.logicalcpu)" -- -quiet +} + +# $1 build_dir, $2 release_subdir, $3 platform (ios|macos), $4 sdk, $5 min_version_flag, $6 archs +make_framework() { + local build_dir=$1 + local release_dir=$2 + local platform=$3 + local sdk=$4 + local min_version_flag=$5 + local archs=$6 + + local fw="${build_dir}/framework/whisper.framework" + local header_path module_path plist_path install_name + + if [[ "$platform" == "macos" ]]; then + mkdir -p "${fw}/Versions/A/Headers" "${fw}/Versions/A/Modules" "${fw}/Versions/A/Resources" + ln -sfh A "${fw}/Versions/Current" + ln -sfh Versions/Current/Headers "${fw}/Headers" + ln -sfh Versions/Current/Modules "${fw}/Modules" + ln -sfh Versions/Current/Resources "${fw}/Resources" + ln -sfh Versions/Current/whisper "${fw}/whisper" + header_path="${fw}/Versions/A/Headers" + module_path="${fw}/Versions/A/Modules" + plist_path="${fw}/Versions/A/Resources/Info.plist" + install_name="@rpath/whisper.framework/Versions/Current/whisper" + local output_lib="${fw}/Versions/A/whisper" + local supported_platform="MacOSX" platform_name="macosx" min_os="${MACOS_MIN_OS_VERSION}" + local device_family="" + else + mkdir -p "${fw}/Headers" "${fw}/Modules" + header_path="${fw}/Headers" + module_path="${fw}/Modules" + plist_path="${fw}/Info.plist" + install_name="@rpath/whisper.framework/whisper" + local output_lib="${fw}/whisper" + local supported_platform="iPhoneOS" platform_name="iphoneos" min_os="${IOS_MIN_OS_VERSION}" + local device_family=' + UIDeviceFamily + + 1 + 2 + ' + if [[ "$sdk" == "iphonesimulator" ]]; then + supported_platform="iPhoneSimulator" + platform_name="iphonesimulator" + fi + fi + + cp "${FRAMEWORK_HEADERS[@]}" "${header_path}/" + + cat > "${module_path}/module.modulemap" << 'EOF' +framework module whisper { + umbrella "Headers" + + link "c++" + link framework "Accelerate" + link framework "Metal" + link framework "Foundation" + + export * +} +EOF + + cat > "${plist_path}" << EOF + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + whisper + CFBundleIdentifier + org.ggml.whisper + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + whisper + CFBundlePackageType + FMWK + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + MinimumOSVersion + ${min_os} + CFBundleSupportedPlatforms + + ${supported_platform} + ${device_family} + DTPlatformName + ${platform_name} + + +EOF + + local libs=( + "${build_dir}/src/${release_dir}/libwhisper.a" + "${build_dir}/ggml/src/${release_dir}/libggml.a" + "${build_dir}/ggml/src/${release_dir}/libggml-base.a" + "${build_dir}/ggml/src/${release_dir}/libggml-cpu.a" + "${build_dir}/ggml/src/ggml-metal/${release_dir}/libggml-metal.a" + "${build_dir}/ggml/src/ggml-blas/${release_dir}/libggml-blas.a" + ) + + local temp_dir="${build_dir}/temp" + mkdir -p "${temp_dir}" + xcrun libtool -static -o "${temp_dir}/combined.a" "${libs[@]}" 2> /dev/null + + local arch_flags="" + for arch in ${archs}; do + arch_flags+=" -arch ${arch}" + done + + echo "Creating dynamic library for ${sdk} (${archs})..." + xcrun -sdk "$sdk" clang++ -dynamiclib \ + -isysroot "$(xcrun --sdk "$sdk" --show-sdk-path)" \ + ${arch_flags} \ + ${min_version_flag} \ + -Wl,-force_load,"${temp_dir}/combined.a" \ + -framework Foundation -framework Metal -framework Accelerate \ + -install_name "${install_name}" \ + -o "${output_lib}" + + if [[ "$sdk" == "iphoneos" ]]; then + xcrun vtool -set-build-version ios ${IOS_MIN_OS_VERSION} ${IOS_MIN_OS_VERSION} -replace \ + -output "${output_lib}" "${output_lib}" + fi + + mkdir -p "${build_dir}/dSYMs" + xcrun dsymutil "${output_lib}" -o "${build_dir}/dSYMs/whisper.dSYM" + xcrun strip -S "${output_lib}" -o "${temp_dir}/stripped" + mv "${temp_dir}/stripped" "${output_lib}" + rm -rf "${output_lib}.dSYM" "${temp_dir}" +} + +echo "== Configuring & building: iOS simulator (arm64 + x86_64) ==" +configure_and_build "${BUILD_ROOT}/whisper-ios-sim" \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \ + -DIOS=ON \ + -DCMAKE_SYSTEM_NAME=iOS \ + -DCMAKE_OSX_SYSROOT=iphonesimulator \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ + -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphonesimulator + +echo "== Configuring & building: iOS device (arm64) ==" +configure_and_build "${BUILD_ROOT}/whisper-ios-device" \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \ + -DCMAKE_SYSTEM_NAME=iOS \ + -DCMAKE_OSX_SYSROOT=iphoneos \ + -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphoneos + +echo "== Configuring & building: macOS (arm64 + x86_64) ==" +configure_and_build "${BUILD_ROOT}/whisper-macos" \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOS_MIN_OS_VERSION} \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" + +echo "== Assembling framework slices ==" +make_framework "${BUILD_ROOT}/whisper-ios-sim" "Release-iphonesimulator" "ios" "iphonesimulator" "-mios-simulator-version-min=${IOS_MIN_OS_VERSION}" "arm64 x86_64" +make_framework "${BUILD_ROOT}/whisper-ios-device" "Release-iphoneos" "ios" "iphoneos" "-mios-version-min=${IOS_MIN_OS_VERSION}" "arm64" +make_framework "${BUILD_ROOT}/whisper-macos" "Release" "macos" "macosx" "-mmacosx-version-min=${MACOS_MIN_OS_VERSION}" "arm64 x86_64" + +echo "== Creating XCFramework ==" +rm -rf "${OUT_DIR}/whisper.xcframework" +mkdir -p "${OUT_DIR}" +xcrun xcodebuild -create-xcframework \ + -framework "${BUILD_ROOT}/whisper-ios-sim/framework/whisper.framework" \ + -debug-symbols "${BUILD_ROOT}/whisper-ios-sim/dSYMs/whisper.dSYM" \ + -framework "${BUILD_ROOT}/whisper-ios-device/framework/whisper.framework" \ + -debug-symbols "${BUILD_ROOT}/whisper-ios-device/dSYMs/whisper.dSYM" \ + -framework "${BUILD_ROOT}/whisper-macos/framework/whisper.framework" \ + -debug-symbols "${BUILD_ROOT}/whisper-macos/dSYMs/whisper.dSYM" \ + -output "${OUT_DIR}/whisper.xcframework" + +echo "Done: ${OUT_DIR}/whisper.xcframework" diff --git a/scripts/lint-cpp.sh b/scripts/lint-cpp.sh index 2d23445..c4a2e00 100755 --- a/scripts/lint-cpp.sh +++ b/scripts/lint-cpp.sh @@ -15,8 +15,14 @@ FILES=( SynapLink/Core/Engine/synap_engine.h SynapLink/Core/Engine/synap_engine.cpp SynapLink/Core/Engine/synap_engine_internal.hpp + SynapLink/Core/Engine/synap_whisper.h + SynapLink/Core/Engine/synap_whisper.cpp + SynapLink/Core/Engine/synap_sd.h + SynapLink/Core/Engine/synap_sd.cpp SynapLink/Core/Engine/SynapLink-Bridging-Header.h tests/engine-smoke/main.cpp + tests/whisper-smoke/main.cpp + tests/sd-smoke/main.cpp ) echo "== clang-format ==" diff --git a/scripts/test-sd-macos.sh b/scripts/test-sd-macos.sh new file mode 100755 index 0000000..83c32c6 --- /dev/null +++ b/scripts/test-sd-macos.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Builds and runs the desktop image-generation smoke test against the macOS +# slice of sd.xcframework. Validates the synap_sd bridge the app ships. +# +# Usage: scripts/test-sd-macos.sh "" [size] [steps] [out.ppm] + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FW_DIR="${REPO_ROOT}/Frameworks/sd.xcframework/macos-arm64_x86_64" +OUT="${REPO_ROOT}/build/sd-smoke" + +mkdir -p "${REPO_ROOT}/build" + +clang++ -std=c++17 -O1 -g \ + -I "${REPO_ROOT}/SynapLink/Core/Engine" \ + -F "${FW_DIR}" \ + -framework sd \ + -rpath "${FW_DIR}" \ + "${REPO_ROOT}/SynapLink/Core/Engine/synap_sd.cpp" \ + "${REPO_ROOT}/tests/sd-smoke/main.cpp" \ + -o "${OUT}" + +exec "${OUT}" "$@" diff --git a/scripts/test-whisper-macos.sh b/scripts/test-whisper-macos.sh new file mode 100755 index 0000000..2676ee4 --- /dev/null +++ b/scripts/test-whisper-macos.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Builds and runs the desktop whisper smoke test against the macOS slice of +# whisper.xcframework. Validates the exact synap_whisper bridge the app ships. +# +# Usage: scripts/test-whisper-macos.sh [expected-substr] + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FW_DIR="${REPO_ROOT}/Frameworks/whisper.xcframework/macos-arm64_x86_64" +OUT="${REPO_ROOT}/build/whisper-smoke" + +mkdir -p "${REPO_ROOT}/build" + +clang++ -std=c++17 -O1 -g \ + -I "${REPO_ROOT}/SynapLink/Core/Engine" \ + -F "${FW_DIR}" \ + -framework whisper \ + -rpath "${FW_DIR}" \ + "${REPO_ROOT}/SynapLink/Core/Engine/synap_whisper.cpp" \ + "${REPO_ROOT}/tests/whisper-smoke/main.cpp" \ + -o "${OUT}" + +exec "${OUT}" "$@" diff --git a/tests/sd-smoke/main.cpp b/tests/sd-smoke/main.cpp new file mode 100644 index 0000000..3dadb0e --- /dev/null +++ b/tests/sd-smoke/main.cpp @@ -0,0 +1,59 @@ +// +// main.cpp — desktop smoke test for synap_sd +// +// Links the macOS slice of sd.xcframework + the synap_sd bridge, generates a +// small image from a prompt, and writes it as a PPM so it can be eyeballed. +// +// Usage: sd-smoke [size] [steps] [out.ppm] +// Exit codes: 0 ok, 1 usage, 2 load failed, 3 generate failed +// + +#include "synap_sd.h" + +#include +#include +#include + +int main(int argc, char** argv) { + setvbuf(stdout, nullptr, _IOLBF, 0); + if (argc < 3) { + fprintf(stderr, "usage: %s [size] [steps] [out.ppm]\n", argv[0]); + return 1; + } + const char* model = argv[1]; + const char* prompt = argv[2]; + const int32_t size = argc > 3 ? atoi(argv[3]) : 256; + const int32_t steps = argc > 4 ? atoi(argv[4]) : 12; + const char* outPath = argc > 5 ? argv[5] : "/tmp/sd-out.ppm"; + + printf("== load ==\n"); + const char* taesd = getenv("SD_TAESD"); // optional tiny VAE override + SynapSD* sd = synap_sd_create(model, taesd, 4, false); + if (!sd) { + fprintf(stderr, "FAIL: sd model load: %s\n", model); + return 2; + } + + printf("== generate %dx%d, %d steps ==\n%s\n", size, size, steps, prompt); + int32_t w = 0, h = 0; + uint8_t* rgb = synap_sd_generate(sd, prompt, "", size, size, steps, 7.0f, + /*EULER_A=*/1, /*seed=*/42, &w, &h); + synap_sd_free(sd); + + if (!rgb || w <= 0 || h <= 0) { + fprintf(stderr, "FAIL: generate\n"); + return 3; + } + printf("generated %dx%d\n", w, h); + + FILE* f = fopen(outPath, "wb"); + if (f) { + fprintf(f, "P6\n%d %d\n255\n", w, h); + fwrite(rgb, 1, static_cast(w) * h * 3, f); + fclose(f); + printf("wrote %s\n", outPath); + } + synap_sd_free_rgb(rgb); + printf("OK: image generation succeeded\n"); + return 0; +} diff --git a/tests/whisper-smoke/main.cpp b/tests/whisper-smoke/main.cpp new file mode 100644 index 0000000..21fd80c --- /dev/null +++ b/tests/whisper-smoke/main.cpp @@ -0,0 +1,104 @@ +// +// main.cpp — desktop smoke test for synap_whisper +// +// Links the macOS slice of whisper.xcframework and the synap_whisper bridge, +// loads a whisper GGML model, transcribes a 16 kHz mono s16le WAV, and checks +// the text is non-empty (and optionally contains an expected substring). +// +// Usage: whisper-smoke [expected-substr] +// Exit codes: 0 ok, 1 usage, 2 load failed, 3 wav read failed, 4 transcribe failed, +// 5 expected substring missing +// + +#include "synap_whisper.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +// Minimal WAV reader for PCM s16le mono at SYNAP_WHISPER_SAMPLE_RATE. Returns +// float samples in [-1, 1]; empty on failure. +std::vector read_wav_s16_mono(const char* path) { + std::ifstream in(path, std::ios::binary); + if (!in) { return {}; } + std::vector bytes((std::istreambuf_iterator(in)), {}); + if (bytes.size() < 44 || memcmp(bytes.data(), "RIFF", 4) != 0) { return {}; } + + // Walk chunks to find "data". + size_t pos = 12; + while (pos + 8 <= bytes.size()) { + const char* id = reinterpret_cast(bytes.data() + pos); + uint32_t sz; + memcpy(&sz, bytes.data() + pos + 4, 4); + if (memcmp(id, "data", 4) == 0) { + const size_t start = pos + 8; + const size_t n = std::min(sz, bytes.size() - start) / 2; + std::vector samples(n); + for (size_t i = 0; i < n; ++i) { + int16_t s; + memcpy(&s, bytes.data() + start + i * 2, 2); + samples[i] = static_cast(s) / 32768.0f; + } + return samples; + } + pos += 8 + sz + (sz & 1); + } + return {}; +} + +std::string lower(std::string s) { + for (auto& c : s) { c = static_cast(std::tolower(static_cast(c))); } + return s; +} + +} // namespace + +int main(int argc, char** argv) { + setvbuf(stdout, nullptr, _IOLBF, 0); + if (argc < 3) { + fprintf(stderr, "usage: %s [expected-substr]\n", argv[0]); + return 1; + } + + std::vector samples = read_wav_s16_mono(argv[2]); + if (samples.empty()) { + fprintf(stderr, "FAIL: could not read 16k mono s16 WAV: %s\n", argv[2]); + return 3; + } + printf("loaded %zu samples (%.1f s @ 16kHz)\n", samples.size(), samples.size() / 16000.0); + + const bool use_gpu = false; // CPU on the Intel dev Mac + SynapWhisper* w = synap_whisper_create(argv[1], use_gpu, 4); + if (!w) { + fprintf(stderr, "FAIL: whisper model load: %s\n", argv[1]); + return 2; + } + + std::vector buf(8 * 1024); + const int32_t n = synap_whisper_transcribe(w, samples.data(), static_cast(samples.size()), "en", + buf.data(), static_cast(buf.size())); + synap_whisper_free(w); + + if (n < 0) { + fprintf(stderr, "FAIL: transcribe rc=%d\n", n); + return 4; + } + printf("transcript: \"%s\"\n", buf.data()); + + if (argc > 3) { + if (lower(buf.data()).find(lower(argv[3])) == std::string::npos) { + fprintf(stderr, "FAIL: expected substring '%s' not found\n", argv[3]); + return 5; + } + printf("substring '%s' found\n", argv[3]); + } + + printf("OK: whisper transcription succeeded\n"); + return 0; +} diff --git a/third_party/stable-diffusion.cpp b/third_party/stable-diffusion.cpp new file mode 160000 index 0000000..bb90bfa --- /dev/null +++ b/third_party/stable-diffusion.cpp @@ -0,0 +1 @@ +Subproject commit bb90bfa00f858c7df6502e75f31c4440d4d11fde diff --git a/third_party/whisper.cpp b/third_party/whisper.cpp new file mode 160000 index 0000000..23ee035 --- /dev/null +++ b/third_party/whisper.cpp @@ -0,0 +1 @@ +Subproject commit 23ee03506a91ac3d3f0071b40e66a430eebdfa1d