Enhance voice transcription, image generation, and attachment handling

.gitignore

@@ -69,4 +69,6 @@ PLAN.md
 # Third Party: tracked as git submodules (see .gitmodules), not ignored.
 
 # Built artifacts (reproduce with scripts/build-llama-xcframework.sh)
-Frameworks/llama.xcframework/
+Frameworks/llama.xcframework/
+Frameworks/whisper.xcframework/
+Frameworks/sd.xcframework/

.gitmodules

@@ -2,3 +2,11 @@
 	path = third_party/llama.cpp
 	url = https://github.com/ggml-org/llama.cpp
 	shallow = true
+[submodule "third_party/whisper.cpp"]
+	path = third_party/whisper.cpp
+	url = https://github.com/ggml-org/whisper.cpp
+	shallow = true
+[submodule "third_party/stable-diffusion.cpp"]
+	path = third_party/stable-diffusion.cpp
+	url = https://github.com/leejet/stable-diffusion.cpp
+	shallow = true

README.md

@@ -35,17 +35,25 @@ Baseline device: **iPhone 11 (A13, 4 GB RAM)**.
 ## Layout
 
 - `SynapLink/` — app sources (SwiftUI + Swift service layer + C/C++ engine)
-- `scripts/build-llama-xcframework.sh` — builds `Frameworks/llama.xcframework`
-  (Metal + libmtmd) from a pinned llama.cpp tag
-- `scripts/test-engine-macos.sh` — desktop functional test of the engine core
+- `scripts/build-frameworks.sh` — builds all three native xcframeworks (run this first)
+- `scripts/build-{llama,whisper,sd}-xcframework.sh` — individual framework builds
+  (llama.cpp + libmtmd, whisper.cpp ASR, stable-diffusion.cpp image-gen)
+- `scripts/test-{engine,whisper,sd}-macos.sh` — desktop functional tests of the bridges
 - `docs/PHASE_0.md` — scaffolding & inference core: build + smoke-test instructions
 - `docs/PHASE_1.md` — text chat MVP: architecture, model catalog, chat pipeline
 - `docs/PHASE_2.md` — multimodal input: attachments, capture, media validation
+- `docs/PHASE_3.md` — sidecar specialists: whisper ASR, SmolVLM vision, SD image-gen
 
 ## Quick start
 
+The `*.xcframework`s under `Frameworks/` are **gitignored build artifacts** —
+you must build them before opening Xcode, or the build fails with
+"There is no XCFramework found at …". One command does it (≈40–60 min first run;
+each is skipped if already present):
+
 ```sh
-scripts/build-llama-xcframework.sh   
+git submodule update --init        # llama.cpp, whisper.cpp, stable-diffusion.cpp
+scripts/build-frameworks.sh        # → Frameworks/{llama,whisper,sd}.xcframework
 open SynapLink.xcodeproj
 ```
 

SynapLink.xcodeproj/project.pbxproj

@@ -10,6 +10,10 @@
 		AA0000000000000000000031 /* llama.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000030 /* llama.xcframework */; };
 		AA0000000000000000000032 /* llama.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000030 /* llama.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
 		AA0000000000000000000062 /* Hub in Frameworks */ = {isa = PBXBuildFile; productRef = AA0000000000000000000061 /* Hub */; };
+		AA0000000000000000000071 /* whisper.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000070 /* whisper.xcframework */; };
+		AA0000000000000000000072 /* whisper.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000070 /* whisper.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
+		AA0000000000000000000081 /* sd.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000080 /* sd.xcframework */; };
+		AA0000000000000000000082 /* sd.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000080 /* sd.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
 /* End PBXBuildFile section */
 
 /* Begin PBXCopyFilesBuildPhase section */
@@ -20,6 +24,8 @@
 			dstSubfolderSpec = 10;
 			files = (
 				AA0000000000000000000032 /* llama.xcframework in Embed Frameworks */,
+				AA0000000000000000000072 /* whisper.xcframework in Embed Frameworks */,
+				AA0000000000000000000082 /* sd.xcframework in Embed Frameworks */,
 			);
 			name = "Embed Frameworks";
 			runOnlyForDeploymentPostprocessing = 0;
@@ -29,6 +35,8 @@
 /* Begin PBXFileReference section */
 		AA0000000000000000000020 /* SynapLink.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = SynapLink.app; sourceTree = BUILT_PRODUCTS_DIR; };
 		AA0000000000000000000030 /* llama.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = llama.xcframework; path = Frameworks/llama.xcframework; sourceTree = "<group>"; };
+		AA0000000000000000000070 /* whisper.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = whisper.xcframework; path = Frameworks/whisper.xcframework; sourceTree = "<group>"; };
+		AA0000000000000000000080 /* sd.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = sd.xcframework; path = Frameworks/sd.xcframework; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFileSystemSynchronizedRootGroup section */
@@ -45,6 +53,8 @@
 			buildActionMask = 2147483647;
 			files = (
 				AA0000000000000000000031 /* llama.xcframework in Frameworks */,
+				AA0000000000000000000071 /* whisper.xcframework in Frameworks */,
+				AA0000000000000000000081 /* sd.xcframework in Frameworks */,
 				AA0000000000000000000062 /* Hub in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -73,6 +83,8 @@
 			isa = PBXGroup;
 			children = (
 				AA0000000000000000000030 /* llama.xcframework */,
+				AA0000000000000000000070 /* whisper.xcframework */,
+				AA0000000000000000000080 /* sd.xcframework */,
 			);
 			name = Frameworks;
 			sourceTree = "<group>";

SynapLink/Core/Engine/SynapLink-Bridging-Header.h

@@ -9,6 +9,8 @@
 #pragma once
 
 #include "synap_engine.h"
+#include "synap_whisper.h"
+#include "synap_sd.h"
 
 #if __has_include(<os/proc.h>)
 #include <os/proc.h> // os_proc_available_memory() — jetsam headroom (iOS only)

SynapLink/Core/Engine/synap_engine.cpp

@@ -20,11 +20,25 @@
 #include <llama/mtmd-helper.h>
 
 #include <algorithm>
+#include <atomic>
 #include <chrono>
 #include <cstring>
 
 // MARK: - §1 Lifecycle
 
+// Refcount the global llama backend so multiple engines can coexist (main
+// text model + an on-demand vision specialist). Without this, freeing one
+// engine's backend would pull the rug out from under the other.
+static std::atomic<int> g_backend_refcount{0};
+
+static void backend_retain() {
+    if (g_backend_refcount.fetch_add(1) == 0) { llama_backend_init(); }
+}
+
+static void backend_release() {
+    if (g_backend_refcount.fetch_sub(1) == 1) { llama_backend_free(); }
+}
+
 SynapEngineParams synap_engine_params_default(void) {
     SynapEngineParams p = {};
     p.model_path        = nullptr;
@@ -64,14 +78,14 @@ static llama_sampler* build_sampler(const SynapEngineParams& p) {
 SynapEngine* synap_engine_create(const SynapEngineParams* params) {
     if (!params || !params->model_path) { return nullptr; }
 
-    llama_backend_init();
+    backend_retain();
 
     llama_model_params mp = llama_model_default_params();
     mp.n_gpu_layers       = params->n_gpu_layers;
 
     llama_model* model = llama_model_load_from_file(params->model_path, mp);
     if (!model) {
-        llama_backend_free();
+        backend_release();
         return nullptr;
     }
 
@@ -87,15 +101,15 @@ SynapEngine* synap_engine_create(const SynapEngineParams* params) {
     llama_context* ctx = llama_init_from_model(model, cp);
     if (!ctx) {
         llama_model_free(model);
-        llama_backend_free();
+        backend_release();
         return nullptr;
     }
 
     llama_sampler* sampler = build_sampler(*params);
     if (!sampler) {
         llama_free(ctx);
         llama_model_free(model);
-        llama_backend_free();
+        backend_release();
         return nullptr;
     }
 
@@ -113,7 +127,7 @@ SynapEngine* synap_engine_create(const SynapEngineParams* params) {
             llama_sampler_free(sampler);
             llama_free(ctx);
             llama_model_free(model);
-            llama_backend_free();
+            backend_release();
             return nullptr;
         }
     }
@@ -132,7 +146,7 @@ void synap_engine_free(SynapEngine* engine) {
     if (engine->sampler) { llama_sampler_free(engine->sampler); }
     if (engine->ctx) { llama_free(engine->ctx); }
     if (engine->model) { llama_model_free(engine->model); }
-    llama_backend_free();
+    backend_release();
     delete engine;
 }
 

SynapLink/Core/Engine/synap_sd.cpp

@@ -0,0 +1,114 @@
+//
+//  synap_sd.cpp
+//  SynapLink
+//
+//  C++ implementation of the synap_sd bridge over stable-diffusion.cpp.
+//
+
+#include "synap_sd.h"
+
+#include <sd/stable-diffusion.h>
+
+#include <cstdlib>
+#include <cstring>
+
+struct SynapSD {
+    sd_ctx_t* ctx = nullptr;
+};
+
+// stable-diffusion.cpp logs verbosely (per-step, per-tensor) to stderr by
+// default. Route everything to a no-op so it doesn't flood logs / temp space.
+static void sd_silent_log(enum sd_log_level_t level, const char* text, void* data) {
+    (void)level;
+    (void)text;
+    (void)data;
+}
+
+static void sd_silent_progress(int step, int steps, float time, void* data) {
+    (void)step;
+    (void)steps;
+    (void)time;
+    (void)data;
+}
+
+SynapSD* synap_sd_create(const char* model_path, const char* taesd_path, int32_t n_threads, bool use_gpu) {
+    if (!model_path) { return nullptr; }
+
+    sd_set_log_callback(sd_silent_log, nullptr);
+    sd_set_progress_callback(sd_silent_progress, nullptr);
+
+    sd_ctx_params_t params;
+    sd_ctx_params_init(&params);
+    params.model_path = model_path;
+    if (taesd_path && taesd_path[0] != '\0') { params.taesd_path = taesd_path; }
+    params.n_threads = n_threads > 0 ? n_threads : 4;
+    // mmap avoids loading all weights into anonymous RAM. Flash attention is
+    // left at the init default (off): forcing it produced blank output on the
+    // CPU backend, and it is not reliably supported across SD backends.
+    params.enable_mmap = true;
+    // Backend selection: default resolves GPU-first, which is correct on an
+    // Apple-GPU device (Metal). On hosts whose only GPU is non-Apple (the
+    // Intel dev Mac, the simulator) Metal produces garbage, so pin to CPU.
+    params.backend = use_gpu ? nullptr : "CPU";
+
+    sd_ctx_t* ctx = new_sd_ctx(&params);
+    if (!ctx) { return nullptr; }
+
+    auto* sd = new SynapSD();
+    sd->ctx  = ctx;
+    return sd;
+}
+
+void synap_sd_free(SynapSD* sd) {
+    if (!sd) { return; }
+    if (sd->ctx) { free_sd_ctx(sd->ctx); }
+    delete sd;
+}
+
+uint8_t* synap_sd_generate(SynapSD* sd, const char* prompt, const char* negative_prompt, int32_t width, int32_t height,
+                           int32_t steps, float cfg_scale, int32_t sample_method, int64_t seed, int32_t* out_width,
+                           int32_t* out_height) {
+    if (!sd || !sd->ctx || !prompt) { return nullptr; }
+
+    sd_img_gen_params_t g;
+    sd_img_gen_params_init(&g);
+    g.prompt                         = prompt;
+    g.negative_prompt                = negative_prompt ? negative_prompt : "";
+    g.width                          = width;
+    g.height                         = height;
+    g.sample_params.sample_steps     = steps;
+    g.sample_params.guidance.txt_cfg = cfg_scale;
+    g.sample_params.sample_method    = static_cast<enum sample_method_t>(sample_method);
+    g.seed                           = seed;
+    g.batch_count                    = 1;
+
+    sd_image_t* images = generate_image(sd->ctx, &g);
+    if (!images || !images[0].data) {
+        if (images) { free_sd_images(images, 1); }
+        return nullptr;
+    }
+
+    const sd_image_t img = images[0];
+    const size_t bytes   = static_cast<size_t>(img.width) * img.height * img.channel;
+    // Normalize to 3-channel RGB for the caller.
+    uint8_t* out = static_cast<uint8_t*>(malloc(static_cast<size_t>(img.width) * img.height * 3));
+    if (out) {
+        if (img.channel == 3) {
+            memcpy(out, img.data, bytes);
+        } else {
+            for (uint32_t i = 0; i < img.width * img.height; ++i) {
+                out[i * 3 + 0] = img.data[i * img.channel + 0];
+                out[i * 3 + 1] = img.data[i * img.channel + (img.channel > 1 ? 1 : 0)];
+                out[i * 3 + 2] = img.data[i * img.channel + (img.channel > 2 ? 2 : 0)];
+            }
+        }
+        if (out_width) { *out_width = static_cast<int32_t>(img.width); }
+        if (out_height) { *out_height = static_cast<int32_t>(img.height); }
+    }
+    free_sd_images(images, 1);
+    return out;
+}
+
+void synap_sd_free_rgb(uint8_t* rgb) {
+    free(rgb);
+}

SynapLink/Core/Engine/synap_sd.h

@@ -0,0 +1,43 @@
+//
+//  synap_sd.h
+//  SynapLink
+//
+//  Pure-C bridge to stable-diffusion.cpp — the experimental on-device image
+//  generation specialist. Swift imports this via the bridging header; the
+//  sd/ggml C++ symbols stay inside sd.xcframework.
+//
+//  Output is raw RGB (width*height*3 bytes); Swift JPEG-encodes it.
+//
+
+#pragma once
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SynapSD SynapSD;
+
+/// Load a Stable Diffusion GGUF (full checkpoint). `taesd_path` is an optional
+/// tiny VAE for memory-light decoding (NULL to use the checkpoint's VAE).
+/// Returns NULL on failure.
+SynapSD* synap_sd_create(const char* model_path, const char* taesd_path, int32_t n_threads, bool use_gpu);
+
+void synap_sd_free(SynapSD* sd);
+
+/// Generate one image. `sample_method` is a sample_method_t raw value
+/// (0 = EULER, 1 = EULER_A, 9 = LCM). Returns a malloc'd RGB buffer of
+/// (*out_width * *out_height * 3) bytes — free with synap_sd_free_rgb — or
+/// NULL on failure.
+uint8_t* synap_sd_generate(SynapSD* sd, const char* prompt, const char* negative_prompt, int32_t width, int32_t height,
+                           int32_t steps, float cfg_scale, int32_t sample_method, int64_t seed, int32_t* out_width,
+                           int32_t* out_height);
+
+void synap_sd_free_rgb(uint8_t* rgb);
+
+#ifdef __cplusplus
+}
+#endif