From 40d22750e9dbef82e5e65cb76ce4eee383e333ce Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Sun, 2 Aug 2026 13:27:36 -0700 Subject: [PATCH 01/15] spike(plugins): pure plugin-system core + meme generator logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Foundation-only half of the plugin spike, all covered by `swift test`: * `PluginManifest` — id/name/version/symbol/entry kind + a network-hosts disclosure (the app is local-first, so a plugin that reaches out says so). Id validation is strict because the id becomes a path component under Application Support: traversal-shaped ids are refused before they are ever joined onto a URL. * `PluginDiscovery` — merges the compile-time registry with `~/Library/Application Support/OpenWhisp/Plugins//manifest.json`. Built-in ALWAYS wins an id collision so a writable directory can never shadow a reviewed in-repo plugin. External plugins are listed but never runnable — a manifest cannot promote itself. * `PluginEnablement` — the enabled set, default-OFF, on its own UserDefaults key rather than as new `@Published` state on AppState (MAK-32 ratchet is at zero headroom). Prunes ids that disappeared, so removing and reinstalling a plugin means re-consenting rather than silently restoring a surface. * `PluginRegistry` — the compile-time list of in-repo plugins. * Meme generator logic: `MemeAI` (prompt + a forgiving-about-packaging, strict-about-content JSON parser), `MemeTemplateMatcher` (local lexical match against the imgflip catalog, honest fallback when nothing scores), `MemeCaptionLayout` (uppercase, greedy wrap, shrink-to-fit). 58 new tests. `plugins/MemeGenerator/manifest.json` is checked in as the authored source of truth, with a test pinning it against the shipped literal. Co-Authored-By: Claude --- OpenWhisp/Services/MemeAI.swift | 171 +++++++++++ OpenWhisp/Services/MemeCaptionLayout.swift | 143 +++++++++ OpenWhisp/Services/MemeTemplateMatcher.swift | 157 ++++++++++ OpenWhisp/Services/PluginDiscovery.swift | 137 +++++++++ OpenWhisp/Services/PluginEnablement.swift | 86 ++++++ OpenWhisp/Services/PluginManifest.swift | 156 ++++++++++ OpenWhisp/Services/PluginRegistry.swift | 50 ++++ Package.swift | 12 +- .../MemeGeneratorTests.swift | 276 +++++++++++++++++ .../PluginSystemTests.swift | 281 ++++++++++++++++++ plugins/MemeGenerator/manifest.json | 9 + 11 files changed, 1477 insertions(+), 1 deletion(-) create mode 100644 OpenWhisp/Services/MemeAI.swift create mode 100644 OpenWhisp/Services/MemeCaptionLayout.swift create mode 100644 OpenWhisp/Services/MemeTemplateMatcher.swift create mode 100644 OpenWhisp/Services/PluginDiscovery.swift create mode 100644 OpenWhisp/Services/PluginEnablement.swift create mode 100644 OpenWhisp/Services/PluginManifest.swift create mode 100644 OpenWhisp/Services/PluginRegistry.swift create mode 100644 Tests/OpenWhispCoreTests/MemeGeneratorTests.swift create mode 100644 Tests/OpenWhispCoreTests/PluginSystemTests.swift create mode 100644 plugins/MemeGenerator/manifest.json diff --git a/OpenWhisp/Services/MemeAI.swift b/OpenWhisp/Services/MemeAI.swift new file mode 100644 index 0000000..41c5b63 --- /dev/null +++ b/OpenWhisp/Services/MemeAI.swift @@ -0,0 +1,171 @@ +import Foundation + +/// The pure rules behind the Meme Generator plugin's LLM step (spike). +/// +/// One round-trip turns a spoken description ("make me the distracted boyfriend one +/// where the guy is looking at Rust and his girlfriend is Python") into three fields: +/// a **template query** used to pick a base image, and the **top/bottom captions**. +/// +/// Foundation-only, so both halves — what we ask for and what we're willing to +/// accept back — are pinned by `swift test`. The app layer owns only the LLM +/// round-trip, the image fetch, and the drawing. +/// +/// The parser is deliberately forgiving about *packaging* and strict about +/// *content*: small local models wrap JSON in prose or code fences constantly, so we +/// dig the object out (`ScratchpadAI`'s posture), but a response missing the fields +/// is REJECTED rather than silently rendered as an empty meme. +public enum MemeAI { + + // MARK: - Request + + /// The instruction sent with the user's spoken description. + /// + /// Language note, same as the Scratchpad's AI prompts (PR #157): the captions + /// must stay in the user's language. Tiny local models translate their input + /// unprompted, and a meme dictated in Ukrainian coming back in English is a bug. + /// The `template_query` is the exception — it is matched against an + /// English-titled catalog, so it is pinned to English on purpose. + public static let prompt = """ + You turn a spoken description of a meme into the fields needed to build it. + + Reply with ONLY a JSON object, no preamble and no code fence, with exactly these keys: + {"template_query": "...", "top_text": "...", "bottom_text": "..."} + + Rules: + 1. "template_query" names the meme template in ENGLISH — the well-known name if \ + the description implies one (e.g. "drake hotline bling", "distracted boyfriend", \ + "two buttons"), otherwise two or three words describing the image that would fit. + 2. "top_text" and "bottom_text" are the caption lines, in the SAME LANGUAGE as \ + the description. Do not translate them. + 3. Keep each caption short — a few words, meme-style. Do not add quotation marks \ + around them. + 4. If the description only calls for one line, put it in "top_text" and leave \ + "bottom_text" as an empty string. + 5. Never invent a caption that contradicts the description. + """ + + /// Build the user payload for a description. Trimmed; the caller guarantees + /// non-empty (the Generate button is disabled otherwise). + public static func userPayload(description: String) -> String { + "Meme description:\n" + description.trimmingCharacters(in: .whitespacesAndNewlines) + } + + // MARK: - Response + + /// The three fields a successful round-trip yields. + public struct MemeSpec: Equatable, Sendable, Codable { + /// English-ish search phrase used to pick a template from the catalog. + public let templateQuery: String + /// Upper caption, in the user's language. May be empty. + public let topText: String + /// Lower caption, in the user's language. May be empty. + public let bottomText: String + + public init(templateQuery: String, topText: String, bottomText: String) { + self.templateQuery = templateQuery + self.topText = topText + self.bottomText = bottomText + } + + private enum CodingKeys: String, CodingKey { + case templateQuery = "template_query" + case topText = "top_text" + case bottomText = "bottom_text" + } + } + + /// Why a response was refused. + public enum Rejection: Error, Equatable, Sendable { + /// Nothing usable came back (empty / whitespace only). + case empty + /// No JSON object could be found in the response. + case notJSON + /// JSON parsed, but there was no template query and no caption at all — + /// rendering this would produce a blank image on a random template. + case missingFields + + public var reason: String { + switch self { + case .empty: return "the model returned nothing" + case .notJSON: return "the model didn't return the expected JSON" + case .missingFields: return "the model left the meme fields empty" + } + } + } + + /// Parse a raw completion into a `MemeSpec`. + /// + /// Accepts a bare object, a fenced ```json block, and an object embedded in + /// surrounding prose — all three are things small local models actually emit. + public static func parse(_ raw: String) -> Result { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return .failure(.empty) } + + guard let object = firstJSONObject(in: trimmed) else { return .failure(.notJSON) } + guard let data = object.data(using: .utf8), + let spec = try? JSONDecoder().decode(MemeSpec.self, from: data) + else { return .failure(.notJSON) } + + let cleaned = MemeSpec( + templateQuery: clean(spec.templateQuery), + topText: clean(spec.topText), + bottomText: clean(spec.bottomText)) + + // A spec with no query AND no captions is not a meme. One empty caption is + // fine (rule 4 in the prompt asks for exactly that on single-line memes). + if cleaned.templateQuery.isEmpty, cleaned.topText.isEmpty, cleaned.bottomText.isEmpty { + return .failure(.missingFields) + } + return .success(cleaned) + } + + /// Strip the wrapping models add around caption values: surrounding quotes and + /// stray whitespace. Keeps interior punctuation untouched. + private static func clean(_ value: String) -> String { + var s = value.trimmingCharacters(in: .whitespacesAndNewlines) + while s.count >= 2, + let first = s.first, let last = s.last, + (first == "\"" && last == "\"") || (first == "'" && last == "'") { + s = String(s.dropFirst().dropLast()).trimmingCharacters(in: .whitespacesAndNewlines) + } + return s + } + + /// Extract the first balanced `{...}` run from a string, ignoring braces that + /// appear inside JSON string literals (so a caption containing `{` can't end the + /// scan early). Returns nil when there is no balanced object. + private static func firstJSONObject(in text: String) -> String? { + var depth = 0 + var start: String.Index? + var inString = false + var escaped = false + + for index in text.indices { + let char = text[index] + + if inString { + if escaped { escaped = false } + else if char == "\\" { escaped = true } + else if char == "\"" { inString = false } + continue + } + + switch char { + case "\"": + inString = true + case "{": + if depth == 0 { start = index } + depth += 1 + case "}": + guard depth > 0 else { break } + depth -= 1 + if depth == 0, let start { + return String(text[start...index]) + } + default: + break + } + } + return nil + } +} diff --git a/OpenWhisp/Services/MemeCaptionLayout.swift b/OpenWhisp/Services/MemeCaptionLayout.swift new file mode 100644 index 0000000..7087a49 --- /dev/null +++ b/OpenWhisp/Services/MemeCaptionLayout.swift @@ -0,0 +1,143 @@ +import Foundation + +/// The pure text rules behind classic meme captioning (spike). +/// +/// Everything here is Foundation-only arithmetic and string work so `swift test` +/// pins it; the app layer owns only the actual CoreGraphics drawing. The split is +/// deliberate — line breaking and font sizing are where meme rendering actually goes +/// wrong (a long caption overflowing the image), and those are exactly the parts +/// that don't need a graphics context to verify. +public enum MemeCaptionLayout { + + /// Classic meme captions are UPPERCASE. Applied here rather than at the drawing + /// site so the transformation is tested and so the LLM's casing never leaks + /// through inconsistently. + /// + /// Uppercasing is locale-aware, which matters for the non-English captions the + /// prompt deliberately preserves. + public static func displayText(_ caption: String) -> String { + caption.trimmingCharacters(in: .whitespacesAndNewlines).uppercased() + } + + /// Break a caption into lines that each fit `maxWidth`, given a function that + /// measures a candidate line at the current font size. + /// + /// Greedy word wrapping: fill a line until the next word wouldn't fit. A single + /// word longer than `maxWidth` is NOT broken mid-word — it gets its own line and + /// the caller shrinks the font instead, which is what preserves readability + /// (hyphenating "AAAAAAAAAA" helps nobody). + /// + /// `measure` is injected so this stays pure: tests pass a deterministic + /// width-per-character stub, the app passes real font metrics. + public static func wrap( + _ text: String, + maxWidth: Double, + measure: (String) -> Double + ) -> [String] { + let words = text.split(separator: " ", omittingEmptySubsequences: true).map(String.init) + guard !words.isEmpty else { return [] } + + var lines: [String] = [] + var current = "" + + for word in words { + let candidate = current.isEmpty ? word : current + " " + word + if current.isEmpty || measure(candidate) <= maxWidth { + current = candidate + } else { + lines.append(current) + current = word + } + } + if !current.isEmpty { lines.append(current) } + return lines + } + + /// The result of fitting a caption into a box. + public struct Fit: Equatable, Sendable { + /// The wrapped lines to draw, already uppercased. + public let lines: [String] + /// The font size that made them fit. + public let fontSize: Double + + public init(lines: [String], fontSize: Double) { + self.lines = lines + self.fontSize = fontSize + } + } + + /// Shrink-to-fit: find the largest font size (stepping down from `maxFontSize`) + /// at which the wrapped caption fits within `maxWidth` × `maxHeight`. + /// + /// Meme captions must never overflow the image, but they also shouldn't be + /// needlessly tiny, so this walks DOWN from the ideal size and takes the first + /// size that fits rather than solving analytically (line count changes + /// discontinuously with font size, so there's no clean closed form). + /// + /// If even `minFontSize` overflows — a genuinely enormous caption — the result is + /// returned AT `minFontSize` anyway: clipping a too-long caption is a better + /// failure than rendering nothing, and the caller has already been told the text + /// is long. Honest degradation over silent emptiness. + /// + /// - Parameters: + /// - measure: `(text, fontSize) -> width` at that size. + /// - lineHeight: `fontSize -> line height`, so callers can pass real metrics. + public static func fit( + caption: String, + maxWidth: Double, + maxHeight: Double, + maxFontSize: Double, + minFontSize: Double, + step: Double = 2, + measure: (String, Double) -> Double, + lineHeight: (Double) -> Double + ) -> Fit { + let text = displayText(caption) + guard !text.isEmpty else { return Fit(lines: [], fontSize: maxFontSize) } + guard maxFontSize >= minFontSize, step > 0 else { + return Fit(lines: [text], fontSize: minFontSize) + } + + var size = maxFontSize + while size > minFontSize { + let lines = wrap(text, maxWidth: maxWidth) { measure($0, size) } + let totalHeight = Double(lines.count) * lineHeight(size) + let widest = lines.map { measure($0, size) }.max() ?? 0 + if totalHeight <= maxHeight && widest <= maxWidth { + return Fit(lines: lines, fontSize: size) + } + size -= step + } + + // Floor: wrap at the smallest allowed size and accept it. + let lines = wrap(text, maxWidth: maxWidth) { measure($0, minFontSize) } + return Fit(lines: lines, fontSize: minFontSize) + } + + // MARK: - Geometry + + /// Where a caption block sits on the image. + public enum Position: String, Equatable, Sendable { + case top + case bottom + } + + /// The share of the image height one caption block may occupy before it must + /// shrink. Classic memes keep captions in the top/bottom third. + public static let captionHeightShare: Double = 0.32 + + /// The horizontal inset applied to both sides, as a share of image width — the + /// caption never runs to the bezel. + public static let horizontalInsetShare: Double = 0.05 + + /// Ideal font size for an image height, before shrink-to-fit. Scaled to the + /// image so a 1200px template and a 400px one look the same when displayed. + public static func idealFontSize(imageHeight: Double) -> Double { + max(12, imageHeight * 0.11) + } + + /// Smallest acceptable font size for an image height. + public static func minimumFontSize(imageHeight: Double) -> Double { + max(8, imageHeight * 0.04) + } +} diff --git a/OpenWhisp/Services/MemeTemplateMatcher.swift b/OpenWhisp/Services/MemeTemplateMatcher.swift new file mode 100644 index 0000000..5f83764 --- /dev/null +++ b/OpenWhisp/Services/MemeTemplateMatcher.swift @@ -0,0 +1,157 @@ +import Foundation + +/// A meme template from the Imgflip public catalog (`https://api.imgflip.com/get_memes`). +/// +/// That endpoint is free, key-less, and read-only: it returns the ~100 most popular +/// templates as `{id, name, url, width, height, box_count}`. The plugin uses it ONLY +/// to find a base image — captioning happens locally with CoreGraphics, so no text, +/// no audio, and no LLM output is ever sent to imgflip. +public struct MemeTemplate: Equatable, Sendable, Codable, Identifiable { + public let id: String + /// Display name, e.g. "Distracted Boyfriend". English — this is what the LLM's + /// `template_query` is matched against. + public let name: String + /// Direct image URL (jpg/png) for the blank template. + public let url: String + public let width: Int + public let height: Int + + public init(id: String, name: String, url: String, width: Int, height: Int) { + self.id = id + self.name = name + self.url = url + self.width = width + self.height = height + } +} + +/// The Imgflip `get_memes` response envelope. +public struct MemeTemplateCatalogResponse: Decodable, Sendable { + public struct Payload: Decodable, Sendable { + public let memes: [MemeTemplate] + } + public let success: Bool + public let data: Payload? + + /// The templates, or empty when the API reported failure. + public var templates: [MemeTemplate] { success ? (data?.memes ?? []) : [] } +} + +/// Picks the template whose name best matches the LLM's `template_query` (spike). +/// +/// Matching is LOCAL and lexical — the catalog is ~100 short English names, so a +/// token-overlap score beats anything heavier and keeps the whole decision pure and +/// testable. No second network call, no embedding model. +/// +/// Scoring, highest first: +/// 1. **Exact** name match (case/punctuation-insensitive) — 1000. +/// 2. **Full-phrase containment** either way ("drake" ⊂ "Drake Hotline Bling") — +/// 500 plus a closeness bonus, so the shortest containing name wins. +/// 3. **Token overlap** — 100 per query token found in the name, plus a small bonus +/// for a tighter name, so "Two Buttons" beats "Two Buttons But Worse" on "two buttons". +/// +/// Ties break on the catalog's own order, which is popularity-ranked — the more +/// famous template is the better guess. +public enum MemeTemplateMatcher { + + /// The minimum score worth accepting. Below this we'd be picking essentially at + /// random, and a confidently-wrong template is worse than an honest fallback, so + /// the caller falls back to the catalog's most popular entry instead. + public static let minimumScore = 100 + + /// Find the best template for a query. Returns nil only for an empty catalog. + /// + /// A query that matches nothing yields the catalog's FIRST entry (most popular) + /// rather than nil: the user dictated a meme and should get a meme. The caller + /// surfaces that this was a fallback. + public static func bestMatch( + for query: String, in catalog: [MemeTemplate] + ) -> Match? { + guard !catalog.isEmpty else { return nil } + + let queryTokens = tokens(in: query) + guard !queryTokens.isEmpty else { + return Match(template: catalog[0], score: 0, isFallback: true) + } + + var best: (template: MemeTemplate, score: Int)? + + for template in catalog { + let score = score(query: query, queryTokens: queryTokens, name: template.name) + // Strictly greater keeps the earlier (more popular) template on a tie. + if score > (best?.score ?? Int.min) { + best = (template, score) + } + } + + guard let best, best.score >= minimumScore else { + return Match(template: catalog[0], score: best?.score ?? 0, isFallback: true) + } + return Match(template: best.template, score: best.score, isFallback: false) + } + + /// The chosen template plus why. + public struct Match: Equatable, Sendable { + public let template: MemeTemplate + public let score: Int + /// True when nothing scored well enough and the most-popular template was + /// substituted — the UI says so rather than pretending it understood. + public let isFallback: Bool + + public init(template: MemeTemplate, score: Int, isFallback: Bool) { + self.template = template + self.score = score + self.isFallback = isFallback + } + } + + // MARK: - Scoring + + static func score(query: String, queryTokens: [String], name: String) -> Int { + let normalizedQuery = normalize(query) + let normalizedName = normalize(name) + + if normalizedQuery == normalizedName { return 1000 } + + let nameTokens = tokens(in: name) + + // Whole-phrase containment in either direction. The closeness bonus favors + // the shortest name that still contains the query. + if !normalizedQuery.isEmpty, + normalizedName.contains(normalizedQuery) || normalizedQuery.contains(normalizedName) { + let lengthGap = abs(normalizedName.count - normalizedQuery.count) + return 500 + max(0, 50 - lengthGap) + } + + let nameTokenSet = Set(nameTokens) + let overlap = queryTokens.filter { nameTokenSet.contains($0) }.count + guard overlap > 0 else { return 0 } + + // Prefer a name that is mostly the matched tokens over one that buries them + // among many others. + let tightness = max(0, 20 - (nameTokens.count - overlap) * 4) + return overlap * 100 + tightness + } + + /// Lowercase, strip punctuation, collapse whitespace. + static func normalize(_ value: String) -> String { + let folded = value.folding(options: [.diacriticInsensitive, .caseInsensitive], + locale: Locale(identifier: "en_US")) + let scalars = folded.unicodeScalars.map { scalar -> Character in + CharacterSet.alphanumerics.contains(scalar) ? Character(scalar) : " " + } + return String(scalars) + .split(separator: " ", omittingEmptySubsequences: true) + .joined(separator: " ") + } + + /// Meaningful tokens, with English stopwords dropped so "the drake meme" and + /// "drake" score the same. + static func tokens(in value: String) -> [String] { + let stopwords: Set = ["the", "a", "an", "of", "and", "meme", "guy", "man"] + return normalize(value) + .split(separator: " ") + .map(String.init) + .filter { !stopwords.contains($0) && $0.count > 1 } + } +} diff --git a/OpenWhisp/Services/PluginDiscovery.swift b/OpenWhisp/Services/PluginDiscovery.swift new file mode 100644 index 0000000..5afbd9c --- /dev/null +++ b/OpenWhisp/Services/PluginDiscovery.swift @@ -0,0 +1,137 @@ +import Foundation + +/// Finds the plugins available to the host and decides which ones win (spike). +/// +/// Two sources, merged with a fixed precedence: +/// +/// 1. **Built-in** — the compile-time list handed in by `PluginRegistry`. These are +/// in-repo plugins under `plugins//`, reviewed and maintained alongside the +/// app, and are the only ones that can actually RUN in the spike. +/// 2. **External** — `~/Library/Application Support/OpenWhisp/Plugins//manifest.json`. +/// Discovered and listed so the pane can show the user what's on disk, but flagged +/// non-runnable (no loader exists). +/// +/// **Built-in always wins a conflicting id.** A dropped-in folder must never be able +/// to shadow a reviewed in-repo plugin — that would turn a writable directory into +/// code-substitution against a mic-and-Accessibility-entitled app. The spike can't +/// execute external plugins at all, so this is belt-and-braces today, but the +/// precedence is the part worth pinning now because a future loader inherits it. +/// +/// Foundation-only: every rule here (validation rejects, precedence, sort order, +/// malformed-JSON tolerance) is covered by `swift test`. +public enum PluginDiscovery { + + /// A discovered plugin plus where it came from. + public struct Discovered: Equatable, Sendable, Identifiable { + public let manifest: PluginManifest + public let source: Source + + public var id: String { manifest.id } + + public init(manifest: PluginManifest, source: Source) { + self.manifest = manifest + self.source = source + } + + /// Whether the host can actually open this plugin. External plugins are + /// listed but never runnable in the spike, regardless of what their manifest + /// claims its entry kind is — a manifest cannot promote itself. + public var isRunnable: Bool { + source == .builtIn && manifest.entry.isRunnable + } + + /// Why this plugin can't run, if it can't. + public var unavailableReason: String? { + if source == .external { + return "Installed plugins can't be loaded yet — this prototype only runs plugins that ship with the app." + } + return manifest.entry.unavailableReason + } + } + + public enum Source: String, Equatable, Sendable { + /// Compiled into the app from `plugins//`. + case builtIn + /// Found on disk under Application Support. + case external + } + + /// The directory external plugins are discovered from. + public static func externalPluginsDirectory( + applicationSupport: URL + ) -> URL { + applicationSupport + .appendingPathComponent("OpenWhisp", isDirectory: true) + .appendingPathComponent("Plugins", isDirectory: true) + } + + /// Merge built-in and external manifests into the host's plugin list. + /// + /// - Invalid manifests are dropped (a plugin with no name/symbol or a + /// traversal-shaped id is not listable). + /// - Built-in beats external on id collision. + /// - Result is sorted by display name, then id, so the pane's order is stable + /// across launches and independent of filesystem enumeration order. + public static func merge( + builtIn: [PluginManifest], + external: [PluginManifest] + ) -> [Discovered] { + var byID: [String: Discovered] = [:] + + // External first so a built-in with the same id overwrites it. + for manifest in external where manifest.isValid { + byID[manifest.id] = Discovered(manifest: manifest, source: .external) + } + for manifest in builtIn where manifest.isValid { + byID[manifest.id] = Discovered(manifest: manifest, source: .builtIn) + } + + return byID.values.sorted { + if $0.manifest.name != $1.manifest.name { + return $0.manifest.name.localizedCaseInsensitiveCompare($1.manifest.name) == .orderedAscending + } + return $0.manifest.id < $1.manifest.id + } + } + + /// Read every `//manifest.json` under an external plugins directory. + /// + /// Tolerant by design: a malformed or unreadable manifest is SKIPPED, never + /// thrown — one bad third-party folder must not stop the Plugins pane from + /// listing the good ones (the same fail-soft posture `JSONStore` takes). + /// A manifest whose `id` disagrees with its containing directory name is + /// rejected, so a folder can't claim to be a different plugin than where it sits. + public static func loadExternalManifests( + in directory: URL, + fileManager: FileManager = .default + ) -> [PluginManifest] { + guard let entries = try? fileManager.contentsOfDirectory( + at: directory, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + ) else { + return [] + } + + let decoder = JSONDecoder() + var manifests: [PluginManifest] = [] + + for entry in entries { + var isDirectory: ObjCBool = false + guard fileManager.fileExists(atPath: entry.path, isDirectory: &isDirectory), + isDirectory.boolValue else { continue } + + let manifestURL = entry.appendingPathComponent("manifest.json") + guard let data = try? Data(contentsOf: manifestURL), + let manifest = try? decoder.decode(PluginManifest.self, from: data), + manifest.isValid, + // The directory name is the authority on identity. + manifest.id == entry.lastPathComponent + else { continue } + + manifests.append(manifest) + } + + return manifests + } +} diff --git a/OpenWhisp/Services/PluginEnablement.swift b/OpenWhisp/Services/PluginEnablement.swift new file mode 100644 index 0000000..225b394 --- /dev/null +++ b/OpenWhisp/Services/PluginEnablement.swift @@ -0,0 +1,86 @@ +import Foundation + +/// Which plugins the user has turned on (spike). +/// +/// Plugins are OPTIONAL and **off by default**: installing the app must not silently +/// add surfaces, network calls, or menu rows the user never asked for. A plugin only +/// appears as a tab/menu entry after it is explicitly enabled in Settings → Plugins. +/// +/// The enabled set lives on this type's OWN UserDefaults key rather than as a +/// `@Published` property on AppState — the MAK-32 AppState LOC ratchet is at its +/// budget, and the established dodge (see `ScratchpadWindowController`'s AI-model +/// overrides and `TranslationPreviewController`) is to keep new storage on the +/// feature's own type. `PluginHost` owns the observable wrapper for the UI. +/// +/// Foundation-only, and the store is injected, so `swift test` pins the default-off +/// rule, the round-trip, and the sanitization without touching real user defaults. +public struct PluginEnablement: Equatable, Sendable { + + /// The single defaults key holding the enabled ids (an array of strings; a Set + /// isn't a plist type). Namespaced like the app's other feature keys. + public static let defaultsKey = "openwhisp.plugins.enabledIDs" + + private var enabled: Set + + public init(enabled: Set = []) { + self.enabled = enabled + } + + /// Whether a plugin id is turned on. Unknown ids are off — the default-off rule. + public func isEnabled(_ id: String) -> Bool { enabled.contains(id) } + + /// The enabled ids, sorted for a stable persisted representation (so writing + /// unchanged state can't churn the defaults file). + public var enabledIDs: [String] { enabled.sorted() } + + public mutating func setEnabled(_ isEnabled: Bool, for id: String) { + if isEnabled { enabled.insert(id) } else { enabled.remove(id) } + } + + /// Drop ids that no longer correspond to an available plugin. + /// + /// Without this, uninstalling a plugin and reinstalling it later would silently + /// come back ENABLED, re-adding a surface (and possibly network access) the user + /// last saw disappear. Prune on load so re-appearing means re-consenting. + public mutating func prune(toAvailable availableIDs: Set) { + enabled.formIntersection(availableIDs) + } + + /// The subset of `discovered` the host should surface as active tabs/menu rows: + /// enabled AND actually runnable. An external plugin can be toggled on in the + /// pane, but the spike still won't open a window for it — being enabled is not + /// the same as being loadable, and conflating the two is how a prototype starts + /// lying about what it can do. + public func activePlugins( + from discovered: [PluginDiscovery.Discovered] + ) -> [PluginDiscovery.Discovered] { + discovered.filter { isEnabled($0.id) && $0.isRunnable } + } + + // MARK: - Persistence + + /// The minimal slice of UserDefaults this store needs, so tests can supply a + /// dictionary-backed fake instead of polluting the real domain. + public protocol Store: AnyObject { + func stringArray(forKey key: String) -> [String]? + func set(_ value: Any?, forKey key: String) + } + + /// Load the enabled set, pruned to what's actually available. + public static func load( + from store: Store, + availableIDs: Set + ) -> PluginEnablement { + var state = PluginEnablement( + enabled: Set(store.stringArray(forKey: defaultsKey) ?? [])) + state.prune(toAvailable: availableIDs) + return state + } + + /// Persist the enabled set. + public func save(to store: Store) { + store.set(enabledIDs, forKey: Self.defaultsKey) + } +} + +extension UserDefaults: PluginEnablement.Store {} diff --git a/OpenWhisp/Services/PluginManifest.swift b/OpenWhisp/Services/PluginManifest.swift new file mode 100644 index 0000000..3fb6472 --- /dev/null +++ b/OpenWhisp/Services/PluginManifest.swift @@ -0,0 +1,156 @@ +import Foundation + +/// The declarative description of an OpenWhisp plugin (spike: `spike/plugin-system`). +/// +/// Plugins are OPTIONAL surfaces layered on top of the app: each one contributes a +/// tab/window of its own plus its own configuration, and none of them are part of +/// the base dictation pipeline. A manifest is the contract between a plugin and the +/// host — enough for the host to LIST a plugin (name, icon, version, what it needs) +/// without knowing anything about what the plugin does. +/// +/// Two provenances, both surfaced identically in the UI: +/// +/// - **Built-in** — an in-repo plugin under `plugins//`, compiled INTO the app +/// and declared in the compile-time `PluginRegistry`. Its manifest ships as a +/// literal so it can never go missing at runtime. This is what the spike uses. +/// - **External** — a manifest discovered on disk at +/// `~/Library/Application Support/OpenWhisp/Plugins//manifest.json`. The +/// spike DISCOVERS and LISTS these but cannot execute them: there is no loader. +/// True out-of-process/dylib loading is future work (see `PluginEntryKind`). +/// +/// Foundation-only, so the manifest schema, its validation, and the discovery +/// merge/precedence rules are all pinned by `swift test`. +public struct PluginManifest: Codable, Equatable, Sendable, Identifiable { + + /// Reverse-DNS-ish stable identifier, e.g. `meme-generator`. Also the on-disk + /// directory name and the key the enabled-set is stored under, so it must stay + /// stable across versions — renaming an id silently disables the plugin. + public let id: String + + /// Human-readable name shown in the Plugins pane and the menu-bar submenu. + public let name: String + + /// Semver-ish display string. The host does not currently gate on it; it exists + /// so the pane can show what's installed and so a future loader has something to + /// compare against a compatibility floor. + public let version: String + + /// One-line description of what the plugin does, shown under the name. + public let summary: String + + /// SF Symbol name for every surface that renders this plugin (pane row, menu + /// item, window). Menu rows in this app always carry a symbol — never an emoji + /// inlined into the title. + public let symbol: String + + /// How the host is expected to run this plugin. + public let entry: PluginEntryKind + + /// Whether the plugin talks to the network, and to whom. The app is local-first, + /// so a plugin that reaches out MUST say so: the Plugins pane renders this + /// verbatim as a disclosure next to the enable toggle. Empty = fully local. + /// + /// This is an HONEST LABEL, not a sandbox — nothing enforces it in the spike. + /// A real third-party plugin system would need the enforcement to live outside + /// the plugin's own manifest (see the PR's security notes). + public let networkHosts: [String] + + public init( + id: String, + name: String, + version: String, + summary: String, + symbol: String, + entry: PluginEntryKind, + networkHosts: [String] = [] + ) { + self.id = id + self.name = name + self.version = version + self.summary = summary + self.symbol = symbol + self.entry = entry + self.networkHosts = networkHosts + } + + /// Whether this plugin uses the network at all — drives the pane's disclosure row. + public var usesNetwork: Bool { !networkHosts.isEmpty } + + /// The disclosure sentence shown in the Plugins pane. Kept here (not in the view) + /// so `swift test` pins the wording of a privacy-facing string. + public var networkDisclosure: String? { + guard usesNetwork else { return nil } + return "Connects to \(networkHosts.joined(separator: ", ")) when you use it." + } + + // MARK: - Validation + + /// Why a decoded manifest was rejected. + public enum ValidationError: Equatable, Sendable { + case emptyID + case invalidID(String) + case emptyName + case emptySymbol + } + + /// Characters allowed in an id: lowercase alphanumerics plus `-` and `.`. + /// Deliberately strict — the id becomes a PATH COMPONENT under Application + /// Support, so anything that could traverse (`/`, `..`, NUL) must be rejected + /// before it is ever joined onto a directory URL. + private static let allowedIDCharacters = CharacterSet( + charactersIn: "abcdefghijklmnopqrstuvwxyz0123456789-.") + + /// Validate a manifest's invariants. Returns `nil` when the manifest is usable. + public func validate() -> ValidationError? { + if id.isEmpty { return .emptyID } + if id.unicodeScalars.contains(where: { !Self.allowedIDCharacters.contains($0) }) { + return .invalidID(id) + } + // `.` is allowed inside an id (reverse-DNS style) but an id that is ONLY + // dots — `.` / `..` — is a path-traversal component, never a plugin. + if id.allSatisfy({ $0 == "." }) { return .invalidID(id) } + if name.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { return .emptyName } + if symbol.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { return .emptySymbol } + return nil + } + + public var isValid: Bool { validate() == nil } +} + +/// How a plugin's code is expected to be executed by the host. +/// +/// The spike implements exactly ONE of these (`builtIn`). The other cases exist so +/// the manifest schema doesn't have to change when a real loader lands, and so the +/// Plugins pane can honestly tell the user that a discovered external plugin is +/// listed but NOT runnable. +public enum PluginEntryKind: String, Codable, Equatable, Sendable, CaseIterable { + + /// Compiled into the app from `plugins//` and declared in `PluginRegistry`. + /// The only kind the host can actually run today. + case builtIn + + /// A dynamically-loaded bundle. NOT IMPLEMENTED — loading third-party native + /// code into a signed, entitled, mic-and-Accessibility-holding app inherits every + /// one of those entitlements, so this needs a real security story first. + case dynamicLibrary + + /// An out-of-process helper spoken to over the existing agent-bridge/MCP wire. + /// NOT IMPLEMENTED — the likeliest real answer (it sandboxes naturally), but out + /// of scope for the spike. + case externalProcess + + /// Whether the host can run this kind of plugin today. + public var isRunnable: Bool { self == .builtIn } + + /// Why a non-runnable plugin can't run, shown in the Plugins pane. + public var unavailableReason: String? { + switch self { + case .builtIn: + return nil + case .dynamicLibrary: + return "Loadable plugin bundles aren't supported yet — this prototype only runs plugins compiled into the app." + case .externalProcess: + return "Out-of-process plugins aren't supported yet — this prototype only runs plugins compiled into the app." + } + } +} diff --git a/OpenWhisp/Services/PluginRegistry.swift b/OpenWhisp/Services/PluginRegistry.swift new file mode 100644 index 0000000..18a8dfb --- /dev/null +++ b/OpenWhisp/Services/PluginRegistry.swift @@ -0,0 +1,50 @@ +import Foundation + +/// The compile-time list of in-repo plugins (spike). +/// +/// In-repo plugins live under `plugins//`, are reviewed and maintained in this +/// repository, and are compiled INTO the app. This registry is the single place that +/// knows they exist — the host asks it for manifests and never hardcodes a plugin id +/// anywhere else. +/// +/// Why a compile-time list and not a loader: the app holds Accessibility, microphone, +/// and clipboard rights, so running third-party native code in-process would inherit +/// all of them (`docs/ROADMAP.md` §6 rejects SwiftPM/dylib plugins outright for this +/// reason). Compiling reviewed plugins in keeps the spike honest about what it is — +/// a UI/architecture prototype for the plugin *surface*, not a third-party code +/// distribution mechanism. The manifest schema is forward-compatible with an +/// out-of-process loader (`PluginEntryKind.externalProcess`), which is the likelier +/// real answer. +/// +/// Manifests live here as literals rather than being parsed from +/// `plugins//manifest.json` at runtime. The JSON files are checked in as the +/// authored source of truth and as the schema example for external plugins, but a +/// built-in plugin must not be able to go missing because a resource didn't get +/// copied into the bundle. `PluginRegistryTests` asserts the two agree. +public enum PluginRegistry { + + /// Every plugin compiled into this build. + /// + /// The meme generator is the spike's first and only plugin. Adding a second one + /// means adding a manifest here and a window in the host's `open` switch — the + /// two places a built-in plugin is wired. + public static let builtInManifests: [PluginManifest] = [ + memeGenerator + ] + + /// The Meme Generator plugin (`plugins/MemeGenerator/`). + /// + /// Voice-first: dictate a description of the meme you want, and it picks a + /// template, writes the captions, and renders them locally. + public static let memeGenerator = PluginManifest( + id: "meme-generator", + name: "Meme Generator", + version: "0.1.0", + summary: "Dictate a meme description — the AI picks a template and writes the captions.", + symbol: "photo.badge.plus", + entry: .builtIn, + // Template images are downloaded from imgflip's public, key-less catalog. + // Captions are rendered locally, so no text ever leaves the Mac. + networkHosts: ["api.imgflip.com", "i.imgflip.com"] + ) +} diff --git a/Package.swift b/Package.swift index b9b8eba..8193f99 100644 --- a/Package.swift +++ b/Package.swift @@ -165,7 +165,17 @@ let package = Package( "TranscriptInterleaver.swift", "MeetingOrphanScan.swift", "StreamOverlay.swift", - "StreamIngest.swift" + "StreamIngest.swift", + // Plugin system spike (spike/plugin-system) + the in-repo meme plugin's + // pure rules. Sources live under plugins/MemeGenerator/ for the app-layer + // UI; the testable logic sits here so `swift test` covers it. + "PluginManifest.swift", + "PluginDiscovery.swift", + "PluginEnablement.swift", + "PluginRegistry.swift", + "MemeAI.swift", + "MemeTemplateMatcher.swift", + "MemeCaptionLayout.swift" ] ), // The Agent Bridge client + MCP stdio adapter. A library (not folded into diff --git a/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift b/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift new file mode 100644 index 0000000..e6c660a --- /dev/null +++ b/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift @@ -0,0 +1,276 @@ +import XCTest +@testable import OpenWhispCore + +/// Covers the Meme Generator plugin's pure layer (spike/plugin-system): the LLM +/// response parser, the template matcher, and the caption layout rules. +final class MemeGeneratorTests: XCTestCase { + + // MARK: - MemeAI.parse + + func testParsesBareJSONObject() { + let result = MemeAI.parse(""" + {"template_query":"drake hotline bling","top_text":"writing tests","bottom_text":"shipping on friday"} + """) + guard case .success(let spec) = result else { return XCTFail("expected success, got \(result)") } + XCTAssertEqual(spec.templateQuery, "drake hotline bling") + XCTAssertEqual(spec.topText, "writing tests") + XCTAssertEqual(spec.bottomText, "shipping on friday") + } + + /// Small local models fence their JSON constantly; that's packaging, not content. + func testParsesFencedJSON() { + let result = MemeAI.parse(""" + Sure! Here you go: + + ```json + {"template_query":"two buttons","top_text":"A","bottom_text":"B"} + ``` + """) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.templateQuery, "two buttons") + XCTAssertEqual(spec.topText, "A") + } + + func testParsesJSONEmbeddedInProse() { + let result = MemeAI.parse( + "I think this works: {\"template_query\":\"success kid\",\"top_text\":\"x\",\"bottom_text\":\"y\"} — enjoy!") + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.templateQuery, "success kid") + } + + /// A caption containing a brace must not end the object scan early. + func testBracesInsideStringValuesDoNotTruncateTheScan() { + let result = MemeAI.parse( + "{\"template_query\":\"code\",\"top_text\":\"when you write {\",\"bottom_text\":\"and forget }\"}") + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.topText, "when you write {") + XCTAssertEqual(spec.bottomText, "and forget }") + } + + func testStripsSurroundingQuotesFromCaptions() { + let result = MemeAI.parse( + "{\"template_query\":\"x\",\"top_text\":\"\\\"quoted\\\"\",\"bottom_text\":\"'single'\"}") + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.topText, "quoted") + XCTAssertEqual(spec.bottomText, "single") + } + + /// Rule 4 in the prompt asks for exactly this on single-line memes. + func testOneEmptyCaptionIsAccepted() { + let result = MemeAI.parse( + "{\"template_query\":\"success kid\",\"top_text\":\"it compiled\",\"bottom_text\":\"\"}") + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.bottomText, "") + } + + func testEmptyResponseIsRejected() { + XCTAssertEqual(rejection(MemeAI.parse(" \n ")), .empty) + } + + func testNonJSONResponseIsRejected() { + XCTAssertEqual(rejection(MemeAI.parse("I'm sorry, I can't help with that.")), .notJSON) + } + + func testUnbalancedBracesAreRejected() { + XCTAssertEqual(rejection(MemeAI.parse("{\"template_query\": \"x\"")), .notJSON) + } + + /// An all-empty spec would render a blank caption on an arbitrary template — + /// worse than an honest error. + func testAllEmptyFieldsAreRejected() { + XCTAssertEqual( + rejection(MemeAI.parse("{\"template_query\":\"\",\"top_text\":\"\",\"bottom_text\":\"\"}")), + .missingFields) + } + + func testRejectionReasonsAreHumanReadable() { + for r in [MemeAI.Rejection.empty, .notJSON, .missingFields] { + XCTAssertFalse(r.reason.isEmpty) + } + } + + /// The prompt must keep captions in the user's language (PR #157 guard) while + /// pinning the template query to English for catalog matching. + func testPromptPinsCaptionLanguageAndEnglishTemplateQuery() { + let prompt = MemeAI.prompt.lowercased() + XCTAssertTrue(prompt.contains("same language")) + XCTAssertTrue(prompt.contains("do not translate")) + XCTAssertTrue(prompt.contains("english")) + } + + func testUserPayloadTrimsAndLabelsTheDescription() { + XCTAssertEqual( + MemeAI.userPayload(description: " a cat "), + "Meme description:\na cat") + } + + private func rejection(_ result: Result) -> MemeAI.Rejection? { + if case .failure(let r) = result { return r } + return nil + } + + // MARK: - Template matching + + private let catalog: [MemeTemplate] = [ + MemeTemplate(id: "1", name: "Drake Hotline Bling", url: "u1", width: 1200, height: 1200), + MemeTemplate(id: "2", name: "Distracted Boyfriend", url: "u2", width: 1200, height: 800), + MemeTemplate(id: "3", name: "Two Buttons", url: "u3", width: 600, height: 908), + MemeTemplate(id: "4", name: "Success Kid", url: "u4", width: 500, height: 500), + ] + + func testExactNameMatchWins() { + let match = MemeTemplateMatcher.bestMatch(for: "Two Buttons", in: catalog) + XCTAssertEqual(match?.template.id, "3") + XCTAssertFalse(match?.isFallback ?? true) + } + + func testMatchIsCaseAndPunctuationInsensitive() { + XCTAssertEqual( + MemeTemplateMatcher.bestMatch(for: "two-buttons!", in: catalog)?.template.id, "3") + } + + /// A partial name the user actually says ("the drake one") must find the template. + func testPartialPhraseMatches() { + XCTAssertEqual( + MemeTemplateMatcher.bestMatch(for: "drake", in: catalog)?.template.id, "1") + XCTAssertEqual( + MemeTemplateMatcher.bestMatch(for: "distracted boyfriend", in: catalog)?.template.id, "2") + } + + func testStopwordsAndTheWordMemeAreIgnored() { + XCTAssertEqual( + MemeTemplateMatcher.bestMatch(for: "the success kid meme", in: catalog)?.template.id, "4") + } + + /// A confidently-wrong template is worse than an admitted guess. + func testUnmatchableQueryFallsBackToMostPopularAndSaysSo() { + let match = MemeTemplateMatcher.bestMatch(for: "zzzz qqqq", in: catalog) + XCTAssertEqual(match?.template.id, "1", "falls back to the catalog's first (most popular) entry") + XCTAssertTrue(match?.isFallback ?? false) + } + + func testEmptyQueryFallsBack() { + let match = MemeTemplateMatcher.bestMatch(for: " ", in: catalog) + XCTAssertTrue(match?.isFallback ?? false) + } + + func testEmptyCatalogYieldsNoMatch() { + XCTAssertNil(MemeTemplateMatcher.bestMatch(for: "drake", in: [])) + } + + /// Ties break on catalog order, which imgflip returns popularity-ranked. + func testTieBreaksTowardTheMorePopularTemplate() { + let tied = [ + MemeTemplate(id: "popular", name: "Angry Cat", url: "u", width: 10, height: 10), + MemeTemplate(id: "less", name: "Angry Dog", url: "u", width: 10, height: 10), + ] + XCTAssertEqual(MemeTemplateMatcher.bestMatch(for: "angry", in: tied)?.template.id, "popular") + } + + func testCatalogResponseDecodesImgflipShape() throws { + let json = """ + {"success":true,"data":{"memes":[ + {"id":"181913649","name":"Drake Hotline Bling","url":"https://i.imgflip.com/30b1gx.jpg", + "width":1200,"height":1200,"box_count":2}]}} + """ + let decoded = try JSONDecoder().decode( + MemeTemplateCatalogResponse.self, from: Data(json.utf8)) + XCTAssertEqual(decoded.templates.count, 1) + XCTAssertEqual(decoded.templates[0].name, "Drake Hotline Bling") + } + + func testFailedCatalogResponseYieldsNoTemplates() throws { + let decoded = try JSONDecoder().decode( + MemeTemplateCatalogResponse.self, from: Data("{\"success\":false}".utf8)) + XCTAssertEqual(decoded.templates.count, 0) + } + + // MARK: - Caption layout + + /// Deterministic metrics: every character is `size * 0.5` wide. + private func measure(_ text: String, _ size: Double) -> Double { + Double(text.count) * size * 0.5 + } + private func lineHeight(_ size: Double) -> Double { size * 1.2 } + + func testCaptionsAreUppercased() { + XCTAssertEqual(MemeCaptionLayout.displayText(" hello there "), "HELLO THERE") + } + + func testWrapBreaksOnWordBoundaries() { + let lines = MemeCaptionLayout.wrap("one two three four", maxWidth: 10) { + Double($0.count) + } + XCTAssertEqual(lines, ["one two", "three four"]) + } + + /// A single over-long word gets its own line rather than being cut mid-word — + /// the font shrinks instead, which is what stays readable. + func testOverlongWordIsNotBrokenMidWord() { + let lines = MemeCaptionLayout.wrap("supercalifragilistic", maxWidth: 5) { + Double($0.count) + } + XCTAssertEqual(lines, ["supercalifragilistic"]) + } + + func testWrapOfEmptyTextYieldsNoLines() { + XCTAssertEqual(MemeCaptionLayout.wrap(" ", maxWidth: 100) { Double($0.count) }, []) + } + + func testShortCaptionKeepsTheIdealFontSize() { + let fit = MemeCaptionLayout.fit( + caption: "yes", maxWidth: 400, maxHeight: 200, + maxFontSize: 40, minFontSize: 10, + measure: measure, lineHeight: lineHeight) + XCTAssertEqual(fit.fontSize, 40) + XCTAssertEqual(fit.lines, ["YES"]) + } + + /// The whole point: a long caption must shrink rather than overflow the image. + func testLongCaptionShrinksToFit() { + let long = "this is a very long meme caption that will absolutely not fit on one line" + let fit = MemeCaptionLayout.fit( + caption: long, maxWidth: 300, maxHeight: 120, + maxFontSize: 48, minFontSize: 8, + measure: measure, lineHeight: lineHeight) + + XCTAssertLessThan(fit.fontSize, 48, "should have shrunk") + let widest = fit.lines.map { measure($0, fit.fontSize) }.max() ?? 0 + XCTAssertLessThanOrEqual(widest, 300) + XCTAssertLessThanOrEqual(Double(fit.lines.count) * lineHeight(fit.fontSize), 120) + } + + /// Clipping an absurd caption beats rendering nothing at all. + func testImpossibleCaptionDegradesToMinimumFontRatherThanEmpty() { + let fit = MemeCaptionLayout.fit( + caption: String(repeating: "word ", count: 400), + maxWidth: 50, maxHeight: 20, + maxFontSize: 40, minFontSize: 10, + measure: measure, lineHeight: lineHeight) + XCTAssertEqual(fit.fontSize, 10) + XCTAssertFalse(fit.lines.isEmpty) + } + + func testEmptyCaptionProducesNoLines() { + let fit = MemeCaptionLayout.fit( + caption: "", maxWidth: 300, maxHeight: 100, + maxFontSize: 40, minFontSize: 10, + measure: measure, lineHeight: lineHeight) + XCTAssertEqual(fit.lines, []) + } + + /// Font sizes scale with the image so a 1200px and a 400px template look alike. + func testFontSizesScaleWithImageHeight() { + XCTAssertGreaterThan( + MemeCaptionLayout.idealFontSize(imageHeight: 1200), + MemeCaptionLayout.idealFontSize(imageHeight: 400)) + XCTAssertGreaterThan( + MemeCaptionLayout.idealFontSize(imageHeight: 500), + MemeCaptionLayout.minimumFontSize(imageHeight: 500)) + } + + func testFontSizesHaveAbsoluteFloors() { + XCTAssertGreaterThanOrEqual(MemeCaptionLayout.idealFontSize(imageHeight: 1), 12) + XCTAssertGreaterThanOrEqual(MemeCaptionLayout.minimumFontSize(imageHeight: 1), 8) + } +} diff --git a/Tests/OpenWhispCoreTests/PluginSystemTests.swift b/Tests/OpenWhispCoreTests/PluginSystemTests.swift new file mode 100644 index 0000000..5ac50b6 --- /dev/null +++ b/Tests/OpenWhispCoreTests/PluginSystemTests.swift @@ -0,0 +1,281 @@ +import XCTest +@testable import OpenWhispCore + +/// Covers the plugin system's pure layer (spike/plugin-system): manifest validation, +/// discovery precedence, and the enabled-set store. +final class PluginSystemTests: XCTestCase { + + // MARK: - Helpers + + private func manifest( + id: String = "demo", + name: String = "Demo", + symbol: String = "puzzlepiece.extension", + entry: PluginEntryKind = .builtIn, + networkHosts: [String] = [] + ) -> PluginManifest { + PluginManifest( + id: id, name: name, version: "1.0.0", summary: "A demo plugin.", + symbol: symbol, entry: entry, networkHosts: networkHosts) + } + + /// Dictionary-backed `PluginEnablement.Store` so tests never touch the real + /// UserDefaults domain. + private final class FakeStore: PluginEnablement.Store { + var values: [String: [String]] = [:] + func stringArray(forKey key: String) -> [String]? { values[key] } + func set(_ value: Any?, forKey key: String) { values[key] = value as? [String] } + } + + // MARK: - Manifest validation + + func testValidManifestPasses() { + XCTAssertNil(manifest().validate()) + XCTAssertTrue(manifest().isValid) + } + + func testEmptyIDIsRejected() { + XCTAssertEqual(manifest(id: "").validate(), .emptyID) + } + + /// The id becomes a path component under Application Support, so anything that + /// could traverse out of the plugins directory must be refused before it is ever + /// joined onto a URL. + func testPathTraversalShapedIDsAreRejected() { + for bad in ["..", ".", "../evil", "foo/bar", "foo bar", "Foo", "a\\b"] { + XCTAssertEqual( + manifest(id: bad).validate(), .invalidID(bad), + "expected \(bad) to be rejected as an id") + } + } + + func testReverseDNSStyleIDIsAllowed() { + XCTAssertNil(manifest(id: "app.openwhisp.meme-generator").validate()) + } + + func testEmptyNameAndSymbolAreRejected() { + XCTAssertEqual(manifest(name: " ").validate(), .emptyName) + XCTAssertEqual(manifest(symbol: "").validate(), .emptySymbol) + } + + // MARK: - Network disclosure + + func testLocalPluginHasNoNetworkDisclosure() { + let local = manifest() + XCTAssertFalse(local.usesNetwork) + XCTAssertNil(local.networkDisclosure) + } + + /// The app is local-first, so a plugin that reaches out must say so in the pane. + func testNetworkPluginDisclosesEveryHost() { + let net = manifest(networkHosts: ["api.imgflip.com", "i.imgflip.com"]) + XCTAssertTrue(net.usesNetwork) + XCTAssertEqual( + net.networkDisclosure, + "Connects to api.imgflip.com, i.imgflip.com when you use it.") + } + + // MARK: - Entry kinds + + func testOnlyBuiltInIsRunnable() { + XCTAssertTrue(PluginEntryKind.builtIn.isRunnable) + XCTAssertFalse(PluginEntryKind.dynamicLibrary.isRunnable) + XCTAssertFalse(PluginEntryKind.externalProcess.isRunnable) + XCTAssertNil(PluginEntryKind.builtIn.unavailableReason) + XCTAssertNotNil(PluginEntryKind.dynamicLibrary.unavailableReason) + XCTAssertNotNil(PluginEntryKind.externalProcess.unavailableReason) + } + + // MARK: - Discovery merge + + func testMergeListsBothSourcesSortedByName() { + let merged = PluginDiscovery.merge( + builtIn: [manifest(id: "zebra", name: "Zebra")], + external: [manifest(id: "alpha", name: "Alpha")]) + XCTAssertEqual(merged.map(\.id), ["alpha", "zebra"]) + XCTAssertEqual(merged.map(\.source), [.external, .builtIn]) + } + + /// A folder dropped into a user-writable directory must never shadow a reviewed + /// in-repo plugin — that would be code substitution against an entitled app. + func testBuiltInWinsIDCollisionWithExternal() { + let merged = PluginDiscovery.merge( + builtIn: [manifest(id: "meme-generator", name: "Real")], + external: [manifest(id: "meme-generator", name: "Impostor")]) + XCTAssertEqual(merged.count, 1) + XCTAssertEqual(merged[0].source, .builtIn) + XCTAssertEqual(merged[0].manifest.name, "Real") + } + + func testInvalidManifestsAreDroppedFromMerge() { + let merged = PluginDiscovery.merge( + builtIn: [manifest(id: "..", name: "Traversal"), manifest(id: "ok")], + external: [manifest(id: "bad", name: "")]) + XCTAssertEqual(merged.map(\.id), ["ok"]) + } + + /// Being enabled is not the same as being loadable: the spike lists external + /// plugins but refuses to claim it can run them, whatever their manifest says. + func testExternalPluginIsNeverRunnableEvenWhenItClaimsBuiltIn() { + let merged = PluginDiscovery.merge( + builtIn: [], external: [manifest(id: "sneaky", entry: .builtIn)]) + XCTAssertEqual(merged.count, 1) + XCTAssertFalse(merged[0].isRunnable) + XCTAssertNotNil(merged[0].unavailableReason) + } + + func testBuiltInPluginIsRunnable() { + let merged = PluginDiscovery.merge(builtIn: [manifest()], external: []) + XCTAssertTrue(merged[0].isRunnable) + XCTAssertNil(merged[0].unavailableReason) + } + + // MARK: - Discovery from disk + + func testLoadExternalManifestsReadsWellFormedFolders() throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("PluginDiscoveryTests-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + + func write(id: String, json: String) throws { + let dir = root.appendingPathComponent(id) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + try json.write(to: dir.appendingPathComponent("manifest.json"), + atomically: true, encoding: .utf8) + } + + try write(id: "good", json: """ + {"id":"good","name":"Good","version":"1.0.0","summary":"s", + "symbol":"star","entry":"externalProcess","networkHosts":[]} + """) + // Malformed JSON must be skipped, not thrown — one bad folder can't take the + // whole pane down. + try write(id: "broken", json: "{not json") + // A folder whose manifest claims a different id is refused: the directory + // name is the authority on identity. + try write(id: "liar", json: """ + {"id":"someone-else","name":"Liar","version":"1.0.0","summary":"s", + "symbol":"star","entry":"builtIn","networkHosts":[]} + """) + + let found = PluginDiscovery.loadExternalManifests(in: root) + XCTAssertEqual(found.map(\.id), ["good"]) + } + + func testLoadExternalManifestsToleratesMissingDirectory() { + let missing = FileManager.default.temporaryDirectory + .appendingPathComponent("definitely-not-there-\(UUID().uuidString)") + XCTAssertEqual(PluginDiscovery.loadExternalManifests(in: missing).count, 0) + } + + func testExternalPluginsDirectoryPath() { + let support = URL(fileURLWithPath: "/Users/x/Library/Application Support") + XCTAssertEqual( + PluginDiscovery.externalPluginsDirectory(applicationSupport: support).path, + "/Users/x/Library/Application Support/OpenWhisp/Plugins") + } + + // MARK: - Enablement + + /// Plugins are optional surfaces; installing the app must not silently add them. + func testPluginsAreDisabledByDefault() { + let state = PluginEnablement() + XCTAssertFalse(state.isEnabled("meme-generator")) + XCTAssertEqual(state.enabledIDs, []) + } + + func testToggleRoundTripsThroughStore() { + let store = FakeStore() + var state = PluginEnablement.load(from: store, availableIDs: ["a", "b"]) + state.setEnabled(true, for: "a") + state.save(to: store) + + let reloaded = PluginEnablement.load(from: store, availableIDs: ["a", "b"]) + XCTAssertTrue(reloaded.isEnabled("a")) + XCTAssertFalse(reloaded.isEnabled("b")) + } + + func testDisablingRemovesFromPersistedSet() { + let store = FakeStore() + var state = PluginEnablement(enabled: ["a", "b"]) + state.setEnabled(false, for: "a") + state.save(to: store) + XCTAssertEqual(store.values[PluginEnablement.defaultsKey], ["b"]) + } + + /// Persisted order is sorted so writing unchanged state can't churn the defaults. + func testPersistedIDsAreSorted() { + let store = FakeStore() + PluginEnablement(enabled: ["zebra", "alpha", "mid"]).save(to: store) + XCTAssertEqual(store.values[PluginEnablement.defaultsKey], ["alpha", "mid", "zebra"]) + } + + /// A plugin that disappears and later comes back must come back OFF — otherwise + /// removing it and reinstalling silently restores a surface (and its network + /// access) the user last saw gone. + func testPruneDropsUnavailablePlugins() { + let store = FakeStore() + PluginEnablement(enabled: ["gone", "still-here"]).save(to: store) + + let loaded = PluginEnablement.load(from: store, availableIDs: ["still-here"]) + XCTAssertFalse(loaded.isEnabled("gone")) + XCTAssertTrue(loaded.isEnabled("still-here")) + } + + /// Enabled + runnable is the bar for getting a tab. An enabled-but-unloadable + /// external plugin must not produce a menu row the host can't service. + func testActivePluginsRequireEnabledAndRunnable() { + let discovered = PluginDiscovery.merge( + builtIn: [manifest(id: "built-in", name: "Built In")], + external: [manifest(id: "external", name: "External")]) + + var state = PluginEnablement() + state.setEnabled(true, for: "built-in") + state.setEnabled(true, for: "external") + + XCTAssertEqual(state.activePlugins(from: discovered).map(\.id), ["built-in"]) + } + + func testDisabledBuiltInIsNotActive() { + let discovered = PluginDiscovery.merge(builtIn: [manifest(id: "x")], external: []) + XCTAssertEqual(PluginEnablement().activePlugins(from: discovered).count, 0) + } + + // MARK: - Registry + + func testRegistryShipsTheMemeGeneratorAndItIsValid() { + let ids = PluginRegistry.builtInManifests.map(\.id) + XCTAssertTrue(ids.contains("meme-generator")) + for m in PluginRegistry.builtInManifests { + XCTAssertNil(m.validate(), "built-in manifest \(m.id) is invalid") + XCTAssertEqual(m.entry, .builtIn) + } + } + + /// The registry's manifest literal and the checked-in + /// `plugins/meme-generator/manifest.json` must not drift: the JSON is the + /// authored source of truth and the schema example external plugins copy, while + /// the literal is what actually ships (a built-in plugin must not be able to go + /// missing because a resource wasn't bundled). + func testCheckedInManifestJSONMatchesTheRegistryLiteral() throws { + // Tests/OpenWhispCoreTests/ -> repo root + let repoRoot = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // OpenWhispCoreTests + .deletingLastPathComponent() // Tests + .deletingLastPathComponent() // repo root + let url = repoRoot + .appendingPathComponent("plugins/MemeGenerator/manifest.json") + + let data = try Data(contentsOf: url) + let fromDisk = try JSONDecoder().decode(PluginManifest.self, from: data) + XCTAssertEqual(fromDisk, PluginRegistry.memeGenerator) + } + + /// The meme plugin uses the network, so it must declare it — the Plugins pane + /// renders this and the app is local-first. + func testMemeGeneratorDeclaresItsNetworkHosts() { + XCTAssertTrue(PluginRegistry.memeGenerator.usesNetwork) + XCTAssertNotNil(PluginRegistry.memeGenerator.networkDisclosure) + XCTAssertTrue(PluginRegistry.memeGenerator.networkHosts.contains("api.imgflip.com")) + } +} diff --git a/plugins/MemeGenerator/manifest.json b/plugins/MemeGenerator/manifest.json new file mode 100644 index 0000000..6842353 --- /dev/null +++ b/plugins/MemeGenerator/manifest.json @@ -0,0 +1,9 @@ +{ + "id": "meme-generator", + "name": "Meme Generator", + "version": "0.1.0", + "summary": "Dictate a meme description — the AI picks a template and writes the captions.", + "symbol": "photo.badge.plus", + "entry": "builtIn", + "networkHosts": ["api.imgflip.com", "i.imgflip.com"] +} From fc8abc5b38959f8558662ddb796bd8a9e68aec42 Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Sun, 2 Aug 2026 13:46:49 -0700 Subject: [PATCH 02/15] spike(plugins): plugin host, Plugins pane, and the voice-driven meme generator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The app layer on top of the pure core, plus the provider seam that keeps the design honest about being hot-swappable. Host + UI: * `PluginHost` — discovers plugins through an ORDERED PROVIDER LIST, tracks the enabled set, and owns plugin windows. The compile-time registry is just ONE provider; the manifest-on-disk provider re-reads the filesystem on every reload, so a real installation path is a new provider + a runner, not a restructure. Earlier providers win id collisions (descending trust). * Settings → Plugins — per-plugin enable toggle, the network disclosure next to the switch that turns it on, and an honest callout when a discovered plugin can't be loaded by this build. * Menu bar → Plugins submenu, present only when something is enabled. * Dictation lands in a focused plugin window via `PluginDictationSink`, wired at BOTH AppState call sites (the liveChunks branch is the one MAK-49 had to go back and fix). AppState is unchanged in NET LOC — the ratchet still reports exactly 7051 — because the checks fold into the existing conditionals. Meme plugin (`plugins/MemeGenerator/`, compiled in with `PLUGINS=1 ./build.sh`): dictate a description -> LLM (reusing `summarizeResolved`, inheriting its busy-reject/fail-closed guarantees) -> imgflip's key-less public catalog -> template matched LOCALLY -> captions drawn LOCALLY with AppKit -> preview, Export PNG, Share. No text ever leaves the Mac; only the blank template comes down. Fails honestly offline. `PLUGINS` defaults to OFF, so a stock build carries no plugin code at all. 2368 tests (2305 baseline + 63). Ratchet OK. Default, plugins, and lean builds all compile. Co-Authored-By: Claude --- OpenWhisp/AppMain.swift | 29 +++ OpenWhisp/Models/AppState.swift | 32 +-- OpenWhisp/Services/MemeCaptionLayout.swift | 28 +++ OpenWhisp/Services/PluginDiscovery.swift | 56 ++++- OpenWhisp/Views/PluginHost.swift | 178 ++++++++++++++ OpenWhisp/Views/Settings/PluginsPane.swift | 133 ++++++++++ OpenWhisp/Views/SettingsView.swift | 9 +- .../PluginSystemTests.swift | 40 +++ build.sh | 26 ++ .../MemeGenerator/MemeGeneratorModel.swift | 230 ++++++++++++++++++ plugins/MemeGenerator/MemeGeneratorView.swift | 149 ++++++++++++ .../MemeGeneratorWindowController.swift | 94 +++++++ plugins/MemeGenerator/MemeRenderer.swift | 153 ++++++++++++ .../MemeGenerator/MemeTemplateService.swift | 77 ++++++ 14 files changed, 1205 insertions(+), 29 deletions(-) create mode 100644 OpenWhisp/Views/PluginHost.swift create mode 100644 OpenWhisp/Views/Settings/PluginsPane.swift create mode 100644 plugins/MemeGenerator/MemeGeneratorModel.swift create mode 100644 plugins/MemeGenerator/MemeGeneratorView.swift create mode 100644 plugins/MemeGenerator/MemeGeneratorWindowController.swift create mode 100644 plugins/MemeGenerator/MemeRenderer.swift create mode 100644 plugins/MemeGenerator/MemeTemplateService.swift diff --git a/OpenWhisp/AppMain.swift b/OpenWhisp/AppMain.swift index 32e7e5e..2a3abfb 100644 --- a/OpenWhisp/AppMain.swift +++ b/OpenWhisp/AppMain.swift @@ -381,6 +381,28 @@ class OpenWhispApp: NSObject, NSApplicationDelegate { // Floating Scratchpad (MAK-49): a target-free surface to dictate into. menu.addItem(menuItem("Scratchpad", symbol: "note.text", action: #selector(openScratchpad), keyEquivalent: "s")) + // Plugins (spike/plugin-system): one row per ENABLED plugin, folded into a + // submenu so an optional feature never crowds the main menu. Absent entirely + // when nothing is enabled — which is the default. + let activePlugins = PluginHost.shared.activePlugins + if !activePlugins.isEmpty { + let pluginsItem = NSMenuItem(title: "Plugins", action: nil, keyEquivalent: "") + pluginsItem.image = NSImage( + systemSymbolName: "puzzlepiece.extension", accessibilityDescription: nil) + let submenu = NSMenu() + for plugin in activePlugins { + let item = menuItem( + plugin.manifest.name, + symbol: plugin.manifest.symbol, + action: #selector(openPlugin(_:))) + item.representedObject = plugin.id + item.target = self + submenu.addItem(item) + } + pluginsItem.submenu = submenu + menu.addItem(pluginsItem) + } + menu.addItem(.separator()) // Quick mid-use toggles only. Engine, live-chunk plumbing, etc. live in @@ -586,6 +608,13 @@ class OpenWhispApp: NSObject, NSApplicationDelegate { } @objc private func openScratchpad() { appState.openScratchpad() } + + /// Open an enabled plugin's window. The id rides on `representedObject` so one + /// selector serves every plugin row (the host re-checks enabled+runnable). + @objc private func openPlugin(_ sender: NSMenuItem) { + guard let id = sender.representedObject as? String else { return } + PluginHost.shared.open(pluginID: id) + } @objc private func startDictation() { appState.startDictation() } @objc private func stopDictation() { appState.stopDictation() } @objc private func cancelDictation() { appState.cancelDictation() } diff --git a/OpenWhisp/Models/AppState.swift b/OpenWhisp/Models/AppState.swift index 645cc53..92b6309 100644 --- a/OpenWhisp/Models/AppState.swift +++ b/OpenWhisp/Models/AppState.swift @@ -4659,14 +4659,14 @@ class AppState: ObservableObject { let pastesWholeOnce = !isLiveChunkSession || isPreviewSession if pastesWholeOnce { let insertion = addTrailingSpace ? "\(text) " : text - // Scratchpad (MAK-49): when our own floating pad is the frontmost key - // window, the user is dictating INTO it with no other target. The - // focused-app insert path can't serve this (its paste fallback - // deliberately declines when OUR app is frontmost), so append straight - // into the active note's model + text view instead. This is the - // target-free capture the pad exists for; it never touches the - // clipboard and always lands the text. - if scratchpadController.appendDictationIfKey(text) { + // Scratchpad (MAK-49) / plugin windows (spike/plugin-system): when one of + // OUR windows is the frontmost key window, the user is dictating INTO it + // with no other target. The focused-app insert path can't serve this (its + // paste fallback deliberately declines when OUR app is frontmost), so + // append straight into that window instead — target-free capture that + // never touches the clipboard and always lands the text. Only one window + // can be key, so at most one of these accepts. + if scratchpadController.appendDictationIfKey(text) || PluginHost.shared.appendDictationIfKey(text) { lastInsertedIntoFocusedApp = nil } else { // Output target (MAK-11..14): when the user has selected AND configured a @@ -4717,14 +4717,14 @@ class AppState: ObservableObject { router.route(payload) { _ in } } } // end: not routed to the Scratchpad - } else if scratchpadController.appendDictationIfKey(text) { - // Scratchpad (MAK-49) in liveChunks mode: the per-chunk live pastes all - // fell back to the clipboard because OUR pad is frontmost (the focused-app - // insert declines when OpenWhisp is key), so NOTHING landed in the note. - // Route the WHOLE session text into the active note once here — the honest - // completion-time fix for the data loss (per-chunk live typing into the pad - // is out of scope). Skip the clipboard-only finish below entirely; the pad - // owns the text and never touches the clipboard. + } else if scratchpadController.appendDictationIfKey(text) || PluginHost.shared.appendDictationIfKey(text) { + // Scratchpad (MAK-49) / plugin window in liveChunks mode: the per-chunk + // live pastes all fell back to the clipboard because OUR window is + // frontmost (the focused-app insert declines when OpenWhisp is key), so + // NOTHING landed in it. Route the WHOLE session text there once here — + // the honest completion-time fix for the data loss (per-chunk live typing + // is out of scope). Skip the clipboard-only finish below; the window owns + // the text and never touches the clipboard. lastInsertedIntoFocusedApp = nil } else { // liveChunks: the text was already pasted incrementally (no trailing space). diff --git a/OpenWhisp/Services/MemeCaptionLayout.swift b/OpenWhisp/Services/MemeCaptionLayout.swift index 7087a49..ab4e543 100644 --- a/OpenWhisp/Services/MemeCaptionLayout.swift +++ b/OpenWhisp/Services/MemeCaptionLayout.swift @@ -140,4 +140,32 @@ public enum MemeCaptionLayout { public static func minimumFontSize(imageHeight: Double) -> Double { max(8, imageHeight * 0.04) } + + // MARK: - Export naming + + /// A PNG filename derived from the captions, so saved memes are findable later + /// rather than a wall of `meme.png`, `meme-1.png`. + /// + /// Slugified conservatively: non-alphanumerics collapse to single hyphens, so a + /// caption in any script degrades to something a filesystem is happy with, and + /// an all-punctuation caption still yields a usable `meme.png`. + public static func suggestedFileName(topText: String, bottomText: String) -> String { + let joined = [topText, bottomText] + .filter { !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } + .joined(separator: " ") + + let slug = joined + .folding(options: [.diacriticInsensitive], locale: Locale(identifier: "en_US")) + .lowercased() + .map { $0.isLetter || $0.isNumber ? $0 : "-" } + .reduce(into: "") { partial, char in + // Collapse runs of separators instead of emitting `a---b`. + if char == "-", partial.last == "-" { return } + partial.append(char) + } + .trimmingCharacters(in: CharacterSet(charactersIn: "-")) + .prefix(48) + + return (slug.isEmpty ? "meme" : String(slug)) + ".png" + } } diff --git a/OpenWhisp/Services/PluginDiscovery.swift b/OpenWhisp/Services/PluginDiscovery.swift index 5afbd9c..9e4d553 100644 --- a/OpenWhisp/Services/PluginDiscovery.swift +++ b/OpenWhisp/Services/PluginDiscovery.swift @@ -65,25 +65,46 @@ public enum PluginDiscovery { .appendingPathComponent("Plugins", isDirectory: true) } - /// Merge built-in and external manifests into the host's plugin list. + /// One source of plugin manifests. + /// + /// The host enumerates an ORDERED list of these rather than knowing about any + /// particular source, so the compile-time registry is just one provider among + /// others. This is the seam that keeps the design honest about being + /// hot-swappable: adding a real installation path (a downloaded bundle + /// directory, a per-user plugins folder, an out-of-process helper that + /// advertises itself) means adding a provider, not changing the host. + public struct Provider: Sendable { + public let source: Source + /// Produce the manifests this source currently offers. Called on every + /// `reload`, so a provider backed by the filesystem picks up installs + /// without an app restart. + public let manifests: @Sendable () -> [PluginManifest] + + public init(source: Source, manifests: @escaping @Sendable () -> [PluginManifest]) { + self.source = source + self.manifests = manifests + } + } + + /// Merge an ordered list of providers into the host's plugin list. /// /// - Invalid manifests are dropped (a plugin with no name/symbol or a /// traversal-shaped id is not listable). - /// - Built-in beats external on id collision. + /// - **Later providers lose an id collision to earlier ones.** Callers pass + /// providers in DESCENDING trust order, so a lower-trust source can never + /// shadow a higher-trust one — the property that stops a writable directory + /// from substituting code into an app holding Accessibility + mic rights. /// - Result is sorted by display name, then id, so the pane's order is stable /// across launches and independent of filesystem enumeration order. - public static func merge( - builtIn: [PluginManifest], - external: [PluginManifest] - ) -> [Discovered] { + public static func merge(providers: [Provider]) -> [Discovered] { var byID: [String: Discovered] = [:] - // External first so a built-in with the same id overwrites it. - for manifest in external where manifest.isValid { - byID[manifest.id] = Discovered(manifest: manifest, source: .external) - } - for manifest in builtIn where manifest.isValid { - byID[manifest.id] = Discovered(manifest: manifest, source: .builtIn) + for provider in providers { + for manifest in provider.manifests() where manifest.isValid { + // First provider to claim an id keeps it. + guard byID[manifest.id] == nil else { continue } + byID[manifest.id] = Discovered(manifest: manifest, source: provider.source) + } } return byID.values.sorted { @@ -94,6 +115,17 @@ public enum PluginDiscovery { } } + /// Convenience for the two sources the spike has, in trust order. + public static func merge( + builtIn: [PluginManifest], + external: [PluginManifest] + ) -> [Discovered] { + merge(providers: [ + Provider(source: .builtIn) { builtIn }, + Provider(source: .external) { external }, + ]) + } + /// Read every `//manifest.json` under an external plugins directory. /// /// Tolerant by design: a malformed or unreadable manifest is SKIPPED, never diff --git a/OpenWhisp/Views/PluginHost.swift b/OpenWhisp/Views/PluginHost.swift new file mode 100644 index 0000000..58c4e72 --- /dev/null +++ b/OpenWhisp/Views/PluginHost.swift @@ -0,0 +1,178 @@ +import AppKit +import SwiftUI + +/// The app-side plugin host (spike/plugin-system). +/// +/// Owns everything about plugins that AppState would otherwise have to: the +/// discovered list, the enabled set, and the windows enabled plugins open. AppState +/// gains exactly ONE line for this feature (a `lazy var pluginHost`) because the +/// MAK-32 ratchet is at zero headroom — same reason `ScratchpadWindowController` and +/// `TranslationPreviewController` keep their own storage. +/// +/// A singleton because the menu bar, the Settings pane, and the plugin windows all +/// need the same enabled set, and none of them share an owner. +/// +/// **What this is not:** a loader. It enumerates a compile-time registry plus any +/// manifests dropped on disk, and can open windows only for the former. See +/// `PluginRegistry` for why (the app holds Accessibility + mic + clipboard rights). +@MainActor +final class PluginHost: ObservableObject { + + static let shared = PluginHost() + + /// Every plugin the host knows about, built-in and discovered-on-disk. + @Published private(set) var discovered: [PluginDiscovery.Discovered] = [] + + /// The enabled set. Published so the pane's toggles and the menu bar stay in + /// sync; persisted through `PluginEnablement` on its own defaults key. + @Published private(set) var enablement = PluginEnablement() + + /// One window per plugin id, created on first open and reused after. + private var windows: [String: NSWindowController] = [:] + + private init() { + reload() + } + + // MARK: - Discovery + + /// Re-enumerate plugins and re-load the (pruned) enabled set. + /// + /// Called at init and whenever the Plugins pane appears, so dropping a folder + /// into the plugins directory shows up without relaunching. + func reload() { + discovered = PluginDiscovery.merge(providers: Self.providers) + enablement = PluginEnablement.load( + from: UserDefaults.standard, + availableIDs: Set(discovered.map(\.id))) + } + + /// The manifest sources, in DESCENDING trust order (earlier wins an id + /// collision, so a writable directory can never shadow a reviewed plugin). + /// + /// The compile-time registry is deliberately just ONE entry here. The + /// install-a-folder provider below is the real hot-swap story: it re-reads the + /// filesystem on every `reload()`, so an installed plugin appears without + /// rebuilding — and, once a loader exists, without relaunching either. Shipping + /// hot-swappable plugins means adding a provider and a runner, not restructuring + /// the host. See the PR's "Path to hot-swappable" section. + private static var providers: [PluginDiscovery.Provider] { + [ + .init(source: .builtIn) { PluginRegistry.builtInManifests }, + .init(source: .external) { + PluginDiscovery.loadExternalManifests(in: PluginHost.externalDirectory) + }, + ] + } + + /// `~/Library/Application Support/OpenWhisp/Plugins`. + /// + /// `nonisolated` because the discovery providers read it from a `@Sendable` + /// closure: it only derives a path from the filesystem and touches no host state. + nonisolated static var externalDirectory: URL { + let support = FileManager.default.urls( + for: .applicationSupportDirectory, in: .userDomainMask).first + ?? URL(fileURLWithPath: NSHomeDirectory()) + .appendingPathComponent("Library/Application Support") + return PluginDiscovery.externalPluginsDirectory(applicationSupport: support) + } + + // MARK: - Enablement + + func isEnabled(_ id: String) -> Bool { enablement.isEnabled(id) } + + /// Turn a plugin on or off and persist immediately. + /// + /// Disabling closes any window the plugin had open: a disabled plugin should not + /// keep a surface alive, and leaving a stale window up is how a user ends up + /// interacting with something they just switched off. + func setEnabled(_ isEnabled: Bool, for id: String) { + enablement.setEnabled(isEnabled, for: id) + enablement.save(to: UserDefaults.standard) + if !isEnabled { closeWindow(for: id) } + } + + /// The plugins that should appear as tabs / menu rows: enabled AND runnable. + var activePlugins: [PluginDiscovery.Discovered] { + enablement.activePlugins(from: discovered) + } + + // MARK: - Windows + + /// Open (or focus) a plugin's window. + /// + /// Refuses anything not enabled AND runnable, so a stale menu row or a + /// hand-crafted call can't surface a plugin the user hasn't turned on. + func open(pluginID: String) { + guard let plugin = activePlugins.first(where: { $0.id == pluginID }) else { return } + + if let existing = windows[pluginID] { + existing.window?.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + return + } + + guard let controller = Self.makeWindowController(for: plugin) else { return } + windows[pluginID] = controller + controller.window?.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + } + + private func closeWindow(for id: String) { + windows[id]?.window?.close() + windows[id] = nil + } + + /// Map a built-in plugin id to its window. The second of the two places a + /// built-in plugin is wired (the first is `PluginRegistry`). + private static func makeWindowController( + for plugin: PluginDiscovery.Discovered + ) -> NSWindowController? { + switch plugin.id { + #if OPENWHISP_PLUGINS + case PluginRegistry.memeGenerator.id: + return MemeGeneratorWindowController() + #endif + default: + // Either the plugin's sources weren't compiled into this build + // (PLUGINS=1 ./build.sh) or the id has no window. Both are honest + // no-ops rather than a crash. + return nil + } + } + + // MARK: - Dictation seam + + /// Offer a completed dictation to whichever plugin window is frontmost. + /// + /// Mirrors `ScratchpadWindowController.appendDictationIfKey`: returns whether a + /// plugin took the text, so AppState skips its focused-app insert. The + /// focused-app paste path deliberately declines while OUR app is frontmost, so + /// without this a dictation aimed at a plugin window would be lost. + /// + /// Only ONE window can be key, so at most one plugin can accept. + @discardableResult + func appendDictationIfKey(_ text: String) -> Bool { + guard !text.isEmpty else { return false } + for controller in windows.values { + if let sink = controller as? PluginDictationSink, + sink.appendDictationIfKey(text) { + return true + } + } + return false + } +} + +/// A plugin window that can receive dictation when it is the key window. +/// +/// This is the seam the Scratchpad hardcodes on AppState, generalized just enough +/// for plugins — a plugin surface with a text field wants dictation to land in it +/// exactly the way the Scratchpad does. +@MainActor +protocol PluginDictationSink: AnyObject { + /// Append a completed dictation IF this window is key. Returns whether it took + /// the text. + @discardableResult + func appendDictationIfKey(_ text: String) -> Bool +} diff --git a/OpenWhisp/Views/Settings/PluginsPane.swift b/OpenWhisp/Views/Settings/PluginsPane.swift new file mode 100644 index 0000000..4e8385a --- /dev/null +++ b/OpenWhisp/Views/Settings/PluginsPane.swift @@ -0,0 +1,133 @@ +import SwiftUI + +/// Settings → Plugins (spike/plugin-system). +/// +/// Lists every discovered plugin with an enable toggle, its network disclosure, and +/// its per-plugin configuration surface. Plugins are OPTIONAL and off by default, so +/// this pane is the only thing standing between a stock install and an extra surface. +/// +/// State lives on `PluginHost` (its own defaults key), not AppState — the MAK-32 +/// ratchet is at zero headroom, so this pane observes the host directly rather than +/// mirroring into `@State` the way single-key features do. +struct PluginsPane: View { + + @ObservedObject var host: PluginHost + + var body: some View { + Form { + introSection + + if host.discovered.isEmpty { + emptySection + } else { + ForEach(host.discovered) { plugin in + pluginSection(plugin) + } + } + + installSection + } + .formStyle(.grouped) + // Re-enumerate on appear so a folder dropped into the plugins directory + // shows up without relaunching the app. + .onAppear { host.reload() } + } + + private var introSection: some View { + Section { + SettingsFootnote( + "Plugins are optional add-ons. Each one you enable gets its own window, " + + "reachable from the menu bar under Plugins. They're off until you turn them on.") + } header: { + Text("Plugins") + } + } + + private var emptySection: some View { + Section { + Text("No plugins are available in this build.") + .foregroundStyle(.secondary) + } footer: { + SettingsFootnote( + "In-repo plugins are compiled in with PLUGINS=1 ./build.sh.") + } + } + + @ViewBuilder + private func pluginSection(_ plugin: PluginDiscovery.Discovered) -> some View { + Section { + SubtitledToggle( + plugin.manifest.name, + subtitle: plugin.manifest.summary, + isOn: Binding( + get: { host.isEnabled(plugin.id) }, + set: { host.setEnabled($0, for: plugin.id) }) + ) + .disabled(!plugin.isRunnable) + + // Honest about what this build can actually run: an external plugin is + // listed so the user knows it was found, but the spike has no loader. + if let reason = plugin.unavailableReason { + SettingsCallout(.warning, reason) + } + + // The app is local-first. A plugin that reaches out says so, right next + // to the switch that turns it on. + if let disclosure = plugin.manifest.networkDisclosure { + SettingsCallout(.info, disclosure) + } + + if host.isEnabled(plugin.id), plugin.isRunnable { + configuration(for: plugin) + } + } header: { + Label(plugin.manifest.name, systemImage: plugin.manifest.symbol) + } footer: { + SettingsFootnote( + "Version \(plugin.manifest.version) · " + + (plugin.source == .builtIn ? "Built in" : "Installed")) + } + } + + /// A plugin's own configuration surface. + /// + /// In the spike this is a fixed switch on the plugin id — the honest shape for a + /// prototype with one plugin. A real system would have the manifest declare its + /// settings schema (or the plugin vend its own view), which is exactly the part + /// that gets hard once plugins are third-party. + @ViewBuilder + private func configuration(for plugin: PluginDiscovery.Discovered) -> some View { + if plugin.id == PluginRegistry.memeGenerator.id { + LabeledContent("Language model") { + Text("Follows Settings → Cleanup") + .foregroundStyle(.secondary) + } + SettingsFootnote( + "The meme generator uses your configured cleanup model to turn a spoken " + + "description into a template choice and captions. Captions are drawn on " + + "your Mac — only the blank template image is downloaded.") + } + } + + private var installSection: some View { + Section { + HStack { + Text("Plugins folder") + Spacer() + Button("Show in Finder") { + let dir = PluginHost.externalDirectory + try? FileManager.default.createDirectory( + at: dir, withIntermediateDirectories: true) + NSWorkspace.shared.activateFileViewerSelecting([dir]) + } + } + } header: { + Text("Installing plugins") + } footer: { + SettingsFootnote( + "Drop a plugin folder containing manifest.json here and reopen this pane to " + + "see it listed. This prototype lists installed plugins but can't load them — " + + "only plugins that ship with the app can run.") + } + } +} diff --git a/OpenWhisp/Views/SettingsView.swift b/OpenWhisp/Views/SettingsView.swift index bcac1c0..96af8c8 100644 --- a/OpenWhisp/Views/SettingsView.swift +++ b/OpenWhisp/Views/SettingsView.swift @@ -37,6 +37,7 @@ enum SettingsPane: String, CaseIterable, Identifiable { case agentBridge case sync case streamOverlay + case plugins var id: String { rawValue } @@ -51,7 +52,8 @@ enum SettingsPane: String, CaseIterable, Identifiable { switch self { case .general, .dictation, .models, .cleanup, .output, .privacy, .advanced: return .setup - case .insights, .modes, .rules, .files, .meetings, .profiles, .agentBridge, .sync, .streamOverlay: + case .insights, .modes, .rules, .files, .meetings, .profiles, .agentBridge, .sync, + .streamOverlay, .plugins: return .moreFeatures } } @@ -75,6 +77,7 @@ enum SettingsPane: String, CaseIterable, Identifiable { case .agentBridge: return "Agent Bridge" case .sync: return "Sync" case .streamOverlay: return "Stream Overlay" + case .plugins: return "Plugins" case .privacy: return "Privacy & Permissions" case .advanced: return "Advanced" } @@ -96,6 +99,7 @@ enum SettingsPane: String, CaseIterable, Identifiable { case .agentBridge: return "point.3.connected.trianglepath.dotted" case .sync: return "arrow.triangle.2.circlepath" case .streamOverlay: return "captions.bubble" + case .plugins: return "puzzlepiece.extension" case .privacy: return "lock.shield" case .advanced: return "wrench.and.screwdriver" } @@ -198,6 +202,9 @@ struct SettingsView: View { case .agentBridge: AgentBridgePane(appState: appState) case .sync: SyncPane(appState: appState) case .streamOverlay: StreamOverlayPane(overlay: appState.streamOverlay, dictationLanguage: appState.language) + // Plugin state lives on PluginHost (its own defaults key), not AppState — + // the MAK-32 ratchet is at zero headroom. + case .plugins: PluginsPane(host: PluginHost.shared) case .privacy: PrivacyPane(appState: appState) case .advanced: AdvancedPane(appState: appState) } diff --git a/Tests/OpenWhispCoreTests/PluginSystemTests.swift b/Tests/OpenWhispCoreTests/PluginSystemTests.swift index 5ac50b6..8617ab7 100644 --- a/Tests/OpenWhispCoreTests/PluginSystemTests.swift +++ b/Tests/OpenWhispCoreTests/PluginSystemTests.swift @@ -107,6 +107,46 @@ final class PluginSystemTests: XCTestCase { XCTAssertEqual(merged[0].manifest.name, "Real") } + // MARK: - Providers (hot-swap seam) + + /// The host enumerates an ordered provider list, so the compile-time registry is + /// just one source among others — the seam a real installation path plugs into + /// without changing the host. + func testProvidersAreMergedInOrder() { + let merged = PluginDiscovery.merge(providers: [ + .init(source: .builtIn) { [self.manifest(id: "b", name: "Bravo")] }, + .init(source: .external) { [self.manifest(id: "a", name: "Alpha")] }, + ]) + XCTAssertEqual(merged.map(\.id), ["a", "b"]) + } + + /// Providers are passed in DESCENDING trust order, so a lower-trust source can + /// never shadow a higher-trust one. + func testEarlierProviderWinsIDCollision() { + let merged = PluginDiscovery.merge(providers: [ + .init(source: .builtIn) { [self.manifest(id: "x", name: "Trusted")] }, + .init(source: .external) { [self.manifest(id: "x", name: "Untrusted")] }, + ]) + XCTAssertEqual(merged.count, 1) + XCTAssertEqual(merged[0].manifest.name, "Trusted") + XCTAssertEqual(merged[0].source, .builtIn) + } + + /// A provider is re-invoked on every merge, which is what lets a filesystem-backed + /// source pick up an installed plugin without a rebuild. + func testProviderIsReEvaluatedOnEachMerge() { + var available: [PluginManifest] = [] + let provider = PluginDiscovery.Provider(source: .external) { available } + + XCTAssertEqual(PluginDiscovery.merge(providers: [provider]).count, 0) + available = [manifest(id: "installed-later")] + XCTAssertEqual(PluginDiscovery.merge(providers: [provider]).map(\.id), ["installed-later"]) + } + + func testMergeWithNoProvidersIsEmpty() { + XCTAssertEqual(PluginDiscovery.merge(providers: []).count, 0) + } + func testInvalidManifestsAreDroppedFromMerge() { let merged = PluginDiscovery.merge( builtIn: [manifest(id: "..", name: "Traversal"), manifest(id: "ok")], diff --git a/build.sh b/build.sh index 9e99999..571871c 100755 --- a/build.sh +++ b/build.sh @@ -35,6 +35,31 @@ done < <(find "$PROJECT_DIR/OpenWhisp" -name "*.swift" -not -path "*/SyncLoopbac # harness). Its main.swift has top-level executable code + an OpenWhispCore import, # so it must NOT be folded into the mac app's single-module glob. +# In-repo plugins (spike/plugin-system). OFF BY DEFAULT: plugins are an OPTIONAL +# surface, so a stock build carries none of their code at all — `PLUGINS=1 +# ./build.sh` compiles the in-repo plugins under plugins/ into the app, where the +# compile-time PluginRegistry picks them up and the Plugins settings pane lists +# them. The pure plugin core (PluginManifest/Discovery/Enablement/Registry and the +# meme rules) lives under OpenWhisp/Services and is always compiled + always +# tested; this flag only controls the plugins' own AppKit/SwiftUI surfaces. +# +# NOTE (spike honesty): this is a compile-time toggle, not a plugin loader. It +# exists so the prototype can demo an optional surface without pretending it can +# load third-party code — docs/ROADMAP.md §6 rejects in-process third-party +# plugins outright for an app holding Accessibility + clipboard rights. +PLUGIN_DEFINE_ARGS=() +if [ "${PLUGINS:-0}" != "0" ]; then + plugin_count=0 + while IFS= read -r f; do + SWIFT_FILES+=("$f") + plugin_count=$((plugin_count + 1)) + done < <(find "$PROJECT_DIR/plugins" -name "*.swift") + PLUGIN_DEFINE_ARGS=( -DOPENWHISP_PLUGINS ) + echo "Plugins: ENABLED (${plugin_count} source file(s) from plugins/)" +else + echo "Plugins: disabled (PLUGINS=1 ./build.sh to compile in-repo plugins)" +fi + # Stamp the build with its git commit (shown in Settings › Advanced) — the # generated file lives in build/, outside the source glob, and is regenerated # every build. @@ -111,6 +136,7 @@ if xcrun swiftc \ "${FLUIDAUDIO_ARGS[@]+"${FLUIDAUDIO_ARGS[@]}"}" \ "${SPARKLE_ARGS[@]+"${SPARKLE_ARGS[@]}"}" \ "${INSTRUMENTATION_ARGS[@]+"${INSTRUMENTATION_ARGS[@]}"}" \ + "${PLUGIN_DEFINE_ARGS[@]+"${PLUGIN_DEFINE_ARGS[@]}"}" \ "${SWIFT_FILES[@]}" \ -o "$BUILD_DIR/OpenWhisp" \ 2>&1 diff --git a/plugins/MemeGenerator/MemeGeneratorModel.swift b/plugins/MemeGenerator/MemeGeneratorModel.swift new file mode 100644 index 0000000..6e97192 --- /dev/null +++ b/plugins/MemeGenerator/MemeGeneratorModel.swift @@ -0,0 +1,230 @@ +import AppKit +import SwiftUI + +/// The observable state behind the Meme Generator window (spike). +/// +/// The pipeline, end to end: +/// +/// 1. The user DICTATES a description (or types it) into `description`. +/// 2. `generate()` sends it to the configured LLM via the injected `aiCall` seam — +/// the same closure-injection `ScratchpadModel` uses, so this model never touches +/// AppState and stays stubbable. +/// 3. The reply is parsed by the pure `MemeAI.parse` into a template query + captions. +/// 4. The imgflip template catalog is fetched (once per session, then cached) and the +/// query is matched locally by `MemeTemplateMatcher`. +/// 5. The template image is downloaded and captioned LOCALLY by `MemeRenderer`. +/// +/// Only steps 4–5 touch the network, and only to GET images — no user text is ever +/// sent to imgflip. +@MainActor +final class MemeGeneratorModel: ObservableObject { + + /// What the user described, and what dictation lands in. + @Published var description: String = "" + + /// The finished meme, if one has been generated. + @Published private(set) var meme: NSImage? + + /// Human-readable status, shown under the buttons. Doubles as the error channel — + /// this is a prototype surface, so failures are stated plainly rather than + /// swallowed. + @Published private(set) var status: String = "" + + /// True while a generate round-trip is in flight (disables the button). + @Published private(set) var isBusy: Bool = false + + /// The captions of the current meme, surfaced so the user can see what the model + /// wrote (and tell a bad caption from a bad template). + @Published private(set) var topText: String = "" + @Published private(set) var bottomText: String = "" + + /// The template that was chosen, for the "used X" line. + @Published private(set) var templateName: String = "" + + // MARK: - Injected seams + + /// One LLM round-trip: (instruction, input, resolved model) -> output. Injected + /// by the window controller, exactly like `ScratchpadModel.AICall`. + typealias AICall = ( + _ instruction: String, _ input: String, _ resolved: SummaryModelResolver.Resolved + ) async throws -> String + + private var aiCall: AICall? + private var resolveAIModel: () -> SummaryModelResolver.Resolved = { + .init(provider: "", model: "", endpoint: "") + } + + func configureAI( + call: @escaping AICall, + resolveModel: @escaping () -> SummaryModelResolver.Resolved + ) { + aiCall = call + resolveAIModel = resolveModel + } + + /// The template catalog, fetched once and reused for the rest of the session. + private var catalog: [MemeTemplate] = [] + + /// Monotonic ticket so a late result from a superseded request can't overwrite a + /// newer meme (the `ScratchpadAISession` fence, simplified for one surface). + private var ticket: Int = 0 + + /// True once the window is gone — stops a late async result touching the UI. + private var isCancelled = false + + // MARK: - Dictation + + /// Append dictated text to the description, matching the Scratchpad's join rule + /// (a space between the existing text and the new words). + func appendDictation(_ text: String) { + let incoming = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !incoming.isEmpty else { return } + if description.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + description = incoming + } else { + description += " " + incoming + } + status = "" + } + + var canGenerate: Bool { + !isBusy && !description.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } + + // MARK: - Generate + + func generate() { + guard canGenerate else { return } + guard let aiCall else { + status = "No LLM is configured — set one up in Settings → Cleanup." + return + } + + // Fail closed BEFORE the request: an agent-CLI resolution would otherwise + // fall through to a cloud endpoint the user never chose (the MAK-53 hazard + // ScratchpadModel guards the same way). + let resolved = resolveAIModel() + guard ScratchpadAIModel.isUsable(resolved) else { + status = ScratchpadAIModel.unusableProviderMessage + return + } + + ticket += 1 + let myTicket = ticket + isBusy = true + status = "Asking the model…" + let prompt = MemeAI.userPayload(description: description) + + Task { [weak self] in + do { + let raw = try await aiCall(MemeAI.prompt, prompt, resolved) + guard let self, !self.isCancelled, myTicket == self.ticket else { return } + + switch MemeAI.parse(raw) { + case .failure(let rejection): + self.finish(myTicket, status: "Couldn't build the meme — \(rejection.reason).") + case .success(let spec): + await self.buildMeme(spec, ticket: myTicket) + } + } catch { + guard let self, !self.isCancelled, myTicket == self.ticket else { return } + self.finish(myTicket, status: "The model request failed — \(Self.reason(error))") + } + } + } + + /// Fetch the catalog (if needed), match a template, download it, caption it. + private func buildMeme(_ spec: MemeAI.MemeSpec, ticket myTicket: Int) async { + status = "Finding a template…" + + if catalog.isEmpty { + do { + catalog = try await MemeTemplateService.fetchCatalog() + } catch { + finish(myTicket, status: + "Couldn't reach imgflip.com for templates — \(Self.reason(error)) " + + "The meme generator needs a connection to fetch template images.") + return + } + } + guard !isCancelled, myTicket == ticket else { return } + + guard let match = MemeTemplateMatcher.bestMatch(for: spec.templateQuery, in: catalog) else { + finish(myTicket, status: "imgflip returned no templates to choose from.") + return + } + + status = "Downloading \(match.template.name)…" + let template: NSImage + do { + template = try await MemeTemplateService.fetchImage(match.template) + } catch { + finish(myTicket, status: "Couldn't download the template image — \(Self.reason(error))") + return + } + guard !isCancelled, myTicket == ticket else { return } + + let rendered = MemeRenderer.render( + template: template, topText: spec.topText, bottomText: spec.bottomText) + + meme = rendered + topText = spec.topText + bottomText = spec.bottomText + templateName = match.template.name + + // Say so when the template was a guess — a confidently-wrong pick the user + // can't explain is worse than an admitted fallback. + let note = match.isFallback + ? "Couldn't match \"\(spec.templateQuery)\" — used the most popular template instead." + : "Used \(match.template.name)." + finish(myTicket, status: note) + } + + private func finish(_ myTicket: Int, status newStatus: String) { + guard myTicket == ticket else { return } + isBusy = false + status = newStatus + } + + private static func reason(_ error: Error) -> String { + if let wire = error as? BridgeWire.ErrorObject, !wire.message.isEmpty { + return wire.message + } + let described = (error as NSError).localizedDescription + return described.isEmpty ? "the request failed." : described + } + + /// Stop accepting async results — called when the window closes. + func cancel() { + isCancelled = true + isBusy = false + } + + // MARK: - Export / share + + /// Write the meme to a user-chosen file. Returns whether anything was written. + @discardableResult + func exportPNG() -> Bool { + guard let meme, let data = MemeRenderer.pngData(for: meme) else { + status = "Nothing to export yet." + return false + } + + let panel = NSSavePanel() + panel.allowedContentTypes = [.png] + panel.nameFieldStringValue = MemeCaptionLayout.suggestedFileName( + topText: topText, bottomText: bottomText) + panel.canCreateDirectories = true + + guard panel.runModal() == .OK, let url = panel.url else { return false } + do { + try data.write(to: url) + status = "Saved to \(url.lastPathComponent)." + return true + } catch { + status = "Couldn't save — \(error.localizedDescription)" + return false + } + } + +} diff --git a/plugins/MemeGenerator/MemeGeneratorView.swift b/plugins/MemeGenerator/MemeGeneratorView.swift new file mode 100644 index 0000000..5cb4c21 --- /dev/null +++ b/plugins/MemeGenerator/MemeGeneratorView.swift @@ -0,0 +1,149 @@ +import AppKit +import SwiftUI + +/// The Meme Generator plugin's window content (spike). +/// +/// Voice-first by construction: the description field is the first responder when the +/// window opens, so pressing the dictation hotkey and speaking lands the words here +/// with no typing and no clicking. "As fast as possible" is the whole point. +struct MemeGeneratorView: View { + + @ObservedObject var model: MemeGeneratorModel + + /// Focuses the description editor on open so a dictation lands immediately. + @FocusState private var descriptionFocused: Bool + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + header + descriptionEditor + controls + if !model.status.isEmpty { statusLine } + preview + } + .padding(16) + .frame(minWidth: 520, minHeight: 560) + .onAppear { descriptionFocused = true } + } + + private var header: some View { + HStack(spacing: 8) { + Image(systemName: PluginRegistry.memeGenerator.symbol) + .foregroundStyle(.secondary) + Text("Describe the meme out loud — the model picks a template and writes the captions.") + .font(.callout) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + } + + private var descriptionEditor: some View { + VStack(alignment: .leading, spacing: 4) { + TextEditor(text: $model.description) + .font(.body) + .frame(minHeight: 70, maxHeight: 110) + .overlay( + RoundedRectangle(cornerRadius: 6) + .stroke(Color.secondary.opacity(0.3))) + .focused($descriptionFocused) + .overlay(alignment: .topLeading) { + // TextEditor has no placeholder; this is the standard workaround. + if model.description.isEmpty { + Text("e.g. \"distracted boyfriend, but he's looking at Rust and his girlfriend is Python\"") + .font(.body) + .foregroundStyle(.tertiary) + .padding(.top, 8) + .padding(.leading, 5) + .allowsHitTesting(false) + } + } + + Text("Dictate into this window and your words land here.") + .font(.caption) + .foregroundStyle(.tertiary) + } + } + + private var controls: some View { + HStack(spacing: 8) { + Button { + model.generate() + } label: { + Label(model.isBusy ? "Generating…" : "Generate", + systemImage: "wand.and.stars") + } + .keyboardShortcut(.return, modifiers: .command) + .disabled(!model.canGenerate) + + if model.isBusy { + ProgressView().controlSize(.small) + } + + Spacer() + + Button { + model.exportPNG() + } label: { + Label("Export PNG…", systemImage: "square.and.arrow.down") + } + .disabled(model.meme == nil) + + Button { + share() + } label: { + Label("Share", systemImage: "square.and.arrow.up") + } + .disabled(model.meme == nil) + } + } + + private var statusLine: some View { + Text(model.status) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + + @ViewBuilder + private var preview: some View { + if let meme = model.meme { + Image(nsImage: meme) + .resizable() + .scaledToFit() + .frame(maxWidth: .infinity, maxHeight: .infinity) + .accessibilityLabel( + "Meme preview. Top caption: \(model.topText). Bottom caption: \(model.bottomText).") + } else { + RoundedRectangle(cornerRadius: 8) + .fill(Color.secondary.opacity(0.08)) + .overlay( + VStack(spacing: 6) { + Image(systemName: "photo") + .font(.largeTitle) + .foregroundStyle(.tertiary) + Text("Your meme will appear here.") + .font(.callout) + .foregroundStyle(.tertiary) + }) + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + } + + /// Share the rendered PNG through the system picker. Writes to a temp file first + /// because `NSSharingServicePicker` shares URLs far more widely than raw images + /// (Mail/Messages/AirDrop all want a file). + private func share() { + guard let meme = model.meme, + let data = MemeRenderer.pngData(for: meme) else { return } + + let url = FileManager.default.temporaryDirectory + .appendingPathComponent( + MemeCaptionLayout.suggestedFileName( + topText: model.topText, bottomText: model.bottomText)) + guard (try? data.write(to: url)) != nil else { return } + + guard let view = NSApp.keyWindow?.contentView else { return } + let picker = NSSharingServicePicker(items: [url]) + picker.show(relativeTo: .zero, of: view, preferredEdge: .minY) + } +} diff --git a/plugins/MemeGenerator/MemeGeneratorWindowController.swift b/plugins/MemeGenerator/MemeGeneratorWindowController.swift new file mode 100644 index 0000000..af364d3 --- /dev/null +++ b/plugins/MemeGenerator/MemeGeneratorWindowController.swift @@ -0,0 +1,94 @@ +import AppKit +import SwiftUI + +/// The Meme Generator plugin's window (spike). +/// +/// Deliberately thin, like `ScratchpadWindowController`: it owns the window's +/// lifecycle, the LLM wiring, and the dictation seam. All state lives in +/// `MemeGeneratorModel`; all rules live in OpenWhispCore (`MemeAI`, +/// `MemeTemplateMatcher`, `MemeCaptionLayout`) where `swift test` covers them. +/// +/// A normal titled `NSWindow` rather than a floating panel: this is a workspace you +/// look at, not an always-on-top scratch surface. It still needs to become KEY so +/// dictation can land in it — that is what makes `appendDictationIfKey` fire. +@MainActor +final class MemeGeneratorWindowController: NSWindowController, NSWindowDelegate, PluginDictationSink { + + private let model = MemeGeneratorModel() + + convenience init() { + let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 560, height: 620), + styleMask: [.titled, .closable, .miniaturizable, .resizable], + backing: .buffered, + defer: false) + window.title = PluginRegistry.memeGenerator.name + window.isReleasedWhenClosed = false + window.setFrameAutosaveName("OpenWhispMemeGeneratorWindow") + window.center() + + self.init(window: window) + + window.contentViewController = NSHostingController( + rootView: MemeGeneratorView(model: model)) + window.delegate = self + wireAI() + } + + // MARK: - LLM seam + + /// Wire the plugin's Generate action to the app's configured LLM. + /// + /// The same bridge `ScratchpadWindowController.wireAI` uses: both closures are + /// re-evaluated per call so a mid-session settings change lands, and the model + /// never sees AppState. Reusing `summarizeResolved` means the plugin inherits its + /// guarantees for free — busy-reject while dictating, bundled-engine bracketing, + /// fail-closed on the agent-CLI provider, and exactly-once delivery. + /// + /// Note the plugin does NOT get its own model picker in this spike: it follows + /// the Scratchpad's resolution (its override, else the cleanup settings). A real + /// plugin config surface would put that on the plugin's own defaults keys. + private func wireAI() { + model.configureAI( + call: { instruction, input, resolved in + try await withCheckedThrowingContinuation { cont in + Task { @MainActor in + AppState.shared.summarizeResolved( + text: input, instruction: instruction, resolved: resolved + ) { result in + switch result { + case .success(let out): cont.resume(returning: out) + case .failure(let err): cont.resume(throwing: err) + } + } + } + } + }, + resolveModel: { ScratchpadWindowController.resolvedAIModel() }) + } + + // MARK: - PluginDictationSink + + /// True while this window is frontmost — the signal that a completed dictation + /// belongs here rather than in the focused app. + var isKeyWindow: Bool { window?.isKeyWindow ?? false } + + /// Append a completed dictation to the meme description IF this window is key. + /// + /// Returns whether it handled the text so AppState skips its focused-app insert. + /// Without this the words would be lost: the focused-app paste path deliberately + /// declines while OpenWhisp itself is frontmost. + @discardableResult + func appendDictationIfKey(_ text: String) -> Bool { + guard isKeyWindow, !text.isEmpty else { return false } + model.appendDictation(text) + return true + } + + // MARK: - NSWindowDelegate + + func windowWillClose(_ notification: Notification) { + // A closed window must never be mutated by a late generate result. + model.cancel() + } +} diff --git a/plugins/MemeGenerator/MemeRenderer.swift b/plugins/MemeGenerator/MemeRenderer.swift new file mode 100644 index 0000000..36bcb06 --- /dev/null +++ b/plugins/MemeGenerator/MemeRenderer.swift @@ -0,0 +1,153 @@ +import AppKit +import Foundation + +/// Draws classic meme captions onto a template image (spike). +/// +/// Rendering is entirely LOCAL — there is no captioning API and no API key. We +/// download the blank template from imgflip's CDN and do the text ourselves with +/// AppKit, which is also why the plugin never sends the user's words anywhere: only +/// the template image comes over the wire. +/// +/// The classic look: heavy condensed sans, WHITE fill with a BLACK outline, centered, +/// wrapped, top and bottom blocks. Impact is not present on macOS, so we fall back +/// through the closest system faces (see `captionFont`). +/// +/// All the *decisions* (uppercasing, line breaking, shrink-to-fit) live in the pure, +/// tested `MemeCaptionLayout`; this type owns only the drawing and the font metrics +/// it feeds back into that layout. +@MainActor +enum MemeRenderer { + + /// Compose a meme: template image + top/bottom captions. + /// + /// Returns a new image at the template's pixel dimensions, so exports are + /// full-resolution regardless of how the preview is scaled. + static func render( + template: NSImage, + topText: String, + bottomText: String + ) -> NSImage { + // Work in PIXELS, not points: NSImage.size is point-based and would render + // captions at the wrong scale on a template whose rep is 2x. + let pixelSize = pixelDimensions(of: template) + let width = pixelSize.width + let height = pixelSize.height + + let output = NSImage(size: NSSize(width: width, height: height)) + output.lockFocus() + defer { output.unlockFocus() } + + NSGraphicsContext.current?.imageInterpolation = .high + template.draw( + in: NSRect(x: 0, y: 0, width: width, height: height), + from: .zero, operation: .copy, fraction: 1.0) + + let inset = width * MemeCaptionLayout.horizontalInsetShare + let maxTextWidth = width - inset * 2 + let maxBlockHeight = height * MemeCaptionLayout.captionHeightShare + + draw( + caption: topText, position: .top, + imageWidth: width, imageHeight: height, + maxTextWidth: maxTextWidth, maxBlockHeight: maxBlockHeight) + draw( + caption: bottomText, position: .bottom, + imageWidth: width, imageHeight: height, + maxTextWidth: maxTextWidth, maxBlockHeight: maxBlockHeight) + + return output + } + + /// The image's size in PIXELS, preferring the largest bitmap rep so a retina + /// template exports at full resolution. + private static func pixelDimensions(of image: NSImage) -> (width: CGFloat, height: CGFloat) { + let reps = image.representations.compactMap { $0 as? NSBitmapImageRep } + if let best = reps.max(by: { $0.pixelsWide * $0.pixelsHigh < $1.pixelsWide * $1.pixelsHigh }), + best.pixelsWide > 0, best.pixelsHigh > 0 { + return (CGFloat(best.pixelsWide), CGFloat(best.pixelsHigh)) + } + // No bitmap rep (e.g. a vector or a placeholder): fall back to the point size, + // guarding against a degenerate zero that would make an undrawable image. + return (max(image.size.width, 1), max(image.size.height, 1)) + } + + private static func draw( + caption: String, + position: MemeCaptionLayout.Position, + imageWidth: CGFloat, + imageHeight: CGFloat, + maxTextWidth: CGFloat, + maxBlockHeight: CGFloat + ) { + guard !caption.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return } + + // Shrink-to-fit using REAL font metrics: the pure layout does the deciding, + // we supply the measurements. + let fit = MemeCaptionLayout.fit( + caption: caption, + maxWidth: Double(maxTextWidth), + maxHeight: Double(maxBlockHeight), + maxFontSize: MemeCaptionLayout.idealFontSize(imageHeight: Double(imageHeight)), + minFontSize: MemeCaptionLayout.minimumFontSize(imageHeight: Double(imageHeight)), + measure: { text, size in + Double(text.size(withAttributes: [.font: captionFont(size: size)]).width) + }, + lineHeight: { size in size * 1.15 }) + + guard !fit.lines.isEmpty else { return } + + let font = captionFont(size: fit.fontSize) + let lineHeight = CGFloat(fit.fontSize) * 1.15 + let blockHeight = lineHeight * CGFloat(fit.lines.count) + let margin = imageHeight * 0.02 + + // AppKit's image coordinate space is bottom-left origin. + let blockTopY: CGFloat + switch position { + case .top: blockTopY = imageHeight - margin + case .bottom: blockTopY = blockHeight + margin + } + + let style = NSMutableParagraphStyle() + style.alignment = .center + + // The classic look is a black OUTLINE around white glyphs. A negative + // `.strokeWidth` tells AppKit to stroke AND fill (a positive value strokes + // only, which would render hollow letters). + let attributes: [NSAttributedString.Key: Any] = [ + .font: font, + .foregroundColor: NSColor.white, + .strokeColor: NSColor.black, + .strokeWidth: -CGFloat(fit.fontSize) * 0.14, + .paragraphStyle: style, + ] + + for (index, line) in fit.lines.enumerated() { + let y = blockTopY - lineHeight * CGFloat(index + 1) + let rect = NSRect( + x: 0, y: y + (lineHeight - font.ascender + font.descender) / 2, + width: imageWidth, height: lineHeight) + line.draw(in: rect, withAttributes: attributes) + } + } + + /// The heaviest condensed face available. Impact — the canonical meme font — does + /// not ship with macOS, so this walks a preference list and ends at a bold system + /// font, which is always present. + static func captionFont(size: Double) -> NSFont { + let candidates = ["Impact", "Haettenschweiler", "Arial Black", "HelveticaNeue-CondensedBlack"] + for name in candidates { + if let font = NSFont(name: name, size: CGFloat(size)) { return font } + } + return NSFont.systemFont(ofSize: CGFloat(size), weight: .black) + } + + // MARK: - Export + + /// PNG data for an image, for both the save panel and the share sheet. + static func pngData(for image: NSImage) -> Data? { + guard let tiff = image.tiffRepresentation, + let rep = NSBitmapImageRep(data: tiff) else { return nil } + return rep.representation(using: .png, properties: [:]) + } +} diff --git a/plugins/MemeGenerator/MemeTemplateService.swift b/plugins/MemeGenerator/MemeTemplateService.swift new file mode 100644 index 0000000..f918dd0 --- /dev/null +++ b/plugins/MemeGenerator/MemeTemplateService.swift @@ -0,0 +1,77 @@ +import AppKit +import Foundation + +/// The Meme Generator's only network access (spike). +/// +/// Two GETs against imgflip's public, key-less endpoints: +/// +/// * `https://api.imgflip.com/get_memes` — the template catalog (~100 popular +/// templates as JSON). No auth, no account, read-only. +/// * `https://i.imgflip.com/.jpg` — the blank template image. +/// +/// **Nothing is ever uploaded.** The user's dictation, the LLM's captions, and the +/// finished meme all stay on the Mac — captioning is done locally by `MemeRenderer` +/// precisely so no text has to leave. The app is local-first; this plugin's Settings +/// row states its network use, and the plugin is off by default. +enum MemeTemplateService { + + /// Failures worth telling the user apart. + enum ServiceError: LocalizedError { + case badResponse(status: Int) + case emptyCatalog + case undecodableImage + + var errorDescription: String? { + switch self { + case .badResponse(let status): + return "imgflip.com replied with HTTP \(status)." + case .emptyCatalog: + return "imgflip.com returned no templates." + case .undecodableImage: + return "the downloaded template wasn't a readable image." + } + } + } + + static let catalogURL = URL(string: "https://api.imgflip.com/get_memes")! + + /// A short timeout: this sits in front of a user waiting on a meme, so failing + /// fast and saying so beats a long hang. + private static let session: URLSession = { + let config = URLSessionConfiguration.ephemeral + config.timeoutIntervalForRequest = 15 + config.timeoutIntervalForResource = 30 + return URLSession(configuration: config) + }() + + /// Fetch the template catalog. Throws on transport failure, non-200, or an + /// `{"success": false}` envelope — all of which surface to the user verbatim. + static func fetchCatalog() async throws -> [MemeTemplate] { + let (data, response) = try await session.data(from: catalogURL) + try check(response) + + let decoded = try JSONDecoder().decode(MemeTemplateCatalogResponse.self, from: data) + let templates = decoded.templates + guard !templates.isEmpty else { throw ServiceError.emptyCatalog } + return templates + } + + /// Download a template's blank image. + static func fetchImage(_ template: MemeTemplate) async throws -> NSImage { + guard let url = URL(string: template.url) else { + throw ServiceError.undecodableImage + } + let (data, response) = try await session.data(from: url) + try check(response) + + guard let image = NSImage(data: data) else { throw ServiceError.undecodableImage } + return image + } + + private static func check(_ response: URLResponse) throws { + guard let http = response as? HTTPURLResponse else { return } + guard (200...299).contains(http.statusCode) else { + throw ServiceError.badResponse(status: http.statusCode) + } + } +} From 60494d65e9273884ea9202c6a686c47b44532e8f Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Sun, 2 Aug 2026 14:04:42 -0700 Subject: [PATCH 03/15] =?UTF-8?q?fix(meme-plugin):=20unreadable=20captions?= =?UTF-8?q?=20=E2=80=94=20strokeWidth=20is=20a=20percentage,=20not=20point?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit -fontSize*0.14 double-scaled the outline to ~12% of the font size; at caption sizes the neighboring glyphs' black strokes swallowed each other's white fill and lines rendered as solid merged blobs (owner screenshot). The attribute is already font-size-relative — use the classic ~4% outline. Co-Authored-By: Claude --- plugins/MemeGenerator/MemeRenderer.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/MemeGenerator/MemeRenderer.swift b/plugins/MemeGenerator/MemeRenderer.swift index 36bcb06..560581d 100644 --- a/plugins/MemeGenerator/MemeRenderer.swift +++ b/plugins/MemeGenerator/MemeRenderer.swift @@ -113,12 +113,16 @@ enum MemeRenderer { // The classic look is a black OUTLINE around white glyphs. A negative // `.strokeWidth` tells AppKit to stroke AND fill (a positive value strokes - // only, which would render hollow letters). + // only, which would render hollow letters). CRUCIALLY the value is a + // PERCENTAGE of the font size, not points — scaling it by the font size + // double-scaled it to ~12%, thick enough that neighboring glyphs' black + // outlines swallowed each other's white interiors (the unreadable-blob + // bug). ~4% is the classic meme outline weight at any size. let attributes: [NSAttributedString.Key: Any] = [ .font: font, .foregroundColor: NSColor.white, .strokeColor: NSColor.black, - .strokeWidth: -CGFloat(fit.fontSize) * 0.14, + .strokeWidth: -4.0, .paragraphStyle: style, ] From 8da9c84c3bcad13a446513998f167b3769044133 Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Sun, 2 Aug 2026 14:56:31 -0700 Subject: [PATCH 04/15] spike(meme-plugin): ranked template candidates + manual caption editor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner feedback after live testing: "yoda meme" found nothing in imgflip's top 100 and silently rendered onto Drake, with no way to tell that had happened. Two changes, both aimed at replacing a confident guess with a visible choice. **Template selection v2.** The LLM now receives the ACTUAL catalog names and returns a RANKED list of up to five, copied verbatim. `parseRanked` drops any name that isn't in the catalog, so an invented "Yoda" is discarded rather than fuzzy-matched onto something unrelated — and an empty candidate list is a SUCCESS, the honest "not in this corpus" answer. The UI shows the candidates as a thumbnail strip (clicking one re-renders the same captions onto it) plus a searchable "Browse all" grid over the whole catalog. Falling back still happens — the user asked for a meme — but never without the strip, a warning, and a status line naming the corpus. v1's `MemeAI.prompt`/`parse` and `MemeTemplateMatcher.bestMatch` are DELETED rather than kept. `bestMatch`'s built-in "or the most popular one" is precisely the reported bug, and leaving a second tested-but-dead parser behind is the trap this spike should expose, not commit. Its replacement `ranked` refuses to guess: no match returns an empty list and the fallback POLICY moves to the call site, where it is visible. **Manual editor.** Captions are now an array of `CaptionBox` with NORMALIZED geometry (0-1, top-left origin) and font size as a share of image height — which is what lets a box dragged on the preview render identically into a full-resolution export, and lets positions survive a switch to a differently-sized template. Each box is a draggable handle over the preview, with text, size, width and font controls in a side panel, plus add/delete. The preview IS the export: both go through `MemeRenderer.render(template:boxes:)`, and the AppKit coordinate flip happens in exactly one place. Layout math stays host-independent and tested per the hot-swap note; the renderer supplies font metrics and does nothing else. Gates: swift test 2395 passed (0 failures) · appstate ratchet OK at 7051 · PLUGINS=1 ./build.sh, plain ./build.sh, and lean (WHISPERKIT=0 PARAKEET=0 SPARKLE=0 PLUGINS=1) all compile · no new warnings in the touched files. Co-Authored-By: Claude --- OpenWhisp/Services/MemeAI.swift | 275 ++++++--- OpenWhisp/Services/MemeCaptionLayout.swift | 233 +++++++- OpenWhisp/Services/MemeTemplateMatcher.swift | 120 ++-- .../MemeGeneratorTests.swift | 565 +++++++++++++----- .../MemeGenerator/MemeGeneratorModel.swift | 323 ++++++++-- plugins/MemeGenerator/MemeGeneratorView.swift | 457 +++++++++++++- plugins/MemeGenerator/MemeRenderer.swift | 218 ++++--- 7 files changed, 1751 insertions(+), 440 deletions(-) diff --git a/OpenWhisp/Services/MemeAI.swift b/OpenWhisp/Services/MemeAI.swift index 41c5b63..3c6139b 100644 --- a/OpenWhisp/Services/MemeAI.swift +++ b/OpenWhisp/Services/MemeAI.swift @@ -3,8 +3,8 @@ import Foundation /// The pure rules behind the Meme Generator plugin's LLM step (spike). /// /// One round-trip turns a spoken description ("make me the distracted boyfriend one -/// where the guy is looking at Rust and his girlfriend is Python") into three fields: -/// a **template query** used to pick a base image, and the **top/bottom captions**. +/// where the guy is looking at Rust and his girlfriend is Python") into a **ranked +/// list of template candidates** plus the **top/bottom captions**. /// /// Foundation-only, so both halves — what we ask for and what we're willing to /// accept back — are pinned by `swift test`. The app layer owns only the LLM @@ -14,121 +14,246 @@ import Foundation /// *content*: small local models wrap JSON in prose or code fences constantly, so we /// dig the object out (`ScratchpadAI`'s posture), but a response missing the fields /// is REJECTED rather than silently rendered as an empty meme. +/// +/// ## Why ranked candidates (v2) +/// +/// v1 asked for a single free-text `template_query` and matched it lexically against +/// the catalog. That silently produced nonsense: "yoda meme" isn't in imgflip's top +/// 100, scored below the matcher's threshold, and the user got Drake with no +/// indication that the corpus simply doesn't contain Yoda. +/// +/// v2 gives the model the ACTUAL catalog names and asks for a RANKED list of up to +/// five, verbatim. The parser then drops any name that isn't in that list — a model +/// that invents "Yoda" gets the candidate discarded rather than fuzzy-matched onto +/// something unrelated. When every candidate is dropped, that is reported as a +/// SUCCESS with an empty `templateNames`, and the UI states the corpus plainly +/// instead of quietly substituting a popular template. +/// +/// The v1 single-query prompt and parser were deleted rather than kept as a +/// fallback: two parsers where only one runs is exactly the dead-wiring trap this +/// spike is supposed to expose, not commit. public enum MemeAI { - // MARK: - Request + // MARK: - Response + + /// Why a response was refused. + public enum Rejection: Error, Equatable, Sendable { + /// Nothing usable came back (empty / whitespace only). + case empty + /// No JSON object could be found in the response. + case notJSON + /// JSON parsed, but there was no usable template AND no caption at all — + /// rendering this would produce a blank image on a random template. Note a + /// missing template with captions present is NOT this: that is the honest + /// "nothing in the corpus fits" answer and it succeeds. + case missingFields - /// The instruction sent with the user's spoken description. + public var reason: String { + switch self { + case .empty: return "the model returned nothing" + case .notJSON: return "the model didn't return the expected JSON" + case .missingFields: return "the model left the meme fields empty" + } + } + } + + /// Strip the wrapping models add around caption values: surrounding quotes and + /// stray whitespace. Keeps interior punctuation untouched. + private static func clean(_ value: String) -> String { + var s = value.trimmingCharacters(in: .whitespacesAndNewlines) + while s.count >= 2, + let first = s.first, let last = s.last, + (first == "\"" && last == "\"") || (first == "'" && last == "'") { + s = String(s.dropFirst().dropLast()).trimmingCharacters(in: .whitespacesAndNewlines) + } + return s + } + + // MARK: - v2: ranked candidates against the real catalog + + /// The maximum number of candidates we ask for (and accept). /// - /// Language note, same as the Scratchpad's AI prompts (PR #157): the captions - /// must stay in the user's language. Tiny local models translate their input - /// unprompted, and a meme dictated in Ukrainian coming back in English is a bug. - /// The `template_query` is the exception — it is matched against an - /// English-titled catalog, so it is pinned to English on purpose. - public static let prompt = """ - You turn a spoken description of a meme into the fields needed to build it. + /// Five is the most that fits a thumbnail strip without scrolling and the most a + /// user will actually consider before reaching for "Browse all". + public static let maxCandidates = 5 + + /// The instruction for the ranked-candidate round-trip. + /// + /// The catalog names are appended by `rankedUserPayload` rather than baked in + /// here, so this constant stays testable and the prompt stays one place. + /// + /// "Think about which templates fit before answering" is deliberate: the owner + /// asked for the model to consider alternatives rather than snap to one. It + /// costs a few tokens of reasoning in the response we then discard — the parser + /// digs the JSON out of prose regardless. + public static let rankedPrompt = """ + You pick meme templates for a spoken description. + + You are given a numbered list of the ONLY templates available. Think about which \ + ones could carry the joke, then rank your best options. - Reply with ONLY a JSON object, no preamble and no code fence, with exactly these keys: - {"template_query": "...", "top_text": "...", "bottom_text": "..."} + Reply with ONLY a JSON object, no preamble and no code fence: + {"templates": ["...", "...", "..."], "top_text": "...", "bottom_text": "..."} Rules: - 1. "template_query" names the meme template in ENGLISH — the well-known name if \ - the description implies one (e.g. "drake hotline bling", "distracted boyfriend", \ - "two buttons"), otherwise two or three words describing the image that would fit. - 2. "top_text" and "bottom_text" are the caption lines, in the SAME LANGUAGE as \ + 1. "templates" lists 1 to 5 template names COPIED EXACTLY from the list, best \ + first. Do not invent a name, do not reword one, do not use a name that is not on \ + the list. + 2. If nothing on the list fits the description well, still return the closest \ + options — but put the genuinely closest first. + 3. "top_text" and "bottom_text" are the caption lines, in the SAME LANGUAGE as \ the description. Do not translate them. - 3. Keep each caption short — a few words, meme-style. Do not add quotation marks \ - around them. - 4. If the description only calls for one line, put it in "top_text" and leave \ + 4. Keep each caption short — a few words, meme-style. No quotation marks around \ + them. + 5. If the description only calls for one line, put it in "top_text" and leave \ "bottom_text" as an empty string. - 5. Never invent a caption that contradicts the description. + 6. Never invent a caption that contradicts the description. """ - /// Build the user payload for a description. Trimmed; the caller guarantees - /// non-empty (the Generate button is disabled otherwise). - public static func userPayload(description: String) -> String { - "Meme description:\n" + description.trimmingCharacters(in: .whitespacesAndNewlines) - } + /// Build the user payload: the description plus the catalog the model must + /// choose from. + /// + /// The list is numbered because small models copy list items more reliably when + /// the items are visually delimited, and truncated to `limit` because a 100-name + /// list is already ~1k tokens and some local models have short contexts. The + /// catalog is popularity-ordered, so truncating drops the least likely picks. + public static func rankedUserPayload( + description: String, templateNames: [String], limit: Int = 100 + ) -> String { + let names = templateNames.prefix(max(0, limit)) + let list = names.enumerated() + .map { "\($0.offset + 1). \($0.element)" } + .joined(separator: "\n") - // MARK: - Response + return """ + Available templates: + \(list) + + Meme description: + \(description.trimmingCharacters(in: .whitespacesAndNewlines)) + """ + } - /// The three fields a successful round-trip yields. - public struct MemeSpec: Equatable, Sendable, Codable { - /// English-ish search phrase used to pick a template from the catalog. - public let templateQuery: String - /// Upper caption, in the user's language. May be empty. + /// A ranked pick: template names that exist in the catalog, plus the captions. + public struct RankedSpec: Equatable, Sendable { + /// Template names, best first, each guaranteed to appear in the catalog that + /// was passed to `parseRanked`. May be EMPTY when the model named only + /// templates that don't exist — the honest "not in this corpus" signal. + public let templateNames: [String] public let topText: String - /// Lower caption, in the user's language. May be empty. public let bottomText: String - public init(templateQuery: String, topText: String, bottomText: String) { - self.templateQuery = templateQuery + public init(templateNames: [String], topText: String, bottomText: String) { + self.templateNames = templateNames self.topText = topText self.bottomText = bottomText } + /// True when the model produced captions but no usable template — the caller + /// must show the corpus rather than silently substituting a popular template. + public var hasNoUsableTemplate: Bool { templateNames.isEmpty } + } + + /// The wire shape. Decoded leniently: `templates` may arrive as an array or, from + /// a model that ignored the schema, as a single string. + private struct RankedWire: Decodable { + let templates: [String] + let topText: String + let bottomText: String + private enum CodingKeys: String, CodingKey { - case templateQuery = "template_query" + case templates case topText = "top_text" case bottomText = "bottom_text" + // Aliases small models reach for when they drift off the schema. + case template + case templateQuery = "template_query" } - } - /// Why a response was refused. - public enum Rejection: Error, Equatable, Sendable { - /// Nothing usable came back (empty / whitespace only). - case empty - /// No JSON object could be found in the response. - case notJSON - /// JSON parsed, but there was no template query and no caption at all — - /// rendering this would produce a blank image on a random template. - case missingFields + init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) - public var reason: String { - switch self { - case .empty: return "the model returned nothing" - case .notJSON: return "the model didn't return the expected JSON" - case .missingFields: return "the model left the meme fields empty" + var names: [String] = [] + if let list = try? c.decode([String].self, forKey: .templates) { + names = list + } else if let single = try? c.decode(String.self, forKey: .templates) { + names = [single] + } else if let single = try? c.decode(String.self, forKey: .template) { + names = [single] + } else if let single = try? c.decode(String.self, forKey: .templateQuery) { + names = [single] } + templates = names + + topText = (try? c.decode(String.self, forKey: .topText)) ?? "" + bottomText = (try? c.decode(String.self, forKey: .bottomText)) ?? "" } } - /// Parse a raw completion into a `MemeSpec`. + /// Parse a ranked-candidate completion, keeping ONLY names that exist in + /// `catalogNames`. /// - /// Accepts a bare object, a fenced ```json block, and an object embedded in - /// surrounding prose — all three are things small local models actually emit. - public static func parse(_ raw: String) -> Result { + /// Validation is the point of this function. A model that answers "Yoda" for a + /// catalog without Yoda must have that candidate DROPPED, not fuzzy-matched — + /// fuzzy matching an invented name is exactly how v1 produced a confident Drake. + /// Matching against the catalog is case/punctuation-insensitive (models + /// re-capitalize constantly) but otherwise exact, and the returned names are the + /// catalog's own spelling so callers can look them up directly. + /// + /// Duplicates are collapsed, order is preserved, and the result is capped at + /// `maxCandidates`. + /// + /// Rejects only when the response isn't JSON at all or carries neither a caption + /// nor a candidate. An empty `templateNames` with captions present is a SUCCESS — + /// it is the "nothing in the corpus fits" answer the UI needs to state plainly. + public static func parseRanked( + _ raw: String, catalogNames: [String] + ) -> Result { let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return .failure(.empty) } - guard let object = firstJSONObject(in: trimmed) else { return .failure(.notJSON) } - guard let data = object.data(using: .utf8), - let spec = try? JSONDecoder().decode(MemeSpec.self, from: data) + guard let object = firstJSONObject(in: trimmed), + let data = object.data(using: .utf8), + let wire = try? JSONDecoder().decode(RankedWire.self, from: data) else { return .failure(.notJSON) } - let cleaned = MemeSpec( - templateQuery: clean(spec.templateQuery), - topText: clean(spec.topText), - bottomText: clean(spec.bottomText)) + let names = validate(wire.templates, against: catalogNames) + let top = clean(wire.topText) + let bottom = clean(wire.bottomText) - // A spec with no query AND no captions is not a meme. One empty caption is - // fine (rule 4 in the prompt asks for exactly that on single-line memes). - if cleaned.templateQuery.isEmpty, cleaned.topText.isEmpty, cleaned.bottomText.isEmpty { + if names.isEmpty, top.isEmpty, bottom.isEmpty { return .failure(.missingFields) } - return .success(cleaned) + return .success(RankedSpec(templateNames: names, topText: top, bottomText: bottom)) } - /// Strip the wrapping models add around caption values: surrounding quotes and - /// stray whitespace. Keeps interior punctuation untouched. - private static func clean(_ value: String) -> String { - var s = value.trimmingCharacters(in: .whitespacesAndNewlines) - while s.count >= 2, - let first = s.first, let last = s.last, - (first == "\"" && last == "\"") || (first == "'" && last == "'") { - s = String(s.dropFirst().dropLast()).trimmingCharacters(in: .whitespacesAndNewlines) + /// Keep the proposed names that exist in the catalog, in the model's order, + /// de-duplicated, capped, and returned in the CATALOG's spelling. + /// + /// Exposed for tests: this is the whole anti-hallucination rule. + public static func validate( + _ proposed: [String], against catalogNames: [String] + ) -> [String] { + // Key on the normalized form so "DRAKE HOTLINE BLING" and "Drake Hotline + // Bling" resolve to the same catalog entry. First occurrence wins, which + // matches the catalog's popularity order on a duplicate name. + var byNormalized: [String: String] = [:] + for name in catalogNames { + let key = MemeTemplateMatcher.normalize(name) + guard !key.isEmpty, byNormalized[key] == nil else { continue } + byNormalized[key] = name } - return s + + var kept: [String] = [] + var seen = Set() + for candidate in proposed { + let key = MemeTemplateMatcher.normalize(clean(candidate)) + guard let canonical = byNormalized[key], !seen.contains(key) else { continue } + seen.insert(key) + kept.append(canonical) + if kept.count == maxCandidates { break } + } + return kept } /// Extract the first balanced `{...}` run from a string, ignoring braces that diff --git a/OpenWhisp/Services/MemeCaptionLayout.swift b/OpenWhisp/Services/MemeCaptionLayout.swift index ab4e543..17a1da8 100644 --- a/OpenWhisp/Services/MemeCaptionLayout.swift +++ b/OpenWhisp/Services/MemeCaptionLayout.swift @@ -114,31 +114,218 @@ public enum MemeCaptionLayout { return Fit(lines: lines, fontSize: minFontSize) } - // MARK: - Geometry + // MARK: - Caption boxes (manual editor) - /// Where a caption block sits on the image. - public enum Position: String, Equatable, Sendable { - case top - case bottom + /// One editable caption on the image. + /// + /// **Coordinates are NORMALIZED** (0…1, origin TOP-LEFT like every UI framework + /// the editor draws in) rather than pixels. That is what lets a box dragged on a + /// 520pt-wide preview render identically into a 1200px export, and what lets a + /// box survive being moved to a template with different dimensions when the user + /// picks another candidate — the whole point of the candidate strip is that the + /// captions carry over. + /// + /// `fontSize` is likewise a SHARE of image height, not points, for the same + /// reason: a 0.11 caption looks the same on a 400px and a 1200px template. + /// + /// `fontName` is a face name resolved by the renderer (`nil` = the renderer's + /// default meme face). Kept as a string rather than a font object so the box + /// model stays Foundation-only and testable. + public struct CaptionBox: Equatable, Sendable, Identifiable, Codable { + public let id: UUID + /// The caption text as the user typed it. Uppercasing happens at render time + /// so the editor shows what was typed. + public var text: String + /// Horizontal center, 0 = left edge, 1 = right edge. + public var centerX: Double + /// Vertical center, 0 = TOP edge, 1 = bottom edge. + public var centerY: Double + /// Font size as a share of image height. + public var fontSizeShare: Double + /// Width available to the box, as a share of image width. + public var widthShare: Double + /// Face name, or nil for the renderer's default. + public var fontName: String? + + public init( + id: UUID = UUID(), + text: String, + centerX: Double, + centerY: Double, + fontSizeShare: Double = CaptionBox.defaultFontSizeShare, + widthShare: Double = CaptionBox.defaultWidthShare, + fontName: String? = nil + ) { + self.id = id + self.text = text + self.centerX = centerX + self.centerY = centerY + self.fontSizeShare = fontSizeShare + self.widthShare = widthShare + self.fontName = fontName + } + + /// The classic caption size — carried over from v1's fixed layout so an + /// AI-seeded meme looks the same as it did before the editor existed. + public static let defaultFontSizeShare: Double = 0.11 + /// Full width minus a 5% margin per side: the caption never runs to the bezel. + public static let defaultWidthShare: Double = 0.90 + + /// Slider bounds for the editor. Below the floor text is unreadable; above + /// the ceiling a single word fills the image. + public static let minimumFontSizeShare: Double = 0.02 + public static let maximumFontSizeShare: Double = 0.30 + } + + /// Clamp a box's geometry into the image. + /// + /// Applied on every drag and every slider move so a box can never be parked + /// outside the canvas (where it would render invisibly and look like data loss). + /// The center is clamped to the edges rather than inset by half the box height — + /// letting a caption bleed off the edge is a legitimate meme look, and a caption + /// the user can still grab matters more than one that is fully contained. + public static func clamped(_ box: CaptionBox) -> CaptionBox { + var out = box + out.centerX = min(max(box.centerX, 0), 1) + out.centerY = min(max(box.centerY, 0), 1) + out.fontSizeShare = min( + max(box.fontSizeShare, CaptionBox.minimumFontSizeShare), + CaptionBox.maximumFontSizeShare) + out.widthShare = min(max(box.widthShare, 0.05), 1) + return out + } + + /// The two boxes the AI path seeds: classic top and bottom captions. + /// + /// Positioned at 0.12 / 0.88 of the height — far enough in that a two-line + /// caption still sits inside the image, which is where v1's top/bottom blocks + /// landed. An empty caption still gets a box so the editor has something to type + /// into rather than making the user hunt for "Add text box". + public static func seedBoxes(topText: String, bottomText: String) -> [CaptionBox] { + [ + CaptionBox(text: topText, centerX: 0.5, centerY: 0.12), + CaptionBox(text: bottomText, centerX: 0.5, centerY: 0.88), + ] + } + + /// A box resolved into pixels for one specific image size, with its text already + /// wrapped and shrunk to fit. + public struct BoxLayout: Equatable, Sendable { + public let id: UUID + /// Wrapped, uppercased lines. + public let lines: [String] + /// Font size in PIXELS for this image. + public let fontSize: Double + /// Face name, or nil for the renderer's default. + public let fontName: String? + /// Box center in pixels, origin TOP-LEFT. + public let centerX: Double + public let centerY: Double + /// Total height of the wrapped block, in pixels. + public let blockHeight: Double + /// Available width in pixels (what the text was wrapped to). + public let maxWidth: Double + + public init( + id: UUID, lines: [String], fontSize: Double, fontName: String?, + centerX: Double, centerY: Double, blockHeight: Double, maxWidth: Double + ) { + self.id = id + self.lines = lines + self.fontSize = fontSize + self.fontName = fontName + self.centerX = centerX + self.centerY = centerY + self.blockHeight = blockHeight + self.maxWidth = maxWidth + } + + /// Y of the block's TOP edge, origin top-left. The renderer flips this into + /// AppKit's bottom-left space; the editor uses it directly. + public var blockTopY: Double { centerY - blockHeight / 2 } } - /// The share of the image height one caption block may occupy before it must - /// shrink. Classic memes keep captions in the top/bottom third. - public static let captionHeightShare: Double = 0.32 + /// The multiplier from font size to line height, shared by the layout math, the + /// renderer, and the editor's hit-testing so all three agree on box height. + public static let lineHeightRatio: Double = 1.15 + + /// Resolve one box against an image size: normalized geometry → pixels, text + /// wrapped and shrunk to fit the box's width. + /// + /// The user's `fontSizeShare` is the CEILING, not a fixed size — a caption too + /// long for its box still shrinks rather than overflowing, exactly like the AI + /// path. Setting a size and getting overflowing text would be a worse editor + /// than one that quietly keeps the caption inside its box; the user can widen + /// the box or shorten the text if they want it bigger. + /// + /// `measure` and `lineHeight` are injected so this stays pure (tests pass a + /// deterministic stub, the renderer passes real font metrics). `measure` + /// receives the box's `fontName` so per-box faces are measured with the right + /// metrics. + public static func layout( + box: CaptionBox, + imageWidth: Double, + imageHeight: Double, + measure: (_ text: String, _ fontSize: Double, _ fontName: String?) -> Double + ) -> BoxLayout { + let safe = clamped(box) + let maxWidth = max(1, imageWidth * safe.widthShare) + let ceiling = max(1, imageHeight * safe.fontSizeShare) + // Floor at a quarter of the requested size: a caption may shrink to stay in + // its box, but never to the point of being unreadable — beyond that it + // clips, which `fit` already degrades to honestly. + let floor = max(1, ceiling * 0.25) + + let fit = fit( + caption: safe.text, + maxWidth: maxWidth, + // Vertically the box is free to grow — it's the WIDTH the user controls. + // Capping at the image height stops an absurd caption from becoming a + // block taller than the canvas. + maxHeight: imageHeight, + maxFontSize: ceiling, + minFontSize: floor, + step: max(1, ceiling * 0.05), + measure: { text, size in measure(text, size, safe.fontName) }, + lineHeight: { $0 * lineHeightRatio }) - /// The horizontal inset applied to both sides, as a share of image width — the - /// caption never runs to the bezel. - public static let horizontalInsetShare: Double = 0.05 + let blockHeight = Double(fit.lines.count) * fit.fontSize * lineHeightRatio - /// Ideal font size for an image height, before shrink-to-fit. Scaled to the - /// image so a 1200px template and a 400px one look the same when displayed. - public static func idealFontSize(imageHeight: Double) -> Double { - max(12, imageHeight * 0.11) + return BoxLayout( + id: safe.id, + lines: fit.lines, + fontSize: fit.fontSize, + fontName: safe.fontName, + centerX: safe.centerX * imageWidth, + centerY: safe.centerY * imageHeight, + blockHeight: blockHeight, + maxWidth: maxWidth) } - /// Smallest acceptable font size for an image height. - public static func minimumFontSize(imageHeight: Double) -> Double { - max(8, imageHeight * 0.04) + /// Resolve every box, dropping the ones with nothing to draw. + /// + /// Empty boxes are dropped at RENDER time only — the editor keeps them so the + /// user has a handle to type into. This is why the export and the preview can + /// disagree by exactly the empty boxes, which is the correct behaviour. + public static func layout( + boxes: [CaptionBox], + imageWidth: Double, + imageHeight: Double, + measure: (_ text: String, _ fontSize: Double, _ fontName: String?) -> Double + ) -> [BoxLayout] { + boxes + .map { layout(box: $0, imageWidth: imageWidth, imageHeight: imageHeight, measure: measure) } + .filter { !$0.lines.isEmpty } + } + + /// Where "Add text box" drops a new caption. + /// + /// Stacked down the middle so successive adds don't land on top of each other + /// (which would look like the button did nothing). Wraps back to the top after + /// filling the column rather than marching off the bottom edge. + public static func newBoxCenter(existingCount: Int) -> (x: Double, y: Double) { + let slots = [0.5, 0.3, 0.7, 0.2, 0.8, 0.4, 0.6] + return (0.5, slots[existingCount % slots.count]) } // MARK: - Export naming @@ -168,4 +355,14 @@ public enum MemeCaptionLayout { return (slug.isEmpty ? "meme" : String(slug)) + ".png" } + + /// The same naming rule, driven by the box model so an edited meme exports under + /// the name the user actually sees rather than the AI's original captions. + public static func suggestedFileName(boxes: [CaptionBox]) -> String { + let joined = boxes + .map(\.text) + .filter { !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } + .joined(separator: " ") + return suggestedFileName(topText: joined, bottomText: "") + } } diff --git a/OpenWhisp/Services/MemeTemplateMatcher.swift b/OpenWhisp/Services/MemeTemplateMatcher.swift index 5f83764..5f653b3 100644 --- a/OpenWhisp/Services/MemeTemplateMatcher.swift +++ b/OpenWhisp/Services/MemeTemplateMatcher.swift @@ -8,8 +8,8 @@ import Foundation /// no audio, and no LLM output is ever sent to imgflip. public struct MemeTemplate: Equatable, Sendable, Codable, Identifiable { public let id: String - /// Display name, e.g. "Distracted Boyfriend". English — this is what the LLM's - /// `template_query` is matched against. + /// Display name, e.g. "Distracted Boyfriend". English — this is the string the + /// LLM is asked to copy verbatim, and the key candidates are validated against. public let name: String /// Direct image URL (jpg/png) for the blank template. public let url: String @@ -37,11 +37,11 @@ public struct MemeTemplateCatalogResponse: Decodable, Sendable { public var templates: [MemeTemplate] { success ? (data?.memes ?? []) : [] } } -/// Picks the template whose name best matches the LLM's `template_query` (spike). +/// Local, lexical template lookup over the catalog (spike). /// -/// Matching is LOCAL and lexical — the catalog is ~100 short English names, so a -/// token-overlap score beats anything heavier and keeps the whole decision pure and -/// testable. No second network call, no embedding model. +/// Matching is LOCAL — the catalog is ~100 short English names, so a token-overlap +/// score beats anything heavier and keeps the whole decision pure and testable. No +/// second network call, no embedding model. /// /// Scoring, highest first: /// 1. **Exact** name match (case/punctuation-insensitive) — 1000. @@ -52,56 +52,88 @@ public struct MemeTemplateCatalogResponse: Decodable, Sendable { /// /// Ties break on the catalog's own order, which is popularity-ranked — the more /// famous template is the better guess. +/// +/// **v2 note.** v1's `bestMatch` — "return the single best template, or the most +/// popular one if nothing scores" — was DELETED. That built-in fallback is precisely +/// the reported bug: it turned "yoda meme" into a confident Drake with no way for the +/// caller to know. The two survivors both refuse to guess, and each returns an empty +/// result the UI must handle: `ranked` (score-ordered candidates) and `search` (the +/// user's own query over the whole corpus). public enum MemeTemplateMatcher { - /// The minimum score worth accepting. Below this we'd be picking essentially at - /// random, and a confidently-wrong template is worse than an honest fallback, so - /// the caller falls back to the catalog's most popular entry instead. - public static let minimumScore = 100 + // MARK: - Fallback ranking (v2) - /// Find the best template for a query. Returns nil only for an empty catalog. + /// Rank the catalog by how well each name scores against a free-text query, + /// keeping only entries that score at all. /// - /// A query that matches nothing yields the catalog's FIRST entry (most popular) - /// rather than nil: the user dictated a meme and should get a meme. The caller - /// surfaces that this was a fallback. - public static func bestMatch( - for query: String, in catalog: [MemeTemplate] - ) -> Match? { - guard !catalog.isEmpty else { return nil } - + /// This is the v2 fallback: when the model proposes only template names that + /// don't exist (the "yoda" case), the candidate strip would otherwise be pure + /// popularity — the same blind guess v1 made, just with more thumbnails. Scoring + /// the user's own description against the catalog at least puts anything lexically + /// related in front of them first. + /// + /// This NEVER substitutes a popular template for a zero score: + /// a query matching nothing returns an EMPTY list, and the caller is responsible + /// for deciding what to show and for saying that nothing matched. That split — + /// ranking here, fallback policy at the call site — is what makes the fallback + /// visible instead of silent. + public static func ranked( + for query: String, in catalog: [MemeTemplate], limit: Int + ) -> [MemeTemplate] { let queryTokens = tokens(in: query) - guard !queryTokens.isEmpty else { - return Match(template: catalog[0], score: 0, isFallback: true) + guard !queryTokens.isEmpty, limit > 0 else { return [] } + + // A named struct rather than a tuple chain: the inferred-tuple version was + // too much for the type checker ("unable to type-check in reasonable time"). + struct Scored { + let index: Int + let template: MemeTemplate + let score: Int } - var best: (template: MemeTemplate, score: Int)? - - for template in catalog { - let score = score(query: query, queryTokens: queryTokens, name: template.name) - // Strictly greater keeps the earlier (more popular) template on a tie. - if score > (best?.score ?? Int.min) { - best = (template, score) - } + // `index` preserves the catalog's popularity order as the tie-break, since + // `sorted(by:)` is not guaranteed stable. + var scored: [Scored] = [] + for (index, template) in catalog.enumerated() { + let value = score(query: query, queryTokens: queryTokens, name: template.name) + guard value > 0 else { continue } + scored.append(Scored(index: index, template: template, score: value)) } - guard let best, best.score >= minimumScore else { - return Match(template: catalog[0], score: best?.score ?? 0, isFallback: true) + scored.sort { left, right in + left.score == right.score ? left.index < right.index : left.score > right.score } - return Match(template: best.template, score: best.score, isFallback: false) + return scored.prefix(limit).map(\.template) } - /// The chosen template plus why. - public struct Match: Equatable, Sendable { - public let template: MemeTemplate - public let score: Int - /// True when nothing scored well enough and the most-popular template was - /// substituted — the UI says so rather than pretending it understood. - public let isFallback: Bool - - public init(template: MemeTemplate, score: Int, isFallback: Bool) { - self.template = template - self.score = score - self.isFallback = isFallback + // MARK: - Browse all (manual override) + + /// Filter the catalog by a user-typed search string, for the "Browse all" grid. + /// + /// This is the honest answer to "yoda isn't in the corpus": the user can see and + /// search every template the plugin has, and pick one the model never proposed. + /// + /// Deliberately unlike v1's deleted `bestMatch`: no scoring, no threshold, and + /// above all **no fallback**. A search that matches nothing returns nothing, because an + /// empty grid saying "no templates match" is the entire point — silently + /// substituting popular templates is the bug this whole change exists to fix. + /// + /// Matching is substring-based over the normalized name so it is + /// case/diacritic/punctuation-insensitive, and multi-word queries match when + /// EVERY token appears somewhere in the name (so "drake bling" finds "Drake + /// Hotline Bling"). Catalog order — imgflip's popularity ranking — is preserved. + public static func search(_ query: String, in catalog: [MemeTemplate]) -> [MemeTemplate] { + let normalizedQuery = normalize(query) + guard !normalizedQuery.isEmpty else { return catalog } + + // Split on the normalized form, not `tokens`: stopword removal is right for + // matching an LLM's phrase, wrong for a user typing letter by letter — a + // search for "the" should still narrow the list rather than reset it. + let needles = normalizedQuery.split(separator: " ").map(String.init) + + return catalog.filter { template in + let haystack = normalize(template.name) + return needles.allSatisfy { haystack.contains($0) } } } diff --git a/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift b/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift index e6c660a..b20fce3 100644 --- a/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift +++ b/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift @@ -1,115 +1,12 @@ import XCTest @testable import OpenWhispCore -/// Covers the Meme Generator plugin's pure layer (spike/plugin-system): the LLM -/// response parser, the template matcher, and the caption layout rules. +/// Covers the Meme Generator plugin's pure layer (spike/plugin-system): the ranked +/// candidate parser, lexical ranking and search over the template catalog, the +/// caption layout rules, and the editable caption-box model. final class MemeGeneratorTests: XCTestCase { - // MARK: - MemeAI.parse - - func testParsesBareJSONObject() { - let result = MemeAI.parse(""" - {"template_query":"drake hotline bling","top_text":"writing tests","bottom_text":"shipping on friday"} - """) - guard case .success(let spec) = result else { return XCTFail("expected success, got \(result)") } - XCTAssertEqual(spec.templateQuery, "drake hotline bling") - XCTAssertEqual(spec.topText, "writing tests") - XCTAssertEqual(spec.bottomText, "shipping on friday") - } - - /// Small local models fence their JSON constantly; that's packaging, not content. - func testParsesFencedJSON() { - let result = MemeAI.parse(""" - Sure! Here you go: - - ```json - {"template_query":"two buttons","top_text":"A","bottom_text":"B"} - ``` - """) - guard case .success(let spec) = result else { return XCTFail("expected success") } - XCTAssertEqual(spec.templateQuery, "two buttons") - XCTAssertEqual(spec.topText, "A") - } - - func testParsesJSONEmbeddedInProse() { - let result = MemeAI.parse( - "I think this works: {\"template_query\":\"success kid\",\"top_text\":\"x\",\"bottom_text\":\"y\"} — enjoy!") - guard case .success(let spec) = result else { return XCTFail("expected success") } - XCTAssertEqual(spec.templateQuery, "success kid") - } - - /// A caption containing a brace must not end the object scan early. - func testBracesInsideStringValuesDoNotTruncateTheScan() { - let result = MemeAI.parse( - "{\"template_query\":\"code\",\"top_text\":\"when you write {\",\"bottom_text\":\"and forget }\"}") - guard case .success(let spec) = result else { return XCTFail("expected success") } - XCTAssertEqual(spec.topText, "when you write {") - XCTAssertEqual(spec.bottomText, "and forget }") - } - - func testStripsSurroundingQuotesFromCaptions() { - let result = MemeAI.parse( - "{\"template_query\":\"x\",\"top_text\":\"\\\"quoted\\\"\",\"bottom_text\":\"'single'\"}") - guard case .success(let spec) = result else { return XCTFail("expected success") } - XCTAssertEqual(spec.topText, "quoted") - XCTAssertEqual(spec.bottomText, "single") - } - - /// Rule 4 in the prompt asks for exactly this on single-line memes. - func testOneEmptyCaptionIsAccepted() { - let result = MemeAI.parse( - "{\"template_query\":\"success kid\",\"top_text\":\"it compiled\",\"bottom_text\":\"\"}") - guard case .success(let spec) = result else { return XCTFail("expected success") } - XCTAssertEqual(spec.bottomText, "") - } - - func testEmptyResponseIsRejected() { - XCTAssertEqual(rejection(MemeAI.parse(" \n ")), .empty) - } - - func testNonJSONResponseIsRejected() { - XCTAssertEqual(rejection(MemeAI.parse("I'm sorry, I can't help with that.")), .notJSON) - } - - func testUnbalancedBracesAreRejected() { - XCTAssertEqual(rejection(MemeAI.parse("{\"template_query\": \"x\"")), .notJSON) - } - - /// An all-empty spec would render a blank caption on an arbitrary template — - /// worse than an honest error. - func testAllEmptyFieldsAreRejected() { - XCTAssertEqual( - rejection(MemeAI.parse("{\"template_query\":\"\",\"top_text\":\"\",\"bottom_text\":\"\"}")), - .missingFields) - } - - func testRejectionReasonsAreHumanReadable() { - for r in [MemeAI.Rejection.empty, .notJSON, .missingFields] { - XCTAssertFalse(r.reason.isEmpty) - } - } - - /// The prompt must keep captions in the user's language (PR #157 guard) while - /// pinning the template query to English for catalog matching. - func testPromptPinsCaptionLanguageAndEnglishTemplateQuery() { - let prompt = MemeAI.prompt.lowercased() - XCTAssertTrue(prompt.contains("same language")) - XCTAssertTrue(prompt.contains("do not translate")) - XCTAssertTrue(prompt.contains("english")) - } - - func testUserPayloadTrimsAndLabelsTheDescription() { - XCTAssertEqual( - MemeAI.userPayload(description: " a cat "), - "Meme description:\na cat") - } - - private func rejection(_ result: Result) -> MemeAI.Rejection? { - if case .failure(let r) = result { return r } - return nil - } - - // MARK: - Template matching + // MARK: - Lexical ranking (the fallback when the model names nothing real) private let catalog: [MemeTemplate] = [ MemeTemplate(id: "1", name: "Drake Hotline Bling", url: "u1", width: 1200, height: 1200), @@ -118,53 +15,66 @@ final class MemeGeneratorTests: XCTestCase { MemeTemplate(id: "4", name: "Success Kid", url: "u4", width: 500, height: 500), ] - func testExactNameMatchWins() { - let match = MemeTemplateMatcher.bestMatch(for: "Two Buttons", in: catalog) - XCTAssertEqual(match?.template.id, "3") - XCTAssertFalse(match?.isFallback ?? true) + func testRankedPutsAnExactNameFirst() { + XCTAssertEqual( + MemeTemplateMatcher.ranked(for: "Two Buttons", in: catalog, limit: 5).first?.id, "3") } - func testMatchIsCaseAndPunctuationInsensitive() { + func testRankedIsCaseAndPunctuationInsensitive() { XCTAssertEqual( - MemeTemplateMatcher.bestMatch(for: "two-buttons!", in: catalog)?.template.id, "3") + MemeTemplateMatcher.ranked(for: "two-buttons!", in: catalog, limit: 5).first?.id, "3") } /// A partial name the user actually says ("the drake one") must find the template. - func testPartialPhraseMatches() { + func testRankedFindsPartialPhrases() { XCTAssertEqual( - MemeTemplateMatcher.bestMatch(for: "drake", in: catalog)?.template.id, "1") + MemeTemplateMatcher.ranked(for: "drake", in: catalog, limit: 5).first?.id, "1") XCTAssertEqual( - MemeTemplateMatcher.bestMatch(for: "distracted boyfriend", in: catalog)?.template.id, "2") + MemeTemplateMatcher.ranked(for: "distracted boyfriend", in: catalog, limit: 5).first?.id, "2") } - func testStopwordsAndTheWordMemeAreIgnored() { + func testRankedIgnoresStopwordsAndTheWordMeme() { XCTAssertEqual( - MemeTemplateMatcher.bestMatch(for: "the success kid meme", in: catalog)?.template.id, "4") + MemeTemplateMatcher.ranked(for: "the success kid meme", in: catalog, limit: 5).first?.id, "4") } - /// A confidently-wrong template is worse than an admitted guess. - func testUnmatchableQueryFallsBackToMostPopularAndSaysSo() { - let match = MemeTemplateMatcher.bestMatch(for: "zzzz qqqq", in: catalog) - XCTAssertEqual(match?.template.id, "1", "falls back to the catalog's first (most popular) entry") - XCTAssertTrue(match?.isFallback ?? false) + /// The core v2 rule: ranking REFUSES to guess. v1's `bestMatch` answered this + /// same query with a confident Drake, which is the reported bug. + func testRankedReturnsNothingWhenNothingScores() { + XCTAssertEqual(MemeTemplateMatcher.ranked(for: "yoda", in: catalog, limit: 5), []) + XCTAssertEqual(MemeTemplateMatcher.ranked(for: "zzzz qqqq", in: catalog, limit: 5), []) } - func testEmptyQueryFallsBack() { - let match = MemeTemplateMatcher.bestMatch(for: " ", in: catalog) - XCTAssertTrue(match?.isFallback ?? false) + func testRankedReturnsNothingForAnEmptyQuery() { + XCTAssertEqual(MemeTemplateMatcher.ranked(for: " ", in: catalog, limit: 5), []) } - func testEmptyCatalogYieldsNoMatch() { - XCTAssertNil(MemeTemplateMatcher.bestMatch(for: "drake", in: [])) + func testRankedHandlesAnEmptyCatalog() { + XCTAssertEqual(MemeTemplateMatcher.ranked(for: "drake", in: [], limit: 5), []) + } + + func testRankedRespectsTheLimit() { + let many = (1...10).map { + MemeTemplate(id: "\($0)", name: "Angry Cat \($0)", url: "u", width: 10, height: 10) + } + XCTAssertEqual(MemeTemplateMatcher.ranked(for: "angry cat", in: many, limit: 3).count, 3) + XCTAssertEqual(MemeTemplateMatcher.ranked(for: "angry", in: many, limit: 0), []) } /// Ties break on catalog order, which imgflip returns popularity-ranked. - func testTieBreaksTowardTheMorePopularTemplate() { + func testRankedTieBreaksTowardTheMorePopularTemplate() { let tied = [ MemeTemplate(id: "popular", name: "Angry Cat", url: "u", width: 10, height: 10), MemeTemplate(id: "less", name: "Angry Dog", url: "u", width: 10, height: 10), ] - XCTAssertEqual(MemeTemplateMatcher.bestMatch(for: "angry", in: tied)?.template.id, "popular") + XCTAssertEqual( + MemeTemplateMatcher.ranked(for: "angry", in: tied, limit: 5).map(\.id), + ["popular", "less"]) + } + + func testRankedOrdersBetterMatchesFirst() { + let ranked = MemeTemplateMatcher.ranked(for: "success kid", in: catalog, limit: 5) + XCTAssertEqual(ranked.first?.id, "4", "the exact match outranks any partial") } func testCatalogResponseDecodesImgflipShape() throws { @@ -259,18 +169,391 @@ final class MemeGeneratorTests: XCTestCase { XCTAssertEqual(fit.lines, []) } - /// Font sizes scale with the image so a 1200px and a 400px template look alike. - func testFontSizesScaleWithImageHeight() { - XCTAssertGreaterThan( - MemeCaptionLayout.idealFontSize(imageHeight: 1200), - MemeCaptionLayout.idealFontSize(imageHeight: 400)) - XCTAssertGreaterThan( - MemeCaptionLayout.idealFontSize(imageHeight: 500), - MemeCaptionLayout.minimumFontSize(imageHeight: 500)) + + // MARK: - v2: ranked candidate parsing + // + // The rule under test is the fix for the "yoda meme → silent Drake" report: a + // template name the model invents must be DROPPED, never fuzzy-matched. + + private let catalogNames = [ + "Drake Hotline Bling", "Distracted Boyfriend", "Two Buttons", "Success Kid", + ] + + func testRankedParsePreservesModelOrder() { + let result = MemeAI.parseRanked(""" + {"templates":["Two Buttons","Drake Hotline Bling"],"top_text":"ship it","bottom_text":"test it"} + """, catalogNames: catalogNames) + guard case .success(let spec) = result else { return XCTFail("expected success, got \(result)") } + XCTAssertEqual(spec.templateNames, ["Two Buttons", "Drake Hotline Bling"]) + XCTAssertEqual(spec.topText, "ship it") + XCTAssertEqual(spec.bottomText, "test it") + } + + /// The headline bug: "yoda" is not in imgflip's top 100. The candidate must be + /// dropped so the UI can say the corpus doesn't contain it, rather than matched + /// onto an unrelated popular template. + func testHallucinatedTemplateNameIsDropped() { + let result = MemeAI.parseRanked(""" + {"templates":["Yoda","Baby Yoda"],"top_text":"do or do not","bottom_text":""} + """, catalogNames: catalogNames) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertTrue(spec.templateNames.isEmpty) + XCTAssertTrue(spec.hasNoUsableTemplate, "the UI needs this to show the corpus honestly") + XCTAssertEqual(spec.topText, "do or do not", "captions survive an unusable template") + } + + func testInventedNamesAreDroppedButValidOnesSurvive() { + let result = MemeAI.parseRanked(""" + {"templates":["Yoda","Success Kid","Gandalf"],"top_text":"a","bottom_text":"b"} + """, catalogNames: catalogNames) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.templateNames, ["Success Kid"]) + } + + /// Models re-capitalize and re-punctuate names constantly; that's packaging. + func testCandidateMatchingIsCaseAndPunctuationInsensitiveAndReturnsCatalogSpelling() { + let result = MemeAI.parseRanked(""" + {"templates":["drake hotline bling!","TWO BUTTONS"],"top_text":"x","bottom_text":""} + """, catalogNames: catalogNames) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual( + spec.templateNames, ["Drake Hotline Bling", "Two Buttons"], + "returned in the catalog's own spelling so callers can look them up") + } + + func testDuplicateCandidatesAreCollapsed() { + XCTAssertEqual( + MemeAI.validate(["Success Kid", "success kid", "Success Kid"], against: catalogNames), + ["Success Kid"]) + } + + func testCandidateListIsCappedAtFive() { + let names = (1...10).map { "T\($0)" } + let kept = MemeAI.validate(names, against: names) + XCTAssertEqual(kept.count, MemeAI.maxCandidates) + XCTAssertEqual(kept, ["T1", "T2", "T3", "T4", "T5"], "keeps the model's ranking") + } + + /// A model that ignores the array schema and sends one string still works. + func testSingleStringTemplateFieldIsAccepted() { + for key in ["templates", "template", "template_query"] { + let raw = "{\"\(key)\":\"Success Kid\",\"top_text\":\"x\",\"bottom_text\":\"\"}" + guard case .success(let spec) = MemeAI.parseRanked(raw, catalogNames: catalogNames) else { + return XCTFail("expected success for key \(key)") + } + XCTAssertEqual(spec.templateNames, ["Success Kid"], "key: \(key)") + } + } + + func testRankedParseDigsJSONOutOfFencedProse() { + let result = MemeAI.parseRanked(""" + Let me think — Drake fits best here. + + ```json + {"templates":["Drake Hotline Bling"],"top_text":"no","bottom_text":"yes"} + ``` + """, catalogNames: catalogNames) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.templateNames, ["Drake Hotline Bling"]) + } + + func testRankedParseRejectsNonJSON() { + XCTAssertEqual( + rejection(MemeAI.parseRanked("I can't help with that.", catalogNames: catalogNames)), + .notJSON) + } + + func testRankedParseRejectsEmpty() { + XCTAssertEqual(rejection(MemeAI.parseRanked(" ", catalogNames: catalogNames)), .empty) + } + + /// No template AND no captions is nothing at all — reject rather than render. + func testRankedParseRejectsWhenNothingUsableCameBack() { + XCTAssertEqual( + rejection(MemeAI.parseRanked( + "{\"templates\":[\"Yoda\"],\"top_text\":\"\",\"bottom_text\":\"\"}", + catalogNames: catalogNames)), + .missingFields) + } + + func testEmptyCatalogDropsEveryCandidate() { + XCTAssertEqual(MemeAI.validate(["Drake Hotline Bling"], against: []), []) + } + + private func rejection(_ result: Result) -> MemeAI.Rejection? { + if case .failure(let r) = result { return r } + return nil + } + + // MARK: - v2: the ranked prompt + + func testRankedPromptForbidsInventingNamesAndPinsCaptionLanguage() { + let prompt = MemeAI.rankedPrompt.lowercased() + XCTAssertTrue(prompt.contains("copied exactly")) + XCTAssertTrue(prompt.contains("do not invent")) + XCTAssertTrue(prompt.contains("same language")) + XCTAssertTrue(prompt.contains("do not translate")) + } + + func testRankedPayloadNumbersTheCatalogAndCarriesTheDescription() { + let payload = MemeAI.rankedUserPayload( + description: " two buttons about deploys ", + templateNames: ["Drake Hotline Bling", "Two Buttons"]) + XCTAssertTrue(payload.contains("1. Drake Hotline Bling")) + XCTAssertTrue(payload.contains("2. Two Buttons")) + XCTAssertTrue(payload.contains("two buttons about deploys")) + XCTAssertFalse(payload.contains(" two buttons about deploys "), "description is trimmed") + } + + /// Short-context local models can't take a 100-name list; truncation drops the + /// least popular entries because the catalog is popularity-ordered. + func testRankedPayloadTruncatesTheCatalogToTheLimit() { + let names = (1...20).map { "Template \($0)" } + let payload = MemeAI.rankedUserPayload(description: "x", templateNames: names, limit: 5) + XCTAssertTrue(payload.contains("5. Template 5")) + XCTAssertFalse(payload.contains("6. Template 6")) + } + + // MARK: - Browse all: search filter + + func testSearchIsCaseInsensitiveSubstringOverNames() { + XCTAssertEqual( + MemeTemplateMatcher.search("DRAKE", in: catalog).map(\.id), ["1"]) + XCTAssertEqual( + MemeTemplateMatcher.search("button", in: catalog).map(\.id), ["3"]) + } + + func testSearchRequiresEveryTokenToAppear() { + XCTAssertEqual( + MemeTemplateMatcher.search("drake bling", in: catalog).map(\.id), ["1"]) + XCTAssertEqual( + MemeTemplateMatcher.search("drake buttons", in: catalog).map(\.id), []) + } + + func testSearchIgnoresPunctuation() { + XCTAssertEqual( + MemeTemplateMatcher.search("two-buttons!", in: catalog).map(\.id), ["3"]) + } + + func testEmptySearchReturnsTheWholeCatalogInOrder() { + XCTAssertEqual( + MemeTemplateMatcher.search(" ", in: catalog).map(\.id), ["1", "2", "3", "4"]) + } + + /// The whole point of Browse all: no fallback, ever. An empty grid is the honest + /// answer for a query the corpus can't serve — this is the "yoda" case, and + /// neither surviving entry point is allowed to invent a substitute. + func testSearchNeverFallsBackToAPopularTemplate() { + XCTAssertEqual(MemeTemplateMatcher.search("yoda", in: catalog), []) + XCTAssertEqual( + MemeTemplateMatcher.ranked(for: "yoda", in: catalog, limit: 5), [], + "ranking refuses too — v1's bestMatch answered Drake here, which was the bug") + } + + func testSearchPreservesPopularityOrder() { + let hits = MemeTemplateMatcher.search("t", in: catalog).map(\.id) + XCTAssertEqual(hits, catalog.filter { hits.contains($0.id) }.map(\.id)) + } + + // MARK: - Caption box model + + /// Deterministic metrics for the box layout: every character is `size * 0.5` + /// wide, and a named font is 20% wider so per-box faces are observably used. + private func boxMeasure(_ text: String, _ size: Double, _ fontName: String?) -> Double { + Double(text.count) * size * 0.5 * (fontName == nil ? 1.0 : 1.2) + } + + func testSeedBoxesArePlacedTopAndBottom() { + let boxes = MemeCaptionLayout.seedBoxes(topText: "up", bottomText: "down") + XCTAssertEqual(boxes.count, 2) + XCTAssertEqual(boxes[0].text, "up") + XCTAssertEqual(boxes[1].text, "down") + XCTAssertLessThan(boxes[0].centerY, 0.5, "origin is TOP-left, so the top box has small y") + XCTAssertGreaterThan(boxes[1].centerY, 0.5) + XCTAssertEqual(boxes[0].centerX, 0.5) + } + + /// Empty captions still get a box — the editor needs a handle to type into. + func testSeedBoxesExistEvenForEmptyCaptions() { + XCTAssertEqual(MemeCaptionLayout.seedBoxes(topText: "", bottomText: "").count, 2) } - func testFontSizesHaveAbsoluteFloors() { - XCTAssertGreaterThanOrEqual(MemeCaptionLayout.idealFontSize(imageHeight: 1), 12) - XCTAssertGreaterThanOrEqual(MemeCaptionLayout.minimumFontSize(imageHeight: 1), 8) + /// Normalized coordinates are the whole reason the box model exists: the same box + /// must land proportionally identically on a preview and on a full-res export. + func testNormalizedGeometryScalesWithImageSize() { + let box = MemeCaptionLayout.CaptionBox(text: "hi", centerX: 0.25, centerY: 0.75) + + let small = MemeCaptionLayout.layout( + box: box, imageWidth: 400, imageHeight: 400, measure: boxMeasure) + let large = MemeCaptionLayout.layout( + box: box, imageWidth: 1200, imageHeight: 1200, measure: boxMeasure) + + XCTAssertEqual(small.centerX, 100) + XCTAssertEqual(large.centerX, 300) + XCTAssertEqual(large.centerX / small.centerX, 3, accuracy: 0.0001) + XCTAssertEqual(large.fontSize / small.fontSize, 3, accuracy: 0.0001, + "font size is a share of height, so it scales too") + } + + func testBoxCenterIsInPixelsWithTopLeftOrigin() { + let box = MemeCaptionLayout.CaptionBox(text: "hi", centerX: 0.5, centerY: 0.1) + let layout = MemeCaptionLayout.layout( + box: box, imageWidth: 1000, imageHeight: 500, measure: boxMeasure) + XCTAssertEqual(layout.centerX, 500) + XCTAssertEqual(layout.centerY, 50, "0.1 of the height, measured from the TOP") + XCTAssertLessThan(layout.blockTopY, layout.centerY) + } + + func testBoxTextIsUppercasedAndWrappedToTheBoxWidth() { + var box = MemeCaptionLayout.CaptionBox(text: "one two three four five", centerX: 0.5, centerY: 0.5) + box.widthShare = 0.5 + let layout = MemeCaptionLayout.layout( + box: box, imageWidth: 400, imageHeight: 400, measure: boxMeasure) + + XCTAssertGreaterThan(layout.lines.count, 1, "should have wrapped") + XCTAssertTrue(layout.lines.allSatisfy { $0 == $0.uppercased() }) + let widest = layout.lines.map { boxMeasure($0, layout.fontSize, nil) }.max() ?? 0 + XCTAssertLessThanOrEqual(widest, layout.maxWidth) + } + + /// The user's font size is a ceiling: a caption too long for its box shrinks + /// rather than overflowing. + func testOversizedCaptionShrinksBelowTheRequestedFontSize() { + var box = MemeCaptionLayout.CaptionBox( + text: "a very long caption that cannot possibly fit at full size", + centerX: 0.5, centerY: 0.5) + box.fontSizeShare = 0.3 + box.widthShare = 0.3 + + let layout = MemeCaptionLayout.layout( + box: box, imageWidth: 400, imageHeight: 400, measure: boxMeasure) + XCTAssertLessThan(layout.fontSize, 400 * 0.3) + XCTAssertGreaterThan(layout.fontSize, 0) + } + + func testPerBoxFontNameIsPassedToTheMeasurer() { + var plain = MemeCaptionLayout.CaptionBox(text: "one two three", centerX: 0.5, centerY: 0.5) + plain.widthShare = 0.4 + var named = plain + named.fontName = "Impact" + + let a = MemeCaptionLayout.layout(box: plain, imageWidth: 400, imageHeight: 400, measure: boxMeasure) + let b = MemeCaptionLayout.layout(box: named, imageWidth: 400, imageHeight: 400, measure: boxMeasure) + + XCTAssertEqual(b.fontName, "Impact") + XCTAssertNotEqual( + a.lines, b.lines, + "the wider face must wrap differently — proof the font name reached the metrics") + } + + func testClampKeepsBoxesOnTheCanvas() { + let wild = MemeCaptionLayout.CaptionBox( + text: "x", centerX: -3, centerY: 9, + fontSizeShare: 99, widthShare: 50) + let safe = MemeCaptionLayout.clamped(wild) + XCTAssertEqual(safe.centerX, 0) + XCTAssertEqual(safe.centerY, 1) + XCTAssertEqual(safe.fontSizeShare, MemeCaptionLayout.CaptionBox.maximumFontSizeShare) + XCTAssertEqual(safe.widthShare, 1) + } + + func testClampRaisesATinyFontToTheReadableFloor() { + let tiny = MemeCaptionLayout.CaptionBox( + text: "x", centerX: 0.5, centerY: 0.5, fontSizeShare: 0.0001) + XCTAssertEqual( + MemeCaptionLayout.clamped(tiny).fontSizeShare, + MemeCaptionLayout.CaptionBox.minimumFontSizeShare) + } + + func testClampPreservesIdentityAndText() { + let box = MemeCaptionLayout.CaptionBox(text: "keep me", centerX: 5, centerY: 0.5) + let safe = MemeCaptionLayout.clamped(box) + XCTAssertEqual(safe.id, box.id) + XCTAssertEqual(safe.text, "keep me") + } + + /// Out-of-range geometry must not survive into the render either — layout + /// clamps on the way through, so a corrupt box can't draw off-canvas. + func testLayoutClampsBeforeResolvingPixels() { + let box = MemeCaptionLayout.CaptionBox(text: "x", centerX: 4, centerY: -1) + let layout = MemeCaptionLayout.layout( + box: box, imageWidth: 200, imageHeight: 200, measure: boxMeasure) + XCTAssertEqual(layout.centerX, 200) + XCTAssertEqual(layout.centerY, 0) + } + + /// Empty boxes are dropped at render time but kept in the editor — the export + /// and the box list are allowed to differ by exactly the empty ones. + func testEmptyBoxesAreDroppedFromTheRenderList() { + let boxes = [ + MemeCaptionLayout.CaptionBox(text: "visible", centerX: 0.5, centerY: 0.2), + MemeCaptionLayout.CaptionBox(text: " ", centerX: 0.5, centerY: 0.8), + ] + let layouts = MemeCaptionLayout.layout( + boxes: boxes, imageWidth: 400, imageHeight: 400, measure: boxMeasure) + XCTAssertEqual(layouts.count, 1) + XCTAssertEqual(layouts[0].lines, ["VISIBLE"]) + } + + func testLayoutPreservesBoxIdentityForHitTesting() { + let boxes = MemeCaptionLayout.seedBoxes(topText: "a", bottomText: "b") + let layouts = MemeCaptionLayout.layout( + boxes: boxes, imageWidth: 400, imageHeight: 400, measure: boxMeasure) + XCTAssertEqual(layouts.map(\.id), boxes.map(\.id)) + } + + func testBlockHeightMatchesLineCountTimesLineHeight() { + var box = MemeCaptionLayout.CaptionBox(text: "one two three four", centerX: 0.5, centerY: 0.5) + box.widthShare = 0.4 + let layout = MemeCaptionLayout.layout( + box: box, imageWidth: 400, imageHeight: 400, measure: boxMeasure) + XCTAssertEqual( + layout.blockHeight, + Double(layout.lines.count) * layout.fontSize * MemeCaptionLayout.lineHeightRatio, + accuracy: 0.0001) + } + + /// Successive "Add text box" clicks must not stack invisibly on top of each other. + func testNewBoxCentersDoNotCollideForSuccessiveAdds() { + let ys = (0..<5).map { MemeCaptionLayout.newBoxCenter(existingCount: $0).y } + XCTAssertEqual(Set(ys).count, ys.count, "each new box lands somewhere free") + XCTAssertTrue(ys.allSatisfy { $0 > 0 && $0 < 1 }) + } + + func testNewBoxCentersWrapRatherThanRunOffTheCanvas() { + for count in 0..<40 { + let center = MemeCaptionLayout.newBoxCenter(existingCount: count) + XCTAssertTrue(center.y > 0 && center.y < 1, "count \(count)") + XCTAssertEqual(center.x, 0.5) + } + } + + /// An edited meme exports under the name the user sees, not the AI's originals. + func testFileNameFollowsTheEditedBoxes() { + let boxes = [ + MemeCaptionLayout.CaptionBox(text: "Ship It", centerX: 0.5, centerY: 0.2), + MemeCaptionLayout.CaptionBox(text: " ", centerX: 0.5, centerY: 0.5), + MemeCaptionLayout.CaptionBox(text: "On Friday", centerX: 0.5, centerY: 0.8), + ] + XCTAssertEqual(MemeCaptionLayout.suggestedFileName(boxes: boxes), "ship-it-on-friday.png") + } + + func testFileNameFallsBackWhenEveryBoxIsEmpty() { + XCTAssertEqual( + MemeCaptionLayout.suggestedFileName(boxes: [ + MemeCaptionLayout.CaptionBox(text: "", centerX: 0.5, centerY: 0.5) + ]), + "meme.png") + } + + /// Boxes are persisted/carried between templates, so the Codable shape matters. + func testCaptionBoxRoundTripsThroughCodable() throws { + var box = MemeCaptionLayout.CaptionBox(text: "keep", centerX: 0.3, centerY: 0.7) + box.fontName = "Impact" + box.fontSizeShare = 0.09 + let decoded = try JSONDecoder().decode( + MemeCaptionLayout.CaptionBox.self, + from: try JSONEncoder().encode(box)) + XCTAssertEqual(decoded, box) } } diff --git a/plugins/MemeGenerator/MemeGeneratorModel.swift b/plugins/MemeGenerator/MemeGeneratorModel.swift index 6e97192..10999d5 100644 --- a/plugins/MemeGenerator/MemeGeneratorModel.swift +++ b/plugins/MemeGenerator/MemeGeneratorModel.swift @@ -6,16 +6,28 @@ import SwiftUI /// The pipeline, end to end: /// /// 1. The user DICTATES a description (or types it) into `description`. -/// 2. `generate()` sends it to the configured LLM via the injected `aiCall` seam — -/// the same closure-injection `ScratchpadModel` uses, so this model never touches -/// AppState and stays stubbable. -/// 3. The reply is parsed by the pure `MemeAI.parse` into a template query + captions. -/// 4. The imgflip template catalog is fetched (once per session, then cached) and the -/// query is matched locally by `MemeTemplateMatcher`. -/// 5. The template image is downloaded and captioned LOCALLY by `MemeRenderer`. +/// 2. The imgflip template catalog is fetched (once per session, then cached). +/// 3. `generate()` sends the description **plus the catalog's actual template names** +/// to the configured LLM via the injected `aiCall` seam — the same +/// closure-injection `ScratchpadModel` uses, so this model never touches AppState +/// and stays stubbable. +/// 4. The reply is parsed by the pure `MemeAI.parseRanked`, which keeps only names +/// that really exist in the catalog. The result is a RANKED candidate list. +/// 5. The best candidate auto-renders; the rest sit in a thumbnail strip. The user +/// can click another candidate, or open "Browse all" and pick any of the 100. +/// 6. The chosen template is downloaded and captioned LOCALLY by `MemeRenderer` from +/// the caption BOX model, which the manual editor then mutates. /// -/// Only steps 4–5 touch the network, and only to GET images — no user text is ever -/// sent to imgflip. +/// Only steps 2 and 6 touch the network, and only to GET images — no user text is +/// ever sent to imgflip. +/// +/// ## v2 — why candidates +/// +/// v1 asked the model for a free-text template name and lexically matched it. "yoda +/// meme" scored below threshold and silently rendered onto Drake. v2 constrains the +/// model to the real corpus and, when nothing fits, SAYS SO — the fallback still +/// happens (a meme is better than an error) but never without the candidate strip and +/// Browse-all visible, so the user immediately sees it was a guess. @MainActor final class MemeGeneratorModel: ObservableObject { @@ -33,13 +45,45 @@ final class MemeGeneratorModel: ObservableObject { /// True while a generate round-trip is in flight (disables the button). @Published private(set) var isBusy: Bool = false - /// The captions of the current meme, surfaced so the user can see what the model - /// wrote (and tell a bad caption from a bad template). - @Published private(set) var topText: String = "" - @Published private(set) var bottomText: String = "" + /// The editable caption boxes. This is the SINGLE source of truth for what the + /// meme says and where — the AI seeds it, the editor mutates it, and both the + /// preview and the export render from it (WYSIWYG). + @Published var boxes: [MemeCaptionLayout.CaptionBox] = [] + + /// The box the editor's side panel is editing, if any. + @Published var selectedBoxID: UUID? + + /// The ranked template candidates the model proposed, best first. Empty until a + /// generate has run. + @Published private(set) var candidates: [MemeTemplate] = [] + + /// The template currently rendered. + @Published private(set) var selectedTemplate: MemeTemplate? - /// The template that was chosen, for the "used X" line. - @Published private(set) var templateName: String = "" + /// The whole catalog, for "Browse all". Empty until fetched. + @Published private(set) var catalog: [MemeTemplate] = [] + + /// True when the model named only templates that don't exist in the corpus and we + /// fell back. Drives the honest "not in this corpus" warning — the candidate strip + /// and Browse all stay visible so the fallback is never silent. Cleared once the + /// user picks a template themselves; the choice is theirs from then on. + @Published private(set) var didFallBack: Bool = false + + /// True when the CURRENT candidate strip is a fallback list rather than the + /// model's own picks. Unlike `didFallBack` this describes the strip's contents, so + /// it survives the user selecting a template and the strip's label stays accurate. + @Published private(set) var candidatesAreFallback: Bool = false + + /// The user's Browse-all search text. Filtering is local and pure. + @Published var searchText: String = "" + + /// The templates matching `searchText`, in popularity order. + var searchResults: [MemeTemplate] { + MemeTemplateMatcher.search(searchText, in: catalog) + } + + /// The template name shown in the "used X" line. + var templateName: String { selectedTemplate?.name ?? "" } // MARK: - Injected seams @@ -62,8 +106,13 @@ final class MemeGeneratorModel: ObservableObject { resolveAIModel = resolveModel } - /// The template catalog, fetched once and reused for the rest of the session. - private var catalog: [MemeTemplate] = [] + /// Downloaded template images, keyed by template id, so clicking back and forth + /// between candidates doesn't re-download. + private var imageCache: [String: NSImage] = [:] + + /// The base image of the currently selected template — kept so an editor tweak + /// re-renders instantly without a network round-trip. + private var baseImage: NSImage? /// Monotonic ticket so a late result from a superseded request can't overwrite a /// newer meme (the `ScratchpadAISession` fence, simplified for one surface). @@ -91,6 +140,9 @@ final class MemeGeneratorModel: ObservableObject { !isBusy && !description.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } + /// True once there is something to browse — gates the "Browse all" affordance. + var canBrowse: Bool { !catalog.isEmpty } + // MARK: - Generate func generate() { @@ -112,72 +164,223 @@ final class MemeGeneratorModel: ObservableObject { ticket += 1 let myTicket = ticket isBusy = true - status = "Asking the model…" - let prompt = MemeAI.userPayload(description: description) Task { [weak self] in + guard let self else { return } + + // The catalog is fetched BEFORE the LLM call now — v2 puts the real + // template names in the prompt, so we can't ask until we have them. + guard await self.ensureCatalog(ticket: myTicket) else { return } + guard !self.isCancelled, myTicket == self.ticket else { return } + + self.status = "Asking the model…" + let names = self.catalog.map(\.name) + let payload = MemeAI.rankedUserPayload( + description: self.description, templateNames: names) + do { - let raw = try await aiCall(MemeAI.prompt, prompt, resolved) - guard let self, !self.isCancelled, myTicket == self.ticket else { return } + let raw = try await aiCall(MemeAI.rankedPrompt, payload, resolved) + guard !self.isCancelled, myTicket == self.ticket else { return } - switch MemeAI.parse(raw) { + switch MemeAI.parseRanked(raw, catalogNames: names) { case .failure(let rejection): self.finish(myTicket, status: "Couldn't build the meme — \(rejection.reason).") case .success(let spec): - await self.buildMeme(spec, ticket: myTicket) + await self.applyRanked(spec, ticket: myTicket) } } catch { - guard let self, !self.isCancelled, myTicket == self.ticket else { return } + guard !self.isCancelled, myTicket == self.ticket else { return } self.finish(myTicket, status: "The model request failed — \(Self.reason(error))") } } } - /// Fetch the catalog (if needed), match a template, download it, caption it. - private func buildMeme(_ spec: MemeAI.MemeSpec, ticket myTicket: Int) async { - status = "Finding a template…" + /// Fetch the catalog once per session. Returns false when it failed (and has + /// already reported why). + private func ensureCatalog(ticket myTicket: Int) async -> Bool { + guard catalog.isEmpty else { return true } + + status = "Loading templates…" + do { + let fetched = try await MemeTemplateService.fetchCatalog() + guard !isCancelled, myTicket == ticket else { return false } + catalog = fetched + return true + } catch { + finish(myTicket, status: + "Couldn't reach imgflip.com for templates — \(Self.reason(error)) " + + "The meme generator needs a connection to fetch template images.") + return false + } + } + + /// Turn a validated ranked spec into candidates + a rendered best guess. + private func applyRanked(_ spec: MemeAI.RankedSpec, ticket myTicket: Int) async { + // Resolve names back to templates. `parseRanked` guarantees every name is in + // the catalog, so a nil here would be a bug rather than model noise. + var picks = spec.templateNames.compactMap { name in + catalog.first { $0.name == name } + } + + didFallBack = picks.isEmpty + candidatesAreFallback = picks.isEmpty + if picks.isEmpty { + // Nothing the model named exists in the corpus. Still render something — + // the user asked for a meme — but ONLY alongside a visible candidate strip + // and Browse all, so the substitution is obvious rather than silent. That + // is the whole fix for the "yoda → Drake with no explanation" report. + // + // Rank by the user's OWN words first: if the description happens to + // mention something that is in the catalog, that beats blind popularity. + // Popularity is the last resort, and only to fill the strip. + let lexical = MemeTemplateMatcher.ranked( + for: description, in: catalog, limit: MemeAI.maxCandidates) + let popular = catalog.prefix(MemeAI.maxCandidates) + picks = lexical + popular.filter { candidate in + !lexical.contains { $0.id == candidate.id } + } + picks = Array(picks.prefix(MemeAI.maxCandidates)) + } + + candidates = picks + boxes = MemeCaptionLayout.seedBoxes(topText: spec.topText, bottomText: spec.bottomText) + selectedBoxID = boxes.first?.id + + guard let best = picks.first else { + finish(myTicket, status: "imgflip returned no templates to choose from.") + return + } + + await renderTemplate(best, ticket: myTicket, isNewGeneration: true) + } + + // MARK: - Template selection + + /// Re-render the current captions onto another template. + /// + /// The captions carry over unchanged — that is the point of the candidate strip + /// and of normalized box geometry: switching templates is a *presentation* choice, + /// not a reason to lose the user's text or their box positions. + func select(template: MemeTemplate) { + guard !isBusy else { return } + guard template.id != selectedTemplate?.id else { return } + + // Seed boxes if the user picked a template before ever generating (Browse all + // is reachable as soon as the catalog is loaded). + if boxes.isEmpty { + boxes = MemeCaptionLayout.seedBoxes(topText: "", bottomText: "") + selectedBoxID = boxes.first?.id + } + + // The user has now chosen deliberately, so this is no longer a fallback — + // leaving the warning up would be nagging about a decision they just made. + didFallBack = false - if catalog.isEmpty { + ticket += 1 + let myTicket = ticket + isBusy = true + Task { [weak self] in + await self?.renderTemplate(template, ticket: myTicket, isNewGeneration: false) + } + } + + /// Load a template's image (cached) and render the current boxes onto it. + private func renderTemplate( + _ template: MemeTemplate, ticket myTicket: Int, isNewGeneration: Bool + ) async { + let image: NSImage + if let cached = imageCache[template.id] { + image = cached + } else { + status = "Downloading \(template.name)…" do { - catalog = try await MemeTemplateService.fetchCatalog() + image = try await MemeTemplateService.fetchImage(template) } catch { finish(myTicket, status: - "Couldn't reach imgflip.com for templates — \(Self.reason(error)) " - + "The meme generator needs a connection to fetch template images.") + "Couldn't download the template image — \(Self.reason(error))") return } + guard !isCancelled, myTicket == ticket else { return } + imageCache[template.id] = image } guard !isCancelled, myTicket == ticket else { return } - guard let match = MemeTemplateMatcher.bestMatch(for: spec.templateQuery, in: catalog) else { - finish(myTicket, status: "imgflip returned no templates to choose from.") - return - } + baseImage = image + selectedTemplate = template + redraw() - status = "Downloading \(match.template.name)…" - let template: NSImage - do { - template = try await MemeTemplateService.fetchImage(match.template) - } catch { - finish(myTicket, status: "Couldn't download the template image — \(Self.reason(error))") - return + finish(myTicket, status: statusLine(isNewGeneration: isNewGeneration)) + } + + /// The line under the controls: honest about a fallback, quiet otherwise. + private func statusLine(isNewGeneration: Bool) -> String { + guard let selectedTemplate else { return "" } + if didFallBack, isNewGeneration { + return "Nothing in the corpus matched — this is imgflip's top 100, and your " + + "meme isn't one of them. Showing \(selectedTemplate.name); pick another " + + "below or Browse all." } - guard !isCancelled, myTicket == ticket else { return } + return "Used \(selectedTemplate.name). Drag the captions to reposition them." + } - let rendered = MemeRenderer.render( - template: template, topText: spec.topText, bottomText: spec.bottomText) + // MARK: - Manual editor - meme = rendered - topText = spec.topText - bottomText = spec.bottomText - templateName = match.template.name + /// Re-render the current boxes onto the current template. + /// + /// Called on every editor mutation. Rendering is local CoreGraphics onto an + /// already-downloaded image, so this is cheap enough to run per keystroke and per + /// drag frame — no debounce, and the preview is therefore literally the export. + func redraw() { + guard let baseImage else { return } + meme = MemeRenderer.render(template: baseImage, boxes: boxes) + } - // Say so when the template was a guess — a confidently-wrong pick the user - // can't explain is worse than an admitted fallback. - let note = match.isFallback - ? "Couldn't match \"\(spec.templateQuery)\" — used the most popular template instead." - : "Used \(match.template.name)." - finish(myTicket, status: note) + /// Mutate one box and re-render. The single funnel for every editor edit, so + /// clamping and redrawing can't be forgotten at a call site. + func updateBox(id: UUID, _ mutate: (inout MemeCaptionLayout.CaptionBox) -> Void) { + guard let index = boxes.firstIndex(where: { $0.id == id }) else { return } + var box = boxes[index] + mutate(&box) + boxes[index] = MemeCaptionLayout.clamped(box) + redraw() + } + + /// Add an empty caption box, placed so it doesn't land on the previous one. + func addBox() { + let center = MemeCaptionLayout.newBoxCenter(existingCount: boxes.count) + let box = MemeCaptionLayout.CaptionBox( + text: "New text", centerX: center.x, centerY: center.y) + boxes.append(box) + selectedBoxID = box.id + redraw() + } + + func deleteBox(id: UUID) { + boxes.removeAll { $0.id == id } + if selectedBoxID == id { selectedBoxID = boxes.first?.id } + redraw() + } + + /// The currently selected box, for the side panel's bindings. + var selectedBox: MemeCaptionLayout.CaptionBox? { + guard let selectedBoxID else { return nil } + return boxes.first { $0.id == selectedBoxID } + } + + // MARK: - Browse all + + /// Load the catalog so "Browse all" works before any generate has run. + func loadCatalogIfNeeded() { + guard catalog.isEmpty, !isBusy else { return } + ticket += 1 + let myTicket = ticket + isBusy = true + Task { [weak self] in + guard let self else { return } + if await self.ensureCatalog(ticket: myTicket) { + self.finish(myTicket, status: "Loaded \(self.catalog.count) templates.") + } + } } private func finish(_ myTicket: Int, status newStatus: String) { @@ -202,6 +405,12 @@ final class MemeGeneratorModel: ObservableObject { // MARK: - Export / share + /// The filename an export should suggest — driven by the EDITED boxes, so a meme + /// the user rewrote saves under what it actually says. + var suggestedFileName: String { + MemeCaptionLayout.suggestedFileName(boxes: boxes) + } + /// Write the meme to a user-chosen file. Returns whether anything was written. @discardableResult func exportPNG() -> Bool { @@ -212,8 +421,7 @@ final class MemeGeneratorModel: ObservableObject { let panel = NSSavePanel() panel.allowedContentTypes = [.png] - panel.nameFieldStringValue = MemeCaptionLayout.suggestedFileName( - topText: topText, bottomText: bottomText) + panel.nameFieldStringValue = suggestedFileName panel.canCreateDirectories = true guard panel.runModal() == .OK, let url = panel.url else { return false } @@ -226,5 +434,4 @@ final class MemeGeneratorModel: ObservableObject { return false } } - } diff --git a/plugins/MemeGenerator/MemeGeneratorView.swift b/plugins/MemeGenerator/MemeGeneratorView.swift index 5cb4c21..ad552f4 100644 --- a/plugins/MemeGenerator/MemeGeneratorView.swift +++ b/plugins/MemeGenerator/MemeGeneratorView.swift @@ -6,6 +6,18 @@ import SwiftUI /// Voice-first by construction: the description field is the first responder when the /// window opens, so pressing the dictation hotkey and speaking lands the words here /// with no typing and no clicking. "As fast as possible" is the whole point. +/// +/// ## v2 — "AI makes its best guess, human makes it perfect" +/// +/// Three surfaces sit between the model's answer and the finished meme: +/// +/// * **The candidate strip** — the model's ranked template picks as thumbnails. The +/// best one auto-renders; clicking another re-renders the SAME captions onto it. +/// * **Browse all** — a searchable grid of the whole corpus, so the user can override +/// the model entirely. This is the honest answer when the corpus doesn't contain +/// what they asked for. +/// * **The editor** — every caption is a draggable box over the preview, with text, +/// size and font controls in a side panel. struct MemeGeneratorView: View { @ObservedObject var model: MemeGeneratorModel @@ -13,24 +25,35 @@ struct MemeGeneratorView: View { /// Focuses the description editor on open so a dictation lands immediately. @FocusState private var descriptionFocused: Bool + /// Whether the Browse-all grid is showing. + @State private var isBrowsing = false + var body: some View { VStack(alignment: .leading, spacing: 12) { header descriptionEditor controls if !model.status.isEmpty { statusLine } - preview + if !model.candidates.isEmpty { candidateStrip } + + HStack(alignment: .top, spacing: 12) { + preview + if !model.boxes.isEmpty { editorPanel } + } } .padding(16) - .frame(minWidth: 520, minHeight: 560) + .frame(minWidth: 760, minHeight: 640) .onAppear { descriptionFocused = true } + .sheet(isPresented: $isBrowsing) { + BrowseAllSheet(model: model, isPresented: $isBrowsing) + } } private var header: some View { HStack(spacing: 8) { Image(systemName: PluginRegistry.memeGenerator.symbol) .foregroundStyle(.secondary) - Text("Describe the meme out loud — the model picks a template and writes the captions.") + Text("Describe the meme out loud — the model proposes templates and writes the captions. Then edit anything.") .font(.callout) .foregroundStyle(.secondary) .fixedSize(horizontal: false, vertical: true) @@ -41,7 +64,7 @@ struct MemeGeneratorView: View { VStack(alignment: .leading, spacing: 4) { TextEditor(text: $model.description) .font(.body) - .frame(minHeight: 70, maxHeight: 110) + .frame(minHeight: 54, maxHeight: 80) .overlay( RoundedRectangle(cornerRadius: 6) .stroke(Color.secondary.opacity(0.3))) @@ -79,6 +102,15 @@ struct MemeGeneratorView: View { ProgressView().controlSize(.small) } + // Always available (it loads the catalog on demand), so the user is never + // stuck with the model's idea of which templates exist. + Button { + model.loadCatalogIfNeeded() + isBrowsing = true + } label: { + Label("Browse all…", systemImage: "square.grid.2x2") + } + Spacer() Button { @@ -98,21 +130,79 @@ struct MemeGeneratorView: View { } private var statusLine: some View { - Text(model.status) - .font(.caption) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) + HStack(alignment: .top, spacing: 6) { + // A fallback is called out with a symbol as well as words — the owner's + // complaint was that the substitution was invisible. + if model.didFallBack { + Image(systemName: "exclamationmark.triangle") + .foregroundStyle(.orange) + } + Text(model.status) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } } + // MARK: - Candidate strip + + /// The model's ranked picks. Clicking one re-renders the same captions onto it. + private var candidateStrip: some View { + VStack(alignment: .leading, spacing: 4) { + Text(model.candidatesAreFallback + ? "Nothing in the corpus matched — closest and most popular instead:" + : "The model's picks, best first:") + .font(.caption) + .foregroundStyle(.secondary) + + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 8) { + ForEach(model.candidates) { template in + TemplateThumbnail( + template: template, + isSelected: template.id == model.selectedTemplate?.id, + width: 96) + .onTapGesture { model.select(template: template) } + } + } + .padding(.vertical, 2) + } + } + } + + // MARK: - Preview + editor overlay + @ViewBuilder private var preview: some View { if let meme = model.meme { - Image(nsImage: meme) - .resizable() - .scaledToFit() - .frame(maxWidth: .infinity, maxHeight: .infinity) - .accessibilityLabel( - "Meme preview. Top caption: \(model.topText). Bottom caption: \(model.bottomText).") + // The base image is the RENDERED meme (captions already burned in by the + // same code path the export uses, so this is genuinely WYSIWYG). The + // overlaid boxes are invisible drag handles positioned by the same + // normalized coordinates, which is why a handle always sits exactly on + // the text it moves. + GeometryReader { geo in + let fitted = Self.fittedRect( + imageSize: meme.size, in: geo.size) + + ZStack(alignment: .topLeading) { + Image(nsImage: meme) + .resizable() + .scaledToFit() + .frame(width: geo.size.width, height: geo.size.height) + .accessibilityLabel( + "Meme preview. Captions: " + + model.boxes.map(\.text).filter { !$0.isEmpty }.joined(separator: ", ")) + + ForEach(model.boxes) { box in + DragHandle( + model: model, + box: box, + canvas: fitted, + isSelected: box.id == model.selectedBoxID) + } + } + } + .frame(maxWidth: .infinity, maxHeight: .infinity) } else { RoundedRectangle(cornerRadius: 8) .fill(Color.secondary.opacity(0.08)) @@ -129,6 +219,59 @@ struct MemeGeneratorView: View { } } + /// Where a `scaledToFit` image actually lands inside its frame. + /// + /// SwiftUI letterboxes the image, so the drag handles must be positioned against + /// the IMAGE's rect, not the frame's — otherwise a handle drifts off the text on + /// any template whose aspect ratio differs from the pane's. + static func fittedRect(imageSize: CGSize, in container: CGSize) -> CGRect { + guard imageSize.width > 0, imageSize.height > 0, + container.width > 0, container.height > 0 else { + return CGRect(origin: .zero, size: container) + } + let scale = min(container.width / imageSize.width, container.height / imageSize.height) + let size = CGSize(width: imageSize.width * scale, height: imageSize.height * scale) + return CGRect( + x: (container.width - size.width) / 2, + y: (container.height - size.height) / 2, + width: size.width, height: size.height) + } + + // MARK: - Editor side panel + + private var editorPanel: some View { + VStack(alignment: .leading, spacing: 10) { + HStack { + Text("Text boxes").font(.headline) + Spacer() + Button { + model.addBox() + } label: { + Image(systemName: "plus") + } + .help("Add a text box") + } + + // The box list doubles as the selector for the controls below. + ForEach(model.boxes) { box in + BoxRow(model: model, box: box) + } + + Divider() + + if let selected = model.selectedBox { + BoxControls(model: model, box: selected) + } else { + Text("Select a text box to edit it.") + .font(.caption) + .foregroundStyle(.tertiary) + } + + Spacer() + } + .frame(width: 240) + } + /// Share the rendered PNG through the system picker. Writes to a temp file first /// because `NSSharingServicePicker` shares URLs far more widely than raw images /// (Mail/Messages/AirDrop all want a file). @@ -137,9 +280,7 @@ struct MemeGeneratorView: View { let data = MemeRenderer.pngData(for: meme) else { return } let url = FileManager.default.temporaryDirectory - .appendingPathComponent( - MemeCaptionLayout.suggestedFileName( - topText: model.topText, bottomText: model.bottomText)) + .appendingPathComponent(model.suggestedFileName) guard (try? data.write(to: url)) != nil else { return } guard let view = NSApp.keyWindow?.contentView else { return } @@ -147,3 +288,285 @@ struct MemeGeneratorView: View { picker.show(relativeTo: .zero, of: view, preferredEdge: .minY) } } + +// MARK: - Drag handle + +/// The invisible, draggable hit area sitting on top of one rendered caption. +/// +/// Deliberately NOT a second copy of the text: drawing the caption again in SwiftUI +/// would mean two renderers to keep in agreement, and they would drift (different +/// wrapping, different outline). Instead the burned-in render IS the visual, and this +/// is a positioned outline the user grabs. The tradeoff is that the handle is a +/// rectangle rather than glyph-tight, which is fine at spike quality. +private struct DragHandle: View { + + @ObservedObject var model: MemeGeneratorModel + let box: MemeCaptionLayout.CaptionBox + /// Where the image actually sits inside the preview frame. + let canvas: CGRect + let isSelected: Bool + + /// Live drag offset in points, applied on top of the box's committed position so + /// the handle tracks the cursor without a re-render per frame fighting it. + @State private var dragOffset: CGSize = .zero + + var body: some View { + // Height comes from the same lineHeightRatio the renderer uses; the handle is + // sized for a single line, which is enough to grab. Width follows the box. + let width = canvas.width * box.widthShare + let height = max(24, canvas.height * box.fontSizeShare * MemeCaptionLayout.lineHeightRatio) + let x = canvas.minX + canvas.width * box.centerX + dragOffset.width + let y = canvas.minY + canvas.height * box.centerY + dragOffset.height + + RoundedRectangle(cornerRadius: 4) + .strokeBorder( + isSelected ? Color.accentColor : Color.white.opacity(0.5), + style: StrokeStyle(lineWidth: isSelected ? 2 : 1, dash: [4, 3])) + .background( + RoundedRectangle(cornerRadius: 4) + .fill(Color.accentColor.opacity(isSelected ? 0.10 : 0.001))) + .frame(width: width, height: height) + .position(x: x, y: y) + .gesture( + DragGesture() + .onChanged { value in + model.selectedBoxID = box.id + dragOffset = value.translation + } + .onEnded { value in + // Commit in NORMALIZED units so the move survives the export's + // full-resolution render and a switch to another template. + guard canvas.width > 0, canvas.height > 0 else { return } + let dx = value.translation.width / canvas.width + let dy = value.translation.height / canvas.height + model.updateBox(id: box.id) { b in + b.centerX += dx + b.centerY += dy + } + dragOffset = .zero + }) + .onTapGesture { model.selectedBoxID = box.id } + .help("Drag to move this caption") + } +} + +// MARK: - Editor rows + +/// One row in the box list: selects the box and shows what it says. +private struct BoxRow: View { + @ObservedObject var model: MemeGeneratorModel + let box: MemeCaptionLayout.CaptionBox + + var body: some View { + HStack(spacing: 6) { + Image(systemName: box.id == model.selectedBoxID + ? "textformat.abc" : "textformat") + .foregroundStyle(box.id == model.selectedBoxID ? Color.accentColor : .secondary) + Text(box.text.isEmpty ? "(empty)" : box.text) + .lineLimit(1) + .truncationMode(.tail) + .foregroundStyle(box.text.isEmpty ? .tertiary : .primary) + Spacer() + Button { + model.deleteBox(id: box.id) + } label: { + Image(systemName: "trash") + } + .buttonStyle(.borderless) + .help("Delete this text box") + } + .contentShape(Rectangle()) + .onTapGesture { model.selectedBoxID = box.id } + .padding(.vertical, 2) + } +} + +/// Text / size / font controls for the selected box. +/// +/// Every control funnels through `model.updateBox`, so clamping and re-rendering +/// happen in one place and the preview can never fall out of sync with the boxes. +private struct BoxControls: View { + @ObservedObject var model: MemeGeneratorModel + let box: MemeCaptionLayout.CaptionBox + + /// The size slider's range, hoisted out of the view builder: a multi-line range + /// expression inside a `Slider(...)` argument list doesn't parse. + private static let sizeRange = + MemeCaptionLayout.CaptionBox.minimumFontSizeShare + ... MemeCaptionLayout.CaptionBox.maximumFontSizeShare + + /// One binding factory for every numeric control. Written with explicit + /// statement bodies because a bare `model.updateBox { ... }` in a `set:` closure + /// makes Swift infer the binding's value type as `Void`. + private func binding( + _ get: @escaping (MemeCaptionLayout.CaptionBox) -> Double, + _ set: @escaping (inout MemeCaptionLayout.CaptionBox, Double) -> Void + ) -> Binding { + Binding( + get: { get(box) }, + set: { newValue in + model.updateBox(id: box.id) { set(&$0, newValue) } + }) + } + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + Text("Text").font(.caption).foregroundStyle(.secondary) + TextField("Caption", text: Binding( + get: { box.text }, + set: { newValue in + model.updateBox(id: box.id) { $0.text = newValue } + })) + .textFieldStyle(.roundedBorder) + + HStack { + Text("Size").font(.caption).foregroundStyle(.secondary) + Spacer() + Button { + model.updateBox(id: box.id) { $0.fontSizeShare -= 0.01 } + } label: { Image(systemName: "minus") } + Button { + model.updateBox(id: box.id) { $0.fontSizeShare += 0.01 } + } label: { Image(systemName: "plus") } + } + Slider( + value: binding({ $0.fontSizeShare }, { $0.fontSizeShare = $1 }), + in: Self.sizeRange) + + HStack { + Text("Width").font(.caption).foregroundStyle(.secondary) + Slider( + value: binding({ $0.widthShare }, { $0.widthShare = $1 }), + in: 0.1...1.0) + } + + Text("Font").font(.caption).foregroundStyle(.secondary) + Picker("", selection: Binding( + get: { box.fontName ?? "" }, + set: { newValue in + model.updateBox(id: box.id) { $0.fontName = newValue.isEmpty ? nil : newValue } + })) { + Text(MemeRenderer.defaultFontLabel).tag("") + // Only faces actually installed are listed — a picker offering a + // font that silently resolves to something else is a lie. + ForEach(MemeRenderer.availableCaptionFonts, id: \.self) { name in + Text(name).tag(name) + } + Text("System (bold)").tag(MemeRenderer.systemFontToken) + } + .labelsHidden() + } + } +} + +// MARK: - Browse all + +/// A searchable grid over the WHOLE catalog. +/// +/// The honest override: when the model's picks are all wrong — or the meme the user +/// wanted simply isn't in imgflip's top 100 — this is where they say so themselves. +/// The search is local, case-insensitive and pure (`MemeTemplateMatcher.search`), and +/// crucially it does NOT fall back: an empty result stays empty and says why. +private struct BrowseAllSheet: View { + @ObservedObject var model: MemeGeneratorModel + @Binding var isPresented: Bool + + private let columns = [GridItem(.adaptive(minimum: 120), spacing: 10)] + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + HStack { + Text("Browse templates").font(.headline) + Spacer() + Button("Done") { isPresented = false } + .keyboardShortcut(.cancelAction) + } + + TextField("Search templates", text: $model.searchText) + .textFieldStyle(.roundedBorder) + + // State the corpus plainly. This is the sentence that would have saved + // the "yoda meme" session. + Text("This is imgflip's public top-100 template list — \(model.catalog.count) templates. " + + "If what you want isn't here, it isn't in the corpus.") + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + + if model.catalog.isEmpty { + ProgressView("Loading templates…") + .frame(maxWidth: .infinity, maxHeight: .infinity) + } else if model.searchResults.isEmpty { + VStack(spacing: 6) { + Image(systemName: "magnifyingglass") + .font(.largeTitle).foregroundStyle(.tertiary) + Text("No template matches \"\(model.searchText)\".") + .foregroundStyle(.secondary) + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + } else { + ScrollView { + LazyVGrid(columns: columns, spacing: 10) { + ForEach(model.searchResults) { template in + TemplateThumbnail( + template: template, + isSelected: template.id == model.selectedTemplate?.id, + width: 120) + .onTapGesture { + model.select(template: template) + isPresented = false + } + } + } + } + } + } + .padding(16) + .frame(width: 640, height: 560) + } +} + +// MARK: - Thumbnail + +/// A template preview image plus its name. +/// +/// `AsyncImage` rather than the plugin's own `MemeTemplateService.fetchImage`: the +/// grid can show 100 of these, and `AsyncImage` already handles per-view cancellation +/// and URLCache reuse. It's the same imgflip CDN GET either way — still no upload. +private struct TemplateThumbnail: View { + let template: MemeTemplate + let isSelected: Bool + let width: CGFloat + + var body: some View { + VStack(spacing: 4) { + AsyncImage(url: URL(string: template.url)) { phase in + switch phase { + case .success(let image): + image.resizable().scaledToFill() + case .failure: + Image(systemName: "photo") + .foregroundStyle(.tertiary) + default: + ProgressView().controlSize(.small) + } + } + .frame(width: width, height: width * 0.75) + .clipped() + .background(Color.secondary.opacity(0.1)) + .clipShape(RoundedRectangle(cornerRadius: 6)) + .overlay( + RoundedRectangle(cornerRadius: 6) + .stroke(isSelected ? Color.accentColor : Color.clear, lineWidth: 3)) + + Text(template.name) + .font(.caption2) + .lineLimit(2) + .multilineTextAlignment(.center) + .foregroundStyle(isSelected ? Color.accentColor : .secondary) + .frame(width: width) + } + .contentShape(Rectangle()) + .help(template.name) + } +} diff --git a/plugins/MemeGenerator/MemeRenderer.swift b/plugins/MemeGenerator/MemeRenderer.swift index 560581d..def781e 100644 --- a/plugins/MemeGenerator/MemeRenderer.swift +++ b/plugins/MemeGenerator/MemeRenderer.swift @@ -12,20 +12,28 @@ import Foundation /// wrapped, top and bottom blocks. Impact is not present on macOS, so we fall back /// through the closest system faces (see `captionFont`). /// -/// All the *decisions* (uppercasing, line breaking, shrink-to-fit) live in the pure, -/// tested `MemeCaptionLayout`; this type owns only the drawing and the font metrics -/// it feeds back into that layout. +/// All the *decisions* (uppercasing, line breaking, shrink-to-fit, and — since v2 — +/// where every box sits) live in the pure, tested `MemeCaptionLayout`; this type owns +/// only the drawing and the font metrics it feeds back into that layout. +/// +/// ## v2 — the box model +/// +/// Rendering is driven by `[MemeCaptionLayout.CaptionBox]` rather than a fixed +/// top/bottom pair. The AI path seeds two boxes, the manual editor mutates them, and +/// BOTH the on-screen preview and the exported PNG go through `render(template: +/// boxes:)` — that's what makes the editor WYSIWYG. Because box geometry is +/// normalized, the same boxes render correctly onto a differently-sized template when +/// the user picks another candidate. @MainActor enum MemeRenderer { - /// Compose a meme: template image + top/bottom captions. + /// Compose a meme from a template and an array of caption boxes. /// /// Returns a new image at the template's pixel dimensions, so exports are /// full-resolution regardless of how the preview is scaled. static func render( template: NSImage, - topText: String, - bottomText: String + boxes: [MemeCaptionLayout.CaptionBox] ) -> NSImage { // Work in PIXELS, not points: NSImage.size is point-based and would render // captions at the wrong scale on a template whose rep is 2x. @@ -42,106 +50,142 @@ enum MemeRenderer { in: NSRect(x: 0, y: 0, width: width, height: height), from: .zero, operation: .copy, fraction: 1.0) - let inset = width * MemeCaptionLayout.horizontalInsetShare - let maxTextWidth = width - inset * 2 - let maxBlockHeight = height * MemeCaptionLayout.captionHeightShare - - draw( - caption: topText, position: .top, - imageWidth: width, imageHeight: height, - maxTextWidth: maxTextWidth, maxBlockHeight: maxBlockHeight) - draw( - caption: bottomText, position: .bottom, - imageWidth: width, imageHeight: height, - maxTextWidth: maxTextWidth, maxBlockHeight: maxBlockHeight) + // The pure layer resolves normalized geometry into pixels and does the + // wrapping/shrinking; we hand it real font metrics and then draw what comes + // back. Empty boxes are dropped there, not here. + let layouts = MemeCaptionLayout.layout( + boxes: boxes, + imageWidth: Double(width), + imageHeight: Double(height), + measure: { text, size, fontName in + Double(text.size(withAttributes: [.font: captionFont(size: size, name: fontName)]).width) + }) + + for layout in layouts { + draw(layout, imageHeight: height) + } return output } - /// The image's size in PIXELS, preferring the largest bitmap rep so a retina - /// template exports at full resolution. - private static func pixelDimensions(of image: NSImage) -> (width: CGFloat, height: CGFloat) { - let reps = image.representations.compactMap { $0 as? NSBitmapImageRep } - if let best = reps.max(by: { $0.pixelsWide * $0.pixelsHigh < $1.pixelsWide * $1.pixelsHigh }), - best.pixelsWide > 0, best.pixelsHigh > 0 { - return (CGFloat(best.pixelsWide), CGFloat(best.pixelsHigh)) - } - // No bitmap rep (e.g. a vector or a placeholder): fall back to the point size, - // guarding against a degenerate zero that would make an undrawable image. - return (max(image.size.width, 1), max(image.size.height, 1)) + /// v1 entry point, kept so the AI path reads the same as before: seed two boxes + /// and render them. The seeded geometry reproduces the classic top/bottom look. + static func render( + template: NSImage, + topText: String, + bottomText: String + ) -> NSImage { + render( + template: template, + boxes: MemeCaptionLayout.seedBoxes(topText: topText, bottomText: bottomText)) } - private static func draw( - caption: String, - position: MemeCaptionLayout.Position, - imageWidth: CGFloat, - imageHeight: CGFloat, - maxTextWidth: CGFloat, - maxBlockHeight: CGFloat - ) { - guard !caption.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return } - - // Shrink-to-fit using REAL font metrics: the pure layout does the deciding, - // we supply the measurements. - let fit = MemeCaptionLayout.fit( - caption: caption, - maxWidth: Double(maxTextWidth), - maxHeight: Double(maxBlockHeight), - maxFontSize: MemeCaptionLayout.idealFontSize(imageHeight: Double(imageHeight)), - minFontSize: MemeCaptionLayout.minimumFontSize(imageHeight: Double(imageHeight)), - measure: { text, size in - Double(text.size(withAttributes: [.font: captionFont(size: size)]).width) - }, - lineHeight: { size in size * 1.15 }) - - guard !fit.lines.isEmpty else { return } - - let font = captionFont(size: fit.fontSize) - let lineHeight = CGFloat(fit.fontSize) * 1.15 - let blockHeight = lineHeight * CGFloat(fit.lines.count) - let margin = imageHeight * 0.02 - - // AppKit's image coordinate space is bottom-left origin. - let blockTopY: CGFloat - switch position { - case .top: blockTopY = imageHeight - margin - case .bottom: blockTopY = blockHeight + margin - } + /// Draw one resolved box. + /// + /// The pure layer works in a TOP-LEFT origin (what every UI framework, and the + /// editor's drag gestures, use). AppKit's image space is BOTTOM-LEFT, so the + /// single conversion `imageHeight - y` happens here and nowhere else — keeping + /// the flip in one place is what stops the editor and the export from disagreeing + /// about which way is up. + private static func draw(_ layout: MemeCaptionLayout.BoxLayout, imageHeight: CGFloat) { + guard !layout.lines.isEmpty else { return } + + let font = captionFont(size: layout.fontSize, name: layout.fontName) + let lineHeight = CGFloat(layout.fontSize * MemeCaptionLayout.lineHeightRatio) let style = NSMutableParagraphStyle() style.alignment = .center - // The classic look is a black OUTLINE around white glyphs. A negative - // `.strokeWidth` tells AppKit to stroke AND fill (a positive value strokes - // only, which would render hollow letters). CRUCIALLY the value is a - // PERCENTAGE of the font size, not points — scaling it by the font size - // double-scaled it to ~12%, thick enough that neighboring glyphs' black - // outlines swallowed each other's white interiors (the unreadable-blob - // bug). ~4% is the classic meme outline weight at any size. - let attributes: [NSAttributedString.Key: Any] = [ + let attributes = captionAttributes(font: font, paragraphStyle: style) + + // Flip the block's top edge into AppKit's bottom-left space. + let blockTopFlipped = imageHeight - CGFloat(layout.blockTopY) + let boxLeft = CGFloat(layout.centerX - layout.maxWidth / 2) + + for (index, line) in layout.lines.enumerated() { + let y = blockTopFlipped - lineHeight * CGFloat(index + 1) + let rect = NSRect( + x: boxLeft, + y: y + (lineHeight - font.ascender + font.descender) / 2, + width: CGFloat(layout.maxWidth), height: lineHeight) + line.draw(in: rect, withAttributes: attributes) + } + } + + /// The classic look is a black OUTLINE around white glyphs. A negative + /// `.strokeWidth` tells AppKit to stroke AND fill (a positive value strokes only, + /// which would render hollow letters). CRUCIALLY the value is a PERCENTAGE of the + /// font size, not points — scaling it by the font size double-scaled it to ~12%, + /// thick enough that neighboring glyphs' black outlines swallowed each other's + /// white interiors (the unreadable-blob bug). ~4% is the classic meme outline + /// weight at any size. + /// + /// Shared by the renderer and any preview that wants to match it, so the two can + /// never drift apart. + static func captionAttributes( + font: NSFont, paragraphStyle: NSParagraphStyle + ) -> [NSAttributedString.Key: Any] { + [ .font: font, .foregroundColor: NSColor.white, .strokeColor: NSColor.black, .strokeWidth: -4.0, - .paragraphStyle: style, + .paragraphStyle: paragraphStyle, ] + } - for (index, line) in fit.lines.enumerated() { - let y = blockTopY - lineHeight * CGFloat(index + 1) - let rect = NSRect( - x: 0, y: y + (lineHeight - font.ascender + font.descender) / 2, - width: imageWidth, height: lineHeight) - line.draw(in: rect, withAttributes: attributes) + /// The image's size in PIXELS, preferring the largest bitmap rep so a retina + /// template exports at full resolution. + private static func pixelDimensions(of image: NSImage) -> (width: CGFloat, height: CGFloat) { + let reps = image.representations.compactMap { $0 as? NSBitmapImageRep } + if let best = reps.max(by: { $0.pixelsWide * $0.pixelsHigh < $1.pixelsWide * $1.pixelsHigh }), + best.pixelsWide > 0, best.pixelsHigh > 0 { + return (CGFloat(best.pixelsWide), CGFloat(best.pixelsHigh)) } + // No bitmap rep (e.g. a vector or a placeholder): fall back to the point size, + // guarding against a degenerate zero that would make an undrawable image. + return (max(image.size.width, 1), max(image.size.height, 1)) } - /// The heaviest condensed face available. Impact — the canonical meme font — does - /// not ship with macOS, so this walks a preference list and ends at a bold system - /// font, which is always present. - static func captionFont(size: Double) -> NSFont { - let candidates = ["Impact", "Haettenschweiler", "Arial Black", "HelveticaNeue-CondensedBlack"] - for name in candidates { - if let font = NSFont(name: name, size: CGFloat(size)) { return font } + // MARK: - Fonts + + /// The faces offered in the editor's font picker, best-meme-first. + /// + /// Impact — the canonical meme font — does not ship with macOS, so the list walks + /// down through the closest heavy/condensed system faces. Only the ones actually + /// installed are offered (`availableCaptionFonts`), because a picker listing a + /// font that silently falls back to something else is worse than a short list. + static let captionFontCandidates = [ + "Impact", "Haettenschweiler", "Arial Black", "HelveticaNeue-CondensedBlack", + ] + + /// The label used for "no explicit face" — the renderer's own default. + static let defaultFontLabel = "Default (meme)" + + /// The subset of `captionFontCandidates` present on this Mac, plus the always- + /// available bold system font. Computed once; font availability doesn't change + /// mid-session in any way that matters here. + static let availableCaptionFonts: [String] = { + captionFontCandidates.filter { NSFont(name: $0, size: 12) != nil } + }() + + /// Sentinel meaning "the bold system font", which has no stable PostScript name + /// to put in `NSFont(name:)`. Stored in the box like any other face name so the + /// box model stays a plain `String?` and remains Codable. + static let systemFontToken = "__system__" + + /// Resolve a face name to a font, falling back the same way at every call site. + /// + /// `name == nil` (or a name that isn't installed) walks the candidate list and + /// ends at a bold system font, which is always present — so a meme still renders + /// in a meme-ish face on a Mac with none of the candidates installed. + static func captionFont(size: Double, name: String? = nil) -> NSFont { + if name == systemFontToken { + return NSFont.systemFont(ofSize: CGFloat(size), weight: .black) + } + if let name, let font = NSFont(name: name, size: CGFloat(size)) { return font } + for candidate in captionFontCandidates { + if let font = NSFont(name: candidate, size: CGFloat(size)) { return font } } return NSFont.systemFont(ofSize: CGFloat(size), weight: .black) } From 3bf1d80da2db02e2a431a223ce418214d0c1ef9d Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Sun, 2 Aug 2026 16:54:46 -0700 Subject: [PATCH 05/15] spike(meme-plugin): multi-source template providers + a real busy-state machine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pure half of v3, from owner live-testing feedback. Four new Foundation-only files, all covered by `swift test`. **Template providers (feedback #1 — "too limited and America-centric").** One catalog cannot fix a corpus problem: imgflip's top 100 is an English-language US popularity list, and any curated remote list is somebody else's culture. So the corpus becomes a MERGE of three sources — imgflip, memegen.link (~200 more, key-less, and it ships KEYWORDS), and the user's own imported library. `MemeTemplateCatalog.merge` puts the USER FIRST: their imported "Drake" beats imgflip's. De-duplication is by NORMALIZED NAME rather than id, because imgflip and memegen genuinely both carry Distracted Boyfriend under different ids and a by-id de-dup would silently do nothing. Ids are source-qualified so two providers can never collide into one image-cache entry. Search now spans names AND keywords, and tokens may span the two fields — memegen names a template "Sweet Brown" and keys the phrase people actually type ("Ain't Nobody Got Time For That") in `keywords`, so a name-only search misses the exact query a user would write. It still NEVER falls back: no match stays empty. **The user library is the actual answer to "worldwide".** No remote catalog contains a Russian or Ukrainian meme nobody uploaded to it, so any image can become a template. `suggestedName` preserves the script — "кот-в-шоке.png" becomes "кот в шоке", with no transliteration, which would defeat the point. The user's name never touches the filesystem (opaque UUID filenames), and `file` values read back off disk are validated before being joined onto a URL, the same rule `PluginManifest` applies to plugin ids and for the same reason. **Busy state (feedback #3 — "stuck loading, can't switch templates").** The cause was structural, not a missed line. v2 tracked in-flight work with a Bool cleared by a `finish()` that several exits never reached: every superseded-ticket bail read `guard ... else { return }` and returned WITHOUT clearing it. That is correct only when another task owns the flag — false after a window close, or when the LLM threw between two guards. Either left `isBusy` true forever, which disabled Generate AND (because `select` began with `guard !isBusy`) froze the candidate strip and Browse — exactly the two reported symptoms. `MemeGenerationState` makes the phase a value with one transition function. `finish` is idempotent, total, and ticket-guarded, so a stale result cannot un-stick newer work and a redundant finish cannot overwrite a status that already landed. `canSelectTemplate` is unconditionally true and says why: switching templates is a local re-render, never an LLM round-trip, so gating it on the busy flag was reflex — and that reflex is what turned a stuck flag into a frozen window. **Cache policy (feedback #1's "instant + offline").** `MemeCatalogCache.decide` always shows what is on disk first and treats the network as a background refresh. A refresh that fails while templates are on screen returns NO message: reporting every fetch failure is what made a cold start look broken. 60 new tests: merge precedence and cross-source name collisions, keyword/Cyrillic search, the live memegen wire shape, cache staleness (including a skewed clock and a future-version cache), library naming/uniqueness/pruning/traversal refusal, and the state machine's out-of-order, duplicate, and superseded orderings. Co-Authored-By: Claude --- OpenWhisp/Services/MemeCatalogCache.swift | 101 ++++ OpenWhisp/Services/MemeGenerationState.swift | 186 ++++++ OpenWhisp/Services/MemeTemplateMatcher.swift | 121 +++- OpenWhisp/Services/MemeTemplateProvider.swift | 135 +++++ OpenWhisp/Services/MemeUserLibrary.swift | 242 ++++++++ Package.swift | 6 +- .../MemeProviderTests.swift | 528 ++++++++++++++++++ 7 files changed, 1312 insertions(+), 7 deletions(-) create mode 100644 OpenWhisp/Services/MemeCatalogCache.swift create mode 100644 OpenWhisp/Services/MemeGenerationState.swift create mode 100644 OpenWhisp/Services/MemeTemplateProvider.swift create mode 100644 OpenWhisp/Services/MemeUserLibrary.swift create mode 100644 Tests/OpenWhispCoreTests/MemeProviderTests.swift diff --git a/OpenWhisp/Services/MemeCatalogCache.swift b/OpenWhisp/Services/MemeCatalogCache.swift new file mode 100644 index 0000000..d8366da --- /dev/null +++ b/OpenWhisp/Services/MemeCatalogCache.swift @@ -0,0 +1,101 @@ +import Foundation + +/// The disk-cache policy for the merged template catalog (spike v3). +/// +/// Two owner requirements drive this: browsing must be **instant**, and the plugin +/// must **work offline after the first fetch**. Both fall out of the same rule — +/// always read the cache first, and treat the network as a background refresh rather +/// than a precondition. +/// +/// The pure part (this file) decides *whether* to refresh and *what to do when the +/// refresh fails*. The app layer does the reading and writing. +public enum MemeCatalogCache { + + /// The cached catalog file's shape. + /// + /// Versioned and stamped: the version lets a format change migrate instead of + /// silently mis-decoding, and the timestamp is what `shouldRefresh` reasons over. + public struct Cached: Equatable, Sendable, Codable { + public var version: Int + public var fetchedAt: Date + public var templates: [MemeTemplate] + + public init(version: Int = MemeCatalogCache.currentVersion, fetchedAt: Date, templates: [MemeTemplate]) { + self.version = version + self.fetchedAt = fetchedAt + self.templates = templates + } + } + + public static let currentVersion = 1 + + public static let fileName = "catalog-cache.json" + + /// How long a cached catalog is considered fresh. + /// + /// Meme template catalogs change on the order of months, and a stale entry costs + /// the user nothing — the template still renders. A day balances "picks up new + /// templates eventually" against "never blocks the UI on a network call the user + /// didn't ask for". + public static let maxAge: TimeInterval = 60 * 60 * 24 + + /// What to do on window open, given what is on disk. + public enum Decision: Equatable, Sendable { + /// Nothing usable cached — fetch before the user can browse. + case fetchNow + /// Cache is usable and fresh; use it and don't touch the network. + case useCache + /// Cache is usable but stale; show it IMMEDIATELY and refresh behind it. + /// + /// This is the case that makes browsing feel instant: the user never waits on + /// a refresh, and a failed one costs them nothing because they are already + /// looking at the cached corpus. + case useCacheAndRefresh + } + + /// Decide how to open the catalog. + /// + /// A cache from a FUTURE version is treated as absent rather than trusted: a + /// downgrade must not read a format it doesn't understand. An empty cache is + /// likewise treated as absent — persisting a zero-template catalog and then + /// honouring it would present the offline state as a legitimately empty corpus. + public static func decide(cached: Cached?, now: Date) -> Decision { + guard let cached, cached.version <= currentVersion, !cached.templates.isEmpty else { + return .fetchNow + } + // A timestamp in the future (clock skew, a restored backup) is treated as + // stale rather than infinitely fresh, so a bad clock can't pin the catalog. + let age = now.timeIntervalSince(cached.fetchedAt) + return (age >= 0 && age < maxAge) ? .useCache : .useCacheAndRefresh + } + + /// What the user should be told when a refresh fails. + /// + /// The distinction is the whole point of the cache. With templates already on + /// screen a failed refresh is a NON-EVENT and must not raise an error — v2's + /// habit of reporting every fetch failure is what made a cold start look broken. + /// With nothing on screen the failure is the only thing the user needs to know, + /// and it must come with the fact that retrying is possible. + public static func refreshFailureMessage(hasCachedTemplates: Bool, reason: String) -> String? { + guard !hasCachedTemplates else { return nil } + return "Couldn't load meme templates — \(reason) " + + "Check your connection and press Retry, or import your own template." + } + + /// The status line for a successful catalog open. + /// + /// Names the per-source counts because the corpus size IS the feature the owner + /// asked for, and because seeing "0 from your library" is the discoverability + /// nudge toward importing one. + public static func summary(_ templates: [MemeTemplate]) -> String { + guard !templates.isEmpty else { return "No templates available." } + var counts: [MemeTemplateSource: Int] = [:] + for template in templates { counts[template.source, default: 0] += 1 } + + let parts = MemeTemplateSource.allCases.compactMap { source -> String? in + guard let count = counts[source], count > 0 else { return nil } + return "\(count) \(source.label)" + } + return "\(templates.count) templates (\(parts.joined(separator: ", ")))." + } +} diff --git a/OpenWhisp/Services/MemeGenerationState.swift b/OpenWhisp/Services/MemeGenerationState.swift new file mode 100644 index 0000000..a000f35 --- /dev/null +++ b/OpenWhisp/Services/MemeGenerationState.swift @@ -0,0 +1,186 @@ +import Foundation + +/// The Meme Generator's busy-state machine (spike v3). +/// +/// ## Why this is a type instead of a `Bool` +/// +/// The owner's report was "stuck loading, and I can't switch templates during or +/// after". The cause was structural rather than a single missed line: v2 tracked +/// in-flight work with a `Bool` cleared by a `finish()` that several exit paths never +/// reached. Specifically, every superseded-ticket bail read +/// +/// ```swift +/// guard !self.isCancelled, myTicket == self.ticket else { return } +/// ``` +/// +/// and returned WITHOUT clearing `isBusy`. That is correct only if some other task +/// owns the flag — true when a newer request superseded this one, false when the +/// window was cancelled and re-shown, or when the LLM call threw between the two +/// guards. Any of those left `isBusy == true` forever, which disabled Generate AND +/// (because `select(template:)` began with `guard !isBusy`) froze the candidate strip +/// and the Browse grid — exactly the two symptoms reported. +/// +/// Making the state a value with ONE transition function fixes the class of bug: a +/// phase can only change through `begin`/`finish`/`cancel`/`timeout`, each of which +/// is total, and `swift test` can drive every ordering including the out-of-order and +/// duplicate ones that a `Bool` gets wrong. +public struct MemeGenerationState: Equatable, Sendable { + + /// What the surface is doing. + public enum Phase: Equatable, Sendable { + /// Nothing in flight. + case idle + /// The LLM is being warmed at window-open. The user may still browse and + /// switch templates; only Generate waits. + case warming + /// Loading the template catalog. + case loadingCatalog + /// Waiting on the LLM. + case asking + /// Downloading a template image. + case downloading(templateName: String) + + /// True when a generate round-trip is in flight. + /// + /// Note `warming` is NOT busy. Warming happens on window open, before the + /// user has asked for anything; blocking the UI on it would trade a cold + /// first Generate for a frozen window, which is a worse bug than the one + /// being fixed. + public var isGenerating: Bool { + switch self { + case .idle, .warming: return false + case .loadingCatalog, .asking, .downloading: return true + } + } + + /// The line shown under the controls while this phase runs. + public var statusText: String { + switch self { + case .idle: return "" + case .warming: return "Preparing model…" + case .loadingCatalog: return "Loading templates…" + case .asking: return "Asking the model…" + case .downloading(let name): return "Downloading \(name)…" + } + } + } + + public private(set) var phase: Phase + /// The ticket of the work that currently owns the phase. A result carrying an + /// older ticket is ignored — including its attempt to clear the phase, which is + /// what stops a late failure from unsticking a newer, legitimately-running + /// request. + public private(set) var ticket: Int + + public init(phase: Phase = .idle, ticket: Int = 0) { + self.phase = phase + self.ticket = ticket + } + + public var isGenerating: Bool { phase.isGenerating } + + /// True when the user may pick a different template right now. + /// + /// **Always true.** This is a deliberate answer to "can't switch templates during + /// or after": switching templates re-renders the SAME caption boxes onto an image + /// that is either already cached or a single GET away — it involves no LLM + /// round-trip and no catalog fetch, so there is no reason a generation in flight + /// should block it. v2 gated it on `!isBusy` purely by reflex, and that reflex is + /// what made a stuck flag freeze the whole surface. + /// + /// Kept as a named property rather than inlining `true` at the call site so the + /// decision is stated once, testable, and hard to silently regress. + public var canSelectTemplate: Bool { true } + + /// True when Generate should be offered. Warming blocks it — but see + /// `generateBlockedReason`, which makes that wait honest rather than a dead button. + public var canGenerate: Bool { phase == .idle } + + /// Why Generate is unavailable, or nil when it is available. + /// + /// The v2 bug report was "first Generate fails with a network error and model + /// loading". The model was simply not up yet, and the surface let the user fire a + /// request into a socket nothing was listening on. Saying "Preparing model…" and + /// waiting is the honest version of the same moment. + public func generateBlockedReason() -> String? { + switch phase { + case .idle: return nil + case .warming: return "Preparing model…" + default: return phase.statusText + } + } + + // MARK: - Transitions + + /// Begin a new unit of work, taking a fresh ticket. + /// + /// Returns the ticket the caller must carry through its async work and present + /// back on every subsequent transition. Incrementing on every begin is what makes + /// a superseded result identifiable. + public mutating func begin(_ phase: Phase) -> Int { + ticket += 1 + self.phase = phase + return ticket + } + + /// Move to another phase WITHIN the same unit of work (catalog → asking → + /// downloading), keeping the ticket. + /// + /// Returns false — and changes nothing — when the ticket is stale, so a + /// superseded task can't drag the UI back to its own phase. + @discardableResult + public mutating func advance(_ phase: Phase, ticket incoming: Int) -> Bool { + guard incoming == ticket else { return false } + self.phase = phase + return true + } + + /// End the unit of work identified by `incoming`. + /// + /// This is THE fix for the stuck state, and its contract is the important part: + /// finishing is **idempotent and total**. Calling it twice is safe, calling it + /// from an error path is safe, and calling it with a stale ticket is a no-op that + /// leaves the newer work's phase intact. Every exit path in the model calls this + /// — success, parse rejection, transport failure, timeout, and cancel — so there + /// is no path left that can end without clearing the phase. + /// + /// The return value says whether THIS call is the one that ended the work, so a + /// caller can write the final status exactly once. A redundant second finish + /// returns false rather than letting, say, a timeout overwrite the success + /// message that already landed. + @discardableResult + public mutating func finish(ticket incoming: Int) -> Bool { + guard incoming == ticket, phase != .idle else { return false } + phase = .idle + return true + } + + /// Abandon whatever is in flight and refuse its result. + /// + /// Used by the Cancel button and by window close. Bumping the ticket is what + /// makes the abandoned work's later `finish` a no-op *and* stops it writing a + /// meme into a closed window. + public mutating func cancel() { + ticket += 1 + phase = .idle + } + + /// Whether a result carrying `incoming` is still wanted. + public func accepts(ticket incoming: Int) -> Bool { incoming == ticket } + + // MARK: - Timeout + + /// The hard ceiling on one generate round-trip. + /// + /// A local model on a cold cache can genuinely take a while, so this is generous + /// — but it is FINITE, which is the point. v2 had no ceiling at all: an LLM call + /// that never returned left the surface busy forever with no way back except + /// closing the window. Whatever the number, "eventually recovers by itself" beats + /// "waits forever". + public static let generateTimeout: TimeInterval = 120 + + /// The message shown when the ceiling is hit. + public static let timeoutMessage = + "The model didn't answer within \(Int(generateTimeout)) seconds. It may still be " + + "loading — try Generate again, or pick a template yourself with Browse all." +} diff --git a/OpenWhisp/Services/MemeTemplateMatcher.swift b/OpenWhisp/Services/MemeTemplateMatcher.swift index 5f653b3..b66811c 100644 --- a/OpenWhisp/Services/MemeTemplateMatcher.swift +++ b/OpenWhisp/Services/MemeTemplateMatcher.swift @@ -7,34 +7,143 @@ import Foundation /// to find a base image — captioning happens locally with CoreGraphics, so no text, /// no audio, and no LLM output is ever sent to imgflip. public struct MemeTemplate: Equatable, Sendable, Codable, Identifiable { + /// Source-qualified id (`"imgflip:181913649"`, `"userLibrary:"`) — see + /// `MemeTemplateCatalog.qualifiedID`. Qualified so two providers can never + /// collide into one image-cache entry. public let id: String - /// Display name, e.g. "Distracted Boyfriend". English — this is the string the - /// LLM is asked to copy verbatim, and the key candidates are validated against. + /// Display name, e.g. "Distracted Boyfriend". This is the string the LLM is asked + /// to copy verbatim and the key candidates are validated against. For a + /// user-library template it is whatever the user named it, in their own script. public let name: String - /// Direct image URL (jpg/png) for the blank template. + /// Where the blank template image lives: an `https:` URL for the remote + /// providers, a `file:` URL for the user's own library. Both are just "a string + /// that locates the image", which is what lets all three sources share one + /// fetch/render path. public let url: String public let width: Int public let height: Int + /// Which provider contributed this template. Drives the Browse grid's badge and + /// whether the image is loaded from disk or the network. + public let source: MemeTemplateSource + /// Alternate search terms. memegen ships these ("Ain't Nobody Got Time For That" + /// on a template *named* "Sweet Brown"); imgflip has none; the user library + /// carries the original filename. Searching them is what makes a merged, + /// multi-lingual corpus findable — see `MemeTemplateCatalog.search`. + public let keywords: [String] - public init(id: String, name: String, url: String, width: Int, height: Int) { + public init( + id: String, name: String, url: String, width: Int, height: Int, + source: MemeTemplateSource = .imgflip, keywords: [String] = [] + ) { self.id = id self.name = name self.url = url self.width = width self.height = height + self.source = source + self.keywords = keywords + } + + /// Decoding tolerates a missing `source`/`keywords` so a catalog cache written by + /// an older build still loads instead of being discarded — the cache is a + /// performance artifact, but throwing it away on every upgrade would make the + /// first launch after an update look like the offline bug this release fixes. + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + id = try c.decode(String.self, forKey: .id) + name = try c.decode(String.self, forKey: .name) + url = try c.decode(String.self, forKey: .url) + width = (try? c.decode(Int.self, forKey: .width)) ?? 0 + height = (try? c.decode(Int.self, forKey: .height)) ?? 0 + source = (try? c.decode(MemeTemplateSource.self, forKey: .source)) ?? .imgflip + keywords = (try? c.decode([String].self, forKey: .keywords)) ?? [] } } /// The Imgflip `get_memes` response envelope. public struct MemeTemplateCatalogResponse: Decodable, Sendable { + /// The raw wire shape. Decoded into a separate type rather than straight into + /// `MemeTemplate` because the id has to be SOURCE-QUALIFIED before it becomes a + /// catalog id, and a `Decodable` conformance can't know which provider it is + /// being decoded for. + public struct Wire: Decodable, Sendable { + public let id: String + public let name: String + public let url: String + public let width: Int + public let height: Int + } public struct Payload: Decodable, Sendable { - public let memes: [MemeTemplate] + public let memes: [Wire] } public let success: Bool public let data: Payload? /// The templates, or empty when the API reported failure. - public var templates: [MemeTemplate] { success ? (data?.memes ?? []) : [] } + public var templates: [MemeTemplate] { + guard success else { return [] } + return (data?.memes ?? []).map { wire in + MemeTemplate( + id: MemeTemplateCatalog.qualifiedID(.imgflip, wire.id), + name: wire.name, url: wire.url, + width: wire.width, height: wire.height, + source: .imgflip, keywords: []) + } + } +} + +/// The memegen.link `/templates` response (spike v3). +/// +/// A second key-less, read-only catalog — ~200 templates, many of which imgflip's +/// top-100 popularity list doesn't carry. Like imgflip it is used ONLY to locate a +/// blank image: captioning stays local, so memegen's own caption-rendering URL API +/// (`/images///.jpg`) is deliberately NOT used. Routing the user's +/// text through a URL would put their words on someone else's server, which is +/// exactly what this plugin avoids. +/// +/// The response is a bare JSON ARRAY, not an envelope, so failure shows up as a +/// decode error rather than a `success: false` flag. +public struct MemegenTemplateResponse: Decodable, Sendable { + public struct Wire: Decodable, Sendable { + public let id: String + public let name: String + /// The blank (caption-less) image URL. + public let blank: String + /// Alternate names — the field that makes a merged corpus searchable. + public let keywords: [String]? + + private enum CodingKeys: String, CodingKey { case id, name, blank, keywords } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + id = try c.decode(String.self, forKey: .id) + name = try c.decode(String.self, forKey: .name) + blank = try c.decode(String.self, forKey: .blank) + keywords = try? c.decode([String].self, forKey: .keywords) + } + } + + public let templates: [MemeTemplate] + + public init(from decoder: Decoder) throws { + let wires = try [Wire](from: decoder) + templates = wires.compactMap { wire in + // A template with no name can't be searched, de-duplicated, or copied + // verbatim by the LLM — drop it at the boundary rather than letting it + // occupy a grid cell nobody can reach. + let name = wire.name.trimmingCharacters(in: .whitespacesAndNewlines) + guard !name.isEmpty, !wire.blank.isEmpty else { return nil } + return MemeTemplate( + id: MemeTemplateCatalog.qualifiedID(.memegen, wire.id), + name: name, url: wire.blank, + // memegen doesn't report dimensions; 0 means "ask the image". + // Nothing in the render path uses these (the layout works off the + // decoded NSImage's real pixel size), so they stay honest zeros + // rather than invented defaults. + width: 0, height: 0, + source: .memegen, keywords: wire.keywords ?? []) + } + } } /// Local, lexical template lookup over the catalog (spike). diff --git a/OpenWhisp/Services/MemeTemplateProvider.swift b/OpenWhisp/Services/MemeTemplateProvider.swift new file mode 100644 index 0000000..6eb32b4 --- /dev/null +++ b/OpenWhisp/Services/MemeTemplateProvider.swift @@ -0,0 +1,135 @@ +import Foundation + +/// Where a template came from (spike v3). +/// +/// The owner's report was that the corpus was "too limited and America-centric". +/// One catalog can't fix that — imgflip's top 100 is an English-language popularity +/// list, and *any* curated remote list will be someone else's culture. So the corpus +/// becomes a MERGE of sources, and the one that actually answers "worldwide" is the +/// user's own library: an image the user imported is by definition a template from +/// their culture, and it needs no API, no key, and no network. +/// +/// Order matters and is deliberate — see `MemeTemplateCatalog.merge`. +public enum MemeTemplateSource: String, Equatable, Sendable, Codable, CaseIterable { + /// imgflip's key-less `get_memes` top 100. + case imgflip + /// memegen.link's key-less `/templates` list (~200 more). + case memegen + /// Images the user imported themselves, stored under Application Support. + case userLibrary + + /// The label shown next to a template in the Browse grid, so the user can tell + /// where a template came from — particularly which ones are theirs. + public var label: String { + switch self { + case .imgflip: return "imgflip" + case .memegen: return "memegen" + case .userLibrary: return "My library" + } + } +} + +/// A merged, de-duplicated, searchable template catalog (spike v3). +/// +/// This is the pure half of the provider system: given templates from any number of +/// sources, it decides which survive, in what order, and which ones a query matches. +/// The IO — HTTP, disk, thumbnails — lives in the app layer, so all the *policy* +/// here is pinned by `swift test`. +public enum MemeTemplateCatalog { + + /// Merge templates from several sources into one catalog. + /// + /// **Precedence is user-first.** The user's own library wins every collision, + /// then imgflip (popularity-ranked and the corpus the LLM prompt was tuned on), + /// then memegen. If the user imported their own "Drake", theirs is the Drake — + /// a remote catalog must never shadow a local file the user deliberately added. + /// This is the same "earlier provider wins" rule `PluginDiscovery` already uses, + /// pointed the other way on purpose: there, trust decreases with writability; + /// here, the writable source IS the trusted one because the user put it there. + /// + /// De-duplication is by NORMALIZED NAME, not by id: imgflip and memegen both + /// carry "Distracted Boyfriend" under completely different ids, and showing the + /// user the same meme twice in a grid is the visible bug. Ids stay unique across + /// sources because `MemeTemplate.id` is prefixed at the provider (see + /// `qualifiedID`), so a de-dup by id would silently do nothing. + /// + /// Within a source the incoming order is preserved — imgflip's order is its + /// popularity ranking, and the LLM payload truncates from the end, so scrambling + /// it would quietly degrade the model's picks. + public static func merge(_ groups: [[MemeTemplate]]) -> [MemeTemplate] { + var out: [MemeTemplate] = [] + var seen = Set() + + for group in groups { + for template in group { + let key = MemeTemplateMatcher.normalize(template.name) + // A template with an unusable name can't be de-duplicated or searched + // for, and can't be copied verbatim by the LLM. Drop it rather than + // letting it occupy a grid cell nobody can reach. + guard !key.isEmpty, !seen.contains(key) else { continue } + seen.insert(key) + out.append(template) + } + } + return out + } + + /// Namespace a provider's raw id so ids stay unique across sources. + /// + /// imgflip ids are numeric ("181913649") and memegen's are slugs ("drake"), so + /// they don't collide *today* — but the user library mints its own ids, and two + /// sources agreeing on an id would make the image cache serve one template's + /// picture for another. Prefixing is cheap insurance against a bug that would + /// look like a rendering glitch rather than an id collision. + public static func qualifiedID(_ source: MemeTemplateSource, _ rawID: String) -> String { + "\(source.rawValue):\(rawID)" + } + + /// The source a qualified id came from, or nil when the id isn't qualified. + /// + /// Used to decide whether a template's image is a local file (user library) or a + /// URL to fetch, without threading the source through every call site. + public static func source(ofQualifiedID id: String) -> MemeTemplateSource? { + guard let separator = id.firstIndex(of: ":") else { return nil } + return MemeTemplateSource(rawValue: String(id[id.startIndex.. [MemeTemplate] { + let normalizedQuery = MemeTemplateMatcher.normalize(query) + guard !normalizedQuery.isEmpty else { return catalog } + + let needles = normalizedQuery.split(separator: " ").map(String.init) + + return catalog.filter { template in + // Every token must appear SOMEWHERE across the name and keywords + // combined, so "russian cat" matches a template named "Cat" with the + // keyword "Russian" — the cross-field case that a per-field search + // would miss. + let haystack = ([template.name] + template.keywords) + .map { MemeTemplateMatcher.normalize($0) } + .joined(separator: " ") + return needles.allSatisfy { haystack.contains($0) } + } + } + + /// The names handed to the LLM, capped so a merged ~300-name corpus doesn't blow + /// a small local model's context. + /// + /// The cap is applied AFTER the merge so the user's own templates — which sort + /// first — are always in the prompt. That is the point: a user who imported ten + /// Russian templates should have the model able to pick them, even if imgflip's + /// hundred would otherwise fill the budget. + public static func promptNames(_ catalog: [MemeTemplate], limit: Int) -> [String] { + Array(catalog.prefix(max(0, limit)).map(\.name)) + } +} diff --git a/OpenWhisp/Services/MemeUserLibrary.swift b/OpenWhisp/Services/MemeUserLibrary.swift new file mode 100644 index 0000000..3846cec --- /dev/null +++ b/OpenWhisp/Services/MemeUserLibrary.swift @@ -0,0 +1,242 @@ +import Foundation + +/// The user's own imported meme templates (spike v3). +/// +/// This is the plugin's answer to "the corpus is America-centric". imgflip's top 100 +/// and memegen's ~200 are both English-language, US-internet lists; no amount of +/// merging remote catalogs produces a Russian, Ukrainian, or Brazilian template that +/// nobody uploaded to those services. The only source that can is the user, so the +/// plugin lets them import any image as a template. +/// +/// It also happens to be the only source that works with the network off, which is +/// the local-first posture the rest of the app already takes. +/// +/// ## Layout on disk +/// +/// ``` +/// ~/Library/Application Support/OpenWhisp/Plugins/MemeGenerator/templates/ +/// index.json <- MemeUserLibrary.Index +/// .png <- the imported images, copied in +/// ``` +/// +/// The index is a small JSON file of `{name, file}` records rather than a directory +/// scan, because the NAME is the load-bearing part: it is what the user searches for, +/// and what the LLM is asked to copy verbatim. A filename can't carry a name with a +/// slash, a colon, or an emoji in it — all of which are perfectly reasonable in the +/// languages this feature exists to serve — so the display name is stored as data and +/// the file on disk gets an opaque UUID name. +/// +/// Everything here is pure: paths in, records out. The app layer does the actual +/// copying and reading (`MemeUserLibraryStore`). +public enum MemeUserLibrary { + + /// One imported template. + public struct Entry: Equatable, Sendable, Codable, Identifiable { + /// Stable id, also used as the qualified catalog id's raw part. + public let id: String + /// The display name the user typed or that was derived from the filename. + /// This is what search matches and what the LLM is asked to copy. + public var name: String + /// The image's filename WITHIN the templates directory. Deliberately a bare + /// filename, never a path: the index must not be able to point outside its + /// own directory (see `imageURL`), and a relative name survives the whole + /// Application Support folder being moved or restored from a backup. + public var file: String + /// Pixel dimensions, captured at import so the Browse grid can lay out + /// without decoding every image. + public var width: Int + public var height: Int + + public init(id: String = UUID().uuidString, name: String, file: String, width: Int, height: Int) { + self.id = id + self.name = name + self.file = file + self.width = width + self.height = height + } + } + + /// The on-disk index file's shape. + /// + /// Versioned from the start: this is a user-authored data store — the images are + /// theirs and are not re-downloadable — so a future format change has to migrate + /// rather than discard. Same posture as the profiles/vocabulary stores the app + /// already treats as a versioned contract. + public struct Index: Equatable, Sendable, Codable { + public var version: Int + public var entries: [Entry] + + public init(version: Int = MemeUserLibrary.currentVersion, entries: [Entry] = []) { + self.version = version + self.entries = entries + } + } + + public static let currentVersion = 1 + + /// The index file's name within the templates directory. + public static let indexFileName = "index.json" + + /// Image types the importer accepts. Anything `NSImage` can decode would work, + /// but the picker needs a concrete list and these cover what people actually have + /// saved from a chat app. + public static let acceptedExtensions: Set = ["png", "jpg", "jpeg", "gif", "webp", "heic", "tiff", "bmp"] + + public static func isAcceptedImage(fileName: String) -> Bool { + let ext = (fileName as NSString).pathExtension.lowercased() + return !ext.isEmpty && acceptedExtensions.contains(ext) + } + + // MARK: - Naming + + /// Derive a display name from an imported file's name. + /// + /// "кот-в-шоке.png" → "кот в шоке". Separators become spaces and the result is + /// trimmed, but the SCRIPT IS PRESERVED — no transliteration, no ASCII folding, + /// no case forcing beyond capitalizing a leading letter. Mangling a Cyrillic + /// filename into Latin would defeat the entire purpose of this feature. + /// + /// An unnameable file (all separators, or no stem) falls back to a generic name + /// so the entry is still visible and renameable rather than blank. + public static func suggestedName(fromFileName fileName: String) -> String { + let stem = (fileName as NSString).deletingPathExtension + let spaced = stem + .replacingOccurrences(of: "_", with: " ") + .replacingOccurrences(of: "-", with: " ") + .split(separator: " ", omittingEmptySubsequences: true) + .joined(separator: " ") + .trimmingCharacters(in: .whitespacesAndNewlines) + + return spaced.isEmpty ? "Imported template" : spaced + } + + /// The on-disk filename for a newly imported image: an opaque id plus the + /// original extension. + /// + /// The user's name never touches the filesystem. That avoids every path-injection + /// and encoding question a user-supplied, possibly non-Latin name would raise, + /// and means renaming a template is a pure index edit rather than a file move. + public static func storageFileName(id: String, sourceExtension: String) -> String { + let ext = sourceExtension.lowercased() + let safeExt = acceptedExtensions.contains(ext) ? ext : "png" + return "\(id).\(safeExt)" + } + + /// Make a name unique within the library by suffixing " 2", " 3", … + /// + /// Names must be distinct because the LLM picks templates BY NAME and the merged + /// catalog de-duplicates by name — two templates called "кот" would mean one of + /// them is unreachable. Comparison is on the normalized form so "Кот" and "кот " + /// are treated as the same name, matching how search and the LLM validator behave. + public static func uniqueName(_ desired: String, existing: [String]) -> String { + let trimmed = desired.trimmingCharacters(in: .whitespacesAndNewlines) + let base = trimmed.isEmpty ? "Imported template" : trimmed + + var taken = Set(existing.map { MemeTemplateMatcher.normalize($0) }) + // An empty normalization (a name that is entirely punctuation or emoji) + // can't be compared meaningfully, so it never blocks another name. + taken.remove("") + + guard taken.contains(MemeTemplateMatcher.normalize(base)) else { return base } + + var suffix = 2 + while true { + let candidate = "\(base) \(suffix)" + if !taken.contains(MemeTemplateMatcher.normalize(candidate)) { return candidate } + suffix += 1 + // Defensive ceiling: an unbounded loop here would hang the import on a + // pathological library rather than failing visibly. + if suffix > 9999 { return "\(base) \(UUID().uuidString.prefix(8))" } + } + } + + // MARK: - Index rules + + /// Add an entry, keeping names unique. + public static func adding(_ entry: Entry, to index: Index) -> Index { + var out = index + var newEntry = entry + newEntry.name = uniqueName(entry.name, existing: index.entries.map(\.name)) + out.entries.append(newEntry) + out.version = currentVersion + return out + } + + public static func removing(id: String, from index: Index) -> Index { + var out = index + out.entries.removeAll { $0.id == id } + return out + } + + /// Rename an entry, keeping the new name unique against the OTHER entries. + /// + /// The entry's own current name is excluded from the uniqueness check, so + /// re-saving a name unchanged doesn't turn it into "кот 2". + public static func renaming(id: String, to newName: String, in index: Index) -> Index { + guard let position = index.entries.firstIndex(where: { $0.id == id }) else { return index } + var out = index + let others = index.entries.filter { $0.id != id }.map(\.name) + out.entries[position].name = uniqueName(newName, existing: others) + return out + } + + /// Drop entries whose image file is gone. + /// + /// The user can delete files out from under the index in Finder; an entry + /// pointing at a missing file would render as a broken cell in the grid and fail + /// at generate time. Pruning at load turns that into a quiet self-heal. + /// `existingFiles` is supplied by the app layer so this stays pure. + public static func pruned(_ index: Index, existingFiles: Set) -> Index { + var out = index + out.entries.removeAll { !existingFiles.contains($0.file) } + return out + } + + /// Validate a `file` value read off disk before it is joined onto a directory URL. + /// + /// The index is a plain JSON file in a user-writable directory, so it is UNTRUSTED + /// input even though the user owns it: a hand-edited (or maliciously supplied) + /// `"file": "../../../../etc/passwd"` must not become a readable path. Same rule + /// `PluginManifest` applies to plugin ids for the same reason — a string that + /// becomes a path component gets checked before it is joined, never after. + public static func isSafeFileName(_ file: String) -> Bool { + guard !file.isEmpty, file.count <= 255 else { return false } + guard !file.contains("/"), !file.contains("\\"), !file.contains("\0") else { return false } + guard file != ".", file != ".." else { return false } + // A leading dot would hide the file from the user in Finder, which makes the + // library's contents dishonest about what it holds. + guard !file.hasPrefix(".") else { return false } + return true + } + + /// The entries safe to use, with unsafe ones dropped. + public static func safeEntries(_ index: Index) -> [Entry] { + index.entries.filter { isSafeFileName($0.file) && !$0.name.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } + } + + // MARK: - Catalog projection + + /// Project the library into catalog templates. + /// + /// `url` is a FILE url string here rather than an http one — the renderer and the + /// thumbnail view both take "a string that locates the image", and keeping the + /// shape identical to a remote template is what lets the user's own images flow + /// through the exact same merge, search, prompt, and render path as imgflip's. + /// One code path, three sources. + /// + /// The filename stem rides along as a keyword so a user who imported + /// "kot-v-shoke.png" and renamed it "Кот в шоке" can still find it by typing the + /// Latin filename they remember. + public static func templates(from index: Index, directory: URL) -> [MemeTemplate] { + safeEntries(index).map { entry in + MemeTemplate( + id: MemeTemplateCatalog.qualifiedID(.userLibrary, entry.id), + name: entry.name, + url: directory.appendingPathComponent(entry.file).absoluteString, + width: entry.width, + height: entry.height, + source: .userLibrary, + keywords: [suggestedName(fromFileName: entry.file)].filter { $0 != "Imported template" }) + } + } +} diff --git a/Package.swift b/Package.swift index 8193f99..3b04ae7 100644 --- a/Package.swift +++ b/Package.swift @@ -175,7 +175,11 @@ let package = Package( "PluginRegistry.swift", "MemeAI.swift", "MemeTemplateMatcher.swift", - "MemeCaptionLayout.swift" + "MemeCaptionLayout.swift", + "MemeTemplateProvider.swift", + "MemeUserLibrary.swift", + "MemeGenerationState.swift", + "MemeCatalogCache.swift" ] ), // The Agent Bridge client + MCP stdio adapter. A library (not folded into diff --git a/Tests/OpenWhispCoreTests/MemeProviderTests.swift b/Tests/OpenWhispCoreTests/MemeProviderTests.swift new file mode 100644 index 0000000..fafa015 --- /dev/null +++ b/Tests/OpenWhispCoreTests/MemeProviderTests.swift @@ -0,0 +1,528 @@ +import XCTest +@testable import OpenWhispCore + +/// Covers the Meme Generator plugin's v3 pure layer (spike/plugin-system): the +/// multi-source template providers and their merge/precedence rules, the keyword +/// search that makes a merged corpus findable, the catalog disk-cache policy, the +/// user template library's index, and the busy-state machine behind the "stuck +/// loading" report. +final class MemeProviderTests: XCTestCase { + + private func template( + _ source: MemeTemplateSource, _ rawID: String, _ name: String, + keywords: [String] = [] + ) -> MemeTemplate { + MemeTemplate( + id: MemeTemplateCatalog.qualifiedID(source, rawID), + name: name, url: "https://example.test/\(rawID).jpg", + width: 100, height: 100, source: source, keywords: keywords) + } + + // MARK: - Merge + precedence + + func testMergeConcatenatesSourcesInOrder() { + let merged = MemeTemplateCatalog.merge([ + [template(.userLibrary, "u1", "Кот в шоке")], + [template(.imgflip, "i1", "Drake Hotline Bling")], + [template(.memegen, "m1", "Ancient Aliens Guy")], + ]) + XCTAssertEqual(merged.map(\.name), + ["Кот в шоке", "Drake Hotline Bling", "Ancient Aliens Guy"]) + } + + /// The rule that matters: a template the USER imported outranks a remote one with + /// the same name. Their file, their corpus, their meme. + func testUserLibraryWinsANameCollisionAgainstRemoteSources() { + let merged = MemeTemplateCatalog.merge([ + [template(.userLibrary, "u1", "Drake Hotline Bling")], + [template(.imgflip, "i1", "Drake Hotline Bling")], + ]) + XCTAssertEqual(merged.count, 1) + XCTAssertEqual(merged[0].source, .userLibrary) + } + + /// imgflip and memegen genuinely both carry Distracted Boyfriend under different + /// ids — de-duplicating by id would silently do nothing and show it twice. + func testDuplicateNamesAcrossRemoteSourcesCollapseDespiteDifferentIDs() { + let merged = MemeTemplateCatalog.merge([ + [template(.imgflip, "112126428", "Distracted Boyfriend")], + [template(.memegen, "db", "Distracted Boyfriend")], + ]) + XCTAssertEqual(merged.count, 1) + XCTAssertEqual(merged[0].source, .imgflip) + } + + func testMergeDeduplicatesCaseAndPunctuationInsensitively() { + let merged = MemeTemplateCatalog.merge([ + [template(.imgflip, "i1", "Two Buttons")], + [template(.memegen, "m1", "two buttons!")], + ]) + XCTAssertEqual(merged.count, 1) + } + + func testMergeDropsUnnameableTemplates() { + let merged = MemeTemplateCatalog.merge([[ + template(.memegen, "m1", " "), + template(.memegen, "m2", "Success Kid"), + ]]) + XCTAssertEqual(merged.map(\.name), ["Success Kid"]) + } + + func testMergePreservesPopularityOrderWithinASource() { + let merged = MemeTemplateCatalog.merge([[ + template(.imgflip, "1", "First"), + template(.imgflip, "2", "Second"), + template(.imgflip, "3", "Third"), + ]]) + XCTAssertEqual(merged.map(\.name), ["First", "Second", "Third"]) + } + + func testQualifiedIDsKeepSourcesFromCollidingOnTheImageCache() { + let a = MemeTemplateCatalog.qualifiedID(.imgflip, "drake") + let b = MemeTemplateCatalog.qualifiedID(.memegen, "drake") + XCTAssertNotEqual(a, b) + XCTAssertEqual(MemeTemplateCatalog.source(ofQualifiedID: a), .imgflip) + XCTAssertEqual(MemeTemplateCatalog.source(ofQualifiedID: b), .memegen) + } + + func testSourceOfUnqualifiedIDIsNil() { + XCTAssertNil(MemeTemplateCatalog.source(ofQualifiedID: "181913649")) + } + + // MARK: - Keyword search (the merged corpus has to be findable) + + /// memegen names this template "Sweet Brown"; nobody searches for that. The + /// keyword is the phrase people actually type. + func testSearchMatchesKeywordsAsWellAsNames() { + let catalog = [template(.memegen, "aint-got-time", "Sweet Brown", + keywords: ["Ain't Nobody Got Time For That"])] + XCTAssertEqual(MemeTemplateCatalog.search("nobody got time", in: catalog).count, 1) + } + + /// The cross-field case a naive per-field search misses. + func testSearchTokensMaySpanNameAndKeywords() { + let catalog = [template(.userLibrary, "u1", "Кот", keywords: ["russian cat"])] + XCTAssertEqual(MemeTemplateCatalog.search("кот cat", in: catalog).count, 1) + } + + func testSearchFindsCyrillicNamesInTheirOwnScript() { + let catalog = [ + template(.userLibrary, "u1", "Кот в шоке"), + template(.imgflip, "i1", "Drake Hotline Bling"), + ] + let hits = MemeTemplateCatalog.search("шоке", in: catalog) + XCTAssertEqual(hits.map(\.name), ["Кот в шоке"]) + } + + func testSearchNeverSubstitutesAPopularTemplateForNoMatch() { + let catalog = [template(.imgflip, "i1", "Drake Hotline Bling")] + XCTAssertTrue(MemeTemplateCatalog.search("yoda", in: catalog).isEmpty) + } + + func testEmptySearchReturnsTheWholeMergedCatalog() { + let catalog = [template(.imgflip, "i1", "A"), template(.memegen, "m1", "B")] + XCTAssertEqual(MemeTemplateCatalog.search(" ", in: catalog).count, 2) + } + + /// The user's own templates sort first, so the prompt cap can never exclude them. + func testPromptNamesCapTheCorpusButKeepUserTemplates() { + let merged = MemeTemplateCatalog.merge([ + [template(.userLibrary, "u1", "Кот в шоке")], + (0..<200).map { template(.imgflip, "i\($0)", "Template \($0)") }, + ]) + let names = MemeTemplateCatalog.promptNames(merged, limit: 100) + XCTAssertEqual(names.count, 100) + XCTAssertEqual(names.first, "Кот в шоке") + } + + // MARK: - Provider wire shapes + + func testImgflipTemplatesAreSourceQualified() throws { + let json = """ + {"success":true,"data":{"memes":[ + {"id":"181913649","name":"Drake Hotline Bling","url":"https://i.imgflip.com/30b1gx.jpg", + "width":1200,"height":1200,"box_count":2}]}} + """ + let decoded = try JSONDecoder().decode( + MemeTemplateCatalogResponse.self, from: Data(json.utf8)) + XCTAssertEqual(decoded.templates[0].id, "imgflip:181913649") + XCTAssertEqual(decoded.templates[0].source, .imgflip) + } + + /// Pinned against the real memegen.link `/templates` payload shape. + func testMemegenResponseDecodesTheLiveShape() throws { + let json = """ + [{"id":"aint-got-time","name":"Sweet Brown","lines":2,"overlays":0,"styles":[], + "blank":"https://api.memegen.link/images/aint-got-time.jpg", + "keywords":["Ain't Nobody Got Time For That"], + "_self":"https://api.memegen.link/templates/aint-got-time"}] + """ + let decoded = try JSONDecoder().decode( + MemegenTemplateResponse.self, from: Data(json.utf8)) + XCTAssertEqual(decoded.templates.count, 1) + XCTAssertEqual(decoded.templates[0].id, "memegen:aint-got-time") + XCTAssertEqual(decoded.templates[0].name, "Sweet Brown") + XCTAssertEqual(decoded.templates[0].url, + "https://api.memegen.link/images/aint-got-time.jpg") + XCTAssertEqual(decoded.templates[0].keywords, ["Ain't Nobody Got Time For That"]) + } + + func testMemegenTemplatesWithoutKeywordsStillDecode() throws { + let json = """ + [{"id":"aag","name":"Ancient Aliens Guy","blank":"https://api.memegen.link/images/aag.jpg"}] + """ + let decoded = try JSONDecoder().decode( + MemegenTemplateResponse.self, from: Data(json.utf8)) + XCTAssertEqual(decoded.templates[0].keywords, []) + } + + func testMemegenTemplatesWithoutAUsableNameOrImageAreDropped() throws { + let json = """ + [{"id":"a","name":" ","blank":"https://x/a.jpg"}, + {"id":"b","name":"Fine","blank":""}, + {"id":"c","name":"Kept","blank":"https://x/c.jpg"}] + """ + let decoded = try JSONDecoder().decode( + MemegenTemplateResponse.self, from: Data(json.utf8)) + XCTAssertEqual(decoded.templates.map(\.name), ["Kept"]) + } + + /// A cache written before v3 has no `source`/`keywords`. It must still load — + /// discarding it would reintroduce the cold-start failure on every upgrade. + func testTemplateFromAnOlderCacheDecodesWithDefaults() throws { + let json = #"{"id":"imgflip:1","name":"Drake","url":"u","width":10,"height":10}"# + let decoded = try JSONDecoder().decode(MemeTemplate.self, from: Data(json.utf8)) + XCTAssertEqual(decoded.source, .imgflip) + XCTAssertEqual(decoded.keywords, []) + } + + // MARK: - Catalog cache policy + + private func cached(ageSeconds: TimeInterval, count: Int = 3, version: Int = 1) + -> MemeCatalogCache.Cached { + MemeCatalogCache.Cached( + version: version, + fetchedAt: Date(timeIntervalSince1970: 10_000 - ageSeconds), + templates: (0.. MemeUserLibrary.Entry { + MemeUserLibrary.Entry(id: id, name: name, file: file ?? "\(id).png", + width: 100, height: 100) + } + + func testAddingKeepsNamesUnique() { + var index = MemeUserLibrary.Index() + index = MemeUserLibrary.adding(entry("1", "Кот"), to: index) + index = MemeUserLibrary.adding(entry("2", "Кот"), to: index) + XCTAssertEqual(index.entries.map(\.name), ["Кот", "Кот 2"]) + } + + func testRemovingDropsOnlyTheNamedEntry() { + var index = MemeUserLibrary.Index(entries: [entry("1", "A"), entry("2", "B")]) + index = MemeUserLibrary.removing(id: "1", from: index) + XCTAssertEqual(index.entries.map(\.id), ["2"]) + } + + func testRenamingAnEntryToItsOwnNameDoesNotSuffixIt() { + var index = MemeUserLibrary.Index(entries: [entry("1", "Кот"), entry("2", "Пёс")]) + index = MemeUserLibrary.renaming(id: "1", to: "Кот", in: index) + XCTAssertEqual(index.entries[0].name, "Кот") + } + + func testRenamingOntoAnotherEntrysNameIsSuffixed() { + var index = MemeUserLibrary.Index(entries: [entry("1", "Кот"), entry("2", "Пёс")]) + index = MemeUserLibrary.renaming(id: "2", to: "Кот", in: index) + XCTAssertEqual(index.entries[1].name, "Кот 2") + } + + /// The user can delete files in Finder; the index self-heals rather than showing + /// broken cells. + func testPruningDropsEntriesWhoseImageIsGone() { + let index = MemeUserLibrary.Index(entries: [entry("1", "A"), entry("2", "B")]) + let pruned = MemeUserLibrary.pruned(index, existingFiles: ["1.png"]) + XCTAssertEqual(pruned.entries.map(\.id), ["1"]) + } + + /// The index is a plain JSON file in a user-writable directory — untrusted input + /// that becomes a path component, so it is validated before it is ever joined. + func testTraversalShapedFileNamesAreRefused() { + XCTAssertFalse(MemeUserLibrary.isSafeFileName("../../../../etc/passwd")) + XCTAssertFalse(MemeUserLibrary.isSafeFileName("..")) + XCTAssertFalse(MemeUserLibrary.isSafeFileName("sub/dir.png")) + XCTAssertFalse(MemeUserLibrary.isSafeFileName("back\\slash.png")) + XCTAssertFalse(MemeUserLibrary.isSafeFileName(".hidden.png")) + XCTAssertFalse(MemeUserLibrary.isSafeFileName("")) + XCTAssertTrue(MemeUserLibrary.isSafeFileName("abc-123.png")) + } + + func testUnsafeAndUnnamedEntriesAreExcludedFromTheCatalog() { + let index = MemeUserLibrary.Index(entries: [ + entry("1", "Good"), + entry("2", "Traversal", file: "../evil.png"), + entry("3", " "), + ]) + XCTAssertEqual(MemeUserLibrary.safeEntries(index).map(\.name), ["Good"]) + } + + func testLibraryProjectsIntoFileURLTemplates() { + let directory = URL(fileURLWithPath: "/tmp/templates", isDirectory: true) + let index = MemeUserLibrary.Index(entries: [entry("abc", "Кот в шоке")]) + let templates = MemeUserLibrary.templates(from: index, directory: directory) + + XCTAssertEqual(templates.count, 1) + XCTAssertEqual(templates[0].id, "userLibrary:abc") + XCTAssertEqual(templates[0].name, "Кот в шоке") + XCTAssertEqual(templates[0].source, .userLibrary) + XCTAssertTrue(templates[0].url.hasPrefix("file://")) + XCTAssertTrue(templates[0].url.hasSuffix("/tmp/templates/abc.png")) + } + + func testLibraryIndexRoundTripsThroughCodable() throws { + let index = MemeUserLibrary.Index(entries: [entry("1", "Кот в шоке")]) + let data = try JSONEncoder().encode(index) + let decoded = try JSONDecoder().decode(MemeUserLibrary.Index.self, from: data) + XCTAssertEqual(decoded, index) + } + + // MARK: - Busy-state machine (the "stuck loading" report) + + func testAFreshStateIsIdleAndCanGenerate() { + let state = MemeGenerationState() + XCTAssertFalse(state.isGenerating) + XCTAssertTrue(state.canGenerate) + XCTAssertNil(state.generateBlockedReason()) + } + + func testBeginTakesAFreshTicketAndMarksTheSurfaceBusy() { + var state = MemeGenerationState() + let ticket = state.begin(.loadingCatalog) + XCTAssertTrue(state.isGenerating) + XCTAssertFalse(state.canGenerate) + XCTAssertTrue(state.accepts(ticket: ticket)) + } + + func testAdvanceMovesPhaseWithinTheSameUnitOfWork() { + var state = MemeGenerationState() + let ticket = state.begin(.loadingCatalog) + XCTAssertTrue(state.advance(.asking, ticket: ticket)) + XCTAssertEqual(state.phase, .asking) + } + + func testFinishClearsTheBusyFlag() { + var state = MemeGenerationState() + let ticket = state.begin(.asking) + XCTAssertTrue(state.finish(ticket: ticket)) + XCTAssertFalse(state.isGenerating) + XCTAssertTrue(state.canGenerate) + } + + /// The regression this whole type exists for: EVERY exit path clears the flag, so + /// a failure, a rejection, or a timeout can't leave the surface stuck. + func testFinishingIsIdempotentSoDoubleExitPathsCannotStick() { + var state = MemeGenerationState() + let ticket = state.begin(.asking) + XCTAssertTrue(state.finish(ticket: ticket)) + XCTAssertFalse(state.finish(ticket: ticket)) + XCTAssertFalse(state.isGenerating) + } + + /// A late result from superseded work must not clear the NEWER work's phase — + /// the other half of the stuck-state bug, in the opposite direction. + func testAStaleFinishCannotUnstickNewerWork() { + var state = MemeGenerationState() + let stale = state.begin(.asking) + let current = state.begin(.downloading(templateName: "Drake")) + + XCTAssertFalse(state.finish(ticket: stale)) + XCTAssertTrue(state.isGenerating) + XCTAssertTrue(state.accepts(ticket: current)) + } + + func testAStaleAdvanceCannotDragTheUIBackToItsOwnPhase() { + var state = MemeGenerationState() + let stale = state.begin(.loadingCatalog) + _ = state.begin(.asking) + XCTAssertFalse(state.advance(.downloading(templateName: "X"), ticket: stale)) + XCTAssertEqual(state.phase, .asking) + } + + func testCancelReturnsToIdleAndRefusesTheAbandonedResult() { + var state = MemeGenerationState() + let ticket = state.begin(.asking) + state.cancel() + + XCTAssertFalse(state.isGenerating) + XCTAssertTrue(state.canGenerate) + XCTAssertFalse(state.accepts(ticket: ticket)) + XCTAssertFalse(state.finish(ticket: ticket)) + } + + /// Feedback #3: the candidate strip and the Browse grid stay live while a + /// generation runs. Switching templates is a local re-render, not an LLM call. + func testTemplateSelectionStaysAvailableInEveryPhase() { + var state = MemeGenerationState() + XCTAssertTrue(state.canSelectTemplate) + _ = state.begin(.loadingCatalog) + XCTAssertTrue(state.canSelectTemplate) + _ = state.begin(.asking) + XCTAssertTrue(state.canSelectTemplate) + _ = state.begin(.downloading(templateName: "Drake")) + XCTAssertTrue(state.canSelectTemplate) + } + + /// Feedback #2: warming is honest and non-blocking — the user can browse, and + /// Generate says what it is waiting for instead of failing. + func testWarmingBlocksGenerateHonestlyButNotBrowsing() { + var state = MemeGenerationState() + _ = state.begin(.warming) + + XCTAssertFalse(state.isGenerating) + XCTAssertFalse(state.canGenerate) + XCTAssertEqual(state.generateBlockedReason(), "Preparing model…") + XCTAssertTrue(state.canSelectTemplate) + } + + func testEveryBusyPhaseNamesItselfInTheStatusLine() { + XCTAssertEqual(MemeGenerationState.Phase.idle.statusText, "") + XCTAssertEqual(MemeGenerationState.Phase.warming.statusText, "Preparing model…") + XCTAssertEqual(MemeGenerationState.Phase.loadingCatalog.statusText, "Loading templates…") + XCTAssertEqual(MemeGenerationState.Phase.asking.statusText, "Asking the model…") + XCTAssertEqual( + MemeGenerationState.Phase.downloading(templateName: "Drake").statusText, + "Downloading Drake…") + } + + func testTimeoutMessageOffersBothRecoveries() { + XCTAssertTrue(MemeGenerationState.timeoutMessage.contains("Generate again")) + XCTAssertTrue(MemeGenerationState.timeoutMessage.contains("Browse all")) + XCTAssertGreaterThan(MemeGenerationState.generateTimeout, 0) + } +} From 7fc7dbd48ad389808202fcb540a6c8af5e181e84 Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Sun, 2 Aug 2026 16:55:09 -0700 Subject: [PATCH 06/15] spike(meme-plugin): warm the LLM on window open, import your own templates, imgflip-shaped editor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The app half of v3. Wires the new pure layer live and fixes the three remaining owner reports. **"First Generate fails: network error and model loading" (#2).** Neither string exists in the sources — it was a request hitting a llama-server that had not started, surfacing connection-refused as a network error. Nothing warmed the LLM on window open (Scratchpad doesn't either), so the first Generate always paid the cold start. `MemeGeneratorWindowController` now calls `model.windowDidOpen()`, which warms the model and opens the catalog. Generate WAITS behind an honest "Preparing model…" instead of firing into a dead socket. `AppState.warmLlamaServerIfPossible` gained a `provider:` parameter, because the global version only fires when Settings → Cleanup is ITSELF set to the bundled provider — a plugin resolved to bundled would never have warmed. That is the same split `ensureBundledLLMReady(provider:)` already makes for MAK-53. Paid for in the AppState ratchet by folding two comments and dropping a `warmLlamaServerIfPossible` call that `ensureLLMModelExists` already makes on both its exit paths; the ratchet still reports exactly 7051. **Catalog at window open, cached thereafter.** Disk cache paints first, the network only upgrades it, and a failed refresh with templates already on screen is silent. Retry is offered when — and only when — there is nothing to show. Thumbnails are cached as small JPEGs so a second open paints instantly and works offline. **Import your own template.** File picker, drag-drop onto the template column, or ⌘V. Images are COPIED into Application Support rather than referenced, because a template pointing at ~/Downloads breaks the first time the user tidies up. A user-library template is a `file:` URL flowing through the SAME fetch, merge, prompt, and render path as a remote one — which is why importing needed no changes anywhere in the render or export code. **Editor polish (#4).** "Add text" is now outside every conditional. v2 rendered the whole editor panel only when `boxes` was non-empty, so deleting the last box removed the only control that could add one back — a dead end with no way out but regenerating. The empty canvas and empty box list both say what to do next. The window is three columns toward the imgflip.com shape: template search/browse and import on the left (prominent, not behind a sheet), canvas center, box controls right. User templates carry a badge so a mixed grid is legible. Nothing local shows a spinner. **Cancel + timeout.** A Cancel button appears only while work is in flight, and a 120s ceiling recovers a hung request — v2 had neither, so a model that never answered left the surface busy until the window closed. The manifest declares api.memegen.link. A plugin quietly contacting an undeclared host is exactly what the `networkHosts` label exists to prevent, so the new disclosure string is pinned by a test. memegen's server-side captioning API is deliberately NOT used: it would put the user's words on someone else's server. Co-Authored-By: Claude --- OpenWhisp/Models/AppState.swift | 28 +- OpenWhisp/Services/PluginRegistry.swift | 12 +- .../PluginSystemTests.swift | 11 + .../MemeGenerator/MemeGeneratorModel.swift | 522 +++++++++++++----- plugins/MemeGenerator/MemeGeneratorView.swift | 490 ++++++++++------ .../MemeGeneratorWindowController.swift | 18 +- plugins/MemeGenerator/MemeLibraryStore.swift | 257 +++++++++ .../MemeGenerator/MemeTemplateService.swift | 119 +++- plugins/MemeGenerator/manifest.json | 4 +- 9 files changed, 1111 insertions(+), 350 deletions(-) create mode 100644 plugins/MemeGenerator/MemeLibraryStore.swift diff --git a/OpenWhisp/Models/AppState.swift b/OpenWhisp/Models/AppState.swift index 92b6309..3e7e9e9 100644 --- a/OpenWhisp/Models/AppState.swift +++ b/OpenWhisp/Models/AppState.swift @@ -385,8 +385,7 @@ class AppState: ObservableObject { // First enable with the built-in provider should work with zero // setup: provision the model if it isn't on disk yet (no-ops and // warms when it is). - if llmProvider == "bundled" { ensureLLMModelExists() } - else { warmLlamaServerIfPossible() } + if llmProvider == "bundled" { ensureLLMModelExists() } else { warmLlamaServerIfPossible() } } else { llamaEngine?.stopServer() } @@ -443,10 +442,8 @@ class AppState: ObservableObject { // Provision/warm only when AI cleanup is actually on — switching // the provider (or Reset All Settings) with cleanup off must not // kick off a model download behind the user's back. - if openAIEnhancementEnabled { - ensureLLMModelExists() - warmLlamaServerIfPossible() - } + // `ensureLLMModelExists` warms on completion, so it is the only call. + if openAIEnhancementEnabled { ensureLLMModelExists() } } else { // Free the ~0.7-1.5 GB the built-in LLM holds when it's not the // active provider. @@ -2786,22 +2783,25 @@ class AppState: ObservableObject { cachedBundledLLMManifest } - /// True when a resident whisper.cpp server is held in memory at the same time - /// the built-in LLM would run. Both load a model, so on small-RAM Macs they - /// can race for memory. (WhisperKit/AppleSpeech keep no resident server.) + /// True when a resident whisper.cpp server is held in memory at the same time the + /// built-in LLM would run. (WhisperKit/AppleSpeech keep no resident server.) private var whisperServerResident: Bool { transcriptionEngine == "whisper" && whisperBackend == "serverAPI" } /// Start the bundled llama-server if the built-in provider is the active, /// enabled, downloaded one. Idempotent (the engine no-ops if already healthy). - func warmLlamaServerIfPossible() { - guard llmProvider == "bundled", - openAIEnhancementEnabled, - bundledLLMModelInstalled else { return } + /// `provider` defaults to the global cleanup one; a caller with its OWN resolved + /// provider passes it (the MAK-53 split `ensureBundledLLMReady` already makes) so + /// a surface resolved to `bundled` still warms when cleanup uses something else. + /// The cleanup on/off toggle then gates only the implicit, global case. + func warmLlamaServerIfPossible(provider explicitProvider: String? = nil) { + let provider = explicitProvider ?? llmProvider + guard provider == "bundled", bundledLLMModelInstalled else { return } + guard explicitProvider != nil || openAIEnhancementEnabled else { return } let engine = ensureLlamaEngine() // Shorter idle teardown when a whisper-server is also resident, to relieve - // dual-engine memory pressure sooner. + // dual-engine memory pressure sooner (small-RAM Macs run both models). engine.idleTimeout = whisperServerResident ? 30 : 90 engine.ensureRunning(modelPath: selectedLLMModelPath()) { _ in } } diff --git a/OpenWhisp/Services/PluginRegistry.swift b/OpenWhisp/Services/PluginRegistry.swift index 18a8dfb..5d31e11 100644 --- a/OpenWhisp/Services/PluginRegistry.swift +++ b/OpenWhisp/Services/PluginRegistry.swift @@ -39,12 +39,16 @@ public enum PluginRegistry { public static let memeGenerator = PluginManifest( id: "meme-generator", name: "Meme Generator", - version: "0.1.0", + version: "0.3.0", summary: "Dictate a meme description — the AI picks a template and writes the captions.", symbol: "photo.badge.plus", entry: .builtIn, - // Template images are downloaded from imgflip's public, key-less catalog. - // Captions are rendered locally, so no text ever leaves the Mac. - networkHosts: ["api.imgflip.com", "i.imgflip.com"] + // Template images are downloaded from two public, key-less catalogs. Captions + // are rendered LOCALLY, so no text ever leaves the Mac — note memegen.link + // also offers server-side captioning by URL and this plugin deliberately does + // not use it. Templates the user imports themselves need no network at all. + networkHosts: [ + "api.imgflip.com", "i.imgflip.com", "api.memegen.link", + ] ) } diff --git a/Tests/OpenWhispCoreTests/PluginSystemTests.swift b/Tests/OpenWhispCoreTests/PluginSystemTests.swift index 8617ab7..d186142 100644 --- a/Tests/OpenWhispCoreTests/PluginSystemTests.swift +++ b/Tests/OpenWhispCoreTests/PluginSystemTests.swift @@ -318,4 +318,15 @@ final class PluginSystemTests: XCTestCase { XCTAssertNotNil(PluginRegistry.memeGenerator.networkDisclosure) XCTAssertTrue(PluginRegistry.memeGenerator.networkHosts.contains("api.imgflip.com")) } + + /// v3 added a SECOND template provider, so the disclosure had to grow with it. + /// A plugin that quietly contacts a host it never declared is precisely the + /// failure the `networkHosts` label exists to prevent, and the pane renders this + /// list verbatim — so the new host is pinned rather than left to review. + func testMemeGeneratorDeclaresTheMemegenProviderItAdded() { + XCTAssertTrue(PluginRegistry.memeGenerator.networkHosts.contains("api.memegen.link")) + XCTAssertEqual( + PluginRegistry.memeGenerator.networkDisclosure, + "Connects to api.imgflip.com, i.imgflip.com, api.memegen.link when you use it.") + } } diff --git a/plugins/MemeGenerator/MemeGeneratorModel.swift b/plugins/MemeGenerator/MemeGeneratorModel.swift index 10999d5..16f56af 100644 --- a/plugins/MemeGenerator/MemeGeneratorModel.swift +++ b/plugins/MemeGenerator/MemeGeneratorModel.swift @@ -1,33 +1,41 @@ import AppKit import SwiftUI -/// The observable state behind the Meme Generator window (spike). +/// The observable state behind the Meme Generator window (spike v3). /// /// The pipeline, end to end: /// -/// 1. The user DICTATES a description (or types it) into `description`. -/// 2. The imgflip template catalog is fetched (once per session, then cached). -/// 3. `generate()` sends the description **plus the catalog's actual template names** +/// 1. The window OPENS: the LLM is warmed and the template catalog is opened from +/// disk (then refreshed in the background). Neither blocks the user. +/// 2. The user DICTATES a description (or types it) into `description`. +/// 3. `generate()` sends the description **plus the merged catalog's template names** /// to the configured LLM via the injected `aiCall` seam — the same /// closure-injection `ScratchpadModel` uses, so this model never touches AppState /// and stays stubbable. /// 4. The reply is parsed by the pure `MemeAI.parseRanked`, which keeps only names /// that really exist in the catalog. The result is a RANKED candidate list. /// 5. The best candidate auto-renders; the rest sit in a thumbnail strip. The user -/// can click another candidate, or open "Browse all" and pick any of the 100. -/// 6. The chosen template is downloaded and captioned LOCALLY by `MemeRenderer` from -/// the caption BOX model, which the manual editor then mutates. +/// can click another candidate, or open "Browse" and pick any of the ~300. +/// 6. The chosen template is loaded (from disk or the network) and captioned LOCALLY +/// by `MemeRenderer` from the caption BOX model, which the editor then mutates. /// -/// Only steps 2 and 6 touch the network, and only to GET images — no user text is -/// ever sent to imgflip. +/// Only the catalog fetch and the image GETs touch the network, and only to READ — +/// no user text is ever sent anywhere. /// -/// ## v2 — why candidates +/// ## v3 — what the owner's live testing changed /// -/// v1 asked the model for a free-text template name and lexically matched it. "yoda -/// meme" scored below threshold and silently rendered onto Drake. v2 constrains the -/// model to the real corpus and, when nothing fits, SAYS SO — the fallback still -/// happens (a meme is better than an error) but never without the candidate strip and -/// Browse-all visible, so the user immediately sees it was a guess. +/// * **The corpus is now three providers** (imgflip + memegen + the user's own +/// imported library), merged and cached on disk. The user library is the answer to +/// "the templates are America-centric": any image can become a template, in any +/// language, and it works offline. +/// * **The LLM is warmed on window open.** The reported "first Generate fails with a +/// network error and model loading" was a request hitting a llama-server that +/// hadn't started. Generate now WAITS behind an honest "Preparing model…" instead. +/// * **The busy flag is a state machine** (`MemeGenerationState`). The reported stuck +/// spinner was a `Bool` that several exit paths never cleared; every exit path now +/// goes through one idempotent `finish`, plus a Cancel button and a hard timeout. +/// * **Template switching is never blocked** — it is a local re-render, so it stays +/// live even while a generation is in flight. @MainActor final class MemeGeneratorModel: ObservableObject { @@ -42,8 +50,9 @@ final class MemeGeneratorModel: ObservableObject { /// swallowed. @Published private(set) var status: String = "" - /// True while a generate round-trip is in flight (disables the button). - @Published private(set) var isBusy: Bool = false + /// The busy-state machine. Replaces v2's `isBusy` Bool — see + /// `MemeGenerationState` for why the stuck-spinner bug was structural. + @Published private(set) var state = MemeGenerationState() /// The editable caption boxes. This is the SINGLE source of truth for what the /// meme says and where — the AI seeds it, the editor mutates it, and both the @@ -53,38 +62,57 @@ final class MemeGeneratorModel: ObservableObject { /// The box the editor's side panel is editing, if any. @Published var selectedBoxID: UUID? - /// The ranked template candidates the model proposed, best first. Empty until a - /// generate has run. + /// The ranked template candidates the model proposed, best first. @Published private(set) var candidates: [MemeTemplate] = [] /// The template currently rendered. @Published private(set) var selectedTemplate: MemeTemplate? - /// The whole catalog, for "Browse all". Empty until fetched. + /// The whole merged catalog, for Browse. @Published private(set) var catalog: [MemeTemplate] = [] /// True when the model named only templates that don't exist in the corpus and we - /// fell back. Drives the honest "not in this corpus" warning — the candidate strip - /// and Browse all stay visible so the fallback is never silent. Cleared once the - /// user picks a template themselves; the choice is theirs from then on. + /// fell back. Drives the honest "not in this corpus" warning. @Published private(set) var didFallBack: Bool = false /// True when the CURRENT candidate strip is a fallback list rather than the - /// model's own picks. Unlike `didFallBack` this describes the strip's contents, so - /// it survives the user selecting a template and the strip's label stays accurate. + /// model's own picks. @Published private(set) var candidatesAreFallback: Bool = false - /// The user's Browse-all search text. Filtering is local and pure. + /// The user's Browse search text. Filtering is local and pure. @Published var searchText: String = "" - /// The templates matching `searchText`, in popularity order. + /// Set when the catalog couldn't be loaded AND nothing was cached — the only case + /// where the user must act. Drives the Retry affordance. + @Published private(set) var catalogFailed: Bool = false + + /// The templates matching `searchText`, across names AND keywords so a merged, + /// multi-lingual corpus is actually findable. var searchResults: [MemeTemplate] { - MemeTemplateMatcher.search(searchText, in: catalog) + MemeTemplateCatalog.search(searchText, in: catalog) } - /// The template name shown in the "used X" line. var templateName: String { selectedTemplate?.name ?? "" } + // MARK: - Derived UI state + + var isBusy: Bool { state.isGenerating } + + var canGenerate: Bool { + state.canGenerate && !description.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } + + /// Why Generate is unavailable right now, for the button's tooltip and the status + /// line. Nil when it is available. + var generateBlockedReason: String? { state.generateBlockedReason() } + + /// Browse is available as soon as ANYTHING is loadable — including a user library + /// with the network off. + var canBrowse: Bool { !catalog.isEmpty } + + /// The user's own imported templates, for the library management UI. + var userTemplates: [MemeTemplate] { catalog.filter { $0.source == .userLibrary } } + // MARK: - Injected seams /// One LLM round-trip: (instruction, input, resolved model) -> output. Injected @@ -97,13 +125,18 @@ final class MemeGeneratorModel: ObservableObject { private var resolveAIModel: () -> SummaryModelResolver.Resolved = { .init(provider: "", model: "", endpoint: "") } + /// Warms the LLM without running a completion. Injected so this model stays + /// AppState-free and the warm is stubbable. + private var warmModel: (SummaryModelResolver.Resolved) -> Void = { _ in } func configureAI( call: @escaping AICall, - resolveModel: @escaping () -> SummaryModelResolver.Resolved + resolveModel: @escaping () -> SummaryModelResolver.Resolved, + warm: @escaping (SummaryModelResolver.Resolved) -> Void = { _ in } ) { aiCall = call resolveAIModel = resolveModel + warmModel = warm } /// Downloaded template images, keyed by template id, so clicking back and forth @@ -114,17 +147,145 @@ final class MemeGeneratorModel: ObservableObject { /// re-renders instantly without a network round-trip. private var baseImage: NSImage? - /// Monotonic ticket so a late result from a superseded request can't overwrite a - /// newer meme (the `ScratchpadAISession` fence, simplified for one surface). - private var ticket: Int = 0 - /// True once the window is gone — stops a late async result touching the UI. private var isCancelled = false + // MARK: - Window lifecycle + + /// Everything that must happen when the window OPENS, so the first Generate is + /// never the thing that pays for a cold start. + /// + /// Both halves of the owner's report #2 are addressed here: the LLM is warmed, and + /// the catalog is opened from disk immediately (refreshing behind the UI when + /// stale). Neither blocks the user — they can browse and edit while both run. + func windowDidOpen() { + isCancelled = false + warmLLM() + openCatalog() + } + + /// Start the local model loading in the background. + /// + /// Deliberately fire-and-forget with a phase, not an awaited call: the warm has no + /// completion we can observe from here (the engine's readiness is AppState's), so + /// we show "Preparing model…" for a bounded moment and then allow Generate. If the + /// model is still loading when Generate runs, `summarizeResolved` blocks on the + /// same engine anyway — the warm has simply removed most of that wait. + private func warmLLM() { + let resolved = resolveAIModel() + guard ScratchpadAIModel.isUsable(resolved) else { return } + warmModel(resolved) + + let ticket = state.begin(.warming) + Task { [weak self] in + // A short, fixed window. The point is to stop the user firing a request + // into a socket nothing is listening on yet, not to gate the UI on a + // completion we can't see. + try? await Task.sleep(nanoseconds: 2_500_000_000) + guard let self, !self.isCancelled else { return } + if self.state.finish(ticket: ticket), self.status == MemeGenerationState.Phase.warming.statusText { + self.status = "" + } + } + status = MemeGenerationState.Phase.warming.statusText + } + + // MARK: - Catalog + + /// Open the catalog: disk cache first, network second. + /// + /// This is what makes browsing instant and the plugin usable offline. The cache is + /// authoritative for what the user SEES immediately; the network only ever + /// upgrades it. A refresh that fails while templates are on screen is silent — + /// reporting it would make a working plugin look broken. + func openCatalog(forceRefresh: Bool = false) { + let cached = MemeLibraryStore.loadCachedCatalog() + let library = MemeLibraryStore.libraryTemplates() + let decision = forceRefresh + ? MemeCatalogCache.Decision.fetchNow + : MemeCatalogCache.decide(cached: cached, now: Date()) + + // Show whatever we already have RIGHT NOW, before any network work. + if let cached, decision != .fetchNow { + catalog = MemeTemplateCatalog.merge([library, cached.templates]) + catalogFailed = false + } else if !library.isEmpty { + // Offline with an empty cache but a populated library: still a usable + // corpus, and the whole point of letting people import their own. + catalog = library + } + + guard decision != .useCache else { + if status.isEmpty { status = MemeCatalogCache.summary(catalog) } + return + } + refreshCatalog(showProgress: catalog.isEmpty) + } + + /// Fetch the remote catalogs and merge them in. + /// + /// `showProgress` distinguishes the two callers, and with them the TICKET + /// OWNERSHIP — the distinction the v2 stuck-state bug came from getting muddled: + /// + /// * **Blocking** (nothing on screen yet) — takes its own ticket, shows a phase, + /// and must clear it on every exit path. + /// * **Background refresh** (a cached catalog is already visible) — owns NO + /// ticket and never touches the phase. It is invisible by design: the user is + /// already browsing, and a background refresh must not be able to disturb, or + /// worse un-stick, whatever they are doing in the meantime. + private func refreshCatalog(showProgress: Bool) { + let ticket: Int? = showProgress ? state.begin(.loadingCatalog) : nil + if showProgress { status = MemeGenerationState.Phase.loadingCatalog.statusText } + + Task { [weak self] in + guard let self else { return } + let library = MemeLibraryStore.libraryTemplates() + do { + let merged = try await MemeTemplateService.fetchMergedCatalog( + userTemplates: library) + guard !self.isCancelled else { return } + + self.catalog = merged + self.catalogFailed = false + MemeLibraryStore.saveCachedCatalog(merged) + + if let ticket { + if self.state.finish(ticket: ticket) { + self.status = MemeCatalogCache.summary(merged) + } + } else if self.status.isEmpty { + self.status = MemeCatalogCache.summary(merged) + } + } catch { + guard !self.isCancelled else { return } + // With templates already on screen this is a non-event — the cache is + // doing exactly its job. + let message = MemeCatalogCache.refreshFailureMessage( + hasCachedTemplates: !self.catalog.isEmpty, + reason: MemeTemplateService.reason(error)) + if let ticket { self.state.finish(ticket: ticket) } + if let message { + self.catalogFailed = true + self.status = message + } + } + } + } + + /// Re-read the user library and merge it into the live catalog, without a fetch. + /// + /// Called after every import/delete so the grid updates instantly — an import + /// that required a network round-trip to become visible would be absurd. + func reloadUserLibrary() { + let library = MemeLibraryStore.libraryTemplates() + let remote = catalog.filter { $0.source != .userLibrary } + catalog = MemeTemplateCatalog.merge([library, remote]) + catalogFailed = catalog.isEmpty + } + // MARK: - Dictation - /// Append dictated text to the description, matching the Scratchpad's join rule - /// (a space between the existing text and the new words). + /// Append dictated text to the description, matching the Scratchpad's join rule. func appendDictation(_ text: String) { let incoming = text.trimmingCharacters(in: .whitespacesAndNewlines) guard !incoming.isEmpty else { return } @@ -136,17 +297,17 @@ final class MemeGeneratorModel: ObservableObject { status = "" } - var canGenerate: Bool { - !isBusy && !description.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - } - - /// True once there is something to browse — gates the "Browse all" affordance. - var canBrowse: Bool { !catalog.isEmpty } - // MARK: - Generate func generate() { - guard canGenerate else { return } + guard !description.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return } + + // Honest wait instead of a failed request: the v2 report was a Generate fired + // at a model that hadn't finished loading. + if let blocked = state.generateBlockedReason() { + status = blocked + return + } guard let aiCall else { status = "No LLM is configured — set one up in Settings → Cleanup." return @@ -161,63 +322,90 @@ final class MemeGeneratorModel: ObservableObject { return } - ticket += 1 - let myTicket = ticket - isBusy = true + let ticket = state.begin(.loadingCatalog) + status = MemeGenerationState.Phase.loadingCatalog.statusText + startTimeout(ticket: ticket) Task { [weak self] in guard let self else { return } - // The catalog is fetched BEFORE the LLM call now — v2 puts the real - // template names in the prompt, so we can't ask until we have them. - guard await self.ensureCatalog(ticket: myTicket) else { return } - guard !self.isCancelled, myTicket == self.ticket else { return } + // The catalog must exist before we can ask — the prompt carries the real + // template names. Normally it is already warm from window open. + guard await self.ensureCatalogForGenerate(ticket: ticket) else { return } + guard !self.isCancelled, self.state.accepts(ticket: ticket) else { return } - self.status = "Asking the model…" - let names = self.catalog.map(\.name) + self.state.advance(.asking, ticket: ticket) + self.status = MemeGenerationState.Phase.asking.statusText + + let names = MemeTemplateCatalog.promptNames(self.catalog, limit: 100) let payload = MemeAI.rankedUserPayload( description: self.description, templateNames: names) do { let raw = try await aiCall(MemeAI.rankedPrompt, payload, resolved) - guard !self.isCancelled, myTicket == self.ticket else { return } + guard !self.isCancelled, self.state.accepts(ticket: ticket) else { return } switch MemeAI.parseRanked(raw, catalogNames: names) { case .failure(let rejection): - self.finish(myTicket, status: "Couldn't build the meme — \(rejection.reason).") + self.finish(ticket, status: "Couldn't build the meme — \(rejection.reason).") case .success(let spec): - await self.applyRanked(spec, ticket: myTicket) + await self.applyRanked(spec, ticket: ticket) } } catch { - guard !self.isCancelled, myTicket == self.ticket else { return } - self.finish(myTicket, status: "The model request failed — \(Self.reason(error))") + guard !self.isCancelled, self.state.accepts(ticket: ticket) else { return } + self.finish(ticket, status: "The model request failed — \(Self.reason(error))") } } } - /// Fetch the catalog once per session. Returns false when it failed (and has - /// already reported why). - private func ensureCatalog(ticket myTicket: Int) async -> Bool { + /// A hard ceiling on one generate. + /// + /// v2 had none: an LLM call that never returned left the surface busy forever with + /// no way back but closing the window. `finish` is ticket-guarded and idempotent, + /// so this fires harmlessly when the work already completed. + private func startTimeout(ticket: Int) { + Task { [weak self] in + try? await Task.sleep( + nanoseconds: UInt64(MemeGenerationState.generateTimeout * 1_000_000_000)) + guard let self, !self.isCancelled else { return } + if self.state.finish(ticket: ticket) { + self.status = MemeGenerationState.timeoutMessage + } + } + } + + /// Cancel whatever is in flight. The user's escape hatch from a slow model. + func cancelGeneration() { + guard state.isGenerating else { return } + state.cancel() + status = "Cancelled." + } + + /// Make sure there is a catalog to prompt with. Returns false when it failed and + /// has already reported why. + private func ensureCatalogForGenerate(ticket: Int) async -> Bool { guard catalog.isEmpty else { return true } - status = "Loading templates…" + let library = MemeLibraryStore.libraryTemplates() do { - let fetched = try await MemeTemplateService.fetchCatalog() - guard !isCancelled, myTicket == ticket else { return false } - catalog = fetched + let merged = try await MemeTemplateService.fetchMergedCatalog(userTemplates: library) + guard !isCancelled, state.accepts(ticket: ticket) else { return false } + catalog = merged + catalogFailed = false + MemeLibraryStore.saveCachedCatalog(merged) return true } catch { - finish(myTicket, status: - "Couldn't reach imgflip.com for templates — \(Self.reason(error)) " - + "The meme generator needs a connection to fetch template images.") + guard !isCancelled, state.accepts(ticket: ticket) else { return false } + catalogFailed = true + finish(ticket, status: + "Couldn't load meme templates — \(MemeTemplateService.reason(error)) " + + "Press Retry, or import your own template to work offline.") return false } } /// Turn a validated ranked spec into candidates + a rendered best guess. - private func applyRanked(_ spec: MemeAI.RankedSpec, ticket myTicket: Int) async { - // Resolve names back to templates. `parseRanked` guarantees every name is in - // the catalog, so a nil here would be a bug rather than model noise. + private func applyRanked(_ spec: MemeAI.RankedSpec, ticket: Int) async { var picks = spec.templateNames.compactMap { name in catalog.first { $0.name == name } } @@ -227,12 +415,7 @@ final class MemeGeneratorModel: ObservableObject { if picks.isEmpty { // Nothing the model named exists in the corpus. Still render something — // the user asked for a meme — but ONLY alongside a visible candidate strip - // and Browse all, so the substitution is obvious rather than silent. That - // is the whole fix for the "yoda → Drake with no explanation" report. - // - // Rank by the user's OWN words first: if the description happens to - // mention something that is in the catalog, that beats blind popularity. - // Popularity is the last resort, and only to fill the strip. + // and Browse, so the substitution is obvious rather than silent. let lexical = MemeTemplateMatcher.ranked( for: description, in: catalog, limit: MemeAI.maxCandidates) let popular = catalog.prefix(MemeAI.maxCandidates) @@ -247,78 +430,80 @@ final class MemeGeneratorModel: ObservableObject { selectedBoxID = boxes.first?.id guard let best = picks.first else { - finish(myTicket, status: "imgflip returned no templates to choose from.") + finish(ticket, status: "There are no templates to choose from — import one to get started.") return } - await renderTemplate(best, ticket: myTicket, isNewGeneration: true) + await renderTemplate(best, ticket: ticket, isNewGeneration: true) } // MARK: - Template selection /// Re-render the current captions onto another template. /// - /// The captions carry over unchanged — that is the point of the candidate strip - /// and of normalized box geometry: switching templates is a *presentation* choice, - /// not a reason to lose the user's text or their box positions. + /// **Never blocked by a generation in flight** (feedback #3). Switching templates + /// re-renders the same boxes onto an image that is cached or one GET away — no + /// LLM round-trip — so gating it on the busy flag was pure reflex, and it is what + /// turned a stuck flag into a frozen window. + /// + /// It runs on its OWN ticket, so picking a template while the model is thinking + /// supersedes the generation rather than racing it: the user's explicit choice + /// wins over the machine's pending guess. func select(template: MemeTemplate) { - guard !isBusy else { return } guard template.id != selectedTemplate?.id else { return } - // Seed boxes if the user picked a template before ever generating (Browse all - // is reachable as soon as the catalog is loaded). + // Seed boxes if the user picked a template before ever generating. if boxes.isEmpty { boxes = MemeCaptionLayout.seedBoxes(topText: "", bottomText: "") selectedBoxID = boxes.first?.id } - // The user has now chosen deliberately, so this is no longer a fallback — - // leaving the warning up would be nagging about a decision they just made. + // The user has now chosen deliberately, so this is no longer a fallback. didFallBack = false - ticket += 1 - let myTicket = ticket - isBusy = true + let ticket = state.begin(.downloading(templateName: template.name)) Task { [weak self] in - await self?.renderTemplate(template, ticket: myTicket, isNewGeneration: false) + await self?.renderTemplate(template, ticket: ticket, isNewGeneration: false) } } - /// Load a template's image (cached) and render the current boxes onto it. + /// Load a template's image (cached, from disk or network) and render the boxes. private func renderTemplate( - _ template: MemeTemplate, ticket myTicket: Int, isNewGeneration: Bool + _ template: MemeTemplate, ticket: Int, isNewGeneration: Bool ) async { let image: NSImage if let cached = imageCache[template.id] { image = cached } else { - status = "Downloading \(template.name)…" + state.advance(.downloading(templateName: template.name), ticket: ticket) + status = MemeGenerationState.Phase.downloading(templateName: template.name).statusText do { image = try await MemeTemplateService.fetchImage(template) } catch { - finish(myTicket, status: - "Couldn't download the template image — \(Self.reason(error))") + finish(ticket, status: + "Couldn't load the template image — \(Self.reason(error))") return } - guard !isCancelled, myTicket == ticket else { return } + guard !isCancelled, state.accepts(ticket: ticket) else { return } imageCache[template.id] = image + MemeLibraryStore.storeThumbnail(image, for: template.id) } - guard !isCancelled, myTicket == ticket else { return } + guard !isCancelled, state.accepts(ticket: ticket) else { return } baseImage = image selectedTemplate = template redraw() - finish(myTicket, status: statusLine(isNewGeneration: isNewGeneration)) + finish(ticket, status: statusLine(isNewGeneration: isNewGeneration)) } /// The line under the controls: honest about a fallback, quiet otherwise. private func statusLine(isNewGeneration: Bool) -> String { guard let selectedTemplate else { return "" } if didFallBack, isNewGeneration { - return "Nothing in the corpus matched — this is imgflip's top 100, and your " - + "meme isn't one of them. Showing \(selectedTemplate.name); pick another " - + "below or Browse all." + return "Nothing in the corpus matched your description. Showing " + + "\(selectedTemplate.name) — pick another below, Browse all " + + "\(catalog.count) templates, or import your own." } return "Used \(selectedTemplate.name). Drag the captions to reposition them." } @@ -327,16 +512,15 @@ final class MemeGeneratorModel: ObservableObject { /// Re-render the current boxes onto the current template. /// - /// Called on every editor mutation. Rendering is local CoreGraphics onto an - /// already-downloaded image, so this is cheap enough to run per keystroke and per - /// drag frame — no debounce, and the preview is therefore literally the export. + /// Local CoreGraphics onto an already-loaded image, so this is cheap enough to run + /// per keystroke and per drag frame — no debounce, and the preview is therefore + /// literally the export. func redraw() { guard let baseImage else { return } meme = MemeRenderer.render(template: baseImage, boxes: boxes) } - /// Mutate one box and re-render. The single funnel for every editor edit, so - /// clamping and redrawing can't be forgotten at a call site. + /// Mutate one box and re-render. The single funnel for every editor edit. func updateBox(id: UUID, _ mutate: (inout MemeCaptionLayout.CaptionBox) -> Void) { guard let index = boxes.firstIndex(where: { $0.id == id }) else { return } var box = boxes[index] @@ -346,6 +530,10 @@ final class MemeGeneratorModel: ObservableObject { } /// Add an empty caption box, placed so it doesn't land on the previous one. + /// + /// **Always available** (feedback #4). v2 only rendered the editor panel when + /// `boxes` was non-empty, so deleting the last box deleted the only Add button + /// with it — a dead end with no way back except regenerating. func addBox() { let center = MemeCaptionLayout.newBoxCenter(existingCount: boxes.count) let box = MemeCaptionLayout.CaptionBox( @@ -361,31 +549,116 @@ final class MemeGeneratorModel: ObservableObject { redraw() } - /// The currently selected box, for the side panel's bindings. var selectedBox: MemeCaptionLayout.CaptionBox? { guard let selectedBoxID else { return nil } return boxes.first { $0.id == selectedBoxID } } - // MARK: - Browse all + // MARK: - User library - /// Load the catalog so "Browse all" works before any generate has run. - func loadCatalogIfNeeded() { - guard catalog.isEmpty, !isBusy else { return } - ticket += 1 - let myTicket = ticket - isBusy = true - Task { [weak self] in - guard let self else { return } - if await self.ensureCatalog(ticket: myTicket) { - self.finish(myTicket, status: "Loaded \(self.catalog.count) templates.") + /// Import image files as templates, then show the first one. + /// + /// Returns how many were imported so the caller can report a partial failure — + /// dropping five files and silently getting four templates would be the kind of + /// quiet data loss this spike keeps trying to eliminate. + @discardableResult + func importTemplates(from urls: [URL]) -> Int { + var imported: MemeUserLibrary.Entry? + var count = 0 + for url in urls { + if let entry = MemeLibraryStore.importImage(at: url) { + imported = imported ?? entry + count += 1 + } + } + reloadUserLibrary() + + if count == 0 { + status = "Couldn't import — pick a PNG, JPEG, GIF, WebP, or HEIC image." + } else { + status = count == urls.count + ? "Imported \(count) template\(count == 1 ? "" : "s")." + : "Imported \(count) of \(urls.count) — the rest weren't readable images." + if let imported, let template = catalog.first(where: { + $0.id == MemeTemplateCatalog.qualifiedID(.userLibrary, imported.id) + }) { + select(template: template) } } + return count } - private func finish(_ myTicket: Int, status newStatus: String) { - guard myTicket == ticket else { return } - isBusy = false + /// Import an image straight from the pasteboard (⌘V into the Browse grid). + @discardableResult + func importFromPasteboard() -> Bool { + let pasteboard = NSPasteboard.general + + // A file URL on the pasteboard is preferred over raw image data: it keeps the + // original filename, which becomes the template's name and a search keyword. + if let urls = pasteboard.readObjects(forClasses: [NSURL.self]) as? [URL], + !urls.isEmpty, + urls.contains(where: { MemeUserLibrary.isAcceptedImage(fileName: $0.lastPathComponent) }) { + return importTemplates(from: urls) > 0 + } + + guard let image = NSImage(pasteboard: pasteboard) else { + status = "Nothing on the clipboard to import." + return false + } + let name = MemeUserLibrary.uniqueName( + "Pasted template", existing: userTemplates.map(\.name)) + guard let entry = MemeLibraryStore.importImage(image, name: name) else { + status = "Couldn't import the pasted image." + return false + } + reloadUserLibrary() + status = "Imported \(entry.name)." + if let template = catalog.first(where: { + $0.id == MemeTemplateCatalog.qualifiedID(.userLibrary, entry.id) + }) { + select(template: template) + } + return true + } + + /// Delete a user-library template. + func deleteUserTemplate(_ template: MemeTemplate) { + guard template.source == .userLibrary, + let separator = template.id.firstIndex(of: ":") else { return } + let rawID = String(template.id[template.id.index(after: separator)...]) + + MemeLibraryStore.remove(id: rawID) + imageCache[template.id] = nil + reloadUserLibrary() + + // Deleting the template that is on screen must not leave a meme rendered from + // a template the user just removed. + if selectedTemplate?.id == template.id { + selectedTemplate = nil + baseImage = nil + meme = nil + } + candidates.removeAll { $0.id == template.id } + status = "Deleted \(template.name)." + } + + func renameUserTemplate(_ template: MemeTemplate, to newName: String) { + guard template.source == .userLibrary, + let separator = template.id.firstIndex(of: ":"), + !newName.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return } + let rawID = String(template.id[template.id.index(after: separator)...]) + + MemeLibraryStore.rename(id: rawID, to: newName) + reloadUserLibrary() + if selectedTemplate?.id == template.id { + selectedTemplate = catalog.first { $0.id == template.id } + } + } + + // MARK: - Lifecycle + + private func finish(_ ticket: Int, status newStatus: String) { + guard state.finish(ticket: ticket) else { return } status = newStatus } @@ -400,13 +673,12 @@ final class MemeGeneratorModel: ObservableObject { /// Stop accepting async results — called when the window closes. func cancel() { isCancelled = true - isBusy = false + state.cancel() } // MARK: - Export / share - /// The filename an export should suggest — driven by the EDITED boxes, so a meme - /// the user rewrote saves under what it actually says. + /// The filename an export should suggest — driven by the EDITED boxes. var suggestedFileName: String { MemeCaptionLayout.suggestedFileName(boxes: boxes) } diff --git a/plugins/MemeGenerator/MemeGeneratorView.swift b/plugins/MemeGenerator/MemeGeneratorView.swift index ad552f4..8f0eb0b 100644 --- a/plugins/MemeGenerator/MemeGeneratorView.swift +++ b/plugins/MemeGenerator/MemeGeneratorView.swift @@ -1,23 +1,36 @@ import AppKit import SwiftUI +import UniformTypeIdentifiers -/// The Meme Generator plugin's window content (spike). +/// The Meme Generator plugin's window content (spike v3). /// /// Voice-first by construction: the description field is the first responder when the /// window opens, so pressing the dictation hotkey and speaking lands the words here -/// with no typing and no clicking. "As fast as possible" is the whole point. +/// with no typing and no clicking. /// -/// ## v2 — "AI makes its best guess, human makes it perfect" +/// ## v3 layout — toward the imgflip.com editor shape /// -/// Three surfaces sit between the model's answer and the finished meme: +/// The owner asked for the familiar meme-editor arrangement, and for everything local +/// to feel instant. The window is now three columns: /// -/// * **The candidate strip** — the model's ranked template picks as thumbnails. The -/// best one auto-renders; clicking another re-renders the SAME captions onto it. -/// * **Browse all** — a searchable grid of the whole corpus, so the user can override -/// the model entirely. This is the honest answer when the corpus doesn't contain -/// what they asked for. -/// * **The editor** — every caption is a draggable box over the preview, with text, -/// size and font controls in a side panel. +/// ``` +/// ┌──────────────┬─────────────────────────┬──────────────┐ +/// │ TEMPLATES │ CANVAS │ TEXT BOXES │ +/// │ search + │ (the rendered meme, │ add/delete, │ +/// │ browse grid │ drag the captions) │ text, size, │ +/// │ + import │ │ width, font │ +/// └──────────────┴─────────────────────────┴──────────────┘ +/// ``` +/// +/// * **Left** — template search and browse are PROMINENT rather than behind a sheet, +/// because picking the template is the decision the corpus expansion exists to +/// serve. Import lives here too (button, drag-drop, or ⌘V). +/// * **Center** — the canvas, with the description and Generate above it. +/// * **Right** — per-box controls, with **Add text always visible** (feedback #4: +/// deleting the last box used to remove the only way to add one back). +/// +/// Nothing local shows a spinner: switching templates, editing text, dragging a box, +/// and searching are all synchronous re-renders. struct MemeGeneratorView: View { @ObservedObject var model: MemeGeneratorModel @@ -25,46 +38,156 @@ struct MemeGeneratorView: View { /// Focuses the description editor on open so a dictation lands immediately. @FocusState private var descriptionFocused: Bool - /// Whether the Browse-all grid is showing. - @State private var isBrowsing = false + /// True while a drag of image files is hovering the template column. + @State private var isDropTargeted = false var body: some View { - VStack(alignment: .leading, spacing: 12) { - header - descriptionEditor - controls - if !model.status.isEmpty { statusLine } - if !model.candidates.isEmpty { candidateStrip } - - HStack(alignment: .top, spacing: 12) { - preview - if !model.boxes.isEmpty { editorPanel } + HSplitView { + templateColumn + .frame(minWidth: 240, idealWidth: 280, maxWidth: 380) + + VStack(alignment: .leading, spacing: 10) { + descriptionEditor + controls + statusLine + if !model.candidates.isEmpty { candidateStrip } + canvas } + .padding(12) + .frame(minWidth: 380) + + editorPanel + .frame(minWidth: 230, idealWidth: 250, maxWidth: 320) } - .padding(16) - .frame(minWidth: 760, minHeight: 640) + .frame(minWidth: 980, minHeight: 660) .onAppear { descriptionFocused = true } - .sheet(isPresented: $isBrowsing) { - BrowseAllSheet(model: model, isPresented: $isBrowsing) + } + + // MARK: - Left column: templates + + private var templateColumn: some View { + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Templates").font(.headline) + Spacer() + Menu { + Button("Import images…") { importViaPanel() } + Button("Paste image") { model.importFromPasteboard() } + Divider() + Button("Show library in Finder") { revealLibrary() } + } label: { + Label("Import", systemImage: "plus.rectangle.on.folder") + } + .menuStyle(.borderlessButton) + .fixedSize() + .help("Add your own image as a template") + } + + TextField("Search all templates", text: $model.searchText) + .textFieldStyle(.roundedBorder) + + // The corpus is the feature — say how big it is and where it came from. + HStack(spacing: 4) { + Text(model.catalog.isEmpty + ? "No templates loaded." + : "\(model.searchResults.count) of \(model.catalog.count) templates") + .font(.caption) + .foregroundStyle(.secondary) + Spacer() + if model.catalogFailed { + Button("Retry") { model.openCatalog(forceRefresh: true) } + .buttonStyle(.link) + .font(.caption) + } + } + + templateGrid + } + .padding(12) + .background(isDropTargeted ? Color.accentColor.opacity(0.12) : Color.clear) + .overlay { + if isDropTargeted { + RoundedRectangle(cornerRadius: 8) + .strokeBorder(Color.accentColor, style: StrokeStyle(lineWidth: 2, dash: [6, 4])) + .padding(4) + } + } + // Drag any image file onto the column to make it a template. The most direct + // path from "I have a meme picture" to "it's in my corpus". + .onDrop(of: [.fileURL], isTargeted: $isDropTargeted) { providers in + handleDrop(providers) } } - private var header: some View { - HStack(spacing: 8) { - Image(systemName: PluginRegistry.memeGenerator.symbol) - .foregroundStyle(.secondary) - Text("Describe the meme out loud — the model proposes templates and writes the captions. Then edit anything.") - .font(.callout) + @ViewBuilder + private var templateGrid: some View { + let columns = [GridItem(.adaptive(minimum: 104), spacing: 8)] + + if model.catalog.isEmpty { + emptyTemplatesHint + } else if model.searchResults.isEmpty { + // The honest no-match state, unchanged from v2: never a substitution. + VStack(spacing: 6) { + Image(systemName: "magnifyingglass") + .font(.title).foregroundStyle(.tertiary) + Text("No template matches \"\(model.searchText)\".") + .font(.caption) + .multilineTextAlignment(.center) + .foregroundStyle(.secondary) + Text("Import your own image to add it to the corpus.") + .font(.caption2) + .multilineTextAlignment(.center) + .foregroundStyle(.tertiary) + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + } else { + ScrollView { + LazyVGrid(columns: columns, spacing: 8) { + ForEach(model.searchResults) { template in + TemplateThumbnail( + template: template, + isSelected: template.id == model.selectedTemplate?.id, + width: 104) + .onTapGesture { model.select(template: template) } + .contextMenu { + if template.source == .userLibrary { + Button("Delete from my library", role: .destructive) { + model.deleteUserTemplate(template) + } + } + } + } + } + .padding(.vertical, 2) + } + } + } + + private var emptyTemplatesHint: some View { + VStack(spacing: 8) { + Image(systemName: "photo.on.rectangle.angled") + .font(.largeTitle).foregroundStyle(.tertiary) + Text(model.catalogFailed + ? "Couldn't reach the template services." + : "Loading templates…") + .font(.caption) .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) + Text("Drag an image here, or use Import — your own templates work offline.") + .font(.caption2) + .multilineTextAlignment(.center) + .foregroundStyle(.tertiary) } + .frame(maxWidth: .infinity, maxHeight: .infinity) + .padding(.horizontal, 8) } + // MARK: - Center column + private var descriptionEditor: some View { VStack(alignment: .leading, spacing: 4) { TextEditor(text: $model.description) .font(.body) - .frame(minHeight: 54, maxHeight: 80) + .frame(minHeight: 50, maxHeight: 72) .overlay( RoundedRectangle(cornerRadius: 6) .stroke(Color.secondary.opacity(0.3))) @@ -72,7 +195,7 @@ struct MemeGeneratorView: View { .overlay(alignment: .topLeading) { // TextEditor has no placeholder; this is the standard workaround. if model.description.isEmpty { - Text("e.g. \"distracted boyfriend, but he's looking at Rust and his girlfriend is Python\"") + Text("Describe the meme out loud — dictate into this window.") .font(.body) .foregroundStyle(.tertiary) .padding(.top, 8) @@ -80,10 +203,6 @@ struct MemeGeneratorView: View { .allowsHitTesting(false) } } - - Text("Dictate into this window and your words land here.") - .font(.caption) - .foregroundStyle(.tertiary) } } @@ -92,23 +211,18 @@ struct MemeGeneratorView: View { Button { model.generate() } label: { - Label(model.isBusy ? "Generating…" : "Generate", - systemImage: "wand.and.stars") + Label(model.isBusy ? "Generating…" : "Generate", systemImage: "wand.and.stars") } .keyboardShortcut(.return, modifiers: .command) .disabled(!model.canGenerate) + .help(model.generateBlockedReason ?? "Ask the model for templates and captions") + // The escape hatch v2 lacked entirely. Only shown while something is + // actually in flight, so it never reads as a dead control. if model.isBusy { ProgressView().controlSize(.small) - } - - // Always available (it loads the catalog on demand), so the user is never - // stuck with the model's idea of which templates exist. - Button { - model.loadCatalogIfNeeded() - isBrowsing = true - } label: { - Label("Browse all…", systemImage: "square.grid.2x2") + Button("Cancel") { model.cancelGeneration() } + .keyboardShortcut(.cancelAction) } Spacer() @@ -129,28 +243,31 @@ struct MemeGeneratorView: View { } } + @ViewBuilder private var statusLine: some View { - HStack(alignment: .top, spacing: 6) { - // A fallback is called out with a symbol as well as words — the owner's - // complaint was that the substitution was invisible. - if model.didFallBack { - Image(systemName: "exclamationmark.triangle") - .foregroundStyle(.orange) + if !model.status.isEmpty { + HStack(alignment: .top, spacing: 6) { + // A fallback is called out with a symbol as well as words — the + // owner's complaint was that the substitution was invisible. + if model.didFallBack || model.catalogFailed { + Image(systemName: "exclamationmark.triangle") + .foregroundStyle(.orange) + } + Text(model.status) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + Spacer() } - Text(model.status) - .font(.caption) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) } } - // MARK: - Candidate strip - - /// The model's ranked picks. Clicking one re-renders the same captions onto it. + /// The model's ranked picks. Clicking one re-renders the same captions onto it — + /// instantly, and even while a generation is still running. private var candidateStrip: some View { VStack(alignment: .leading, spacing: 4) { Text(model.candidatesAreFallback - ? "Nothing in the corpus matched — closest and most popular instead:" + ? "Nothing matched — closest and most popular instead:" : "The model's picks, best first:") .font(.caption) .foregroundStyle(.secondary) @@ -161,7 +278,7 @@ struct MemeGeneratorView: View { TemplateThumbnail( template: template, isSelected: template.id == model.selectedTemplate?.id, - width: 96) + width: 88) .onTapGesture { model.select(template: template) } } } @@ -170,19 +287,15 @@ struct MemeGeneratorView: View { } } - // MARK: - Preview + editor overlay - @ViewBuilder - private var preview: some View { + private var canvas: some View { if let meme = model.meme { // The base image is the RENDERED meme (captions already burned in by the // same code path the export uses, so this is genuinely WYSIWYG). The // overlaid boxes are invisible drag handles positioned by the same - // normalized coordinates, which is why a handle always sits exactly on - // the text it moves. + // normalized coordinates. GeometryReader { geo in - let fitted = Self.fittedRect( - imageSize: meme.size, in: geo.size) + let fitted = Self.fittedRect(imageSize: meme.size, in: geo.size) ZStack(alignment: .topLeading) { Image(nsImage: meme) @@ -211,9 +324,11 @@ struct MemeGeneratorView: View { Image(systemName: "photo") .font(.largeTitle) .foregroundStyle(.tertiary) - Text("Your meme will appear here.") + Text("Pick a template on the left, or describe a meme and press Generate.") .font(.callout) + .multilineTextAlignment(.center) .foregroundStyle(.tertiary) + .padding(.horizontal, 24) }) .frame(maxWidth: .infinity, maxHeight: .infinity) } @@ -237,39 +352,98 @@ struct MemeGeneratorView: View { width: size.width, height: size.height) } - // MARK: - Editor side panel + // MARK: - Right column: the box editor private var editorPanel: some View { VStack(alignment: .leading, spacing: 10) { + // "Add text" is ALWAYS here — outside every conditional. v2 rendered this + // whole panel only when `boxes` was non-empty, so deleting the last box + // removed the only control that could add one back (feedback #4). HStack { Text("Text boxes").font(.headline) Spacer() Button { model.addBox() } label: { - Image(systemName: "plus") + Label("Add text", systemImage: "plus") } - .help("Add a text box") + .help("Add a caption box") } - // The box list doubles as the selector for the controls below. - ForEach(model.boxes) { box in - BoxRow(model: model, box: box) - } + if model.boxes.isEmpty { + VStack(spacing: 6) { + Image(systemName: "textformat") + .font(.title).foregroundStyle(.tertiary) + Text("No caption boxes.") + .font(.caption).foregroundStyle(.secondary) + Text("Press Add text to put a caption on the meme.") + .font(.caption2) + .multilineTextAlignment(.center) + .foregroundStyle(.tertiary) + } + .frame(maxWidth: .infinity) + .padding(.vertical, 12) + } else { + ForEach(model.boxes) { box in + BoxRow(model: model, box: box) + } - Divider() + Divider() - if let selected = model.selectedBox { - BoxControls(model: model, box: selected) - } else { - Text("Select a text box to edit it.") - .font(.caption) - .foregroundStyle(.tertiary) + if let selected = model.selectedBox { + BoxControls(model: model, box: selected) + } else { + Text("Select a text box to edit it.") + .font(.caption) + .foregroundStyle(.tertiary) + } } Spacer() } - .frame(width: 240) + .padding(12) + } + + // MARK: - Import + + private func importViaPanel() { + let panel = NSOpenPanel() + panel.allowsMultipleSelection = true + panel.canChooseDirectories = false + panel.allowedContentTypes = [.png, .jpeg, .gif, .heic, .tiff, .bmp, .webP] + panel.message = "Choose images to add as meme templates." + guard panel.runModal() == .OK else { return } + model.importTemplates(from: panel.urls) + } + + private func handleDrop(_ providers: [NSItemProvider]) -> Bool { + // Resolve every provider, then import once: importing per-callback would + // reload the library N times and race the status line. + let group = DispatchGroup() + var urls: [URL] = [] + let lock = NSLock() + + for provider in providers { + group.enter() + _ = provider.loadObject(ofClass: URL.self) { url, _ in + if let url, MemeUserLibrary.isAcceptedImage(fileName: url.lastPathComponent) { + lock.lock(); urls.append(url); lock.unlock() + } + group.leave() + } + } + + group.notify(queue: .main) { + guard !urls.isEmpty else { return } + model.importTemplates(from: urls) + } + return true + } + + private func revealLibrary() { + let directory = MemeLibraryStore.templatesDirectory + try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + NSWorkspace.shared.activateFileViewerSelecting([directory]) } /// Share the rendered PNG through the system picker. Writes to a temp file first @@ -296,8 +470,7 @@ struct MemeGeneratorView: View { /// Deliberately NOT a second copy of the text: drawing the caption again in SwiftUI /// would mean two renderers to keep in agreement, and they would drift (different /// wrapping, different outline). Instead the burned-in render IS the visual, and this -/// is a positioned outline the user grabs. The tradeoff is that the handle is a -/// rectangle rather than glyph-tight, which is fine at spike quality. +/// is a positioned outline the user grabs. private struct DragHandle: View { @ObservedObject var model: MemeGeneratorModel @@ -311,8 +484,6 @@ private struct DragHandle: View { @State private var dragOffset: CGSize = .zero var body: some View { - // Height comes from the same lineHeightRatio the renderer uses; the handle is - // sized for a single line, which is enough to grab. Width follows the box. let width = canvas.width * box.widthShare let height = max(24, canvas.height * box.fontSizeShare * MemeCaptionLayout.lineHeightRatio) let x = canvas.minX + canvas.width * box.centerX + dragOffset.width @@ -459,105 +630,40 @@ private struct BoxControls: View { } } -// MARK: - Browse all - -/// A searchable grid over the WHOLE catalog. -/// -/// The honest override: when the model's picks are all wrong — or the meme the user -/// wanted simply isn't in imgflip's top 100 — this is where they say so themselves. -/// The search is local, case-insensitive and pure (`MemeTemplateMatcher.search`), and -/// crucially it does NOT fall back: an empty result stays empty and says why. -private struct BrowseAllSheet: View { - @ObservedObject var model: MemeGeneratorModel - @Binding var isPresented: Bool - - private let columns = [GridItem(.adaptive(minimum: 120), spacing: 10)] - - var body: some View { - VStack(alignment: .leading, spacing: 10) { - HStack { - Text("Browse templates").font(.headline) - Spacer() - Button("Done") { isPresented = false } - .keyboardShortcut(.cancelAction) - } - - TextField("Search templates", text: $model.searchText) - .textFieldStyle(.roundedBorder) - - // State the corpus plainly. This is the sentence that would have saved - // the "yoda meme" session. - Text("This is imgflip's public top-100 template list — \(model.catalog.count) templates. " - + "If what you want isn't here, it isn't in the corpus.") - .font(.caption) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) - - if model.catalog.isEmpty { - ProgressView("Loading templates…") - .frame(maxWidth: .infinity, maxHeight: .infinity) - } else if model.searchResults.isEmpty { - VStack(spacing: 6) { - Image(systemName: "magnifyingglass") - .font(.largeTitle).foregroundStyle(.tertiary) - Text("No template matches \"\(model.searchText)\".") - .foregroundStyle(.secondary) - } - .frame(maxWidth: .infinity, maxHeight: .infinity) - } else { - ScrollView { - LazyVGrid(columns: columns, spacing: 10) { - ForEach(model.searchResults) { template in - TemplateThumbnail( - template: template, - isSelected: template.id == model.selectedTemplate?.id, - width: 120) - .onTapGesture { - model.select(template: template) - isPresented = false - } - } - } - } - } - } - .padding(16) - .frame(width: 640, height: 560) - } -} - // MARK: - Thumbnail -/// A template preview image plus its name. +/// A template preview image plus its name and source badge. /// -/// `AsyncImage` rather than the plugin's own `MemeTemplateService.fetchImage`: the -/// grid can show 100 of these, and `AsyncImage` already handles per-view cancellation -/// and URLCache reuse. It's the same imgflip CDN GET either way — still no upload. +/// Loads from the on-disk thumbnail cache first so a second open of the window paints +/// instantly and works with the network off; `AsyncImage` is the fallback for a +/// template whose thumbnail hasn't been cached yet. User-library templates are +/// `file:` URLs, which `AsyncImage` handles natively — one code path, three sources. private struct TemplateThumbnail: View { let template: MemeTemplate let isSelected: Bool let width: CGFloat var body: some View { - VStack(spacing: 4) { - AsyncImage(url: URL(string: template.url)) { phase in - switch phase { - case .success(let image): - image.resizable().scaledToFill() - case .failure: - Image(systemName: "photo") - .foregroundStyle(.tertiary) - default: - ProgressView().controlSize(.small) + VStack(spacing: 3) { + thumbnailImage + .frame(width: width, height: width * 0.75) + .clipped() + .background(Color.secondary.opacity(0.1)) + .clipShape(RoundedRectangle(cornerRadius: 6)) + .overlay( + RoundedRectangle(cornerRadius: 6) + .stroke(isSelected ? Color.accentColor : Color.clear, lineWidth: 3)) + .overlay(alignment: .topTrailing) { + // The user's own templates are badged so a mixed grid is legible + // at a glance — "which of these are mine" is the question a merged + // corpus creates. + if template.source == .userLibrary { + Image(systemName: "person.crop.circle.fill") + .font(.caption2) + .foregroundStyle(.white, Color.accentColor) + .padding(3) + } } - } - .frame(width: width, height: width * 0.75) - .clipped() - .background(Color.secondary.opacity(0.1)) - .clipShape(RoundedRectangle(cornerRadius: 6)) - .overlay( - RoundedRectangle(cornerRadius: 6) - .stroke(isSelected ? Color.accentColor : Color.clear, lineWidth: 3)) Text(template.name) .font(.caption2) @@ -567,6 +673,24 @@ private struct TemplateThumbnail: View { .frame(width: width) } .contentShape(Rectangle()) - .help(template.name) + .help("\(template.name) — \(template.source.label)") + } + + @ViewBuilder + private var thumbnailImage: some View { + if let cached = MemeLibraryStore.cachedThumbnail(for: template.id) { + Image(nsImage: cached).resizable().scaledToFill() + } else { + AsyncImage(url: URL(string: template.url)) { phase in + switch phase { + case .success(let image): + image.resizable().scaledToFill() + case .failure: + Image(systemName: "photo").foregroundStyle(.tertiary) + default: + ProgressView().controlSize(.small) + } + } + } } } diff --git a/plugins/MemeGenerator/MemeGeneratorWindowController.swift b/plugins/MemeGenerator/MemeGeneratorWindowController.swift index af364d3..83bb03d 100644 --- a/plugins/MemeGenerator/MemeGeneratorWindowController.swift +++ b/plugins/MemeGenerator/MemeGeneratorWindowController.swift @@ -33,6 +33,14 @@ final class MemeGeneratorWindowController: NSWindowController, NSWindowDelegate, rootView: MemeGeneratorView(model: model)) window.delegate = self wireAI() + + // v3: warm the LLM and open the template catalog the moment the window + // exists, NOT on the first Generate. The owner's report — "first Generate + // fails: network error and model loading" — was a request hitting a + // llama-server that hadn't started yet; the request surfaced connection- + // refused as a network error. Doing this work at open turns that failure + // into a brief, honest "Preparing model…". + model.windowDidOpen() } // MARK: - LLM seam @@ -64,7 +72,15 @@ final class MemeGeneratorWindowController: NSWindowController, NSWindowDelegate, } } }, - resolveModel: { ScratchpadWindowController.resolvedAIModel() }) + resolveModel: { ScratchpadWindowController.resolvedAIModel() }, + // Start the bundled llama-server WITHOUT running a completion, passing + // the plugin's RESOLVED provider: the global `warmLlamaServerIfPossible()` + // only fires when Settings → Cleanup is itself set to the bundled + // provider, so a plugin resolved to bundled would otherwise never warm. + // Same MAK-53 split `ensureBundledLLMReady(provider:)` already makes. + warm: { resolved in + AppState.shared.warmLlamaServerIfPossible(provider: resolved.provider) + }) } // MARK: - PluginDictationSink diff --git a/plugins/MemeGenerator/MemeLibraryStore.swift b/plugins/MemeGenerator/MemeLibraryStore.swift new file mode 100644 index 0000000..b3a4d42 --- /dev/null +++ b/plugins/MemeGenerator/MemeLibraryStore.swift @@ -0,0 +1,257 @@ +import AppKit +import Foundation + +/// Disk IO for the Meme Generator's user template library and catalog cache (v3). +/// +/// The *policy* lives in the pure, tested `MemeUserLibrary` and `MemeCatalogCache`; +/// this type does only the reading, writing, and image copying. That split is the same +/// one the rest of the plugin uses, and it is what lets `swift test` cover the index +/// rules (uniqueness, pruning, traversal refusal) without touching a filesystem. +/// +/// Everything lives under +/// `~/Library/Application Support/OpenWhisp/Plugins/MemeGenerator/`, which is the +/// directory `PluginHost.externalDirectory` already establishes for per-plugin data. +@MainActor +enum MemeLibraryStore { + + /// `…/Plugins/MemeGenerator`. + static var pluginDirectory: URL { + PluginHost.externalDirectory + .appendingPathComponent(PluginRegistry.memeGenerator.id, isDirectory: true) + } + + /// `…/Plugins/MemeGenerator/templates` — the imported images plus `index.json`. + static var templatesDirectory: URL { + pluginDirectory.appendingPathComponent("templates", isDirectory: true) + } + + private static var indexURL: URL { + templatesDirectory.appendingPathComponent(MemeUserLibrary.indexFileName) + } + + private static var cacheURL: URL { + pluginDirectory.appendingPathComponent(MemeCatalogCache.fileName) + } + + // MARK: - User library + + /// Read the index, dropping entries whose image file has disappeared. + /// + /// A missing or unreadable index is an EMPTY library, never an error: the library + /// is empty on first run by definition, and a corrupt index must not block the + /// plugin from opening — the remote providers still work. + static func loadIndex() -> MemeUserLibrary.Index { + guard let data = try? Data(contentsOf: indexURL), + let decoded = try? JSONDecoder().decode(MemeUserLibrary.Index.self, from: data) + else { return MemeUserLibrary.Index() } + + let files = (try? FileManager.default.contentsOfDirectory( + atPath: templatesDirectory.path)) ?? [] + return MemeUserLibrary.pruned(decoded, existingFiles: Set(files)) + } + + @discardableResult + static func save(_ index: MemeUserLibrary.Index) -> Bool { + do { + try FileManager.default.createDirectory( + at: templatesDirectory, withIntermediateDirectories: true) + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + // The index is small and hand-inspectable on purpose: this is a spike, and + // a user who wants to rename twenty templates should be able to open the + // file and do it. + try encoder.encode(index).write(to: indexURL, options: .atomic) + return true + } catch { + return false + } + } + + /// Import an image file as a user template. + /// + /// The source file is COPIED rather than referenced. A template that points at + /// `~/Downloads` breaks the moment the user tidies up, and the whole promise of + /// the library is that it keeps working offline and indefinitely. + /// + /// Returns the new entry, or nil when the file isn't a readable image. + @discardableResult + static func importImage(at source: URL, name: String? = nil) -> MemeUserLibrary.Entry? { + guard MemeUserLibrary.isAcceptedImage(fileName: source.lastPathComponent) else { + return nil + } + // Decode BEFORE copying: an unreadable file must not leave a stray image in + // the library directory that pruning would then have to clean up. + guard let image = NSImage(contentsOf: source), image.size.width > 0 else { return nil } + + let id = UUID().uuidString + let file = MemeUserLibrary.storageFileName( + id: id, sourceExtension: source.pathExtension) + let destination = templatesDirectory.appendingPathComponent(file) + + do { + try FileManager.default.createDirectory( + at: templatesDirectory, withIntermediateDirectories: true) + try FileManager.default.copyItem(at: source, to: destination) + } catch { + return nil + } + + let pixels = pixelSize(of: image) + let entry = MemeUserLibrary.Entry( + id: id, + name: name ?? MemeUserLibrary.suggestedName(fromFileName: source.lastPathComponent), + file: file, + width: pixels.width, height: pixels.height) + + var index = loadIndex() + index = MemeUserLibrary.adding(entry, to: index) + guard save(index) else { + try? FileManager.default.removeItem(at: destination) + return nil + } + // `adding` may have uniquified the name, so return what was actually stored + // rather than what was requested. + return index.entries.last + } + + /// Import an image already in memory — the paste and drag-drop paths, where there + /// is no source file to copy. + @discardableResult + static func importImage(_ image: NSImage, name: String) -> MemeUserLibrary.Entry? { + guard let data = MemeRenderer.pngData(for: image) else { return nil } + + let id = UUID().uuidString + let file = MemeUserLibrary.storageFileName(id: id, sourceExtension: "png") + let destination = templatesDirectory.appendingPathComponent(file) + + do { + try FileManager.default.createDirectory( + at: templatesDirectory, withIntermediateDirectories: true) + try data.write(to: destination, options: .atomic) + } catch { + return nil + } + + let pixels = pixelSize(of: image) + let entry = MemeUserLibrary.Entry( + id: id, name: name, file: file, width: pixels.width, height: pixels.height) + + var index = loadIndex() + index = MemeUserLibrary.adding(entry, to: index) + guard save(index) else { + try? FileManager.default.removeItem(at: destination) + return nil + } + return index.entries.last + } + + /// Delete a template and its image. + static func remove(id: String) { + let index = loadIndex() + if let entry = index.entries.first(where: { $0.id == id }), + MemeUserLibrary.isSafeFileName(entry.file) { + try? FileManager.default.removeItem( + at: templatesDirectory.appendingPathComponent(entry.file)) + } + save(MemeUserLibrary.removing(id: id, from: index)) + } + + static func rename(id: String, to newName: String) { + save(MemeUserLibrary.renaming(id: id, to: newName, in: loadIndex())) + } + + /// The library projected into catalog templates. + static func libraryTemplates() -> [MemeTemplate] { + MemeUserLibrary.templates(from: loadIndex(), directory: templatesDirectory) + } + + private static func pixelSize(of image: NSImage) -> (width: Int, height: Int) { + let reps = image.representations.compactMap { $0 as? NSBitmapImageRep } + if let best = reps.max(by: { $0.pixelsWide * $0.pixelsHigh < $1.pixelsWide * $1.pixelsHigh }) { + return (best.pixelsWide, best.pixelsHigh) + } + return (Int(image.size.width), Int(image.size.height)) + } + + // MARK: - Catalog cache + + static func loadCachedCatalog() -> MemeCatalogCache.Cached? { + guard let data = try? Data(contentsOf: cacheURL) else { return nil } + return try? JSONDecoder().decode(MemeCatalogCache.Cached.self, from: data) + } + + /// Persist the REMOTE catalog only. + /// + /// User-library templates are deliberately excluded: they are already durable in + /// `index.json`, and caching them would mean a deleted import could come back + /// from the cache. The cache's job is to stand in for the network, nothing else. + static func saveCachedCatalog(_ templates: [MemeTemplate], now: Date = Date()) { + let remote = templates.filter { $0.source != .userLibrary } + guard !remote.isEmpty else { return } + let payload = MemeCatalogCache.Cached(fetchedAt: now, templates: remote) + do { + try FileManager.default.createDirectory( + at: pluginDirectory, withIntermediateDirectories: true) + try JSONEncoder().encode(payload).write(to: cacheURL, options: .atomic) + } catch { + // A cache that can't be written costs a fetch next launch — not worth + // interrupting the user over. + } + } + + // MARK: - Thumbnails + + /// `…/Plugins/MemeGenerator/thumbnails` — downscaled template previews. + /// + /// Browsing ~300 templates means ~300 image loads; caching a small JPEG per + /// template is what makes the grid instant on the second open and functional with + /// the network off. Keyed by the QUALIFIED id, hashed so a slug id with awkward + /// characters can't shape the filename. + static var thumbnailsDirectory: URL { + pluginDirectory.appendingPathComponent("thumbnails", isDirectory: true) + } + + static func thumbnailURL(for templateID: String) -> URL { + // A stable, filesystem-safe name derived from the id. Hex of the id's UTF-8 + // bytes: reversible, collision-free, and free of path characters — the + // property that matters, since this string becomes a path component. + let hex = templateID.utf8.map { String(format: "%02x", $0) }.joined() + // Long ids would exceed the 255-byte filename limit; the suffix keeps the + // tail (where the distinguishing part of an id lives) rather than the head. + let safe = hex.count <= 200 ? hex : String(hex.suffix(200)) + return thumbnailsDirectory.appendingPathComponent("\(safe).jpg") + } + + static func cachedThumbnail(for templateID: String) -> NSImage? { + NSImage(contentsOf: thumbnailURL(for: templateID)) + } + + /// Downscale and store a thumbnail. Failures are silent — a missing thumbnail + /// just means the grid refetches next time. + static func storeThumbnail(_ image: NSImage, for templateID: String) { + guard let scaled = downscaled(image, maxDimension: 320), + let tiff = scaled.tiffRepresentation, + let rep = NSBitmapImageRep(data: tiff), + let data = rep.representation(using: .jpeg, properties: [.compressionFactor: 0.7]) + else { return } + + try? FileManager.default.createDirectory( + at: thumbnailsDirectory, withIntermediateDirectories: true) + try? data.write(to: thumbnailURL(for: templateID), options: .atomic) + } + + private static func downscaled(_ image: NSImage, maxDimension: CGFloat) -> NSImage? { + let size = image.size + guard size.width > 0, size.height > 0 else { return nil } + let scale = min(1, maxDimension / max(size.width, size.height)) + guard scale < 1 else { return image } + + let target = NSSize(width: size.width * scale, height: size.height * scale) + let output = NSImage(size: target) + output.lockFocus() + NSGraphicsContext.current?.imageInterpolation = .high + image.draw(in: NSRect(origin: .zero, size: target)) + output.unlockFocus() + return output + } +} diff --git a/plugins/MemeGenerator/MemeTemplateService.swift b/plugins/MemeGenerator/MemeTemplateService.swift index f918dd0..d068667 100644 --- a/plugins/MemeGenerator/MemeTemplateService.swift +++ b/plugins/MemeGenerator/MemeTemplateService.swift @@ -1,39 +1,50 @@ import AppKit import Foundation -/// The Meme Generator's only network access (spike). +/// The Meme Generator's network access (spike v3). /// -/// Two GETs against imgflip's public, key-less endpoints: +/// Read-only GETs against two public, key-less catalogs plus their image CDNs: /// -/// * `https://api.imgflip.com/get_memes` — the template catalog (~100 popular -/// templates as JSON). No auth, no account, read-only. -/// * `https://i.imgflip.com/.jpg` — the blank template image. +/// * `https://api.imgflip.com/get_memes` — ~100 popular templates. +/// * `https://api.memegen.link/templates` — ~200 more, with keywords. +/// * the templates' own blank-image URLs. /// /// **Nothing is ever uploaded.** The user's dictation, the LLM's captions, and the /// finished meme all stay on the Mac — captioning is done locally by `MemeRenderer` -/// precisely so no text has to leave. The app is local-first; this plugin's Settings -/// row states its network use, and the plugin is off by default. +/// precisely so no text has to leave. Note memegen.link *offers* server-side +/// captioning via URL (`/images///.jpg`) and this plugin deliberately +/// does NOT use it: that would put the user's words on someone else's server, which is +/// exactly what the local-first posture rules out. +/// +/// The third provider, the user's own library, needs no network at all — it is read +/// from disk by `MemeLibraryStore`. enum MemeTemplateService { /// Failures worth telling the user apart. enum ServiceError: LocalizedError { - case badResponse(status: Int) + case badResponse(host: String, status: Int) case emptyCatalog case undecodableImage + /// Every remote provider failed. Carries the first reason so the user gets a + /// cause rather than a generic "couldn't load". + case allProvidersFailed(reason: String) var errorDescription: String? { switch self { - case .badResponse(let status): - return "imgflip.com replied with HTTP \(status)." + case .badResponse(let host, let status): + return "\(host) replied with HTTP \(status)." case .emptyCatalog: - return "imgflip.com returned no templates." + return "the template service returned no templates." case .undecodableImage: return "the downloaded template wasn't a readable image." + case .allProvidersFailed(let reason): + return reason } } } - static let catalogURL = URL(string: "https://api.imgflip.com/get_memes")! + static let imgflipCatalogURL = URL(string: "https://api.imgflip.com/get_memes")! + static let memegenCatalogURL = URL(string: "https://api.memegen.link/templates")! /// A short timeout: this sits in front of a user waiting on a meme, so failing /// fast and saying so beats a long hang. @@ -44,11 +55,46 @@ enum MemeTemplateService { return URLSession(configuration: config) }() - /// Fetch the template catalog. Throws on transport failure, non-200, or an - /// `{"success": false}` envelope — all of which surface to the user verbatim. - static func fetchCatalog() async throws -> [MemeTemplate] { - let (data, response) = try await session.data(from: catalogURL) - try check(response) + // MARK: - Providers + + /// Fetch every remote catalog, then merge with the user's library. + /// + /// The providers run CONCURRENTLY and are tolerant of partial failure: if + /// memegen is down, imgflip's hundred still arrive and the user still gets a + /// working plugin. Only when EVERY remote provider fails does this throw — and + /// even then the caller can fall back to the disk cache or the user's library, + /// which is what makes the plugin work offline. + /// + /// Merge order encodes precedence (`MemeTemplateCatalog.merge`): the user's own + /// templates win, then imgflip (popularity-ranked, and the corpus the prompt was + /// tuned against), then memegen. + static func fetchMergedCatalog(userTemplates: [MemeTemplate]) async throws -> [MemeTemplate] { + async let imgflip = fetchImgflip() + async let memegen = fetchMemegen() + + var groups: [[MemeTemplate]] = [userTemplates] + var firstFailure: String? + + do { groups.append(try await imgflip) } + catch { firstFailure = reason(error) } + + do { groups.append(try await memegen) } + catch { firstFailure = firstFailure ?? reason(error) } + + let merged = MemeTemplateCatalog.merge(groups) + + // The user's own library alone is a legitimate corpus — an offline user with + // imported templates is fully functional and must not see an error. + if merged.isEmpty, let firstFailure { + throw ServiceError.allProvidersFailed(reason: firstFailure) + } + guard !merged.isEmpty else { throw ServiceError.emptyCatalog } + return merged + } + + static func fetchImgflip() async throws -> [MemeTemplate] { + let (data, response) = try await session.data(from: imgflipCatalogURL) + try check(response, host: "imgflip.com") let decoded = try JSONDecoder().decode(MemeTemplateCatalogResponse.self, from: data) let templates = decoded.templates @@ -56,22 +102,53 @@ enum MemeTemplateService { return templates } - /// Download a template's blank image. + static func fetchMemegen() async throws -> [MemeTemplate] { + let (data, response) = try await session.data(from: memegenCatalogURL) + try check(response, host: "memegen.link") + + let decoded = try JSONDecoder().decode(MemegenTemplateResponse.self, from: data) + guard !decoded.templates.isEmpty else { throw ServiceError.emptyCatalog } + return decoded.templates + } + + // MARK: - Images + + /// Load a template's blank image — from disk for the user's library, over the + /// network for the remote providers. + /// + /// One entry point for all three sources: the difference is a URL scheme, which + /// is exactly the abstraction `MemeTemplate.url` was widened to carry. A + /// user-library template therefore renders through the same path as an imgflip + /// one, which is why importing your own template needs no changes anywhere in the + /// render or export code. static func fetchImage(_ template: MemeTemplate) async throws -> NSImage { guard let url = URL(string: template.url) else { throw ServiceError.undecodableImage } + + if url.isFileURL { + guard let image = NSImage(contentsOf: url) else { + throw ServiceError.undecodableImage + } + return image + } + let (data, response) = try await session.data(from: url) - try check(response) + try check(response, host: url.host ?? "the template host") guard let image = NSImage(data: data) else { throw ServiceError.undecodableImage } return image } - private static func check(_ response: URLResponse) throws { + private static func check(_ response: URLResponse, host: String) throws { guard let http = response as? HTTPURLResponse else { return } guard (200...299).contains(http.statusCode) else { - throw ServiceError.badResponse(status: http.statusCode) + throw ServiceError.badResponse(host: host, status: http.statusCode) } } + + static func reason(_ error: Error) -> String { + let described = (error as NSError).localizedDescription + return described.isEmpty ? "the request failed." : described + } } diff --git a/plugins/MemeGenerator/manifest.json b/plugins/MemeGenerator/manifest.json index 6842353..7e17f7f 100644 --- a/plugins/MemeGenerator/manifest.json +++ b/plugins/MemeGenerator/manifest.json @@ -1,9 +1,9 @@ { "id": "meme-generator", "name": "Meme Generator", - "version": "0.1.0", + "version": "0.3.0", "summary": "Dictate a meme description — the AI picks a template and writes the captions.", "symbol": "photo.badge.plus", "entry": "builtIn", - "networkHosts": ["api.imgflip.com", "i.imgflip.com"] + "networkHosts": ["api.imgflip.com", "i.imgflip.com", "api.memegen.link"] } From 99be37d417f156d397cc30c6cb07a7b2bc1844b3 Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Sun, 2 Aug 2026 20:25:59 -0700 Subject: [PATCH 07/15] spike(meme-plugin): ranked search, unstuck downloads, real LLM readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects from the owner's live pass on v3. 1. SEARCH couldn't find a template by describing it. Both `MemeTemplateMatcher. search` and `MemeTemplateCatalog.search` required EVERY query token to appear (`needles.allSatisfy`), so "the worst day for the planet" returned nothing: "planet" is in neither the Bart template's name nor its keywords, and one unmatched token vetoed the three that matched perfectly. A content description can essentially never satisfy an all-tokens rule. Search is now SCORED over name + keywords (`MemeTemplateCatalog.score`): exact name > whole-phrase containment > per-token hits, where a name-token hit outranks a keyword hit and a prefix hit counts for less. Results are ordered best-first with the catalog's popularity order as the tie-break. It still never falls back — no match at all is still an empty result. The LLM path benefits too: `prefilter` scores the whole merged corpus against the user's own words and hands the model the top 30 WITH their keywords (`promptLines`), instead of v3's first-100-by-popularity name-only list. The relevant template can no longer be truncated off the end of the prompt. 2. TEMPLATE DOWNLOAD stuck on "Downloading " forever. `renderTemplate` had two BARE returns on a stale/cancelled ticket that never touched the state machine, and `select()` began a `.downloading` ticket with no timeout at all. Closing and reopening the window then revived the stranded phase, because `windowDidOpen` cleared `isCancelled` while the phase survived — leaving a busy surface with no task, no timer and no Retry behind it. Every exit now ends at a ticket-guarded, idempotent `finish`; downloads get their own finite ceiling (`downloadTimeout`, well under the generate one); `windowDidOpen` calls the new `state.reset()` so a reopened window can never inherit a phase; and an image failure surfaces an honest error plus its own Retry (`imageFailed` / `retryTemplate`), separate from the catalog's. 3. FIRST GENERATES still failed with a raw network error. v3 gated Generate on a guessed 2.5s sleep, which expires long before a cold llama-server binds its port. The real readiness signal already existed and was being discarded: `ensureRunning` polls `/health` and calls back only when it answers, but `warmLlamaServerIfPossible` threw that completion away (`{ _ in }`). It now forwards readiness, the plugin's `warm` seam carries it, and "Preparing model…" lasts until the model can actually take a request. A request that still hits a refused connection retries with backoff (`MemeGenerateRetry`, 3 attempts, matched on URL error CODES so a localized Mac doesn't silently stop retrying) before anything is reported. Warm POLICY moved into the pure `LLMWarmReadiness` resolver and the engine call into an `AppState` extension, so AppState SHRANK by 15 lines; the ratchet budget is lowered to 7036 to lock the win in. Tests: 2456 -> 2485. Includes the exact repro (`testWorstDayDescriptionFindsTheBartTemplate`), verified to fail against the v3 all-tokens rule before the fix. Co-Authored-By: Claude --- OpenWhisp/Models/AppState.swift | 23 +- OpenWhisp/Services/LLMWarmReadiness.swift | 56 ++++ OpenWhisp/Services/LLMWarmService.swift | 48 +++ OpenWhisp/Services/MemeAI.swift | 36 ++- OpenWhisp/Services/MemeGenerationState.swift | 103 ++++++ OpenWhisp/Services/MemeTemplateMatcher.swift | 45 +-- OpenWhisp/Services/MemeTemplateProvider.swift | 219 ++++++++++++- Package.swift | 3 +- .../MemeGeneratorTests.swift | 26 +- .../MemeProviderTests.swift | 302 ++++++++++++++++++ .../MemeGenerator/MemeGeneratorModel.swift | 231 ++++++++++++-- plugins/MemeGenerator/MemeGeneratorView.swift | 10 +- .../MemeGeneratorWindowController.swift | 11 +- .../MemeGenerator/MemeTemplateService.swift | 14 +- scripts/appstate-loc-budget.txt | 2 +- 15 files changed, 1023 insertions(+), 106 deletions(-) create mode 100644 OpenWhisp/Services/LLMWarmReadiness.swift create mode 100644 OpenWhisp/Services/LLMWarmService.swift diff --git a/OpenWhisp/Models/AppState.swift b/OpenWhisp/Models/AppState.swift index 3e7e9e9..047a132 100644 --- a/OpenWhisp/Models/AppState.swift +++ b/OpenWhisp/Models/AppState.swift @@ -575,7 +575,7 @@ class AppState: ObservableObject { /// Lazily-created engine that manages the bundled llama-server subprocess. private var llamaEngine: LlamaServerEngine? - private func ensureLlamaEngine() -> LlamaServerEngine { + func ensureLlamaEngine() -> LlamaServerEngine { if let engine = llamaEngine { return engine } let engine = LlamaServerEngine() llamaEngine = engine @@ -2785,27 +2785,12 @@ class AppState: ObservableObject { /// True when a resident whisper.cpp server is held in memory at the same time the /// built-in LLM would run. (WhisperKit/AppleSpeech keep no resident server.) - private var whisperServerResident: Bool { + /// Internal rather than private: the warm path lives in `LLMWarmReadiness.swift` + /// (MAK-32 — new AppState logic goes to core, not into the god object). + var whisperServerResident: Bool { transcriptionEngine == "whisper" && whisperBackend == "serverAPI" } - /// Start the bundled llama-server if the built-in provider is the active, - /// enabled, downloaded one. Idempotent (the engine no-ops if already healthy). - /// `provider` defaults to the global cleanup one; a caller with its OWN resolved - /// provider passes it (the MAK-53 split `ensureBundledLLMReady` already makes) so - /// a surface resolved to `bundled` still warms when cleanup uses something else. - /// The cleanup on/off toggle then gates only the implicit, global case. - func warmLlamaServerIfPossible(provider explicitProvider: String? = nil) { - let provider = explicitProvider ?? llmProvider - guard provider == "bundled", bundledLLMModelInstalled else { return } - guard explicitProvider != nil || openAIEnhancementEnabled else { return } - let engine = ensureLlamaEngine() - // Shorter idle teardown when a whisper-server is also resident, to relieve - // dual-engine memory pressure sooner (small-RAM Macs run both models). - engine.idleTimeout = whisperServerResident ? 30 : 90 - engine.ensureRunning(modelPath: selectedLLMModelPath()) { _ in } - } - /// Gate a refinement call behind the bundled server being healthy. For the /// bundled provider it lazily starts llama-server, then runs `work` on /// success, or `fallback` (insert the raw local text — never drop it) on diff --git a/OpenWhisp/Services/LLMWarmReadiness.swift b/OpenWhisp/Services/LLMWarmReadiness.swift new file mode 100644 index 0000000..6b24fa5 --- /dev/null +++ b/OpenWhisp/Services/LLMWarmReadiness.swift @@ -0,0 +1,56 @@ +import Foundation + +/// Whether warming a provider means WAITING for a local server, and what "ready" +/// means when it doesn't (plugin spike v4). +/// +/// ## Why this is a resolver and not three guards inside AppState +/// +/// The meme plugin's report was "the first two Generates fail with a network error". +/// The cause was that the warm had no readiness signal: `warmLlamaServerIfPossible` +/// discarded `ensureRunning`'s completion — the one thing that actually knows the +/// server answered its `/health` poll — and the plugin slept a guessed 2.5 seconds +/// instead. Fixing that means a caller can now ask "is it ready?", and the answer +/// depends on WHICH provider is resolved: +/// +/// * **bundled** — there is a local llama-server, so readiness is its health check. +/// * **anything else** (a cloud or remote endpoint) — there is no local server to +/// start, so it is ready by definition. Gating a cloud provider on a llama-server +/// that will never launch would leave "Preparing model…" on screen forever, which +/// is the stuck-state bug wearing a different hat. +/// * **bundled but not installed / not enabled** — genuinely not ready, and the +/// caller must say so rather than firing a request into nothing. +/// +/// That decision is pure policy, so it lives here where `swift test` pins it, and +/// AppState keeps only the engine call (MAK-32 ratchet: new logic goes to core, not +/// into the god object). +public enum LLMWarmReadiness { + + /// What a caller should do to warm `provider`. + public enum Decision: Equatable, Sendable { + /// Start the local server and report its health-check result. + case awaitLocalServer + /// Nothing to start — report ready immediately. + case alreadyReady + /// Nothing to start and it will never be ready; report not-ready. + case unavailable + } + + /// Decide how to warm. + /// + /// `isExplicit` marks a caller that resolved its OWN provider (the MAK-53 split a + /// plugin or the Scratchpad makes) rather than inheriting the global cleanup one. + /// That distinction is why `cleanupEnabled` gates only the implicit case: a + /// surface deliberately resolved to the bundled provider must still warm even when + /// Settings → Cleanup is switched off or pointed elsewhere. + public static func decide( + provider: String, + isExplicit: Bool, + modelInstalled: Bool, + cleanupEnabled: Bool + ) -> Decision { + guard provider == "bundled" else { return .alreadyReady } + guard modelInstalled else { return .unavailable } + guard isExplicit || cleanupEnabled else { return .unavailable } + return .awaitLocalServer + } +} diff --git a/OpenWhisp/Services/LLMWarmService.swift b/OpenWhisp/Services/LLMWarmService.swift new file mode 100644 index 0000000..8c0a07e --- /dev/null +++ b/OpenWhisp/Services/LLMWarmService.swift @@ -0,0 +1,48 @@ +import Foundation + +/// The AppState half of the LLM warm path (plugin spike v4). +/// +/// Lives outside `AppState.swift` deliberately: MAK-32's ratchet says new AppState +/// logic goes into core or an extension rather than growing the god object. The +/// *policy* — which providers need a local server at all — is the pure +/// `LLMWarmReadiness` resolver next door, pinned by `swift test`; this file holds only +/// the part that must touch the engine. +extension AppState { + + /// Start the bundled llama-server when it is the active, enabled, downloaded + /// provider. Idempotent (the engine no-ops if already healthy). Which providers + /// warm, and why an explicitly-resolved one bypasses the cleanup toggle (MAK-53), + /// is decided by `LLMWarmReadiness.decide` — see it for the rules. + /// + /// `completion` (v4) reports REAL readiness. `ensureRunning` polls llama-server's + /// `/health` and calls back only once it answers, so a caller can gate a button on + /// "the model can actually take a request" instead of guessing a duration — which + /// is exactly what the meme plugin was doing when its first two Generates failed + /// with a raw network error. Passing nil keeps the historic fire-and-forget + /// behaviour for the callers that don't wait. + func warmLlamaServerIfPossible( + provider explicitProvider: String? = nil, + completion: ((Bool) -> Void)? = nil + ) { + let decision = LLMWarmReadiness.decide( + provider: explicitProvider ?? llmProvider, + isExplicit: explicitProvider != nil, + modelInstalled: bundledLLMModelInstalled, + cleanupEnabled: openAIEnhancementEnabled) + + guard decision == .awaitLocalServer else { + completion?(decision == .alreadyReady) + return + } + + let engine = ensureLlamaEngine() + // Shorter idle teardown when a whisper-server is also resident, to relieve + // dual-engine memory pressure sooner (small-RAM Macs run both models). + engine.idleTimeout = whisperServerResident ? 30 : 90 + engine.ensureRunning(modelPath: selectedLLMModelPath()) { result in + guard let completion else { return } + // `ensureRunning` completes off the main thread; callers are main-actor. + Task { @MainActor in completion((try? result.get()) != nil) } + } + } +} diff --git a/OpenWhisp/Services/MemeAI.swift b/OpenWhisp/Services/MemeAI.swift index 3c6139b..b1cb550 100644 --- a/OpenWhisp/Services/MemeAI.swift +++ b/OpenWhisp/Services/MemeAI.swift @@ -89,16 +89,20 @@ public enum MemeAI { public static let rankedPrompt = """ You pick meme templates for a spoken description. - You are given a numbered list of the ONLY templates available. Think about which \ - ones could carry the joke, then rank your best options. + You are given a numbered list of the ONLY templates available. Each line is a \ + template name, optionally followed by alternate names in parentheses — the \ + parentheses describe what the meme is about, so use them to match a description \ + of the meme's CONTENT, but never copy them into your answer. + + Think about which ones could carry the joke, then rank your best options. Reply with ONLY a JSON object, no preamble and no code fence: {"templates": ["...", "...", "..."], "top_text": "...", "bottom_text": "..."} Rules: 1. "templates" lists 1 to 5 template names COPIED EXACTLY from the list, best \ - first. Do not invent a name, do not reword one, do not use a name that is not on \ - the list. + first — the part BEFORE any parentheses, without the number. Do not invent a \ + name, do not reword one, do not use a name that is not on the list. 2. If nothing on the list fits the description well, still return the closest \ options — but put the genuinely closest first. 3. "top_text" and "bottom_text" are the caption lines, in the SAME LANGUAGE as \ @@ -110,15 +114,31 @@ public enum MemeAI { 6. Never invent a caption that contradicts the description. """ + /// How many templates the LLM is asked to rank (v4). + /// + /// The shortlist is built LOCALLY by `MemeTemplateCatalog.prefilter` scoring the + /// user's own description against the whole merged corpus, so this cap is no + /// longer "the first N by popularity" — it is "the N most relevant". That makes it + /// safe to be much smaller than v3's 100, which matters: a tiny local model + /// attends to a 30-line list far better than a 100-line one, and the relevant + /// template is now guaranteed to be IN the list rather than truncated off the end + /// at position 180 of a merged ~300-template catalog. + public static let candidateShortlist = 30 + /// Build the user payload: the description plus the catalog the model must /// choose from. /// /// The list is numbered because small models copy list items more reliably when - /// the items are visually delimited, and truncated to `limit` because a 100-name - /// list is already ~1k tokens and some local models have short contexts. The - /// catalog is popularity-ordered, so truncating drops the least likely picks. + /// the items are visually delimited, and truncated to `limit` because a long list + /// blows a small local model's context. + /// + /// Each line may carry the template's KEYWORDS after its name (v4, see + /// `MemeTemplateCatalog.promptLines`) — that is what lets the model connect a + /// description of meme CONTENT to a template whose name shares no words with it. + /// The name stays first and unadorned so the model can copy it verbatim, which is + /// what `validate` checks. public static func rankedUserPayload( - description: String, templateNames: [String], limit: Int = 100 + description: String, templateNames: [String], limit: Int = candidateShortlist ) -> String { let names = templateNames.prefix(max(0, limit)) let list = names.enumerated() diff --git a/OpenWhisp/Services/MemeGenerationState.swift b/OpenWhisp/Services/MemeGenerationState.swift index a000f35..273f9da 100644 --- a/OpenWhisp/Services/MemeGenerationState.swift +++ b/OpenWhisp/Services/MemeGenerationState.swift @@ -165,6 +165,18 @@ public struct MemeGenerationState: Equatable, Sendable { phase = .idle } + /// Return to idle unconditionally, refusing every outstanding result (v4). + /// + /// Distinct from `cancel()` only in intent, and worth its own name for that + /// reason: `cancel` is "the user or the window stopped this", `reset` is "this + /// surface is starting fresh and must not inherit anything". Used by + /// `windowDidOpen`, where a phase left over from a previous session had no task, + /// no timeout, and no owner — the shape of the stuck-download report. + public mutating func reset() { + ticket += 1 + phase = .idle + } + /// Whether a result carrying `incoming` is still wanted. public func accepts(ticket incoming: Int) -> Bool { incoming == ticket } @@ -183,4 +195,95 @@ public struct MemeGenerationState: Equatable, Sendable { public static let timeoutMessage = "The model didn't answer within \(Int(generateTimeout)) seconds. It may still be " + "loading — try Generate again, or pick a template yourself with Browse all." + + /// The ceiling on ONE template-image download (v4). + /// + /// Much tighter than `generateTimeout`, because the two waits are not comparable: + /// a local model loading a multi-gigabyte file legitimately takes a minute, while + /// a template image is a few hundred KB over HTTP or a file read off the local + /// disk. Anything past this is a hang, not slowness. + /// + /// v3 applied NO ceiling to a download ticket at all — `select()` began a + /// `.downloading` phase and started no timer — which is why the owner's + /// "Downloading " could sit there indefinitely. + public static let downloadTimeout: TimeInterval = 30 + + /// The message shown when a template download hits the ceiling. + public static func downloadTimeoutMessage(_ templateName: String) -> String { + "\(templateName) didn't finish downloading within \(Int(downloadTimeout)) seconds. " + + "Press Retry, or pick another template." + } +} + +/// When a failed generate request is worth retrying rather than reporting (v4). +/// +/// ## Why a policy type +/// +/// The owner's report was that the FIRST TWO generates after opening the window fail +/// with a raw "network error". The cause is a race the UI cannot see: llama-server +/// binds its port slightly after it starts, so a request fired into that gap is +/// refused by the OS — not by the model. `NSURLErrorCannotConnectToHost` / +/// `ECONNREFUSED` is therefore not really an error yet, it is "too early"; the honest +/// response is to wait a moment and try again, exactly as a human would. +/// +/// v3 attacked the same race by sleeping a guessed 2.5 seconds before allowing +/// Generate. Guessing is what made it fail twice on a slow cold start and wait +/// pointlessly on a warm one. v4 gates on REAL readiness instead, and keeps this +/// retry as the second line of defence for the gap that readiness can't cover: the +/// server can pass a health check and still refuse the very next connection if it is +/// mid-restart (an idle-teardown relaunch, a model swap). +/// +/// Pure and Foundation-only so every decision is pinned by `swift test` — the +/// alternative is a retry loop nobody can test without a real socket. +public enum MemeGenerateRetry { + + /// How many ATTEMPTS one generate gets in total (the first try plus retries). + public static let maxAttempts = 3 + + /// The delay before attempt `attempt` (1-based: the delay before attempt 2). + /// + /// Backoff, not a fixed sleep: a server that is one instant from binding recovers + /// on the first short retry, and one that is genuinely still loading gets a longer + /// second wait rather than burning both attempts in half a second. + public static func delay(beforeAttempt attempt: Int) -> TimeInterval { + switch attempt { + case ..<2: return 0 + case 2: return 0.75 + default: return 2.0 + } + } + + /// True when `error` looks like "the server isn't accepting connections YET" + /// rather than a real failure. + /// + /// Matched on the URL-loading error codes rather than on message text, because the + /// text is localized — a Russian-locale Mac would silently stop retrying if this + /// keyed on English. `cannotConnectToHost` is the refused-connection case, + /// `networkConnectionLost` and `cannotFindHost` cover a server that dropped the + /// connection mid-restart, and `timedOut` covers one still paging its model in. + public static func isNotReadyYet(_ error: Error) -> Bool { + let ns = error as NSError + guard ns.domain == NSURLErrorDomain else { return false } + switch ns.code { + case NSURLErrorCannotConnectToHost, + NSURLErrorCannotFindHost, + NSURLErrorNetworkConnectionLost, + NSURLErrorTimedOut, + NSURLErrorNotConnectedToInternet: + return true + default: + return false + } + } + + /// Whether to retry: the error is a not-ready one AND attempts remain. + public static func shouldRetry(_ error: Error, attempt: Int) -> Bool { + attempt < maxAttempts && isNotReadyYet(error) + } + + /// The status shown while waiting to retry, so the wait is visible rather than a + /// frozen button. + public static func retryingMessage(attempt: Int) -> String { + "The model isn't ready yet — retrying (\(attempt) of \(maxAttempts))…" + } } diff --git a/OpenWhisp/Services/MemeTemplateMatcher.swift b/OpenWhisp/Services/MemeTemplateMatcher.swift index b66811c..e57a7b2 100644 --- a/OpenWhisp/Services/MemeTemplateMatcher.swift +++ b/OpenWhisp/Services/MemeTemplateMatcher.swift @@ -222,28 +222,18 @@ public enum MemeTemplateMatcher { /// This is the honest answer to "yoda isn't in the corpus": the user can see and /// search every template the plugin has, and pick one the model never proposed. /// - /// Deliberately unlike v1's deleted `bestMatch`: no scoring, no threshold, and - /// above all **no fallback**. A search that matches nothing returns nothing, because an + /// **v4: ranked, not all-or-nothing.** This now delegates to + /// `MemeTemplateCatalog.search`, which scores name AND keywords and orders the + /// results best-first. The v3 rule here — every query token had to appear in the + /// name — is what made a content description ("the worst day for the planet") + /// return nothing at all: one unmatched token vetoed every token that did match. + /// See `MemeTemplateCatalog.score` for the tiers. + /// + /// Still **no fallback**: a query matching nothing returns nothing, because an /// empty grid saying "no templates match" is the entire point — silently /// substituting popular templates is the bug this whole change exists to fix. - /// - /// Matching is substring-based over the normalized name so it is - /// case/diacritic/punctuation-insensitive, and multi-word queries match when - /// EVERY token appears somewhere in the name (so "drake bling" finds "Drake - /// Hotline Bling"). Catalog order — imgflip's popularity ranking — is preserved. public static func search(_ query: String, in catalog: [MemeTemplate]) -> [MemeTemplate] { - let normalizedQuery = normalize(query) - guard !normalizedQuery.isEmpty else { return catalog } - - // Split on the normalized form, not `tokens`: stopword removal is right for - // matching an LLM's phrase, wrong for a user typing letter by letter — a - // search for "the" should still narrow the list rather than reset it. - let needles = normalizedQuery.split(separator: " ").map(String.init) - - return catalog.filter { template in - let haystack = normalize(template.name) - return needles.allSatisfy { haystack.contains($0) } - } + MemeTemplateCatalog.search(query, in: catalog) } // MARK: - Scoring @@ -275,7 +265,7 @@ public enum MemeTemplateMatcher { } /// Lowercase, strip punctuation, collapse whitespace. - static func normalize(_ value: String) -> String { + public static func normalize(_ value: String) -> String { let folded = value.folding(options: [.diacriticInsensitive, .caseInsensitive], locale: Locale(identifier: "en_US")) let scalars = folded.unicodeScalars.map { scalar -> Character in @@ -286,9 +276,22 @@ public enum MemeTemplateMatcher { .joined(separator: " ") } + /// Tokens for SEARCH scoring (v4). + /// + /// Like `tokens`, but it degrades gracefully instead of vanishing: a query made + /// entirely of stopwords ("the man") would tokenize to nothing and silently reset + /// the grid to the whole catalog, so when stopword removal empties the query we + /// fall back to the raw normalized words. Single characters survive here too — a + /// user typing "x" is narrowing, not searching for nothing. + public static func searchTokens(in value: String) -> [String] { + let meaningful = tokens(in: value) + guard meaningful.isEmpty else { return meaningful } + return normalize(value).split(separator: " ").map(String.init) + } + /// Meaningful tokens, with English stopwords dropped so "the drake meme" and /// "drake" score the same. - static func tokens(in value: String) -> [String] { + public static func tokens(in value: String) -> [String] { let stopwords: Set = ["the", "a", "an", "of", "and", "meme", "guy", "man"] return normalize(value) .split(separator: " ") diff --git a/OpenWhisp/Services/MemeTemplateProvider.swift b/OpenWhisp/Services/MemeTemplateProvider.swift index 6eb32b4..0e15c28 100644 --- a/OpenWhisp/Services/MemeTemplateProvider.swift +++ b/OpenWhisp/Services/MemeTemplateProvider.swift @@ -94,7 +94,8 @@ public enum MemeTemplateCatalog { return MemeTemplateSource(rawValue: String(id[id.startIndex.. [MemeTemplate] { + ranked(query, in: catalog, limit: catalog.count).map(\.template) + } + + /// A scored search hit. Exposed so callers that need the score (the LLM + /// prefilter, tests) don't have to re-derive it. + public struct Match: Equatable, Sendable { + public let template: MemeTemplate + public let score: Int + public init(template: MemeTemplate, score: Int) { + self.template = template + self.score = score + } + } + + /// The scored, ordered matches for a query — the engine behind both `search` and + /// `prefilter`. + /// + /// Ordering is score descending, then the catalog's own index ascending. The + /// index tie-break matters twice: `sorted(by:)` is not guaranteed stable, and the + /// catalog order IS the popularity ranking, so it is the right thing to fall back + /// on when two templates match a query equally well. + public static func ranked( + _ query: String, in catalog: [MemeTemplate], limit: Int + ) -> [Match] { let normalizedQuery = MemeTemplateMatcher.normalize(query) - guard !normalizedQuery.isEmpty else { return catalog } - - let needles = normalizedQuery.split(separator: " ").map(String.init) - - return catalog.filter { template in - // Every token must appear SOMEWHERE across the name and keywords - // combined, so "russian cat" matches a template named "Cat" with the - // keyword "Russian" — the cross-field case that a per-field search - // would miss. - let haystack = ([template.name] + template.keywords) - .map { MemeTemplateMatcher.normalize($0) } - .joined(separator: " ") - return needles.allSatisfy { haystack.contains($0) } + let cap = max(0, limit) + guard cap > 0 else { return [] } + // An empty query isn't a failed search — it's "no filter applied", which the + // Browse grid renders as the whole corpus in popularity order. + guard !normalizedQuery.isEmpty else { + return catalog.prefix(cap).map { Match(template: $0, score: 0) } } + + let queryTokens = MemeTemplateMatcher.searchTokens(in: query) + guard !queryTokens.isEmpty else { + return catalog.prefix(cap).map { Match(template: $0, score: 0) } + } + + // A named struct rather than a tuple chain: the inferred-tuple version of this + // sort was too much for the type checker in `MemeTemplateMatcher.ranked`, and + // there is no reason to rediscover that here. + struct Scored { + let index: Int + let template: MemeTemplate + let score: Int + } + + var scored: [Scored] = [] + for (index, template) in catalog.enumerated() { + let value = score(queryTokens: queryTokens, normalizedQuery: normalizedQuery, + template: template) + guard value > 0 else { continue } + scored.append(Scored(index: index, template: template, score: value)) + } + + scored.sort { left, right in + left.score == right.score ? left.index < right.index : left.score > right.score + } + return scored.prefix(cap).map { Match(template: $0.template, score: $0.score) } + } + + /// Score one template against a query, over its name AND its keywords. + /// + /// The tiers, and why each exists: + /// + /// * **Exact name** (10_000) — typing a template's name means you want that + /// template, full stop. + /// * **Whole-phrase containment** in the name (5_000 + closeness) — "drake" ⊂ + /// "Drake Hotline Bling". The closeness bonus prefers the shortest name that + /// still contains the phrase. + /// * **Per-token matches** — this is the tier that fixes the owner's repro. Each + /// query token is worth its best match anywhere in the template: + /// 100 for a whole-token hit in the NAME, 60 for one in a KEYWORD (a name match + /// is stronger evidence than an alias), and 25/15 for a PREFIX hit + /// ("planetary" → "planet"), which counts for less precisely because it is + /// weaker evidence. Summing over tokens means more matched tokens ranks higher, + /// which is the whole ordering the owner asked for. + /// * **Coverage bonus** — a template matching a larger FRACTION of the query is + /// worth more than one matching the same count out of a longer query, so short + /// precise queries stay precise. + /// + /// Returns 0 when nothing matched, which is what keeps "no results" possible. + public static func score( + queryTokens: [String], normalizedQuery: String, template: MemeTemplate + ) -> Int { + let normalizedName = MemeTemplateMatcher.normalize(template.name) + if !normalizedName.isEmpty, normalizedName == normalizedQuery { return 10_000 } + + if !normalizedQuery.isEmpty, !normalizedName.isEmpty, + normalizedName.contains(normalizedQuery) || normalizedQuery.contains(normalizedName) { + let lengthGap = abs(normalizedName.count - normalizedQuery.count) + return 5_000 + max(0, 100 - lengthGap) + } + + let nameTokens = Set(MemeTemplateMatcher.normalize(template.name).split(separator: " ").map(String.init)) + let keywordTokens = Set( + template.keywords + .flatMap { MemeTemplateMatcher.normalize($0).split(separator: " ").map(String.init) }) + + var total = 0 + var matchedTokens = 0 + for token in queryTokens { + var best = 0 + if nameTokens.contains(token) { best = 100 } + else if keywordTokens.contains(token) { best = 60 } + else if nameTokens.contains(where: { $0.hasPrefix(token) || token.hasPrefix($0) }) { best = 25 } + else if keywordTokens.contains(where: { $0.hasPrefix(token) || token.hasPrefix($0) }) { best = 15 } + + if best > 0 { + total += best + matchedTokens += 1 + } + } + guard matchedTokens > 0 else { return 0 } + + // Reward matching a larger share of what the user actually said. + let coverage = (matchedTokens * 50) / queryTokens.count + return total + coverage + } + + /// The templates handed to the LLM to rank, chosen by LOCAL relevance to the + /// user's description rather than by raw popularity. + /// + /// ## Why this exists (v4) + /// + /// v3 gave the model `promptNames(catalog, limit: 100)` — the first hundred + /// templates in popularity order, names only. That has two failures the owner hit: + /// the genuinely relevant template can sit at position 180 of a merged ~300 corpus + /// and never enter the prompt at all, and a model given bare NAMES cannot connect + /// "the worst day for the planet" to a template whose relevance lives in its + /// KEYWORDS. + /// + /// So the catalog is prefiltered locally first: score every template against the + /// user's own words, keep the top `limit`, and hand the model that shortlist WITH + /// its keywords (see `promptLines`). Describing meme CONTENT now finds templates + /// through their keywords, and the shortlist is small enough that a tiny local + /// model can actually attend to all of it. + /// + /// Falls back to popularity order when the description matches nothing — the model + /// still deserves a corpus to choose from, and the UI already states plainly when + /// nothing matched. + public static func prefilter( + for description: String, in catalog: [MemeTemplate], limit: Int + ) -> [MemeTemplate] { + let cap = max(0, limit) + guard cap > 0 else { return [] } + + let hits = ranked(description, in: catalog, limit: cap).map(\.template) + guard !hits.isEmpty else { return Array(catalog.prefix(cap)) } + + // Top up with popular templates when the query was narrow, so the model always + // sees a full shortlist rather than the two things that happened to match. + guard hits.count < cap else { return hits } + var out = hits + var seen = Set(hits.map(\.id)) + for template in catalog where out.count < cap { + guard !seen.contains(template.id) else { continue } + seen.insert(template.id) + out.append(template) + } + return out } /// The names handed to the LLM, capped so a merged ~300-name corpus doesn't blow @@ -132,4 +297,26 @@ public enum MemeTemplateCatalog { public static func promptNames(_ catalog: [MemeTemplate], limit: Int) -> [String] { Array(catalog.prefix(max(0, limit)).map(\.name)) } + + /// One prompt line per template: the name, plus its keywords in parentheses. + /// + /// The keywords are the point (v4). "Worst Day Of My Life So Far" carries aliases + /// a user's description will hit even when the NAME shares no words with it, so + /// showing the model only names throws away the very signal that connects a + /// content description to a template. Templates with no keywords render as a bare + /// name, so nothing is padded with noise. + /// + /// The name is always FIRST on the line and unadorned, because the prompt asks the + /// model to copy the name verbatim and `MemeAI.validate` checks it against the + /// catalog — a line the model can't cleanly copy a name out of would be rejected + /// as a hallucination. + public static func promptLines(_ catalog: [MemeTemplate], limit: Int) -> [String] { + catalog.prefix(max(0, limit)).map { template in + let keywords = template.keywords + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + guard !keywords.isEmpty else { return template.name } + return "\(template.name) (\(keywords.prefix(6).joined(separator: ", ")))" + } + } } diff --git a/Package.swift b/Package.swift index 3b04ae7..15c99df 100644 --- a/Package.swift +++ b/Package.swift @@ -179,7 +179,8 @@ let package = Package( "MemeTemplateProvider.swift", "MemeUserLibrary.swift", "MemeGenerationState.swift", - "MemeCatalogCache.swift" + "MemeCatalogCache.swift", + "LLMWarmReadiness.swift" ] ), // The Agent Bridge client + MCP stdio adapter. A library (not folded into diff --git a/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift b/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift index b20fce3..894392f 100644 --- a/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift +++ b/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift @@ -323,11 +323,23 @@ final class MemeGeneratorTests: XCTestCase { MemeTemplateMatcher.search("button", in: catalog).map(\.id), ["3"]) } - func testSearchRequiresEveryTokenToAppear() { + /// v3 REQUIRED every token to appear, which is what made a content description + /// unsearchable (see `testWorstDayDescriptionFindsTheBartTemplate`). v4 ranks + /// instead: a query spanning two templates surfaces BOTH rather than neither. + func testAPartialTokenMatchStillSurfacesTheTemplate() { XCTAssertEqual( MemeTemplateMatcher.search("drake bling", in: catalog).map(\.id), ["1"]) - XCTAssertEqual( - MemeTemplateMatcher.search("drake buttons", in: catalog).map(\.id), []) + + let both = MemeTemplateMatcher.search("drake buttons", in: catalog).map(\.id) + XCTAssertTrue(both.contains("1"), "the Drake half of the query must still match") + XCTAssertTrue(both.contains("3"), "the Buttons half must too — v3 returned nothing here") + } + + /// More matched tokens ranks higher. This is the ordering the whole v4 change + /// exists to produce. + func testMoreMatchedTokensRanksHigher() { + let hits = MemeTemplateMatcher.search("drake hotline bling", in: catalog).map(\.id) + XCTAssertEqual(hits.first, "1") } func testSearchIgnoresPunctuation() { @@ -350,9 +362,11 @@ final class MemeGeneratorTests: XCTestCase { "ranking refuses too — v1's bestMatch answered Drake here, which was the bug") } - func testSearchPreservesPopularityOrder() { - let hits = MemeTemplateMatcher.search("t", in: catalog).map(\.id) - XCTAssertEqual(hits, catalog.filter { hits.contains($0.id) }.map(\.id)) + /// Equal scores fall back to the catalog's own order, which IS the popularity + /// ranking — so ranking never reshuffles templates it has no reason to separate. + func testEquallyScoringHitsKeepPopularityOrder() { + let hits = MemeTemplateMatcher.search("bling boyfriend", in: catalog).map(\.id) + XCTAssertEqual(hits, ["1", "2"], "same score (one token each) -> catalog order") } // MARK: - Caption box model diff --git a/Tests/OpenWhispCoreTests/MemeProviderTests.swift b/Tests/OpenWhispCoreTests/MemeProviderTests.swift index fafa015..5ee6e4a 100644 --- a/Tests/OpenWhispCoreTests/MemeProviderTests.swift +++ b/Tests/OpenWhispCoreTests/MemeProviderTests.swift @@ -124,6 +124,130 @@ final class MemeProviderTests: XCTestCase { XCTAssertEqual(MemeTemplateCatalog.search(" ", in: catalog).count, 2) } + // MARK: - v4: ranked search (the owner's "worst day" repro) + + /// A corpus shaped like the real merged one: the Bart template is present, buried + /// well down the popularity order, and its relevance lives partly in keywords. + private var worstDayCatalog: [MemeTemplate] { + var out: [MemeTemplate] = (0..<40).map { + template(.imgflip, "i\($0)", "Popular Template \($0)") + } + out.append(template(.memegen, "worst-day", "Worst Day Of My Life So Far", + keywords: ["Bart Simpson", "chalkboard", "bad day"])) + return out + } + + /// **The owner's exact report.** "the worst day for the planet" must surface the + /// Bart template. v3 returned NOTHING: its all-tokens rule required "planet" to + /// appear in the name or keywords, so one unmatched token vetoed the three that + /// matched perfectly. + func testWorstDayDescriptionFindsTheBartTemplate() { + let hits = MemeTemplateCatalog.search("the worst day for the planet", in: worstDayCatalog) + XCTAssertEqual(hits.first?.name, "Worst Day Of My Life So Far", + "describing the meme's content must find it, and rank it first") + } + + /// The owner's second phrasing of the same query. + func testWorstDaySoFarDescriptionFindsTheBartTemplate() { + let hits = MemeTemplateCatalog.search("the worst day so far", in: worstDayCatalog) + XCTAssertEqual(hits.first?.name, "Worst Day Of My Life So Far") + } + + /// Relevance beats popularity: the Bart template is at index 40 and still wins. + func testARelevantTemplateOutranksPopularOnesThatDoNotMatch() { + let hits = MemeTemplateCatalog.search("worst day", in: worstDayCatalog) + XCTAssertEqual(hits.first?.name, "Worst Day Of My Life So Far") + XCTAssertFalse(hits.contains { $0.name.hasPrefix("Popular Template") }, + "templates that match nothing must not be padded in") + } + + /// A name-token match is stronger evidence than a keyword match, so the template + /// actually NAMED for the query ranks above one that merely lists it as an alias. + func testANameMatchOutranksAKeywordMatch() { + let catalog = [ + template(.memegen, "alias", "Something Else", keywords: ["chalkboard"]), + template(.imgflip, "named", "Chalkboard"), + ] + XCTAssertEqual(MemeTemplateCatalog.search("chalkboard", in: catalog).first?.name, + "Chalkboard") + } + + /// Partial/prefix matches count, but count LESS — they are weaker evidence. + func testAPrefixMatchScoresBelowAWholeTokenMatch() { + let whole = template(.imgflip, "w", "Planet") + let prefix = template(.imgflip, "p", "Planetarium Nights") + let hits = MemeTemplateCatalog.search("planet", in: [prefix, whole]) + XCTAssertEqual(hits.first?.name, "Planet", + "exact token beats prefix even though prefix came first in the catalog") + XCTAssertEqual(hits.count, 2, "the prefix match is still shown, just ranked lower") + } + + /// v4 ranks partial matches — it must still never INVENT one. Ranking and + /// falling back are different things, and the fallback is the original bug. + func testRankedSearchStillReturnsNothingWhenNothingMatchesAtAll() { + XCTAssertTrue( + MemeTemplateCatalog.search("zzzzz qqqqq", in: worstDayCatalog).isEmpty) + } + + /// A query of pure stopwords must still narrow rather than silently resetting the + /// grid to the whole catalog. + func testAStopwordOnlyQueryStillFilters() { + let catalog = [ + template(.imgflip, "i1", "The Rock Driving"), + template(.imgflip, "i2", "Success Kid"), + ] + XCTAssertEqual(MemeTemplateCatalog.search("the", in: catalog).map(\.name), + ["The Rock Driving"]) + } + + // MARK: - v4: the LLM shortlist + + /// The prefilter is what lets the LLM benefit from the same scoring: the relevant + /// template is at index 40 of the corpus and would be truncated off a + /// popularity-ordered prompt, but it leads the shortlist. + func testPrefilterPutsTheRelevantTemplateInFrontOfTheModel() { + let shortlist = MemeTemplateCatalog.prefilter( + for: "the worst day for the planet", in: worstDayCatalog, limit: 30) + XCTAssertEqual(shortlist.first?.name, "Worst Day Of My Life So Far") + XCTAssertEqual(shortlist.count, 30, "the model still gets a full shortlist to rank") + } + + /// A description matching nothing still gets the model a corpus to choose from — + /// the UI, not the prompt, is where "nothing matched" is stated. + func testPrefilterFallsBackToPopularityWhenNothingMatches() { + let shortlist = MemeTemplateCatalog.prefilter( + for: "zzzzz qqqqq", in: worstDayCatalog, limit: 5) + XCTAssertEqual(shortlist.map(\.name), (0..<5).map { "Popular Template \($0)" }) + } + + func testPrefilterNeverRepeatsATemplateWhenToppingUp() { + let shortlist = MemeTemplateCatalog.prefilter( + for: "worst day", in: worstDayCatalog, limit: 10) + XCTAssertEqual(Set(shortlist.map(\.id)).count, shortlist.count) + } + + /// The model is shown keywords — that is what connects a CONTENT description to a + /// template — but the name stays first and unadorned so it can be copied verbatim + /// and validated against the catalog. + func testPromptLinesCarryKeywordsAfterAnUnadornedName() { + let catalog = [ + template(.memegen, "m1", "Sweet Brown", keywords: ["Ain't Nobody Got Time For That"]), + template(.imgflip, "i1", "Drake Hotline Bling"), + ] + let lines = MemeTemplateCatalog.promptLines(catalog, limit: 10) + XCTAssertEqual(lines[0], "Sweet Brown (Ain't Nobody Got Time For That)") + XCTAssertEqual(lines[1], "Drake Hotline Bling", "no keywords -> no empty parentheses") + } + + /// The name a `promptLines` entry starts with must be the one `MemeAI.validate` + /// accepts — otherwise every keyword-carrying template would read as a + /// hallucination and be dropped. + func testAModelCopyingTheNameOffAPromptLineValidates() { + let catalog = [template(.memegen, "m1", "Sweet Brown", keywords: ["no time"])] + let names = MemeTemplateCatalog.promptNames(catalog, limit: 10) + XCTAssertEqual(MemeAI.validate(["Sweet Brown"], against: names), ["Sweet Brown"]) + } + /// The user's own templates sort first, so the prompt cap can never exclude them. func testPromptNamesCapTheCorpusButKeepUserTemplates() { let merged = MemeTemplateCatalog.merge([ @@ -525,4 +649,182 @@ final class MemeProviderTests: XCTestCase { XCTAssertTrue(MemeGenerationState.timeoutMessage.contains("Browse all")) XCTAssertGreaterThan(MemeGenerationState.generateTimeout, 0) } + + // MARK: - v4: the stuck-download orderings + + /// **The owner's report #2 ordering.** A download that is superseded/abandoned and + /// then followed by a window REOPEN must not leave the surface parked in + /// `.downloading`. v3's `renderTemplate` returned bare on a stale ticket and + /// `windowDidOpen` only cleared `isCancelled`, so the phase survived with no task, + /// no timeout and no Retry behind it — "Downloading " forever. + func testAReopenedWindowNeverInheritsADownloadingPhase() { + var state = MemeGenerationState() + _ = state.begin(.downloading(templateName: "Drake")) + XCTAssertTrue(state.isGenerating) + + state.reset() + + XCTAssertEqual(state.phase, .idle) + XCTAssertFalse(state.isGenerating) + XCTAssertNil(state.generateBlockedReason(), "Generate must be live again after a reopen") + } + + /// `reset` also refuses the abandoned work's late result, so a download that + /// completes after the reopen can't drag the fresh window back into its phase. + func testResetRefusesTheAbandonedDownloadsLateResult() { + var state = MemeGenerationState() + let stale = state.begin(.downloading(templateName: "Drake")) + state.reset() + + XCTAssertFalse(state.accepts(ticket: stale)) + XCTAssertFalse(state.finish(ticket: stale)) + XCTAssertFalse(state.advance(.asking, ticket: stale)) + XCTAssertEqual(state.phase, .idle) + } + + /// Every download exit — success, failure, timeout, supersession — ends at a + /// `finish` for its OWN ticket, and finishing a superseded ticket is a harmless + /// no-op. This is the property that makes "no exit can leave the phase set" true + /// without a superseded task being able to unstick newer work. + func testFinishingASupersededDownloadCannotDisturbTheNewerOne() { + var state = MemeGenerationState() + let first = state.begin(.downloading(templateName: "Drake")) + let second = state.begin(.downloading(templateName: "Bart")) + + XCTAssertFalse(state.finish(ticket: first), "the superseded exit is a no-op") + XCTAssertEqual(state.phase, .downloading(templateName: "Bart"), + "the newer download still owns the phase") + + XCTAssertTrue(state.finish(ticket: second)) + XCTAssertEqual(state.phase, .idle) + } + + /// A download ticket now carries a FINITE ceiling — v3 started one with no timer + /// at all, which is why a wedged GET hung the surface indefinitely. + func testADownloadHasItsOwnFiniteCeilingShorterThanAGenerates() { + XCTAssertGreaterThan(MemeGenerationState.downloadTimeout, 0) + XCTAssertLessThan(MemeGenerationState.downloadTimeout, + MemeGenerationState.generateTimeout, + "an image is not a model load — it must give up much sooner") + } + + /// The download give-up message names the template and offers a way out, rather + /// than leaving a spinner with no explanation. + func testDownloadTimeoutMessageNamesTheTemplateAndOffersRetry() { + let message = MemeGenerationState.downloadTimeoutMessage("Drake Hotline Bling") + XCTAssertTrue(message.contains("Drake Hotline Bling")) + XCTAssertTrue(message.contains("Retry")) + } + + // MARK: - v4: generate retry (the "first two generates fail" report) + + /// A refused connection is "the server isn't up YET", not a failure to report — + /// this is exactly what surfaced as the owner's raw "network error". + func testARefusedConnectionIsTreatedAsNotReadyYet() { + let refused = NSError(domain: NSURLErrorDomain, + code: NSURLErrorCannotConnectToHost, userInfo: nil) + XCTAssertTrue(MemeGenerateRetry.isNotReadyYet(refused)) + XCTAssertTrue(MemeGenerateRetry.shouldRetry(refused, attempt: 1)) + } + + /// A real model error is reported immediately — retrying it would only make the + /// user wait longer for the same message. + func testARealFailureIsNotRetried() { + let real = NSError(domain: "OpenWhisp", code: 42, userInfo: nil) + XCTAssertFalse(MemeGenerateRetry.isNotReadyYet(real)) + XCTAssertFalse(MemeGenerateRetry.shouldRetry(real, attempt: 1)) + } + + /// The retry budget is finite: the third refusal surfaces honestly instead of + /// looping forever. + func testRetriesAreBoundedAndThenReportHonestly() { + let refused = NSError(domain: NSURLErrorDomain, + code: NSURLErrorCannotConnectToHost, userInfo: nil) + XCTAssertTrue(MemeGenerateRetry.shouldRetry(refused, attempt: 2)) + XCTAssertFalse( + MemeGenerateRetry.shouldRetry(refused, attempt: MemeGenerateRetry.maxAttempts), + "the budget is spent — say so rather than retrying forever") + XCTAssertGreaterThanOrEqual(MemeGenerateRetry.maxAttempts, 2) + } + + /// Backoff, not a fixed sleep: the first retry is quick (the server is usually one + /// instant from binding) and later ones wait longer. + func testRetryDelaysBackOff() { + XCTAssertEqual(MemeGenerateRetry.delay(beforeAttempt: 1), 0, + "the first attempt never waits") + XCTAssertGreaterThan(MemeGenerateRetry.delay(beforeAttempt: 2), 0) + XCTAssertGreaterThan(MemeGenerateRetry.delay(beforeAttempt: 3), + MemeGenerateRetry.delay(beforeAttempt: 2)) + } + + /// Matching is on the URL error CODE, not on message text — the text is localized, + /// so a non-English Mac would silently stop retrying if this keyed on English. + func testNotReadyDetectionDoesNotDependOnLocalizedText() { + let localized = NSError( + domain: NSURLErrorDomain, code: NSURLErrorCannotConnectToHost, + userInfo: [NSLocalizedDescriptionKey: "Не удалось подключиться к серверу"]) + XCTAssertTrue(MemeGenerateRetry.isNotReadyYet(localized)) + } + + /// The wait is visible rather than a frozen button. + func testRetryStatusNamesTheAttempt() { + let message = MemeGenerateRetry.retryingMessage(attempt: 2) + XCTAssertTrue(message.contains("2")) + XCTAssertTrue(message.contains("\(MemeGenerateRetry.maxAttempts)")) + } + + // MARK: - v4: what "warmed" means per provider + + /// The bundled provider has a local server, so readiness is its health check — + /// this is the case the plugin must WAIT for rather than guess at. + func testTheBundledProviderIsWarmedByWaitingForItsLocalServer() { + XCTAssertEqual( + LLMWarmReadiness.decide(provider: "bundled", isExplicit: true, + modelInstalled: true, cleanupEnabled: false), + .awaitLocalServer) + } + + /// A cloud/remote provider has no local server to start, so it is ready by + /// definition. Gating it on a llama-server that will never launch would leave + /// "Preparing model…" on screen forever — the stuck-state bug in a new costume. + func testANonBundledProviderIsReadyImmediately() { + XCTAssertEqual( + LLMWarmReadiness.decide(provider: "openai", isExplicit: true, + modelInstalled: false, cleanupEnabled: false), + .alreadyReady) + } + + /// Bundled but never downloaded is genuinely unavailable — the caller must say so + /// rather than blocking Generate behind a warm that can't finish. + func testTheBundledProviderWithoutItsModelIsUnavailable() { + XCTAssertEqual( + LLMWarmReadiness.decide(provider: "bundled", isExplicit: true, + modelInstalled: false, cleanupEnabled: true), + .unavailable) + } + + /// The MAK-53 split: a surface that resolved its OWN provider to bundled warms + /// even when Settings → Cleanup is off, while the implicit global case still + /// respects the toggle. + func testAnExplicitlyResolvedProviderBypassesTheCleanupToggle() { + XCTAssertEqual( + LLMWarmReadiness.decide(provider: "bundled", isExplicit: true, + modelInstalled: true, cleanupEnabled: false), + .awaitLocalServer) + XCTAssertEqual( + LLMWarmReadiness.decide(provider: "bundled", isExplicit: false, + modelInstalled: true, cleanupEnabled: false), + .unavailable) + } + + /// Warming blocks Generate with an honest reason — and `reset` clears it, so a + /// warm that never completed can't leave the button permanently "Preparing model…". + func testAWarmThatNeverCompletesCannotBlockGenerateForever() { + var state = MemeGenerationState() + _ = state.begin(.warming) + XCTAssertEqual(state.generateBlockedReason(), "Preparing model…") + + state.reset() + XCTAssertNil(state.generateBlockedReason()) + } } diff --git a/plugins/MemeGenerator/MemeGeneratorModel.swift b/plugins/MemeGenerator/MemeGeneratorModel.swift index 16f56af..d9f123d 100644 --- a/plugins/MemeGenerator/MemeGeneratorModel.swift +++ b/plugins/MemeGenerator/MemeGeneratorModel.swift @@ -86,6 +86,19 @@ final class MemeGeneratorModel: ObservableObject { /// where the user must act. Drives the Retry affordance. @Published private(set) var catalogFailed: Bool = false + /// Set when a TEMPLATE IMAGE failed to load. Drives its own Retry affordance + /// (v4). + /// + /// Separate from `catalogFailed` because they fail independently and recover + /// differently: a catalog failure retries the fetch, an image failure retries one + /// template. v3 had no image-failure signal at all, so a failed download had + /// nowhere to surface — which is half of why "Downloading…" looked like it hung + /// rather than like it had failed. + @Published private(set) var imageFailed: Bool = false + + /// The template whose image failed, so Retry knows what to re-fetch. + @Published private(set) var failedTemplate: MemeTemplate? + /// The templates matching `searchText`, across names AND keywords so a merged, /// multi-lingual corpus is actually findable. var searchResults: [MemeTemplate] { @@ -125,14 +138,22 @@ final class MemeGeneratorModel: ObservableObject { private var resolveAIModel: () -> SummaryModelResolver.Resolved = { .init(provider: "", model: "", endpoint: "") } - /// Warms the LLM without running a completion. Injected so this model stays - /// AppState-free and the warm is stubbable. - private var warmModel: (SummaryModelResolver.Resolved) -> Void = { _ in } + /// Warms the LLM and REPORTS READINESS (v4). + /// + /// The completion carries whether the model can actually take a request — it is + /// driven by llama-server's `/health` poll, not by a timer. v3's seam returned + /// `Void`, which is precisely why this model had to guess how long to wait. + /// Injected so the model stays AppState-free and the readiness is stubbable. + typealias WarmCall = ( + _ resolved: SummaryModelResolver.Resolved, _ ready: @escaping (Bool) -> Void + ) -> Void + + private var warmModel: WarmCall = { _, ready in ready(true) } func configureAI( call: @escaping AICall, resolveModel: @escaping () -> SummaryModelResolver.Resolved, - warm: @escaping (SummaryModelResolver.Resolved) -> Void = { _ in } + warm: @escaping WarmCall = { _, ready in ready(true) } ) { aiCall = call resolveAIModel = resolveModel @@ -160,36 +181,72 @@ final class MemeGeneratorModel: ObservableObject { /// stale). Neither blocks the user — they can browse and edit while both run. func windowDidOpen() { isCancelled = false + // v4: a reopened window must never INHERIT a phase. `cancel()` on close bumps + // the ticket and returns to idle, but any exit path that failed to finish left + // the machine parked — and clearing `isCancelled` here is what used to revive + // a stranded `.downloading` with no task, no timeout, and no Retry behind it. + // Resetting is cheap and makes the window's initial state unconditional. + state.reset() + imageFailed = false + failedTemplate = nil + status = "" warmLLM() openCatalog() } - /// Start the local model loading in the background. + /// Start the local model loading, and hold "Preparing model…" until it is REALLY + /// ready. + /// + /// ## v4 — readiness, not a stopwatch + /// + /// v3 showed the warming phase for a guessed 2.5 seconds and then allowed Generate + /// regardless. That is why the owner still saw the first TWO generates fail with a + /// raw network error: on a cold start llama-server needs far longer than 2.5s to + /// bind its port, so the guess expired while the socket was still refusing + /// connections, and the UI cheerfully fired into it. /// - /// Deliberately fire-and-forget with a phase, not an awaited call: the warm has no - /// completion we can observe from here (the engine's readiness is AppState's), so - /// we show "Preparing model…" for a bounded moment and then allow Generate. If the - /// model is still loading when Generate runs, `summarizeResolved` blocks on the - /// same engine anyway — the warm has simply removed most of that wait. + /// The readiness signal existed all along and was being discarded: + /// `LlamaServerEngine.ensureRunning` polls the server's `/health` endpoint and + /// calls back only when it answers. `AppState.warmLlamaServerIfPossible` dropped + /// that completion (`{ _ in }`); it now forwards it, and the `warm` seam carries it + /// here. So the warming phase ends when the model can actually take a request — + /// immediately on a warm server, a minute later on a cold one, and never on a + /// guess. + /// + /// A warm that FAILS is stated rather than hidden, because the alternative is the + /// original bug in a new costume: an available-looking Generate button in front of + /// a model that isn't there. private func warmLLM() { let resolved = resolveAIModel() guard ScratchpadAIModel.isUsable(resolved) else { return } - warmModel(resolved) let ticket = state.begin(.warming) - Task { [weak self] in - // A short, fixed window. The point is to stop the user firing a request - // into a socket nothing is listening on yet, not to gate the UI on a - // completion we can't see. - try? await Task.sleep(nanoseconds: 2_500_000_000) + status = MemeGenerationState.Phase.warming.statusText + + // A ceiling on the warm itself: a server that never becomes healthy must not + // leave Generate blocked behind "Preparing model…" forever — that would be the + // stuck-state bug moved into the warm path. On expiry we simply stop blocking; + // the request's own retry (`MemeGenerateRetry`) becomes the safety net. + startTimeout(ticket: ticket, + after: Self.warmTimeout, + message: "The model is taking a while to start — Generate will try anyway.") + + warmModel(resolved) { [weak self] ready in guard let self, !self.isCancelled else { return } - if self.state.finish(ticket: ticket), self.status == MemeGenerationState.Phase.warming.statusText { - self.status = "" + guard self.state.finish(ticket: ticket) else { return } + // Only clear a status we ourselves wrote; anything newer wins. + if ready { + if self.status == MemeGenerationState.Phase.warming.statusText { self.status = "" } + } else { + self.status = "The built-in model isn't available — check Settings → Cleanup." } } - status = MemeGenerationState.Phase.warming.statusText } + /// How long the warm may block Generate before it gives up and lets the request + /// try on its own. + private static let warmTimeout: TimeInterval = 180 + // MARK: - Catalog /// Open the catalog: disk cache first, network second. @@ -337,12 +394,25 @@ final class MemeGeneratorModel: ObservableObject { self.state.advance(.asking, ticket: ticket) self.status = MemeGenerationState.Phase.asking.statusText - let names = MemeTemplateCatalog.promptNames(self.catalog, limit: 100) + // v4: shortlist LOCALLY before asking. Scoring the user's own words + // against the whole merged corpus (name AND keywords) is what lets a + // description of the meme's CONTENT reach a template whose name shares no + // words with it — and it guarantees the relevant template is in the + // prompt at all, rather than truncated off the end of a popularity list. + let shortlist = MemeTemplateCatalog.prefilter( + for: self.description, in: self.catalog, limit: MemeAI.candidateShortlist) + // The model is shown names + keywords, but validated against NAMES — + // `promptLines` keeps the name first and unadorned precisely so the two + // stay in sync. + let names = MemeTemplateCatalog.promptNames(shortlist, limit: shortlist.count) + let lines = MemeTemplateCatalog.promptLines(shortlist, limit: shortlist.count) let payload = MemeAI.rankedUserPayload( - description: self.description, templateNames: names) + description: self.description, templateNames: lines, + limit: MemeAI.candidateShortlist) do { - let raw = try await aiCall(MemeAI.rankedPrompt, payload, resolved) + let raw = try await self.askWithRetry( + aiCall, payload: payload, resolved: resolved, ticket: ticket) guard !self.isCancelled, self.state.accepts(ticket: ticket) else { return } switch MemeAI.parseRanked(raw, catalogNames: names) { @@ -358,18 +428,60 @@ final class MemeGeneratorModel: ObservableObject { } } + /// Run the LLM round-trip, retrying while the failure is "the server isn't + /// accepting connections yet" (v4). + /// + /// This is the second line of defence behind the readiness gate. Readiness covers + /// the cold start; this covers the gap readiness cannot see — llama-server can + /// pass a health check and still refuse the very next connection when it is + /// mid-restart (idle teardown, a model swap). Without it that shows up as the raw + /// "network error" the owner reported. + /// + /// The decision of WHETHER to retry lives in `MemeGenerateRetry` so it is pinned + /// by `swift test`; only the sleeping and the status writing happen here. A + /// non-transport failure (a real model error) is rethrown on the first attempt — + /// retrying it would just make the user wait longer for the same message. + private func askWithRetry( + _ aiCall: AICall, payload: String, + resolved: SummaryModelResolver.Resolved, ticket: Int + ) async throws -> String { + var attempt = 1 + while true { + do { + return try await aiCall(MemeAI.rankedPrompt, payload, resolved) + } catch { + guard MemeGenerateRetry.shouldRetry(error, attempt: attempt) else { throw error } + // A superseded or cancelled request must not keep retrying in the + // background — rethrow and let the caller's guards drop it. + guard !isCancelled, state.accepts(ticket: ticket) else { throw error } + + attempt += 1 + status = MemeGenerateRetry.retryingMessage(attempt: attempt) + let delay = MemeGenerateRetry.delay(beforeAttempt: attempt) + if delay > 0 { + try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000)) + } + guard !isCancelled, state.accepts(ticket: ticket) else { throw error } + status = MemeGenerationState.Phase.asking.statusText + } + } + } + /// A hard ceiling on one generate. /// /// v2 had none: an LLM call that never returned left the surface busy forever with /// no way back but closing the window. `finish` is ticket-guarded and idempotent, /// so this fires harmlessly when the work already completed. - private func startTimeout(ticket: Int) { + private func startTimeout( + ticket: Int, + after seconds: TimeInterval = MemeGenerationState.generateTimeout, + message: String = MemeGenerationState.timeoutMessage + ) { Task { [weak self] in - try? await Task.sleep( - nanoseconds: UInt64(MemeGenerationState.generateTimeout * 1_000_000_000)) + try? await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) guard let self, !self.isCancelled else { return } if self.state.finish(ticket: ticket) { - self.status = MemeGenerationState.timeoutMessage + self.status = message } } } @@ -462,12 +574,48 @@ final class MemeGeneratorModel: ObservableObject { didFallBack = false let ticket = state.begin(.downloading(templateName: template.name)) + // v4: an image download gets the SAME hard ceiling a generate does. v3 started + // this ticket with no timeout at all, so a download that never came back left + // "Downloading …" on screen forever — the owner's report #2. + startTimeout(ticket: ticket, + after: MemeGenerationState.downloadTimeout, + message: MemeGenerationState.downloadTimeoutMessage(template.name)) Task { [weak self] in await self?.renderTemplate(template, ticket: ticket, isNewGeneration: false) } } + /// Re-run the last template load that failed. The Retry affordance's action. + func retryTemplate() { + guard let template = failedTemplate else { return } + // Clear it first: `select` early-returns when the template is already + // selected, and a stale failure must not survive a successful retry. + failedTemplate = nil + imageFailed = false + // The failed template is NOT the selected one (the load never completed), so + // `select` will proceed — but drop any cached corpse just in case. + imageCache[template.id] = nil + select(template: template) + } + /// Load a template's image (cached, from disk or network) and render the boxes. + /// + /// ## v4 — every exit clears the phase + /// + /// The owner's "Downloading forever" was caused by the two BARE `return`s + /// this function used to have: a superseded/cancelled ticket returned without ever + /// touching the state machine. That is only safe if some other task owns the + /// phase, which is exactly the assumption v3's own doc comment identified as the + /// bug class — and it is false in the ordering the owner hit: closing the window + /// (`cancel()` sets `isCancelled`) and reopening it (`windowDidOpen` resets + /// `isCancelled = false`) left the machine parked in `.downloading` with no + /// in-flight task left to clear it, no timeout, and no Retry. + /// + /// Now every path — cache hit, fetch failure, decode failure, stale ticket, + /// cancellation — ends at a `finish` for its OWN ticket. `finish` is + /// ticket-guarded and idempotent, so finishing a superseded ticket is a harmless + /// no-op that cannot disturb newer work; what it CANNOT do any more is leave the + /// phase set with nobody responsible for it. private func renderTemplate( _ template: MemeTemplate, ticket: Int, isNewGeneration: Bool ) async { @@ -480,23 +628,46 @@ final class MemeGeneratorModel: ObservableObject { do { image = try await MemeTemplateService.fetchImage(template) } catch { - finish(ticket, status: - "Couldn't load the template image — \(Self.reason(error))") + // Honest error + a way out, for EVERY failure the fetch can produce: + // transport, non-2xx, an undecodable image, and the timeout. + noteImageFailure(template, ticket: ticket, error: error) + return + } + guard !isCancelled, state.accepts(ticket: ticket) else { + finish(ticket, status: status) return } - guard !isCancelled, state.accepts(ticket: ticket) else { return } imageCache[template.id] = image MemeLibraryStore.storeThumbnail(image, for: template.id) } - guard !isCancelled, state.accepts(ticket: ticket) else { return } + guard !isCancelled, state.accepts(ticket: ticket) else { + finish(ticket, status: status) + return + } baseImage = image selectedTemplate = template + imageFailed = false + failedTemplate = nil redraw() finish(ticket, status: statusLine(isNewGeneration: isNewGeneration)) } + /// Record a template-image failure: clear the phase, say what happened, and offer + /// Retry. One funnel so no image failure can reach the UI without all three. + private func noteImageFailure(_ template: MemeTemplate, ticket: Int, error: Error) { + guard state.accepts(ticket: ticket) else { + // Superseded — still end our own ticket rather than returning bare. + finish(ticket, status: status) + return + } + failedTemplate = template + imageFailed = true + finish(ticket, status: + "Couldn't load \(template.name) — \(Self.reason(error)) Press Retry, or pick another template.") + } + /// The line under the controls: honest about a fallback, quiet otherwise. private func statusLine(isNewGeneration: Bool) -> String { guard let selectedTemplate else { return "" } diff --git a/plugins/MemeGenerator/MemeGeneratorView.swift b/plugins/MemeGenerator/MemeGeneratorView.swift index 8f0eb0b..dfccfc1 100644 --- a/plugins/MemeGenerator/MemeGeneratorView.swift +++ b/plugins/MemeGenerator/MemeGeneratorView.swift @@ -249,7 +249,7 @@ struct MemeGeneratorView: View { HStack(alignment: .top, spacing: 6) { // A fallback is called out with a symbol as well as words — the // owner's complaint was that the substitution was invisible. - if model.didFallBack || model.catalogFailed { + if model.didFallBack || model.catalogFailed || model.imageFailed { Image(systemName: "exclamationmark.triangle") .foregroundStyle(.orange) } @@ -257,6 +257,14 @@ struct MemeGeneratorView: View { .font(.caption) .foregroundStyle(.secondary) .fixedSize(horizontal: false, vertical: true) + // A failed template image gets its OWN Retry (v4) — the catalog Retry + // above re-fetches the catalog, which does nothing for an image that + // failed to download. + if model.imageFailed { + Button("Retry") { model.retryTemplate() } + .buttonStyle(.link) + .font(.caption) + } Spacer() } } diff --git a/plugins/MemeGenerator/MemeGeneratorWindowController.swift b/plugins/MemeGenerator/MemeGeneratorWindowController.swift index 83bb03d..bb3a818 100644 --- a/plugins/MemeGenerator/MemeGeneratorWindowController.swift +++ b/plugins/MemeGenerator/MemeGeneratorWindowController.swift @@ -78,8 +78,15 @@ final class MemeGeneratorWindowController: NSWindowController, NSWindowDelegate, // only fires when Settings → Cleanup is itself set to the bundled // provider, so a plugin resolved to bundled would otherwise never warm. // Same MAK-53 split `ensureBundledLLMReady(provider:)` already makes. - warm: { resolved in - AppState.shared.warmLlamaServerIfPossible(provider: resolved.provider) + // + // v4: the completion carries REAL readiness — `ensureRunning` polls + // llama-server's `/health` and calls back only when it answers. v3 + // discarded that signal and had the plugin sleep a guessed 2.5s instead, + // which is why the first generates still hit a socket nothing was + // listening on. + warm: { resolved, ready in + AppState.shared.warmLlamaServerIfPossible( + provider: resolved.provider, completion: ready) }) } diff --git a/plugins/MemeGenerator/MemeTemplateService.swift b/plugins/MemeGenerator/MemeTemplateService.swift index d068667..8228134 100644 --- a/plugins/MemeGenerator/MemeTemplateService.swift +++ b/plugins/MemeGenerator/MemeTemplateService.swift @@ -133,13 +133,25 @@ enum MemeTemplateService { return image } - let (data, response) = try await session.data(from: url) + // A per-request ceiling on top of the session's, so one wedged image GET can't + // outlive the UI's own download timeout and land a result into a surface that + // has already recovered. + var request = URLRequest(url: url) + request.timeoutInterval = imageTimeout + + let (data, response) = try await session.data(for: request) try check(response, host: url.host ?? "the template host") guard let image = NSImage(data: data) else { throw ServiceError.undecodableImage } return image } + /// The ceiling on one template-image GET. Deliberately shorter than + /// `MemeGenerationState.downloadTimeout` so the transport fails FIRST and the user + /// gets a real reason ("the request timed out") rather than the UI's generic + /// give-up message. + static let imageTimeout: TimeInterval = 20 + private static func check(_ response: URLResponse, host: String) throws { guard let http = response as? HTTPURLResponse else { return } guard (200...299).contains(http.statusCode) else { diff --git a/scripts/appstate-loc-budget.txt b/scripts/appstate-loc-budget.txt index 23595f8..6ec0b28 100644 --- a/scripts/appstate-loc-budget.txt +++ b/scripts/appstate-loc-budget.txt @@ -1 +1 @@ -7051 +7036 From 36eec3d54f7f66d4f398070a48feb177ae2a02be Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Mon, 3 Aug 2026 12:41:05 -0700 Subject: [PATCH 08/15] spike(meme-plugin): reopen-poisoned downloads, recyclable session, New meme MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two live-soak defects from leaving the app running for a day, plus a manifest-declared menu shortcut. ## 1. Downloads stop working after ~a day of uptime Two independent causes, both requiring uptime + a close to appear. ROOT CAUSE A (primary) — the window lifecycle was unbalanced. `PluginHost.open()` caches a plugin's window controller for the app's lifetime and reuses it, so `MemeGeneratorWindowController.init` — the only caller of `model.windowDidOpen()` — ran EXACTLY ONCE. But `windowWillClose` ran on every close, and it calls `model.cancel()`, which sets `isCancelled = true`. Only `windowDidOpen` clears that flag. So the first time the user closed the window, every subsequent async result was dropped by `guard !isCancelled` — permanently, for the rest of the launch. Downloads did not hang; they completed and were thrown away. Fix: a `PluginWindowLifecycle` seam. `PluginHost` tells a cached controller it is being shown again, and the meme controller re-runs `windowDidOpen()`. Setup and teardown are now balanced however many times the window opens. ROOT CAUSE B — the URLSession was unreplaceable, which made Retry a no-op. `MemeTemplateService.session` was a process-lifetime `static let`. A URLSession is a connection pool, and a pooled connection can outlive its validity across a sleep/wake, a network change, or an expired captive-portal lease — after which every request through it fails identically until relaunch. Retry re-ran the request through that same session, so it inherited exactly the broken pool: a no-op by construction. Fix: the session is rebuildable. A transport-shaped failure invalidates it (`invalidateAndCancel`) and the next request builds a fresh pool. Both Retry paths (image and catalog) invalidate first, so a retry is always a genuinely fresh session AND a fresh `URLRequest` rather than a replay. Which failures count is the pure, tested `MemeGenerationState.isTransportFailure` — narrow on purpose: a 404 or an undecodable image keeps the pool. AUDITED AND CLEARED — the catalog cache TTL. An expired cache resolves to `.useCacheAndRefresh` (show it, refresh behind it), never to a wedge, and a background refresh failure is silent only while templates are on screen. A week-old cache still serves. Pinned with clock-injected tests at both TTL boundaries so this stays true. ## 2. No way to start from scratch "New meme" (`arrow.counterclockwise`, ⌘N) clears the description, caption boxes, candidate strip, selected template, rendered meme, search text, and every error — and calls `state.reset()` FIRST, so in-flight work is refused by the existing ticket guard instead of landing on the cleared surface. The clearing is a pure `MemeComposition` value whose `reset()` returns `.empty` wholesale, so a field added later is reset by construction rather than by remembering a line. The test asserts a fully-populated composition comes back exactly equal to `.empty` — the property a partial reset fails. Deliberately survives: the catalog (a corpus, not this meme — clearing it would make New meme a network round-trip) and the image cache. ## 3. ⌘M opens the Meme Generator Manifests now carry an optional `keyEquivalent`. The plugin ASKS; the host DECIDES (`PluginKeyEquivalent`), because only the host can see the whole menu — a plugin must not be able to shadow ⌘Q. Collisions resolve against the app's reserved set then by list order, first-wins, matching the precedence `PluginDiscovery` already uses. A refusal is silent and costs only the shortcut, never the row. The Plugins pane shows the GRANTED shortcut, resolved through the same pass, so it can't advertise a refused key. Decode is forward-compatible; a malformed shortcut is reported by `validate()` but is not fatal. Gates: swift test 2520 passing (2485 baseline + 35 new); ratchet 7036/7036; PLUGINS=1 ./build.sh, plain ./build.sh, and lean all compile with no new warnings in touched files. Co-Authored-By: Claude --- OpenWhisp/AppMain.swift | 10 +- OpenWhisp/Services/MemeGenerationState.swift | 154 ++++++ OpenWhisp/Services/PluginManifest.swift | 145 +++++- OpenWhisp/Services/PluginRegistry.swift | 8 +- OpenWhisp/Views/PluginHost.swift | 29 ++ OpenWhisp/Views/Settings/PluginsPane.swift | 21 + .../MemeRecoveryTests.swift | 449 ++++++++++++++++++ .../MemeGenerator/MemeGeneratorModel.swift | 90 +++- plugins/MemeGenerator/MemeGeneratorView.swift | 24 +- .../MemeGeneratorWindowController.swift | 18 +- .../MemeGenerator/MemeTemplateService.swift | 129 ++++- plugins/MemeGenerator/manifest.json | 5 +- 12 files changed, 1058 insertions(+), 24 deletions(-) create mode 100644 Tests/OpenWhispCoreTests/MemeRecoveryTests.swift diff --git a/OpenWhisp/AppMain.swift b/OpenWhisp/AppMain.swift index 2a3abfb..5f35d99 100644 --- a/OpenWhisp/AppMain.swift +++ b/OpenWhisp/AppMain.swift @@ -390,11 +390,19 @@ class OpenWhispApp: NSObject, NSApplicationDelegate { pluginsItem.image = NSImage( systemSymbolName: "puzzlepiece.extension", accessibilityDescription: nil) let submenu = NSMenu() + // Shortcuts are DECLARED by each manifest and GRANTED by the host (v5): + // the plugin can't see the app's own menu, so it asks for a key and + // `PluginKeyEquivalent` resolves it against what's already bound — the + // app's reserved set first, then earlier plugins in this same list. A + // refusal is silent; the row still opens on a click. + let shortcuts = PluginKeyEquivalent.assign( + requests: activePlugins.map { ($0.id, $0.manifest.keyEquivalent) }) for plugin in activePlugins { let item = menuItem( plugin.manifest.name, symbol: plugin.manifest.symbol, - action: #selector(openPlugin(_:))) + action: #selector(openPlugin(_:)), + keyEquivalent: shortcuts[plugin.id] ?? "") item.representedObject = plugin.id item.target = self submenu.addItem(item) diff --git a/OpenWhisp/Services/MemeGenerationState.swift b/OpenWhisp/Services/MemeGenerationState.swift index 273f9da..dee2ed7 100644 --- a/OpenWhisp/Services/MemeGenerationState.swift +++ b/OpenWhisp/Services/MemeGenerationState.swift @@ -180,6 +180,46 @@ public struct MemeGenerationState: Equatable, Sendable { /// Whether a result carrying `incoming` is still wanted. public func accepts(ticket incoming: Int) -> Bool { incoming == ticket } + // MARK: - Transport health (v5) + + /// Whether a failure means the shared HTTP session should be THROWN AWAY before + /// the next attempt. + /// + /// The v5 report was "downloads stop working after about a day of uptime, and + /// Retry does nothing". A `URLSession` is a connection pool, and a pooled + /// connection can outlive its own validity — the Mac sleeps and wakes on a + /// different network, a captive portal expires, an interface changes — after + /// which every request handed to that session fails the same way, forever. + /// Retrying on the SAME session is then a no-op by construction: the retry + /// inherits exactly the pool that is broken. + /// + /// So a transport-shaped failure invalidates the session and the next request + /// builds a new one. The predicate is deliberately narrow: an HTTP 404 or an + /// undecodable image says nothing about the transport, and tearing the pool down + /// for those would just throw away working connections. + /// + /// Pure and matched on URL-loading error CODES rather than message text — the + /// same reason `MemeGenerateRetry.isNotReadyYet` does: the text is localized, so + /// keying on English would silently stop recycling on a non-English Mac. + public static func isTransportFailure(_ error: Error) -> Bool { + let ns = error as NSError + guard ns.domain == NSURLErrorDomain else { return false } + switch ns.code { + case NSURLErrorTimedOut, + NSURLErrorCannotConnectToHost, + NSURLErrorCannotFindHost, + NSURLErrorDNSLookupFailed, + NSURLErrorNetworkConnectionLost, + NSURLErrorNotConnectedToInternet, + NSURLErrorInternationalRoamingOff, + NSURLErrorSecureConnectionFailed, + NSURLErrorResourceUnavailable: + return true + default: + return false + } + } + // MARK: - Timeout /// The hard ceiling on one generate round-trip. @@ -215,6 +255,120 @@ public struct MemeGenerationState: Equatable, Sendable { } } +/// The Meme Generator's composition — everything "this meme" consists of (v5). +/// +/// ## Why this is a type +/// +/// The owner asked for a way to start from scratch, and "start from scratch" is only +/// trustworthy if it is TOTAL: a New-meme button that clears the canvas but leaves the +/// prompt text, or clears the candidate strip but leaves a stale error and a Retry +/// pointing at a template the user has moved on from, is worse than no button — it +/// looks reset while carrying the previous meme's state forward. +/// +/// Spreading that clearing over a dozen assignments in an `@MainActor` AppKit class +/// makes it untestable and, worse, easy to under-do: adding a `@Published` next month +/// and forgetting one line is a silent regression nothing catches. Gathering the +/// resettable fields into ONE Foundation-only value means `reset()` is a single +/// expression, `swift test` can assert that EVERY field returned to its initial value, +/// and the model's job shrinks to projecting this into its published properties. +/// +/// The image cache is deliberately NOT part of this: it is a performance detail keyed +/// by template id, holds nothing about the current meme, and discarding it would make +/// New-meme re-download templates the user already has. +public struct MemeComposition: Equatable, Sendable { + + /// What the user described / dictated. + public var description: String + + /// The caption boxes. + public var boxes: [MemeCaptionLayout.CaptionBox] + + /// The box the editor panel is editing. + public var selectedBoxID: UUID? + + /// The ranked candidate strip. + public var candidateIDs: [String] + + /// The rendered template's id, if one is selected. + public var selectedTemplateID: String? + + /// The status / error line. + public var status: String + + /// Whether the candidate strip is a lexical fallback rather than the model's picks. + public var didFallBack: Bool + + /// As above, for the CURRENT strip. + public var candidatesAreFallback: Bool + + /// Whether the catalog failed to load with nothing cached. + public var catalogFailed: Bool + + /// Whether a template IMAGE failed to load. + public var imageFailed: Bool + + /// The id of the template whose image failed, so Retry knows what to re-fetch. + public var failedTemplateID: String? + + /// Whether a meme is currently rendered. + public var hasMeme: Bool + + public init( + description: String = "", + boxes: [MemeCaptionLayout.CaptionBox] = [], + selectedBoxID: UUID? = nil, + candidateIDs: [String] = [], + selectedTemplateID: String? = nil, + status: String = "", + didFallBack: Bool = false, + candidatesAreFallback: Bool = false, + catalogFailed: Bool = false, + imageFailed: Bool = false, + failedTemplateID: String? = nil, + hasMeme: Bool = false + ) { + self.description = description + self.boxes = boxes + self.selectedBoxID = selectedBoxID + self.candidateIDs = candidateIDs + self.selectedTemplateID = selectedTemplateID + self.status = status + self.didFallBack = didFallBack + self.candidatesAreFallback = candidatesAreFallback + self.catalogFailed = catalogFailed + self.imageFailed = imageFailed + self.failedTemplateID = failedTemplateID + self.hasMeme = hasMeme + } + + /// The state a freshly-opened, never-used window is in. + /// + /// Note the CATALOG is not here. Resetting the meme must not throw away the ~300 + /// templates the user is browsing: they are a corpus, not part of this meme, and + /// re-fetching them would turn New-meme into a network round-trip and a spinner. + public static let empty = MemeComposition() + + /// Everything the user was making, cleared. + /// + /// Total by construction — it returns `.empty` rather than assigning field by + /// field, so a field added to this type is reset automatically instead of being + /// forgotten. `MemeCompositionResetTests` asserts a fully-populated composition + /// comes back exactly equal to `.empty`. + public mutating func reset() { self = .empty } + + /// Whether there is anything to clear — drives whether New meme is offered. + /// + /// A New-meme button that is live on an untouched window is a control that + /// visibly does nothing, so it is disabled until the surface actually holds + /// something. An error alone counts: clearing a failed state is exactly the + /// moment the user most wants a way back to a clean sheet. + public var isEmpty: Bool { self == .empty } + + /// The hint shown on the empty canvas — an INVITATION rather than a blank pane. + public static let emptyHint = + "Describe the meme out loud, or pick a template on the left." +} + /// When a failed generate request is worth retrying rather than reporting (v4). /// /// ## Why a policy type diff --git a/OpenWhisp/Services/PluginManifest.swift b/OpenWhisp/Services/PluginManifest.swift index 3fb6472..4c70da9 100644 --- a/OpenWhisp/Services/PluginManifest.swift +++ b/OpenWhisp/Services/PluginManifest.swift @@ -55,6 +55,24 @@ public struct PluginManifest: Codable, Equatable, Sendable, Identifiable { /// the plugin's own manifest (see the PR's security notes). public let networkHosts: [String] + /// The single character this plugin would like as its ⌘-shortcut in the menu bar + /// (v5), e.g. `"m"` → ⌘M opens the Meme Generator's window. + /// + /// OPTIONAL, and a REQUEST rather than a grant. The host is the only thing that + /// knows the app's own menu shortcuts, so it — not the plugin — decides whether + /// the request is honoured (`PluginKeyEquivalent.assignable`). A plugin that asks + /// for ⌘Q does not get to shadow Quit. + /// + /// It lives on the MANIFEST rather than being hardcoded next to the one plugin + /// that wants it, because the manifest is already the place a plugin declares how + /// the host should present it (name, symbol, disclosure). That is the MAK-100 + /// "manifests carry host metadata" direction, and it means a second plugin needs + /// no change in `AppMain` at all. + /// + /// Decoded with a default so every manifest written before this field existed — + /// including any already sitting in the user's plugins folder — still decodes. + public let keyEquivalent: String? + public init( id: String, name: String, @@ -62,7 +80,8 @@ public struct PluginManifest: Codable, Equatable, Sendable, Identifiable { summary: String, symbol: String, entry: PluginEntryKind, - networkHosts: [String] = [] + networkHosts: [String] = [], + keyEquivalent: String? = nil ) { self.id = id self.name = name @@ -71,6 +90,30 @@ public struct PluginManifest: Codable, Equatable, Sendable, Identifiable { self.symbol = symbol self.entry = entry self.networkHosts = networkHosts + self.keyEquivalent = keyEquivalent + } + + /// Forward-compatible decode: `networkHosts` and `keyEquivalent` are optional in + /// the JSON, so an older manifest (and a hand-written one) decodes rather than + /// failing the whole plugin out of the list over a missing key. + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + id = try container.decode(String.self, forKey: .id) + name = try container.decode(String.self, forKey: .name) + version = try container.decodeIfPresent(String.self, forKey: .version) ?? "0.0.0" + summary = try container.decodeIfPresent(String.self, forKey: .summary) ?? "" + symbol = try container.decode(String.self, forKey: .symbol) + entry = try container.decodeIfPresent(PluginEntryKind.self, forKey: .entry) ?? .builtIn + networkHosts = try container.decodeIfPresent([String].self, forKey: .networkHosts) ?? [] + keyEquivalent = try container.decodeIfPresent(String.self, forKey: .keyEquivalent) + } + + /// The shortcut as it should be DISPLAYED, e.g. `"⌘M"`, or nil when this manifest + /// asks for none / asks for something unusable. Kept here so the Plugins pane and + /// any future surface render it identically, and so `swift test` pins it. + public var keyEquivalentDisplay: String? { + guard let key = PluginKeyEquivalent.normalized(keyEquivalent) else { return nil } + return "⌘\(key.uppercased())" } /// Whether this plugin uses the network at all — drives the pane's disclosure row. @@ -91,6 +134,8 @@ public struct PluginManifest: Codable, Equatable, Sendable, Identifiable { case invalidID(String) case emptyName case emptySymbol + /// The manifest asked for a shortcut that isn't a single character (v5). + case invalidKeyEquivalent(String) } /// Characters allowed in an id: lowercase alphanumerics plus `-` and `.`. @@ -111,10 +156,106 @@ public struct PluginManifest: Codable, Equatable, Sendable, Identifiable { if id.allSatisfy({ $0 == "." }) { return .invalidID(id) } if name.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { return .emptyName } if symbol.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { return .emptySymbol } + // A malformed shortcut is reported but is NOT fatal — see `isValid`. Losing a + // whole working plugin over a cosmetic field would be a bad trade, and the id, + // name, and symbol are the fields the host genuinely cannot proceed without. + if let requested = keyEquivalent, + PluginKeyEquivalent.normalized(requested) == nil { + return .invalidKeyEquivalent(requested) + } return nil } - public var isValid: Bool { validate() == nil } + /// Whether the host can LIST and run this manifest. + /// + /// Deliberately more permissive than `validate() == nil`: only the structural + /// failures disqualify a plugin. An unusable `keyEquivalent` costs the plugin its + /// shortcut (`keyEquivalentDisplay` returns nil, and the menu assigns nothing) and + /// nothing else. + public var isValid: Bool { + switch validate() { + case nil, .invalidKeyEquivalent: return true + default: return false + } + } +} + +/// Who gets a ⌘-shortcut in the menu bar, and who is refused (v5). +/// +/// A plugin ASKS for a shortcut in its manifest; this decides. The host owns the +/// keyboard because only the host can see the whole menu — a plugin cannot know that +/// ⌘S is the Scratchpad or that ⌘, is Settings, and a plugin that could silently +/// shadow Quit would be a genuine hazard rather than a papercut. +/// +/// Pure and Foundation-only so every rule here is pinned by `swift test` rather than +/// discovered by a user whose ⌘Q stopped quitting. +public enum PluginKeyEquivalent { + + /// The shortcuts the app itself already owns, which no plugin may take. + /// + /// Sourced from `AppMain`'s ACTUAL menu construction, and nothing beyond it: + /// Quit (q), Scratchpad (s), Settings (,), Copy-last (c), and the Edit-menu verbs + /// cut/paste/select-all/undo (x, v, a, z). The Edit ones matter most — PR #242 was + /// the bug where those shortcuts were MISSING app-wide, and letting a plugin + /// re-take one would reintroduce it for the price of a line in a JSON file. + /// + /// Kept to what the app really binds rather than padded with plausible-looking + /// extras: every speculative entry here is a shortcut silently denied to a plugin + /// for no reason. ⌘M is free precisely because this app has no Window menu. + public static let reserved: Set = [ + "q", "s", ",", "c", "x", "v", "a", "z", + ] + + /// Normalize a requested shortcut, or nil when it isn't usable. + /// + /// Usable means: exactly ONE character after trimming, and a letter, digit, or + /// `,`. Lowercased, because `NSMenuItem` treats an uppercase key equivalent as + /// ⇧⌘ — a manifest saying `"M"` means ⌘M, not ⇧⌘M, and silently promoting it + /// would hand out a different shortcut than the one declared. + public static func normalized(_ requested: String?) -> String? { + guard let requested else { return nil } + let trimmed = requested.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + guard trimmed.count == 1, let character = trimmed.first else { return nil } + guard character.isLetter || character.isNumber || character == "," else { return nil } + return trimmed + } + + /// The shortcut a plugin may actually be given, or nil to assign none. + /// + /// `taken` carries the shortcuts already handed out in THIS menu build — the + /// app's reserved set plus anything an earlier plugin in the list already got — + /// so two plugins both asking for `"m"` resolve deterministically by list order + /// instead of both rendering ⌘M and one of them silently never firing. + /// + /// A refusal is SILENT by design: the plugin still appears in the menu and still + /// opens by clicking. Dropping the whole row, or surfacing an error to the user + /// about a collision they did not cause and cannot fix, would both be worse. + public static func assignable( + _ requested: String?, taken: Set + ) -> String? { + guard let key = normalized(requested) else { return nil } + guard !reserved.contains(key), !taken.contains(key) else { return nil } + return key + } + + /// Resolve shortcuts for a whole ordered menu in one pass. + /// + /// Returns plugin id → assigned key for the plugins that got one. Earlier entries + /// win, matching the list order the menu renders in — the same first-wins rule + /// `PluginDiscovery` already uses for id collisions, so the host has ONE + /// precedence story rather than two. + public static func assign( + requests: [(id: String, keyEquivalent: String?)] + ) -> [String: String] { + var taken = reserved + var assigned: [String: String] = [:] + for request in requests { + guard let key = assignable(request.keyEquivalent, taken: taken) else { continue } + assigned[request.id] = key + taken.insert(key) + } + return assigned + } } /// How a plugin's code is expected to be executed by the host. diff --git a/OpenWhisp/Services/PluginRegistry.swift b/OpenWhisp/Services/PluginRegistry.swift index 5d31e11..d9bd7fd 100644 --- a/OpenWhisp/Services/PluginRegistry.swift +++ b/OpenWhisp/Services/PluginRegistry.swift @@ -39,7 +39,7 @@ public enum PluginRegistry { public static let memeGenerator = PluginManifest( id: "meme-generator", name: "Meme Generator", - version: "0.3.0", + version: "0.5.0", summary: "Dictate a meme description — the AI picks a template and writes the captions.", symbol: "photo.badge.plus", entry: .builtIn, @@ -49,6 +49,10 @@ public enum PluginRegistry { // not use it. Templates the user imports themselves need no network at all. networkHosts: [ "api.imgflip.com", "i.imgflip.com", "api.memegen.link", - ] + ], + // ⌘M opens the window from the menu bar, the way ⌘S opens the Scratchpad. + // Declared here rather than hardcoded in `AppMain` so a second plugin needs no + // host change — the host resolves collisions (`PluginKeyEquivalent`). + keyEquivalent: "m" ) } diff --git a/OpenWhisp/Views/PluginHost.swift b/OpenWhisp/Views/PluginHost.swift index 58c4e72..a0c832e 100644 --- a/OpenWhisp/Views/PluginHost.swift +++ b/OpenWhisp/Views/PluginHost.swift @@ -109,6 +109,17 @@ final class PluginHost: ObservableObject { if let existing = windows[pluginID] { existing.window?.makeKeyAndOrderFront(nil) NSApp.activate(ignoringOtherApps: true) + // v5: tell the controller it is being SHOWN AGAIN. + // + // This is the fix for "template downloads stop working after a day". The + // controller is cached for the app's lifetime and reused on every open, so + // a plugin that did its open-time work in `init` did it exactly ONCE. The + // Meme Generator closes over that: `windowWillClose` sets its model's + // `isCancelled = true` to stop a late result touching a dead window, and + // only `windowDidOpen` clears it — which, reached from `init` alone, never + // ran again. Every download after the first close therefore returned + // through a guard that dropped it, silently and permanently. + (existing as? PluginWindowLifecycle)?.pluginWindowWillShow() return } @@ -164,6 +175,24 @@ final class PluginHost: ObservableObject { } } +/// A plugin window that needs to know when it is shown again (v5). +/// +/// Plugin window controllers are created once and REUSED — `PluginHost` caches them so +/// reopening restores the user's window rather than throwing their work away. That +/// makes `init` the wrong place for anything that must be true on every open, and the +/// wrong-place-ness is invisible until a plugin also does teardown on close: the +/// teardown then runs N times against exactly one setup. +/// +/// This seam is the missing half. A controller adopting it gets told about every +/// subsequent show, so "prepare to be used" and "stop touching me" stay balanced no +/// matter how many times the user opens and closes the window. +@MainActor +protocol PluginWindowLifecycle: AnyObject { + /// The window is about to be brought forward again after having been created. + /// Not called for the first show — `init` already covers that. + func pluginWindowWillShow() +} + /// A plugin window that can receive dictation when it is the key window. /// /// This is the seam the Scratchpad hardcodes on AppState, generalized just enough diff --git a/OpenWhisp/Views/Settings/PluginsPane.swift b/OpenWhisp/Views/Settings/PluginsPane.swift index 4e8385a..27ee751 100644 --- a/OpenWhisp/Views/Settings/PluginsPane.swift +++ b/OpenWhisp/Views/Settings/PluginsPane.swift @@ -77,7 +77,15 @@ struct PluginsPane: View { SettingsCallout(.info, disclosure) } + // Where to find it, and the shortcut that opens it (v5). A shortcut the + // user is never told about may as well not exist, and the menu row it + // appears on is two clicks away inside a submenu. if host.isEnabled(plugin.id), plugin.isRunnable { + LabeledContent("Open from the menu bar") { + Text(shortcutSubtitle(for: plugin)) + .foregroundStyle(.secondary) + } + configuration(for: plugin) } } header: { @@ -89,6 +97,19 @@ struct PluginsPane: View { } } + /// "Plugins › Meme Generator (⌘M)" — the GRANTED shortcut, not the requested one. + /// + /// Resolved through the same `PluginKeyEquivalent.assign` pass the menu itself + /// uses, over the same active-plugin list, so the pane can never advertise a + /// shortcut the menu refused on a collision. Advertising a key that does nothing + /// would be worse than saying nothing at all. + private func shortcutSubtitle(for plugin: PluginDiscovery.Discovered) -> String { + let granted = PluginKeyEquivalent.assign( + requests: host.activePlugins.map { ($0.id, $0.manifest.keyEquivalent) }) + guard let key = granted[plugin.id] else { return "Plugins › \(plugin.manifest.name)" } + return "Plugins › \(plugin.manifest.name) ⌘\(key.uppercased())" + } + /// A plugin's own configuration surface. /// /// In the spike this is a fixed switch on the plugin id — the honest shape for a diff --git a/Tests/OpenWhispCoreTests/MemeRecoveryTests.swift b/Tests/OpenWhispCoreTests/MemeRecoveryTests.swift new file mode 100644 index 0000000..08c2e5f --- /dev/null +++ b/Tests/OpenWhispCoreTests/MemeRecoveryTests.swift @@ -0,0 +1,449 @@ +import XCTest +@testable import OpenWhispCore + +/// The v5 live-soak defects: template downloads that stop working after ~a day of app +/// uptime, and the absence of any way to start a meme from scratch. +/// +/// Both reports came from leaving the app RUNNING for a day, which is the one thing a +/// unit test cannot literally do. What it CAN do is pin the mechanisms behind each +/// report — the cache's staleness arithmetic against an injected clock, the predicate +/// that decides a connection pool is suspect, and the totality of the reset — so the +/// long-idle behaviour is a consequence of tested rules rather than of a hope. +final class MemeRecoveryTests: XCTestCase { + + // MARK: - Cache staleness against an injected clock + // + // Suspect (a) from the report: does an expired cache plus a failing refresh wedge + // the catalog? These pin the arithmetic at the exact boundaries a day-long uptime + // walks through, using an injected `now` rather than a real clock. + + private let templates = [ + MemeTemplate(id: "imgflip:1", name: "Drake Hotline Bling", url: "u1", + width: 1200, height: 1200, source: .imgflip) + ] + + private func cached(ageSeconds: TimeInterval, now: Date) -> MemeCatalogCache.Cached { + MemeCatalogCache.Cached( + fetchedAt: now.addingTimeInterval(-ageSeconds), templates: templates) + } + + /// A day and a second of uptime makes the cache stale — but stale means SHOW IT + /// AND REFRESH, never "stop working". This is the decision the owner's day-long + /// session crosses, and the one a wedged-catalog theory would have to blame. + func testCacheOneSecondPastTheTTLIsShownAndRefreshedRatherThanDiscarded() { + let now = Date(timeIntervalSince1970: 2_000_000) + let decision = MemeCatalogCache.decide( + cached: cached(ageSeconds: MemeCatalogCache.maxAge + 1, now: now), now: now) + XCTAssertEqual(decision, .useCacheAndRefresh) + } + + /// One second BEFORE the TTL is still fresh — the other side of the same boundary, + /// so an off-by-one can't silently turn every open into a fetch. + func testCacheOneSecondBeforeTheTTLStillAvoidsTheNetwork() { + let now = Date(timeIntervalSince1970: 2_000_000) + let decision = MemeCatalogCache.decide( + cached: cached(ageSeconds: MemeCatalogCache.maxAge - 1, now: now), now: now) + XCTAssertEqual(decision, .useCache) + } + + /// A WEEK of uptime is still only "stale", not "unusable". The catalog must never + /// degrade into `.fetchNow` with age alone: that would make an offline user's + /// working plugin start demanding the network, which is the failure mode the disk + /// cache exists to prevent. + func testAWeekOldCacheIsStillServedRatherThanForcingAFetch() { + let now = Date(timeIntervalSince1970: 2_000_000) + let decision = MemeCatalogCache.decide( + cached: cached(ageSeconds: MemeCatalogCache.maxAge * 7, now: now), now: now) + XCTAssertEqual(decision, .useCacheAndRefresh) + XCTAssertNotEqual(decision, .fetchNow) + } + + /// A refresh that FAILS while a stale cache is on screen stays silent. + /// + /// This is the "is refresh failure silent in a way that becomes a permanent + /// download failure?" question from the report, answered directly: silence here is + /// correct and deliberate, because the user is still looking at a usable corpus. + /// The permanence came from elsewhere (the session and the lifecycle), not here. + func testAFailedRefreshBehindAStaleCacheStaysSilent() { + XCTAssertNil(MemeCatalogCache.refreshFailureMessage( + hasCachedTemplates: true, reason: "The request timed out.")) + } + + /// ...but with NOTHING on screen the same failure must name the Retry. A silent + /// failure there is indistinguishable from a hang, which is exactly what the owner + /// reported seeing. + func testAFailedRefreshWithNothingCachedSurfacesTheErrorAndTheRetry() { + let message = MemeCatalogCache.refreshFailureMessage( + hasCachedTemplates: false, reason: "The request timed out.") + XCTAssertNotNil(message) + XCTAssertTrue(message!.contains("Retry")) + XCTAssertTrue(message!.contains("The request timed out.")) + } + + // MARK: - Transport health: which failures throw the connection pool away + // + // Suspect (b): the process-lifetime URLSession. A pooled connection can outlive + // its validity across a sleep/wake, after which every request through that session + // fails identically — forever. These pin WHICH errors mean "the pool is suspect". + + private func urlError(_ code: Int) -> NSError { + NSError(domain: NSURLErrorDomain, code: code) + } + + /// The sleep/wake signatures. Each of these is a real thing a Mac that has been + /// awake for a day produces, and each must recycle the session rather than being + /// retried forever through the same broken pool. + func testSleepWakeTransportFailuresRecycleTheSession() { + for code in [ + NSURLErrorTimedOut, + NSURLErrorCannotConnectToHost, + NSURLErrorCannotFindHost, + NSURLErrorDNSLookupFailed, + NSURLErrorNetworkConnectionLost, + NSURLErrorNotConnectedToInternet, + NSURLErrorSecureConnectionFailed, + ] { + XCTAssertTrue( + MemeGenerationState.isTransportFailure(urlError(code)), + "URL error \(code) should be treated as a suspect transport") + } + } + + /// A 404, a cancelled request, or an unparseable body says NOTHING about the + /// transport. Recycling on these would throw away healthy connections on every + /// missing template — a fix that costs more than the bug. + func testNonTransportFailuresKeepTheSession() { + XCTAssertFalse(MemeGenerationState.isTransportFailure(urlError(NSURLErrorCancelled))) + XCTAssertFalse(MemeGenerationState.isTransportFailure( + urlError(NSURLErrorBadServerResponse))) + XCTAssertFalse(MemeGenerationState.isTransportFailure( + NSError(domain: NSCocoaErrorDomain, code: 4))) + } + + /// Errors from another domain are not transport verdicts. Matched on CODE within + /// `NSURLErrorDomain` rather than on message text, because the text is localized — + /// a Russian-locale Mac must recycle exactly like an English one. + func testForeignErrorDomainsAreNotTreatedAsTransportFailures() { + XCTAssertFalse(MemeGenerationState.isTransportFailure( + NSError(domain: "MemeGenerator", code: NSURLErrorTimedOut))) + } + + // MARK: - The state machine across a long idle + // + // Suspect (c): the download ceiling and `windowDidOpen`'s reset. + + /// A `.downloading` phase stranded by a close/reopen is cleared unconditionally. + /// + /// This is the state half of the owner's report. `reset` must not be + /// ticket-guarded: the whole point is that the stranded phase's owner is GONE, so + /// there is no correct ticket left to present. + func testResetClearsAStrandedDownloadingPhaseWithoutATicket() { + var state = MemeGenerationState() + _ = state.begin(.downloading(templateName: "Drake Hotline Bling")) + XCTAssertTrue(state.isGenerating) + + state.reset() + + XCTAssertEqual(state.phase, .idle) + XCTAssertFalse(state.isGenerating) + XCTAssertTrue(state.canGenerate) + } + + /// The work stranded by a reset can never come back and re-wedge the surface: its + /// ticket is dead, so its `finish` is a no-op. Without this a download that + /// returns after the reset would write its status over a fresh window. + func testWorkStrandedByAResetCannotFinishOverTheFreshState() { + var state = MemeGenerationState() + let stranded = state.begin(.downloading(templateName: "Drake Hotline Bling")) + + state.reset() + let newWork = state.begin(.asking) + + XCTAssertFalse(state.finish(ticket: stranded), + "a reset-stranded ticket must not be able to finish anything") + XCTAssertEqual(state.phase, .asking, "the stranded finish must not clear newer work") + XCTAssertTrue(state.finish(ticket: newWork)) + } + + /// A download that fails and is then retried recovers: the failure clears the + /// phase (so the surface isn't stuck), and the retry takes a NEW ticket that can + /// legitimately complete. This is the failure-then-recovery ordering the report + /// asks to be pinned. + func testADownloadFailureThenRetryRecoversTheSurface() { + var state = MemeGenerationState() + + // Attempt 1: begins, then fails. + let first = state.begin(.downloading(templateName: "Drake Hotline Bling")) + XCTAssertTrue(state.finish(ticket: first), "the failure path must clear the phase") + XCTAssertFalse(state.isGenerating) + XCTAssertTrue(state.canGenerate, "a failed download must leave the surface usable") + + // Attempt 2 (Retry): a fresh ticket, which completes. + let retry = state.begin(.downloading(templateName: "Drake Hotline Bling")) + XCTAssertNotEqual(retry, first, "Retry must not reuse the failed attempt's ticket") + XCTAssertTrue(state.finish(ticket: retry)) + XCTAssertEqual(state.phase, .idle) + } + + /// The download ceiling is finite and far tighter than the generate one. An image + /// is a few hundred KB; anything past this is a hang, not slowness — and a finite + /// ceiling is what turns "Downloading… forever" into an error with a Retry. + func testTheDownloadCeilingIsFiniteAndTighterThanTheGenerateCeiling() { + XCTAssertLessThan( + MemeGenerationState.downloadTimeout, MemeGenerationState.generateTimeout) + XCTAssertGreaterThan(MemeGenerationState.downloadTimeout, 0) + XCTAssertTrue( + MemeGenerationState.downloadTimeoutMessage("Drake").contains("Retry"), + "the timeout message must point at the way out") + } + + /// Template selection stays live through a failed download. The owner's earlier + /// report was a surface frozen by a stuck flag; a v5 download failure must not + /// reintroduce it by any other route. + func testTemplateSelectionSurvivesAFailedDownload() { + var state = MemeGenerationState() + let ticket = state.begin(.downloading(templateName: "Drake Hotline Bling")) + XCTAssertTrue(state.canSelectTemplate) + _ = state.finish(ticket: ticket) + XCTAssertTrue(state.canSelectTemplate) + } +} + +/// "Start from scratch" — the v5 reset, proved TOTAL. +/// +/// The value of testing `MemeComposition` rather than the AppKit model is that this +/// asserts the property that actually matters: after a reset, EVERY field equals its +/// initial value. A test that checked six named fields would pass while a seventh +/// silently survived, which is the exact bug a "New meme" button must not have. +final class MemeCompositionResetTests: XCTestCase { + + /// A composition with every single field dirtied, so nothing is reset by accident. + private func fullyPopulated() -> MemeComposition { + let boxes = MemeCaptionLayout.seedBoxes(topText: "when the", bottomText: "build is green") + return MemeComposition( + description: "a meme about flaky tests", + boxes: boxes, + selectedBoxID: boxes.first?.id, + candidateIDs: ["imgflip:1", "imgflip:2", "memegen:drake"], + selectedTemplateID: "imgflip:1", + status: "Couldn't load Drake Hotline Bling — The request timed out. Press Retry.", + didFallBack: true, + candidatesAreFallback: true, + catalogFailed: true, + imageFailed: true, + failedTemplateID: "imgflip:1", + hasMeme: true) + } + + /// Nothing survives. One equality against `.empty` covers every field there is — + /// including any added later, which is the whole reason this is a value type. + func testResetReturnsEveryFieldToTheInitialEmptyState() { + var composition = fullyPopulated() + XCTAssertNotEqual(composition, .empty, "the fixture must actually be dirty") + + composition.reset() + + XCTAssertEqual(composition, .empty) + } + + /// The individually-named checks, so a failure says WHICH field survived rather + /// than only that the whole value differed. + func testResetClearsThePromptCaptionsCandidatesAndErrors() { + var composition = fullyPopulated() + composition.reset() + + XCTAssertEqual(composition.description, "") + XCTAssertTrue(composition.boxes.isEmpty) + XCTAssertNil(composition.selectedBoxID) + XCTAssertTrue(composition.candidateIDs.isEmpty) + XCTAssertNil(composition.selectedTemplateID) + XCTAssertEqual(composition.status, "") + XCTAssertFalse(composition.didFallBack) + XCTAssertFalse(composition.candidatesAreFallback) + XCTAssertFalse(composition.catalogFailed) + XCTAssertFalse(composition.imageFailed) + XCTAssertNil(composition.failedTemplateID) + XCTAssertFalse(composition.hasMeme) + } + + /// Resetting twice is the same as resetting once — the button is idempotent, so a + /// double-press (or ⌘N on an already-clear window) can't reach a different state. + func testResetIsIdempotent() { + var composition = fullyPopulated() + composition.reset() + composition.reset() + XCTAssertEqual(composition, .empty) + } + + /// An untouched surface reports itself empty, which is what dims the New meme + /// button. A control that is live but does nothing is the papercut this avoids. + func testAnUntouchedCompositionIsEmpty() { + XCTAssertTrue(MemeComposition.empty.isEmpty) + } + + /// An error ALONE counts as something to clear. This is the state a user most + /// wants a way out of, so New meme must be offered even when nothing was produced. + func testACompositionHoldingOnlyAnErrorIsNotEmpty() { + let failed = MemeComposition( + status: "Couldn't load Drake — the request timed out.", + imageFailed: true, + failedTemplateID: "imgflip:1") + XCTAssertFalse(failed.isEmpty) + } + + /// A dictated description alone counts too — the commonest "I misspoke, start + /// over" case, and the one voice-first users hit most. + func testACompositionHoldingOnlyADictatedDescriptionIsNotEmpty() { + XCTAssertFalse(MemeComposition(description: "a meme about mondays").isEmpty) + } + + /// The empty state INVITES rather than just being blank, and names both routes in + /// (dictate, or pick a template) because the window offers both. + func testTheEmptyHintNamesBothWaysToStart() { + let hint = MemeComposition.emptyHint + XCTAssertFalse(hint.isEmpty) + XCTAssertTrue(hint.lowercased().contains("describe")) + XCTAssertTrue(hint.lowercased().contains("template")) + } +} + +/// Manifest-declared menu shortcuts (v5, item 3), and who is allowed one. +/// +/// The host owns the keyboard because only the host can see the whole menu. A plugin +/// asks; `PluginKeyEquivalent` decides. These pin the decision rather than leaving it +/// to be discovered by a user whose ⌘Q stopped quitting. +final class PluginKeyEquivalentTests: XCTestCase { + + func testASingleLowercaseLetterIsAccepted() { + XCTAssertEqual(PluginKeyEquivalent.normalized("m"), "m") + } + + /// An uppercase request means ⌘M, not ⇧⌘M — `NSMenuItem` reads an uppercase key + /// equivalent as shifted, so normalizing is what stops a manifest silently getting + /// a different shortcut than the one it declared. + func testAnUppercaseRequestIsLowercasedRatherThanBecomingAShiftShortcut() { + XCTAssertEqual(PluginKeyEquivalent.normalized("M"), "m") + } + + func testSurroundingWhitespaceIsTolerated() { + XCTAssertEqual(PluginKeyEquivalent.normalized(" m "), "m") + } + + /// Multi-character, empty, and absent requests are all "no shortcut" rather than + /// errors — the field is optional and cosmetic. + func testUnusableRequestsYieldNoShortcut() { + XCTAssertNil(PluginKeyEquivalent.normalized("mm")) + XCTAssertNil(PluginKeyEquivalent.normalized("")) + XCTAssertNil(PluginKeyEquivalent.normalized(" ")) + XCTAssertNil(PluginKeyEquivalent.normalized(nil)) + XCTAssertNil(PluginKeyEquivalent.normalized("⌘")) + } + + /// A plugin may not shadow the app's own shortcuts. ⌘Q is the one that matters + /// most — a manifest that could take it would be a genuine hazard, not a papercut. + func testAPluginCannotTakeAShortcutTheAppAlreadyOwns() { + for reserved in ["q", "s", ",", "c", "x", "v", "a", "z"] { + XCTAssertNil( + PluginKeyEquivalent.assignable(reserved, taken: []), + "\(reserved) is the app's and must not be grantable") + } + } + + /// ⌘M specifically IS free — this app has no Window menu, which is what makes the + /// meme plugin's request grantable. Pinned so padding the reserved set later + /// doesn't silently revoke a shipped shortcut. + func testTheMemePluginsRequestedShortcutIsGrantable() { + XCTAssertEqual(PluginKeyEquivalent.assignable("m", taken: []), "m") + } + + /// Two plugins asking for the same key resolve by list order — deterministically, + /// rather than both rendering ⌘M and one of them silently never firing. + func testTwoPluginsAskingForTheSameKeyResolveByListOrder() { + let assigned = PluginKeyEquivalent.assign(requests: [ + (id: "meme-generator", keyEquivalent: "m"), + (id: "metronome", keyEquivalent: "m"), + ]) + XCTAssertEqual(assigned["meme-generator"], "m") + XCTAssertNil(assigned["metronome"], "the loser gets no shortcut, not a duplicate") + } + + /// A refused plugin still appears — it just gets no key. Losing the whole row over + /// a shortcut collision would be a far worse trade than losing the shortcut. + func testARefusedShortcutDoesNotRemoveThePlugin() { + let assigned = PluginKeyEquivalent.assign(requests: [ + (id: "shadow", keyEquivalent: "q"), + (id: "meme-generator", keyEquivalent: "m"), + ]) + XCTAssertNil(assigned["shadow"]) + XCTAssertEqual(assigned["meme-generator"], "m", + "a refusal must not disturb the next plugin's grant") + } + + func testPluginsAskingForNothingGetNothing() { + let assigned = PluginKeyEquivalent.assign(requests: [ + (id: "quiet", keyEquivalent: nil), + ]) + XCTAssertTrue(assigned.isEmpty) + } +} + +/// The manifest side of the shortcut field: decode, validation, and display. +final class PluginManifestKeyEquivalentTests: XCTestCase { + + private func manifest(keyEquivalent: String?) -> PluginManifest { + PluginManifest( + id: "test-plugin", name: "Test", version: "1.0", summary: "s", + symbol: "gear", entry: .builtIn, keyEquivalent: keyEquivalent) + } + + /// The field is OPTIONAL: a manifest written before it existed — including one + /// already sitting in a user's plugins folder — must still decode. + func testAManifestWithoutTheFieldStillDecodes() throws { + let json = """ + {"id":"legacy","name":"Legacy","version":"1.0","summary":"s", + "symbol":"gear","entry":"builtIn"} + """ + let decoded = try JSONDecoder().decode(PluginManifest.self, from: Data(json.utf8)) + XCTAssertNil(decoded.keyEquivalent) + XCTAssertTrue(decoded.isValid) + } + + func testTheFieldRoundTripsThroughJSON() throws { + let original = manifest(keyEquivalent: "m") + let decoded = try JSONDecoder().decode( + PluginManifest.self, from: try JSONEncoder().encode(original)) + XCTAssertEqual(decoded.keyEquivalent, "m") + XCTAssertEqual(decoded, original) + } + + /// A malformed shortcut is REPORTED but not fatal: the plugin still lists and + /// still runs, it just gets no key. Losing a working plugin over a cosmetic field + /// would be the wrong trade. + func testAMalformedShortcutIsReportedButDoesNotInvalidateThePlugin() { + let bad = manifest(keyEquivalent: "cmd+m") + XCTAssertEqual(bad.validate(), .invalidKeyEquivalent("cmd+m")) + XCTAssertTrue(bad.isValid, "a bad shortcut must not disqualify the plugin") + XCTAssertNil(bad.keyEquivalentDisplay) + } + + /// The structural failures still ARE fatal — the permissiveness above is scoped to + /// the shortcut and must not have leaked into id/name/symbol validation. + func testStructuralFailuresRemainFatal() { + let traversal = PluginManifest( + id: "../escape", name: "Bad", version: "1", summary: "", + symbol: "gear", entry: .builtIn) + XCTAssertFalse(traversal.isValid) + } + + func testDisplayFormIsTheUppercasedCommandForm() { + XCTAssertEqual(manifest(keyEquivalent: "m").keyEquivalentDisplay, "⌘M") + XCTAssertNil(manifest(keyEquivalent: nil).keyEquivalentDisplay) + } + + /// The shipped plugin declares ⌘M, and the registry literal is what actually runs. + func testTheMemeGeneratorDeclaresCommandM() { + XCTAssertEqual(PluginRegistry.memeGenerator.keyEquivalent, "m") + XCTAssertEqual(PluginRegistry.memeGenerator.keyEquivalentDisplay, "⌘M") + XCTAssertNil(PluginRegistry.memeGenerator.validate()) + } +} diff --git a/plugins/MemeGenerator/MemeGeneratorModel.swift b/plugins/MemeGenerator/MemeGeneratorModel.swift index d9f123d..c7beaad 100644 --- a/plugins/MemeGenerator/MemeGeneratorModel.swift +++ b/plugins/MemeGenerator/MemeGeneratorModel.swift @@ -256,6 +256,11 @@ final class MemeGeneratorModel: ObservableObject { /// upgrades it. A refresh that fails while templates are on screen is silent — /// reporting it would make a working plugin look broken. func openCatalog(forceRefresh: Bool = false) { + // An explicit Retry / force-refresh throws the HTTP session away first (v5). + // Same reason `retryTemplate` does: a refresh that reuses a wedged connection + // pool is a no-op however many times the user presses it. + if forceRefresh { MemeTemplateService.invalidateSession() } + let cached = MemeLibraryStore.loadCachedCatalog() let library = MemeLibraryStore.libraryTemplates() let decision = forceRefresh @@ -493,6 +498,72 @@ final class MemeGeneratorModel: ObservableObject { status = "Cancelled." } + // MARK: - New meme (v5) + + /// The current composition, projected into the pure, testable value. + /// + /// The projection exists so `startNewMeme` can be proved TOTAL by `swift test` + /// without linking AppKit: the test builds a fully-populated `MemeComposition`, + /// resets it, and asserts it equals `.empty`. A field added to the surface later + /// gets reset by construction rather than by remembering to add a line. + var composition: MemeComposition { + MemeComposition( + description: description, + boxes: boxes, + selectedBoxID: selectedBoxID, + candidateIDs: candidates.map(\.id), + selectedTemplateID: selectedTemplate?.id, + status: status, + didFallBack: didFallBack, + candidatesAreFallback: candidatesAreFallback, + catalogFailed: catalogFailed, + imageFailed: imageFailed, + failedTemplateID: failedTemplate?.id, + hasMeme: meme != nil) + } + + /// Whether New meme has anything to do — drives the button's enabled state. + var canStartNewMeme: Bool { !composition.isEmpty } + + /// Start from scratch (v5). + /// + /// The owner asked for a way back to an empty sheet, and the important word is + /// BACK: this has to abandon in-flight work as well as clear what is on screen. + /// `state.reset()` bumps the ticket, so a download or an LLM round-trip already + /// running is refused when it lands — the same ticket guard every other exit path + /// uses — rather than completing a moment later and repopulating the surface the + /// user just cleared. + /// + /// What deliberately SURVIVES: the template catalog (a corpus, not part of this + /// meme — clearing it would make New meme a network round-trip), the downloaded + /// image cache (keyed by template id, holds nothing about this meme, and dropping + /// it would re-download templates the user already has), and the user's library. + func startNewMeme() { + // Refuse every outstanding result BEFORE clearing, so nothing in flight can + // land between the reset and the next user action. + state.reset() + + let empty = MemeComposition.empty + description = empty.description + boxes = empty.boxes + selectedBoxID = empty.selectedBoxID + candidates = [] + selectedTemplate = nil + status = empty.status + didFallBack = empty.didFallBack + candidatesAreFallback = empty.candidatesAreFallback + imageFailed = empty.imageFailed + failedTemplate = nil + meme = nil + baseImage = nil + searchText = "" + + // The catalog stays, so a catalog failure is only cleared when there is in + // fact a catalog — clearing the flag with an empty corpus would hide a real + // problem behind a fresh-looking empty state. + catalogFailed = catalog.isEmpty && catalogFailed + } + /// Make sure there is a catalog to prompt with. Returns false when it failed and /// has already reported why. private func ensureCatalogForGenerate(ticket: Int) async -> Bool { @@ -586,6 +657,16 @@ final class MemeGeneratorModel: ObservableObject { } /// Re-run the last template load that failed. The Retry affordance's action. + /// + /// ## v5 — Retry must be able to succeed + /// + /// The owner reported Retry doing nothing once downloads had gone bad. It was a + /// no-op by construction: it re-ran the request through the same process-lifetime + /// `URLSession`, so it inherited exactly the connection pool that was broken. + /// `MemeTemplateService.invalidateSession()` throws that pool away first, so the + /// retry builds a FRESH session and a fresh request. Unconditional here rather + /// than only on transport errors: Retry is an explicit "try properly this time", + /// it happens at most once per user click, and a new pool costs a handshake. func retryTemplate() { guard let template = failedTemplate else { return } // Clear it first: `select` early-returns when the template is already @@ -595,6 +676,7 @@ final class MemeGeneratorModel: ObservableObject { // The failed template is NOT the selected one (the load never completed), so // `select` will proceed — but drop any cached corpse just in case. imageCache[template.id] = nil + MemeTemplateService.invalidateSession() select(template: template) } @@ -664,8 +746,14 @@ final class MemeGeneratorModel: ObservableObject { } failedTemplate = template imageFailed = true + // Name the transport's history when it has one (v5). After a day of uptime the + // useful question is whether the connection has already been rebuilt — that is + // the difference between "the network blipped" and "something is genuinely + // wrong", and it is the datum the owner's next report will need. + let diagnostic = MemeTemplateService.sessionDiagnostic.map { " \($0)" } ?? "" finish(ticket, status: - "Couldn't load \(template.name) — \(Self.reason(error)) Press Retry, or pick another template.") + "Couldn't load \(template.name) — \(Self.reason(error))\(diagnostic) " + + "Press Retry, or pick another template.") } /// The line under the controls: honest about a fallback, quiet otherwise. diff --git a/plugins/MemeGenerator/MemeGeneratorView.swift b/plugins/MemeGenerator/MemeGeneratorView.swift index dfccfc1..dd74768 100644 --- a/plugins/MemeGenerator/MemeGeneratorView.swift +++ b/plugins/MemeGenerator/MemeGeneratorView.swift @@ -225,6 +225,20 @@ struct MemeGeneratorView: View { .keyboardShortcut(.cancelAction) } + // Start from scratch (v5). ALWAYS present rather than appearing once + // there's something to clear — a control that materializes is harder to + // find than one that is simply dimmed, and this is the button a user + // reaches for when the surface is in a state they don't understand. + // Disabled (not hidden) on an untouched window so it never reads as broken. + Button { + model.startNewMeme() + } label: { + Label("New meme", systemImage: "arrow.counterclockwise") + } + .keyboardShortcut("n", modifiers: .command) + .disabled(!model.canStartNewMeme) + .help("Clear the description, captions, and template and start over (⌘N)") + Spacer() Button { @@ -332,11 +346,19 @@ struct MemeGeneratorView: View { Image(systemName: "photo") .font(.largeTitle) .foregroundStyle(.tertiary) - Text("Pick a template on the left, or describe a meme and press Generate.") + // The empty state INVITES the next action rather than just + // being blank (v5) — this is what the user is looking at + // straight after ⌘N, so it has to say what to do next. + Text(MemeComposition.emptyHint) .font(.callout) .multilineTextAlignment(.center) .foregroundStyle(.tertiary) .padding(.horizontal, 24) + Text("Press ⌘⏎ to generate once you've described one.") + .font(.caption) + .multilineTextAlignment(.center) + .foregroundStyle(.quaternary) + .padding(.horizontal, 24) }) .frame(maxWidth: .infinity, maxHeight: .infinity) } diff --git a/plugins/MemeGenerator/MemeGeneratorWindowController.swift b/plugins/MemeGenerator/MemeGeneratorWindowController.swift index bb3a818..a903d60 100644 --- a/plugins/MemeGenerator/MemeGeneratorWindowController.swift +++ b/plugins/MemeGenerator/MemeGeneratorWindowController.swift @@ -12,7 +12,8 @@ import SwiftUI /// look at, not an always-on-top scratch surface. It still needs to become KEY so /// dictation can land in it — that is what makes `appendDictationIfKey` fire. @MainActor -final class MemeGeneratorWindowController: NSWindowController, NSWindowDelegate, PluginDictationSink { +final class MemeGeneratorWindowController: NSWindowController, NSWindowDelegate, + PluginDictationSink, PluginWindowLifecycle { private let model = MemeGeneratorModel() @@ -90,6 +91,21 @@ final class MemeGeneratorWindowController: NSWindowController, NSWindowDelegate, }) } + // MARK: - PluginWindowLifecycle + + /// The window is being shown again after a close (v5). + /// + /// `PluginHost` caches this controller forever, so without this hook the model's + /// `windowDidOpen` ran exactly once — at `init` — while `windowWillClose` ran on + /// every close. The two are a matched pair: close sets `isCancelled = true` so a + /// late result can't write into a dead window, and only `windowDidOpen` clears it. + /// Unbalanced, the FIRST close permanently poisoned every later download; the + /// symptom the owner saw was a plugin that worked all day and then stopped, since + /// closing the window at some point during that day is what armed it. + func pluginWindowWillShow() { + model.windowDidOpen() + } + // MARK: - PluginDictationSink /// True while this window is frontmost — the signal that a completed dictation diff --git a/plugins/MemeGenerator/MemeTemplateService.swift b/plugins/MemeGenerator/MemeTemplateService.swift index 8228134..99d9cca 100644 --- a/plugins/MemeGenerator/MemeTemplateService.swift +++ b/plugins/MemeGenerator/MemeTemplateService.swift @@ -46,14 +46,94 @@ enum MemeTemplateService { static let imgflipCatalogURL = URL(string: "https://api.imgflip.com/get_memes")! static let memegenCatalogURL = URL(string: "https://api.memegen.link/templates")! - /// A short timeout: this sits in front of a user waiting on a meme, so failing - /// fast and saying so beats a long hang. - private static let session: URLSession = { + // MARK: - Session + + /// The HTTP session, REPLACEABLE (v5). + /// + /// ## Why this stopped being a `static let` + /// + /// The owner's v5 report was "template downloads stop working after about a day + /// of uptime, and Retry does nothing". A `URLSession` is a connection pool, and + /// v4's was a process-lifetime `static let` that nothing could ever replace. A + /// pooled connection can outlive its own validity — the Mac sleeps and wakes on a + /// different network, a VPN comes up, a captive portal's lease expires, an + /// interface changes — and once the pool is in that state EVERY request handed to + /// the session fails identically, for as long as the app stays running. That is + /// precisely the reported shape: fine all day, then permanently broken, with a + /// relaunch as the only cure. + /// + /// It also explains the second half of the report. Retry re-ran the request + /// through the SAME session, so it inherited exactly the pool that was broken — + /// a no-op by construction, however many times the user pressed it. + /// + /// So the session is now rebuildable, and a transport-shaped failure throws it + /// away (`invalidate`). The next request — including a Retry — builds a fresh one + /// with a fresh pool. The decision of WHICH failures count is the pure, + /// `swift test`-pinned `MemeGenerationState.isTransportFailure`; a 404 or an + /// undecodable image says nothing about the transport and keeps the pool. + private static var _session: URLSession? + + /// How many times the pool has been thrown away this launch. + /// + /// Not test-reachable (this file is behind `PLUGINS=1`, outside the `swift test` + /// target), so it earns its place as a DIAGNOSTIC instead: if the owner reports + /// downloads dying again, this number distinguishes "the pool was never recycled, + /// so the recycle predicate is too narrow" from "it recycled repeatedly and still + /// failed, so the problem is not the pool". Surfaced through `sessionDiagnostic`. + private(set) static var sessionGeneration = 0 + + /// A one-line description of the transport's history, for the status line when a + /// download fails after the session has already been recycled at least once. + static var sessionDiagnostic: String? { + guard sessionGeneration > 0 else { return nil } + return sessionGeneration == 1 + ? "(the connection was reset once)" + : "(the connection was reset \(sessionGeneration) times)" + } + + static var session: URLSession { + if let existing = _session { return existing } + let created = makeSession() + _session = created + return created + } + + private static func makeSession() -> URLSession { let config = URLSessionConfiguration.ephemeral + // A short timeout: this sits in front of a user waiting on a meme, so failing + // fast and saying so beats a long hang. config.timeoutIntervalForRequest = 15 config.timeoutIntervalForResource = 30 + // Never hand back a response cached before the network went bad — the whole + // point of rebuilding is to re-ask reality. + config.requestCachePolicy = .reloadIgnoringLocalCacheData + // Fail rather than park a request until connectivity returns: this sits in + // front of a waiting user, and a visible error with a Retry beats a spinner. + config.waitsForConnectivity = false return URLSession(configuration: config) - }() + } + + /// Throw the current session away so the next request builds a fresh one. + /// + /// `invalidateAndCancel` rather than a bare drop: it tears down the pooled + /// connections instead of leaving them alive until ARC gets around to the + /// session, which matters because those connections are the thing being + /// discarded. + static func invalidateSession() { + guard let existing = _session else { return } + _session = nil + sessionGeneration += 1 + existing.invalidateAndCancel() + } + + /// Drop the session if `error` says the transport itself is suspect. + /// + /// One funnel, called from every fetch path, so no request can fail on a wedged + /// pool without the pool being reconsidered. + static func recycleSessionIfNeeded(after error: Error) { + guard MemeGenerationState.isTransportFailure(error) else { return } + invalidateSession() + } // MARK: - Providers @@ -93,8 +173,8 @@ enum MemeTemplateService { } static func fetchImgflip() async throws -> [MemeTemplate] { - let (data, response) = try await session.data(from: imgflipCatalogURL) - try check(response, host: "imgflip.com") + let (data, response) = try await get(imgflipCatalogURL, host: "imgflip.com") + _ = response let decoded = try JSONDecoder().decode(MemeTemplateCatalogResponse.self, from: data) let templates = decoded.templates @@ -103,14 +183,37 @@ enum MemeTemplateService { } static func fetchMemegen() async throws -> [MemeTemplate] { - let (data, response) = try await session.data(from: memegenCatalogURL) - try check(response, host: "memegen.link") + let (data, response) = try await get(memegenCatalogURL, host: "memegen.link") + _ = response let decoded = try JSONDecoder().decode(MemegenTemplateResponse.self, from: data) guard !decoded.templates.isEmpty else { throw ServiceError.emptyCatalog } return decoded.templates } + /// One GET, through the current session, recycling it on a transport failure. + /// + /// Every network read in this file goes through here so the recycle rule cannot be + /// forgotten on a path — which is how v4 ended up with a session nothing could + /// replace. Each call builds its OWN `URLRequest` rather than reusing a stored one, + /// so a Retry is a genuinely fresh request and not a replay of the wedged attempt. + private static func get( + _ url: URL, host: String, timeout: TimeInterval? = nil + ) async throws -> (Data, URLResponse) { + var request = URLRequest(url: url) + request.cachePolicy = .reloadIgnoringLocalCacheData + if let timeout { request.timeoutInterval = timeout } + + do { + let (data, response) = try await session.data(for: request) + try check(response, host: host) + return (data, response) + } catch { + recycleSessionIfNeeded(after: error) + throw error + } + } + // MARK: - Images /// Load a template's blank image — from disk for the user's library, over the @@ -135,12 +238,10 @@ enum MemeTemplateService { // A per-request ceiling on top of the session's, so one wedged image GET can't // outlive the UI's own download timeout and land a result into a surface that - // has already recovered. - var request = URLRequest(url: url) - request.timeoutInterval = imageTimeout - - let (data, response) = try await session.data(for: request) - try check(response, host: url.host ?? "the template host") + // has already recovered. A FRESH `URLRequest` every call (v5) — see `get` — + // so pressing Retry re-asks rather than replaying the attempt that hung. + let (data, _) = try await get( + url, host: url.host ?? "the template host", timeout: imageTimeout) guard let image = NSImage(data: data) else { throw ServiceError.undecodableImage } return image diff --git a/plugins/MemeGenerator/manifest.json b/plugins/MemeGenerator/manifest.json index 7e17f7f..a1c3aeb 100644 --- a/plugins/MemeGenerator/manifest.json +++ b/plugins/MemeGenerator/manifest.json @@ -1,9 +1,10 @@ { "id": "meme-generator", "name": "Meme Generator", - "version": "0.3.0", + "version": "0.5.0", "summary": "Dictate a meme description — the AI picks a template and writes the captions.", "symbol": "photo.badge.plus", "entry": "builtIn", - "networkHosts": ["api.imgflip.com", "i.imgflip.com", "api.memegen.link"] + "networkHosts": ["api.imgflip.com", "i.imgflip.com", "api.memegen.link"], + "keyEquivalent": "m" } From 3247bdf1fa823b3e5f740bc255b84f2c5be92324 Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Mon, 3 Aug 2026 13:36:57 -0700 Subject: [PATCH 09/15] spike(meme-plugin): per-template caption structure, numbered picks, refit, learned ranking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v5 assumed every meme is a top line and a bottom line, and asked the model to copy template names verbatim. Both assumptions were wrong for most of the corpus, and both had the data to do better sitting unused on the wire. Per-template caption STRUCTURE. imgflip ships `box_count` and memegen ships `lines`; v5 decoded neither. Both now become `MemeTemplate.captionSlots` (clamped 1...8, defaulting to 2 so a v5 cache and the user library are unchanged), the LLM is asked for a captions ARRAY sized to the top candidate's slots, and boxes are seeded per slot. Drake gets 2, Distracted Boyfriend 3, Expanding Brain 4 — instead of two captions on a four-panel joke. Slot GEOMETRY is synthesized, not sourced — verified against both live APIs on 2026-08-03: memegen's `/templates` and `/templates/` return `lines` as a COUNT with no geometry field anywhere, and imgflip's key-less `get_memes` is the same. (imgflip exposes per-box rectangles only through the authenticated captioning endpoint, which this plugin deliberately doesn't use — that would put the user's words on someone else's server.) So: 1 = centered, 2 = the classic top/bottom (unchanged — two thirds of the corpus), 3-4 = a stacked left column at narrower width and smaller type, 5+ = an even full-width column. The 3-4 case is an explicit spike-grade compromise: it lands captions roughly right for vertically-panelled templates and only approximately right for horizontal ones. Approximately-right and draggable beats confidently-wrong and invisible. Numbered candidate references. The shortlist was already numbered and the numbers went unused. Asking a tiny local model to transcribe names exactly is the most fragile thing you can ask it for — it re-capitalizes, expands, folds in the parenthesized keywords, and every one of those was a silently dropped candidate. It now answers `{"templates": [3, 17, 1]}`; the parser accepts numbers, numeric strings, exact names, and any mix, validates the range (out-of-range is DROPPED, never clamped — clamping would be v1's confident-Drake bug wearing a number), and dedupes on the resolved template so a number and its name can't occupy two strip slots. Captions re-fit on a candidate switch. A slot-count change runs a small second LLM call ("same joke, N slots") on its own ticket through the existing state machine — cancellable, never blocking the strip, silent on failure so a failed nicety can't make a working switch look broken. Same count takes a fast path with no round-trip, which is the common case. Generate no longer clobbers manual work. The rule, chosen over a confirmation dialog: Generate replaces AI-seeded boxes and PRESERVES boxes the user added. A dialog would tax every regenerate to protect the rare one; this is a rule a user can predict, and an unwanted surviving box is one click to delete whereas a destroyed caption is retyped from memory. Edits to a seeded box are still replaced — that box is the AI's answer to the old description. Cheap learning signal. Clicking past the model's first pick is a correction and the cheapest supervision this plugin will get. Each one adds a small capped boost (12/pick, ceiling 120) applied in the prefilter ranking, persisted per plugin. Bounded so it can't hijack ranking: it is smaller than one keyword-token match, it saturates after ten picks, it only ever boosts templates the query ALREADY matched — so "nothing matches" stays reachable — and it does not reorder an unfiltered Browse grid. Also drops v5's "think about which templates fit" invitation, which asked for reasoning the parser then discarded — pure token cost on the models least able to afford it. The model now returns one short `reason` that the candidate strip shows as a tooltip: the same request, routed somewhere the user can read it. swift test 2593 (2520 + 73 new in MemeStructureTests). One v5 test renamed — it pinned the "copied exactly" wording; the guard it protects is now the numbering, so the assertion moved rather than being dropped. Co-Authored-By: Claude --- OpenWhisp/Services/MemeAI.swift | 408 +++++++-- OpenWhisp/Services/MemeCaptionLayout.swift | 163 +++- OpenWhisp/Services/MemeTemplateAffinity.swift | 110 +++ OpenWhisp/Services/MemeTemplateMatcher.swift | 100 ++- OpenWhisp/Services/MemeTemplateProvider.swift | 44 +- Package.swift | 1 + .../MemeGeneratorTests.swift | 13 +- .../MemeStructureTests.swift | 773 ++++++++++++++++++ .../MemeGenerator/MemeGeneratorModel.swift | 176 +++- plugins/MemeGenerator/MemeGeneratorView.swift | 51 +- plugins/MemeGenerator/MemeLibraryStore.swift | 32 + 11 files changed, 1783 insertions(+), 88 deletions(-) create mode 100644 OpenWhisp/Services/MemeTemplateAffinity.swift create mode 100644 Tests/OpenWhispCoreTests/MemeStructureTests.swift diff --git a/OpenWhisp/Services/MemeAI.swift b/OpenWhisp/Services/MemeAI.swift index b1cb550..c688aa1 100644 --- a/OpenWhisp/Services/MemeAI.swift +++ b/OpenWhisp/Services/MemeAI.swift @@ -79,39 +79,61 @@ public enum MemeAI { /// The instruction for the ranked-candidate round-trip. /// - /// The catalog names are appended by `rankedUserPayload` rather than baked in + /// The catalog lines are appended by `rankedUserPayload` rather than baked in /// here, so this constant stays testable and the prompt stays one place. /// - /// "Think about which templates fit before answering" is deliberate: the owner - /// asked for the model to consider alternatives rather than snap to one. It - /// costs a few tokens of reasoning in the response we then discard — the parser - /// digs the JSON out of prose regardless. + /// ## v6 — three changes, each fixing something the previous prompt got wrong + /// + /// **1. Candidates are NUMBERS, not copied names.** v5 asked the model to copy + /// names "EXACTLY", then dropped anything that didn't match the catalog. That is a + /// transcription task, and it is the single most fragile thing you can ask a tiny + /// local model to do: it re-capitalizes, it expands "Y U No" to "Why You No", it + /// drops the parenthesized keywords or folds them in, and every one of those is a + /// silently discarded candidate. The shortlist was already numbered for exactly + /// this reason and the numbers went unused. Returning `[3, 17, 1]` makes the answer + /// a one-token-per-pick lookup that either indexes a real template or is out of + /// range — no fuzzy middle ground. The parser still accepts names (see + /// `parseRanked`), so a model that ignores this is no worse off than in v5. + /// + /// **2. Captions are an ARRAY sized to the template.** `top_text`/`bottom_text` + /// hard-coded the assumption that every meme is a two-liner. Drake is two side + /// labels, Distracted Boyfriend is three, Expanding Brain is four — so the payload + /// now states the top candidate's slot count and asks for that many captions, in + /// panel order. + /// + /// **3. "Think about which ones could carry the joke" is GONE.** It asked for + /// reasoning that the parser then threw away — pure token cost with no consumer, + /// on the models least able to afford it. In its place the model returns ONE short + /// `reason`, which the candidate strip actually shows as a tooltip. That is the + /// same request turned into something the user can see: if the model picked Drake + /// for a bad reason, the user now reads the bad reason instead of guessing why the + /// thumbnail is there. Reasoning that is displayed earns its tokens; reasoning that + /// is discarded does not. public static let rankedPrompt = """ You pick meme templates for a spoken description. You are given a numbered list of the ONLY templates available. Each line is a \ - template name, optionally followed by alternate names in parentheses — the \ + number, a template name, and optionally alternate names in parentheses — the \ parentheses describe what the meme is about, so use them to match a description \ - of the meme's CONTENT, but never copy them into your answer. - - Think about which ones could carry the joke, then rank your best options. + of the meme's CONTENT. Reply with ONLY a JSON object, no preamble and no code fence: - {"templates": ["...", "...", "..."], "top_text": "...", "bottom_text": "..."} + {"templates": [3, 17, 1], "captions": ["...", "..."], "reason": "..."} Rules: - 1. "templates" lists 1 to 5 template names COPIED EXACTLY from the list, best \ - first — the part BEFORE any parentheses, without the number. Do not invent a \ - name, do not reword one, do not use a name that is not on the list. + 1. "templates" lists 1 to 5 NUMBERS from the list, best first. Use the number \ + only — do not write the name, do not invent a number that is not on the list. 2. If nothing on the list fits the description well, still return the closest \ options — but put the genuinely closest first. - 3. "top_text" and "bottom_text" are the caption lines, in the SAME LANGUAGE as \ - the description. Do not translate them. - 4. Keep each caption short — a few words, meme-style. No quotation marks around \ + 3. "captions" are the caption lines for your FIRST choice, in order from the top \ + (or left) panel to the last. The description above says how many that template \ + takes — return exactly that many. + 4. Captions must be in the SAME LANGUAGE as the description. Do not translate them. + 5. Keep each caption short — a few words, meme-style. No quotation marks around \ them. - 5. If the description only calls for one line, put it in "top_text" and leave \ - "bottom_text" as an empty string. 6. Never invent a caption that contradicts the description. + 7. "reason" is ONE short sentence saying why your first choice fits. It is shown \ + to the user, so write it for them, not for yourself. """ /// How many templates the LLM is asked to rank (v4). @@ -154,59 +176,193 @@ public enum MemeAI { """ } + /// The numbering the model answers with — 1-BASED, matching what it sees. + /// + /// Stated once, here, because the offset is the whole contract between + /// `rankedUserPayload` (which prints `index + 1`) and `resolve` (which subtracts + /// it). An off-by-one between those two would silently return the model's + /// neighbour on every pick — a bug that produces plausible-looking wrong templates + /// rather than an error, which is the worst kind to hunt. + public static let firstCandidateNumber = 1 + + /// One prompt line per shortlisted template, carrying its caption-slot count (v6). + /// + /// The count has to be IN the list because the model chooses its first candidate + /// and writes that candidate's captions in the same reply — it cannot be told the + /// slot count in advance without knowing what it will pick. Printing it per line + /// lets the model read the number off the row it just chose. + /// + /// Only non-default counts are annotated. Tagging all 166 two-slot templates with + /// "(2 captions)" would be 166 lines of noise for the case that is already the + /// default, on models whose attention is the scarce resource — so a bare line MEANS + /// two, which rule 3 of the prompt and the sentence below both state. + public static func slotAnnotatedLines(_ lines: [String], slots: [Int]) -> [String] { + lines.enumerated().map { index, line in + let count = index < slots.count + ? MemeCaptionSlots.clamp(slots[index]) + : MemeCaptionSlots.default + guard count != MemeCaptionSlots.default else { return line } + return "\(line) [\(count) captions]" + } + } + + /// The payload for the v6 round-trip: the numbered shortlist with slot counts, the + /// description, and the sentence that ties the two together. + public static func rankedUserPayload( + description: String, templateLines: [String], slots: [Int], + limit: Int = candidateShortlist + ) -> String { + let annotated = slotAnnotatedLines(templateLines, slots: slots) + let base = rankedUserPayload( + description: description, templateNames: annotated, limit: limit) + + return base + """ + + + A template marked [N captions] takes N caption lines; an unmarked one takes \ + \(MemeCaptionSlots.default). Return exactly as many captions as your FIRST \ + choice takes. + """ + } + /// A ranked pick: template names that exist in the catalog, plus the captions. public struct RankedSpec: Equatable, Sendable { /// Template names, best first, each guaranteed to appear in the catalog that /// was passed to `parseRanked`. May be EMPTY when the model named only /// templates that don't exist — the honest "not in this corpus" signal. public let templateNames: [String] - public let topText: String - public let bottomText: String - public init(templateNames: [String], topText: String, bottomText: String) { + /// The captions in PANEL ORDER (v6), one per slot the model was asked for. + /// + /// Replaces v5's `topText`/`bottomText` pair. The array is the general case: + /// a two-slot response is `[top, bottom]`, which is exactly what the legacy + /// keys decode into, so nothing about the classic path changed except its + /// spelling. + public let captions: [String] + + /// The model's one-line justification for its first pick, shown in the + /// candidate strip's tooltip. Empty when the model didn't give one — it is a + /// nicety, never a reason to reject a response. + public let reason: String + + public init(templateNames: [String], captions: [String], reason: String = "") { self.templateNames = templateNames - self.topText = topText - self.bottomText = bottomText + self.captions = captions + self.reason = reason } + /// The classic two-slot spelling, kept so existing call sites and tests that + /// think in top/bottom still read naturally. + public init(templateNames: [String], topText: String, bottomText: String) { + self.init(templateNames: templateNames, captions: [topText, bottomText]) + } + + public var topText: String { captions.first ?? "" } + public var bottomText: String { captions.count > 1 ? captions[1] : "" } + /// True when the model produced captions but no usable template — the caller /// must show the corpus rather than silently substituting a popular template. public var hasNoUsableTemplate: Bool { templateNames.isEmpty } + + /// True when nothing at all was said — the reject condition, stated once. + public var isEmpty: Bool { + templateNames.isEmpty && captions.allSatisfy(\.isEmpty) + } } - /// The wire shape. Decoded leniently: `templates` may arrive as an array or, from - /// a model that ignored the schema, as a single string. + /// One candidate reference as the model wrote it: a number or a name (v6). + /// + /// Modelled explicitly rather than collapsing both into a string, because the two + /// resolve against completely different things — a number indexes the shortlist, + /// a name is matched against the catalog — and conflating them is how "17" would + /// end up being fuzzy-matched against a template called "17 Again". + public enum CandidateRef: Equatable, Sendable { + case index(Int) + case name(String) + } + + /// The wire shape. Decoded leniently on purpose: `templates` may arrive as an + /// array of numbers (the v6 contract), an array of strings, a mix of both, a bare + /// number, or a single string; captions may arrive as an array (v6) or as + /// `top_text`/`bottom_text` (v5 and any model that saw a two-line meme and reached + /// for the classic keys). private struct RankedWire: Decodable { - let templates: [String] - let topText: String - let bottomText: String + let templates: [CandidateRef] + let captions: [String] + let reason: String private enum CodingKeys: String, CodingKey { case templates + case captions + case reason case topText = "top_text" case bottomText = "bottom_text" // Aliases small models reach for when they drift off the schema. case template case templateQuery = "template_query" + case texts + case lines + } + + /// Decode one element of `templates`, which may be a number or a string. + /// + /// A string that is ENTIRELY a number ("3") is treated as an index: a model + /// asked for numbers and answering `["3"]` meant the third template, and + /// looking for a catalog entry named "3" would throw that away. + private struct AnyRef: Decodable { + let ref: CandidateRef? + + init(from decoder: Decoder) throws { + let c = try decoder.singleValueContainer() + if let number = try? c.decode(Int.self) { + ref = .index(number) + } else if let text = try? c.decode(String.self) { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + if let number = Int(trimmed) { + ref = .index(number) + } else { + ref = trimmed.isEmpty ? nil : .name(trimmed) + } + } else { + // A shape we don't understand (an object, a null) is DROPPED rather + // than failing the whole decode — one weird element must not cost + // the user the four good candidates beside it. + ref = nil + } + } } init(from decoder: Decoder) throws { let c = try decoder.container(keyedBy: CodingKeys.self) - var names: [String] = [] - if let list = try? c.decode([String].self, forKey: .templates) { - names = list - } else if let single = try? c.decode(String.self, forKey: .templates) { - names = [single] - } else if let single = try? c.decode(String.self, forKey: .template) { - names = [single] + var refs: [CandidateRef] = [] + if let list = try? c.decode([AnyRef].self, forKey: .templates) { + refs = list.compactMap(\.ref) + } else if let single = try? c.decode(AnyRef.self, forKey: .templates), let ref = single.ref { + refs = [ref] + } else if let single = try? c.decode(AnyRef.self, forKey: .template), let ref = single.ref { + refs = [ref] } else if let single = try? c.decode(String.self, forKey: .templateQuery) { - names = [single] + refs = [.name(single)] + } + templates = refs + + // v6 array first, then the aliases, then the v5 legacy pair. The legacy + // branch is LAST so a response carrying both an array and stray top/bottom + // keys keeps the array — the richer answer wins. + if let list = try? c.decode([String].self, forKey: .captions) { + captions = list + } else if let list = try? c.decode([String].self, forKey: .texts) { + captions = list + } else if let list = try? c.decode([String].self, forKey: .lines) { + captions = list + } else { + let top = (try? c.decode(String.self, forKey: .topText)) ?? "" + let bottom = (try? c.decode(String.self, forKey: .bottomText)) ?? "" + captions = [top, bottom] } - templates = names - topText = (try? c.decode(String.self, forKey: .topText)) ?? "" - bottomText = (try? c.decode(String.self, forKey: .bottomText)) ?? "" + reason = (try? c.decode(String.self, forKey: .reason)) ?? "" } } @@ -237,28 +393,46 @@ public enum MemeAI { let wire = try? JSONDecoder().decode(RankedWire.self, from: data) else { return .failure(.notJSON) } - let names = validate(wire.templates, against: catalogNames) - let top = clean(wire.topText) - let bottom = clean(wire.bottomText) - - if names.isEmpty, top.isEmpty, bottom.isEmpty { - return .failure(.missingFields) - } - return .success(RankedSpec(templateNames: names, topText: top, bottomText: bottom)) + let names = resolve(wire.templates, shortlist: catalogNames) + // Trailing empties are dropped so a model padding a 2-slot answer out to four + // strings doesn't seed two blank boxes — but INTERIOR empties are kept, because + // in a panel meme "" at slot 2 means "this panel has no caption", and shifting + // slot 3 up into its place would relabel the wrong panel. + var captions = wire.captions.map(clean) + while let last = captions.last, last.isEmpty { captions.removeLast() } + + let spec = RankedSpec( + templateNames: names, captions: captions, reason: clean(wire.reason)) + guard !spec.isEmpty else { return .failure(.missingFields) } + return .success(spec) } - /// Keep the proposed names that exist in the catalog, in the model's order, - /// de-duplicated, capped, and returned in the CATALOG's spelling. + /// Resolve the model's candidate references against the shortlist it was shown. /// - /// Exposed for tests: this is the whole anti-hallucination rule. - public static func validate( - _ proposed: [String], against catalogNames: [String] + /// This is the whole anti-hallucination rule, now covering both answer shapes: + /// + /// * **A number** indexes the shortlist, 1-based (`firstCandidateNumber`), and is + /// REJECTED when out of range. There is no clamping and no nearest-match: a model + /// answering `47` for a 30-line list has miscounted or invented, and quietly + /// handing back template 30 would be v1's confident-Drake bug wearing a number. + /// * **A name** is matched case/punctuation-insensitively against the shortlist and + /// returned in the SHORTLIST's spelling. Kept from v5 for backward compatibility: + /// a model that ignores the numbering is exactly as well served as before. + /// + /// Order is the model's, duplicates collapse (including a number and a name that + /// resolve to the SAME template — the dedupe is on the resolved name, not on how it + /// was written), and the result is capped at `maxCandidates`. + /// + /// `shortlist` must be the same list, in the same order, that + /// `rankedUserPayload` numbered — that is what makes an index meaningful. + public static func resolve( + _ refs: [CandidateRef], shortlist: [String] ) -> [String] { // Key on the normalized form so "DRAKE HOTLINE BLING" and "Drake Hotline // Bling" resolve to the same catalog entry. First occurrence wins, which // matches the catalog's popularity order on a duplicate name. var byNormalized: [String: String] = [:] - for name in catalogNames { + for name in shortlist { let key = MemeTemplateMatcher.normalize(name) guard !key.isEmpty, byNormalized[key] == nil else { continue } byNormalized[key] = name @@ -266,9 +440,20 @@ public enum MemeAI { var kept: [String] = [] var seen = Set() - for candidate in proposed { - let key = MemeTemplateMatcher.normalize(clean(candidate)) - guard let canonical = byNormalized[key], !seen.contains(key) else { continue } + + for ref in refs { + let canonical: String? + switch ref { + case .index(let number): + let offset = number - firstCandidateNumber + canonical = shortlist.indices.contains(offset) ? shortlist[offset] : nil + case .name(let raw): + canonical = byNormalized[MemeTemplateMatcher.normalize(clean(raw))] + } + + guard let canonical else { continue } + let key = MemeTemplateMatcher.normalize(canonical) + guard !key.isEmpty, !seen.contains(key) else { continue } seen.insert(key) kept.append(canonical) if kept.count == maxCandidates { break } @@ -276,6 +461,117 @@ public enum MemeAI { return kept } + /// Keep the proposed names that exist in the catalog (the v5 name-only entry + /// point), expressed in terms of `resolve` so there is exactly one rule. + public static func validate( + _ proposed: [String], against catalogNames: [String] + ) -> [String] { + resolve(proposed.map { CandidateRef.name($0) }, shortlist: catalogNames) + } + + // MARK: - v6: refitting captions to another template's structure + + /// The instruction for the caption REFIT round-trip. + /// + /// ## Why a second call exists at all + /// + /// The candidate strip's promise is "same joke, different template". That held + /// while every template was two-slot: the boxes carried over verbatim. It breaks + /// the moment structure varies — clicking from Drake (2) to Expanding Brain (4) + /// used to leave two captions on a four-panel meme, and clicking back left four + /// captions stacked on a two-panel one. Neither is the same joke. + /// + /// Redistributing captions LOCALLY can't work: going 2 → 4 needs two new lines + /// invented in the user's language and in the joke's voice, which is a language + /// task. So it is a second, deliberately small LLM call — no catalog in the + /// payload, no ranking, just the captions and a target count. + /// + /// It is only ever reached when the count actually CHANGES (`needsRefit`); a + /// same-count switch reuses the captions instantly, as it always did. + public static let refitPrompt = """ + You rewrite meme captions to fit a different meme template. + + You are given the original description, the captions as they stand, and how many \ + caption slots the new template has. + + Reply with ONLY a JSON object, no preamble and no code fence: + {"captions": ["...", "..."]} + + Rules: + 1. Return EXACTLY the requested number of captions, in order from the top (or \ + left) panel to the last. + 2. Keep the SAME joke and the SAME language as the captions you were given. Do \ + not translate them and do not change the subject. + 3. When there are more slots than before, split or extend the joke across them — \ + do not repeat a caption or pad with empty strings. + 4. When there are fewer, condense — keep the punchline. + 5. Keep each caption short, meme-style. No quotation marks around them. + """ + + /// The payload for a refit: the joke as it stands plus the target structure. + public static func refitUserPayload( + description: String, captions: [String], slots: Int, templateName: String + ) -> String { + let count = MemeCaptionSlots.clamp(slots) + let current = captions.isEmpty + ? "(none yet)" + : captions.enumerated().map { "\($0.offset + 1). \($0.element)" }.joined(separator: "\n") + + return """ + Meme description: + \(description.trimmingCharacters(in: .whitespacesAndNewlines)) + + Current captions: + \(current) + + The new template is "\(templateName)" and it takes \(count) caption\(count == 1 ? "" : "s"). + Return exactly \(count). + """ + } + + /// Whether switching to a `slots`-slot template needs the refit round-trip. + /// + /// The fast path is the point: a same-count switch is the overwhelmingly common + /// one (two thirds of the corpus is two-slot), and paying an LLM call to be told + /// the captions are fine would make the strip feel slower than v5 for no gain. + /// Also false when there are no captions to refit — an empty box set is seeded + /// locally, not rewritten. + public static func needsRefit(captions: [String], slots: Int) -> Bool { + let meaningful = captions.filter { !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } + guard !meaningful.isEmpty else { return false } + return captions.count != MemeCaptionSlots.clamp(slots) + } + + /// Parse a refit reply into exactly `slots` captions. + /// + /// Returns nil rather than throwing a typed error: a refit that fails must leave + /// the user with the captions they already had, silently. The switch itself already + /// succeeded (the template is rendering) — turning a failed nicety into a visible + /// error would make a working action look broken. + /// + /// The result is always exactly `slots` long: padded with empty boxes when the + /// model returned too few, truncated when it returned too many. The caller seeds + /// boxes from this directly, so a length mismatch here would silently produce the + /// wrong number of boxes — the very bug being fixed. + public static func parseRefit(_ raw: String, slots: Int) -> [String]? { + let count = MemeCaptionSlots.clamp(slots) + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty, + let object = firstJSONObject(in: trimmed), + let data = object.data(using: .utf8), + let wire = try? JSONDecoder().decode(RankedWire.self, from: data) + else { return nil } + + let captions = wire.captions.map(clean) + // Nothing usable came back — the legacy top/bottom fallback in `RankedWire` + // means an object with no caption keys at all decodes to ["", ""], so an + // all-empty result has to be refused here rather than becoming blank boxes. + guard captions.contains(where: { !$0.isEmpty }) else { return nil } + + if captions.count >= count { return Array(captions.prefix(count)) } + return captions + Array(repeating: "", count: count - captions.count) + } + /// Extract the first balanced `{...}` run from a string, ignoring braces that /// appear inside JSON string literals (so a caption containing `{` can't end the /// scan early). Returns nil when there is no balanced object. diff --git a/OpenWhisp/Services/MemeCaptionLayout.swift b/OpenWhisp/Services/MemeCaptionLayout.swift index 17a1da8..67ba3ba 100644 --- a/OpenWhisp/Services/MemeCaptionLayout.swift +++ b/OpenWhisp/Services/MemeCaptionLayout.swift @@ -201,11 +201,166 @@ public enum MemeCaptionLayout { /// caption still sits inside the image, which is where v1's top/bottom blocks /// landed. An empty caption still gets a box so the editor has something to type /// into rather than making the user hunt for "Add text box". + /// + /// Kept as the 2-slot special case of `seedBoxes(captions:slots:)` so the classic + /// path is literally the same code and can't drift from it. public static func seedBoxes(topText: String, bottomText: String) -> [CaptionBox] { - [ - CaptionBox(text: topText, centerX: 0.5, centerY: 0.12), - CaptionBox(text: bottomText, centerX: 0.5, centerY: 0.88), - ] + seedBoxes(captions: [topText, bottomText], slots: 2) + } + + // MARK: - Per-template caption slots (v6) + + /// Where a template's `n` caption slots sit, as normalized centers. + /// + /// ## The honest state of the source data + /// + /// The task hoped memegen would supply real box POSITIONS. It does not — verified + /// against the live API on 2026-08-03. `GET /templates` returns + /// `{id, name, lines, overlays, styles, blank, example, source, keywords, _self}` + /// and the per-template `GET /templates/` returns the same shape; `lines` is a + /// COUNT and there is no geometry field anywhere in either payload. imgflip's + /// `get_memes` is the same story — `box_count`, no rectangles. (imgflip *does* + /// expose per-box geometry, but only through the authenticated `caption_image` + /// endpoint, which is a captioning API we deliberately don't use: sending the + /// user's words to someone else's server is exactly the local-first line this + /// plugin holds.) + /// + /// So EVERY position here is synthesized from the count. The doc comment says so + /// rather than implying the layout is template-accurate, because a wrong claim + /// about provenance is how the next person ships a "fix" that removes a fallback + /// that was never a fallback. + /// + /// ## The synthesized layouts, and why each shape + /// + /// * **1** — one centered caption near the top: the impact-font one-liner. + /// * **2** — classic top/bottom at 0.12 / 0.88. Unchanged from v1, because this is + /// two thirds of the corpus (166 of memegen's 212, 66 of imgflip's 100) and + /// regressing the common case to gain the rare one is a bad trade. + /// * **3 and 4** — a STACKED LEFT COLUMN: evenly spaced rows, left-aligned by + /// sitting at x = 0.30 with a narrower width. This is the spike-grade choice and + /// it is a genuine compromise, so here is the reasoning. The 3- and 4-slot + /// templates that matter (Drake, Distracted Boyfriend, Expanding Brain, Galaxy + /// Brain) are PANEL memes: their captions belong beside or inside stacked panels, + /// never spread top-to-bottom across the whole frame. A stacked column lands the + /// captions in roughly the right band for a vertically-panelled template (Drake, + /// Expanding Brain — the most common panel layout by far) and merely + /// *approximately* right for a horizontally-panelled one (Distracted Boyfriend). + /// Approximately-right and draggable beats confidently-wrong and invisible: the + /// user sees N captions in N distinct places and moves them, instead of getting + /// two captions for a four-panel joke. + /// * **5+** — the same even column at full width, since past four slots there is no + /// dominant convention left to approximate. + /// + /// The real fix is per-template geometry, which needs a data source none of the + /// key-less APIs provide — a bundled table of hand-measured boxes for the top ~30 + /// templates would do it, and that is a deliberate non-goal for a spike. + public static func slotCenters(slots: Int) -> [(x: Double, y: Double)] { + let count = MemeCaptionSlots.clamp(slots) + switch count { + case 1: + return [(0.5, 0.12)] + case 2: + return [(0.5, 0.12), (0.5, 0.88)] + default: + // Evenly spaced rows inside the frame, inset so the first and last aren't + // welded to the bezel. Panel memes read top-to-bottom, so slot order is + // top-to-bottom too — which is also the order the LLM returns captions in. + let top = 0.14 + let bottom = 0.86 + let span = bottom - top + let x = count <= 4 ? 0.30 : 0.5 + return (0.. Double { + MemeCaptionSlots.clamp(slots) <= 2 ? CaptionBox.defaultWidthShare : 0.46 + } + + /// The font share for an `n`-slot template. + /// + /// Stacked captions have to share the frame's height, so they start smaller — at + /// the classic 0.11 four captions would overlap before the user typed anything. + /// This is a CEILING (`layout` shrinks further to fit), so a short caption still + /// renders as large as it can. + public static func slotFontSizeShare(slots: Int) -> Double { + MemeCaptionSlots.clamp(slots) <= 2 ? CaptionBox.defaultFontSizeShare : 0.07 + } + + /// Seed one box per caption slot, laid out for a template of that structure (v6). + /// + /// The caption list is fitted to the slot count rather than trusted: a model that + /// returns three captions for a two-slot template has its extra dropped, and one + /// that returns two for a four-slot template gets two empty boxes to type into. + /// Both are better than the alternatives — rendering captions the template has no + /// room for, or silently losing the ones it does. + /// + /// Empty captions still get boxes, for the same reason `seedBoxes(topText:)` always + /// did: the editor needs a handle to type into. + public static func seedBoxes(captions: [String], slots: Int) -> [CaptionBox] { + let count = MemeCaptionSlots.clamp(slots) + let centers = slotCenters(slots: count) + let width = slotWidthShare(slots: count) + let fontSize = slotFontSizeShare(slots: count) + + return centers.enumerated().map { index, center in + CaptionBox( + text: index < captions.count ? captions[index] : "", + centerX: center.x, + centerY: center.y, + fontSizeShare: fontSize, + widthShare: width) + } + } + + // MARK: - What a regenerate is allowed to destroy (v6) + + /// Merge a fresh AI seed over the boxes already on the canvas. + /// + /// ## The rule, stated once: Generate replaces AI-seeded boxes and PRESERVES boxes + /// the user added. + /// + /// v5's Generate assigned `boxes = seedBoxes(...)` outright, so a user who had + /// pressed "Add text", typed a third caption, and positioned it, lost it the moment + /// they tweaked the description and regenerated. Silent destruction of manual work + /// is the worst class of bug in an editor, and it has no undo here. + /// + /// Of the two options in the brief — confirm before replacing, or preserve + /// user-added boxes — this takes the second, deliberately: + /// + /// * A confirmation dialog charges EVERY regenerate (the common, harmless case: + /// nothing was hand-added and the user just wants a rewrite) to protect the rare + /// one. Generate is the plugin's primary verb and putting a modal in front of it + /// would be felt on every single use. + /// * "Your own boxes survive, the AI's are rewritten" is a rule a user can hold in + /// their head and predict, which a dialog they dismiss reflexively is not. + /// * It is reversible in the direction that matters: an unwanted surviving box is + /// one click on the trash icon, whereas a destroyed caption is retyped and + /// repositioned from memory. + /// + /// A box counts as user-added when its id is not among `seededIDs` — the ids the + /// previous seed minted. EDITS to a seeded box (retyping it, dragging it, resizing + /// it) are NOT preserved: that box is the AI's answer to the old description, and a + /// regenerate is a request for a new answer, so keeping the old text would make + /// Generate look broken. The distinction is deliberate and is what keeps the rule + /// to one sentence. + /// + /// Preserved boxes keep their identity, text, and geometry exactly, and are + /// appended AFTER the new seed so panel order still reads top-to-bottom for the + /// slots the template actually has. + public static func merging( + seed: [CaptionBox], into existing: [CaptionBox], seededIDs: Set + ) -> [CaptionBox] { + let userAdded = existing.filter { !seededIDs.contains($0.id) } + return seed + userAdded } /// A box resolved into pixels for one specific image size, with its text already diff --git a/OpenWhisp/Services/MemeTemplateAffinity.swift b/OpenWhisp/Services/MemeTemplateAffinity.swift new file mode 100644 index 0000000..990bdda --- /dev/null +++ b/OpenWhisp/Services/MemeTemplateAffinity.swift @@ -0,0 +1,110 @@ +import Foundation + +/// What the user's own picks teach the ranker (spike v6). +/// +/// ## The signal, and why it is worth having +/// +/// Every time the user clicks past the model's first candidate — a different thumbnail +/// in the strip, or something else entirely from Browse — they have made a correction. +/// It is the cheapest supervision in the whole plugin: unambiguous, free, and produced +/// by the action the user was going to take anyway. v5 discarded all of it, so a user +/// who reached for the same template on every third meme got no better ranking on the +/// hundredth. +/// +/// So each such pick BOOSTS that template's prefilter score a little. The effect is +/// deliberately small and strictly bounded, because the failure mode of a learning +/// signal is worse than the failure mode of not having one: a boost that can outrun the +/// lexical score would eventually put the user's favourite template at the top of every +/// shortlist regardless of what they said, which is a personalized version of exactly +/// the confident-Drake bug this spike exists to kill. +/// +/// ## The bounds, and why each one +/// +/// * **`boostPerPick` (12)** is smaller than one keyword-token match (60) and far +/// smaller than a name-token match (100). One correction can therefore reorder +/// templates that scored *nearly the same*, and can never promote an unrelated +/// template over a relevant one. +/// * **`maximumBoost` (120)** caps the total at roughly two name-token matches, reached +/// after ten picks. Past that, picking the same template again changes nothing — the +/// signal saturates instead of compounding, which is what stops a long-lived store +/// from slowly taking over the ranking. +/// * **Only templates that ALREADY MATCH are boosted.** The boost is added inside +/// `MemeTemplateCatalog.ranked`, to templates whose lexical score is already above +/// zero — never to a zero-scoring one. This is the load-bearing rule: it means the +/// "no template matches" answer stays reachable, search never invents a hit, and a +/// favourite template cannot appear for a query that has nothing to do with it. +/// +/// Foundation-only and pure, so every one of those bounds is pinned by `swift test` +/// rather than asserted in a comment. The app layer owns only the JSON file. +public struct MemeTemplateAffinity: Equatable, Sendable, Codable { + + /// Accumulated boost per template id. Ids are source-qualified + /// (`MemeTemplateCatalog.qualifiedID`) so the same-named template from two + /// providers doesn't share a score. + public private(set) var boosts: [String: Int] + + public init(boosts: [String: Int] = [:]) { + self.boosts = boosts.compactMapValues { value in + let clamped = min(max(value, 0), Self.maximumBoost) + // A zero carries no information and would grow the file forever. + return clamped > 0 ? clamped : nil + } + } + + /// What one correction is worth. + public static let boostPerPick = 12 + + /// The ceiling on any single template's accumulated boost. + public static let maximumBoost = 120 + + /// How many picks it takes to saturate — derived, not a second constant that could + /// drift out of step with the two above. + public static var picksToSaturate: Int { + Int((Double(maximumBoost) / Double(boostPerPick)).rounded(.up)) + } + + /// The boost for one template. Zero for anything never picked. + public func boost(for templateID: String) -> Int { + boosts[templateID] ?? 0 + } + + /// Record that the user chose this template over the one that was offered first. + /// + /// Saturating rather than wrapping or growing: at the ceiling this is a no-op, so + /// a user who picks the same template a thousand times ends up exactly where they + /// were after ten. + public mutating func record(pick templateID: String) { + guard !templateID.isEmpty else { return } + boosts[templateID] = min(boost(for: templateID) + Self.boostPerPick, Self.maximumBoost) + } + + /// The same, as a value — for the pure call sites and the tests. + public func recording(pick templateID: String) -> MemeTemplateAffinity { + var copy = self + copy.record(pick: templateID) + return copy + } + + /// Forget everything learned. Wired to nothing yet; it exists so the store has an + /// honest way back, and so "the boosts got weird" is a recoverable state rather + /// than a reason to hand-edit JSON. + public mutating func reset() { boosts = [:] } + + /// Decoding re-applies the clamp, so a hand-edited or corrupt file can't inject a + /// boost large enough to dominate the ranking. The store is user-writable by + /// design (it's a spike, and the file is meant to be inspectable), which makes this + /// the boundary where the cap has to be enforced rather than assumed. + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let raw = (try? container.decode([String: Int].self)) ?? [:] + self.init(boosts: raw) + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(boosts) + } + + /// The filename under the plugin's own directory. + public static let fileName = "template-affinity.json" +} diff --git a/OpenWhisp/Services/MemeTemplateMatcher.swift b/OpenWhisp/Services/MemeTemplateMatcher.swift index e57a7b2..d6c0efb 100644 --- a/OpenWhisp/Services/MemeTemplateMatcher.swift +++ b/OpenWhisp/Services/MemeTemplateMatcher.swift @@ -31,9 +31,32 @@ public struct MemeTemplate: Equatable, Sendable, Codable, Identifiable { /// multi-lingual corpus findable — see `MemeTemplateCatalog.search`. public let keywords: [String] + /// How many caption slots this template actually has (v6). + /// + /// ## Why this exists + /// + /// Up to v5 every meme was captioned top-and-bottom, because that is what the LLM + /// was asked for and what `seedBoxes` produced. That is *wrong for most of the + /// corpus*: Drake is two SIDE labels, Distracted Boyfriend is three, Expanding + /// Brain is four. Rendering a four-panel meme with a top line and a bottom line + /// isn't a stylistic choice, it's a broken meme — the joke lives in the per-panel + /// captions. + /// + /// Both remote sources carried this all along and v5 discarded it: memegen's + /// `/templates` ships `lines`, imgflip's `get_memes` ships `box_count`. Now they + /// are decoded into this field, the LLM is asked for exactly this many captions, + /// and `MemeCaptionLayout.seedBoxes(slots:)` lays out that many boxes. + /// + /// Defaults to `MemeCaptionSlots.default` (2) so an older cache, a user-library + /// import, or a source that doesn't report it still behaves exactly as it did. + /// Clamped at construction — see `MemeCaptionSlots.clamp` — because a wire value + /// of 0 (or 40) must not become 0 caption boxes (or 40). + public let captionSlots: Int + public init( id: String, name: String, url: String, width: Int, height: Int, - source: MemeTemplateSource = .imgflip, keywords: [String] = [] + source: MemeTemplateSource = .imgflip, keywords: [String] = [], + captionSlots: Int = MemeCaptionSlots.default ) { self.id = id self.name = name @@ -42,6 +65,7 @@ public struct MemeTemplate: Equatable, Sendable, Codable, Identifiable { self.height = height self.source = source self.keywords = keywords + self.captionSlots = MemeCaptionSlots.clamp(captionSlots) } /// Decoding tolerates a missing `source`/`keywords` so a catalog cache written by @@ -57,6 +81,41 @@ public struct MemeTemplate: Equatable, Sendable, Codable, Identifiable { height = (try? c.decode(Int.self, forKey: .height)) ?? 0 source = (try? c.decode(MemeTemplateSource.self, forKey: .source)) ?? .imgflip keywords = (try? c.decode([String].self, forKey: .keywords)) ?? [] + // A cache written by a v5 build has no `captionSlots`. Defaulting rather than + // failing keeps the offline path working across the upgrade — the cache is a + // performance artifact, and discarding it would make the first launch after an + // update look like the offline bug this plugin already fixed once. + captionSlots = MemeCaptionSlots.clamp( + (try? c.decode(Int.self, forKey: .captionSlots)) ?? MemeCaptionSlots.default) + } +} + +/// How many caption slots a template has, and what a sane value looks like (v6). +/// +/// A tiny namespace rather than loose constants because the clamp is a RULE with a +/// reason, applied at three boundaries (imgflip's `box_count`, memegen's `lines`, and +/// the cache decoder) and it must agree at all three. +public enum MemeCaptionSlots { + + /// What a template gets when its source doesn't say — the classic top/bottom meme. + /// + /// Two, because that is what every caption path did before slots existed: a + /// template with unknown structure must degrade to v5's behaviour, not to a guess. + public static let `default` = 2 + + /// The floor. A template with zero caption slots would seed zero boxes and give + /// the user a picture with no way to type on it — the source reporting `0` (or a + /// negative, or a corrupt cache) must never produce that dead end. + public static let minimum = 1 + + /// The ceiling. memegen reports up to 8 `lines`; the cap exists so a wire value + /// nobody anticipated can't seed a screenful of boxes the user has to delete by + /// hand. Real templates top out at 8, so this clips nothing that exists today. + public static let maximum = 8 + + /// Bring any reported count into range. + public static func clamp(_ raw: Int) -> Int { + min(max(raw, minimum), maximum) } } @@ -72,6 +131,27 @@ public struct MemeTemplateCatalogResponse: Decodable, Sendable { public let url: String public let width: Int public let height: Int + /// How many caption boxes the template really has (v6). imgflip has shipped + /// this on `get_memes` all along and v5 threw it away, which is why Distracted + /// Boyfriend (3) and Expanding Brain (4) were captioned top-and-bottom. + /// Optional so a response missing it decodes to the 2-slot default rather than + /// failing the whole catalog. + public let boxCount: Int? + + private enum CodingKeys: String, CodingKey { + case id, name, url, width, height + case boxCount = "box_count" + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + id = try c.decode(String.self, forKey: .id) + name = try c.decode(String.self, forKey: .name) + url = try c.decode(String.self, forKey: .url) + width = (try? c.decode(Int.self, forKey: .width)) ?? 0 + height = (try? c.decode(Int.self, forKey: .height)) ?? 0 + boxCount = try? c.decode(Int.self, forKey: .boxCount) + } } public struct Payload: Decodable, Sendable { public let memes: [Wire] @@ -87,7 +167,8 @@ public struct MemeTemplateCatalogResponse: Decodable, Sendable { id: MemeTemplateCatalog.qualifiedID(.imgflip, wire.id), name: wire.name, url: wire.url, width: wire.width, height: wire.height, - source: .imgflip, keywords: []) + source: .imgflip, keywords: [], + captionSlots: wire.boxCount ?? MemeCaptionSlots.default) } } } @@ -111,8 +192,17 @@ public struct MemegenTemplateResponse: Decodable, Sendable { public let blank: String /// Alternate names — the field that makes a merged corpus searchable. public let keywords: [String]? + /// How many caption lines the template takes (v6). + /// + /// memegen's `/templates` reports this per template — verified against the live + /// API: of its 212 templates, 166 are 2-line, 23 are 3-line, 7 are 4-line, and + /// the rest spread over 1/5/6/8. It ships the COUNT only; there is no box + /// geometry anywhere in the payload (the fields are `lines`, `overlays`, + /// `styles`, `blank`, `example`, `source`, `keywords`), so positions have to be + /// synthesized — see `MemeCaptionLayout.slotCenters`. + public let lines: Int? - private enum CodingKeys: String, CodingKey { case id, name, blank, keywords } + private enum CodingKeys: String, CodingKey { case id, name, blank, keywords, lines } public init(from decoder: Decoder) throws { let c = try decoder.container(keyedBy: CodingKeys.self) @@ -120,6 +210,7 @@ public struct MemegenTemplateResponse: Decodable, Sendable { name = try c.decode(String.self, forKey: .name) blank = try c.decode(String.self, forKey: .blank) keywords = try? c.decode([String].self, forKey: .keywords) + lines = try? c.decode(Int.self, forKey: .lines) } } @@ -141,7 +232,8 @@ public struct MemegenTemplateResponse: Decodable, Sendable { // decoded NSImage's real pixel size), so they stay honest zeros // rather than invented defaults. width: 0, height: 0, - source: .memegen, keywords: wire.keywords ?? []) + source: .memegen, keywords: wire.keywords ?? [], + captionSlots: wire.lines ?? MemeCaptionSlots.default) } } } diff --git a/OpenWhisp/Services/MemeTemplateProvider.swift b/OpenWhisp/Services/MemeTemplateProvider.swift index 0e15c28..7d3039a 100644 --- a/OpenWhisp/Services/MemeTemplateProvider.swift +++ b/OpenWhisp/Services/MemeTemplateProvider.swift @@ -146,13 +146,18 @@ public enum MemeTemplateCatalog { /// catalog order IS the popularity ranking, so it is the right thing to fall back /// on when two templates match a query equally well. public static func ranked( - _ query: String, in catalog: [MemeTemplate], limit: Int + _ query: String, in catalog: [MemeTemplate], limit: Int, + affinity: MemeTemplateAffinity = MemeTemplateAffinity() ) -> [Match] { let normalizedQuery = MemeTemplateMatcher.normalize(query) let cap = max(0, limit) guard cap > 0 else { return [] } // An empty query isn't a failed search — it's "no filter applied", which the - // Browse grid renders as the whole corpus in popularity order. + // Browse grid renders as the whole corpus in popularity order. Note the + // affinity boost deliberately does NOT reorder this: an unfiltered Browse grid + // is the corpus in popularity order, and quietly floating the user's favourites + // to the top of it would make the grid's order mean two different things + // depending on whether the search box happened to be empty. guard !normalizedQuery.isEmpty else { return catalog.prefix(cap).map { Match(template: $0, score: 0) } } @@ -175,8 +180,15 @@ public enum MemeTemplateCatalog { for (index, template) in catalog.enumerated() { let value = score(queryTokens: queryTokens, normalizedQuery: normalizedQuery, template: template) + // The affinity boost applies ONLY to a template that already matched (v6). + // This guard is the whole safety property: a learned preference can reorder + // things the query already found, and can never conjure a hit for a query + // that found nothing. "No template matches" therefore stays reachable no + // matter how much the store has learned. guard value > 0 else { continue } - scored.append(Scored(index: index, template: template, score: value)) + scored.append(Scored( + index: index, template: template, + score: value + affinity.boost(for: template.id))) } scored.sort { left, right in @@ -265,13 +277,22 @@ public enum MemeTemplateCatalog { /// Falls back to popularity order when the description matches nothing — the model /// still deserves a corpus to choose from, and the UI already states plainly when /// nothing matched. + /// + /// ## v6 — the user's own picks tilt this + /// + /// `affinity` adds a small, capped boost to templates the user has previously + /// chosen over the model's first suggestion. It is applied inside `ranked`, only to + /// templates that already matched the description, so it changes the ORDER of the + /// shortlist and never its membership rule. See `MemeTemplateAffinity` for the + /// bounds and why each exists. public static func prefilter( - for description: String, in catalog: [MemeTemplate], limit: Int + for description: String, in catalog: [MemeTemplate], limit: Int, + affinity: MemeTemplateAffinity = MemeTemplateAffinity() ) -> [MemeTemplate] { let cap = max(0, limit) guard cap > 0 else { return [] } - let hits = ranked(description, in: catalog, limit: cap).map(\.template) + let hits = ranked(description, in: catalog, limit: cap, affinity: affinity).map(\.template) guard !hits.isEmpty else { return Array(catalog.prefix(cap)) } // Top up with popular templates when the query was narrow, so the model always @@ -319,4 +340,17 @@ public enum MemeTemplateCatalog { return "\(template.name) (\(keywords.prefix(6).joined(separator: ", ")))" } } + + /// The caption-slot count per shortlisted template, positionally aligned with + /// `promptLines` and `promptNames` (v6). + /// + /// Three parallel arrays rather than one array of triples because the call site + /// hands each to a different consumer (the prompt gets lines, the resolver gets + /// names, the annotator gets slots) and they must all be sliced by the same limit. + /// `slotAnnotatedLines` is the only thing that zips two of them, and it tolerates a + /// short `slots` array by defaulting — so a future limit mismatch degrades to "2 + /// captions" rather than crashing. + public static func promptSlots(_ catalog: [MemeTemplate], limit: Int) -> [Int] { + catalog.prefix(max(0, limit)).map(\.captionSlots) + } } diff --git a/Package.swift b/Package.swift index 15c99df..8cfeb8f 100644 --- a/Package.swift +++ b/Package.swift @@ -180,6 +180,7 @@ let package = Package( "MemeUserLibrary.swift", "MemeGenerationState.swift", "MemeCatalogCache.swift", + "MemeTemplateAffinity.swift", "LLMWarmReadiness.swift" ] ), diff --git a/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift b/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift index 894392f..31bc15b 100644 --- a/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift +++ b/Tests/OpenWhispCoreTests/MemeGeneratorTests.swift @@ -287,9 +287,18 @@ final class MemeGeneratorTests: XCTestCase { // MARK: - v2: the ranked prompt - func testRankedPromptForbidsInventingNamesAndPinsCaptionLanguage() { + /// The two guards that must survive every prompt revision: the model may not + /// invent a candidate, and it may not translate the captions. + /// + /// **v6 note.** The anti-invention guard used to be "names COPIED EXACTLY from the + /// list". v6 asks for NUMBERS instead — the same guard, expressed in a form a small + /// local model can actually satisfy (see `MemeAI.rankedPrompt`), so the assertion + /// moved to the numbering rather than being dropped. The language guard is + /// unchanged and is the one that stops a tiny model translating a Ukrainian + /// dictation into English (`llm-cleanup-language-guard`). + func testRankedPromptForbidsInventingCandidatesAndPinsCaptionLanguage() { let prompt = MemeAI.rankedPrompt.lowercased() - XCTAssertTrue(prompt.contains("copied exactly")) + XCTAssertTrue(prompt.contains("numbers from the list")) XCTAssertTrue(prompt.contains("do not invent")) XCTAssertTrue(prompt.contains("same language")) XCTAssertTrue(prompt.contains("do not translate")) diff --git a/Tests/OpenWhispCoreTests/MemeStructureTests.swift b/Tests/OpenWhispCoreTests/MemeStructureTests.swift new file mode 100644 index 0000000..822f9a0 --- /dev/null +++ b/Tests/OpenWhispCoreTests/MemeStructureTests.swift @@ -0,0 +1,773 @@ +import XCTest +@testable import OpenWhispCore + +/// Covers the v6 algorithm upgrade to the Meme Generator plugin (spike/plugin-system): +/// per-template caption STRUCTURE, numbered candidate references, the caption refit on +/// a candidate switch, the regenerate-preservation rule, and the learned per-template +/// affinity boost. +/// +/// The through-line for all five: v5's pipeline hard-coded "every meme is top and +/// bottom, and the model copies names verbatim". Both assumptions were wrong for most +/// of the corpus, and both had the data to do better sitting unused on the wire. +final class MemeStructureTests: XCTestCase { + + // MARK: - Slot counts off the wire + // + // Both catalogs have carried the structure all along; v5 decoded it into nothing. + + func testImgflipBoxCountBecomesCaptionSlots() throws { + let json = """ + {"success":true,"data":{"memes":[ + {"id":"1","name":"Distracted Boyfriend","url":"u","width":1200,"height":800,"box_count":3}, + {"id":"2","name":"Expanding Brain","url":"u","width":857,"height":1202,"box_count":4} + ]}} + """ + let decoded = try JSONDecoder().decode( + MemeTemplateCatalogResponse.self, from: Data(json.utf8)) + XCTAssertEqual(decoded.templates.map(\.captionSlots), [3, 4]) + } + + /// A response without the field must not fail the whole catalog — it degrades to + /// the classic two-slot meme, which is what v5 did for everything. + func testImgflipWithoutBoxCountDefaultsToTwoSlots() throws { + let json = """ + {"success":true,"data":{"memes":[ + {"id":"1","name":"Drake","url":"u","width":1200,"height":1200} + ]}} + """ + let decoded = try JSONDecoder().decode( + MemeTemplateCatalogResponse.self, from: Data(json.utf8)) + XCTAssertEqual(decoded.templates.first?.captionSlots, MemeCaptionSlots.default) + XCTAssertEqual(MemeCaptionSlots.default, 2) + } + + /// memegen calls the same thing `lines`. Shapes taken from the live API. + func testMemegenLinesBecomeCaptionSlots() throws { + let json = """ + [ + {"id":"db","name":"Distracted Boyfriend","blank":"b","keywords":[],"lines":3}, + {"id":"gb","name":"Galaxy Brain","blank":"b","keywords":[],"lines":4}, + {"id":"drake","name":"Drakeposting","blank":"b","keywords":[],"lines":2} + ] + """ + let decoded = try JSONDecoder().decode( + MemegenTemplateResponse.self, from: Data(json.utf8)) + XCTAssertEqual(decoded.templates.map(\.captionSlots), [3, 4, 2]) + } + + func testMemegenWithoutLinesDefaultsToTwoSlots() throws { + let json = """ + [{"id":"x","name":"Something","blank":"b","keywords":[]}] + """ + let decoded = try JSONDecoder().decode( + MemegenTemplateResponse.self, from: Data(json.utf8)) + XCTAssertEqual(decoded.templates.first?.captionSlots, 2) + } + + /// A source reporting 0 must never produce a template with no way to type on it. + func testZeroAndNegativeSlotCountsAreClampedUp() throws { + let json = """ + {"success":true,"data":{"memes":[ + {"id":"1","name":"Zero","url":"u","width":1,"height":1,"box_count":0}, + {"id":"2","name":"Negative","url":"u","width":1,"height":1,"box_count":-4} + ]}} + """ + let decoded = try JSONDecoder().decode( + MemeTemplateCatalogResponse.self, from: Data(json.utf8)) + XCTAssertEqual(decoded.templates.map(\.captionSlots), + [MemeCaptionSlots.minimum, MemeCaptionSlots.minimum]) + } + + /// And an absurd one must not seed a screenful of boxes to delete by hand. + func testAbsurdSlotCountIsClampedDown() { + let template = MemeTemplate( + id: "x", name: "X", url: "u", width: 1, height: 1, captionSlots: 400) + XCTAssertEqual(template.captionSlots, MemeCaptionSlots.maximum) + XCTAssertEqual(MemeCaptionSlots.maximum, 8) + } + + /// A cache written by a v5 build has no `captionSlots`. Dropping it would make the + /// first launch after the upgrade look like the offline bug the plugin already fixed. + func testAV5CacheWithoutSlotsStillDecodesAtTheDefault() throws { + let json = """ + {"id":"imgflip:1","name":"Drake","url":"u","width":1200,"height":1200, + "source":"imgflip","keywords":[]} + """ + let decoded = try JSONDecoder().decode(MemeTemplate.self, from: Data(json.utf8)) + XCTAssertEqual(decoded.captionSlots, MemeCaptionSlots.default) + } + + func testCaptionSlotsRoundTripThroughTheCache() throws { + let original = MemeTemplate( + id: "memegen:gb", name: "Galaxy Brain", url: "u", width: 0, height: 0, + source: .memegen, keywords: [], captionSlots: 4) + let data = try JSONEncoder().encode(original) + let decoded = try JSONDecoder().decode(MemeTemplate.self, from: data) + XCTAssertEqual(decoded.captionSlots, 4) + } + + // MARK: - Slot geometry + // + // Every position is SYNTHESIZED from the count: neither key-less API ships box + // geometry (verified against both live endpoints). These tests pin the shapes so a + // later change to them is a deliberate act. + + func testTwoSlotsKeepTheClassicTopAndBottomLayout() { + let centers = MemeCaptionLayout.slotCenters(slots: 2) + XCTAssertEqual(centers.count, 2) + XCTAssertEqual(centers[0].x, 0.5) + XCTAssertEqual(centers[0].y, 0.12) + XCTAssertEqual(centers[1].y, 0.88) + } + + func testOneSlotIsASingleCenteredCaption() { + let centers = MemeCaptionLayout.slotCenters(slots: 1) + XCTAssertEqual(centers.count, 1) + XCTAssertEqual(centers[0].x, 0.5) + } + + /// The panel-meme layout: N distinct, evenly spaced positions inside the frame. + func testPanelSlotsAreDistinctEvenlySpacedAndInsideTheFrame() { + for count in 3...MemeCaptionSlots.maximum { + let centers = MemeCaptionLayout.slotCenters(slots: count) + XCTAssertEqual(centers.count, count, "slot count \(count)") + + let ys = centers.map(\.y) + XCTAssertEqual(Set(ys.map { Int($0 * 1000) }).count, count, + "slot \(count): every caption needs its own position") + XCTAssertTrue(ys.allSatisfy { $0 > 0 && $0 < 1 }, + "slot \(count): captions must sit inside the image") + XCTAssertEqual(ys, ys.sorted(), + "slot \(count): panel order must read top-to-bottom") + + // Evenly spaced: every gap the same, so no two captions crowd. + let gaps = zip(ys.dropFirst(), ys).map { $0 - $1 } + for gap in gaps { XCTAssertEqual(gap, gaps[0], accuracy: 0.0001) } + } + } + + /// Three and four slots are the panel-meme case: narrower boxes in a left column, + /// smaller type so four captions don't overlap before anything is typed. + func testPanelLayoutsUseNarrowerBoxesAndSmallerTypeThanTheClassicPair() { + XCTAssertEqual(MemeCaptionLayout.slotWidthShare(slots: 2), + MemeCaptionLayout.CaptionBox.defaultWidthShare) + XCTAssertEqual(MemeCaptionLayout.slotFontSizeShare(slots: 2), + MemeCaptionLayout.CaptionBox.defaultFontSizeShare) + + XCTAssertLessThan(MemeCaptionLayout.slotWidthShare(slots: 4), + MemeCaptionLayout.slotWidthShare(slots: 2)) + XCTAssertLessThan(MemeCaptionLayout.slotFontSizeShare(slots: 4), + MemeCaptionLayout.slotFontSizeShare(slots: 2)) + } + + func testSlotGeometryIsClampedLikeEveryOtherSlotCount() { + XCTAssertEqual(MemeCaptionLayout.slotCenters(slots: 0).count, MemeCaptionSlots.minimum) + XCTAssertEqual(MemeCaptionLayout.slotCenters(slots: 99).count, MemeCaptionSlots.maximum) + } + + // MARK: - Seeding boxes from slots + + func testSeedingProducesOneBoxPerSlotInPanelOrder() { + let boxes = MemeCaptionLayout.seedBoxes( + captions: ["one", "two", "three", "four"], slots: 4) + XCTAssertEqual(boxes.map(\.text), ["one", "two", "three", "four"]) + XCTAssertEqual(boxes.map(\.centerY), boxes.map(\.centerY).sorted()) + } + + /// A model that returns too many captions for the template must not render captions + /// the template has no room for. + func testExtraCaptionsBeyondTheSlotCountAreDropped() { + let boxes = MemeCaptionLayout.seedBoxes( + captions: ["a", "b", "c", "d"], slots: 2) + XCTAssertEqual(boxes.map(\.text), ["a", "b"]) + } + + /// And one that returns too few must leave empty boxes to type into, not fewer boxes. + func testTooFewCaptionsStillFillEverySlotWithAnEmptyBox() { + let boxes = MemeCaptionLayout.seedBoxes(captions: ["only"], slots: 4) + XCTAssertEqual(boxes.count, 4) + XCTAssertEqual(boxes.map(\.text), ["only", "", "", ""]) + } + + /// The classic entry point must be exactly the 2-slot case, so the common path + /// can't drift away from the general one. + func testTheClassicTopBottomSeedIsTheTwoSlotCase() { + let classic = MemeCaptionLayout.seedBoxes(topText: "up", bottomText: "down") + let general = MemeCaptionLayout.seedBoxes(captions: ["up", "down"], slots: 2) + XCTAssertEqual(classic.map(\.text), general.map(\.text)) + XCTAssertEqual(classic.map(\.centerY), general.map(\.centerY)) + XCTAssertEqual(classic.map(\.centerX), general.map(\.centerX)) + XCTAssertEqual(classic.count, 2) + } + + // MARK: - Numbered candidate references + // + // The v6 contract: the model answers with INDICES into the shortlist it was shown. + // Copying names verbatim is a transcription task, and it is the single most fragile + // thing a tiny local model can be asked to do. + + private let shortlist = [ + "Drake Hotline Bling", "Distracted Boyfriend", "Two Buttons", "Success Kid", + ] + + func testNumberedCandidatesResolveToTheShortlistEntriesTheyIndex() { + let result = MemeAI.parseRanked(""" + {"templates":[3,1],"captions":["ship it","test it"]} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.templateNames, ["Two Buttons", "Drake Hotline Bling"]) + } + + /// The numbering is 1-based because that is what the payload prints. An off-by-one + /// here would return the model's neighbour on every pick — plausible-looking wrong + /// templates rather than an error, which is the worst kind to hunt. + func testTheNumberingIsOneBasedMatchingWhatThePayloadPrints() { + let payload = MemeAI.rankedUserPayload( + description: "anything", templateNames: shortlist) + XCTAssertTrue(payload.contains("1. Drake Hotline Bling")) + + let result = MemeAI.parseRanked( + "{\"templates\":[1],\"captions\":[\"x\"]}", catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.templateNames, ["Drake Hotline Bling"]) + XCTAssertEqual(MemeAI.firstCandidateNumber, 1) + } + + /// A number past the end of the list is a miscount or an invention. Clamping it to + /// the last entry would be v1's confident-Drake bug wearing a number. + func testOutOfRangeNumbersAreDroppedRatherThanClamped() { + let result = MemeAI.parseRanked(""" + {"templates":[99,0,-3,2],"captions":["x"]} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.templateNames, ["Distracted Boyfriend"]) + } + + func testAnAllOutOfRangeAnswerLeavesNoUsableTemplate() { + let result = MemeAI.parseRanked(""" + {"templates":[40,41],"captions":["still a joke"]} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertTrue(spec.hasNoUsableTemplate) + XCTAssertEqual(spec.captions, ["still a joke"]) + } + + /// Backward compatibility: a model that ignores the numbering is no worse off. + func testExactNamesAreStillAccepted() { + let result = MemeAI.parseRanked(""" + {"templates":["Two Buttons","Success Kid"],"captions":["a","b"]} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.templateNames, ["Two Buttons", "Success Kid"]) + } + + func testNumbersAndNamesMayBeMixedInOneAnswer() { + let result = MemeAI.parseRanked(""" + {"templates":[3,"Success Kid"],"captions":["a"]} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.templateNames, ["Two Buttons", "Success Kid"]) + } + + /// A model asked for numbers answering `["3"]` meant the third template — looking + /// for a catalog entry NAMED "3" would silently throw the pick away. + func testANumericStringIsTreatedAsAnIndexNotAName() { + let result = MemeAI.parseRanked(""" + {"templates":["2"],"captions":["a"]} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.templateNames, ["Distracted Boyfriend"]) + } + + /// The dedupe is on the RESOLVED template, not on how it was written — otherwise a + /// number and its name would occupy two slots in a five-slot strip. + func testANumberAndItsNameCollapseToOneCandidate() { + let result = MemeAI.parseRanked(""" + {"templates":[3,"two buttons",3],"captions":["a"]} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.templateNames, ["Two Buttons"]) + } + + func testNumberedCandidatesAreCappedAtFive() { + let long = (1...12).map { "T\($0)" } + let result = MemeAI.parseRanked(""" + {"templates":[1,2,3,4,5,6,7,8],"captions":["a"]} + """, catalogNames: long) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.templateNames.count, MemeAI.maxCandidates) + XCTAssertEqual(spec.templateNames, ["T1", "T2", "T3", "T4", "T5"]) + } + + /// One element of a shape we don't understand must not cost the user the good + /// candidates beside it. + func testAnUnparseableElementIsDroppedWithoutFailingTheWholeAnswer() { + let result = MemeAI.parseRanked(""" + {"templates":[1,{"name":"weird"},null,3],"captions":["a"]} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.templateNames, ["Drake Hotline Bling", "Two Buttons"]) + } + + func testAnEmptyShortlistResolvesNothingRatherThanCrashing() { + XCTAssertTrue(MemeAI.resolve([.index(1), .name("Drake")], shortlist: []).isEmpty) + } + + // MARK: - Caption arrays and the legacy shape + + func testCaptionsArriveAsAnArrayInPanelOrder() { + let result = MemeAI.parseRanked(""" + {"templates":[1],"captions":["one","two","three","four"]} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.captions, ["one", "two", "three", "four"]) + } + + /// v5's shape, and the one any model that has seen a two-line meme reaches for. + func testLegacyTopAndBottomTextDecodeAsATwoSlotResponse() { + let result = MemeAI.parseRanked(""" + {"templates":[1],"top_text":"ship it","bottom_text":"test it"} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.captions, ["ship it", "test it"]) + XCTAssertEqual(spec.topText, "ship it") + XCTAssertEqual(spec.bottomText, "test it") + } + + /// A response carrying BOTH keeps the richer answer. + func testACaptionsArrayWinsOverStrayLegacyKeys() { + let result = MemeAI.parseRanked(""" + {"templates":[1],"captions":["a","b","c"],"top_text":"ignored","bottom_text":"also"} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.captions, ["a", "b", "c"]) + } + + /// A single-line meme in the legacy shape must not seed a trailing blank box. + func testATrailingEmptyCaptionIsDropped() { + let result = MemeAI.parseRanked(""" + {"templates":[1],"top_text":"one liner","bottom_text":""} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.captions, ["one liner"]) + } + + /// An INTERIOR empty means "this panel has no caption" — shifting the next one up + /// into its place would relabel the wrong panel. + func testAnInteriorEmptyCaptionIsKeptSoPanelsDoNotShift() { + let result = MemeAI.parseRanked(""" + {"templates":[1],"captions":["first","","third"]} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.captions, ["first", "", "third"]) + } + + func testTheModelsReasonSurvivesForTheStripTooltip() { + let result = MemeAI.parseRanked(""" + {"templates":[1],"captions":["a"],"reason":"the two-choice shape fits the dilemma"} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.reason, "the two-choice shape fits the dilemma") + } + + /// The reason is a nicety, never a reason to reject an otherwise good answer. + func testAMissingReasonIsNotAFailure() { + let result = MemeAI.parseRanked(""" + {"templates":[1],"captions":["a"]} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertTrue(spec.reason.isEmpty) + } + + func testAnAnswerWithNeitherTemplateNorCaptionIsStillRejected() { + let result = MemeAI.parseRanked(""" + {"templates":[99],"captions":["",""]} + """, catalogNames: shortlist) + guard case .failure(let rejection) = result else { return XCTFail("expected failure") } + XCTAssertEqual(rejection, .missingFields) + } + + // MARK: - The prompt and the payload + + /// The discarded "think first" invitation is gone; the reasoning it asked for is + /// now a `reason` the strip actually shows. + func testThePromptAsksForNumbersACaptionArrayAndAVisibleReason() { + XCTAssertTrue(MemeAI.rankedPrompt.contains("NUMBERS")) + XCTAssertTrue(MemeAI.rankedPrompt.contains("\"captions\"")) + XCTAssertTrue(MemeAI.rankedPrompt.contains("\"reason\"")) + XCTAssertTrue(MemeAI.rankedPrompt.contains("shown"), + "the reason has to be described as user-facing") + XCTAssertFalse(MemeAI.rankedPrompt.contains("Think about which"), + "v5's discarded reasoning invitation must be gone") + } + + /// Only NON-default counts are annotated — tagging every two-slot template would be + /// noise on the models whose attention is the scarce resource. + func testOnlyNonDefaultSlotCountsAreAnnotatedInThePrompt() { + let annotated = MemeAI.slotAnnotatedLines( + ["Drake", "Distracted Boyfriend", "Expanding Brain"], slots: [2, 3, 4]) + XCTAssertEqual(annotated, [ + "Drake", + "Distracted Boyfriend [3 captions]", + "Expanding Brain [4 captions]", + ]) + } + + func testASlotArrayShorterThanTheLinesDegradesToTheDefault() { + let annotated = MemeAI.slotAnnotatedLines(["A", "B"], slots: [4]) + XCTAssertEqual(annotated, ["A [4 captions]", "B"]) + } + + func testThePayloadCarriesTheSlotCountsAndExplainsTheUnmarkedCase() { + let payload = MemeAI.rankedUserPayload( + description: "two brains", templateLines: ["Drake", "Galaxy Brain"], slots: [2, 4]) + XCTAssertTrue(payload.contains("1. Drake\n")) + XCTAssertTrue(payload.contains("2. Galaxy Brain [4 captions]")) + XCTAssertTrue(payload.contains("two brains")) + XCTAssertTrue(payload.contains("unmarked one takes 2")) + } + + // MARK: - Refit on a candidate switch + // + // The strip promises "same joke, different template". That held while everything + // was two-slot and breaks the moment structure varies. + + func testNoRefitIsNeededWhenTheSlotCountMatches() { + XCTAssertFalse(MemeAI.needsRefit(captions: ["a", "b"], slots: 2)) + } + + func testARefitIsNeededWhenTheSlotCountDiffers() { + XCTAssertTrue(MemeAI.needsRefit(captions: ["a", "b"], slots: 4)) + XCTAssertTrue(MemeAI.needsRefit(captions: ["a", "b", "c", "d"], slots: 2)) + } + + /// Nothing to refit: an empty box set is seeded locally, not rewritten by an LLM. + func testNoRefitForCaptionsThatAreAllEmpty() { + XCTAssertFalse(MemeAI.needsRefit(captions: ["", " "], slots: 4)) + XCTAssertFalse(MemeAI.needsRefit(captions: [], slots: 4)) + } + + func testRefitNeedIsJudgedAgainstTheClampedSlotCount() { + // A 1-caption box set against a template reporting 0 slots clamps to 1 — equal, + // so no round-trip. + XCTAssertFalse(MemeAI.needsRefit(captions: ["a"], slots: 0)) + } + + func testTheRefitPayloadCarriesTheJokeTheCurrentCaptionsAndTheTarget() { + let payload = MemeAI.refitUserPayload( + description: "rust versus python", captions: ["rust", "python"], + slots: 4, templateName: "Galaxy Brain") + XCTAssertTrue(payload.contains("rust versus python")) + XCTAssertTrue(payload.contains("1. rust")) + XCTAssertTrue(payload.contains("2. python")) + XCTAssertTrue(payload.contains("Galaxy Brain")) + XCTAssertTrue(payload.contains("4 captions")) + XCTAssertTrue(payload.contains("Return exactly 4")) + } + + func testTheRefitPromptPinsTheLanguageAndForbidsPadding() { + XCTAssertTrue(MemeAI.refitPrompt.contains("SAME language")) + XCTAssertTrue(MemeAI.refitPrompt.contains("EXACTLY")) + XCTAssertTrue(MemeAI.refitPrompt.contains("do not repeat")) + } + + /// The result is always exactly `slots` long — the caller seeds boxes straight from + /// it, so a length mismatch would produce the wrong number of boxes. + func testARefitIsPaddedUpToTheSlotCount() { + let captions = MemeAI.parseRefit(""" + {"captions":["one","two"]} + """, slots: 4) + XCTAssertEqual(captions, ["one", "two", "", ""]) + } + + func testARefitIsTruncatedDownToTheSlotCount() { + let captions = MemeAI.parseRefit(""" + {"captions":["a","b","c","d"]} + """, slots: 2) + XCTAssertEqual(captions, ["a", "b"]) + } + + func testARefitDigsItsJSONOutOfProse() { + let captions = MemeAI.parseRefit(""" + Sure! Here you go: + ```json + {"captions":["small brain","big brain","galaxy brain"]} + ``` + """, slots: 3) + XCTAssertEqual(captions, ["small brain", "big brain", "galaxy brain"]) + } + + /// A failed refit must leave the user with the captions they already had — the + /// switch itself succeeded, so nil means "keep what's there", never an error state. + func testAnUnusableRefitReplyIsRefusedRatherThanBlankingTheCaptions() { + XCTAssertNil(MemeAI.parseRefit("sorry, I can't do that", slots: 3)) + XCTAssertNil(MemeAI.parseRefit("", slots: 3)) + XCTAssertNil(MemeAI.parseRefit("{\"captions\":[]}", slots: 3)) + XCTAssertNil(MemeAI.parseRefit("{\"captions\":[\"\",\"\"]}", slots: 3)) + XCTAssertNil(MemeAI.parseRefit("{\"unrelated\":true}", slots: 3)) + } + + // MARK: - Regenerate preservation + // + // The rule: Generate replaces AI-seeded boxes and PRESERVES boxes the user added. + + func testRegenerateReplacesTheAISeededBoxes() { + let oldSeed = MemeCaptionLayout.seedBoxes(captions: ["old top", "old bottom"], slots: 2) + let newSeed = MemeCaptionLayout.seedBoxes(captions: ["new top", "new bottom"], slots: 2) + + let merged = MemeCaptionLayout.merging( + seed: newSeed, into: oldSeed, seededIDs: Set(oldSeed.map(\.id))) + + XCTAssertEqual(merged.map(\.text), ["new top", "new bottom"]) + } + + /// The headline: a caption the user added by hand survives a regenerate. Silent + /// destruction of manual work is the worst class of bug in an editor with no undo. + func testRegeneratePreservesABoxTheUserAdded() throws { + let oldSeed = MemeCaptionLayout.seedBoxes(captions: ["old"], slots: 1) + let userBox = MemeCaptionLayout.CaptionBox( + text: "mine", centerX: 0.25, centerY: 0.6) + let existing = oldSeed + [userBox] + + let newSeed = MemeCaptionLayout.seedBoxes(captions: ["new"], slots: 1) + let merged = MemeCaptionLayout.merging( + seed: newSeed, into: existing, seededIDs: Set(oldSeed.map(\.id))) + + XCTAssertEqual(merged.map(\.text), ["new", "mine"]) + // Identity and geometry survive exactly — it is the same box, not a copy. + let survivor = try XCTUnwrap(merged.first { $0.id == userBox.id }) + XCTAssertEqual(survivor.centerX, 0.25) + XCTAssertEqual(survivor.centerY, 0.6) + } + + /// A regenerate to a template with MORE slots keeps the user's box on the end, so + /// panel order still reads top-to-bottom for the slots the template has. + func testUserBoxesAreAppendedAfterTheNewSeedWhateverTheSlotCount() { + let oldSeed = MemeCaptionLayout.seedBoxes(captions: ["a", "b"], slots: 2) + let userBox = MemeCaptionLayout.CaptionBox(text: "mine", centerX: 0.5, centerY: 0.5) + let newSeed = MemeCaptionLayout.seedBoxes( + captions: ["1", "2", "3", "4"], slots: 4) + + let merged = MemeCaptionLayout.merging( + seed: newSeed, into: oldSeed + [userBox], seededIDs: Set(oldSeed.map(\.id))) + + XCTAssertEqual(merged.map(\.text), ["1", "2", "3", "4", "mine"]) + } + + /// EDITS to a seeded box are deliberately NOT preserved: that box is the AI's + /// answer to the old description, and a regenerate asks for a new one. + func testAnEditedSeededBoxIsStillReplaced() { + var seed = MemeCaptionLayout.seedBoxes(captions: ["original"], slots: 1) + let seededIDs = Set(seed.map(\.id)) + seed[0].text = "the user retyped this" + + let merged = MemeCaptionLayout.merging( + seed: MemeCaptionLayout.seedBoxes(captions: ["fresh"], slots: 1), + into: seed, seededIDs: seededIDs) + + XCTAssertEqual(merged.map(\.text), ["fresh"]) + } + + /// The first generate on an empty canvas has nothing to preserve. + func testTheFirstGenerateOnAnEmptyCanvasJustSeeds() { + let seed = MemeCaptionLayout.seedBoxes(captions: ["a", "b"], slots: 2) + let merged = MemeCaptionLayout.merging(seed: seed, into: [], seededIDs: []) + XCTAssertEqual(merged.map(\.text), ["a", "b"]) + } + + /// A user who picked a template and typed BEFORE ever generating has boxes that no + /// seed minted — those are theirs and must survive the first Generate. + func testBoxesTypedBeforeTheFirstGenerateAreTreatedAsUserAdded() { + let handmade = [ + MemeCaptionLayout.CaptionBox(text: "typed by hand", centerX: 0.5, centerY: 0.5) + ] + let merged = MemeCaptionLayout.merging( + seed: MemeCaptionLayout.seedBoxes(captions: ["ai"], slots: 1), + into: handmade, seededIDs: []) + XCTAssertEqual(merged.map(\.text), ["ai", "typed by hand"]) + } + + // MARK: - The learned affinity + // + // Cheap supervision: a user clicking past the model's first pick is a correction. + // The bounds matter more than the signal — an unbounded boost is a personalized + // version of the confident-Drake bug. + + func testAPickBoostsThatTemplate() { + var affinity = MemeTemplateAffinity() + affinity.record(pick: "imgflip:1") + XCTAssertEqual(affinity.boost(for: "imgflip:1"), MemeTemplateAffinity.boostPerPick) + XCTAssertEqual(affinity.boost(for: "imgflip:2"), 0) + } + + func testRepeatedPicksAccumulate() { + var affinity = MemeTemplateAffinity() + for _ in 0..<3 { affinity.record(pick: "x") } + XCTAssertEqual(affinity.boost(for: "x"), MemeTemplateAffinity.boostPerPick * 3) + } + + /// The cap is the load-bearing bound: past saturation the signal stops compounding, + /// which is what stops a long-lived store from taking over the ranking. + func testTheBoostSaturatesAtTheCapAndNeverExceedsIt() { + var affinity = MemeTemplateAffinity() + for _ in 0..<500 { affinity.record(pick: "x") } + XCTAssertEqual(affinity.boost(for: "x"), MemeTemplateAffinity.maximumBoost) + } + + func testSaturationTakesTheDocumentedNumberOfPicks() { + var affinity = MemeTemplateAffinity() + for _ in 0.. = [] + + /// The learned per-template boosts, loaded once per window and persisted on every + /// correction (v6). + private var affinity = MemeTemplateAffinity() + /// True when the model named only templates that don't exist in the corpus and we /// fell back. Drives the honest "not in this corpus" warning. @Published private(set) var didFallBack: Bool = false @@ -190,6 +210,10 @@ final class MemeGeneratorModel: ObservableObject { imageFailed = false failedTemplate = nil status = "" + // Re-read on every open rather than caching across sessions: the file is small, + // and a second window (or a hand edit) must not be silently overwritten by a + // stale in-memory copy on the next pick. + affinity = MemeLibraryStore.loadAffinity() warmLLM() openCatalog() } @@ -404,15 +428,21 @@ final class MemeGeneratorModel: ObservableObject { // description of the meme's CONTENT reach a template whose name shares no // words with it — and it guarantees the relevant template is in the // prompt at all, rather than truncated off the end of a popularity list. + // v6: the user's own past corrections tilt this, within a hard cap and + // only among templates the description already matched. let shortlist = MemeTemplateCatalog.prefilter( - for: self.description, in: self.catalog, limit: MemeAI.candidateShortlist) - // The model is shown names + keywords, but validated against NAMES — + for: self.description, in: self.catalog, limit: MemeAI.candidateShortlist, + affinity: self.affinity) + // Three positionally-aligned projections of the SAME shortlist: the model + // sees `lines` (name + keywords + slot count), answers with numbers that + // index it, and any name it writes instead is validated against `names`. // `promptLines` keeps the name first and unadorned precisely so the two // stay in sync. let names = MemeTemplateCatalog.promptNames(shortlist, limit: shortlist.count) let lines = MemeTemplateCatalog.promptLines(shortlist, limit: shortlist.count) + let slots = MemeTemplateCatalog.promptSlots(shortlist, limit: shortlist.count) let payload = MemeAI.rankedUserPayload( - description: self.description, templateNames: lines, + description: self.description, templateLines: lines, slots: slots, limit: MemeAI.candidateShortlist) do { @@ -557,6 +587,16 @@ final class MemeGeneratorModel: ObservableObject { meme = nil baseImage = nil searchText = "" + candidateReason = "" + // No boxes left, so nothing can be "AI-seeded" any more. Leaving stale ids here + // would make the first regenerate after a New meme treat a fresh user-added box + // as seeded if UUIDs ever collided — cheap to clear, and it keeps the invariant + // "seededBoxIDs ⊆ boxes" true at all times. + seededBoxIDs = [] + + // The learned affinity deliberately SURVIVES. It is not part of this meme — it + // is what the user has taught the ranker across all of them, and throwing it + // away on New meme would make the lesson unlearnable in practice. // The catalog stays, so a catalog failure is only cleared when there is in // fact a catalog — clearing the flag with an empty corpus would hide a real @@ -609,8 +649,13 @@ final class MemeGeneratorModel: ObservableObject { } candidates = picks - boxes = MemeCaptionLayout.seedBoxes(topText: spec.topText, bottomText: spec.bottomText) - selectedBoxID = boxes.first?.id + candidateReason = spec.reason + + // v6: the boxes are seeded for the TOP candidate's real structure, not for an + // assumed top/bottom pair. A 4-slot Expanding Brain gets four boxes with four + // captions in panel order; a 2-slot Drake behaves exactly as it did in v5. + let slots = picks.first?.captionSlots ?? MemeCaptionSlots.default + seedBoxes(captions: spec.captions, slots: slots) guard let best = picks.first else { finish(ticket, status: "There are no templates to choose from — import one to get started.") @@ -620,6 +665,22 @@ final class MemeGeneratorModel: ObservableObject { await renderTemplate(best, ticket: ticket, isNewGeneration: true) } + /// Replace the AI-seeded boxes, keeping any the user added by hand (v6). + /// + /// The rule and the reasoning behind choosing it over a confirmation dialog live in + /// `MemeCaptionLayout.merging`; this is only the plumbing. `seededBoxIDs` is + /// re-minted from the NEW seed, so the next regenerate replaces these in turn while + /// the user's own boxes keep surviving indefinitely. + private func seedBoxes(captions: [String], slots: Int) { + let seed = MemeCaptionLayout.seedBoxes(captions: captions, slots: slots) + boxes = MemeCaptionLayout.merging(seed: seed, into: boxes, seededIDs: seededBoxIDs) + seededBoxIDs = Set(seed.map(\.id)) + // Select the first seeded box rather than whatever was selected before: after a + // regenerate the user is looking at new captions, and leaving the editor panel + // pointed at a box that may no longer exist would show an empty panel. + selectedBoxID = boxes.first?.id + } + // MARK: - Template selection /// Re-render the current captions onto another template. @@ -635,10 +696,17 @@ final class MemeGeneratorModel: ObservableObject { func select(template: MemeTemplate) { guard template.id != selectedTemplate?.id else { return } - // Seed boxes if the user picked a template before ever generating. + // v6: the user reaching past the model's first pick is a correction, and the + // cheapest supervision this plugin will ever get. Recorded BEFORE the async + // work so a download that fails still teaches — the user's preference was + // expressed by the click, not by the download succeeding. + recordCorrection(for: template) + + // Seed boxes if the user picked a template before ever generating — now for + // the template's OWN slot count, so picking a 4-panel meme first and typing + // into it works without ever touching the LLM. if boxes.isEmpty { - boxes = MemeCaptionLayout.seedBoxes(topText: "", bottomText: "") - selectedBoxID = boxes.first?.id + seedBoxes(captions: [], slots: template.captionSlots) } // The user has now chosen deliberately, so this is no longer a fallback. @@ -652,7 +720,94 @@ final class MemeGeneratorModel: ObservableObject { after: MemeGenerationState.downloadTimeout, message: MemeGenerationState.downloadTimeoutMessage(template.name)) Task { [weak self] in - await self?.renderTemplate(template, ticket: ticket, isNewGeneration: false) + guard let self else { return } + await self.renderTemplate(template, ticket: ticket, isNewGeneration: false) + await self.refitCaptionsIfNeeded(for: template) + } + } + + // MARK: - Learning signal (v6) + + /// Note that the user picked `template` instead of the candidate on offer. + /// + /// Only a NON-FIRST pick counts. Clicking the candidate the model already put first + /// is agreement, not a correction, and boosting it would just amplify whatever the + /// ranker already believed — the signal has to be about the cases the ranker got + /// wrong, or it is a feedback loop rather than a lesson. A Browse pick always + /// counts: reaching into the full corpus is the strongest correction available. + private func recordCorrection(for template: MemeTemplate) { + guard candidates.first?.id != template.id else { return } + affinity.record(pick: template.id) + MemeLibraryStore.saveAffinity(affinity) + } + + // MARK: - Caption refit (v6) + + /// Re-fit the captions when the newly-chosen template has a DIFFERENT number of + /// slots than the captions currently on the canvas. + /// + /// ## Why this needs the model + /// + /// The candidate strip promises "same joke, different template". Going from a + /// 2-slot Drake to a 4-slot Expanding Brain needs two new lines invented in the + /// user's language and the joke's voice — that is a language task, not a + /// redistribution, so it is a small second round-trip. + /// + /// ## What keeps it from being another stuck-state bug + /// + /// * It runs on its OWN ticket through the same state machine, so Cancel works and + /// a superseded refit can't write over a newer one. + /// * It NEVER blocks the strip. The template has already rendered by the time this + /// starts; the user can click straight past it to a different candidate, and the + /// in-flight refit is refused when it lands. + /// * Every exit path — no-op, no LLM, parse failure, transport failure, stale + /// ticket — ends at `finish` for its own ticket. + /// * A failure is SILENT and leaves the previous captions in place. The switch + /// itself succeeded, so surfacing an error would make a working action look + /// broken; the user keeps captions that are merely the wrong shape, and can edit + /// them by hand exactly as before. + private func refitCaptionsIfNeeded(for template: MemeTemplate) async { + let slots = MemeCaptionSlots.clamp(template.captionSlots) + let current = boxes.map(\.text) + + // The fast path, and the common one: same slot count means the captions carry + // over verbatim, instantly, with no LLM involved — exactly as in v5. + guard MemeAI.needsRefit(captions: current, slots: slots) else { return } + guard let aiCall else { return } + let resolved = resolveAIModel() + guard ScratchpadAIModel.isUsable(resolved) else { return } + + let ticket = state.begin(.asking) + status = "Refitting the captions to \(template.name)…" + startTimeout(ticket: ticket, + after: MemeGenerationState.downloadTimeout, + message: "Used \(template.name). The captions weren't refitted — edit them by hand.") + + let payload = MemeAI.refitUserPayload( + description: description, captions: current, slots: slots, + templateName: template.name) + + do { + let raw = try await aiCall(MemeAI.refitPrompt, payload, resolved) + guard !isCancelled, state.accepts(ticket: ticket) else { + finish(ticket, status: status) + return + } + guard let captions = MemeAI.parseRefit(raw, slots: slots) else { + // Nothing usable came back. Keep what the user has and say nothing + // about the failed nicety. + finish(ticket, status: statusLine(isNewGeneration: false)) + return + } + seedBoxes(captions: captions, slots: slots) + redraw() + finish(ticket, status: statusLine(isNewGeneration: false)) + } catch { + guard !isCancelled, state.accepts(ticket: ticket) else { + finish(ticket, status: status) + return + } + finish(ticket, status: statusLine(isNewGeneration: false)) } } @@ -804,6 +959,9 @@ final class MemeGeneratorModel: ObservableObject { func deleteBox(id: UUID) { boxes.removeAll { $0.id == id } + // Keep the seeded set in step with what's actually on the canvas, so it can't + // accumulate ids for boxes that no longer exist. + seededBoxIDs.remove(id) if selectedBoxID == id { selectedBoxID = boxes.first?.id } redraw() } diff --git a/plugins/MemeGenerator/MemeGeneratorView.swift b/plugins/MemeGenerator/MemeGeneratorView.swift index dd74768..5cf13dc 100644 --- a/plugins/MemeGenerator/MemeGeneratorView.swift +++ b/plugins/MemeGenerator/MemeGeneratorView.swift @@ -288,19 +288,39 @@ struct MemeGeneratorView: View { /// instantly, and even while a generation is still running. private var candidateStrip: some View { VStack(alignment: .leading, spacing: 4) { - Text(model.candidatesAreFallback - ? "Nothing matched — closest and most popular instead:" - : "The model's picks, best first:") - .font(.caption) - .foregroundStyle(.secondary) + HStack(spacing: 4) { + Text(model.candidatesAreFallback + ? "Nothing matched — closest and most popular instead:" + : "The model's picks, best first:") + .font(.caption) + .foregroundStyle(.secondary) + + // The model's own justification for its top pick (v6). This is where + // the reasoning the v5 prompt asked for and threw away now goes: on + // hover the user reads WHY the first thumbnail is first, instead of + // guessing. Only shown when the model actually gave one. + if !model.candidateReason.isEmpty, !model.candidatesAreFallback { + Image(systemName: "info.circle") + .font(.caption) + .foregroundStyle(.tertiary) + .help(model.candidateReason) + .accessibilityLabel("Why this pick: \(model.candidateReason)") + } + Spacer() + } ScrollView(.horizontal, showsIndicators: false) { HStack(spacing: 8) { - ForEach(model.candidates) { template in + ForEach(Array(model.candidates.enumerated()), id: \.element.id) { index, template in TemplateThumbnail( template: template, isSelected: template.id == model.selectedTemplate?.id, - width: 88) + width: 88, + // Only the FIRST candidate carries the reason: the model was + // asked to justify its top pick, so attaching that sentence + // to the others would be attributing an explanation to a + // choice it doesn't describe. + note: index == 0 ? model.candidateReason : "") .onTapGesture { model.select(template: template) } } } @@ -672,6 +692,21 @@ private struct TemplateThumbnail: View { let template: MemeTemplate let isSelected: Bool let width: CGFloat + /// An extra line for the tooltip — the model's reason, on the top candidate (v6). + var note: String = "" + + /// The tooltip: what this template is, where it came from, how many captions it + /// takes, and (on the model's top pick) why it was chosen. + /// + /// The slot count is worth stating because it now has a visible consequence — + /// clicking a 4-caption template turns two captions into four — and a user who can + /// see that coming isn't surprised by it. + private var tooltip: String { + var parts = ["\(template.name) — \(template.source.label)"] + parts.append("\(template.captionSlots) caption\(template.captionSlots == 1 ? "" : "s")") + if !note.isEmpty { parts.append(note) } + return parts.joined(separator: " · ") + } var body: some View { VStack(spacing: 3) { @@ -703,7 +738,7 @@ private struct TemplateThumbnail: View { .frame(width: width) } .contentShape(Rectangle()) - .help("\(template.name) — \(template.source.label)") + .help(tooltip) } @ViewBuilder diff --git a/plugins/MemeGenerator/MemeLibraryStore.swift b/plugins/MemeGenerator/MemeLibraryStore.swift index b3a4d42..1342c35 100644 --- a/plugins/MemeGenerator/MemeLibraryStore.swift +++ b/plugins/MemeGenerator/MemeLibraryStore.swift @@ -199,6 +199,38 @@ enum MemeLibraryStore { } } + // MARK: - Template affinity (v6) + + private static var affinityURL: URL { + pluginDirectory.appendingPathComponent(MemeTemplateAffinity.fileName) + } + + /// Read the learned per-template boosts. + /// + /// A missing or corrupt file is an EMPTY affinity, never an error — it is a + /// ranking nicety, and failing to open the plugin because a preference file went + /// bad would be wildly out of proportion. The decoder re-applies the cap, so even + /// a hand-edited file can't inject a dominating boost. + static func loadAffinity() -> MemeTemplateAffinity { + guard let data = try? Data(contentsOf: affinityURL), + let decoded = try? JSONDecoder().decode(MemeTemplateAffinity.self, from: data) + else { return MemeTemplateAffinity() } + return decoded + } + + /// Persist the boosts. Silent on failure for the same reason as the read. + static func saveAffinity(_ affinity: MemeTemplateAffinity) { + do { + try FileManager.default.createDirectory( + at: pluginDirectory, withIntermediateDirectories: true) + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(affinity).write(to: affinityURL, options: .atomic) + } catch { + // Losing a boost costs nothing the user will notice. + } + } + // MARK: - Thumbnails /// `…/Plugins/MemeGenerator/thumbnails` — downscaled template previews. From e0386cfd00bb08f591843764e6776c110cad92b2 Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Mon, 3 Aug 2026 14:51:52 -0700 Subject: [PATCH 10/15] spike(meme-plugin): read the captions the user already said, and enforce slots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live failure: "expanding brain: typing, dictating, dictating memes, dictating memes by voice" picked Expanding Brain (4 slots) and rendered TWO captions. Three independent things had to be true, and v7 breaks all three. 1. Deterministic caption extraction (highest leverage). The captions were RIGHT THERE in the dictation, comma-separated after a colon, and we asked a 1.5B local model to re-derive them anyway. MemeCaptionExtraction reads list-shaped descriptions directly — colon lists, numbered runs, newline lists, spoken "and"/"then" joiners — and uses the items verbatim, skipping LLM caption-writing entirely. The LLM keeps the one job it is needed for: picking a template. Deliberately narrow (an enumeration signal, a plausible count, and caption-sized items are all required), so prose with commas — "a drake meme about rust, python and go" — falls through to the v6 path unchanged. 2. Host-side slot enforcement. MemeAI.fit is now the single rule: a caption count that doesn't match the template REFITS via the existing ticket-guarded call, with an honest "Model wrote 2 of 4 — refitting…". The legacy top/bottom form is accepted as final ONLY for 2-slot templates — on any N≠2 template it is the bug signature it was. Captions always seed into the template's own slot geometry, so a 4-slot template can never render as a classic two-liner. The refit path is extracted so generate and template-switch share one implementation. 3. Constrained decoding (the systemic fix). Every other guard here is a parser deciding after the fact; that is a losing game one shape at a time. The seam now carries a JSON schema down to llama-server, which compiles it to a GBNF grammar and constrains the sampler — so `templates` typed as integers makes an invented name unrepresentable, and the refit schema's minItems == maxItems == N makes "wrote 2 of 4" impossible to emit. Optional and omitted when nil, so every existing caller's request bytes are unchanged. Template search also prefers matching slot counts when the item count is known — reordering only, never filtering, since hiding a described template is the confident-Drake bug in a new costume. swift test 2624 (was 2593); AppState ratchet unchanged at 7036 (the plumbing adds zero net lines); PLUGINS=1, plain, and lean builds all clean with no new warnings in touched files. Co-Authored-By: Claude --- OpenWhisp/Models/AppState.swift | 6 +- OpenWhisp/Services/MemeAI.swift | 229 ++++++++++- .../Services/MemeCaptionExtraction.swift | 256 +++++++++++++ OpenWhisp/Services/MemeTemplateProvider.swift | 41 +- .../Services/OpenAITranslationService.swift | 59 ++- Package.swift | 1 + .../MemeSlotEnforcementTests.swift | 361 ++++++++++++++++++ .../MemeStructureTests.swift | 18 + .../MemeGenerator/MemeGeneratorModel.swift | 97 ++++- .../MemeGeneratorWindowController.swift | 12 +- 10 files changed, 1053 insertions(+), 27 deletions(-) create mode 100644 OpenWhisp/Services/MemeCaptionExtraction.swift create mode 100644 Tests/OpenWhispCoreTests/MemeSlotEnforcementTests.swift diff --git a/OpenWhisp/Models/AppState.swift b/OpenWhisp/Models/AppState.swift index 047a132..9b87ead 100644 --- a/OpenWhisp/Models/AppState.swift +++ b/OpenWhisp/Models/AppState.swift @@ -6514,8 +6514,8 @@ extension AppState: AgentBridgeHost { /// bundled engine when the resolved provider is bundled. func summarizeResolved( text: String, instruction: String, resolved: SummaryModelResolver.Resolved, - completion: @escaping (Result) -> Void - ) { + responseFormat: ResponseFormat? = nil, // v7: grammar-constrained decoding + completion: @escaping (Result) -> Void) { // Busy-reject while dictating (same guarantee refineText gives): warming // the bundled LLM would stop a live whisper-server. guard !sessionActive, !isRecording, !isTranscribing else { @@ -6558,7 +6558,7 @@ extension AppState: AgentBridgeHost { targetLanguage: self.translationTargetLanguage, endpoint: endpoint, model: model, - customInstruction: systemDirective + customInstruction: systemDirective, responseFormat: responseFormat ) { [weak self] result in Task { @MainActor in done() diff --git a/OpenWhisp/Services/MemeAI.swift b/OpenWhisp/Services/MemeAI.swift index c688aa1..47f31f1 100644 --- a/OpenWhisp/Services/MemeAI.swift +++ b/OpenWhisp/Services/MemeAI.swift @@ -1,5 +1,31 @@ import Foundation +/// A minimal JSON value, so a schema can be built in Swift and encoded verbatim. +/// +/// Written by hand rather than reaching for `[String: Any]` because `Any` is not +/// `Encodable` — and rather than a raw JSON string, because a string would put the +/// schema beyond the reach of the type checker AND of `swift test`. It lives in +/// OpenWhispCore (not beside the HTTP client) for exactly that reason: the schemas +/// are logic, and logic in this project is testable by construction. +public indirect enum JSONValue: Encodable, Equatable, Sendable { + case string(String) + case int(Int) + case bool(Bool) + case array([JSONValue]) + case object([String: JSONValue]) + + public func encode(to encoder: Encoder) throws { + var c = encoder.singleValueContainer() + switch self { + case .string(let v): try c.encode(v) + case .int(let v): try c.encode(v) + case .bool(let v): try c.encode(v) + case .array(let v): try c.encode(v) + case .object(let v): try c.encode(v) + } + } +} + /// The pure rules behind the Meme Generator plugin's LLM step (spike). /// /// One round-trip turns a spoken description ("make me the distracted boyfriend one @@ -245,10 +271,25 @@ public enum MemeAI { /// nicety, never a reason to reject a response. public let reason: String - public init(templateNames: [String], captions: [String], reason: String = "") { + /// True when the captions came from the legacy `top_text`/`bottom_text` pair + /// rather than a `captions` array (v7). + /// + /// Recorded because the two shapes mean different things even when they carry + /// the same two strings: an ARRAY of two is a model answering a 2-slot question, + /// while the legacy pair is a model that never engaged with the slot count. On a + /// 2-slot template both are correct; on any other, the legacy pair is the v6 bug + /// signature. `fit` treats them identically today — the count is what decides — + /// and this flag is what lets the host say WHICH happened without re-parsing. + public let wasLegacyShape: Bool + + public init( + templateNames: [String], captions: [String], reason: String = "", + wasLegacyShape: Bool = false + ) { self.templateNames = templateNames self.captions = captions self.reason = reason + self.wasLegacyShape = wasLegacyShape } /// The classic two-slot spelling, kept so existing call sites and tests that @@ -268,6 +309,19 @@ public enum MemeAI { public var isEmpty: Bool { templateNames.isEmpty && captions.allSatisfy(\.isEmpty) } + + /// The same pick, with the captions replaced by the user's own words (v7). + /// + /// Used when `MemeCaptionExtraction` read a list out of the description: the + /// model's TEMPLATE choice is kept (that is the judgement we wanted from it) and + /// its captions are discarded in favour of what the user actually said. Marks + /// the result as non-legacy because these captions came from the user, not from + /// a `top_text` pair — the distinction the status line reads. + public func replacingCaptions(with captions: [String]) -> RankedSpec { + RankedSpec( + templateNames: templateNames, captions: captions, reason: reason, + wasLegacyShape: false) + } } /// One candidate reference as the model wrote it: a number or a name (v6). @@ -290,6 +344,8 @@ public enum MemeAI { let templates: [CandidateRef] let captions: [String] let reason: String + /// Whether `captions` was reconstructed from `top_text`/`bottom_text` (v7). + let wasLegacyShape: Bool private enum CodingKeys: String, CodingKey { case templates @@ -352,14 +408,18 @@ public enum MemeAI { // keys keeps the array — the richer answer wins. if let list = try? c.decode([String].self, forKey: .captions) { captions = list + wasLegacyShape = false } else if let list = try? c.decode([String].self, forKey: .texts) { captions = list + wasLegacyShape = false } else if let list = try? c.decode([String].self, forKey: .lines) { captions = list + wasLegacyShape = false } else { let top = (try? c.decode(String.self, forKey: .topText)) ?? "" let bottom = (try? c.decode(String.self, forKey: .bottomText)) ?? "" captions = [top, bottom] + wasLegacyShape = true } reason = (try? c.decode(String.self, forKey: .reason)) ?? "" @@ -402,7 +462,8 @@ public enum MemeAI { while let last = captions.last, last.isEmpty { captions.removeLast() } let spec = RankedSpec( - templateNames: names, captions: captions, reason: clean(wire.reason)) + templateNames: names, captions: captions, reason: clean(wire.reason), + wasLegacyShape: wire.wasLegacyShape) guard !spec.isEmpty else { return .failure(.missingFields) } return .success(spec) } @@ -542,6 +603,86 @@ public enum MemeAI { return captions.count != MemeCaptionSlots.clamp(slots) } + // MARK: - v7: the host decides what a caption count MEANS + + /// What to do with a caption response, given the template it has to fill (v7). + /// + /// ## The v6 bug this type exists to make impossible + /// + /// v6 had no such decision. `parseRanked` returned whatever the model wrote, + /// `applyRanked` handed it to `seedBoxes`, and a 2-caption answer for a 4-slot + /// Expanding Brain was padded with two blank boxes and rendered — the reported + /// failure. The legacy `top_text`/`bottom_text` fallback made that the DEFAULT + /// outcome for any model that reached for the classic keys, because those keys can + /// only ever produce two. + /// + /// The rule is now stated once, here, and it is the host's, not the model's: + /// + /// * **The count matches** → render. + /// * **The count doesn't match** → REFIT. Never render a mismatch silently; the + /// refit call already exists (`refitPrompt`) and says "same joke, exactly N". + /// * **The legacy two-caption shape** is accepted as final for a **2-slot template + /// only**. That is the one case where `top_text`/`bottom_text` is genuinely the + /// right answer rather than a model that ignored the slot count. On any N≠2 + /// template it is treated as the mismatch it is. + public enum CaptionFit: Equatable, Sendable { + /// The captions fill the template — render them as they are. + case ready([String]) + /// The count is wrong; run the refit round-trip to `slots` captions. + case refit(from: [String], to: Int) + + /// The captions to seed right now, in either case. + /// + /// A refit still SEEDS first: the template has already been chosen and the user + /// should see the joke land while the refit runs, rather than an empty canvas. + /// `seedBoxes` pads or truncates to the slot count, so this is always safe. + public var captions: [String] { + switch self { + case .ready(let captions): return captions + case .refit(let captions, _): return captions + } + } + + /// True when a second round-trip is owed. + public var needsRefit: Bool { + if case .refit = self { return true } + return false + } + } + + /// Decide whether `captions` may be rendered on a `slots`-slot template. + /// + /// `wasLegacyShape` is what makes the 2-slot exception decidable: a `["a","b"]` that + /// came from a `captions` ARRAY is the model answering a 2-slot question correctly, + /// while the same pair from `top_text`/`bottom_text` is a model that never engaged + /// with the slot count at all. Both are fine on a 2-slot template and neither is + /// fine on a 4-slot one, so the flag doesn't change THIS rule — but it is carried on + /// `RankedSpec` so the status line can be honest about which happened, and so a + /// future rule can tell them apart without re-deriving it. + /// + /// Empty captions never refit: there is no joke to preserve, and asking a model to + /// rewrite nothing into four somethings is how you get four hallucinations. They + /// seed as empty boxes for the user to type into, exactly as `select` already does. + public static func fit(captions: [String], slots: Int, wasLegacyShape: Bool = false) -> CaptionFit { + let target = MemeCaptionSlots.clamp(slots) + let meaningful = captions.filter { + !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } + guard !meaningful.isEmpty else { return .ready(captions) } + guard captions.count != target else { return .ready(captions) } + return .refit(from: captions, to: target) + } + + /// The status line shown while a mismatch is being refitted. + /// + /// Honest about the shortfall rather than a generic spinner: the user watched the + /// model answer and is about to watch the captions CHANGE, and "Refitting…" alone + /// would make that look like a glitch. Naming the numbers makes the second call + /// legible as a correction. + public static func refitStatus(wrote: Int, of slots: Int) -> String { + "Model wrote \(wrote) of \(slots) — refitting…" + } + /// Parse a refit reply into exactly `slots` captions. /// /// Returns nil rather than throwing a typed error: a refit that fails must leave @@ -572,6 +713,90 @@ public enum MemeAI { return captions + Array(repeating: "", count: count - captions.count) } + // MARK: - v7: constrained decoding (the systemic fix) + + /// JSON schemas that FORCE the response shape, for servers that support + /// constrained decoding. + /// + /// ## Why this is the real fix + /// + /// Every other guard in this file is a parser: the model writes whatever it wants + /// and we decide afterwards whether to accept it. That is a losing game against a + /// 1.5B local model — v5 fixed name transcription, v6 fixed the caption array, and + /// v6 STILL shipped the bug this file's `fit` now catches, because there was always + /// one more shape to mis-write. + /// + /// llama-server compiles a `json_schema` into a GBNF grammar and constrains the + /// SAMPLER with it. A response missing `captions`, or carrying `top_text` instead, + /// or returning three strings where four were required, is then not rejected — it + /// is unrepresentable, because no token sequence that produces it is reachable. + /// The whole class of bug goes away rather than being caught one shape at a time. + /// + /// These are built as values rather than raw JSON strings so `swift test` can + /// assert their contents; a schema stored as a string literal would be exactly the + /// kind of untested wiring this spike exists to avoid shipping. + public enum Schema { + + /// The ranked-pick schema: numeric template indices plus a caption array. + /// + /// `templates` is `integer`-typed, which is what makes v6's numbered-reference + /// idea airtight: a model CANNOT answer with a name it invented, because a + /// string is not a representable token at that position. + /// + /// Captions are deliberately NOT pinned to a count here — the model chooses its + /// first candidate in the same reply, so the required count isn't known when the + /// request is built. The count is enforced host-side by `fit`, and the refit call + /// (which DOES know the number) is schema-pinned by `refit(slots:)`. + public static func ranked(maxCaptions: Int = MemeCaptionSlots.maximum) -> JSONValue { + .object([ + "type": .string("object"), + "properties": .object([ + "templates": .object([ + "type": .string("array"), + "items": .object(["type": .string("integer")]), + "minItems": .int(1), + "maxItems": .int(maxCandidates), + ]), + "captions": .object([ + "type": .string("array"), + "items": .object(["type": .string("string")]), + "minItems": .int(1), + "maxItems": .int(MemeCaptionSlots.clamp(maxCaptions)), + ]), + "reason": .object(["type": .string("string")]), + ]), + "required": .array([ + .string("templates"), .string("captions"), .string("reason"), + ]), + "additionalProperties": .bool(false), + ]) + } + + /// The refit schema: EXACTLY `slots` captions, and nothing else. + /// + /// `minItems == maxItems == slots` is the whole point. The refit call exists + /// precisely because a count was wrong, so it is the one call where the required + /// count is known up front — and pinning both bounds makes "wrote 2 of 4" a + /// shape the sampler cannot emit. On a server with constrained decoding this + /// makes the refit succeed on the first attempt by construction. + public static func refit(slots: Int) -> JSONValue { + let count = MemeCaptionSlots.clamp(slots) + return .object([ + "type": .string("object"), + "properties": .object([ + "captions": .object([ + "type": .string("array"), + "items": .object(["type": .string("string")]), + "minItems": .int(count), + "maxItems": .int(count), + ]), + ]), + "required": .array([.string("captions")]), + "additionalProperties": .bool(false), + ]) + } + } + /// Extract the first balanced `{...}` run from a string, ignoring braces that /// appear inside JSON string literals (so a caption containing `{` can't end the /// scan early). Returns nil when there is no balanced object. diff --git a/OpenWhisp/Services/MemeCaptionExtraction.swift b/OpenWhisp/Services/MemeCaptionExtraction.swift new file mode 100644 index 0000000..219a634 --- /dev/null +++ b/OpenWhisp/Services/MemeCaptionExtraction.swift @@ -0,0 +1,256 @@ +import Foundation + +/// Reading the captions straight out of what the user said (spike v7). +/// +/// ## Why this exists — the v6 failure it removes +/// +/// The v6 report: "expanding brain: typing, dictating, dictating memes, dictating +/// memes by voice" picked Expanding Brain correctly (4 slots) and then rendered TWO +/// captions. The model had answered in the legacy `top_text`/`bottom_text` shape, the +/// backward-compatible parser accepted it, and nothing checked the count against the +/// template's structure. +/// +/// The prompt could be tightened again — v6 already tightened it twice — but that is +/// treating a symptom. Look at what the user actually said: the four captions are +/// RIGHT THERE, comma-separated, in order, after a colon. Asking a 1.5B local model to +/// re-derive from that dictation the four strings it was handed is inventing a +/// language task where none exists, and every such round-trip is a chance to get two +/// back instead of four. +/// +/// So v7 reads them directly. When the description is LIST-SHAPED, the items become +/// the captions verbatim and the LLM never writes captions at all — it is left with +/// the one job it is actually needed for, picking a template. A model cannot return +/// the wrong number of captions for a request that was never made. +/// +/// ## What counts as list-shaped +/// +/// Deliberately narrow. This runs BEFORE the LLM on every generate, so a false +/// positive would hijack ordinary prose ("make me a drake meme about rust, python and +/// go" is prose *about* three things, not a three-caption list) and produce a meme of +/// fragments. The rules that follow are what keep it honest: +/// +/// * There must be a **separator that means enumeration** — a colon introducing the +/// list, an explicit numbering, or newlines. A bare comma run inside a sentence is +/// NOT enough on its own, because that is how people write ordinary prose. +/// * The item count must be **plausible** (`minimumItems`…`maximumItems`). One item is +/// not a list, and past eight it is a monologue that no template has slots for. +/// * Every item must be **caption-sized**. A "list" whose entries run to sentence +/// length is prose with commas in it, not a set of captions. +/// +/// Prose that fails any of these falls through to the v6 LLM caption path completely +/// unchanged. That is the design: this is a fast path over the existing one, never a +/// replacement for it. +public enum MemeCaptionExtraction { + + /// The fewest items that can be a list. Two — a one-item "list" is a phrase, and + /// treating it as one would hijack every ordinary description ending in a colon. + public static let minimumItems = 2 + + /// The most items we will read out. Matches `MemeCaptionSlots.maximum`: past the + /// slot ceiling there is no template that could hold them, so the extraction would + /// be discarded anyway. + public static let maximumItems = MemeCaptionSlots.maximum + + /// The longest an item may be, in WORDS, before it stops looking like a caption. + /// + /// Six is generous for meme text (real captions run one to four words) and still + /// tight enough to reject prose. The check is on words rather than characters so a + /// language with long compounds isn't penalised for being itself — the concern is + /// clause structure, not orthography. + public static let maximumWordsPerItem = 6 + + /// A description read as a list of captions. + public struct Extraction: Equatable, Sendable { + /// The caption items, in the order the user said them, cleaned but NOT + /// uppercased — display casing stays `MemeCaptionLayout.displayText`'s job so + /// the editor shows the user their own words. + public let captions: [String] + + /// The text BEFORE the colon, when there was one: "expanding brain" from + /// "expanding brain: typing, dictating, …". + /// + /// This is the template query, and separating it from the items is half the + /// value of parsing the colon at all — without it the theme's words would be + /// scored as though they were caption content, and with it the template search + /// gets exactly the phrase the user used to name the meme. + public let theme: String + + public init(captions: [String], theme: String = "") { + self.captions = captions + self.theme = theme + } + + /// How many caption slots a template needs to hold this extraction. + public var slotCount: Int { captions.count } + } + + /// Read `description` as a list of captions, or return nil to use the LLM path. + /// + /// Nil is the common answer and the safe one: everything that isn't clearly a list + /// falls through to v6's behaviour untouched. + public static func extract(from description: String) -> Extraction? { + let trimmed = description.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + + let (theme, body) = splitTheme(trimmed) + // A colon is what licenses the comma form — see `commaItems`. Passed explicitly + // rather than re-derived, because `body` has already had the theme removed and + // can no longer answer the question about itself. + guard let items = items(in: body, introducedByColon: !theme.isEmpty) else { return nil } + + return Extraction(captions: items, theme: theme) + } + + /// Split "theme: a, b, c" into its theme and its list body. + /// + /// Only the FIRST colon splits, and only when something precedes it — a + /// description that opens with a colon has no theme, and a colon inside an item + /// ("me: no") must not re-split the list that already started. + /// + /// A colon is not required. "typing, dictating, dictating memes" with no theme is + /// still a list; it just gives the template search nothing extra to work with. + private static func splitTheme(_ text: String) -> (theme: String, body: String) { + // Only ASCII ':' and its full-width counterpart. Not every punctuation mark + // that resembles one — a stray ';' in prose would split sentences into + // "themes" and turn ordinary text into a list. + guard let range = text.rangeOfCharacter(from: CharacterSet(charactersIn: ":：")) else { + return ("", text) + } + let theme = String(text[text.startIndex.. [String]? { + let candidates = [ + newlineItems(body), + numberedItems(body), + introducedByColon ? commaItems(body) : nil, + ] + for candidate in candidates { + guard let items = candidate, isPlausibleList(items) else { continue } + return items + } + return nil + } + + /// One item per line. Leading bullets and numbers are stripped so a pasted list + /// doesn't caption itself "1." — the marker is the syntax, never the content. + private static func newlineItems(_ body: String) -> [String]? { + let lines = body + .split(whereSeparator: \.isNewline) + .map { stripMarker(String($0)) } + .filter { !$0.isEmpty } + guard lines.count >= minimumItems else { return nil } + return lines + } + + /// Items introduced by "1.", "2)", "(3)" — the shape a model or a careful user + /// writes when the order matters. + /// + /// Requires the numbering to START the list (the first marker must be at the + /// beginning), so a sentence merely CONTAINING "2." isn't cut in half. + private static func numberedItems(_ body: String) -> [String]? { + let pattern = #"(?:^|\s)\(?(\d{1,2})[.):]\s+"# + guard let regex = try? NSRegularExpression(pattern: pattern) else { return nil } + + let full = NSRange(body.startIndex..= minimumItems else { return nil } + // The list must OPEN with its first marker; anything before it is prose that + // happens to precede a numbered run. + guard matches[0].range.location <= 1 else { return nil } + + var items: [String] = [] + for (offset, match) in matches.enumerated() { + let start = match.range.upperBound + let end = offset + 1 < matches.count + ? matches[offset + 1].range.lowerBound + : full.upperBound + guard start <= end, + let range = Range(NSRange(location: start, length: end - start), in: body) + else { continue } + let item = String(body[range]).trimmingCharacters(in: .whitespacesAndNewlines) + if !item.isEmpty { items.append(item) } + } + guard items.count >= minimumItems else { return nil } + return items + } + + /// Comma-separated items, accepting the spoken final joiners. + /// + /// **Only reached when a colon introduced the list.** This is the deliberate + /// asymmetry that keeps prose safe: "make me a drake meme about rust, python and + /// go" has commas but no introducer, and cutting it into three captions would be + /// exactly the false positive this type must not produce. A colon is the user + /// saying "here comes the list", and that is the signal we require. + private static func commaItems(_ body: String) -> [String]? { + let separated = body.replacingOccurrences( + of: #"[,;]\s*(?:and|then|und|и|затем|потом)\s+"#, + with: ",", + options: [.regularExpression, .caseInsensitive]) + + var items = separated + .split(separator: ",") + .map { stripMarker(String($0)) } + .filter { !$0.isEmpty } + + // "a, b and c" — the last comma-free joiner, split only when it yields a + // caption-sized tail. Prose ends in "and " far more often than a list + // does, so the size check is what makes this safe. + if items.count >= minimumItems - 1, let last = items.last { + let tail = last.replacingOccurrences( + of: #"^(.*?)\s+(?:and|then|und|и|затем|потом)\s+(.+)$"#, + with: "$1\u{0}$2", + options: [.regularExpression, .caseInsensitive]) + let halves = tail.split(separator: "\u{0}").map(String.init) + if halves.count == 2, halves.allSatisfy({ isCaptionSized($0) }) { + items.removeLast() + items.append(contentsOf: halves) + } + } + + guard items.count >= minimumItems else { return nil } + return items + } + + /// Strip a leading list marker: "1.", "2)", "-", "•", "*". + private static func stripMarker(_ item: String) -> String { + item.replacingOccurrences( + of: #"^\s*(?:\(?\d{1,2}[.):]|[-–—•*])\s*"#, + with: "", + options: .regularExpression) + .trimmingCharacters(in: .whitespacesAndNewlines) + } + + /// Whether one item is short enough to be a caption rather than a clause. + private static func isCaptionSized(_ item: String) -> Bool { + let words = item.split(whereSeparator: { $0 == " " || $0.isNewline }) + return !words.isEmpty && words.count <= maximumWordsPerItem + } + + /// Whether a set of items is a plausible caption list: a sane count, and every + /// item caption-sized. + /// + /// The all-items rule is strict on purpose. A "list" with one sentence-length entry + /// is prose that happens to contain commas, and accepting it would caption the meme + /// with a fragment of a sentence — a worse outcome than falling through to the LLM. + private static func isPlausibleList(_ items: [String]) -> Bool { + guard items.count >= minimumItems, items.count <= maximumItems else { return false } + return items.allSatisfy(isCaptionSized) + } +} diff --git a/OpenWhisp/Services/MemeTemplateProvider.swift b/OpenWhisp/Services/MemeTemplateProvider.swift index 7d3039a..387dbe8 100644 --- a/OpenWhisp/Services/MemeTemplateProvider.swift +++ b/OpenWhisp/Services/MemeTemplateProvider.swift @@ -285,19 +285,35 @@ public enum MemeTemplateCatalog { /// templates that already matched the description, so it changes the ORDER of the /// shortlist and never its membership rule. See `MemeTemplateAffinity` for the /// bounds and why each exists. + /// ## v7 — `preferringSlots` + /// + /// When the description was a LIST, we already know how many captions the meme + /// needs, and a template with exactly that many slots is a better fit than one that + /// merely shares words. So a known slot count STABLY REORDERS the shortlist to put + /// exact-slot matches first. + /// + /// Reordering, never filtering: a 4-item list whose best lexical match is a 2-slot + /// template should still see that template (the user may want it, and the refit path + /// handles the mismatch), it just shouldn't be the first thing offered. Filtering + /// here would be the confident-Drake bug in a new costume — silently hiding the + /// template the user actually described. public static func prefilter( for description: String, in catalog: [MemeTemplate], limit: Int, - affinity: MemeTemplateAffinity = MemeTemplateAffinity() + affinity: MemeTemplateAffinity = MemeTemplateAffinity(), + preferringSlots: Int? = nil ) -> [MemeTemplate] { let cap = max(0, limit) guard cap > 0 else { return [] } - let hits = ranked(description, in: catalog, limit: cap, affinity: affinity).map(\.template) - guard !hits.isEmpty else { return Array(catalog.prefix(cap)) } + let hits = ranked(description, in: catalog, limit: cap, affinity: affinity) + .map(\.template) + guard !hits.isEmpty else { + return reorder(Array(catalog.prefix(cap)), preferringSlots: preferringSlots) + } // Top up with popular templates when the query was narrow, so the model always // sees a full shortlist rather than the two things that happened to match. - guard hits.count < cap else { return hits } + guard hits.count < cap else { return reorder(hits, preferringSlots: preferringSlots) } var out = hits var seen = Set(hits.map(\.id)) for template in catalog where out.count < cap { @@ -305,7 +321,22 @@ public enum MemeTemplateCatalog { seen.insert(template.id) out.append(template) } - return out + return reorder(out, preferringSlots: preferringSlots) + } + + /// Stable-partition a shortlist so templates with exactly `slots` caption slots + /// come first, preserving relevance order within each group. + /// + /// Stability is the requirement: the ranker's ordering is the primary signal and + /// slot count is a tiebreaker, so a sort that reshuffled equal-slot templates would + /// throw away the relevance work `ranked` just did. + static func reorder(_ templates: [MemeTemplate], preferringSlots slots: Int?) -> [MemeTemplate] { + guard let slots else { return templates } + let wanted = MemeCaptionSlots.clamp(slots) + let matching = templates.filter { MemeCaptionSlots.clamp($0.captionSlots) == wanted } + guard !matching.isEmpty else { return templates } + let rest = templates.filter { MemeCaptionSlots.clamp($0.captionSlots) != wanted } + return matching + rest } /// The names handed to the LLM, capped so a merged ~300-name corpus doesn't blow diff --git a/OpenWhisp/Services/OpenAITranslationService.swift b/OpenWhisp/Services/OpenAITranslationService.swift index 9eed75b..0b98183 100644 --- a/OpenWhisp/Services/OpenAITranslationService.swift +++ b/OpenWhisp/Services/OpenAITranslationService.swift @@ -109,6 +109,7 @@ final class OpenAITranslationService { endpoint: LLMEndpoint, model: String, customInstruction: String? = nil, + responseFormat: ResponseFormat? = nil, completion: @escaping (Result) -> Void ) { let key = endpoint.apiKey.trimmingCharacters(in: .whitespacesAndNewlines) @@ -133,7 +134,8 @@ final class OpenAITranslationService { messages: [ Message(role: "system", content: instruction), Message(role: "user", content: trimmedText) - ] + ], + responseFormat: responseFormat ) var request = URLRequest(url: url) @@ -235,8 +237,63 @@ private struct ChatCompletionRequest: Encodable { let model: String let temperature: Double let messages: [Message] + /// OpenAI-compatible constrained decoding (spike v7). + /// + /// llama-server implements `response_format: {"type": "json_schema", ...}` by + /// compiling the schema to a GBNF grammar and constraining the sampler, so a + /// response that violates the schema is not merely rejected after the fact — it + /// is UNREPRESENTABLE. That is the difference between the parser catching a + /// bad shape and the bad shape never existing. + /// + /// Optional and omitted when nil (`encodeIfPresent`), so every existing caller's + /// request bytes are byte-identical to v6. Endpoints that don't understand the + /// key are unaffected because they never receive it. + let responseFormat: ResponseFormat? + + private enum CodingKeys: String, CodingKey { + case model, temperature, messages + case responseFormat = "response_format" + } + + func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(model, forKey: .model) + try c.encode(temperature, forKey: .temperature) + try c.encode(messages, forKey: .messages) + // Omitted rather than null: a null `response_format` is a different request + // than an absent one to some servers, and "unchanged for everyone who didn't + // ask" is the whole safety property of this addition. + try c.encodeIfPresent(responseFormat, forKey: .responseFormat) + } } +/// The `response_format` envelope llama-server and the OpenAI API both accept. +struct ResponseFormat: Encodable { + let type: String + let jsonSchema: SchemaEnvelope + + private enum CodingKeys: String, CodingKey { + case type + case jsonSchema = "json_schema" + } + + struct SchemaEnvelope: Encodable { + let name: String + let strict: Bool + let schema: JSONValue + } + + /// Build a `json_schema` response format around a raw schema. + static func jsonSchema(name: String, schema: JSONValue) -> ResponseFormat { + ResponseFormat( + type: "json_schema", + jsonSchema: SchemaEnvelope(name: name, strict: true, schema: schema)) + } +} + +// `JSONValue` lives in MemeAI.swift — it has to be inside OpenWhispCore so the +// schemas that use it are reachable from `swift test`. + private struct Message: Codable { let role: String let content: String diff --git a/Package.swift b/Package.swift index 8cfeb8f..5b26fa6 100644 --- a/Package.swift +++ b/Package.swift @@ -174,6 +174,7 @@ let package = Package( "PluginEnablement.swift", "PluginRegistry.swift", "MemeAI.swift", + "MemeCaptionExtraction.swift", "MemeTemplateMatcher.swift", "MemeCaptionLayout.swift", "MemeTemplateProvider.swift", diff --git a/Tests/OpenWhispCoreTests/MemeSlotEnforcementTests.swift b/Tests/OpenWhispCoreTests/MemeSlotEnforcementTests.swift new file mode 100644 index 0000000..a1d1a01 --- /dev/null +++ b/Tests/OpenWhispCoreTests/MemeSlotEnforcementTests.swift @@ -0,0 +1,361 @@ +import XCTest +@testable import OpenWhispCore + +/// Covers the v7 algorithm change (spike/plugin-system): deterministic caption +/// extraction, host-side slot enforcement, and the constrained-decoding schemas. +/// +/// ## The failure these exist to prevent +/// +/// The v6 live report: "expanding brain: typing, dictating, dictating memes, dictating +/// memes by voice" picked Expanding Brain (4 slots) and rendered TWO captions. The model +/// answered in the legacy `top_text`/`bottom_text` shape, v6's backward-compatible +/// parser accepted it, and nothing compared the caption count to the template's slot +/// count. Three independent things had to be true for that bug to ship, and v7 breaks +/// all three: +/// +/// 1. The captions were re-derived by an LLM even though the user had literally listed +/// them — `MemeCaptionExtraction` now reads them directly. +/// 2. A count mismatch was silently padded — `MemeAI.fit` now refits instead. +/// 3. The legacy two-caption shape was accepted anywhere — it is now accepted as final +/// only on a 2-slot template. +final class MemeSlotEnforcementTests: XCTestCase { + + // MARK: - Deterministic caption extraction + // + // The highest-leverage half of v7: when the user has already said the captions, + // asking a 1.5B model to reproduce them is inventing a failure mode. + + /// THE REPRO. The exact prompt from the owner's screenshot must yield exactly four + /// captions, in the order they were spoken, with the theme split off. + func testTheScreenshotPromptYieldsExactlyFourCaptionsInOrder() { + let extracted = MemeCaptionExtraction.extract( + from: "expanding brain: typing, dictating, dictating memes, dictating memes by voice") + + guard let extracted else { return XCTFail("expected the list to be extracted") } + XCTAssertEqual(extracted.captions, [ + "typing", "dictating", "dictating memes", "dictating memes by voice", + ]) + XCTAssertEqual(extracted.captions.count, 4) + XCTAssertEqual(extracted.slotCount, 4) + // The theme is the template query, kept separate so the search isn't polluted + // by the caption words. + XCTAssertEqual(extracted.theme, "expanding brain") + } + + /// The same list said with a spoken final joiner — "a, b, c and d" — is the same + /// four captions, not three with a run-on last one. + func testASpokenAndJoinerStillSplitsTheFinalItem() { + let extracted = MemeCaptionExtraction.extract( + from: "expanding brain: typing, dictating, dictating memes and dictating memes by voice") + XCTAssertEqual(extracted?.captions, [ + "typing", "dictating", "dictating memes", "dictating memes by voice", + ]) + } + + func testATwoItemListIsExtracted() { + let extracted = MemeCaptionExtraction.extract(from: "drake: manual testing, automated testing") + XCTAssertEqual(extracted?.captions, ["manual testing", "automated testing"]) + XCTAssertEqual(extracted?.theme, "drake") + } + + /// A numbered list needs no colon — the numbering IS the enumeration signal. + func testANumberedListIsExtractedWithoutAColon() { + let extracted = MemeCaptionExtraction.extract(from: "1. wake up 2. write code 3. sleep") + XCTAssertEqual(extracted?.captions, ["wake up", "write code", "sleep"]) + XCTAssertTrue(extracted?.theme.isEmpty ?? false) + } + + /// A dictated or pasted list, one item per line. + func testANewlineSeparatedListIsExtracted() { + let extracted = MemeCaptionExtraction.extract( + from: "expanding brain:\ntyping\ndictating\ndictating memes\ndictating memes by voice") + XCTAssertEqual(extracted?.captions.count, 4) + XCTAssertEqual(extracted?.captions.first, "typing") + XCTAssertEqual(extracted?.theme, "expanding brain") + } + + /// Bullet markers are syntax, never caption text. + func testBulletMarkersAreStrippedFromItems() { + let extracted = MemeCaptionExtraction.extract(from: "steps:\n- plan\n- build\n- ship") + XCTAssertEqual(extracted?.captions, ["plan", "build", "ship"]) + } + + // MARK: - What must NOT be extracted + // + // A false positive here hijacks ordinary prose and captions the meme with sentence + // fragments — worse than the LLM path it would be replacing. + + /// The critical negative case: commas inside a sentence are prose, not a list. + /// Without the colon requirement this would become three captions. + func testProseWithCommasIsNotTreatedAsAList() { + XCTAssertNil(MemeCaptionExtraction.extract( + from: "make me a drake meme about rust, python and go")) + } + + func testOrdinaryProseWithNoEnumerationFallsThroughToTheLLM() { + XCTAssertNil(MemeCaptionExtraction.extract( + from: "a meme where the guy is looking at rust and his girlfriend is python")) + } + + /// A colon followed by clause-length items is prose that happens to have a colon — + /// captions are short by definition. + func testAColonFollowedBySentenceLengthItemsIsNotAList() { + XCTAssertNil(MemeCaptionExtraction.extract( + from: "steps: first you plan the whole thing out carefully, " + + "then you throw it away entirely")) + } + + /// One item is a phrase, not a list — otherwise every description ending in a colon + /// would be hijacked. + func testASingleItemIsNotAList() { + XCTAssertNil(MemeCaptionExtraction.extract(from: "drake: shipping on friday")) + } + + func testAnEmptyDescriptionExtractsNothing() { + XCTAssertNil(MemeCaptionExtraction.extract(from: " ")) + } + + /// Past the slot ceiling no template could hold the captions, so the extraction + /// would be discarded anyway. + func testAListLongerThanTheSlotCeilingIsRefused() { + let items = (1...12).map { "item\($0)" }.joined(separator: ", ") + XCTAssertNil(MemeCaptionExtraction.extract(from: "many: \(items)")) + } + + // MARK: - Host-side slot enforcement + // + // The rule: a caption count that doesn't match the template is REFITTED, never + // silently rendered. + + /// The v6 bug, now caught. Two captions on a four-slot template must refit. + func testTwoCaptionsOnAFourSlotTemplateRefitRatherThanRender() { + let fit = MemeAI.fit(captions: ["ship it", "test it"], slots: 4, wasLegacyShape: true) + guard case .refit(let from, let to) = fit else { + return XCTFail("expected a refit, got \(fit)") + } + XCTAssertEqual(from, ["ship it", "test it"]) + XCTAssertEqual(to, 4) + XCTAssertTrue(fit.needsRefit) + } + + /// The legacy shape is final ONLY when the template really is two-slot. + func testTheLegacyShapeIsAcceptedAsFinalOnATwoSlotTemplate() { + let fit = MemeAI.fit(captions: ["ship it", "test it"], slots: 2, wasLegacyShape: true) + guard case .ready(let captions) = fit else { + return XCTFail("expected ready, got \(fit)") + } + XCTAssertEqual(captions, ["ship it", "test it"]) + XCTAssertFalse(fit.needsRefit) + } + + /// A matching count renders as-is regardless of which wire shape produced it — the + /// COUNT is what decides, not the spelling. + func testAMatchingCountIsReadyWhicheverShapeItCameFrom() { + let array = MemeAI.fit(captions: ["a", "b", "c", "d"], slots: 4, wasLegacyShape: false) + XCTAssertFalse(array.needsRefit) + let legacy = MemeAI.fit(captions: ["a", "b"], slots: 2, wasLegacyShape: true) + XCTAssertFalse(legacy.needsRefit) + } + + /// Too MANY captions is also a mismatch — four captions on a 2-slot Drake would + /// otherwise silently drop the punchline. + func testTooManyCaptionsAlsoRefit() { + let fit = MemeAI.fit(captions: ["a", "b", "c", "d"], slots: 2) + guard case .refit(_, let to) = fit else { return XCTFail("expected a refit") } + XCTAssertEqual(to, 2) + } + + /// Empty captions never refit: there is no joke to preserve, and asking a model to + /// turn nothing into four somethings is a hallucination generator. + func testEmptyCaptionsSeedBlankBoxesRatherThanRefitting() { + XCTAssertFalse(MemeAI.fit(captions: [], slots: 4).needsRefit) + XCTAssertFalse(MemeAI.fit(captions: ["", " "], slots: 4).needsRefit) + } + + /// The refit target is clamped like every other slot count, so a corrupt cache + /// can't ask for 900 captions. + func testTheRefitTargetIsClamped() { + let fit = MemeAI.fit(captions: ["a"], slots: 900) + guard case .refit(_, let to) = fit else { return XCTFail("expected a refit") } + XCTAssertEqual(to, MemeCaptionSlots.maximum) + } + + /// The status line is honest about the shortfall rather than a generic spinner — + /// the user is about to watch the captions change and deserves to know why. + func testTheRefitStatusNamesBothCounts() { + let status = MemeAI.refitStatus(wrote: 2, of: 4) + XCTAssertTrue(status.contains("2")) + XCTAssertTrue(status.contains("4")) + XCTAssertTrue(status.lowercased().contains("refitting")) + } + + /// A refit still seeds the captions it has, so the user sees the joke land while + /// the second round-trip runs rather than staring at an empty canvas. + func testARefitStillCarriesTheCaptionsToSeedMeanwhile() { + let fit = MemeAI.fit(captions: ["ship it", "test it"], slots: 4) + XCTAssertEqual(fit.captions, ["ship it", "test it"]) + } + + // MARK: - Slot GEOMETRY for N≠2 (the screenshot's second bug) + // + // Captions must ALWAYS seed into the template's own slot geometry. A 4-slot + // template can never render as a classic top/bottom pair, whatever the model wrote. + + /// Two captions arriving for a four-slot template still produce FOUR boxes in the + /// four-panel layout — never two boxes at the classic 0.12/0.88 positions. + func testTwoCaptionsOnAFourSlotTemplateStillSeedFourPanelBoxes() { + let boxes = MemeCaptionLayout.seedBoxes(captions: ["ship it", "test it"], slots: 4) + + XCTAssertEqual(boxes.count, 4, "a 4-slot template must always get 4 boxes") + // The classic pair — the wrong answer — is exactly these two centers. + let classic = MemeCaptionLayout.seedBoxes(captions: ["a", "b"], slots: 2) + XCTAssertNotEqual(boxes.prefix(2).map(\.centerY), classic.map(\.centerY), + "must not fall back to classic top/bottom geometry") + // Panel layout: distinct rows, left column, smaller type. + XCTAssertEqual(Set(boxes.map(\.centerY)).count, 4, "each panel needs its own row") + XCTAssertTrue(boxes.allSatisfy { $0.centerX == 0.30 }) + XCTAssertTrue(boxes.allSatisfy { $0.fontSizeShare < MemeCaptionLayout.CaptionBox.defaultFontSizeShare }) + // The captions that DID arrive land in panel order; the rest are typeable blanks. + XCTAssertEqual(boxes.map(\.text), ["ship it", "test it", "", ""]) + } + + /// The four extracted captions fill all four panels in the order the user said them. + func testTheExtractedListSeedsEveryPanelInSpokenOrder() { + let extracted = MemeCaptionExtraction.extract( + from: "expanding brain: typing, dictating, dictating memes, dictating memes by voice") + let boxes = MemeCaptionLayout.seedBoxes( + captions: extracted?.captions ?? [], slots: 4) + + XCTAssertEqual(boxes.map(\.text), [ + "typing", "dictating", "dictating memes", "dictating memes by voice", + ]) + XCTAssertEqual(boxes.map(\.centerY), boxes.map(\.centerY).sorted(), + "panel order must read top-to-bottom") + } + + // MARK: - Slot-count preference in the shortlist + + /// A known caption count puts exact-slot templates first — but never removes the + /// others, because hiding a template the user described is the bug this plugin + /// already fixed once. + func testAKnownSlotCountReordersWithoutFiltering() { + let two = MemeTemplate( + id: "a", name: "Drake Hotline Bling", url: "u", width: 1, height: 1, + source: .imgflip, keywords: [], captionSlots: 2) + let four = MemeTemplate( + id: "b", name: "Expanding Brain", url: "u", width: 1, height: 1, + source: .imgflip, keywords: [], captionSlots: 4) + + let ordered = MemeTemplateCatalog.reorder([two, four], preferringSlots: 4) + XCTAssertEqual(ordered.map(\.id), ["b", "a"], "the 4-slot template comes first") + XCTAssertEqual(ordered.count, 2, "nothing is filtered out") + + // No preference expressed leaves the order exactly as the ranker produced it. + XCTAssertEqual( + MemeTemplateCatalog.reorder([two, four], preferringSlots: nil).map(\.id), ["a", "b"]) + } + + /// Reordering is STABLE: relevance order survives within the matching group. + func testReorderingIsStableWithinEachGroup() { + let templates = (1...4).map { index in + MemeTemplate( + id: "t\(index)", name: "T\(index)", url: "u", width: 1, height: 1, + source: .imgflip, keywords: [], captionSlots: index.isMultiple(of: 2) ? 4 : 2) + } + let ordered = MemeTemplateCatalog.reorder(templates, preferringSlots: 4) + XCTAssertEqual(ordered.map(\.id), ["t2", "t4", "t1", "t3"]) + } + + // MARK: - Constrained decoding schemas + // + // The systemic fix: a schema the sampler enforces makes the bad shapes + // unrepresentable rather than merely rejected. + + /// The refit schema pins the count on BOTH ends — that is what makes + /// "wrote 2 of 4" impossible to emit. + func testTheRefitSchemaPinsExactlyTheRequestedCaptionCount() throws { + let json = try encoded(MemeAI.Schema.refit(slots: 4)) + + XCTAssertTrue(json.contains("\"minItems\":4"), json) + XCTAssertTrue(json.contains("\"maxItems\":4"), json) + XCTAssertTrue(json.contains("\"captions\"")) + // Nothing but captions may come back. + XCTAssertTrue(json.contains("\"additionalProperties\":false")) + } + + func testTheRefitSchemaClampsAnAbsurdSlotCount() throws { + let json = try encoded(MemeAI.Schema.refit(slots: 900)) + XCTAssertTrue(json.contains("\"minItems\":\(MemeCaptionSlots.maximum)"), json) + } + + /// The ranked schema types `templates` as INTEGERS, which is what makes an invented + /// template name unrepresentable rather than merely dropped by the parser. + func testTheRankedSchemaForcesNumericTemplateReferences() throws { + let json = try encoded(MemeAI.Schema.ranked()) + + XCTAssertTrue(json.contains("\"integer\""), json) + XCTAssertTrue(json.contains("\"templates\"")) + XCTAssertTrue(json.contains("\"captions\"")) + XCTAssertTrue(json.contains("\"reason\"")) + // The legacy keys are not in the schema at all, so a constrained model cannot + // reach for them — the v6 bug's entry point is closed. + XCTAssertFalse(json.contains("top_text")) + XCTAssertFalse(json.contains("bottom_text")) + } + + /// The schemas are values, not strings, so they encode to real JSON. + func testSchemasEncodeToWellFormedJSON() throws { + for schema in [MemeAI.Schema.ranked(), MemeAI.Schema.refit(slots: 3)] { + let data = try JSONEncoder().encode(schema) + let parsed = try JSONSerialization.jsonObject(with: data) + XCTAssertTrue(parsed is [String: Any]) + } + } + + // MARK: - The spec carries the user's own captions + + /// When a list was extracted, the model's TEMPLATE choice survives and its captions + /// are replaced by the user's own words. + func testReplacingCaptionsKeepsTheTemplatePickAndClearsTheLegacyFlag() { + let spec = MemeAI.RankedSpec( + templateNames: ["Expanding Brain"], captions: ["ship it", "test it"], + reason: "four panels fit the escalation", wasLegacyShape: true) + + let replaced = spec.replacingCaptions(with: ["a", "b", "c", "d"]) + + XCTAssertEqual(replaced.templateNames, ["Expanding Brain"]) + XCTAssertEqual(replaced.captions, ["a", "b", "c", "d"]) + XCTAssertEqual(replaced.reason, "four panels fit the escalation") + XCTAssertFalse(replaced.wasLegacyShape, "these captions came from the user") + } + + /// End to end on the repro, minus the network: extract → replace → fit → seed must + /// produce four filled boxes and NO refit. + func testTheReproEndsWithFourFilledBoxesAndNoRefit() { + let extracted = MemeCaptionExtraction.extract( + from: "expanding brain: typing, dictating, dictating memes, dictating memes by voice") + guard let extracted else { return XCTFail("expected extraction") } + + // The model answered in the legacy shape — the exact v6 failure. + let spec = MemeAI.RankedSpec( + templateNames: ["Expanding Brain"], captions: ["typing", "by voice"], + wasLegacyShape: true) + let resolved = spec.replacingCaptions(with: extracted.captions) + + let fit = MemeAI.fit( + captions: resolved.captions, slots: 4, wasLegacyShape: resolved.wasLegacyShape) + XCTAssertFalse(fit.needsRefit, "the user's own four captions already fit") + + let boxes = MemeCaptionLayout.seedBoxes(captions: fit.captions, slots: 4) + XCTAssertEqual(boxes.count, 4) + XCTAssertTrue(boxes.allSatisfy { !$0.text.isEmpty }, "every panel is captioned") + } + + // MARK: - Helper + + private func encoded(_ value: JSONValue) throws -> String { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return String(decoding: try encoder.encode(value), as: UTF8.self) + } +} diff --git a/Tests/OpenWhispCoreTests/MemeStructureTests.swift b/Tests/OpenWhispCoreTests/MemeStructureTests.swift index 822f9a0..0d23547 100644 --- a/Tests/OpenWhispCoreTests/MemeStructureTests.swift +++ b/Tests/OpenWhispCoreTests/MemeStructureTests.swift @@ -324,6 +324,10 @@ final class MemeStructureTests: XCTestCase { } /// v5's shape, and the one any model that has seen a two-line meme reaches for. + /// + /// v7 keeps DECODING it — the parser is still forgiving about packaging — but the + /// decision about whether it may be RENDERED moved to `MemeAI.fit`, which accepts it + /// only on a 2-slot template. See `MemeSlotEnforcementTests`. func testLegacyTopAndBottomTextDecodeAsATwoSlotResponse() { let result = MemeAI.parseRanked(""" {"templates":[1],"top_text":"ship it","bottom_text":"test it"} @@ -332,6 +336,20 @@ final class MemeStructureTests: XCTestCase { XCTAssertEqual(spec.captions, ["ship it", "test it"]) XCTAssertEqual(spec.topText, "ship it") XCTAssertEqual(spec.bottomText, "test it") + // v7: the shape is RECORDED so the host can tell a deliberate 2-slot answer + // from a model that never engaged with the slot count. + XCTAssertTrue(spec.wasLegacyShape) + } + + /// The counterpart: a two-element ARRAY is not the legacy shape, even though it + /// carries the same two strings. + func testACaptionsArrayOfTwoIsNotFlaggedAsTheLegacyShape() { + let result = MemeAI.parseRanked(""" + {"templates":[1],"captions":["ship it","test it"]} + """, catalogNames: shortlist) + guard case .success(let spec) = result else { return XCTFail("expected success") } + XCTAssertEqual(spec.captions, ["ship it", "test it"]) + XCTAssertFalse(spec.wasLegacyShape) } /// A response carrying BOTH keeps the richer answer. diff --git a/plugins/MemeGenerator/MemeGeneratorModel.swift b/plugins/MemeGenerator/MemeGeneratorModel.swift index 311f456..0f33490 100644 --- a/plugins/MemeGenerator/MemeGeneratorModel.swift +++ b/plugins/MemeGenerator/MemeGeneratorModel.swift @@ -148,10 +148,18 @@ final class MemeGeneratorModel: ObservableObject { // MARK: - Injected seams - /// One LLM round-trip: (instruction, input, resolved model) -> output. Injected - /// by the window controller, exactly like `ScratchpadModel.AICall`. + /// One LLM round-trip: (instruction, input, resolved model, schema) -> output. + /// Injected by the window controller, like `ScratchpadModel.AICall`. + /// + /// v7 adds `schema`: an optional JSON schema the endpoint should CONSTRAIN the + /// response to (llama-server compiles it to a GBNF grammar). It rides the seam + /// rather than being applied at the AppState layer because only this model knows + /// which of the two meme shapes a given call expects — and because a seam that + /// can't express the constraint would force the schema to be hardcoded next to + /// the transport, out of reach of `swift test`. typealias AICall = ( - _ instruction: String, _ input: String, _ resolved: SummaryModelResolver.Resolved + _ instruction: String, _ input: String, _ resolved: SummaryModelResolver.Resolved, + _ schema: JSONValue? ) async throws -> String private var aiCall: AICall? @@ -430,9 +438,18 @@ final class MemeGeneratorModel: ObservableObject { // prompt at all, rather than truncated off the end of a popularity list. // v6: the user's own past corrections tilt this, within a hard cap and // only among templates the description already matched. + // v7: read the captions straight out of the description when it is + // list-shaped ("expanding brain: a, b, c, d"). Done BEFORE the shortlist so + // the template search can prefer templates with the matching slot count, + // and so the theme ("expanding brain") rather than the whole sentence drives + // the match. Nil for ordinary prose, which falls through to v6 unchanged. + let extracted = MemeCaptionExtraction.extract(from: self.description) + let searchText = extracted.flatMap { $0.theme.isEmpty ? nil : $0.theme } + ?? self.description + let shortlist = MemeTemplateCatalog.prefilter( - for: self.description, in: self.catalog, limit: MemeAI.candidateShortlist, - affinity: self.affinity) + for: searchText, in: self.catalog, limit: MemeAI.candidateShortlist, + affinity: self.affinity, preferringSlots: extracted?.slotCount) // Three positionally-aligned projections of the SAME shortlist: the model // sees `lines` (name + keywords + slot count), answers with numbers that // index it, and any name it writes instead is validated against `names`. @@ -454,7 +471,14 @@ final class MemeGeneratorModel: ObservableObject { case .failure(let rejection): self.finish(ticket, status: "Couldn't build the meme — \(rejection.reason).") case .success(let spec): - await self.applyRanked(spec, ticket: ticket) + // v7: when the description WAS a list, the user's own items are the + // captions and the model's are discarded. The model is still asked + // for both (one round-trip, and its pick is what we want) — but the + // captions it writes are the part it gets wrong, and the part we + // already have verbatim. + let resolvedSpec = extracted.map { spec.replacingCaptions(with: $0.captions) } + ?? spec + await self.applyRanked(resolvedSpec, ticket: ticket) } } catch { guard !self.isCancelled, self.state.accepts(ticket: ticket) else { return } @@ -483,7 +507,11 @@ final class MemeGeneratorModel: ObservableObject { var attempt = 1 while true { do { - return try await aiCall(MemeAI.rankedPrompt, payload, resolved) + // v7: constrain the reply to the ranked schema. On llama-server this + // compiles to a grammar, so `templates` cannot come back as invented + // names and `captions` cannot come back as a top/bottom pair. + return try await aiCall( + MemeAI.rankedPrompt, payload, resolved, MemeAI.Schema.ranked()) } catch { guard MemeGenerateRetry.shouldRetry(error, attempt: attempt) else { throw error } // A superseded or cancelled request must not keep retrying in the @@ -651,11 +679,20 @@ final class MemeGeneratorModel: ObservableObject { candidates = picks candidateReason = spec.reason - // v6: the boxes are seeded for the TOP candidate's real structure, not for an - // assumed top/bottom pair. A 4-slot Expanding Brain gets four boxes with four - // captions in panel order; a 2-slot Drake behaves exactly as it did in v5. - let slots = picks.first?.captionSlots ?? MemeCaptionSlots.default - seedBoxes(captions: spec.captions, slots: slots) + // The boxes are seeded for the TOP candidate's real structure, never for an + // assumed top/bottom pair. This is the second half of the v6 report: the + // geometry ALWAYS comes from the template's own slot count, so an N≠2 template + // can never render as a classic two-liner regardless of what the model wrote. + let slots = MemeCaptionSlots.clamp(picks.first?.captionSlots ?? MemeCaptionSlots.default) + + // v7: the host decides whether these captions may be rendered. A count that + // doesn't match the template is NOT silently padded out with blanks — it is + // refitted, because "Expanding Brain with two captions" is not the joke the + // user asked for. The seed still happens first so the user sees the meme land + // while the refit runs. + let fit = MemeAI.fit( + captions: spec.captions, slots: slots, wasLegacyShape: spec.wasLegacyShape) + seedBoxes(captions: fit.captions, slots: slots) guard let best = picks.first else { finish(ticket, status: "There are no templates to choose from — import one to get started.") @@ -663,6 +700,14 @@ final class MemeGeneratorModel: ObservableObject { } await renderTemplate(best, ticket: ticket, isNewGeneration: true) + + // Ordering matters: the template is on screen before the refit starts, so the + // second round-trip is a visible correction rather than a longer wait. + if case .refit(let current, let target) = fit { + await refitCaptions( + to: best, slots: target, from: current, + status: MemeAI.refitStatus(wrote: current.count, of: target)) + } } /// Replace the AI-seeded boxes, keeping any the user added by hand (v6). @@ -773,12 +818,32 @@ final class MemeGeneratorModel: ObservableObject { // The fast path, and the common one: same slot count means the captions carry // over verbatim, instantly, with no LLM involved — exactly as in v5. guard MemeAI.needsRefit(captions: current, slots: slots) else { return } + await refitCaptions( + to: template, slots: slots, from: current, + status: "Refitting the captions to \(template.name)…") + } + + /// Run the refit round-trip: rewrite `current` into exactly `slots` captions (v7). + /// + /// Extracted from `refitCaptionsIfNeeded` so the GENERATE path can reach it too. + /// That is the point of the v7 change: a caption count that doesn't match the + /// template is the same problem whether it arrived from a template switch or from + /// the model's first answer, so it must have the same fix — and one shared + /// implementation means the generate path can't drift from the tested switch path. + /// + /// `status` is the caller's, because the two entry points mean different things to + /// the user: switching templates says "refitting to X", while a short first answer + /// says "Model wrote 2 of 4 — refitting…", which is an honest account of why the + /// captions are about to change. + private func refitCaptions( + to template: MemeTemplate, slots: Int, from current: [String], status refitStatus: String + ) async { guard let aiCall else { return } let resolved = resolveAIModel() guard ScratchpadAIModel.isUsable(resolved) else { return } let ticket = state.begin(.asking) - status = "Refitting the captions to \(template.name)…" + status = refitStatus startTimeout(ticket: ticket, after: MemeGenerationState.downloadTimeout, message: "Used \(template.name). The captions weren't refitted — edit them by hand.") @@ -788,7 +853,11 @@ final class MemeGeneratorModel: ObservableObject { templateName: template.name) do { - let raw = try await aiCall(MemeAI.refitPrompt, payload, resolved) + // v7: the refit is the ONE call where the required count is known up front, + // so the schema pins minItems == maxItems == slots. With constrained + // decoding a short answer is unrepresentable rather than retried. + let raw = try await aiCall( + MemeAI.refitPrompt, payload, resolved, MemeAI.Schema.refit(slots: slots)) guard !isCancelled, state.accepts(ticket: ticket) else { finish(ticket, status: status) return diff --git a/plugins/MemeGenerator/MemeGeneratorWindowController.swift b/plugins/MemeGenerator/MemeGeneratorWindowController.swift index a903d60..03d2c9c 100644 --- a/plugins/MemeGenerator/MemeGeneratorWindowController.swift +++ b/plugins/MemeGenerator/MemeGeneratorWindowController.swift @@ -59,11 +59,19 @@ final class MemeGeneratorWindowController: NSWindowController, NSWindowDelegate, /// plugin config surface would put that on the plugin's own defaults keys. private func wireAI() { model.configureAI( - call: { instruction, input, resolved in + call: { instruction, input, resolved, schema in try await withCheckedThrowingContinuation { cont in Task { @MainActor in + // v7: when the model gives us a schema, ask the endpoint to + // CONSTRAIN the response to it. llama-server turns this into a + // GBNF grammar, so an off-schema reply becomes unrepresentable + // rather than merely rejected downstream. + let format = schema.map { + ResponseFormat.jsonSchema(name: "meme_response", schema: $0) + } AppState.shared.summarizeResolved( - text: input, instruction: instruction, resolved: resolved + text: input, instruction: instruction, resolved: resolved, + responseFormat: format ) { result in switch result { case .success(let out): cont.resume(returning: out) From 263e8e1e640aa0643a75fda866f0909c0cc3e96e Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Mon, 3 Aug 2026 15:29:31 -0700 Subject: [PATCH 11/15] =?UTF-8?q?spike(meme-plugin):=20move=20the=20captio?= =?UTF-8?q?ns=E2=86=92boxes=20decision=20into=20tested=20core?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The owner's v7 screenshot (four-item Expanding Brain rendered as two boxes carrying the first and last items) is v6 code. The v7 commit e0386cf that fixes it landed at 14:51 today, after that build. Reproduced the exact output against v6's applyRanked: let slots = picks.first?.captionSlots ?? MemeCaptionSlots.default seedBoxes(captions: spec.captions, slots: slots) // no extraction, no fit with the legacy top_text/bottom_text pair decoding to ["typing", "dictating memes by voice"], padded to ["typing", "dictating memes by voice", "", ""]. But the real defect is structural and outlived the fix: v7's tests passed while the app was broken, because they re-implemented the app's sequence in the test body (with `slots: 4` as a literal) while the code that CHAINS extraction → replace → fit → seed lived in plugins/MemeGeneratorModel.swift — PLUGINS=1 only, outside the swift test target. The chain was untested by construction. So the chain moves to core: * NEW MemeCaptionSeeding.resolve(description:specCaptions:wasLegacyShape: templateSlots:) owns the whole decision and returns boxes + an owed refit. Three rules, each closing one way the bug returns: the user's own list beats the model's captions; the geometry always comes from the TEMPLATE's slot count; a count mismatch refits instead of padding blanks. * MemeCaptionSeeding.templateQuery shares that one extraction with the template search, so query and slot preference can't disagree about list-shapedness. * applyRanked is now a call to resolve + UI glue. `apply(seed:)` is the only place boxes are assigned; the description-reading logic is gone from plugins/. Two-caption entry points closed so this can't regress silently: * MemeRenderer.render(template:topText:bottomText:) — @available unavailable (it had no callers; any future one would collapse N→2 at the boundary). * MemeCaptionLayout.seedBoxes(topText:bottomText:) and RankedSpec(templateNames:topText:bottomText:) — deprecated, no production callers left; kept for tests that are legitimately about the 2-slot layout. Tests: 10 new in MemeCaptionSeedingTests drive the REAL function with a raw legacy-shape model reply — text, order, slot geometry, refit reachability. Verified they fail against the v6 behaviour, reproducing the screenshot output exactly, rather than passing vacuously. Re-verified the other v7 wiring claims against the app path: both aiCall sites pass a schema, and it survives configureAI → summarizeResolved → processFinalText → encodeIfPresent(responseFormat); the refit is reachable from applyRanked's real caller via seed.refit. swift test 2634 green (2624 + 10), ratchet 7036/7036, PLUGINS=1 + plain + lean build clean with no new warnings in touched files. Co-Authored-By: Claude --- OpenWhisp/Services/MemeAI.swift | 10 + OpenWhisp/Services/MemeCaptionLayout.swift | 12 ++ OpenWhisp/Services/MemeCaptionSeeding.swift | 157 +++++++++++++++ Package.swift | 1 + .../MemeCaptionSeedingTests.swift | 183 ++++++++++++++++++ .../MemeGenerator/MemeGeneratorModel.swift | 98 ++++++---- plugins/MemeGenerator/MemeRenderer.swift | 24 ++- 7 files changed, 433 insertions(+), 52 deletions(-) create mode 100644 OpenWhisp/Services/MemeCaptionSeeding.swift create mode 100644 Tests/OpenWhispCoreTests/MemeCaptionSeedingTests.swift diff --git a/OpenWhisp/Services/MemeAI.swift b/OpenWhisp/Services/MemeAI.swift index 47f31f1..6749093 100644 --- a/OpenWhisp/Services/MemeAI.swift +++ b/OpenWhisp/Services/MemeAI.swift @@ -294,6 +294,16 @@ public enum MemeAI { /// The classic two-slot spelling, kept so existing call sites and tests that /// think in top/bottom still read naturally. + /// + /// **Deprecated in v8**, for the same reason as `MemeCaptionLayout`'s + /// top/bottom seed: a two-caption constructor is the shape the v6 bug came in, + /// and every production path now carries N captions from `parseRanked` straight + /// into `MemeCaptionSeeding.resolve`. Kept for the tests that assert the legacy + /// wire shape still decodes. + @available(*, deprecated, message: """ + Two-caption shape. Use init(templateNames:captions:) — pass a 2-element \ + array when the template really has two slots. + """) public init(templateNames: [String], topText: String, bottomText: String) { self.init(templateNames: templateNames, captions: [topText, bottomText]) } diff --git a/OpenWhisp/Services/MemeCaptionLayout.swift b/OpenWhisp/Services/MemeCaptionLayout.swift index 67ba3ba..d395711 100644 --- a/OpenWhisp/Services/MemeCaptionLayout.swift +++ b/OpenWhisp/Services/MemeCaptionLayout.swift @@ -204,6 +204,18 @@ public enum MemeCaptionLayout { /// /// Kept as the 2-slot special case of `seedBoxes(captions:slots:)` so the classic /// path is literally the same code and can't drift from it. + /// + /// **Deprecated in v8.** It hard-codes `slots: 2`, so reaching for it on ANY code + /// path that doesn't already know the template is 2-slot is the v6 bug's exact + /// shape: four captions in, two boxes out, no test failing. It has no production + /// callers left — the app seeds through `MemeCaptionSeeding.resolve`, which takes + /// the slot count from the template. Retained (deprecated rather than unavailable) + /// because the geometry assertions in the test suite are legitimately ABOUT the + /// 2-slot layout, and rewriting them would lose that coverage. + @available(*, deprecated, message: """ + Hard-codes 2 slots. Use seedBoxes(captions:slots:) with the template's own \ + captionSlots, or MemeCaptionSeeding.resolve for the full decision. + """) public static func seedBoxes(topText: String, bottomText: String) -> [CaptionBox] { seedBoxes(captions: [topText, bottomText], slots: 2) } diff --git a/OpenWhisp/Services/MemeCaptionSeeding.swift b/OpenWhisp/Services/MemeCaptionSeeding.swift new file mode 100644 index 0000000..f44dc6a --- /dev/null +++ b/OpenWhisp/Services/MemeCaptionSeeding.swift @@ -0,0 +1,157 @@ +import Foundation + +/// The whole "captions → boxes" decision, in one pure place (spike v8). +/// +/// ## Why this type exists — the test gap that let v6 ship +/// +/// v6 rendered the owner's four-item Expanding Brain prompt as TWO captions. The fix +/// (v7) was real, but it was verified by tests that re-implemented the app's steps +/// rather than calling the app's code: each core piece — `MemeCaptionExtraction`, +/// `RankedSpec.replacingCaptions`, `MemeAI.fit`, `MemeCaptionLayout.seedBoxes` — was +/// proved correct in isolation, while the code that CHAINS them lived in +/// `MemeGeneratorModel.applyRanked`, inside `plugins/`, which compiles only under +/// `PLUGINS=1` and is outside the `swift test` target. +/// +/// So the sequence was untested by construction, and a test asserting "extract, then +/// replace, then fit with slots: 4" could pass forever while the app passed +/// `spec.captions` straight to `seedBoxes` — which is precisely what v6 did: +/// +/// ```swift +/// let slots = picks.first?.captionSlots ?? MemeCaptionSlots.default +/// seedBoxes(captions: spec.captions, slots: slots) // v6 — no extraction, no fit +/// ``` +/// +/// The two filled boxes were the legacy `top_text`/`bottom_text` pair, padded out with +/// blanks by `seedBoxes` and rendered without complaint. +/// +/// This type moves that chain OUT of the plugin. `resolve` takes everything the +/// decision depends on — the user's description, the model's answer, and the chosen +/// template's slot count — and returns the boxes plus whether a refit is owed. The +/// plugin keeps only what genuinely needs AppKit: assigning `boxes`, merging the +/// user's hand-added boxes, and running the async refit round-trip. +/// +/// The property that matters: a caption-count regression now fails `swift test` +/// against the SAME function the app calls, on a stock build, with no `PLUGINS=1`. +public enum MemeCaptionSeeding { + + /// A resolved seeding decision: the boxes to show now, and what is still owed. + public struct Seed: Equatable, Sendable { + /// The boxes to put on the canvas, one per slot the template really has. + public let boxes: [MemeCaptionLayout.CaptionBox] + + /// The captions those boxes carry, before layout — kept so a caller (and a + /// test) can assert the TEXT decision separately from the geometry. + public let captions: [String] + + /// The slot count the boxes were laid out for: the template's own structure, + /// clamped. Never an assumed pair. + public let slots: Int + + /// The refit round-trip owed for this seed, or nil when the captions already + /// fill the template. Non-nil is not an error — the boxes are still valid and + /// are shown immediately; the refit is a visible correction that follows. + public let refit: Refit? + + /// True when the captions came from the user's own words rather than the + /// model's. Drives the status line, and is the signal that no LLM caption + /// round-trip was needed at all. + public let captionsCameFromUser: Bool + + public init( + boxes: [MemeCaptionLayout.CaptionBox], captions: [String], slots: Int, + refit: Refit? = nil, captionsCameFromUser: Bool = false + ) { + self.boxes = boxes + self.captions = captions + self.slots = slots + self.refit = refit + self.captionsCameFromUser = captionsCameFromUser + } + } + + /// A second round-trip owed because the caption count didn't match the template. + public struct Refit: Equatable, Sendable { + /// The captions to rewrite. + public let from: [String] + /// How many captions the template needs. + public let slots: Int + /// The status line to show while it runs — honest about the shortfall. + public var status: String { MemeAI.refitStatus(wrote: from.count, of: slots) } + + public init(from: [String], slots: Int) { + self.from = from + self.slots = slots + } + } + + /// Decide the boxes for a ranked answer, given the template that will hold them. + /// + /// The order of the three rules is the whole design, and each one removes a way the + /// v6 bug could come back: + /// + /// 1. **The user's own words win.** When the description was list-shaped + /// ("expanding brain: a, b, c, d") those items ARE the captions, verbatim and in + /// order, and the model's captions are discarded. A model cannot return the wrong + /// number of captions for a question that was never asked. + /// 2. **The geometry comes from the template**, never from the caption count. A + /// 4-slot template lays out four boxes whether the model wrote one caption or + /// seven, so an N≠2 template can never render as a classic two-liner. + /// 3. **A count mismatch refits rather than padding.** Two captions on a four-slot + /// template is not "two captions and two blanks", it is the wrong answer, and it + /// is sent back to be rewritten. + /// + /// - Parameters: + /// - description: what the user typed or dictated — read for a caption list first. + /// - specCaptions: the captions the model returned. + /// - wasLegacyShape: whether those came from `top_text`/`bottom_text`. + /// - templateSlots: the chosen template's own slot count, or nil when there is no + /// template yet (falls back to the classic default, as every path always has). + public static func resolve( + description: String, + specCaptions: [String], + wasLegacyShape: Bool = false, + templateSlots: Int? + ) -> Seed { + let extracted = MemeCaptionExtraction.extract(from: description) + + // Rule 1: the user's own list wins over whatever the model wrote. + let captions = extracted?.captions ?? specCaptions + // Captions taken from the user are never "legacy shaped" — they didn't come + // from a top/bottom pair, and treating them as such would misreport the status. + let legacy = extracted == nil ? wasLegacyShape : false + + // Rule 2: the geometry is the TEMPLATE's, always. + let slots = MemeCaptionSlots.clamp(templateSlots ?? MemeCaptionSlots.default) + + // Rule 3: a mismatch is refitted, not padded. + let fit = MemeAI.fit(captions: captions, slots: slots, wasLegacyShape: legacy) + + let boxes = MemeCaptionLayout.seedBoxes(captions: fit.captions, slots: slots) + let refit: Refit? = { + guard case .refit(let from, let target) = fit else { return nil } + return Refit(from: from, slots: target) + }() + + return Seed( + boxes: boxes, captions: fit.captions, slots: slots, refit: refit, + captionsCameFromUser: extracted != nil) + } + + /// The template search query for a description, and the slot count to prefer. + /// + /// Lives here rather than in the plugin for the same reason as `resolve`: it is a + /// pure decision derived from the SAME extraction, and splitting the two across the + /// test boundary is how they would drift. When the description is list-shaped the + /// theme ("expanding brain") is the query — searching with the caption words in it + /// would score the template against text that is about to become its captions — and + /// the item count is the slot count to prefer. + public static func templateQuery(for description: String) -> (query: String, preferredSlots: Int?) { + guard let extracted = MemeCaptionExtraction.extract(from: description) else { + return (description, nil) + } + // A themeless list ("1. a 2. b") still tells us the slot count, but has no + // better query than the description itself. + let query = extracted.theme.isEmpty ? description : extracted.theme + return (query, extracted.slotCount) + } +} diff --git a/Package.swift b/Package.swift index 5b26fa6..b59cb21 100644 --- a/Package.swift +++ b/Package.swift @@ -175,6 +175,7 @@ let package = Package( "PluginRegistry.swift", "MemeAI.swift", "MemeCaptionExtraction.swift", + "MemeCaptionSeeding.swift", "MemeTemplateMatcher.swift", "MemeCaptionLayout.swift", "MemeTemplateProvider.swift", diff --git a/Tests/OpenWhispCoreTests/MemeCaptionSeedingTests.swift b/Tests/OpenWhispCoreTests/MemeCaptionSeedingTests.swift new file mode 100644 index 0000000..6951992 --- /dev/null +++ b/Tests/OpenWhispCoreTests/MemeCaptionSeedingTests.swift @@ -0,0 +1,183 @@ +import XCTest +@testable import OpenWhispCore + +/// The captions→boxes decision, tested against the SAME function the app calls (v8). +/// +/// ## The gap these close +/// +/// v7 fixed the four-captions bug and shipped tests that passed — but those tests +/// re-implemented the app's sequence (extract → replace → fit → seed) inside the test +/// body, with `slots: 4` written as a literal. The code that actually chained those +/// steps lived in `plugins/MemeGenerator/MemeGeneratorModel.swift`, which compiles only +/// under `PLUGINS=1` and is outside the `swift test` target, so the chain itself was +/// untested by construction. A test spelling out the right sequence proves nothing about +/// an app that performs a different one — which is exactly what v6 did: +/// +/// ```swift +/// seedBoxes(captions: spec.captions, slots: slots) // v6: no extraction, no fit +/// ``` +/// +/// `MemeCaptionSeeding.resolve` now owns that chain, and the app is a call to it. These +/// tests drive `resolve` with a raw model reply, so a regression anywhere in the +/// sequence fails here on a stock `swift test` run. +final class MemeCaptionSeedingTests: XCTestCase { + + /// THE REPRO, end to end through the real decision. + /// + /// The owner's prompt, the model answering in the legacy `top_text`/`bottom_text` + /// shape with the user's FIRST and LAST items (the screenshot exactly), and a 4-slot + /// Expanding Brain. Four boxes, four captions, in order, no refit. + func testTheScreenshotReproSeedsFourBoxesFromTheUsersOwnWords() { + let description = + "expanding brain: typing, dictating, dictating memes, dictating memes by voice" + + // What the v6 failure looked like on the wire: two captions, first and last. + let raw = #""" + {"templates":["Expanding Brain"],"top_text":"typing", + "bottom_text":"dictating memes by voice","reason":"escalation"} + """# + guard case .success(let spec) = MemeAI.parseRanked( + raw, catalogNames: ["Expanding Brain", "Drake Hotline Bling"]) + else { return XCTFail("the legacy shape must still parse") } + + // Precondition: the model really did return only two captions. + XCTAssertEqual(spec.captions.count, 2) + XCTAssertTrue(spec.wasLegacyShape) + + let seed = MemeCaptionSeeding.resolve( + description: description, + specCaptions: spec.captions, + wasLegacyShape: spec.wasLegacyShape, + templateSlots: 4) + + // The heart of it: four boxes carrying the user's own four items, in order. + XCTAssertEqual(seed.boxes.count, 4, "a 4-slot template gets four boxes") + XCTAssertEqual(seed.boxes.map(\.text), [ + "typing", "dictating", "dictating memes", "dictating memes by voice", + ]) + XCTAssertEqual(seed.slots, 4) + XCTAssertTrue(seed.boxes.allSatisfy { !$0.text.isEmpty }, "no blank panels") + XCTAssertNil(seed.refit, "the user's own four captions already fit") + XCTAssertTrue(seed.captionsCameFromUser) + + // The exact v6 signature must be gone: never two filled boxes carrying the + // first and last items. + let filled = seed.boxes.filter { !$0.text.isEmpty } + XCTAssertNotEqual( + filled.map(\.text), ["typing", "dictating memes by voice"], + "the v6 collapse: first and last item, seeded top/bottom") + } + + /// Slot GEOMETRY, not just the text: four boxes must be four distinct stacked + /// positions, not two top/bottom ones with extras piled on. + func testTheReproSeedsFourDistinctStackedSlots() { + let seed = MemeCaptionSeeding.resolve( + description: "expanding brain: typing, dictating, dictating memes, dictating memes by voice", + specCaptions: ["typing", "by voice"], + wasLegacyShape: true, + templateSlots: 4) + + let centersY = seed.boxes.map(\.centerY) + XCTAssertEqual(Set(centersY).count, 4, "four distinct vertical positions") + XCTAssertEqual(centersY, centersY.sorted(), "panel order runs top to bottom") + + // The classic 2-slot layout puts captions at 0.12/0.88. A 4-slot template must + // NOT reuse those — that is the visual signature of the bug. + XCTAssertNotEqual(centersY.first, 0.12, "not the classic top caption position") + XCTAssertNotEqual(centersY.last, 0.88, "not the classic bottom caption position") + } + + /// The other half of rule 2: the geometry comes from the TEMPLATE, so the same + /// four-item description on a 2-slot template does not invent four boxes. + func testTheTemplateOwnsTheSlotCountNotTheCaptionList() { + let seed = MemeCaptionSeeding.resolve( + description: "drake: manual testing, automated testing, prod, prayer", + specCaptions: ["a", "b"], + wasLegacyShape: true, + templateSlots: 2) + + XCTAssertEqual(seed.boxes.count, 2, "a 2-slot template gets exactly two boxes") + // Four captions for two slots is a mismatch, and a mismatch REFITS rather than + // dropping the extras silently. + XCTAssertEqual(seed.refit?.slots, 2) + XCTAssertEqual(seed.refit?.from.count, 4) + } + + /// Ordinary prose still goes through the model, unchanged: no extraction, and the + /// model's captions are what get seeded. + func testProseKeepsTheModelsCaptions() { + let seed = MemeCaptionSeeding.resolve( + description: "make me a drake meme about rust and go", + specCaptions: ["rust", "go"], + wasLegacyShape: true, + templateSlots: 2) + + XCTAssertEqual(seed.boxes.map(\.text), ["rust", "go"]) + XCTAssertFalse(seed.captionsCameFromUser) + XCTAssertNil(seed.refit, "two captions fill a two-slot template") + } + + /// A short model answer on a 4-slot template — prose, so no extraction to save it — + /// must refit rather than render two captions and two blanks. This is the v6 bug's + /// other entry point, and the refit must be REACHABLE from the resolve the app calls. + func testAShortProseAnswerOnAFourSlotTemplateOwesARefit() { + let seed = MemeCaptionSeeding.resolve( + description: "an expanding brain meme about testing", + specCaptions: ["typing", "by voice"], + wasLegacyShape: true, + templateSlots: 4) + + XCTAssertEqual(seed.boxes.count, 4, "the boxes still match the template") + guard let refit = seed.refit else { + return XCTFail("a 2-of-4 answer must owe a refit, not render two blanks") + } + XCTAssertEqual(refit.slots, 4) + XCTAssertEqual(refit.from, ["typing", "by voice"]) + XCTAssertEqual(refit.status, "Model wrote 2 of 4 — refitting…") + } + + /// No template chosen yet falls back to the classic default rather than crashing or + /// seeding zero boxes. + func testNoTemplateFallsBackToTheClassicDefault() { + let seed = MemeCaptionSeeding.resolve( + description: "something funny", specCaptions: ["a", "b"], templateSlots: nil) + XCTAssertEqual(seed.slots, MemeCaptionSlots.default) + XCTAssertEqual(seed.boxes.count, MemeCaptionSlots.default) + } + + /// An empty answer seeds empty boxes to type into and owes NO refit — asking a model + /// to rewrite nothing into four somethings is how you get four hallucinations. + func testAnEmptyAnswerSeedsEmptyBoxesWithoutARefit() { + let seed = MemeCaptionSeeding.resolve( + description: "an expanding brain meme", specCaptions: [], templateSlots: 4) + XCTAssertEqual(seed.boxes.count, 4) + XCTAssertTrue(seed.boxes.allSatisfy { $0.text.isEmpty }) + XCTAssertNil(seed.refit) + } + + // MARK: - The template query comes from the same extraction + + /// A list-shaped description searches on its THEME and prefers the item count, so + /// the caption words don't pollute the template match. + func testAListSearchesOnItsThemeAndPrefersItsItemCount() { + let search = MemeCaptionSeeding.templateQuery( + for: "expanding brain: typing, dictating, dictating memes, dictating memes by voice") + XCTAssertEqual(search.query, "expanding brain") + XCTAssertEqual(search.preferredSlots, 4) + } + + /// Prose searches on itself and expresses no slot preference. + func testProseSearchesOnTheWholeDescription() { + let search = MemeCaptionSeeding.templateQuery(for: "a drake meme about rust and go") + XCTAssertEqual(search.query, "a drake meme about rust and go") + XCTAssertNil(search.preferredSlots) + } + + /// A themeless list still reports its slot count — the numbering is the enumeration + /// signal — but has no better query than the description itself. + func testAThemelessListStillReportsItsSlotCount() { + let search = MemeCaptionSeeding.templateQuery(for: "1. wake up 2. write code 3. sleep") + XCTAssertEqual(search.query, "1. wake up 2. write code 3. sleep") + XCTAssertEqual(search.preferredSlots, 3) + } +} diff --git a/plugins/MemeGenerator/MemeGeneratorModel.swift b/plugins/MemeGenerator/MemeGeneratorModel.swift index 0f33490..b9c84f7 100644 --- a/plugins/MemeGenerator/MemeGeneratorModel.swift +++ b/plugins/MemeGenerator/MemeGeneratorModel.swift @@ -443,13 +443,14 @@ final class MemeGeneratorModel: ObservableObject { // the template search can prefer templates with the matching slot count, // and so the theme ("expanding brain") rather than the whole sentence drives // the match. Nil for ordinary prose, which falls through to v6 unchanged. - let extracted = MemeCaptionExtraction.extract(from: self.description) - let searchText = extracted.flatMap { $0.theme.isEmpty ? nil : $0.theme } - ?? self.description + // v8: the query and the preferred slot count come from the same core + // decision that later seeds the boxes, so the two can't disagree about + // whether the description was a list. + let search = MemeCaptionSeeding.templateQuery(for: self.description) let shortlist = MemeTemplateCatalog.prefilter( - for: searchText, in: self.catalog, limit: MemeAI.candidateShortlist, - affinity: self.affinity, preferringSlots: extracted?.slotCount) + for: search.query, in: self.catalog, limit: MemeAI.candidateShortlist, + affinity: self.affinity, preferringSlots: search.preferredSlots) // Three positionally-aligned projections of the SAME shortlist: the model // sees `lines` (name + keywords + slot count), answers with numbers that // index it, and any name it writes instead is validated against `names`. @@ -471,14 +472,10 @@ final class MemeGeneratorModel: ObservableObject { case .failure(let rejection): self.finish(ticket, status: "Couldn't build the meme — \(rejection.reason).") case .success(let spec): - // v7: when the description WAS a list, the user's own items are the - // captions and the model's are discarded. The model is still asked - // for both (one round-trip, and its pick is what we want) — but the - // captions it writes are the part it gets wrong, and the part we - // already have verbatim. - let resolvedSpec = extracted.map { spec.replacingCaptions(with: $0.captions) } - ?? spec - await self.applyRanked(resolvedSpec, ticket: ticket) + // v8: the "user's own list beats the model's captions" rule moved + // into `MemeCaptionSeeding.resolve`, which `applyRanked` calls — + // one tested place instead of a step here and a step there. + await self.applyRanked(spec, ticket: ticket) } } catch { guard !self.isCancelled, self.state.accepts(ticket: ticket) else { return } @@ -679,20 +676,20 @@ final class MemeGeneratorModel: ObservableObject { candidates = picks candidateReason = spec.reason - // The boxes are seeded for the TOP candidate's real structure, never for an - // assumed top/bottom pair. This is the second half of the v6 report: the - // geometry ALWAYS comes from the template's own slot count, so an N≠2 template - // can never render as a classic two-liner regardless of what the model wrote. - let slots = MemeCaptionSlots.clamp(picks.first?.captionSlots ?? MemeCaptionSlots.default) - - // v7: the host decides whether these captions may be rendered. A count that - // doesn't match the template is NOT silently padded out with blanks — it is - // refitted, because "Expanding Brain with two captions" is not the joke the - // user asked for. The seed still happens first so the user sees the meme land - // while the refit runs. - let fit = MemeAI.fit( - captions: spec.captions, slots: slots, wasLegacyShape: spec.wasLegacyShape) - seedBoxes(captions: fit.captions, slots: slots) + // v8: the ENTIRE captions→boxes decision is one pure core call. Extraction of + // the user's own list, the template's slot geometry, and the refit rule used to + // be three steps chained HERE, in a file `swift test` cannot compile — so the + // chain was untested by construction even while each link had tests. That is + // exactly how v6 shipped `seedBoxes(captions: spec.captions, slots:)` with no + // fit at all and rendered the owner's four-item prompt as two captions. + // + // Everything below is UI glue: assign the boxes, render, run the owed refit. + let seed = MemeCaptionSeeding.resolve( + description: description, + specCaptions: spec.captions, + wasLegacyShape: spec.wasLegacyShape, + templateSlots: picks.first?.captionSlots) + apply(seed: seed) guard let best = picks.first else { finish(ticket, status: "There are no templates to choose from — import one to get started.") @@ -703,29 +700,46 @@ final class MemeGeneratorModel: ObservableObject { // Ordering matters: the template is on screen before the refit starts, so the // second round-trip is a visible correction rather than a longer wait. - if case .refit(let current, let target) = fit { + if let refit = seed.refit { await refitCaptions( - to: best, slots: target, from: current, - status: MemeAI.refitStatus(wrote: current.count, of: target)) + to: best, slots: refit.slots, from: refit.from, status: refit.status) } } - /// Replace the AI-seeded boxes, keeping any the user added by hand (v6). + /// Put a resolved core seed onto the canvas — the ONLY place boxes are seeded. /// - /// The rule and the reasoning behind choosing it over a confirmation dialog live in - /// `MemeCaptionLayout.merging`; this is only the plumbing. `seededBoxIDs` is - /// re-minted from the NEW seed, so the next regenerate replaces these in turn while - /// the user's own boxes keep surviving indefinitely. - private func seedBoxes(captions: [String], slots: Int) { - let seed = MemeCaptionLayout.seedBoxes(captions: captions, slots: slots) - boxes = MemeCaptionLayout.merging(seed: seed, into: boxes, seededIDs: seededBoxIDs) - seededBoxIDs = Set(seed.map(\.id)) - // Select the first seeded box rather than whatever was selected before: after a - // regenerate the user is looking at new captions, and leaving the editor panel - // pointed at a box that may no longer exist would show an empty panel. + /// This is the app-layer half of seeding, and all of it is state mutation: merge the + /// new seed over the user's hand-added boxes (the rule and the reasoning live in + /// `MemeCaptionLayout.merging`), re-mint `seededBoxIDs` from the NEW seed so the + /// next regenerate replaces these in turn while the user's own boxes survive + /// indefinitely, and move the selection to the first box — after a regenerate the + /// user is looking at new captions, and leaving the editor panel pointed at a box + /// that may no longer exist would show an empty panel. + /// + /// Every decision ABOUT the seed — which captions, how many slots, whether a refit + /// is owed — was already made in `MemeCaptionSeeding`, under test. + private func apply(seed: MemeCaptionSeeding.Seed) { + boxes = MemeCaptionLayout.merging( + seed: seed.boxes, into: boxes, seededIDs: seededBoxIDs) + seededBoxIDs = Set(seed.boxes.map(\.id)) selectedBoxID = boxes.first?.id } + /// Seed boxes for a known caption list and slot count, with no description to read. + /// + /// The two callers that legitimately have no description in play: picking a template + /// before ever generating (empty captions, the template's own slots), and applying a + /// completed refit (captions already exactly `slots` long). Both still go through + /// the core layout + the single `apply` path, so there is no second way to put boxes + /// on the canvas. + private func seedBoxes(captions: [String], slots: Int) { + let count = MemeCaptionSlots.clamp(slots) + apply(seed: MemeCaptionSeeding.Seed( + boxes: MemeCaptionLayout.seedBoxes(captions: captions, slots: count), + captions: captions, + slots: count)) + } + // MARK: - Template selection /// Re-render the current captions onto another template. diff --git a/plugins/MemeGenerator/MemeRenderer.swift b/plugins/MemeGenerator/MemeRenderer.swift index def781e..1a37242 100644 --- a/plugins/MemeGenerator/MemeRenderer.swift +++ b/plugins/MemeGenerator/MemeRenderer.swift @@ -68,16 +68,20 @@ enum MemeRenderer { return output } - /// v1 entry point, kept so the AI path reads the same as before: seed two boxes - /// and render them. The seeded geometry reproduces the classic top/bottom look. - static func render( - template: NSImage, - topText: String, - bottomText: String - ) -> NSImage { - render( - template: template, - boxes: MemeCaptionLayout.seedBoxes(topText: topText, bottomText: bottomText)) + /// v1's two-caption entry point. **Removed in v8 — do not reintroduce.** + /// + /// It had no callers left (the render path takes `boxes:`), and leaving a + /// top/bottom-shaped overload in reach is precisely how a 4-slot template gets + /// rendered as a classic two-liner: any future call site that reached for it would + /// collapse N captions to 2 at the boundary, silently and without failing a test. + /// The `boxes:` overload above is the only entry point; a caller that genuinely has + /// two captions passes a 2-element array through `MemeCaptionSeeding`. + @available(*, unavailable, message: """ + Removed in v8: renders exactly two captions and silently drops the rest. \ + Use render(template:boxes:) with boxes from MemeCaptionSeeding.resolve. + """) + static func render(template: NSImage, topText: String, bottomText: String) -> NSImage { + fatalError("unavailable") } /// Draw one resolved box. From 9323a5aa16e1ee1fc38c3ef9bc1041b732c944d7 Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Mon, 3 Aug 2026 16:24:24 -0700 Subject: [PATCH 12/15] spike(meme-plugin): a stale catalog cache said every meme was 2 slots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The owner's four-item Expanding Brain prompt rendered as two captions for the third time. v7 and v8 each traced the caption wiring by eye, found it correct, and shipped; the caption wiring WAS correct. It was being handed `slots: 2`. `MemeTemplate.captionSlots` arrived in v6 without a bump to `MemeCatalogCache.currentVersion`, and `decide` admitted anything `<= currentVersion`. So a catalog cached by a v5-era build stayed version 1 and was accepted as current, while `MemeTemplate.init(from:)` — deliberately tolerant so an old cache still loads — defaulted the missing field to 2. Every template in the corpus reported two caption slots, Expanding Brain included. Extraction then read four captions correctly, matched the right template, and the count-mismatch rule refit them DOWN to two — which is why the meme kept the first and last items and dropped the middle two. Fixed by bumping the version to 2 and gating on equality: a pre-captionSlots cache is refetched, because the slot counts were never written to that file and there is nothing to migrate them from. Proved at runtime rather than by reading, since reading had failed twice. `MemeTrace` emits a breadcrumb at each decision, and a launch-gated probe (OPENWHISP_MEME_PROBE_PROMPT) drives the real `generate()` through `PluginHost`, so a build can capture what the shipping binary decides. Before, against the owner's own stale cache: extraction fired: 4 items, theme: "expanding brain" resolve(..., specCaptions: 4, slots: 2) -> 2 boxes, refit: 4->2 probe result: 2 boxes, texts=["typing", "dictating memes by voice"] After, same binary, same prompt, cache refetched: resolve(..., specCaptions: 4, slots: 4) -> 4 boxes, refit: none probe result: 4 boxes, texts=["typing", "dictating", "dictating memes", "dictating memes by voice"] Also fixes the drag: the caption is burned into the preview, so dragging moved an empty dashed outline while the text stayed behind until drop. The dragged box now renders empty (`MemeCaptionLayout.hidingText`) while the handle carries a live copy, so the text travels with the cursor. The duplicate is explicit and lasts only for the gesture — on drop the real renderer's output is what remains, so it can never reach the export. Tests: the cache-policy helper defaulted to a literal `version: 1`, so the suite was asserting the stale-cache behaviour was correct. It now defaults to `currentVersion`, plus regressions for the older-version gate, the tolerant decode, a slot-count round-trip, the owner's prompt at 4 and at 2 slots, and the drag helper. 2645 pass. Co-Authored-By: Claude --- OpenWhisp/AppMain.swift | 37 +++++++ OpenWhisp/Services/MemeCaptionLayout.swift | 21 ++++ OpenWhisp/Services/MemeCaptionSeeding.swift | 13 ++- OpenWhisp/Services/MemeCatalogCache.swift | 49 ++++++++- OpenWhisp/Services/MemeTrace.swift | 78 +++++++++++++ OpenWhisp/Views/PluginHost.swift | 8 ++ Package.swift | 1 + .../MemeProviderTests.swift | 103 +++++++++++++++++- .../MemeStructureTests.swift | 67 ++++++++++++ .../MemeGenerator/MemeGeneratorModel.swift | 37 ++++++- plugins/MemeGenerator/MemeGeneratorView.swift | 67 +++++++++++- .../MemeGeneratorWindowController.swift | 39 +++++++ 12 files changed, 504 insertions(+), 16 deletions(-) create mode 100644 OpenWhisp/Services/MemeTrace.swift diff --git a/OpenWhisp/AppMain.swift b/OpenWhisp/AppMain.swift index 5f35d99..286731d 100644 --- a/OpenWhisp/AppMain.swift +++ b/OpenWhisp/AppMain.swift @@ -120,6 +120,43 @@ class OpenWhispApp: NSObject, NSApplicationDelegate { // First-run onboarding showOnboardingIfNeeded() print("[OpenWhisp] Ready") + + // v9 spike: the meme plugin's runtime-proof probe. Opens the plugin window and + // drives the real Generate from `OPENWHISP_MEME_PROBE_PROMPT`, so a build + // script can capture what the SHIPPING binary decides instead of another + // reading of the source. Absent the env var this is a no-op. + startMemeProbeIfRequested() + } + + /// Launch-time entry for the meme plugin's runtime proof (spike v9). + /// + /// Deliberately routed through `PluginHost.open` rather than constructing the + /// controller directly: the probe is only worth anything if it exercises the same + /// window the menu item opens, including the enablement gate and the cached-window + /// reuse. A probe with its own construction path could pass while the real one + /// failed — which is the exact class of mistake this whole exercise is about. + private func startMemeProbeIfRequested() { + let env = ProcessInfo.processInfo.environment + guard let prompt = env["OPENWHISP_MEME_PROBE_PROMPT"], !prompt.isEmpty else { return } + MemeTrace.log("probe requested at launch") + + // The catalog and the LLM warm on window open; give them a moment before + // firing Generate, exactly as a human opening the window and speaking would. + let delay = Double(env["OPENWHISP_MEME_PROBE_DELAY"] ?? "") ?? 6 + DispatchQueue.main.asyncAfter(deadline: .now() + delay) { + PluginHost.shared.open(pluginID: PluginRegistry.memeGenerator.id) + #if OPENWHISP_PLUGINS + guard let controller = PluginHost.shared.windowController( + for: PluginRegistry.memeGenerator.id) as? MemeGeneratorWindowController + else { + MemeTrace.log("probe ABORTED: no meme window controller (plugin disabled?)") + return + } + controller.runTraceProbeIfRequested() + #else + MemeTrace.log("probe ABORTED: build has no plugins (PLUGINS=1 ./build.sh)") + #endif + } } /// Re-check permissions whenever the app becomes active. This is what makes diff --git a/OpenWhisp/Services/MemeCaptionLayout.swift b/OpenWhisp/Services/MemeCaptionLayout.swift index d395711..4005947 100644 --- a/OpenWhisp/Services/MemeCaptionLayout.swift +++ b/OpenWhisp/Services/MemeCaptionLayout.swift @@ -195,6 +195,27 @@ public enum MemeCaptionLayout { return out } + /// Blank the TEXT of one box, keeping every box and all geometry (v9). + /// + /// Used for the live drag: the box being dragged has its caption drawn by the drag + /// handle, travelling with the cursor, so the burned-in render must not draw it a + /// second time at the position the user is moving it away from. + /// + /// The box is emptied rather than REMOVED, and that distinction is the whole point: + /// `boxes` is the document, indices and ids are referenced by the editor's + /// selection, and dropping an element mid-gesture to achieve a visual effect would + /// make a rendering concern edit the user's data. Emptying `text` changes only what + /// is painted. `nil` returns the boxes untouched, so the resting path is identity. + public static func hidingText(of id: UUID?, in boxes: [CaptionBox]) -> [CaptionBox] { + guard let id else { return boxes } + return boxes.map { box in + guard box.id == id else { return box } + var hidden = box + hidden.text = "" + return hidden + } + } + /// The two boxes the AI path seeds: classic top and bottom captions. /// /// Positioned at 0.12 / 0.88 of the height — far enough in that a two-line diff --git a/OpenWhisp/Services/MemeCaptionSeeding.swift b/OpenWhisp/Services/MemeCaptionSeeding.swift index f44dc6a..11db820 100644 --- a/OpenWhisp/Services/MemeCaptionSeeding.swift +++ b/OpenWhisp/Services/MemeCaptionSeeding.swift @@ -132,9 +132,20 @@ public enum MemeCaptionSeeding { return Refit(from: from, slots: target) }() - return Seed( + let seed = Seed( boxes: boxes, captions: fit.captions, slots: slots, refit: refit, captionsCameFromUser: extracted != nil) + + // v9: breadcrumbs at the decision itself, not at the call site. Two rounds of + // "the wiring reads correct" were wrong about the running app, so the trace + // has to come from the function that actually made the decision — a call-site + // log can only prove that the call site ran. + MemeTrace.log(MemeTrace.extractionLine(extracted)) + MemeTrace.log(MemeTrace.seedLine( + description: description, specCaptions: specCaptions, + slots: templateSlots, seed: seed)) + + return seed } /// The template search query for a description, and the slot count to prefer. diff --git a/OpenWhisp/Services/MemeCatalogCache.swift b/OpenWhisp/Services/MemeCatalogCache.swift index d8366da..56ced2a 100644 --- a/OpenWhisp/Services/MemeCatalogCache.swift +++ b/OpenWhisp/Services/MemeCatalogCache.swift @@ -27,7 +27,32 @@ public enum MemeCatalogCache { } } - public static let currentVersion = 1 + /// The cache format version. + /// + /// ## v9: why this is 2, and what a stale 1 actually cost + /// + /// v6 added `MemeTemplate.captionSlots` — the field the whole "a 4-panel meme gets + /// 4 captions" behaviour hangs off — and did NOT bump this. So every catalog cached + /// by a v5-era build stayed `version: 1`, `decide` accepted it as current, and + /// `MemeTemplate.init(from:)` — deliberately tolerant, so an old cache still loads — + /// defaulted the missing field to `MemeCaptionSlots.default`, i.e. 2. + /// + /// The result was invisible and total: EVERY template in the corpus reported two + /// slots. Expanding Brain reported two slots. The owner's four-item prompt then + /// extracted four captions correctly, matched the right template, and was refit + /// DOWN to two by the rule that a count mismatch must be refitted — which is why + /// the rendered meme kept the first and last items and dropped the middle two. + /// + /// That is the bug v7 and v8 both hunted in the caption code and could not find by + /// reading it: the caption code was right the whole time, and was being handed + /// `slots: 2` by a cache file older than the feature. A version bump discards those + /// entries and refetches, which is the only honest fix — the slot counts are simply + /// not in that file, so there is nothing to migrate them from. + /// + /// **The rule this encodes:** adding a field to `MemeTemplate` that any DECISION + /// reads is a format change, and it must bump this number. The tolerant decoder + /// makes a stale cache load; it cannot make it correct. + public static let currentVersion = 2 public static let fileName = "catalog-cache.json" @@ -55,12 +80,24 @@ public enum MemeCatalogCache { /// Decide how to open the catalog. /// - /// A cache from a FUTURE version is treated as absent rather than trusted: a - /// downgrade must not read a format it doesn't understand. An empty cache is - /// likewise treated as absent — persisting a zero-template catalog and then - /// honouring it would present the offline state as a legitimately empty corpus. + /// Any cache whose version is not EXACTLY `currentVersion` is treated as absent. + /// + /// A future version must not be trusted (a downgrade would read a format it doesn't + /// understand), and — the v9 fix — neither must an older one. The previous + /// `<= currentVersion` test is what let a v1 cache survive the arrival of + /// `captionSlots`: it loaded, every template silently defaulted to 2 slots, and the + /// owner's 4-panel meme was refit down to 2 captions. See `currentVersion`. + /// + /// Refetching is cheap (one key-less GET, already backed by the offline fallback) + /// and correctness here is not optional, so equality is the right test even though + /// it discards a cache that a migration could in principle have salvaged. There is + /// nothing to salvage: the missing field was never written to that file. + /// + /// An empty cache is likewise treated as absent — persisting a zero-template + /// catalog and then honouring it would present the offline state as a legitimately + /// empty corpus. public static func decide(cached: Cached?, now: Date) -> Decision { - guard let cached, cached.version <= currentVersion, !cached.templates.isEmpty else { + guard let cached, cached.version == currentVersion, !cached.templates.isEmpty else { return .fetchNow } // A timestamp in the future (clock skew, a restored backup) is treated as diff --git a/OpenWhisp/Services/MemeTrace.swift b/OpenWhisp/Services/MemeTrace.swift new file mode 100644 index 0000000..9362561 --- /dev/null +++ b/OpenWhisp/Services/MemeTrace.swift @@ -0,0 +1,78 @@ +import Foundation + +/// Runtime breadcrumbs for the meme generate path (spike v9). +/// +/// ## Why this exists — reading the wiring failed twice +/// +/// v7 and v8 both diagnosed the "four items render as two captions" report by TRACING +/// THE WIRING BY EYE, concluded it was correct, and shipped. The owner then ran a +/// hash-verified v8 binary with the exact repro prompt and still got two boxes. A +/// static read of the same code will keep saying the same thing; what was missing was +/// evidence from the RUNNING APP about which branch actually executed. +/// +/// So the decision points now emit a line each. The rule this encodes: when a +/// user-visible outcome contradicts what the code appears to say, the next move is a +/// breadcrumb, not another read. +/// +/// Lines go to `NSLog`, so they land in unified logging AND on stderr — the app is +/// launched by `open`, whose stderr is redirected to a file by the runtime harness in +/// `scripts/meme-runtime-proof.sh`. That belt-and-braces matters because this app's +/// NSLog output has historically not been reliably visible to `log stream`. +public enum MemeTrace { + + /// Whether breadcrumbs are emitted. Off by default so a normal run is quiet; + /// the harness (and a curious owner) turns it on with `OPENWHISP_MEME_TRACE=1`. + /// + /// Read once — an env var cannot change under a running process, and re-reading it + /// per line would put a `getenv` in the middle of the render loop. + public static let isEnabled: Bool = { + let raw = ProcessInfo.processInfo.environment["OPENWHISP_MEME_TRACE"] + return raw == "1" || raw == "true" || raw == "YES" + }() + + /// Emit one breadcrumb, prefixed so `grep '\[MemeGen\]'` finds the whole trace. + public static func log(_ message: @autoclosure () -> String) { + guard isEnabled else { return } + NSLog("[MemeGen] %@", message()) + } + + /// The breadcrumb for a resolved seeding decision. + /// + /// Built as a pure function so `swift test` can assert the LINE ITSELF, not just + /// the decision behind it. A trace that silently stopped describing the code would + /// be worse than no trace: the next debugging round would trust it. + public static func seedLine( + description: String, specCaptions: [String], slots: Int?, + seed: MemeCaptionSeeding.Seed + ) -> String { + "resolve(prompt: \(quoted(description)), specCaptions: \(specCaptions.count), " + + "slots: \(slots.map(String.init) ?? "nil")) -> \(seed.boxes.count) boxes, " + + "captions: \(seed.captions.count), fromUser: \(seed.captionsCameFromUser), " + + "refit: \(seed.refit.map { "\($0.from.count)->\($0.slots)" } ?? "none")" + } + + /// The breadcrumb for the extraction step, before any LLM involvement. + public static func extractionLine(_ extraction: MemeCaptionExtraction.Extraction?) -> String { + guard let extraction else { return "extraction fired: 0 items (not list-shaped)" } + return "extraction fired: \(extraction.captions.count) items, " + + "theme: \(quoted(extraction.theme))" + } + + /// The breadcrumb for the parsed LLM answer. + public static func llmLine(captions: [String], wasLegacyShape: Bool, schema: Bool) -> String { + "LLM path, schema=\(schema), captions=\(captions.count), legacyShape=\(wasLegacyShape)" + } + + /// The breadcrumb for what actually reached the canvas. + public static func seedingLine(boxes: Int, merged: Int) -> String { + "seeding \(boxes) boxes (canvas now \(merged))" + } + + /// Truncated + quoted, so a long dictation can't flood the log and an empty string + /// is visibly empty rather than invisible. + private static func quoted(_ text: String) -> String { + let limit = 120 + let clipped = text.count > limit ? String(text.prefix(limit)) + "…" : text + return "\"\(clipped)\"" + } +} diff --git a/OpenWhisp/Views/PluginHost.swift b/OpenWhisp/Views/PluginHost.swift index a0c832e..4340d0c 100644 --- a/OpenWhisp/Views/PluginHost.swift +++ b/OpenWhisp/Views/PluginHost.swift @@ -129,6 +129,14 @@ final class PluginHost: ObservableObject { NSApp.activate(ignoringOtherApps: true) } + /// The live window controller for a plugin, if one has been opened. + /// + /// Exists for the v9 runtime probe, which must reach the SAME cached controller + /// `open(pluginID:)` created rather than build a parallel one. + func windowController(for pluginID: String) -> NSWindowController? { + windows[pluginID] + } + private func closeWindow(for id: String) { windows[id]?.window?.close() windows[id] = nil diff --git a/Package.swift b/Package.swift index b59cb21..e388da0 100644 --- a/Package.swift +++ b/Package.swift @@ -176,6 +176,7 @@ let package = Package( "MemeAI.swift", "MemeCaptionExtraction.swift", "MemeCaptionSeeding.swift", + "MemeTrace.swift", "MemeTemplateMatcher.swift", "MemeCaptionLayout.swift", "MemeTemplateProvider.swift", diff --git a/Tests/OpenWhispCoreTests/MemeProviderTests.swift b/Tests/OpenWhispCoreTests/MemeProviderTests.swift index 5ee6e4a..b8c80e8 100644 --- a/Tests/OpenWhispCoreTests/MemeProviderTests.swift +++ b/Tests/OpenWhispCoreTests/MemeProviderTests.swift @@ -322,8 +322,15 @@ final class MemeProviderTests: XCTestCase { // MARK: - Catalog cache policy - private func cached(ageSeconds: TimeInterval, count: Int = 3, version: Int = 1) - -> MemeCatalogCache.Cached { + /// Defaults to the CURRENT version rather than a literal `1`. + /// + /// The literal is what let the v9 bug through: these tests all passed a v1 cache and + /// asserted it was honoured, so when `captionSlots` arrived and the format really did + /// change, the suite was actively asserting the stale-cache behaviour was correct. + private func cached( + ageSeconds: TimeInterval, count: Int = 3, + version: Int = MemeCatalogCache.currentVersion + ) -> MemeCatalogCache.Cached { MemeCatalogCache.Cached( version: version, fetchedAt: Date(timeIntervalSince1970: 10_000 - ageSeconds), @@ -359,6 +366,98 @@ final class MemeProviderTests: XCTestCase { .fetchNow) } + /// **The v9 regression.** A cache written before `captionSlots` existed must be + /// refetched, not honoured. + /// + /// This is the test that would have caught the owner's bug three rounds earlier. + /// The old policy was `cached.version <= currentVersion`, so a v1 file — written by + /// a v5-era build, with no `captionSlots` key on any template — was accepted as + /// current. `MemeTemplate.init(from:)` is deliberately tolerant of the missing + /// field and defaulted it to 2, so EVERY template in the corpus reported two + /// caption slots, including Expanding Brain. The caption code then did exactly what + /// it is supposed to do with a 4-caption answer for a 2-slot template: refit it + /// down to 2. Hence "typing" + "dictating memes by voice", the first and last of + /// four items, from a pipeline every unit test agreed was correct. + func testCacheFromAnOlderVersionIsRefetchedRatherThanTrusted() { + XCTAssertEqual( + MemeCatalogCache.decide(cached: cached(ageSeconds: 10, version: 1), now: now), + .fetchNow, + "A pre-captionSlots cache must be refetched — honouring it silently " + + "reports every template as a 2-slot meme.") + } + + /// The mechanism behind that regression, pinned directly: a template JSON with no + /// `captionSlots` decodes to the 2-slot default. + /// + /// The tolerance itself is correct and stays — the point is that it is a LOADING + /// convenience and never a source of truth, which is why the version gate above has + /// to keep such a file from reaching the decision in the first place. + func testTemplateWithoutCaptionSlotsDecodesToTheDefaultAndIsThereforeNotTrustworthy() throws { + let json = Data(""" + {"id":"imgflip:1","name":"Expanding Brain","url":"https://e.example/x.jpg", + "width":857,"height":1202,"source":"imgflip","keywords":[]} + """.utf8) + let decoded = try JSONDecoder().decode(MemeTemplate.self, from: json) + XCTAssertEqual( + decoded.captionSlots, MemeCaptionSlots.default, + "A missing captionSlots defaults to 2 — which is WHY a stale cache must " + + "never be honoured: the real Expanding Brain has four.") + } + + /// A cache written by THIS build round-trips its slot counts. + /// + /// The other half of the story: nothing was wrong with encoding, so the fix is a + /// version bump rather than a serialization change. Pinned so a future refactor + /// that drops the field from the wire format fails here. + func testCurrentCacheRoundTripsCaptionSlots() throws { + let four = MemeTemplate( + id: "imgflip:1", name: "Expanding Brain", url: "https://e.example/x.jpg", + width: 857, height: 1202, source: .imgflip, keywords: [], captionSlots: 4) + let payload = MemeCatalogCache.Cached(fetchedAt: now, templates: [four]) + + let data = try JSONEncoder().encode(payload) + let back = try JSONDecoder().decode(MemeCatalogCache.Cached.self, from: data) + + XCTAssertEqual(back.version, MemeCatalogCache.currentVersion) + XCTAssertEqual(back.templates.first?.captionSlots, 4) + XCTAssertEqual(MemeCatalogCache.decide(cached: back, now: now), .useCache) + } + + /// End to end over the SAME core call the app makes: the owner's exact prompt, on a + /// template whose slot count survived the cache, yields four boxes. + /// + /// Paired deliberately with the failing case below, because the difference between + /// them is the entire bug: identical prompt, identical model answer, and the only + /// variable is what the catalog said about the template. + func testOwnerPromptSeedsFourBoxesWhenTheTemplateKeepsItsSlots() { + let seed = MemeCaptionSeeding.resolve( + description: "expanding brain: typing, dictating, dictating memes, dictating memes by voice", + specCaptions: ["typing", "dictating", "dictating memes", "dictating memes by voice"], + templateSlots: 4) + + XCTAssertEqual(seed.boxes.count, 4) + XCTAssertNil(seed.refit, "Four captions on a four-slot template need no refit.") + XCTAssertEqual( + seed.captions, + ["typing", "dictating", "dictating memes", "dictating memes by voice"]) + } + + /// The bug, reproduced as a unit test: the same prompt with a 2-slot template + /// refits down to two — which is CORRECT behaviour for the input it was given, and + /// why no amount of reading the caption code could find the fault. + func testOwnerPromptRefitsToTwoWhenAStaleCacheReportsTwoSlots() { + let seed = MemeCaptionSeeding.resolve( + description: "expanding brain: typing, dictating, dictating memes, dictating memes by voice", + specCaptions: ["typing", "dictating", "dictating memes", "dictating memes by voice"], + templateSlots: 2) + + XCTAssertEqual(seed.boxes.count, 2) + XCTAssertEqual(seed.refit?.slots, 2) + XCTAssertEqual(seed.refit?.from.count, 4, + "Four captions are sent back to be squeezed into two — the " + + "observed 'first and last item' meme.") + } + /// A restored backup or a skewed clock must not pin the catalog forever. func testFutureTimestampIsTreatedAsStale() { XCTAssertEqual(MemeCatalogCache.decide(cached: cached(ageSeconds: -5_000), now: now), diff --git a/Tests/OpenWhispCoreTests/MemeStructureTests.swift b/Tests/OpenWhispCoreTests/MemeStructureTests.swift index 0d23547..c4ed085 100644 --- a/Tests/OpenWhispCoreTests/MemeStructureTests.swift +++ b/Tests/OpenWhispCoreTests/MemeStructureTests.swift @@ -788,4 +788,71 @@ final class MemeStructureTests: XCTestCase { XCTAssertEqual(MemeTemplateCatalog.promptSlots(catalog, limit: 2).count, 2) XCTAssertEqual(MemeTemplateCatalog.promptSlots(catalog, limit: 0).count, 0) } + + // MARK: - Live drag (v9) + + private func box(_ text: String) -> MemeCaptionLayout.CaptionBox { + MemeCaptionLayout.CaptionBox(text: text, centerX: 0.5, centerY: 0.5) + } + + /// The dragged box renders EMPTY so its caption isn't painted twice — once burned + /// in at the old spot and once travelling under the cursor. + func testDraggedBoxHasItsTextHiddenFromTheRender() { + let boxes = [box("TOP"), box("BOTTOM")] + let hidden = MemeCaptionLayout.hidingText(of: boxes[0].id, in: boxes) + + XCTAssertEqual(hidden.map(\.text), ["", "BOTTOM"]) + } + + /// Hiding is a RENDERING concern and must not edit the document: the box survives, + /// with its id and geometry, so the editor's selection stays valid mid-drag. + func testHidingTextKeepsTheBoxItsIDAndItsGeometry() { + let boxes = [box("TOP"), box("BOTTOM")] + let hidden = MemeCaptionLayout.hidingText(of: boxes[0].id, in: boxes) + + XCTAssertEqual(hidden.count, 2) + XCTAssertEqual(hidden.map(\.id), boxes.map(\.id)) + XCTAssertEqual(hidden[0].centerX, boxes[0].centerX) + XCTAssertEqual(hidden[0].centerY, boxes[0].centerY) + } + + /// At rest — nothing being dragged — the render is untouched. + func testHidingNothingIsIdentity() { + let boxes = [box("TOP"), box("BOTTOM")] + XCTAssertEqual(MemeCaptionLayout.hidingText(of: nil, in: boxes), boxes) + } + + /// An id that no longer exists (the box was deleted mid-drag) leaves every caption + /// visible rather than blanking an arbitrary one. + func testHidingAnUnknownIDLeavesEveryCaptionVisible() { + let boxes = [box("TOP"), box("BOTTOM")] + XCTAssertEqual(MemeCaptionLayout.hidingText(of: UUID(), in: boxes), boxes) + } + + // MARK: - Runtime trace (v9) + + /// The breadcrumb must report the SLOT COUNT, because that is the field that was + /// wrong and the trace is what finally showed it. + /// + /// Asserting the line's content keeps the trace honest: a trace that quietly + /// stopped describing the decision would send the next debugging round the same + /// way the last three went. + func testSeedTraceReportsSlotsAndBoxCount() { + let seed = MemeCaptionSeeding.resolve( + description: "expanding brain: a, b, c, d", + specCaptions: ["a", "b", "c", "d"], templateSlots: 2) + let line = MemeTrace.seedLine( + description: "expanding brain: a, b, c, d", + specCaptions: ["a", "b", "c", "d"], slots: 2, seed: seed) + + XCTAssertTrue(line.contains("slots: 2"), line) + XCTAssertTrue(line.contains("-> 2 boxes"), line) + XCTAssertTrue(line.contains("refit: 4->2"), line) + } + + func testExtractionTraceDistinguishesAListFromProse() { + let list = MemeCaptionExtraction.extract(from: "brain: a, b, c, d") + XCTAssertTrue(MemeTrace.extractionLine(list).contains("4 items")) + XCTAssertTrue(MemeTrace.extractionLine(nil).contains("not list-shaped")) + } } diff --git a/plugins/MemeGenerator/MemeGeneratorModel.swift b/plugins/MemeGenerator/MemeGeneratorModel.swift index b9c84f7..b45571d 100644 --- a/plugins/MemeGenerator/MemeGeneratorModel.swift +++ b/plugins/MemeGenerator/MemeGeneratorModel.swift @@ -79,6 +79,10 @@ final class MemeGeneratorModel: ObservableObject { /// the user can read it instead of into tokens the parser dropped. @Published private(set) var candidateReason: String = "" + /// The box currently being dragged, whose burned-in caption is suppressed while the + /// drag handle carries the text (v9). Nil at rest. + private var draggingBoxID: UUID? + /// The ids of the boxes the last AI seed minted (v6). /// /// This is the whole mechanism behind "Generate replaces AI boxes and preserves @@ -472,6 +476,9 @@ final class MemeGeneratorModel: ObservableObject { case .failure(let rejection): self.finish(ticket, status: "Couldn't build the meme — \(rejection.reason).") case .success(let spec): + MemeTrace.log(MemeTrace.llmLine( + captions: spec.captions, wasLegacyShape: spec.wasLegacyShape, + schema: true)) // v8: the "user's own list beats the model's captions" rule moved // into `MemeCaptionSeeding.resolve`, which `applyRanked` calls — // one tested place instead of a step here and a step there. @@ -723,6 +730,7 @@ final class MemeGeneratorModel: ObservableObject { seed: seed.boxes, into: boxes, seededIDs: seededBoxIDs) seededBoxIDs = Set(seed.boxes.map(\.id)) selectedBoxID = boxes.first?.id + MemeTrace.log(MemeTrace.seedingLine(boxes: seed.boxes.count, merged: boxes.count)) } /// Seed boxes for a known caption list and slot count, with no description to read. @@ -1014,7 +1022,34 @@ final class MemeGeneratorModel: ObservableObject { /// literally the export. func redraw() { guard let baseImage else { return } - meme = MemeRenderer.render(template: baseImage, boxes: boxes) + // v9: the box being dragged is rendered EMPTY, because the drag handle is + // drawing that caption itself and travelling with the cursor. Without this the + // caption would appear twice mid-drag — burned in at the old position and live + // under the cursor — which reads as a duplicate rather than a move. + meme = MemeRenderer.render( + template: baseImage, boxes: MemeCaptionLayout.hidingText(of: draggingBoxID, in: boxes)) + } + + // MARK: - Drag (v9) + + /// Begin dragging a caption box: hide its burned-in copy so the handle's own copy + /// is the only one on screen. + /// + /// Re-rendering ONCE at the start (and once at the drop) is what makes the live + /// drag cheap. The alternative — re-rendering the meme on every gesture frame — + /// would tie the drag's frame rate to a full-resolution CoreGraphics pass. + func beginDragging(id: UUID) { + guard draggingBoxID != id else { return } + draggingBoxID = id + redraw() + } + + /// End the drag. The caller commits the new position immediately after, which + /// re-renders with the caption at its new home. + func endDragging() { + guard draggingBoxID != nil else { return } + draggingBoxID = nil + redraw() } /// Mutate one box and re-render. The single funnel for every editor edit. diff --git a/plugins/MemeGenerator/MemeGeneratorView.swift b/plugins/MemeGenerator/MemeGeneratorView.swift index 5cf13dc..a0c7cd4 100644 --- a/plugins/MemeGenerator/MemeGeneratorView.swift +++ b/plugins/MemeGenerator/MemeGeneratorView.swift @@ -515,12 +515,28 @@ struct MemeGeneratorView: View { // MARK: - Drag handle -/// The invisible, draggable hit area sitting on top of one rendered caption. +/// The draggable hit area sitting on top of one rendered caption. /// -/// Deliberately NOT a second copy of the text: drawing the caption again in SwiftUI -/// would mean two renderers to keep in agreement, and they would drift (different -/// wrapping, different outline). Instead the burned-in render IS the visual, and this -/// is a positioned outline the user grabs. +/// ## v9: the text travels with the box +/// +/// Until v9 this was an outline and nothing else, on the reasoning that drawing the +/// caption a second time in SwiftUI would mean two renderers to keep in agreement. The +/// reasoning was sound and the result was still wrong: the caption is BURNED INTO the +/// preview image, so dragging moved an empty dashed rectangle while the text stayed +/// behind, and it only jumped to the new position when the drag ended and the renderer +/// ran. The user is aiming text at a spot; a handle that doesn't carry the text gives +/// them nothing to aim. +/// +/// The fix keeps a single source of truth by making the duplication EXPLICIT and +/// strictly temporary: while (and only while) this box is being dragged, the burned-in +/// copy is masked out and a SwiftUI approximation rides along inside the handle. On +/// drop the mask lifts and the real renderer's output is what remains, so the +/// approximation is never what the user keeps — it cannot drift into the export, +/// because it never reaches it. +/// +/// Re-rendering the whole meme per frame was the other option and is the worse one: +/// `MemeRenderer` redraws a full-resolution image, and driving that from a gesture +/// would make the drag's smoothness depend on the template's pixel count. private struct DragHandle: View { @ObservedObject var model: MemeGeneratorModel @@ -533,6 +549,11 @@ private struct DragHandle: View { /// the handle tracks the cursor without a re-render per frame fighting it. @State private var dragOffset: CGSize = .zero + /// True from the first `onChanged` until the drop. Drives BOTH the travelling text + /// and the mask over the burned-in copy, so the two can never disagree about + /// whether a drag is in progress. + @State private var isDragging = false + var body: some View { let width = canvas.width * box.widthShare let height = max(24, canvas.height * box.fontSizeShare * MemeCaptionLayout.lineHeightRatio) @@ -546,20 +567,54 @@ private struct DragHandle: View { .background( RoundedRectangle(cornerRadius: 4) .fill(Color.accentColor.opacity(isSelected ? 0.10 : 0.001))) + .overlay { + // The travelling caption. Only while dragging — at rest the burned-in + // render is the one true visual, exactly as before. + if isDragging { + Text(MemeCaptionLayout.displayText(box.text)) + .font(.system( + size: canvas.height * box.fontSizeShare, + weight: .heavy)) + .foregroundStyle(.white) + .shadow(color: .black, radius: 1, x: 1, y: 1) + .shadow(color: .black, radius: 1, x: -1, y: -1) + .minimumScaleFactor(0.4) + .lineLimit(3) + .multilineTextAlignment(.center) + .allowsHitTesting(false) + } + } .frame(width: width, height: height) .position(x: x, y: y) .gesture( DragGesture() .onChanged { value in model.selectedBoxID = box.id + // Announce the drag BEFORE the offset so the burned-in copy is + // masked on the same frame the text starts moving — setting it + // after would flash both copies for one frame. + if !isDragging { + isDragging = true + model.beginDragging(id: box.id) + } dragOffset = value.translation } .onEnded { value in // Commit in NORMALIZED units so the move survives the export's // full-resolution render and a switch to another template. - guard canvas.width > 0, canvas.height > 0 else { return } + guard canvas.width > 0, canvas.height > 0 else { + isDragging = false + model.endDragging() + dragOffset = .zero + return + } let dx = value.translation.width / canvas.width let dy = value.translation.height / canvas.height + isDragging = false + // Clear the mask and commit in one step: `updateBox` re-renders + // with the caption at its NEW home, so there is no frame in + // which the burned-in copy is visible at the old position. + model.endDragging() model.updateBox(id: box.id) { b in b.centerX += dx b.centerY += dy diff --git a/plugins/MemeGenerator/MemeGeneratorWindowController.swift b/plugins/MemeGenerator/MemeGeneratorWindowController.swift index 03d2c9c..23d5730 100644 --- a/plugins/MemeGenerator/MemeGeneratorWindowController.swift +++ b/plugins/MemeGenerator/MemeGeneratorWindowController.swift @@ -99,6 +99,45 @@ final class MemeGeneratorWindowController: NSWindowController, NSWindowDelegate, }) } + // MARK: - Runtime proof hook (v9) + + /// Drive the REAL generate path from a launch argument, and log what it produced. + /// + /// ## Why a hook rather than another read of the code + /// + /// v7 and v8 each fixed the "four items, two captions" report by tracing the wiring + /// and declaring it correct. The owner then ran a hash-verified v8 build and got + /// two boxes anyway. At that point the code's appearance had been wrong twice, so + /// the only evidence worth anything is what the running binary does — and driving + /// Generate by hand through the UI is not something a build script can do. + /// + /// This runs `model.generate()` — the same method the Generate button calls, with + /// no branch of its own — after seeding `description` exactly as the user's typing + /// would. Everything downstream (extraction, shortlist, the LLM round-trip, the + /// seeding) is the production path, and `MemeTrace` reports what each step decided. + /// + /// It is gated on an env var, so a normal launch never touches it. + func runTraceProbeIfRequested() { + let env = ProcessInfo.processInfo.environment + guard let prompt = env["OPENWHISP_MEME_PROBE_PROMPT"], !prompt.isEmpty else { return } + + MemeTrace.log("probe start, prompt=\"\(prompt)\"") + model.description = prompt + model.generate() + + // Report the canvas after the generate settles. The probe cannot know when the + // LLM answers, so it samples on a deadline and states what it found — a probe + // that reported nothing would look like a crash. + let deadline = Double(env["OPENWHISP_MEME_PROBE_SECONDS"] ?? "") ?? 90 + Task { @MainActor [model] in + try? await Task.sleep(nanoseconds: UInt64(deadline * 1_000_000_000)) + MemeTrace.log( + "probe result: \(model.boxes.count) boxes on canvas, " + + "texts=\(model.boxes.map(\.text))") + MemeTrace.log("probe done") + } + } + // MARK: - PluginWindowLifecycle /// The window is being shown again after a close (v5). From f26912fa752c337b18f2bcb00432aef5f637d1da Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Mon, 3 Aug 2026 17:39:17 -0700 Subject: [PATCH 13/15] spike(meme-plugin): manifest-declared voice triggers + a pure command router MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trigger layer for MAK-100: a plugin declares the spoken phrases that should route a refine instruction to it, and a pure core router decides which (if any) plugin claims an instruction. Nothing is hardcoded next to the meme plugin — a second plugin gains voice commands by shipping a manifest. `PluginManifest.voiceTriggers` decodes forward-compatibly (a manifest written before this field still decodes) and normalizes to trimmed, lowercased, de-duped phrases. An all-empty list is REPORTED (`.emptyVoiceTriggers`) but never fatal — same trade `keyEquivalent` already makes, since losing a working plugin over a stray `""` in a JSON file would be a bad deal. Crucially, normalization is what stops an empty prefix from matching EVERY instruction the user ever speaks. `PluginVoiceCommandRouter.match` is deliberately strict, because a match REDIRECTS a dictation away from the user's editor and a false positive costs them text: - PREFIX only, so "summarize this, then create a meme" stays a normal refine - WORD BOUNDARY, so "create a memo about Q3" does not match "create a mem"+"o" - exact phrases, case/whitespace-insensitive; no fuzzy or edit-distance matching - longest trigger wins, so a specific phrase can't be shadowed by a general one The remainder is sliced from the ORIGINAL text (not the lowercased form) so the user's capitalization reaches the rendered captions, and only leading separator punctuation is trimmed — the interior commas in "typing, dictating, …" ARE the list the extraction step later reads. The meme manifest declares EN + RU ("сделай мем", "создай мем") because the owner dictates in both; the checked-in manifest.json matches the registry literal, which the existing parity test pins. Tests: 22 router cases, negatives carrying most of the weight (memo, mid-sentence mention, "create a memes", empty instruction), plus enablement gating, precedence, normalization, and forward-compat decode. Co-Authored-By: Claude --- OpenWhisp/Services/PluginManifest.swift | 52 +++- OpenWhisp/Services/PluginRegistry.swift | 15 +- .../Services/PluginVoiceCommandRouter.swift | 212 +++++++++++++++ Package.swift | 1 + .../PluginVoiceCommandRouterTests.swift | 255 ++++++++++++++++++ plugins/MemeGenerator/manifest.json | 9 +- 6 files changed, 540 insertions(+), 4 deletions(-) create mode 100644 OpenWhisp/Services/PluginVoiceCommandRouter.swift create mode 100644 Tests/OpenWhispCoreTests/PluginVoiceCommandRouterTests.swift diff --git a/OpenWhisp/Services/PluginManifest.swift b/OpenWhisp/Services/PluginManifest.swift index 4c70da9..9e38d72 100644 --- a/OpenWhisp/Services/PluginManifest.swift +++ b/OpenWhisp/Services/PluginManifest.swift @@ -73,6 +73,24 @@ public struct PluginManifest: Codable, Equatable, Sendable, Identifiable { /// including any already sitting in the user's plugins folder — still decodes. public let keyEquivalent: String? + /// Spoken PREFIX phrases that route a refine instruction to this plugin (v10), + /// e.g. `["create a meme", "make a meme", "сделай мем"]`. + /// + /// This is the MAK-100 trigger layer: the refine pipeline asks + /// `PluginVoiceCommandRouter` which plugin (if any) claims an instruction, and the + /// answer comes from THESE strings rather than from a hardcoded list next to the + /// one plugin that wants them. A second plugin gains voice commands by shipping a + /// manifest — no host change. + /// + /// Deliberately PREFIX-only and exact-phrase: an instruction is routed away from + /// the user's normal refine, so a loose match (substring/fuzzy) would hijack + /// dictations the user meant to keep. See `PluginVoiceCommandRouter` for the + /// matching rules and `normalizedVoiceTriggers` for what survives validation. + /// + /// Decoded with a default, like every field added after v1 — a manifest written + /// before this existed still decodes rather than dropping the plugin from the list. + public let voiceTriggers: [String] + public init( id: String, name: String, @@ -81,7 +99,8 @@ public struct PluginManifest: Codable, Equatable, Sendable, Identifiable { symbol: String, entry: PluginEntryKind, networkHosts: [String] = [], - keyEquivalent: String? = nil + keyEquivalent: String? = nil, + voiceTriggers: [String] = [] ) { self.id = id self.name = name @@ -91,6 +110,7 @@ public struct PluginManifest: Codable, Equatable, Sendable, Identifiable { self.entry = entry self.networkHosts = networkHosts self.keyEquivalent = keyEquivalent + self.voiceTriggers = voiceTriggers } /// Forward-compatible decode: `networkHosts` and `keyEquivalent` are optional in @@ -106,6 +126,23 @@ public struct PluginManifest: Codable, Equatable, Sendable, Identifiable { entry = try container.decodeIfPresent(PluginEntryKind.self, forKey: .entry) ?? .builtIn networkHosts = try container.decodeIfPresent([String].self, forKey: .networkHosts) ?? [] keyEquivalent = try container.decodeIfPresent(String.self, forKey: .keyEquivalent) + voiceTriggers = try container.decodeIfPresent([String].self, forKey: .voiceTriggers) ?? [] + } + + /// The voice triggers this manifest may actually be routed on: trimmed, + /// lowercased, de-duplicated, and with anything empty dropped. + /// + /// The router consumes THIS rather than the raw array, so a manifest carrying + /// `["", " ", "Create A Meme"]` contributes exactly one usable phrase instead of + /// matching every instruction on the empty string — an empty prefix matches + /// EVERYTHING, which would silently swallow every refine the user ever spoke. + public var normalizedVoiceTriggers: [String] { + var seen = Set() + return voiceTriggers.compactMap { raw in + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + guard !trimmed.isEmpty, seen.insert(trimmed).inserted else { return nil } + return trimmed + } } /// The shortcut as it should be DISPLAYED, e.g. `"⌘M"`, or nil when this manifest @@ -136,6 +173,10 @@ public struct PluginManifest: Codable, Equatable, Sendable, Identifiable { case emptySymbol /// The manifest asked for a shortcut that isn't a single character (v5). case invalidKeyEquivalent(String) + /// The manifest declared `voiceTriggers` but not one of them survived + /// normalization — e.g. `[""]` or `[" "]` (v10). Reported so a plugin + /// author sees it, but never fatal: see `isValid`. + case emptyVoiceTriggers } /// Characters allowed in an id: lowercase alphanumerics plus `-` and `.`. @@ -163,6 +204,13 @@ public struct PluginManifest: Codable, Equatable, Sendable, Identifiable { PluginKeyEquivalent.normalized(requested) == nil { return .invalidKeyEquivalent(requested) } + // Same trade as the shortcut: a manifest whose declared triggers all normalize + // away loses its VOICE ROUTE (the router simply never matches it) and keeps + // everything else. Fatal here would mean a stray `""` in a JSON file costs the + // user a whole working plugin. + if !voiceTriggers.isEmpty, normalizedVoiceTriggers.isEmpty { + return .emptyVoiceTriggers + } return nil } @@ -174,7 +222,7 @@ public struct PluginManifest: Codable, Equatable, Sendable, Identifiable { /// nothing else. public var isValid: Bool { switch validate() { - case nil, .invalidKeyEquivalent: return true + case nil, .invalidKeyEquivalent, .emptyVoiceTriggers: return true default: return false } } diff --git a/OpenWhisp/Services/PluginRegistry.swift b/OpenWhisp/Services/PluginRegistry.swift index d9bd7fd..13907d4 100644 --- a/OpenWhisp/Services/PluginRegistry.swift +++ b/OpenWhisp/Services/PluginRegistry.swift @@ -53,6 +53,19 @@ public enum PluginRegistry { // ⌘M opens the window from the menu bar, the way ⌘S opens the Scratchpad. // Declared here rather than hardcoded in `AppMain` so a second plugin needs no // host change — the host resolves collisions (`PluginKeyEquivalent`). - keyEquivalent: "m" + keyEquivalent: "m", + // v10: the spoken phrases that route a REFINE instruction here instead of to + // the refine LLM (MAK-100 trigger layer). Declared on the manifest — the + // refine pipeline asks `PluginVoiceCommandRouter`, which knows only about + // manifests, so a second plugin gains voice commands without a host change. + // + // English + Russian because the owner dictates in both. Kept to the natural + // imperative openings for "make me a meme" and nothing looser: each phrase + // here REDIRECTS a dictation away from the user's editor, so a phrase that + // could plausibly begin an ordinary sentence would cost them text. + voiceTriggers: [ + "create a meme", "make a meme", "generate a meme", + "сделай мем", "создай мем", + ] ) } diff --git a/OpenWhisp/Services/PluginVoiceCommandRouter.swift b/OpenWhisp/Services/PluginVoiceCommandRouter.swift new file mode 100644 index 0000000..e31a5e9 --- /dev/null +++ b/OpenWhisp/Services/PluginVoiceCommandRouter.swift @@ -0,0 +1,212 @@ +import Foundation + +/// Routes a spoken REFINE instruction to a plugin that claims it (spike v10). +/// +/// ## What this is for +/// +/// Refine is "apply this spoken instruction to that text". v10 adds a second +/// destination: an instruction that STARTS with a phrase a plugin declared in its +/// manifest (`PluginManifest.voiceTriggers`) is handed to that plugin instead of the +/// refine LLM. The owner's two flows: +/// +/// - Select text anywhere, dictate, tap Refine, say *"create a meme based on that"* — +/// the selection is the material and the meme window is the output. +/// - Refine with nothing selected, say *"create a meme expanding brain: typing, +/// dictating, …"* — the spoken remainder is the whole input. +/// +/// ## Why the matching is deliberately STRICT +/// +/// A match REDIRECTS the user's dictation: the refine LLM never runs and nothing is +/// inserted into the focused app. That makes a false positive expensive — it silently +/// swallows text the user meant to keep — so the rules are the narrowest ones that +/// still serve the two flows: +/// +/// - **Prefix only.** "create a meme …" routes; "…, then create a meme" does not. A +/// substring match would hijack any instruction that merely MENTIONS a meme +/// ("rewrite this so it doesn't sound like a meme"). +/// - **Whole-word boundary.** "create a memo about Q3" must NOT match "create a mem"- +/// anything; the character after the phrase has to be a separator, not a letter. +/// This is the near-miss the runtime probe pins. +/// - **Exact phrases**, case- and punctuation-insensitive, with runs of whitespace +/// collapsed — dictation output varies in capitalization and trailing commas, and +/// the user should not have to hit one byte-for-byte. +/// - **No fuzzy/edit-distance matching.** Cheap to add, impossible to reason about, +/// and every false positive costs a dictation. +/// +/// Non-match returns nil, and the caller runs the instruction as a NORMAL refine with +/// byte-identical behavior — the fallback is the default, not an error path. +/// +/// ## Languages +/// +/// Matching is language-agnostic (plain Unicode prefix comparison), so a manifest can +/// declare phrases in any language. The meme plugin ships EN + RU (`сделай мем`, +/// `создай мем`) because the owner dictates in both; adding a language is a manifest +/// edit, not a code change. +/// +/// Pure and Foundation-only so `swift test` pins every rule here — this is the gate +/// that decides whether a dictation reaches the user's editor or a plugin window. +public enum PluginVoiceCommandRouter { + + /// A matched voice command: which plugin claimed it, and what was left over. + public struct Match: Equatable, Sendable { + /// The `PluginManifest.id` that declared the matched phrase. + public let pluginID: String + /// The trigger phrase that matched, normalized (useful for logging/tests). + public let trigger: String + /// The instruction with the trigger phrase (and any leading separator + /// punctuation) removed — the material the plugin should act on. + /// + /// EMPTY is a legitimate outcome: "create a meme" on its own is a valid + /// command whose material comes from the refine CONTENT (the user's + /// selection) rather than from the spoken remainder. + public let remainder: String + + public init(pluginID: String, trigger: String, remainder: String) { + self.pluginID = pluginID + self.trigger = trigger + self.remainder = remainder + } + } + + /// The characters allowed to sit between the trigger phrase and the remainder — + /// i.e. what proves the phrase ended on a WORD BOUNDARY. + /// + /// Whitespace plus the punctuation dictation actually emits after a lead-in + /// clause. `:` matters most: the owner's own prompt is + /// "create a meme expanding brain: typing, …", and `,`/`.`/`—` cover the rest. + private static let boundaryCharacters = CharacterSet(charactersIn: " \t\n\r:,.;!?-—–") + + /// Ask which enabled plugin claims `instruction`, if any. + /// + /// - Parameters: + /// - instruction: the spoken refine instruction, raw from the pipeline. + /// - enabledPlugins: manifests of plugins that are ENABLED and runnable. The + /// caller passes only enabled plugins, so a disabled plugin cannot claim a + /// dictation — see `matchIgnoringEnablement` for the disabled-hint case. + /// - Returns: the match, or nil to proceed as a normal refine. + /// + /// Ties are resolved by LONGEST trigger first, then by the order + /// `enabledPlugins` arrives in (the same first-wins rule `PluginDiscovery` and + /// `PluginKeyEquivalent` already use). Longest-first matters when one plugin + /// declares "create a meme" and another "create a meme poster": the more specific + /// phrase must win regardless of list order, or the specific plugin is + /// unreachable. + public static func match( + instruction: String, enabledPlugins: [PluginManifest] + ) -> Match? { + let normalized = normalize(instruction) + guard !normalized.isEmpty else { return nil } + + // (plugin index, trigger) pairs, longest trigger first so a more specific + // phrase always beats a shorter one that prefixes it. + let candidates = enabledPlugins.enumerated() + .flatMap { index, manifest in + manifest.normalizedVoiceTriggers.map { (index: index, manifest: manifest, trigger: $0) } + } + .sorted { lhs, rhs in + lhs.trigger.count != rhs.trigger.count + ? lhs.trigger.count > rhs.trigger.count + : lhs.index < rhs.index + } + + for candidate in candidates { + guard let remainder = remainderAfterPrefix( + candidate.trigger, in: normalized, original: instruction) + else { continue } + return Match( + pluginID: candidate.manifest.id, + trigger: candidate.trigger, + remainder: remainder) + } + return nil + } + + /// Whether `instruction` would match `manifest` if it were enabled. + /// + /// Exists for exactly one thing: the caller shows "… is disabled" ONLY when a + /// disabled plugin would otherwise have claimed the instruction. Without this the + /// hint would either never appear or appear on every unrelated refine. + public static func matchIgnoringEnablement( + instruction: String, plugins: [PluginManifest] + ) -> Match? { + match(instruction: instruction, enabledPlugins: plugins) + } + + // MARK: - User-facing strings + + /// The overlay acknowledgment shown the moment a voice command is claimed, e.g. + /// `"Meme Generator — creating…"`. + /// + /// Named after the PLUGIN, because the whole point of the acknowledgment is to + /// tell the user WHERE their words just went: a routed dictation produces nothing + /// in the focused app, and an overlay that still said "Refining…" would look like + /// the refine silently ate it. + /// + /// It reaches the overlay through `AppState.statusMessage`, which + /// `FinalizingCaption.resolve` already surfaces verbatim as the finalize caption — + /// so this needs no new `OverlayPhase` case and no view change. + public static func acknowledgment(pluginName: String) -> String { + "\(pluginName) — creating…" + } + + /// The one-line hint shown when a voice command matched EXACTLY but the plugin is + /// switched off. The instruction still runs as a normal refine; this only explains + /// why the plugin window didn't appear. + public static func disabledHint(pluginName: String) -> String { + "\(pluginName) plugin is disabled" + } + + // MARK: - Rules + + /// Lowercase, collapse whitespace runs, and trim. Applied to BOTH sides so the + /// comparison is stable against dictation's capitalization and spacing. + /// + /// Note this does NOT strip punctuation from the middle of the string — the + /// boundary check handles the one place punctuation matters (right after the + /// trigger), and stripping it wholesale would corrupt the remainder the plugin + /// receives ("typing, dictating" is a LIST; losing the commas loses the items). + private static func normalize(_ text: String) -> String { + text.lowercased() + .components(separatedBy: .whitespacesAndNewlines) + .filter { !$0.isEmpty } + .joined(separator: " ") + } + + /// The remainder after `prefix`, or nil when `normalized` doesn't start with it on + /// a word boundary. + /// + /// The remainder is sliced from the ORIGINAL instruction (not the lowercased + /// form) so the plugin receives the user's real words — the meme captions are + /// rendered from this text, and lowercasing them would be visible in the output. + private static func remainderAfterPrefix( + _ prefix: String, in normalized: String, original: String + ) -> String? { + guard normalized.hasPrefix(prefix) else { return nil } + + let afterIndex = normalized.index(normalized.startIndex, offsetBy: prefix.count) + // Exact match ("create a meme") — a valid command with no remainder. + if afterIndex == normalized.endIndex { return "" } + // Word boundary: the phrase must END here. "create a memo" must not match + // "create a mem" + "o". + guard let next = normalized[afterIndex].unicodeScalars.first, + boundaryCharacters.contains(next) else { return nil } + + // Slice the ORIGINAL by counting the same number of significant words the + // trigger consumed, so casing/spacing in the user's remainder is preserved. + return originalRemainder(original: original, triggerWordCount: prefix.split(separator: " ").count) + } + + /// Drop the first `triggerWordCount` whitespace-separated words from `original` + /// and return what's left, minus any leading separator punctuation. + private static func originalRemainder(original: String, triggerWordCount: Int) -> String { + let words = original + .components(separatedBy: .whitespacesAndNewlines) + .filter { !$0.isEmpty } + guard words.count > triggerWordCount else { return "" } + let rest = words.dropFirst(triggerWordCount).joined(separator: " ") + // The trigger's own trailing punctuation belongs to the TRIGGER, not the + // material: "create a meme: typing, dictating" hands over "typing, dictating". + // Only leading separators are trimmed — interior commas are the list. + return rest.trimmingCharacters(in: boundaryCharacters) + } +} diff --git a/Package.swift b/Package.swift index e388da0..5671b01 100644 --- a/Package.swift +++ b/Package.swift @@ -170,6 +170,7 @@ let package = Package( // pure rules. Sources live under plugins/MemeGenerator/ for the app-layer // UI; the testable logic sits here so `swift test` covers it. "PluginManifest.swift", + "PluginVoiceCommandRouter.swift", "PluginDiscovery.swift", "PluginEnablement.swift", "PluginRegistry.swift", diff --git a/Tests/OpenWhispCoreTests/PluginVoiceCommandRouterTests.swift b/Tests/OpenWhispCoreTests/PluginVoiceCommandRouterTests.swift new file mode 100644 index 0000000..89bc6f7 --- /dev/null +++ b/Tests/OpenWhispCoreTests/PluginVoiceCommandRouterTests.swift @@ -0,0 +1,255 @@ +import XCTest +@testable import OpenWhispCore + +/// The v10 voice-command trigger layer: which spoken refine instructions get routed +/// to a plugin, and — just as important — which ones must NOT be. +/// +/// The negatives carry most of the weight here. A false positive silently redirects a +/// dictation away from the user's editor into a plugin window, so "create a memo +/// about Q3" staying a normal refine is the property that keeps this feature safe. +final class PluginVoiceCommandRouterTests: XCTestCase { + + private func manifest( + id: String, triggers: [String], name: String = "Test Plugin" + ) -> PluginManifest { + PluginManifest( + id: id, name: name, version: "1.0.0", summary: "", + symbol: "star", entry: .builtIn, voiceTriggers: triggers) + } + + private var meme: PluginManifest { + manifest(id: "meme-generator", + triggers: ["create a meme", "make a meme", "сделай мем"], + name: "Meme Generator") + } + + // MARK: - Positive matches + + func testMatchesBarePrefixWithEmptyRemainder() { + let match = PluginVoiceCommandRouter.match( + instruction: "create a meme", enabledPlugins: [meme]) + XCTAssertEqual(match?.pluginID, "meme-generator") + XCTAssertEqual(match?.trigger, "create a meme") + // Empty remainder is legitimate: the material comes from the refine CONTENT. + XCTAssertEqual(match?.remainder, "") + } + + /// CASE 1: selection is the material, the spoken words are just the trigger plus + /// a pointer back at the selection. + func testMatchesSelectionPhrasingAndKeepsRemainder() { + let match = PluginVoiceCommandRouter.match( + instruction: "Create a meme based on that", enabledPlugins: [meme]) + XCTAssertEqual(match?.pluginID, "meme-generator") + XCTAssertEqual(match?.remainder, "based on that") + } + + /// CASE 2: the owner's real prompt. The colon is a boundary, and the list after + /// it must survive intact — the commas ARE the items. + func testMatchesOwnersExpandingBrainPromptPreservingTheList() { + let match = PluginVoiceCommandRouter.match( + instruction: "create a meme expanding brain: typing, dictating, dictating memes, dictating memes by voice", + enabledPlugins: [meme]) + XCTAssertEqual(match?.pluginID, "meme-generator") + XCTAssertEqual( + match?.remainder, + "expanding brain: typing, dictating, dictating memes, dictating memes by voice") + } + + /// The trigger may be followed immediately by a colon — that punctuation belongs + /// to the trigger, not to the material. + func testTriggerTrailingPunctuationIsNotPartOfTheRemainder() { + let match = PluginVoiceCommandRouter.match( + instruction: "Create a meme: typing, dictating", enabledPlugins: [meme]) + XCTAssertEqual(match?.remainder, "typing, dictating") + } + + func testMatchIsCaseAndWhitespaceInsensitive() { + let match = PluginVoiceCommandRouter.match( + instruction: " MAKE A MEME about deadlines ", enabledPlugins: [meme]) + XCTAssertEqual(match?.trigger, "make a meme") + XCTAssertEqual(match?.remainder, "about deadlines") + } + + /// The remainder is sliced from the ORIGINAL text, so the user's capitalization + /// survives into the rendered captions. + func testRemainderPreservesOriginalCasing() { + let match = PluginVoiceCommandRouter.match( + instruction: "create a meme about Kubernetes and YAML", enabledPlugins: [meme]) + XCTAssertEqual(match?.remainder, "about Kubernetes and YAML") + } + + /// Russian, because the owner dictates in it. + func testMatchesRussianTrigger() { + let match = PluginVoiceCommandRouter.match( + instruction: "Сделай мем про дедлайны", enabledPlugins: [meme]) + XCTAssertEqual(match?.pluginID, "meme-generator") + XCTAssertEqual(match?.remainder, "про дедлайны") + } + + // MARK: - Negatives (a false positive costs the user their dictation) + + /// The near-miss the runtime probe also pins: "memo" is not "meme". + func testDoesNotMatchCreateAMemo() { + XCTAssertNil(PluginVoiceCommandRouter.match( + instruction: "create a memo about the Q3 numbers", enabledPlugins: [meme])) + } + + /// A word-boundary check, not a substring check. + func testDoesNotMatchWhenTriggerRunsIntoAnotherWord() { + XCTAssertNil(PluginVoiceCommandRouter.match( + instruction: "create a memes", enabledPlugins: [meme])) + } + + /// PREFIX only — a mention mid-sentence is an ordinary refine instruction. + func testDoesNotMatchTriggerInTheMiddle() { + XCTAssertNil(PluginVoiceCommandRouter.match( + instruction: "rewrite this so it doesn't sound like a meme", enabledPlugins: [meme])) + XCTAssertNil(PluginVoiceCommandRouter.match( + instruction: "summarize this, then create a meme", enabledPlugins: [meme])) + } + + func testDoesNotMatchUnrelatedInstruction() { + XCTAssertNil(PluginVoiceCommandRouter.match( + instruction: "make this more concise", enabledPlugins: [meme])) + } + + func testEmptyInstructionDoesNotMatch() { + XCTAssertNil(PluginVoiceCommandRouter.match(instruction: "", enabledPlugins: [meme])) + XCTAssertNil(PluginVoiceCommandRouter.match(instruction: " ", enabledPlugins: [meme])) + } + + // MARK: - Enablement gating + + /// The gate: the caller passes only ENABLED plugins, so a disabled plugin cannot + /// claim a dictation. + func testDisabledPluginDoesNotMatchWhenAbsentFromEnabledList() { + XCTAssertNil(PluginVoiceCommandRouter.match( + instruction: "create a meme about deadlines", enabledPlugins: [])) + } + + /// …but the caller can still ask whether it WOULD have matched, which is what + /// gates the "plugin is disabled" hint so it never fires on an unrelated refine. + func testMatchIgnoringEnablementDrivesTheDisabledHint() { + XCTAssertNotNil(PluginVoiceCommandRouter.matchIgnoringEnablement( + instruction: "create a meme about deadlines", plugins: [meme])) + XCTAssertNil(PluginVoiceCommandRouter.matchIgnoringEnablement( + instruction: "make this more concise", plugins: [meme])) + } + + /// A plugin with no declared triggers is simply not routable. + func testPluginWithoutTriggersNeverMatches() { + let silent = manifest(id: "quiet", triggers: []) + XCTAssertNil(PluginVoiceCommandRouter.match( + instruction: "create a meme", enabledPlugins: [silent])) + } + + // MARK: - Precedence + + /// The more SPECIFIC phrase wins regardless of list order, or the plugin that + /// declared it would be unreachable. + func testLongestTriggerWinsRegardlessOfOrder() { + let general = manifest(id: "general", triggers: ["create a meme"]) + let specific = manifest(id: "specific", triggers: ["create a meme poster"]) + let match = PluginVoiceCommandRouter.match( + instruction: "create a meme poster about deadlines", + enabledPlugins: [general, specific]) + XCTAssertEqual(match?.pluginID, "specific") + XCTAssertEqual(match?.remainder, "about deadlines") + } + + /// Equal-length triggers resolve by list order — the same first-wins rule + /// `PluginDiscovery` uses, so the host has one precedence story. + func testEqualLengthTriggersResolveByListOrder() { + let first = manifest(id: "first", triggers: ["create a meme"]) + let second = manifest(id: "second", triggers: ["create a meme"]) + XCTAssertEqual( + PluginVoiceCommandRouter.match( + instruction: "create a meme now", enabledPlugins: [first, second])?.pluginID, + "first") + } + + // MARK: - Manifest trigger normalization + + /// An empty prefix matches EVERYTHING — it must never survive into the router, or + /// a stray `""` in a JSON file would swallow every refine the user ever spoke. + func testEmptyTriggersAreNormalizedAwayAndNeverMatchEverything() { + let broken = manifest(id: "broken", triggers: ["", " "]) + XCTAssertEqual(broken.normalizedVoiceTriggers, []) + XCTAssertNil(PluginVoiceCommandRouter.match( + instruction: "anything at all", enabledPlugins: [broken])) + // Reported to the author, but never fatal — the plugin still lists and runs. + XCTAssertEqual(broken.validate(), .emptyVoiceTriggers) + XCTAssertTrue(broken.isValid) + } + + func testTriggersAreLowercasedTrimmedAndDeduplicated() { + let messy = manifest(id: "messy", triggers: [" Create A Meme ", "create a meme"]) + XCTAssertEqual(messy.normalizedVoiceTriggers, ["create a meme"]) + } + + /// Forward-compatible decode: a manifest written before `voiceTriggers` existed + /// still decodes (and simply has no voice route). + func testManifestWithoutVoiceTriggersStillDecodes() throws { + let json = """ + {"id":"legacy","name":"Legacy","symbol":"star","entry":"builtIn"} + """.data(using: .utf8)! + let decoded = try JSONDecoder().decode(PluginManifest.self, from: json) + XCTAssertEqual(decoded.voiceTriggers, []) + XCTAssertTrue(decoded.isValid) + } + + func testManifestDecodesDeclaredVoiceTriggers() throws { + let json = """ + {"id":"p","name":"P","symbol":"star","entry":"builtIn", + "voiceTriggers":["Create A Meme","make a meme"]} + """.data(using: .utf8)! + let decoded = try JSONDecoder().decode(PluginManifest.self, from: json) + XCTAssertEqual(decoded.normalizedVoiceTriggers, ["create a meme", "make a meme"]) + } + + // MARK: - Overlay acknowledgment + + /// The routed dictation produces NOTHING in the focused app, so the overlay has + /// to say where the words went — and it must name the plugin. + func testAcknowledgmentNamesThePlugin() { + XCTAssertEqual( + PluginVoiceCommandRouter.acknowledgment(pluginName: "Meme Generator"), + "Meme Generator — creating…") + } + + func testDisabledHintNamesThePlugin() { + XCTAssertEqual( + PluginVoiceCommandRouter.disabledHint(pluginName: "Meme Generator"), + "Meme Generator plugin is disabled") + } + + /// The acknowledgment reaches the overlay through `statusMessage`, which + /// `FinalizingCaption` surfaces verbatim — so no new OverlayPhase case is needed. + /// This pins that the two agree; a change to either side that broke the caption + /// would otherwise only show up on screen. + func testAcknowledgmentSurvivesAsTheOverlayFinalizeCaption() { + let ack = PluginVoiceCommandRouter.acknowledgment(pluginName: "Meme Generator") + XCTAssertEqual( + FinalizingCaption.resolve( + isTranscribing: true, statusMessage: ack, + workerStatus: "", usesWhisperKit: false), + "Meme Generator — creating…") + } + + // MARK: - The shipping manifest + + /// The meme plugin actually declares the phrases the owner speaks. + func testShippingMemeManifestRoutesBothOwnerFlows() { + let plugins = PluginRegistry.builtInManifests + XCTAssertEqual( + PluginVoiceCommandRouter.match( + instruction: "create a meme based on that", enabledPlugins: plugins)?.pluginID, + PluginRegistry.memeGenerator.id) + XCTAssertEqual( + PluginVoiceCommandRouter.match( + instruction: "Сделай мем про дедлайны", enabledPlugins: plugins)?.pluginID, + PluginRegistry.memeGenerator.id) + XCTAssertNil(PluginVoiceCommandRouter.match( + instruction: "create a memo about the Q3 numbers", enabledPlugins: plugins)) + } +} diff --git a/plugins/MemeGenerator/manifest.json b/plugins/MemeGenerator/manifest.json index a1c3aeb..bdbe4d6 100644 --- a/plugins/MemeGenerator/manifest.json +++ b/plugins/MemeGenerator/manifest.json @@ -6,5 +6,12 @@ "symbol": "photo.badge.plus", "entry": "builtIn", "networkHosts": ["api.imgflip.com", "i.imgflip.com", "api.memegen.link"], - "keyEquivalent": "m" + "keyEquivalent": "m", + "voiceTriggers": [ + "create a meme", + "make a meme", + "generate a meme", + "сделай мем", + "создай мем" + ] } From a4fa6cf92c8155cddbbb78c1a6f649776d0b7fc8 Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Mon, 3 Aug 2026 17:39:37 -0700 Subject: [PATCH 14/15] spike(meme-plugin): route refine voice commands to the plugin, no insertion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the trigger layer into the refine pipeline for the owner's two flows: CASE 1 — select text anywhere, dictate, tap Refine, say "create a meme based on that": the SELECTION (the refine content snapshot) is the material. CASE 2 — Refine with nothing selected, say "create a meme expanding brain: typing, dictating, …": the spoken remainder is. CASE 2 needed a real fix, not just a call. `armRefineMidSession` bailed with "Nothing to refine yet" whenever there was no in-session text, no selection, and no last dictation — so `refineContentSnapshot` stayed nil, `deliverFinalText`'s refine branch never ran, and the command could never reach a router. It now arms with an EMPTY snapshot, but only when an enabled plugin actually declares triggers (`PluginHost.armsWithoutContent`); with no such plugin the old status is byte-for- byte what it was. When the instruction turns out NOT to be a command, RefineFlow's existing empty-content rule produces the same "Nothing to refine" outcome as before. NO INSERTION on the routed path. The route is consulted in `deliverFinalText` BEFORE `runLLM` and before any insert, and a claimed command returns `RefineFlow.Effect.finishQuietly` — the same teardown a contentless refine already uses, so the session ends through one auditable no-insert path instead of a second hand-rolled one. Verified against the real delivery path: nothing reaches `insertCompletedText`, so nothing lands in the focused app. Failure never eats the dictation. A non-match, a disabled plugin, or a window that can't take the command all fall through to the untouched normal refine. A disabled plugin additionally sets a one-line hint, and ONLY when the trigger matched exactly (`matchIgnoringEnablement`) — so it can't fire on an unrelated refine. Overlay: the acknowledgment names the plugin ("Meme Generator — creating…") and reaches the overlay via `statusMessage`, which `FinalizingCaption.resolve` already surfaces verbatim — no new OverlayPhase case, no view change. A test pins that the two agree. AppState pays for itself: the feature adds ~10 lines there (one `if let` and one guard, both delegating to PluginHost), and the ratchet BUDGET DROPS 7036 -> 7024 because the consent-decision, dictation-stats-event, rules-firing, and refinement- mode constructions moved into the pure core where swift test can reach them. Co-Authored-By: Claude --- OpenWhisp/Models/AppState.swift | 124 +++++++------- OpenWhisp/Services/AgentCLIProvider.swift | 11 ++ OpenWhisp/Services/AgentClientStore.swift | 43 +++++ OpenWhisp/Services/DictationStats.swift | 43 +++++ OpenWhisp/Services/Rules.swift | 22 +++ .../Views/PluginVoiceCommandDispatch.swift | 157 ++++++++++++++++++ .../MemeGeneratorWindowController.swift | 39 ++++- scripts/appstate-loc-budget.txt | 2 +- 8 files changed, 366 insertions(+), 75 deletions(-) create mode 100644 OpenWhisp/Views/PluginVoiceCommandDispatch.swift diff --git a/OpenWhisp/Models/AppState.swift b/OpenWhisp/Models/AppState.swift index 9b87ead..fbcc0ef 100644 --- a/OpenWhisp/Models/AppState.swift +++ b/OpenWhisp/Models/AppState.swift @@ -1519,8 +1519,7 @@ class AppState: ObservableObject { /// provider is active, so tiny on-device models get the terser, stricter /// system prompt (see OpenAITranslationService.instructionForMode). private func refinementMode(_ mode: String) -> String { - guard llmProvider == "bundled" else { return mode } - return mode == "rephrase" ? "bundled-rephrase" : "bundled-improve" + EnhancementProvider.refinementMode(mode, llmProvider: llmProvider) } /// The whole-text final AI step, expressed as the `AsyncTextRefiner` seam @@ -4223,6 +4222,17 @@ class AppState: ObservableObject { let instruction = instructionSuffix(fullFinal: finalText, content: content) refineActiveInstruction = instruction refineDebug("completeFinalText MID-REFINE content=\"\(content.prefix(20))\" instr=\"\(instruction.prefix(20))\" fromSelection=\(fromSelection)") + + // v10: a plugin may claim this instruction by its spoken PREFIX ("create a + // meme …") — asked BEFORE the refine LLM and before any insert, because a + // claimed command delivers NOTHING to the focused app. A non-match falls + // through to the normal refine below, so the words are never lost. + if let quietly = PluginHost.shared.routeVoiceCommand( + instruction: instruction, content: content, on: self) { + refineFlow.reset(); refineActiveInstruction = nil + executeRefineEffects([quietly]); return + } + isTranscribing = true // Drive the machine: engage with the content as step-1, then feed the // instruction. @@ -4421,8 +4431,15 @@ class AppState: ObservableObject { content = sel fromSelection = true } else { - statusMessage = "Nothing to refine yet — dictate first, then tap Refine" - return + // v10: no content still arms IF a plugin can claim the instruction by + // voice — that command carries its own material. See + // `PluginHost.armsWithoutContent`. + guard PluginHost.shared.armsWithoutContent else { + statusMessage = "Nothing to refine yet — dictate first, then tap Refine" + return + } + content = "" + fromSelection = false } refineContentSnapshot = content refineContentFromSelection = fromSelection @@ -4850,23 +4867,18 @@ class AppState: ObservableObject { /// fires on a `suppressOutput` (agent) session if it explicitly opted in. private func fireRules(hook: RuleHook, text: String) { guard !ruleSet.rules.isEmpty else { return } - let context = RuleContext( - hook: hook, - text: text, - appBundleID: targetApplication?.bundleIdentifier, - isAgentSession: suppressOutput - ) - let payload = OutputPayload( - text: text, - language: outputLanguageForCleaning, - targetAppBundleID: targetApplication?.bundleIdentifier, - isLiveChunk: false - ) // Planning happens on the runner's queue, not here: matching can evaluate a // user-supplied regex, and even the matcher's backtracking time budget must // never be spent on the finalize path. `ruleSet` is a value type — the - // runner gets an immutable snapshot. - ruleEngineRunner.planAndRun(rules: ruleSet, context: context, payload: payload) + // runner gets an immutable snapshot. The (context, payload) construction is + // pure and lives in `RuleContext.firing` where `swift test` pins it. + let firing = RuleContext.firing( + hook: hook, text: text, + appBundleID: targetApplication?.bundleIdentifier, + isAgentSession: suppressOutput, + language: outputLanguageForCleaning) + ruleEngineRunner.planAndRun( + rules: ruleSet, context: firing.context, payload: firing.payload) } /// Instance method (called only from `screenContext`'s didSet, where `self` @@ -5075,16 +5087,21 @@ class AppState: ObservableObject { } } + /// The learner's state as the pure pipeline models it — the (proposals, + /// confidence) pair is always read and written together. + private var correctionState: CorrectionLearningPipeline.State { + get { .init(proposals: correctionProposals, confidence: correctionConfidence) } + set { correctionProposals = newValue.proposals; correctionConfidence = newValue.confidence } + } + /// A captured (inserted, surviving) edit — single- or multi-word (MAK-86): the /// pure `CorrectionLearningPipeline` decides ignore / propose / auto-add. Only a /// repeat-corroborated auto-add mutates the dictionary (never a one-off). private func handleObservedCorrection(inserted: String, surviving: String) { let (newState, outcome) = CorrectionLearningPipeline.decide( inserted: inserted, surviving: surviving, - existingSubstitutions: vocabulary.substitutions, - state: .init(proposals: correctionProposals, confidence: correctionConfidence)) - correctionProposals = newState.proposals - correctionConfidence = newState.confidence + existingSubstitutions: vocabulary.substitutions, state: correctionState) + correctionState = newState vocabulary = CorrectionLearningPipeline.applying(outcome, to: vocabulary) } @@ -5100,10 +5117,7 @@ class AppState: ObservableObject { /// Reject a pending correction proposal: dequeue it and remember not to re-offer /// the same fix. Does not touch the dictionary. func rejectCorrectionProposal(_ id: CorrectionProposal.ID) { - let next = CorrectionLearningPipeline.rejecting( - id, state: .init(proposals: correctionProposals, confidence: correctionConfidence)) - correctionProposals = next.proposals - correctionConfidence = next.confidence + correctionState = CorrectionLearningPipeline.rejecting(id, state: correctionState) } /// File-tagging (MAK-48) fires ONLY when the user opted in AND the app being @@ -5930,26 +5944,11 @@ class AppState: ObservableObject { guard !trimmed.isEmpty else { return } guard let startedAt = recordingStartedAt else { return } - let now = Date() - let model: String? = switch transcriptionEngine { - case "whisperKit": whisperKitModel - case "parakeet": parakeetVariant - case "appleSpeech": nil - case "speechAnalyzer": nil - default: modelName - } - let latency = transcriptionStartedAt.map { now.timeIntervalSince($0) } - - let event = DictationEvent( - date: now, - wordCount: DictationEvent.words(in: trimmed), - charCount: trimmed.count, - durationSeconds: now.timeIntervalSince(startedAt), - engine: transcriptionEngine, - model: model, - outputMode: outputMode, - appBundleID: targetApplication?.bundleIdentifier, - transcriptionLatencySeconds: latency + let event = DictationEvent.make( + trimmedText: trimmed, now: Date(), startedAt: startedAt, transcriptionStartedAt: transcriptionStartedAt, + engine: transcriptionEngine, whisperKitModel: whisperKitModel, + parakeetVariant: parakeetVariant, modelName: modelName, + outputMode: outputMode, appBundleID: targetApplication?.bundleIdentifier ) dictationStats.record(event) // Same off-main-actor write as persistHistory(): stats save on every @@ -6826,38 +6825,27 @@ extension AppState: AgentBridgeHost { consentDecision(record: agentClients.record(for: clientName), clientName: clientName, scope: scope) } - /// Same, with the record already fetched — bridge.hello resolves every scope - /// at once and must not re-scan the records array once per scope. + /// Same, with the record already fetched (bridge.hello resolves every scope at + /// once, so it must not re-scan records per scope); supplies the this-run + /// grant to the pure `AgentScope.consentDecision`. private func consentDecision( record: AgentClientRecord?, clientName: String, scope: AgentScope ) -> AgentConsentDecision { - guard let policy = record?.policy(for: scope) else { return .prompt } let grantedThisRun = consentGrantedThisRun[clientName]?.contains(scope) ?? false - return policy.decision(grantedThisRun: grantedThisRun) + return AgentScope.consentDecision(record: record, grantedThisRun: grantedThisRun, scope: scope) } - /// The posture advertised in `bridge.hello` (never prompts): a per-scope map - /// plus a summary scalar — `.granted` only if EVERY scope is already allowed, - /// `.denied` only if every scope is denied, else `.pending`. The scalar alone - /// is too lossy for real clients (a dictate-only agent with an explicit deny - /// would read "pending" forever); adapters that care which capability is - /// usable read the map. Prompting still happens per call. + /// The posture advertised in `bridge.hello` (never prompts). Resolves every + /// scope's decision here (the client record + this-run grants AppState owns) + /// and hands the pure aggregation to `AgentScope.consentSnapshot` (see + /// its doc for the summary-scalar semantics). Prompting still happens per call. func bridgeConsentSnapshot(clientName: String) -> (summary: BridgeWire.ConsentState, scopes: [String: BridgeWire.ConsentState]) { let record = agentClients.record(for: clientName) - var scopes: [String: BridgeWire.ConsentState] = [:] - var allAllow = true - var allDeny = true + var decisions: [AgentScope: AgentConsentDecision] = [:] for scope in AgentScope.allCases { - let decision = consentDecision(record: record, clientName: clientName, scope: scope) - switch decision { - case .allow: scopes[scope.rawValue] = .granted - case .deny: scopes[scope.rawValue] = .denied - case .prompt: scopes[scope.rawValue] = .pending - } - allAllow = allAllow && decision == .allow - allDeny = allDeny && decision == .deny + decisions[scope] = consentDecision(record: record, clientName: clientName, scope: scope) } - return (allAllow ? .granted : (allDeny ? .denied : .pending), scopes) + return AgentScope.consentSnapshot(decisions: decisions) } /// Note a completed agent call on the client's record (for the settings pane). diff --git a/OpenWhisp/Services/AgentCLIProvider.swift b/OpenWhisp/Services/AgentCLIProvider.swift index 40dd140..7135b08 100644 --- a/OpenWhisp/Services/AgentCLIProvider.swift +++ b/OpenWhisp/Services/AgentCLIProvider.swift @@ -343,6 +343,17 @@ public enum EnhancementProvider: Equatable { providerID == agentCLIID } + /// Remap a generic refine mode ("rephrase" / "improve") onto the bundled + /// llama.cpp model's own prompt-variant ids when `llmProvider == "bundled"`; + /// every other provider passes `mode` through unchanged. The bundled model + /// ships two dedicated prompt variants (tuned smaller/cheaper than the cloud + /// prompts), so the caller's generic mode string needs translating only for + /// that one provider. + public static func refinementMode(_ mode: String, llmProvider: String) -> String { + guard llmProvider == "bundled" else { return mode } + return mode == "rephrase" ? "bundled-rephrase" : "bundled-improve" + } + /// The command the app would spawn for the agent-CLI provider, given the user's /// persisted selection — or a `BuildError` if the config is unusable (empty /// command, transcript-in-argv). Pure: this is the seam a test drives to prove diff --git a/OpenWhisp/Services/AgentClientStore.swift b/OpenWhisp/Services/AgentClientStore.swift index 5c956a4..23db61d 100644 --- a/OpenWhisp/Services/AgentClientStore.swift +++ b/OpenWhisp/Services/AgentClientStore.swift @@ -41,6 +41,49 @@ public enum AgentScope: String, Codable, Equatable, Sendable, CaseIterable { /// to every migrated "always allow" client. public static let legacyV1Scopes: [AgentScope] = [.dictate, .history, .refine] + /// The consent decision for one client/scope, given its (possibly absent) + /// stored record and whether a `.whileRunning` grant already happened this app + /// run. A record with no stored policy for `scope` has no decision yet, so it + /// always prompts — matching a brand-new scope added after the client was + /// first seen. Pure: callers own the record lookup and the this-run grant + /// lookup (both are AppState-owned state), so this stays testable in + /// isolation from the store's on-disk shape. + public static func consentDecision( + record: AgentClientRecord?, grantedThisRun: Bool, scope: AgentScope + ) -> AgentConsentDecision { + guard let policy = record?.policy(for: scope) else { return .prompt } + return policy.decision(grantedThisRun: grantedThisRun) + } + + /// Aggregate a per-scope ``AgentConsentDecision`` map (already resolved by the + /// caller — this function does no record lookups, so it stays pure and + /// Foundation-only) into the `bridge.hello` consent posture: a per-scope wire + /// map plus a summary scalar. The summary is `.granted` only if EVERY scope + /// decided `.allow`, `.denied` only if every scope decided `.deny`, and + /// `.pending` otherwise — a mixed bag (e.g. dictate allowed, refine denied) + /// reads as `.pending` because no single scalar can represent it, and callers + /// that care which capability is usable should read the per-scope map instead. + /// `decisions` should have one entry per `AgentScope.allCases`; a missing scope + /// is simply omitted from the returned map (never surfaced as `.pending` on + /// its own) and does not affect the summary aggregation. + public static func consentSnapshot( + decisions: [AgentScope: AgentConsentDecision] + ) -> (summary: BridgeWire.ConsentState, scopes: [String: BridgeWire.ConsentState]) { + var scopes: [String: BridgeWire.ConsentState] = [:] + var allAllow = true + var allDeny = true + for (scope, decision) in decisions { + switch decision { + case .allow: scopes[scope.rawValue] = .granted + case .deny: scopes[scope.rawValue] = .denied + case .prompt: scopes[scope.rawValue] = .pending + } + allAllow = allAllow && decision == .allow + allDeny = allDeny && decision == .deny + } + return (allAllow ? .granted : (allDeny ? .denied : .pending), scopes) + } + /// A short human label for the consent window / settings pane. public var title: String { switch self { diff --git a/OpenWhisp/Services/DictationStats.swift b/OpenWhisp/Services/DictationStats.swift index 5827a23..027d26c 100644 --- a/OpenWhisp/Services/DictationStats.swift +++ b/OpenWhisp/Services/DictationStats.swift @@ -29,6 +29,49 @@ struct DictationEvent: Equatable { static func words(in text: String) -> Int { text.split(whereSeparator: { $0.isWhitespace || $0.isNewline }).count } + + /// Which model identifier to record for a completed dictation, given the + /// active `engine` id and each engine's own model-selection setting. Engines + /// that don't have a distinct model concept (Apple's on-device engines) record + /// `nil` rather than a misleading placeholder. Kept as a single pure switch so + /// `recordStats` and any future stats consumer resolve the model the same way; + /// static + pure so it's unit-tested directly against fixture engine ids. + static func engineModel( + engine: String, whisperKitModel: String, parakeetVariant: String, modelName: String + ) -> String? { + switch engine { + case "whisperKit": return whisperKitModel + case "parakeet": return parakeetVariant + case "appleSpeech": return nil + case "speechAnalyzer": return nil + default: return modelName + } + } + + /// Build the event for a just-completed dictation from the already-trimmed + /// final `text` and the session's timing/engine inputs. Pure aggregation of + /// `words(in:)` + `engineModel` + the two elapsed-time computations + /// (`recordStats` still owns the pre-checks: secure field, empty transcript, + /// unknown start time — those are early-return GUARDS, not part of the value + /// being built, so they stay in AppState). `now` is threaded in rather than + /// read internally so callers and tests can pin the clock. + static func make( + trimmedText: String, now: Date, startedAt: Date, transcriptionStartedAt: Date?, + engine: String, whisperKitModel: String, parakeetVariant: String, modelName: String, + outputMode: String, appBundleID: String? + ) -> DictationEvent { + DictationEvent( + date: now, + wordCount: words(in: trimmedText), + charCount: trimmedText.count, + durationSeconds: now.timeIntervalSince(startedAt), + engine: engine, + model: engineModel(engine: engine, whisperKitModel: whisperKitModel, parakeetVariant: parakeetVariant, modelName: modelName), + outputMode: outputMode, + appBundleID: appBundleID, + transcriptionLatencySeconds: transcriptionStartedAt.map { now.timeIntervalSince($0) } + ) + } } /// Per-day rollup bucket. Keyed by an ISO `yyyy-MM-dd` day string (UTC) in the diff --git a/OpenWhisp/Services/Rules.swift b/OpenWhisp/Services/Rules.swift index 1aa3ab3..51a6bfd 100644 --- a/OpenWhisp/Services/Rules.swift +++ b/OpenWhisp/Services/Rules.swift @@ -262,6 +262,28 @@ struct RuleContext: Equatable { self.appBundleID = appBundleID self.isAgentSession = isAgentSession } + + /// Build the (context, payload) pair one rules-engine firing needs. + /// + /// Pure, so the shape of what the engine receives is pinned by `swift test` + /// rather than only by reading AppState's finalize path — and so AppState carries + /// the call, not the construction (MAK-32 ratchet). + /// + /// `isLiveChunk` is deliberately fixed to `false`: every caller fires this from a + /// FINAL transcript, never a streaming chunk. + static func firing( + hook: RuleHook, text: String, appBundleID: String?, + isAgentSession: Bool, language: String + ) -> (context: RuleContext, payload: OutputPayload) { + ( + RuleContext( + hook: hook, text: text, + appBundleID: appBundleID, isAgentSession: isAgentSession), + OutputPayload( + text: text, language: language, + targetAppBundleID: appBundleID, isLiveChunk: false) + ) + } } // MARK: - Matcher diff --git a/OpenWhisp/Views/PluginVoiceCommandDispatch.swift b/OpenWhisp/Views/PluginVoiceCommandDispatch.swift new file mode 100644 index 0000000..7177aa2 --- /dev/null +++ b/OpenWhisp/Views/PluginVoiceCommandDispatch.swift @@ -0,0 +1,157 @@ +import AppKit + +/// The app-side half of the v10 voice-command route: turn a `PluginVoiceCommandRouter` +/// match into an open plugin window seeded with the user's material. +/// +/// Lives on `PluginHost` (not `AppState`) for the MAK-32 ratchet reason the host was +/// created for in the first place — AppState is at zero headroom, so the feature's +/// logic sits here and AppState keeps a single call. The DECISION (does this +/// instruction route, and to whom) is pure and already lives in +/// `PluginVoiceCommandRouter`; this file is only the side effect. +@MainActor +extension PluginHost { + + /// What the refine pipeline should do with an instruction, after consulting the + /// plugins. + enum VoiceCommandOutcome: Equatable { + /// No plugin claimed it — run the NORMAL refine, byte-identical to v9. + case notHandled + /// A plugin took it. The refine LLM must not run and NOTHING may be inserted + /// into the focused app: the plugin window is the output. + /// `status` is the overlay acknowledgment. + case handled(status: String) + /// The command matched a plugin that is switched OFF. The instruction still + /// runs as a normal refine; `hint` explains the missing window. + case disabled(hint: String) + + /// The refine effect that ends a ROUTED session, or nil to keep refining. + /// + /// Reusing `RefineFlow.Effect.finishQuietly` rather than open-coding the + /// teardown in AppState is deliberate twice over: it keeps the ratchet paid, + /// and it means the routed path tears down through the exact sequence every + /// other no-insert refine outcome already uses — one auditable place where a + /// session ends without delivering text. + var finishQuietlyEffect: RefineFlow.Effect? { + guard case let .handled(status) = self else { return nil } + return .finishQuietly(status: status) + } + + /// A transient status line to show while STILL running the normal refine. + var statusHint: String? { + guard case let .disabled(hint) = self else { return nil } + return hint + } + } + + /// Whether ANY enabled plugin declares voice triggers. + /// + /// The refine key uses this to decide whether a no-content tap is worth arming + /// (CASE 2). Without a routable plugin there is genuinely nothing to do with an + /// instruction that has no content, and arming would replace the honest "Nothing + /// to refine yet" with a silent dead end — so a build with no such plugin keeps + /// v9's behavior exactly. + var armsWithoutContent: Bool { + activePlugins.contains { !$0.manifest.normalizedVoiceTriggers.isEmpty } + } + + /// The refine pipeline's single entry point (v10). + /// + /// Consults the plugins and applies the only side effect that belongs to the + /// CALLER's state — the disabled hint — returning the teardown effect when a + /// plugin took the command, or nil to continue with a normal refine. + /// + /// Shaped this way so `AppState.deliverFinalText` carries one `if let`: the MAK-32 + /// ratchet is at zero headroom, and a feature that spends its budget on a switch + /// statement in the god object is a feature that makes the next one harder. + func routeVoiceCommand( + instruction: String, content: String?, on appState: AppState + ) -> RefineFlow.Effect? { + let outcome = handleVoiceCommand(instruction: instruction, content: content) + if let hint = outcome.statusHint { appState.statusMessage = hint } + return outcome.finishQuietlyEffect + } + + /// Offer a spoken refine instruction to the plugins. + /// + /// - Parameters: + /// - instruction: the spoken words (the trigger phrase plus any material). + /// - content: the refine CONTENT snapshot — the user's selection or prior + /// dictation. This is CASE 1's material: "create a meme based on that" carries + /// no description of its own, and `that` is the selection. + /// - Returns: what the caller should do next. + /// + /// ## How the two flows converge + /// + /// Both end up calling the plugin with ONE string. The remainder (what the user + /// said after the trigger) and the content (what they had selected) are joined + /// when both exist, because they are both material: a user who selects a paragraph + /// AND says "create a meme about the deadline" meant both to count. Remainder + /// first — it is the more specific instruction. + func handleVoiceCommand(instruction: String, content: String?) -> VoiceCommandOutcome { + // Enabled AND runnable only: `activePlugins` is the same list the menu bar + // and the window opener use, so a plugin the user switched off cannot claim a + // dictation. This is the enablement gate (requirement 3). + let enabled = activePlugins.map(\.manifest) + guard let match = PluginVoiceCommandRouter.match( + instruction: instruction, enabledPlugins: enabled) + else { + // Not claimed by an enabled plugin. Before falling through, check whether a + // DISABLED one would have taken it — that, and only that, earns the hint. + let all = discovered.map(\.manifest) + if let offMatch = PluginVoiceCommandRouter.matchIgnoringEnablement( + instruction: instruction, plugins: all), + !isEnabled(offMatch.pluginID) { + let name = all.first { $0.id == offMatch.pluginID }?.name ?? offMatch.pluginID + MemeTrace.log("voice command matched but plugin disabled -> normal refine") + return .disabled(hint: PluginVoiceCommandRouter.disabledHint(pluginName: name)) + } + return .notHandled + } + + let manifest = enabled.first { $0.id == match.pluginID } + let name = manifest?.name ?? match.pluginID + MemeTrace.log( + "voice command MATCHED plugin=\(match.pluginID) trigger=\"\(match.trigger)\" " + + "remainder=\"\(match.remainder)\" content=\(content?.count ?? 0) chars") + + // Material: what they said after the trigger, plus what they had selected. + let selection = content?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let material = [match.remainder, selection] + .filter { !$0.isEmpty } + .joined(separator: "\n") + + guard !material.isEmpty else { + // "create a meme" with nothing selected and nothing said after it. Opening + // an empty window is still the right answer — the user asked for the meme + // window — but say so rather than looking like a silent no-op. + open(pluginID: match.pluginID) + MemeTrace.log("voice command opened \(match.pluginID) with NO material") + return .handled(status: "\(name) — say what the meme should be") + } + + open(pluginID: match.pluginID) + guard let sink = windowController(for: match.pluginID) as? PluginVoiceCommandSink else { + // The window exists but doesn't take commands (or this build has no + // plugins compiled in). Fail LOUDLY rather than eating the dictation: the + // caller falls back to a normal refine, so the words are not lost. + MemeTrace.log("voice command ABORTED: \(match.pluginID) has no command sink") + return .notHandled + } + sink.runVoiceCommand(material: material) + MemeTrace.log("voice command dispatched to \(match.pluginID), material=\"\(material)\"") + return .handled(status: PluginVoiceCommandRouter.acknowledgment(pluginName: name)) + } +} + +/// A plugin window that can be driven by a spoken command (v10). +/// +/// The third plugin seam, alongside `PluginWindowLifecycle` and `PluginDictationSink`. +/// Distinct from the dictation sink on purpose: that one APPENDS text to whatever the +/// user is editing when the window is already key, whereas this one arrives from a +/// refine the user spoke into another app entirely, and means "start a new one from +/// this material." +@MainActor +protocol PluginVoiceCommandSink: AnyObject { + /// Seed the plugin with `material` and run its primary action. + func runVoiceCommand(material: String) +} diff --git a/plugins/MemeGenerator/MemeGeneratorWindowController.swift b/plugins/MemeGenerator/MemeGeneratorWindowController.swift index 23d5730..a20bf5c 100644 --- a/plugins/MemeGenerator/MemeGeneratorWindowController.swift +++ b/plugins/MemeGenerator/MemeGeneratorWindowController.swift @@ -13,7 +13,7 @@ import SwiftUI /// dictation can land in it — that is what makes `appendDictationIfKey` fire. @MainActor final class MemeGeneratorWindowController: NSWindowController, NSWindowDelegate, - PluginDictationSink, PluginWindowLifecycle { + PluginDictationSink, PluginWindowLifecycle, PluginVoiceCommandSink { private let model = MemeGeneratorModel() @@ -125,12 +125,18 @@ final class MemeGeneratorWindowController: NSWindowController, NSWindowDelegate, model.description = prompt model.generate() - // Report the canvas after the generate settles. The probe cannot know when the - // LLM answers, so it samples on a deadline and states what it found — a probe - // that reported nothing would look like a crash. - let deadline = Double(env["OPENWHISP_MEME_PROBE_SECONDS"] ?? "") ?? 90 + reportCanvasAfter(seconds: Double(env["OPENWHISP_MEME_PROBE_SECONDS"] ?? "") ?? 90) + } + + /// Report the canvas after the generate settles. + /// + /// The probe cannot know when the LLM answers, so it samples on a deadline and + /// states what it found — a probe that reported nothing would look like a crash. + /// Shared by the v9 prompt probe and the v10 voice-command probe so both prove the + /// outcome the same way: N boxes, and the text in them. + func reportCanvasAfter(seconds: Double) { Task { @MainActor [model] in - try? await Task.sleep(nanoseconds: UInt64(deadline * 1_000_000_000)) + try? await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) MemeTrace.log( "probe result: \(model.boxes.count) boxes on canvas, " + "texts=\(model.boxes.map(\.text))") @@ -153,6 +159,27 @@ final class MemeGeneratorWindowController: NSWindowController, NSWindowDelegate, model.windowDidOpen() } + // MARK: - PluginVoiceCommandSink (v10) + + /// Run a meme from a spoken refine command ("create a meme …"). + /// + /// Unlike `appendDictationIfKey`, this arrives from a dictation the user spoke + /// into ANOTHER app, so it must not append to whatever is sitting in the window + /// from an earlier session — `startNewMeme()` clears first, then the material + /// becomes the description and Generate runs. That is the same + /// `description` + `generate()` pair the Generate button and the v9 probe use, so + /// the voice route inherits the whole production path (extraction, shortlist, LLM, + /// seeding) with no branch of its own. + func runVoiceCommand(material: String) { + MemeTrace.log("runVoiceCommand material=\"\(material)\"") + // The window was just opened/focused by the host; make sure the model is in + // its open state (clears `isCancelled` if the window had been closed before). + model.windowDidOpen() + model.startNewMeme() + model.description = material + model.generate() + } + // MARK: - PluginDictationSink /// True while this window is frontmost — the signal that a completed dictation diff --git a/scripts/appstate-loc-budget.txt b/scripts/appstate-loc-budget.txt index 6ec0b28..cd85f97 100644 --- a/scripts/appstate-loc-budget.txt +++ b/scripts/appstate-loc-budget.txt @@ -1 +1 @@ -7036 +7024 From 65f7077ebc7fac2bfbf87161ea043189478828bc Mon Sep 17 00:00:00 2001 From: Maksym Naboka Date: Mon, 3 Aug 2026 17:39:47 -0700 Subject: [PATCH 15/15] spike(meme-plugin): runtime proof for the voice-command route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v9's lesson was that reading the wiring is not evidence. `swift test` proves the ROUTER but cannot prove the pipeline ever REACHES it — the refine path lives on AppState, which the core test target doesn't compile, and this project's wiring bugs have all passed their unit tests while the live gate was dead. So the launch-gated probe now drives the route itself. `startRefineRouteProbe` calls the SAME `PluginHost.routeVoiceCommand` that `AppState.deliverFinalText` calls when a mid-dictation refine finalizes, with the same (instruction, content) pair — only the source of those two strings differs (env vars instead of the mic). Setting `..._REFINE_CONTENT` expresses CASE 1; omitting it expresses CASE 2. One harness gotcha worth recording: the probe MUST run from a bundle. `build.sh` emits a loose executable, and AppKit never services the main run loop for one, so every `asyncAfter` — the v9 probe's included — sits queued forever and the run looks like a silent hang. The v9 probe has this same defect; the script now hand-assembles a minimal ad-hoc-signed .app (binary + Info.plist + Resources), which is enough since no third_party runtime matters to the trigger layer. The canvas report is factored out of the v9 probe so both prove the outcome the same way: N boxes, and the text in them. Co-Authored-By: Claude --- OpenWhisp/AppMain.swift | 61 +++++++++++++++ scripts/meme-voice-command-proof.sh | 110 ++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100755 scripts/meme-voice-command-proof.sh diff --git a/OpenWhisp/AppMain.swift b/OpenWhisp/AppMain.swift index 286731d..1145706 100644 --- a/OpenWhisp/AppMain.swift +++ b/OpenWhisp/AppMain.swift @@ -137,6 +137,19 @@ class OpenWhispApp: NSObject, NSApplicationDelegate { /// failed — which is the exact class of mistake this whole exercise is about. private func startMemeProbeIfRequested() { let env = ProcessInfo.processInfo.environment + + // v10: the VOICE-COMMAND probe. Drives the refine route — the same + // `PluginHost.routeVoiceCommand` AppState calls on a real dictation — so the + // proof covers the TRIGGER LAYER, not just Generate. Both of the owner's + // flows are expressible: `..._REFINE_CONTENT` set = CASE 1 (a selection is + // the material), unset = CASE 2 (the spoken remainder is). + if let instruction = env["OPENWHISP_MEME_PROBE_REFINE"], !instruction.isEmpty { + startRefineRouteProbe(instruction: instruction, + content: env["OPENWHISP_MEME_PROBE_REFINE_CONTENT"], + delay: Double(env["OPENWHISP_MEME_PROBE_DELAY"] ?? "") ?? 6) + return + } + guard let prompt = env["OPENWHISP_MEME_PROBE_PROMPT"], !prompt.isEmpty else { return } MemeTrace.log("probe requested at launch") @@ -159,6 +172,54 @@ class OpenWhispApp: NSObject, NSApplicationDelegate { } } + /// Drive the v10 voice-command route from launch and report what it decided. + /// + /// Calls `PluginHost.routeVoiceCommand` — the SAME entry point + /// `AppState.deliverFinalText` uses the moment a mid-dictation refine finalizes, + /// with the same (instruction, content) pair. Nothing about the routing decision, + /// the enablement gate, the window open, or the generate is probe-specific; only + /// the source of the two strings differs (env vars instead of the mic). + /// + /// That matters because the wiring is exactly what a source read keeps getting + /// wrong: a trigger that matches in `swift test` proves the ROUTER, not that the + /// refine pipeline ever reaches it. This drives the pipeline's own seam. + private func startRefineRouteProbe(instruction: String, content: String?, delay: Double) { + MemeTrace.log( + "refine-route probe requested: instruction=\"\(instruction)\" " + + "content=\(content.map { "\"\($0)\"" } ?? "nil")") + DispatchQueue.main.asyncAfter(deadline: .now() + delay) { + guard let appState = self.appState else { + MemeTrace.log("refine-route probe ABORTED: no appState") + return + } + let effect = PluginHost.shared.routeVoiceCommand( + instruction: instruction, content: content, on: appState) + // nil = the router declined and the pipeline would run a NORMAL refine. + // That is the near-miss case's expected outcome, and it must be visible. + if let effect { + MemeTrace.log("refine-route probe: ROUTED, refine effect=\(effect)") + } else { + MemeTrace.log( + "refine-route probe: NOT ROUTED -> normal refine " + + "(status=\"\(appState.statusMessage)\")") + } + // Report the canvas after the generate settles, like the v9 probe. + let deadline = Double( + ProcessInfo.processInfo.environment["OPENWHISP_MEME_PROBE_SECONDS"] ?? "") ?? 90 + #if OPENWHISP_PLUGINS + guard let controller = PluginHost.shared.windowController( + for: PluginRegistry.memeGenerator.id) as? MemeGeneratorWindowController + else { + MemeTrace.log("refine-route probe done (no meme window was opened)") + return + } + controller.reportCanvasAfter(seconds: deadline) + #else + MemeTrace.log("refine-route probe: build has no plugins (PLUGINS=1 ./build.sh)") + #endif + } + } + /// Re-check permissions whenever the app becomes active. This is what makes /// the missing-permission banner AUTO-CLEAR: the user grants the permission /// in System Settings, clicks back into OpenWhisp, and the live recheck diff --git a/scripts/meme-voice-command-proof.sh b/scripts/meme-voice-command-proof.sh new file mode 100755 index 0000000..f783432 --- /dev/null +++ b/scripts/meme-voice-command-proof.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Runtime proof for the v10 plugin voice-command route (spike/plugin-system). +# +# ## Why this exists +# +# `swift test` proves the ROUTER — which instructions match, which don't. It cannot +# prove the pipeline ever REACHES the router: the refine path lives on AppState, +# which the core test target doesn't compile. Every wiring bug this project has hit +# (see the wiring-review lessons) passed its unit tests while the live gate was dead. +# +# So this drives the shipping binary. `AppMain.startRefineRouteProbe` calls the SAME +# `PluginHost.routeVoiceCommand` that `AppState.deliverFinalText` calls the moment a +# mid-dictation refine finalizes, with the same (instruction, content) pair — only +# the source of those two strings differs (env vars instead of the mic). +# +# ## Cases +# +# case1 — selection is the material ("create a meme based on that" + content) +# case2 — the spoken remainder is ("create a meme expanding brain: …", no content) +# nearmiss — "create a memo about …" must NOT route; it logs a normal refine +# +# Usage: scripts/meme-voice-command-proof.sh [case1|case2|nearmiss|all] +# +# Requires: PLUGINS=1 ./build.sh, and the meme plugin ENABLED in Settings → Plugins. +# Runs the WORKTREE binary only — it never touches an installed /Applications copy. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUT="${OUT_DIR:-$ROOT/build/voice-proof}" +SETTLE="${OPENWHISP_MEME_PROBE_SECONDS:-75}" +DELAY="${OPENWHISP_MEME_PROBE_DELAY:-6}" + +[ -x "$ROOT/build/OpenWhisp" ] || { + echo "✗ no binary at $ROOT/build/OpenWhisp — run: PLUGINS=1 ./build.sh" >&2; exit 1; } +mkdir -p "$OUT" + +# The probe MUST run from a bundle, not the bare binary. +# +# `build.sh` emits a loose executable, and AppKit never services the main run loop +# for one: `applicationDidFinishLaunching` completes but every `asyncAfter` — the v9 +# probe's included — is left queued forever, so the run looks like a silent hang. A +# minimal hand-assembled bundle (binary + Info.plist + Resources, ad-hoc signed) is +# enough; none of the third_party runtimes matter to the trigger layer, which is why +# this doesn't need the full `package.sh`. +APP="$ROOT/build/ProbeApp.app" +rm -rf "$APP" +mkdir -p "$APP/Contents/MacOS" "$APP/Contents/Resources" +cp "$ROOT/build/OpenWhisp" "$APP/Contents/MacOS/OpenWhisp" +cp "$ROOT/OpenWhisp/Info.plist" "$APP/Contents/" +[ -d "$ROOT/OpenWhisp/Resources" ] && cp -R "$ROOT/OpenWhisp/Resources/." "$APP/Contents/Resources/" +codesign --force --deep --sign - "$APP" >/dev/null 2>&1 || true +BIN="$APP/Contents/MacOS/OpenWhisp" + +run_case() { + local name="$1" instruction="$2" content="${3:-}" + local log="$OUT/$name.log" + + echo "── $name ─────────────────────────────────────────────" + echo " instruction: \"$instruction\"" + [ -n "$content" ] && echo " content: \"$content\"" + + # The probe fires `delay` seconds after launch, then samples the canvas after + # `SETTLE`. Give the process both plus headroom, then stop it. + local budget=$(( ${DELAY%.*} + ${SETTLE%.*} + 15 )) + + env OPENWHISP_MEME_TRACE=1 \ + OPENWHISP_MEME_PROBE_REFINE="$instruction" \ + ${content:+OPENWHISP_MEME_PROBE_REFINE_CONTENT="$content"} \ + OPENWHISP_MEME_PROBE_DELAY="$DELAY" \ + OPENWHISP_MEME_PROBE_SECONDS="$SETTLE" \ + "$BIN" >"$log" 2>&1 & + local pid=$! + + # Wait for the probe to finish rather than a fixed sleep, so a fast case doesn't + # burn the full budget — but never wait past it. + local waited=0 + while [ "$waited" -lt "$budget" ]; do + grep -q "probe done\|NOT ROUTED" "$log" 2>/dev/null && break + sleep 2; waited=$((waited + 2)) + done + + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + + echo + grep '\[MemeGen\]' "$log" || echo " (no breadcrumbs — is the plugin enabled?)" + echo +} + +WHICH="${1:-all}" +case "$WHICH" in + case1|case2|nearmiss|all) ;; + *) echo "usage: $0 [case1|case2|nearmiss|all]" >&2; exit 2 ;; +esac + +if [ "$WHICH" = case1 ] || [ "$WHICH" = all ]; then + run_case case1 "create a meme based on that" \ + "Our deploy pipeline takes 45 minutes and fails on the last step half the time." +fi +if [ "$WHICH" = case2 ] || [ "$WHICH" = all ]; then + run_case case2 \ + "create a meme expanding brain: typing, dictating, dictating memes, dictating memes by voice" +fi +if [ "$WHICH" = nearmiss ] || [ "$WHICH" = all ]; then + run_case nearmiss "create a memo about the Q3 numbers" \ + "Revenue was up 12 percent." +fi + +echo "Logs: $OUT"