fix(server): fix SMT multimodal crash on hybrid-recurrent models + drop temp-file round-trip Two related fixes for the SMT vision multimodal server path

VERSION_NUMBER

@@ -1 +1 @@
-0.1.5
+0.1.6

tools/mtmd/smt-vision-wrapper.cpp

@@ -5,10 +5,12 @@
 #include "ggml-profile.h"
 #include "onnxruntime_cxx_api.h"
 
+#include <dlfcn.h>
+
 #include <cctype>
 #include <cstdio>
 #include <cstdlib>
-#include <dlfcn.h>
+#include <cstring>
 #include <fstream>
 #include <iostream>
 #include <limits>
@@ -164,10 +166,10 @@ class smt_ort_vision_engine {
         return session_;
     }
 
-    Ort::Value & set_input_tensor(const std::string & input_binary_path) {
-        auto                      type_info   = session_.GetInputTypeInfo(0);
-        auto                      tensor_info = type_info.GetTensorTypeAndShapeInfo();
-        const std::vector<int64_t> input_shape = tensor_info.GetShape();
+    void query_input_layout(std::vector<int64_t> & input_shape, size_t & expected_bytes) {
+        auto type_info   = session_.GetInputTypeInfo(0);
+        auto tensor_info = type_info.GetTensorTypeAndShapeInfo();
+        input_shape      = tensor_info.GetShape();
 
         if (tensor_info.GetElementType() != ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) {
             throw std::runtime_error("SMT vision expects float32 input tensor");
@@ -184,20 +186,48 @@ class smt_ort_vision_engine {
             input_size *= static_cast<size_t>(dim);
         }
 
-        input_data_.resize(input_size);
+        if (input_size > std::numeric_limits<size_t>::max() / sizeof(float)) {
+            throw std::runtime_error("SMT vision input tensor is too large");
+        }
+
+        expected_bytes = input_size * sizeof(float);
+    }
+
+    Ort::Value & set_input_tensor_from_memory(const uint8_t * data, size_t len) {
+        std::vector<int64_t> input_shape;
+        size_t               expected_bytes = 0;
+        query_input_layout(input_shape, expected_bytes);
+
+        if (data == nullptr || len != expected_bytes) {
+            throw std::runtime_error("SMT vision input size mismatch: expected " + std::to_string(expected_bytes) +
+                                     ", actual " + std::to_string(len));
+        }
+
+        input_data_.resize(expected_bytes / sizeof(float));
+        std::memcpy(input_data_.data(), data, expected_bytes);
+
+        input_tensor_ = make_tensor_f32(input_shape, input_data_);
+        return input_tensor_;
+    }
+
+    Ort::Value & set_input_tensor(const std::string & input_binary_path) {
+        std::vector<int64_t> input_shape;
+        size_t               expected_bytes = 0;
+        query_input_layout(input_shape, expected_bytes);
+
         std::ifstream file(input_binary_path, std::ios::binary | std::ios::ate);
         if (!file.is_open()) {
             throw std::runtime_error("failed to open SMT vision input binary: " + input_binary_path);
         }
 
-        const std::streamoff actual_bytes   = file.tellg();
-        const size_t         expected_bytes = input_data_.size() * sizeof(float);
+        const std::streamoff actual_bytes = file.tellg();
         if (actual_bytes < 0 || static_cast<size_t>(actual_bytes) != expected_bytes) {
             throw std::runtime_error("SMT vision input binary size mismatch: expected " +
                                      std::to_string(expected_bytes) + ", actual " +
                                      std::to_string(actual_bytes < 0 ? 0 : static_cast<size_t>(actual_bytes)));
         }
 
+        input_data_.resize(expected_bytes / sizeof(float));
         file.seekg(0, std::ios::beg);
         file.read(reinterpret_cast<char *>(input_data_.data()), static_cast<std::streamsize>(expected_bytes));
         if (!file) {
@@ -829,6 +859,24 @@ std::vector<float> smt_vision_context::encode_image(const std::string & binary_p
     return result;
 }
 
+std::vector<float> smt_vision_context::encode_image_mem(const uint8_t * data, size_t len) {
+    auto & d = *pimpl_;
+
+    ggml_trace_log_begin("encode_image", "Vision", NULL);
+
+    ggml_trace_log_begin("set_input_tensor", "Vision", NULL);
+    Ort::Value & input_tensor = d.vision_engine->set_input_tensor_from_memory(data, len);
+    ggml_trace_log_end("set_input_tensor", "Vision", NULL);
+
+    ggml_trace_log_begin("vision_session_run", "Vision", NULL);
+    std::vector<float> result = d.vision_engine->run_session(input_tensor);
+    ggml_trace_log_end("vision_session_run", "Vision", NULL);
+
+    ggml_trace_log_end("encode_image", "Vision", NULL);
+    ggml_profile_flush_tls();
+    return result;
+}
+
 int64_t smt_vision_context::hidden_size() const {
     return pimpl_->config.hidden_size;
 }

tools/mtmd/smt-vision-wrapper.h

@@ -5,6 +5,7 @@
 
 #include "smt-vision-preprocess.h"
 
+#include <cstddef>
 #include <cstdint>
 #include <memory>
 #include <string>
@@ -22,6 +23,8 @@ struct smt_vision_context {
     // Returns image embedding vector (n_tokens * hidden_size floats)
     std::vector<float> encode_image(const std::string & binary_path);
 
+    std::vector<float> encode_image_mem(const uint8_t * data, size_t len);
+
     // Get the hidden size (embedding dimension)
     int64_t hidden_size() const;
 

tools/server/server-context.cpp

@@ -752,6 +752,11 @@ struct server_context_impl {
 
     bool has_multimodal() const { return mctx != nullptr || server_smt_vision_supports_prompt_embeddings(smt_ctx); }
 
+    bool prompt_cache_reuse_unsafe() const {
+        return has_multimodal() && model_tgt != nullptr &&
+               (llama_model_is_recurrent(model_tgt) || llama_model_is_hybrid(model_tgt));
+    }
+
     const char * vision_backend_name() const {
         switch (vision_backend) {
             case SERVER_VISION_BACKEND_MTMD:
@@ -1444,7 +1449,7 @@ struct server_context_impl {
         bool update_cache = false;
 
         // find the slot that has at least n% prompt similarity
-        if (ret == nullptr && slot_prompt_similarity != 0.0f) {
+        if (ret == nullptr && slot_prompt_similarity != 0.0f && !prompt_cache_reuse_unsafe()) {
             float sim_best = 0;
 
             for (server_slot & slot : slots) {
@@ -2900,7 +2905,7 @@ struct server_context_impl {
                                 continue;
                             }
 
-                            if (slot.task->params.cache_prompt) {
+                            if (slot.task->params.cache_prompt && !prompt_cache_reuse_unsafe()) {
                                 // reuse any previously computed tokens that are common with the new prompt
                                 n_past = slot.prompt.tokens.get_common_prefix(input_tokens);
 

tools/server/server-smt-vision.cpp

@@ -1330,18 +1330,12 @@ server_smt_image_chunk server_smt_vision_encode_image_bin(server_smt_vision_cont
         smt_input = std::move(preproc.tensor_bytes);
     }
 
-    const std::string tmp_file = write_temp_bin_file(smt_input);
-
     server_smt_image_chunk out;
     out.type = server_smt_media_type::image;
-    try {
+    {
         const int64_t t0 = ggml_time_us();
-        out.embd         = ctx->smt_vision->encode_image(tmp_file);
+        out.embd         = ctx->smt_vision->encode_image_mem(smt_input.data(), smt_input.size());
         out.t_encode_ms  = (ggml_time_us() - t0) / 1e3;
-        std::remove(tmp_file.c_str());
-    } catch (...) {
-        std::remove(tmp_file.c_str());
-        throw;
     }
 
     if (ctx->hidden_size <= 0 || out.embd.empty() || out.embd.size() % (size_t) ctx->hidden_size != 0) {