From 7937b47e8082f305caa34a0baef6f3664d874b87 Mon Sep 17 00:00:00 2001 From: co-seven Date: Mon, 6 Jul 2026 13:16:44 +0000 Subject: [PATCH] fix(server): fix SMT multimodal crash on hybrid-recurrent models + drop temp-file round-trip Two related fixes for the SMT vision multimodal server path, plus a version bump. 1. Prompt-cache reuse crash on hybrid/recurrent multimodal models (e.g. Qwen3.5VL). A multi-turn chat answered turn 1 but aborted on turn 2 with: common.cpp:1495: failed to remove sequence N with p0=M, p1=-1 Root cause: the text prompt-cache reuse path (LCP-similarity slot selection + get_common_prefix) was applied to multimodal requests on hybrid-recurrent models. Their llama_memory_recurrent::seq_rm() cannot partially remove a tail range outside the bounded per-token snapshot window (n_rs_seq), so the reuse-computed KV position makes seq_rm return false -> GGML_ABORT. Pure-attention multimodal models (e.g. LLaVA-style fastvlm) are unaffected. Gate prompt-cache reuse off precisely for multimodal AND hybrid/recurrent models (prompt_cache_reuse_unsafe()), so those prompts are fully re-processed while pure-attention multimodal models keep their cache reuse. 2. Remove the pointless temp-file round-trip in SMT vision encode. The preprocessed float32 tensor was written to /tmp/llama-server-smt-XXXXXX and immediately read back into memory by the ONNX engine. Feed the in-memory blob directly (encode_image_mem / set_input_tensor_from_memory). This removes a per-image disk write+read and fixes intermittent "failed to write temp file" failures on boards whose root (/tmp) partition is full. The file-path API is kept as a thin wrapper for other callers; the audio path is unchanged. 3. Bump VERSION_NUMBER 0.1.5 -> 0.1.6. --- VERSION_NUMBER | 2 +- tools/mtmd/smt-vision-wrapper.cpp | 64 ++++++++++++++++++++++++++---- tools/mtmd/smt-vision-wrapper.h | 3 ++ tools/server/server-context.cpp | 9 ++++- tools/server/server-smt-vision.cpp | 10 +---- 5 files changed, 69 insertions(+), 19 deletions(-) diff --git a/VERSION_NUMBER b/VERSION_NUMBER index 9faa1b7a7339..c946ee6160c2 100644 --- a/VERSION_NUMBER +++ b/VERSION_NUMBER @@ -1 +1 @@ -0.1.5 +0.1.6 diff --git a/tools/mtmd/smt-vision-wrapper.cpp b/tools/mtmd/smt-vision-wrapper.cpp index f0f866e62491..54b4e695e569 100644 --- a/tools/mtmd/smt-vision-wrapper.cpp +++ b/tools/mtmd/smt-vision-wrapper.cpp @@ -5,10 +5,12 @@ #include "ggml-profile.h" #include "onnxruntime_cxx_api.h" +#include + #include #include #include -#include +#include #include #include #include @@ -164,10 +166,10 @@ class smt_ort_vision_engine { return session_; } - Ort::Value & set_input_tensor(const std::string & input_binary_path) { - auto type_info = session_.GetInputTypeInfo(0); - auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - const std::vector input_shape = tensor_info.GetShape(); + void query_input_layout(std::vector & input_shape, size_t & expected_bytes) { + auto type_info = session_.GetInputTypeInfo(0); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + input_shape = tensor_info.GetShape(); if (tensor_info.GetElementType() != ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) { throw std::runtime_error("SMT vision expects float32 input tensor"); @@ -184,20 +186,48 @@ class smt_ort_vision_engine { input_size *= static_cast(dim); } - input_data_.resize(input_size); + if (input_size > std::numeric_limits::max() / sizeof(float)) { + throw std::runtime_error("SMT vision input tensor is too large"); + } + + expected_bytes = input_size * sizeof(float); + } + + Ort::Value & set_input_tensor_from_memory(const uint8_t * data, size_t len) { + std::vector input_shape; + size_t expected_bytes = 0; + query_input_layout(input_shape, expected_bytes); + + if (data == nullptr || len != expected_bytes) { + throw std::runtime_error("SMT vision input size mismatch: expected " + std::to_string(expected_bytes) + + ", actual " + std::to_string(len)); + } + + input_data_.resize(expected_bytes / sizeof(float)); + std::memcpy(input_data_.data(), data, expected_bytes); + + input_tensor_ = make_tensor_f32(input_shape, input_data_); + return input_tensor_; + } + + Ort::Value & set_input_tensor(const std::string & input_binary_path) { + std::vector input_shape; + size_t expected_bytes = 0; + query_input_layout(input_shape, expected_bytes); + std::ifstream file(input_binary_path, std::ios::binary | std::ios::ate); if (!file.is_open()) { throw std::runtime_error("failed to open SMT vision input binary: " + input_binary_path); } - const std::streamoff actual_bytes = file.tellg(); - const size_t expected_bytes = input_data_.size() * sizeof(float); + const std::streamoff actual_bytes = file.tellg(); if (actual_bytes < 0 || static_cast(actual_bytes) != expected_bytes) { throw std::runtime_error("SMT vision input binary size mismatch: expected " + std::to_string(expected_bytes) + ", actual " + std::to_string(actual_bytes < 0 ? 0 : static_cast(actual_bytes))); } + input_data_.resize(expected_bytes / sizeof(float)); file.seekg(0, std::ios::beg); file.read(reinterpret_cast(input_data_.data()), static_cast(expected_bytes)); if (!file) { @@ -829,6 +859,24 @@ std::vector smt_vision_context::encode_image(const std::string & binary_p return result; } +std::vector smt_vision_context::encode_image_mem(const uint8_t * data, size_t len) { + auto & d = *pimpl_; + + ggml_trace_log_begin("encode_image", "Vision", NULL); + + ggml_trace_log_begin("set_input_tensor", "Vision", NULL); + Ort::Value & input_tensor = d.vision_engine->set_input_tensor_from_memory(data, len); + ggml_trace_log_end("set_input_tensor", "Vision", NULL); + + ggml_trace_log_begin("vision_session_run", "Vision", NULL); + std::vector result = d.vision_engine->run_session(input_tensor); + ggml_trace_log_end("vision_session_run", "Vision", NULL); + + ggml_trace_log_end("encode_image", "Vision", NULL); + ggml_profile_flush_tls(); + return result; +} + int64_t smt_vision_context::hidden_size() const { return pimpl_->config.hidden_size; } diff --git a/tools/mtmd/smt-vision-wrapper.h b/tools/mtmd/smt-vision-wrapper.h index 1afd4e7f6e1b..925b67893567 100644 --- a/tools/mtmd/smt-vision-wrapper.h +++ b/tools/mtmd/smt-vision-wrapper.h @@ -5,6 +5,7 @@ #include "smt-vision-preprocess.h" +#include #include #include #include @@ -22,6 +23,8 @@ struct smt_vision_context { // Returns image embedding vector (n_tokens * hidden_size floats) std::vector encode_image(const std::string & binary_path); + std::vector encode_image_mem(const uint8_t * data, size_t len); + // Get the hidden size (embedding dimension) int64_t hidden_size() const; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index b5eaab05a327..41221278ee58 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -752,6 +752,11 @@ struct server_context_impl { bool has_multimodal() const { return mctx != nullptr || server_smt_vision_supports_prompt_embeddings(smt_ctx); } + bool prompt_cache_reuse_unsafe() const { + return has_multimodal() && model_tgt != nullptr && + (llama_model_is_recurrent(model_tgt) || llama_model_is_hybrid(model_tgt)); + } + const char * vision_backend_name() const { switch (vision_backend) { case SERVER_VISION_BACKEND_MTMD: @@ -1444,7 +1449,7 @@ struct server_context_impl { bool update_cache = false; // find the slot that has at least n% prompt similarity - if (ret == nullptr && slot_prompt_similarity != 0.0f) { + if (ret == nullptr && slot_prompt_similarity != 0.0f && !prompt_cache_reuse_unsafe()) { float sim_best = 0; for (server_slot & slot : slots) { @@ -2900,7 +2905,7 @@ struct server_context_impl { continue; } - if (slot.task->params.cache_prompt) { + if (slot.task->params.cache_prompt && !prompt_cache_reuse_unsafe()) { // reuse any previously computed tokens that are common with the new prompt n_past = slot.prompt.tokens.get_common_prefix(input_tokens); diff --git a/tools/server/server-smt-vision.cpp b/tools/server/server-smt-vision.cpp index 8c7ab49d1dd8..0cfbcce9dc74 100644 --- a/tools/server/server-smt-vision.cpp +++ b/tools/server/server-smt-vision.cpp @@ -1330,18 +1330,12 @@ server_smt_image_chunk server_smt_vision_encode_image_bin(server_smt_vision_cont smt_input = std::move(preproc.tensor_bytes); } - const std::string tmp_file = write_temp_bin_file(smt_input); - server_smt_image_chunk out; out.type = server_smt_media_type::image; - try { + { const int64_t t0 = ggml_time_us(); - out.embd = ctx->smt_vision->encode_image(tmp_file); + out.embd = ctx->smt_vision->encode_image_mem(smt_input.data(), smt_input.size()); out.t_encode_ms = (ggml_time_us() - t0) / 1e3; - std::remove(tmp_file.c_str()); - } catch (...) { - std::remove(tmp_file.c_str()); - throw; } if (ctx->hidden_size <= 0 || out.embd.empty() || out.embd.size() % (size_t) ctx->hidden_size != 0) {