bug fix

Author: ext.xingsilan1

xllm/models/vlm/glm4v.h

@@ -648,9 +648,9 @@ class Glm4VisionTransformerImpl : public torch::nn::Module {
        const ModelInputParams& input_params) {
     LOG(INFO) << " Glm4VisionTransformerImpl forward beging ";
     hidden_states = patch_embed_(hidden_states);
-    // at_npu::native::custom_ops::npu_rms_norm()
+    LOG(INFO) << " Glm4VisionTransformerImpl patch_embed_ beging ";
     hidden_states = post_conv_layernorm_(hidden_states);
-    // hidden_states = at_npu::native::custom_ops::npu_rms_norm(hidden_states);
+    LOG(INFO) << " Glm4VisionTransformerImpl post_conv_layernorm_ beging ";
 
     auto [rotary_pos_emb, image_type_ids] = rot_pos_emb(grid_thw);
     auto emb = torch::cat({rotary_pos_emb, rotary_pos_emb}, -1);
@@ -665,9 +665,7 @@ class Glm4VisionTransformerImpl : public torch::nn::Module {
     auto repeats = grid_t.squeeze(1);
     auto repeated = torch::repeat_interleave(h_times_w, repeats, 0);
     c10::optional<torch::ScalarType> cumsum_dtype;
-    // if (torch::jit::is_tracing()) {
-    //     cumsum_dtype = grid_thw.scalar_type();
-    // } else {
+
     cumsum_dtype = torch::kInt32;
     auto cu_seqlens = torch::cumsum(repeated, 0, cumsum_dtype);
     namespace F = torch::nn::functional;

xllm/models/vlm/glm4v_moe.h

@@ -42,6 +42,7 @@ class Glm4vMoeForConditionalGenerationImpl : public torch::nn::Module {
   Glm4vMoeForConditionalGenerationImpl(const ModelContext& context)
       : model_args_(context.get_model_args()),
         options_(context.get_tensor_options()) {
+    std::cout << "----------------Glm4vMoeForConditionalGenerationImpl init begin ----------------- " << std::endl;
     visual_ = register_module("visual", Glm4VisionTransformer(context));
 
     language_model_ =
@@ -53,25 +54,28 @@ class Glm4vMoeForConditionalGenerationImpl : public torch::nn::Module {
       const std::optional<Glm4VImageInputs>& image_input,
       const std::optional<Glm4VVideoInputs>& video_input,
       const ModelInputParams& input_params) {
+    // visual
+    LOG(INFO) << " Glm4vMoeForConditionalGenerationImpl forward get_input_embeddings  visual_ begin ";
+    torch::Tensor pixel = image_input->pixel_values.to(options_);
+    LOG(INFO) << " Glm4vMoeForConditionalGenerationImpl forward get_input_embeddings  pixel aft ";
+    auto image_embeds =
+        visual_(pixel,
+                image_input->image_grid_thw,
+                input_params);
+    LOG(INFO) << " Glm4vMoeForConditionalGenerationImpl forward get_input_embeddings  visual_ end ";
     auto inputs_embeds = language_model_->get_input_embeddings(input_ids);
-    if (image_input) {
-      // visual
-      auto image_embeds =
-          visual_(image_input->pixel_values.to(options_),
-                  image_input->image_grid_thw,
-                  input_params);
-      // merge
-      auto is_multimodal = torch::isin(input_ids,
-      model_args_.image_token_id()); input_params.visual_pos_masks =
-      is_multimodal; inputs_embeds.index_put_({is_multimodal}, image_embeds);
-    }
+    // merge
+    auto is_multimodal = torch::isin(input_ids,
+    model_args_.image_token_id()); input_params.visual_pos_masks =
+    is_multimodal; inputs_embeds.index_put_({is_multimodal}, image_embeds);
     return inputs_embeds;
   }
 
   torch::Tensor forward(const torch::Tensor& tokens,
                         const torch::Tensor& positions,
                         std::vector<KVCache>& kv_caches,
                         const ModelInputParams& input_params) {
+    std::cout << "----------------Glm4vMoeForConditionalGenerationImpl beging ----------------- " << std::endl;
     LOG(INFO) << " Glm4vMoeForConditionalGenerationImpl beging ";
     torch::NoGradGuard no_grad;
     const auto& mm_data = input_params.mm_data;
@@ -87,6 +91,8 @@ class Glm4vMoeForConditionalGenerationImpl : public torch::nn::Module {
     LOG(INFO) << " Glm4vMoeForConditionalGenerationImpl forward Glm4VImageInputs  beging ";
     if (pixel_values.defined() && image_grid_thw.defined())
       image_inputs = Glm4VImageInputs{pixel_values, image_grid_thw};
+    else
+      LOG(FATAL) << "Pixel value or image grid thw is null";
 
     LOG(INFO) << " Glm4vMoeForConditionalGenerationImpl forward get_input_embeddings  beging ";