feat: Add Qwen2.5-VL support

README.md

@@ -127,7 +127,12 @@ bash scripts/train.sh
 ```
 
 ## :checkered_flag: Evaluation on GUI Grounding Benchmarks
-For evaluation on ScreenSpot and ScreenSpot-v2, you can directly run the scripts under the `scripts/` folder like `python eval/screenSpot.py` or `python eval/screenSpot_v2.py`.
+For evaluation on ScreenSpot and ScreenSpot-v2, you can directly run the scripts under the `scripts/` folder like:
+```bash
+# model_type: qwen2vl or qwen25vl
+python eval/screenSpot.py --model_type qwen2vl --model_name_or_path microsoft/GUI-Actor-2B-Qwen2-VL
+python eval/screenSpot_v2.py --model_type qwen25vl --model_name_or_path microsoft/GUI-Actor-3B-Qwen2.5-VL
+```
 
 For evaluation on ScreenSpot-Pro, you first need to download the data from [here](https://huggingface.co/datasets/likaixin/ScreenSpot-Pro), then run the following command:
 ```bash

eval/screenSpot.py

@@ -9,6 +9,7 @@
 
 from gui_actor.constants import chat_template
 from gui_actor.modeling import Qwen2VLForConditionalGenerationWithPointer
+from gui_actor.modeling_qwen25vl import Qwen2_5_VLForConditionalGenerationWithPointer
 from gui_actor.inference import inference, ForceFollowTokensLogitsProcessor
 from gui_actor.utils import do_boxes_overlap
 from gui_actor.constants import DEFAULT_POINTER_PAD_TOKEN, DEFAULT_POINTER_END_TOKEN, grounding_system_message
@@ -27,19 +28,32 @@ def normalize_bbox(bbox_x1y1x2y2, img_width, img_height):
         y2 = y2 / img_height
         return x1, y1, x2, y2
 
-def evaluate(model_name_or_path, use_placeholder, topk):
+def evaluate(model_name_or_path, use_placeholder, topk, model_type="qwen2vl"):
     # initialize model
     data_processor = Qwen2VLProcessor.from_pretrained(model_name_or_path)
     tokenizer = data_processor.tokenizer
     for k, v in tokenizer.added_tokens_encoder.items():
         print(v, k)
 
-    model = Qwen2VLForConditionalGenerationWithPointer.from_pretrained(
-        model_name_or_path,
-        torch_dtype=torch.bfloat16,
-        device_map="cuda:0",
-        attn_implementation="flash_attention_2"
-    ).eval()
+    if model_type == "qwen2vl":
+        print("Loading Qwen2-VL")
+        model = Qwen2VLForConditionalGenerationWithPointer.from_pretrained(
+            model_name_or_path,
+            torch_dtype=torch.bfloat16,
+            device_map="cuda:0",
+            attn_implementation="flash_attention_2"
+        ).eval()
+    elif model_type == "qwen25vl":
+        print("Loading Qwen2.5-VL")
+        model = Qwen2_5_VLForConditionalGenerationWithPointer.from_pretrained(
+            model_name_or_path,
+            torch_dtype=torch.bfloat16,
+            device_map="cuda:0",
+            attn_implementation="flash_attention_2"
+        ).eval()
+    else:
+        print("undefine model type")
+        input()
     print(f"Loaded model from {model_name_or_path}")
 
     logits_processor_pointer = ForceFollowTokensLogitsProcessor(
@@ -61,6 +75,12 @@ def evaluate(model_name_or_path, use_placeholder, topk):
             "forum": "web"
         }
 
+    # special system message for our different models
+    if model_type == "qwen2vl":
+        grounding_system_message = "You are a GUI agent. You are given a task and a screenshot of the screen. You need to perform a series of pyautogui actions to complete the task."
+    elif model_type == "qwen25vl":
+        grounding_system_message =  "You are a GUI agent. Given a screenshot of the current GUI and a human instruction, your task is to locate the screen element that corresponds to the instruction. You should output a PyAutoGUI action that performs a click on the correct position. To indicate the click location, we will use some special tokens, which is used to refer to a visual patch later. For example, you can output: pyautogui.click(<your_special_token_here>)."
+
     results = []
     for i, example in tqdm(enumerate(dataset), total=len(dataset)):
         ele = {
@@ -248,6 +268,7 @@ def format_cell(cell):
 """
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
+    parser.add_argument("--model_type", type=str, default="qwen2vl", choices=["qwen2vl", "qwen25vl"])
     parser.add_argument("--model_name_or_path", type=str, default="microsoft/GUI-Actor-2B-Qwen2-VL")
     parser.add_argument("--save_path", type=str, default="./")
     parser.add_argument('--topk', type=int, default=3, help='Topk')
@@ -271,7 +292,7 @@ def format_cell(cell):
             results = json.load(f)
     else:
         print(f"Evaluating {args.model_name_or_path}...")
-        results = evaluate(args.model_name_or_path, args.use_placeholder, args.topk)
+        results = evaluate(args.model_name_or_path, args.use_placeholder, args.topk, args.model_type)
         with open(pred_path, "w") as f:
             json.dump(results, f)
         print(f"Saved {len(results)} predictions to {pred_path}")

eval/screenSpot_pro.py

@@ -9,6 +9,7 @@
 from PIL import Image
 from gui_actor.constants import chat_template
 from gui_actor.modeling import Qwen2VLForConditionalGenerationWithPointer
+from gui_actor.modeling_qwen25vl import Qwen2_5_VLForConditionalGenerationWithPointer
 from gui_actor.inference import inference, ForceFollowTokensLogitsProcessor
 from gui_actor.utils import do_boxes_overlap
 from gui_actor.constants import DEFAULT_POINTER_PAD_TOKEN, DEFAULT_POINTER_END_TOKEN, grounding_system_message
@@ -27,19 +28,32 @@ def normalize_bbox(bbox_x1y1x2y2, img_width, img_height):
         y2 = y2 / img_height
         return x1, y1, x2, y2
 
-def evaluate(model_name_or_path, data_fn, image_dir, use_placeholder, topk, resize_to_pixels=None):
+def evaluate(model_name_or_path, data_fn, image_dir, use_placeholder, topk, resize_to_pixels=None, model_type="qwen2vl"):
     # initialize model
     data_processor = Qwen2VLProcessor.from_pretrained(model_name_or_path)
     tokenizer = data_processor.tokenizer
     for k, v in tokenizer.added_tokens_encoder.items():
         print(v, k)
 
-    model = Qwen2VLForConditionalGenerationWithPointer.from_pretrained(
-        model_name_or_path,
-        torch_dtype=torch.bfloat16,
-        device_map="cuda:0",
-        attn_implementation="flash_attention_2"
-    ).eval()
+    if model_type == "qwen2vl":
+        print("Loading Qwen2-VL")
+        model = Qwen2VLForConditionalGenerationWithPointer.from_pretrained(
+            model_name_or_path,
+            torch_dtype=torch.bfloat16,
+            device_map="cuda:0",
+            attn_implementation="flash_attention_2"
+        ).eval()
+    elif model_type == "qwen25vl":
+        print("Loading Qwen2.5-VL")
+        model = Qwen2_5_VLForConditionalGenerationWithPointer.from_pretrained(
+            model_name_or_path,
+            torch_dtype=torch.bfloat16,
+            device_map="cuda:0",
+            attn_implementation="flash_attention_2"
+        ).eval()
+    else:
+        print("undefine model type")
+        input()
     print(f"Loaded model from {model_name_or_path}")
 
     logits_processor_pointer = ForceFollowTokensLogitsProcessor(
@@ -54,6 +68,12 @@ def evaluate(model_name_or_path, data_fn, image_dir, use_placeholder, topk, resi
         data = json.load(f)
     print(f"Loaded {len(data)} examples from {data_fn}")
 
+    # special system message for our different models
+    if model_type == "qwen2vl":
+        grounding_system_message = "You are a GUI agent. You are given a task and a screenshot of the screen. You need to perform a series of pyautogui actions to complete the task."
+    elif model_type == "qwen25vl":
+        grounding_system_message =  "You are a GUI agent. Given a screenshot of the current GUI and a human instruction, your task is to locate the screen element that corresponds to the instruction. You should output a PyAutoGUI action that performs a click on the correct position. To indicate the click location, we will use some special tokens, which is used to refer to a visual patch later. For example, you can output: pyautogui.click(<your_special_token_here>)."
+
     results = []
     for i, example in tqdm(enumerate(data), total=len(data)):
         ele = {
@@ -253,6 +273,7 @@ def format_cell(cell):
 """
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
+    parser.add_argument("--model_type", type=str, default="qwen2vl", choices=["qwen2vl", "qwen25vl"])
     parser.add_argument("--model_name_or_path", type=str, default="microsoft/GUI-Actor-2B-Qwen2-VL")
     parser.add_argument("--save_path", type=str, default="./")
     parser.add_argument("--data_path", type=str, default="/mnt/data/ScreenSpot-Pro")
@@ -281,7 +302,7 @@ def format_cell(cell):
             results = json.load(f)
     else:
         print(f"Evaluating {args.model_name_or_path}...")
-        results = evaluate(args.model_name_or_path, data_fn, image_dir, args.use_placeholder, args.topk, resize_to_pixels)
+        results = evaluate(args.model_name_or_path, data_fn, image_dir, args.use_placeholder, args.topk, resize_to_pixels, args.model_type)
         with open(pred_path, "w") as f:
             json.dump(results, f)
         print(f"Saved {len(results)} predictions to {pred_path}")

eval/screenSpot_v2.py

@@ -9,6 +9,7 @@
 
 from gui_actor.constants import chat_template
 from gui_actor.modeling import Qwen2VLForConditionalGenerationWithPointer
+from gui_actor.modeling_qwen25vl import Qwen2_5_VLForConditionalGenerationWithPointer
 from gui_actor.inference import inference, ForceFollowTokensLogitsProcessor
 from gui_actor.utils import do_boxes_overlap
 from gui_actor.constants import DEFAULT_POINTER_PAD_TOKEN, DEFAULT_POINTER_END_TOKEN, grounding_system_message
@@ -27,19 +28,32 @@ def normalize_bbox(bbox_x1y1x2y2, img_width, img_height):
         y2 = y2 / img_height
         return x1, y1, x2, y2
 
-def evaluate(model_name_or_path, use_placeholder, topk):
+def evaluate(model_name_or_path, use_placeholder, topk, model_type="qwen2vl"):
     # initialize model
     data_processor = Qwen2VLProcessor.from_pretrained(model_name_or_path)
     tokenizer = data_processor.tokenizer
     for k, v in tokenizer.added_tokens_encoder.items():
         print(v, k)
 
-    model = Qwen2VLForConditionalGenerationWithPointer.from_pretrained(
-        model_name_or_path,
-        torch_dtype=torch.bfloat16,
-        device_map="cuda:0",
-        attn_implementation="flash_attention_2"
-    ).eval()
+    if model_type == "qwen2vl":
+        print("Loading Qwen2-VL")
+        model = Qwen2VLForConditionalGenerationWithPointer.from_pretrained(
+            model_name_or_path,
+            torch_dtype=torch.bfloat16,
+            device_map="cuda:0",
+            attn_implementation="flash_attention_2"
+        ).eval()
+    elif model_type == "qwen25vl":
+        print("Loading Qwen2.5-VL")
+        model = Qwen2_5_VLForConditionalGenerationWithPointer.from_pretrained(
+            model_name_or_path,
+            torch_dtype=torch.bfloat16,
+            device_map="cuda:0",
+            attn_implementation="flash_attention_2"
+        ).eval()
+    else:
+        print("undefine model type")
+        input()
     print(f"Loaded model from {model_name_or_path}")
 
     logits_processor_pointer = ForceFollowTokensLogitsProcessor(
@@ -61,6 +75,12 @@ def evaluate(model_name_or_path, use_placeholder, topk):
             "forum": "web"
         }
 
+    # special system message for our different models
+    if model_type == "qwen2vl":
+        grounding_system_message = "You are a GUI agent. You are given a task and a screenshot of the screen. You need to perform a series of pyautogui actions to complete the task."
+    elif model_type == "qwen25vl":
+        grounding_system_message =  "You are a GUI agent. Given a screenshot of the current GUI and a human instruction, your task is to locate the screen element that corresponds to the instruction. You should output a PyAutoGUI action that performs a click on the correct position. To indicate the click location, we will use some special tokens, which is used to refer to a visual patch later. For example, you can output: pyautogui.click(<your_special_token_here>)."
+
     results = []
     for i, example in tqdm(enumerate(dataset), total=len(dataset)):
         ele = {
@@ -248,6 +268,7 @@ def format_cell(cell):
 """
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
+    parser.add_argument("--model_type", type=str, default="qwen2vl", choices=["qwen2vl", "qwen25vl"])
     parser.add_argument("--model_name_or_path", type=str, default="microsoft/GUI-Actor-2B-Qwen2-VL")
     parser.add_argument("--save_path", type=str, default="./")
     parser.add_argument('--topk', type=int, default=3, help='Topk')
@@ -271,7 +292,7 @@ def format_cell(cell):
             results = json.load(f)
     else:
         print(f"Evaluating {args.model_name_or_path}...")
-        results = evaluate(args.model_name_or_path, args.use_placeholder, args.topk)
+        results = evaluate(args.model_name_or_path, args.use_placeholder, args.topk, args.model_type)
         with open(pred_path, "w") as f:
             json.dump(results, f)
         print(f"Saved {len(results)} predictions to {pred_path}")

pyproject.toml

@@ -14,9 +14,10 @@ dependencies = [
     "accelerate==1.1.1",
     "qwen-vl-utils==0.0.8",
     "deepspeed==0.16.0",
-    "transformers==4.50.0",
+    "transformers==4.51.3",
     "flash-attn",
-    "wandb==0.18.3"
+    "wandb==0.18.3",
+    "datasets>=2.18.0"
 ]
 requires-python = ">=3.10,<3.13"
 readme = "README.md"

scripts/train.sh

@@ -8,6 +8,7 @@ torchrun --nproc_per_node=4 train.py \
   --data_path data/data_config.yaml \
   --image_folder "" \
   --model_name_or_path ${llm_model} \
+  --model_type "qwen2vl" \
   --group_by_modality_length True \
   --bf16 True \
   --output_dir ${output_dir} \

src/gui_actor/modeling.py

@@ -166,7 +166,7 @@ def forward(self,
         if inputs_embeds is None:
             inputs_embeds = self.model.embed_tokens(input_ids) # shape: (batch_size, seq_len, d_model)
             if pixel_values is not None:
-                pixel_values = pixel_values.type(self.visual.get_dtype())
+                pixel_values = pixel_values.type(self.visual.dtype)
                 image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw)
                 n_image_tokens = (input_ids == self.config.image_token_id).sum().item()
                 n_image_features = image_embeds.shape[0]
@@ -184,7 +184,7 @@ def forward(self,
                 inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)
 
             if pixel_values_videos is not None:
-                pixel_values_videos = pixel_values_videos.type(self.visual.get_dtype())
+                pixel_values_videos = pixel_values_videos.type(self.visual.dtype)
                 video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw)
                 n_video_tokens = (input_ids == self.config.video_token_id).sum().item()
                 n_video_features = video_embeds.shape[0]