WIP fix image loading

app.py

@@ -1,13 +1,15 @@
 import gradio as gr
 import spaces
-from transformers import AutoModelForCausalLM, AutoProcessor
+from transformers import Qwen2VLForConditionalGeneration, AutoTokenizer, AutoProcessor
+from qwen_vl_utils import process_vision_info
 import torch
+import base64
 from PIL import Image
-import subprocess
+from io import BytesIO
 
 
 models = {
-    "Qwen/Qwen2-VL-7B-Instruct": AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-VL-7B-Instruct", torch_dtype="auto", device_map="auto")
+    "Qwen/Qwen2-VL-7B-Instruct": Qwen2VLForConditionalGeneration.from_pretrained("Qwen/Qwen2-VL-7B-Instruct") #, torch_dtype="auto", device_map="auto")
 }
 
 processors = {
@@ -17,6 +19,13 @@ processors = {
 DESCRIPTION = "# Qwen2-VL Object Localization Demo"
 
 
+def image_to_base64(image):
+    buffered = BytesIO()
+    image.save(buffered, format="PNG")  # Save the image in memory as PNG
+    img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")  # Encode image to base64
+    return img_str
+
+
 @spaces.GPU
 def run_example(image, text_input, model_id="Qwen/Qwen2-VL-7B-Instruct"):
     model = models[model_id].eval().cuda()
@@ -26,7 +35,7 @@ def run_example(image, text_input, model_id="Qwen/Qwen2-VL-7B-Instruct"):
         {
             "role": "user",
             "content": [
-                {"type": "image", "image": image},
+                {"type": "image", "image": f"data:image;base64,{image_to_base64(image)}"},
                 {"type": "text", "text": f"Give a bounding box for {text_input}"},
             ],
         }
@@ -67,7 +76,7 @@ with gr.Blocks(css=css) as demo:
     with gr.Tab(label="Qwen2-VL Input"):
         with gr.Row():
             with gr.Column():
-                input_img = gr.Image(label="Input Picture")
+                input_img = gr.Image(label="Input Picture", type="pil")
                 model_selector = gr.Dropdown(choices=list(models.keys()), label="Model", value="Qwen/Qwen2-VL-7B-Instruct")
                 text_input = gr.Textbox(label="Description of Localization Target")
                 submit_btn = gr.Button(value="Submit")

requirements.txt

@@ -3,6 +3,6 @@ Pillow==10.3.0
 Requests==2.31.0
 torch
 torchvision
-transformers
+git+https://github.com/huggingface/transformers.git@main
 accelerate==0.30.0
 qwen-vl-utils