update

.gitignore

@@ -0,0 +1 @@
+.idea

app.py

@@ -11,12 +11,11 @@ from threading import Thread
 
 import gradio as gr
 import torch
-import spaces
 from qwen_vl_utils import process_vision_info
 from transformers import AutoProcessor, Qwen2_5_VLForConditionalGeneration, TextIteratorStreamer
 
-# DEFAULT_CKPT_PATH = 'farrell236/test_model'
-DEFAULT_CKPT_PATH = 'Qwen/Qwen2.5-VL-32B-Instruct'
+DEFAULT_CKPT_PATH = 'farrell236/test_model'
+# DEFAULT_CKPT_PATH = '/scratch/llm-weights/Qwen/Qwen2.5-VL-7B-Instruct'
 AUTH_TOKEN = os.environ.get("HF_spaces")
 
 def _get_args():
@@ -148,10 +147,11 @@ def _transform_messages(original_messages):
 
 def _launch_demo(args, model, processor):
 
-    # @spaces.GPU
     def call_local_model(model, processor, messages,
-                         max_tokens=1024, temperature=0.6,
-                         top_p=0.9, top_k=50,
+                         max_tokens=1024,
+                         temperature=0.6,
+                         top_p=0.9,
+                         top_k=50,
                          repetition_penalty=1.2):
 
         messages = _transform_messages(messages)
@@ -164,8 +164,8 @@ def _launch_demo(args, model, processor):
         tokenizer = processor.tokenizer
         streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
 
-        gen_kwargs = {'max_new_tokens': max_tokens,
-                      'streamer': streamer,
+        gen_kwargs = {'streamer': streamer,
+                      'max_new_tokens': max_tokens,
                       'temperature': temperature,
                       'top_p': top_p,
                       'top_k': top_k,
@@ -209,7 +209,12 @@ def _launch_demo(args, model, processor):
                     content = []
             messages.pop()
 
-            for response in call_local_model(model, processor, messages):
+            for response in call_local_model(model, processor, messages,
+                                             max_tokens=max_tokens,
+                                             temperature=temperature,
+                                             top_p=top_p,
+                                             top_k=top_k,
+                                             repetition_penalty=repetition_penalty):
                 _chatbot[-1] = (_parse_text(chat_query), _remove_image_special(_parse_text(response)))
 
                 yield _chatbot
@@ -281,7 +286,7 @@ def _launch_demo(args, model, processor):
         chatbot = gr.Chatbot(label='Qwen2.5-VL', elem_classes='control-height', height=500)
 
         with gr.Accordion("Generation Parameters", open=False):
-            max_tokens = gr.Slider(64, 4096, value=512, step=64, label="Max Tokens")
+            max_tokens = gr.Slider(64, 4096, value=1024, step=64, label="Max Tokens")
             temperature = gr.Slider(0.0, 2.0, value=0.6, step=0.1, label="Temperature")
             top_p = gr.Slider(0.0, 1.0, value=0.9, step=0.05, label="Top-p (nucleus sampling)")
             top_k = gr.Slider(0, 100, value=50, step=1, label="Top-k")
@@ -299,8 +304,8 @@ def _launch_demo(args, model, processor):
         submit_btn.click(add_text,
                          [chatbot, task_history, query],
                          [chatbot, task_history]).then(predict,
-                                                       [chatbot, task_history, max_tokens,
-                                                        temperature, top_p, top_k, repetition_penalty],
+                                                       [chatbot, task_history,
+                                                        max_tokens, temperature, top_p, top_k, repetition_penalty],
                                                        [chatbot], show_progress=True)
         submit_btn.click(reset_user_input, [], [query])
         empty_bin.click(reset_state, [chatbot, task_history], [chatbot], show_progress=True)