Update app.py

I would like to propose adding support for multimodal capabilities to the Hugging Face API, enabling the use of vision models alongside text models. This would allow developers to pass both image and text data into models that can process both modalities

app.py

@@ -1,8 +1,6 @@
 import os
 import gradio as gr
-
 from openai import OpenAI
-
 from optillm.cot_reflection import cot_reflection
 from optillm.rto import round_trip_optimization
 from optillm.z3_solver import Z3SymPySolverSystem
@@ -11,7 +9,6 @@ from optillm.plansearch import plansearch
 from optillm.leap import leap
 from optillm.reread import re2_approach
 
-
 API_KEY = os.environ.get("OPENROUTER_API_KEY")
 
 def compare_responses(message, model1, approach1, model2, approach2, system_message, max_tokens, temperature, top_p):
@@ -35,11 +32,12 @@ def parse_conversation(messages):
     initial_query = "\n".join(conversation)
     return system_prompt, initial_query
 
-def respond(message, history, model, approach, system_message, max_tokens, temperature, top_p):
+def respond(message, history, model, approach, system_message, max_tokens, temperature, top_p, image=None):
     try:
         client = OpenAI(api_key=API_KEY, base_url="https://openrouter.ai/api/v1")
         messages = [{"role": "system", "content": system_message}]
         
+        # Add history if available
         for val in history:
             if val[0]:
                 messages.append({"role": "user", "content": val[0]})
@@ -49,21 +47,29 @@ def respond(message, history, model, approach, system_message, max_tokens, tempe
         messages.append({"role": "user", "content": message})
         
         if approach == "none":
+            # Prepare the API request data
+            data = {
+                "model": model,
+                "messages": messages,
+                "max_tokens": max_tokens,
+                "temperature": temperature,
+                "top_p": top_p,
+            }
+            if image:
+                data["image"] = image  # Add image if provided
+
             response = client.chat.completions.create(
                 extra_headers={
                     "HTTP-Referer": "https://github.com/codelion/optillm",
                     "X-Title": "optillm"
                 },
-                model=model,
-                messages=messages,
-                max_tokens=max_tokens,
-                temperature=temperature,
-                top_p=top_p,
+                **data
             )
             return response.choices[0].message.content
         else:
             system_prompt, initial_query = parse_conversation(messages)
             
+            # Handle different approaches
             if approach == 'rto':
                 final_response, _ = round_trip_optimization(system_prompt, initial_query, client, model)
             elif approach == 'z3':
@@ -87,22 +93,10 @@ def respond(message, history, model, approach, system_message, max_tokens, tempe
         error_message = f"Error in respond function: {str(e)}\nType: {type(e).__name__}"
         print(error_message)
 
-    # for message in client.chat_completion(
-    #     messages,
-    #     max_tokens=max_tokens,
-    #     stream=True,
-    #     temperature=temperature,
-    #     top_p=top_p,
-    # ):
-    #     token = message.choices[0].delta.content
-
-    #     response += token
-    #     yield response
-
 def create_model_dropdown():
     return gr.Dropdown(
-        [ "meta-llama/llama-3.1-8b-instruct:free", "nousresearch/hermes-3-llama-3.1-405b:free","meta-llama/llama-3.2-1b-instruct:free",
-         "mistralai/mistral-7b-instruct:free","mistralai/pixtral-12b:free","meta-llama/llama-3.1-70b-instruct:free",
+        [ "meta-llama/llama-3.1-8b-instruct:free", "nousresearch/hermes-3-llama-3.1-405b:free", "meta-llama/llama-3.2-1b-instruct:free",
+         "mistralai/mistral-7b-instruct:free", "mistralai/pixtral-12b:free", "meta-llama/llama-3.1-70b-instruct:free",
          "qwen/qwen-2-7b-instruct:free", "qwen/qwen-2-vl-7b-instruct:free", "google/gemma-2-9b-it:free", "liquid/lfm-40b:free", "meta-llama/llama-3.1-405b-instruct:free",
          "openchat/openchat-7b:free", "meta-llama/llama-3.2-90b-vision-instruct:free", "meta-llama/llama-3.2-11b-vision-instruct:free",
          "meta-llama/llama-3-8b-instruct:free", "meta-llama/llama-3.2-3b-instruct:free", "microsoft/phi-3-medium-128k-instruct:free",
@@ -135,24 +129,25 @@ with gr.Blocks() as demo:
             approach = create_approach_dropdown()
             chatbot = gr.Chatbot()
             msg = gr.Textbox()
+            image = gr.Image(type="pil", label="Upload Image (optional)", optional=True)
             with gr.Row():
                 submit = gr.Button("Submit")
                 clear = gr.Button("Clear")
 
-            def user(user_message, history):
-                return "", history + [[user_message, None]]
+            def user(user_message, history, uploaded_image):
+                return "", history + [[user_message, None]], uploaded_image
 
-            def bot(history, model, approach, system_message, max_tokens, temperature, top_p):
+            def bot(history, model, approach, system_message, max_tokens, temperature, top_p, uploaded_image):
                 user_message = history[-1][0]
-                bot_message = respond(user_message, history[:-1], model, approach, system_message, max_tokens, temperature, top_p)
+                bot_message = respond(user_message, history[:-1], model, approach, system_message, max_tokens, temperature, top_p, image=uploaded_image)
                 history[-1][1] = bot_message
                 return history
 
-            msg.submit(user, [msg, chatbot], [msg, chatbot]).then(
-                bot, [chatbot, model, approach, system_message, max_tokens, temperature, top_p], chatbot
+            msg.submit(user, [msg, chatbot, image], [msg, chatbot, image]).then(
+                bot, [chatbot, model, approach, system_message, max_tokens, temperature, top_p, image], chatbot
             )
-            submit.click(user, [msg, chatbot], [msg, chatbot]).then(
-                bot, [chatbot, model, approach, system_message, max_tokens, temperature, top_p], chatbot
+            submit.click(user, [msg, chatbot, image], [msg, chatbot, image]).then(
+                bot, [chatbot, model, approach, system_message, max_tokens, temperature, top_p, image], chatbot
             )
             clear.click(lambda: None, None, chatbot, queue=False)
 
@@ -165,6 +160,7 @@ with gr.Blocks() as demo:
             
             compare_input = gr.Textbox(label="Enter your message for comparison")
             compare_button = gr.Button("Compare")
+            compare_image = gr.Image(type="pil", label="Upload Image for Comparison", optional=True)
             
             with gr.Row():
                 output1 = gr.Textbox(label="Response 1")
@@ -177,4 +173,4 @@ with gr.Blocks() as demo:
             )
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()