✨ Feature: Add support for v1/audio/transcriptions endpoint

main.py

@@ -12,7 +12,7 @@ from fastapi.responses import StreamingResponse, JSONResponse
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 from fastapi.exceptions import RequestValidationError
 
-from models import RequestModel, ImageGenerationRequest
+from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest
 from request import get_payload
 from response import fetch_response, fetch_response_stream
 from utils import error_handling_wrapper, post_all_models, load_config, safe_get, circular_list_encoder
@@ -191,7 +191,7 @@ app.add_middleware(
 app.add_middleware(StatsMiddleware)
 
 # 在 process_request 函数中更新成功和失败计数
-async def process_request(request: Union[RequestModel, ImageGenerationRequest], provider: Dict, endpoint=None, token=None):
+async def process_request(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest], provider: Dict, endpoint=None, token=None):
     url = provider['base_url']
     parsed_url = urlparse(url)
     # print("parsed_url", parsed_url)
@@ -233,6 +233,10 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest],
         engine = "dalle"
         request.stream = False
 
+    if endpoint == "/v1/audio/transcriptions":
+        engine = "whisper"
+        request.stream = False
+
     if provider.get("engine"):
         engine = provider["engine"]
 
@@ -241,7 +245,10 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest],
     url, headers, payload = await get_payload(request, engine, provider)
     if is_debug:
         logger.info(json.dumps(headers, indent=4, ensure_ascii=False))
-        logger.info(json.dumps(payload, indent=4, ensure_ascii=False))
+        if payload.get("file"):
+            pass
+        else:
+            logger.info(json.dumps(payload, indent=4, ensure_ascii=False))
     try:
         if request.stream:
             model = provider['model'][request.model]
@@ -356,7 +363,7 @@ class ModelRequestHandler:
                 print(json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder))
         return provider_list
 
-    async def request_model(self, request: Union[RequestModel, ImageGenerationRequest], token: str, endpoint=None):
+    async def request_model(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest], token: str, endpoint=None):
         config = app.state.config
         # api_keys_db = app.state.api_keys_db
         api_list = app.state.api_list
@@ -399,7 +406,7 @@ class ModelRequestHandler:
         return await self.try_all_providers(request, matching_providers, use_round_robin, auto_retry, endpoint, token)
 
     # 在 try_all_providers 函数中处理失败的情况
-    async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None, token: str = None):
+    async def try_all_providers(self, request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest], providers: List[Dict], use_round_robin: bool, auto_retry: bool, endpoint: str = None, token: str = None):
         status_code = 500
         error_message = None
         num_providers = len(providers)
@@ -421,6 +428,9 @@ class ModelRequestHandler:
                     raise HTTPException(status_code=500, detail=f"Error: Current provider response failed: {error_message}")
             except (Exception, asyncio.CancelledError, httpx.ReadError, httpx.RemoteProtocolError) as e:
                 logger.error(f"Error with provider {provider['provider']}: {str(e)}")
+                if is_debug:
+                    import traceback
+                    traceback.print_exc()
                 error_message = str(e)
                 if auto_retry:
                     continue
@@ -523,7 +533,7 @@ def verify_admin_api_key(credentials: HTTPAuthorizationCredentials = Depends(sec
     return token
 
 @app.post("/v1/chat/completions", dependencies=[Depends(rate_limit_dependency)])
-async def request_model(request: Union[RequestModel, ImageGenerationRequest], token: str = Depends(verify_api_key)):
+async def request_model(request: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest], token: str = Depends(verify_api_key)):
     # logger.info(f"Request received: {request}")
     return await model_handler.request_model(request, token)
 
@@ -546,6 +556,34 @@ async def images_generations(
 ):
     return await model_handler.request_model(request, token, endpoint="/v1/images/generations")
 
+from fastapi import UploadFile, File, Form, HTTPException
+import io
+@app.post("/v1/audio/transcriptions", dependencies=[Depends(rate_limit_dependency)])
+async def audio_transcriptions(
+    file: UploadFile = File(...),
+    model: str = Form(...),
+    token: str = Depends(verify_api_key)
+):
+    try:
+        # 读取上传的文件内容
+        content = await file.read()
+        file_obj = io.BytesIO(content)
+
+        # 创建AudioTranscriptionRequest对象
+        request = AudioTranscriptionRequest(
+            file=(file.filename, file_obj, file.content_type),
+            model=model
+        )
+
+        return await model_handler.request_model(request, token, endpoint="/v1/audio/transcriptions")
+    except UnicodeDecodeError:
+        raise HTTPException(status_code=400, detail="Invalid audio file encoding")
+    except Exception as e:
+        if is_debug:
+            import traceback
+            traceback.print_exc()
+        raise HTTPException(status_code=500, detail=f"Error processing audio file: {str(e)}")
+
 @app.get("/generate-api-key", dependencies=[Depends(rate_limit_dependency)])
 def generate_api_key():
     api_key = "sk-" + secrets.token_urlsafe(36)

models.py

@@ -1,5 +1,6 @@
+from io import IOBase
 from pydantic import BaseModel, Field
-from typing import List, Dict, Optional, Union
+from typing import List, Dict, Optional, Union, Tuple
 
 class ImageGenerationRequest(BaseModel):
     model: str
@@ -8,6 +9,18 @@ class ImageGenerationRequest(BaseModel):
     size: str
     stream: bool = False
 
+class AudioTranscriptionRequest(BaseModel):
+    file: Tuple[str, IOBase, str]
+    model: str
+    language: Optional[str] = None
+    prompt: Optional[str] = None
+    response_format: Optional[str] = None
+    temperature: Optional[float] = None
+    stream: bool = False
+
+    class Config:
+        arbitrary_types_allowed = True
+
 class FunctionParameter(BaseModel):
     type: str
     properties: Dict[str, Dict[str, Union[str, Dict[str, str]]]]

request.py

@@ -1040,6 +1040,32 @@ async def get_dalle_payload(request, engine, provider):
 
     return url, headers, payload
 
+async def get_whisper_payload(request, engine, provider):
+    model = provider['model'][request.model]
+    headers = {
+        "Content-Type": "application/json",
+    }
+    if provider.get("api"):
+        headers['Authorization'] = f"Bearer {provider['api'].next()}"
+    url = provider['base_url']
+    url = BaseAPI(url).audio_transcriptions
+
+    payload = {
+        "model": model,
+        "file": request.file,
+    }
+
+    if request.prompt:
+        payload["prompt"] = request.prompt
+    if request.response_format:
+        payload["response_format"] = request.response_format
+    if request.temperature:
+        payload["temperature"] = request.temperature
+    if request.language:
+        payload["language"] = request.language
+
+    return url, headers, payload
+
 async def get_payload(request: RequestModel, engine, provider):
     if engine == "gemini":
         return await get_gemini_payload(request, engine, provider)
@@ -1061,5 +1087,7 @@ async def get_payload(request: RequestModel, engine, provider):
         return await get_cohere_payload(request, engine, provider)
     elif engine == "dalle":
         return await get_dalle_payload(request, engine, provider)
+    elif engine == "whisper":
+        return await get_whisper_payload(request, engine, provider)
     else:
         raise ValueError("Unknown payload")

response.py

@@ -1,6 +1,7 @@
 import json
 import httpx
 from datetime import datetime
+from io import BytesIO
 
 from log_config import logger
 
@@ -41,7 +42,7 @@ async def generate_sse_response(timestamp, model, content=None, tools_id=None, f
     return sse_response
 
 async def check_response(response, error_log):
-    if response.status_code != 200:
+    if response and response.status_code != 200:
         error_message = await response.aread()
         error_str = error_message.decode('utf-8', errors='replace')
         try:
@@ -269,7 +270,13 @@ async def fetch_claude_response_stream(client, url, headers, payload, model):
         yield "data: [DONE]\n\r\n"
 
 async def fetch_response(client, url, headers, payload):
-    response = await client.post(url, headers=headers, json=payload)
+    response = None
+    if payload.get("file"):
+        file = payload.pop("file")
+        headers.pop("Content-Type")
+        response = await client.post(url, headers=headers, data=payload, files={"file": file})
+    else:
+        response = await client.post(url, headers=headers, json=payload)
     error_message = await check_response(response, "fetch_response")
     if error_message:
         yield error_message