✨ Feature: Add feature: Support for text-to-speech endpoint /v1/audio/speech

main.py

@@ -15,7 +15,7 @@ from starlette.responses import StreamingResponse as StarletteStreamingResponse
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 from fastapi.exceptions import RequestValidationError
 
-from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, UnifiedRequest, EmbeddingRequest
+from models import RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, TextToSpeechRequest, UnifiedRequest, EmbeddingRequest
 from request import get_payload
 from response import fetch_response, fetch_response_stream
 from utils import (
@@ -392,6 +392,9 @@ class LoggingStreamingResponse(Response):
             async for chunk in self.body_iterator:
                 if isinstance(chunk, str):
                     chunk = chunk.encode('utf-8')
+                if isinstance(chunk, bytes):
+                    yield chunk
+                    continue
                 line = chunk.decode('utf-8')
                 if is_debug:
                     logger.info(f"{line.encode('utf-8').decode('unicode_escape')}")
@@ -504,6 +507,8 @@ class StatsMiddleware(BaseHTTPMiddleware):
                     moderated_content = request_model.get_last_text_message()
                 elif request_model.request_type == "image":
                     moderated_content = request_model.prompt
+                elif request_model.request_type == "tts":
+                    moderated_content = request_model.input
                 elif request_model.request_type == "moderation":
                     pass
                 elif request_model.request_type == "embedding":
@@ -817,6 +822,9 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A
 
     if endpoint == "/v1/embeddings":
         engine = "embedding"
+
+    if endpoint == "/v1/audio/speech":
+        engine = "tts"
         request.stream = False
 
     if provider.get("engine"):
@@ -848,19 +856,6 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A
 
     try:
         async with app.state.client_manager.get_client(timeout_value, url, proxy) as client:
-            # 打印client配置信息
-            # logger.info(f"Client config - Timeout: {client.timeout}")
-            # logger.info(f"Client config - Headers: {client.headers}")
-            # if hasattr(client, '_transport'):
-            #     if hasattr(client._transport, 'proxy_url'):
-            #         logger.info(f"Client config - Proxy: {client._transport.proxy_url}")
-            #     elif hasattr(client._transport, 'proxies'):
-            #         logger.info(f"Client config - Proxies: {client._transport.proxies}")
-            #     else:
-            #         logger.info("Client config - No proxy configured")
-            # else:
-            #     logger.info("Client config - No transport configured")
-            # logger.info(f"Client config - Follow Redirects: {client.follow_redirects}")
             if request.stream:
                 generator = fetch_response_stream(client, url, headers, payload, engine, original_model)
                 wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id)
@@ -868,12 +863,16 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest, A
             else:
                 generator = fetch_response(client, url, headers, payload, engine, original_model)
                 wrapped_generator, first_response_time = await error_handling_wrapper(generator, channel_id)
-                first_element = await anext(wrapped_generator)
-                first_element = first_element.lstrip("data: ")
-                # print("first_element", first_element)
-                first_element = json.loads(first_element)
-                response = StarletteStreamingResponse(iter([json.dumps(first_element)]), media_type="application/json")
-                # response = JSONResponse(first_element)
+
+                # 处理音频和其他二进制响应
+                if endpoint == "/v1/audio/speech":
+                    if isinstance(wrapped_generator, bytes):
+                        response = Response(content=wrapped_generator, media_type="audio/mpeg")
+                else:
+                    first_element = await anext(wrapped_generator)
+                    first_element = first_element.lstrip("data: ")
+                    first_element = json.loads(first_element)
+                    response = StarletteStreamingResponse(iter([json.dumps(first_element)]), media_type="application/json")
 
             # 更新成功计数和首次响应时间
             await update_channel_stats(current_info["request_id"], channel_id, request.model, current_info["api_key"], success=True)
@@ -1269,6 +1268,13 @@ async def embeddings(
 ):
     return await model_handler.request_model(request, api_index, endpoint="/v1/embeddings")
 
+@app.post("/v1/audio/speech", dependencies=[Depends(rate_limit_dependency)])
+async def audio_speech(
+    request: TextToSpeechRequest,
+    api_index: str = Depends(verify_api_key)
+):
+    return await model_handler.request_model(request, api_index, endpoint="/v1/audio/speech")
+
 @app.post("/v1/moderations", dependencies=[Depends(rate_limit_dependency)])
 async def moderations(
     request: ModerationRequest,

models.py

@@ -136,8 +136,16 @@ class ModerationRequest(BaseRequest):
     model: Optional[str] = "text-moderation-latest"
     stream: bool = False
 
+class TextToSpeechRequest(BaseRequest):
+    model: str
+    input: str
+    voice: str
+    response_format: Optional[str] = "mp3"
+    speed: Optional[float] = 1.0
+    stream: Optional[bool] = False  # Add this line
+
 class UnifiedRequest(BaseModel):
-    data: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest]
+    data: Union[RequestModel, ImageGenerationRequest, AudioTranscriptionRequest, ModerationRequest, EmbeddingRequest, TextToSpeechRequest]
 
     @model_validator(mode='before')
     @classmethod
@@ -152,6 +160,10 @@ class UnifiedRequest(BaseModel):
             elif "file" in values:
                 values["data"] = AudioTranscriptionRequest(**values)
                 values["data"].request_type = "audio"
+            elif "tts" in values.get("model", ""):
+                logger.info(f"TextToSpeechRequest: {values}")
+                values["data"] = TextToSpeechRequest(**values)
+                values["data"].request_type = "tts"
             elif "text-embedding" in values.get("model", ""):
                 values["data"] = EmbeddingRequest(**values)
                 values["data"].request_type = "embedding"
@@ -160,4 +172,4 @@ class UnifiedRequest(BaseModel):
                 values["data"].request_type = "moderation"
             else:
                 raise ValueError("无法确定请求类型")
-        return values
+        return values

request.py

@@ -1145,6 +1145,33 @@ async def get_embedding_payload(request, engine, provider):
 
     return url, headers, payload
 
+async def get_tts_payload(request, engine, provider):
+    model_dict = get_model_dict(provider)
+    model = model_dict[request.model]
+    headers = {
+        "Content-Type": "application/json",
+    }
+    if provider.get("api"):
+        headers['Authorization'] = f"Bearer {await provider_api_circular_list[provider['provider']].next(model)}"
+    url = provider['base_url']
+    url = BaseAPI(url).audio_speech
+
+    payload = {
+        "model": model,
+        "input": request.input,
+        "voice": request.voice,
+    }
+
+    if request.response_format:
+        payload["response_format"] = request.response_format
+    if request.speed:
+        payload["speed"] = request.speed
+    if request.stream is not None:
+        payload["stream"] = request.stream
+
+    return url, headers, payload
+
+
 async def get_payload(request: RequestModel, engine, provider):
     if engine == "gemini":
         return await get_gemini_payload(request, engine, provider)
@@ -1168,6 +1195,8 @@ async def get_payload(request: RequestModel, engine, provider):
         return await get_dalle_payload(request, engine, provider)
     elif engine == "whisper":
         return await get_whisper_payload(request, engine, provider)
+    elif engine == "tts":
+        return await get_tts_payload(request, engine, provider)
     elif engine == "moderation":
         return await get_moderation_payload(request, engine, provider)
     elif engine == "embedding":

response.py

@@ -326,8 +326,12 @@ async def fetch_response(client, url, headers, payload, engine, model):
     if error_message:
         yield error_message
         return
-    response_json = response.json()
-    if engine == "gemini" or engine == "vertex-gemini":
+
+    if engine == "tts":
+        yield response.read()
+
+    elif engine == "gemini" or engine == "vertex-gemini":
+        response_json = response.json()
 
         if isinstance(response_json, str):
             import ast
@@ -361,6 +365,7 @@ async def fetch_response(client, url, headers, payload, engine, model):
         yield await generate_no_stream_response(timestamp, model, content=content, tools_id=None, function_call_name=None, function_call_content=None, role=role, total_tokens=total_tokens, prompt_tokens=prompt_tokens, completion_tokens=candidates_tokens)
 
     else:
+        response_json = response.json()
         yield response_json
 
 async def fetch_response_stream(client, url, headers, payload, engine, model):

utils.py

@@ -416,6 +416,24 @@ def ensure_string(item):
     else:
         return str(item)
 
+def identify_audio_format(file_bytes):
+    # 读取开头的字节
+    if file_bytes.startswith(b'\xFF\xFB') or file_bytes.startswith(b'\xFF\xF3'):
+        return "MP3"
+    elif file_bytes.startswith(b'ID3'):
+        return "MP3 with ID3"
+    elif file_bytes.startswith(b'OpusHead'):
+        return "OPUS"
+    elif file_bytes.startswith(b'ADIF'):
+        return "AAC (ADIF)"
+    elif file_bytes.startswith(b'\xFF\xF1') or file_bytes.startswith(b'\xFF\xF9'):
+        return "AAC (ADTS)"
+    elif file_bytes.startswith(b'fLaC'):
+        return "FLAC"
+    elif file_bytes.startswith(b'RIFF') and file_bytes[8:12] == b'WAVE':
+        return "WAV"
+    return "Unknown/PCM"
+
 import asyncio
 import time as time_module
 async def error_handling_wrapper(generator, channel_id):
@@ -426,7 +444,10 @@ async def error_handling_wrapper(generator, channel_id):
         first_item_str = first_item
         # logger.info("first_item_str: %s", first_item_str)
         if isinstance(first_item_str, (bytes, bytearray)):
-            first_item_str = first_item_str.decode("utf-8")
+            if identify_audio_format(first_item_str) in ["MP3", "MP3 with ID3", "OPUS", "AAC (ADIF)", "AAC (ADTS)", "FLAC", "WAV"]:
+                return first_item, first_response_time
+            else:
+                first_item_str = first_item_str.decode("utf-8")
         if isinstance(first_item_str, str):
             if first_item_str.startswith("data:"):
                 first_item_str = first_item_str.lstrip("data: ")
@@ -598,6 +619,7 @@ class BaseAPI:
         self.audio_transcriptions: str = urlunparse(parsed_url[:2] + (before_v1 + "audio/transcriptions",) + ("",) * 3)
         self.moderations: str = urlunparse(parsed_url[:2] + (before_v1 + "moderations",) + ("",) * 3)
         self.embeddings: str = urlunparse(parsed_url[:2] + (before_v1 + "embeddings",) + ("",) * 3)
+        self.audio_speech: str = urlunparse(parsed_url[:2] + (before_v1 + "audio/speech",) + ("",) * 3)
 
 def safe_get(data, *keys, default=None):
     for key in keys: