Fix the bug that prevents automatic polling.

README.md

@@ -72,6 +72,7 @@ api_keys:
       - anthropic/claude-3-5-sonnet # 可以使用的模型名称，仅可以使用名为 anthropic 提供商提供的 claude-3-5-sonnet 模型。其他提供商的 claude-3-5-sonnet 模型不可以使用。
     preferences:
       USE_ROUND_ROBIN: true # 是否使用轮询负载均衡，true 为使用，false 为不使用，默认为 true
+      AUTO_RETRY: true # 是否自动重试，自动重试下一个提供商，true 为自动重试，false 为不自动重试，默认为 true
 ```
 
 ## Docker Local Deployment

main.py

@@ -133,12 +133,14 @@ class ModelRequestHandler:
         # 检查是否启用轮询
         api_index = api_list.index(token)
         use_round_robin = False
+        auto_retry = False
         if config['api_keys'][api_index].get("preferences"):
             use_round_robin = config['api_keys'][api_index]["preferences"].get("USE_ROUND_ROBIN")
+            auto_retry = config['api_keys'][api_index]["preferences"].get("AUTO_RETRY")
 
-        return await self.try_all_providers(request, matching_providers, use_round_robin)
+        return await self.try_all_providers(request, matching_providers, use_round_robin, auto_retry)
 
-    async def try_all_providers(self, request: RequestModel, providers: List[Dict], use_round_robin: bool):
+    async def try_all_providers(self, request: RequestModel, providers: List[Dict], use_round_robin: bool, auto_retry: bool):
         num_providers = len(providers)
         start_index = self.last_provider_index + 1 if use_round_robin else 0
 
@@ -153,7 +155,7 @@ class ModelRequestHandler:
                 print(f"Error with provider {provider['provider']}: {str(e)}")
                 # traceback.print_exc()
                 print('\033[0m')
-                if use_round_robin:
+                if auto_retry:
                     continue
                 else:
                     raise HTTPException(status_code=500, detail="Error: Current provider response failed!")

response.py

@@ -155,26 +155,28 @@ async def fetch_response(client, url, headers, payload):
             continue
 
 async def fetch_response_stream(client, url, headers, payload, engine, model):
-    for _ in range(2):
-        try:
-            if engine == "gemini":
-                async for chunk in fetch_gemini_response_stream(client, url, headers, payload, model):
-                    yield chunk
-            elif engine == "claude":
-                async for chunk in fetch_claude_response_stream(client, url, headers, payload, model):
-                    yield chunk
-            elif engine == "gpt":
-                async for chunk in fetch_gpt_response_stream(client, url, headers, payload):
-                    yield chunk
-            elif engine == "openrouter":
-                async for chunk in fetch_gpt_response_stream(client, url, headers, payload):
-                    yield chunk
-            else:
-                raise ValueError("Unknown response")
-            break
-        except httpx.ConnectError as e:
-            print(f"fetch_response_stream 连接错误： {e}")
-            continue
-        except httpx.ReadTimeout as e:
-            print(f"fetch_response_stream 读取响应超时： {e}")
-            continue
+    # for _ in range(2):
+    try:
+        if engine == "gemini":
+            async for chunk in fetch_gemini_response_stream(client, url, headers, payload, model):
+                yield chunk
+        elif engine == "claude":
+            async for chunk in fetch_claude_response_stream(client, url, headers, payload, model):
+                yield chunk
+        elif engine == "gpt":
+            async for chunk in fetch_gpt_response_stream(client, url, headers, payload):
+                yield chunk
+        elif engine == "openrouter":
+            async for chunk in fetch_gpt_response_stream(client, url, headers, payload):
+                yield chunk
+        else:
+            raise ValueError("Unknown response")
+        # break
+    except httpx.ConnectError as e:
+        # print(f"fetch_response_stream 连接错误： {e}")
+        yield {"error": f"500", "details": "fetch_response_stream Connect Error"}
+        # continue
+    except httpx.ReadTimeout as e:
+        # print(f"fetch_response_stream 读取响应超时 {e}")
+        yield {"error": f"500", "details": "fetch_response_stream Read Response Timeout"}
+        # continue