Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Fix logging issue
Browse files- backend/app/api/dependencies.py +1 -1
- backend/app/api/endpoints/leaderboard.py +1 -1
- backend/app/api/endpoints/votes.py +1 -1
- backend/app/asgi.py +1 -1
- backend/app/config/hf_config.py +0 -4
- backend/app/core/fastapi_cache.py +1 -1
- backend/app/core/formatting.py +104 -0
- backend/app/services/hf_service.py +1 -1
- backend/app/services/leaderboard.py +1 -1
- backend/app/services/models.py +18 -10
- backend/app/services/votes.py +2 -3
- backend/app/utils/logging.py +2 -104
backend/app/api/dependencies.py
CHANGED
@@ -2,7 +2,7 @@ from fastapi import Depends, HTTPException
|
|
2 |
import logging
|
3 |
from app.services.models import ModelService
|
4 |
from app.services.votes import VoteService
|
5 |
-
from app.
|
6 |
|
7 |
logger = logging.getLogger(__name__)
|
8 |
|
|
|
2 |
import logging
|
3 |
from app.services.models import ModelService
|
4 |
from app.services.votes import VoteService
|
5 |
+
from app.core.formatting import LogFormatter
|
6 |
|
7 |
logger = logging.getLogger(__name__)
|
8 |
|
backend/app/api/endpoints/leaderboard.py
CHANGED
@@ -3,7 +3,7 @@ from typing import List, Dict, Any
|
|
3 |
from app.services.leaderboard import LeaderboardService
|
4 |
from app.core.fastapi_cache import cached, build_cache_key
|
5 |
import logging
|
6 |
-
from app.
|
7 |
|
8 |
logger = logging.getLogger(__name__)
|
9 |
router = APIRouter()
|
|
|
3 |
from app.services.leaderboard import LeaderboardService
|
4 |
from app.core.fastapi_cache import cached, build_cache_key
|
5 |
import logging
|
6 |
+
from app.core.formatting import LogFormatter
|
7 |
|
8 |
logger = logging.getLogger(__name__)
|
9 |
router = APIRouter()
|
backend/app/api/endpoints/votes.py
CHANGED
@@ -3,7 +3,7 @@ from typing import Dict, Any, List
|
|
3 |
from app.services.votes import VoteService
|
4 |
from app.core.fastapi_cache import cached, build_cache_key, invalidate_cache_key
|
5 |
import logging
|
6 |
-
from app.
|
7 |
|
8 |
logger = logging.getLogger(__name__)
|
9 |
router = APIRouter()
|
|
|
3 |
from app.services.votes import VoteService
|
4 |
from app.core.fastapi_cache import cached, build_cache_key, invalidate_cache_key
|
5 |
import logging
|
6 |
+
from app.core.formatting import LogFormatter
|
7 |
|
8 |
logger = logging.getLogger(__name__)
|
9 |
router = APIRouter()
|
backend/app/asgi.py
CHANGED
@@ -12,7 +12,7 @@ import sys
|
|
12 |
|
13 |
from app.api.router import router
|
14 |
from app.core.fastapi_cache import setup_cache
|
15 |
-
from app.
|
16 |
from app.config import hf_config
|
17 |
|
18 |
# Configure logging before anything else
|
|
|
12 |
|
13 |
from app.api.router import router
|
14 |
from app.core.fastapi_cache import setup_cache
|
15 |
+
from app.core.formatting import LogFormatter
|
16 |
from app.config import hf_config
|
17 |
|
18 |
# Configure logging before anything else
|
backend/app/config/hf_config.py
CHANGED
@@ -1,13 +1,9 @@
|
|
1 |
-
"""
|
2 |
-
Hugging Face configuration module
|
3 |
-
"""
|
4 |
import os
|
5 |
import logging
|
6 |
from typing import Optional
|
7 |
from huggingface_hub import HfApi
|
8 |
from pathlib import Path
|
9 |
from app.core.cache import cache_config
|
10 |
-
from app.utils.logging import LogFormatter
|
11 |
|
12 |
logger = logging.getLogger(__name__)
|
13 |
|
|
|
|
|
|
|
|
|
1 |
import os
|
2 |
import logging
|
3 |
from typing import Optional
|
4 |
from huggingface_hub import HfApi
|
5 |
from pathlib import Path
|
6 |
from app.core.cache import cache_config
|
|
|
7 |
|
8 |
logger = logging.getLogger(__name__)
|
9 |
|
backend/app/core/fastapi_cache.py
CHANGED
@@ -4,7 +4,7 @@ from fastapi_cache.decorator import cache
|
|
4 |
from datetime import timedelta
|
5 |
from app.config import CACHE_TTL
|
6 |
import logging
|
7 |
-
from app.
|
8 |
|
9 |
logger = logging.getLogger(__name__)
|
10 |
|
|
|
4 |
from datetime import timedelta
|
5 |
from app.config import CACHE_TTL
|
6 |
import logging
|
7 |
+
from app.core.formatting import LogFormatter
|
8 |
|
9 |
logger = logging.getLogger(__name__)
|
10 |
|
backend/app/core/formatting.py
ADDED
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import logging
|
2 |
+
from typing import Dict, Any, List, Optional
|
3 |
+
|
4 |
+
logger = logging.getLogger(__name__)
|
5 |
+
|
6 |
+
class LogFormatter:
|
7 |
+
"""Utility class for consistent log formatting across the application"""
|
8 |
+
|
9 |
+
@staticmethod
|
10 |
+
def section(title: str) -> str:
|
11 |
+
"""Create a section header"""
|
12 |
+
return f"\n{'='*20} {title.upper()} {'='*20}"
|
13 |
+
|
14 |
+
@staticmethod
|
15 |
+
def subsection(title: str) -> str:
|
16 |
+
"""Create a subsection header"""
|
17 |
+
return f"\n{'β'*20} {title} {'β'*20}"
|
18 |
+
|
19 |
+
@staticmethod
|
20 |
+
def tree(items: Dict[str, Any], title: str = None) -> List[str]:
|
21 |
+
"""Create a tree view of dictionary data"""
|
22 |
+
lines = []
|
23 |
+
if title:
|
24 |
+
lines.append(f"π {title}:")
|
25 |
+
|
26 |
+
# Get the maximum length for alignment
|
27 |
+
max_key_length = max(len(str(k)) for k in items.keys())
|
28 |
+
|
29 |
+
# Format each item
|
30 |
+
for i, (key, value) in enumerate(items.items()):
|
31 |
+
prefix = "βββ" if i == len(items) - 1 else "βββ"
|
32 |
+
if isinstance(value, (int, float)):
|
33 |
+
value = f"{value:,}" # Add thousand separators
|
34 |
+
lines.append(f"{prefix} {str(key):<{max_key_length}}: {value}")
|
35 |
+
|
36 |
+
return lines
|
37 |
+
|
38 |
+
@staticmethod
|
39 |
+
def stats(stats: Dict[str, int], title: str = None) -> List[str]:
|
40 |
+
"""Format statistics with icons"""
|
41 |
+
lines = []
|
42 |
+
if title:
|
43 |
+
lines.append(f"π {title}:")
|
44 |
+
|
45 |
+
# Get the maximum length for alignment
|
46 |
+
max_key_length = max(len(str(k)) for k in stats.keys())
|
47 |
+
|
48 |
+
# Format each stat with an appropriate icon
|
49 |
+
icons = {
|
50 |
+
"total": "π",
|
51 |
+
"success": "β
",
|
52 |
+
"error": "β",
|
53 |
+
"pending": "β³",
|
54 |
+
"processing": "βοΈ",
|
55 |
+
"finished": "β¨",
|
56 |
+
"evaluating": "π",
|
57 |
+
"downloads": "β¬οΈ",
|
58 |
+
"files": "π",
|
59 |
+
"cached": "πΎ",
|
60 |
+
"size": "π",
|
61 |
+
"time": "β±οΈ",
|
62 |
+
"rate": "π"
|
63 |
+
}
|
64 |
+
|
65 |
+
# Format each item
|
66 |
+
for i, (key, value) in enumerate(stats.items()):
|
67 |
+
prefix = "βββ" if i == len(stats) - 1 else "βββ"
|
68 |
+
icon = icons.get(key.lower().split('_')[0], "β’")
|
69 |
+
if isinstance(value, (int, float)):
|
70 |
+
value = f"{value:,}" # Add thousand separators
|
71 |
+
lines.append(f"{prefix} {icon} {str(key):<{max_key_length}}: {value}")
|
72 |
+
|
73 |
+
return lines
|
74 |
+
|
75 |
+
@staticmethod
|
76 |
+
def progress_bar(current: int, total: int, width: int = 20) -> str:
|
77 |
+
"""Create a progress bar"""
|
78 |
+
percentage = (current * 100) // total
|
79 |
+
filled = "β" * (percentage * width // 100)
|
80 |
+
empty = "β" * (width - len(filled))
|
81 |
+
return f"{filled}{empty} {percentage:3d}%"
|
82 |
+
|
83 |
+
@staticmethod
|
84 |
+
def error(message: str, error: Optional[Exception] = None) -> str:
|
85 |
+
"""Format error message"""
|
86 |
+
error_msg = f"\nβ Error: {message}"
|
87 |
+
if error:
|
88 |
+
error_msg += f"\n βββ Details: {str(error)}"
|
89 |
+
return error_msg
|
90 |
+
|
91 |
+
@staticmethod
|
92 |
+
def success(message: str) -> str:
|
93 |
+
"""Format success message"""
|
94 |
+
return f"β
{message}"
|
95 |
+
|
96 |
+
@staticmethod
|
97 |
+
def warning(message: str) -> str:
|
98 |
+
"""Format warning message"""
|
99 |
+
return f"β οΈ {message}"
|
100 |
+
|
101 |
+
@staticmethod
|
102 |
+
def info(message: str) -> str:
|
103 |
+
"""Format info message"""
|
104 |
+
return f"βΉοΈ {message}"
|
backend/app/services/hf_service.py
CHANGED
@@ -2,7 +2,7 @@ from typing import Optional
|
|
2 |
from huggingface_hub import HfApi
|
3 |
from app.config import HF_TOKEN, API
|
4 |
from app.core.cache import cache_config
|
5 |
-
from app.
|
6 |
import logging
|
7 |
|
8 |
logger = logging.getLogger(__name__)
|
|
|
2 |
from huggingface_hub import HfApi
|
3 |
from app.config import HF_TOKEN, API
|
4 |
from app.core.cache import cache_config
|
5 |
+
from app.core.formatting import LogFormatter
|
6 |
import logging
|
7 |
|
8 |
logger = logging.getLogger(__name__)
|
backend/app/services/leaderboard.py
CHANGED
@@ -5,7 +5,7 @@ import datasets
|
|
5 |
from fastapi import HTTPException
|
6 |
import logging
|
7 |
from app.config.base import HF_ORGANIZATION
|
8 |
-
from app.
|
9 |
|
10 |
logger = logging.getLogger(__name__)
|
11 |
|
|
|
5 |
from fastapi import HTTPException
|
6 |
import logging
|
7 |
from app.config.base import HF_ORGANIZATION
|
8 |
+
from app.core.formatting import LogFormatter
|
9 |
|
10 |
logger = logging.getLogger(__name__)
|
11 |
|
backend/app/services/models.py
CHANGED
@@ -25,7 +25,7 @@ from app.services.hf_service import HuggingFaceService
|
|
25 |
from app.utils.model_validation import ModelValidator
|
26 |
from app.services.votes import VoteService
|
27 |
from app.core.cache import cache_config
|
28 |
-
from app.
|
29 |
|
30 |
# Disable datasets progress bars globally
|
31 |
disable_progress_bar()
|
@@ -409,6 +409,14 @@ class ModelService(HuggingFaceService):
|
|
409 |
logger.info(LogFormatter.subsection("CHECKING EXISTING SUBMISSIONS"))
|
410 |
existing_models = await self.get_models()
|
411 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
412 |
# Check in all statuses (pending, evaluating, finished)
|
413 |
for status, models in existing_models.items():
|
414 |
for model in models:
|
@@ -516,15 +524,15 @@ class ModelService(HuggingFaceService):
|
|
516 |
temp_file.flush()
|
517 |
temp_path = temp_file.name
|
518 |
|
519 |
-
# Upload file directly
|
520 |
-
self.hf_api.upload_file(
|
521 |
-
|
522 |
-
|
523 |
-
|
524 |
-
|
525 |
-
|
526 |
-
|
527 |
-
)
|
528 |
|
529 |
# Clean up temp file
|
530 |
os.unlink(temp_path)
|
|
|
25 |
from app.utils.model_validation import ModelValidator
|
26 |
from app.services.votes import VoteService
|
27 |
from app.core.cache import cache_config
|
28 |
+
from app.core.formatting import LogFormatter
|
29 |
|
30 |
# Disable datasets progress bars globally
|
31 |
disable_progress_bar()
|
|
|
409 |
logger.info(LogFormatter.subsection("CHECKING EXISTING SUBMISSIONS"))
|
410 |
existing_models = await self.get_models()
|
411 |
|
412 |
+
# Call the official provider status check
|
413 |
+
is_valid, error_message = await self.validator.check_official_provider_status(
|
414 |
+
model_data["model_id"],
|
415 |
+
existing_models
|
416 |
+
)
|
417 |
+
if not is_valid:
|
418 |
+
raise ValueError(error_message)
|
419 |
+
|
420 |
# Check in all statuses (pending, evaluating, finished)
|
421 |
for status, models in existing_models.items():
|
422 |
for model in models:
|
|
|
524 |
temp_file.flush()
|
525 |
temp_path = temp_file.name
|
526 |
|
527 |
+
# # Upload file directly
|
528 |
+
# self.hf_api.upload_file(
|
529 |
+
# path_or_fileobj=temp_path,
|
530 |
+
# path_in_repo=relative_path,
|
531 |
+
# repo_id=f"{HF_ORGANIZATION}/requests",
|
532 |
+
# repo_type="dataset",
|
533 |
+
# commit_message=f"Add {model_data['model_id']} to eval queue",
|
534 |
+
# token=self.token
|
535 |
+
# )
|
536 |
|
537 |
# Clean up temp file
|
538 |
os.unlink(temp_path)
|
backend/app/services/votes.py
CHANGED
@@ -4,16 +4,15 @@ import json
|
|
4 |
import logging
|
5 |
import asyncio
|
6 |
from pathlib import Path
|
7 |
-
import os
|
8 |
import aiohttp
|
9 |
from huggingface_hub import HfApi
|
10 |
import datasets
|
11 |
|
12 |
from app.services.hf_service import HuggingFaceService
|
13 |
-
from app.config import HF_TOKEN
|
14 |
from app.config.hf_config import HF_ORGANIZATION
|
15 |
from app.core.cache import cache_config
|
16 |
-
from app.
|
17 |
|
18 |
logger = logging.getLogger(__name__)
|
19 |
|
|
|
4 |
import logging
|
5 |
import asyncio
|
6 |
from pathlib import Path
|
|
|
7 |
import aiohttp
|
8 |
from huggingface_hub import HfApi
|
9 |
import datasets
|
10 |
|
11 |
from app.services.hf_service import HuggingFaceService
|
12 |
+
from app.config import HF_TOKEN
|
13 |
from app.config.hf_config import HF_ORGANIZATION
|
14 |
from app.core.cache import cache_config
|
15 |
+
from app.core.formatting import LogFormatter
|
16 |
|
17 |
logger = logging.getLogger(__name__)
|
18 |
|
backend/app/utils/logging.py
CHANGED
@@ -1,105 +1,3 @@
|
|
1 |
-
import
|
2 |
-
import sys
|
3 |
-
from typing import Dict, Any, List, Optional
|
4 |
|
5 |
-
|
6 |
-
|
7 |
-
class LogFormatter:
|
8 |
-
"""Utility class for consistent log formatting across the application"""
|
9 |
-
|
10 |
-
@staticmethod
|
11 |
-
def section(title: str) -> str:
|
12 |
-
"""Create a section header"""
|
13 |
-
return f"\n{'='*20} {title.upper()} {'='*20}"
|
14 |
-
|
15 |
-
@staticmethod
|
16 |
-
def subsection(title: str) -> str:
|
17 |
-
"""Create a subsection header"""
|
18 |
-
return f"\n{'β'*20} {title} {'β'*20}"
|
19 |
-
|
20 |
-
@staticmethod
|
21 |
-
def tree(items: Dict[str, Any], title: str = None) -> List[str]:
|
22 |
-
"""Create a tree view of dictionary data"""
|
23 |
-
lines = []
|
24 |
-
if title:
|
25 |
-
lines.append(f"π {title}:")
|
26 |
-
|
27 |
-
# Get the maximum length for alignment
|
28 |
-
max_key_length = max(len(str(k)) for k in items.keys())
|
29 |
-
|
30 |
-
# Format each item
|
31 |
-
for i, (key, value) in enumerate(items.items()):
|
32 |
-
prefix = "βββ" if i == len(items) - 1 else "βββ"
|
33 |
-
if isinstance(value, (int, float)):
|
34 |
-
value = f"{value:,}" # Add thousand separators
|
35 |
-
lines.append(f"{prefix} {str(key):<{max_key_length}}: {value}")
|
36 |
-
|
37 |
-
return lines
|
38 |
-
|
39 |
-
@staticmethod
|
40 |
-
def stats(stats: Dict[str, int], title: str = None) -> List[str]:
|
41 |
-
"""Format statistics with icons"""
|
42 |
-
lines = []
|
43 |
-
if title:
|
44 |
-
lines.append(f"π {title}:")
|
45 |
-
|
46 |
-
# Get the maximum length for alignment
|
47 |
-
max_key_length = max(len(str(k)) for k in stats.keys())
|
48 |
-
|
49 |
-
# Format each stat with an appropriate icon
|
50 |
-
icons = {
|
51 |
-
"total": "π",
|
52 |
-
"success": "β
",
|
53 |
-
"error": "β",
|
54 |
-
"pending": "β³",
|
55 |
-
"processing": "βοΈ",
|
56 |
-
"finished": "β¨",
|
57 |
-
"evaluating": "π",
|
58 |
-
"downloads": "β¬οΈ",
|
59 |
-
"files": "π",
|
60 |
-
"cached": "πΎ",
|
61 |
-
"size": "π",
|
62 |
-
"time": "β±οΈ",
|
63 |
-
"rate": "π"
|
64 |
-
}
|
65 |
-
|
66 |
-
# Format each item
|
67 |
-
for i, (key, value) in enumerate(stats.items()):
|
68 |
-
prefix = "βββ" if i == len(stats) - 1 else "βββ"
|
69 |
-
icon = icons.get(key.lower().split('_')[0], "β’")
|
70 |
-
if isinstance(value, (int, float)):
|
71 |
-
value = f"{value:,}" # Add thousand separators
|
72 |
-
lines.append(f"{prefix} {icon} {str(key):<{max_key_length}}: {value}")
|
73 |
-
|
74 |
-
return lines
|
75 |
-
|
76 |
-
@staticmethod
|
77 |
-
def progress_bar(current: int, total: int, width: int = 20) -> str:
|
78 |
-
"""Create a progress bar"""
|
79 |
-
percentage = (current * 100) // total
|
80 |
-
filled = "β" * (percentage * width // 100)
|
81 |
-
empty = "β" * (width - len(filled))
|
82 |
-
return f"{filled}{empty} {percentage:3d}%"
|
83 |
-
|
84 |
-
@staticmethod
|
85 |
-
def error(message: str, error: Optional[Exception] = None) -> str:
|
86 |
-
"""Format error message"""
|
87 |
-
error_msg = f"\nβ Error: {message}"
|
88 |
-
if error:
|
89 |
-
error_msg += f"\n βββ Details: {str(error)}"
|
90 |
-
return error_msg
|
91 |
-
|
92 |
-
@staticmethod
|
93 |
-
def success(message: str) -> str:
|
94 |
-
"""Format success message"""
|
95 |
-
return f"β
{message}"
|
96 |
-
|
97 |
-
@staticmethod
|
98 |
-
def warning(message: str) -> str:
|
99 |
-
"""Format warning message"""
|
100 |
-
return f"β οΈ {message}"
|
101 |
-
|
102 |
-
@staticmethod
|
103 |
-
def info(message: str) -> str:
|
104 |
-
"""Format info message"""
|
105 |
-
return f"βΉοΈ {message}"
|
|
|
1 |
+
from app.core.formatting import LogFormatter
|
|
|
|
|
2 |
|
3 |
+
__all__ = ['LogFormatter']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|