fix: auto-fix code issues (cron)
- 修复重复导入/字段 - 修复异常处理 - 修复PEP8格式问题 - 添加类型注解
This commit is contained in:
@@ -9,18 +9,19 @@ Phase 7 Task 8: Performance Optimization & Scaling
|
||||
4. PerformanceMonitor - 性能监控(API响应、查询性能、缓存命中率)
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Any, Callable, Tuple
|
||||
from datetime import datetime
|
||||
from collections import OrderedDict
|
||||
from functools import wraps
|
||||
import time
|
||||
import uuid
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from functools import wraps
|
||||
from typing import Any
|
||||
|
||||
# 尝试导入 Redis
|
||||
try:
|
||||
@@ -67,7 +68,7 @@ class CacheEntry:
|
||||
key: str
|
||||
value: Any
|
||||
created_at: float
|
||||
expires_at: Optional[float]
|
||||
expires_at: float | None
|
||||
access_count: int = 0
|
||||
last_accessed: float = 0
|
||||
size_bytes: int = 0
|
||||
@@ -79,12 +80,12 @@ class PerformanceMetric:
|
||||
|
||||
id: str
|
||||
metric_type: str # api_response, db_query, cache_operation
|
||||
endpoint: Optional[str]
|
||||
endpoint: str | None
|
||||
duration_ms: float
|
||||
timestamp: str
|
||||
metadata: Dict = field(default_factory=dict)
|
||||
metadata: dict = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict:
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"metric_type": self.metric_type,
|
||||
@@ -102,16 +103,16 @@ class TaskInfo:
|
||||
id: str
|
||||
task_type: str
|
||||
status: str # pending, running, success, failed, retrying
|
||||
payload: Dict
|
||||
payload: dict
|
||||
created_at: str
|
||||
started_at: Optional[str] = None
|
||||
completed_at: Optional[str] = None
|
||||
result: Optional[Any] = None
|
||||
error_message: Optional[str] = None
|
||||
started_at: str | None = None
|
||||
completed_at: str | None = None
|
||||
result: Any | None = None
|
||||
error_message: str | None = None
|
||||
retry_count: int = 0
|
||||
max_retries: int = 3
|
||||
|
||||
def to_dict(self) -> Dict:
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"task_type": self.task_type,
|
||||
@@ -132,7 +133,7 @@ class ShardInfo:
|
||||
"""分片信息数据模型"""
|
||||
|
||||
shard_id: str
|
||||
shard_key_range: Tuple[str, str] # (start, end)
|
||||
shard_key_range: tuple[str, str] # (start, end)
|
||||
db_path: str
|
||||
entity_count: int = 0
|
||||
is_active: bool = True
|
||||
@@ -160,7 +161,7 @@ class CacheManager:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
redis_url: Optional[str] = None,
|
||||
redis_url: str | None = None,
|
||||
max_memory_size: int = 100 * 1024 * 1024, # 100MB
|
||||
default_ttl: int = 3600, # 1小时
|
||||
db_path: str = "insightflow.db",
|
||||
@@ -242,7 +243,7 @@ class CacheManager:
|
||||
self.current_memory_size -= oldest_entry.size_bytes
|
||||
self.stats.evictions += 1
|
||||
|
||||
def get(self, key: str) -> Optional[Any]:
|
||||
def get(self, key: str) -> Any | None:
|
||||
"""
|
||||
获取缓存值
|
||||
|
||||
@@ -291,7 +292,7 @@ class CacheManager:
|
||||
self.stats.misses += 1
|
||||
return None
|
||||
|
||||
def set(self, key: str, value: Any, ttl: Optional[int] = None) -> bool:
|
||||
def set(self, key: str, value: Any, ttl: int | None = None) -> bool:
|
||||
"""
|
||||
设置缓存值
|
||||
|
||||
@@ -373,7 +374,7 @@ class CacheManager:
|
||||
self.current_memory_size = 0
|
||||
return True
|
||||
|
||||
def get_many(self, keys: List[str]) -> Dict[str, Any]:
|
||||
def get_many(self, keys: list[str]) -> dict[str, Any]:
|
||||
"""批量获取缓存"""
|
||||
results = {}
|
||||
|
||||
@@ -397,7 +398,7 @@ class CacheManager:
|
||||
|
||||
return results
|
||||
|
||||
def set_many(self, mapping: Dict[str, Any], ttl: Optional[int] = None) -> bool:
|
||||
def set_many(self, mapping: dict[str, Any], ttl: int | None = None) -> bool:
|
||||
"""批量设置缓存"""
|
||||
ttl = ttl or self.default_ttl
|
||||
|
||||
@@ -417,7 +418,7 @@ class CacheManager:
|
||||
self.set(key, value, ttl)
|
||||
return True
|
||||
|
||||
def get_stats(self) -> Dict:
|
||||
def get_stats(self) -> dict:
|
||||
"""获取缓存统计"""
|
||||
self.stats.update_hit_rate()
|
||||
|
||||
@@ -468,7 +469,7 @@ class CacheManager:
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def warm_up(self, project_id: str) -> Dict:
|
||||
def warm_up(self, project_id: str) -> dict:
|
||||
"""
|
||||
缓存预热 - 加载项目的热点数据
|
||||
|
||||
@@ -612,7 +613,7 @@ class DatabaseSharding:
|
||||
os.makedirs(shard_db_dir, exist_ok=True)
|
||||
|
||||
# 分片映射
|
||||
self.shard_map: Dict[str, ShardInfo] = {}
|
||||
self.shard_map: dict[str, ShardInfo] = {}
|
||||
|
||||
# 初始化分片
|
||||
self._init_shards()
|
||||
@@ -708,7 +709,7 @@ class DatabaseSharding:
|
||||
|
||||
return conn
|
||||
|
||||
def get_all_shards(self) -> List[ShardInfo]:
|
||||
def get_all_shards(self) -> list[ShardInfo]:
|
||||
"""获取所有分片信息"""
|
||||
return list(self.shard_map.values())
|
||||
|
||||
@@ -805,7 +806,7 @@ class DatabaseSharding:
|
||||
|
||||
conn.close()
|
||||
|
||||
def cross_shard_query(self, query_func: Callable) -> List[Dict]:
|
||||
def cross_shard_query(self, query_func: Callable) -> list[dict]:
|
||||
"""
|
||||
跨分片查询
|
||||
|
||||
@@ -831,7 +832,7 @@ class DatabaseSharding:
|
||||
|
||||
return results
|
||||
|
||||
def get_shard_stats(self) -> List[Dict]:
|
||||
def get_shard_stats(self) -> list[dict]:
|
||||
"""获取所有分片的统计信息"""
|
||||
stats = []
|
||||
|
||||
@@ -852,7 +853,7 @@ class DatabaseSharding:
|
||||
|
||||
return stats
|
||||
|
||||
def rebalance_shards(self) -> Dict:
|
||||
def rebalance_shards(self) -> dict:
|
||||
"""
|
||||
重新平衡分片
|
||||
|
||||
@@ -899,15 +900,15 @@ class TaskQueue:
|
||||
- 任务状态追踪和重试机制
|
||||
"""
|
||||
|
||||
def __init__(self, redis_url: Optional[str] = None, db_path: str = "insightflow.db"):
|
||||
def __init__(self, redis_url: str | None = None, db_path: str = "insightflow.db"):
|
||||
self.db_path = db_path
|
||||
self.redis_url = redis_url
|
||||
self.celery_app = None
|
||||
self.use_celery = False
|
||||
|
||||
# 内存任务存储(非 Celery 模式)
|
||||
self.tasks: Dict[str, TaskInfo] = {}
|
||||
self.task_handlers: Dict[str, Callable] = {}
|
||||
self.tasks: dict[str, TaskInfo] = {}
|
||||
self.task_handlers: dict[str, Callable] = {}
|
||||
self.task_lock = threading.RLock()
|
||||
|
||||
# 初始化任务队列表
|
||||
@@ -956,7 +957,7 @@ class TaskQueue:
|
||||
"""注册任务处理器"""
|
||||
self.task_handlers[task_type] = handler
|
||||
|
||||
def submit(self, task_type: str, payload: Dict, max_retries: int = 3) -> str:
|
||||
def submit(self, task_type: str, payload: dict, max_retries: int = 3) -> str:
|
||||
"""
|
||||
提交任务
|
||||
|
||||
@@ -1112,7 +1113,7 @@ class TaskQueue:
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def get_status(self, task_id: str) -> Optional[TaskInfo]:
|
||||
def get_status(self, task_id: str) -> TaskInfo | None:
|
||||
"""获取任务状态"""
|
||||
if self.use_celery:
|
||||
try:
|
||||
@@ -1143,8 +1144,8 @@ class TaskQueue:
|
||||
return self.tasks.get(task_id)
|
||||
|
||||
def list_tasks(
|
||||
self, status: Optional[str] = None, task_type: Optional[str] = None, limit: int = 100
|
||||
) -> List[TaskInfo]:
|
||||
self, status: str | None = None, task_type: str | None = None, limit: int = 100
|
||||
) -> list[TaskInfo]:
|
||||
"""列出任务"""
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
@@ -1233,7 +1234,7 @@ class TaskQueue:
|
||||
self._update_task_status(task)
|
||||
return True
|
||||
|
||||
def get_stats(self) -> Dict:
|
||||
def get_stats(self) -> dict:
|
||||
"""获取任务队列统计"""
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
|
||||
@@ -1290,15 +1291,15 @@ class PerformanceMonitor:
|
||||
self.alert_threshold = alert_threshold
|
||||
|
||||
# 内存中的指标缓存
|
||||
self.metrics_buffer: List[PerformanceMetric] = []
|
||||
self.metrics_buffer: list[PerformanceMetric] = []
|
||||
self.buffer_lock = threading.RLock()
|
||||
self.buffer_size = 100
|
||||
|
||||
# 告警回调
|
||||
self.alert_handlers: List[Callable] = []
|
||||
self.alert_handlers: list[Callable] = []
|
||||
|
||||
def record_metric(
|
||||
self, metric_type: str, duration_ms: float, endpoint: Optional[str] = None, metadata: Optional[Dict] = None
|
||||
self, metric_type: str, duration_ms: float, endpoint: str | None = None, metadata: dict | None = None
|
||||
):
|
||||
"""
|
||||
记录性能指标
|
||||
@@ -1385,7 +1386,7 @@ class PerformanceMonitor:
|
||||
"""注册告警处理器"""
|
||||
self.alert_handlers.append(handler)
|
||||
|
||||
def get_stats(self, hours: int = 24) -> Dict:
|
||||
def get_stats(self, hours: int = 24) -> dict:
|
||||
"""
|
||||
获取性能统计
|
||||
|
||||
@@ -1504,7 +1505,7 @@ class PerformanceMonitor:
|
||||
],
|
||||
}
|
||||
|
||||
def get_api_performance(self, endpoint: Optional[str] = None, hours: int = 24) -> Dict:
|
||||
def get_api_performance(self, endpoint: str | None = None, hours: int = 24) -> dict:
|
||||
"""获取 API 性能详情"""
|
||||
self._flush_metrics()
|
||||
|
||||
@@ -1584,7 +1585,7 @@ class PerformanceMonitor:
|
||||
# ==================== 性能装饰器 ====================
|
||||
|
||||
|
||||
def cached(cache_manager: CacheManager, key_prefix: str = "", ttl: int = 3600, key_func: Optional[Callable] = None):
|
||||
def cached(cache_manager: CacheManager, key_prefix: str = "", ttl: int = 3600, key_func: Callable | None = None):
|
||||
"""
|
||||
缓存装饰器
|
||||
|
||||
@@ -1624,7 +1625,7 @@ def cached(cache_manager: CacheManager, key_prefix: str = "", ttl: int = 3600, k
|
||||
return decorator
|
||||
|
||||
|
||||
def monitored(monitor: PerformanceMonitor, metric_type: str, endpoint: Optional[str] = None):
|
||||
def monitored(monitor: PerformanceMonitor, metric_type: str, endpoint: str | None = None):
|
||||
"""
|
||||
性能监控装饰器
|
||||
|
||||
@@ -1662,7 +1663,7 @@ class PerformanceManager:
|
||||
整合缓存管理、数据库分片、任务队列和性能监控功能
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str = "insightflow.db", redis_url: Optional[str] = None, enable_sharding: bool = False):
|
||||
def __init__(self, db_path: str = "insightflow.db", redis_url: str | None = None, enable_sharding: bool = False):
|
||||
self.db_path = db_path
|
||||
|
||||
# 初始化各模块
|
||||
@@ -1674,7 +1675,7 @@ class PerformanceManager:
|
||||
|
||||
self.monitor = PerformanceMonitor(db_path=db_path)
|
||||
|
||||
def get_health_status(self) -> Dict:
|
||||
def get_health_status(self) -> dict:
|
||||
"""获取系统健康状态"""
|
||||
return {
|
||||
"cache": {
|
||||
@@ -1698,7 +1699,7 @@ class PerformanceManager:
|
||||
},
|
||||
}
|
||||
|
||||
def get_full_stats(self) -> Dict:
|
||||
def get_full_stats(self) -> dict:
|
||||
"""获取完整统计信息"""
|
||||
stats = {
|
||||
"cache": self.cache.get_stats(),
|
||||
@@ -1717,7 +1718,7 @@ _performance_manager = None
|
||||
|
||||
|
||||
def get_performance_manager(
|
||||
db_path: str = "insightflow.db", redis_url: Optional[str] = None, enable_sharding: bool = False
|
||||
db_path: str = "insightflow.db", redis_url: str | None = None, enable_sharding: bool = False
|
||||
) -> PerformanceManager:
|
||||
"""获取性能管理器单例"""
|
||||
global _performance_manager
|
||||
|
||||
Reference in New Issue
Block a user