claw/insightflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: auto-fix code issues (cron)

Browse Source 
- 修复重复导入/字段
- 修复异常处理
- 修复PEP8格式问题
- 添加类型注解
- 修复重复函数定义 (health_check, create_webhook_endpoint, etc)
- 修复未定义名称 (SearchOperator, TenantTier, Query, Body, logger)
- 修复 workflow_manager.py 的类定义重复问题
- 添加缺失的导入
This commit is contained in:
OpenClaw Bot
2026-02-27 09:18:58 +08:00
parent 1d55ae8f1e
commit be22b763fa
39 changed files with 12535 additions and 10327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

357
backend/security_manager.py
View File

@@ -3,7 +3,6 @@ InsightFlow Phase 7 Task 3: 数据安全与合规模块
Security Manager - 端到端加密、数据脱敏、审计日志
"""
import os
import json
import hashlib
import secrets
@@ -83,7 +82,7 @@ class AuditLog:
success: bool = True
error_message: Optional[str] = None
created_at: str = field(default_factory=lambda: datetime.now().isoformat())
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
@@ -100,7 +99,7 @@ class EncryptionConfig:
salt: Optional[str] = None
created_at: str = field(default_factory=lambda: datetime.now().isoformat())
updated_at: str = field(default_factory=lambda: datetime.now().isoformat())
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
@@ -119,7 +118,7 @@ class MaskingRule:
description: Optional[str] = None
created_at: str = field(default_factory=lambda: datetime.now().isoformat())
updated_at: str = field(default_factory=lambda: datetime.now().isoformat())
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
@@ -140,7 +139,7 @@ class DataAccessPolicy:
is_active: bool = True
created_at: str = field(default_factory=lambda: datetime.now().isoformat())
updated_at: str = field(default_factory=lambda: datetime.now().isoformat())
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
@@ -157,14 +156,14 @@ class AccessRequest:
approved_at: Optional[str] = None
expires_at: Optional[str] = None
created_at: str = field(default_factory=lambda: datetime.now().isoformat())
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
class SecurityManager:
"""安全管理器"""
# 预定义脱敏规则
DEFAULT_MASKING_RULES = {
MaskingRuleType.PHONE: {
@@ -192,17 +191,20 @@ class SecurityManager:
"replacement": r"\1\2***"
}
}
def __init__(self, db_path: str = "insightflow.db"):
self.db_path = db_path
self.db_path = db_path
# 预编译正则缓存
self._compiled_patterns: Dict[str, re.Pattern] = {}
self._local = {}
self._init_db()
def _init_db(self):
"""初始化数据库表"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# 审计日志表
cursor.execute("""
CREATE TABLE IF NOT EXISTS audit_logs (
@@ -221,7 +223,7 @@ class SecurityManager:
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
# 加密配置表
cursor.execute("""
CREATE TABLE IF NOT EXISTS encryption_configs (
@@ -237,7 +239,7 @@ class SecurityManager:
FOREIGN KEY (project_id) REFERENCES projects(id)
)
""")
# 脱敏规则表
cursor.execute("""
CREATE TABLE IF NOT EXISTS masking_rules (
@@ -255,7 +257,7 @@ class SecurityManager:
FOREIGN KEY (project_id) REFERENCES projects(id)
)
""")
# 数据访问策略表
cursor.execute("""
CREATE TABLE IF NOT EXISTS data_access_policies (
@@ -275,7 +277,7 @@ class SecurityManager:
FOREIGN KEY (project_id) REFERENCES projects(id)
)
""")
# 访问请求表
cursor.execute("""
CREATE TABLE IF NOT EXISTS access_requests (
@@ -291,7 +293,7 @@ class SecurityManager:
FOREIGN KEY (policy_id) REFERENCES data_access_policies(id)
)
""")
# 创建索引
cursor.execute("CREATE INDEX IF NOT EXISTS idx_audit_logs_user ON audit_logs(user_id)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_audit_logs_resource ON audit_logs(resource_type, resource_id)")
@@ -300,18 +302,18 @@ class SecurityManager:
cursor.execute("CREATE INDEX IF NOT EXISTS idx_encryption_project ON encryption_configs(project_id)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_masking_project ON masking_rules(project_id)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_access_policy_project ON data_access_policies(project_id)")
conn.commit()
conn.close()
def _generate_id(self) -> str:
"""生成唯一ID"""
return hashlib.sha256(
f"{datetime.now().isoformat()}{secrets.token_hex(16)}".encode()
).hexdigest()[:32]
# ==================== 审计日志 ====================
def log_audit(
self,
action_type: AuditActionType,
@@ -341,11 +343,11 @@ class SecurityManager:
success=success,
error_message=error_message
)
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
INSERT INTO audit_logs
INSERT INTO audit_logs
(id, action_type, user_id, user_ip, user_agent, resource_type, resource_id,
action_details, before_value, after_value, success, error_message, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
@@ -357,9 +359,9 @@ class SecurityManager:
))
conn.commit()
conn.close()
return log
def get_audit_logs(
self,
user_id: Optional[str] = None,
@@ -375,10 +377,10 @@ class SecurityManager:
"""查询审计日志"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
query = "SELECT * FROM audit_logs WHERE 1=1"
params = []
if user_id:
query += " AND user_id = ?"
params.append(user_id)
@@ -400,26 +402,19 @@ class SecurityManager:
if success is not None:
query += " AND success = ?"
params.append(int(success))
query += " ORDER BY created_at DESC LIMIT ? OFFSET ?"
params.extend([limit, offset])
cursor.execute(query, params)
rows = cursor.fetchall()
conn.close()
logs = []
for row in cursor.description:
col_names = [desc[0] for desc in cursor.description]
break
else:
col_names = [desc[0] for desc in cursor.description] if cursor.description else []
if not col_names:
return logs
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute(query, params)
rows = cursor.fetchall()
for row in rows:
log = AuditLog(
id=row[0],
@@ -437,10 +432,10 @@ class SecurityManager:
created_at=row[12]
)
logs.append(log)
conn.close()
return logs
def get_audit_stats(
self,
start_time: Optional[str] = None,
@@ -449,54 +444,54 @@ class SecurityManager:
"""获取审计统计"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
query = "SELECT action_type, success, COUNT(*) FROM audit_logs WHERE 1=1"
params = []
if start_time:
query += " AND created_at >= ?"
params.append(start_time)
if end_time:
query += " AND created_at <= ?"
params.append(end_time)
query += " GROUP BY action_type, success"
cursor.execute(query, params)
rows = cursor.fetchall()
stats = {
"total_actions": 0,
"success_count": 0,
"failure_count": 0,
"action_breakdown": {}
}
for action_type, success, count in rows:
stats["total_actions"] += count
if success:
stats["success_count"] += count
else:
stats["failure_count"] += count
if action_type not in stats["action_breakdown"]:
stats["action_breakdown"][action_type] = {"success": 0, "failure": 0}
if success:
stats["action_breakdown"][action_type]["success"] += count
else:
stats["action_breakdown"][action_type]["failure"] += count
conn.close()
return stats
# ==================== 端到端加密 ====================
def _derive_key(self, password: str, salt: bytes) -> bytes:
"""从密码派生密钥"""
if not CRYPTO_AVAILABLE:
raise RuntimeError("cryptography library not available")
kdf = PBKDF2HMAC(
algorithm=hashes.SHA256(),
length=32,
@@ -504,7 +499,7 @@ class SecurityManager:
iterations=100000,
)
return base64.urlsafe_b64encode(kdf.derive(password.encode()))
def enable_encryption(
self,
project_id: str,
@@ -513,14 +508,14 @@ class SecurityManager:
"""启用项目加密"""
if not CRYPTO_AVAILABLE:
raise RuntimeError("cryptography library not available")
# 生成盐值
salt = secrets.token_hex(16)
# 派生密钥并哈希（用于验证）
key = self._derive_key(master_password, salt.encode())
key_hash = hashlib.sha256(key).hexdigest()
config = EncryptionConfig(
id=self._generate_id(),
project_id=project_id,
@@ -530,20 +525,20 @@ class SecurityManager:
master_key_hash=key_hash,
salt=salt
)
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# 检查是否已存在配置
cursor.execute(
"SELECT id FROM encryption_configs WHERE project_id = ?",
(project_id,)
)
existing = cursor.fetchone()
if existing:
cursor.execute("""
UPDATE encryption_configs
UPDATE encryption_configs
SET is_enabled = 1, encryption_type = ?, key_derivation = ?,
master_key_hash = ?, salt = ?, updated_at = ?
WHERE project_id = ?
@@ -555,7 +550,7 @@ class SecurityManager:
config.id = existing[0]
else:
cursor.execute("""
INSERT INTO encryption_configs
INSERT INTO encryption_configs
(id, project_id, is_enabled, encryption_type, key_derivation,
master_key_hash, salt, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
@@ -565,10 +560,10 @@ class SecurityManager:
config.master_key_hash, config.salt,
config.created_at, config.updated_at
))
conn.commit()
conn.close()
# 记录审计日志
self.log_audit(
action_type=AuditActionType.ENCRYPTION_ENABLE,
@@ -576,9 +571,9 @@ class SecurityManager:
resource_id=project_id,
action_details={"encryption_type": config.encryption_type}
)
return config
def disable_encryption(
self,
project_id: str,
@@ -588,28 +583,28 @@ class SecurityManager:
# 验证密码
if not self.verify_encryption_password(project_id, master_password):
return False
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
UPDATE encryption_configs
UPDATE encryption_configs
SET is_enabled = 0, updated_at = ?
WHERE project_id = ?
""", (datetime.now().isoformat(), project_id))
conn.commit()
conn.close()
# 记录审计日志
self.log_audit(
action_type=AuditActionType.ENCRYPTION_DISABLE,
resource_type="project",
resource_id=project_id
)
return True
def verify_encryption_password(
self,
project_id: str,
@@ -618,41 +613,41 @@ class SecurityManager:
"""验证加密密码"""
if not CRYPTO_AVAILABLE:
return False
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute(
"SELECT master_key_hash, salt FROM encryption_configs WHERE project_id = ?",
(project_id,)
)
row = cursor.fetchone()
conn.close()
if not row:
return False
stored_hash, salt = row
key = self._derive_key(password, salt.encode())
key_hash = hashlib.sha256(key).hexdigest()
return key_hash == stored_hash
def get_encryption_config(self, project_id: str) -> Optional[EncryptionConfig]:
"""获取加密配置"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute(
"SELECT * FROM encryption_configs WHERE project_id = ?",
(project_id,)
)
row = cursor.fetchone()
conn.close()
if not row:
return None
return EncryptionConfig(
id=row[0],
project_id=row[1],
@@ -664,7 +659,7 @@ class SecurityManager:
created_at=row[7],
updated_at=row[8]
)
def encrypt_data(
self,
data: str,
@@ -674,16 +669,16 @@ class SecurityManager:
"""加密数据"""
if not CRYPTO_AVAILABLE:
raise RuntimeError("cryptography library not available")
if salt is None:
salt = secrets.token_hex(16)
key = self._derive_key(password, salt.encode())
f = Fernet(key)
encrypted = f.encrypt(data.encode())
return base64.b64encode(encrypted).decode(), salt
def decrypt_data(
self,
encrypted_data: str,
@@ -693,15 +688,15 @@ class SecurityManager:
"""解密数据"""
if not CRYPTO_AVAILABLE:
raise RuntimeError("cryptography library not available")
key = self._derive_key(password, salt.encode())
f = Fernet(key)
decrypted = f.decrypt(base64.b64decode(encrypted_data))
return decrypted.decode()
# ==================== 数据脱敏 ====================
def create_masking_rule(
self,
project_id: str,
@@ -718,7 +713,7 @@ class SecurityManager:
default = self.DEFAULT_MASKING_RULES[rule_type]
pattern = default["pattern"]
replacement = replacement or default["replacement"]
rule = MaskingRule(
id=self._generate_id(),
project_id=project_id,
@@ -729,12 +724,12 @@ class SecurityManager:
description=description,
priority=priority
)
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
INSERT INTO masking_rules
INSERT INTO masking_rules
(id, project_id, name, rule_type, pattern, replacement,
is_active, priority, description, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
@@ -743,10 +738,10 @@ class SecurityManager:
rule.pattern, rule.replacement, int(rule.is_active),
rule.priority, rule.description, rule.created_at, rule.updated_at
))
conn.commit()
conn.close()
# 记录审计日志
self.log_audit(
action_type=AuditActionType.DATA_MASKING,
@@ -754,9 +749,9 @@ class SecurityManager:
resource_id=project_id,
action_details={"action": "create_rule", "rule_name": name}
)
return rule
def get_masking_rules(
self,
project_id: str,
@@ -765,19 +760,19 @@ class SecurityManager:
"""获取脱敏规则"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
query = "SELECT * FROM masking_rules WHERE project_id = ?"
params = [project_id]
if active_only:
query += " AND is_active = 1"
query += " ORDER BY priority DESC"
cursor.execute(query, params)
rows = cursor.fetchall()
conn.close()
rules = []
for row in rows:
rules.append(MaskingRule(
@@ -793,9 +788,9 @@ class SecurityManager:
created_at=row[9],
updated_at=row[10]
))
return rules
def update_masking_rule(
self,
rule_id: str,
@@ -803,45 +798,45 @@ class SecurityManager:
) -> Optional[MaskingRule]:
"""更新脱敏规则"""
allowed_fields = ["name", "pattern", "replacement", "is_active", "priority", "description"]
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
set_clauses = []
params = []
for key, value in kwargs.items():
if key in allowed_fields:
set_clauses.append(f"{key} = ?")
params.append(int(value) if key == "is_active" else value)
if not set_clauses:
conn.close()
return None
set_clauses.append("updated_at = ?")
params.append(datetime.now().isoformat())
params.append(rule_id)
cursor.execute(f"""
UPDATE masking_rules
UPDATE masking_rules
SET {', '.join(set_clauses)}
WHERE id = ?
""", params)
conn.commit()
conn.close()
# 获取更新后的规则
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("SELECT * FROM masking_rules WHERE id = ?", (rule_id,))
row = cursor.fetchone()
conn.close()
if not row:
return None
return MaskingRule(
id=row[0],
project_id=row[1],
@@ -855,20 +850,20 @@ class SecurityManager:
created_at=row[9],
updated_at=row[10]
)
def delete_masking_rule(self, rule_id: str) -> bool:
"""删除脱敏规则"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("DELETE FROM masking_rules WHERE id = ?", (rule_id,))
success = cursor.rowcount > 0
conn.commit()
conn.close()
return success
def apply_masking(
self,
text: str,
@@ -877,17 +872,17 @@ class SecurityManager:
) -> str:
"""应用脱敏规则到文本"""
rules = self.get_masking_rules(project_id)
if not rules:
return text
masked_text = text
for rule in rules:
# 如果指定了规则类型，只应用指定类型的规则
if rule_types and MaskingRuleType(rule.rule_type) not in rule_types:
continue
try:
masked_text = re.sub(
rule.pattern,
@@ -897,9 +892,9 @@ class SecurityManager:
except re.error:
# 忽略无效的正则表达式
continue
return masked_text
def apply_masking_to_entity(
self,
entity_data: Dict[str, Any],
@@ -907,18 +902,18 @@ class SecurityManager:
) -> Dict[str, Any]:
"""对实体数据应用脱敏"""
masked_data = entity_data.copy()
# 对可能包含敏感信息的字段进行脱敏
sensitive_fields = ["name", "definition", "description", "value"]
for field in sensitive_fields:
if field in masked_data and isinstance(masked_data[field], str):
masked_data[field] = self.apply_masking(masked_data[field], project_id)
return masked_data
# ==================== 数据访问策略 ====================
def create_access_policy(
self,
project_id: str,
@@ -944,12 +939,12 @@ class SecurityManager:
max_access_count=max_access_count,
require_approval=require_approval
)
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
INSERT INTO data_access_policies
INSERT INTO data_access_policies
(id, project_id, name, description, allowed_users, allowed_roles,
allowed_ips, time_restrictions, max_access_count, require_approval,
is_active, created_at, updated_at)
@@ -961,12 +956,12 @@ class SecurityManager:
int(policy.require_approval), int(policy.is_active),
policy.created_at, policy.updated_at
))
conn.commit()
conn.close()
return policy
def get_access_policies(
self,
project_id: str,
@@ -975,17 +970,17 @@ class SecurityManager:
"""获取数据访问策略"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
query = "SELECT * FROM data_access_policies WHERE project_id = ?"
params = [project_id]
if active_only:
query += " AND is_active = 1"
cursor.execute(query, params)
rows = cursor.fetchall()
conn.close()
policies = []
for row in rows:
policies.append(DataAccessPolicy(
@@ -1003,9 +998,9 @@ class SecurityManager:
created_at=row[11],
updated_at=row[12]
))
return policies
def check_access_permission(
self,
policy_id: str,
@@ -1015,17 +1010,17 @@ class SecurityManager:
"""检查访问权限"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute(
"SELECT * FROM data_access_policies WHERE id = ? AND is_active = 1",
(policy_id,)
)
row = cursor.fetchone()
conn.close()
if not row:
return False, "Policy not found or inactive"
policy = DataAccessPolicy(
id=row[0],
project_id=row[1],
@@ -1041,13 +1036,13 @@ class SecurityManager:
created_at=row[11],
updated_at=row[12]
)
# 检查用户白名单
if policy.allowed_users:
allowed = json.loads(policy.allowed_users)
if user_id not in allowed:
return False, "User not in allowed list"
# 检查IP白名单
if policy.allowed_ips and user_ip:
allowed_ips = json.loads(policy.allowed_ips)
@@ -1058,45 +1053,45 @@ class SecurityManager:
break
if not ip_allowed:
return False, "IP not in allowed list"
# 检查时间限制
if policy.time_restrictions:
restrictions = json.loads(policy.time_restrictions)
now = datetime.now()
if "start_time" in restrictions and "end_time" in restrictions:
current_time = now.strftime("%H:%M")
if not (restrictions["start_time"] <= current_time <= restrictions["end_time"]):
return False, "Access not allowed at this time"
if "days_of_week" in restrictions:
if now.weekday() not in restrictions["days_of_week"]:
return False, "Access not allowed on this day"
# 检查是否需要审批
if policy.require_approval:
# 检查是否有有效的访问请求
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
SELECT * FROM access_requests
SELECT * FROM access_requests
WHERE policy_id = ? AND user_id = ? AND status = 'approved'
AND (expires_at IS NULL OR expires_at > ?)
""", (policy_id, user_id, datetime.now().isoformat()))
request = cursor.fetchone()
conn.close()
if not request:
return False, "Access requires approval"
return True, None
def _match_ip_pattern(self, ip: str, pattern: str) -> bool:
"""匹配IP模式（支持CIDR）"""
import ipaddress
try:
if "/" in pattern:
# CIDR 表示法
@@ -1107,7 +1102,7 @@ class SecurityManager:
return ip == pattern
except ValueError:
return ip == pattern
def create_access_request(
self,
policy_id: str,
@@ -1123,12 +1118,12 @@ class SecurityManager:
request_reason=request_reason,
expires_at=(datetime.now() + timedelta(hours=expires_hours)).isoformat()
)
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
INSERT INTO access_requests
INSERT INTO access_requests
(id, policy_id, user_id, request_reason, status, expires_at, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
""", (
@@ -1136,12 +1131,12 @@ class SecurityManager:
request.request_reason, request.status, request.expires_at,
request.created_at
))
conn.commit()
conn.close()
return request
def approve_access_request(
self,
request_id: str,
@@ -1151,26 +1146,26 @@ class SecurityManager:
"""批准访问请求"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
expires_at = (datetime.now() + timedelta(hours=expires_hours)).isoformat()
approved_at = datetime.now().isoformat()
cursor.execute("""
UPDATE access_requests
UPDATE access_requests
SET status = 'approved', approved_by = ?, approved_at = ?, expires_at = ?
WHERE id = ?
""", (approved_by, approved_at, expires_at, request_id))
conn.commit()
# 获取更新后的请求
cursor.execute("SELECT * FROM access_requests WHERE id = ?", (request_id,))
row = cursor.fetchone()
conn.close()
if not row:
return None
return AccessRequest(
id=row[0],
policy_id=row[1],
@@ -1182,7 +1177,7 @@ class SecurityManager:
expires_at=row[7],
created_at=row[8]
)
def reject_access_request(
self,
request_id: str,
@@ -1191,22 +1186,22 @@ class SecurityManager:
"""拒绝访问请求"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
UPDATE access_requests
UPDATE access_requests
SET status = 'rejected', approved_by = ?
WHERE id = ?
""", (rejected_by, request_id))
conn.commit()
cursor.execute("SELECT * FROM access_requests WHERE id = ?", (request_id,))
row = cursor.fetchone()
conn.close()
if not row:
return None
return AccessRequest(
id=row[0],
policy_id=row[1],