insightflow/backend/schema_multimodal.sql

-- Phase 7: 多模态支持相关表

-- 视频表
CREATE TABLE IF NOT EXISTS videos (
    id TEXT PRIMARY KEY,
    project_id TEXT NOT NULL,
    filename TEXT NOT NULL,
    file_path TEXT,
    duration REAL,  -- 视频时长（秒）
    width INTEGER,  -- 视频宽度
    height INTEGER,  -- 视频高度
    fps REAL,  -- 帧率
    audio_extracted INTEGER DEFAULT 0,  -- 是否已提取音频
    audio_path TEXT,  -- 提取的音频文件路径
    transcript_id TEXT,  -- 关联的转录记录ID
    status TEXT DEFAULT 'pending',  -- pending, processing, completed, failed
    error_message TEXT,
    metadata TEXT,  -- JSON: 其他元数据
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    FOREIGN KEY (project_id) REFERENCES projects(id),
    FOREIGN KEY (transcript_id) REFERENCES transcripts(id)
);

-- 视频关键帧表
CREATE TABLE IF NOT EXISTS video_frames (
    id TEXT PRIMARY KEY,
    video_id TEXT NOT NULL,
    frame_number INTEGER NOT NULL,
    timestamp REAL NOT NULL,  -- 帧时间戳（秒）
    frame_path TEXT NOT NULL,  -- 帧图片路径
    ocr_text TEXT,  -- OCR识别的文字
    ocr_confidence REAL,  -- OCR置信度
    entities_detected TEXT,  -- JSON: 检测到的实体
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    FOREIGN KEY (video_id) REFERENCES videos(id) ON DELETE CASCADE
);

-- 图片表
CREATE TABLE IF NOT EXISTS images (
    id TEXT PRIMARY KEY,
    project_id TEXT NOT NULL,
    filename TEXT NOT NULL,
    file_path TEXT,
    image_type TEXT,  -- whiteboard, ppt, handwritten, screenshot, other
    width INTEGER,
    height INTEGER,
    ocr_text TEXT,  -- OCR识别的文字
    description TEXT,  -- 图片描述（LLM生成）
    entities_detected TEXT,  -- JSON: 检测到的实体
    relations_detected TEXT,  -- JSON: 检测到的关系
    transcript_id TEXT,  -- 关联的转录记录ID（可选）
    status TEXT DEFAULT 'pending',  -- pending, processing, completed, failed
    error_message TEXT,
    metadata TEXT,  -- JSON: 其他元数据
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    FOREIGN KEY (project_id) REFERENCES projects(id),
    FOREIGN KEY (transcript_id) REFERENCES transcripts(id)
);

-- 多模态实体关联表
CREATE TABLE IF NOT EXISTS multimodal_entities (
    id TEXT PRIMARY KEY,
    project_id TEXT NOT NULL,
    entity_id TEXT NOT NULL,  -- 关联的实体ID
    source_type TEXT NOT NULL,  -- audio, video, image, document
    source_id TEXT NOT NULL,  -- 来源ID（transcript_id, video_id, image_id）
    mention_context TEXT,  -- 提及上下文
    confidence REAL DEFAULT 1.0,
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    FOREIGN KEY (project_id) REFERENCES projects(id),
    FOREIGN KEY (entity_id) REFERENCES entities(id),
    UNIQUE(entity_id, source_type, source_id)
);

-- 多模态实体对齐表（跨模态实体关联）
CREATE TABLE IF NOT EXISTS multimodal_entity_links (
    id TEXT PRIMARY KEY,
    project_id TEXT NOT NULL,
    source_entity_id TEXT NOT NULL,  -- 源实体ID
    target_entity_id TEXT NOT NULL,  -- 目标实体ID
    link_type TEXT NOT NULL,  -- same_as, related_to, part_of
    source_modality TEXT NOT NULL,  -- audio, video, image, document
    target_modality TEXT NOT NULL,  -- audio, video, image, document
    confidence REAL DEFAULT 1.0,
    evidence TEXT,  -- 关联证据
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    FOREIGN KEY (project_id) REFERENCES projects(id),
    FOREIGN KEY (source_entity_id) REFERENCES entities(id),
    FOREIGN KEY (target_entity_id) REFERENCES entities(id)
);

-- 创建索引
CREATE INDEX IF NOT EXISTS idx_videos_project ON videos(project_id);
CREATE INDEX IF NOT EXISTS idx_videos_status ON videos(status);
CREATE INDEX IF NOT EXISTS idx_video_frames_video ON video_frames(video_id);
CREATE INDEX IF NOT EXISTS idx_video_frames_timestamp ON video_frames(timestamp);
CREATE INDEX IF NOT EXISTS idx_images_project ON images(project_id);
CREATE INDEX IF NOT EXISTS idx_images_type ON images(image_type);
CREATE INDEX IF NOT EXISTS idx_images_status ON images(status);
CREATE INDEX IF NOT EXISTS idx_multimodal_entities_project ON multimodal_entities(project_id);
CREATE INDEX IF NOT EXISTS idx_multimodal_entities_entity ON multimodal_entities(entity_id);
CREATE INDEX IF NOT EXISTS idx_multimodal_entity_links_project ON multimodal_entity_links(project_id);