diff --git a/README.md b/README.md index 70720d1..af4451d 100644 --- a/README.md +++ b/README.md @@ -332,12 +332,18 @@ docker compose run --rm api Useful API endpoints for generated files: - `POST /webdav/scan` +- `GET /runs` +- `GET /videos/{video_id}/runs` - `GET /videos/{video_id}/artifacts` - `GET /clips/{clip_id}/artifacts` - `GET /artifacts/{artifact_id}` +- `GET /system/logs` +- `GET /system/logs/{service}?tail=300` Thumbnail outputs are visible through the artifact endpoints. Upload command payloads include `thumbnail` when a `thumbnail_final` artifact exists. +The frontend operations view uses the run, artifact, and log endpoints to show per-video pipeline history and recent Compose container logs. The API service mounts `/var/run/docker.sock` read-only so it can read logs for `api`, `frontend`, `scheduler`, `worker-ai`, `worker-media`, and `redis`; keep that endpoint LAN-only or behind trusted access. + ## Custom LLM Prompt Edit `[llm_prompt]` in your private `config.toml` to tune clip selection without rebuilding containers: diff --git a/docker-compose.yml b/docker-compose.yml index c4b3b77..c4d7752 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,7 @@ services: volumes: - ./config.toml:/etc/evanescere/config.toml:ro - ./storage:/data/evanescere + - /var/run/docker.sock:/var/run/docker.sock:ro depends_on: - redis diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 5719c80..04e87b7 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,21 +1,29 @@ import { + Activity, Check, Clapperboard, + FileArchive, FolderSync, Play, RefreshCw, Save, Send, Settings2, + Terminal, Upload, } from "lucide-react"; import { useCallback, useEffect, useMemo, useState } from "react"; import { apiBaseUrl, approveClip, + getContainerLog, getClips, + getLogServices, + getRuns, getSettings, getTranscript, + getVideoArtifacts, + getVideoRuns, getVideos, patchSettings, renderClip, @@ -23,7 +31,16 @@ import { scanWebDav, uploadClip, } from "./api"; -import type { ClipSuggestion, PipelineSettings, TranscriptSegment, Video } from "./types"; +import type { + Artifact, + ClipSuggestion, + ContainerLog, + LogService, + PipelineRun, + PipelineSettings, + TranscriptSegment, + Video, +} from "./types"; const settingLabels: Record = { suggest_enabled: "Suggest", @@ -48,6 +65,10 @@ function formatTimeRange(start: number, end: number) { return `${formatDuration(start)}-${formatDuration(end)}`; } +function formatDateTime(value: string) { + return new Date(value).toLocaleString(); +} + function statusTone(value: string) { if (["done", "stable", "existing_done", "approved", "auto_approved"].includes(value)) return "good"; if (["failed", "error"].includes(value)) return "bad"; @@ -61,6 +82,13 @@ export function App() { const [selectedVideoId, setSelectedVideoId] = useState(null); const [transcript, setTranscript] = useState([]); const [clips, setClips] = useState([]); + const [runs, setRuns] = useState([]); + const [latestRuns, setLatestRuns] = useState([]); + const [artifacts, setArtifacts] = useState([]); + const [logServices, setLogServices] = useState([]); + const [selectedLogService, setSelectedLogService] = useState("worker-media"); + const [containerLog, setContainerLog] = useState(null); + const [logsLoading, setLogsLoading] = useState(false); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); const [notice, setNotice] = useState(null); @@ -71,18 +99,45 @@ export function App() { ); const loadSelected = useCallback(async (videoId: number) => { - const [nextTranscript, nextClips] = await Promise.all([getTranscript(videoId), getClips(videoId)]); + const [nextTranscript, nextClips, nextRuns, nextArtifacts] = await Promise.all([ + getTranscript(videoId), + getClips(videoId), + getVideoRuns(videoId), + getVideoArtifacts(videoId), + ]); setTranscript(nextTranscript); setClips(nextClips); + setRuns(nextRuns); + setArtifacts(nextArtifacts); }, []); + const loadLogs = useCallback(async (service = selectedLogService) => { + setLogsLoading(true); + setError(null); + try { + const nextLog = await getContainerLog(service, 300); + setContainerLog(nextLog); + } catch (caught) { + setError(caught instanceof Error ? caught.message : "Unknown error"); + } finally { + setLogsLoading(false); + } + }, [selectedLogService]); + const refresh = useCallback(async () => { setLoading(true); setError(null); try { - const [nextSettings, nextVideos] = await Promise.all([getSettings(), getVideos()]); + const [nextSettings, nextVideos, nextLogServices, nextLatestRuns] = await Promise.all([ + getSettings(), + getVideos(), + getLogServices(), + getRuns(20), + ]); setSettings(nextSettings); setVideos(nextVideos); + setLogServices(nextLogServices); + setLatestRuns(nextLatestRuns); const targetVideoId = selectedVideoId ?? nextVideos[0]?.id ?? null; setSelectedVideoId(targetVideoId); if (targetVideoId !== null) { @@ -90,6 +145,8 @@ export function App() { } else { setTranscript([]); setClips([]); + setRuns([]); + setArtifacts([]); } } catch (caught) { setError(caught instanceof Error ? caught.message : "Unknown error"); @@ -102,6 +159,13 @@ export function App() { void refresh(); }, [refresh]); + useEffect(() => { + if (logServices.length === 0) return; + if (!logServices.some((entry) => entry.service === selectedLogService)) { + setSelectedLogService(logServices[0].service); + } + }, [logServices, selectedLogService]); + async function handleSelect(videoId: number) { setSelectedVideoId(videoId); setLoading(true); @@ -152,6 +216,11 @@ export function App() { await withRefresh(() => patchSettings(settings)); } + function chooseLogService(service: string) { + setSelectedLogService(service); + void loadLogs(service); + } + return (
@@ -198,6 +267,28 @@ export function App() { +
+
+ +

Recent Jobs

+
+
+ {latestRuns.slice(0, 8).map((run) => ( + + ))} + {latestRuns.length === 0 && No jobs recorded yet.} +
+
+
diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 7fed10a..1dcc3e5 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -1,6 +1,9 @@ import type { Artifact, ClipSuggestion, + ContainerLog, + LogService, + PipelineRun, PipelineSettings, TranscriptSegment, Video, @@ -60,6 +63,18 @@ export function getClips(videoId: number): Promise { return request(`/videos/${videoId}/clips`); } +export function getVideoRuns(videoId: number): Promise { + return request(`/videos/${videoId}/runs`); +} + +export function getRuns(limit = 50): Promise { + return request(`/runs?limit=${limit}`); +} + +export function getVideoArtifacts(videoId: number): Promise { + return request(`/videos/${videoId}/artifacts`); +} + export function getClipArtifacts(clipId: number): Promise { return request(`/clips/${clipId}/artifacts`); } @@ -75,3 +90,11 @@ export function renderClip(clipId: number): Promise { export function uploadClip(clipId: number): Promise { return request(`/clips/${clipId}/upload`, { method: "POST" }); } + +export function getLogServices(): Promise { + return request("/system/logs"); +} + +export function getContainerLog(service: string, tail = 200): Promise { + return request(`/system/logs/${service}?tail=${tail}`); +} diff --git a/frontend/src/styles.css b/frontend/src/styles.css index e80d525..41b923e 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -152,7 +152,8 @@ button { .section-title h2, .detail-header h2, .transcript-pane h3, -.clips-pane h3 { +.clips-pane h3, +.section-title h3 { margin: 0; font-size: 16px; font-weight: 690; @@ -183,6 +184,50 @@ button { accent-color: #21756b; } +.jobs-band { + display: grid; + gap: 12px; +} + +.job-strip { + display: flex; + gap: 8px; + flex-wrap: wrap; + align-items: center; +} + +.job-chip { + display: inline-flex; + align-items: center; + gap: 7px; + min-height: 34px; + max-width: 280px; + border: 1px solid #ccd3db; + border-radius: 6px; + background: #ffffff; + color: #2f3b46; + padding: 0 9px; + cursor: pointer; +} + +.job-chip:hover { + border-color: #21756b; + background: #e9f3f1; +} + +.job-chip span { + color: #66737f; + font-size: 12px; +} + +.job-chip strong { + overflow: hidden; + font-size: 13px; + font-weight: 680; + text-overflow: ellipsis; + white-space: nowrap; +} + .workbench { display: grid; grid-template-columns: minmax(340px, 0.36fr) minmax(0, 1fr); @@ -267,6 +312,7 @@ tr.selected td { .metric-row { display: flex; gap: 10px; + flex-wrap: wrap; } .metric { @@ -290,6 +336,100 @@ tr.selected td { font-weight: 720; } +.ops-grid { + display: grid; + grid-template-columns: minmax(320px, 1fr) minmax(280px, 0.8fr); + gap: 18px; + margin-bottom: 18px; +} + +.ops-panel, +.logs-panel { + min-width: 0; + border: 1px solid #d8dee6; + border-radius: 8px; + background: #ffffff; + padding: 12px; +} + +.logs-panel { + margin-top: 18px; +} + +.panel-heading { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + margin-bottom: 10px; +} + +.compact-table { + overflow: auto; +} + +.compact-table table { + min-width: 520px; +} + +.run-error { + max-width: 420px; + margin-top: 6px; + color: #a02717; + font-size: 12px; + overflow-wrap: anywhere; +} + +.empty-cell, +.empty-block { + color: #66737f; + font-size: 13px; +} + +.empty-block { + border: 1px dashed #ccd3db; + border-radius: 6px; + padding: 14px; +} + +.artifact-list { + display: grid; + gap: 8px; + max-height: 252px; + overflow: auto; +} + +.artifact-row { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 12px; + border-top: 1px solid #edf0f3; + padding-top: 8px; +} + +.artifact-row:first-child { + border-top: 0; + padding-top: 0; +} + +.artifact-row strong { + color: #18222b; + font-size: 13px; +} + +.artifact-row p { + margin: 3px 0 0; + color: #4f5c68; + font-size: 12px; + overflow-wrap: anywhere; +} + +.artifact-row span { + color: #66737f; + font-size: 12px; + white-space: nowrap; +} + .split { display: grid; grid-template-columns: minmax(280px, 0.95fr) minmax(320px, 1.05fr); @@ -412,9 +552,50 @@ tr.selected td { color: #6f4d00; } +.log-service-row { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-bottom: 8px; +} + +.service-tab { + min-height: 32px; + border: 1px solid #ccd3db; + border-radius: 6px; + background: #fbfcfd; + color: #2f3b46; + padding: 0 10px; + cursor: pointer; +} + +.service-tab.active { + border-color: #21756b; + background: #e9f3f1; + color: #155c54; +} + +.log-output { + min-height: 240px; + max-height: 480px; + margin: 10px 0 0; + overflow: auto; + border: 1px solid #263340; + border-radius: 8px; + background: #101820; + color: #d9e7ee; + padding: 12px; + font-family: "JetBrains Mono", "SFMono-Regular", Consolas, monospace; + font-size: 12px; + line-height: 1.45; + white-space: pre-wrap; + overflow-wrap: anywhere; +} + @media (max-width: 1100px) { .workbench, - .split { + .split, + .ops-grid { grid-template-columns: 1fr; } diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 411a9bf..ef25ac0 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -48,6 +48,17 @@ export interface PipelineSettings { bake_subtitles: boolean; } +export interface PipelineRun { + id: number; + video_id: number; + trigger: string; + stage: string; + status: string; + error: string | null; + created_at: string; + updated_at: string; +} + export interface Artifact { id: number; video_id: number | null; @@ -67,3 +78,14 @@ export interface WebDavScanResult { baseline_inserted: number; baseline_marked_existing: number; } + +export interface LogService { + service: string; + container: string; +} + +export interface ContainerLog { + service: string; + container: string; + logs: string; +} diff --git a/src/evanescere/api.py b/src/evanescere/api.py index 59e12da..6f051b1 100644 --- a/src/evanescere/api.py +++ b/src/evanescere/api.py @@ -1,9 +1,8 @@ import logging import time -from fastapi import Depends, FastAPI, HTTPException +from fastapi import Depends, FastAPI, HTTPException, Query, Request from fastapi.middleware.cors import CORSMiddleware -from fastapi import Request from sqlalchemy import select from sqlalchemy.orm import Session @@ -16,6 +15,8 @@ from evanescere.models import Artifact, ClipSuggestion, PipelineRun, TranscriptS from evanescere.schemas import ( ArtifactRead, ClipSuggestionRead, + ContainerLogRead, + ContainerLogServiceRead, RunRead, SettingsPatch, SettingsRead, @@ -23,6 +24,11 @@ from evanescere.schemas import ( VideoRead, WebDavScanRead, ) +from evanescere.services.docker_logs import ( + DockerLogsError, + list_log_services, + read_container_logs, +) from evanescere.services.webdav import WebDavClient, safe_scan_once from evanescere.settings_store import get_pipeline_settings, patch_pipeline_settings @@ -111,6 +117,22 @@ def video_clips(video_id: int, db: Session = Depends(get_db)) -> list[ClipSugges ) +@app.get("/videos/{video_id}/runs", response_model=list[RunRead]) +def video_runs( + video_id: int, + limit: int = Query(default=50, ge=1, le=200), + db: Session = Depends(get_db), +) -> list[PipelineRun]: + return list( + db.scalars( + select(PipelineRun) + .where(PipelineRun.video_id == video_id) + .order_by(PipelineRun.updated_at.desc()) + .limit(limit) + ).all() + ) + + @app.get("/videos/{video_id}/artifacts", response_model=list[ArtifactRead]) def video_artifacts(video_id: int, db: Session = Depends(get_db)) -> list[Artifact]: return list( @@ -162,6 +184,23 @@ def upload_clip(clip_id: int, db: Session = Depends(get_db)) -> ClipSuggestion: return clip +@app.get("/runs", response_model=list[RunRead]) +def list_runs( + video_id: int | None = None, + limit: int = Query(default=50, ge=1, le=200), + db: Session = Depends(get_db), +) -> list[PipelineRun]: + query = select(PipelineRun).order_by(PipelineRun.updated_at.desc()).limit(limit) + if video_id is not None: + query = ( + select(PipelineRun) + .where(PipelineRun.video_id == video_id) + .order_by(PipelineRun.updated_at.desc()) + .limit(limit) + ) + return list(db.scalars(query).all()) + + @app.get("/runs/{run_id}", response_model=RunRead) def get_run(run_id: int, db: Session = Depends(get_db)) -> PipelineRun: run = db.get(PipelineRun, run_id) @@ -178,6 +217,28 @@ def get_artifact(artifact_id: int, db: Session = Depends(get_db)) -> Artifact: return artifact +@app.get("/system/logs", response_model=list[ContainerLogServiceRead]) +def system_log_services() -> list[ContainerLogServiceRead]: + return [ + ContainerLogServiceRead(service=entry.service, container=entry.container) + for entry in list_log_services() + ] + + +@app.get("/system/logs/{service}", response_model=ContainerLogRead) +def system_logs( + service: str, + tail: int = Query(default=200, ge=1, le=2000), +) -> ContainerLogRead: + try: + result = read_container_logs(service, tail=tail) + except KeyError as exc: + raise HTTPException(status_code=404, detail=f"Unknown log service: {service}") from exc + except DockerLogsError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + return ContainerLogRead(service=result.service, container=result.container, logs=result.logs) + + @app.get("/settings", response_model=SettingsRead) def read_settings(db: Session = Depends(get_db)) -> SettingsRead: return get_pipeline_settings(db) diff --git a/src/evanescere/schemas.py b/src/evanescere/schemas.py index 1efc2ba..eb0c870 100644 --- a/src/evanescere/schemas.py +++ b/src/evanescere/schemas.py @@ -103,6 +103,17 @@ class WebDavScanRead(BaseModel): baseline_marked_existing: int = 0 +class ContainerLogServiceRead(BaseModel): + service: str + container: str + + +class ContainerLogRead(BaseModel): + service: str + container: str + logs: str + + class ClipCandidate(BaseModel): start_sec: float = Field(ge=0) end_sec: float = Field(gt=0) diff --git a/src/evanescere/services/docker_logs.py b/src/evanescere/services/docker_logs.py new file mode 100644 index 0000000..7e80bbe --- /dev/null +++ b/src/evanescere/services/docker_logs.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass + +import httpx + + +DEFAULT_DOCKER_SOCKET = "/var/run/docker.sock" + +SERVICE_CONTAINERS: dict[str, str] = { + "api": "evanescere-api-1", + "frontend": "evanescere-frontend-1", + "scheduler": "evanescere-scheduler-1", + "worker-ai": "evanescere-worker-ai-1", + "worker-media": "evanescere-worker-media-1", + "redis": "evanescere-redis-1", +} + + +class DockerLogsError(RuntimeError): + pass + + +@dataclass(frozen=True) +class LogService: + service: str + container: str + + +@dataclass(frozen=True) +class ContainerLogs: + service: str + container: str + logs: str + + +def list_log_services() -> list[LogService]: + return [ + LogService(service=service, container=container) + for service, container in SERVICE_CONTAINERS.items() + ] + + +def read_container_logs( + service: str, + *, + tail: int = 200, + socket_path: str = DEFAULT_DOCKER_SOCKET, +) -> ContainerLogs: + container = SERVICE_CONTAINERS.get(service) + if container is None: + raise KeyError(service) + + if not os.path.exists(socket_path): + raise DockerLogsError( + f"Docker socket {socket_path} is not mounted into the API container." + ) + + transport = httpx.HTTPTransport(uds=socket_path) + params = { + "stdout": "1", + "stderr": "1", + "timestamps": "1", + "tail": str(tail), + } + with httpx.Client(transport=transport, base_url="http://docker", timeout=10.0) as client: + response = client.get(f"/containers/{container}/logs", params=params) + + if response.status_code == 404: + raise DockerLogsError(f"Container {container} was not found.") + if response.status_code >= 400: + raise DockerLogsError( + f"Docker logs request failed with HTTP {response.status_code}: {response.text}" + ) + + return ContainerLogs( + service=service, + container=container, + logs=_decode_docker_logs(response.content), + ) + + +def _decode_docker_logs(content: bytes) -> str: + frames: list[bytes] = [] + index = 0 + while index + 8 <= len(content): + frame_size = int.from_bytes(content[index + 4 : index + 8], "big") + frame_start = index + 8 + frame_end = frame_start + frame_size + if frame_size <= 0 or frame_end > len(content): + break + frames.append(content[frame_start:frame_end]) + index = frame_end + + if frames and index == len(content): + return b"".join(frames).decode("utf-8", errors="replace") + return content.decode("utf-8", errors="replace")