operation scripts

README.md

@@ -69,8 +69,8 @@ The checked-in `docker-compose.yml` mounts local `./config.toml` there for the A
 ## Services And Ports
 
 - Frontend: `http://localhost:3000`
-- Backend API: `http://localhost:8080`
+- Backend API: `http://localhost:8081`
-- API docs: `http://localhost:8080/docs`
+- API docs: `http://localhost:8081/docs`
 - Redis: `localhost:6379` when exposed by Compose
 - PostgreSQL: only started by Compose when using the `local-db` profile
 
@@ -145,7 +145,7 @@ The frontend reads the backend URL from `frontend/public/config.js` at runtime:
 
 ```js
 window.__EVANESCERE_FRONTEND_CONFIG__ = {
-  apiBaseUrl: "http://192.168.1.44:8080"
+  apiBaseUrl: "http://192.168.1.44:8081"
 };
 ```
 
@@ -332,12 +332,18 @@ docker compose run --rm api
 Useful API endpoints for generated files:
 
 - `POST /webdav/scan`
+- `GET /runs`
+- `GET /videos/{video_id}/runs`
 - `GET /videos/{video_id}/artifacts`
 - `GET /clips/{clip_id}/artifacts`
 - `GET /artifacts/{artifact_id}`
+- `GET /system/logs`
+- `GET /system/logs/{service}?tail=300`
 
 Thumbnail outputs are visible through the artifact endpoints. Upload command payloads include `thumbnail` when a `thumbnail_final` artifact exists.
 
+The frontend operations view uses the run, artifact, and log endpoints to show per-video pipeline history and recent Compose container logs. The API service mounts `/var/run/docker.sock` read-only so it can read logs for `api`, `frontend`, `scheduler`, `worker-ai`, `worker-media`, and `redis`; keep that endpoint LAN-only or behind trusted access.
+
 ## Custom LLM Prompt
 
 Edit `[llm_prompt]` in your private `config.toml` to tune clip selection without rebuilding containers:

docker-compose.yml

@@ -10,10 +10,11 @@ services:
     build: .
     command: uvicorn evanescere.api:app --host 0.0.0.0 --port 8000
     ports:
-      - "8080:8000"
+      - "8081:8000"
     volumes:
       - ./config.toml:/etc/evanescere/config.toml:ro
       - ./storage:/data/evanescere
+      - /var/run/docker.sock:/var/run/docker.sock:ro
     depends_on:
       - redis
 

frontend/public/config.js

@@ -1,3 +1,3 @@
 window.__EVANESCERE_FRONTEND_CONFIG__ = {
-  apiBaseUrl: "http://192.168.1.44:8080"
+  apiBaseUrl: "http://192.168.1.44:8081"
 };

frontend/src/App.tsx

@@ -1,21 +1,29 @@
 import {
+  Activity,
   Check,
   Clapperboard,
+  FileArchive,
   FolderSync,
   Play,
   RefreshCw,
   Save,
   Send,
   Settings2,
+  Terminal,
   Upload,
 } from "lucide-react";
 import { useCallback, useEffect, useMemo, useState } from "react";
 import {
   apiBaseUrl,
   approveClip,
+  getContainerLog,
   getClips,
+  getLogServices,
+  getRuns,
   getSettings,
   getTranscript,
+  getVideoArtifacts,
+  getVideoRuns,
   getVideos,
   patchSettings,
   renderClip,
@@ -23,7 +31,16 @@ import {
   scanWebDav,
   uploadClip,
 } from "./api";
-import type { ClipSuggestion, PipelineSettings, TranscriptSegment, Video } from "./types";
+import type {
+  Artifact,
+  ClipSuggestion,
+  ContainerLog,
+  LogService,
+  PipelineRun,
+  PipelineSettings,
+  TranscriptSegment,
+  Video,
+} from "./types";
 
 const settingLabels: Record<keyof PipelineSettings, string> = {
   suggest_enabled: "Suggest",
@@ -48,6 +65,10 @@ function formatTimeRange(start: number, end: number) {
   return `${formatDuration(start)}-${formatDuration(end)}`;
 }
 
+function formatDateTime(value: string) {
+  return new Date(value).toLocaleString();
+}
+
 function statusTone(value: string) {
   if (["done", "stable", "existing_done", "approved", "auto_approved"].includes(value)) return "good";
   if (["failed", "error"].includes(value)) return "bad";
@@ -61,6 +82,13 @@ export function App() {
   const [selectedVideoId, setSelectedVideoId] = useState<number | null>(null);
   const [transcript, setTranscript] = useState<TranscriptSegment[]>([]);
   const [clips, setClips] = useState<ClipSuggestion[]>([]);
+  const [runs, setRuns] = useState<PipelineRun[]>([]);
+  const [latestRuns, setLatestRuns] = useState<PipelineRun[]>([]);
+  const [artifacts, setArtifacts] = useState<Artifact[]>([]);
+  const [logServices, setLogServices] = useState<LogService[]>([]);
+  const [selectedLogService, setSelectedLogService] = useState("worker-media");
+  const [containerLog, setContainerLog] = useState<ContainerLog | null>(null);
+  const [logsLoading, setLogsLoading] = useState(false);
   const [loading, setLoading] = useState(false);
   const [error, setError] = useState<string | null>(null);
   const [notice, setNotice] = useState<string | null>(null);
@@ -71,18 +99,45 @@ export function App() {
   );
 
   const loadSelected = useCallback(async (videoId: number) => {
-    const [nextTranscript, nextClips] = await Promise.all([getTranscript(videoId), getClips(videoId)]);
+    const [nextTranscript, nextClips, nextRuns, nextArtifacts] = await Promise.all([
+      getTranscript(videoId),
+      getClips(videoId),
+      getVideoRuns(videoId),
+      getVideoArtifacts(videoId),
+    ]);
     setTranscript(nextTranscript);
     setClips(nextClips);
+    setRuns(nextRuns);
+    setArtifacts(nextArtifacts);
   }, []);
 
+  const loadLogs = useCallback(async (service = selectedLogService) => {
+    setLogsLoading(true);
+    setError(null);
+    try {
+      const nextLog = await getContainerLog(service, 300);
+      setContainerLog(nextLog);
+    } catch (caught) {
+      setError(caught instanceof Error ? caught.message : "Unknown error");
+    } finally {
+      setLogsLoading(false);
+    }
+  }, [selectedLogService]);
+
   const refresh = useCallback(async () => {
     setLoading(true);
     setError(null);
     try {
-      const [nextSettings, nextVideos] = await Promise.all([getSettings(), getVideos()]);
+      const [nextSettings, nextVideos, nextLogServices, nextLatestRuns] = await Promise.all([
+        getSettings(),
+        getVideos(),
+        getLogServices(),
+        getRuns(20),
+      ]);
       setSettings(nextSettings);
       setVideos(nextVideos);
+      setLogServices(nextLogServices);
+      setLatestRuns(nextLatestRuns);
       const targetVideoId = selectedVideoId ?? nextVideos[0]?.id ?? null;
       setSelectedVideoId(targetVideoId);
       if (targetVideoId !== null) {
@@ -90,6 +145,8 @@ export function App() {
       } else {
         setTranscript([]);
         setClips([]);
+        setRuns([]);
+        setArtifacts([]);
       }
     } catch (caught) {
       setError(caught instanceof Error ? caught.message : "Unknown error");
@@ -102,6 +159,13 @@ export function App() {
     void refresh();
   }, [refresh]);
 
+  useEffect(() => {
+    if (logServices.length === 0) return;
+    if (!logServices.some((entry) => entry.service === selectedLogService)) {
+      setSelectedLogService(logServices[0].service);
+    }
+  }, [logServices, selectedLogService]);
+
   async function handleSelect(videoId: number) {
     setSelectedVideoId(videoId);
     setLoading(true);
@@ -152,6 +216,11 @@ export function App() {
     await withRefresh(() => patchSettings(settings));
   }
 
+  function chooseLogService(service: string) {
+    setSelectedLogService(service);
+    void loadLogs(service);
+  }
+
   return (
     <main className="app-shell">
       <header className="topbar">
@@ -198,6 +267,28 @@ export function App() {
         </div>
       </section>
 
+      <section className="band jobs-band">
+        <div className="section-title">
+          <Activity size={18} />
+          <h2>Recent Jobs</h2>
+        </div>
+        <div className="job-strip">
+          {latestRuns.slice(0, 8).map((run) => (
+            <button
+              type="button"
+              className="job-chip"
+              key={run.id}
+              onClick={() => void handleSelect(run.video_id)}
+            >
+              <span>#{run.id}</span>
+              <strong>{run.stage}</strong>
+              <StatusPill value={run.status} />
+            </button>
+          ))}
+          {latestRuns.length === 0 && <span className="muted">No jobs recorded yet.</span>}
+        </div>
+      </section>
+
       <section className="workbench">
         <aside className="video-list">
           <div className="list-header">
@@ -260,9 +351,87 @@ export function App() {
             <div className="metric-row">
               <Metric label="Transcript" value={transcript.length.toString()} />
               <Metric label="Clips" value={clips.length.toString()} />
+              <Metric label="Runs" value={runs.length.toString()} />
+              <Metric label="Artifacts" value={artifacts.length.toString()} />
             </div>
           </div>
 
+          <div className="ops-grid">
+            <section className="ops-panel">
+              <div className="panel-heading">
+                <div className="section-title">
+                  <Activity size={17} />
+                  <h3>Runs</h3>
+                </div>
+                <button
+                  type="button"
+                  className="icon-button"
+                  onClick={() => selectedVideoId !== null && void loadSelected(selectedVideoId)}
+                  disabled={loading || selectedVideoId === null}
+                >
+                  <RefreshCw size={15} />
+                  Refresh
+                </button>
+              </div>
+              <div className="compact-table">
+                <table>
+                  <thead>
+                    <tr>
+                      <th>ID</th>
+                      <th>Stage</th>
+                      <th>Status</th>
+                      <th>Updated</th>
+                    </tr>
+                  </thead>
+                  <tbody>
+                    {runs.map((run) => (
+                      <tr key={run.id}>
+                        <td>{run.id}</td>
+                        <td>
+                          <div>{run.stage}</div>
+                          <div className="muted">{run.trigger}</div>
+                        </td>
+                        <td>
+                          <StatusPill value={run.status} />
+                          {run.error && <div className="run-error">{run.error}</div>}
+                        </td>
+                        <td>{formatDateTime(run.updated_at)}</td>
+                      </tr>
+                    ))}
+                    {runs.length === 0 && (
+                      <tr>
+                        <td colSpan={4} className="empty-cell">
+                          No runs recorded for this video.
+                        </td>
+                      </tr>
+                    )}
+                  </tbody>
+                </table>
+              </div>
+            </section>
+
+            <section className="ops-panel">
+              <div className="panel-heading">
+                <div className="section-title">
+                  <FileArchive size={17} />
+                  <h3>Artifacts</h3>
+                </div>
+              </div>
+              <div className="artifact-list">
+                {artifacts.slice(0, 12).map((artifact) => (
+                  <div className="artifact-row" key={artifact.id}>
+                    <div>
+                      <strong>{artifact.artifact_type}</strong>
+                      <p>{artifact.local_path}</p>
+                    </div>
+                    <span>{formatDateTime(artifact.created_at)}</span>
+                  </div>
+                ))}
+                {artifacts.length === 0 && <div className="empty-block">No artifacts yet.</div>}
+              </div>
+            </section>
+          </div>
+
           <div className="split">
             <div className="transcript-pane">
               <h3>Transcript</h3>
@@ -313,6 +482,40 @@ export function App() {
               </div>
             </div>
           </div>
+
+          <section className="logs-panel">
+            <div className="panel-heading">
+              <div className="section-title">
+                <Terminal size={17} />
+                <h3>Container Logs</h3>
+              </div>
+              <button
+                type="button"
+                className="icon-button"
+                onClick={() => void loadLogs()}
+                disabled={logsLoading}
+              >
+                <RefreshCw size={15} />
+                Refresh
+              </button>
+            </div>
+            <div className="log-service-row">
+              {logServices.map((entry) => (
+                <button
+                  type="button"
+                  className={`service-tab ${entry.service === selectedLogService ? "active" : ""}`}
+                  key={entry.service}
+                  onClick={() => chooseLogService(entry.service)}
+                >
+                  {entry.service}
+                </button>
+              ))}
+            </div>
+            <div className="muted">
+              {containerLog ? `${containerLog.container}, latest 300 lines` : "Pick a service to load logs."}
+            </div>
+            <pre className="log-output">{containerLog?.logs || ""}</pre>
+          </section>
         </section>
       </section>
     </main>

frontend/src/api.ts

@@ -1,6 +1,9 @@
 import type {
   Artifact,
   ClipSuggestion,
+  ContainerLog,
+  LogService,
+  PipelineRun,
   PipelineSettings,
   TranscriptSegment,
   Video,
@@ -9,7 +12,7 @@ import type {
 
 const runtimeApiBase = window.__EVANESCERE_FRONTEND_CONFIG__?.apiBaseUrl;
 
-export const apiBaseUrl = (runtimeApiBase || "http://localhost:8080").replace(
+export const apiBaseUrl = (runtimeApiBase || "http://localhost:8081").replace(
   /\/$/,
   "",
 );
@@ -60,6 +63,18 @@ export function getClips(videoId: number): Promise<ClipSuggestion[]> {
   return request<ClipSuggestion[]>(`/videos/${videoId}/clips`);
 }
 
+export function getVideoRuns(videoId: number): Promise<PipelineRun[]> {
+  return request<PipelineRun[]>(`/videos/${videoId}/runs`);
+}
+
+export function getRuns(limit = 50): Promise<PipelineRun[]> {
+  return request<PipelineRun[]>(`/runs?limit=${limit}`);
+}
+
+export function getVideoArtifacts(videoId: number): Promise<Artifact[]> {
+  return request<Artifact[]>(`/videos/${videoId}/artifacts`);
+}
+
 export function getClipArtifacts(clipId: number): Promise<Artifact[]> {
   return request<Artifact[]>(`/clips/${clipId}/artifacts`);
 }
@@ -75,3 +90,11 @@ export function renderClip(clipId: number): Promise<ClipSuggestion> {
 export function uploadClip(clipId: number): Promise<ClipSuggestion> {
   return request<ClipSuggestion>(`/clips/${clipId}/upload`, { method: "POST" });
 }
+
+export function getLogServices(): Promise<LogService[]> {
+  return request<LogService[]>("/system/logs");
+}
+
+export function getContainerLog(service: string, tail = 200): Promise<ContainerLog> {
+  return request<ContainerLog>(`/system/logs/${service}?tail=${tail}`);
+}

frontend/src/styles.css

@@ -152,7 +152,8 @@ button {
 .section-title h2,
 .detail-header h2,
 .transcript-pane h3,
-.clips-pane h3 {
+.clips-pane h3,
+.section-title h3 {
   margin: 0;
   font-size: 16px;
   font-weight: 690;
@@ -183,6 +184,50 @@ button {
   accent-color: #21756b;
 }
 
+.jobs-band {
+  display: grid;
+  gap: 12px;
+}
+
+.job-strip {
+  display: flex;
+  gap: 8px;
+  flex-wrap: wrap;
+  align-items: center;
+}
+
+.job-chip {
+  display: inline-flex;
+  align-items: center;
+  gap: 7px;
+  min-height: 34px;
+  max-width: 280px;
+  border: 1px solid #ccd3db;
+  border-radius: 6px;
+  background: #ffffff;
+  color: #2f3b46;
+  padding: 0 9px;
+  cursor: pointer;
+}
+
+.job-chip:hover {
+  border-color: #21756b;
+  background: #e9f3f1;
+}
+
+.job-chip span {
+  color: #66737f;
+  font-size: 12px;
+}
+
+.job-chip strong {
+  overflow: hidden;
+  font-size: 13px;
+  font-weight: 680;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+
 .workbench {
   display: grid;
   grid-template-columns: minmax(340px, 0.36fr) minmax(0, 1fr);
@@ -267,6 +312,7 @@ tr.selected td {
 .metric-row {
   display: flex;
   gap: 10px;
+  flex-wrap: wrap;
 }
 
 .metric {
@@ -290,6 +336,100 @@ tr.selected td {
   font-weight: 720;
 }
 
+.ops-grid {
+  display: grid;
+  grid-template-columns: minmax(320px, 1fr) minmax(280px, 0.8fr);
+  gap: 18px;
+  margin-bottom: 18px;
+}
+
+.ops-panel,
+.logs-panel {
+  min-width: 0;
+  border: 1px solid #d8dee6;
+  border-radius: 8px;
+  background: #ffffff;
+  padding: 12px;
+}
+
+.logs-panel {
+  margin-top: 18px;
+}
+
+.panel-heading {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 12px;
+  margin-bottom: 10px;
+}
+
+.compact-table {
+  overflow: auto;
+}
+
+.compact-table table {
+  min-width: 520px;
+}
+
+.run-error {
+  max-width: 420px;
+  margin-top: 6px;
+  color: #a02717;
+  font-size: 12px;
+  overflow-wrap: anywhere;
+}
+
+.empty-cell,
+.empty-block {
+  color: #66737f;
+  font-size: 13px;
+}
+
+.empty-block {
+  border: 1px dashed #ccd3db;
+  border-radius: 6px;
+  padding: 14px;
+}
+
+.artifact-list {
+  display: grid;
+  gap: 8px;
+  max-height: 252px;
+  overflow: auto;
+}
+
+.artifact-row {
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) auto;
+  gap: 12px;
+  border-top: 1px solid #edf0f3;
+  padding-top: 8px;
+}
+
+.artifact-row:first-child {
+  border-top: 0;
+  padding-top: 0;
+}
+
+.artifact-row strong {
+  color: #18222b;
+  font-size: 13px;
+}
+
+.artifact-row p {
+  margin: 3px 0 0;
+  color: #4f5c68;
+  font-size: 12px;
+  overflow-wrap: anywhere;
+}
+
+.artifact-row span {
+  color: #66737f;
+  font-size: 12px;
+  white-space: nowrap;
+}
+
 .split {
   display: grid;
   grid-template-columns: minmax(280px, 0.95fr) minmax(320px, 1.05fr);
@@ -412,9 +552,50 @@ tr.selected td {
   color: #6f4d00;
 }
 
+.log-service-row {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  margin-bottom: 8px;
+}
+
+.service-tab {
+  min-height: 32px;
+  border: 1px solid #ccd3db;
+  border-radius: 6px;
+  background: #fbfcfd;
+  color: #2f3b46;
+  padding: 0 10px;
+  cursor: pointer;
+}
+
+.service-tab.active {
+  border-color: #21756b;
+  background: #e9f3f1;
+  color: #155c54;
+}
+
+.log-output {
+  min-height: 240px;
+  max-height: 480px;
+  margin: 10px 0 0;
+  overflow: auto;
+  border: 1px solid #263340;
+  border-radius: 8px;
+  background: #101820;
+  color: #d9e7ee;
+  padding: 12px;
+  font-family: "JetBrains Mono", "SFMono-Regular", Consolas, monospace;
+  font-size: 12px;
+  line-height: 1.45;
+  white-space: pre-wrap;
+  overflow-wrap: anywhere;
+}
+
 @media (max-width: 1100px) {
   .workbench,
-  .split {
+  .split,
+  .ops-grid {
     grid-template-columns: 1fr;
   }
 

frontend/src/types.ts

@@ -48,6 +48,17 @@ export interface PipelineSettings {
   bake_subtitles: boolean;
 }
 
+export interface PipelineRun {
+  id: number;
+  video_id: number;
+  trigger: string;
+  stage: string;
+  status: string;
+  error: string | null;
+  created_at: string;
+  updated_at: string;
+}
+
 export interface Artifact {
   id: number;
   video_id: number | null;
@@ -67,3 +78,14 @@ export interface WebDavScanResult {
   baseline_inserted: number;
   baseline_marked_existing: number;
 }
+
+export interface LogService {
+  service: string;
+  container: string;
+}
+
+export interface ContainerLog {
+  service: string;
+  container: string;
+  logs: string;
+}

scripts/deploy-rsync.zsh

@@ -0,0 +1,23 @@
+#!/usr/bin/env zsh
+set -euo pipefail
+
+repo_root="$(cd -- "$(dirname -- "$0")/.." && pwd)"
+remote="${EVANESCERE_DEPLOY_REMOTE:-felis@192.168.1.44}"
+remote_dir="${EVANESCERE_DEPLOY_DIR:-~/evanescere/}"
+
+print "Syncing Evanescere to ${remote}:${remote_dir}"
+print "Keeping remote config.toml and storage/ untouched."
+
+rsync -az --delete --progress \
+  --exclude ".git/" \
+  --exclude "config.toml" \
+  --exclude "storage/" \
+  --exclude "frontend/node_modules/" \
+  --exclude "frontend/dist/" \
+  --exclude "__pycache__/" \
+  --exclude ".pytest_cache/" \
+  "${repo_root}/" "${remote}:${remote_dir}"
+
+print
+print "Sync complete. Rebuild on the Framework box with:"
+print "  ssh ${remote} 'cd ${remote_dir} && docker compose up -d --build'"

scripts/reset-prod-queue.zsh

@@ -0,0 +1,143 @@
+#!/usr/bin/env zsh
+set -euo pipefail
+
+remote="${EVANESCERE_DEPLOY_REMOTE:-felis@192.168.1.44}"
+remote_dir="${EVANESCERE_DEPLOY_DIR:-/home/felis/evanescere}"
+reset_mode="${EVANESCERE_RESET_MODE:-baseline}"
+assume_yes=false
+
+while (( $# > 0 )); do
+  case "$1" in
+    --yes|-y)
+      assume_yes=true
+      shift
+      ;;
+    --pending)
+      reset_mode="pending"
+      shift
+      ;;
+    --baseline)
+      reset_mode="baseline"
+      shift
+      ;;
+    *)
+      print -u2 "Unknown argument: $1"
+      print -u2 "Usage: $0 [--yes] [--baseline|--pending]"
+      exit 2
+      ;;
+  esac
+done
+
+if [[ "$reset_mode" != "baseline" && "$reset_mode" != "pending" ]]; then
+  print -u2 "EVANESCERE_RESET_MODE must be baseline or pending."
+  exit 2
+fi
+
+print "This will reset queued/running Evanescere jobs on ${remote}:${remote_dir}"
+print
+print "Actions:"
+print "  - stop scheduler, worker-ai, and worker-media"
+print "  - flush the project Redis database"
+print "  - mark non-terminal pipeline runs as cancelled"
+print "  - reset queued/running clip render and upload statuses to pending"
+if [[ "$reset_mode" == "baseline" ]]; then
+  print "  - mark affected queued/running videos as existing_done/done so they will not auto-requeue"
+else
+  print "  - mark affected queued/running videos as stable/pending so scheduler can queue them again"
+fi
+print "  - restart the Compose stack"
+print
+
+if [[ "$assume_yes" != true ]]; then
+  printf "Continue? [y/N] "
+  read -r reply
+  if [[ ! "$reply" =~ '^[Yy]$' ]]; then
+    print "Cancelled."
+    exit 0
+  fi
+fi
+
+ssh "$remote" \
+  "EVANESCERE_DEPLOY_DIR=${remote_dir:q} EVANESCERE_RESET_MODE=${reset_mode:q} zsh -s" <<'REMOTE'
+set -euo pipefail
+
+cd "$EVANESCERE_DEPLOY_DIR"
+
+docker compose stop scheduler worker-ai worker-media
+docker compose exec -T redis redis-cli FLUSHDB </dev/null
+
+docker compose run --rm -e EVANESCERE_RESET_MODE="$EVANESCERE_RESET_MODE" api python - <<'PY'
+import os
+
+from sqlalchemy import or_
+
+from evanescere.db import session_scope
+from evanescere.models import ClipSuggestion, PipelineRun, Video
+
+active_statuses = {"queued", "running"}
+terminal_run_statuses = {"done", "failed", "cancelled"}
+mode = os.environ["EVANESCERE_RESET_MODE"]
+
+with session_scope() as session:
+    runs = (
+        session.query(PipelineRun)
+        .filter(PipelineRun.status.notin_(terminal_run_statuses))
+        .all()
+    )
+    affected_video_ids = {run.video_id for run in runs}
+    video_filters = [
+        Video.processing_status.in_(active_statuses),
+        Video.ingest_status == "queued",
+    ]
+    if affected_video_ids:
+        video_filters.append(Video.id.in_(affected_video_ids))
+    videos = (
+        session.query(Video)
+        .filter(or_(*video_filters))
+        .all()
+    )
+    render_clips = (
+        session.query(ClipSuggestion)
+        .filter(ClipSuggestion.render_status.in_(active_statuses))
+        .all()
+    )
+    upload_clips = (
+        session.query(ClipSuggestion)
+        .filter(ClipSuggestion.upload_status.in_(active_statuses))
+        .all()
+    )
+
+    for run in runs:
+        run.status = "cancelled"
+        if run.stage == "queued":
+            run.stage = "cancelled"
+        run.error = "Cancelled by reset-prod-queue."
+
+    for video in videos:
+        if mode == "baseline":
+            video.ingest_status = "existing_done"
+            video.processing_status = "done"
+        else:
+            if video.ingest_status == "queued":
+                video.ingest_status = "stable"
+            video.processing_status = "pending"
+
+    for clip in render_clips:
+        clip.render_status = "pending"
+
+    for clip in upload_clips:
+        clip.upload_status = "pending"
+
+    print(
+        "reset complete:",
+        f"runs={len(runs)}",
+        f"videos={len(videos)}",
+        f"render_clips={len(render_clips)}",
+        f"upload_clips={len(upload_clips)}",
+        f"mode={mode}",
+    )
+PY
+
+docker compose up -d
+docker compose ps
+REMOTE

src/evanescere/api.py

@@ -1,9 +1,8 @@
 import logging
 import time
 
-from fastapi import Depends, FastAPI, HTTPException
+from fastapi import Depends, FastAPI, HTTPException, Query, Request
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi import Request
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
@@ -16,6 +15,8 @@ from evanescere.models import Artifact, ClipSuggestion, PipelineRun, TranscriptS
 from evanescere.schemas import (
     ArtifactRead,
     ClipSuggestionRead,
+    ContainerLogRead,
+    ContainerLogServiceRead,
     RunRead,
     SettingsPatch,
     SettingsRead,
@@ -23,6 +24,11 @@ from evanescere.schemas import (
     VideoRead,
     WebDavScanRead,
 )
+from evanescere.services.docker_logs import (
+    DockerLogsError,
+    list_log_services,
+    read_container_logs,
+)
 from evanescere.services.webdav import WebDavClient, safe_scan_once
 from evanescere.settings_store import get_pipeline_settings, patch_pipeline_settings
 
@@ -111,6 +117,22 @@ def video_clips(video_id: int, db: Session = Depends(get_db)) -> list[ClipSugges
     )
 
 
+@app.get("/videos/{video_id}/runs", response_model=list[RunRead])
+def video_runs(
+    video_id: int,
+    limit: int = Query(default=50, ge=1, le=200),
+    db: Session = Depends(get_db),
+) -> list[PipelineRun]:
+    return list(
+        db.scalars(
+            select(PipelineRun)
+            .where(PipelineRun.video_id == video_id)
+            .order_by(PipelineRun.updated_at.desc())
+            .limit(limit)
+        ).all()
+    )
+
+
 @app.get("/videos/{video_id}/artifacts", response_model=list[ArtifactRead])
 def video_artifacts(video_id: int, db: Session = Depends(get_db)) -> list[Artifact]:
     return list(
@@ -162,6 +184,23 @@ def upload_clip(clip_id: int, db: Session = Depends(get_db)) -> ClipSuggestion:
     return clip
 
 
+@app.get("/runs", response_model=list[RunRead])
+def list_runs(
+    video_id: int | None = None,
+    limit: int = Query(default=50, ge=1, le=200),
+    db: Session = Depends(get_db),
+) -> list[PipelineRun]:
+    query = select(PipelineRun).order_by(PipelineRun.updated_at.desc()).limit(limit)
+    if video_id is not None:
+        query = (
+            select(PipelineRun)
+            .where(PipelineRun.video_id == video_id)
+            .order_by(PipelineRun.updated_at.desc())
+            .limit(limit)
+        )
+    return list(db.scalars(query).all())
+
+
 @app.get("/runs/{run_id}", response_model=RunRead)
 def get_run(run_id: int, db: Session = Depends(get_db)) -> PipelineRun:
     run = db.get(PipelineRun, run_id)
@@ -178,6 +217,28 @@ def get_artifact(artifact_id: int, db: Session = Depends(get_db)) -> Artifact:
     return artifact
 
 
+@app.get("/system/logs", response_model=list[ContainerLogServiceRead])
+def system_log_services() -> list[ContainerLogServiceRead]:
+    return [
+        ContainerLogServiceRead(service=entry.service, container=entry.container)
+        for entry in list_log_services()
+    ]
+
+
+@app.get("/system/logs/{service}", response_model=ContainerLogRead)
+def system_logs(
+    service: str,
+    tail: int = Query(default=200, ge=1, le=2000),
+) -> ContainerLogRead:
+    try:
+        result = read_container_logs(service, tail=tail)
+    except KeyError as exc:
+        raise HTTPException(status_code=404, detail=f"Unknown log service: {service}") from exc
+    except DockerLogsError as exc:
+        raise HTTPException(status_code=503, detail=str(exc)) from exc
+    return ContainerLogRead(service=result.service, container=result.container, logs=result.logs)
+
+
 @app.get("/settings", response_model=SettingsRead)
 def read_settings(db: Session = Depends(get_db)) -> SettingsRead:
     return get_pipeline_settings(db)

src/evanescere/pipeline.py

@@ -232,13 +232,16 @@ def run_pipeline(session: Session, video_id: int, run_id: int | None = None) ->
         video.processing_status = "running"
         update_run(run, "media", "running")
         prepare_media(session, video)
+        session.commit()
 
         update_run(run, "transcribe", "running")
         transcribe_video(session, video)
+        session.commit()
 
         if settings.suggest_enabled:
             update_run(run, "suggest", "running")
             suggest_clips(session, video)
+            session.commit()
 
         if settings.render_enabled:
             clips = session.scalars(
@@ -250,12 +253,15 @@ def run_pipeline(session: Session, video_id: int, run_id: int | None = None) ->
                 render_clip_by_id(session, clip.id)
                 if settings.upload_enabled:
                     upload_clip_by_id(session, clip.id)
+                session.commit()
 
         video.processing_status = "done"
         update_run(run, "done", "done")
+        session.commit()
         logger.info("pipeline done video_id=%s run_id=%s", video_id, run_id)
     except Exception as exc:
         video.processing_status = "failed"
         update_run(run, "failed", "failed", str(exc))
+        session.commit()
         logger.exception("pipeline failed video_id=%s run_id=%s", video_id, run_id)
         raise

src/evanescere/schemas.py

@@ -103,6 +103,17 @@ class WebDavScanRead(BaseModel):
     baseline_marked_existing: int = 0
 
 
+class ContainerLogServiceRead(BaseModel):
+    service: str
+    container: str
+
+
+class ContainerLogRead(BaseModel):
+    service: str
+    container: str
+    logs: str
+
+
 class ClipCandidate(BaseModel):
     start_sec: float = Field(ge=0)
     end_sec: float = Field(gt=0)

src/evanescere/services/asr.py

@@ -35,7 +35,7 @@ class FunAsrClient:
                     "model": self.model,
                     "language": "zh",
                     "response_format": "verbose_json",
-                    "timestamp_granularities[]": "segment",
+                    "timestamp_granularities": "segment",
                 },
                 files={"file": (audio_path.name, audio_file, "audio/wav")},
                 timeout=None,
@@ -45,8 +45,12 @@ class FunAsrClient:
         logger.info("funasr request done audio=%s keys=%s", audio_path, sorted(payload.keys()))
         return payload
 
-
+def normalize_segments(
-def normalize_segments(payload: dict[str, Any]) -> list[dict[str, Any]]:
+    payload: dict[str, Any],
+    *,
+    fallback_start_sec: float = 0.0,
+    fallback_end_sec: float | None = None,
+) -> list[dict[str, Any]]:
     raw_segments = payload.get("segments") or payload.get("sentence_info") or []
     logger.debug("normalizing asr segments raw_count=%s", len(raw_segments))
     normalized: list[dict[str, Any]] = []
@@ -71,10 +75,17 @@ def normalize_segments(payload: dict[str, Any]) -> list[dict[str, Any]]:
             }
         )
     if not normalized and payload.get("text"):
+        logger.warning(
+            "asr response contained text but no timestamped segments; "
+            "falling back to the known audio bounds"
+        )
+        end_sec = fallback_end_sec
+        if end_sec is None:
+            end_sec = float(payload.get("duration") or fallback_start_sec + 0.1)
         normalized.append(
             {
-                "start_sec": 0.0,
+                "start_sec": fallback_start_sec,
-                "end_sec": 0.1,
+                "end_sec": max(fallback_start_sec + 0.1, end_sec),
                 "text": str(payload["text"]).strip(),
                 "speaker": None,
                 "confidence": None,

src/evanescere/services/docker_logs.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+import httpx
+
+
+DEFAULT_DOCKER_SOCKET = "/var/run/docker.sock"
+
+SERVICE_CONTAINERS: dict[str, str] = {
+    "api": "evanescere-api-1",
+    "frontend": "evanescere-frontend-1",
+    "scheduler": "evanescere-scheduler-1",
+    "worker-ai": "evanescere-worker-ai-1",
+    "worker-media": "evanescere-worker-media-1",
+    "redis": "evanescere-redis-1",
+}
+
+
+class DockerLogsError(RuntimeError):
+    pass
+
+
+@dataclass(frozen=True)
+class LogService:
+    service: str
+    container: str
+
+
+@dataclass(frozen=True)
+class ContainerLogs:
+    service: str
+    container: str
+    logs: str
+
+
+def list_log_services() -> list[LogService]:
+    return [
+        LogService(service=service, container=container)
+        for service, container in SERVICE_CONTAINERS.items()
+    ]
+
+
+def read_container_logs(
+    service: str,
+    *,
+    tail: int = 200,
+    socket_path: str = DEFAULT_DOCKER_SOCKET,
+) -> ContainerLogs:
+    container = SERVICE_CONTAINERS.get(service)
+    if container is None:
+        raise KeyError(service)
+
+    if not os.path.exists(socket_path):
+        raise DockerLogsError(
+            f"Docker socket {socket_path} is not mounted into the API container."
+        )
+
+    transport = httpx.HTTPTransport(uds=socket_path)
+    params = {
+        "stdout": "1",
+        "stderr": "1",
+        "timestamps": "1",
+        "tail": str(tail),
+    }
+    with httpx.Client(transport=transport, base_url="http://docker", timeout=10.0) as client:
+        response = client.get(f"/containers/{container}/logs", params=params)
+
+    if response.status_code == 404:
+        raise DockerLogsError(f"Container {container} was not found.")
+    if response.status_code >= 400:
+        raise DockerLogsError(
+            f"Docker logs request failed with HTTP {response.status_code}: {response.text}"
+        )
+
+    return ContainerLogs(
+        service=service,
+        container=container,
+        logs=_decode_docker_logs(response.content),
+    )
+
+
+def _decode_docker_logs(content: bytes) -> str:
+    frames: list[bytes] = []
+    index = 0
+    while index + 8 <= len(content):
+        frame_size = int.from_bytes(content[index + 4 : index + 8], "big")
+        frame_start = index + 8
+        frame_end = frame_start + frame_size
+        if frame_size <= 0 or frame_end > len(content):
+            break
+        frames.append(content[frame_start:frame_end])
+        index = frame_end
+
+    if frames and index == len(content):
+        return b"".join(frames).decode("utf-8", errors="replace")
+    return content.decode("utf-8", errors="replace")

src/evanescere/services/llm.py

@@ -190,6 +190,7 @@ def chunk_transcript(
 def parse_clip_response(content: str) -> list[ClipCandidate]:
     try:
         payload = json.loads(content)
+        payload = normalize_clip_payload(payload)
         parsed = ClipCandidateResponse.model_validate(payload)
     except (json.JSONDecodeError, ValidationError) as exc:
         logger.error("invalid clip json content_prefix=%s", content[:1000])
@@ -197,6 +198,43 @@ def parse_clip_response(content: str) -> list[ClipCandidate]:
     return parsed.clips
 
 
+def normalize_clip_payload(payload: object) -> dict:
+    if not isinstance(payload, dict):
+        return {"clips": []}
+    if "clips" not in payload and "candidates" in payload:
+        payload = payload | {"clips": payload["candidates"]}
+    clips = payload.get("clips")
+    if not isinstance(clips, list):
+        return payload
+    normalized_clips = []
+    for clip in clips:
+        if not isinstance(clip, dict):
+            normalized_clips.append(clip)
+            continue
+        normalized = dict(clip)
+        normalized["score"] = normalize_score(normalized.get("score", 0))
+        normalized_clips.append(normalized)
+    return payload | {"clips": normalized_clips}
+
+
+def normalize_score(value: object) -> float:
+    try:
+        score = float(value)
+    except (TypeError, ValueError):
+        logger.warning("invalid llm score=%r; using 0", value)
+        return 0.0
+    if score > 1:
+        original = score
+        if score <= 10:
+            score = score / 10
+        elif score <= 100:
+            score = score / 100
+        else:
+            score = 1.0
+        logger.debug("normalized llm score original=%s normalized=%s", original, score)
+    return max(0.0, min(1.0, score))
+
+
 def overlap_ratio(left: ClipCandidate, right: ClipCandidate) -> float:
     overlap = max(0, min(left.end_sec, right.end_sec) - max(left.start_sec, right.start_sec))
     shortest = min(left.end_sec - left.start_sec, right.end_sec - right.start_sec)

src/evanescere/services/webdav.py

@@ -182,7 +182,10 @@ def bootstrap_existing(client: WebDavClient, session: Session) -> BootstrapResul
             session.add(video)
             inserted += 1
             logger.debug("bootstrap existing file=%s size=%s", file.filename, file.size_bytes)
-        elif video.ingest_status == "observing" and video.processing_status == "pending":
+        elif (
+            video.ingest_status in {"observing", "stable", "queued"}
+            and video.processing_status in {"pending", "queued"}
+        ):
             video.size_bytes = file.size_bytes
             video.ingest_status = "existing_done"
             video.processing_status = "done"

tests/test_asr.py

@@ -0,0 +1,16 @@
+from evanescere.services.asr import normalize_segments
+
+
+def test_normalize_segments_uses_known_audio_bounds_when_timestamps_are_missing():
+    assert normalize_segments(
+        {"text": "你好", "segments": []}, fallback_start_sec=0, fallback_end_sec=15
+    ) == [
+        {
+            "start_sec": 0,
+            "end_sec": 15,
+            "text": "你好",
+            "speaker": None,
+            "confidence": None,
+            "metadata": {"text": "你好", "segments": []},
+        }
+    ]

tests/test_llm.py

@@ -30,6 +30,50 @@ def test_parse_clip_response():
     assert candidates[0].title_zh == "标题"
 
 
+def test_parse_clip_response_accepts_candidates_key_and_score_out_of_ten():
+    candidates = parse_clip_response(
+        """
+        {
+          "candidates": [
+            {
+              "start_sec": 10,
+              "end_sec": 60,
+              "title_zh": "标题",
+              "summary_zh": "摘要",
+              "reason": "有趣",
+              "score": 8.5,
+              "tags": ["反应"],
+              "subtitle_priority": "high"
+            }
+          ]
+        }
+        """
+    )
+    assert candidates[0].score == 0.85
+
+
+def test_parse_clip_response_normalizes_score_out_of_hundred():
+    candidates = parse_clip_response(
+        """
+        {
+          "clips": [
+            {
+              "start_sec": 10,
+              "end_sec": 60,
+              "title_zh": "标题",
+              "summary_zh": "摘要",
+              "reason": "有趣",
+              "score": 92,
+              "tags": ["反应"],
+              "subtitle_priority": "high"
+            }
+          ]
+        }
+        """
+    )
+    assert candidates[0].score == 0.92
+
+
 def test_dedupe_and_rank_candidates_filters_short_and_overlapping():
     candidates = [
         ClipCandidate(