operation scripts

2026-06-03 00:03:49 -07:00
parent d26342efd6
commit 6ec321f7d2
2 changed files with 166 additions and 0 deletions
@@ -0,0 +1,143 @@
+#!/usr/bin/env zsh
+set -euo pipefail
+
+remote="${EVANESCERE_DEPLOY_REMOTE:-felis@192.168.1.44}"
+remote_dir="${EVANESCERE_DEPLOY_DIR:-/home/felis/evanescere}"
+reset_mode="${EVANESCERE_RESET_MODE:-baseline}"
+assume_yes=false
+
+while (( $# > 0 )); do
+  case "$1" in
+    --yes|-y)
+      assume_yes=true
+      shift
+      ;;
+    --pending)
+      reset_mode="pending"
+      shift
+      ;;
+    --baseline)
+      reset_mode="baseline"
+      shift
+      ;;
+    *)
+      print -u2 "Unknown argument: $1"
+      print -u2 "Usage: $0 [--yes] [--baseline|--pending]"
+      exit 2
+      ;;
+  esac
+done
+
+if [[ "$reset_mode" != "baseline" && "$reset_mode" != "pending" ]]; then
+  print -u2 "EVANESCERE_RESET_MODE must be baseline or pending."
+  exit 2
+fi
+
+print "This will reset queued/running Evanescere jobs on ${remote}:${remote_dir}"
+print
+print "Actions:"
+print "  - stop scheduler, worker-ai, and worker-media"
+print "  - flush the project Redis database"
+print "  - mark non-terminal pipeline runs as cancelled"
+print "  - reset queued/running clip render and upload statuses to pending"
+if [[ "$reset_mode" == "baseline" ]]; then
+  print "  - mark affected queued/running videos as existing_done/done so they will not auto-requeue"
+else
+  print "  - mark affected queued/running videos as stable/pending so scheduler can queue them again"
+fi
+print "  - restart the Compose stack"
+print
+
+if [[ "$assume_yes" != true ]]; then
+  printf "Continue? [y/N] "
+  read -r reply
+  if [[ ! "$reply" =~ '^[Yy]$' ]]; then
+    print "Cancelled."
+    exit 0
+  fi
+fi
+
+ssh "$remote" \
+  "EVANESCERE_DEPLOY_DIR=${remote_dir:q} EVANESCERE_RESET_MODE=${reset_mode:q} zsh -s" <<'REMOTE'
+set -euo pipefail
+
+cd "$EVANESCERE_DEPLOY_DIR"
+
+docker compose stop scheduler worker-ai worker-media
+docker compose exec -T redis redis-cli FLUSHDB </dev/null
+
+docker compose run --rm -e EVANESCERE_RESET_MODE="$EVANESCERE_RESET_MODE" api python - <<'PY'
+import os
+
+from sqlalchemy import or_
+
+from evanescere.db import session_scope
+from evanescere.models import ClipSuggestion, PipelineRun, Video
+
+active_statuses = {"queued", "running"}
+terminal_run_statuses = {"done", "failed", "cancelled"}
+mode = os.environ["EVANESCERE_RESET_MODE"]
+
+with session_scope() as session:
+    runs = (
+        session.query(PipelineRun)
+        .filter(PipelineRun.status.notin_(terminal_run_statuses))
+        .all()
+    )
+    affected_video_ids = {run.video_id for run in runs}
+    video_filters = [
+        Video.processing_status.in_(active_statuses),
+        Video.ingest_status == "queued",
+    ]
+    if affected_video_ids:
+        video_filters.append(Video.id.in_(affected_video_ids))
+    videos = (
+        session.query(Video)
+        .filter(or_(*video_filters))
+        .all()
+    )
+    render_clips = (
+        session.query(ClipSuggestion)
+        .filter(ClipSuggestion.render_status.in_(active_statuses))
+        .all()
+    )
+    upload_clips = (
+        session.query(ClipSuggestion)
+        .filter(ClipSuggestion.upload_status.in_(active_statuses))
+        .all()
+    )
+
+    for run in runs:
+        run.status = "cancelled"
+        if run.stage == "queued":
+            run.stage = "cancelled"
+        run.error = "Cancelled by reset-prod-queue."
+
+    for video in videos:
+        if mode == "baseline":
+            video.ingest_status = "existing_done"
+            video.processing_status = "done"
+        else:
+            if video.ingest_status == "queued":
+                video.ingest_status = "stable"
+            video.processing_status = "pending"
+
+    for clip in render_clips:
+        clip.render_status = "pending"
+
+    for clip in upload_clips:
+        clip.upload_status = "pending"
+
+    print(
+        "reset complete:",
+        f"runs={len(runs)}",
+        f"videos={len(videos)}",
+        f"render_clips={len(render_clips)}",
+        f"upload_clips={len(upload_clips)}",
+        f"mode={mode}",
+    )
+PY
+
+docker compose up -d
+docker compose ps
+REMOTE