144 lines
3.8 KiB
Bash
Executable File
144 lines
3.8 KiB
Bash
Executable File
#!/usr/bin/env zsh
|
|
set -euo pipefail
|
|
|
|
remote="${EVANESCERE_DEPLOY_REMOTE:-felis@192.168.1.44}"
|
|
remote_dir="${EVANESCERE_DEPLOY_DIR:-/home/felis/evanescere}"
|
|
reset_mode="${EVANESCERE_RESET_MODE:-baseline}"
|
|
assume_yes=false
|
|
|
|
while (( $# > 0 )); do
|
|
case "$1" in
|
|
--yes|-y)
|
|
assume_yes=true
|
|
shift
|
|
;;
|
|
--pending)
|
|
reset_mode="pending"
|
|
shift
|
|
;;
|
|
--baseline)
|
|
reset_mode="baseline"
|
|
shift
|
|
;;
|
|
*)
|
|
print -u2 "Unknown argument: $1"
|
|
print -u2 "Usage: $0 [--yes] [--baseline|--pending]"
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ "$reset_mode" != "baseline" && "$reset_mode" != "pending" ]]; then
|
|
print -u2 "EVANESCERE_RESET_MODE must be baseline or pending."
|
|
exit 2
|
|
fi
|
|
|
|
print "This will reset queued/running Evanescere jobs on ${remote}:${remote_dir}"
|
|
print
|
|
print "Actions:"
|
|
print " - stop scheduler, worker-ai, and worker-media"
|
|
print " - flush the project Redis database"
|
|
print " - mark non-terminal pipeline runs as cancelled"
|
|
print " - reset queued/running clip render and upload statuses to pending"
|
|
if [[ "$reset_mode" == "baseline" ]]; then
|
|
print " - mark affected queued/running videos as existing_done/done so they will not auto-requeue"
|
|
else
|
|
print " - mark affected queued/running videos as stable/pending so scheduler can queue them again"
|
|
fi
|
|
print " - restart the Compose stack"
|
|
print
|
|
|
|
if [[ "$assume_yes" != true ]]; then
|
|
printf "Continue? [y/N] "
|
|
read -r reply
|
|
if [[ ! "$reply" =~ '^[Yy]$' ]]; then
|
|
print "Cancelled."
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
ssh "$remote" \
|
|
"EVANESCERE_DEPLOY_DIR=${remote_dir:q} EVANESCERE_RESET_MODE=${reset_mode:q} zsh -s" <<'REMOTE'
|
|
set -euo pipefail
|
|
|
|
cd "$EVANESCERE_DEPLOY_DIR"
|
|
|
|
docker compose stop scheduler worker-ai worker-media
|
|
docker compose exec -T redis redis-cli FLUSHDB </dev/null
|
|
|
|
docker compose run --rm -e EVANESCERE_RESET_MODE="$EVANESCERE_RESET_MODE" api python - <<'PY'
|
|
import os
|
|
|
|
from sqlalchemy import or_
|
|
|
|
from evanescere.db import session_scope
|
|
from evanescere.models import ClipSuggestion, PipelineRun, Video
|
|
|
|
active_statuses = {"queued", "running"}
|
|
terminal_run_statuses = {"done", "failed", "cancelled"}
|
|
mode = os.environ["EVANESCERE_RESET_MODE"]
|
|
|
|
with session_scope() as session:
|
|
runs = (
|
|
session.query(PipelineRun)
|
|
.filter(PipelineRun.status.notin_(terminal_run_statuses))
|
|
.all()
|
|
)
|
|
affected_video_ids = {run.video_id for run in runs}
|
|
video_filters = [
|
|
Video.processing_status.in_(active_statuses),
|
|
Video.ingest_status == "queued",
|
|
]
|
|
if affected_video_ids:
|
|
video_filters.append(Video.id.in_(affected_video_ids))
|
|
videos = (
|
|
session.query(Video)
|
|
.filter(or_(*video_filters))
|
|
.all()
|
|
)
|
|
render_clips = (
|
|
session.query(ClipSuggestion)
|
|
.filter(ClipSuggestion.render_status.in_(active_statuses))
|
|
.all()
|
|
)
|
|
upload_clips = (
|
|
session.query(ClipSuggestion)
|
|
.filter(ClipSuggestion.upload_status.in_(active_statuses))
|
|
.all()
|
|
)
|
|
|
|
for run in runs:
|
|
run.status = "cancelled"
|
|
if run.stage == "queued":
|
|
run.stage = "cancelled"
|
|
run.error = "Cancelled by reset-prod-queue."
|
|
|
|
for video in videos:
|
|
if mode == "baseline":
|
|
video.ingest_status = "existing_done"
|
|
video.processing_status = "done"
|
|
else:
|
|
if video.ingest_status == "queued":
|
|
video.ingest_status = "stable"
|
|
video.processing_status = "pending"
|
|
|
|
for clip in render_clips:
|
|
clip.render_status = "pending"
|
|
|
|
for clip in upload_clips:
|
|
clip.upload_status = "pending"
|
|
|
|
print(
|
|
"reset complete:",
|
|
f"runs={len(runs)}",
|
|
f"videos={len(videos)}",
|
|
f"render_clips={len(render_clips)}",
|
|
f"upload_clips={len(upload_clips)}",
|
|
f"mode={mode}",
|
|
)
|
|
PY
|
|
|
|
docker compose up -d
|
|
docker compose ps
|
|
REMOTE
|