fix pipelines
This commit is contained in:
@@ -232,13 +232,16 @@ def run_pipeline(session: Session, video_id: int, run_id: int | None = None) ->
|
||||
video.processing_status = "running"
|
||||
update_run(run, "media", "running")
|
||||
prepare_media(session, video)
|
||||
session.commit()
|
||||
|
||||
update_run(run, "transcribe", "running")
|
||||
transcribe_video(session, video)
|
||||
session.commit()
|
||||
|
||||
if settings.suggest_enabled:
|
||||
update_run(run, "suggest", "running")
|
||||
suggest_clips(session, video)
|
||||
session.commit()
|
||||
|
||||
if settings.render_enabled:
|
||||
clips = session.scalars(
|
||||
@@ -250,12 +253,15 @@ def run_pipeline(session: Session, video_id: int, run_id: int | None = None) ->
|
||||
render_clip_by_id(session, clip.id)
|
||||
if settings.upload_enabled:
|
||||
upload_clip_by_id(session, clip.id)
|
||||
session.commit()
|
||||
|
||||
video.processing_status = "done"
|
||||
update_run(run, "done", "done")
|
||||
session.commit()
|
||||
logger.info("pipeline done video_id=%s run_id=%s", video_id, run_id)
|
||||
except Exception as exc:
|
||||
video.processing_status = "failed"
|
||||
update_run(run, "failed", "failed", str(exc))
|
||||
session.commit()
|
||||
logger.exception("pipeline failed video_id=%s run_id=%s", video_id, run_id)
|
||||
raise
|
||||
|
||||
@@ -190,6 +190,7 @@ def chunk_transcript(
|
||||
def parse_clip_response(content: str) -> list[ClipCandidate]:
|
||||
try:
|
||||
payload = json.loads(content)
|
||||
payload = normalize_clip_payload(payload)
|
||||
parsed = ClipCandidateResponse.model_validate(payload)
|
||||
except (json.JSONDecodeError, ValidationError) as exc:
|
||||
logger.error("invalid clip json content_prefix=%s", content[:1000])
|
||||
@@ -197,6 +198,43 @@ def parse_clip_response(content: str) -> list[ClipCandidate]:
|
||||
return parsed.clips
|
||||
|
||||
|
||||
def normalize_clip_payload(payload: object) -> dict:
|
||||
if not isinstance(payload, dict):
|
||||
return {"clips": []}
|
||||
if "clips" not in payload and "candidates" in payload:
|
||||
payload = payload | {"clips": payload["candidates"]}
|
||||
clips = payload.get("clips")
|
||||
if not isinstance(clips, list):
|
||||
return payload
|
||||
normalized_clips = []
|
||||
for clip in clips:
|
||||
if not isinstance(clip, dict):
|
||||
normalized_clips.append(clip)
|
||||
continue
|
||||
normalized = dict(clip)
|
||||
normalized["score"] = normalize_score(normalized.get("score", 0))
|
||||
normalized_clips.append(normalized)
|
||||
return payload | {"clips": normalized_clips}
|
||||
|
||||
|
||||
def normalize_score(value: object) -> float:
|
||||
try:
|
||||
score = float(value)
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("invalid llm score=%r; using 0", value)
|
||||
return 0.0
|
||||
if score > 1:
|
||||
original = score
|
||||
if score <= 10:
|
||||
score = score / 10
|
||||
elif score <= 100:
|
||||
score = score / 100
|
||||
else:
|
||||
score = 1.0
|
||||
logger.debug("normalized llm score original=%s normalized=%s", original, score)
|
||||
return max(0.0, min(1.0, score))
|
||||
|
||||
|
||||
def overlap_ratio(left: ClipCandidate, right: ClipCandidate) -> float:
|
||||
overlap = max(0, min(left.end_sec, right.end_sec) - max(left.start_sec, right.start_sec))
|
||||
shortest = min(left.end_sec - left.start_sec, right.end_sec - right.start_sec)
|
||||
|
||||
@@ -182,7 +182,10 @@ def bootstrap_existing(client: WebDavClient, session: Session) -> BootstrapResul
|
||||
session.add(video)
|
||||
inserted += 1
|
||||
logger.debug("bootstrap existing file=%s size=%s", file.filename, file.size_bytes)
|
||||
elif video.ingest_status == "observing" and video.processing_status == "pending":
|
||||
elif (
|
||||
video.ingest_status in {"observing", "stable", "queued"}
|
||||
and video.processing_status in {"pending", "queued"}
|
||||
):
|
||||
video.size_bytes = file.size_bytes
|
||||
video.ingest_status = "existing_done"
|
||||
video.processing_status = "done"
|
||||
|
||||
@@ -30,6 +30,50 @@ def test_parse_clip_response():
|
||||
assert candidates[0].title_zh == "标题"
|
||||
|
||||
|
||||
def test_parse_clip_response_accepts_candidates_key_and_score_out_of_ten():
|
||||
candidates = parse_clip_response(
|
||||
"""
|
||||
{
|
||||
"candidates": [
|
||||
{
|
||||
"start_sec": 10,
|
||||
"end_sec": 60,
|
||||
"title_zh": "标题",
|
||||
"summary_zh": "摘要",
|
||||
"reason": "有趣",
|
||||
"score": 8.5,
|
||||
"tags": ["反应"],
|
||||
"subtitle_priority": "high"
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
)
|
||||
assert candidates[0].score == 0.85
|
||||
|
||||
|
||||
def test_parse_clip_response_normalizes_score_out_of_hundred():
|
||||
candidates = parse_clip_response(
|
||||
"""
|
||||
{
|
||||
"clips": [
|
||||
{
|
||||
"start_sec": 10,
|
||||
"end_sec": 60,
|
||||
"title_zh": "标题",
|
||||
"summary_zh": "摘要",
|
||||
"reason": "有趣",
|
||||
"score": 92,
|
||||
"tags": ["反应"],
|
||||
"subtitle_priority": "high"
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
)
|
||||
assert candidates[0].score == 0.92
|
||||
|
||||
|
||||
def test_dedupe_and_rank_candidates_filters_short_and_overlapping():
|
||||
candidates = [
|
||||
ClipCandidate(
|
||||
|
||||
Reference in New Issue
Block a user