Measure the frame rate rather than believing the container
`probe` took `r_frame_rate` whenever it was at or under the cap, on the grounds that it is the rate that keeps every distinct source frame. It is not a claim about frames at all: ordinary iPhone footage declares 120 over a stream whose timestamps are 1/30s apart, and resampling it up turned an 11-second clip into 1293 proxy frames instead of 323 — four times the encode, four times the tracing stills (91MB against 23MB), four times the blobs and the rows, for 970 frames that are copies of their neighbours. So `_measured_rate` reads the timestamps and `_choose_rate` keeps whichever declared rate they bear out. Two details carry it: the times are sorted before differencing, because an HEVC stream arrives in decode order and differencing that measures the reordering delay instead of the rate; and the statistic is the MEDIAN interval, which is what keeps the property the nominal rate was being taken for — a take held on one frame still reports the rate of the parts that move, so no distinct frame is dropped. A genuine 120fps capture still extracts at 120, and there is a test on that specifically. `Source.probe` also stopped being the place a reading goes to be preserved. The facts are a pure function of bytes that are the row's own identity, so a re-upload re-reads them: otherwise every already-uploaded source would have gone on resampling to four times the frames with no way to correct it short of deleting the row. Already-extracted footage is untouched — `extraction_key` still says scheme 3, so those jobs stay done and reachable. Bumping it re-extracts everything at the corrected rate. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Your Name
parent
8d20097e61
commit
e26ad723fa
3 changed files with 210 additions and 7 deletions
|
|
@ -159,6 +159,16 @@ def _extract_stills(job, proxy_path, frames_dir, frames, root):
|
|||
|
||||
|
||||
MAX_RATE = 120 # a capture rate; past this the container is describing something else
|
||||
# How many packet timestamps `_measured_rate` reads, and the fewest intervals it
|
||||
# will draw a conclusion from. 300 is a flat cost on a long take and still a
|
||||
# wide enough sample for a median; below 8 intervals there is not enough of a
|
||||
# stream to outvote one odd timestamp, so the metadata is left to speak.
|
||||
RATE_SAMPLE = 300
|
||||
RATE_MINIMUM = 8
|
||||
# How far a declared rate may sit from the measured one and still be taken as
|
||||
# what the stream is: 2% covers 30 against 30000/1001 and nothing like 120
|
||||
# against 30.
|
||||
RATE_TOLERANCE = 0.02
|
||||
|
||||
|
||||
def probe_image(path):
|
||||
|
|
@ -184,6 +194,75 @@ def probe_audio(path):
|
|||
return duration
|
||||
|
||||
|
||||
def _measured_rate(path):
|
||||
"""The rate the stream's own packet timestamps imply, or None.
|
||||
|
||||
THE CONTAINER'S SUMMARY OF ITSELF IS NOT EVIDENCE, and this is the function
|
||||
that goes and looks. An iPhone's `r_frame_rate` is 120 on footage whose
|
||||
timestamps are 1/30s apart, which is the difference between 323 frames and
|
||||
1293 — four times the encode, four times the tracing stills, four times the
|
||||
blobs, for 970 frames that are copies of their neighbours.
|
||||
|
||||
It reads TIMESTAMPS, not frames: `-show_entries packet=pts_time` demuxes
|
||||
without decoding, so this costs a file read and no pixels. The times are
|
||||
SORTED before differencing because a stream with B-frames arrives in decode
|
||||
order — an HEVC clip's first packets come out 0, 0.133, 0.067, 0.033 — and
|
||||
differencing that order measures the reordering rather than the rate.
|
||||
|
||||
THE MEDIAN INTERVAL, which is what makes this safe on genuinely variable
|
||||
input. It answers "how far apart are two frames normally", so a take held on
|
||||
one frame for a second still reports the rate of the parts that move, and
|
||||
choosing it keeps every distinct frame — the property `probe` used to reach
|
||||
for by taking the nominal rate. Only the last few intervals of the sample are
|
||||
unreliable (a frame whose turn comes after the window is missing from it), and
|
||||
a median does not care.
|
||||
|
||||
Returning None is the honest answer for a clip too short to sample, and this
|
||||
also swallows a probe that fails outright: the rate the metadata declares is
|
||||
the documented fallback, so an optimisation must not be able to refuse an
|
||||
upload that would otherwise have been accepted.
|
||||
"""
|
||||
try:
|
||||
text = _command(["ffprobe", "-v", "error", "-select_streams", "v:0",
|
||||
"-show_entries", "packet=pts_time", "-of", "json",
|
||||
"-read_intervals", f"%+#{RATE_SAMPLE}", str(path)])
|
||||
packets = json.loads(text).get("packets") or []
|
||||
times = sorted(float(packet["pts_time"]) for packet in packets
|
||||
if (packet.get("pts_time") or "N/A") != "N/A")
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
intervals = sorted(b - a for a, b in zip(times, times[1:]) if b > a)
|
||||
if len(intervals) < RATE_MINIMUM:
|
||||
return None
|
||||
median = intervals[len(intervals) // 2]
|
||||
return 1.0 / median if median > 0 else None
|
||||
|
||||
|
||||
def _choose_rate(nominal, average, measured):
|
||||
"""The rate to resample onto, as an exact Fraction.
|
||||
|
||||
A DECLARED RATE IS PREFERRED WHEN IT AGREES WITH THE TIMESTAMPS, because it is
|
||||
the exact rational the stream was authored at — 30000/1001 is not a float, and
|
||||
`limit_denominator` on a measured 29.97 is a guess at a number the container
|
||||
already states. So the measured rate is used to CHOOSE between what the
|
||||
container declares, and only stands in itself when neither declaration
|
||||
describes the stream.
|
||||
"""
|
||||
candidates = [rate for rate in (nominal, average) if 0 < rate <= MAX_RATE]
|
||||
if measured:
|
||||
agreeing = [rate for rate in candidates
|
||||
if abs(float(rate) - measured) <= RATE_TOLERANCE * measured]
|
||||
if agreeing:
|
||||
return min(agreeing, key=lambda rate: abs(float(rate) - measured))
|
||||
from_timestamps = Fraction(measured).limit_denominator(1001)
|
||||
if 0 < from_timestamps <= MAX_RATE:
|
||||
return from_timestamps
|
||||
# Nothing to go on but the metadata, and nominal first keeps the rate that
|
||||
# drops no distinct frame. An unusable pair falls through to the refusal
|
||||
# below, which names the rate the file claimed rather than one of these.
|
||||
return nominal if 0 < nominal <= MAX_RATE else average
|
||||
|
||||
|
||||
def probe(path):
|
||||
"""What the upload is, as far as choosing a proxy rate goes.
|
||||
|
||||
|
|
@ -201,12 +280,21 @@ def probe(path):
|
|||
of itself disagreed with the container's own contents, so the guard rejected
|
||||
CFR video for being variable.
|
||||
|
||||
THE RATE IS THE NOMINAL ONE. `r_frame_rate` is the rate every timestamp in the
|
||||
stream can be expressed at, which is the rate that keeps every distinct source
|
||||
frame; resampling to the average would drop some. Duration is preserved either
|
||||
way — ffmpeg's CFR conversion is driven by timestamps, so the audio stays in
|
||||
sync at any rate — so this trades a possible duplicated frame against a
|
||||
certainly lost one.
|
||||
THE RATE IS MEASURED AND THE DECLARATIONS ARE VOTED ON, which is the same
|
||||
distrust applied to the one number that still comes from here. This used to
|
||||
take `r_frame_rate` outright — the rate every timestamp in the stream can be
|
||||
expressed at, and so the rate that keeps every distinct source frame. The
|
||||
trouble is that it is not a claim about frames at all: the file above declares
|
||||
120 and holds 30, and resampling it up cost four times the encode, four times
|
||||
the tracing stills and four times the blobs for 970 duplicated frames. So
|
||||
`_measured_rate` reads the timestamps, `_choose_rate` keeps whichever declared
|
||||
rate they bear out, and the nominal rate is believed when it is true rather
|
||||
than because it is nominal.
|
||||
|
||||
Duration is preserved either way — ffmpeg's CFR conversion is driven by
|
||||
timestamps, so the audio stays in sync at any rate — and the median interval
|
||||
keeps the no-distinct-frame-dropped property that taking the nominal rate was
|
||||
reaching for. See `_measured_rate`.
|
||||
"""
|
||||
data = json.loads(_command(["ffprobe", "-v", "error", "-show_streams",
|
||||
"-show_format", "-of", "json", str(path)]))
|
||||
|
|
@ -217,7 +305,8 @@ def probe(path):
|
|||
average = Fraction(video.get("avg_frame_rate") or "0")
|
||||
if nominal <= 0 and average <= 0:
|
||||
raise ValueError("the video's frame rate is unknown")
|
||||
rate = nominal if 0 < nominal <= MAX_RATE else average
|
||||
measured = _measured_rate(path)
|
||||
rate = _choose_rate(nominal, average, measured)
|
||||
if not 0 < rate <= MAX_RATE:
|
||||
raise ValueError(f"the video reports a frame rate of {float(rate):g}, which is "
|
||||
"not a rate footage can be measured at")
|
||||
|
|
@ -230,6 +319,11 @@ def probe(path):
|
|||
# `-r` a rounded one is how a long take drifts out of sync.
|
||||
"rate": f"{rate.numerator}/{rate.denominator}",
|
||||
"nominal_fps": float(nominal), "average_fps": float(average),
|
||||
# What the timestamps said, and null when there were too few to ask.
|
||||
# Recorded because it is the input to a decision this file used not to
|
||||
# make, and the one number that explains a chosen rate matching
|
||||
# neither declaration.
|
||||
"measured_fps": measured,
|
||||
"width": int(video["width"]), "height": int(video["height"]),
|
||||
"duration": duration,
|
||||
# KEPT, AND NO LONGER TRUSTED AS A COUNT. See the docstring: this is
|
||||
|
|
|
|||
|
|
@ -908,6 +908,37 @@ class UploadTests(TestCase):
|
|||
self.assertEqual("image/jpeg", still["Content-Type"])
|
||||
self.assertEqual(200, self.client.get(footage["audio"]).status_code)
|
||||
|
||||
def test_re_uploading_a_source_re_reads_its_facts(self):
|
||||
# A SOURCE ROW HOLDS A READING, NOT A DECISION. The facts are a pure
|
||||
# function of bytes that are themselves this row's identity, so the row
|
||||
# cannot be the place a reading goes to be preserved: `probe` got better
|
||||
# at phone footage — it stopped believing a declared 120 over timestamps
|
||||
# 1/30s apart — and a stored reading that nothing can replace would have
|
||||
# left every already-uploaded source resampling to four times the frames
|
||||
# with no way to correct it short of deleting the row.
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "four-frames.mp4"
|
||||
subprocess.run([
|
||||
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-f", "lavfi", "-i", "color=c=red:s=64x48:r=4:d=1",
|
||||
"-c:v", "mpeg4", str(path),
|
||||
], check=True, capture_output=True)
|
||||
payload = path.read_bytes()
|
||||
|
||||
first = self.client.post("/api/sources", {
|
||||
"file": SimpleUploadedFile("four-frames.mp4", payload, content_type="video/mp4")})
|
||||
self.assertEqual(201, first.status_code, first.content)
|
||||
self.assertEqual(4.0, first.json()["probe"]["fps"])
|
||||
|
||||
better = dict(first.json()["probe"], fps=12.0, rate="12/1", measured_fps=12.0)
|
||||
with patch("clips.extraction.probe", return_value=better):
|
||||
again = self.client.post("/api/sources", {
|
||||
"file": SimpleUploadedFile("same.mp4", payload, content_type="video/mp4")})
|
||||
self.assertEqual(200, again.status_code, again.content)
|
||||
self.assertFalse(again.json()["created"], "the same bytes are the same source")
|
||||
self.assertEqual("12/1", again.json()["probe"]["rate"])
|
||||
self.assertEqual(12.0, Source.objects.get(id=first.json()["id"]).probe["fps"])
|
||||
|
||||
def test_the_proxy_is_re_encoded_rather_than_the_upload_re_served(self):
|
||||
# The footage's identity is the proxy's digest, and the proxy is produced
|
||||
# by one ffmpeg invocation whatever the upload was. If the upload were
|
||||
|
|
@ -955,6 +986,73 @@ class UploadTests(TestCase):
|
|||
self.assertTrue(facts["vfr"], "the disagreement is still recorded, just not fatal")
|
||||
self.assertTrue(facts["has_audio"])
|
||||
|
||||
def test_a_declared_rate_the_timestamps_do_not_bear_out_is_not_resampled_to(self):
|
||||
# THE FOUR-TIMES. An iPhone container declares `r_frame_rate` 120 over a
|
||||
# stream whose frames are 1/30s apart, and taking the declaration at its
|
||||
# word turned an 11-second clip into 1293 proxy frames instead of 323:
|
||||
# four times the encode, four times the tracing stills, four times the
|
||||
# blobs and the rows, for 970 frames that are copies of their neighbours.
|
||||
# The timestamps are the evidence and they say 30.
|
||||
streams = json.dumps({"streams": [
|
||||
{"codec_type": "video", "r_frame_rate": "120/1",
|
||||
"avg_frame_rate": "96900/3233", "nb_frames": "323",
|
||||
"width": 1920, "height": 1440},
|
||||
{"codec_type": "audio"}],
|
||||
"format": {"duration": "10.775"}})
|
||||
# IN DECODE ORDER, which is how an HEVC stream really arrives — the first
|
||||
# packets of the fixture this was found on come out 0, 0.133, 0.067,
|
||||
# 0.033. Differencing that order unsorted measures the reordering delay
|
||||
# and not the rate, so the fixture keeps the hazard in it.
|
||||
shuffled = [0, 4, 2, 1, 3, 8, 6, 5, 7, 12, 10, 9, 11]
|
||||
packets = json.dumps({"packets": [{"pts_time": f"{i / 30:.6f}"} for i in shuffled]})
|
||||
with patch("clips.extraction._command", side_effect=[streams, packets]):
|
||||
facts = extraction.probe(Path("phone.mov"))
|
||||
self.assertEqual("96900/3233", facts["rate"], "resampled to the declared 120")
|
||||
self.assertAlmostEqual(30.0, facts["measured_fps"], places=2)
|
||||
|
||||
def test_a_genuine_high_rate_capture_is_still_taken_at_its_own_rate(self):
|
||||
# The other half of the same decision, and the one that would be easy to
|
||||
# break: a real 120fps capture must not be dragged down to anything. Its
|
||||
# declaration and its timestamps agree, so the declaration — the exact
|
||||
# rational the stream was authored at — is what is used.
|
||||
streams = json.dumps({"streams": [
|
||||
{"codec_type": "video", "r_frame_rate": "120/1", "avg_frame_rate": "120/1",
|
||||
"width": 640, "height": 480}],
|
||||
"format": {"duration": "2"}})
|
||||
packets = json.dumps({"packets": [{"pts_time": f"{i / 120:.6f}"} for i in range(13)]})
|
||||
with patch("clips.extraction._command", side_effect=[streams, packets]):
|
||||
facts = extraction.probe(Path("slowmo.mov"))
|
||||
self.assertEqual("120/1", facts["rate"])
|
||||
|
||||
def test_too_few_timestamps_to_measure_leaves_the_declaration_standing(self):
|
||||
# A clip with nine-ish frames cannot outvote one odd timestamp, so the
|
||||
# measurement declines to have an opinion and the nominal rate — the one
|
||||
# that drops no distinct frame — is used exactly as it was before.
|
||||
streams = json.dumps({"streams": [
|
||||
{"codec_type": "video", "r_frame_rate": "30/1", "avg_frame_rate": "24/1",
|
||||
"width": 640, "height": 480}],
|
||||
"format": {"duration": "0.1"}})
|
||||
packets = json.dumps({"packets": [{"pts_time": f"{i / 30:.6f}"} for i in range(3)]})
|
||||
with patch("clips.extraction._command", side_effect=[streams, packets]):
|
||||
facts = extraction.probe(Path("tiny.mov"))
|
||||
self.assertEqual("30/1", facts["rate"])
|
||||
self.assertIsNone(facts["measured_fps"])
|
||||
|
||||
def test_a_rate_measurement_that_fails_outright_cannot_refuse_an_upload(self):
|
||||
# The measurement is an optimisation. If ffprobe cannot read the packets
|
||||
# of a file whose streams it just read happily, the upload still has to be
|
||||
# accepted on its metadata — an optimisation that can reject work is worse
|
||||
# than no optimisation.
|
||||
streams = json.dumps({"streams": [
|
||||
{"codec_type": "video", "r_frame_rate": "25/1", "avg_frame_rate": "25/1",
|
||||
"width": 640, "height": 480}],
|
||||
"format": {"duration": "4"}})
|
||||
with patch("clips.extraction._command",
|
||||
side_effect=[streams, ValueError("ffprobe fell over")]):
|
||||
facts = extraction.probe(Path("awkward.mov"))
|
||||
self.assertEqual("25/1", facts["rate"])
|
||||
self.assertIsNone(facts["measured_fps"])
|
||||
|
||||
def test_the_proxy_rate_is_exact_rather_than_a_rounded_float(self):
|
||||
# 30000/1001 is not a float. Handing ffmpeg's -r a rounded one is how a
|
||||
# long take drifts out of sync with its own audio.
|
||||
|
|
|
|||
|
|
@ -218,6 +218,17 @@ def sources(request):
|
|||
"media_type": upload.content_type or "video/mp4"})
|
||||
row, created = Source.objects.get_or_create(
|
||||
blob=blob, defaults={"filename": Path(upload.name).name[:255], "probe": facts})
|
||||
if not created and row.probe != facts:
|
||||
# THE FACTS ARE RE-READ, NOT REMEMBERED. They are a pure function of
|
||||
# the bytes, and the bytes are this row's identity — so a
|
||||
# disagreement means the server reads the file differently now from
|
||||
# whenever it first saw it, and the fresh reading is the one to keep.
|
||||
# Storing the first reading forever pins a source to a rate the code
|
||||
# no longer believes in, and makes it unfixable without deleting the
|
||||
# row: `extraction.probe` got better at phone footage and every
|
||||
# already-uploaded source would have gone on being wrong.
|
||||
row.probe = facts
|
||||
row.save(update_fields=["probe"])
|
||||
return JsonResponse({"id": str(row.id), "digest": digest,
|
||||
"filename": row.filename, "probe": row.probe,
|
||||
"created": created}, status=201 if created else 200)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue