add dts extractor #61

This commit is contained in:
Curid 2023-03-26 16:41:10 +00:00
parent 320f208fc7
commit d03baa5689
9 changed files with 1877 additions and 37 deletions

153
Cargo.lock generated
View File

@ -127,7 +127,7 @@ checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.98",
] ]
[[package]] [[package]]
@ -188,9 +188,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]] [[package]]
name = "bitstream-io" name = "bitstream-io"
version = "1.5.0" version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97d524fdb78bf6dc6d2dc4c02043e4b4962ede0a17ae3e13f0ed211a7eda5897" checksum = "9d28070975aaf4ef1fd0bd1f29b739c06c2cdd9972e090617fb6dca3b2cb564e"
[[package]] [[package]]
name = "bitvec" name = "bitvec"
@ -391,7 +391,7 @@ dependencies = [
"proc-macro-error", "proc-macro-error",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.98",
] ]
[[package]] [[package]]
@ -417,13 +417,17 @@ name = "client"
version = "0.0.0" version = "0.0.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bitstream-io",
"bytes", "bytes",
"clap 4.1.4", "clap 4.1.4",
"futures", "futures",
"itertools", "itertools",
"log", "log",
"mylog", "mylog",
"pretty_assertions",
"retina", "retina",
"test-case",
"thiserror",
"tokio", "tokio",
"url", "url",
] ]
@ -606,6 +610,16 @@ dependencies = [
"subtle", "subtle",
] ]
[[package]]
name = "ctor"
version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096"
dependencies = [
"quote",
"syn 1.0.98",
]
[[package]] [[package]]
name = "ctr" name = "ctr"
version = "0.6.0" version = "0.6.0"
@ -658,7 +672,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"strsim", "strsim",
"syn", "syn 1.0.98",
] ]
[[package]] [[package]]
@ -669,7 +683,7 @@ checksum = "29b5acf0dea37a7f66f7b25d2c5e93fd46f8f6968b1a5d7a3e02e97768afc95a"
dependencies = [ dependencies = [
"darling_core", "darling_core",
"quote", "quote",
"syn", "syn 1.0.98",
] ]
[[package]] [[package]]
@ -696,7 +710,7 @@ dependencies = [
"nom 6.1.2", "nom 6.1.2",
"num-bigint", "num-bigint",
"num-traits", "num-traits",
"syn", "syn 1.0.98",
] ]
[[package]] [[package]]
@ -707,7 +721,7 @@ checksum = "c73af209b6a5dc8ca7cbaba720732304792cddc933cfea3d74509c2b1ef2f436"
dependencies = [ dependencies = [
"num-bigint", "num-bigint",
"num-traits", "num-traits",
"syn", "syn 1.0.98",
] ]
[[package]] [[package]]
@ -754,7 +768,7 @@ dependencies = [
"darling", "darling",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.98",
] ]
[[package]] [[package]]
@ -764,9 +778,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58a94ace95092c5acb1e97a7e846b310cfbd499652f72297da7493f618a98d73" checksum = "58a94ace95092c5acb1e97a7e846b310cfbd499652f72297da7493f618a98d73"
dependencies = [ dependencies = [
"derive_builder_core", "derive_builder_core",
"syn", "syn 1.0.98",
] ]
[[package]]
name = "diff"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
[[package]] [[package]]
name = "digest" name = "digest"
version = "0.9.0" version = "0.9.0"
@ -794,7 +814,7 @@ checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.98",
] ]
[[package]] [[package]]
@ -981,7 +1001,7 @@ checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.98",
] ]
[[package]] [[package]]
@ -1545,6 +1565,15 @@ version = "6.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
[[package]]
name = "output_vt100"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66"
dependencies = [
"winapi",
]
[[package]] [[package]]
name = "p256" name = "p256"
version = "0.10.1" version = "0.10.1"
@ -1612,7 +1641,7 @@ checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.98",
] ]
[[package]] [[package]]
@ -1701,6 +1730,18 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5" checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5"
[[package]]
name = "pretty_assertions"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a25e9bcb20aa780fd0bb16b72403a9064d6b3f22f026946029acb941a50af755"
dependencies = [
"ctor",
"diff",
"output_vt100",
"yansi",
]
[[package]] [[package]]
name = "proc-macro-error" name = "proc-macro-error"
version = "1.0.4" version = "1.0.4"
@ -1710,7 +1751,7 @@ dependencies = [
"proc-macro-error-attr", "proc-macro-error-attr",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.98",
"version_check", "version_check",
] ]
@ -1727,18 +1768,18 @@ dependencies = [
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.50" version = "1.0.53"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73"
dependencies = [ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.20" version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
] ]
@ -1884,10 +1925,12 @@ dependencies = [
"once_cell", "once_cell",
"pin-project", "pin-project",
"pretty-hex", "pretty-hex",
"pretty_assertions",
"rand", "rand",
"rtsp-types", "rtsp-types",
"sdp-types", "sdp-types",
"smallvec", "smallvec",
"test-case",
"thiserror", "thiserror",
"time", "time",
"tokio", "tokio",
@ -2139,7 +2182,7 @@ checksum = "dc1d3230c1de7932af58ad8ffbe1d784bd55efd5a9d84ac24f69c72d83543dfb"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.98",
] ]
[[package]] [[package]]
@ -2310,6 +2353,17 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "syn"
version = "2.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aad1363ed6d37b84299588d62d3a7d95b5a5c2d9aad5c85609fda12afaa1f40"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]] [[package]]
name = "synstructure" name = "synstructure"
version = "0.12.6" version = "0.12.6"
@ -2318,7 +2372,7 @@ checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.98",
"unicode-xid", "unicode-xid",
] ]
@ -2337,6 +2391,41 @@ dependencies = [
"winapi-util", "winapi-util",
] ]
[[package]]
name = "test-case"
version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "679b019fb241da62cc449b33b224d19ebe1c6767b495569765115dd7f7f9fba4"
dependencies = [
"test-case-macros",
]
[[package]]
name = "test-case-core"
version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72dc21b5887f4032c4656502d085dc28f2afbb686f25f216472bb0526f4b1b88"
dependencies = [
"cfg-if",
"proc-macro-error",
"proc-macro2",
"quote",
"syn 1.0.98",
]
[[package]]
name = "test-case-macros"
version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3786898e0be151a96f730fd529b0e8a10f5990fa2a7ea14e37ca27613c05190"
dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
"syn 1.0.98",
"test-case-core",
]
[[package]] [[package]]
name = "textwrap" name = "textwrap"
version = "0.16.0" version = "0.16.0"
@ -2345,22 +2434,22 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.31" version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
dependencies = [ dependencies = [
"thiserror-impl", "thiserror-impl",
] ]
[[package]] [[package]]
name = "thiserror-impl" name = "thiserror-impl"
version = "1.0.31" version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 2.0.10",
] ]
[[package]] [[package]]
@ -2427,7 +2516,7 @@ checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.98",
] ]
[[package]] [[package]]
@ -2635,7 +2724,7 @@ dependencies = [
"log", "log",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.98",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -2657,7 +2746,7 @@ checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.98",
"wasm-bindgen-backend", "wasm-bindgen-backend",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -3095,6 +3184,12 @@ dependencies = [
"thiserror", "thiserror",
] ]
[[package]]
name = "yansi"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec"
[[package]] [[package]]
name = "yasna" name = "yasna"
version = "0.4.0" version = "0.4.0"
@ -3121,6 +3216,6 @@ checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 1.0.98",
"synstructure", "synstructure",
] ]

View File

@ -40,6 +40,8 @@ url = "2.2.1"
[dev-dependencies] [dev-dependencies]
criterion = { version = "0.4.0", features = ["async_tokio"] } criterion = { version = "0.4.0", features = ["async_tokio"] }
mylog = { git = "https://github.com/scottlamb/mylog" } mylog = { git = "https://github.com/scottlamb/mylog" }
pretty_assertions = "1.3.0"
test-case = "3.0.0"
tokio = { version = "1.5.0", features = ["io-util", "macros", "rt-multi-thread", "test-util"] } tokio = { version = "1.5.0", features = ["io-util", "macros", "rt-multi-thread", "test-util"] }
[profile.bench] [profile.bench]

View File

@ -118,6 +118,7 @@ impl Timeline {
Ok(( Ok((
Timestamp { Timestamp {
timestamp, timestamp,
dts: None,
clock_rate: self.clock_rate, clock_rate: self.clock_rate,
start, start,
}, },

View File

@ -805,6 +805,7 @@ mod tests {
).unwrap(); ).unwrap();
let timestamp = crate::Timestamp { let timestamp = crate::Timestamp {
timestamp: 42, timestamp: 42,
dts: None,
clock_rate: NonZeroU32::new(48_000).unwrap(), clock_rate: NonZeroU32::new(48_000).unwrap(),
start: 0, start: 0,
}; };
@ -1001,6 +1002,7 @@ mod tests {
).unwrap(); ).unwrap();
let timestamp = crate::Timestamp { let timestamp = crate::Timestamp {
timestamp: 42, timestamp: 42,
dts: None,
clock_rate: NonZeroU32::new(48_000).unwrap(), clock_rate: NonZeroU32::new(48_000).unwrap(),
start: 0, start: 0,
}; };
@ -1105,6 +1107,7 @@ mod tests {
).unwrap(); ).unwrap();
let timestamp = crate::Timestamp { let timestamp = crate::Timestamp {
timestamp: 42, timestamp: 42,
dts: None,
clock_rate: NonZeroU32::new(48_000).unwrap(), clock_rate: NonZeroU32::new(48_000).unwrap(),
start: 0, start: 0,
}; };
@ -1211,6 +1214,7 @@ mod tests {
).unwrap(); ).unwrap();
let timestamp = crate::Timestamp { let timestamp = crate::Timestamp {
timestamp: 42, timestamp: 42,
dts: None,
clock_rate: NonZeroU32::new(48_000).unwrap(), clock_rate: NonZeroU32::new(48_000).unwrap(),
start: 0, start: 0,
}; };
@ -1320,6 +1324,7 @@ mod tests {
).unwrap(); ).unwrap();
let timestamp = crate::Timestamp { let timestamp = crate::Timestamp {
timestamp: 42, timestamp: 42,
dts: None,
clock_rate: NonZeroU32::new(48_000).unwrap(), clock_rate: NonZeroU32::new(48_000).unwrap(),
start: 0, start: 0,
}; };

View File

@ -3,6 +3,10 @@
//! [H.264](https://www.itu.int/rec/T-REC-H.264-201906-I/en)-encoded video. //! [H.264](https://www.itu.int/rec/T-REC-H.264-201906-I/en)-encoded video.
pub mod dts_extractor;
pub mod emulation_prevention;
pub mod sps;
use std::convert::TryFrom; use std::convert::TryFrom;
use std::fmt::Write; use std::fmt::Write;
@ -11,6 +15,7 @@ use h264_reader::nal::{NalHeader, UnitType};
use log::{debug, log_enabled, trace}; use log::{debug, log_enabled, trace};
use crate::{ use crate::{
codec::h264::{dts_extractor::DtsExtractor, sps::Sps},
rtp::{ReceivedPacket, ReceivedPacketBuilder}, rtp::{ReceivedPacket, ReceivedPacketBuilder},
Error, Timestamp, Error, Timestamp,
}; };
@ -43,6 +48,9 @@ pub(crate) struct Depacketizer {
/// In state `PreMark`, an entry for each NAL. /// In state `PreMark`, an entry for each NAL.
/// Kept around (empty) in other states to re-use the backing allocation. /// Kept around (empty) in other states to re-use the backing allocation.
nals: Vec<Nal>, nals: Vec<Nal>,
sps: Option<Sps>,
dts_extractor: Option<DtsExtractor>,
} }
#[derive(Debug)] #[derive(Debug)]
@ -107,13 +115,16 @@ impl Depacketizer {
)); ));
} }
let parameters = match format_specific_params { let (parameters, sps) = match format_specific_params {
None => None, None => (None, None),
Some(fp) => match InternalParameters::parse_format_specific_params(fp) { Some(fp) => match InternalParameters::parse_format_specific_params(fp) {
Ok(p) => Some(p), Ok(p) => {
let sps = Sps::deserialize(&p.sps_nal).map_err(|e| e.to_string())?;
(Some(p), Some(sps))
}
Err(e) => { Err(e) => {
log::warn!("Ignoring bad H.264 format-specific-params {:?}: {}", fp, e); log::warn!("Ignoring bad H.264 format-specific-params {:?}: {}", fp, e);
None (None, None)
} }
}, },
}; };
@ -123,6 +134,8 @@ impl Depacketizer {
pieces: Vec::new(), pieces: Vec::new(),
nals: Vec::new(), nals: Vec::new(),
parameters, parameters,
sps,
dts_extractor: None,
}) })
} }
@ -425,7 +438,11 @@ impl Depacketizer {
} }
} }
fn finalize_access_unit(&mut self, au: AccessUnit, reason: &str) -> Result<VideoFrame, String> { fn finalize_access_unit(
&mut self,
mut au: AccessUnit,
reason: &str,
) -> Result<VideoFrame, String> {
let mut piece_idx = 0; let mut piece_idx = 0;
let mut retained_len = 0usize; let mut retained_len = 0usize;
let mut is_random_access_point = false; let mut is_random_access_point = false;
@ -472,11 +489,23 @@ impl Depacketizer {
} }
let mut data = Vec::with_capacity(retained_len); let mut data = Vec::with_capacity(retained_len);
piece_idx = 0; piece_idx = 0;
let mut nalus: Vec<Vec<u8>> = Vec::new();
for nal in &self.nals { for nal in &self.nals {
let hdr: u8 = nal.hdr.into();
let next_piece_idx = usize::try_from(nal.next_piece_idx).expect("u32 fits in usize"); let next_piece_idx = usize::try_from(nal.next_piece_idx).expect("u32 fits in usize");
let nal_pieces = &self.pieces[piece_idx..next_piece_idx]; let nal_pieces = &self.pieces[piece_idx..next_piece_idx];
let mut nalu = Vec::new();
nalu.push(hdr.to_owned());
for piece in nal_pieces {
nalu.extend_from_slice(&piece[..]);
}
nalus.push(nalu);
data.extend_from_slice(&nal.len.to_be_bytes()[..]); data.extend_from_slice(&nal.len.to_be_bytes()[..]);
data.push(nal.hdr.into()); data.push(hdr);
let mut actual_len = 1; let mut actual_len = 1;
for piece in nal_pieces { for piece in nal_pieces {
data.extend_from_slice(&piece[..]); data.extend_from_slice(&piece[..]);
@ -488,6 +517,30 @@ impl Depacketizer {
); );
piece_idx = next_piece_idx; piece_idx = next_piece_idx;
} }
if let Some(sps) = &self.sps {
let pts = au.timestamp.timestamp();
// If first sync sample has been received.
if let Some(dts_extractor) = &mut self.dts_extractor {
au.timestamp.dts = Some(
dts_extractor
.extract(sps, &nalus, pts)
.map_err(|e| e.to_string())?,
);
// Skip samples silently until we find one with an IDR.
} else if is_random_access_point {
let mut dts_extractor = DtsExtractor::new();
dts_extractor
.extract(sps, &nalus, 0)
.map_err(|e| e.to_string())?;
self.dts_extractor = Some(dts_extractor);
au.timestamp.dts = Some(0);
}
}
debug_assert_eq!(retained_len, data.len()); debug_assert_eq!(retained_len, data.len());
self.nals.clear(); self.nals.clear();
self.pieces.clear(); self.pieces.clear();
@ -511,6 +564,7 @@ impl Depacketizer {
} }
_ => false, _ => false,
}; };
Ok(VideoFrame { Ok(VideoFrame {
has_new_parameters, has_new_parameters,
loss: au.loss, loss: au.loss,
@ -997,6 +1051,7 @@ enum PacketizerState {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use pretty_assertions::assert_eq;
use std::num::NonZeroU32; use std::num::NonZeroU32;
use crate::testutil::init_logging; use crate::testutil::init_logging;
@ -1062,6 +1117,7 @@ mod tests {
let mut d = super::Depacketizer::new(90_000, Some("packetization-mode=1;profile-level-id=64001E;sprop-parameter-sets=Z2QAHqwsaoLA9puCgIKgAAADACAAAAMD0IAA,aO4xshsA")).unwrap(); let mut d = super::Depacketizer::new(90_000, Some("packetization-mode=1;profile-level-id=64001E;sprop-parameter-sets=Z2QAHqwsaoLA9puCgIKgAAADACAAAAMD0IAA,aO4xshsA")).unwrap();
let timestamp = crate::Timestamp { let timestamp = crate::Timestamp {
timestamp: 0, timestamp: 0,
dts: None,
clock_rate: NonZeroU32::new(90_000).unwrap(), clock_rate: NonZeroU32::new(90_000).unwrap(),
start: 0, start: 0,
}; };
@ -1171,11 +1227,13 @@ mod tests {
let mut d = super::Depacketizer::new(90_000, Some("packetization-mode=1;profile-level-id=640033;sprop-parameter-sets=Z2QAM6wVFKCgL/lQ,aO48sA==")).unwrap(); let mut d = super::Depacketizer::new(90_000, Some("packetization-mode=1;profile-level-id=640033;sprop-parameter-sets=Z2QAM6wVFKCgL/lQ,aO48sA==")).unwrap();
let ts1 = crate::Timestamp { let ts1 = crate::Timestamp {
timestamp: 0, timestamp: 0,
dts: None,
clock_rate: NonZeroU32::new(90_000).unwrap(), clock_rate: NonZeroU32::new(90_000).unwrap(),
start: 0, start: 0,
}; };
let ts2 = crate::Timestamp { let ts2 = crate::Timestamp {
timestamp: 1, timestamp: 1,
dts: Some(0),
clock_rate: NonZeroU32::new(90_000).unwrap(), clock_rate: NonZeroU32::new(90_000).unwrap(),
start: 0, start: 0,
}; };
@ -1255,11 +1313,13 @@ mod tests {
let mut d = super::Depacketizer::new(90_000, Some("packetization-mode=1;profile-level-id=640033;sprop-parameter-sets=Z2QAM6wVFKCgL/lQ,aO48sA==")).unwrap(); let mut d = super::Depacketizer::new(90_000, Some("packetization-mode=1;profile-level-id=640033;sprop-parameter-sets=Z2QAM6wVFKCgL/lQ,aO48sA==")).unwrap();
let ts1 = crate::Timestamp { let ts1 = crate::Timestamp {
timestamp: 0, timestamp: 0,
dts: None,
clock_rate: NonZeroU32::new(90_000).unwrap(), clock_rate: NonZeroU32::new(90_000).unwrap(),
start: 0, start: 0,
}; };
let ts2 = crate::Timestamp { let ts2 = crate::Timestamp {
timestamp: 1, timestamp: 1,
dts: Some(0),
clock_rate: NonZeroU32::new(90_000).unwrap(), clock_rate: NonZeroU32::new(90_000).unwrap(),
start: 0, start: 0,
}; };
@ -1361,6 +1421,7 @@ mod tests {
} }
let timestamp = crate::Timestamp { let timestamp = crate::Timestamp {
timestamp: 0, timestamp: 0,
dts: None,
clock_rate: NonZeroU32::new(90_000).unwrap(), clock_rate: NonZeroU32::new(90_000).unwrap(),
start: 0, start: 0,
}; };
@ -1469,6 +1530,7 @@ mod tests {
// The stream should honor in-band parameters. // The stream should honor in-band parameters.
let timestamp = crate::Timestamp { let timestamp = crate::Timestamp {
timestamp: 0, timestamp: 0,
dts: None,
clock_rate: NonZeroU32::new(90_000).unwrap(), clock_rate: NonZeroU32::new(90_000).unwrap(),
start: 0, start: 0,
}; };

View File

@ -0,0 +1,403 @@
// https://github.com/bluenviron/gortsplib/blob/f0540b4eee760583b2d94f022674b0f3b0f8c8b0/pkg/codecs/h264/dts_extractor.go
use crate::codec::h264::{
emulation_prevention::emulation_prevention_remove,
sps::{self, Sps},
};
use bitstream_io::{BitRead, BitReader};
use h264_reader::nal::UnitType;
use std::io::Cursor;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum DtsExtractorError {
#[error("{0}")]
FromInt(#[from] std::num::TryFromIntError),
#[error("frame_mbs_only_flag = 0 is not supported")]
FrameMbsNotSupported,
#[error("POC not found")]
PocNotFound,
#[error("invalid POC")]
PocInvalid,
#[error("{0}")]
SpsUnmarshal(#[from] sps::Error),
#[error("SPS not received yet")]
SpsNotReceivedYet,
#[error("pic_order_cnt_type = 1 is not supported yet")]
PicOrderCntType1Unsupported,
#[error("DTS is greater than PTS: '{0}' vs '{1}'")]
DtsGreaterThanPts(i64, i64),
#[error("DTS is not monotonically increasing, was {0}, now is {1}")]
DtsNotIncreasing(i64, i64),
#[error("read: {0}")]
Read(#[from] std::io::Error),
#[error("unit type value {0} out of range")]
UnitType(u8),
}
fn get_picture_order_count(sps: &Sps, buf: &[u8]) -> Result<u32, DtsExtractorError> {
let buf = emulation_prevention_remove(&buf[..6]);
let mut r = BitReader::new(Cursor::new(buf));
r.skip(8)?;
sps::read_golomb_unsigned(&mut r)?; // first_mb_in_slice
sps::read_golomb_unsigned(&mut r)?; // slice_type
sps::read_golomb_unsigned(&mut r)?; // pic_parameter_set_id
r.skip(sps.log2_max_frame_num_minus4 + 4)?; // frame_num
if !sps.frame_mbs_only_flag {
return Err(DtsExtractorError::FrameMbsNotSupported);
}
let pic_order_cnt_lsb: u32 = r.read(sps.log2_max_pic_order_cnt_lsb_minus4 + 4)?;
Ok(pic_order_cnt_lsb)
}
fn find_picture_order_count(sps: &Sps, au: &[Vec<u8>]) -> Result<u32, DtsExtractorError> {
for nalu in au {
let typ = nalu[0] & 0b00011111;
let typ = UnitType::for_id(typ).map_err(|_| DtsExtractorError::UnitType(typ))?;
if typ == UnitType::SliceLayerWithoutPartitioningNonIdr {
let poc = get_picture_order_count(sps, nalu)?;
return Ok(poc);
}
}
Err(DtsExtractorError::PocNotFound)
}
fn get_picture_order_count_diff(
sps: &Sps,
poc1: u32,
poc2: u32,
) -> Result<i32, std::num::TryFromIntError> {
let diff = i32::try_from(poc1)? - i32::try_from(poc2)?;
if diff < -((1 << (sps.log2_max_pic_order_cnt_lsb_minus4 + 3)) - 1) {
Ok(diff + (1 << (sps.log2_max_pic_order_cnt_lsb_minus4 + 4)))
} else if diff > ((1 << (sps.log2_max_pic_order_cnt_lsb_minus4 + 3)) - 1) {
Ok(diff - (1 << (sps.log2_max_pic_order_cnt_lsb_minus4 + 4)))
} else {
Ok(diff)
}
}
// Allows to extract DTS from PTS.
#[derive(Debug)]
pub struct DtsExtractor {
prev_dts_filled: bool,
prev_dts: i64,
expected_poc: u32,
reordered_frames: isize,
pause_dts: isize,
poc_increment: isize,
}
impl Default for DtsExtractor {
fn default() -> Self {
Self::new()
}
}
impl DtsExtractor {
pub fn new() -> Self {
Self {
poc_increment: 2,
prev_dts_filled: false,
prev_dts: 0,
expected_poc: 0,
reordered_frames: 0,
pause_dts: 0,
}
}
// Extracts the DTS of a access unit.
pub fn extract(
&mut self,
sps: &Sps,
au: &[Vec<u8>],
pts: i64,
) -> Result<i64, DtsExtractorError> {
let dts = self.extract_inner(sps, au, pts)?;
if dts > pts {
return Err(DtsExtractorError::DtsGreaterThanPts(dts, pts));
}
if self.prev_dts_filled && dts <= self.prev_dts {
return Err(DtsExtractorError::DtsNotIncreasing(self.prev_dts, dts));
}
self.prev_dts = dts;
self.prev_dts_filled = true;
Ok(dts)
}
fn extract_inner(
&mut self,
sps: &Sps,
au: &[Vec<u8>],
pts: i64,
) -> Result<i64, DtsExtractorError> {
let mut idr_present = false;
for nalu in au {
let typ = nalu[0] & 0b00011111;
let typ = UnitType::for_id(typ).map_err(|_| DtsExtractorError::UnitType(typ))?;
if typ == UnitType::SliceLayerWithoutPartitioningIdr {
idr_present = true
}
}
if sps.pic_order_cnt_type == 2 {
return Ok(pts);
}
if sps.pic_order_cnt_type == 1 {
return Err(DtsExtractorError::PicOrderCntType1Unsupported);
}
if idr_present {
self.expected_poc = 0;
self.reordered_frames = 0;
self.pause_dts = 0;
self.poc_increment = 2;
return Ok(pts);
}
self.expected_poc += u32::try_from(self.poc_increment)?;
self.expected_poc &= (1 << (sps.log2_max_pic_order_cnt_lsb_minus4 + 4)) - 1;
if self.pause_dts > 0 {
self.pause_dts -= 1;
return Ok(self.prev_dts + 1);
}
let poc = find_picture_order_count(sps, au)?;
if self.poc_increment == 2 && (poc % 2) != 0 {
self.poc_increment = 1;
self.expected_poc /= 2;
}
let poc_diff = isize::try_from(get_picture_order_count_diff(sps, poc, self.expected_poc)?)?
+ self.reordered_frames * self.poc_increment;
if poc_diff < 0 {
return Err(DtsExtractorError::PocInvalid);
}
if poc_diff == 0 {
return Ok(pts);
}
let reordered_frames =
(poc_diff - self.reordered_frames * self.poc_increment) / self.poc_increment;
if reordered_frames > self.reordered_frames {
self.pause_dts = reordered_frames - self.reordered_frames - 1;
self.reordered_frames = reordered_frames;
return Ok(self.prev_dts + 1);
}
Ok(self.prev_dts
+ ((pts - self.prev_dts) * i64::try_from(self.poc_increment)?
/ i64::try_from(poc_diff + self.poc_increment)?))
}
}
#[cfg(test)]
mod tests {
use super::*;
use test_case::test_case;
struct SequenceSample {
nalus: Vec<Vec<u8>>,
dts: i64,
pts: i64,
}
#[test_case(
&[
0x67, 0x64, 0x00, 0x28, 0xac, 0xd9, 0x40, 0x78,
0x02, 0x27, 0xe5, 0x84, 0x00, 0x00, 0x03, 0x00,
0x04, 0x00, 0x00, 0x03, 0x00, 0xf0, 0x3c, 0x60,
0xc6, 0x58,
],
vec![
SequenceSample{
nalus: vec![
vec![ // IDR
0x65, 0x88, 0x84, 0x00, 0x33, 0xff,
],
],
dts: 3333333,
pts: 3333333,
},
SequenceSample{
nalus: vec![vec![0x41, 0x9a, 0x21, 0x6c, 0x45, 0xff]],
dts: 3336666,
pts: 3336666,
},
SequenceSample{
nalus: vec![vec![0x41, 0x9a, 0x42, 0x3c, 0x21, 0x93]],
dts: 3340000,
pts: 3340000,
},
SequenceSample{
nalus: vec![vec![0x41, 0x9a, 0x63, 0x49, 0xe1, 0x0f]],
dts: 3343333,
pts: 3343333,
},
SequenceSample{
nalus: vec![vec![0x41, 0x9a, 0x86, 0x49, 0xe1, 0x0f]],
dts: 3343334,
pts: 3353333,
},
SequenceSample{
nalus: vec![vec![0x41, 0x9e, 0xa5, 0x42, 0x7f, 0xf9]],
dts: 3343335,
pts: 3350000,
},
SequenceSample{
nalus: vec![vec![0x01, 0x9e, 0xc4, 0x69, 0x13, 0xff]],
dts: 3346666,
pts: 3346666,
},
SequenceSample{
nalus: vec![vec![0x41, 0x9a, 0xc8, 0x4b, 0xa8, 0x42]],
dts: 3349999,
pts: 3360000,
},
]; "with timing info"
)]
#[test_case(
&[
0x27, 0x64, 0x00, 0x20, 0xac, 0x52, 0x18, 0x0f,
0x01, 0x17, 0xef, 0xff, 0x00, 0x01, 0x00, 0x01,
0x6a, 0x02, 0x02, 0x03, 0x6d, 0x85, 0x6b, 0xde,
0xf8, 0x08,
],
vec![
SequenceSample{
nalus: vec![
vec![ // IDR
0x25, 0xb8, 0x08, 0x02, 0x1f, 0xff,
],
],
dts: 85000,
pts: 85000,
},
SequenceSample{
nalus: vec![vec![0x21, 0xe1, 0x05, 0xc7, 0x38, 0xbf]],
dts: 86666,
pts: 86666,
},
SequenceSample{
nalus: vec![vec![0x21, 0xe2, 0x09, 0xa1, 0xce, 0x0b]],
dts: 88333,
pts: 88333,
},
SequenceSample{
nalus: vec![vec![0x21, 0xe3, 0x0d, 0xb1, 0xce, 0x02]],
dts: 90000,
pts: 90000,
},
SequenceSample{
nalus: vec![vec![0x21, 0xe4, 0x11, 0x90, 0x73, 0x80]],
dts: 91666,
pts: 91666,
},
SequenceSample{
nalus: vec![vec![0x21, 0xe5, 0x19, 0x0e, 0x70, 0x01]],
dts: 91667,
pts: 95000,
},
SequenceSample{
nalus: vec![vec![0x01, 0xa9, 0x85, 0x7c, 0x93, 0xff]],
dts: 93333,
pts: 93333,
},
SequenceSample{
nalus: vec![vec![0x21, 0xe6, 0x1d, 0x0e, 0x70, 0x01]],
dts: 94999,
pts: 96666,
},
SequenceSample{
nalus: vec![vec![0x21, 0xe7, 0x21, 0x0e, 0x70, 0x01]],
dts: 96666,
pts: 98333,
},
SequenceSample{
nalus: vec![vec![0x21, 0xe8, 0x25, 0x0e, 0x70, 0x01]],
dts: 98333,
pts: 100000,
},
SequenceSample{
nalus: vec![vec![0x21, 0xe9, 0x29, 0x0e, 0x70, 0x01]],
dts: 99999,
pts: 101666,
},
SequenceSample{
nalus: vec![vec![0x21, 0xea, 0x31, 0x0e, 0x70, 0x01]],
dts: 101666,
pts: 105000,
},
SequenceSample{
nalus: vec![vec![0x01, 0xaa, 0xcb, 0x7c, 0x93, 0xff]],
dts: 103333,
pts: 103333,
},
]; "no timing info"
)]
#[test_case(
&[
0x67, 0x64, 0x00, 0x2a, 0xac, 0x2c, 0x6a, 0x81,
0xe0, 0x08, 0x9f, 0x96, 0x6e, 0x02, 0x02, 0x02,
0x80, 0x00, 0x03, 0x84, 0x00, 0x00, 0xaf, 0xc8,
0x02,
],
vec![
SequenceSample{
nalus: vec![
vec![ // IDR
0x65, 0xb8, 0x00, 0x00, 0x0b, 0xc8,
],
],
dts: 61,
pts: 61,
},
SequenceSample{
nalus: vec![vec![0x61, 0xe0, 0x20, 0x00, 0x39, 0x37]],
dts: 101,
pts: 101,
},
SequenceSample{
nalus: vec![vec![0x61, 0xe0, 0x40, 0x00, 0x59, 0x37]],
dts: 141,
pts: 141,
},
SequenceSample{
nalus: vec![vec![0x61, 0xe0, 0x60, 0x00, 0x79, 0x37]],
dts: 181,
pts: 181,
},
]; "poc increment = 1"
)]
fn test_dts_extractor(sps: &[u8], sequence: Vec<SequenceSample>) {
let sps = Sps::deserialize(sps).unwrap();
let mut ex = DtsExtractor::new();
for sample in sequence {
let dts = ex.extract(&sps, &sample.nalus, sample.pts).unwrap();
assert_eq!(sample.dts, dts);
}
}
}

View File

@ -0,0 +1,62 @@
// Removes emulation prevention bytes from a NALU.
pub fn emulation_prevention_remove(nalu: &[u8]) -> Vec<u8> {
// 0x00 0x00 0x03 0x00 -> 0x00 0x00 0x00
// 0x00 0x00 0x03 0x01 -> 0x00 0x00 0x01
// 0x00 0x00 0x03 0x02 -> 0x00 0x00 0x02
// 0x00 0x00 0x03 0x03 -> 0x00 0x00 0x03
let len = nalu.len();
let mut ret_len = len;
for i in 2..len {
if nalu[i - 2] == 0 && nalu[i - 1] == 0 && nalu[i] == 3 {
ret_len -= 1;
}
}
let mut ret = Vec::with_capacity(ret_len);
let mut start = 0;
for i in 2..len {
if nalu[i - 2] == 0 && nalu[i - 1] == 0 && nalu[i] == 3 {
ret.extend_from_slice(&nalu[start..i]);
start = i + 1;
}
}
ret.extend_from_slice(&nalu[start..]);
ret
}
#[cfg(test)]
mod tests {
use super::*;
use test_case::test_case;
#[test_case(
&[
0x00, 0x00, 0x00,
0x00, 0x00, 0x01,
0x00, 0x00, 0x02,
0x00, 0x00, 0x03,
],
&[
0x00, 0x00, 0x03, 0x00,
0x00, 0x00, 0x03, 0x01,
0x00, 0x00, 0x03, 0x02,
0x00, 0x00, 0x03, 0x03,
]; "base"
)]
#[test_case(
&[
0x00, 0x00, 0x00,
0x00, 0x00,
],
&[
0x00, 0x00, 0x03,
0x00, 0x00, 0x03, 0x00,
]; "double emulation byte"
)]
#[test_case(&[0x00, 0x00], &[0x00, 0x00, 0x03]; "terminal emulation byte")]
fn test_emulation_prevention_remove(unproc: &[u8], proc: &[u8]) {
assert_eq!(unproc, emulation_prevention_remove(proc))
}
}

1200
src/codec/h264/sps.rs Normal file

File diff suppressed because it is too large Load Diff

View File

@ -96,6 +96,8 @@ pub struct Timestamp {
/// A timestamp which must be compared to `start`. /// A timestamp which must be compared to `start`.
timestamp: i64, timestamp: i64,
dts: Option<i64>,
/// The codec-specified clock rate, in Hz. Must be non-zero. /// The codec-specified clock rate, in Hz. Must be non-zero.
clock_rate: NonZeroU32, clock_rate: NonZeroU32,
@ -109,6 +111,7 @@ impl Timestamp {
pub fn new(timestamp: i64, clock_rate: NonZeroU32, start: u32) -> Option<Self> { pub fn new(timestamp: i64, clock_rate: NonZeroU32, start: u32) -> Option<Self> {
timestamp.checked_sub(i64::from(start)).map(|_| Timestamp { timestamp.checked_sub(i64::from(start)).map(|_| Timestamp {
timestamp, timestamp,
dts: None,
clock_rate, clock_rate,
start, start,
}) })
@ -120,6 +123,11 @@ impl Timestamp {
self.timestamp self.timestamp
} }
#[inline]
pub fn dts(&self) -> Option<i64> {
self.dts
}
/// Returns timestamp of the start of the stream. /// Returns timestamp of the start of the stream.
#[inline] #[inline]
pub fn start(&self) -> u32 { pub fn start(&self) -> u32 {
@ -153,6 +161,7 @@ impl Timestamp {
.checked_add(i64::from(delta)) .checked_add(i64::from(delta))
.map(|timestamp| Timestamp { .map(|timestamp| Timestamp {
timestamp, timestamp,
dts: self.dts,
clock_rate: self.clock_rate, clock_rate: self.clock_rate,
start: self.start, start: self.start,
}) })
@ -163,10 +172,11 @@ impl Display for Timestamp {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!( write!(
f, f,
"{} (mod-2^32: {}), npt {:.03}", "{} (mod-2^32: {}), npt {:.03}, dts {:?}",
self.timestamp, self.timestamp,
self.timestamp as u32, self.timestamp as u32,
self.elapsed_secs() self.elapsed_secs(),
self.dts,
) )
} }
} }