vidcodec: add 'intra' parameter to decoder api

- the intra flag must be set to true, if the decoded video frame
  is a full intra frame (Key frame)

- the video decoder can use this flag to cancel sending
  of FIR packets

14 files changed, 107 insertions, 30 deletions

diff --git a/include/baresip.h b/include/baresip.h
index 0e27a33..43b38c1 100644
--- a/include/baresip.h
+++ b/include/baresip.h
@@ -843,7 +843,8 @@ typedef int (videnc_encode_h)(struct videnc_state *ves, bool update,
 typedef int (viddec_update_h)(struct viddec_state **vdsp,
 			      const struct vidcodec *vc, const char *fmtp);
 typedef int (viddec_decode_h)(struct viddec_state *vds, struct vidframe *frame,
-			      bool marker, uint16_t seq, struct mbuf *mb);
+                              bool *intra, bool marker, uint16_t seq,
+                              struct mbuf *mb);
 
 struct vidcodec {
 	struct le le;
diff --git a/modules/avcodec/avcodec.h b/modules/avcodec/avcodec.h
index 5ffe1d8..9a0f132 100644
--- a/modules/avcodec/avcodec.h
+++ b/modules/avcodec/avcodec.h
@@ -44,11 +44,11 @@ struct viddec_state;
 int decode_update(struct viddec_state **vdsp, const struct vidcodec *vc,
 		  const char *fmtp);
 int decode_h263(struct viddec_state *st, struct vidframe *frame,
-		bool eof, uint16_t seq, struct mbuf *src);
+		bool *intra, bool eof, uint16_t seq, struct mbuf *src);
 int decode_h264(struct viddec_state *st, struct vidframe *frame,
-		bool eof, uint16_t seq, struct mbuf *src);
+		bool *intra, bool eof, uint16_t seq, struct mbuf *src);
 int decode_mpeg4(struct viddec_state *st, struct vidframe *frame,
-		 bool eof, uint16_t seq, struct mbuf *src);
+		 bool *intra, bool eof, uint16_t seq, struct mbuf *src);
 
 
 int decode_sdpparam_h264(struct videnc_state *st, const struct pl *name,
diff --git a/modules/avcodec/decode.c b/modules/avcodec/decode.c
index cb0446e..9b24586 100644
--- a/modules/avcodec/decode.c
+++ b/modules/avcodec/decode.c
@@ -239,12 +239,14 @@ static int ffdecode(struct viddec_state *st, struct vidframe *frame,
 }
 
 
-static int h264_decode(struct viddec_state *st, struct mbuf *src)
+static int h264_decode(struct viddec_state *st, bool *intra, struct mbuf *src)
 {
 	struct h264_hdr h264_hdr;
 	const uint8_t nal_seq[3] = {0, 0, 1};
 	int err;
 
+	*intra = false;
+
 	err = h264_hdr_decode(&h264_hdr, src);
 	if (err)
 		return err;
@@ -267,6 +269,9 @@ static int h264_decode(struct viddec_state *st, struct mbuf *src)
 			}
 		}
 
+		if (h264_is_keyframe(h264_hdr.type))
+			*intra = true;
+
 		/* prepend H.264 NAL start sequence */
 		mbuf_write_mem(st->mb, nal_seq, 3);
 
@@ -282,6 +287,9 @@ static int h264_decode(struct viddec_state *st, struct mbuf *src)
 		h264_hdr.type = fu.type;
 
 		if (fu.s) {
+			if (h264_is_keyframe(fu.type))
+				*intra = true;
+
 			/* prepend H.264 NAL start sequence */
 			mbuf_write_mem(st->mb, nal_seq, 3);
 
@@ -299,7 +307,7 @@ static int h264_decode(struct viddec_state *st, struct mbuf *src)
 
 
 int decode_h264(struct viddec_state *st, struct vidframe *frame,
-		bool eof, uint16_t seq, struct mbuf *src)
+		bool *intra, bool eof, uint16_t seq, struct mbuf *src)
 {
 	int err;
 
@@ -308,7 +316,7 @@ int decode_h264(struct viddec_state *st, struct vidframe *frame,
 	if (!src)
 		return 0;
 
-	err = h264_decode(st, src);
+	err = h264_decode(st, intra, src);
 	if (err)
 		return err;
 
@@ -317,13 +325,15 @@ int decode_h264(struct viddec_state *st, struct vidframe *frame,
 
 
 int decode_mpeg4(struct viddec_state *st, struct vidframe *frame,
-		 bool eof, uint16_t seq, struct mbuf *src)
+		 bool *intra, bool eof, uint16_t seq, struct mbuf *src)
 {
 	if (!src)
 		return 0;
 
 	(void)seq;
 
+	*intra = false;  /* XXX */
+
 	/* let the decoder handle this */
 	st->got_keyframe = true;
 
@@ -332,14 +342,16 @@ int decode_mpeg4(struct viddec_state *st, struct vidframe *frame,
 
 
 int decode_h263(struct viddec_state *st, struct vidframe *frame,
-		bool marker, uint16_t seq, struct mbuf *src)
+		bool *intra, bool marker, uint16_t seq, struct mbuf *src)
 {
 	struct h263_hdr hdr;
 	int err;
 
-	if (!st || !frame)
+	if (!st || !frame || !intra)
 		return EINVAL;
 
+	*intra = false;
+
 	if (!src)
 		return 0;
 
@@ -359,8 +371,11 @@ int decode_h263(struct viddec_state *st, struct vidframe *frame,
 	      mbuf_get_left(src), st->mb->end);
 #endif
 
-	if (!hdr.i)
+	if (!hdr.i) {
 		st->got_keyframe = true;
+		if (st->mb->pos == 0)
+			*intra = true;
+	}
 
 #if 0
 	if (st->mb->pos == 0) {
diff --git a/modules/daala/daala.h b/modules/daala/daala.h
index 3649253..323ba88 100644
--- a/modules/daala/daala.h
+++ b/modules/daala/daala.h
@@ -17,4 +17,4 @@ int daala_encode(struct videnc_state *ves, bool update,
 int daala_decode_update(struct viddec_state **vdsp, const struct vidcodec *vc,
 			const char *fmtp);
 int daala_decode(struct viddec_state *vds, struct vidframe *frame,
-		 bool marker, uint16_t seq, struct mbuf *mb);
+		 bool *intra, bool marker, uint16_t seq, struct mbuf *mb);
diff --git a/modules/daala/decode.c b/modules/daala/decode.c
index cacd8b5..062fb60 100644
--- a/modules/daala/decode.c
+++ b/modules/daala/decode.c
@@ -91,7 +91,7 @@ int daala_decode_update(struct viddec_state **vdsp, const struct vidcodec *vc,
 
 
 int daala_decode(struct viddec_state *vds, struct vidframe *frame,
-		 bool marker, uint16_t seq, struct mbuf *mb)
+		 bool *intra, bool marker, uint16_t seq, struct mbuf *mb)
 {
 	daala_packet dp;
 	bool ishdr;
@@ -101,6 +101,8 @@ int daala_decode(struct viddec_state *vds, struct vidframe *frame,
 	if (!vds || !frame || !mb)
 		return EINVAL;
 
+	*intra = false;
+
 	++vds->stats.n_packet;
 	++vds->stats.valid;
 
@@ -114,9 +116,10 @@ int daala_decode(struct viddec_state *vds, struct vidframe *frame,
 
 	if (ishdr)
 		++vds->stats.n_header;
-	else if (daala_packet_iskeyframe(&dp) > 0)
+	else if (daala_packet_iskeyframe(&dp) > 0) {
 		++vds->stats.n_keyframe;
-
+		*intra = true;
+	}
 
 	if (daala_packet_isheader(&dp)) {
 
diff --git a/modules/h265/decode.c b/modules/h265/decode.c
index d14a99d..5f444d6 100644
--- a/modules/h265/decode.c
+++ b/modules/h265/decode.c
@@ -150,7 +150,7 @@ static inline void fragment_rewind(struct viddec_state *vds)
 
 
 int h265_decode(struct viddec_state *vds, struct vidframe *frame,
-		bool marker, uint16_t seq, struct mbuf *mb)
+		bool *intra, bool marker, uint16_t seq, struct mbuf *mb)
 {
 	static const uint8_t nal_seq[3] = {0, 0, 1};
 	int err, ret, got_picture, i;
@@ -158,23 +158,24 @@ int h265_decode(struct viddec_state *vds, struct vidframe *frame,
 	AVPacket avpkt;
 	enum vidfmt fmt;
 
-	if (!vds || !frame || !mb)
+	if (!vds || !frame || !intra || !mb)
 		return EINVAL;
 
+	*intra = false;
+
 	err = h265_nal_decode(&hdr, mbuf_buf(mb));
 	if (err)
 		return err;
 
 	mbuf_advance(mb, H265_HDR_SIZE);
 
-#if 1
+#if 0
 	debug("h265: decode: %s type=%2d  %s\n",
 		  h265_is_keyframe(hdr.nal_unit_type) ? "<KEY>" : "     ",
 		  hdr.nal_unit_type,
 		  h265_nalunit_name(hdr.nal_unit_type));
 #endif
 
-
 	if (vds->frag && hdr.nal_unit_type != H265_NAL_FU) {
 		debug("h265: lost fragments; discarding previous NAL\n");
 		fragment_rewind(vds);
@@ -184,6 +185,9 @@ int h265_decode(struct viddec_state *vds, struct vidframe *frame,
 	/* handle NAL types */
 	if (0 <= hdr.nal_unit_type && hdr.nal_unit_type <= 40) {
 
+		if (h265_is_keyframe(hdr.nal_unit_type))
+			*intra = true;
+
 		mb->pos -= H265_HDR_SIZE;
 
 		err  = mbuf_write_mem(vds->mb, nal_seq, 3);
@@ -200,6 +204,9 @@ int h265_decode(struct viddec_state *vds, struct vidframe *frame,
 			return err;
 
 		if (fu.s) {
+			if (h265_is_keyframe(fu.type))
+				*intra = true;
+
 			if (vds->frag) {
 				debug("h265: lost fragments; ignoring NAL\n");
 				fragment_rewind(vds);
diff --git a/modules/h265/encode.c b/modules/h265/encode.c
index 5ff16f9..83264b2 100644
--- a/modules/h265/encode.c
+++ b/modules/h265/encode.c
@@ -253,7 +253,7 @@ int h265_encode(struct videnc_state *st, bool update,
 		size_t len = nal->sizeBytes;
 		bool marker;
 
-#if 1
+#if 0
 		debug("h265: encode: %s type=%2d  %s\n",
 			  h265_is_keyframe(nal->type) ? "<KEY>" : "     ",
 			  nal->type, h265_nalunit_name(nal->type));
diff --git a/modules/h265/h265.h b/modules/h265/h265.h
index 2705cf6..4e44c33 100644
--- a/modules/h265/h265.h
+++ b/modules/h265/h265.h
@@ -66,4 +66,4 @@ int h265_encode(struct videnc_state *ves, bool update,
 int h265_decode_update(struct viddec_state **vdsp, const struct vidcodec *vc,
 		       const char *fmtp);
 int h265_decode(struct viddec_state *vds, struct vidframe *frame,
-		bool marker, uint16_t seq, struct mbuf *mb);
+		bool *intra, bool marker, uint16_t seq, struct mbuf *mb);
diff --git a/modules/vidloop/vidloop.c b/modules/vidloop/vidloop.c
index c4a596a..3a5d38c 100644
--- a/modules/vidloop/vidloop.c
+++ b/modules/vidloop/vidloop.c
@@ -45,6 +45,7 @@ struct vstat {
 	size_t bytes;
 	uint32_t bitrate;
 	double efps;
+	size_t n_intra;
 };
 
 
@@ -127,6 +128,7 @@ static int packet_handler(bool marker, const uint8_t *hdr, size_t hdr_len,
 	struct video_loop *vl = arg;
 	struct vidframe frame;
 	struct mbuf *mb;
+	bool intra;
 	int err = 0;
 
 	mb = mbuf_alloc(hdr_len + pld_len);
@@ -144,11 +146,15 @@ static int packet_handler(bool marker, const uint8_t *hdr, size_t hdr_len,
 	/* decode */
 	frame.data[0] = NULL;
 	if (vl->vc_dec && vl->dec) {
-		err = vl->vc_dec->dech(vl->dec, &frame, marker, vl->seq++, mb);
+		err = vl->vc_dec->dech(vl->dec, &frame, &intra,
+				       marker, vl->seq++, mb);
 		if (err) {
 			warning("vidloop: codec decode: %m\n", err);
 			goto out;
 		}
+
+		if (intra)
+			++vl->stat.n_intra;
 	}
 
 	if (vidframe_isvalid(&frame)) {
@@ -287,9 +293,11 @@ static void print_status(struct video_loop *vl)
 {
 	(void)re_fprintf(stderr,
 			 "\rstatus:"
-			 " [%s] [%s] EFPS=%.1f      %u kbit/s       \r",
+			 " [%s] [%s]  intra=%zu "
+			 " EFPS=%.1f      %u kbit/s       \r",
 			 vl->vc_enc ? vl->vc_enc->name : "",
 			 vl->vc_dec ? vl->vc_dec->name : "",
+			 vl->stat.n_intra,
 			 vl->stat.efps, vl->stat.bitrate);
 }
 
diff --git a/modules/vp8/decode.c b/modules/vp8/decode.c
index 1722eb6..4399c0b 100644
--- a/modules/vp8/decode.c
+++ b/modules/vp8/decode.c
@@ -173,6 +173,18 @@ static inline int hdr_decode(struct hdr *hdr, struct mbuf *mb)
 }
 
 
+static inline bool is_keyframe(struct mbuf *mb)
+{
+	if (mbuf_get_left(mb) < 1)
+		return false;
+
+	if (mb->buf[mb->pos] & 0x01)
+		return false;
+
+	return true;
+}
+
+
 static inline int16_t seq_diff(uint16_t x, uint16_t y)
 {
 	return (int16_t)(y - x);
@@ -180,7 +192,7 @@ static inline int16_t seq_diff(uint16_t x, uint16_t y)
 
 
 int vp8_decode(struct viddec_state *vds, struct vidframe *frame,
-	       bool marker, uint16_t seq, struct mbuf *mb)
+	       bool *intra, bool marker, uint16_t seq, struct mbuf *mb)
 {
 	vpx_codec_iter_t iter = NULL;
 	vpx_codec_err_t res;
@@ -188,9 +200,11 @@ int vp8_decode(struct viddec_state *vds, struct vidframe *frame,
 	struct hdr hdr;
 	int err, i;
 
-	if (!vds || !frame || !mb)
+	if (!vds || !frame || !intra || !mb)
 		return EINVAL;
 
+	*intra = false;
+
 	err = hdr_decode(&hdr, mb);
 	if (err)
 		return err;
@@ -206,6 +220,9 @@ int vp8_decode(struct viddec_state *vds, struct vidframe *frame,
 
 	if (hdr.start && hdr.partid == 0) {
 
+		if (is_keyframe(mb))
+			*intra = true;
+
 		mbuf_rewind(vds->mb);
 		vds->started = true;
 	}
diff --git a/modules/vp8/vp8.h b/modules/vp8/vp8.h
index 934e3ee..c0e262a 100644
--- a/modules/vp8/vp8.h
+++ b/modules/vp8/vp8.h
@@ -21,7 +21,7 @@ int vp8_encode(struct videnc_state *ves, bool update,
 int vp8_decode_update(struct viddec_state **vdsp, const struct vidcodec *vc,
 		      const char *fmtp);
 int vp8_decode(struct viddec_state *vds, struct vidframe *frame,
-	       bool marker, uint16_t seq, struct mbuf *mb);
+	       bool *intra, bool marker, uint16_t seq, struct mbuf *mb);
 
 
 /* SDP */
diff --git a/modules/vp9/decode.c b/modules/vp9/decode.c
index 84f0d49..f69b240 100644
--- a/modules/vp9/decode.c
+++ b/modules/vp9/decode.c
@@ -164,6 +164,24 @@ static inline int hdr_decode(struct hdr *hdr, struct mbuf *mb)
 }
 
 
+static inline bool is_keyframe(const struct mbuf *mb)
+{
+	vpx_codec_stream_info_t si;
+	vpx_codec_err_t ret;
+
+	memset(&si, 0, sizeof(si));
+	si.sz = sizeof(si);
+
+	ret = vpx_codec_peek_stream_info(&vpx_codec_vp9_dx_algo,
+					 mbuf_buf(mb),
+					 (unsigned int)mbuf_get_left(mb), &si);
+	if (ret != VPX_CODEC_OK)
+		return false;
+
+	return si.is_kf;
+}
+
+
 static inline int16_t seq_diff(uint16_t x, uint16_t y)
 {
 	return (int16_t)(y - x);
@@ -171,7 +189,7 @@ static inline int16_t seq_diff(uint16_t x, uint16_t y)
 
 
 int vp9_decode(struct viddec_state *vds, struct vidframe *frame,
-	       bool marker, uint16_t seq, struct mbuf *mb)
+	       bool *intra, bool marker, uint16_t seq, struct mbuf *mb)
 {
 	vpx_codec_iter_t iter = NULL;
 	vpx_codec_err_t res;
@@ -179,9 +197,11 @@ int vp9_decode(struct viddec_state *vds, struct vidframe *frame,
 	struct hdr hdr;
 	int err, i;
 
-	if (!vds || !frame || !mb)
+	if (!vds || !frame || !intra || !mb)
 		return EINVAL;
 
+	*intra = false;
+
 	vds->n_bytes += mbuf_get_left(mb);
 
 	err = hdr_decode(&hdr, mb);
@@ -195,6 +215,9 @@ int vp9_decode(struct viddec_state *vds, struct vidframe *frame,
 
 	if (hdr.b) {
 
+		if (is_keyframe(mb))
+			*intra = true;
+
 		mbuf_rewind(vds->mb);
 		vds->started = true;
 	}
diff --git a/modules/vp9/vp9.h b/modules/vp9/vp9.h
index c60def2..53899a1 100644
--- a/modules/vp9/vp9.h
+++ b/modules/vp9/vp9.h
@@ -21,7 +21,7 @@ int vp9_encode(struct videnc_state *ves, bool update,
 int vp9_decode_update(struct viddec_state **vdsp, const struct vidcodec *vc,
 		      const char *fmtp);
 int vp9_decode(struct viddec_state *vds, struct vidframe *frame,
-	       bool marker, uint16_t seq, struct mbuf *mb);
+	       bool *intra, bool marker, uint16_t seq, struct mbuf *mb);
 
 
 /* SDP */
diff --git a/src/video.c b/src/video.c
index d2587aa..49cc9e4 100644
--- a/src/video.c
+++ b/src/video.c
@@ -515,6 +515,7 @@ static int video_stream_decode(struct vrx *vrx, const struct rtp_header *hdr,
 	struct vidframe *frame_filt = NULL;
 	struct vidframe frame_store, *frame = &frame_store;
 	struct le *le;
+	bool intra;
 	int err = 0;
 
 	if (!hdr || !mbuf_get_left(mb))
@@ -529,7 +530,7 @@ static int video_stream_decode(struct vrx *vrx, const struct rtp_header *hdr,
 	}
 
 	frame->data[0] = NULL;
-	err = vrx->vc->dech(vrx->dec, frame, hdr->m, hdr->seq, mb);
+	err = vrx->vc->dech(vrx->dec, frame, &intra, hdr->m, hdr->seq, mb);
 	if (err) {
 
 		if (err != EPROTO) {
@@ -547,6 +548,8 @@ static int video_stream_decode(struct vrx *vrx, const struct rtp_header *hdr,
 		goto out;
 	}
 
+	/* XXX stop pending FIR if intra=true */
+
 	/* Got a full picture-frame? */
 	if (!vidframe_isvalid(frame))
 		goto out;