h265: more flexible logic for selecting colorspace (pixel-format)

2 files changed, 42 insertions, 5 deletions

diff --git a/modules/h265/decode.c b/modules/h265/decode.c
index f05aafc..ec823b4 100644
--- a/modules/h265/decode.c
+++ b/modules/h265/decode.c
@@ -7,6 +7,7 @@
 #include <re.h>
 #include <rem.h>
 #include <baresip.h>
+#include <libavutil/pixdesc.h>
 #include <libavcodec/avcodec.h>
 #include "h265.h"
 
@@ -149,6 +150,7 @@ int h265_decode(struct viddec_state *vds, struct vidframe *frame,
 	int err, ret, got_picture, i;
 	struct h265_nal hdr;
 	AVPacket avpkt;
+	enum vidfmt fmt;
 
 	if (!vds || !frame || !mb)
 		return EINVAL;
@@ -267,8 +269,22 @@ int h265_decode(struct viddec_state *vds, struct vidframe *frame,
 		goto out;
 	}
 
-	if (vds->pict->format != PIX_FMT_YUV420P) {
-		warning("h265: bad pixel format (%i)\n", vds->pict->format);
+	switch (vds->pict->format) {
+
+	case PIX_FMT_YUV420P:
+		fmt = VID_FMT_YUV420P;
+		break;
+
+#if 0
+	case PIX_FMT_YUV444P:
+		fmt = VID_FMT_YUV444;
+		break;
+#endif
+
+	default:
+		warning("h265: decode: bad pixel format (%i) (%s)\n",
+			vds->pict->format,
+			av_get_pix_fmt_name(vds->pict->format));
 		goto out;
 	}
 
@@ -279,7 +295,7 @@ int h265_decode(struct viddec_state *vds, struct vidframe *frame,
 
 	frame->size.w = vds->ctx->width;
 	frame->size.h = vds->ctx->height;
-	frame->fmt    = VID_FMT_YUV420P;
+	frame->fmt    = fmt;
 
  out:
 	mbuf_rewind(vds->mb);
diff --git a/modules/h265/encode.c b/modules/h265/encode.c
index 18685d6..7a42ea7 100644
--- a/modules/h265/encode.c
+++ b/modules/h265/encode.c
@@ -181,13 +181,34 @@ int h265_encode(struct videnc_state *st, bool update,
 	x265_picture *pic_in = NULL, pic_out;
 	x265_nal *nalv;
 	uint32_t i, nalc = 0;
+	int colorspace;
 	int n, err = 0;
 
-	if (!st || !frame || !pkth || frame->fmt != VID_FMT_YUV420P)
+	if (!st || !frame || !pkth)
 		return EINVAL;
 
+	switch (frame->fmt) {
+
+	case VID_FMT_YUV420P:
+		colorspace = X265_CSP_I420;
+		break;
+
+#if 0
+	case VID_FMT_YUV444:
+		colorspace = X265_CSP_I444;
+		break;
+#endif
+
+	default:
+		warning("h265: encode: pixel format not supported (%s)\n",
+			vidfmt_name(frame->fmt));
+		return EINVAL;
+	}
+
 	if (!st->x265 || !vidsz_cmp(&st->size, &frame->size)) {
 
+		st->param->internalCsp = colorspace;
+
 		err = open_encoder(st, &frame->size);
 		if (err)
 			return err;
@@ -209,7 +230,7 @@ int h265_encode(struct videnc_state *st, bool update,
 
 	pic_in->sliceType  = update ? X265_TYPE_IDR : X265_TYPE_AUTO;
 	pic_in->pts        = ++st->pts;      /* XXX: add PTS to API */
-	pic_in->colorSpace = X265_CSP_I420;
+	pic_in->colorSpace = colorspace;
 
 	for (i=0; i<3; i++) {
 		pic_in->planes[i] = frame->data[i];