From b29b5b9f529bbb10cd9880adebf0fb287dcf233b Mon Sep 17 00:00:00 2001 From: Andriy Gelman Date: Tue, 28 Apr 2020 22:54:21 -0400 Subject: [PATCH 01/11] avcodec/v4l2_m2m: Adapt to call close() on init fail This fixes several mem leaks when init of encoder/decoder failed. Fixes ticket #8285 Signed-off-by: Andriy Gelman --- libavcodec/v4l2_m2m.c | 8 ++++++++ libavcodec/v4l2_m2m_dec.c | 10 ++-------- libavcodec/v4l2_m2m_enc.c | 1 + 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/libavcodec/v4l2_m2m.c b/libavcodec/v4l2_m2m.c index e48b3a8ccf..bfea70ff0c 100644 --- a/libavcodec/v4l2_m2m.c +++ b/libavcodec/v4l2_m2m.c @@ -338,6 +338,13 @@ int ff_v4l2_m2m_codec_end(V4L2m2mPriv *priv) V4L2m2mContext *s = priv->context; int ret; + if (!s) + return 0; + + if (av_codec_is_decoder(s->avctx->codec)) + av_packet_unref(&s->buf_pkt); + + if (s->fd >= 0) { ret = ff_v4l2_context_set_status(&s->output, VIDIOC_STREAMOFF); if (ret) av_log(s->avctx, AV_LOG_ERROR, "VIDIOC_STREAMOFF %s\n", s->output.name); @@ -345,6 +352,7 @@ int ff_v4l2_m2m_codec_end(V4L2m2mPriv *priv) ret = ff_v4l2_context_set_status(&s->capture, VIDIOC_STREAMOFF); if (ret) av_log(s->avctx, AV_LOG_ERROR, "VIDIOC_STREAMOFF %s\n", s->capture.name); + } ff_v4l2_context_release(&s->output); diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c index 3e17e0fcac..a2ea0ff73a 100644 --- a/libavcodec/v4l2_m2m_dec.c +++ b/libavcodec/v4l2_m2m_dec.c @@ -212,9 +212,6 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx) ret = ff_v4l2_m2m_codec_init(priv); if (ret) { av_log(avctx, AV_LOG_ERROR, "can't configure decoder\n"); - s->self_ref = NULL; - av_buffer_unref(&priv->context_ref); - return ret; } @@ -223,10 +220,7 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx) static av_cold int v4l2_decode_close(AVCodecContext *avctx) { - V4L2m2mPriv *priv = avctx->priv_data; - V4L2m2mContext *s = priv->context; - av_packet_unref(&s->buf_pkt); - return ff_v4l2_m2m_codec_end(priv); + return ff_v4l2_m2m_codec_end(avctx->priv_data); } #define OFFSET(x) offsetof(V4L2m2mPriv, x) @@ -261,7 +255,7 @@ static const AVOption options[] = { .close = v4l2_decode_close, \ .bsfs = bsf_name, \ .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING, \ - .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS, \ + .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS | FF_CODEC_CAP_INIT_CLEANUP, \ .wrapper_name = "v4l2m2m", \ } diff --git a/libavcodec/v4l2_m2m_enc.c b/libavcodec/v4l2_m2m_enc.c index 32321f392f..9f1b2c2ffc 100644 --- a/libavcodec/v4l2_m2m_enc.c +++ b/libavcodec/v4l2_m2m_enc.c @@ -416,6 +416,7 @@ static const AVCodecDefault v4l2_m2m_defaults[] = { .close = v4l2_encode_close, \ .defaults = v4l2_m2m_defaults, \ .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY, \ + .caps_internal = FF_CODEC_CAP_INIT_CLEANUP, \ .wrapper_name = "v4l2m2m", \ } From 7aaac68934c0e03a78c9f477ec64e522729a64b7 Mon Sep 17 00:00:00 2001 From: Andriy Gelman Date: Tue, 5 May 2020 01:54:54 -0400 Subject: [PATCH 02/11] avcodec/v4l2_m2m_dec: Use av_packet_move_ref() Signed-off-by: Andriy Gelman --- libavcodec/v4l2_m2m_dec.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c index a2ea0ff73a..45e9a8e9fe 100644 --- a/libavcodec/v4l2_m2m_dec.c +++ b/libavcodec/v4l2_m2m_dec.c @@ -142,8 +142,7 @@ static int v4l2_receive_frame(AVCodecContext *avctx, AVFrame *frame) int ret; if (s->buf_pkt.size) { - avpkt = s->buf_pkt; - memset(&s->buf_pkt, 0, sizeof(AVPacket)); + av_packet_move_ref(&avpkt, &s->buf_pkt); } else { ret = ff_decode_get_packet(avctx, &avpkt); if (ret < 0 && ret != AVERROR_EOF) From c6b85ed30f06ea99513b13cc768a922ebe4d68c2 Mon Sep 17 00:00:00 2001 From: Lukas Rusak Date: Tue, 24 Apr 2018 23:00:23 -0700 Subject: [PATCH 03/11] libavcodec: v4l2m2m: output AVDRMFrameDescriptor This allows for a zero-copy output by exporting the v4l2 buffer then wrapping that buffer in the AVDRMFrameDescriptor like it is done in rkmpp. This has been in use for quite some time with great success on many platforms including: - Amlogic S905 - Raspberry Pi - i.MX6 - Dragonboard 410c This was developed in conjunction with Kodi to allow handling the zero-copy buffer rendering. A simply utility for testing is also available here: https://github.com/BayLibre/ffmpeg-drm todo: - allow selecting pixel format output from decoder - allow configuring amount of output and capture buffers V2: - allow selecting AV_PIX_FMT_DRM_PRIME V3: - use get_format to select AV_PIX_FMT_DRM_PRIME - use hw_configs - add handling of AV_PIX_FMT_YUV420P format (for raspberry pi) - add handling of AV_PIX_FMT_YUYV422 format (for i.MX6 coda decoder) V4: - rebased on 4.2.2 V5: - rebased on 4.3 --- libavcodec/v4l2_buffers.c | 155 ++++++++++++++++++++++++++++++++++++-- libavcodec/v4l2_buffers.h | 4 + libavcodec/v4l2_context.c | 40 +++++++++- libavcodec/v4l2_m2m.h | 3 + libavcodec/v4l2_m2m_dec.c | 23 ++++++ 5 files changed, 213 insertions(+), 12 deletions(-) diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c index 02f23d954b..4bb2bf6f87 100644 --- a/libavcodec/v4l2_buffers.c +++ b/libavcodec/v4l2_buffers.c @@ -21,6 +21,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include #include #include #include @@ -30,6 +31,7 @@ #include "libavcodec/avcodec.h" #include "libavcodec/internal.h" #include "libavutil/pixdesc.h" +#include "libavutil/hwcontext.h" #include "v4l2_context.h" #include "v4l2_buffers.h" #include "v4l2_m2m.h" @@ -210,7 +212,79 @@ static enum AVColorTransferCharacteristic v4l2_get_color_trc(V4L2Buffer *buf) return AVCOL_TRC_UNSPECIFIED; } -static void v4l2_free_buffer(void *opaque, uint8_t *unused) +static uint8_t * v4l2_get_drm_frame(V4L2Buffer *avbuf) +{ + AVDRMFrameDescriptor *drm_desc = &avbuf->drm_frame; + AVDRMLayerDescriptor *layer; + + /* fill the DRM frame descriptor */ + drm_desc->nb_objects = avbuf->num_planes; + drm_desc->nb_layers = 1; + + layer = &drm_desc->layers[0]; + layer->nb_planes = avbuf->num_planes; + + for (int i = 0; i < avbuf->num_planes; i++) { + layer->planes[i].object_index = i; + layer->planes[i].offset = 0; + layer->planes[i].pitch = avbuf->plane_info[i].bytesperline; + } + + switch (avbuf->context->av_pix_fmt) { + case AV_PIX_FMT_YUYV422: + + layer->format = DRM_FORMAT_YUYV; + layer->nb_planes = 1; + + break; + + case AV_PIX_FMT_NV12: + case AV_PIX_FMT_NV21: + + layer->format = avbuf->context->av_pix_fmt == AV_PIX_FMT_NV12 ? + DRM_FORMAT_NV12 : DRM_FORMAT_NV21; + + if (avbuf->num_planes > 1) + break; + + layer->nb_planes = 2; + + layer->planes[1].object_index = 0; + layer->planes[1].offset = avbuf->plane_info[0].bytesperline * + avbuf->context->format.fmt.pix.height; + layer->planes[1].pitch = avbuf->plane_info[0].bytesperline; + break; + + case AV_PIX_FMT_YUV420P: + + layer->format = DRM_FORMAT_YUV420; + + if (avbuf->num_planes > 1) + break; + + layer->nb_planes = 3; + + layer->planes[1].object_index = 0; + layer->planes[1].offset = avbuf->plane_info[0].bytesperline * + avbuf->context->format.fmt.pix.height; + layer->planes[1].pitch = avbuf->plane_info[0].bytesperline >> 1; + + layer->planes[2].object_index = 0; + layer->planes[2].offset = layer->planes[1].offset + + ((avbuf->plane_info[0].bytesperline * + avbuf->context->format.fmt.pix.height) >> 2); + layer->planes[2].pitch = avbuf->plane_info[0].bytesperline >> 1; + break; + + default: + drm_desc->nb_layers = 0; + break; + } + + return (uint8_t *) drm_desc; +} + +static void v4l2_free_buffer(void *opaque, uint8_t *data) { V4L2Buffer* avbuf = opaque; V4L2m2mContext *s = buf_to_m2mctx(avbuf); @@ -234,6 +308,36 @@ static void v4l2_free_buffer(void *opaque, uint8_t *unused) } } +static int v4l2_buffer_export_drm(V4L2Buffer* avbuf) +{ + struct v4l2_exportbuffer expbuf; + int i, ret; + + for (i = 0; i < avbuf->num_planes; i++) { + memset(&expbuf, 0, sizeof(expbuf)); + + expbuf.index = avbuf->buf.index; + expbuf.type = avbuf->buf.type; + expbuf.plane = i; + + ret = ioctl(buf_to_m2mctx(avbuf)->fd, VIDIOC_EXPBUF, &expbuf); + if (ret < 0) + return AVERROR(errno); + + if (V4L2_TYPE_IS_MULTIPLANAR(avbuf->buf.type)) { + /* drm frame */ + avbuf->drm_frame.objects[i].size = avbuf->buf.m.planes[i].length; + avbuf->drm_frame.objects[i].fd = expbuf.fd; + } else { + /* drm frame */ + avbuf->drm_frame.objects[0].size = avbuf->buf.length; + avbuf->drm_frame.objects[0].fd = expbuf.fd; + } + } + + return 0; +} + static int v4l2_buf_increase_ref(V4L2Buffer *in) { V4L2m2mContext *s = buf_to_m2mctx(in); @@ -254,6 +358,24 @@ static int v4l2_buf_increase_ref(V4L2Buffer *in) return 0; } +static int v4l2_buf_to_bufref_drm(V4L2Buffer *in, AVBufferRef **buf) +{ + int ret; + + *buf = av_buffer_create((uint8_t *) &in->drm_frame, + sizeof(in->drm_frame), + v4l2_free_buffer, + in, AV_BUFFER_FLAG_READONLY); + if (!*buf) + return AVERROR(ENOMEM); + + ret = v4l2_buf_increase_ref(in); + if (ret) + av_buffer_unref(buf); + + return ret; +} + static int v4l2_buf_to_bufref(V4L2Buffer *in, int plane, AVBufferRef **buf) { int ret; @@ -303,13 +425,24 @@ static int v4l2_buffer_buf_to_swframe(AVFrame *frame, V4L2Buffer *avbuf) frame->format = avbuf->context->av_pix_fmt; - for (i = 0; i < avbuf->num_planes; i++) { - ret = v4l2_buf_to_bufref(avbuf, i, &frame->buf[i]); + if (buf_to_m2mctx(avbuf)->output_drm) { + /* 1. get references to the actual data */ + ret = v4l2_buf_to_bufref_drm(avbuf, &frame->buf[0]); if (ret) return ret; - frame->linesize[i] = avbuf->plane_info[i].bytesperline; - frame->data[i] = frame->buf[i]->data; + frame->data[0] = (uint8_t *) v4l2_get_drm_frame(avbuf); + frame->format = AV_PIX_FMT_DRM_PRIME; + } else { + /* 1. get references to the actual data */ + for (i = 0; i < avbuf->num_planes; i++) { + ret = v4l2_buf_to_bufref(avbuf, i, &frame->buf[i]); + if (ret) + return ret; + + frame->linesize[i] = avbuf->plane_info[i].bytesperline; + frame->data[i] = frame->buf[i]->data; + } } /* fixup special cases */ @@ -543,9 +676,6 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) avbuf->status = V4L2BUF_AVAILABLE; - if (V4L2_TYPE_IS_OUTPUT(ctx->type)) - return 0; - if (V4L2_TYPE_IS_MULTIPLANAR(ctx->type)) { avbuf->buf.m.planes = avbuf->planes; avbuf->buf.length = avbuf->num_planes; @@ -555,6 +685,15 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) avbuf->buf.length = avbuf->planes[0].length; } + if (V4L2_TYPE_IS_OUTPUT(ctx->type)) + return 0; + + if (buf_to_m2mctx(avbuf)->output_drm) { + ret = v4l2_buffer_export_drm(avbuf); + if (ret) + return ret; + } + return ff_v4l2_buffer_enqueue(avbuf); } diff --git a/libavcodec/v4l2_buffers.h b/libavcodec/v4l2_buffers.h index 8dbc7fc104..037e667997 100644 --- a/libavcodec/v4l2_buffers.h +++ b/libavcodec/v4l2_buffers.h @@ -27,6 +27,7 @@ #include #include +#include "libavutil/hwcontext_drm.h" #include "avcodec.h" enum V4L2Buffer_status { @@ -42,6 +43,9 @@ typedef struct V4L2Buffer { /* each buffer needs to have a reference to its context */ struct V4L2Context *context; + /* DRM descriptor */ + AVDRMFrameDescriptor drm_frame; + /* This object is refcounted per-plane, so we need to keep track * of how many context-refs we are holding. */ AVBufferRef *context_ref; diff --git a/libavcodec/v4l2_context.c b/libavcodec/v4l2_context.c index 29b144ed73..7a92df2c3e 100644 --- a/libavcodec/v4l2_context.c +++ b/libavcodec/v4l2_context.c @@ -455,22 +455,54 @@ static int v4l2_release_buffers(V4L2Context* ctx) struct v4l2_requestbuffers req = { .memory = V4L2_MEMORY_MMAP, .type = ctx->type, - .count = 0, /* 0 -> unmaps buffers from the driver */ + .count = 0, /* 0 -> unmap all buffers from the driver */ }; - int i, j; + int ret, i, j; for (i = 0; i < ctx->num_buffers; i++) { V4L2Buffer *buffer = &ctx->buffers[i]; for (j = 0; j < buffer->num_planes; j++) { struct V4L2Plane_info *p = &buffer->plane_info[j]; + + if (V4L2_TYPE_IS_OUTPUT(ctx->type)) { + /* output buffers are not EXPORTED */ + goto unmap; + } + + if (ctx_to_m2mctx(ctx)->output_drm) { + /* use the DRM frame to close */ + if (buffer->drm_frame.objects[j].fd >= 0) { + if (close(buffer->drm_frame.objects[j].fd) < 0) { + av_log(logger(ctx), AV_LOG_ERROR, "%s close drm fd " + "[buffer=%2d, plane=%d, fd=%2d] - %s \n", + ctx->name, i, j, buffer->drm_frame.objects[j].fd, + av_err2str(AVERROR(errno))); + } + } + } +unmap: if (p->mm_addr && p->length) if (munmap(p->mm_addr, p->length) < 0) - av_log(logger(ctx), AV_LOG_ERROR, "%s unmap plane (%s))\n", ctx->name, av_err2str(AVERROR(errno))); + av_log(logger(ctx), AV_LOG_ERROR, "%s unmap plane (%s))\n", + ctx->name, av_err2str(AVERROR(errno))); } } - return ioctl(ctx_to_m2mctx(ctx)->fd, VIDIOC_REQBUFS, &req); + ret = ioctl(ctx_to_m2mctx(ctx)->fd, VIDIOC_REQBUFS, &req); + if (ret < 0) { + av_log(logger(ctx), AV_LOG_ERROR, "release all %s buffers (%s)\n", + ctx->name, av_err2str(AVERROR(errno))); + + if (ctx_to_m2mctx(ctx)->output_drm) + av_log(logger(ctx), AV_LOG_ERROR, + "Make sure the DRM client releases all FB/GEM objects before closing the codec (ie):\n" + "for all buffers: \n" + " 1. drmModeRmFB(..)\n" + " 2. drmIoctl(.., DRM_IOCTL_GEM_CLOSE,... )\n"); + } + + return ret; } static inline int v4l2_try_raw_format(V4L2Context* ctx, enum AVPixelFormat pixfmt) diff --git a/libavcodec/v4l2_m2m.h b/libavcodec/v4l2_m2m.h index 456281f48c..4ee0be653b 100644 --- a/libavcodec/v4l2_m2m.h +++ b/libavcodec/v4l2_m2m.h @@ -63,6 +63,9 @@ typedef struct V4L2m2mContext { /* reference back to V4L2m2mPriv */ void *priv; + + /* generate DRM frames */ + int output_drm; } V4L2m2mContext; typedef struct V4L2m2mPriv { diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c index 45e9a8e9fe..eb6ecc8ed5 100644 --- a/libavcodec/v4l2_m2m_dec.c +++ b/libavcodec/v4l2_m2m_dec.c @@ -23,6 +23,9 @@ #include #include + +#include "libavutil/hwcontext.h" +#include "libavutil/hwcontext_drm.h" #include "libavutil/pixfmt.h" #include "libavutil/pixdesc.h" #include "libavutil/opt.h" @@ -30,6 +33,9 @@ #include "libavcodec/decode.h" #include "libavcodec/internal.h" +#include "libavcodec/hwaccels.h" +#include "libavcodec/internal.h" + #include "v4l2_context.h" #include "v4l2_m2m.h" #include "v4l2_fmt.h" @@ -207,6 +213,15 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx) capture->av_codec_id = AV_CODEC_ID_RAWVIDEO; capture->av_pix_fmt = avctx->pix_fmt; + /* the client requests the codec to generate DRM frames: + * - data[0] will therefore point to the returned AVDRMFrameDescriptor + * check the ff_v4l2_buffer_to_avframe conversion function. + * - the DRM frame format is passed in the DRM frame descriptor layer. + * check the v4l2_get_drm_frame function. + */ + if (ff_get_format(avctx, avctx->codec->pix_fmts) == AV_PIX_FMT_DRM_PRIME) + s->output_drm = 1; + s->avctx = avctx; ret = ff_v4l2_m2m_codec_init(priv); if (ret) { @@ -232,6 +247,11 @@ static const AVOption options[] = { { NULL}, }; +static const AVCodecHWConfigInternal *v4l2_m2m_hw_configs[] = { + HW_CONFIG_INTERNAL(DRM_PRIME), + NULL +}; + #define M2MDEC_CLASS(NAME) \ static const AVClass v4l2_m2m_ ## NAME ## _dec_class = { \ .class_name = #NAME "_v4l2m2m_decoder", \ @@ -255,6 +275,9 @@ static const AVOption options[] = { .bsfs = bsf_name, \ .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING, \ .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS | FF_CODEC_CAP_INIT_CLEANUP, \ + .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_DRM_PRIME, \ + AV_PIX_FMT_NONE}, \ + .hw_configs = v4l2_m2m_hw_configs, \ .wrapper_name = "v4l2m2m", \ } From 2b5cd753892dceba3b211053a6266c40aab38c55 Mon Sep 17 00:00:00 2001 From: Lukas Rusak Date: Thu, 16 Aug 2018 21:09:40 -0700 Subject: [PATCH 04/11] libavcodec: v4l2m2m: depends on libdrm --- configure | 1 + libavcodec/v4l2_buffers.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/configure b/configure index 8569a60bf8..a049707dd6 100755 --- a/configure +++ b/configure @@ -3401,6 +3401,7 @@ sndio_indev_deps="sndio" sndio_outdev_deps="sndio" v4l2_indev_deps_any="linux_videodev2_h sys_videoio_h" v4l2_indev_suggest="libv4l2" +v4l2_outdev_deps="libdrm" v4l2_outdev_deps_any="linux_videodev2_h sys_videoio_h" v4l2_outdev_suggest="libv4l2" vfwcap_indev_deps="vfw32 vfwcap_defines" diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c index 4bb2bf6f87..c36a73d1fa 100644 --- a/libavcodec/v4l2_buffers.c +++ b/libavcodec/v4l2_buffers.c @@ -21,7 +21,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -#include +#include #include #include #include From 09a0f1b99548a249991891ee4e02ae6613b545d7 Mon Sep 17 00:00:00 2001 From: Lukas Rusak Date: Thu, 16 Aug 2018 21:10:13 -0700 Subject: [PATCH 05/11] libavcodec: v4l2m2m: set format_modifier to DRM_FORMAT_MOD_LINEAR --- libavcodec/v4l2_buffers.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c index c36a73d1fa..072b77bbda 100644 --- a/libavcodec/v4l2_buffers.c +++ b/libavcodec/v4l2_buffers.c @@ -328,10 +328,12 @@ static int v4l2_buffer_export_drm(V4L2Buffer* avbuf) /* drm frame */ avbuf->drm_frame.objects[i].size = avbuf->buf.m.planes[i].length; avbuf->drm_frame.objects[i].fd = expbuf.fd; + avbuf->drm_frame.objects[i].format_modifier = DRM_FORMAT_MOD_LINEAR; } else { /* drm frame */ avbuf->drm_frame.objects[0].size = avbuf->buf.length; avbuf->drm_frame.objects[0].fd = expbuf.fd; + avbuf->drm_frame.objects[0].format_modifier = DRM_FORMAT_MOD_LINEAR; } } From e8df5a982f705aaba1e03aef653169bc17a0d464 Mon Sep 17 00:00:00 2001 From: Lukas Rusak Date: Thu, 16 Aug 2018 21:10:53 -0700 Subject: [PATCH 06/11] libavcodec: v4l2m2m: only mmap the buffer when it is output type and drm prime is used --- libavcodec/v4l2_buffers.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c index 072b77bbda..8162531973 100644 --- a/libavcodec/v4l2_buffers.c +++ b/libavcodec/v4l2_buffers.c @@ -662,14 +662,22 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) if (V4L2_TYPE_IS_MULTIPLANAR(ctx->type)) { avbuf->plane_info[i].length = avbuf->buf.m.planes[i].length; - avbuf->plane_info[i].mm_addr = mmap(NULL, avbuf->buf.m.planes[i].length, - PROT_READ | PROT_WRITE, MAP_SHARED, - buf_to_m2mctx(avbuf)->fd, avbuf->buf.m.planes[i].m.mem_offset); + + if ((V4L2_TYPE_IS_OUTPUT(ctx->type) && buf_to_m2mctx(avbuf)->output_drm) || + !buf_to_m2mctx(avbuf)->output_drm) { + avbuf->plane_info[i].mm_addr = mmap(NULL, avbuf->buf.m.planes[i].length, + PROT_READ | PROT_WRITE, MAP_SHARED, + buf_to_m2mctx(avbuf)->fd, avbuf->buf.m.planes[i].m.mem_offset); + } } else { avbuf->plane_info[i].length = avbuf->buf.length; - avbuf->plane_info[i].mm_addr = mmap(NULL, avbuf->buf.length, - PROT_READ | PROT_WRITE, MAP_SHARED, - buf_to_m2mctx(avbuf)->fd, avbuf->buf.m.offset); + + if ((V4L2_TYPE_IS_OUTPUT(ctx->type) && buf_to_m2mctx(avbuf)->output_drm) || + !buf_to_m2mctx(avbuf)->output_drm) { + avbuf->plane_info[i].mm_addr = mmap(NULL, avbuf->buf.length, + PROT_READ | PROT_WRITE, MAP_SHARED, + buf_to_m2mctx(avbuf)->fd, avbuf->buf.m.offset); + } } if (avbuf->plane_info[i].mm_addr == MAP_FAILED) From 4fb7664bb6be542b691323a03050cdd024585afc Mon Sep 17 00:00:00 2001 From: Lukas Rusak Date: Thu, 16 Aug 2018 21:11:38 -0700 Subject: [PATCH 07/11] libavcodec: v4l2m2m: allow using software pixel formats --- libavcodec/v4l2_m2m_dec.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c index eb6ecc8ed5..3b2449ae6c 100644 --- a/libavcodec/v4l2_m2m_dec.c +++ b/libavcodec/v4l2_m2m_dec.c @@ -219,8 +219,16 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx) * - the DRM frame format is passed in the DRM frame descriptor layer. * check the v4l2_get_drm_frame function. */ - if (ff_get_format(avctx, avctx->codec->pix_fmts) == AV_PIX_FMT_DRM_PRIME) + switch (ff_get_format(avctx, avctx->codec->pix_fmts)) { + case AV_PIX_FMT_DRM_PRIME: s->output_drm = 1; + break; + case AV_PIX_FMT_NONE: + return 0; + break; + default: + break; + } s->avctx = avctx; ret = ff_v4l2_m2m_codec_init(priv); @@ -276,6 +284,7 @@ static const AVCodecHWConfigInternal *v4l2_m2m_hw_configs[] = { .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING, \ .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS | FF_CODEC_CAP_INIT_CLEANUP, \ .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_DRM_PRIME, \ + AV_PIX_FMT_NV12, \ AV_PIX_FMT_NONE}, \ .hw_configs = v4l2_m2m_hw_configs, \ .wrapper_name = "v4l2m2m", \ From 27ae887df07992385b1afc9b532f978066e83774 Mon Sep 17 00:00:00 2001 From: Lukas Rusak Date: Mon, 24 Sep 2018 13:39:31 -0700 Subject: [PATCH 08/11] libavcodec: v4l2m2m: implement hwcontext --- libavcodec/v4l2_buffers.c | 22 ++++++++++++++++++++++ libavcodec/v4l2_context.h | 2 ++ libavcodec/v4l2_m2m.h | 2 ++ libavcodec/v4l2_m2m_dec.c | 11 +++++++++++ 4 files changed, 37 insertions(+) diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c index 8162531973..9c5d471c9b 100644 --- a/libavcodec/v4l2_buffers.c +++ b/libavcodec/v4l2_buffers.c @@ -435,6 +435,7 @@ static int v4l2_buffer_buf_to_swframe(AVFrame *frame, V4L2Buffer *avbuf) frame->data[0] = (uint8_t *) v4l2_get_drm_frame(avbuf); frame->format = AV_PIX_FMT_DRM_PRIME; + frame->hw_frames_ctx = av_buffer_ref(avbuf->context->frames_ref); } else { /* 1. get references to the actual data */ for (i = 0; i < avbuf->num_planes; i++) { @@ -635,6 +636,27 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) avbuf->buf.type = ctx->type; avbuf->buf.index = index; + if (buf_to_m2mctx(avbuf)->output_drm) { + AVHWFramesContext *hwframes; + + av_buffer_unref(&ctx->frames_ref); + + ctx->frames_ref = av_hwframe_ctx_alloc(buf_to_m2mctx(avbuf)->device_ref); + if (!ctx->frames_ref) { + ret = AVERROR(ENOMEM); + return ret; + } + + hwframes = (AVHWFramesContext*)ctx->frames_ref->data; + hwframes->format = AV_PIX_FMT_DRM_PRIME; + hwframes->sw_format = ctx->av_pix_fmt; + hwframes->width = ctx->width; + hwframes->height = ctx->height; + ret = av_hwframe_ctx_init(ctx->frames_ref); + if (ret < 0) + return ret; + } + if (V4L2_TYPE_IS_MULTIPLANAR(ctx->type)) { avbuf->buf.length = VIDEO_MAX_PLANES; avbuf->buf.m.planes = avbuf->planes; diff --git a/libavcodec/v4l2_context.h b/libavcodec/v4l2_context.h index 22a9532444..e804e94131 100644 --- a/libavcodec/v4l2_context.h +++ b/libavcodec/v4l2_context.h @@ -92,6 +92,8 @@ typedef struct V4L2Context { */ int done; + AVBufferRef *frames_ref; + } V4L2Context; /** diff --git a/libavcodec/v4l2_m2m.h b/libavcodec/v4l2_m2m.h index 4ee0be653b..61cb919771 100644 --- a/libavcodec/v4l2_m2m.h +++ b/libavcodec/v4l2_m2m.h @@ -64,6 +64,8 @@ typedef struct V4L2m2mContext { /* reference back to V4L2m2mPriv */ void *priv; + AVBufferRef *device_ref; + /* generate DRM frames */ int output_drm; } V4L2m2mContext; diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c index 3b2449ae6c..c6b865fde8 100644 --- a/libavcodec/v4l2_m2m_dec.c +++ b/libavcodec/v4l2_m2m_dec.c @@ -35,6 +35,7 @@ #include "libavcodec/hwaccels.h" #include "libavcodec/internal.h" +#include "libavcodec/hwconfig.h" #include "v4l2_context.h" #include "v4l2_m2m.h" @@ -230,6 +231,16 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx) break; } + s->device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_DRM); + if (!s->device_ref) { + ret = AVERROR(ENOMEM); + return ret; + } + + ret = av_hwdevice_ctx_init(s->device_ref); + if (ret < 0) + return ret; + s->avctx = avctx; ret = ff_v4l2_m2m_codec_init(priv); if (ret) { From aee464209ec6ad060d352dfb638344a1f4db3ce4 Mon Sep 17 00:00:00 2001 From: Lukas Rusak Date: Mon, 4 May 2020 13:01:29 -0700 Subject: [PATCH 09/11] libavcodec: v4l2m2m: allow lower minimum buffer values There is no reason to enforce a high minimum. In the context of streaming only a few output buffers and capture buffers are even needed for continuous playback. This also helps alleviate memory pressure when decoding 4K media. --- libavcodec/v4l2_m2m.h | 2 +- libavcodec/v4l2_m2m_dec.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/v4l2_m2m.h b/libavcodec/v4l2_m2m.h index 61cb919771..feeb162812 100644 --- a/libavcodec/v4l2_m2m.h +++ b/libavcodec/v4l2_m2m.h @@ -38,7 +38,7 @@ #define V4L_M2M_DEFAULT_OPTS \ { "num_output_buffers", "Number of buffers in the output context",\ - OFFSET(num_output_buffers), AV_OPT_TYPE_INT, { .i64 = 16 }, 6, INT_MAX, FLAGS } + OFFSET(num_output_buffers), AV_OPT_TYPE_INT, { .i64 = 16 }, 2, INT_MAX, FLAGS } typedef struct V4L2m2mContext { char devname[PATH_MAX]; diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c index c6b865fde8..b9725be377 100644 --- a/libavcodec/v4l2_m2m_dec.c +++ b/libavcodec/v4l2_m2m_dec.c @@ -262,7 +262,7 @@ static av_cold int v4l2_decode_close(AVCodecContext *avctx) static const AVOption options[] = { V4L_M2M_DEFAULT_OPTS, { "num_capture_buffers", "Number of buffers in the capture context", - OFFSET(num_capture_buffers), AV_OPT_TYPE_INT, {.i64 = 20}, 20, INT_MAX, FLAGS }, + OFFSET(num_capture_buffers), AV_OPT_TYPE_INT, {.i64 = 20}, 2, INT_MAX, FLAGS }, { NULL}, }; From ffc4419f456c00ab71cf93f792b0473c6de14e64 Mon Sep 17 00:00:00 2001 From: Lukas Rusak Date: Wed, 6 May 2020 11:12:58 -0700 Subject: [PATCH 10/11] libavcodec: v4l2m2m: add option to specify pixel format used by the decoder --- libavcodec/v4l2_context.c | 9 +++++++++ libavcodec/v4l2_m2m.h | 2 ++ libavcodec/v4l2_m2m_dec.c | 1 + 3 files changed, 12 insertions(+) diff --git a/libavcodec/v4l2_context.c b/libavcodec/v4l2_context.c index 7a92df2c3e..fa2deae888 100644 --- a/libavcodec/v4l2_context.c +++ b/libavcodec/v4l2_context.c @@ -531,6 +531,8 @@ static inline int v4l2_try_raw_format(V4L2Context* ctx, enum AVPixelFormat pixfm static int v4l2_get_raw_format(V4L2Context* ctx, enum AVPixelFormat *p) { + V4L2m2mContext* s = ctx_to_m2mctx(ctx); + V4L2m2mPriv *priv = s->avctx->priv_data; enum AVPixelFormat pixfmt = ctx->av_pix_fmt; struct v4l2_fmtdesc fdesc; int ret; @@ -549,6 +551,13 @@ static int v4l2_get_raw_format(V4L2Context* ctx, enum AVPixelFormat *p) if (ret) return AVERROR(EINVAL); + if (priv->pix_fmt != AV_PIX_FMT_NONE) { + if (fdesc.pixelformat != ff_v4l2_format_avfmt_to_v4l2(priv->pix_fmt)) { + fdesc.index++; + continue; + } + } + pixfmt = ff_v4l2_format_v4l2_to_avfmt(fdesc.pixelformat, AV_CODEC_ID_RAWVIDEO); ret = v4l2_try_raw_format(ctx, pixfmt); if (ret){ diff --git a/libavcodec/v4l2_m2m.h b/libavcodec/v4l2_m2m.h index feeb162812..0e88bf9329 100644 --- a/libavcodec/v4l2_m2m.h +++ b/libavcodec/v4l2_m2m.h @@ -30,6 +30,7 @@ #include #include "libavcodec/avcodec.h" +#include "libavutil/pixfmt.h" #include "v4l2_context.h" #define container_of(ptr, type, member) ({ \ @@ -78,6 +79,7 @@ typedef struct V4L2m2mPriv { int num_output_buffers; int num_capture_buffers; + enum AVPixelFormat pix_fmt; } V4L2m2mPriv; /** diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c index b9725be377..6109deee8a 100644 --- a/libavcodec/v4l2_m2m_dec.c +++ b/libavcodec/v4l2_m2m_dec.c @@ -263,6 +263,7 @@ static const AVOption options[] = { V4L_M2M_DEFAULT_OPTS, { "num_capture_buffers", "Number of buffers in the capture context", OFFSET(num_capture_buffers), AV_OPT_TYPE_INT, {.i64 = 20}, 2, INT_MAX, FLAGS }, + { "pixel_format", "Pixel format to be used by the decoder", OFFSET(pix_fmt), AV_OPT_TYPE_PIXEL_FMT, {.i64 = AV_PIX_FMT_NONE}, AV_PIX_FMT_NONE, AV_PIX_FMT_NB, FLAGS }, { NULL}, }; From 8595d06d4909bbec0aa14625fcfc869c6bcef696 Mon Sep 17 00:00:00 2001 From: Lukas Rusak Date: Mon, 24 Sep 2018 13:39:56 -0700 Subject: [PATCH 11/11] libavcodec: v4l2m2m: implement flush --- libavcodec/v4l2_m2m_dec.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c index 6109deee8a..820cdf241f 100644 --- a/libavcodec/v4l2_m2m_dec.c +++ b/libavcodec/v4l2_m2m_dec.c @@ -256,6 +256,41 @@ static av_cold int v4l2_decode_close(AVCodecContext *avctx) return ff_v4l2_m2m_codec_end(avctx->priv_data); } +static void v4l2_decode_flush(AVCodecContext *avctx) +{ + V4L2m2mPriv *priv = avctx->priv_data; + V4L2m2mContext* s = priv->context; + V4L2Context* output = &s->output; + V4L2Context* capture = &s->capture; + int ret, i; + + ret = ff_v4l2_context_set_status(output, VIDIOC_STREAMOFF); + if (ret < 0) + av_log(avctx, AV_LOG_ERROR, "VIDIOC_STREAMOFF %s error: %d\n", output->name, ret); + + ret = ff_v4l2_context_set_status(output, VIDIOC_STREAMON); + if (ret < 0) + av_log(avctx, AV_LOG_ERROR, "VIDIOC_STREAMON %s error: %d\n", output->name, ret); + + for (i = 0; i < output->num_buffers; i++) { + if (output->buffers[i].status == V4L2BUF_IN_DRIVER) + output->buffers[i].status = V4L2BUF_AVAILABLE; + } + + struct v4l2_decoder_cmd cmd = { + .cmd = V4L2_DEC_CMD_START, + .flags = 0, + }; + + ret = ioctl(s->fd, VIDIOC_DECODER_CMD, &cmd); + if (ret < 0) + av_log(avctx, AV_LOG_ERROR, "VIDIOC_DECODER_CMD start error: %d\n", errno); + + s->draining = 0; + output->done = 0; + capture->done = 0; +} + #define OFFSET(x) offsetof(V4L2m2mPriv, x) #define FLAGS AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM @@ -292,6 +327,7 @@ static const AVCodecHWConfigInternal *v4l2_m2m_hw_configs[] = { .init = v4l2_decode_init, \ .receive_frame = v4l2_receive_frame, \ .close = v4l2_decode_close, \ + .flush = v4l2_decode_flush, \ .bsfs = bsf_name, \ .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING, \ .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS | FF_CODEC_CAP_INIT_CLEANUP, \