/* * Copyright 2026 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "pc/encoded_video_frame_injector.h" #include #include #include #include #include #include #include #include #include #include "absl/base/nullability.h" #include "api/encoded_video_frame_injector_interface.h" #include "api/environment/environment.h" #include "api/frame_transformer_interface.h" #include "api/media_stream_interface.h" #include "api/scoped_refptr.h" #include "api/sequence_checker.h" #include "api/task_queue/pending_task_safety_flag.h" #include "api/task_queue/task_queue_base.h" #include "api/units/timestamp.h" #include "api/video/encoded_image.h" #include "api/video/i420_buffer.h" #include "api/video/video_codec_type.h" #include "api/video/video_frame.h" #include "api/video/video_frame_type.h" #include "api/video/video_rotation.h" #include "api/video/video_sink_interface.h" #include "api/video/video_source_interface.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_factory.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/checks.h" #include "rtc_base/ref_counted_object.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/system/no_unique_address.h" #include "rtc_base/thread_annotations.h" #include "rtc_base/time_utils.h" namespace webrtc { // A placeholder video track that generates black frames to keep the WebRTC // engine's capture and processing pipeline active when using // EncodedVideoFrameInjector. class ProxyVideoTrack : public VideoTrackInterface { public: static scoped_refptr Create( TaskQueueBase* absl_nonnull worker_thread) { return scoped_refptr( new RefCountedObject(worker_thread)); } // called on any thread void InjectBlackFrame(uint16_t width, uint16_t height) { if (worker_thread_->IsCurrent()) { InjectBlackFrameInternal(width, height); } else { worker_thread_->PostTask( [scoped_this = scoped_refptr(this), width, height] { scoped_this->InjectBlackFrameInternal(width, height); }); } } // webrtc::VideoSourceInterface void AddOrUpdateSink(VideoSinkInterface* sink, const VideoSinkWants& wants) override { RTC_DCHECK_RUN_ON(&worker_sequence_checker_); RTC_DCHECK(sink != nullptr); if (std::find(sinks_.begin(), sinks_.end(), sink) == sinks_.end()) { sinks_.push_back(sink); } } void RemoveSink(VideoSinkInterface* sink) override { RTC_DCHECK_RUN_ON(&worker_sequence_checker_); RTC_DCHECK(sink != nullptr); std::erase(sinks_, sink); } void RequestRefreshFrame() override { RTC_DCHECK_RUN_ON(&worker_sequence_checker_); } // webrtc::VideoTrackInterface / MediaStreamTrackInterface / NotifierInterface VideoTrackSourceInterface* GetSource() const override { RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); return nullptr; } std::string kind() const override { RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); return "video"; } std::string id() const override { RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); return "proxy_video_track"; } bool enabled() const override { RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); return true; } bool set_enabled(bool enable) override { RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); return true; } TrackState state() const override { RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); return TrackState::kLive; } void RegisterObserver(ObserverInterface* observer) override { RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); } void UnregisterObserver(ObserverInterface* observer) override { RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); } protected: explicit ProxyVideoTrack(TaskQueueBase* absl_nonnull worker_thread) : worker_thread_(worker_thread) { RTC_DCHECK(worker_thread_); } ~ProxyVideoTrack() override = default; private: void InjectBlackFrameInternal(uint16_t width, uint16_t height) { RTC_DCHECK_RUN_ON(&worker_sequence_checker_); if (sinks_.empty()) { return; } if (!black_buffer_ || black_buffer_->width() != width || black_buffer_->height() != height) { black_buffer_ = I420Buffer::Create(width, height); I420Buffer::SetBlack(black_buffer_.get()); } VideoFrame frame = VideoFrame::Builder() .set_video_frame_buffer(black_buffer_) .set_rotation(kVideoRotation_0) .set_timestamp_us(TimeMicros()) .build(); for (auto* sink : sinks_) { sink->OnFrame(frame); } } RTC_NO_UNIQUE_ADDRESS SequenceChecker signaling_sequence_checker_; RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_{ SequenceChecker::kDetached}; TaskQueueBase* absl_nonnull const worker_thread_; std::vector*> sinks_ RTC_GUARDED_BY(worker_sequence_checker_); scoped_refptr black_buffer_ RTC_GUARDED_BY(worker_sequence_checker_); }; // A placeholder video encoder that discards raw frames and instead passes // injected encoded video frames to the WebRTC encoded video pipeline. class ProxyVideoEncoder : public VideoEncoder { public: explicit ProxyVideoEncoder(scoped_refptr injector) : injector_(injector), encoder_queue_(TaskQueueBase::Current()) { RTC_DCHECK(injector); RTC_DCHECK(encoder_queue_); } int InitEncode(const VideoCodec* codec_settings, const VideoEncoder::Settings& settings) override { RTC_DCHECK_RUN_ON(encoder_queue_); codec_type_ = codec_settings->codecType; if (callback_ != nullptr) { injector_->RegisterEncoder(this); } return WEBRTC_VIDEO_CODEC_OK; } int32_t RegisterEncodeCompleteCallback( EncodedImageCallback* callback) override { RTC_DCHECK_RUN_ON(encoder_queue_); callback_ = callback; injector_->RegisterEncoder(this); return WEBRTC_VIDEO_CODEC_OK; } int32_t Release() override { RTC_DCHECK_RUN_ON(encoder_queue_); injector_->UnregisterEncoder(this); return WEBRTC_VIDEO_CODEC_OK; } int32_t Encode(const VideoFrame& frame, const std::vector* frame_types) override { RTC_DCHECK_RUN_ON(encoder_queue_); if (!frame_types) { return WEBRTC_VIDEO_CODEC_OK; } if (std::find(frame_types->begin(), frame_types->end(), VideoFrameType::kVideoFrameKey) != frame_types->end()) { injector_->InvokeKeyFrameCallback(); } return WEBRTC_VIDEO_CODEC_OK; } void SetRates(const RateControlParameters& parameters) override { RTC_DCHECK_RUN_ON(encoder_queue_); injector_->InvokeBitrateInfoCallback( parameters.bitrate.get_sum_bps(), static_cast(parameters.bandwidth_allocation.bps())); } EncoderInfo GetEncoderInfo() const override { RTC_DCHECK_RUN_ON(encoder_queue_); EncoderInfo info; info.implementation_name = "ProxyVideoEncoder"; return info; } void InjectEncodedFrameOnQueue(EncodedImage encoded_image) { RTC_DCHECK_RUN_ON(encoder_queue_); if (!callback_) { return; } CodecSpecificInfo codec_specific_info; codec_specific_info.codecType = codec_type_; callback_->OnEncodedImage(encoded_image, &codec_specific_info); } // can be called on any thread void InjectEncodedFrame( std::unique_ptr frame) { if (!frame) { return; } EncodedImage encoded_image; encoded_image.SetEncodedData(EncodedImageBuffer::Create( frame->GetData().data(), frame->GetData().size())); RtpTimestampInfo rtp_info = frame->GetRtpTimestampInfo(); if (std::holds_alternative(rtp_info)) { encoded_image.SetRtpTimestamp(std::get(rtp_info)); } else { encoded_image.SetRtpTimestamp( std::get(rtp_info)); } std::optional capture_time = frame->CaptureTime(); if (capture_time.has_value() && capture_time->IsFinite()) { encoded_image.capture_time_ms_ = capture_time->ms(); } encoded_image.set_frame_type(frame->IsKeyFrame() ? VideoFrameType::kVideoFrameKey : VideoFrameType::kVideoFrameDelta); encoder_queue_->PostTask( SafeTask(safety_.flag(), [this, encoded_image = std::move(encoded_image)]() mutable { InjectEncodedFrameOnQueue(std::move(encoded_image)); })); } private: const scoped_refptr injector_; EncodedImageCallback* callback_ RTC_GUARDED_BY(encoder_queue_) = nullptr; // Safe reference to the Encoder owner VideoCodecType codec_type_ RTC_GUARDED_BY(encoder_queue_); TaskQueueBase* const encoder_queue_; ScopedTaskSafety const safety_; }; // A video encoder factory that creates ProxyVideoEncoder instances. // Essentially just a wrapper of the EncodedVideoFrameInjector. class ProxyVideoEncoderFactory : public VideoEncoderFactory { public: explicit ProxyVideoEncoderFactory( scoped_refptr injector) : injector_(injector) {} ~ProxyVideoEncoderFactory() override {} std::vector GetSupportedFormats() const override { RTC_DCHECK_RUN_ON(&sequence_checker_); return {}; } std::unique_ptr Create(const Environment& env, const SdpVideoFormat& format) override { RTC_DCHECK_RUN_ON(&sequence_checker_); return std::make_unique(injector_); } private: RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_{ SequenceChecker::kDetached}; const scoped_refptr injector_; }; scoped_refptr EncodedVideoFrameInjector::Create( KeyFrameCallback keyframe_callback, BitrateInfoCallback bitrate_callback, TaskQueueBase* absl_nonnull worker_thread) { return scoped_refptr( new RefCountedObject( std::move(keyframe_callback), std::move(bitrate_callback), worker_thread)); } EncodedVideoFrameInjector::EncodedVideoFrameInjector( KeyFrameCallback keyframe_callback, BitrateInfoCallback bitrate_callback, TaskQueueBase* absl_nonnull worker_thread) : video_track_(ProxyVideoTrack::Create(worker_thread)), keyframe_callback_(std::move(keyframe_callback)), bitrate_callback_(std::move(bitrate_callback)) { RTC_DCHECK(worker_thread); } EncodedVideoFrameInjector::~EncodedVideoFrameInjector() {} // Inject an encoded video frame. // Requires to inject a raw frame into the track to simulate standard WebRTC // video capture and processing. This ensures standard pipeline mechanisms like // bitrate allocation remain active. // Raw frames will ultimately be passed to the encoder and dropped. // It is necessary to buffer the first encoded frames as the encoder is not // allocated until a raw frame is injected. Once the encoder is registered with // the injector, encoded frames are directly passed to it for injection. It is // not strictly necessary to keep a 1:1 correlation between raw and encoded // frames. void EncodedVideoFrameInjector::InjectFrame( std::unique_ptr encoded_frame) { if (!encoded_frame) { return; } uint16_t width = encoded_frame->Metadata().GetWidth(); uint16_t height = encoded_frame->Metadata().GetHeight(); video_track_->InjectBlackFrame(width, height); { MutexLock lock(&encoder_lock_); if (encoder_) { while (!buffered_frames_.empty()) { encoder_->InjectEncodedFrame(std::move(buffered_frames_.front())); buffered_frames_.pop_front(); } encoder_->InjectEncodedFrame(std::move(encoded_frame)); } else { if (buffered_frames_.size() >= kMaxBufferedFrames) { buffered_frames_.pop_front(); } buffered_frames_.push_back(std::move(encoded_frame)); } } } scoped_refptr EncodedVideoFrameInjector::GetVideoTrack() { RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); return video_track_; } absl_nonnull std::unique_ptr EncodedVideoFrameInjector::CreateEncoderFactory() { RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); return std::make_unique( scoped_refptr(this)); } void EncodedVideoFrameInjector::RegisterEncoder(ProxyVideoEncoder* encoder) { RTC_DCHECK_RUN_ON(&encoder_sequence_checker_); MutexLock lock(&encoder_lock_); RTC_CHECK(encoder); RTC_CHECK(!encoder_ || encoder_ == encoder); encoder_ = encoder; } void EncodedVideoFrameInjector::InvokeKeyFrameCallback() { RTC_DCHECK_RUN_ON(&encoder_sequence_checker_); if (keyframe_callback_) { keyframe_callback_(); } } void EncodedVideoFrameInjector::InvokeBitrateInfoCallback( int32_t allocated_bitrate, int32_t available_outgoing_bitrate) { RTC_DCHECK_RUN_ON(&encoder_sequence_checker_); if (bitrate_callback_) { bitrate_callback_(allocated_bitrate, available_outgoing_bitrate); } } void EncodedVideoFrameInjector::UnregisterEncoder(ProxyVideoEncoder* encoder) { RTC_DCHECK_RUN_ON(&encoder_sequence_checker_); MutexLock lock(&encoder_lock_); RTC_CHECK(encoder_); RTC_CHECK(encoder); RTC_CHECK(encoder_ == encoder); encoder_ = nullptr; } } // namespace webrtc