/* * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "test/pc/e2e/analyzer/video/analyzing_video_sink.h" #include #include #include #include #include #include #include #include #include "absl/memory/memory.h" #include "absl/strings/string_view.h" #include "api/scoped_refptr.h" #include "api/test/metrics/metric.h" #include "api/test/metrics/metrics_logger.h" #include "api/test/pclf/media_configuration.h" #include "api/test/video/video_frame_writer.h" #include "api/test/video_quality_analyzer_interface.h" #include "api/units/timestamp.h" #include "api/video/i420_buffer.h" #include "api/video/video_frame.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/synchronization/mutex.h" #include "system_wrappers/include/clock.h" #include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h" #include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" #include "test/pc/e2e/analyzer/video/video_dumping.h" #include "test/pc/e2e/metric_metadata_keys.h" #include "test/testsupport/fixed_fps_video_frame_writer_adapter.h" #include "test/video_renderer.h" namespace webrtc { namespace webrtc_pc_e2e { AnalyzingVideoSink::AnalyzingVideoSink(absl::string_view peer_name, Clock* clock, VideoQualityAnalyzerInterface& analyzer, AnalyzingVideoSinksHelper& sinks_helper, const VideoSubscription& subscription, bool report_infra_stats) : peer_name_(peer_name), report_infra_stats_(report_infra_stats), clock_(clock), analyzer_(&analyzer), sinks_helper_(&sinks_helper), subscription_(subscription) {} void AnalyzingVideoSink::UpdateSubscription( const VideoSubscription& subscription) { // For peers with changed resolutions we need to close current writers and // open new ones. This is done by removing existing sinks, which will force // creation of the new sinks when next frame will be received. std::set writers_to_close; { MutexLock lock(&mutex_); subscription_ = subscription; for (auto it = stream_sinks_.cbegin(); it != stream_sinks_.cend();) { std::optional new_requested_resolution = subscription_.GetResolutionForPeer(it->second.sender_peer_name); if (new_requested_resolution != it->second.resolution) { RTC_LOG(LS_INFO) << peer_name_ << ": Subscribed resolution for stream " << it->first << " from " << it->second.sender_peer_name << " was updated from " << it->second.resolution.ToString() << " to " << (new_requested_resolution.has_value() ? new_requested_resolution->ToString() : "none") << ". Repopulating all video sinks and recreating " << "requested video writers"; writers_to_close.insert(it->second.video_frame_writer); it = stream_sinks_.erase(it); } else { ++it; } } } sinks_helper_->CloseAndRemoveVideoWriters(writers_to_close); } void AnalyzingVideoSink::OnFrame(const VideoFrame& frame) { if (IsDummyFrame(frame)) { // This is dummy frame, so we don't need to process it further. return; } if (frame.id() == VideoFrame::kNotSetId) { // If frame ID is unknown we can't get required render resolution, so pass // to the analyzer in the actual resolution of the frame. AnalyzeFrame(frame); } else { std::string stream_label = analyzer_->GetStreamLabel(frame.id()); MutexLock lock(&mutex_); Timestamp processing_started = clock_->CurrentTime(); SinksDescriptor* sinks_descriptor = PopulateSinks(stream_label); RTC_CHECK(sinks_descriptor != nullptr); VideoFrame scaled_frame = ScaleVideoFrame(frame, sinks_descriptor->resolution); AnalyzeFrame(scaled_frame); for (auto& sink : sinks_descriptor->sinks) { sink->OnFrame(scaled_frame); } Timestamp processing_finished = clock_->CurrentTime(); if (report_infra_stats_) { stats_.analyzing_sink_processing_time_ms.AddSample( {.value = (processing_finished - processing_started).ms(), .time = processing_finished}); } } } void AnalyzingVideoSink::LogMetrics(test::MetricsLogger& metrics_logger, absl::string_view test_case_name) const { if (report_infra_stats_) { MutexLock lock(&mutex_); std::map metadata = { {MetricMetadataKey::kPeerMetadataKey, peer_name_}}; metrics_logger.LogMetric( "analyzing_sink_processing_time_ms", test_case_name, stats_.analyzing_sink_processing_time_ms, test::Unit::kMilliseconds, test::ImprovementDirection::kSmallerIsBetter, metadata); metrics_logger.LogMetric("scaling_tims_ms", test_case_name, stats_.scaling_tims_ms, test::Unit::kMilliseconds, test::ImprovementDirection::kSmallerIsBetter, metadata); } } AnalyzingVideoSink::Stats AnalyzingVideoSink::stats() const { MutexLock lock(&mutex_); return stats_; } VideoFrame AnalyzingVideoSink::ScaleVideoFrame( const VideoFrame& frame, const VideoResolution& required_resolution) { Timestamp processing_started = clock_->CurrentTime(); if ((required_resolution.width() == static_cast(frame.width()) && required_resolution.height() == static_cast(frame.height())) || !required_resolution.IsRegular() || (required_resolution.width() == 0 || required_resolution.height() == 0)) { if (report_infra_stats_) { Timestamp now = clock_->CurrentTime(); stats_.scaling_tims_ms.AddSample( {.value = (now - processing_started).ms(), .time = now}); } return frame; } // We allow some difference in the aspect ration because when decoder // downscales video stream it may round up some dimensions to make them even, // ex: 960x540 -> 480x270 -> 240x136 instead of 240x135. RTC_CHECK_LE(std::abs(static_cast(required_resolution.width()) / required_resolution.height() - static_cast(frame.width()) / frame.height()), 0.1) << peer_name_ << ": Received frame has too different aspect ratio compared to " << "requested video resolution: required resolution=" << required_resolution.ToString() << "; actual resolution=" << frame.width() << "x" << frame.height(); scoped_refptr scaled_buffer(I420Buffer::Create( required_resolution.width(), required_resolution.height())); scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420()); VideoFrame scaled_frame = frame; scaled_frame.set_video_frame_buffer(scaled_buffer); if (report_infra_stats_) { Timestamp now = clock_->CurrentTime(); stats_.scaling_tims_ms.AddSample( {.value = (now - processing_started).ms(), .time = now}); } return scaled_frame; } void AnalyzingVideoSink::AnalyzeFrame(const VideoFrame& frame) { VideoFrame frame_copy = frame; frame_copy.set_video_frame_buffer( I420Buffer::Copy(*frame.video_frame_buffer()->ToI420())); analyzer_->OnFrameRendered(peer_name_, frame_copy); } AnalyzingVideoSink::SinksDescriptor* AnalyzingVideoSink::PopulateSinks( absl::string_view stream_label) { // Fast pass: sinks already exists. auto sinks_it = stream_sinks_.find(std::string(stream_label)); if (sinks_it != stream_sinks_.end()) { return &sinks_it->second; } // Slow pass: we need to create and save sinks std::optional> peer_and_config = sinks_helper_->GetPeerAndConfig(stream_label); RTC_CHECK(peer_and_config.has_value()) << "No video config for stream " << stream_label; const std::string& sender_peer_name = peer_and_config->first; const VideoConfig& config = peer_and_config->second; std::optional resolution = subscription_.GetResolutionForPeer(sender_peer_name); if (!resolution.has_value()) { RTC_LOG(LS_ERROR) << peer_name_ << " received stream " << stream_label << " from " << sender_peer_name << " for which they were not subscribed"; resolution = config.GetResolution(); } if (!resolution->IsRegular()) { RTC_LOG(LS_ERROR) << peer_name_ << " received stream " << stream_label << " from " << sender_peer_name << " for which resolution wasn't resolved"; resolution = config.GetResolution(); } RTC_CHECK(resolution.has_value()); SinksDescriptor sinks_descriptor(sender_peer_name, *resolution); if (config.output_dump_options.has_value()) { std::unique_ptr writer = config.output_dump_options->CreateOutputDumpVideoFrameWriter( stream_label, peer_name_, *resolution); if (config.output_dump_use_fixed_framerate) { writer = std::make_unique( resolution->fps(), clock_, std::move(writer)); } sinks_descriptor.sinks.push_back(std::make_unique( writer.get(), config.output_dump_options->sampling_modulo())); sinks_descriptor.video_frame_writer = sinks_helper_->AddVideoWriter(std::move(writer)); } if (config.show_on_screen) { sinks_descriptor.sinks.push_back( absl::WrapUnique(test::VideoRenderer::Create( (*config.stream_label + "-render").c_str(), resolution->width(), resolution->height()))); } return &stream_sinks_.emplace(stream_label, std::move(sinks_descriptor)) .first->second; } } // namespace webrtc_pc_e2e } // namespace webrtc