/** * @file src/video.h * @brief Declarations for video. */ #pragma once // local includes #include "input.h" #include "platform/common.h" #include "pyrowave_policy.h" #include "video_policy.h" #include "thread_safe.h" #include "video_colorspace.h" // standard includes #include #include #include #include extern "C" { #include #include } struct AVPacket; namespace video { enum class capture_source_e : std::uint8_t { active_output, exact_output, synthetic_black, }; /* Encoding configuration requested by remote client */ struct config_t { // DO NOT CHANGE ORDER OR ADD FIELDS IN THE MIDDLE!!!!! // ONLY APPEND NEW FIELD AFTERWARDS!!!!!!!!! // BIG F WORD to Sunshine!!!!!!!!! int width; // Video width in pixels int height; // Video height in pixels int framerate; // Requested framerate, used in individual frame bitrate budget calculation int framerateX100; // Optional field for streaming at NTSC or similar rates e.g. 59.94 = 5994 int bitrate; // Video bitrate in kilobits (1000 bits) for requested framerate int slicesPerFrame; // Number of slices per frame int numRefFrames; // Max number of reference frames /* Requested color range and SDR encoding colorspace, HDR encoding colorspace is always BT.2020+ST2084 Color range (encoderCscMode & 0x1) : 0 - limited, 1 - full SDR encoding colorspace (encoderCscMode >> 1) : 0 - BT.601, 1 - BT.709, 2 - BT.2020 */ int encoderCscMode; int videoFormat; // 0 - H.264, 1 - HEVC, 2 - AV1, 3 - PyroWave /* Encoding color depth (bit depth): 0 - 8-bit, 1 - 10-bit HDR encoding activates when color depth is higher than 8-bit and the display which is being captured is operating in HDR mode */ int dynamicRange; // Set when we bumped dynamicRange to 10-bit for SDR-only encoding (prefer_10bit_sdr) bool prefer_sdr_10bit; // Force SDR colorspace/signaling independently of negotiated encoder bit depth. bool force_sdr = false; // Stream-scoped TrueHDR-capable pipeline enable. Runtime foreground-window // checks can still bypass per-frame conversion while this remains true, which // keeps capture and encoder format stable across desktop/app focus changes. bool rtx_hdr_active = false; int rtx_hdr_peak_nits = 1000; int chromaSamplingType; // 0 - 4:2:0, 1 - 4:4:4 int enableIntraRefresh; // 0 - disabled, 1 - enabled int encodingFramerate = 0; // Requested display framerate bool input_only = false; // Capture ownership is session-local. Exact remote outputs must not fall // back through the mutable process-wide output selection, while input-only // sessions use an in-memory black producer and never initialize capture. capture_source_e capture_source {capture_source_e::active_output}; std::optional capture_output; // Opt into the smallest supported host-side queues for a launch-qualified VRR // session. This remains false for clients that do not negotiate the mode. bool vrr_low_latency = false; // Original client-requested wire-bandwidth budget in Kbps, before Sunshine // subtracts FEC/audio/control overhead from `bitrate` for the encoder. // Same as `bitrate` for clients that don't send maximumBitrateKbps. int client_requested_bitrate = 0; // PyroWave only (videoFormat 3): the negotiated RTP packet size, which record // framing aligns to, and the framing the client asked for in ANNOUNCE. int packetsize = 0; pyrowave::policy::framing_e pyrowave_framing = pyrowave::policy::framing_e::records; }; platf::mem_type_e map_base_dev_type(AVHWDeviceType type); platf::pix_fmt_e map_pix_fmt(AVPixelFormat fmt); void free_ctx(AVCodecContext *ctx); void free_frame(AVFrame *frame); void free_buffer(AVBufferRef *ref); using avcodec_ctx_t = util::safe_ptr; using avcodec_frame_t = util::safe_ptr; using avcodec_buffer_t = util::safe_ptr; using sws_t = util::safe_ptr; using img_event_t = std::shared_ptr>>; struct encoder_platform_formats_t { virtual ~encoder_platform_formats_t() = default; platf::mem_type_e dev_type; platf::pix_fmt_e pix_fmt_8bit; platf::pix_fmt_e pix_fmt_10bit; platf::pix_fmt_e pix_fmt_yuv444_8bit; platf::pix_fmt_e pix_fmt_yuv444_10bit; }; struct encoder_platform_formats_avcodec: encoder_platform_formats_t { using init_buffer_function_t = std::function(platf::avcodec_encode_device_t *)>; encoder_platform_formats_avcodec( const AVHWDeviceType &avcodec_base_dev_type, const AVHWDeviceType &avcodec_derived_dev_type, const AVPixelFormat &avcodec_dev_pix_fmt, const AVPixelFormat &avcodec_pix_fmt_8bit, const AVPixelFormat &avcodec_pix_fmt_10bit, const AVPixelFormat &avcodec_pix_fmt_yuv444_8bit, const AVPixelFormat &avcodec_pix_fmt_yuv444_10bit, const init_buffer_function_t &init_avcodec_hardware_input_buffer_function ): avcodec_base_dev_type {avcodec_base_dev_type}, avcodec_derived_dev_type {avcodec_derived_dev_type}, avcodec_dev_pix_fmt {avcodec_dev_pix_fmt}, avcodec_pix_fmt_8bit {avcodec_pix_fmt_8bit}, avcodec_pix_fmt_10bit {avcodec_pix_fmt_10bit}, avcodec_pix_fmt_yuv444_8bit {avcodec_pix_fmt_yuv444_8bit}, avcodec_pix_fmt_yuv444_10bit {avcodec_pix_fmt_yuv444_10bit}, init_avcodec_hardware_input_buffer {init_avcodec_hardware_input_buffer_function} { dev_type = map_base_dev_type(avcodec_base_dev_type); pix_fmt_8bit = map_pix_fmt(avcodec_pix_fmt_8bit); pix_fmt_10bit = map_pix_fmt(avcodec_pix_fmt_10bit); pix_fmt_yuv444_8bit = map_pix_fmt(avcodec_pix_fmt_yuv444_8bit); pix_fmt_yuv444_10bit = map_pix_fmt(avcodec_pix_fmt_yuv444_10bit); } AVHWDeviceType avcodec_base_dev_type; AVHWDeviceType avcodec_derived_dev_type; AVPixelFormat avcodec_dev_pix_fmt; AVPixelFormat avcodec_pix_fmt_8bit; AVPixelFormat avcodec_pix_fmt_10bit; AVPixelFormat avcodec_pix_fmt_yuv444_8bit; AVPixelFormat avcodec_pix_fmt_yuv444_10bit; init_buffer_function_t init_avcodec_hardware_input_buffer; }; struct encoder_platform_formats_nvenc: encoder_platform_formats_t { encoder_platform_formats_nvenc( const platf::mem_type_e &dev_type, const platf::pix_fmt_e &pix_fmt_8bit, const platf::pix_fmt_e &pix_fmt_10bit, const platf::pix_fmt_e &pix_fmt_yuv444_8bit, const platf::pix_fmt_e &pix_fmt_yuv444_10bit ) { encoder_platform_formats_t::dev_type = dev_type; encoder_platform_formats_t::pix_fmt_8bit = pix_fmt_8bit; encoder_platform_formats_t::pix_fmt_10bit = pix_fmt_10bit; encoder_platform_formats_t::pix_fmt_yuv444_8bit = pix_fmt_yuv444_8bit; encoder_platform_formats_t::pix_fmt_yuv444_10bit = pix_fmt_yuv444_10bit; } }; struct encoder_platform_formats_amf: encoder_platform_formats_t { encoder_platform_formats_amf( const platf::mem_type_e &dev_type, const platf::pix_fmt_e &pix_fmt_8bit, const platf::pix_fmt_e &pix_fmt_10bit, const platf::pix_fmt_e &pix_fmt_yuv444_8bit, const platf::pix_fmt_e &pix_fmt_yuv444_10bit ) { encoder_platform_formats_t::dev_type = dev_type; encoder_platform_formats_t::pix_fmt_8bit = pix_fmt_8bit; encoder_platform_formats_t::pix_fmt_10bit = pix_fmt_10bit; encoder_platform_formats_t::pix_fmt_yuv444_8bit = pix_fmt_yuv444_8bit; encoder_platform_formats_t::pix_fmt_yuv444_10bit = pix_fmt_yuv444_10bit; } }; struct encoder_t { std::string_view name; enum flag_e { PASSED, ///< Indicates the encoder is supported. REF_FRAMES_RESTRICT, ///< Set maximum reference frames. DYNAMIC_RANGE, ///< HDR support. YUV444, ///< YUV 4:4:4 support. VUI_PARAMETERS, ///< AMD encoder with VAAPI doesn't add VUI parameters to SPS. MAX_FLAGS ///< Maximum number of flags. }; static std::string_view from_flag(flag_e flag) { #define _CONVERT(x) \ case flag_e::x: \ return std::string_view(#x) switch (flag) { _CONVERT(PASSED); _CONVERT(REF_FRAMES_RESTRICT); _CONVERT(DYNAMIC_RANGE); _CONVERT(YUV444); _CONVERT(VUI_PARAMETERS); _CONVERT(MAX_FLAGS); } #undef _CONVERT return {"unknown"}; } struct option_t { KITTY_DEFAULT_CONSTR_MOVE(option_t) option_t(const option_t &) = default; std::string name; struct optional_int_function_t { std::function()> evaluate; }; std::variant *, std::function, optional_int_function_t, std::string, std::string *, std::function> value; option_t(std::string &&name, decltype(value) &&value): name {std::move(name)}, value {std::move(value)} { } }; const std::unique_ptr platform_formats; struct codec_t { std::vector common_options; std::vector sdr_options; std::vector hdr_options; std::vector sdr444_options; std::vector hdr444_options; std::vector fallback_options; std::string name; std::bitset capabilities; bool operator[](flag_e flag) const { return capabilities[(std::size_t) flag]; } std::bitset::reference operator[](flag_e flag) { return capabilities[(std::size_t) flag]; } }; codec_t av1; codec_t hevc; codec_t h264; const codec_t &codec_from_config(const config_t &config) const { switch (config.videoFormat) { default: BOOST_LOG(error) << "Unknown video format " << config.videoFormat << ", falling back to H.264"; // fallthrough case 0: return h264; case 1: return hevc; case 2: return av1; } } uint32_t flags; }; struct encode_session_t { virtual ~encode_session_t() = default; virtual int convert(platf::img_t &img) = 0; virtual void request_idr_frame() = 0; virtual void request_normal_frame() = 0; virtual void invalidate_ref_frames(int64_t first_frame, int64_t last_frame) = 0; /** * @brief Apply a new encoder bitrate to the live encoder without rebuilding it. * @param bitrate_kbps New bitrate in kbps. * @return `true` if the encoder reconfigured itself in place; `false` if the caller must * rebuild the encode session to apply the change. */ virtual bool set_bitrate(int bitrate_kbps) { return false; } /** * @brief Replace static HDR mastering metadata used for subsequent encoded frames. * @param metadata New mastering-display and content-light metadata. */ virtual void set_hdr_metadata(const SS_HDR_METADATA &) { } }; // encoders extern encoder_t software; #if !defined(__APPLE__) extern encoder_t nvenc; // available for windows and linux #endif #if defined(__linux__) extern encoder_t nvenc_legacy; #endif #ifdef _WIN32 extern encoder_t amdvce_experimental; extern encoder_t amdvce_ffmpeg; extern encoder_t quicksync; extern encoder_t mediafoundation; #endif #if defined(__linux__) || defined(linux) || defined(__linux) || defined(__FreeBSD__) extern encoder_t vaapi; #endif #ifdef __APPLE__ extern encoder_t videotoolbox; #endif struct packet_raw_t { virtual ~packet_raw_t() = default; virtual bool is_idr() = 0; virtual int64_t frame_index() = 0; virtual uint8_t *data() = 0; virtual size_t data_size() = 0; struct replace_t { std::string_view old; std::string_view _new; KITTY_DEFAULT_CONSTR_MOVE(replace_t) replace_t(std::string_view old, std::string_view _new) noexcept: old {std::move(old)}, _new {std::move(_new)} { } }; std::vector *replacements = nullptr; void *channel_data = nullptr; bool after_ref_frame_invalidation = false; // Pacing/scheduled timestamp used for transport timing. /// PyroWave record framing: frame bytes through the coarsest wavelet level, whose /// packets stream.cpp protects with parity and announces; 0 when unknown. std::size_t pyrowave_critical_bytes = 0; int pyrowave_detail_fec_percentage = 0; std::size_t pyrowave_frame_wire_budget = 0; std::optional frame_timestamp; // Raw capture/QPC-derived timestamp before pacing adjustments. std::optional capture_timestamp; std::optional host_processing_timestamp; std::chrono::steady_clock::time_point packet_enqueue_timestamp = std::chrono::steady_clock::now(); }; struct packet_raw_avcodec: packet_raw_t { packet_raw_avcodec() { av_packet = av_packet_alloc(); } ~packet_raw_avcodec() { av_packet_free(&this->av_packet); } bool is_idr() override { return av_packet->flags & AV_PKT_FLAG_KEY; } int64_t frame_index() override { return av_packet->pts; } uint8_t *data() override { return av_packet->data; } size_t data_size() override { return av_packet->size; } AVPacket *av_packet; }; struct packet_raw_generic: packet_raw_t { packet_raw_generic(std::vector &&frame_data, int64_t frame_index, bool idr): frame_data {std::move(frame_data)}, index {frame_index}, idr {idr} { } bool is_idr() override { return idr; } int64_t frame_index() override { return index; } uint8_t *data() override { return frame_data.data(); } size_t data_size() override { return frame_data.size(); } std::vector frame_data; int64_t index; bool idr; }; using packet_t = std::unique_ptr; struct hdr_info_raw_t { explicit hdr_info_raw_t(bool enabled): enabled {enabled}, metadata {} {}; explicit hdr_info_raw_t(bool enabled, const SS_HDR_METADATA &metadata): enabled {enabled}, metadata {metadata} {}; bool enabled; SS_HDR_METADATA metadata; }; using hdr_info_t = std::unique_ptr; extern int active_hevc_mode; extern int active_av1_mode; /** * PyroWave encoding on the probed capture adapter: 0 - not probed, 1 - disabled * or unsupported, 2 - available (advertised). PyroWave bypasses the encoder_t * selection, so this is independent of the chosen hardware encoder. */ extern std::atomic_int active_pyrowave_mode; extern bool last_encoder_probe_supported_ref_frames_invalidation; extern std::array last_encoder_probe_supported_yuv444_for_codec; // 0 - H.264, 1 - HEVC, 2 - AV1 bool has_attempted_encoder_probe(); bool has_successful_encoder_probe(); bool last_encoder_probe_failed(); struct advertised_encoder_capabilities_t { int hevc_mode = 0; int av1_mode = 0; int pyrowave_mode = 0; ///< Same values as active_pyrowave_mode. std::array yuv444_for_codec {}; }; advertised_encoder_capabilities_t advertised_encoder_capabilities( bool probe_before_negative = false, bool *probe_complete = nullptr ); #ifdef _WIN32 // Bridge the interval between selecting a virtual-display render adapter and // Windows publishing its replacement output. The lease prevents an older // request from clearing a newer session's identity. using encoder_probe_adapter_hint_lease_t = std::uint64_t; encoder_probe_adapter_hint_lease_t set_pending_virtual_display_adapter_hint(const LUID &adapter_luid); // Once the replacement output is published, capability identity must agree // with its WGC/DXGI adapter before a successful probe can remain cached. bool mark_pending_virtual_display_adapter_hint_ready_for_verification(encoder_probe_adapter_hint_lease_t lease); bool clear_pending_virtual_display_adapter_hint(encoder_probe_adapter_hint_lease_t lease); #endif void capture( safe::mail_t mail, config_t config, void *channel_data ); bool validate_encoder( encoder_t &encoder, bool expect_failure, const std::optional &required_adapter = std::nullopt, std::optional *actual_adapter = nullptr, const std::string &probe_display_name = std::string {} ); /** * @brief Check if we can allow probing for the encoders. * @return True if there should be no issues with the probing, false if we should prevent it. */ bool allow_encoder_probing(); /** * @brief Probe encoders and select the preferred encoder. * This is called once at startup and each time a stream is launched to * ensure the best encoder is selected. Encoder availability can change * at runtime due to all sorts of things from driver updates to eGPUs. * * @warning This is only safe to call when there is no client actively streaming. */ int probe_encoders(); // Several NTSC standard refresh rates are hardcoded here, because their // true rate requires a denominator of 1001. ffmpeg's av_d2q() would assume it could // reduce 29.97 to 2997/100 but this would be slightly wrong. We also include // support for 23.976 film in case someone wants to stream a film at the perfect // framerate. inline AVRational framerateX100_to_rational(const int framerateX100) { const auto rational = policy::framerate_x100_to_rational(framerateX100); return AVRational {rational.numerator, rational.denominator}; } } // namespace video