From 9441ca0eced3b5fc369a3e4c2c0ed69dddd74285 Mon Sep 17 00:00:00 2001 From: David Chen Date: Thu, 25 Jun 2026 09:18:08 -0700 Subject: [PATCH 1/8] av1 encode support for nvenc --- .changeset/nvenc-av1-encoding.md | 8 + webrtc-sys/build.rs | 3 +- ...on_av1_bitstream.cpp => av1_bitstream.cpp} | 2 +- ...jetson_av1_bitstream.h => av1_bitstream.h} | 6 +- webrtc-sys/src/jetson/av1_encoder_impl.cpp | 2 +- webrtc-sys/src/nvidia/av1_encoder_impl.cpp | 470 ++++++++++++++++++ webrtc-sys/src/nvidia/av1_encoder_impl.h | 94 ++++ .../src/nvidia/nvidia_encoder_factory.cpp | 208 ++++++-- .../src/nvidia/nvidia_encoder_factory.h | 2 + webrtc-sys/test/CMakeLists.txt | 4 + 10 files changed, 757 insertions(+), 42 deletions(-) create mode 100644 .changeset/nvenc-av1-encoding.md rename webrtc-sys/src/{jetson/jetson_av1_bitstream.cpp => av1_bitstream.cpp} (99%) rename webrtc-sys/src/{jetson/jetson_av1_bitstream.h => av1_bitstream.h} (94%) create mode 100644 webrtc-sys/src/nvidia/av1_encoder_impl.cpp create mode 100644 webrtc-sys/src/nvidia/av1_encoder_impl.h diff --git a/.changeset/nvenc-av1-encoding.md b/.changeset/nvenc-av1-encoding.md new file mode 100644 index 000000000..a82c8822e --- /dev/null +++ b/.changeset/nvenc-av1-encoding.md @@ -0,0 +1,8 @@ +--- +webrtc-sys: patch +libwebrtc: patch +livekit: patch +livekit-ffi: patch +--- + +Add NVIDIA NVENC AV1 encoding when the GPU reports AV1 encode support. diff --git a/webrtc-sys/build.rs b/webrtc-sys/build.rs index a7e73a6ef..0a95488b7 100644 --- a/webrtc-sys/build.rs +++ b/webrtc-sys/build.rs @@ -94,6 +94,7 @@ fn main() { "src/prohibit_libsrtp_initialization.cpp", "src/apm.cpp", "src/audio_mixer.cpp", + "src/av1_bitstream.cpp", "src/packet_trailer.cpp", "src/packet_trailer_av1.cpp", ]); @@ -231,7 +232,6 @@ fn main() { .file("src/jetson/h264_encoder_impl.cpp") .file("src/jetson/h265_encoder_impl.cpp") .file("src/jetson/av1_encoder_impl.cpp") - .file("src/jetson/jetson_av1_bitstream.cpp") .file("src/jetson/jetson_encoder_factory.cpp") .flag("-DUSE_JETSON_VIDEO_CODEC=1"); @@ -287,6 +287,7 @@ fn main() { .file("src/nvidia/NvCodec/NvCodec/NvEncoder/NvEncoderCuda.cpp") .file("src/nvidia/h264_encoder_impl.cpp") .file("src/nvidia/h265_encoder_impl.cpp") + .file("src/nvidia/av1_encoder_impl.cpp") .file("src/nvidia/h264_decoder_impl.cpp") .file("src/nvidia/h265_decoder_impl.cpp") .file("src/nvidia/nvidia_decoder_factory.cpp") diff --git a/webrtc-sys/src/jetson/jetson_av1_bitstream.cpp b/webrtc-sys/src/av1_bitstream.cpp similarity index 99% rename from webrtc-sys/src/jetson/jetson_av1_bitstream.cpp rename to webrtc-sys/src/av1_bitstream.cpp index bc3ab2206..3adff66bb 100644 --- a/webrtc-sys/src/jetson/jetson_av1_bitstream.cpp +++ b/webrtc-sys/src/av1_bitstream.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "jetson_av1_bitstream.h" +#include "av1_bitstream.h" #include diff --git a/webrtc-sys/src/jetson/jetson_av1_bitstream.h b/webrtc-sys/src/av1_bitstream.h similarity index 94% rename from webrtc-sys/src/jetson/jetson_av1_bitstream.h rename to webrtc-sys/src/av1_bitstream.h index 0af9614e0..0fa039cc7 100644 --- a/webrtc-sys/src/jetson/jetson_av1_bitstream.h +++ b/webrtc-sys/src/av1_bitstream.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef WEBRTC_JETSON_AV1_BITSTREAM_H_ -#define WEBRTC_JETSON_AV1_BITSTREAM_H_ +#ifndef WEBRTC_AV1_BITSTREAM_H_ +#define WEBRTC_AV1_BITSTREAM_H_ #include #include @@ -61,4 +61,4 @@ bool IsWebRtcParseable(const uint8_t* data, size_t len); } // namespace av1 } // namespace livekit -#endif // WEBRTC_JETSON_AV1_BITSTREAM_H_ +#endif // WEBRTC_AV1_BITSTREAM_H_ diff --git a/webrtc-sys/src/jetson/av1_encoder_impl.cpp b/webrtc-sys/src/jetson/av1_encoder_impl.cpp index 9626de4b4..7e1b9d289 100644 --- a/webrtc-sys/src/jetson/av1_encoder_impl.cpp +++ b/webrtc-sys/src/jetson/av1_encoder_impl.cpp @@ -26,7 +26,7 @@ #include "api/video/video_codec_constants.h" #include "api/video_codecs/scalability_mode.h" #include "common_video/libyuv/include/webrtc_libyuv.h" -#include "jetson_av1_bitstream.h" +#include "../av1_bitstream.h" #include "livekit/dmabuf_video_frame_buffer.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/include/video_error_codes.h" diff --git a/webrtc-sys/src/nvidia/av1_encoder_impl.cpp b/webrtc-sys/src/nvidia/av1_encoder_impl.cpp new file mode 100644 index 000000000..5ab74e300 --- /dev/null +++ b/webrtc-sys/src/nvidia/av1_encoder_impl.cpp @@ -0,0 +1,470 @@ +#include "av1_encoder_impl.h" + +#include +#include +#include + +#include "../av1_bitstream.h" +#include "i420_buffer_cuda.h" +#include "api/video/video_codec_constants.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/utility/simulcast_rate_allocator.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +// Used by histograms. Values of entries should not be changed. +enum AV1EncoderImplEvent { + kAV1EncoderEventInit = 0, + kAV1EncoderEventError = 1, + kAV1EncoderEventMax = 16, +}; + +namespace { + +uint32_t ClampToUint32(uint64_t value) { + return static_cast( + std::min(value, std::numeric_limits::max())); +} + +} // namespace + +NvidiaAV1EncoderImpl::NvidiaAV1EncoderImpl( + const webrtc::Environment& env, + CUcontext context, + CUmemorytype memory_type, + NV_ENC_BUFFER_FORMAT nv_format, + const SdpVideoFormat& format) + : env_(env), + encoder_(nullptr), + cu_context_(context), + cu_memory_type_(memory_type), + cu_scaled_array_(nullptr), + nv_format_(nv_format), + format_(format) { + RTC_CHECK_NE(cu_memory_type_, CU_MEMORYTYPE_HOST); +} + +NvidiaAV1EncoderImpl::~NvidiaAV1EncoderImpl() { + Release(); +} + +void NvidiaAV1EncoderImpl::ReportInit() { + if (has_reported_init_) { + return; + } + RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.AV1EncoderImpl.Event", + kAV1EncoderEventInit, kAV1EncoderEventMax); + has_reported_init_ = true; +} + +void NvidiaAV1EncoderImpl::ReportError() { + if (has_reported_error_) { + return; + } + RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.AV1EncoderImpl.Event", + kAV1EncoderEventError, kAV1EncoderEventMax); + has_reported_error_ = true; +} + +int32_t NvidiaAV1EncoderImpl::InitEncode( + const VideoCodec* inst, + const VideoEncoder::Settings& settings) { + (void)settings; + if (!inst || inst->codecType != kVideoCodecAV1) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->maxFramerate == 0) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->width < 1 || inst->height < 1) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->numberOfSimulcastStreams > 1) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + const auto scalability_mode = inst->GetScalabilityMode(); + if (scalability_mode.has_value() && + *scalability_mode != ScalabilityMode::kL1T1) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + int32_t release_ret = Release(); + if (release_ret != WEBRTC_VIDEO_CODEC_OK) { + ReportError(); + return release_ret; + } + + codec_ = *inst; + sent_decodable_keyframe_ = false; + cached_sequence_header_obu_.clear(); + svc_controller_ = ScalableVideoControllerNoLayering(); + if (!codec_.GetScalabilityMode().has_value()) { + codec_.SetScalabilityMode(ScalabilityMode::kL1T1); + } + + if (codec_.numberOfSimulcastStreams == 0) { + codec_.simulcastStream[0].width = codec_.width; + codec_.simulcastStream[0].height = codec_.height; + } + + const size_t new_capacity = + CalcBufferSize(VideoType::kI420, codec_.width, codec_.height); + encoded_image_.SetEncodedData(EncodedImageBuffer::Create(new_capacity)); + encoded_image_._encodedWidth = codec_.width; + encoded_image_._encodedHeight = codec_.height; + encoded_image_.set_size(0); + + configuration_.sending = false; + configuration_.frame_dropping_on = codec_.GetFrameDropEnabled(); + configuration_.key_frame_interval = + std::max(1, static_cast(codec_.maxFramerate)) * 2; + + configuration_.width = codec_.width; + configuration_.height = codec_.height; + + configuration_.max_frame_rate = codec_.maxFramerate; + configuration_.target_bps = codec_.startBitrate * 1000; + configuration_.max_bps = codec_.maxBitrate * 1000; + + const CUresult result = cuCtxSetCurrent(cu_context_); + if (result != CUDA_SUCCESS) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + + try { + if (cu_memory_type_ == CU_MEMORYTYPE_DEVICE) { + encoder_ = std::make_unique(cu_context_, codec_.width, + codec_.height, nv_format_, 0); + } else { + RTC_DCHECK_NOTREACHED(); + } + } catch (const NVENCException& e) { + RTC_LOG(LS_ERROR) << "Failed Initialize NvEncoder " << e.what(); + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + nv_initialize_params_.version = NV_ENC_INITIALIZE_PARAMS_VER; + nv_encode_config_.version = NV_ENC_CONFIG_VER; + nv_initialize_params_.encodeConfig = &nv_encode_config_; + + GUID encodeGuid = NV_ENC_CODEC_AV1_GUID; + GUID presetGuid = NV_ENC_PRESET_P4_GUID; + + encoder_->CreateDefaultEncoderParams(&nv_initialize_params_, encodeGuid, + presetGuid, + NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY); + + nv_initialize_params_.frameRateNum = + static_cast(configuration_.max_frame_rate); + nv_initialize_params_.frameRateDen = 1; + nv_initialize_params_.bufferFormat = nv_format_; + + nv_encode_config_.profileGUID = NV_ENC_AV1_PROFILE_MAIN_GUID; + nv_encode_config_.gopLength = NVENC_INFINITE_GOPLENGTH; + nv_encode_config_.frameIntervalP = 1; + nv_encode_config_.encodeCodecConfig.av1Config.level = + NV_ENC_LEVEL_AV1_AUTOSELECT; + nv_encode_config_.encodeCodecConfig.av1Config.tier = NV_ENC_TIER_AV1_0; + nv_encode_config_.encodeCodecConfig.av1Config.chromaFormatIDC = 1; + nv_encode_config_.encodeCodecConfig.av1Config.inputPixelBitDepthMinus8 = 0; + nv_encode_config_.encodeCodecConfig.av1Config.pixelBitDepthMinus8 = 0; + nv_encode_config_.encodeCodecConfig.av1Config.idrPeriod = + NVENC_INFINITE_GOPLENGTH; + nv_encode_config_.encodeCodecConfig.av1Config.repeatSeqHdr = 1; + nv_encode_config_.encodeCodecConfig.av1Config.maxTemporalLayersMinus1 = 0; + nv_encode_config_.rcParams.version = NV_ENC_RC_PARAMS_VER; + nv_encode_config_.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR; + nv_encode_config_.rcParams.averageBitRate = configuration_.target_bps; + const uint64_t vbv_buffer_size = + (static_cast(nv_encode_config_.rcParams.averageBitRate) * + nv_initialize_params_.frameRateDen / + nv_initialize_params_.frameRateNum) * + 5; + nv_encode_config_.rcParams.vbvBufferSize = + ClampToUint32(vbv_buffer_size); + nv_encode_config_.rcParams.vbvInitialDelay = + nv_encode_config_.rcParams.vbvBufferSize; + + try { + encoder_->CreateEncoder(&nv_initialize_params_); + } catch (const NVENCException& e) { + RTC_LOG(LS_ERROR) << "Failed Initialize NvEncoder " << e.what(); + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + RTC_LOG(LS_INFO) << "NVIDIA AV1 NVENC initialized: " << codec_.width << "x" + << codec_.height << " @ " << codec_.maxFramerate + << "fps, target_bps=" << configuration_.target_bps; + + ReportInit(); + + SimulcastRateAllocator init_allocator(env_, codec_); + VideoBitrateAllocation allocation = + init_allocator.Allocate(VideoBitrateAllocationParameters( + DataRate::KilobitsPerSec(codec_.startBitrate), codec_.maxFramerate)); + SetRates(RateControlParameters(allocation, codec_.maxFramerate)); + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t NvidiaAV1EncoderImpl::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + encoded_image_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t NvidiaAV1EncoderImpl::Release() { + if (encoder_) { + encoder_->DestroyEncoder(); + encoder_ = nullptr; + } + if (cu_scaled_array_) { + cuArrayDestroy(cu_scaled_array_); + cu_scaled_array_ = nullptr; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t NvidiaAV1EncoderImpl::Encode( + const VideoFrame& input_frame, + const std::vector* frame_types) { + if (!encoder_) { + ReportError(); + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (!encoded_image_callback_) { + RTC_LOG(LS_WARNING) + << "InitEncode() has been called, but a callback function " + "has not been set with RegisterEncodeCompleteCallback()"; + ReportError(); + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + webrtc::scoped_refptr frame_buffer = + input_frame.video_frame_buffer()->ToI420(); + if (!frame_buffer) { + RTC_LOG(LS_ERROR) << "Failed to convert " + << VideoFrameBufferTypeToString( + input_frame.video_frame_buffer()->type()) + << " image to I420. Can't encode frame."; + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + RTC_CHECK(frame_buffer->type() == VideoFrameBuffer::Type::kI420); + + bool is_keyframe_needed = false; + if (configuration_.key_frame_request && configuration_.sending) { + is_keyframe_needed = true; + } + if (frame_types && !frame_types->empty()) { + if ((*frame_types)[0] == VideoFrameType::kVideoFrameKey) { + is_keyframe_needed = true; + } + if ((*frame_types)[0] == VideoFrameType::kEmptyFrame) { + return WEBRTC_VIDEO_CODEC_NO_OUTPUT; + } + } + + RTC_DCHECK_EQ(configuration_.width, frame_buffer->width()); + RTC_DCHECK_EQ(configuration_.height, frame_buffer->height()); + + if (!configuration_.sending) { + return WEBRTC_VIDEO_CODEC_NO_OUTPUT; + } + + if (!sent_decodable_keyframe_) { + is_keyframe_needed = true; + } + + try { + const NvEncInputFrame* nv_enc_input_frame = encoder_->GetNextInputFrame(); + + if (cu_memory_type_ == CU_MEMORYTYPE_DEVICE) { + CopyI420BufferToDeviceFrame(cu_context_, *frame_buffer, + *nv_enc_input_frame); + } + + NV_ENC_PIC_PARAMS pic_params = NV_ENC_PIC_PARAMS(); + pic_params.version = NV_ENC_PIC_PARAMS_VER; + pic_params.encodePicFlags = 0; + if (is_keyframe_needed) { + pic_params.encodePicFlags = NV_ENC_PIC_FLAG_FORCEINTRA | + NV_ENC_PIC_FLAG_FORCEIDR | + NV_ENC_PIC_FLAG_OUTPUT_SPSPPS; + } + + std::vector> bit_stream; + encoder_->EncodeFrame(bit_stream, &pic_params); + + for (std::vector& packet : bit_stream) { + if (packet.empty()) { + RTC_LOG(LS_WARNING) + << "NVIDIA AV1 NVENC returned empty packet; skipping output."; + continue; + } + + livekit::av1::StripIvfFrameHeaderIfPresent(&packet); + if (packet.empty()) { + RTC_LOG(LS_ERROR) + << "NVIDIA AV1 NVENC packet contained only IVF framing; skipping."; + continue; + } + livekit::av1::ConvertAnnexBToLowOverheadIfPresent(&packet); + + std::vector sequence_header; + if (livekit::av1::ExtractSequenceHeaderObu( + packet.data(), packet.size(), &sequence_header)) { + cached_sequence_header_obu_ = std::move(sequence_header); + } + + const bool treat_as_keyframe = is_keyframe_needed; + if (treat_as_keyframe) { + livekit::av1::EnsureSequenceHeaderOnKeyframe( + &packet, cached_sequence_header_obu_); + } + + if (!livekit::av1::IsWebRtcParseable(packet.data(), packet.size())) { + RTC_LOG(LS_ERROR) + << "NVIDIA AV1 NVENC bitstream is not parseable by WebRTC; " + "dropping frame (size=" + << packet.size() << ")."; + continue; + } + + if (treat_as_keyframe && + livekit::av1::HasSequenceHeaderObu(packet.data(), packet.size())) { + sent_decodable_keyframe_ = true; + configuration_.key_frame_request = false; + } else if (!sent_decodable_keyframe_) { + RTC_LOG(LS_WARNING) + << "NVIDIA AV1 NVENC keyframe missing sequence header OBU; " + "continuing to force keyframes."; + configuration_.key_frame_request = true; + } else if (is_keyframe_needed) { + configuration_.key_frame_request = false; + } + + int32_t result = + ProcessEncodedFrame(packet, input_frame, treat_as_keyframe); + if (result != WEBRTC_VIDEO_CODEC_OK) { + return result; + } + } + } catch (const NVENCException& e) { + RTC_LOG(LS_ERROR) << "Failed EncodeFrame NvEncoder " << e.what(); + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t NvidiaAV1EncoderImpl::ProcessEncodedFrame( + std::vector& packet, + const ::webrtc::VideoFrame& input_frame, + bool is_keyframe) { + encoded_image_._encodedWidth = encoder_->GetEncodeWidth(); + encoded_image_._encodedHeight = encoder_->GetEncodeHeight(); + encoded_image_.SetRtpTimestamp(input_frame.rtp_timestamp()); + encoded_image_.SetSimulcastIndex(0); + encoded_image_.ntp_time_ms_ = input_frame.ntp_time_ms(); + encoded_image_.capture_time_ms_ = input_frame.render_time_ms(); + encoded_image_.rotation_ = input_frame.rotation(); + encoded_image_.content_type_ = VideoContentType::UNSPECIFIED; + encoded_image_.timing_.flags = VideoSendTiming::kInvalid; + encoded_image_._frameType = + is_keyframe ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta; + encoded_image_.SetColorSpace(input_frame.color_space()); + + encoded_image_.SetEncodedData( + EncodedImageBuffer::Create(packet.data(), packet.size())); + encoded_image_.set_size(packet.size()); + + encoded_image_.qp_ = -1; + + CodecSpecificInfo codecInfo; + codecInfo.codecType = kVideoCodecAV1; + codecInfo.end_of_picture = true; + codecInfo.scalability_mode = ScalabilityMode::kL1T1; + + std::vector layer_frames = + svc_controller_.NextFrameConfig(/*restart=*/is_keyframe); + if (!layer_frames.empty()) { + const ScalableVideoController::LayerFrameConfig& layer_frame = + layer_frames.front(); + codecInfo.generic_frame_info = svc_controller_.OnEncodeDone(layer_frame); + if (layer_frame.IsKeyframe()) { + codecInfo.template_structure = svc_controller_.DependencyStructure(); + } + } + + const auto result = + encoded_image_callback_->OnEncodedImage(encoded_image_, &codecInfo); + if (result.error != EncodedImageCallback::Result::OK) { + RTC_LOG(LS_ERROR) << "Encode m_encodedCompleteCallback failed " + << result.error; + return WEBRTC_VIDEO_CODEC_ERROR; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +VideoEncoder::EncoderInfo NvidiaAV1EncoderImpl::GetEncoderInfo() const { + EncoderInfo info; + info.supports_native_handle = false; + info.implementation_name = "NVIDIA AV1 Encoder"; + info.scaling_settings = VideoEncoder::ScalingSettings::kOff; + info.is_hardware_accelerated = true; + info.supports_simulcast = false; + info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420}; + return info; +} + +void NvidiaAV1EncoderImpl::SetRates( + const RateControlParameters& parameters) { + if (!encoder_) { + RTC_LOG(LS_WARNING) << "SetRates() while uninitialized."; + return; + } + + if (parameters.framerate_fps < 1.0) { + RTC_LOG(LS_WARNING) << "Invalid frame rate: " << parameters.framerate_fps; + return; + } + + if (parameters.bitrate.get_sum_bps() == 0) { + configuration_.SetStreamState(false); + return; + } + + codec_.maxFramerate = static_cast(parameters.framerate_fps); + codec_.maxBitrate = parameters.bitrate.GetSpatialLayerSum(0); + + configuration_.target_bps = parameters.bitrate.GetSpatialLayerSum(0); + configuration_.max_frame_rate = parameters.framerate_fps; + + if (configuration_.target_bps) { + configuration_.SetStreamState(true); + } else { + configuration_.SetStreamState(false); + } +} + +void NvidiaAV1EncoderImpl::LayerConfig::SetStreamState(bool send_stream) { + if (send_stream && !sending) { + key_frame_request = true; + } + sending = send_stream; +} + +} // namespace webrtc diff --git a/webrtc-sys/src/nvidia/av1_encoder_impl.h b/webrtc-sys/src/nvidia/av1_encoder_impl.h new file mode 100644 index 000000000..93c0a63e0 --- /dev/null +++ b/webrtc-sys/src/nvidia/av1_encoder_impl.h @@ -0,0 +1,94 @@ +#ifndef WEBRTC_NVIDIA_AV1_ENCODER_IMPL_H_ +#define WEBRTC_NVIDIA_AV1_ENCODER_IMPL_H_ + +#include + +#include +#include + +#include "NvEncoder/NvEncoder.h" +#include "NvEncoder/NvEncoderCuda.h" + +#include "api/environment/environment.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_codec_constants.h" +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/svc/scalable_video_controller_no_layering.h" + +namespace webrtc { + +class NvidiaAV1EncoderImpl : public VideoEncoder { + public: + struct LayerConfig { + int simulcast_idx = 0; + int width = -1; + int height = -1; + bool sending = true; + bool key_frame_request = false; + float max_frame_rate = 0; + uint32_t target_bps = 0; + uint32_t max_bps = 0; + bool frame_dropping_on = false; + int key_frame_interval = 0; + int num_temporal_layers = 1; + + void SetStreamState(bool send_stream); + }; + + public: + NvidiaAV1EncoderImpl(const webrtc::Environment& env, + CUcontext context, + CUmemorytype memory_type, + NV_ENC_BUFFER_FORMAT nv_format, + const SdpVideoFormat& format); + ~NvidiaAV1EncoderImpl() override; + + int32_t InitEncode(const VideoCodec* codec_settings, + const Settings& settings) override; + + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override; + + int32_t Release() override; + + int32_t Encode(const VideoFrame& frame, + const std::vector* frame_types) override; + + void SetRates(const RateControlParameters& rc_parameters) override; + + EncoderInfo GetEncoderInfo() const override; + + private: + int32_t ProcessEncodedFrame(std::vector& packet, + const ::webrtc::VideoFrame& input_frame, + bool is_keyframe); + + const webrtc::Environment& env_; + EncodedImageCallback* encoded_image_callback_ = nullptr; + + std::unique_ptr encoder_; + CUcontext cu_context_; + CUmemorytype cu_memory_type_; + CUarray cu_scaled_array_; + NV_ENC_BUFFER_FORMAT nv_format_; + NV_ENC_INITIALIZE_PARAMS nv_initialize_params_; + NV_ENC_CONFIG nv_encode_config_; + + LayerConfig configuration_; + EncodedImage encoded_image_; + VideoCodec codec_; + void ReportInit(); + void ReportError(); + bool has_reported_init_ = false; + bool has_reported_error_ = false; + bool sent_decodable_keyframe_ = false; + std::vector cached_sequence_header_obu_; + ScalableVideoControllerNoLayering svc_controller_; + const SdpVideoFormat format_; +}; + +} // namespace webrtc + +#endif // WEBRTC_NVIDIA_AV1_ENCODER_IMPL_H_ diff --git a/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp b/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp index fe2e38752..353d968fa 100644 --- a/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp +++ b/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp @@ -1,7 +1,15 @@ #include "nvidia_encoder_factory.h" +#include +#include +#include #include +#include +#include +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/scalability_mode.h" +#include "av1_encoder_impl.h" #include "cuda_context.h" #include "h264_encoder_impl.h" #include "h265_encoder_impl.h" @@ -12,35 +20,106 @@ namespace webrtc { -NvidiaVideoEncoderFactory::NvidiaVideoEncoderFactory() { - std::map baselineParameters = { - {"profile-level-id", "42e01f"}, - {"level-asymmetry-allowed", "1"}, - {"packetization-mode", "1"}, - }; - supported_formats_.push_back(SdpVideoFormat("H264", baselineParameters)); +namespace { - // Advertise HEVC/H265 with default parameters. - supported_formats_.push_back(SdpVideoFormat("H265")); - // Some stacks use 'HEVC' name. - supported_formats_.push_back(SdpVideoFormat("HEVC")); +struct NvencProbeResult { + bool encoder_supported = false; + bool av1_supported = false; +}; - /*std::map highParameters = { - {"profile-level-id", "4d0032"}, - {"level-asymmetry-allowed", "1"}, - {"packetization-mode", "1"}, - }; +bool GuidEquals(const GUID& lhs, const GUID& rhs) { + return std::memcmp(&lhs, &rhs, sizeof(GUID)) == 0; +} - supported_formats_.push_back(SdpVideoFormat("H264", highParameters)); - */ +bool SupportsEncodeGuid(NV_ENCODE_API_FUNCTION_LIST& fnList, + void* hEncoder, + const GUID& encodeGuid) { + uint32_t guid_count = 0; + NVENCSTATUS nvStatus = + fnList.nvEncGetEncodeGUIDCount(hEncoder, &guid_count); + if (nvStatus != NV_ENC_SUCCESS || guid_count == 0) { + return false; + } + + std::vector guids(guid_count); + uint32_t written_guid_count = 0; + nvStatus = fnList.nvEncGetEncodeGUIDs(hEncoder, guids.data(), guid_count, + &written_guid_count); + if (nvStatus != NV_ENC_SUCCESS) { + return false; + } + + written_guid_count = std::min(written_guid_count, guid_count); + return std::any_of(guids.begin(), guids.begin() + written_guid_count, + [&](const GUID& guid) { + return GuidEquals(guid, encodeGuid); + }); } -NvidiaVideoEncoderFactory::~NvidiaVideoEncoderFactory() {} +bool SupportsInputFormat(NV_ENCODE_API_FUNCTION_LIST& fnList, + void* hEncoder, + const GUID& encodeGuid, + NV_ENC_BUFFER_FORMAT required_format) { + uint32_t format_count = 0; + NVENCSTATUS nvStatus = + fnList.nvEncGetInputFormatCount(hEncoder, encodeGuid, &format_count); + if (nvStatus != NV_ENC_SUCCESS || format_count == 0) { + return false; + } -bool NvidiaVideoEncoderFactory::IsSupported() { + std::vector formats(format_count); + uint32_t written_format_count = 0; + nvStatus = fnList.nvEncGetInputFormats( + hEncoder, encodeGuid, formats.data(), format_count, &written_format_count); + if (nvStatus != NV_ENC_SUCCESS) { + return false; + } + + written_format_count = std::min(written_format_count, format_count); + return std::any_of(formats.begin(), formats.begin() + written_format_count, + [&](NV_ENC_BUFFER_FORMAT format) { + return format == required_format; + }); +} + +bool SupportsPositiveDimensionCaps(NV_ENCODE_API_FUNCTION_LIST& fnList, + void* hEncoder, + const GUID& encodeGuid) { + NV_ENC_CAPS_PARAM capsParam = {NV_ENC_CAPS_PARAM_VER}; + + int max_width = 0; + capsParam.capsToQuery = NV_ENC_CAPS_WIDTH_MAX; + if (fnList.nvEncGetEncodeCaps(hEncoder, encodeGuid, &capsParam, + &max_width) != NV_ENC_SUCCESS || + max_width <= 0) { + return false; + } + + int max_height = 0; + capsParam.capsToQuery = NV_ENC_CAPS_HEIGHT_MAX; + if (fnList.nvEncGetEncodeCaps(hEncoder, encodeGuid, &capsParam, + &max_height) != NV_ENC_SUCCESS || + max_height <= 0) { + return false; + } + + return true; +} + +bool SupportsAv1Encoding(NV_ENCODE_API_FUNCTION_LIST& fnList, + void* hEncoder) { + return SupportsEncodeGuid(fnList, hEncoder, NV_ENC_CODEC_AV1_GUID) && + SupportsInputFormat(fnList, hEncoder, NV_ENC_CODEC_AV1_GUID, + NV_ENC_BUFFER_FORMAT_IYUV) && + SupportsPositiveDimensionCaps(fnList, hEncoder, + NV_ENC_CODEC_AV1_GUID); +} + +NvencProbeResult ProbeNvencSupport() { + NvencProbeResult result; if (!livekit_ffi::CudaContext::IsAvailable()) { RTC_LOG(LS_WARNING) << "CUDA is not available, NVENC disabled."; - return false; + return result; } // CUDA being available does NOT imply NVENC is present. Compute-only GPUs @@ -50,8 +129,9 @@ bool NvidiaVideoEncoderFactory::IsSupported() { if (!hModule) { RTC_LOG(LS_WARNING) << "NVENC library (libnvidia-encode.so.1) not found, " "hardware encoding unavailable."; - return false; + return result; } + auto NvEncodeAPIGetMaxSupportedVersion = (NVENCSTATUS(NVENCAPI*)(uint32_t*))dlsym( hModule, "NvEncodeAPIGetMaxSupportedVersion"); @@ -59,7 +139,9 @@ bool NvidiaVideoEncoderFactory::IsSupported() { (NVENCSTATUS(NVENCAPI*)(NV_ENCODE_API_FUNCTION_LIST*))dlsym( hModule, "NvEncodeAPICreateInstance"); - bool supported = false; + NV_ENCODE_API_FUNCTION_LIST fnList = {NV_ENCODE_API_FUNCTION_LIST_VER}; + CUcontext cuCtx = nullptr; + void* hEncoder = nullptr; do { if (!NvEncodeAPIGetMaxSupportedVersion || !NvEncodeAPICreateInstance) { @@ -81,14 +163,11 @@ bool NvidiaVideoEncoderFactory::IsSupported() { break; } - NV_ENCODE_API_FUNCTION_LIST fnList = {NV_ENCODE_API_FUNCTION_LIST_VER}; if (NvEncodeAPICreateInstance(&fnList) != NV_ENC_SUCCESS) { RTC_LOG(LS_WARNING) << "NvEncodeAPICreateInstance failed."; break; } - // Try opening an encode session with CUDA device 0 to confirm the GPU - // actually has NVENC hardware. CUresult cuRes = cuInit(0); if (cuRes != CUDA_SUCCESS) { RTC_LOG(LS_WARNING) << "cuInit failed during NVENC probe."; @@ -102,7 +181,6 @@ bool NvidiaVideoEncoderFactory::IsSupported() { break; } - CUcontext cuCtx = nullptr; #if CUDA_VERSION >= 13000 cuRes = cuCtxCreate(&cuCtx, nullptr, 0, cuDevice); #else @@ -119,30 +197,81 @@ bool NvidiaVideoEncoderFactory::IsSupported() { sessionParams.deviceType = NV_ENC_DEVICE_TYPE_CUDA; sessionParams.apiVersion = NVENCAPI_VERSION; - void* hEncoder = nullptr; NVENCSTATUS nvStatus = fnList.nvEncOpenEncodeSessionEx(&sessionParams, &hEncoder); - - if (nvStatus == NV_ENC_SUCCESS && hEncoder) { - fnList.nvEncDestroyEncoder(hEncoder); - supported = true; - } else { + if (nvStatus != NV_ENC_SUCCESS || !hEncoder) { char deviceName[80] = {}; cuDeviceGetName(deviceName, sizeof(deviceName), cuDevice); RTC_LOG(LS_WARNING) << "NVENC not available on GPU \"" << deviceName << "\" (status=" << nvStatus << "). This GPU likely lacks encode hardware."; + break; } - cuCtxDestroy(cuCtx); + result.encoder_supported = true; + result.av1_supported = SupportsAv1Encoding(fnList, hEncoder); } while (false); + if (hEncoder) { + fnList.nvEncDestroyEncoder(hEncoder); + } + if (cuCtx) { + cuCtxDestroy(cuCtx); + } dlclose(hModule); - if (supported) { + return result; +} + +} // namespace + +NvidiaVideoEncoderFactory::NvidiaVideoEncoderFactory() { + std::map baselineParameters = { + {"profile-level-id", "42e01f"}, + {"level-asymmetry-allowed", "1"}, + {"packetization-mode", "1"}, + }; + supported_formats_.push_back(SdpVideoFormat("H264", baselineParameters)); + + // Advertise HEVC/H265 with default parameters. + supported_formats_.push_back(SdpVideoFormat("H265")); + // Some stacks use 'HEVC' name. + supported_formats_.push_back(SdpVideoFormat("HEVC")); + + if (IsAv1Supported()) { + absl::InlinedVector + scalability_modes; + scalability_modes.push_back(ScalabilityMode::kL1T1); + supported_formats_.push_back( + SdpVideoFormat(SdpVideoFormat::AV1Profile0(), scalability_modes)); + RTC_LOG(LS_INFO) << "NVIDIA AV1 NVENC encoder is available."; + } else { + RTC_LOG(LS_INFO) + << "NVIDIA AV1 NVENC encoder is not supported on this GPU."; + } + + /*std::map highParameters = { + {"profile-level-id", "4d0032"}, + {"level-asymmetry-allowed", "1"}, + {"packetization-mode", "1"}, + }; + + supported_formats_.push_back(SdpVideoFormat("H264", highParameters)); + */ +} + +NvidiaVideoEncoderFactory::~NvidiaVideoEncoderFactory() {} + +bool NvidiaVideoEncoderFactory::IsSupported() { + const NvencProbeResult probe = ProbeNvencSupport(); + if (probe.encoder_supported) { RTC_LOG(LS_INFO) << "NVIDIA NVENC hardware encoder is available."; } - return supported; + return probe.encoder_supported; +} + +bool NvidiaVideoEncoderFactory::IsAv1Supported() { + return ProbeNvencSupport().av1_supported; } std::unique_ptr NvidiaVideoEncoderFactory::Create( @@ -172,6 +301,13 @@ std::unique_ptr NvidiaVideoEncoderFactory::Create( env, cu_context_->GetContext(), CU_MEMORYTYPE_DEVICE, NV_ENC_BUFFER_FORMAT_IYUV, format); } + + if (format.name == "AV1") { + RTC_LOG(LS_INFO) << "Using NVIDIA HW encoder (NVENC) for AV1"; + return std::make_unique( + env, cu_context_->GetContext(), CU_MEMORYTYPE_DEVICE, + NV_ENC_BUFFER_FORMAT_IYUV, format); + } } } return nullptr; diff --git a/webrtc-sys/src/nvidia/nvidia_encoder_factory.h b/webrtc-sys/src/nvidia/nvidia_encoder_factory.h index fd0391006..87f5a29b4 100644 --- a/webrtc-sys/src/nvidia/nvidia_encoder_factory.h +++ b/webrtc-sys/src/nvidia/nvidia_encoder_factory.h @@ -33,6 +33,8 @@ class NvidiaVideoEncoderFactory : public VideoEncoderFactory { } private: + static bool IsAv1Supported(); + std::vector supported_formats_; livekit_ffi::CudaContext* cu_context_ = nullptr; }; diff --git a/webrtc-sys/test/CMakeLists.txt b/webrtc-sys/test/CMakeLists.txt index ed47a648e..a3a43f55e 100644 --- a/webrtc-sys/test/CMakeLists.txt +++ b/webrtc-sys/test/CMakeLists.txt @@ -67,8 +67,12 @@ add_executable(${BINARY_NAME} "../src/nvidia/NvCodec/NvCodec/NvDecoder/NvDecoder.cpp" "../src/nvidia/NvCodec/NvCodec/NvEncoder/NvEncoder.cpp" "../src/nvidia/NvCodec/NvCodec/NvEncoder/NvEncoderCuda.cpp" + "../src/av1_bitstream.cpp" "../src/nvidia/h264_encoder_impl.cpp" + "../src/nvidia/h265_encoder_impl.cpp" + "../src/nvidia/av1_encoder_impl.cpp" "../src/nvidia/h264_decoder_impl.cpp" + "../src/nvidia/h265_decoder_impl.cpp" "../src/nvidia/nvidia_decoder_factory.cpp" "../src/nvidia/nvidia_encoder_factory.cpp" "../src/nvidia/cuda_context.cpp" From bebba308afe73c5e939b232cbca33c4e315ab3fc Mon Sep 17 00:00:00 2001 From: David Chen Date: Thu, 25 Jun 2026 15:52:12 -0700 Subject: [PATCH 2/8] add linux build instructions --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index f865afcd1..a0a17c1bd 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,29 @@ match event { ## Building +### Linux + +Building on Ubuntu 24 x86_64: + +``` +# install required libs +sudo apt install -y \ + libglib2.0-dev build-essential clang \ + libclang-dev libc6-dev pkg-config libjpeg-turbo8-dev + +# install cuda-toolkit if you have an Nvidia GPU and want to use NVENC for video encoding +wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb +sudo dpkg -i cuda-keyring_1.1-1_all.deb +sudo apt update +sudo apt install -y cuda-toolkit + +# ensure CUDA_HOME env var is set +export CUDA_HOME="$(dirname "$(dirname "$(sudo find /usr/local /usr -path '*/include/cuda.h' -print 2>/dev/null | grep -v '/linux/' | sort -V | tail -1)")")" + +cargo build + +``` + ### MacOS When building on MacOS, `-ObjC` linker flag is needed. LiveKit's WebRTC implementation make use of ObjectiveC libraries on the Mac. You may get the following error if the app isn't linked with ObjC: From 206a0299d5a5e6b1a605406a1350dc4e5606e651 Mon Sep 17 00:00:00 2001 From: David Chen Date: Thu, 25 Jun 2026 15:54:33 -0700 Subject: [PATCH 3/8] add AV1 to encoder support matrix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a0a17c1bd..b416d44bb 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Use this SDK to add realtime video, audio and data features to your Rust app. By - [x] Dynacast - [x] Hardware video enc/dec - [x] H.264, H.265 using VideoToolbox (MacOS/iOS) - - [x] H.264, H.265 on NVidia discrete GPUs (Linux) + - [x] H.264, H.265, AV1 on NVidia discrete GPUs (Linux) - [x] H.264, H.265 on AMD CPUs & GPUs (Linux) - [x] H.264, H.265, AV1 on NVidia Jetson (Linux) - Supported Platforms From eb4e318156a7696f5587094f1f8c5c53b5daf9bc Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 26 Jun 2026 10:43:57 -0700 Subject: [PATCH 4/8] remove comment --- webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp b/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp index 353d968fa..d3c825185 100644 --- a/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp +++ b/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp @@ -249,15 +249,6 @@ NvidiaVideoEncoderFactory::NvidiaVideoEncoderFactory() { RTC_LOG(LS_INFO) << "NVIDIA AV1 NVENC encoder is not supported on this GPU."; } - - /*std::map highParameters = { - {"profile-level-id", "4d0032"}, - {"level-asymmetry-allowed", "1"}, - {"packetization-mode", "1"}, - }; - - supported_formats_.push_back(SdpVideoFormat("H264", highParameters)); - */ } NvidiaVideoEncoderFactory::~NvidiaVideoEncoderFactory() {} From 6ff987eaab3d7fa404522fc419ba4893be25090b Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 26 Jun 2026 10:48:10 -0700 Subject: [PATCH 5/8] new function to cache ProbeNvencSupport result to avoid probing multiple times --- .../src/nvidia/nvidia_encoder_factory.cpp | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp b/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp index d3c825185..ff1cd232a 100644 --- a/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp +++ b/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp @@ -253,16 +253,27 @@ NvidiaVideoEncoderFactory::NvidiaVideoEncoderFactory() { NvidiaVideoEncoderFactory::~NvidiaVideoEncoderFactory() {} +namespace { + +const NvencProbeResult& CachedNvencProbe() { + static const NvencProbeResult probe = [] { + NvencProbeResult result = ProbeNvencSupport(); + RTC_LOG(LS_INFO) << "NVIDIA NVENC hardware encoder " + << (result.encoder_supported ? "is available." + : "is not available."); + return result; + }(); + return probe; +} + +} // namespace + bool NvidiaVideoEncoderFactory::IsSupported() { - const NvencProbeResult probe = ProbeNvencSupport(); - if (probe.encoder_supported) { - RTC_LOG(LS_INFO) << "NVIDIA NVENC hardware encoder is available."; - } - return probe.encoder_supported; + return CachedNvencProbe().encoder_supported; } bool NvidiaVideoEncoderFactory::IsAv1Supported() { - return ProbeNvencSupport().av1_supported; + return CachedNvencProbe().av1_supported; } std::unique_ptr NvidiaVideoEncoderFactory::Create( From e73ad086e08739970505ca0756a19d83be6378ca Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 26 Jun 2026 11:13:04 -0700 Subject: [PATCH 6/8] make checks more robust --- webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp b/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp index ff1cd232a..3e095d579 100644 --- a/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp +++ b/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp @@ -45,7 +45,7 @@ bool SupportsEncodeGuid(NV_ENCODE_API_FUNCTION_LIST& fnList, uint32_t written_guid_count = 0; nvStatus = fnList.nvEncGetEncodeGUIDs(hEncoder, guids.data(), guid_count, &written_guid_count); - if (nvStatus != NV_ENC_SUCCESS) { + if (nvStatus != NV_ENC_SUCCESS || written_guid_count == 0) { return false; } @@ -71,7 +71,7 @@ bool SupportsInputFormat(NV_ENCODE_API_FUNCTION_LIST& fnList, uint32_t written_format_count = 0; nvStatus = fnList.nvEncGetInputFormats( hEncoder, encodeGuid, formats.data(), format_count, &written_format_count); - if (nvStatus != NV_ENC_SUCCESS) { + if (nvStatus != NV_ENC_SUCCESS || written_format_count == 0) { return false; } From 486a153043a8917ea2fab8b2f569f1fe579dcf7c Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 26 Jun 2026 11:22:42 -0700 Subject: [PATCH 7/8] use const where applicable, protect against div by 0 on frame rate --- webrtc-sys/src/nvidia/av1_encoder_impl.cpp | 7 ++++--- webrtc-sys/src/nvidia/av1_encoder_impl.h | 10 +++++----- webrtc-sys/src/nvidia/h264_encoder_impl.cpp | 5 +++-- webrtc-sys/src/nvidia/h265_encoder_impl.cpp | 5 +++-- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/webrtc-sys/src/nvidia/av1_encoder_impl.cpp b/webrtc-sys/src/nvidia/av1_encoder_impl.cpp index 5ab74e300..3a88ceb3e 100644 --- a/webrtc-sys/src/nvidia/av1_encoder_impl.cpp +++ b/webrtc-sys/src/nvidia/av1_encoder_impl.cpp @@ -1,6 +1,7 @@ #include "av1_encoder_impl.h" #include +#include #include #include @@ -166,8 +167,8 @@ int32_t NvidiaAV1EncoderImpl::InitEncode( presetGuid, NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY); - nv_initialize_params_.frameRateNum = - static_cast(configuration_.max_frame_rate); + nv_initialize_params_.frameRateNum = std::max( + 1, static_cast(std::round(configuration_.max_frame_rate))); nv_initialize_params_.frameRateDen = 1; nv_initialize_params_.bufferFormat = nv_format_; @@ -370,7 +371,7 @@ int32_t NvidiaAV1EncoderImpl::Encode( } int32_t NvidiaAV1EncoderImpl::ProcessEncodedFrame( - std::vector& packet, + const std::vector& packet, const ::webrtc::VideoFrame& input_frame, bool is_keyframe) { encoded_image_._encodedWidth = encoder_->GetEncodeWidth(); diff --git a/webrtc-sys/src/nvidia/av1_encoder_impl.h b/webrtc-sys/src/nvidia/av1_encoder_impl.h index 93c0a63e0..f4c26a99e 100644 --- a/webrtc-sys/src/nvidia/av1_encoder_impl.h +++ b/webrtc-sys/src/nvidia/av1_encoder_impl.h @@ -61,7 +61,7 @@ class NvidiaAV1EncoderImpl : public VideoEncoder { EncoderInfo GetEncoderInfo() const override; private: - int32_t ProcessEncodedFrame(std::vector& packet, + int32_t ProcessEncodedFrame(const std::vector& packet, const ::webrtc::VideoFrame& input_frame, bool is_keyframe); @@ -69,10 +69,10 @@ class NvidiaAV1EncoderImpl : public VideoEncoder { EncodedImageCallback* encoded_image_callback_ = nullptr; std::unique_ptr encoder_; - CUcontext cu_context_; - CUmemorytype cu_memory_type_; - CUarray cu_scaled_array_; - NV_ENC_BUFFER_FORMAT nv_format_; + const CUcontext cu_context_; + const CUmemorytype cu_memory_type_; + CUarray cu_scaled_array_ = nullptr; + const NV_ENC_BUFFER_FORMAT nv_format_; NV_ENC_INITIALIZE_PARAMS nv_initialize_params_; NV_ENC_CONFIG nv_encode_config_; diff --git a/webrtc-sys/src/nvidia/h264_encoder_impl.cpp b/webrtc-sys/src/nvidia/h264_encoder_impl.cpp index dde54f41e..7024c6bae 100644 --- a/webrtc-sys/src/nvidia/h264_encoder_impl.cpp +++ b/webrtc-sys/src/nvidia/h264_encoder_impl.cpp @@ -1,6 +1,7 @@ #include "h264_encoder_impl.h" #include +#include #include #include @@ -209,8 +210,8 @@ int32_t NvidiaH264EncoderImpl::InitEncode( presetGuid, NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY); - nv_initialize_params_.frameRateNum = - static_cast(configuration_.max_frame_rate); + nv_initialize_params_.frameRateNum = std::max( + 1, static_cast(std::round(configuration_.max_frame_rate))); nv_initialize_params_.frameRateDen = 1; nv_initialize_params_.bufferFormat = nv_format_; diff --git a/webrtc-sys/src/nvidia/h265_encoder_impl.cpp b/webrtc-sys/src/nvidia/h265_encoder_impl.cpp index fff286bf2..55ed56ba6 100644 --- a/webrtc-sys/src/nvidia/h265_encoder_impl.cpp +++ b/webrtc-sys/src/nvidia/h265_encoder_impl.cpp @@ -1,6 +1,7 @@ #include "h265_encoder_impl.h" #include +#include #include #include @@ -142,8 +143,8 @@ int32_t NvidiaH265EncoderImpl::InitEncode( presetGuid, NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY); - nv_initialize_params_.frameRateNum = - static_cast(configuration_.max_frame_rate); + nv_initialize_params_.frameRateNum = std::max( + 1, static_cast(std::round(configuration_.max_frame_rate))); nv_initialize_params_.frameRateDen = 1; nv_initialize_params_.bufferFormat = nv_format_; From c36328937d8d6cafdca63e1fbfa677be14a54e07 Mon Sep 17 00:00:00 2001 From: David Chen Date: Wed, 1 Jul 2026 08:23:14 -0700 Subject: [PATCH 8/8] add SetRate to nvenc encoder implementations (#1194) --- .../NvCodec/NvCodec/NvEncoder/NvEncoder.cpp | 32 +++++++++++++++++++ .../NvCodec/NvCodec/NvEncoder/NvEncoder.h | 7 ++++ webrtc-sys/src/nvidia/av1_encoder_impl.cpp | 4 +++ webrtc-sys/src/nvidia/h264_encoder_impl.cpp | 4 +++ webrtc-sys/src/nvidia/h265_encoder_impl.cpp | 4 +++ 5 files changed, 51 insertions(+) diff --git a/webrtc-sys/src/nvidia/NvCodec/NvCodec/NvEncoder/NvEncoder.cpp b/webrtc-sys/src/nvidia/NvCodec/NvCodec/NvEncoder/NvEncoder.cpp index 4a530b87b..9351680f6 100644 --- a/webrtc-sys/src/nvidia/NvCodec/NvCodec/NvEncoder/NvEncoder.cpp +++ b/webrtc-sys/src/nvidia/NvCodec/NvCodec/NvEncoder/NvEncoder.cpp @@ -683,6 +683,38 @@ bool NvEncoder::Reconfigure( return true; } +bool NvEncoder::SetRates(uint32_t frameRate, uint32_t averageBitrate) { + if (!IsHWEncoderInitialized()) { + return false; + } + if (frameRate == 0) { + frameRate = 1; + } + + NV_ENC_RECONFIGURE_PARAMS reconfigureParams = {}; + reconfigureParams.version = NV_ENC_RECONFIGURE_PARAMS_VER; + + NV_ENC_CONFIG encodeConfig = {}; + encodeConfig.version = NV_ENC_CONFIG_VER; + reconfigureParams.reInitEncodeParams.version = NV_ENC_INITIALIZE_PARAMS_VER; + reconfigureParams.reInitEncodeParams.encodeConfig = &encodeConfig; + + GetInitializeParams(&reconfigureParams.reInitEncodeParams); + + reconfigureParams.reInitEncodeParams.frameRateNum = frameRate; + reconfigureParams.reInitEncodeParams.frameRateDen = 1; + + encodeConfig.rcParams.averageBitRate = averageBitrate; + encodeConfig.rcParams.vbvBufferSize = (averageBitrate / frameRate) * 5; + encodeConfig.rcParams.vbvInitialDelay = encodeConfig.rcParams.vbvBufferSize; + + try { + return Reconfigure(&reconfigureParams); + } catch (const NVENCException&) { + return false; + } +} + NV_ENC_REGISTERED_PTR NvEncoder::RegisterResource( void* pBuffer, NV_ENC_INPUT_RESOURCE_TYPE eResourceType, diff --git a/webrtc-sys/src/nvidia/NvCodec/NvCodec/NvEncoder/NvEncoder.h b/webrtc-sys/src/nvidia/NvCodec/NvCodec/NvEncoder/NvEncoder.h index 98c17911d..5eb8871f5 100644 --- a/webrtc-sys/src/nvidia/NvCodec/NvCodec/NvEncoder/NvEncoder.h +++ b/webrtc-sys/src/nvidia/NvCodec/NvCodec/NvEncoder/NvEncoder.h @@ -114,6 +114,13 @@ class NvEncoder { */ bool Reconfigure(const NV_ENC_RECONFIGURE_PARAMS* pReconfigureParams); + /** + * @brief Dynamically updates the encode bitrate and framerate on the live + * session. Returns false if the encoder is not initialized or reconfigure + * fails. + */ + bool SetRates(uint32_t frameRate, uint32_t averageBitrate); + /** * @brief This function is used to get the next available input buffer. * Applications must call this function to obtain a pointer to the next diff --git a/webrtc-sys/src/nvidia/av1_encoder_impl.cpp b/webrtc-sys/src/nvidia/av1_encoder_impl.cpp index 3a88ceb3e..3ce3ee943 100644 --- a/webrtc-sys/src/nvidia/av1_encoder_impl.cpp +++ b/webrtc-sys/src/nvidia/av1_encoder_impl.cpp @@ -454,6 +454,10 @@ void NvidiaAV1EncoderImpl::SetRates( configuration_.target_bps = parameters.bitrate.GetSpatialLayerSum(0); configuration_.max_frame_rate = parameters.framerate_fps; + if (!encoder_->SetRates(codec_.maxFramerate, configuration_.target_bps)) { + RTC_LOG(LS_WARNING) << "Failed to reconfigure NVENC rates."; + } + if (configuration_.target_bps) { configuration_.SetStreamState(true); } else { diff --git a/webrtc-sys/src/nvidia/h264_encoder_impl.cpp b/webrtc-sys/src/nvidia/h264_encoder_impl.cpp index 7024c6bae..bc56f8264 100644 --- a/webrtc-sys/src/nvidia/h264_encoder_impl.cpp +++ b/webrtc-sys/src/nvidia/h264_encoder_impl.cpp @@ -439,6 +439,10 @@ void NvidiaH264EncoderImpl::SetRates( configuration_.target_bps = parameters.bitrate.GetSpatialLayerSum(0); configuration_.max_frame_rate = parameters.framerate_fps; + if (!encoder_->SetRates(codec_.maxFramerate, configuration_.target_bps)) { + RTC_LOG(LS_WARNING) << "Failed to reconfigure NVENC rates."; + } + if (configuration_.target_bps) { configuration_.SetStreamState(true); } else { diff --git a/webrtc-sys/src/nvidia/h265_encoder_impl.cpp b/webrtc-sys/src/nvidia/h265_encoder_impl.cpp index 55ed56ba6..8d65a1fb1 100644 --- a/webrtc-sys/src/nvidia/h265_encoder_impl.cpp +++ b/webrtc-sys/src/nvidia/h265_encoder_impl.cpp @@ -360,6 +360,10 @@ void NvidiaH265EncoderImpl::SetRates( configuration_.target_bps = parameters.bitrate.GetSpatialLayerSum(0); configuration_.max_frame_rate = parameters.framerate_fps; + if (!encoder_->SetRates(codec_.maxFramerate, configuration_.target_bps)) { + RTC_LOG(LS_WARNING) << "Failed to reconfigure NVENC rates."; + } + if (configuration_.target_bps) { configuration_.SetStreamState(true); } else {