优化播放音视频不同步问题

Author: LxzBUG

ArLiveLite/ArLive2Engine.cpp

@@ -57,6 +57,7 @@ ArLive2Engine::ArLive2Engine(void)
 	: rtc::Thread(rtc::CreateDefaultSocketServer())
 	, observer_(NULL)
 	, b_running_(false)
+	, b_app_background_(false)
 	, b_aud_cap_exception_(false)
 	, b_aud_ply_exception_(false)
 	, b_video_preview_(false)
@@ -208,6 +209,7 @@ void ArLive2Engine::releaseArLivePlayer(AR::IArLivePlayer* player)
 
 void ArLive2Engine::setAppInBackground(bool bBackground)
 {
+	b_app_background_ = bBackground;
 	rtc::CritScope l(&cs_arlive2_player_);
 	MapArLive2Player::iterator itpr = map_arlive2_player_.begin();
 	while (itpr != map_arlive2_player_.end()) {
@@ -307,12 +309,11 @@ void ArLive2Engine::Run()
 			}
 		}
 
-
-		rtc::Thread::SleepMs(1);
 		rtc::Thread::ProcessMessages(1);
 #ifdef WIN32
 		w32_thread.ProcessMessages(1);
 #endif
+		rtc::Thread::SleepMs(1);
 	}
 }
 

ArLiveLite/ArLive2Engine.h

@@ -122,6 +122,7 @@ class ArLive2Engine : public AR::IArLive2Engine, public rtc::Thread, public MThr
 
 	// Control or logic
 	bool	b_running_;
+	bool	b_app_background_;
 	bool	b_aud_cap_exception_;
 	bool	b_aud_ply_exception_;
 	bool	b_video_preview_;

ArLiveLite/ArLive2Player.cpp

@@ -4,7 +4,6 @@
 #include "rtc_base/bind.h"
 #include "rtc_base/time_utils.h"
 
-
 static const size_t kMaxDataSizeSamples = 3840;
 
 
@@ -302,6 +301,7 @@ void ArLive2Player::OnTick()
 	if (ar_player_ != NULL) {
 		ar_player_->RunOnce();
 	}
+
 	PlayBuffer::DoVidRender(b_video_paused_);
 }
 void ArLive2Player::OnTickUnAttach()
@@ -488,6 +488,10 @@ bool ArLive2Player::OnArPlyNeedMoreVideoData(void*player)
 {
 	return PlayBuffer::NeedMoreVideoPlyData();
 }
+bool ArLive2Player::OnArPlyAppIsBackground(void* player)
+{
+	return PlayBuffer::AppIsBackground();
+}
 void ArLive2Player::OnArPlyAudio(void*player, const char*pData, int nSampleHz, int nChannels, int64_t pts)
 {
 	PcmData *audioData = new PcmData((char*)pData, (nSampleHz / 100)*nChannels * sizeof(short), nSampleHz, nChannels);

ArLiveLite/ArLive2Player.h

@@ -74,6 +74,7 @@ class ArLive2Player : public AR::IArLivePlayer, public RtcTick, public PlayBuffe
 
 	virtual bool OnArPlyNeedMoreAudioData(void*player);
 	virtual bool OnArPlyNeedMoreVideoData(void*player);
+	virtual bool OnArPlyAppIsBackground(void* player);
 	virtual void OnArPlyAudio(void*player, const char*pData, int nSampleHz, int nChannels, int64_t pts);
 	virtual void OnArPlyVideo(void*player, int fmt, int ww, int hh, uint8_t**pData, int*linesize, int64_t pts);
 	virtual void OnArPlySeiData(void* player, const char* pData, int nLen, int64_t pts);

ArLiveLite/PlayBuffer.cpp

@@ -91,11 +91,11 @@ static int MixAudio(int iChannelNum, short* sourceData1, short* sourceData2, flo
 
 #ifdef WEBRTC_ANDROID
 //android 的某些机型，音频播放的线程实时性不高，所以一次性需要更多的数据
-const int kMaxAudioPlaySize = 15;
+const int kMaxAudioPlaySize = 30;
 #else
-const int kMaxAudioPlaySize = 5;
+const int kMaxAudioPlaySize = 20;
 #endif
-const int kMaxVedeoPlaySize = 3;
+const int kMaxVedeoPlaySize = kMaxAudioPlaySize / 2;
 
 
 PlayBuffer::PlayBuffer(void)
@@ -104,6 +104,8 @@ PlayBuffer::PlayBuffer(void)
 	, b_video_decoded_(false)
 	, b_audio_decoded_(false)
 	, b_app_in_background_(false)
+	, n_last_render_video_time_(0)
+	, n_last_render_video_pts_(0)
 {
 	aud_data_resamp_ = new char[kMaxDataSizeSamples];
 	memset(aud_data_resamp_, 0, kMaxDataSizeSamples);
@@ -120,22 +122,40 @@ PlayBuffer::~PlayBuffer(void)
 
 int PlayBuffer::DoVidRender(bool bVideoPaused)
 {
-	VideoData* vidPkt = NULL;
-	{
-		rtc::CritScope cs(&cs_video_play_);
-		if (lst_video_play_.size() > 0) {
-			vidPkt = lst_video_play_.front();
-			lst_video_play_.pop_front();
+	bool bRender = false;
+	while (1) {
+		VideoData* vidPkt = NULL;
+		{
+			rtc::CritScope cs(&cs_video_play_);
+			if (lst_video_play_.size() > 0) {
+				vidPkt = lst_video_play_.front();
+				if (n_last_render_video_pts_ == 0 || n_last_render_video_pts_ > vidPkt->pts_) {
+					n_last_render_video_time_ = rtc::TimeUTCMillis();
+					n_last_render_video_pts_ = vidPkt->pts_;
+				}
+				if (vidPkt->pts_ <= (rtc::TimeUTCMillis() - n_last_render_video_time_) + n_last_render_video_pts_) {
+					lst_video_play_.pop_front();
+				}
+				else {
+					vidPkt = NULL;
+				}
+			}
 		}
-	}
 
-	if (vidPkt != NULL) {
-		//RTC_LOG(LS_INFO) << "DoRender video pts: " << vidPkt->pts_ << " plytime: " << play_pts_time_;
-		if (!bVideoPaused && !b_app_in_background_) {
-			OnBufferVideoRender(vidPkt, vidPkt->pts_);
+		if (vidPkt != NULL) {
+			//RTC_LOG(LS_INFO) << "DoRender video pts: " << vidPkt->pts_ << " plytime: " << play_pts_time_;
+			if (!bVideoPaused && !b_app_in_background_) {
+				if (!bRender) {//@Eric - 跳帧处理，防止一次性输出过多
+					OnBufferVideoRender(vidPkt, vidPkt->pts_);
+				}
+				bRender = true;
+			}
+			delete vidPkt;
+			vidPkt = NULL;
+		}
+		else {
+			break;
 		}
-		delete vidPkt;
-		vidPkt = NULL;
 	}
 
 	return 0;
@@ -148,9 +168,11 @@ int PlayBuffer::DoAudRender(bool mix, void* audioSamples, uint32_t samplesPerSec
 	{//*
 		rtc::CritScope cs(&cs_audio_play_);
 		//RTC_LOG(LS_INFO) << "Audio list size: " << lst_audio_play_.size();
+
 		if (lst_audio_play_.size() > 0) {
 			audPkt = lst_audio_play_.front();
 			lst_audio_play_.pop_front();
+
 		}
 
 	}
@@ -182,6 +204,10 @@ int PlayBuffer::DoAudRender(bool mix, void* audioSamples, uint32_t samplesPerSec
 				memcpy(audioSamples, pOutputPcm, a_frame_size);
 			}
 		}
+		{
+			n_last_render_video_time_ = rtc::TimeUTCMillis();
+			n_last_render_video_pts_ = audPkt->pts_;
+		}
 		delete audPkt;
 		audPkt = NULL;
 	}
@@ -298,6 +324,10 @@ bool PlayBuffer::NeedMoreVideoPlyData()
 	rtc::CritScope cs(&cs_video_play_);
 	return lst_video_play_.size() <= kMaxVedeoPlaySize;
 }
+bool PlayBuffer::AppIsBackground()
+{
+	return b_app_in_background_;
+}
 void PlayBuffer::PlayVideoData(VideoData* videoData)
 {
 	//RTC_LOG(LS_INFO) << "PlayVideoData pts: " << videoData->pts_;
@@ -341,7 +371,7 @@ void PlayBuffer::PlayVideoData(VideoData* videoData)
 	//@Eric - 跳帧处理 - 防止缓存太多：内存报警，渲染延时增大
 	while (lst_video_play_.size() > 1) {
 		int64_t timeGap = lst_video_play_.back()->pts_ - lst_video_play_.front()->pts_;
-		if (timeGap < (kMaxAudioPlaySize << 1) * 10) {
+		if (timeGap < (kMaxVideoPlaySize << 1) * 10) {
 			break;
 		}
 		VideoData* pkt = lst_video_play_.front();
@@ -363,27 +393,39 @@ void PlayBuffer::PlayAudioData(PcmData*pcmData)
 	int64_t dropPts = -1;
 	{
 		rtc::CritScope cs(&cs_audio_play_);
+		lst_audio_play_.push_back(pcmData);
+
 		//@Eric - 跳帧处理 - 防止缓存太多：延时增大
-		while (lst_audio_play_.size() > (kMaxAudioPlaySize << 1)) {
+		if (lst_audio_play_.size() >= kMaxAudioPlaySize) {//清一半缓存
+			while (lst_audio_play_.size() > kMaxAudioPlaySize/2) {
+				PcmData* pkt = lst_audio_play_.front();
+				dropPts = pkt->pts_;
+				lst_audio_play_.pop_front();
+				delete pkt;
+
+				OnBufferAudioDropped();
+			}
+		}
+	}
+
+#if 0
+	if (b_app_in_background_) {
+		if (lst_audio_play_.size() > 0) {
 			PcmData* pkt = lst_audio_play_.front();
 			dropPts = pkt->pts_;
-			lst_audio_play_.pop_front();
-			delete pkt;
-
-			OnBufferAudioDropped();
 		}
-		lst_audio_play_.push_back(pcmData);
 	}
+#endif
 
 	if (dropPts != -1) {
 		//@Eric - 跳帧处理 - 防止缓存太多：内存报警，渲染延时增大
 		while (lst_video_play_.size() > 1) {
 			if (lst_video_play_.front()->pts_ > dropPts) {
 				break;
 			}
-			VideoData* pkt = lst_video_play_.front();
+			VideoData* vidPkt = lst_video_play_.front();
 			lst_video_play_.pop_front();
-			delete pkt;
+			delete vidPkt;
 
 			OnBufferVideoDropped();
 		}

ArLiveLite/PlayBuffer.h

@@ -77,6 +77,7 @@ class PlayBuffer
 
 	bool NeedMoreAudioPlyData();
 	bool NeedMoreVideoPlyData();
+	bool AppIsBackground();
 	void PlayVideoData(VideoData *videoData);
 	void PlayAudioData(PcmData*pcmData);
 
@@ -91,6 +92,8 @@ class PlayBuffer
 	bool	b_video_decoded_;
 	bool	b_audio_decoded_;
 	bool	b_app_in_background_;		// app 是否进入后台
+	int64_t n_last_render_video_time_;
+	int64_t n_last_render_video_pts_;
 
 	//* 显示缓存
 	rtc::RecursiveCriticalSection	cs_audio_play_;

ArLiveLite/player/ARFFPlayer.cpp

@@ -13,10 +13,6 @@ static const AVRational TIMEBASE_MS = { 1, 1000 };
 #define fftime_to_milliseconds(ts) (av_rescale(ts, 1000, AV_TIME_BASE))
 #define milliseconds_to_fftime(ms) (av_rescale(ms, AV_TIME_BASE, 1000))
 
-
-
-
-
 int16_t WebRtcSpl_MaxAbsValueW16_I(const int16_t* vector, size_t length) {
 	size_t i = 0;
 	int absolute = 0, maximum = 0;
@@ -451,7 +447,7 @@ void ARFFPlayer::RunOnce()
 		}
 	}
 	while (callback_.OnArPlyNeedMoreVideoData(this)) {
-		if (!FFBuffer::DoDecodeVideo()) {
+		if (!FFBuffer::DoDecodeVideo(callback_.OnArPlyAppIsBackground(this))) {
 			break;
 		}
 	}
@@ -745,6 +741,13 @@ void ARFFPlayer::OnBufferStatusChanged(PlayStatus playStatus)
 		callback_.OnArPlyPlaying(this, user_set_.b_audio_enabled_, user_set_.b_video_enabled_);
 	}
 }
+bool ARFFPlayer::OnBufferIsKeyFrame(AVPacket* pkt)
+{
+	if (pkt->flags & AV_PKT_FLAG_KEY) {
+		return true;
+	}
+	return false;
+}
 bool ARFFPlayer::OnBufferGetPuased()
 {
 	return !user_set_.b_audio_enabled_ && !user_set_.b_video_enabled_;

ArLiveLite/player/ARFFPlayer.h

@@ -77,6 +77,7 @@ class ARFFPlayer : public ARPlayer, public FFBuffer, public rtc::Thread
 	virtual void OnBufferDecodeAudioData(AVPacket* pkt);
 	virtual void OnBufferDecodeVideoData(AVPacket* pkt);
 	virtual void OnBufferStatusChanged(PlayStatus playStatus);
+	virtual bool OnBufferIsKeyFrame(AVPacket* pkt);
 	virtual bool OnBufferGetPuased();
 	virtual float OnBufferGetSpeed();
 

ArLiveLite/player/ARPlayer.h

@@ -62,6 +62,7 @@ class ARPlayerEvent
 
 	virtual bool OnArPlyNeedMoreAudioData(void*player) { return true; };
 	virtual bool OnArPlyNeedMoreVideoData(void*player) { return true; };
+	virtual bool OnArPlyAppIsBackground(void* player) { return false; };
 	virtual void OnArPlyAudio(void*player, const char*pData, int nSampleHz, int nChannels, int64_t pts) {};
 	virtual void OnArPlyVideo(void*player, int fmt, int ww, int hh, uint8_t**pData, int*linesize, int64_t pts) {};
 	virtual void OnArPlyRawVideo(void* player, const char* pData, int nLen, bool bKeyframe, int64_t pts) {};

ArLiveLite/player/FFBuffer.cpp

@@ -9,6 +9,7 @@ FFBuffer::FFBuffer()
 	: play_status_(PS_Init)
 	, play_mode_(PM_Fluent)
 	, b_reset_time_(false)
+	, b_need_keyframe_(true)
 	, n_cacheing_time_(DFT_CACHEING_TIME)
 	, n_cache_to_play_max_(1000)	
 	, n_cache_to_play_time_(1000)
@@ -283,32 +284,46 @@ void FFBuffer::DoTick()
 
 bool  FFBuffer::DoDecodeAudio()
 {
+	bool bRet = false;
 	rtc::CritScope l(&cs_audio_decode_);
-	while (lst_audio_decode_.size() > 0) {
+	if (lst_audio_decode_.size() > 0) {
 		RecvPacket* audPkt = lst_audio_decode_.front();
 		lst_audio_decode_.pop_front();
 
 		OnBufferDecodeAudioData(audPkt->pkt_);
 		delete audPkt;
 		audPkt = NULL;
-		return true;
+		bRet = true;
 	}
 
-	return false;
+	return bRet;
 }
-bool  FFBuffer::DoDecodeVideo()
+bool  FFBuffer::DoDecodeVideo(bool bBackground)
 {
+	bool bRet = false;
 	rtc::CritScope l(&cs_video_decode_);
-	while (lst_video_decode_.size() > 0) {
+	if (lst_video_decode_.size() > 0) {
 		RecvPacket* vidPkt = lst_video_decode_.front();
 		lst_video_decode_.pop_front();
-		OnBufferDecodeVideoData(vidPkt->pkt_);
+		
+		if (bBackground) {
+			b_need_keyframe_ = true;
+		}
+		else {
+			if (b_need_keyframe_ && OnBufferIsKeyFrame(vidPkt->pkt_)) {
+				b_need_keyframe_ = false;
+			}
+		}
+	
+		if (!b_need_keyframe_ && !bBackground) {
+			OnBufferDecodeVideoData(vidPkt->pkt_);
+		}
 		delete vidPkt;
 		vidPkt = NULL;
-		return true;
+		bRet = true;
 	}
 
-	return false;
+	return bRet;
 }
 
 void FFBuffer::SetPlaySetting(bool bAuto, int nCacheTime, int nMinCacheTime, int nMaxCacheTime, int nVideoBlockThreshold)