增加android 自定义音视频采集示例

LxzBUG · LxzBUG · commit f29996a56656 · 2023-06-12T15:33:34.000+08:00
diff --git a/Prj-Android/app/build.gradle b/Prj-Android/app/build.gradle
@@ -9,7 +9,7 @@ android {
 
     defaultConfig {
         applicationId "io.anyrtc.liveplayer"
-        minSdk 19
+        minSdk 21
         targetSdk 31
         versionCode 1
         versionName "1.0"
@@ -60,5 +60,6 @@ dependencies {
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0'
     implementation("io.coil-kt:coil-gif:1.4.0")
     implementation("io.coil-kt:coil:1.4.0")
+    implementation 'com.github.LxzBUG:ScreenShare:1.1.6'
 
 }
diff --git a/Prj-Android/app/src/main/java/io/anyrtc/liveplayer/InputActivity.kt b/Prj-Android/app/src/main/java/io/anyrtc/liveplayer/InputActivity.kt
@@ -22,6 +22,8 @@ class InputActivity : BaseActivity(),RadioGroup.OnCheckedChangeListener,View.OnC
     private var resolution = 0 //0-720 1-540 2-360
     private val config = Config()
     private var requestPermission: ActivityResultLauncher<Array<String>>?=null
+    val VIDEO_1 = "https://www.apple.com/105/media/us/iphone-x/2017/01df5b43-28e4-4848-bf20-490c34a926a7/films/feature/iphone-x-feature-tpl-cc-us-20170912_1920x1080h.mp4"
+
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
         setContentView(binding.root)
@@ -71,9 +73,12 @@ class InputActivity : BaseActivity(),RadioGroup.OnCheckedChangeListener,View.OnC
             R.id.rb_camera-> {
                 this@InputActivity.pushType = 0
             }
-            else->{
+            R.id.rb_screen->{
                 this@InputActivity.pushType = 1
             }
+            else->{
+                this@InputActivity.pushType = 2
+            }
         }
     }
 
@@ -89,7 +94,7 @@ class InputActivity : BaseActivity(),RadioGroup.OnCheckedChangeListener,View.OnC
                         requestPermission?.launch(arrayOf(android.Manifest.permission.CAMERA,android.Manifest.permission.RECORD_AUDIO))
                     }
                     1->{
-                        go(PullActivity::class.java, Pair("url",binding.etUrl.text.toString()))
+                        go(PullActivity::class.java, Pair("url",VIDEO_1))
                     }
                 }
             }
diff --git a/Prj-Android/app/src/main/java/io/anyrtc/liveplayer/PullActivity.kt b/Prj-Android/app/src/main/java/io/anyrtc/liveplayer/PullActivity.kt
@@ -82,8 +82,6 @@ class PullActivity : AppCompatActivity() {
             ) {
                 super.onReceiveSeiMessage(player, payloadType, data)
 
-                binding.tvStatus.setText(String(data!!))
-
             }
 
             override fun onRenderVideoFrame(
@@ -169,9 +167,12 @@ class PullActivity : AppCompatActivity() {
             }
         }
 
+    }
 
-
-
+    override fun onBackPressed() {
+        player.stopPlay()
+        ArLiveEngine.release()
+        finish()
     }
 
     inner class PlayStatus{
diff --git a/Prj-Android/app/src/main/java/io/anyrtc/liveplayer/PushActivity.kt b/Prj-Android/app/src/main/java/io/anyrtc/liveplayer/PushActivity.kt
@@ -17,19 +17,24 @@ import io.anyrtc.live.ArLiveDef.ArLiveVideoResolution
 import io.anyrtc.live.ArLiveEngine
 import io.anyrtc.live.ArLivePusherObserver
 import io.anyrtc.liveplayer.databinding.ActivityPushBinding
+import org.loka.screensharekit.EncodeBuilder
+import org.loka.screensharekit.ScreenShareKit
+import org.loka.screensharekit.callback.AudioCallBack
+import org.loka.screensharekit.callback.RGBACallBack
+import org.loka.screensharekit.callback.StartCaptureCallback
 
 class PushActivity : BaseActivity() {
     private val binding by lazy { ActivityPushBinding.inflate(layoutInflater) }
     private val liveEngine by lazy { ArLiveEngine.create(this)}
     private val pusher by lazy { liveEngine.createArLivePusher() }
     private var pushUrl = ""
-
+    private var pushType = 0
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
         setContentView(binding.root)
         immersive(darkMode = false)
-        val pushType = intent.getIntExtra("pushType",0);
+        pushType = intent.getIntExtra("pushType",0);
         val resolution = intent.getIntExtra("resolution",0)
         pushUrl = intent.getStringExtra("url").toString()
         when(resolution){
@@ -46,11 +51,13 @@ class PushActivity : BaseActivity() {
                 pusher.setVideoQuality(ArLiveDef.ArLiveVideoEncoderParam(ArLiveVideoResolution.ArLiveVideoResolution1920x1080))
             }
         }
-        pusher.startMicrophone()
+
         if (pushType == 0){
             pusher.setRenderView(binding.videoView)
             pusher.startCamera(true)
-        }else{
+            pusher.startMicrophone()
+            pusher.startPush(pushUrl)
+        }else if (pushType == 1){
             binding.ivBeauty.visibility = View.GONE
             val imageLoader = ImageLoader.Builder(this)
                 .componentRegistry {
@@ -63,8 +70,51 @@ class PushActivity : BaseActivity() {
                 .build()
             binding.ivGif.load("https://gimg2.baidu.com/image_search/src=http%3A%2F%2Fhbimg.b0.upaiyun.com%2F60ed9921abb8a968651aae697626dc816624cc4770c32-uwUmhP_fw658&refer=http%3A%2F%2Fhbimg.b0.upaiyun.com&app=2002&size=f9999,10000&q=a80&n=0&g=0n&fmt=jpeg?sec=1645166781&t=ff39058ea3e782746361d8b2bea68511",imageLoader)
             pusher.startScreenCapture()
+            pusher.startPush(pushUrl)
+        }else{//自定义音视频采集
+            pusher.enableCustomAudioCapture(true)
+            pusher.enableCustomVideoCapture(true)
+            ScreenShareKit.init(this).config(screenDataType = EncodeBuilder.SCREEN_DATA_TYPE.RGBA, audioCapture = true)
+                .onRGBA(object :RGBACallBack{
+                    override fun onRGBA(
+                        rgba: ByteArray,
+                        width: Int,
+                        height: Int,
+                        stride: Int,
+                        rotation: Int,
+                        rotationChanged: Boolean
+                    ) {
+                        pusher.sendCustomVideoFrame(ArLiveDef.ArLiveVideoFrame().apply {
+                            pixelFormat = ArLiveDef.ArLivePixelFormat.ArLivePixelFormatBGRA32
+                            bufferType = ArLiveDef.ArLiveBufferType.ArLiveBufferTypeByteArray
+                            data = rgba
+                            this.width = width
+                            this.height = height
+                            this.rotation = rotation
+                            this.stride = stride*4
+                        })
+                    }
+
+                })
+                .onAudio(object :AudioCallBack{
+                    override fun onAudio(buffer: ByteArray?, ts: Long) {
+                        pusher.sendCustomAudioFrame(ArLiveDef.ArLiveAudioFrame().apply {
+                            data = buffer
+                            sampleRate = 16000
+                            channel = 2
+                        })
+                    }
+
+                })
+                .onStart(object :StartCaptureCallback{
+                    override fun onStart() {
+                        pusher.startPush(pushUrl)
+                    }
+
+                })
+                .start()
+
         }
-        pusher.startPush(pushUrl)
         initView()
 
     }
@@ -142,6 +192,9 @@ class PushActivity : BaseActivity() {
 
     override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean {
         if (keyCode == KeyEvent.KEYCODE_BACK){
+            if (pushType==2){
+                ScreenShareKit.stop()
+            }
             pusher.stopPush()
             ArLiveEngine.release()
             finish()
diff --git a/Prj-Android/app/src/main/res/layout/activity_input.xml b/Prj-Android/app/src/main/res/layout/activity_input.xml
@@ -178,6 +178,21 @@
                     android:layout_height="44dp"
                     android:layout_weight="1"/>
 
+                <Space
+                    android:layout_width="25dp"
+                    android:layout_height="wrap_content"/>
+
+                <RadioButton
+                    android:id="@+id/rb_custom"
+                    android:layout_width="0dp"
+                    android:button="@null"
+                    android:text="自定义采集"
+                    android:textColor="@color/color_options"
+                    android:background="@drawable/select_options"
+                    android:gravity="center"
+                    android:layout_height="44dp"
+                    android:layout_weight="1"/>
+
             </RadioGroup>
 
         </LinearLayout>
diff --git a/Prj-Android/liveplayer/src/main/cpp/jni/LiveEngine.cpp b/Prj-Android/liveplayer/src/main/cpp/jni/LiveEngine.cpp
@@ -400,7 +400,7 @@ Java_io_anyrtc_live_internal_NativeInstance_nativeSendCustomVideoFrame(JNIEnv *e
                                                                        jint buffer_type,
                                                                        jbyteArray data,
                                                                        jobject buffer, jint width,
-                                                                       jint height, jint rotation) {
+                                                                       jint height, jint rotation,jint stride) {
     IArLivePusher* arLivePushKit = reinterpret_cast<IArLivePusher *>(nativePtr);
     jint result = -1;
     if (arLivePushKit!= NULL){
@@ -412,6 +412,7 @@ Java_io_anyrtc_live_internal_NativeInstance_nativeSendCustomVideoFrame(JNIEnv *e
         frame->rotation = static_cast<ArLiveRotation>(rotation);
         jbyte* arrayData=(env)->GetByteArrayElements(data, NULL);
         frame->data = reinterpret_cast<char *>(arrayData);
+        frame->stride = stride;
         result = arLivePushKit->sendCustomVideoFrame(frame);
         env->ReleaseByteArrayElements(data, arrayData, 0);
     }
diff --git a/Prj-Android/liveplayer/src/main/java/io/anyrtc/live/ArLiveDef.java b/Prj-Android/liveplayer/src/main/java/io/anyrtc/live/ArLiveDef.java
@@ -246,6 +246,7 @@ public static final class ArLiveVideoFrame {
         public int width;
         public int height;
         public int rotation;
+        public int stride;
 
 
         public ArLiveVideoFrame() {
@@ -254,7 +255,7 @@ public ArLiveVideoFrame() {
         }
 
         @CalledByNative
-        public ArLiveVideoFrame(ArLivePixelFormat pixelFormat, ArLiveBufferType bufferType, ArLiveTexture texture, byte[] data, ByteBuffer buffer, int width, int height, int rotation) {
+        public ArLiveVideoFrame(ArLivePixelFormat pixelFormat, ArLiveBufferType bufferType, ArLiveTexture texture, byte[] data, ByteBuffer buffer, int width, int height, int rotation,int stride) {
             this.pixelFormat = pixelFormat;
             this.bufferType = bufferType;
             this.texture = texture;
@@ -263,6 +264,7 @@ public ArLiveVideoFrame(ArLivePixelFormat pixelFormat, ArLiveBufferType bufferTy
             this.width = width;
             this.height = height;
             this.rotation = rotation;
+            this.stride = stride;
         }
     }
 
diff --git a/Prj-Android/liveplayer/src/main/java/io/anyrtc/live/internal/ArLivePlayerImpl.java b/Prj-Android/liveplayer/src/main/java/io/anyrtc/live/internal/ArLivePlayerImpl.java
@@ -279,7 +279,7 @@ public void onRenderVideoFrame(int pixelFormat, int bufferType,  byte[] data, By
                 @Override
                 public void run() {
                    if (arLivePlayEvent!=null){
-                       arLivePlayEvent.onRenderVideoFrame(arLivePlayer,new ArLiveDef.ArLiveVideoFrame(ArLiveDef.ArLivePixelFormat.values()[pixelFormat], ArLiveDef.ArLiveBufferType.values()[bufferType],null,data,buffer,width,height,rotation));
+                       arLivePlayEvent.onRenderVideoFrame(arLivePlayer,new ArLiveDef.ArLiveVideoFrame(ArLiveDef.ArLivePixelFormat.values()[pixelFormat], ArLiveDef.ArLiveBufferType.values()[bufferType],null,data,buffer,width,height,rotation,0));
                    }
                 }
             });
diff --git a/Prj-Android/liveplayer/src/main/java/io/anyrtc/live/internal/ArLivePusherImpl.java b/Prj-Android/liveplayer/src/main/java/io/anyrtc/live/internal/ArLivePusherImpl.java
@@ -458,7 +458,7 @@ public int enableCustomVideoCapture(boolean var1) {
 
     @Override
     public int sendCustomVideoFrame(ArLiveDef.ArLiveVideoFrame var1) {
-        return nativeInstance.nativeSendCustomVideoFrame(nativeId,var1.pixelFormat.ordinal(),var1.bufferType.ordinal(),var1.data,var1.buffer,var1.width,var1.height,var1.rotation);
+        return nativeInstance.nativeSendCustomVideoFrame(nativeId,var1.pixelFormat.ordinal(),var1.bufferType.ordinal(),var1.data,var1.buffer,var1.width,var1.height,var1.rotation, var1.stride);
     }
 
     @Override
diff --git a/Prj-Android/liveplayer/src/main/java/io/anyrtc/live/internal/NativeInstance.java b/Prj-Android/liveplayer/src/main/java/io/anyrtc/live/internal/NativeInstance.java
@@ -128,7 +128,7 @@ public static NativeInstance getSharedInstance() {
 
     protected native int nativeEnableCustomVideoCapture(long nativePtr,boolean enable);
 
-    protected native int nativeSendCustomVideoFrame(long nativePtr, int pixelFormat, int bufferType, byte[] data, ByteBuffer buffer,int width,int height,int rotation);
+    protected native int nativeSendCustomVideoFrame(long nativePtr, int pixelFormat, int bufferType, byte[] data, ByteBuffer buffer,int width,int height,int rotation,int stride);
 
     protected native int nativeEnableCustomAudioCapture(long nativePtr,boolean enable);
 
diff --git a/Prj-Android/liveplayer/src/main/java/org/webrtc/DefaultVideoEncoderFactory.java b/Prj-Android/liveplayer/src/main/java/org/webrtc/DefaultVideoEncoderFactory.java
@@ -34,14 +34,15 @@ public DefaultVideoEncoderFactory(
   @Nullable
   @Override
   public VideoEncoder createEncoder(VideoCodecInfo info) {
-    final VideoEncoder softwareEncoder = softwareVideoEncoderFactory.createEncoder(info);
-    final VideoEncoder hardwareEncoder = hardwareVideoEncoderFactory.createEncoder(info);
-    if (hardwareEncoder != null && softwareEncoder != null) {
-      // Both hardware and software supported, wrap it in a software fallback
-      return new VideoEncoderFallback(
-          /* fallback= */ softwareEncoder, /* primary= */ hardwareEncoder);
-    }
-    return hardwareEncoder != null ? hardwareEncoder : softwareEncoder;
+    return null;
+//    final VideoEncoder softwareEncoder = softwareVideoEncoderFactory.createEncoder(info);
+//    final VideoEncoder hardwareEncoder = hardwareVideoEncoderFactory.createEncoder(info);
+//    if (hardwareEncoder != null && softwareEncoder != null) {
+//      // Both hardware and software supported, wrap it in a software fallback
+//      return new VideoEncoderFallback(
+//          /* fallback= */ softwareEncoder, /* primary= */ hardwareEncoder);
+//    }
+//    return hardwareEncoder != null ? hardwareEncoder : softwareEncoder;
   }
 
   @Override

Original file line number	Diff line number	Diff line change
`@@ -22,6 +22,8 @@ class InputActivity : BaseActivity(),RadioGroup.OnCheckedChangeListener,View.OnC`
`22`	`22`	`private var resolution = 0 //0-720 1-540 2-360`
`23`	`23`	`private val config = Config()`
`24`	`24`	`private var requestPermission: ActivityResultLauncher<Array<String>>?=null`
	`25`	`+ val VIDEO_1 = "https://www.apple.com/105/media/us/iphone-x/2017/01df5b43-28e4-4848-bf20-490c34a926a7/films/feature/iphone-x-feature-tpl-cc-us-20170912_1920x1080h.mp4"`
	`26`	`+`
`25`	`27`	`override fun onCreate(savedInstanceState: Bundle?) {`
`26`	`28`	`super.onCreate(savedInstanceState)`
`27`	`29`	`setContentView(binding.root)`
`@@ -71,9 +73,12 @@ class InputActivity : BaseActivity(),RadioGroup.OnCheckedChangeListener,View.OnC`
`71`	`73`	`R.id.rb_camera-> {`
`72`	`74`	`this@InputActivity.pushType = 0`
`73`	`75`	`}`
`74`		`- else->{`
	`76`	`+ R.id.rb_screen->{`
`75`	`77`	`this@InputActivity.pushType = 1`
`76`	`78`	`}`
	`79`	`+ else->{`
	`80`	`+ this@InputActivity.pushType = 2`
	`81`	`+ }`
`77`	`82`	`}`
`78`	`83`	`}`
`79`	`84`
`@@ -89,7 +94,7 @@ class InputActivity : BaseActivity(),RadioGroup.OnCheckedChangeListener,View.OnC`
`89`	`94`	`requestPermission?.launch(arrayOf(android.Manifest.permission.CAMERA,android.Manifest.permission.RECORD_AUDIO))`
`90`	`95`	`}`
`91`	`96`	`1->{`
`92`		`- go(PullActivity::class.java, Pair("url",binding.etUrl.text.toString()))`
	`97`	`+ go(PullActivity::class.java, Pair("url",VIDEO_1))`
`93`	`98`	`}`
`94`	`99`	`}`
`95`	`100`	`}`
Original file line number	Diff line number	Diff line change
`@@ -82,8 +82,6 @@ class PullActivity : AppCompatActivity() {`
`82`	`82`	`) {`
`83`	`83`	`super.onReceiveSeiMessage(player, payloadType, data)`
`84`	`84`
`85`		`- binding.tvStatus.setText(String(data!!))`
`86`		`-`
`87`	`85`	`}`
`88`	`86`
`89`	`87`	`override fun onRenderVideoFrame(`
`@@ -169,9 +167,12 @@ class PullActivity : AppCompatActivity() {`
`169`	`167`	`}`
`170`	`168`	`}`
`171`	`169`
	`170`	`+ }`
`172`	`171`
`173`		`-`
`174`		`-`
	`172`	`+ override fun onBackPressed() {`
	`173`	`+ player.stopPlay()`
	`174`	`+ ArLiveEngine.release()`
	`175`	`+ finish()`
`175`	`176`	`}`
`176`	`177`
`177`	`178`	`inner class PlayStatus{`
Original file line number	Diff line number	Diff line change
`@@ -246,6 +246,7 @@ public static final class ArLiveVideoFrame {`
`246`	`246`	`public int width;`
`247`	`247`	`public int height;`
`248`	`248`	`public int rotation;`
	`249`	`+ public int stride;`
`249`	`250`
`250`	`251`
`251`	`252`	`public ArLiveVideoFrame() {`
`@@ -254,7 +255,7 @@ public ArLiveVideoFrame() {`
`254`	`255`	`}`
`255`	`256`
`256`	`257`	`@CalledByNative`
`257`		`- public ArLiveVideoFrame(ArLivePixelFormat pixelFormat, ArLiveBufferType bufferType, ArLiveTexture texture, byte[] data, ByteBuffer buffer, int width, int height, int rotation) {`
	`258`	`+ public ArLiveVideoFrame(ArLivePixelFormat pixelFormat, ArLiveBufferType bufferType, ArLiveTexture texture, byte[] data, ByteBuffer buffer, int width, int height, int rotation,int stride) {`
`258`	`259`	`this.pixelFormat = pixelFormat;`
`259`	`260`	`this.bufferType = bufferType;`
`260`	`261`	`this.texture = texture;`
`@@ -263,6 +264,7 @@ public ArLiveVideoFrame(ArLivePixelFormat pixelFormat, ArLiveBufferType bufferTy`
`263`	`264`	`this.width = width;`
`264`	`265`	`this.height = height;`
`265`	`266`	`this.rotation = rotation;`
	`267`	`+ this.stride = stride;`
`266`	`268`	`}`
`267`	`269`	`}`
`268`	`270`
Original file line number	Diff line number	Diff line change
`@@ -279,7 +279,7 @@ public void onRenderVideoFrame(int pixelFormat, int bufferType, byte[] data, By`
`279`	`279`	`@Override`
`280`	`280`	`public void run() {`
`281`	`281`	`if (arLivePlayEvent!=null){`
`282`		`- arLivePlayEvent.onRenderVideoFrame(arLivePlayer,new ArLiveDef.ArLiveVideoFrame(ArLiveDef.ArLivePixelFormat.values()[pixelFormat], ArLiveDef.ArLiveBufferType.values()[bufferType],null,data,buffer,width,height,rotation));`
	`282`	`+ arLivePlayEvent.onRenderVideoFrame(arLivePlayer,new ArLiveDef.ArLiveVideoFrame(ArLiveDef.ArLivePixelFormat.values()[pixelFormat], ArLiveDef.ArLiveBufferType.values()[bufferType],null,data,buffer,width,height,rotation,0));`
`283`	`283`	`}`
`284`	`284`	`}`
`285`	`285`	`});`
Original file line number	Diff line number	Diff line change
`@@ -458,7 +458,7 @@ public int enableCustomVideoCapture(boolean var1) {`
`458`	`458`
`459`	`459`	`@Override`
`460`	`460`	`public int sendCustomVideoFrame(ArLiveDef.ArLiveVideoFrame var1) {`
`461`		`- return nativeInstance.nativeSendCustomVideoFrame(nativeId,var1.pixelFormat.ordinal(),var1.bufferType.ordinal(),var1.data,var1.buffer,var1.width,var1.height,var1.rotation);`
	`461`	`+ return nativeInstance.nativeSendCustomVideoFrame(nativeId,var1.pixelFormat.ordinal(),var1.bufferType.ordinal(),var1.data,var1.buffer,var1.width,var1.height,var1.rotation, var1.stride);`
`462`	`462`	`}`
`463`	`463`
`464`	`464`	`@Override`