feat: 集成 rtsp-client-android 零缓冲 RTSP 渲染，替换 ExoPlayer
- 新增本地模块 rtspclientlibrary/（拷贝自 rtsp-client-android v5.6.5 library-client-rtsp） - RtspVideoPlayer.kt 重写：RtspSurfaceView + AndroidView，VideoCodec 枚举映射 DecoderType.HARDWARE/SOFTWARE；指数退避自动重连（1s→30s，URL/codec 变化重置） - 延迟统计：每 2s 轮询 statistics 输出 videoDecoderLatencyMsec/networkLatencyMsec - 本地修复两个已知坑：CSD 参数集顺序 sps+pps+vps -> vps+sps+pps（H.265 起播黑屏根因）， codecType 硬编码 H264 -> 按实际 MIME 标记（H.265 宽高解析） - FrameQueue 60 -> 20 压低积压延迟；剔除 camera 依赖（删除 bitmap 渲染路径） - app 移除 media3-exoplayer-rtsp/ui 依赖（media3-exoplayer 由库模块提供）
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
// rtspclientlibrary：从 rtsp-client-android 拷贝的本地模块（library-client-rtsp）
|
||||
// 源码与上游保持同步，仅改写构建脚本为 Kotlin DSL + 版本目录。
|
||||
plugins {
|
||||
alias(libs.plugins.android.library)
|
||||
}
|
||||
|
||||
android {
|
||||
namespace = "com.alexvas.rtsp"
|
||||
compileSdk = 36
|
||||
|
||||
defaultConfig {
|
||||
minSdk = 24
|
||||
targetSdk = 36
|
||||
}
|
||||
|
||||
buildTypes {
|
||||
release {
|
||||
isMinifyEnabled = false
|
||||
proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.txt")
|
||||
}
|
||||
}
|
||||
|
||||
compileOptions {
|
||||
sourceCompatibility = JavaVersion.VERSION_17
|
||||
targetCompatibility = JavaVersion.VERSION_17
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation(libs.androidx.annotation)
|
||||
// media3 仅用于 MediaCodecUtil 解码器枚举 / NalUnitUtil SPS 解析（编译期必需）
|
||||
implementation(libs.androidx.media3.exoplayer)
|
||||
// jcodec 用于 SPS 低延迟参数重写
|
||||
implementation(libs.jcodec)
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
# Proguard rules.
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
<manifest
|
||||
xmlns:android="http://schemas.android.com/apk/res/android">
|
||||
<uses-permission android:name="android.permission.INTERNET" />
|
||||
</manifest>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,231 @@
|
||||
package com.alexvas.rtsp.codec
|
||||
|
||||
import android.media.*
|
||||
import android.os.Process
|
||||
import android.util.Log
|
||||
import java.nio.ByteBuffer
|
||||
|
||||
|
||||
class AudioDecodeThread (
|
||||
private val mimeType: String,
|
||||
private val sampleRate: Int,
|
||||
private val channelCount: Int,
|
||||
private val codecConfig: ByteArray?,
|
||||
private val audioFrameQueue: AudioFrameQueue) : Thread() {
|
||||
|
||||
private var isRunning = true
|
||||
|
||||
fun stopAsync() {
|
||||
if (DEBUG) Log.v(TAG, "stopAsync()")
|
||||
isRunning = false
|
||||
// Wake up sleep() code
|
||||
interrupt()
|
||||
}
|
||||
|
||||
override fun run() {
|
||||
if (DEBUG) Log.d(TAG, "$name started")
|
||||
|
||||
Process.setThreadPriority(Process.THREAD_PRIORITY_AUDIO)
|
||||
|
||||
// Creating audio decoder
|
||||
val decoder = MediaCodec.createDecoderByType(mimeType)
|
||||
val format = MediaFormat.createAudioFormat(mimeType, sampleRate, channelCount)
|
||||
|
||||
if (mimeType == MediaFormat.MIMETYPE_AUDIO_AAC) {
|
||||
val csd0 = codecConfig ?: getAacDecoderConfigData(MediaCodecInfo.CodecProfileLevel.AACObjectLC, sampleRate, channelCount)
|
||||
format.setByteBuffer("csd-0", ByteBuffer.wrap(csd0))
|
||||
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC)
|
||||
} else if (mimeType == MediaFormat.MIMETYPE_AUDIO_OPUS) {
|
||||
// TODO: Add Opus support
|
||||
|
||||
// val OPUS_IDENTIFICATION_HEADER = "OpusHead".toByteArray()
|
||||
// val OPUS_PRE_SKIP_NSEC = ByteBuffer.allocate(8).putLong(11971).array()
|
||||
// val OPUS_SEEK_PRE_ROLL_NSEC = ByteBuffer.allocate(8).putLong(80000000).array()
|
||||
|
||||
// val csd0 = ByteBuffer.allocate(8+1+1+2+4+2+1)
|
||||
// csd0.put("OpusHead".toByteArray())
|
||||
// // Version
|
||||
// csd0.put(1)
|
||||
// // Number of channels
|
||||
// csd0.put(2)
|
||||
// // Pre-skip
|
||||
// csd0.putShort(0)
|
||||
// csd0.putInt(sampleRate)
|
||||
// // Output Gain
|
||||
// csd0.putShort(0)
|
||||
// // Channel Mapping Family
|
||||
// csd0.put(0)
|
||||
// Buffer buf = new Buffer();
|
||||
// // Magic Signature：固定头，占8个字节，为字符串OpusHead
|
||||
// buf.write("OpusHead".getBytes(StandardCharsets.UTF_8));
|
||||
// // Version：版本号，占1字节，固定为0x01
|
||||
// buf.writeByte(1);
|
||||
// // Channel Count：通道数，占1字节，根据音频流通道自行设置，如0x02
|
||||
// buf.writeByte(1);
|
||||
// // Pre-skip：回放的时候从解码器中丢弃的samples数量，占2字节，为小端模式，默认设置0x00,
|
||||
// buf.writeShortLe(0);
|
||||
// // Input Sample Rate (Hz)：音频流的Sample Rate，占4字节，为小端模式，根据实际情况自行设置
|
||||
// buf.writeIntLe(currentFormat.HZ);
|
||||
// //Output Gain：输出增益，占2字节，为小端模式，没有用到默认设置0x00, 0x00就好
|
||||
// buf.writeShortLe(0);
|
||||
// // Channel Mapping Family：通道映射系列，占1字节，默认设置0x00就好
|
||||
// buf.writeByte(0);
|
||||
// //Channel Mapping Table：可选参数，上面的Family默认设置0x00的时候可忽略
|
||||
// format.setByteBuffer("csd-0", ByteBuffer.wrap(OPUS_IDENTIFICATION_HEADER).order(ByteOrder.BIG_ENDIAN))
|
||||
// format.setByteBuffer("csd-1", ByteBuffer.wrap(OPUS_PRE_SKIP_NSEC).order(ByteOrder.BIG_ENDIAN))
|
||||
// format.setByteBuffer("csd-2", ByteBuffer.wrap(OPUS_SEEK_PRE_ROLL_NSEC).order(ByteOrder.LITTLE_ENDIAN))
|
||||
|
||||
val csd0 = byteArrayOf(
|
||||
0x4f, 0x70, 0x75, 0x73, // "Opus"
|
||||
0x48, 0x65, 0x61, 0x64, // "Head"
|
||||
0x01, // Version
|
||||
0x02, // Channel Count
|
||||
0x00, 0x00, // Pre skip
|
||||
0x80.toByte(), 0xbb.toByte(), 0x00, 0x00, // Sample rate 48000
|
||||
0x00, 0x00, // Output Gain (Q7.8 in dB)
|
||||
0x00, // Mapping Family
|
||||
)
|
||||
val csd1 = byteArrayOf(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00)
|
||||
val csd2 = byteArrayOf(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00)
|
||||
format.setByteBuffer("csd-0", ByteBuffer.wrap(csd0))
|
||||
format.setByteBuffer("csd-1", ByteBuffer.wrap(csd1))
|
||||
format.setByteBuffer("csd-2", ByteBuffer.wrap(csd2))
|
||||
}
|
||||
|
||||
decoder.configure(format, null, null, 0)
|
||||
decoder.start()
|
||||
|
||||
// Creating audio playback device
|
||||
val outChannel = if (channelCount > 1) AudioFormat.CHANNEL_OUT_STEREO else AudioFormat.CHANNEL_OUT_MONO
|
||||
val outAudio = AudioFormat.ENCODING_PCM_16BIT
|
||||
val bufferSize = AudioTrack.getMinBufferSize(sampleRate, outChannel, outAudio)
|
||||
// Log.i(TAG, "sampleRate: $sampleRate, bufferSize: $bufferSize".format(sampleRate, bufferSize))
|
||||
val audioTrack = AudioTrack(
|
||||
AudioAttributes.Builder()
|
||||
.setUsage(AudioAttributes.USAGE_MEDIA)
|
||||
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
||||
.build(),
|
||||
AudioFormat.Builder()
|
||||
.setEncoding(outAudio)
|
||||
.setChannelMask(outChannel)
|
||||
.setSampleRate(sampleRate)
|
||||
.build(),
|
||||
bufferSize,
|
||||
AudioTrack.MODE_STREAM,
|
||||
0)
|
||||
audioTrack.play()
|
||||
|
||||
val bufferInfo = MediaCodec.BufferInfo()
|
||||
while (isRunning) {
|
||||
val inIndex: Int = decoder.dequeueInputBuffer(10000L)
|
||||
if (inIndex >= 0) {
|
||||
// fill inputBuffers[inputBufferIndex] with valid data
|
||||
var byteBuffer: ByteBuffer?
|
||||
try {
|
||||
byteBuffer = decoder.getInputBuffer(inIndex)
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
break
|
||||
}
|
||||
byteBuffer?.rewind()
|
||||
|
||||
// Preventing BufferOverflowException
|
||||
// if (length > byteBuffer.limit()) throw DecoderFatalException("Error")
|
||||
|
||||
val audioFrame: FrameQueue.Frame?
|
||||
try {
|
||||
audioFrame = audioFrameQueue.pop()
|
||||
if (audioFrame == null) {
|
||||
Log.d(TAG, "Empty audio frame")
|
||||
// Release input buffer
|
||||
decoder.queueInputBuffer(inIndex, 0, 0, 0L, 0)
|
||||
} else {
|
||||
byteBuffer?.put(audioFrame.data, audioFrame.offset, audioFrame.length)
|
||||
decoder.queueInputBuffer(inIndex, audioFrame.offset, audioFrame.length, audioFrame.timestampMs, 0)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
// Log.i(TAG, "inIndex: ${inIndex}")
|
||||
|
||||
try {
|
||||
// Log.w(TAG, "outIndex: ${outIndex}")
|
||||
if (!isRunning) break
|
||||
when (val outIndex = decoder.dequeueOutputBuffer(bufferInfo, 10000L)) {
|
||||
MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> Log.d(TAG, "Decoder format changed: ${decoder.outputFormat}")
|
||||
MediaCodec.INFO_TRY_AGAIN_LATER -> if (DEBUG) Log.d(TAG, "No output from decoder available")
|
||||
else -> {
|
||||
if (outIndex >= 0) {
|
||||
val byteBuffer: ByteBuffer? = decoder.getOutputBuffer(outIndex)
|
||||
|
||||
val chunk = ByteArray(bufferInfo.size)
|
||||
byteBuffer?.get(chunk)
|
||||
byteBuffer?.clear()
|
||||
|
||||
if (chunk.isNotEmpty()) {
|
||||
audioTrack.write(chunk, 0, chunk.size)
|
||||
}
|
||||
decoder.releaseOutputBuffer(outIndex, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
// All decoded frames have been rendered, we can stop playing now
|
||||
if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) {
|
||||
Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM")
|
||||
break
|
||||
}
|
||||
}
|
||||
audioTrack.flush()
|
||||
audioTrack.release()
|
||||
|
||||
try {
|
||||
decoder.stop()
|
||||
decoder.release()
|
||||
} catch (_: InterruptedException) {
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
audioFrameQueue.clear()
|
||||
if (DEBUG) Log.d(TAG, "$name stopped")
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val TAG: String = AudioDecodeThread::class.java.simpleName
|
||||
private const val DEBUG = false
|
||||
|
||||
fun getAacDecoderConfigData(audioProfile: Int, sampleRate: Int, channels: Int): ByteArray {
|
||||
// AOT_LC = 2
|
||||
// 0001 0000 0000 0000
|
||||
var extraDataAac = audioProfile shl 11
|
||||
// Sample rate
|
||||
when (sampleRate) {
|
||||
7350 -> extraDataAac = extraDataAac or (0xC shl 7)
|
||||
8000 -> extraDataAac = extraDataAac or (0xB shl 7)
|
||||
11025 -> extraDataAac = extraDataAac or (0xA shl 7)
|
||||
12000 -> extraDataAac = extraDataAac or (0x9 shl 7)
|
||||
16000 -> extraDataAac = extraDataAac or (0x8 shl 7)
|
||||
22050 -> extraDataAac = extraDataAac or (0x7 shl 7)
|
||||
24000 -> extraDataAac = extraDataAac or (0x6 shl 7)
|
||||
32000 -> extraDataAac = extraDataAac or (0x5 shl 7)
|
||||
44100 -> extraDataAac = extraDataAac or (0x4 shl 7)
|
||||
48000 -> extraDataAac = extraDataAac or (0x3 shl 7)
|
||||
64000 -> extraDataAac = extraDataAac or (0x2 shl 7)
|
||||
88200 -> extraDataAac = extraDataAac or (0x1 shl 7)
|
||||
96000 -> extraDataAac = extraDataAac or (0x0 shl 7)
|
||||
}
|
||||
// Channels
|
||||
extraDataAac = extraDataAac or (channels shl 3)
|
||||
val extraData = ByteArray(2)
|
||||
extraData[0] = (extraDataAac and 0xff00 shr 8).toByte() // high byte
|
||||
extraData[1] = (extraDataAac and 0xff).toByte() // low byte
|
||||
return extraData
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package com.alexvas.rtsp.codec
|
||||
|
||||
import java.util.concurrent.ArrayBlockingQueue
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
enum class VideoCodecType {
|
||||
H264, H265, UNKNOWN
|
||||
}
|
||||
|
||||
enum class AudioCodecType {
|
||||
AAC_LC, G711_ALAW, G711_MLAW, UNKNOWN
|
||||
}
|
||||
|
||||
class VideoFrameQueue(frameQueueCapacity: Int): FrameQueue<FrameQueue.VideoFrame>(frameQueueCapacity)
|
||||
class AudioFrameQueue(frameQueueCapacity: Int): FrameQueue<FrameQueue.AudioFrame>(frameQueueCapacity)
|
||||
|
||||
/**
|
||||
* Queue for concurrent adding/removing audio/video frames.
|
||||
*/
|
||||
open class FrameQueue<T>(private val frameQueueCapacity: Int) {
|
||||
|
||||
interface Frame {
|
||||
val data: ByteArray
|
||||
val offset: Int
|
||||
val length: Int
|
||||
val timestampMs: Long // presentation time in msec
|
||||
}
|
||||
|
||||
data class VideoFrame(
|
||||
/** Only H264 codec supported */
|
||||
val codecType: VideoCodecType,
|
||||
/** Indicates whether it is a keyframe or not */
|
||||
val isKeyframe: Boolean,
|
||||
override val data: ByteArray,
|
||||
override val offset: Int,
|
||||
override val length: Int,
|
||||
/** Video frame timestamp (msec) generated by camera */
|
||||
override val timestampMs: Long,
|
||||
/** Captured (received) video frame timestamp (msec). If -1, not supported. */
|
||||
val capturedTimestampMs: Long = -1
|
||||
) : Frame
|
||||
|
||||
data class AudioFrame(
|
||||
val codecType: AudioCodecType,
|
||||
// val sampleRate: Int,
|
||||
override val data: ByteArray,
|
||||
override val offset: Int,
|
||||
override val length: Int,
|
||||
override val timestampMs: Long,
|
||||
) : Frame
|
||||
|
||||
private val queue = ArrayBlockingQueue<T>(frameQueueCapacity)
|
||||
|
||||
val size: Int
|
||||
get() = queue.size
|
||||
|
||||
val capacity: Int
|
||||
get() = frameQueueCapacity
|
||||
|
||||
@Throws(InterruptedException::class)
|
||||
fun push(frame: T): Boolean {
|
||||
if (queue.offer(frame, 5, TimeUnit.MILLISECONDS)) {
|
||||
return true
|
||||
}
|
||||
// Log.w(TAG, "Cannot add frame, queue is full")
|
||||
return false
|
||||
}
|
||||
|
||||
@Throws(InterruptedException::class)
|
||||
open fun pop(timeout: Long = 1000): T? {
|
||||
try {
|
||||
val frame: T? = queue.poll(timeout, TimeUnit.MILLISECONDS)
|
||||
// if (frame == null) {
|
||||
// Log.w(TAG, "Cannot get frame within 1 sec, queue is empty")
|
||||
// }
|
||||
return frame
|
||||
} catch (e: InterruptedException) {
|
||||
Thread.currentThread().interrupt()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
fun clear() {
|
||||
queue.clear()
|
||||
}
|
||||
|
||||
fun copyInto(dstFrameQueue: FrameQueue<T>) {
|
||||
dstFrameQueue.queue.addAll(queue)
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val TAG: String = FrameQueue::class.java.simpleName
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,530 @@
|
||||
package com.alexvas.rtsp.codec
|
||||
|
||||
import android.annotation.SuppressLint
|
||||
import android.media.MediaCodec
|
||||
import android.media.MediaCodec.OnFrameRenderedListener
|
||||
import android.media.MediaFormat
|
||||
import android.os.Build
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.os.Process
|
||||
import android.util.Log
|
||||
import com.alexvas.utils.MediaCodecUtils
|
||||
import com.alexvas.utils.capabilitiesToString
|
||||
import androidx.media3.common.util.Util
|
||||
import com.alexvas.utils.VideoCodecUtils
|
||||
import com.limelight.binding.video.MediaCodecHelper
|
||||
import java.lang.Integer.min
|
||||
import java.nio.ByteBuffer
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
|
||||
abstract class VideoDecodeThread (
|
||||
protected val mimeType: String,
|
||||
protected val width: Int,
|
||||
protected val height: Int,
|
||||
protected val rotation: Int, // 0, 90, 180, 270
|
||||
protected val videoFrameQueue: VideoFrameQueue,
|
||||
protected val videoDecoderListener: VideoDecoderListener,
|
||||
protected var videoDecoderType: DecoderType
|
||||
) : Thread() {
|
||||
|
||||
enum class DecoderType {
|
||||
HARDWARE,
|
||||
SOFTWARE // fallback
|
||||
}
|
||||
|
||||
interface VideoDecoderListener {
|
||||
/** Video decoder successfully started */
|
||||
fun onVideoDecoderStarted() {}
|
||||
/** Video decoder successfully stopped */
|
||||
fun onVideoDecoderStopped() {}
|
||||
/** Fatal error occurred */
|
||||
fun onVideoDecoderFailed(message: String?) {}
|
||||
/** Resolution changed */
|
||||
fun onVideoDecoderFormatChanged(width: Int, height: Int) {}
|
||||
/** First video frame rendered */
|
||||
fun onVideoDecoderFirstFrameRendered() {}
|
||||
}
|
||||
|
||||
protected val uiHandler = Handler(Looper.getMainLooper())
|
||||
protected var exitFlag = AtomicBoolean(false)
|
||||
protected var firstFrameRendered = false
|
||||
|
||||
/** Decoder latency used for statistics */
|
||||
@Volatile private var decoderLatency = -1
|
||||
/** Flag for allowing calculating latency */
|
||||
private var decoderLatencyRequested = false
|
||||
/** Network latency used for statistics */
|
||||
@Volatile private var networkLatency = -1
|
||||
private var videoDecoderName: String? = null
|
||||
private var firstFrameDecoded = false
|
||||
@Volatile private var videoFrameRateStabilization = false
|
||||
|
||||
fun stopAsync() {
|
||||
if (DEBUG) Log.v(TAG, "stopAsync()")
|
||||
exitFlag.set(true)
|
||||
// Wake up sleep() code
|
||||
interrupt()
|
||||
}
|
||||
|
||||
/**
|
||||
* Currently used video decoder. Video decoder can be changed on runtime.
|
||||
* If videoDecoderType set to HARDWARE, it can be switched to SOFTWARE in case of decoding issue
|
||||
* (e.g. hardware decoder does not support the stream resolution).
|
||||
* If videoDecoderType set to SOFTWARE, it will always remain SOFTWARE (no any changes).
|
||||
*/
|
||||
fun getCurrentVideoDecoderType(): DecoderType {
|
||||
return videoDecoderType
|
||||
}
|
||||
|
||||
fun getCurrentVideoDecoderName(): String? {
|
||||
return videoDecoderName
|
||||
}
|
||||
|
||||
/**
|
||||
* Get frames decoding/rendering latency in msec. Returns -1 if not supported.
|
||||
*/
|
||||
fun getCurrentVideoDecoderLatencyMsec(): Int {
|
||||
decoderLatencyRequested = true
|
||||
return decoderLatency
|
||||
}
|
||||
|
||||
/**
|
||||
* Get network latency in msec. Returns -1 if not supported.
|
||||
*/
|
||||
fun getCurrentNetworkLatencyMsec(): Int {
|
||||
return networkLatency
|
||||
}
|
||||
|
||||
fun setVideoFrameRateStabilization(enable: Boolean) {
|
||||
if (DEBUG) Log.v(TAG, "setVideoFrameRateStabilization(enable=$enable)")
|
||||
videoFrameRateStabilization = enable
|
||||
}
|
||||
|
||||
fun hasVideoFrameRateStabilization(): Boolean {
|
||||
return videoFrameRateStabilization
|
||||
}
|
||||
|
||||
@SuppressLint("UnsafeOptInUsageError")
|
||||
private fun getDecoderSafeWidthHeight(decoder: MediaCodec): Pair<Int, Int> {
|
||||
if (DEBUG) Log.v(TAG, "getDecoderSafeWidthHeight()")
|
||||
val capabilities = decoder.codecInfo.getCapabilitiesForType(mimeType).videoCapabilities
|
||||
return if (capabilities == null) {
|
||||
Log.e(TAG, "Not a video decoder")
|
||||
Pair(-1, -1)
|
||||
} else if (capabilities.isSizeSupported(width, height)) {
|
||||
Log.i(TAG, "Video decoder frame size ${width}x${height} supported")
|
||||
Pair(width, height)
|
||||
} else {
|
||||
Log.w(TAG, "Video decoder frame size ${width}x${height} is not supported")
|
||||
val widthAlignment = capabilities.widthAlignment
|
||||
val heightAlignment = capabilities.heightAlignment
|
||||
val w = Util.ceilDivide(width, widthAlignment) * widthAlignment
|
||||
val h = Util.ceilDivide(height, heightAlignment) * heightAlignment
|
||||
if (capabilities.isSizeSupported(w, h)) {
|
||||
Log.i(TAG, "Video decoder frame size ${w}x${h} calculated from alignment ${widthAlignment}x${heightAlignment} and original size ${width}x${height}]")
|
||||
Pair(w, h)
|
||||
} else {
|
||||
val p = Pair(capabilities.supportedWidths.upper, capabilities.supportedHeights.upper)
|
||||
Log.i(TAG, "Video decoder max supported frame size ${w}x${h}")
|
||||
p
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressLint("InlinedApi")
|
||||
private fun getWidthHeight(mediaFormat: MediaFormat): Pair<Int, Int> {
|
||||
// Sometimes height obtained via KEY_HEIGHT is not valid, e.g. can be 1088 instead 1080
|
||||
// (no problems with width though). Use crop parameters to correctly determine height.
|
||||
val hasCrop =
|
||||
mediaFormat.containsKey(MediaFormat.KEY_CROP_RIGHT) && mediaFormat.containsKey(MediaFormat.KEY_CROP_LEFT) &&
|
||||
mediaFormat.containsKey(MediaFormat.KEY_CROP_BOTTOM) && mediaFormat.containsKey(MediaFormat.KEY_CROP_TOP)
|
||||
val width =
|
||||
if (hasCrop)
|
||||
mediaFormat.getInteger(MediaFormat.KEY_CROP_RIGHT) - mediaFormat.getInteger(MediaFormat.KEY_CROP_LEFT) + 1
|
||||
else
|
||||
mediaFormat.getInteger(MediaFormat.KEY_WIDTH)
|
||||
var height =
|
||||
if (hasCrop)
|
||||
mediaFormat.getInteger(MediaFormat.KEY_CROP_BOTTOM) - mediaFormat.getInteger(MediaFormat.KEY_CROP_TOP) + 1
|
||||
else
|
||||
mediaFormat.getInteger(MediaFormat.KEY_HEIGHT)
|
||||
// Fix for 1080p resolution for Samsung S21
|
||||
// {crop-right=1919, max-height=4320, sar-width=1, color-format=2130708361, mime=video/raw,
|
||||
// hdr-static-info=java.nio.HeapByteBuffer[pos=0 lim=25 cap=25],
|
||||
// priority=0, color-standard=1, feature-secure-playback=0, color-transfer=3, sar-height=1,
|
||||
// crop-bottom=1087, max-width=8192, crop-left=0, width=1920, color-range=2, crop-top=0,
|
||||
// rotation-degrees=0, frame-rate=30, height=1088}
|
||||
height = height / 16 * 16 // 1088 -> 1080
|
||||
// if (height == 1088)
|
||||
// height = 1080
|
||||
return Pair(width, height)
|
||||
}
|
||||
|
||||
private fun getDecoderMediaFormat(decoder: MediaCodec): MediaFormat {
|
||||
if (DEBUG) Log.v(TAG, "getDecoderMediaFormat()")
|
||||
val safeWidthHeight = getDecoderSafeWidthHeight(decoder)
|
||||
val format = MediaFormat.createVideoFormat(mimeType, safeWidthHeight.first, safeWidthHeight.second)
|
||||
if (DEBUG)
|
||||
Log.d(TAG, "Configuring surface ${safeWidthHeight.first}x${safeWidthHeight.second} w/ '$mimeType'")
|
||||
else
|
||||
Log.i(TAG, "Configuring surface ${safeWidthHeight.first}x${safeWidthHeight.second} w/ '$mimeType'")
|
||||
format.setInteger(MediaFormat.KEY_ROTATION, rotation)
|
||||
// if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
|
||||
// // format.setFeatureEnabled(android.media.MediaCodecInfo.CodecCapabilities.FEATURE_LowLatency, true)
|
||||
// // Request low-latency for the decoder. Not all of the decoders support that.
|
||||
// format.setInteger(MediaFormat.KEY_LOW_LATENCY, 1)
|
||||
// }
|
||||
|
||||
val succeeded = MediaCodecHelper.setDecoderLowLatencyOptions(format, decoder.codecInfo, 1)
|
||||
Log.i(TAG, "Low-latency: $succeeded")
|
||||
|
||||
return format
|
||||
}
|
||||
|
||||
/** Decoder created */
|
||||
abstract fun decoderCreated(mediaCodec: MediaCodec, mediaFormat: MediaFormat)
|
||||
|
||||
/** Frame processed */
|
||||
abstract fun releaseOutputBuffer(
|
||||
mediaCodec: MediaCodec,
|
||||
outIndex: Int,
|
||||
bufferInfo: MediaCodec.BufferInfo,
|
||||
render: Boolean
|
||||
)
|
||||
|
||||
/** Decoder stopped and released */
|
||||
abstract fun decoderDestroyed(mediaCodec: MediaCodec)
|
||||
|
||||
private fun createVideoDecoderAndStart(decoderType: DecoderType): MediaCodec {
|
||||
if (DEBUG) Log.v(TAG, "createVideoDecoderAndStart(decoderType=$decoderType)")
|
||||
|
||||
@SuppressLint("UnsafeOptInUsageError")
|
||||
val decoder = when (decoderType) {
|
||||
DecoderType.HARDWARE -> {
|
||||
val hwDecoders = MediaCodecUtils.getHardwareDecoders(mimeType)
|
||||
if (hwDecoders.isEmpty()) {
|
||||
Log.w(TAG, "Cannot get hardware video decoders for mime type '$mimeType'. Using default one.")
|
||||
MediaCodec.createDecoderByType(mimeType)
|
||||
} else {
|
||||
val lowLatencyDecoder = MediaCodecUtils.getLowLatencyDecoder(hwDecoders)
|
||||
val name = lowLatencyDecoder?.let {
|
||||
Log.i(TAG, "[$name] Dedicated low-latency decoder found '${lowLatencyDecoder.name}'")
|
||||
lowLatencyDecoder.name
|
||||
} ?: hwDecoders[0].name
|
||||
MediaCodec.createByCodecName(name)
|
||||
}
|
||||
}
|
||||
DecoderType.SOFTWARE -> {
|
||||
val swDecoders = MediaCodecUtils.getSoftwareDecoders(mimeType)
|
||||
if (swDecoders.isEmpty()) {
|
||||
Log.w(TAG, "Cannot get software video decoders for mime type '$mimeType'. Using default one .")
|
||||
MediaCodec.createDecoderByType(mimeType)
|
||||
} else {
|
||||
val name = swDecoders[0].name
|
||||
MediaCodec.createByCodecName(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
this.videoDecoderType = decoderType
|
||||
this.videoDecoderName = decoder.name
|
||||
|
||||
val frameRenderedListener = OnFrameRenderedListener { _, _, _ ->
|
||||
if (!firstFrameRendered) {
|
||||
firstFrameRendered = true
|
||||
uiHandler.post {
|
||||
videoDecoderListener.onVideoDecoderFirstFrameRendered()
|
||||
}
|
||||
}
|
||||
}
|
||||
decoder.setOnFrameRenderedListener(frameRenderedListener, null)
|
||||
val format = getDecoderMediaFormat(decoder)
|
||||
decoderCreated(decoder, format)
|
||||
decoder.start()
|
||||
|
||||
val capabilities = decoder.codecInfo.getCapabilitiesForType(mimeType)
|
||||
val lowLatencySupport = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
|
||||
capabilities.isFeatureSupported(android.media.MediaCodecInfo.CodecCapabilities.FEATURE_LowLatency)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
Log.i(TAG, "[$name] Video decoder '${decoder.name}' started " +
|
||||
"(${if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { if (decoder.codecInfo.isHardwareAccelerated) "hardware" else "software" } else ""}, " +
|
||||
"${capabilities.capabilitiesToString()}, " +
|
||||
"${if (lowLatencySupport) "w/" else "w/o"} low-latency support)")
|
||||
|
||||
return decoder
|
||||
}
|
||||
|
||||
private fun stopAndReleaseVideoDecoder(decoder: MediaCodec) {
|
||||
if (DEBUG) Log.v(TAG, "stopAndReleaseVideoDecoder()")
|
||||
val type = videoDecoderType.toString().lowercase()
|
||||
Log.i(TAG, "Stopping $type video decoder...")
|
||||
try {
|
||||
decoder.stop()
|
||||
Log.i(TAG, "Decoder successfully stopped")
|
||||
} catch (e3: Throwable) {
|
||||
Log.e(TAG, "Failed to stop decoder", e3)
|
||||
}
|
||||
Log.i(TAG, "Releasing decoder...")
|
||||
try {
|
||||
decoder.release()
|
||||
Log.i(TAG, "Decoder successfully released")
|
||||
} catch (e3: Throwable) {
|
||||
Log.e(TAG, "Failed to release decoder", e3)
|
||||
}
|
||||
videoFrameQueue.clear()
|
||||
decoderDestroyed(decoder)
|
||||
}
|
||||
|
||||
override fun run() {
|
||||
if (DEBUG) Log.d(TAG, "$name started")
|
||||
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
|
||||
Process.setThreadPriority(Process.THREAD_PRIORITY_VIDEO)
|
||||
}
|
||||
|
||||
videoDecoderListener.onVideoDecoderStarted()
|
||||
|
||||
try {
|
||||
Log.i(TAG, "Starting hardware video decoder...")
|
||||
var decoder = try {
|
||||
createVideoDecoderAndStart(videoDecoderType)
|
||||
} catch (e: Throwable) {
|
||||
Log.e(TAG, "Failed to start $videoDecoderType video decoder (${e.message})", e)
|
||||
Log.i(TAG, "Starting software video decoder...")
|
||||
try {
|
||||
createVideoDecoderAndStart(DecoderType.SOFTWARE)
|
||||
} catch (e2: Throwable) {
|
||||
Log.e(TAG, "Failed to start video software decoder. Exiting...", e2)
|
||||
// Unexpected behavior
|
||||
videoDecoderListener.onVideoDecoderFailed("Cannot initialize video decoder for mime type '$mimeType'")
|
||||
return
|
||||
}
|
||||
}
|
||||
val bufferInfo = MediaCodec.BufferInfo()
|
||||
|
||||
try {
|
||||
var widthHeightFromStream: Pair<Int, Int>? = null
|
||||
|
||||
// Map for calculating decoder rendering latency.
|
||||
// key - original frame timestamp, value - timestamp when frame was added to the map
|
||||
val keyframesTimestamps = HashMap<Long, Long>()
|
||||
|
||||
var frameQueuedMsec = System.currentTimeMillis()
|
||||
var frameAlreadyDequeued = false
|
||||
|
||||
// Main loop
|
||||
while (!exitFlag.get()) {
|
||||
try {
|
||||
val inIndex: Int = decoder.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT_US)
|
||||
if (inIndex >= 0) {
|
||||
// fill inputBuffers[inputBufferIndex] with valid data
|
||||
val byteBuffer: ByteBuffer? = decoder.getInputBuffer(inIndex)
|
||||
byteBuffer?.rewind()
|
||||
|
||||
// Preventing BufferOverflowException
|
||||
// if (length > byteBuffer.limit()) throw DecoderFatalException("Error")
|
||||
|
||||
val frame = videoFrameQueue.pop()
|
||||
if (frame == null) {
|
||||
Log.d(TAG, "Empty video frame")
|
||||
// Release input buffer
|
||||
decoder.queueInputBuffer(inIndex, 0, 0, 0L, 0)
|
||||
} else {
|
||||
// Add timestamp for keyframe to calculating latency further.
|
||||
if ((DEBUG || decoderLatencyRequested) && frame.isKeyframe) {
|
||||
if (keyframesTimestamps.size > 5) {
|
||||
// Something wrong with map. Allow only 5 map entries.
|
||||
keyframesTimestamps.clear()
|
||||
}
|
||||
val l = System.currentTimeMillis()
|
||||
keyframesTimestamps[frame.timestampMs] = l
|
||||
// Log.d(TAG, "Added $l")
|
||||
}
|
||||
// Calculate network latency
|
||||
networkLatency = if (frame.capturedTimestampMs > -1)
|
||||
(frame.timestampMs - frame.capturedTimestampMs).toInt()
|
||||
else
|
||||
-1
|
||||
|
||||
byteBuffer?.put(frame.data, frame.offset, frame.length)
|
||||
if (DEBUG) {
|
||||
val l = System.currentTimeMillis()
|
||||
Log.i(TAG, "\tFrame queued (${l - frameQueuedMsec}) ${if (frame.isKeyframe) "key frame" else ""}")
|
||||
frameQueuedMsec = l
|
||||
}
|
||||
val flags = if (frame.isKeyframe)
|
||||
(MediaCodec.BUFFER_FLAG_KEY_FRAME /*or MediaCodec.BUFFER_FLAG_CODEC_CONFIG*/) else 0
|
||||
decoder.queueInputBuffer(inIndex, frame.offset, frame.length, frame.timestampMs, flags)
|
||||
|
||||
if (frame.isKeyframe) {
|
||||
// Obtain width and height from stream
|
||||
widthHeightFromStream = try {
|
||||
VideoCodecUtils.getWidthHeightFromArray(
|
||||
frame.data,
|
||||
frame.offset,
|
||||
// Check only first 100 bytes maximum. That's enough for finding SPS NAL unit.
|
||||
min(frame.length, VideoCodecUtils.MAX_NAL_SPS_SIZE),
|
||||
isH265 = frame.codecType == VideoCodecType.H265
|
||||
)
|
||||
} catch (_: Exception) {
|
||||
// Log.e(TAG, "Failed to parse width/height from SPS frame. SPS frame seems to be corrupted.", e)
|
||||
null
|
||||
}
|
||||
// Log.i(TAG, "width/height: ${widthHeightFromStream?.first}x${widthHeightFromStream?.second}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (exitFlag.get()) break
|
||||
|
||||
// Get all output buffer frames until no buffer from decoder available (INFO_TRY_AGAIN_LATER).
|
||||
// Single input buffer frame can contain several frames, e.g. SPS + PPS + IDR.
|
||||
// Thus dequeueOutputBuffer should be called several times.
|
||||
// First time it obtains SPS + PPS, second one - IDR frame.
|
||||
do {
|
||||
// For the first time wait for a frame within 100 msec, next times no timeout
|
||||
val timeout = if (frameAlreadyDequeued || !firstFrameDecoded) 0L else DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US
|
||||
val outIndex = decoder.dequeueOutputBuffer(bufferInfo, timeout)
|
||||
when (outIndex) {
|
||||
// Resolution changed
|
||||
MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED, MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
|
||||
Log.d(TAG, "Decoder format changed: ${decoder.outputFormat}")
|
||||
// Decoder can contain different resolution (it can make downsampling).
|
||||
// If resolution successfully obtained from SPS frame, use it.
|
||||
val widthHeightFromDecoder = getWidthHeight(decoder.outputFormat)
|
||||
val widthHeight = widthHeightFromStream ?: widthHeightFromDecoder
|
||||
Log.i(TAG, "Video decoder resolution: ${widthHeightFromDecoder.first}x${widthHeightFromDecoder.second}, stream resolution: ${widthHeightFromStream?.first}x${widthHeightFromStream?.second}")
|
||||
|
||||
// val widthHeightFromDecoder = getWidthHeight(decoder.outputFormat)
|
||||
val rotation = if (decoder.outputFormat.containsKey(MediaFormat.KEY_ROTATION)) {
|
||||
decoder.outputFormat.getInteger(MediaFormat.KEY_ROTATION)
|
||||
} else {
|
||||
// Some devices like Samsung SM-A505U (Android 11) do not allow
|
||||
// video stream rotation on decoding for hardware decoder
|
||||
Log.w(TAG, "Video stream rotation is not supported by this Android device (${Build.MODEL} - ${Build.DEVICE}, codec: '${decoder.name}')")
|
||||
0
|
||||
}
|
||||
uiHandler.post {
|
||||
// Run in UI thread
|
||||
when (rotation) {
|
||||
90, 270 -> videoDecoderListener.onVideoDecoderFormatChanged(widthHeight.second, widthHeight.first)
|
||||
else -> videoDecoderListener.onVideoDecoderFormatChanged(widthHeight.first, widthHeight.second)
|
||||
}
|
||||
}
|
||||
frameAlreadyDequeued = true
|
||||
}
|
||||
// No any frames in queue
|
||||
MediaCodec.INFO_TRY_AGAIN_LATER -> {
|
||||
if (DEBUG) Log.d(TAG, "No output from decoder available")
|
||||
frameAlreadyDequeued = true
|
||||
}
|
||||
// Frame decoded
|
||||
else -> {
|
||||
if (outIndex >= 0) {
|
||||
if (DEBUG || decoderLatencyRequested) {
|
||||
val ts = bufferInfo.presentationTimeUs
|
||||
keyframesTimestamps.remove(ts)?.apply {
|
||||
decoderLatency = (System.currentTimeMillis() - this).toInt()
|
||||
// Log.d(TAG, "Removed $this")
|
||||
}
|
||||
}
|
||||
|
||||
val render = bufferInfo.size != 0 && !exitFlag.get()
|
||||
if (DEBUG) Log.i(TAG, "\tFrame decoded [outIndex=$outIndex, render=$render]")
|
||||
releaseOutputBuffer(decoder, outIndex, bufferInfo, render)
|
||||
if (!firstFrameDecoded && render) {
|
||||
firstFrameDecoded = true
|
||||
}
|
||||
frameAlreadyDequeued = false
|
||||
} else {
|
||||
Log.e(TAG, "Obtaining frame failed w/ error code $outIndex")
|
||||
}
|
||||
}
|
||||
}
|
||||
// For SPS/PPS frame request another frame (IDR)
|
||||
} while (outIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED || outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED)
|
||||
// } while (outIndex != MediaCodec.INFO_TRY_AGAIN_LATER)
|
||||
|
||||
// All decoded frames have been rendered, we can stop playing now
|
||||
if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) {
|
||||
if (DEBUG) Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM")
|
||||
break
|
||||
}
|
||||
} catch (_: InterruptedException) {
|
||||
} catch (e: IllegalStateException) {
|
||||
// Restarting decoder in software mode
|
||||
Log.e(TAG, "${e.message}", e)
|
||||
stopAndReleaseVideoDecoder(decoder)
|
||||
Log.i(TAG, "Starting software video decoder...")
|
||||
decoder = createVideoDecoderAndStart(DecoderType.SOFTWARE)
|
||||
Log.i(TAG, "Software video decoder '${decoder.name}' started (${decoder.codecInfo.getCapabilitiesForType(mimeType).capabilitiesToString()})")
|
||||
} catch (e: MediaCodec.CodecException) {
|
||||
Log.w(TAG, "${e.diagnosticInfo}\nisRecoverable: ${e.isRecoverable}, isTransient: ${e.isTransient}")
|
||||
if (e.isRecoverable) {
|
||||
// Recoverable error.
|
||||
// Calling stop(), configure(), and start() to recover.
|
||||
Log.i(TAG, "Recovering video decoder...")
|
||||
try {
|
||||
decoder.stop()
|
||||
val format = getDecoderMediaFormat(decoder)
|
||||
decoderCreated(decoder, format)
|
||||
decoder.start()
|
||||
Log.i(TAG, "Video decoder recovering succeeded")
|
||||
} catch (e2: Throwable) {
|
||||
Log.e(TAG, "Video decoder recovering failed")
|
||||
Log.e(TAG, "${e2.message}", e2)
|
||||
}
|
||||
} else if (e.isTransient) {
|
||||
// Transient error. Resources are temporarily unavailable and
|
||||
// the method may be retried at a later time.
|
||||
Log.w(TAG, "Video decoder resource temporarily unavailable")
|
||||
} else {
|
||||
// Fatal error. Restarting decoder in software mode.
|
||||
stopAndReleaseVideoDecoder(decoder)
|
||||
Log.i(TAG, "Starting video software decoder...")
|
||||
decoder = createVideoDecoderAndStart(DecoderType.SOFTWARE)
|
||||
Log.i(TAG, "Software video decoder '${decoder.name}' started (${decoder.codecInfo.getCapabilitiesForType(mimeType).capabilitiesToString()})")
|
||||
}
|
||||
} catch (e: Throwable) {
|
||||
Log.e(TAG, "${e.message}", e)
|
||||
}
|
||||
} // while
|
||||
|
||||
// Drain decoder
|
||||
val inIndex: Int = decoder.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT_US)
|
||||
if (inIndex >= 0) {
|
||||
decoder.queueInputBuffer(inIndex, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
|
||||
} else {
|
||||
Log.w(TAG, "Not able to signal end of stream")
|
||||
}
|
||||
|
||||
} catch (e2: Throwable) {
|
||||
Log.e(TAG, "${e2.message}", e2)
|
||||
} finally {
|
||||
stopAndReleaseVideoDecoder(decoder)
|
||||
}
|
||||
|
||||
} catch (e: Throwable) {
|
||||
Log.e(TAG, "$name stopped due to '${e.message}'")
|
||||
videoDecoderListener.onVideoDecoderFailed(e.message)
|
||||
// While configuring stopAsync can be called and surface released. Just exit.
|
||||
if (!exitFlag.get()) e.printStackTrace()
|
||||
return
|
||||
}
|
||||
|
||||
videoDecoderListener.onVideoDecoderStopped()
|
||||
if (DEBUG) Log.d(TAG, "$name stopped")
|
||||
}
|
||||
|
||||
companion object {
|
||||
internal val TAG: String = VideoDecodeThread::class.java.simpleName
|
||||
internal const val DEBUG = false
|
||||
|
||||
private val DEQUEUE_INPUT_TIMEOUT_US = TimeUnit.MILLISECONDS.toMicros(500)
|
||||
private val DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = TimeUnit.MILLISECONDS.toMicros(100)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package com.alexvas.rtsp.codec
|
||||
|
||||
import android.media.MediaCodec
|
||||
import android.media.MediaFormat
|
||||
import android.util.Log
|
||||
import android.view.Surface
|
||||
import java.util.concurrent.TimeUnit
|
||||
import kotlin.math.max
|
||||
|
||||
class VideoDecoderSurfaceThread(
|
||||
private val surface: Surface,
|
||||
mimeType: String,
|
||||
width: Int,
|
||||
height: Int,
|
||||
rotation: Int, // 0, 90, 180, 270
|
||||
videoFrameQueue: VideoFrameQueue,
|
||||
videoDecoderListener: VideoDecoderListener,
|
||||
videoDecoderType: DecoderType = DecoderType.HARDWARE,
|
||||
videoFrameRateStabilization: Boolean = false,
|
||||
) : VideoDecodeThread(
|
||||
mimeType, width, height, rotation, videoFrameQueue, videoDecoderListener, videoDecoderType
|
||||
) {
|
||||
|
||||
/**
|
||||
* Presentation time (in RTP units converted to microseconds) of the first frame used as the
|
||||
* PTS baseline.
|
||||
*/
|
||||
private var streamStartPtsUs: Long? = null
|
||||
|
||||
/**
|
||||
* Monotonic clock timestamp corresponding to streamStartPtsUs, used to map future frames
|
||||
* to real time.
|
||||
*/
|
||||
private var playbackStartRealtimeNs: Long? = null
|
||||
|
||||
/**
|
||||
* Timestamp of the most recently released frame to enforce minimum spacing between consecutive
|
||||
* frames.
|
||||
*/
|
||||
private var lastFrameReleaseTimeNs: Long = Long.MIN_VALUE
|
||||
|
||||
/**
|
||||
* Last presentation timestamp we processed; used to detect wrap-around or backwards jumps.
|
||||
*/
|
||||
private var lastPresentationTimeUs: Long = Long.MIN_VALUE
|
||||
|
||||
init {
|
||||
setVideoFrameRateStabilization(videoFrameRateStabilization)
|
||||
}
|
||||
|
||||
override fun decoderCreated(mediaCodec: MediaCodec, mediaFormat: MediaFormat) {
|
||||
if (DEBUG) Log.v(TAG, "decoderCreated()")
|
||||
if (!surface.isValid) {
|
||||
Log.e(TAG, "Surface invalid")
|
||||
}
|
||||
mediaCodec.configure(mediaFormat, surface, null, 0)
|
||||
resetFrameTiming()
|
||||
}
|
||||
|
||||
private fun releaseOutputBufferWithFrameRateStabilization(
|
||||
mediaCodec: MediaCodec,
|
||||
outIndex: Int,
|
||||
bufferInfo: MediaCodec.BufferInfo
|
||||
) {
|
||||
if (DEBUG) Log.v(TAG, "releaseOutputBufferWithFrameRateStabilization(outIndex=$outIndex)")
|
||||
|
||||
val ptsUs = bufferInfo.presentationTimeUs
|
||||
val nowNs = System.nanoTime()
|
||||
|
||||
if (streamStartPtsUs == null || playbackStartRealtimeNs == null) {
|
||||
// First frame (or after a reset): initialize all timing anchors.
|
||||
streamStartPtsUs = ptsUs
|
||||
playbackStartRealtimeNs = nowNs
|
||||
lastFrameReleaseTimeNs = nowNs
|
||||
lastPresentationTimeUs = ptsUs
|
||||
mediaCodec.releaseOutputBuffer(outIndex, nowNs)
|
||||
return
|
||||
}
|
||||
|
||||
var targetNs = playbackStartRealtimeNs!! + (ptsUs - streamStartPtsUs!!) * 1000L
|
||||
var adjustedNowNs = System.nanoTime()
|
||||
|
||||
if (lastPresentationTimeUs != Long.MIN_VALUE && ptsUs < lastPresentationTimeUs) {
|
||||
// PTS went backwards (e.g. codec reordering). Re-base the clock to avoid negative deltas.
|
||||
streamStartPtsUs = ptsUs
|
||||
playbackStartRealtimeNs = adjustedNowNs
|
||||
targetNs = adjustedNowNs
|
||||
}
|
||||
|
||||
if (lastFrameReleaseTimeNs != Long.MIN_VALUE) {
|
||||
// Ensure we never schedule two frames closer together than the min spacing.
|
||||
targetNs = max(targetNs, lastFrameReleaseTimeNs + MIN_FRAME_SPACING_NS)
|
||||
}
|
||||
|
||||
adjustedNowNs = System.nanoTime()
|
||||
val latenessNs = adjustedNowNs - targetNs
|
||||
|
||||
if (latenessNs >= FRAME_DROP_THRESHOLD_NS) {
|
||||
// Frame is critically late; drop to keep playback responsive.
|
||||
mediaCodec.releaseOutputBuffer(outIndex, false)
|
||||
lastFrameReleaseTimeNs = adjustedNowNs
|
||||
return
|
||||
}
|
||||
|
||||
var correctedTargetNs = targetNs
|
||||
if (latenessNs > 0) {
|
||||
// For mild lateness, shift the playback baseline forward so future frames stay aligned.
|
||||
val correction = minOf(latenessNs, FRAME_DROP_THRESHOLD_NS)
|
||||
playbackStartRealtimeNs = playbackStartRealtimeNs?.plus(correction)
|
||||
correctedTargetNs += correction
|
||||
}
|
||||
|
||||
if (correctedTargetNs <= adjustedNowNs + RENDER_EARLY_MARGIN_NS) {
|
||||
// Already at/behind the target time: render immediately using the current VSYNC.
|
||||
mediaCodec.releaseOutputBuffer(outIndex, true)
|
||||
lastFrameReleaseTimeNs = adjustedNowNs
|
||||
} else {
|
||||
// Still early enough: hand the desired release timestamp to MediaCodec for VSYNC alignment.
|
||||
mediaCodec.releaseOutputBuffer(outIndex, correctedTargetNs)
|
||||
lastFrameReleaseTimeNs = correctedTargetNs
|
||||
}
|
||||
|
||||
lastPresentationTimeUs = ptsUs
|
||||
}
|
||||
|
||||
override fun releaseOutputBuffer(
|
||||
mediaCodec: MediaCodec,
|
||||
outIndex: Int,
|
||||
bufferInfo: MediaCodec.BufferInfo,
|
||||
render: Boolean
|
||||
) {
|
||||
if (DEBUG) Log.v(TAG, "releaseOutputBuffer(outIndex=$outIndex, render=$render)")
|
||||
if (!render || !surface.isValid) {
|
||||
mediaCodec.releaseOutputBuffer(outIndex, false)
|
||||
return
|
||||
}
|
||||
|
||||
if (!hasVideoFrameRateStabilization()) {
|
||||
mediaCodec.releaseOutputBuffer(outIndex, true)
|
||||
} else {
|
||||
releaseOutputBufferWithFrameRateStabilization(mediaCodec, outIndex, bufferInfo)
|
||||
}
|
||||
}
|
||||
|
||||
override fun decoderDestroyed(mediaCodec: MediaCodec) {
|
||||
if (DEBUG) Log.v(TAG, "decoderDestroyed()")
|
||||
resetFrameTiming()
|
||||
}
|
||||
|
||||
private fun resetFrameTiming() {
|
||||
if (DEBUG) Log.v(TAG, "resetFrameTiming()")
|
||||
streamStartPtsUs = null
|
||||
playbackStartRealtimeNs = null
|
||||
lastFrameReleaseTimeNs = Long.MIN_VALUE
|
||||
lastPresentationTimeUs = Long.MIN_VALUE
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val FRAME_DROP_THRESHOLD_NS = TimeUnit.MILLISECONDS.toNanos(80)
|
||||
private val MIN_FRAME_SPACING_NS = TimeUnit.MILLISECONDS.toNanos(1)
|
||||
private val RENDER_EARLY_MARGIN_NS = TimeUnit.MILLISECONDS.toNanos(2)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package com.alexvas.rtsp.parser;
|
||||
|
||||
import android.annotation.SuppressLint;
|
||||
import android.util.Log;
|
||||
|
||||
import androidx.annotation.NonNull;
|
||||
import androidx.annotation.Nullable;
|
||||
|
||||
import androidx.media3.common.util.ParsableBitArray;
|
||||
import androidx.media3.common.util.ParsableByteArray;
|
||||
|
||||
// https://tools.ietf.org/html/rfc3640
|
||||
// +---------+-----------+-----------+---------------+
|
||||
// | RTP | AU Header | Auxiliary | Access Unit |
|
||||
// | Header | Section | Section | Data Section |
|
||||
// +---------+-----------+-----------+---------------+
|
||||
//
|
||||
// <----------RTP Packet Payload----------->
|
||||
@SuppressLint("UnsafeOptInUsageError")
|
||||
public class AacParser extends AudioParser {
|
||||
|
||||
private static final String TAG = AacParser.class.getSimpleName();
|
||||
private static final boolean DEBUG = false;
|
||||
|
||||
private final ParsableBitArray headerScratchBits;
|
||||
private final ParsableByteArray headerScratchBytes;
|
||||
|
||||
private static final int MODE_LBR = 0;
|
||||
private static final int MODE_HBR = 1;
|
||||
|
||||
// Number of bits for AAC AU sizes, indexed by mode (LBR and HBR)
|
||||
private static final int[] NUM_BITS_AU_SIZES = {6, 13};
|
||||
|
||||
// Number of bits for AAC AU index(-delta), indexed by mode (LBR and HBR)
|
||||
private static final int[] NUM_BITS_AU_INDEX = {2, 3};
|
||||
|
||||
// Frame Sizes for AAC AU fragments, indexed by mode (LBR and HBR)
|
||||
private static final int[] FRAME_SIZES = {63, 8191};
|
||||
|
||||
private final int _aacMode;
|
||||
private boolean completeFrameIndicator = true;
|
||||
|
||||
public AacParser(@NonNull String aacMode) {
|
||||
_aacMode = aacMode.equalsIgnoreCase("AAC-lbr") ? MODE_LBR : MODE_HBR;
|
||||
|
||||
headerScratchBits = new ParsableBitArray();
|
||||
headerScratchBytes = new ParsableByteArray();
|
||||
}
|
||||
|
||||
@Override
|
||||
@Nullable
|
||||
public byte[] processRtpPacketAndGetSample(@NonNull byte[] data, int length) {
|
||||
if (DEBUG)
|
||||
Log.v(TAG, "processRtpPacketAndGetSample(length=" + length + ")");
|
||||
int auHeadersCount = 1;
|
||||
int numBitsAuSize = NUM_BITS_AU_SIZES[_aacMode];
|
||||
int numBitsAuIndex = NUM_BITS_AU_INDEX[_aacMode];
|
||||
|
||||
ParsableByteArray packet = new ParsableByteArray(data, length);
|
||||
|
||||
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- .. -+-+-+-+-+-+-+-+-+-+
|
||||
// |AU-headers-length|AU-header|AU-header| |AU-header|padding|
|
||||
// | | (1) | (2) | | (n) | bits |
|
||||
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- .. -+-+-+-+-+-+-+-+-+-+
|
||||
int auHeadersLength = packet.readShort();//((data[0] & 0xFF) << 8) | (data[1] & 0xFF);
|
||||
int auHeadersLengthBytes = (auHeadersLength + 7) / 8;
|
||||
|
||||
headerScratchBytes.reset(auHeadersLengthBytes);
|
||||
packet.readBytes(headerScratchBytes.getData(), 0, auHeadersLengthBytes);
|
||||
headerScratchBits.reset(headerScratchBytes.getData());
|
||||
|
||||
int bitsAvailable = auHeadersLength - (numBitsAuSize + numBitsAuIndex);
|
||||
|
||||
if (bitsAvailable > 0) {// && (numBitsAuSize + numBitsAuSize) > 0) {
|
||||
auHeadersCount += bitsAvailable / (numBitsAuSize + numBitsAuIndex);
|
||||
}
|
||||
|
||||
if (auHeadersCount == 1) {
|
||||
int auSize = headerScratchBits.readBits(numBitsAuSize);
|
||||
int auIndex = headerScratchBits.readBits(numBitsAuIndex);
|
||||
|
||||
if (completeFrameIndicator) {
|
||||
if (auIndex == 0) {
|
||||
if (packet.bytesLeft() == auSize) {
|
||||
return handleSingleAacFrame(packet);
|
||||
|
||||
} else {
|
||||
// handleFragmentationAacFrame(packet, auSize);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// handleFragmentationAacFrame(packet, auSize);
|
||||
}
|
||||
|
||||
} else {
|
||||
if (completeFrameIndicator) {
|
||||
// handleMultipleAacFrames(packet, auHeadersLength);
|
||||
}
|
||||
}
|
||||
// byte[] auHeader = new byte[length-2-auHeadersLengthBytes];
|
||||
// System.arraycopy(data,2-auHeadersLengthBytes, auHeader,0, auHeader.length);
|
||||
// if (DEBUG)
|
||||
// Log.d(TAG, "AU headers size: " + auHeadersLengthBytes + ", AU headers: " + auHeadersCount + ", sample length: " + auHeader.length);
|
||||
// return auHeader;
|
||||
return new byte[0];
|
||||
}
|
||||
|
||||
private byte[] handleSingleAacFrame(ParsableByteArray packet) {
|
||||
int length = packet.bytesLeft();
|
||||
byte[] data = new byte[length];
|
||||
System.arraycopy(packet.getData(), packet.getPosition(), data,0, data.length);
|
||||
return data;
|
||||
}
|
||||
|
||||
// private static final class AUHeader {
|
||||
// private int size;
|
||||
// private int index;
|
||||
//
|
||||
// public AUHeader(int size, int index) {
|
||||
// this.size = size;
|
||||
// this.index = index;
|
||||
// }
|
||||
//
|
||||
// public int size() { return size; }
|
||||
//
|
||||
// public int index() { return index; }
|
||||
// }
|
||||
|
||||
// /**
|
||||
// * Stores the consecutive fragment AU to reconstruct an AAC-Frame
|
||||
// */
|
||||
// private static final class FragmentedAacFrame {
|
||||
// public byte[] auData;
|
||||
// public int auLength;
|
||||
// public int auSize;
|
||||
//
|
||||
// private int sequence;
|
||||
//
|
||||
// public FragmentedAacFrame(int frameSize) {
|
||||
// // Initialize data
|
||||
// auData = new byte[frameSize];
|
||||
// sequence = -1;
|
||||
// }
|
||||
//
|
||||
// /**
|
||||
// * Resets the buffer, clearing any data that it holds.
|
||||
// */
|
||||
// public void reset() {
|
||||
// auLength = 0;
|
||||
// auSize = 0;
|
||||
// sequence = -1;
|
||||
// }
|
||||
//
|
||||
// public void sequence(int sequence) {
|
||||
// this.sequence = sequence;
|
||||
// }
|
||||
//
|
||||
// public int sequence() {
|
||||
// return sequence;
|
||||
// }
|
||||
//
|
||||
// /**
|
||||
// * Called to add a fragment unit to fragmented AU.
|
||||
// *
|
||||
// * @param fragment Holds the data of fragment unit being passed.
|
||||
// * @param offset The offset of the data in {@code fragment}.
|
||||
// * @param limit The limit (exclusive) of the data in {@code fragment}.
|
||||
// */
|
||||
// public void appendFragment(byte[] fragment, int offset, int limit) {
|
||||
// if (auSize == 0) {
|
||||
// auSize = limit;
|
||||
// } else if (auSize != limit) {
|
||||
// reset();
|
||||
// }
|
||||
//
|
||||
// if (auData.length < auLength + limit) {
|
||||
// auData = Arrays.copyOf(auData, (auLength + limit) * 2);
|
||||
// }
|
||||
//
|
||||
// System.arraycopy(fragment, offset, auData, auLength, limit);
|
||||
// auLength += limit;
|
||||
// }
|
||||
//
|
||||
// public boolean isCompleted() {
|
||||
// return auSize == auLength;
|
||||
// }
|
||||
// }
|
||||
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package com.alexvas.rtsp.parser
|
||||
|
||||
abstract class AudioParser {
|
||||
abstract fun processRtpPacketAndGetSample(
|
||||
data: ByteArray,
|
||||
length: Int
|
||||
): ByteArray?
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package com.alexvas.rtsp.parser
|
||||
|
||||
class G711Parser() : AudioParser() {
|
||||
override fun processRtpPacketAndGetSample(
|
||||
data: ByteArray,
|
||||
length: Int
|
||||
): ByteArray? {
|
||||
val g711Payload = data.copyOfRange(0, length)
|
||||
return g711Payload
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package com.alexvas.rtsp.parser
|
||||
|
||||
import android.util.Log
|
||||
import com.alexvas.utils.VideoCodecUtils
|
||||
import com.alexvas.utils.VideoCodecUtils.getH264NalUnitTypeString
|
||||
import java.io.ByteArrayOutputStream
|
||||
|
||||
class RtpH264Parser: RtpParser() {
|
||||
|
||||
private var stream = ByteArrayOutputStream()
|
||||
|
||||
override fun processRtpPacketAndGetNalUnit(data: ByteArray, length: Int, marker: Boolean): ByteArray? {
|
||||
if (DEBUG) Log.v(TAG, "processRtpPacketAndGetNalUnit(data.size=${data.size}, length=$length, marker=$marker)")
|
||||
|
||||
val nalType = (data[0].toInt() and 0x1F).toByte()
|
||||
val packFlag = data[1].toInt() and 0xC0
|
||||
var nalUnit: ByteArray? = null
|
||||
|
||||
if (DEBUG)
|
||||
Log.d(TAG, "\t\tNAL type: ${getH264NalUnitTypeString(nalType)}, pack flag: 0x${Integer.toHexString(packFlag).lowercase()}")
|
||||
|
||||
when (nalType) {
|
||||
VideoCodecUtils.NAL_STAP_A, VideoCodecUtils.NAL_STAP_B -> {
|
||||
// Not supported
|
||||
}
|
||||
|
||||
VideoCodecUtils.NAL_MTAP16, VideoCodecUtils.NAL_MTAP24 -> {
|
||||
// Not supported
|
||||
}
|
||||
|
||||
VideoCodecUtils.NAL_FU_A -> {
|
||||
when (packFlag) {
|
||||
0x80 -> {
|
||||
addStartFragmentedPacket(data, length)
|
||||
}
|
||||
|
||||
0x00 -> {
|
||||
if (marker) {
|
||||
// Sometimes 0x40 end packet is not arrived. Use marker bit in this case
|
||||
// to finish fragmented packet.
|
||||
nalUnit = addEndFragmentedPacketAndCombine(data, length)
|
||||
} else {
|
||||
addMiddleFragmentedPacket(data, length)
|
||||
}
|
||||
}
|
||||
|
||||
0x40 -> {
|
||||
nalUnit = addEndFragmentedPacketAndCombine(data, length)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
VideoCodecUtils.NAL_FU_B -> {
|
||||
// Not supported
|
||||
}
|
||||
|
||||
else -> {
|
||||
nalUnit = processSingleFramePacket(data, length)
|
||||
clearFragmentedBuffer()
|
||||
if (DEBUG) Log.d(TAG, "Single NAL (${nalUnit.size})")
|
||||
}
|
||||
}
|
||||
nalUnit?.let { stream.write(it) }
|
||||
if (marker) {
|
||||
val result = stream.toByteArray()
|
||||
stream = ByteArrayOutputStream()
|
||||
return result
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun addStartFragmentedPacket(data: ByteArray, length: Int) {
|
||||
if (DEBUG) Log.v(TAG, "addStartFragmentedPacket(data.size=${data.size}, length=$length)")
|
||||
fragmentedPackets = 0
|
||||
fragmentedBufferLength = length - 1
|
||||
fragmentedBuffer[0] = ByteArray(fragmentedBufferLength).apply {
|
||||
this[0] = ((data[0].toInt() and 0xE0) or (data[1].toInt() and 0x1F)).toByte()
|
||||
}
|
||||
System.arraycopy(data, 2, fragmentedBuffer[0]!!, 1, length - 2)
|
||||
}
|
||||
|
||||
private fun addMiddleFragmentedPacket(data: ByteArray, length: Int) {
|
||||
if (DEBUG) Log.v(TAG, "addMiddleFragmentedPacket(data.size=${data.size}, length=$length)")
|
||||
fragmentedPackets++
|
||||
if (fragmentedPackets >= fragmentedBuffer.size) {
|
||||
Log.e(TAG, "Too many middle packets. No NAL FU_A end packet received. Skipped RTP packet.")
|
||||
fragmentedBuffer[0] = null
|
||||
} else {
|
||||
fragmentedBufferLength += length - 2
|
||||
fragmentedBuffer[fragmentedPackets] = ByteArray(length - 2)
|
||||
System.arraycopy(data, 2, fragmentedBuffer[fragmentedPackets]!!, 0, length - 2)
|
||||
}
|
||||
}
|
||||
|
||||
private fun addEndFragmentedPacketAndCombine(data: ByteArray, length: Int): ByteArray? {
|
||||
if (DEBUG) Log.v(TAG, "addEndFragmentedPacketAndCombine(data.size=${data.size}, length=$length)")
|
||||
var nalUnit: ByteArray? = null
|
||||
var tmpLen: Int
|
||||
if (fragmentedBuffer[0] == null) {
|
||||
Log.e(TAG, "No NAL FU_A start packet received. Skipped RTP packet.")
|
||||
} else {
|
||||
nalUnit = ByteArray(fragmentedBufferLength + length + 2)
|
||||
writeNalPrefix0001(nalUnit)
|
||||
tmpLen = 4
|
||||
// Write start and middle packets
|
||||
for (i in 0 until fragmentedPackets + 1) {
|
||||
fragmentedBuffer[i]!!.apply {
|
||||
System.arraycopy(
|
||||
this,
|
||||
0,
|
||||
nalUnit,
|
||||
tmpLen,
|
||||
this.size
|
||||
)
|
||||
tmpLen += this.size
|
||||
}
|
||||
}
|
||||
// Write end packet
|
||||
System.arraycopy(data, 2, nalUnit, tmpLen, length - 2)
|
||||
clearFragmentedBuffer()
|
||||
if (DEBUG) Log.d(TAG, "Fragmented NAL (${nalUnit.size})")
|
||||
}
|
||||
return nalUnit
|
||||
}
|
||||
|
||||
private fun clearFragmentedBuffer() {
|
||||
if (DEBUG) Log.v(TAG, "clearFragmentedBuffer()")
|
||||
for (i in 0 until fragmentedPackets + 1) {
|
||||
fragmentedBuffer[i] = null
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val TAG: String = RtpH264Parser::class.java.simpleName
|
||||
private const val DEBUG = false
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package com.alexvas.rtsp.parser
|
||||
|
||||
import android.util.Log
|
||||
import java.io.ByteArrayOutputStream
|
||||
|
||||
class RtpH265Parser: RtpParser() {
|
||||
|
||||
private var stream = ByteArrayOutputStream()
|
||||
|
||||
override fun processRtpPacketAndGetNalUnit(data: ByteArray, length: Int, marker: Boolean): ByteArray? {
|
||||
if (DEBUG) Log.v(TAG, "processRtpPacketAndGetNalUnit(length=$length, marker=$marker)")
|
||||
|
||||
// NAL Unit Header.type (RFC7798 Section 1.1.4).
|
||||
val nalType = ((data[0].toInt() shr 1) and 0x3F).toByte()
|
||||
var nalUnit: ByteArray? = null
|
||||
|
||||
// Log.d(TAG, "\t\tNAL type: ${VideoCodecUtils.getH265NalUnitTypeString(nalType)}")
|
||||
|
||||
if (nalType in 0..<RTP_PACKET_TYPE_AP) {
|
||||
nalUnit = processSingleFramePacket(data, length)
|
||||
clearFragmentedBuffer()
|
||||
if (DEBUG) Log.d(TAG, "Single NAL (${nalUnit.size})")
|
||||
} else if (nalType == RTP_PACKET_TYPE_AP) {
|
||||
// TODO: Support AggregationPacket mode.
|
||||
Log.e(TAG, "need to implement processAggregationPacket")
|
||||
} else if (nalType == RTP_PACKET_TYPE_FU) {
|
||||
nalUnit = processFragmentationUnitPacket(data, length, marker)
|
||||
} else {
|
||||
Log.e(TAG, "RTP H265 payload type [${nalType}] not supported.")
|
||||
}
|
||||
nalUnit?.let { stream.write(it) }
|
||||
if (marker) {
|
||||
val result = stream.toByteArray()
|
||||
stream = ByteArrayOutputStream()
|
||||
return result
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun processFragmentationUnitPacket(data: ByteArray, length: Int, marker: Boolean): ByteArray? {
|
||||
if (DEBUG) Log.v(TAG, "processFragmentationUnitPacket(length=$length, marker=$marker)")
|
||||
|
||||
val fuHeader = data[2].toInt()
|
||||
val isFirstFuPacket = (fuHeader and 0x80) > 0
|
||||
val isLastFuPacket = (fuHeader and 0x40) > 0
|
||||
|
||||
if (isFirstFuPacket) {
|
||||
addStartFragmentedPacket(data, length)
|
||||
} else if (isLastFuPacket || marker) {
|
||||
return addEndFragmentedPacketAndCombine(data, length)
|
||||
} else {
|
||||
addMiddleFragmentedPacket(data, length)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun addStartFragmentedPacket(data: ByteArray, length: Int) {
|
||||
if (DEBUG) Log.v(TAG, "addStartFragmentedPacket(data.size=${data.size}, length=$length)")
|
||||
fragmentedPackets = 0
|
||||
fragmentedBufferLength = length - 1
|
||||
fragmentedBuffer[0] = ByteArray(fragmentedBufferLength).apply {
|
||||
|
||||
val tid = (data[1].toInt() and 0x7)
|
||||
val fuHeader = data[2].toInt()
|
||||
val nalUnitType = fuHeader and 0x3F
|
||||
|
||||
// Convert RTP header into HEVC NAL Unit header accoding to RFC7798 Section 1.1.4.
|
||||
// RTP byte 0: ignored.
|
||||
// RTP byte 1: repurposed as HEVC HALU byte 0, copy NALU type.
|
||||
// RTP Byte 2: repurposed as HEVC HALU byte 1, layerId required to be zero, copying only tid.
|
||||
// Set data position from byte 1 as byte 0 is ignored.
|
||||
this[0] = (((nalUnitType shl 1) and 0x7F).toByte())
|
||||
this[1] = tid.toByte()
|
||||
}
|
||||
System.arraycopy(data, 3, fragmentedBuffer[0]!!, 2, length - 3)
|
||||
}
|
||||
|
||||
private fun addMiddleFragmentedPacket(data: ByteArray, length: Int) {
|
||||
if (DEBUG) Log.v(TAG, "addMiddleFragmentedPacket(data.size=${data.size}, length=$length)")
|
||||
fragmentedPackets++
|
||||
if (fragmentedPackets >= fragmentedBuffer.size) {
|
||||
Log.e(TAG, "Too many middle packets. No RTP_PACKET_TYPE_FU end packet received. Skipped RTP packet.")
|
||||
fragmentedBuffer[0] = null
|
||||
} else {
|
||||
fragmentedBufferLength += length - 3
|
||||
fragmentedBuffer[fragmentedPackets] = ByteArray(length - 3).apply {
|
||||
System.arraycopy(data, 3, this, 0, length - 3)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun addEndFragmentedPacketAndCombine(data: ByteArray, length: Int): ByteArray? {
|
||||
if (DEBUG) Log.v(TAG, "addEndFragmentedPacketAndCombine(data.size=${data.size}, length=$length)")
|
||||
var nalUnit: ByteArray? = null
|
||||
if (fragmentedBuffer[0] == null) {
|
||||
Log.e(TAG, "No NAL FU_A start packet received. Skipped RTP packet.")
|
||||
} else {
|
||||
nalUnit = ByteArray(fragmentedBufferLength + length + 3)
|
||||
writeNalPrefix0001(nalUnit)
|
||||
var tmpLen = 4
|
||||
// Write start and middle packets
|
||||
for (i in 0 until fragmentedPackets + 1) {
|
||||
fragmentedBuffer[i]!!.apply {
|
||||
System.arraycopy(
|
||||
this,
|
||||
0,
|
||||
nalUnit,
|
||||
tmpLen,
|
||||
this.size
|
||||
)
|
||||
tmpLen += this.size
|
||||
}
|
||||
}
|
||||
// Write end packet
|
||||
System.arraycopy(data, 3, nalUnit, tmpLen, length - 3)
|
||||
clearFragmentedBuffer()
|
||||
if (DEBUG) Log.d(TAG, "Fragmented NAL (${nalUnit.size})")
|
||||
}
|
||||
return nalUnit
|
||||
}
|
||||
|
||||
private fun clearFragmentedBuffer() {
|
||||
if (DEBUG) Log.v(TAG, "clearFragmentedBuffer()")
|
||||
for (i in 0 until fragmentedPackets + 1) {
|
||||
fragmentedBuffer[i] = null
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val TAG: String = RtpH265Parser::class.java.simpleName
|
||||
private const val DEBUG = false
|
||||
|
||||
/** Aggregation Packet. RFC7798 Section 4.4.2. */
|
||||
private const val RTP_PACKET_TYPE_AP: Byte = 48
|
||||
/** Fragmentation Unit. RFC7798 Section 4.4.3. */
|
||||
private const val RTP_PACKET_TYPE_FU: Byte = 49
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package com.alexvas.rtsp.parser;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
import androidx.annotation.NonNull;
|
||||
import androidx.annotation.Nullable;
|
||||
|
||||
import com.alexvas.utils.NetUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
public class RtpHeaderParser {
|
||||
|
||||
private static final String TAG = RtpHeaderParser.class.getSimpleName();
|
||||
private static final boolean DEBUG = false;
|
||||
|
||||
private final static int RTP_HEADER_SIZE = 12;
|
||||
|
||||
public static class RtpHeader {
|
||||
public int version;
|
||||
public int padding;
|
||||
public int extension;
|
||||
public int cc;
|
||||
public int marker;
|
||||
public int payloadType;
|
||||
public int sequenceNumber;
|
||||
public long timeStamp;
|
||||
public long ssrc;
|
||||
public int payloadSize;
|
||||
|
||||
public long getTimestampMsec() {
|
||||
return (long)(timeStamp * 11.111111);
|
||||
}
|
||||
|
||||
// If RTP header found, return 4 bytes of the header
|
||||
private static boolean searchForNextRtpHeader(@NonNull InputStream inputStream, @NonNull byte[] header /*out*/) throws IOException {
|
||||
if (header.length < 4)
|
||||
throw new IOException("Invalid allocated buffer size");
|
||||
|
||||
int bytesRemaining = 100000; // 100 KB max to check
|
||||
boolean foundFirstByte = false;
|
||||
boolean foundSecondByte = false;
|
||||
byte[] oneByte = new byte[1];
|
||||
// Search for {0x24, 0x00}
|
||||
do {
|
||||
if (bytesRemaining-- < 0)
|
||||
return false;
|
||||
// Read 1 byte
|
||||
NetUtils.readData(inputStream, oneByte, 0, 1);
|
||||
if (foundFirstByte) {
|
||||
// Found 0x24. Checking for 0x00-0x02.
|
||||
if (oneByte[0] == 0x00)
|
||||
foundSecondByte = true;
|
||||
else
|
||||
foundFirstByte = false;
|
||||
}
|
||||
if (!foundFirstByte && oneByte[0] == 0x24) {
|
||||
// Found 0x24
|
||||
foundFirstByte = true;
|
||||
}
|
||||
} while (!foundSecondByte);
|
||||
header[0] = 0x24;
|
||||
header[1] = oneByte[0];
|
||||
// Read 2 bytes more (packet size)
|
||||
NetUtils.readData(inputStream, header, 2, 2);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
private static RtpHeader parseData(@NonNull byte[] header, int packetSize) {
|
||||
RtpHeader rtpHeader = new RtpHeader();
|
||||
rtpHeader.version = (header[0] & 0xFF) >> 6;
|
||||
if (rtpHeader.version != 2) {
|
||||
if (DEBUG)
|
||||
Log.e(TAG,"Not a RTP packet (" + rtpHeader.version + ")");
|
||||
return null;
|
||||
}
|
||||
|
||||
// 80 60 40 91 fd ab d4 2a
|
||||
// 80 c8 00 06
|
||||
rtpHeader.padding = (header[0] & 0x20) >> 5; // 0b00100100
|
||||
rtpHeader.extension = (header[0] & 0x10) >> 4;
|
||||
rtpHeader.marker = (header[1] & 0x80) >> 7;
|
||||
rtpHeader.payloadType = header[1] & 0x7F;
|
||||
rtpHeader.sequenceNumber = (header[3] & 0xFF) + ((header[2] & 0xFF) << 8);
|
||||
rtpHeader.timeStamp = (header[7] & 0xFF) + ((header[6] & 0xFF) << 8) + ((header[5] & 0xFF) << 16) + ((header[4] & 0xFF) << 24) & 0xffffffffL;
|
||||
rtpHeader.ssrc = (header[7] & 0xFF) + ((header[6] & 0xFF) << 8) + ((header[5] & 0xFF) << 16) + ((header[4] & 0xFF) << 24) & 0xffffffffL;
|
||||
rtpHeader.payloadSize = packetSize - RTP_HEADER_SIZE;
|
||||
return rtpHeader;
|
||||
}
|
||||
|
||||
private static int getPacketSize(@NonNull byte[] header) {
|
||||
int packetSize = ((header[2] & 0xFF) << 8) | (header[3] & 0xFF);
|
||||
if (DEBUG)
|
||||
Log.d(TAG, "Packet size: " + packetSize);
|
||||
return packetSize;
|
||||
}
|
||||
|
||||
public void dumpHeader() {
|
||||
Log.d("RTP","\t\tRTP header version: " + version
|
||||
+ ", padding: " + padding
|
||||
+ ", ext: " + extension
|
||||
+ ", cc: " + cc
|
||||
+ ", marker: " + marker
|
||||
+ ", payload type: " + payloadType
|
||||
+ ", seq num: " + sequenceNumber
|
||||
+ ", ts: " + timeStamp
|
||||
+ ", ssrc: " + ssrc
|
||||
+ ", payload size: " + payloadSize);
|
||||
}
|
||||
}
|
||||
|
||||
@Nullable
|
||||
public static RtpHeader readHeader(@NonNull InputStream inputStream) throws IOException {
|
||||
// 24 01 00 1c 80 c8 00 06 7f 1d d2 c4
|
||||
// 24 01 00 1c 80 c8 00 06 13 9b cf 60
|
||||
// 24 02 01 12 80 e1 01 d2 00 07 43 f0
|
||||
byte[] header = new byte[RTP_HEADER_SIZE];
|
||||
// Skip 4 bytes (TCP only). No those bytes in UDP.
|
||||
NetUtils.readData(inputStream, header, 0, 4);
|
||||
if (DEBUG && header[0] == 0x24)
|
||||
Log.d(TAG, header[1] == 0 ? "RTP packet" : "RTCP packet");
|
||||
|
||||
int packetSize = RtpHeader.getPacketSize(header);
|
||||
if (DEBUG)
|
||||
Log.d(TAG, "Packet size: " + packetSize);
|
||||
|
||||
if (NetUtils.readData(inputStream, header, 0, header.length) == header.length) {
|
||||
RtpHeader rtpHeader = RtpHeader.parseData(header, packetSize);
|
||||
if (rtpHeader == null) {
|
||||
// Header not found. Possible keep-alive response. Search for another RTP header.
|
||||
boolean foundHeader = RtpHeader.searchForNextRtpHeader(inputStream, header);
|
||||
if (foundHeader) {
|
||||
packetSize = RtpHeader.getPacketSize(header);
|
||||
if (NetUtils.readData(inputStream, header, 0, header.length) == header.length)
|
||||
return RtpHeader.parseData(header, packetSize);
|
||||
}
|
||||
} else {
|
||||
return rtpHeader;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package com.alexvas.rtsp.parser
|
||||
|
||||
abstract class RtpParser {
|
||||
|
||||
abstract fun processRtpPacketAndGetNalUnit(data: ByteArray, length: Int, marker: Boolean): ByteArray?
|
||||
|
||||
// TODO Use already allocated buffer with RtpPacket.MAX_SIZE = 65507
|
||||
// Used only for fragmented packets
|
||||
protected val fragmentedBuffer = arrayOfNulls<ByteArray>(1024)
|
||||
protected var fragmentedBufferLength = 0
|
||||
protected var fragmentedPackets = 0
|
||||
|
||||
protected fun writeNalPrefix0001(buffer: ByteArray) {
|
||||
buffer[0] = 0x00
|
||||
buffer[1] = 0x00
|
||||
buffer[2] = 0x00
|
||||
buffer[3] = 0x01
|
||||
}
|
||||
|
||||
protected fun processSingleFramePacket(data: ByteArray, length: Int): ByteArray {
|
||||
return ByteArray(4 + length).apply {
|
||||
writeNalPrefix0001(this)
|
||||
System.arraycopy(data, 0, this, 4, length)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.alexvas.rtsp.widget
|
||||
|
||||
/**
|
||||
* Listener for getting RTSP status update.
|
||||
*/
|
||||
interface RtspStatusListener {
|
||||
fun onRtspStatusConnecting() {}
|
||||
fun onRtspStatusConnected() {}
|
||||
fun onRtspStatusDisconnecting() {}
|
||||
fun onRtspStatusDisconnected() {}
|
||||
fun onRtspStatusFailedUnauthorized() {}
|
||||
fun onRtspStatusFailed(message: String?) {}
|
||||
fun onRtspFirstFrameRendered() {}
|
||||
fun onRtspFrameSizeChanged(width: Int, height: Int) {}
|
||||
}
|
||||
|
||||
/**
|
||||
* Listener for getting RTSP raw data, e.g. for recording.
|
||||
*/
|
||||
interface RtspDataListener {
|
||||
fun onRtspDataVideoNalUnitReceived(data: ByteArray, offset: Int, length: Int, timestamp: Long) {}
|
||||
fun onRtspDataAudioSampleReceived(data: ByteArray, offset: Int, length: Int, timestamp: Long) {}
|
||||
fun onRtspDataApplicationDataReceived(data: ByteArray, offset: Int, length: Int, timestamp: Long) {}
|
||||
}
|
||||
@@ -0,0 +1,623 @@
|
||||
package com.alexvas.rtsp.widget
|
||||
|
||||
import android.annotation.SuppressLint
|
||||
import android.media.MediaFormat
|
||||
import android.net.Uri
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.util.Log
|
||||
import androidx.media3.container.NalUnitUtil
|
||||
import com.alexvas.rtsp.RtspClient
|
||||
import com.alexvas.rtsp.RtspClient.SdpInfo
|
||||
import com.alexvas.rtsp.codec.AudioCodecType
|
||||
import com.alexvas.rtsp.codec.AudioDecodeThread
|
||||
import com.alexvas.rtsp.codec.AudioFrameQueue
|
||||
import com.alexvas.rtsp.codec.FrameQueue
|
||||
import com.alexvas.rtsp.codec.VideoCodecType
|
||||
import com.alexvas.rtsp.codec.VideoDecodeThread
|
||||
import com.alexvas.rtsp.codec.VideoDecodeThread.DecoderType
|
||||
import com.alexvas.rtsp.codec.VideoDecodeThread.VideoDecoderListener
|
||||
import com.alexvas.rtsp.codec.VideoFrameQueue
|
||||
import com.alexvas.utils.NetUtils
|
||||
import com.alexvas.utils.VideoCodecUtils
|
||||
import org.jcodec.codecs.h264.io.model.SeqParameterSet
|
||||
import org.jcodec.codecs.h264.io.model.VUIParameters
|
||||
import java.net.Socket
|
||||
import java.nio.ByteBuffer
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import kotlin.math.min
|
||||
|
||||
class RtspProcessor(
|
||||
private var onVideoDecoderCreateRequested: ((
|
||||
videoMimeType: String,
|
||||
videoRotation: Int, // 0, 90, 180, 270
|
||||
videoFrameQueue: VideoFrameQueue,
|
||||
videoDecoderListener: VideoDecoderListener,
|
||||
videoDecoderType: DecoderType,
|
||||
videoFrameRateStabilization: Boolean,
|
||||
) -> VideoDecodeThread)
|
||||
) {
|
||||
|
||||
class Statistics {
|
||||
var videoDecoderType = DecoderType.HARDWARE
|
||||
var videoDecoderName: String? = null
|
||||
var videoDecoderLatencyMsec = -1
|
||||
var networkLatencyMsec = -1
|
||||
}
|
||||
|
||||
private lateinit var uri: Uri
|
||||
private var username: String? = null
|
||||
private var password: String? = null
|
||||
private var userAgent: String? = null
|
||||
private var requestVideo = true
|
||||
private var requestAudio = true
|
||||
private var requestApplication = false
|
||||
private var rtspThread: RtspThread? = null
|
||||
// 本地修改（相对上游）：队列容量 60 -> 20，压缩解码侧缓冲。解码线程无播放时钟、
|
||||
// 按最快速度取帧，队列正常情况下近乎为空，容量仅作抖动兜底，调小不丢帧但压低积压延迟。
|
||||
private var videoFrameQueue = VideoFrameQueue(20)
|
||||
private var audioFrameQueue = AudioFrameQueue(10)
|
||||
private var videoDecodeThread: VideoDecodeThread? = null
|
||||
private var audioDecodeThread: AudioDecodeThread? = null
|
||||
private val uiHandler = Handler(Looper.getMainLooper())
|
||||
private var videoMimeType: String = "video/avc"
|
||||
private var audioMimeType: String = ""
|
||||
private var audioSampleRate: Int = 0
|
||||
private var audioChannelCount: Int = 0
|
||||
private var audioCodecConfig: ByteArray? = null
|
||||
private var firstFrameRendered = false
|
||||
var statistics = Statistics()
|
||||
get() {
|
||||
videoDecodeThread?.let { decoder ->
|
||||
field.apply {
|
||||
networkLatencyMsec = decoder.getCurrentNetworkLatencyMsec()
|
||||
videoDecoderLatencyMsec = decoder.getCurrentVideoDecoderLatencyMsec()
|
||||
videoDecoderType = decoder.getCurrentVideoDecoderType()
|
||||
videoDecoderName = decoder.getCurrentVideoDecoderName()
|
||||
}
|
||||
}
|
||||
return field
|
||||
}
|
||||
private set
|
||||
|
||||
/** Read and connect timeout for socket in msec. */
|
||||
private var socketTimeoutMsec: Int = 5000
|
||||
|
||||
/**
|
||||
* Show more debug info on console on runtime.
|
||||
*/
|
||||
var debug = false
|
||||
|
||||
/**
|
||||
* Video rotation in degrees. Allowed values: 0, 90, 180, 270.
|
||||
* Note that not all hardware video decoders support rotation.
|
||||
*/
|
||||
var videoRotation = 0
|
||||
set(value) {
|
||||
if (value == 0 || value == 90 || value == 180 || value == 270)
|
||||
field = value
|
||||
}
|
||||
|
||||
/**
|
||||
* Requested video decoder type.
|
||||
*/
|
||||
var videoDecoderType = DecoderType.HARDWARE
|
||||
|
||||
/**
|
||||
* Try to modify SPS frame coming from camera with low-latency parameters to decrease video
|
||||
* decoding latency.
|
||||
* If SPS frame param num_ref_frames is equal to 1 or more, set it to 0. That should decrease
|
||||
* decoder latency by 2x times on some hardware decoders.
|
||||
*/
|
||||
var experimentalUpdateSpsFrameWithLowLatencyParams = false
|
||||
|
||||
/**
|
||||
* Enables the playback smoothing logic inside the video decoder.
|
||||
*/
|
||||
var videoFrameRateStabilization: Boolean = false
|
||||
set(value) {
|
||||
field = value
|
||||
videoDecodeThread?.setVideoFrameRateStabilization(value)
|
||||
}
|
||||
|
||||
/**
|
||||
* Status listener for getting RTSP event updates.
|
||||
*/
|
||||
var statusListener: RtspStatusListener? = null
|
||||
|
||||
/**
|
||||
* Listener for getting raw data, e.g. for recording.
|
||||
*/
|
||||
var dataListener: RtspDataListener? = null
|
||||
|
||||
private val proxyClientListener = object: RtspClient.RtspClientListener {
|
||||
|
||||
override fun onRtspConnecting() {
|
||||
if (DEBUG) Log.v(TAG, "onRtspConnecting()")
|
||||
uiHandler.post {
|
||||
statusListener?.onRtspStatusConnecting()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onRtspConnected(sdpInfo: SdpInfo) {
|
||||
if (DEBUG) Log.v(TAG, "onRtspConnected()")
|
||||
if (sdpInfo.videoTrack != null) {
|
||||
videoFrameQueue.clear()
|
||||
when (sdpInfo.videoTrack?.videoCodec) {
|
||||
RtspClient.VIDEO_CODEC_H264 -> videoMimeType = MediaFormat.MIMETYPE_VIDEO_AVC
|
||||
RtspClient.VIDEO_CODEC_H265 -> videoMimeType = MediaFormat.MIMETYPE_VIDEO_HEVC
|
||||
}
|
||||
when (sdpInfo.audioTrack?.audioCodec) {
|
||||
RtspClient.AUDIO_CODEC_AAC -> audioMimeType = MediaFormat.MIMETYPE_AUDIO_AAC
|
||||
RtspClient.AUDIO_CODEC_OPUS -> audioMimeType = MediaFormat.MIMETYPE_AUDIO_OPUS
|
||||
RtspClient.AUDIO_CODEC_G711_ULAW -> audioMimeType = MediaFormat.MIMETYPE_AUDIO_G711_MLAW
|
||||
RtspClient.AUDIO_CODEC_G711_ALAW -> audioMimeType = MediaFormat.MIMETYPE_AUDIO_G711_ALAW
|
||||
}
|
||||
val sps: ByteArray? = sdpInfo.videoTrack?.sps
|
||||
val pps: ByteArray? = sdpInfo.videoTrack?.pps
|
||||
// Initialize decoder
|
||||
@SuppressLint("UnsafeOptInUsageError")
|
||||
if (sps != null && pps != null) {
|
||||
// 本地修改（相对上游）：H.265 的 CSD 参数集顺序必须为 VPS→SPS→PPS（ISO/IEC 14496-15），
|
||||
// 上游拼接顺序 sps+pps+vps 会导致 H.265 起播黑屏。H.264 无 VPS，顺序不变。
|
||||
val vps: ByteArray = sdpInfo.videoTrack?.vps ?: ByteArray(0)
|
||||
val data = ByteArray(sps.size + pps.size + vps.size)
|
||||
var offset = 0
|
||||
vps.copyInto(data, offset, 0, vps.size)
|
||||
offset += vps.size
|
||||
sps.copyInto(data, offset, 0, sps.size)
|
||||
offset += sps.size
|
||||
pps.copyInto(data, offset, 0, pps.size)
|
||||
offset += pps.size
|
||||
videoFrameQueue.push(
|
||||
FrameQueue.VideoFrame(
|
||||
// 本地修改（相对上游）：按实际 MIME 标记 codecType，上游硬编码 H264
|
||||
// 会导致 H.265 流无法从 SPS 解析宽高（解码线程按 codecType 决定解析方式）。
|
||||
if (videoMimeType == MediaFormat.MIMETYPE_VIDEO_HEVC) VideoCodecType.H265 else VideoCodecType.H264,
|
||||
isKeyframe = true,
|
||||
data,
|
||||
0,
|
||||
data.size,
|
||||
0
|
||||
)
|
||||
)
|
||||
try {
|
||||
val startNalOffset = if (sps[3] == 1.toByte()) 5 else 4
|
||||
val spsData = NalUnitUtil.parseSpsNalUnitPayload(
|
||||
data, startNalOffset, data.size - startNalOffset)
|
||||
if (spsData.maxNumReorderFrames > 0) {
|
||||
Log.w(
|
||||
TAG, "SPS frame param max_num_reorder_frames=" +
|
||||
"${spsData.maxNumReorderFrames} is too high" +
|
||||
" for low latency decoding (expecting 0)."
|
||||
)
|
||||
}
|
||||
if (debug) {
|
||||
Log.d(TAG, "SPS frame: ${sps.toHexString(0, sps.size)}")
|
||||
Log.d(TAG, "\t${spsData.spsDataToString()}")
|
||||
Log.d(TAG, "PPS frame: ${pps.toHexString(0, pps.size)}")
|
||||
if (vps.isNotEmpty())
|
||||
Log.d(TAG, "VPS frame: ${vps.toHexString(0, vps.size)}")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
} else {
|
||||
if (DEBUG) Log.d(TAG, "RTSP SPS and PPS NAL units missed in SDP")
|
||||
}
|
||||
}
|
||||
if (sdpInfo.audioTrack != null) {
|
||||
audioFrameQueue.clear()
|
||||
when (sdpInfo.audioTrack?.audioCodec) {
|
||||
RtspClient.AUDIO_CODEC_AAC -> audioMimeType = MediaFormat.MIMETYPE_AUDIO_AAC
|
||||
RtspClient.AUDIO_CODEC_OPUS -> audioMimeType = MediaFormat.MIMETYPE_AUDIO_OPUS
|
||||
}
|
||||
audioSampleRate = sdpInfo.audioTrack?.sampleRateHz!!
|
||||
audioChannelCount = sdpInfo.audioTrack?.channels!!
|
||||
audioCodecConfig = sdpInfo.audioTrack?.config
|
||||
}
|
||||
onRtspClientConnected()
|
||||
uiHandler.post {
|
||||
statusListener?.onRtspStatusConnected()
|
||||
}
|
||||
}
|
||||
|
||||
private var framesPerGop = 0
|
||||
|
||||
override fun onRtspVideoNalUnitReceived(data: ByteArray, offset: Int, length: Int, timestamp: Long) {
|
||||
if (DEBUG) Log.v(TAG, "onRtspVideoNalUnitReceived(data.size=${data.size}, length=$length, timestamp=$timestamp)")
|
||||
|
||||
val isH265 = videoMimeType == MediaFormat.MIMETYPE_VIDEO_HEVC
|
||||
// Search for NAL_IDR_SLICE within first 1KB maximum
|
||||
val isKeyframe = VideoCodecUtils.isAnyKeyFrame(data, offset, min(length, 1000), isH265)
|
||||
|
||||
var videoFrame = FrameQueue.VideoFrame(
|
||||
VideoCodecType.H264,
|
||||
isKeyframe,
|
||||
data,
|
||||
offset,
|
||||
length,
|
||||
timestamp,
|
||||
capturedTimestampMs = System.currentTimeMillis()
|
||||
)
|
||||
if (isKeyframe && experimentalUpdateSpsFrameWithLowLatencyParams) {
|
||||
videoFrame = getNewLowLatencyFrameFromKeyFrame(videoFrame)
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
nalUnitsFound.clear()
|
||||
VideoCodecUtils.getNalUnits(videoFrame.data, videoFrame.offset, videoFrame.length, nalUnitsFound, isH265)
|
||||
var b = StringBuilder()
|
||||
for (nal in nalUnitsFound) {
|
||||
b
|
||||
.append(if (isH265)
|
||||
VideoCodecUtils.getH265NalUnitTypeString(nal.type)
|
||||
else
|
||||
VideoCodecUtils.getH264NalUnitTypeString(nal.type))
|
||||
.append(" (${nal.length}), ")
|
||||
}
|
||||
if (b.length > 2)
|
||||
b = b.removeRange(b.length - 2, b.length) as StringBuilder
|
||||
Log.d(TAG, "NALs: $b")
|
||||
@SuppressLint("UnsafeOptInUsageError")
|
||||
if (isKeyframe) {
|
||||
val sps = VideoCodecUtils.getSpsNalUnitFromArray(
|
||||
videoFrame.data,
|
||||
videoFrame.offset,
|
||||
// Check only first 100 bytes maximum. That's enough for finding SPS NAL unit.
|
||||
Integer.min(videoFrame.length, VideoCodecUtils.MAX_NAL_SPS_SIZE),
|
||||
isH265
|
||||
)
|
||||
Log.d(TAG,
|
||||
"\tKey frame received (${videoFrame.length} bytes, ts=$timestamp," +
|
||||
" ${sps?.width}x${sps?.height}," +
|
||||
" GoP=$framesPerGop," +
|
||||
" profile=${sps?.profileIdc}, level=${sps?.levelIdc})")
|
||||
framesPerGop = 0
|
||||
} else {
|
||||
framesPerGop++
|
||||
}
|
||||
}
|
||||
|
||||
videoFrameQueue.push(videoFrame)
|
||||
dataListener?.onRtspDataVideoNalUnitReceived(
|
||||
videoFrame.data,
|
||||
videoFrame.offset,
|
||||
videoFrame.length,
|
||||
timestamp)
|
||||
}
|
||||
|
||||
override fun onRtspAudioSampleReceived(data: ByteArray, offset: Int, length: Int, timestamp: Long) {
|
||||
if (DEBUG) Log.v(TAG, "onRtspAudioSampleReceived(length=$length, timestamp=$timestamp)")
|
||||
if (length > 0) {
|
||||
audioFrameQueue.push(
|
||||
FrameQueue.AudioFrame(
|
||||
AudioCodecType.AAC_LC,
|
||||
data, offset,
|
||||
length,
|
||||
timestamp
|
||||
)
|
||||
)
|
||||
}
|
||||
dataListener?.onRtspDataAudioSampleReceived(data, offset, length, timestamp)
|
||||
}
|
||||
|
||||
override fun onRtspApplicationDataReceived(data: ByteArray, offset: Int, length: Int, timestamp: Long) {
|
||||
if (DEBUG) Log.v(TAG, "onRtspApplicationDataReceived(length=$length, timestamp=$timestamp)")
|
||||
dataListener?.onRtspDataApplicationDataReceived(data, offset, length, timestamp)
|
||||
}
|
||||
|
||||
override fun onRtspDisconnecting() {
|
||||
if (DEBUG) Log.v(TAG, "onRtspDisconnecting()")
|
||||
uiHandler.post {
|
||||
statusListener?.onRtspStatusDisconnecting()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onRtspDisconnected() {
|
||||
if (DEBUG) Log.v(TAG, "onRtspDisconnected()")
|
||||
uiHandler.post {
|
||||
statusListener?.onRtspStatusDisconnected()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onRtspFailedUnauthorized() {
|
||||
if (DEBUG) Log.v(TAG, "onRtspFailedUnauthorized()")
|
||||
uiHandler.post {
|
||||
statusListener?.onRtspStatusFailedUnauthorized()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onRtspFailed(message: String?) {
|
||||
if (DEBUG) Log.v(TAG, "onRtspFailed(message='$message')")
|
||||
uiHandler.post {
|
||||
statusListener?.onRtspStatusFailed(message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inner class RtspThread: Thread() {
|
||||
private var rtspStopped = AtomicBoolean(false)
|
||||
|
||||
fun stopAsync() {
|
||||
if (DEBUG) Log.v(TAG, "stopAsync()")
|
||||
rtspStopped.set(true)
|
||||
// Wake up sleep() code
|
||||
interrupt()
|
||||
}
|
||||
|
||||
override fun run() {
|
||||
onRtspClientStarted()
|
||||
val port = if (uri.port == -1) DEFAULT_RTSP_PORT else uri.port
|
||||
var socket: Socket? = null
|
||||
try {
|
||||
if (DEBUG) Log.d(TAG, "Connecting to ${uri.host.toString()}:$port...")
|
||||
|
||||
socket = if (uri.scheme?.lowercase() == "rtsps")
|
||||
NetUtils.createSslSocketAndConnect(
|
||||
uri.host.toString(),
|
||||
port,
|
||||
socketTimeoutMsec
|
||||
)
|
||||
else
|
||||
NetUtils.createSocketAndConnect(
|
||||
uri.host.toString(),
|
||||
port,
|
||||
socketTimeoutMsec
|
||||
)
|
||||
|
||||
// Blocking call until stopped variable is true or connection failed
|
||||
val rtspClient = RtspClient.Builder(socket, uri.toString(), rtspStopped, proxyClientListener)
|
||||
.requestVideo(requestVideo)
|
||||
.requestAudio(requestAudio)
|
||||
.requestApplication(requestApplication)
|
||||
.withDebug(debug)
|
||||
.withUserAgent(userAgent)
|
||||
.withCredentials(username, password)
|
||||
.build()
|
||||
rtspClient.execute()
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
uiHandler.post { proxyClientListener.onRtspFailed(e.message) }
|
||||
} finally {
|
||||
NetUtils.closeSocket(socket)
|
||||
}
|
||||
onRtspClientStopped()
|
||||
}
|
||||
}
|
||||
|
||||
private val videoDecoderListener = object: VideoDecoderListener {
|
||||
override fun onVideoDecoderStarted() {
|
||||
if (DEBUG) Log.v(TAG, "onVideoDecoderStarted()")
|
||||
}
|
||||
|
||||
override fun onVideoDecoderStopped() {
|
||||
if (DEBUG) Log.v(TAG, "onVideoDecoderStopped()")
|
||||
}
|
||||
|
||||
override fun onVideoDecoderFailed(message: String?) {
|
||||
if (DEBUG) Log.e(TAG, "onVideoDecoderFailed(message='$message')")
|
||||
}
|
||||
|
||||
override fun onVideoDecoderFormatChanged(width: Int, height: Int) {
|
||||
if (DEBUG) Log.v(TAG, "onVideoDecoderFormatChanged(width=$width, height=$height)")
|
||||
statusListener?.onRtspFrameSizeChanged(width, height)
|
||||
}
|
||||
|
||||
override fun onVideoDecoderFirstFrameRendered() {
|
||||
if (DEBUG) Log.v(TAG, "onVideoDecoderFirstFrameDecoded()")
|
||||
if (!firstFrameRendered) statusListener?.onRtspFirstFrameRendered()
|
||||
firstFrameRendered = true
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun onRtspClientStarted() {
|
||||
if (DEBUG) Log.v(TAG, "onRtspClientStarted()")
|
||||
// uiHandler.post { statusListener?.onRtspStatusConnected() }
|
||||
}
|
||||
|
||||
private fun onRtspClientConnected() {
|
||||
if (DEBUG) Log.v(TAG, "onRtspClientConnected()")
|
||||
if (videoMimeType.isNotEmpty()) {
|
||||
firstFrameRendered = false
|
||||
Log.i(TAG, "Starting video decoder with mime type \"$videoMimeType\"")
|
||||
videoDecodeThread = onVideoDecoderCreateRequested.invoke(
|
||||
videoMimeType,
|
||||
videoRotation,
|
||||
videoFrameQueue,
|
||||
videoDecoderListener,
|
||||
videoDecoderType,
|
||||
videoFrameRateStabilization,
|
||||
)
|
||||
videoDecodeThread!!.apply {
|
||||
name = "RTSP video thread [${getUriName()}]"
|
||||
start()
|
||||
}
|
||||
}
|
||||
if (audioMimeType.isNotEmpty() /*&& checkAudio!!.isChecked*/) {
|
||||
Log.i(TAG, "Starting audio decoder with mime type \"$audioMimeType\"")
|
||||
audioDecodeThread = AudioDecodeThread(
|
||||
audioMimeType, audioSampleRate, audioChannelCount, audioCodecConfig, audioFrameQueue)
|
||||
audioDecodeThread!!.apply {
|
||||
name = "RTSP audio thread [${getUriName()}]"
|
||||
start()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun onRtspClientStopped() {
|
||||
if (DEBUG) Log.v(TAG, "onRtspClientStopped()")
|
||||
stopDecoders()
|
||||
rtspThread = null
|
||||
// uiHandler.post { statusListener?.onRtspStatusDisconnected() }
|
||||
}
|
||||
|
||||
fun init(uri: Uri, username: String?, password: String?, userAgent: String? = null, socketTimeout: Int = DEFAULT_SOCKET_TIMEOUT) {
|
||||
if (DEBUG) Log.v(TAG, "init(uri='$uri', username='$username', password='$password', userAgent='$userAgent', socketTimeout=$socketTimeout)")
|
||||
this.uri = uri
|
||||
this.username = username
|
||||
this.password = password
|
||||
this.userAgent = userAgent
|
||||
this.socketTimeoutMsec = socketTimeout
|
||||
}
|
||||
|
||||
fun start(requestVideo: Boolean, requestAudio: Boolean, requestApplication: Boolean = false) {
|
||||
if (DEBUG) Log.v(TAG, "start(requestVideo=$requestVideo, requestAudio=$requestAudio, requestApplication=$requestApplication)")
|
||||
if (rtspThread != null) rtspThread?.stopAsync()
|
||||
this.requestVideo = requestVideo
|
||||
this.requestAudio = requestAudio
|
||||
this.requestApplication = requestApplication
|
||||
rtspThread = RtspThread().apply {
|
||||
name = "RTSP IO thread [${getUriName()}]"
|
||||
start()
|
||||
}
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
if (DEBUG) Log.v(TAG, "stop()")
|
||||
rtspThread?.stopAsync()
|
||||
rtspThread = null
|
||||
}
|
||||
|
||||
fun isStarted(): Boolean {
|
||||
return rtspThread != null
|
||||
}
|
||||
|
||||
fun stopDecoders() {
|
||||
if (DEBUG) Log.v(TAG, "stopDecoders()")
|
||||
videoDecodeThread?.stopAsync()
|
||||
videoDecodeThread = null
|
||||
audioDecodeThread?.stopAsync()
|
||||
audioDecodeThread = null
|
||||
}
|
||||
|
||||
// Cached values
|
||||
private val nalUnitsFound = ArrayList<VideoCodecUtils.NalUnit>()
|
||||
private val spsBufferReadFrame = ByteBuffer.allocate(VideoCodecUtils.MAX_NAL_SPS_SIZE)
|
||||
private val spsBufferWriteFrame = ByteBuffer.allocate(VideoCodecUtils.MAX_NAL_SPS_SIZE)
|
||||
|
||||
/**
|
||||
* Try to get a new frame keyframe (SPS+PPS+IDR) with low latency modified SPS frame.
|
||||
* If modification failed, original frame will be returned.
|
||||
* Inspired by https://webrtc.googlesource.com/src/+/refs/heads/main/common_video/h264/sps_vui_rewriter.cc#400
|
||||
*/
|
||||
private fun getNewLowLatencyFrameFromKeyFrame(frame: FrameQueue.VideoFrame): FrameQueue.VideoFrame {
|
||||
try {
|
||||
// Support only H264 for now
|
||||
if (frame.codecType == VideoCodecType.H265)
|
||||
return frame
|
||||
|
||||
nalUnitsFound.clear()
|
||||
VideoCodecUtils.getNalUnits(frame.data, frame.offset, frame.length, nalUnitsFound, isH265 = false)
|
||||
|
||||
val oldSpsNalUnit = nalUnitsFound.firstOrNull { it.type == VideoCodecUtils.NAL_SPS }
|
||||
|
||||
// SPS frame not found. Return original frame.
|
||||
if (oldSpsNalUnit == null)
|
||||
return frame
|
||||
|
||||
spsBufferReadFrame.apply {
|
||||
rewind()
|
||||
put(frame.data, oldSpsNalUnit.offset + 5,
|
||||
Integer.min(oldSpsNalUnit.length, VideoCodecUtils.MAX_NAL_SPS_SIZE)
|
||||
)
|
||||
rewind()
|
||||
}
|
||||
// Read SPS frame
|
||||
val spsSet = SeqParameterSet.read(spsBufferReadFrame)
|
||||
|
||||
// adding VUI might decrease latency for some streams, if max_dec_frame_buffering is set properly
|
||||
// https://community.intel.com/t5/Media-Intel-oneAPI-Video/h-264-decoder-gives-two-frames-latency-while-decoding-a-stream/td-p/1099694
|
||||
// https://github.com/Consti10/LiveVideo10ms/blob/master/VideoCore/src/main/cpp/NALU/H26X.hpp
|
||||
fun modifyVui() {
|
||||
// spsSet.vuiParams = VUIParameters()
|
||||
spsSet.vuiParams.apply {
|
||||
// videoSignalTypePresentFlag = true
|
||||
// videoFormat = 5
|
||||
// colourDescriptionPresentFlag = true
|
||||
// matrixCoefficients = 5
|
||||
// timingInfoPresentFlag = true
|
||||
// numUnitsInTick = 1
|
||||
// timeScale = 120
|
||||
// fixedFrameRateFlag = true
|
||||
bitstreamRestriction = VUIParameters.BitstreamRestriction().apply {
|
||||
// motionVectorsOverPicBoundariesFlag = true
|
||||
// log2MaxMvLengthHorizontal = 16
|
||||
// log2MaxMvLengthVertical = 16
|
||||
maxDecFrameBuffering = 1
|
||||
numReorderFrames = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
modifyVui()
|
||||
|
||||
// Write SPS frame
|
||||
spsBufferWriteFrame.rewind()
|
||||
spsSet.write(spsBufferWriteFrame)
|
||||
|
||||
val newSpsNalUnitSize = spsBufferWriteFrame.position()
|
||||
|
||||
if (oldSpsNalUnit.length > -1) {
|
||||
val newSize = frame.length - oldSpsNalUnit.length + newSpsNalUnitSize
|
||||
val newData = ByteArray(newSize + 5)
|
||||
var newDataOffset = 0
|
||||
|
||||
for (nalUnit in nalUnitsFound) {
|
||||
when (nalUnit.type) {
|
||||
VideoCodecUtils.NAL_SPS -> {
|
||||
// Write NAL header + SPS frame type
|
||||
val b = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x27)
|
||||
b.copyInto(newData, newDataOffset, 0, b.size)
|
||||
newDataOffset += b.size
|
||||
// Write SPS frame body
|
||||
spsBufferWriteFrame.apply {
|
||||
rewind()
|
||||
get(newData, newDataOffset, newSpsNalUnitSize)
|
||||
}
|
||||
newDataOffset += newSpsNalUnitSize
|
||||
}
|
||||
|
||||
else -> {
|
||||
frame.data.copyInto(
|
||||
newData,
|
||||
newDataOffset,
|
||||
nalUnit.offset,
|
||||
nalUnit.offset + nalUnit.length
|
||||
)
|
||||
newDataOffset += nalUnit.length
|
||||
}
|
||||
}
|
||||
}
|
||||
// Create SPS+PPS+IDR frame with newly modified SPS frame data
|
||||
return FrameQueue.VideoFrame(
|
||||
frame.codecType,
|
||||
frame.isKeyframe,
|
||||
newData,
|
||||
0,
|
||||
newData.size,
|
||||
frame.timestampMs,
|
||||
frame.capturedTimestampMs
|
||||
)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Failed to create low-latency keyframe", e)
|
||||
}
|
||||
return frame
|
||||
}
|
||||
|
||||
private fun getUriName(): String {
|
||||
val port = if (uri.port == -1) DEFAULT_RTSP_PORT else uri.port
|
||||
return "${uri.host.toString()}:$port"
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val TAG: String = RtspProcessor::class.java.simpleName
|
||||
private const val DEBUG = false
|
||||
|
||||
private const val DEFAULT_RTSP_PORT = 554
|
||||
|
||||
const val DEFAULT_SOCKET_TIMEOUT = 5000
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package com.alexvas.rtsp.widget
|
||||
|
||||
import android.content.Context
|
||||
import android.net.Uri
|
||||
import android.util.AttributeSet
|
||||
import android.util.Log
|
||||
import android.view.SurfaceHolder
|
||||
import android.view.SurfaceView
|
||||
import androidx.annotation.OptIn
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import androidx.media3.container.NalUnitUtil
|
||||
import com.alexvas.rtsp.codec.VideoDecodeThread.DecoderType
|
||||
import com.alexvas.rtsp.codec.VideoDecoderSurfaceThread
|
||||
import com.alexvas.rtsp.widget.RtspProcessor.Statistics
|
||||
import com.limelight.binding.video.MediaCodecHelper
|
||||
|
||||
/**
|
||||
* Low latency RTSP stream playback on surface view.
|
||||
*/
|
||||
open class RtspSurfaceView: SurfaceView {
|
||||
|
||||
private var surfaceWidth = 1920
|
||||
private var surfaceHeight = 1080
|
||||
|
||||
private var rtspProcessor = RtspProcessor(
|
||||
onVideoDecoderCreateRequested = {
|
||||
videoMimeType,
|
||||
videoRotation,
|
||||
videoFrameQueue,
|
||||
videoDecoderListener,
|
||||
videoDecoderType,
|
||||
videoFrameRateStabilization,
|
||||
->
|
||||
VideoDecoderSurfaceThread(
|
||||
holder.surface,
|
||||
videoMimeType,
|
||||
surfaceWidth,
|
||||
surfaceHeight,
|
||||
videoRotation,
|
||||
videoFrameQueue,
|
||||
videoDecoderListener,
|
||||
videoDecoderType,
|
||||
videoFrameRateStabilization,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
var statistics = Statistics()
|
||||
get() = rtspProcessor.statistics
|
||||
private set
|
||||
|
||||
var videoRotation: Int
|
||||
get() = rtspProcessor.videoRotation
|
||||
set(value) { rtspProcessor.videoRotation = value }
|
||||
|
||||
var videoDecoderType: DecoderType
|
||||
get() = rtspProcessor.videoDecoderType
|
||||
set(value) { rtspProcessor.videoDecoderType = value }
|
||||
|
||||
var experimentalUpdateSpsFrameWithLowLatencyParams: Boolean
|
||||
get() = rtspProcessor.experimentalUpdateSpsFrameWithLowLatencyParams
|
||||
set(value) { rtspProcessor.experimentalUpdateSpsFrameWithLowLatencyParams = value }
|
||||
|
||||
var debug: Boolean
|
||||
get() = rtspProcessor.debug
|
||||
set(value) { rtspProcessor.debug = value }
|
||||
|
||||
/** Enables decoder-side playback smoothing. Disabled by default. */
|
||||
var videoFrameRateStabilization: Boolean
|
||||
get() = rtspProcessor.videoFrameRateStabilization
|
||||
set(value) { rtspProcessor.videoFrameRateStabilization = value }
|
||||
|
||||
private val surfaceCallback = object: SurfaceHolder.Callback {
|
||||
override fun surfaceCreated(holder: SurfaceHolder) {
|
||||
if (DEBUG) Log.v(TAG, "surfaceCreated()")
|
||||
}
|
||||
|
||||
override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {
|
||||
if (DEBUG) Log.v(TAG, "surfaceChanged(format=$format, width=$width, height=$height)")
|
||||
surfaceWidth = width
|
||||
surfaceHeight = height
|
||||
}
|
||||
|
||||
override fun surfaceDestroyed(holder: SurfaceHolder) {
|
||||
if (DEBUG) Log.v(TAG, "surfaceDestroyed()")
|
||||
rtspProcessor.stopDecoders()
|
||||
}
|
||||
}
|
||||
|
||||
constructor(context: Context) : super(context) {
|
||||
initView(context, null, 0)
|
||||
}
|
||||
|
||||
constructor(context: Context, attrs: AttributeSet?) : super(context, attrs) {
|
||||
initView(context, attrs, 0)
|
||||
}
|
||||
|
||||
constructor(context: Context, attrs: AttributeSet?, defStyleAttr: Int) : super(context, attrs, defStyleAttr) {
|
||||
initView(context, attrs, defStyleAttr)
|
||||
}
|
||||
|
||||
private fun initView(context: Context, attrs: AttributeSet?, defStyleAttr: Int) {
|
||||
if (DEBUG) Log.v(TAG, "initView()")
|
||||
MediaCodecHelper.initialize(context, /*glRenderer*/ "")
|
||||
holder.addCallback(surfaceCallback)
|
||||
}
|
||||
|
||||
fun init(
|
||||
uri: Uri,
|
||||
username: String? = null,
|
||||
password: String? = null,
|
||||
userAgent: String? = null,
|
||||
socketTimeout: Int? = null
|
||||
) {
|
||||
if (DEBUG) Log.v(TAG, "init(uri='$uri', username='$username', password='$password', userAgent='$userAgent', socketTimeout=$socketTimeout)")
|
||||
rtspProcessor.init(
|
||||
uri,
|
||||
username,
|
||||
password,
|
||||
userAgent,
|
||||
socketTimeout ?: RtspProcessor.DEFAULT_SOCKET_TIMEOUT
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Start RTSP client.
|
||||
*
|
||||
* @param requestVideo request video track
|
||||
* @param requestAudio request audio track
|
||||
* @param requestApplication request application track
|
||||
* @see https://datatracker.ietf.org/doc/html/rfc4566#section-5.14
|
||||
*/
|
||||
fun start(requestVideo: Boolean, requestAudio: Boolean, requestApplication: Boolean = false) {
|
||||
if (DEBUG) Log.v(TAG, "start(requestVideo=$requestVideo, requestAudio=$requestAudio, requestApplication=$requestApplication)")
|
||||
rtspProcessor.start(requestVideo, requestAudio, requestApplication)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop RTSP client.
|
||||
*/
|
||||
fun stop() {
|
||||
if (DEBUG) Log.v(TAG, "stop()")
|
||||
rtspProcessor.stop()
|
||||
}
|
||||
|
||||
fun isStarted(): Boolean {
|
||||
return rtspProcessor.isStarted()
|
||||
}
|
||||
|
||||
fun setStatusListener(listener: RtspStatusListener?) {
|
||||
if (DEBUG) Log.v(TAG, "setStatusListener()")
|
||||
rtspProcessor.statusListener = listener
|
||||
}
|
||||
|
||||
fun setDataListener(listener: RtspDataListener?) {
|
||||
if (DEBUG) Log.v(TAG, "setDataListener()")
|
||||
rtspProcessor.dataListener = listener
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val TAG: String = RtspSurfaceView::class.java.simpleName
|
||||
private const val DEBUG = false
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@OptIn(UnstableApi::class)
|
||||
fun NalUnitUtil.SpsData.spsDataToString(): String {
|
||||
return "" +
|
||||
"width=${this.width}, " +
|
||||
"height=${this.height}, " +
|
||||
"profile_idc=${this.profileIdc}, " +
|
||||
"constraint_set_flags=${this.constraintsFlagsAndReservedZero2Bits}, " +
|
||||
"level_idc=${this.levelIdc}, " +
|
||||
"max_num_ref_frames=${this.maxNumRefFrames}, " +
|
||||
"frame_mbs_only_flag=${this.frameMbsOnlyFlag}, " +
|
||||
"log2_max_frame_num=${this.frameNumLength}, " +
|
||||
"pic_order_cnt_type=${this.picOrderCountType}, " +
|
||||
"log2_max_pic_order_cnt_lsb=${this.picOrderCntLsbLength}, " +
|
||||
"delta_pic_order_always_zero_flag=${this.deltaPicOrderAlwaysZeroFlag}, " +
|
||||
"max_reorder_frames=${this.maxNumReorderFrames}"
|
||||
}
|
||||
|
||||
fun ByteArray.toHexString(offset: Int, maxLength: Int): String {
|
||||
val length = minOf(maxLength, size - offset)
|
||||
return sliceArray(offset until (offset + length))
|
||||
.joinToString(separator = "") { byte ->
|
||||
"%02x ".format(byte).uppercase()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.alexvas.utils;
|
||||
|
||||
import androidx.annotation.NonNull;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
|
||||
public class ByteUtils {
|
||||
|
||||
// int memcmp ( const void * ptr1, const void * ptr2, size_t num );
|
||||
public static boolean memcmp(
|
||||
@NonNull byte[] source1,
|
||||
int offsetSource1,
|
||||
@NonNull byte[] source2,
|
||||
int offsetSource2,
|
||||
int num) {
|
||||
if (source1.length - offsetSource1 < num)
|
||||
return false;
|
||||
if (source2.length - offsetSource2 < num)
|
||||
return false;
|
||||
|
||||
for (int i = 0; i < num; i++) {
|
||||
if (source1[offsetSource1 + i] != source2[offsetSource2 + i])
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public static byte[] copy(@NonNull byte[] src) {
|
||||
byte[] dest = new byte[src.length];
|
||||
System.arraycopy(src, 0, dest, 0, src.length);
|
||||
return dest;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package com.alexvas.utils
|
||||
|
||||
import android.annotation.SuppressLint
|
||||
import android.util.Log
|
||||
import android.util.Range
|
||||
import androidx.annotation.OptIn
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import androidx.media3.exoplayer.mediacodec.MediaCodecInfo
|
||||
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil
|
||||
import java.lang.Exception
|
||||
|
||||
@SuppressLint("UnsafeOptInUsageError")
|
||||
object MediaCodecUtils {
|
||||
|
||||
// key - codecs mime type
|
||||
// value - list of codecs able to handle this mime type
|
||||
private val decoderInfosMap = HashMap<String, List<MediaCodecInfo>>()
|
||||
|
||||
private val TAG: String = MediaCodecUtils::class.java.simpleName
|
||||
|
||||
private fun getDecoderInfos(mimeType: String): List<MediaCodecInfo> {
|
||||
val list = decoderInfosMap[mimeType]
|
||||
return if (list.isNullOrEmpty()) {
|
||||
val decoderInfos = try {
|
||||
MediaCodecUtil.getDecoderInfos(mimeType, false, false)
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Failed to initialize '$mimeType' decoders list (${e.message})", e)
|
||||
ArrayList()
|
||||
}
|
||||
decoderInfosMap[mimeType] = decoderInfos
|
||||
decoderInfos
|
||||
} else {
|
||||
list
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get software decoders list. Usually used as fallback.
|
||||
*/
|
||||
@Synchronized
|
||||
fun getSoftwareDecoders(mimeType: String): List<MediaCodecInfo> {
|
||||
val decoderInfos = getDecoderInfos(mimeType)
|
||||
val list = ArrayList<MediaCodecInfo>()
|
||||
for (codec in decoderInfos) {
|
||||
if (codec.softwareOnly)
|
||||
list.add(codec)
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
/**
|
||||
* Get hardware accelerated decoders list. Used as default.
|
||||
*/
|
||||
@Synchronized
|
||||
fun getHardwareDecoders(mimeType: String): List<MediaCodecInfo> {
|
||||
val decoderInfos = getDecoderInfos(mimeType)
|
||||
val list = ArrayList<MediaCodecInfo>()
|
||||
for (codec in decoderInfos) {
|
||||
if (codec.hardwareAccelerated)
|
||||
list.add(codec)
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
/**
|
||||
* Look through all decoders (if there are multiple)
|
||||
* and select the one which supports low-latency.
|
||||
*/
|
||||
@OptIn(UnstableApi::class)
|
||||
fun getLowLatencyDecoder(decoders: List<MediaCodecInfo>): MediaCodecInfo? {
|
||||
// Some devices can have several decoders, e.g.
|
||||
// Samsung Fold 5:
|
||||
// "c2.qti.avc.decoder"
|
||||
// "c2.qti.avc.decoder.low_latency"
|
||||
for (decoder in decoders) {
|
||||
if (decoder.name.contains("low_latency"))
|
||||
return decoder
|
||||
}
|
||||
// Another approach to find decoder with low-latency is to call
|
||||
// MediaCodec.createByCodecName(name) for every decoder to get decoder instance and then call
|
||||
// decoder.codecInfo.getCapabilitiesForType(mimeType).isFeatureSupported(MediaCodecInfo.CodecCapabilities.FEATURE_LowLatency)
|
||||
|
||||
// No low-latency decoder found.
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
fun android.media.MediaCodecInfo.CodecCapabilities.capabilitiesToString(): String {
|
||||
var heights = videoCapabilities?.supportedHeights
|
||||
if (heights == null)
|
||||
heights = Range(-1, -1)
|
||||
var widths = videoCapabilities?.supportedWidths
|
||||
if (widths == null)
|
||||
widths = Range(-1, -1)
|
||||
return "max instances: ${maxSupportedInstances}, max resolution: ${heights.upper}x${widths.upper}"
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
package com.alexvas.utils;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
import androidx.annotation.NonNull;
|
||||
import androidx.annotation.Nullable;
|
||||
|
||||
import java.io.EOFException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.Socket;
|
||||
import java.security.cert.CertificateException;
|
||||
import java.security.cert.X509Certificate;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
import javax.net.ssl.SSLSocket;
|
||||
import javax.net.ssl.TrustManager;
|
||||
import javax.net.ssl.X509TrustManager;
|
||||
|
||||
public class NetUtils {
|
||||
|
||||
private static final String TAG = NetUtils.class.getSimpleName();
|
||||
private static final boolean DEBUG = false;
|
||||
private final static int MAX_LINE_SIZE = 4098;
|
||||
|
||||
public static final class FakeX509TrustManager implements X509TrustManager {
|
||||
|
||||
/**
|
||||
* Accepted issuers for fake trust manager
|
||||
*/
|
||||
final static private X509Certificate[] mAcceptedIssuers = new X509Certificate[]{};
|
||||
|
||||
/**
|
||||
* Constructor for FakeX509TrustManager.
|
||||
*/
|
||||
public FakeX509TrustManager() {
|
||||
}
|
||||
|
||||
/**
|
||||
* @see javax.net.ssl.X509TrustManager#checkClientTrusted(X509Certificate[],String authType)
|
||||
*/
|
||||
public void checkClientTrusted(X509Certificate[] certificates, String authType)
|
||||
throws CertificateException {
|
||||
}
|
||||
|
||||
/**
|
||||
* @see javax.net.ssl.X509TrustManager#checkServerTrusted(X509Certificate[],String authType)
|
||||
*/
|
||||
public void checkServerTrusted(X509Certificate[] certificates, String authType)
|
||||
throws CertificateException {
|
||||
}
|
||||
|
||||
// https://github.com/square/okhttp/issues/4669
|
||||
// Called by Android via reflection in X509TrustManagerExtensions.
|
||||
@SuppressWarnings("unused")
|
||||
public List<X509Certificate> checkServerTrusted(X509Certificate[] chain, String authType, String host) throws CertificateException {
|
||||
return Arrays.asList(chain);
|
||||
}
|
||||
|
||||
/**
|
||||
* @see javax.net.ssl.X509TrustManager#getAcceptedIssuers()
|
||||
*/
|
||||
public X509Certificate[] getAcceptedIssuers() {
|
||||
return mAcceptedIssuers;
|
||||
}
|
||||
}
|
||||
|
||||
@NonNull
|
||||
public static SSLSocket createSslSocketAndConnect(@NonNull String dstName, int dstPort, int timeout) throws Exception {
|
||||
if (DEBUG)
|
||||
Log.v(TAG, "createSslSocketAndConnect(dstName=" + dstName + ", dstPort=" + dstPort + ", timeout=" + timeout + ")");
|
||||
|
||||
// TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
|
||||
// trustManagerFactory.init((KeyStore) null);
|
||||
// TrustManager[] trustManagers = trustManagerFactory.getTrustManagers();
|
||||
// if (trustManagers.length != 1 || !(trustManagers[0] instanceof X509TrustManager)) {
|
||||
// throw new IllegalStateException("Unexpected default trust managers:" + Arrays.toString(trustManagers));
|
||||
// }
|
||||
// X509TrustManager trustManager = (X509TrustManager) trustManagers[0];
|
||||
|
||||
SSLContext sslContext = SSLContext.getInstance("TLS");
|
||||
sslContext.init(null, new TrustManager[] { new FakeX509TrustManager() }, null);
|
||||
SSLSocket sslSocket = (SSLSocket) sslContext.getSocketFactory().createSocket();
|
||||
sslSocket.connect(new InetSocketAddress(dstName, dstPort), timeout);
|
||||
sslSocket.setSoLinger(false, 1);
|
||||
sslSocket.setSoTimeout(timeout);
|
||||
return sslSocket;
|
||||
}
|
||||
|
||||
@NonNull
|
||||
public static Socket createSocketAndConnect(@NonNull String dstName, int dstPort, int timeout) throws IOException {
|
||||
if (DEBUG)
|
||||
Log.v(TAG, "createSocketAndConnect(dstName=" + dstName + ", dstPort=" + dstPort + ", timeout=" + timeout + ")");
|
||||
Socket socket = new Socket();
|
||||
socket.connect(new InetSocketAddress(dstName, dstPort), timeout);
|
||||
socket.setSoLinger(false, 1);
|
||||
socket.setSoTimeout(timeout);
|
||||
return socket;
|
||||
}
|
||||
|
||||
@NonNull
|
||||
public static Socket createSocket(int timeout) throws IOException {
|
||||
Socket socket = new Socket();
|
||||
socket.setSoLinger(false, 1);// 1 sec for flush() before close()
|
||||
socket.setSoTimeout(timeout); // 10 sec timeout for read(), not for write()
|
||||
return socket;
|
||||
}
|
||||
|
||||
public static void closeSocket(@Nullable Socket socket) throws IOException {
|
||||
if (DEBUG)
|
||||
Log.v(TAG, "closeSocket()");
|
||||
if (socket != null) {
|
||||
try {
|
||||
socket.shutdownInput();
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
try {
|
||||
socket.shutdownOutput();
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
socket.close();
|
||||
}
|
||||
}
|
||||
|
||||
@NonNull
|
||||
public static ArrayList<String> readResponseHeaders(@NonNull InputStream inputStream) throws IOException {
|
||||
// Assert.assertNotNull("Input stream should not be null", inputStream);
|
||||
ArrayList<String> headers = new ArrayList<>();
|
||||
String line;
|
||||
while (true) {
|
||||
line = readLine(inputStream);
|
||||
if (line != null) {
|
||||
if (line.equals("\r\n"))
|
||||
return headers;
|
||||
else
|
||||
headers.add(line);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
public static String readLine(@NonNull InputStream inputStream) throws IOException {
|
||||
// Assert.assertNotNull("Input stream should not be null", inputStream);
|
||||
byte[] bufferLine = new byte[MAX_LINE_SIZE];
|
||||
int offset = 0;
|
||||
int readBytes;
|
||||
do {
|
||||
// Didn't find "\r\n" within 4K bytes
|
||||
if (offset >= MAX_LINE_SIZE) {
|
||||
throw new IOException("Invalid headers");
|
||||
}
|
||||
|
||||
// Read 1 byte
|
||||
readBytes = inputStream.read(bufferLine, offset, 1);
|
||||
if (readBytes == 1) {
|
||||
// Check for EOL
|
||||
// Some cameras like Linksys WVC200 do not send \n instead of \r\n
|
||||
if (offset > 0 && /*bufferLine[offset-1] == '\r' &&*/ bufferLine[offset] == '\n') {
|
||||
// Found empty EOL. End of header section
|
||||
if (offset == 1)
|
||||
break;
|
||||
|
||||
// Found EOL. Add to array.
|
||||
return new String(bufferLine, 0, offset-1);
|
||||
} else {
|
||||
offset++;
|
||||
}
|
||||
}
|
||||
} while (readBytes > 0);
|
||||
return null;
|
||||
}
|
||||
|
||||
public static int getResponseStatusCode(@NonNull ArrayList<String> headers) {
|
||||
// Assert.assertNotNull("Headers should not be null", headers);
|
||||
// Search for HTTP status code header
|
||||
for (String header: headers) {
|
||||
int indexHttp = header.indexOf("HTTP/1.1 "); // 9 characters
|
||||
if (indexHttp == -1)
|
||||
indexHttp = header.indexOf("HTTP/1.0 ");
|
||||
if (indexHttp >= 0) {
|
||||
int indexCode = header.indexOf(' ', 9);
|
||||
String code = header.substring(9, indexCode);
|
||||
try {
|
||||
return Integer.parseInt(code);
|
||||
} catch (NumberFormatException e) {
|
||||
// Does not fulfill standard "HTTP/1.1 200 Ok" token
|
||||
// Continue search for
|
||||
}
|
||||
}
|
||||
}
|
||||
// Not found
|
||||
return -1;
|
||||
}
|
||||
|
||||
// @Nullable
|
||||
// static String readContentAsText(@Nullable InputStream inputStream) throws IOException {
|
||||
// if (inputStream == null)
|
||||
// return null;
|
||||
// BufferedReader r = new BufferedReader(new InputStreamReader(inputStream));
|
||||
// StringBuilder total = new StringBuilder();
|
||||
// String line;
|
||||
// while ((line = r.readLine()) != null) {
|
||||
// total.append(line);
|
||||
// total.append("\r\n");
|
||||
// }
|
||||
// return total.toString();
|
||||
// }
|
||||
|
||||
@NonNull
|
||||
public static String readContentAsText(@NonNull InputStream inputStream, int length) throws IOException {
|
||||
// Assert.assertNotNull("Input stream should not be null", inputStream);
|
||||
if (length <= 0)
|
||||
return "";
|
||||
byte[] b = new byte[length];
|
||||
int read = readData(inputStream, b, 0, length);
|
||||
return new String(b, 0, read);
|
||||
}
|
||||
|
||||
public static int readData(@NonNull InputStream inputStream, @NonNull byte[] buffer, int offset, int length) throws IOException {
|
||||
int readBytes;
|
||||
int totalReadBytes = 0;
|
||||
do {
|
||||
readBytes = inputStream.read(buffer, offset + totalReadBytes, length - totalReadBytes);
|
||||
if (readBytes == -1) {
|
||||
throw new EOFException("Stream closed, read " + totalReadBytes + " of " + length + " bytes");
|
||||
}
|
||||
totalReadBytes += readBytes;
|
||||
} while (readBytes >= 0 && totalReadBytes < length);
|
||||
return totalReadBytes;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
package com.alexvas.utils
|
||||
|
||||
import android.annotation.SuppressLint
|
||||
import android.util.Log
|
||||
import androidx.media3.container.NalUnitUtil
|
||||
import androidx.media3.container.NalUnitUtil.SpsData
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import kotlin.experimental.and
|
||||
|
||||
|
||||
object VideoCodecUtils {
|
||||
|
||||
private val TAG = VideoCodecUtils::class.java.simpleName
|
||||
|
||||
/** Max possible NAL SPS size in bytes */
|
||||
const val MAX_NAL_SPS_SIZE: Int = 500
|
||||
|
||||
const val NAL_SLICE: Byte = 1
|
||||
const val NAL_DPA: Byte = 2
|
||||
const val NAL_DPB: Byte = 3
|
||||
const val NAL_DPC: Byte = 4
|
||||
const val NAL_IDR_SLICE: Byte = 5
|
||||
const val NAL_SEI: Byte = 6
|
||||
const val NAL_SPS: Byte = 7
|
||||
const val NAL_PPS: Byte = 8
|
||||
const val NAL_AUD: Byte = 9
|
||||
const val NAL_END_SEQUENCE: Byte = 10
|
||||
const val NAL_END_STREAM: Byte = 11
|
||||
const val NAL_FILLER_DATA: Byte = 12
|
||||
const val NAL_SPS_EXT: Byte = 13
|
||||
const val NAL_AUXILIARY_SLICE: Byte = 19
|
||||
const val NAL_STAP_A: Byte = 24 // https://tools.ietf.org/html/rfc3984 5.7.1
|
||||
const val NAL_STAP_B: Byte = 25 // 5.7.1
|
||||
const val NAL_MTAP16: Byte = 26 // 5.7.2
|
||||
const val NAL_MTAP24: Byte = 27 // 5.7.2
|
||||
const val NAL_FU_A: Byte = 28 // 5.8 fragmented unit
|
||||
const val NAL_FU_B: Byte = 29 // 5.8
|
||||
|
||||
// Table 7-3: NAL unit type codes
|
||||
const val H265_NAL_TRAIL_N: Byte = 0
|
||||
const val H265_NAL_TRAIL_R: Byte = 1
|
||||
const val H265_NAL_TSA_N: Byte = 2
|
||||
const val H265_NAL_TSA_R: Byte = 3
|
||||
const val H265_NAL_STSA_N: Byte = 4
|
||||
const val H265_NAL_STSA_R: Byte = 5
|
||||
const val H265_NAL_RADL_N: Byte = 6
|
||||
const val H265_NAL_RADL_R: Byte = 7
|
||||
const val H265_NAL_RASL_N: Byte = 8
|
||||
const val H265_NAL_RASL_R: Byte = 9
|
||||
const val H265_NAL_BLA_W_LP: Byte = 16
|
||||
const val H265_NAL_BLA_W_RADL: Byte = 17
|
||||
const val H265_NAL_BLA_N_LP: Byte = 18
|
||||
const val H265_NAL_IDR_W_RADL: Byte = 19
|
||||
const val H265_NAL_IDR_N_LP: Byte = 20
|
||||
const val H265_NAL_CRA_NUT: Byte = 21
|
||||
const val H265_NAL_VPS: Byte = 32
|
||||
const val H265_NAL_SPS: Byte = 33
|
||||
const val H265_NAL_PPS: Byte = 34
|
||||
const val H265_NAL_AUD: Byte = 35
|
||||
const val H265_NAL_EOS_NUT: Byte = 36
|
||||
const val H265_NAL_EOB_NUT: Byte = 37
|
||||
const val H265_NAL_FD_NUT: Byte = 38
|
||||
const val H265_NAL_SEI_PREFIX: Byte = 39
|
||||
const val H265_NAL_SEI_SUFFIX: Byte = 40
|
||||
|
||||
private val NAL_PREFIX1 = byteArrayOf(0x00, 0x00, 0x00, 0x01)
|
||||
private val NAL_PREFIX2 = byteArrayOf(0x00, 0x00, 0x01)
|
||||
|
||||
|
||||
/**
|
||||
* Search for 00 00 01 or 00 00 00 01 in byte stream.
|
||||
* @return offset to the start of NAL unit if found, otherwise -1
|
||||
*/
|
||||
fun searchForNalUnitStart(
|
||||
data: ByteArray,
|
||||
offset: Int,
|
||||
length: Int,
|
||||
prefixSize: AtomicInteger
|
||||
): Int {
|
||||
if (offset >= data.size - 3) return -1
|
||||
for (pos in 0 until length) {
|
||||
val prefix: Int = getNalUnitStartCodePrefixSize(data, pos + offset, length)
|
||||
if (prefix >= 0) {
|
||||
prefixSize.set(prefix)
|
||||
return pos + offset
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
fun searchForH264NalUnitByType(
|
||||
data: ByteArray,
|
||||
offset: Int,
|
||||
length: Int,
|
||||
byUnitType: Int
|
||||
): Int {
|
||||
var off = offset
|
||||
val nalUnitPrefixSize = AtomicInteger(-1)
|
||||
val timestamp = System.currentTimeMillis()
|
||||
while (true) {
|
||||
val nalUnitIndex = searchForNalUnitStart(data, off, length, nalUnitPrefixSize)
|
||||
if (nalUnitIndex >= 0) {
|
||||
val nalUnitOffset = nalUnitIndex + nalUnitPrefixSize.get()
|
||||
if (nalUnitOffset >= data.size)
|
||||
break
|
||||
val nalUnitTypeOctet = data[nalUnitOffset]
|
||||
if ((nalUnitTypeOctet and 0x1f).toInt() == byUnitType) {
|
||||
return nalUnitIndex
|
||||
}
|
||||
off = nalUnitOffset
|
||||
|
||||
// Check that we are not too long here
|
||||
if (System.currentTimeMillis() - timestamp > 100) {
|
||||
Log.w(TAG, "Cannot process data within 100 msec in $length bytes")
|
||||
break
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
fun getNalUnitType(data: ByteArray?, offset: Int, length: Int, isH265: Boolean): Byte {
|
||||
if (data == null || length <= NAL_PREFIX1.size) return (-1).toByte()
|
||||
var nalUnitTypeOctetOffset = -1
|
||||
if (data[offset + NAL_PREFIX2.size - 1] == 1.toByte())
|
||||
nalUnitTypeOctetOffset =
|
||||
offset + NAL_PREFIX2.size - 1
|
||||
else if (data[offset + NAL_PREFIX1.size - 1] == 1.toByte())
|
||||
nalUnitTypeOctetOffset = offset + NAL_PREFIX1.size - 1
|
||||
|
||||
return if (nalUnitTypeOctetOffset != -1) {
|
||||
val nalUnitTypeOctet = data[nalUnitTypeOctetOffset + 1]
|
||||
if (isH265)
|
||||
((nalUnitTypeOctet.toInt() shr 1) and 0x3F).toByte()
|
||||
else
|
||||
(nalUnitTypeOctet and 0x1f)
|
||||
} else {
|
||||
(-1).toByte()
|
||||
}
|
||||
}
|
||||
|
||||
private fun getNalUnitStartCodePrefixSize(
|
||||
data: ByteArray,
|
||||
offset: Int,
|
||||
length: Int
|
||||
): Int {
|
||||
if (length < 4) return -1
|
||||
return if (memcmp(data, offset, NAL_PREFIX1, 0, NAL_PREFIX1.size))
|
||||
NAL_PREFIX1.size else
|
||||
if (memcmp(data, offset, NAL_PREFIX2, 0, NAL_PREFIX2.size))
|
||||
NAL_PREFIX2.size else
|
||||
-1
|
||||
}
|
||||
|
||||
private fun memcmp(
|
||||
source1: ByteArray,
|
||||
offsetSource1: Int,
|
||||
source2: ByteArray,
|
||||
offsetSource2: Int,
|
||||
num: Int
|
||||
): Boolean {
|
||||
if (source1.size - offsetSource1 < num) return false
|
||||
if (source2.size - offsetSource2 < num) return false
|
||||
for (i in 0 until num) {
|
||||
if (source1[offsetSource1 + i] != source2[offsetSource2 + i]) return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
data class NalUnit (val type: Byte, val offset: Int, val length: Int)
|
||||
|
||||
|
||||
fun getNalUnits(
|
||||
data: ByteArray,
|
||||
dataOffset: Int,
|
||||
length: Int,
|
||||
foundNals: ArrayList<NalUnit>,
|
||||
isH265: Boolean
|
||||
): Int {
|
||||
foundNals.clear()
|
||||
var nalUnits = 0
|
||||
val nextNalOffset = 0
|
||||
val nalUnitPrefixSize = AtomicInteger(-1)
|
||||
val timestamp = System.currentTimeMillis()
|
||||
var offset = dataOffset
|
||||
var stopped = false
|
||||
while (!stopped) {
|
||||
|
||||
// Search for first NAL unit
|
||||
val nalUnitIndex = searchForNalUnitStart(
|
||||
data,
|
||||
offset + nextNalOffset,
|
||||
length - nextNalOffset,
|
||||
nalUnitPrefixSize
|
||||
)
|
||||
|
||||
// NAL unit found
|
||||
if (nalUnitIndex >= 0) {
|
||||
nalUnits++
|
||||
val nalUnitOffset = offset + nextNalOffset + nalUnitPrefixSize.get()
|
||||
val nalUnitTypeOctet = data[nalUnitOffset]
|
||||
val nalUnitType = if (isH265)
|
||||
((nalUnitTypeOctet.toInt() shr 1) and 0x3F).toByte()
|
||||
else
|
||||
(nalUnitTypeOctet and 0x1F)
|
||||
|
||||
// Search for second NAL unit (optional)
|
||||
var nextNalUnitStartIndex = searchForNalUnitStart(
|
||||
data,
|
||||
nalUnitOffset,
|
||||
length - nalUnitOffset,
|
||||
nalUnitPrefixSize
|
||||
)
|
||||
|
||||
// Second NAL unit not found. Use till the end.
|
||||
if (nextNalUnitStartIndex < 0) {
|
||||
// Not found next NAL unit. Use till the end.
|
||||
// nextNalUnitStartIndex = length - nextNalOffset + dataOffset;
|
||||
nextNalUnitStartIndex = length + dataOffset
|
||||
stopped = true
|
||||
}
|
||||
val l = nextNalUnitStartIndex - offset
|
||||
// if (DEBUG) Log.d(
|
||||
// TAG,
|
||||
// "NAL unit type: " + getH264NalUnitTypeString(nalUnitType.toInt()) +
|
||||
// " (" + nalUnitType + ") - " + l + " bytes, offset " + offset
|
||||
// )
|
||||
foundNals.add(NalUnit(nalUnitType, offset, l))
|
||||
offset = nextNalUnitStartIndex
|
||||
|
||||
// Check that we are not too long here
|
||||
if (System.currentTimeMillis() - timestamp > 200) {
|
||||
Log.w(TAG, "Cannot process data within 200 msec in $length bytes (NALs found: " + foundNals.size + ")")
|
||||
break
|
||||
}
|
||||
} else {
|
||||
stopped = true
|
||||
}
|
||||
}
|
||||
return nalUnits
|
||||
}
|
||||
|
||||
private fun getNalUnitStartLengthFromArray(
|
||||
src: ByteArray, offset: Int, length: Int,
|
||||
isH265: Boolean,
|
||||
nalUnitType: Byte
|
||||
): Pair<Int, Int>? {
|
||||
val nalUnitsFound = ArrayList<NalUnit>()
|
||||
if (getNalUnits(src, offset, length, nalUnitsFound, isH265) > 0) {
|
||||
for (nalUnit in nalUnitsFound) {
|
||||
if (nalUnit.type == nalUnitType) {
|
||||
val prefixSize = AtomicInteger()
|
||||
val nalUnitIndex = searchForNalUnitStart(
|
||||
src,
|
||||
nalUnit.offset,
|
||||
nalUnit.length,
|
||||
prefixSize
|
||||
)
|
||||
val nalOffset = nalUnitIndex + prefixSize.get() + 1 /* NAL unit type */
|
||||
return Pair(nalOffset, nalUnit.length)
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
@SuppressLint("UnsafeOptInUsageError")
|
||||
fun getSpsNalUnitFromArray(src: ByteArray, offset: Int, length: Int, isH265: Boolean): SpsData? {
|
||||
val spsStartLength = getNalUnitStartLengthFromArray(src, offset, length, isH265, NAL_SPS)
|
||||
spsStartLength?.let {
|
||||
return NalUnitUtil.parseSpsNalUnitPayload(
|
||||
src, spsStartLength.first, spsStartLength.first + spsStartLength.second)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
@SuppressLint("UnsafeOptInUsageError")
|
||||
fun getWidthHeightFromArray(src: ByteArray, offset: Int, length: Int, isH265: Boolean): Pair<Int, Int>? {
|
||||
val sps = getSpsNalUnitFromArray(src, offset, length, isH265)
|
||||
sps?.let {
|
||||
return Pair(sps.width, sps.height)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
// private fun isH265IRAP(nalUnitType: Byte): Boolean {
|
||||
// return nalUnitType in 16..23
|
||||
// }
|
||||
|
||||
fun isAnyKeyFrame(data: ByteArray?, offset: Int, length: Int, isH265: Boolean): Boolean {
|
||||
if (data == null || length <= 0) return false
|
||||
var currOffset = offset
|
||||
|
||||
val nalUnitPrefixSize = AtomicInteger(-1)
|
||||
val timestamp = System.currentTimeMillis()
|
||||
while (true) {
|
||||
val nalUnitIndex = searchForNalUnitStart(
|
||||
data,
|
||||
currOffset,
|
||||
length,
|
||||
nalUnitPrefixSize
|
||||
)
|
||||
|
||||
if (nalUnitIndex >= 0) {
|
||||
val nalUnitOffset = nalUnitIndex + nalUnitPrefixSize.get()
|
||||
if (nalUnitOffset >= data.size)
|
||||
return false
|
||||
val nalUnitTypeOctet = data[nalUnitOffset]
|
||||
|
||||
if (isH265) {
|
||||
val nalUnitType = ((nalUnitTypeOctet.toInt() and 0x7E) shr 1).toByte()
|
||||
// Treat SEI_PREFIX as key frame.
|
||||
if (nalUnitType == H265_NAL_IDR_W_RADL || nalUnitType == H265_NAL_IDR_N_LP)
|
||||
return true
|
||||
} else {
|
||||
val nalUnitType = (nalUnitTypeOctet.toInt() and 0x1f).toByte()
|
||||
when (nalUnitType) {
|
||||
NAL_IDR_SLICE -> return true
|
||||
NAL_SLICE -> return false
|
||||
}
|
||||
}
|
||||
// Continue searching
|
||||
currOffset = nalUnitOffset
|
||||
|
||||
// Check that we are not too long here
|
||||
if (System.currentTimeMillis() - timestamp > 100) {
|
||||
Log.w(TAG, "Cannot process data within 100 msec in $length bytes (index=$nalUnitIndex)")
|
||||
break
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
fun getH264NalUnitTypeString(nalUnitType: Byte): String {
|
||||
return when (nalUnitType) {
|
||||
NAL_SLICE -> "NAL_SLICE"
|
||||
NAL_DPA -> "NAL_DPA"
|
||||
NAL_DPB -> "NAL_DPB"
|
||||
NAL_DPC -> "NAL_DPC"
|
||||
NAL_IDR_SLICE -> "NAL_IDR_SLICE"
|
||||
NAL_SEI -> "NAL_SEI"
|
||||
NAL_SPS -> "NAL_SPS"
|
||||
NAL_PPS -> "NAL_PPS"
|
||||
NAL_AUD -> "NAL_AUD"
|
||||
NAL_END_SEQUENCE -> "NAL_END_SEQUENCE"
|
||||
NAL_END_STREAM -> "NAL_END_STREAM"
|
||||
NAL_FILLER_DATA -> "NAL_FILLER_DATA"
|
||||
NAL_SPS_EXT -> "NAL_SPS_EXT"
|
||||
NAL_AUXILIARY_SLICE -> "NAL_AUXILIARY_SLICE"
|
||||
NAL_STAP_A -> "NAL_STAP_A"
|
||||
NAL_STAP_B -> "NAL_STAP_B"
|
||||
NAL_MTAP16 -> "NAL_MTAP16"
|
||||
NAL_MTAP24 -> "NAL_MTAP24"
|
||||
NAL_FU_A -> "NAL_FU_A"
|
||||
NAL_FU_B -> "NAL_FU_B"
|
||||
else -> "unknown - $nalUnitType"
|
||||
}
|
||||
}
|
||||
|
||||
fun getH265NalUnitTypeString(nalUnitType: Byte): String {
|
||||
return when (nalUnitType) {
|
||||
H265_NAL_TRAIL_N -> "NAL_TRAIL_N"
|
||||
H265_NAL_TRAIL_R -> "NAL_TRAIL_R"
|
||||
H265_NAL_TSA_N -> "NAL_TSA_N"
|
||||
H265_NAL_TSA_R -> "NAL_TSA_R"
|
||||
H265_NAL_STSA_N -> "NAL_STSA_N"
|
||||
H265_NAL_STSA_R -> "NAL_STSA_R"
|
||||
H265_NAL_RADL_N -> "NAL_RADL_N"
|
||||
H265_NAL_RADL_R -> "NAL_RADL_R"
|
||||
H265_NAL_RASL_N -> "NAL_RASL_N"
|
||||
H265_NAL_RASL_R -> "NAL_RASL_R"
|
||||
H265_NAL_BLA_W_LP -> "NAL_BLA_W_LP"
|
||||
H265_NAL_BLA_W_RADL -> "NAL_BLA_W_RADL"
|
||||
H265_NAL_BLA_N_LP -> "NAL_BLA_N_LP"
|
||||
H265_NAL_IDR_W_RADL -> "NAL_IDR_W_RADL"
|
||||
H265_NAL_IDR_N_LP -> "NAL_IDR_N_LP"
|
||||
H265_NAL_CRA_NUT -> "NAL_CRA_NUT"
|
||||
H265_NAL_VPS -> "NAL_VPS"
|
||||
H265_NAL_SPS -> "NAL_SPS"
|
||||
H265_NAL_PPS -> "NAL_PPS"
|
||||
H265_NAL_AUD -> "NAL_AUD"
|
||||
H265_NAL_EOS_NUT -> "NAL_EOS_NUT"
|
||||
H265_NAL_EOB_NUT -> "NAL_EOB_NUT"
|
||||
H265_NAL_FD_NUT -> "NAL_FD_NUT"
|
||||
H265_NAL_SEI_PREFIX -> "NAL_SEI_PREFIX"
|
||||
H265_NAL_SEI_SUFFIX -> "NAL_SEI_SUFFIX"
|
||||
else -> "unknown - $nalUnitType"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user