From 1a59a848636f88e67791f5da60302f315dfb9451 Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Mon, 25 Dec 2017 16:24:42 +0300 Subject: [PATCH] Video player changes by Roma Elizarov. (#1190) --- .../src/main/kotlin/DecoderWorker.kt | 643 +++++++++--------- .../videoplayer/src/main/kotlin/Dimensions.kt | 5 + .../videoplayer/src/main/kotlin/Disposable.kt | 63 ++ samples/videoplayer/src/main/kotlin/Queue.kt | 25 +- .../videoplayer/src/main/kotlin/SDLAudio.kt | 106 +-- .../main/kotlin/{SDLBase.kt => SDLErrors.kt} | 10 +- .../videoplayer/src/main/kotlin/SDLInput.kt | 21 +- .../videoplayer/src/main/kotlin/SDLVideo.kt | 177 +++-- .../src/main/kotlin/VideoPlayer.kt | 233 +++---- 9 files changed, 661 insertions(+), 622 deletions(-) create mode 100644 samples/videoplayer/src/main/kotlin/Dimensions.kt create mode 100644 samples/videoplayer/src/main/kotlin/Disposable.kt rename samples/videoplayer/src/main/kotlin/{SDLBase.kt => SDLErrors.kt} (76%) diff --git a/samples/videoplayer/src/main/kotlin/DecoderWorker.kt b/samples/videoplayer/src/main/kotlin/DecoderWorker.kt index ce79eade209..ed54285a1f0 100644 --- a/samples/videoplayer/src/main/kotlin/DecoderWorker.kt +++ b/samples/videoplayer/src/main/kotlin/DecoderWorker.kt @@ -15,31 +15,30 @@ */ import ffmpeg.* -import kotlin.system.* import kotlinx.cinterop.* import konan.worker.* import platform.posix.memcpy // This global variable only set to != null value in the decoding worker. -private var state: DecodeWorkerState? = null +private var decoder: Decoder? = null -data class OutputInfo(val width: Int, val height: Int, val pixelFormat: AVPixelFormat) -data class VideoInfo(val width: Int, val height: Int, val fps: Double, - val sampleRate: Int, val channels: Int) +data class VideoInfo(val size: Dimensions, val fps: Double) +data class AudioInfo(val sampleRate: Int, val channels: Int) -data class VideoFrame(val buffer: CPointer, val lineSize: Int, val timeStamp: Double) { - fun unref() { - av_buffer_unref2(buffer) - } +data class CodecInfo(val video: VideoInfo?, val audio: AudioInfo?) { + val hasVideo = video != null + val hasAudio = audio != null } -data class AudioFrame(val buffer: CPointer, var position: Int, val size: Int, val timeStamp: Double) { - fun unref() { - av_buffer_unref2(buffer) - } +class VideoFrame(val buffer: CPointer, val lineSize: Int, val timeStamp: Double) { + fun unref() = av_buffer_unref2(buffer) } -private fun Int.checkError() { +class AudioFrame(val buffer: CPointer, var position: Int, val size: Int, val timeStamp: Double) { + fun unref() = av_buffer_unref2(buffer) +} + +private fun Int.checkAVError() { if (this != 0) { val buffer = ByteArray(1024) av_strerror(this, buffer.refTo(0), buffer.size.signExtend()) @@ -47,208 +46,196 @@ private fun Int.checkError() { } } -class DecodeWorkerState(val formatContext: CPointer, - val videoStreamIndex: Int, - val audioStreamIndex: Int, - val videoCodecContext: CPointer?, - val audioCodecContext: CPointer?) { - var videoFrame: CPointer? = null - var scaledVideoFrame: CPointer? = null - var audioFrame: CPointer? = null - var resampledAudioFrame: CPointer? = null - var softwareScalingContext: CPointer? = null - var resampleContext: CPointer? = null - val videoQueue = Queue(100, null) - val audioQueue = Queue(100, null) - var buffer: ByteArray? = null - var videoWidth = 0 - var videoHeight = 0 - var windowWidth = 0 - var windowHeight = 0 - var scaledFrameSize = 0 - var noMoreFrames = false - val minAudioFrames = 2 - val maxAudioFrames = 5 - val minVideoFrames = 5 +private val AVFormatContext.codecs: List + get() = List(nb_streams) { streams?.get(it)?.pointed?.codec?.pointed } - fun makeVideoFrame(): VideoFrame { - // TODO: reuse buffers! - // Convert the frame from its movie format to window pixel format. - sws_scale(softwareScalingContext, videoFrame!!.pointed.data, - videoFrame!!.pointed.linesize, 0, videoHeight, - scaledVideoFrame!!.pointed.data, scaledVideoFrame!!.pointed.linesize) - val buffer = av_buffer_alloc(scaledFrameSize)!! - val ts = av_frame_get_best_effort_timestamp(videoFrame)* av_q2d( - videoCodecContext!!.pointed.time_base.readValue()) - memcpy(buffer.pointed.data, scaledVideoFrame!!.pointed.data[0], scaledFrameSize.signExtend()) - return VideoFrame(buffer, scaledVideoFrame!!.pointed.linesize[0], ts) +private fun AVFormatContext.streamAt(index: Int): AVStream? = + if (index < 0) null else streams?.get(index)?.pointed + +private fun AVStream.openCodec(tag: String): AVCodecContext { + // Get codec context for the video stream. + val codecContext = codec!!.pointed + val codec = avcodec_find_decoder(codecContext.codec_id)?.pointed ?: + throw Error("Unsupported $tag codec with id ${codecContext.codec_id}...") + // Open codec. + if (avcodec_open2(codecContext.ptr, codec.ptr, null) < 0) + throw Error("Couldn't open $tag codec with id ${codecContext.codec_id}") + return codecContext +} + +class AVFile(private val fileName: String) : DisposableContainer() { + private val contextPtrPtr = arena.alloc>().ptr + private val contextPtr: CPointer + + init { + avformat_open_input(contextPtrPtr, fileName, null, null).checkAVError() + contextPtr = contextPtrPtr.pointed.value ?: throw Error("Failed to open AV file") + tryConstruct { + if (avformat_find_stream_info(contextPtr, null) < 0) + throw Error("Couldn't find stream information") + } } - fun makeAudioFrame(): AudioFrame { - avresample_convert_frame(resampleContext, resampledAudioFrame, audioFrame).checkError() - val audioFrameSize = av_samples_get_buffer_size( - null, - resampledAudioFrame!!.pointed.channels, - resampledAudioFrame!!.pointed.nb_samples, - resampledAudioFrame!!.pointed.format, - 1) - val ts = av_frame_get_best_effort_timestamp(audioFrame) * av_q2d( - audioCodecContext!!.pointed.time_base.readValue()) - val buffer = av_buffer_alloc(audioFrameSize)!! - memcpy(buffer.pointed.data, resampledAudioFrame!!.pointed.data[0], audioFrameSize.signExtend()) - return AudioFrame(buffer, 0, audioFrameSize, ts) + override fun dispose() { + avformat_close_input(contextPtrPtr) + super.dispose() + } + + fun dumpFormat() = av_dump_format(contextPtr, 0, fileName, 0) + + val context get() = contextPtr.pointed +} + +private fun PixelFormat.toAVPixelFormat(): AVPixelFormat? = when (this) { + PixelFormat.RGB24 -> AV_PIX_FMT_RGB24 + PixelFormat.ARGB32 -> AV_PIX_FMT_RGB32 + PixelFormat.INVALID -> null +} + +private data class VideoDecoderOutput(val size: Dimensions, val avPixelFormat: AVPixelFormat) + +// Performs data type conversion and copy to transfer data to DecoderWorker +private fun VideoOutput.toVideoDecoderOutput(): VideoDecoderOutput? { + val avPixelFormat = pixelFormat.toAVPixelFormat() ?: return null + return VideoDecoderOutput(size.copy(), avPixelFormat) +} + +private class VideoDecoder( + private val videoCodecContext: AVCodecContext, + output: VideoDecoderOutput +) : DisposableContainer() { + private val windowSize = output.size + private val avPixelFormat = output.avPixelFormat + private val videoSize = Dimensions(videoCodecContext.width, videoCodecContext.height) + private val videoFrame: AVFrame = + disposable("av_frame_alloc", ::av_frame_alloc, ::av_frame_unref).pointed + private val scaledVideoFrame: AVFrame = + disposable("av_frame_alloc", ::av_frame_alloc, ::av_frame_unref).pointed + private val softwareScalingContext: CPointer = disposable( + message = "sws_getContext", + create = { + sws_getContext( + videoSize.w, videoSize.h, + videoCodecContext.pix_fmt, + windowSize.w, windowSize.h, avPixelFormat, + SWS_BILINEAR, null, null, null) + }, + dispose = ::sws_freeContext + ) + private val scaledFrameSize = avpicture_get_size(avPixelFormat, windowSize.w, windowSize.h) + private val buffer: ByteArray = ByteArray(scaledFrameSize) + + private val videoQueue = Queue(100) + + private val minVideoFrames = 5 + + init { + avpicture_fill(scaledVideoFrame.ptr.reinterpret(), buffer.refTo(0), + avPixelFormat, windowSize.w, windowSize.h) + } + + override fun dispose() { + super.dispose() + while (!videoQueue.isEmpty()) videoQueue.pop().unref() + } + + fun isQueueEmpty() = videoQueue.isEmpty() + fun isQueueAlmostFull() = videoQueue.size() > videoQueue.maxSize - 5 + fun needMoreFrames() = videoQueue.size() < minVideoFrames + fun nextFrame() = videoQueue.popOrNull() + + fun decodeVideoPacket(packet: AVPacket, frameFinished: IntVar) { + // Decode video frame. + avcodec_decode_video2(videoCodecContext.ptr, videoFrame.ptr, frameFinished.ptr, packet.ptr) + // Did we get a video frame? + if (frameFinished.value != 0) { + // Convert the frame from its movie format to window pixel format. + sws_scale(softwareScalingContext, videoFrame.data, + videoFrame.linesize, 0, videoSize.h, + scaledVideoFrame.data, scaledVideoFrame.linesize) + // TODO: reuse buffers! + val buffer = av_buffer_alloc(scaledFrameSize)!! + val ts = av_frame_get_best_effort_timestamp(videoFrame.ptr) * + av_q2d(videoCodecContext.time_base.readValue()) + memcpy(buffer.pointed.data, scaledVideoFrame.data[0], scaledFrameSize.signExtend()) + videoQueue.push(VideoFrame(buffer, scaledVideoFrame.linesize[0], ts)) + } + } + +} + +private fun SampleFormat.toAVSampleFormat(): AVSampleFormat? = when (this) { + SampleFormat.S16 -> AV_SAMPLE_FMT_S16 + SampleFormat.INVALID -> null +} + +private data class AudioDecoderOutput( + val sampleRate: Int, + val channels: Int, + val channelLayout: Int, + val sampleFormat: AVSampleFormat) + +// Performs data type conversion and copy to transfer data to DecoderWorker +private fun AudioOutput.toAudioDecoderOutput(): AudioDecoderOutput? { + val avSampleFormat = sampleFormat.toAVSampleFormat() ?: return null + if (channels != 2) return null // only stereo output is supported for now + return AudioDecoderOutput(sampleRate, channels, AV_CH_LAYOUT_STEREO, avSampleFormat) +} + +private class AudioDecoder( + private val audioCodecContext: AVCodecContext, + output: AudioDecoderOutput +): DisposableContainer() { + private val audioFrame: AVFrame = + disposable(create = ::av_frame_alloc, dispose = ::av_frame_unref).pointed + private val resampledAudioFrame: AVFrame = + disposable(create = ::av_frame_alloc, dispose = ::av_frame_unref).pointed + private val resampleContext: CPointer = + disposable(create = ::avresample_alloc_context, dispose = ::avresample_free2) + + private val audioQueue = Queue(100) + + private val minAudioFrames = 2 + private val maxAudioFrames = 5 + + init { + with (resampledAudioFrame) { + channels = output.channels + sample_rate = output.sampleRate + format = output.sampleFormat + channel_layout = output.channelLayout.signExtend() + } + + with (audioCodecContext) { + var inputChannelLayout = if (channel_layout != 0L) + channel_layout else av_get_default_channel_layout(audioCodecContext.channels) + setResampleOpt("in_channel_layout", channel_layout.narrow()) + setResampleOpt("out_channel_layout", output.channelLayout) + setResampleOpt("in_sample_rate", sample_rate) + setResampleOpt("out_sample_rate", output.sampleRate) + setResampleOpt("in_sample_fmt", sample_fmt) + setResampleOpt("out_sample_fmt", output.sampleFormat) + } + avresample_open(resampleContext) } private fun setResampleOpt(name: String, value: Int) = - av_opt_set_int(resampleContext, name, value.signExtend(), 0) + av_opt_set_int(resampleContext, name, value.signExtend(), 0) - - fun start(output: OutputInfo) { - if (videoCodecContext != null) { - videoWidth = videoCodecContext.pointed.width - videoHeight = videoCodecContext.pointed.height - windowWidth = output.width - windowHeight = output.height - videoFrame = av_frame_alloc()!! - scaledVideoFrame = av_frame_alloc()!! - softwareScalingContext = sws_getContext( - videoWidth, - videoHeight, - videoCodecContext.pointed.pix_fmt, - windowWidth, windowHeight, output.pixelFormat, - SWS_BILINEAR, null, null, null)!! - - scaledFrameSize = avpicture_get_size(output.pixelFormat, output.width, output.height) - buffer = ByteArray(scaledFrameSize) - - avpicture_fill(scaledVideoFrame!!.reinterpret(), buffer!!.refTo(0), - output.pixelFormat, output.width, output.height) - } - - if (audioCodecContext != null) { - audioFrame = av_frame_alloc()!! - resampledAudioFrame = av_frame_alloc()!! - resampledAudioFrame!!.pointed.format = AV_SAMPLE_FMT_S16 - resampledAudioFrame!!.pointed.channels = 2 - resampledAudioFrame!!.pointed.channel_layout = AV_CH_LAYOUT_STEREO.signExtend() - resampledAudioFrame!!.pointed.sample_rate = 44100 - resampleContext = avresample_alloc_context() - setResampleOpt("in_channel_layout", audioCodecContext.pointed.channel_layout.narrow()) - setResampleOpt("out_channel_layout", AV_CH_LAYOUT_STEREO) - setResampleOpt("in_sample_rate", audioCodecContext.pointed.sample_rate) - setResampleOpt("out_sample_rate", 44100) - setResampleOpt("in_sample_fmt", audioCodecContext.pointed.sample_fmt) - setResampleOpt("out_sample_fmt", AV_SAMPLE_FMT_S16) - avresample_open(resampleContext) - } - - noMoreFrames = false - - decodeIfNeeded() + override fun dispose() { + super.dispose() + while (!audioQueue.isEmpty()) audioQueue.pop().unref() } - fun done() = noMoreFrames && videoQueue.isEmpty() && audioQueue.isEmpty() + fun isSynced(): Boolean = audioQueue.size() < maxAudioFrames - fun stop() { - while (!videoQueue.isEmpty()) { - videoQueue.pop()?.unref() - } - while (!audioQueue.isEmpty()) { - audioQueue.pop()?.unref() - } - if (videoFrame != null) { - av_frame_unref(videoFrame) - videoFrame = null - } - if (scaledVideoFrame != null) { - av_frame_unref(scaledVideoFrame) - scaledVideoFrame = null - } - if (audioFrame != null) { - av_frame_unref(audioFrame) - audioFrame = null - } - if (resampledAudioFrame != null) { - av_frame_unref(resampledAudioFrame) - resampledAudioFrame = null - } - if (softwareScalingContext != null) { - sws_freeContext(softwareScalingContext) - softwareScalingContext = null - } - if (resampleContext != null) { - avresample_free2(resampleContext) - resampleContext = null - } - if (videoCodecContext != null) { - avcodec_free_context2(videoCodecContext) - } - if (audioCodecContext != null) { - avcodec_free_context2(audioCodecContext) - } - //avformat_free_context2(formatContext) - } + fun isQueueEmpty() = audioQueue.isEmpty() + fun isQueueAlmostFull() = audioQueue.size() > audioQueue.maxSize - 20 + fun needMoreFrames() = audioQueue.size() < minAudioFrames - fun needMoreBuffers(): Boolean { - return ((videoStreamIndex != -1) && (videoQueue.size() < minVideoFrames)) || - ((audioStreamIndex != -1) && (audioQueue.size() < minAudioFrames)) - } - - fun decodeIfNeeded() { - if (!needMoreBuffers() || audioQueue.size() > audioQueue.maxSize - 20 || - videoQueue.size() > videoQueue.maxSize - 5) return - - memScoped { - val packet = alloc() - val frameFinished = alloc() - while (needMoreBuffers() && av_read_frame(formatContext, packet.ptr) >= 0) { - when (packet.stream_index) { - videoStreamIndex -> { - // Decode video frame. - avcodec_decode_video2(videoCodecContext, videoFrame, frameFinished.ptr, packet.ptr) - // Did we get a video frame? - if (frameFinished.value != 0) { - videoQueue.push(makeVideoFrame()) - } - } - audioStreamIndex -> { - while (packet.size > 0) { - val size = avcodec_decode_audio4( - audioCodecContext, audioFrame, frameFinished.ptr, packet.ptr) - - if (frameFinished.value != 0) { - // Put audio frame to decoder's queue. - audioQueue.push(makeAudioFrame()) - } - packet.size -= size - packet.data += size - } - } - } - av_packet_unref(packet.ptr) - } - if (needMoreBuffers()) noMoreFrames = true - } - } - - fun nextVideoFrame(): VideoFrame? { - decodeIfNeeded() - - if (videoQueue.isEmpty()) { - return null - } - val frame = videoQueue.pop()!! - return frame - } - - fun nextAudioFrame(size: Int): AudioFrame? { - decodeIfNeeded() - - if (audioQueue.isEmpty()) { - return null - } - val frame = audioQueue.peek()!! - var realSize = if (frame.position + size > frame.size) frame.size - frame.position else size + fun nextFrame(size: Int): AudioFrame? { + val frame = audioQueue.peek() ?: return null + val realSize = if (frame.position + size > frame.size) frame.size - frame.position else size if (frame.position + realSize == frame.size) { return audioQueue.pop() } else { @@ -258,144 +245,164 @@ class DecodeWorkerState(val formatContext: CPointer, } } - fun audioVideoSynced() = (audioQueue.size() < maxAudioFrames) || done() + fun decodeAudioPacket(packet: AVPacket, frameFinished: IntVar) { + while (packet.size > 0) { + val size = avcodec_decode_audio4(audioCodecContext.ptr, audioFrame.ptr, frameFinished.ptr, packet.ptr) + if (frameFinished.value != 0) { + // Put audio frame to decoder's queue. + avresample_convert_frame(resampleContext, resampledAudioFrame.ptr, audioFrame.ptr).checkAVError() + with (resampledAudioFrame) { + val audioFrameSize = av_samples_get_buffer_size(null, channels, nb_samples, format, 1) + val buffer = av_buffer_alloc(audioFrameSize)!! + val ts = av_frame_get_best_effort_timestamp(audioFrame.ptr) * + av_q2d(audioCodecContext.time_base.readValue()) + memcpy(buffer.pointed.data, data[0], audioFrameSize.signExtend()) + audioQueue.push(AudioFrame(buffer, 0, audioFrameSize, ts)) + } + } + packet.size -= size + packet.data += size + } + } } -class DecodeWorker { - // This class must have no other state, but this worker object. - // All the real state must be stored on the worker's side. - private val decodeWorker: Worker +private class Decoder( + private val formatContext: CPointer, + private val videoStreamIndex: Int, + private val audioStreamIndex: Int, + private val videoCodecContext: AVCodecContext?, + private val audioCodecContext: AVCodecContext? +) { + private var video: VideoDecoder? = null + private var audio: AudioDecoder? = null - constructor() { - decodeWorker = konan.worker.startWorker() + var noMoreFrames = false + + fun start(videoOutput: VideoDecoderOutput?, audioOutput: AudioDecoderOutput?) { + video = videoCodecContext?.let { ctx -> videoOutput?.let { VideoDecoder(ctx, it) } } + audio = audioCodecContext?.let { ctx -> audioOutput?.let { AudioDecoder(ctx, it) } } + noMoreFrames = false + decodeIfNeeded() } - constructor(id: WorkerId) { - decodeWorker = Worker(id) + fun done() = noMoreFrames && (video?.isQueueEmpty() ?: true) && (audio?.isQueueEmpty() ?: true) + + fun dispose() { + video?.dispose() + audio?.dispose() } - fun workerId() = decodeWorker.id + private fun needMoreFrames(): Boolean = + (video?.needMoreFrames() ?: false) || (audio?.needMoreFrames() ?: false) - fun renderPixelFormat(pixelFormat: PixelFormat) = when (pixelFormat) { - PixelFormat.RGB24 -> AV_PIX_FMT_RGB24 - PixelFormat.ARGB32 -> AV_PIX_FMT_RGB32 - PixelFormat.INVALID -> AV_PIX_FMT_NONE - } - - private fun findStream(useStream: Boolean, formatContext: CPointer, streamIndex: Int, tag: String): - Pair?, CPointer?> { - if (streamIndex < 0 || !useStream) return null to null - val stream = formatContext.pointed.streams!!.get(streamIndex)!! - // Get codec context for the video stream. - val codecContext = stream.pointed.codec!! - val codec = avcodec_find_decoder(codecContext.pointed.codec_id) - if (codec == null) - throw Error("Unsupported $tag codec...") - // Open codec. - if (avcodec_open2(codecContext, codec, null) < 0) - throw Error("Couldn't open $tag codec") - return stream to codecContext - } - - fun initDecode(file: String, useVideo: Boolean = true, useAudio: Boolean = true): VideoInfo { + fun decodeIfNeeded() { + if (!needMoreFrames()) return + if (video?.isQueueAlmostFull() == true) return + if (audio?.isQueueAlmostFull() == true) return memScoped { - try { - val formatContextPtr = alloc>() - if (avformat_open_input(formatContextPtr.ptr, file, null, null) != 0) - throw Error("Cannot open video file") - val formatContext = formatContextPtr.value!! - if (avformat_find_stream_info(formatContext, null) < 0) - throw Error("Couldn't find stream information") - av_dump_format(formatContext, 0, file, 0) - - // Find the first video/audio streams. - var videoStreamIndex = -1 - var audioStreamIndex = -1 - for (i in 0 until formatContext.pointed.nb_streams) { - val stream = formatContext.pointed.streams!!.get(i) - val codec = stream!!.pointed.codec!!.pointed - if (codec.codec_type == AVMEDIA_TYPE_VIDEO && videoStreamIndex == -1) { - videoStreamIndex = i - } - if (codec.codec_type == AVMEDIA_TYPE_AUDIO && audioStreamIndex == -1) { - audioStreamIndex = i - } + val packet = alloc() + val frameFinished = alloc() + while (needMoreFrames() && av_read_frame(formatContext, packet.ptr) >= 0) { + when (packet.stream_index) { + videoStreamIndex -> video?.decodeVideoPacket(packet, frameFinished) + audioStreamIndex -> audio?.decodeAudioPacket(packet, frameFinished) } - - val (videoStream, videoCodecContext) = findStream(useVideo, formatContext, videoStreamIndex, "video") - val (_, audioCodecContext) = findStream(useAudio, formatContext, audioStreamIndex, "audio") - - // Extract video info. - val (videoWidth, videoHeight, fps) = if (videoCodecContext != null) { - Triple(videoCodecContext.pointed.width, videoCodecContext.pointed.height, - av_q2d(av_stream_get_r_frame_rate(videoStream))) - } else { - Triple(-1, -1, 0.0) - } - val (sampleRate, channels) = if (audioCodecContext != null) { - Pair(audioCodecContext.pointed.sample_rate, audioCodecContext.pointed.channels) - } else { - Pair(0, 0) - } - - // Pack all inited state and pass it to the worker. - decodeWorker.schedule(TransferMode.CHECKED, { - DecodeWorkerState(formatContext, - videoStreamIndex, audioStreamIndex, - videoCodecContext, audioCodecContext) - }) { input -> - state = input - null - } - return VideoInfo(videoWidth, videoHeight, fps, sampleRate, channels) - } finally { - // TODO: clean up whatever we allocated. + av_packet_unref(packet.ptr) } + if (needMoreFrames()) noMoreFrames = true } } - fun init() { + fun nextVideoFrame(): VideoFrame? { + decodeIfNeeded() + return video?.nextFrame() } - fun deinit() { - decodeWorker.requestTermination().result() + fun nextAudioFrame(size: Int): AudioFrame? { + decodeIfNeeded() + return audio?.nextFrame(size) } - fun start(width: Int, height: Int, pixelFormat: PixelFormat) { - decodeWorker.schedule(TransferMode.CHECKED, { - OutputInfo(width, height, renderPixelFormat(pixelFormat)) - }) { input -> state!!.start(input) } + fun audioVideoSynced() = (audio?.isSynced() ?: true) || done() +} + +class DecoderWorker : Disposable { + // This class must have no other state, but this worker object. + // All the real state must be stored on the worker's side. + private val worker: Worker + + constructor() { worker = konan.worker.startWorker() } + constructor(id: WorkerId) { worker = Worker(id) } + + override fun dispose() { + worker.requestTermination().result() + } + + val workerId get() = worker.id + + fun initDecode(context: AVFormatContext, useVideo: Boolean = true, useAudio: Boolean = true): CodecInfo { + // Find the first video/audio streams. + val videoStreamIndex = + if (useVideo) context.codecs.indexOfFirst { it?.codec_type == AVMEDIA_TYPE_VIDEO } else -1 + val audioStreamIndex = + if (useAudio) context.codecs.indexOfFirst { it?.codec_type == AVMEDIA_TYPE_AUDIO } else -1 + + val videoStream = context.streamAt(videoStreamIndex) + val audioStream = context.streamAt(audioStreamIndex) + + val videoContext = videoStream?.openCodec("video") + val audioContext = audioStream?.openCodec("audio") + + // Extract video info. + val video = videoContext?.run { + VideoInfo(Dimensions(width, height), av_q2d(av_stream_get_r_frame_rate(videoStream.ptr))) + } + // Extract audio info. + val audio = audioContext?.run { + AudioInfo(sample_rate, channels) + } + + // Pack all state and pass it to the worker. + worker.schedule(TransferMode.CHECKED, { + Decoder(context.ptr, + videoStreamIndex, audioStreamIndex, + videoContext, audioContext) + }) { decoder = it } + return CodecInfo(video, audio) + } + + fun start(videoOutput: VideoOutput, audioOutput: AudioOutput) { + worker.schedule(TransferMode.CHECKED, + { Pair( + videoOutput.toVideoDecoderOutput(), + audioOutput.toAudioDecoderOutput()) + }) { + decoder?.start(it.first, it.second) + } } fun stop() { - decodeWorker.schedule( - TransferMode.CHECKED, - { null }) { _ -> - state?.stop() - state = null - }.result() + worker.schedule(TransferMode.CHECKED, { null }) { + decoder?.run { + dispose() + decoder = null + } + }.result() } - // TODO: we manually box returned primitive value, - // fix by autoboxing schedule()'s result in the compiler. - fun done() = decodeWorker.schedule(TransferMode.CHECKED, - { null }) { _ -> (state == null || state!!.done()) as Boolean? - }.consume { it -> it!! } + fun done(): Boolean = + worker.schedule(TransferMode.CHECKED, { null }) { decoder?.done() ?: true }.result() + fun requestDecodeChunk() { + worker.schedule(TransferMode.CHECKED, { null }) { decoder?.decodeIfNeeded() }.result() + } - fun requestDecodeChunk() = decodeWorker.schedule( - TransferMode.CHECKED, - { null }) { _ -> state!!.decodeIfNeeded() } + fun nextVideoFrame(): VideoFrame? = + worker.schedule(TransferMode.CHECKED, { null }) { decoder?.nextVideoFrame() }.result() - fun nextVideoFrame(): VideoFrame? = decodeWorker.schedule( - TransferMode.CHECKED, - { null }) { _ -> state!!.nextVideoFrame() }.result() + fun nextAudioFrame(size: Int): AudioFrame? = + worker.schedule(TransferMode.CHECKED, { size }) { decoder?.nextAudioFrame(it) }.result() - fun nextAudioFrame(size: Int) = decodeWorker.schedule( - TransferMode.CHECKED, - { size }) { input -> state!!.nextAudioFrame(input) }.result() - - fun audioVideoSynced() = decodeWorker.schedule( - TransferMode.CHECKED, - { null }) { _ -> state!!.audioVideoSynced() }.result() + fun audioVideoSynced(): Boolean = + worker.schedule(TransferMode.CHECKED, { null }) { decoder?.audioVideoSynced() ?: true }.result() } diff --git a/samples/videoplayer/src/main/kotlin/Dimensions.kt b/samples/videoplayer/src/main/kotlin/Dimensions.kt new file mode 100644 index 00000000000..fa6dc0be7fd --- /dev/null +++ b/samples/videoplayer/src/main/kotlin/Dimensions.kt @@ -0,0 +1,5 @@ + +data class Dimensions(val w: Int, val h: Int) { + operator fun minus(other: Dimensions) = Dimensions(w - other.w, h - other.h) + operator fun div(b: Int) = Dimensions(w / b, h / b) +} diff --git a/samples/videoplayer/src/main/kotlin/Disposable.kt b/samples/videoplayer/src/main/kotlin/Disposable.kt new file mode 100644 index 00000000000..257ce6f72d7 --- /dev/null +++ b/samples/videoplayer/src/main/kotlin/Disposable.kt @@ -0,0 +1,63 @@ + +import kotlinx.cinterop.* + +/** + * Disposable class manages and owns all its native resources. It allocates some resources + * during construction and may allocate some additional resources during operation. + * It must free all its resource once [dispose] is invoked. "Disposed" is a final state of the disposable + * class. It is not supposed to be used after being disposed. + */ +interface Disposable { + /** + * Disposes all native resources owned by this class. This function must be invoked + * exactly once as the last operation on the corresponding class. + */ + fun dispose() +} + +/** + * Helper class to implement [Disposable] interface. It contains an [arena] for native + * memory allocations and a number of helper methods to simplify management of other + * kinds of native resources. + * + * It is important to wrap all potentially exception-throwing code in the class constructor + * into [tryConstruct] invocation, because, when object construction fails with exception, + * it ensures that all the resource that were allocated so far will get freed. + */ +abstract class DisposableContainer : Disposable { + val arena = Arena() + + override fun dispose() { + arena.clear() + } + + inline fun tryConstruct(init: () -> T): T = + try { init() } + catch (e: Throwable) { + dispose() + throw e + } + + // TODO: It shall be inline, but crashes backend + fun disposable( + message: String = "disposable", + create: () -> T?, + dispose: (T) -> Unit + ): T = + tryConstruct { + create()?.also { + arena.defer { dispose(it) } + } ?: throw Error(message) + } + + // TODO: It shall be inline, but crashes backend + fun disposable(create: () -> T): T = + disposable( + create = create, + dispose = { it.dispose() }) + + fun sdlDisposable(message: String, ptr: CPointer?, dispose: (CPointer) -> Unit): CPointer = + disposable( + create = { ptr ?: throwSDLError(message) }, + dispose = dispose) +} diff --git a/samples/videoplayer/src/main/kotlin/Queue.kt b/samples/videoplayer/src/main/kotlin/Queue.kt index 86a944b4187..b83b60434e1 100644 --- a/samples/videoplayer/src/main/kotlin/Queue.kt +++ b/samples/videoplayer/src/main/kotlin/Queue.kt @@ -14,10 +14,10 @@ * limitations under the License. */ -class Queue(val maxSize: Int, val none: T) { - val array = Array(maxSize, { _ -> none}) - var head = 0 - var tail = 0 +class Queue(val maxSize: Int) { + private val array = kotlin.arrayOfNulls(maxSize) + private var head = 0 + private var tail = 0 fun push(element: T) { if ((tail + 1) % maxSize == head) @@ -26,22 +26,25 @@ class Queue(val maxSize: Int, val none: T) { tail = (tail + 1) % maxSize } - fun pop() : T { + @Suppress("UNCHECKED_CAST") + fun pop(): T { if (tail == head) throw Error("queue underflow") - val result = array[head] - array[head] = none + val result = array[head] as T + array[head] = null head = (head + 1) % maxSize return result } - fun peek() : T { - if (tail == head) - throw Error("queue underflow") - return array[head] + @Suppress("UNCHECKED_CAST") + fun peek() : T? { + if (isEmpty()) return null + return array[head] as T } fun size() = if (tail >= head) tail - head else maxSize - (head - tail) fun isEmpty() = head == tail + + fun popOrNull(): T? = if (isEmpty()) null else pop() } diff --git a/samples/videoplayer/src/main/kotlin/SDLAudio.kt b/samples/videoplayer/src/main/kotlin/SDLAudio.kt index a116ee4f41f..a4299104c2b 100644 --- a/samples/videoplayer/src/main/kotlin/SDLAudio.kt +++ b/samples/videoplayer/src/main/kotlin/SDLAudio.kt @@ -16,85 +16,85 @@ import kotlinx.cinterop.* import sdl.* -import konan.worker.WorkerId import platform.posix.memset import platform.posix.memcpy -class SDLAudio(val player: VideoPlayer) : SDLBase() { +enum class SampleFormat { + INVALID, + S16 +} - private var threadData: CPointer? = null +data class AudioOutput(val sampleRate: Int, val channels: Int, val sampleFormat: SampleFormat) - override fun init() { - threadData = nativeHeap.alloc().ptr - } +private fun SampleFormat.toSDLFormat(): SDL_AudioFormat? = when (this) { + SampleFormat.S16 -> AUDIO_S16SYS.narrow() + SampleFormat.INVALID -> null +} - override fun deinit() { - if (threadData != null) { - nativeHeap.free(threadData!!) - threadData = null - } - } +class SDLAudio(val player: VideoPlayer) : DisposableContainer() { + private val threadData = arena.alloc().ptr + private var state = State.STOPPED - fun start(sampleRate: Int, channels: Int) { - println("Audio: $channels channels, $sampleRate samples per second") + fun start(audio: AudioOutput) { + stop() + val audioFormat = audio.sampleFormat.toSDLFormat() ?: return + println("SDL Audio: Playing output with ${audio.channels} channels, ${audio.sampleRate} samples per second") memScoped { // TODO: better mechanisms to ensure we have same output format here and in resampler of the decoder. - val spec = alloc() - spec.freq = 44100 - spec.format = AUDIO_S16SYS.narrow() - spec.channels = 2.narrow() - spec.silence = 0 - spec.samples = 4096 - spec.callback = staticCFunction { - userdata, buffer, length -> - // This handler will be invoked in the audio thread, so reinit runtime. - konan.initRuntimeIfNeeded() - - if (decoder == null) { - val callbackData = userdata!!.reinterpret() - decoder = DecodeWorker(callbackData.pointed.value) - } - var outPosition = 0 - while (outPosition < length) { - val frame = decoder!!.nextAudioFrame(length - outPosition) - if (frame != null) { - val toCopy = min(length - outPosition, frame.size - frame.position) - memcpy(buffer + outPosition, frame.buffer.pointed.data + frame.position, toCopy.signExtend()) - frame.unref() - outPosition += toCopy - } else { - println("Decoder returned nothing!") - memset(buffer + outPosition, 0, (length - outPosition).signExtend()) - break - } - } + val spec = alloc().apply { + freq = audio.sampleRate + format = audioFormat + channels = audio.channels.narrow() + silence = 0 + samples = 4096 + userdata = threadData + callback = staticCFunction(::audioCallback) } - threadData!!.pointed.value = player.decoder.workerId() - spec.userdata = threadData + threadData.pointed.value = player.workerId val realSpec = alloc() if (SDL_OpenAudio(spec.ptr, realSpec.ptr) < 0) - throw Error("SDL_OpenAudio: ${get_SDL_Error()}") + throwSDLError("SDL_OpenAudio") // TODO: ensure real spec matches what we asked for. - + state = State.PAUSED resume() } } fun pause() { - SDL_PauseAudio(1) + state = state.transition(State.PLAYING, State.PAUSED) { SDL_PauseAudio(1) } } fun resume() { - SDL_PauseAudio(0) + state = state.transition(State.PAUSED, State.PLAYING) { SDL_PauseAudio(0) } } fun stop() { pause() - SDL_CloseAudio() - + state = state.transition(State.PAUSED, State.STOPPED) { SDL_CloseAudio() } } } -// This global is only set in the audio thread. -var decoder: DecodeWorker? = null +// Only set in the audio thread +private var decoder: DecoderWorker? = null + +private fun audioCallback(userdata: COpaquePointer?, buffer: CPointer?, length: Int) { + // This handler will be invoked in the audio thread, so reinit runtime. + konan.initRuntimeIfNeeded() + val decoder = decoder ?: + DecoderWorker(userdata!!.reinterpret().pointed.value).also { decoder = it } + var outPosition = 0 + while (outPosition < length) { + val frame = decoder.nextAudioFrame(length - outPosition) + if (frame != null) { + val toCopy = min(length - outPosition, frame.size - frame.position) + memcpy(buffer + outPosition, frame.buffer.pointed.data + frame.position, toCopy.signExtend()) + frame.unref() + outPosition += toCopy + } else { + // println("Decoder returned nothing!") + memset(buffer + outPosition, 0, (length - outPosition).signExtend()) + break + } + } +} \ No newline at end of file diff --git a/samples/videoplayer/src/main/kotlin/SDLBase.kt b/samples/videoplayer/src/main/kotlin/SDLErrors.kt similarity index 76% rename from samples/videoplayer/src/main/kotlin/SDLBase.kt rename to samples/videoplayer/src/main/kotlin/SDLErrors.kt index b18ba6f013d..18a02f33335 100644 --- a/samples/videoplayer/src/main/kotlin/SDLBase.kt +++ b/samples/videoplayer/src/main/kotlin/SDLErrors.kt @@ -17,9 +17,9 @@ import sdl.SDL_GetError import kotlinx.cinterop.* -open class SDLBase { - protected fun get_SDL_Error() = SDL_GetError()!!.toKString() +fun throwSDLError(name: String): Nothing = + throw Error("SDL_$name Error: ${SDL_GetError()!!.toKString()}") - open fun init() {} - open fun deinit() {} -} \ No newline at end of file +fun checkSDLError(name: String, result: Int) { + if (result != 0) throwSDLError(name) +} diff --git a/samples/videoplayer/src/main/kotlin/SDLInput.kt b/samples/videoplayer/src/main/kotlin/SDLInput.kt index a5833590076..f7cc2986afd 100644 --- a/samples/videoplayer/src/main/kotlin/SDLInput.kt +++ b/samples/videoplayer/src/main/kotlin/SDLInput.kt @@ -17,26 +17,15 @@ import kotlinx.cinterop.* import sdl.* -class SDLInput(val player: VideoPlayer) : SDLBase() { - var event: CPointer? = null - - override fun init() { - event = nativeHeap.alloc().ptr - } - - override fun deinit() { - if (event != null) { - nativeHeap.free(event!!) - event = null - } - } +class SDLInput(val player: VideoPlayer) : DisposableContainer() { + private val event = arena.alloc().ptr fun check() { - while (SDL_PollEvent(event!!.reinterpret()) != 0) { - when (event!!.pointed.type) { + while (SDL_PollEvent(event.reinterpret()) != 0) { + when (event.pointed.type) { SDL_QUIT -> player.stop() SDL_KEYDOWN -> { - val keyboardEvent = event!!.reinterpret().pointed + val keyboardEvent = event.reinterpret().pointed when (keyboardEvent.keysym.scancode) { SDL_SCANCODE_ESCAPE -> player.stop() SDL_SCANCODE_SPACE -> player.pause() diff --git a/samples/videoplayer/src/main/kotlin/SDLVideo.kt b/samples/videoplayer/src/main/kotlin/SDLVideo.kt index 9c657bcf681..0e24c328060 100644 --- a/samples/videoplayer/src/main/kotlin/SDLVideo.kt +++ b/samples/videoplayer/src/main/kotlin/SDLVideo.kt @@ -14,117 +14,98 @@ * limitations under the License. */ +import ffmpeg.AV_PIX_FMT_NONE +import ffmpeg.AV_PIX_FMT_RGB24 +import ffmpeg.AV_PIX_FMT_RGB32 import kotlinx.cinterop.* import sdl.* -class SDLVideo(val player: VideoPlayer) : SDLBase() { - var displayWidth = 0 - var displayHeight = 0 - var windowWidth = 0 - var windowHeight = 0 +enum class PixelFormat { + INVALID, + RGB24, + ARGB32 +} - private var window: CPointer? = null - private var renderer: CPointer? = null - private var surface: CPointer? = null - private var texture: CPointer? = null - private var rect: CPointer? = null +data class VideoOutput(val size: Dimensions, val pixelFormat: PixelFormat) - override fun init() { - if (SDL_Init(SDL_INIT_EVERYTHING) != 0) { - println("SDL_Init Error: ${get_SDL_Error()}") - throw Error() - } +class SDLVideo : DisposableContainer() { + private val displaySize: Dimensions + private var window: SDLRendererWindow? = null - memScoped { - val displayMode = alloc() - if (SDL_GetCurrentDisplayMode(0, displayMode.ptr.reinterpret()) != 0) { - println("SDL_GetCurrentDisplayMode Error: ${get_SDL_Error()}") - SDL_Quit() - throw Error() - } - displayWidth = displayMode.w - displayHeight = displayMode.h - } - - rect = SDL_calloc(1, SDL_Rect.size)!!.reinterpret() - } - - override fun deinit() { - stop() - - if (rect != null) { - SDL_free(rect) - rect = null - } - - SDL_Quit() - } - - fun start(videoWidth: Int, videoHeight: Int) { - // To free resources from previous playbacks. - stop() - - windowWidth = videoWidth - windowHeight = videoHeight - - rect!!.pointed.let { - it.x = 0 - it.y = 0 - it.w = windowWidth - it.h = windowHeight - } - - val windowX = (displayWidth - windowWidth) / 2 - val windowY = (displayHeight - windowHeight) / 2 - - val window = SDL_CreateWindow("KoPlayer", windowX, windowY, windowWidth, windowHeight, SDL_WINDOW_SHOWN) - if (window == null) { - println("SDL_CreateWindow Error: ${get_SDL_Error()}") - SDL_Quit() - throw Error() - } - this.window = window - - val renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED or SDL_RENDERER_PRESENTVSYNC) - if (renderer == null) { - SDL_DestroyWindow(window) - println("SDL_CreateRenderer Error: ${get_SDL_Error()}") - SDL_Quit() - throw Error() - } - this.renderer = renderer - - this.texture = SDL_CreateTexture(renderer, SDL_GetWindowPixelFormat(window), 0, videoWidth, videoHeight) - } - - fun pixelFormat(): PixelFormat { - if (window == null) - return PixelFormat.INVALID - return when (SDL_GetWindowPixelFormat(window)) { - SDL_PIXELFORMAT_RGB24 -> PixelFormat.RGB24 - SDL_PIXELFORMAT_ARGB8888, SDL_PIXELFORMAT_RGB888 -> PixelFormat.ARGB32 - else -> { - println("Pixel format ${SDL_GetWindowPixelFormat(window)} unknown") - TODO() + init { + disposable( + create = { checkSDLError("Init", SDL_Init(SDL_INIT_EVERYTHING)) }, + dispose = { SDL_Quit() }) + displaySize = tryConstruct { + memScoped { + alloc().run { + checkSDLError("GetCurrentDisplayMode", SDL_GetCurrentDisplayMode(0, ptr.reinterpret())) + Dimensions(w, h) + } } } } - fun nextFrame(frameData: CPointer?, linesize: Int) { + override fun dispose() { + stop() + super.dispose() + } + + fun start(videoSize: Dimensions) { + stop() // To free resources from previous playbacks. + println("SDL Video: Playing output with ${videoSize.w} x ${videoSize.h} pixels") + window = SDLRendererWindow((displaySize - videoSize) / 2, videoSize) + } + + fun pixelFormat(): PixelFormat = window?.pixelFormat() ?: PixelFormat.INVALID + + fun nextFrame(frameData: CPointer, linesize: Int) = + window?.nextFrame(frameData, linesize) + + fun stop() { + window?.let { + it.dispose() + window = null + } + } +} + +class SDLRendererWindow(windowPos: Dimensions, videoSize: Dimensions) : DisposableContainer() { + private val window = sdlDisposable("CreateWindow", + SDL_CreateWindow("KoPlayer", windowPos.w, windowPos.h, videoSize.w, videoSize.h, SDL_WINDOW_SHOWN), + ::SDL_DestroyWindow) + private val renderer = sdlDisposable("CreateRenderer", + SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED or SDL_RENDERER_PRESENTVSYNC), + ::SDL_DestroyRenderer) + private val texture = sdlDisposable("CreateTexture", + SDL_CreateTexture(renderer, SDL_GetWindowPixelFormat(window), 0, videoSize.w, videoSize.h), + ::SDL_DestroyTexture) + private val rect = sdlDisposable("calloc(SDL_Rect)", + SDL_calloc(1, SDL_Rect.size), ::SDL_free) + .reinterpret() + + init { + rect.pointed.apply { + x = 0 + y = 0 + w = videoSize.w + h = videoSize.h + } + } + + fun pixelFormat(): PixelFormat = when (SDL_GetWindowPixelFormat(window)) { + SDL_PIXELFORMAT_RGB24 -> PixelFormat.RGB24 + SDL_PIXELFORMAT_ARGB8888, SDL_PIXELFORMAT_RGB888 -> PixelFormat.ARGB32 + else -> { + println("Pixel format ${SDL_GetWindowPixelFormat(window)} unknown") + PixelFormat.INVALID + } + } + + fun nextFrame(frameData: CPointer, linesize: Int) { SDL_UpdateTexture(texture, rect, frameData, linesize) SDL_RenderClear(renderer) SDL_RenderCopy(renderer, texture, rect, rect) SDL_RenderPresent(renderer) } - - fun stop() { - if (renderer != null) { - SDL_DestroyRenderer(renderer) - renderer = null - } - if (window != null) { - SDL_DestroyWindow(window) - window = null - } - } } diff --git a/samples/videoplayer/src/main/kotlin/VideoPlayer.kt b/samples/videoplayer/src/main/kotlin/VideoPlayer.kt index ff0d56983a8..a47a67b9d6c 100644 --- a/samples/videoplayer/src/main/kotlin/VideoPlayer.kt +++ b/samples/videoplayer/src/main/kotlin/VideoPlayer.kt @@ -22,25 +22,36 @@ import platform.posix.* enum class State { PLAYING, STOPPED, - PAUSED + PAUSED; + + inline fun transition(from: State, to: State, block: () -> Unit): State = + if (this == from) { + block() + to + } else this } -enum class PixelFormat { - INVALID, - RGB24, - ARGB32 +enum class PlayMode { + VIDEO, + AUDIO, + BOTH; + + val useVideo: Boolean get() = this != AUDIO + val useAudio: Boolean get() = this != VIDEO } -class VideoPlayer(val requestedWidth: Int, val requestedHeight: Int) { - val video = SDLVideo(this) - val audio = SDLAudio(this) - val input = SDLInput(this) +class VideoPlayer(val requestedSize: Dimensions?) : DisposableContainer() { + private val video = disposable { SDLVideo() } + private val audio = disposable { SDLAudio(this) } + private val input = disposable { SDLInput(this) } + private val decoder = disposable { DecoderWorker() } + private val now = arena.alloc().ptr - var decoder = DecodeWorker() - var state: State = State.STOPPED - var hasAudio = false - var hasVideo = false + private var state = State.STOPPED + val workerId get() = decoder.workerId + var lastFrameTime = 0.0 + fun stop() { state = State.STOPPED } @@ -59,132 +70,112 @@ class VideoPlayer(val requestedWidth: Int, val requestedHeight: Int) { } } - private var now: CPointer? = null - private fun getTime(): Double { clock_gettime(platform.posix.CLOCK_MONOTONIC, now) - return now!!.pointed.tv_sec + now!!.pointed.tv_nsec / 1_000_000_000.0 + return now.pointed.tv_sec + now.pointed.tv_nsec / 1e9 } - fun init() { - // Prealloc this one, to avoid frequent allocation. - if (now == null) - now = nativeHeap.alloc().ptr - } - - fun deinit() { - if (now != null) { - nativeHeap.free(now!!) - now = null + fun playFile(fileName: String, mode: PlayMode) { + println("playFile $fileName") + val file = AVFile(fileName) + try { + file.dumpFormat() + val info = decoder.initDecode(file.context, mode.useVideo, mode.useAudio) + val videoSize = requestedSize ?: info.video?.size ?: Dimensions(400, 200) + // Use requested video size to start SDLVideo + info.video?.let { video.start(videoSize) } + // Configure decoder output based on actual SDLVideo pixel format + val videoOutput = VideoOutput(videoSize, video.pixelFormat()) + // Use fixed audio output format + val audioOutput = AudioOutput(44100, 2, SampleFormat.S16) + // Start decoder + decoder.start(videoOutput, audioOutput) + // Start SDLAudio player + info.audio?.let { audio.start(audioOutput) } + // Main player loop + lastFrameTime = getTime() + state = State.PLAYING + decoder.requestDecodeChunk() // Fill in frame caches + while (state != State.STOPPED) { + // Fetch video + info.video?.let { playVideoFrame(it) } + // Audio is being auto-fetched by the audio thread + // Check if there are any input + input.check() + // Pause support + checkPause() + // Inter-frame pause, may lead to broken A/V sync, think of better approach + if (state == State.PLAYING) syncAV(info) + if (decoder.done()) stop() + } + } finally { + stop() + audio.stop() + video.stop() + decoder.stop() + file.dispose() } } - fun playFile(file: String) { - println("playFile $file") + private fun playVideoFrame(videoInfo: VideoInfo) { + // Fetch next frame + val frame = decoder.nextVideoFrame() ?: return + // Use video FPS to maintain frame rate + val now = getTime() + val frameDuration = 1.0 / videoInfo.fps + val passedTime = now - lastFrameTime + lastFrameTime += frameDuration // try to maintain perfect frame rate + // Wait for next frame, if needed + if (passedTime < frameDuration) { + usleep((1000_000 * (frameDuration - passedTime)).toInt()) + } else if (passedTime > frameDuration * 1.5){ + lastFrameTime = now // we fell behind more than half frame, reset time + } + // Play frame + video.nextFrame(frame.buffer.pointed.data!!, frame.lineSize) + frame.unref() + } - this.init() - decoder.init() - video.init() - audio.init() - input.init() - - try { - val info = decoder.initDecode(file, true, true) - val windowWidth = if (requestedWidth == 0) { - if (info.width < 0) 400 else info.width - } else requestedWidth - val windowHeight = if (requestedHeight == 0) { - if (info.height < 0) 200 else info.height - } else requestedHeight - hasVideo = info.width > 0 && info.height > 0 - hasAudio = info.sampleRate > 0 && info.channels > 0 - if (hasVideo) - video.start(windowWidth, windowHeight) - decoder.start(windowWidth, windowHeight, video.pixelFormat()) - if (hasAudio) - audio.start(info.sampleRate, info.channels) - var lastTimeStamp = getTime() - state = State.PLAYING - // Fill in frame caches. - decoder.requestDecodeChunk().result() - while (state != State.STOPPED) { - if (hasVideo) { - val frame = decoder.nextVideoFrame() - if (frame == null) { - state = State.STOPPED - continue - } - video.nextFrame(frame.buffer.pointed.data!!, frame.lineSize) - frame.unref() - } - // Audio is being auto-fetched by the audio thread. - - // Check if there are any input. - input.check() - - // Pause support. - while (state == State.PAUSED) { - if (hasAudio) audio.pause() - input.check() - usleep(1 * 1000) - } - if (hasAudio) audio.resume() - - // Interframe pause, may lead to broken A/V sync, think of better approach. - if (state == State.PLAYING) { - // Request decoding. - decoder.requestDecodeChunk() - if (hasVideo) { - if (hasAudio) { - // Use sound for A/V sync. - while (!decoder.audioVideoSynced() && state == State.PLAYING) { - usleep(500) - input.check() - } - } else { - // Use video FPS for frame rate. - val now = getTime() - val delta = now - lastTimeStamp - if (delta < 1.0 / info.fps) { - usleep((1000 * 1000 * (1.0 / info.fps - delta)).toInt()) - } - lastTimeStamp = now - } - } else { - // For pure sound, playback is driven by demand. - usleep(10 * 1000) - } - - if (decoder.done()) { - state = State.STOPPED + private fun checkPause() { + while (state == State.PAUSED) { + audio.pause() + input.check() + usleep(1 * 1000) + } + audio.resume() + } + + private fun syncAV(info: CodecInfo) { + if (info.hasVideo) { + if (info.hasAudio) { + // Use sound for A/V sync. + if (!decoder.audioVideoSynced()) { + println("Resynchronizing video with audio") + while (!decoder.audioVideoSynced() && state == State.PLAYING) { + usleep(500) + input.check() } } } - if (hasAudio) - audio.stop() - if (hasVideo) - video.stop() - decoder.stop() - } finally { - input.deinit() - audio.deinit() - video.deinit() - decoder.deinit() - this.deinit() + } else { + // For pure sound, playback is driven by demand. + usleep(10 * 1000) } } } fun main(args: Array) { if (args.size < 1) { - println("usage: koplayer file.ext ") + println("usage: koplayer file.ext [ | 'video' | 'audio' | 'both']") exitProcess(1) } - av_register_all() - - val width = if (args.size < 3) 0 else args[1].toInt() - val height = if (args.size < 3) 0 else args[2].toInt() - val player = VideoPlayer(width, height) - player.playFile(args[0]) + val mode = if (args.size == 2) PlayMode.valueOf(args[1].toUpperCase()) else PlayMode.BOTH + val requestedSize = if (args.size < 3) null else Dimensions(args[1].toInt(), args[2].toInt()) + val player = VideoPlayer(requestedSize) + try { + player.playFile(args[0], mode) + } finally { + player.dispose() + } }