From 131c99d5354a2c2a97f26fdf6e67256d0bccc9fa Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Tue, 14 Nov 2017 19:39:52 +0300 Subject: [PATCH] Videoplayer example (#1031) --- .../src/main/kotlin/konan/worker/Worker.kt | 4 + samples/settings.gradle | 1 + samples/videoplayer/README.md | 16 + samples/videoplayer/build.gradle | 45 ++ .../videoplayer/src/main/c_interop/ffmpeg.def | 27 ++ .../videoplayer/src/main/c_interop/sdl.def | 8 + .../src/main/kotlin/DecoderWorker.kt | 394 ++++++++++++++++++ samples/videoplayer/src/main/kotlin/Queue.kt | 47 +++ .../videoplayer/src/main/kotlin/SDLAudio.kt | 100 +++++ .../videoplayer/src/main/kotlin/SDLBase.kt | 25 ++ .../videoplayer/src/main/kotlin/SDLInput.kt | 48 +++ .../videoplayer/src/main/kotlin/SDLVideo.kt | 130 ++++++ .../src/main/kotlin/VideoPlayer.kt | 188 +++++++++ 13 files changed, 1033 insertions(+) create mode 100644 samples/videoplayer/README.md create mode 100644 samples/videoplayer/build.gradle create mode 100644 samples/videoplayer/src/main/c_interop/ffmpeg.def create mode 100644 samples/videoplayer/src/main/c_interop/sdl.def create mode 100644 samples/videoplayer/src/main/kotlin/DecoderWorker.kt create mode 100644 samples/videoplayer/src/main/kotlin/Queue.kt create mode 100644 samples/videoplayer/src/main/kotlin/SDLAudio.kt create mode 100644 samples/videoplayer/src/main/kotlin/SDLBase.kt create mode 100644 samples/videoplayer/src/main/kotlin/SDLInput.kt create mode 100644 samples/videoplayer/src/main/kotlin/SDLVideo.kt create mode 100644 samples/videoplayer/src/main/kotlin/VideoPlayer.kt diff --git a/runtime/src/main/kotlin/konan/worker/Worker.kt b/runtime/src/main/kotlin/konan/worker/Worker.kt index 57954011f5c..62f81ad6b3c 100644 --- a/runtime/src/main/kotlin/konan/worker/Worker.kt +++ b/runtime/src/main/kotlin/konan/worker/Worker.kt @@ -101,6 +101,10 @@ class Future internal constructor(val id: FutureId) { throw IllegalStateException("Future is cancelled") } + fun result(): T { + return consume { it -> it } + } + val state: FutureState get() = FutureState.values()[stateOfFuture(id)] diff --git a/samples/settings.gradle b/samples/settings.gradle index 5c21f0f663e..718b34daa59 100644 --- a/samples/settings.gradle +++ b/samples/settings.gradle @@ -14,4 +14,5 @@ include ':tensorflow' include ':objc' include ':uikit' include ':win32' +include ':videoplayer' includeBuild '../' diff --git a/samples/videoplayer/README.md b/samples/videoplayer/README.md new file mode 100644 index 00000000000..c73166aacb5 --- /dev/null +++ b/samples/videoplayer/README.md @@ -0,0 +1,16 @@ +# Simple video player + + This example shows how one could implement a video player in Kotlin. +Almost any video file supported by ffmpeg could be played with it. +ffmpeg and SDL2 is needed for that to work, i.e. + + port install ffmpeg-devel + apt install libavcodec-dev libavformat-dev libavutil-dev libswscale-dev \ + libavresample-dev + apt install libsdl2-dev + +To build use `../gradlew build`. + +To run use `./build/konan/bin//Player.kexe file.mp4`. + + diff --git a/samples/videoplayer/build.gradle b/samples/videoplayer/build.gradle new file mode 100644 index 00000000000..169e452babf --- /dev/null +++ b/samples/videoplayer/build.gradle @@ -0,0 +1,45 @@ +apply plugin: 'konan' + +konan.targets = ['macbook', 'linux'] + +konanArtifacts { + interop('ffmpeg') { + defFile 'src/main/c_interop/ffmpeg.def' + + target 'linux', { + includeDirs '/usr/include', '/usr/include/x86_64-linux-gnu' + } + target 'macbook', { + includeDirs '/opt/local/include', '/usr/local/include' + } + } + + interop ('sdl') { + defFile 'src/main/c_interop/sdl.def' + + target 'macbook', { + includeDirs '/Library/Frameworks/SDL2.framework/Headers', + "${System.getProperty("user.home")}/Library/Frameworks/SDL2.framework/Headers", + '/opt/local/include/SDL2', + '/usr/local/include/SDL2' + } + + target 'linux', { + includeDirs '/usr/include/SDL2' + } + } + + program('Player') { + libraries { + artifact 'ffmpeg' + artifact 'sdl' + } + target 'macbook', { + linkerOpts "-F ${System.getProperty("user.home")}/Library/Frameworks -F /Library/Frameworks -L/opt/local/lib" + } + + target 'linux', { + linkerOpts '-L/usr/lib/x86_64-linux-gnu' + } + } +} diff --git a/samples/videoplayer/src/main/c_interop/ffmpeg.def b/samples/videoplayer/src/main/c_interop/ffmpeg.def new file mode 100644 index 00000000000..28321009864 --- /dev/null +++ b/samples/videoplayer/src/main/c_interop/ffmpeg.def @@ -0,0 +1,27 @@ +package = ffmpeg +headers = libavcodec/avcodec.h libavformat/avformat.h libavutil/pixfmt.h libavutil/opt.h \ + libswscale/swscale.h libavresample/avresample.h +headerFilter = libavcodec/** libavformat/** libavutil/** \ + libswscale/** libavresample/** +linkerOpts = -lavutil -lavformat -lavcodec -lswscale -lavresample +--- + +static void av_buffer_unref2(AVBufferRef* ref) { + AVBufferRef* copy = ref; + av_buffer_unref(©); +} + +static void avcodec_free_context2(AVCodecContext* ref) { + AVCodecContext* copy = ref; + avcodec_free_context(©); +} + +static void avformat_free_context2(AVFormatContext* ref) { + AVFormatContext* copy = ref; + avformat_free_context(©); +} + +static void avresample_free2(AVAudioResampleContext* ref) { + AVAudioResampleContext* copy = ref; + avresample_free(©); +} diff --git a/samples/videoplayer/src/main/c_interop/sdl.def b/samples/videoplayer/src/main/c_interop/sdl.def new file mode 100644 index 00000000000..b11c16c89be --- /dev/null +++ b/samples/videoplayer/src/main/c_interop/sdl.def @@ -0,0 +1,8 @@ +package = sdl +headers = SDL.h +entryPoint = SDL_main + +headerFilter = SDL* + +linkerOpts.osx = -lSDL2 +linkerOpts.linux = -lSDL2 \ No newline at end of file diff --git a/samples/videoplayer/src/main/kotlin/DecoderWorker.kt b/samples/videoplayer/src/main/kotlin/DecoderWorker.kt new file mode 100644 index 00000000000..6a704db6de8 --- /dev/null +++ b/samples/videoplayer/src/main/kotlin/DecoderWorker.kt @@ -0,0 +1,394 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ffmpeg.* +import kotlin.system.* +import kotlinx.cinterop.* +import konan.worker.* +import platform.posix.memcpy + +// This global variable only set to != null value in the decoding worker. +private var state: DecodeWorkerState? = null + +data class OutputInfo(val width: Int, val height: Int, val pixelFormat: AVPixelFormat) +data class VideoInfo(val width: Int, val height: Int, val fps: Double, + val sampleRate: Int, val channels: Int) + +private data class AskNextAudioFrame(val size: Int) +data class VideoFrame(val buffer: CPointer, val lineSize: Int) { + fun unref() { + av_buffer_unref2(buffer) + } +} + +data class AudioFrame(val buffer: CPointer, var position: Int, val size: Int) { + fun unref() { + av_buffer_unref2(buffer) + } +} + +private fun Int.checkError() { + if (this != 0) { + val buffer = ByteArray(1024) + av_strerror(this, buffer.refTo(0), buffer.size.signExtend()) + throw Error("AVError: ${kotlin.text.fromUtf8Array(buffer, 0, buffer.size)}") + } +} + +class DecodeWorkerState(val formatContext: CPointer, + val videoStreamIndex: Int, + val audioStreamIndex: Int, + val videoCodecContext: CPointer?, + val audioCodecContext: CPointer?) { + var videoFrame: CPointer? = null + var scaledVideoFrame: CPointer? = null + var audioFrame: CPointer? = null + var resampledAudioFrame: CPointer? = null + var softwareScalingContext: CPointer? = null + var resampleContext: CPointer? = null + val videoQueue = Queue(100, null) + val audioQueue = Queue(100, null) + var buffer: ByteArray? = null + var videoWidth = 0 + var videoHeight = 0 + var windowWidth = 0 + var windowHeight = 0 + var scaledFrameSize = 0 + var noMoreFrames = false + val minAudioFrames = 1 + val maxAudioFrames = 3 + val minVideoFrames = 5 + + fun makeVideoFrame(): VideoFrame { + // TODO: reuse buffers! + val buffer = av_buffer_alloc(scaledFrameSize)!! + memcpy(buffer.pointed.data, scaledVideoFrame!!.pointed.data[0], scaledFrameSize.signExtend()) + return VideoFrame(buffer, scaledVideoFrame!!.pointed.linesize[0]) + } + + fun makeAudioFrame(): AudioFrame { + avresample_convert_frame(resampleContext, resampledAudioFrame, audioFrame).checkError() + val audioFrameSize = av_samples_get_buffer_size( + null, + resampledAudioFrame!!.pointed.channels, + resampledAudioFrame!!.pointed.nb_samples, + resampledAudioFrame!!.pointed.format, + 1) + val buffer = av_buffer_alloc(audioFrameSize)!! + memcpy(buffer.pointed.data, resampledAudioFrame!!.pointed.data[0], audioFrameSize.signExtend()) + return AudioFrame(buffer, 0, audioFrameSize) + } + + private fun setResampleOpt(name: String, value: Int) = + av_opt_set_int(resampleContext, name, value.signExtend(), 0) + + + fun start(output: OutputInfo) { + if (videoCodecContext != null) { + videoWidth = videoCodecContext.pointed.width + videoHeight = videoCodecContext.pointed.height + windowWidth = output.width + windowHeight = output.height + videoFrame = av_frame_alloc()!! + scaledVideoFrame = av_frame_alloc()!! + softwareScalingContext = sws_getContext( + videoWidth, + videoHeight, + videoCodecContext.pointed.pix_fmt, + windowWidth, windowHeight, output.pixelFormat, + SWS_BILINEAR, null, null, null)!! + + scaledFrameSize = avpicture_get_size(output.pixelFormat, output.width, output.height) + buffer = ByteArray(scaledFrameSize) + + avpicture_fill(scaledVideoFrame!!.reinterpret(), buffer!!.refTo(0), + output.pixelFormat, output.width, output.height) + } + + if (audioCodecContext != null) { + audioFrame = av_frame_alloc()!! + resampledAudioFrame = av_frame_alloc()!! + resampledAudioFrame!!.pointed.format = AV_SAMPLE_FMT_S16 + resampledAudioFrame!!.pointed.channels = 2 + resampledAudioFrame!!.pointed.channel_layout = AV_CH_LAYOUT_STEREO.signExtend() + resampledAudioFrame!!.pointed.sample_rate = 44100 + resampleContext = avresample_alloc_context() + setResampleOpt("in_channel_layout", audioCodecContext.pointed.channel_layout.narrow()) + setResampleOpt("out_channel_layout", AV_CH_LAYOUT_STEREO) + setResampleOpt("in_sample_rate", audioCodecContext.pointed.sample_rate) + setResampleOpt("out_sample_rate", 44100) + setResampleOpt("in_sample_fmt", audioCodecContext.pointed.sample_fmt) + setResampleOpt("out_sample_fmt", AV_SAMPLE_FMT_S16) + avresample_open(resampleContext) + } + + noMoreFrames = false + + decodeIfNeeded() + } + + fun done() = noMoreFrames && videoQueue.isEmpty() && audioQueue.isEmpty() + + fun stop() { + while (!videoQueue.isEmpty()) { + videoQueue.pop()?.unref() + } + while (!audioQueue.isEmpty()) { + audioQueue.pop()?.unref() + } + if (videoFrame != null) { + av_frame_unref(videoFrame) + videoFrame = null + } + if (scaledVideoFrame != null) { + av_frame_unref(scaledVideoFrame) + scaledVideoFrame = null + } + if (audioFrame != null) { + av_frame_unref(audioFrame) + audioFrame = null + } + if (resampledAudioFrame != null) { + av_frame_unref(resampledAudioFrame) + resampledAudioFrame = null + } + if (softwareScalingContext != null) { + sws_freeContext(softwareScalingContext) + softwareScalingContext = null + } + if (resampleContext != null) { + avresample_free2(resampleContext) + resampleContext = null + } + if (videoCodecContext != null) { + avcodec_free_context2(videoCodecContext) + } + if (audioCodecContext != null) { + avcodec_free_context2(audioCodecContext) + } + //avformat_free_context2(formatContext) + } + + fun needMoreBuffers(): Boolean { + return ((videoStreamIndex != -1) && (videoQueue.size() < minVideoFrames)) || + ((audioStreamIndex != -1) && (audioQueue.size() < minAudioFrames)) + } + + fun decodeIfNeeded() { + if (!needMoreBuffers()) return + + memScoped { + val packet = alloc() + val frameFinished = alloc() + while (needMoreBuffers() && av_read_frame(formatContext, packet.ptr) >= 0) { + when (packet.stream_index) { + videoStreamIndex -> { + // Decode video frame. + avcodec_decode_video2(videoCodecContext, videoFrame, frameFinished.ptr, packet.ptr) + // Did we get a video frame? + if (frameFinished.value != 0) { + // Convert the frame from its movie format to window pixel format. + sws_scale(softwareScalingContext, videoFrame!!.pointed.data, + videoFrame!!.pointed.linesize, 0, videoHeight, + scaledVideoFrame!!.pointed.data, scaledVideoFrame!!.pointed.linesize) + videoQueue.push(makeVideoFrame()) + } + } + audioStreamIndex -> { + while (packet.size > 0) { + val size = avcodec_decode_audio4( + audioCodecContext, audioFrame, frameFinished.ptr, packet.ptr) + + if (frameFinished.value != 0) { + // Put audio frame to decoder's queue. + audioQueue.push(makeAudioFrame()) + } + packet.size -= size + packet.data += size + } + } + } + av_packet_unref(packet.ptr) + } + if (needMoreBuffers()) noMoreFrames = true + } + } + + fun nextVideoFrame(): VideoFrame? { + decodeIfNeeded() + + if (videoQueue.isEmpty()) { + return null + } + return videoQueue.pop() + } + + fun nextAudioFrame(size: Int): AudioFrame? { + decodeIfNeeded() + + if (audioQueue.isEmpty()) { + return null + } + val frame = audioQueue.peek()!! + var realSize = if (frame.position + size > frame.size) frame.size - frame.position else size + if (frame.position + realSize == frame.size) { + return audioQueue.pop() + } else { + val result = AudioFrame(av_buffer_ref(frame.buffer)!!, frame.position, frame.size) + frame.position += realSize + return result + } + } + + fun audioVideoSynced() = (audioQueue.size() < maxAudioFrames) || done() +} + +class DecodeWorker { + // This class must have no other state, but this worker object. + // All the real state must be stored on the worker's side. + private val decodeWorker: Worker + + constructor() { + decodeWorker = konan.worker.startWorker() + } + + constructor(id: WorkerId) { + decodeWorker = Worker(id) + } + + fun workerId() = decodeWorker.id + + fun init() {} + + fun renderPixelFormat(pixelFormat: PixelFormat) = when (pixelFormat) { + PixelFormat.RGB24 -> AV_PIX_FMT_RGB24 + PixelFormat.ARGB32 -> AV_PIX_FMT_RGB32 + PixelFormat.INVALID -> AV_PIX_FMT_NONE + } + + private fun findStream(useStream: Boolean, formatContext: CPointer, streamIndex: Int, tag: String): + Pair?, CPointer?> { + if (streamIndex < 0 || !useStream) return null to null + val stream = formatContext.pointed.streams!!.get(streamIndex)!! + // Get codec context for the video stream. + val codecContext = stream.pointed.codec!! + val codec = avcodec_find_decoder(codecContext.pointed.codec_id) + if (codec == null) + throw Error("Unsupported $tag codec...") + // Open codec. + if (avcodec_open2(codecContext, codec, null) < 0) + throw Error("Couldn't open $tag codec") + return stream to codecContext + } + + fun initDecode(file: String, useVideo: Boolean = true, useAudio: Boolean = true): VideoInfo { + memScoped { + try { + val formatContextPtr = alloc>() + if (avformat_open_input(formatContextPtr.ptr, file, null, null) != 0) + throw Error("Cannot open video file") + val formatContext = formatContextPtr.value!! + if (avformat_find_stream_info(formatContext, null) < 0) + throw Error("Couldn't find stream information") + av_dump_format(formatContext, 0, file, 0) + + // Find the first video/audio streams. + var videoStreamIndex = -1 + var audioStreamIndex = -1 + for (i in 0 until formatContext.pointed.nb_streams) { + val stream = formatContext.pointed.streams!!.get(i) + val codec = stream!!.pointed.codec!!.pointed + if (codec.codec_type == AVMEDIA_TYPE_VIDEO && videoStreamIndex == -1) { + videoStreamIndex = i + } + if (codec.codec_type == AVMEDIA_TYPE_AUDIO && audioStreamIndex == -1) { + audioStreamIndex = i + } + } + + val (videoStream, videoCodecContext) = findStream(useVideo, formatContext, videoStreamIndex, "video") + val (_, audioCodecContext) = findStream(useAudio, formatContext, audioStreamIndex, "audio") + + // Extract video info. + val (videoWidth, videoHeight, fps) = if (videoCodecContext != null) { + Triple(videoCodecContext.pointed.width, videoCodecContext.pointed.height, + av_q2d(av_stream_get_r_frame_rate(videoStream))) + } else { + Triple(-1, -1, 0.0) + } + val (sampleRate, channels) = if (audioCodecContext != null) { + Pair(audioCodecContext.pointed.sample_rate, audioCodecContext.pointed.channels) + } else { + Pair(0, 0) + } + + // Pack all inited state and pass it to the worker. + decodeWorker.schedule(TransferMode.CHECKED, { + DecodeWorkerState(formatContext, + videoStreamIndex, audioStreamIndex, + videoCodecContext, audioCodecContext) + }) { input -> + state = input + null + } + return VideoInfo(videoWidth, videoHeight, fps, sampleRate, channels) + } finally { + // TODO: clean up whatever we allocated. + } + } + } + + fun start(width: Int, height: Int, pixelFormat: PixelFormat) { + decodeWorker.schedule(TransferMode.CHECKED, { + OutputInfo(width, height, renderPixelFormat(pixelFormat)) + }) { input -> state!!.start(input) } + } + + fun stop() { + decodeWorker.schedule( + TransferMode.CHECKED, + { null }) { _ -> + state!!.stop() + state = null + }.result() + } + + // TODO: we manually box returned primitive value, + // fix by autoboxing schedule()'s result in the compiler. + fun done() = decodeWorker.schedule(TransferMode.CHECKED, + { null }) { _ -> (state == null || state!!.done()) as Boolean? + }.consume { it -> it!! } + + fun deinit() { + decodeWorker.requestTermination().result() + } + + fun requestDecodeChunk() = decodeWorker.schedule( + TransferMode.CHECKED, + { null }) { _ -> state!!.decodeIfNeeded() } + + fun nextVideoFrame(): VideoFrame? = decodeWorker.schedule( + TransferMode.CHECKED, + { null }) { _ -> state!!.nextVideoFrame() }.result() + + fun nextAudioFrame(size: Int) = decodeWorker.schedule( + TransferMode.CHECKED, + { size as Int? }) { input -> state!!.nextAudioFrame(input!!) }.result() + + fun audioVideoSynced() = decodeWorker.schedule( + TransferMode.CHECKED, + { null }) { _ -> state!!.audioVideoSynced() as Boolean? }.consume { it -> it!! } +} diff --git a/samples/videoplayer/src/main/kotlin/Queue.kt b/samples/videoplayer/src/main/kotlin/Queue.kt new file mode 100644 index 00000000000..86a944b4187 --- /dev/null +++ b/samples/videoplayer/src/main/kotlin/Queue.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +class Queue(val maxSize: Int, val none: T) { + val array = Array(maxSize, { _ -> none}) + var head = 0 + var tail = 0 + + fun push(element: T) { + if ((tail + 1) % maxSize == head) + throw Error("queue overflow: $tail $head") + array[tail] = element + tail = (tail + 1) % maxSize + } + + fun pop() : T { + if (tail == head) + throw Error("queue underflow") + val result = array[head] + array[head] = none + head = (head + 1) % maxSize + return result + } + + fun peek() : T { + if (tail == head) + throw Error("queue underflow") + return array[head] + } + + fun size() = if (tail >= head) tail - head else maxSize - (head - tail) + + fun isEmpty() = head == tail +} diff --git a/samples/videoplayer/src/main/kotlin/SDLAudio.kt b/samples/videoplayer/src/main/kotlin/SDLAudio.kt new file mode 100644 index 00000000000..a116ee4f41f --- /dev/null +++ b/samples/videoplayer/src/main/kotlin/SDLAudio.kt @@ -0,0 +1,100 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import kotlinx.cinterop.* +import sdl.* +import konan.worker.WorkerId +import platform.posix.memset +import platform.posix.memcpy + +class SDLAudio(val player: VideoPlayer) : SDLBase() { + + private var threadData: CPointer? = null + + override fun init() { + threadData = nativeHeap.alloc().ptr + } + + override fun deinit() { + if (threadData != null) { + nativeHeap.free(threadData!!) + threadData = null + } + } + + fun start(sampleRate: Int, channels: Int) { + println("Audio: $channels channels, $sampleRate samples per second") + + memScoped { + // TODO: better mechanisms to ensure we have same output format here and in resampler of the decoder. + val spec = alloc() + spec.freq = 44100 + spec.format = AUDIO_S16SYS.narrow() + spec.channels = 2.narrow() + spec.silence = 0 + spec.samples = 4096 + spec.callback = staticCFunction { + userdata, buffer, length -> + // This handler will be invoked in the audio thread, so reinit runtime. + konan.initRuntimeIfNeeded() + + if (decoder == null) { + val callbackData = userdata!!.reinterpret() + decoder = DecodeWorker(callbackData.pointed.value) + } + var outPosition = 0 + while (outPosition < length) { + val frame = decoder!!.nextAudioFrame(length - outPosition) + if (frame != null) { + val toCopy = min(length - outPosition, frame.size - frame.position) + memcpy(buffer + outPosition, frame.buffer.pointed.data + frame.position, toCopy.signExtend()) + frame.unref() + outPosition += toCopy + } else { + println("Decoder returned nothing!") + memset(buffer + outPosition, 0, (length - outPosition).signExtend()) + break + } + } + } + threadData!!.pointed.value = player.decoder.workerId() + spec.userdata = threadData + val realSpec = alloc() + if (SDL_OpenAudio(spec.ptr, realSpec.ptr) < 0) + throw Error("SDL_OpenAudio: ${get_SDL_Error()}") + // TODO: ensure real spec matches what we asked for. + + resume() + } + } + + fun pause() { + SDL_PauseAudio(1) + } + + fun resume() { + SDL_PauseAudio(0) + } + + fun stop() { + pause() + SDL_CloseAudio() + + } +} + +// This global is only set in the audio thread. +var decoder: DecodeWorker? = null diff --git a/samples/videoplayer/src/main/kotlin/SDLBase.kt b/samples/videoplayer/src/main/kotlin/SDLBase.kt new file mode 100644 index 00000000000..b18ba6f013d --- /dev/null +++ b/samples/videoplayer/src/main/kotlin/SDLBase.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import sdl.SDL_GetError +import kotlinx.cinterop.* + +open class SDLBase { + protected fun get_SDL_Error() = SDL_GetError()!!.toKString() + + open fun init() {} + open fun deinit() {} +} \ No newline at end of file diff --git a/samples/videoplayer/src/main/kotlin/SDLInput.kt b/samples/videoplayer/src/main/kotlin/SDLInput.kt new file mode 100644 index 00000000000..a5833590076 --- /dev/null +++ b/samples/videoplayer/src/main/kotlin/SDLInput.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import kotlinx.cinterop.* +import sdl.* + +class SDLInput(val player: VideoPlayer) : SDLBase() { + var event: CPointer? = null + + override fun init() { + event = nativeHeap.alloc().ptr + } + + override fun deinit() { + if (event != null) { + nativeHeap.free(event!!) + event = null + } + } + + fun check() { + while (SDL_PollEvent(event!!.reinterpret()) != 0) { + when (event!!.pointed.type) { + SDL_QUIT -> player.stop() + SDL_KEYDOWN -> { + val keyboardEvent = event!!.reinterpret().pointed + when (keyboardEvent.keysym.scancode) { + SDL_SCANCODE_ESCAPE -> player.stop() + SDL_SCANCODE_SPACE -> player.pause() + } + } + } + } + } +} \ No newline at end of file diff --git a/samples/videoplayer/src/main/kotlin/SDLVideo.kt b/samples/videoplayer/src/main/kotlin/SDLVideo.kt new file mode 100644 index 00000000000..9c657bcf681 --- /dev/null +++ b/samples/videoplayer/src/main/kotlin/SDLVideo.kt @@ -0,0 +1,130 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import kotlinx.cinterop.* +import sdl.* + +class SDLVideo(val player: VideoPlayer) : SDLBase() { + var displayWidth = 0 + var displayHeight = 0 + var windowWidth = 0 + var windowHeight = 0 + + private var window: CPointer? = null + private var renderer: CPointer? = null + private var surface: CPointer? = null + private var texture: CPointer? = null + private var rect: CPointer? = null + + override fun init() { + if (SDL_Init(SDL_INIT_EVERYTHING) != 0) { + println("SDL_Init Error: ${get_SDL_Error()}") + throw Error() + } + + memScoped { + val displayMode = alloc() + if (SDL_GetCurrentDisplayMode(0, displayMode.ptr.reinterpret()) != 0) { + println("SDL_GetCurrentDisplayMode Error: ${get_SDL_Error()}") + SDL_Quit() + throw Error() + } + displayWidth = displayMode.w + displayHeight = displayMode.h + } + + rect = SDL_calloc(1, SDL_Rect.size)!!.reinterpret() + } + + override fun deinit() { + stop() + + if (rect != null) { + SDL_free(rect) + rect = null + } + + SDL_Quit() + } + + fun start(videoWidth: Int, videoHeight: Int) { + // To free resources from previous playbacks. + stop() + + windowWidth = videoWidth + windowHeight = videoHeight + + rect!!.pointed.let { + it.x = 0 + it.y = 0 + it.w = windowWidth + it.h = windowHeight + } + + val windowX = (displayWidth - windowWidth) / 2 + val windowY = (displayHeight - windowHeight) / 2 + + val window = SDL_CreateWindow("KoPlayer", windowX, windowY, windowWidth, windowHeight, SDL_WINDOW_SHOWN) + if (window == null) { + println("SDL_CreateWindow Error: ${get_SDL_Error()}") + SDL_Quit() + throw Error() + } + this.window = window + + val renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED or SDL_RENDERER_PRESENTVSYNC) + if (renderer == null) { + SDL_DestroyWindow(window) + println("SDL_CreateRenderer Error: ${get_SDL_Error()}") + SDL_Quit() + throw Error() + } + this.renderer = renderer + + this.texture = SDL_CreateTexture(renderer, SDL_GetWindowPixelFormat(window), 0, videoWidth, videoHeight) + } + + fun pixelFormat(): PixelFormat { + if (window == null) + return PixelFormat.INVALID + return when (SDL_GetWindowPixelFormat(window)) { + SDL_PIXELFORMAT_RGB24 -> PixelFormat.RGB24 + SDL_PIXELFORMAT_ARGB8888, SDL_PIXELFORMAT_RGB888 -> PixelFormat.ARGB32 + else -> { + println("Pixel format ${SDL_GetWindowPixelFormat(window)} unknown") + TODO() + } + } + } + + fun nextFrame(frameData: CPointer?, linesize: Int) { + SDL_UpdateTexture(texture, rect, frameData, linesize) + SDL_RenderClear(renderer) + SDL_RenderCopy(renderer, texture, rect, rect) + SDL_RenderPresent(renderer) + } + + fun stop() { + if (renderer != null) { + SDL_DestroyRenderer(renderer) + renderer = null + } + if (window != null) { + SDL_DestroyWindow(window) + window = null + } + } +} diff --git a/samples/videoplayer/src/main/kotlin/VideoPlayer.kt b/samples/videoplayer/src/main/kotlin/VideoPlayer.kt new file mode 100644 index 00000000000..3b5c7930d3f --- /dev/null +++ b/samples/videoplayer/src/main/kotlin/VideoPlayer.kt @@ -0,0 +1,188 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ffmpeg.* +import kotlin.system.* +import kotlinx.cinterop.* +import platform.posix.* + +enum class State { + PLAYING, + STOPPED, + PAUSED +} + +enum class PixelFormat { + INVALID, + RGB24, + ARGB32 +} + +class VideoPlayer(val requestedWidth: Int, val requestedHeight: Int) { + val video = SDLVideo(this) + val audio = SDLAudio(this) + val input = SDLInput(this) + + var decoder = DecodeWorker() + var state: State = State.STOPPED + var hasAudio = false + var hasVideo = false + + fun stop() { + state = State.STOPPED + } + + fun pause() { + when (state) { + State.PAUSED -> { + state = State.PLAYING + audio.resume() + } + State.PLAYING -> { + state = State.PAUSED + audio.pause() + } + State.STOPPED -> throw Error("Cannot pause in stopped state") + } + } + + private var now: CPointer? = null + + private fun getTime(): Double { + clock_gettime(platform.posix.CLOCK_MONOTONIC, now) + return now!!.pointed.tv_sec + now!!.pointed.tv_nsec / 1_000_000_000.0 + } + + fun init() { + // Prealloc this one, to avoid frequent allocation. + if (now == null) + now = nativeHeap.alloc().ptr + } + + fun deinit() { + if (now != null) { + nativeHeap.free(now!!) + now = null + } + } + + fun playFile(file: String) { + println("playFile $file") + + this.init() + decoder.init() + video.init() + audio.init() + input.init() + + try { + val info = decoder.initDecode(file, true, true) + val windowWidth = if (requestedWidth == 0) { + if (info.width < 0) 400 else info.width + } else requestedWidth + val windowHeight = if (requestedHeight == 0) { + if (info.height < 0) 200 else info.height + } else requestedHeight + hasVideo = info.width > 0 && info.height > 0 + hasAudio = info.sampleRate > 0 && info.channels > 0 + if (hasVideo) + video.start(windowWidth, windowHeight) + decoder.start(windowWidth, windowHeight, video.pixelFormat()) + if (hasAudio) + audio.start(info.sampleRate, info.channels) + var lastTimeStamp = getTime() + state = State.PLAYING + // Fill in frame caches. + decoder.requestDecodeChunk().result() + while (state != State.STOPPED) { + if (hasVideo) { + val frame = decoder.nextVideoFrame() + if (frame == null) { + state = State.STOPPED + continue + } + video.nextFrame(frame.buffer.pointed.data!!, frame.lineSize) + frame.unref() + } + // Audio is being auto-fetched by the audio thread. + + // Check if there are any input. + input.check() + + // Pause support. + while (state == State.PAUSED) { + if (hasAudio) audio.pause() + input.check() + usleep(1 * 1000) + } + if (hasAudio) audio.resume() + + // Interframe pause, may lead to broken A/V sync, think of better approach. + if (state == State.PLAYING) { + if (hasVideo) { + if (hasAudio) { + // Use sound for A/V sync. + while (!decoder.audioVideoSynced() && state == State.PLAYING) { + usleep(500) + input.check() + } + } else { + // Use video FPS for frame rate. + val now = getTime() + val delta = now - lastTimeStamp + if (delta < 1.0 / info.fps) { + usleep((1000 * 1000 * (1.0 / info.fps - delta)).toInt()) + } + lastTimeStamp = now + } + } else { + // For pure sound, playback is driven by demand. + usleep(10 * 1000) + } + + if (decoder.done()) { + state = State.STOPPED + } + } + } + if (hasAudio) + audio.stop() + if (hasVideo) + video.stop() + decoder.stop() + } finally { + input.deinit() + audio.deinit() + video.deinit() + decoder.deinit() + this.deinit() + } + } +} + +fun main(args: Array) { + if (args.size < 1) { + println("usage: koplayer file.ext ") + exitProcess(1) + } + + av_register_all() + + val width = if (args.size < 3) 0 else args[1].toInt() + val height = if (args.size < 3) 0 else args[2].toInt() + val player = VideoPlayer(width, height) + player.playFile(args[0]) +}