From bf29dd2a203e47ce47c9f63288c0b7ab9577dafb Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Tue, 2 Jan 2024 17:40:34 +0800 Subject: [PATCH] improve canvas player speed for images --- public/canvasPlayer.js | 9 ++--- public/ffmpeg.js | 74 +++++++++++++++++++++++++----------------- src/CanvasPlayer.js | 21 ++++++++++-- 3 files changed, 68 insertions(+), 36 deletions(-) diff --git a/public/canvasPlayer.js b/public/canvasPlayer.js index 427f59d7..64b7ed63 100644 --- a/public/canvasPlayer.js +++ b/public/canvasPlayer.js @@ -5,7 +5,7 @@ const { getOneRawFrame, encodeLiveRawStream } = require('./ffmpeg'); let aborters = []; -async function command({ path, inWidth, inHeight, streamIndex, seekTo: commandedTime, onFrame, playing }) { +async function command({ path, inWidth, inHeight, streamIndex, seekTo: commandedTime, onRawFrame, onJpegFrame, playing }) { let process; let aborted = false; @@ -40,14 +40,15 @@ async function command({ path, inWidth, inHeight, streamIndex, seekTo: commanded // eslint-disable-next-line no-await-in-loop await tokenizer.readBuffer(rgbaImage, { length: size }); if (aborted) return; - onFrame(rgbaImage, width, height); + // eslint-disable-next-line no-await-in-loop + await onRawFrame(rgbaImage, width, height); } } else { const { process: processIn, width, height } = getOneRawFrame({ path, inWidth, inHeight, streamIndex, seekTo: commandedTime, outSize: 1000 }); process = processIn; - const { stdout: rgbaImage } = await process; + const { stdout: jpegImage } = await process; if (aborted) return; - onFrame(rgbaImage, width, height); + onJpegFrame(jpegImage, width, height); } } catch (err) { if (!err.killed) console.warn(err.message); diff --git a/public/ffmpeg.js b/public/ffmpeg.js index 39d0b198..046b177e 100644 --- a/public/ffmpeg.js +++ b/public/ffmpeg.js @@ -485,25 +485,26 @@ async function html5ify({ outPath, filePath: filePathArg, speed, hasAudio, hasVi console.log(stdout); } -function createRawFfmpeg({ fps = 25, path, inWidth, inHeight, seekTo, oneFrameOnly, execaOpts, streamIndex, outSize = 320 }) { - // const fps = 25; // TODO - +function calcSize({ inWidth, inHeight, outSize }) { const aspectRatio = inWidth / inHeight; - let newWidth; - let newHeight; if (inWidth > inHeight) { - newWidth = outSize; - newHeight = Math.floor(newWidth / aspectRatio); - } else { - newHeight = outSize; - newWidth = Math.floor(newHeight * aspectRatio); + return { + newWidth: outSize, + newHeight: Math.floor(outSize / aspectRatio), + }; } + return { + newHeight: outSize, + newWidth: Math.floor(outSize * aspectRatio), + }; +} - const args = [ - '-hide_banner', '-loglevel', 'panic', +function getOneRawFrame({ path, inWidth, inHeight, seekTo, streamIndex, outSize }) { + const { newWidth, newHeight } = calcSize({ inWidth, inHeight, outSize }); - '-re', + const args = [ + '-hide_banner', '-loglevel', 'error', '-ss', seekTo, @@ -511,12 +512,11 @@ function createRawFfmpeg({ fps = 25, path, inWidth, inHeight, seekTo, oneFrameOn '-i', path, - '-vf', `fps=${fps},scale=${newWidth}:${newHeight}:flags=lanczos`, + '-vf', `scale=${newWidth}:${newHeight}:flags=lanczos`, '-map', `0:${streamIndex}`, - '-vcodec', 'rawvideo', - '-pix_fmt', 'rgba', + '-vcodec', 'mjpeg', - ...(oneFrameOnly ? ['-frames:v', '1'] : []), + '-frames:v', '1', '-f', 'image2pipe', '-', @@ -525,26 +525,42 @@ function createRawFfmpeg({ fps = 25, path, inWidth, inHeight, seekTo, oneFrameOn // console.log(args); return { - process: runFfmpegProcess(args, execaOpts, { logCli: false }), + process: runFfmpegProcess(args, { encoding: 'buffer' }, { logCli: true }), width: newWidth, height: newHeight, - channels: 4, }; } -function getOneRawFrame({ path, inWidth, inHeight, seekTo, streamIndex, outSize }) { - const { process, width, height, channels } = createRawFfmpeg({ path, inWidth, inHeight, seekTo, streamIndex, oneFrameOnly: true, execaOpts: { encoding: null }, outSize }); - return { process, width, height, channels }; -} +function encodeLiveRawStream({ path, inWidth, inHeight, seekTo, streamIndex, fps = 25 }) { + const { newWidth, newHeight } = calcSize({ inWidth, inHeight, outSize: 320 }); + + const args = [ + '-hide_banner', '-loglevel', 'panic', -function encodeLiveRawStream({ path, inWidth, inHeight, seekTo, streamIndex }) { - const { process, width, height, channels } = createRawFfmpeg({ path, inWidth, inHeight, seekTo, streamIndex, execaOpts: { encoding: null, buffer: false } }); + '-re', + + '-ss', seekTo, + + '-noautorotate', + + '-i', path, + + '-vf', `fps=${fps},scale=${newWidth}:${newHeight}:flags=lanczos`, + '-map', `0:${streamIndex}`, + '-vcodec', 'rawvideo', + '-pix_fmt', 'rgba', + + '-f', 'image2pipe', + '-', + ]; + + // console.log(args); return { - process, - width, - height, - channels, + process: runFfmpegProcess(args, { encoding: null, buffer: false }, { logCli: true }), + width: newWidth, + height: newHeight, + channels: 4, }; } diff --git a/src/CanvasPlayer.js b/src/CanvasPlayer.js index 9928626b..ef61083b 100644 --- a/src/CanvasPlayer.js +++ b/src/CanvasPlayer.js @@ -5,7 +5,7 @@ const { command, abortAll } = remote.require('./canvasPlayer'); export default ({ path, width: inWidth, height: inHeight, streamIndex, getCanvas }) => { let terminated; - function drawOnCanvas(rgbaImage, width, height) { + async function drawRawFrame(rgbaImage, width, height) { const canvas = getCanvas(); if (!canvas || rgbaImage.length === 0) return; @@ -18,16 +18,31 @@ export default ({ path, width: inWidth, height: inHeight, streamIndex, getCanvas ctx.putImageData(new ImageData(Uint8ClampedArray.from(rgbaImage), width, height), 0, 0); } + function drawJpegFrame(jpegImage, width, height) { + const canvas = getCanvas(); + if (!canvas) return; + + canvas.width = width; + canvas.height = height; + + const ctx = canvas.getContext('2d'); + + const img = new Image(); + img.onload = () => ctx.drawImage(img, 0, 0, canvas.width, canvas.height); + img.onerror = (error) => console.error('Canvas JPEG image error', error); + img.src = `data:image/jpeg;base64,${jpegImage.toString('base64')}`; + } + function pause(seekTo) { if (terminated) return; abortAll(); - command({ path, inWidth, inHeight, streamIndex, seekTo, onFrame: drawOnCanvas, playing: false }); + command({ path, inWidth, inHeight, streamIndex, seekTo, onJpegFrame: drawJpegFrame, onRawFrame: drawRawFrame, playing: false }); } function play(playFrom) { if (terminated) return; abortAll(); - command({ path, inWidth, inHeight, streamIndex, seekTo: playFrom, onFrame: drawOnCanvas, playing: true }); + command({ path, inWidth, inHeight, streamIndex, seekTo: playFrom, onJpegFrame: drawJpegFrame, onRawFrame: drawRawFrame, playing: true }); } function terminate() {