improve canvas player speed for images

public/canvasPlayer.js

@@ -5,7 +5,7 @@ const { getOneRawFrame, encodeLiveRawStream } = require('./ffmpeg');
 
 let aborters = [];
 
-async function command({ path, inWidth, inHeight, streamIndex, seekTo: commandedTime, onFrame, playing }) {
+async function command({ path, inWidth, inHeight, streamIndex, seekTo: commandedTime, onRawFrame, onJpegFrame, playing }) {
   let process;
   let aborted = false;
 
@@ -40,14 +40,15 @@ async function command({ path, inWidth, inHeight, streamIndex, seekTo: commanded
         // eslint-disable-next-line no-await-in-loop
         await tokenizer.readBuffer(rgbaImage, { length: size });
         if (aborted) return;
-        onFrame(rgbaImage, width, height);
+        // eslint-disable-next-line no-await-in-loop
+        await onRawFrame(rgbaImage, width, height);
       }
     } else {
       const { process: processIn, width, height } = getOneRawFrame({ path, inWidth, inHeight, streamIndex, seekTo: commandedTime, outSize: 1000 });
       process = processIn;
-      const { stdout: rgbaImage } = await process;
+      const { stdout: jpegImage } = await process;
       if (aborted) return;
-      onFrame(rgbaImage, width, height);
+      onJpegFrame(jpegImage, width, height);
     }
   } catch (err) {
     if (!err.killed) console.warn(err.message);

public/ffmpeg.js

@@ -485,25 +485,26 @@ async function html5ify({ outPath, filePath: filePathArg, speed, hasAudio, hasVi
   console.log(stdout);
 }
 
-function createRawFfmpeg({ fps = 25, path, inWidth, inHeight, seekTo, oneFrameOnly, execaOpts, streamIndex, outSize = 320 }) {
-  // const fps = 25; // TODO
-
+function calcSize({ inWidth, inHeight, outSize }) {
   const aspectRatio = inWidth / inHeight;
 
-  let newWidth;
-  let newHeight;
   if (inWidth > inHeight) {
-    newWidth = outSize;
-    newHeight = Math.floor(newWidth / aspectRatio);
-  } else {
-    newHeight = outSize;
-    newWidth = Math.floor(newHeight * aspectRatio);
+    return {
+      newWidth: outSize,
+      newHeight: Math.floor(outSize / aspectRatio),
+    };
   }
+  return {
+    newHeight: outSize,
+    newWidth: Math.floor(outSize * aspectRatio),
+  };
+}
 
-  const args = [
-    '-hide_banner', '-loglevel', 'panic',
+function getOneRawFrame({ path, inWidth, inHeight, seekTo, streamIndex, outSize }) {
+  const { newWidth, newHeight } = calcSize({ inWidth, inHeight, outSize });
 
-    '-re',
+  const args = [
+    '-hide_banner', '-loglevel', 'error',
 
     '-ss', seekTo,
 
@@ -511,12 +512,11 @@ function createRawFfmpeg({ fps = 25, path, inWidth, inHeight, seekTo, oneFrameOn
 
     '-i', path,
 
-    '-vf', `fps=${fps},scale=${newWidth}:${newHeight}:flags=lanczos`,
+    '-vf', `scale=${newWidth}:${newHeight}:flags=lanczos`,
     '-map', `0:${streamIndex}`,
-    '-vcodec', 'rawvideo',
-    '-pix_fmt', 'rgba',
+    '-vcodec', 'mjpeg',
 
-    ...(oneFrameOnly ? ['-frames:v', '1'] : []),
+    '-frames:v', '1',
 
     '-f', 'image2pipe',
     '-',
@@ -525,26 +525,42 @@ function createRawFfmpeg({ fps = 25, path, inWidth, inHeight, seekTo, oneFrameOn
   // console.log(args);
 
   return {
-    process: runFfmpegProcess(args, execaOpts, { logCli: false }),
+    process: runFfmpegProcess(args, { encoding: 'buffer' }, { logCli: true }),
     width: newWidth,
     height: newHeight,
-    channels: 4,
   };
 }
 
-function getOneRawFrame({ path, inWidth, inHeight, seekTo, streamIndex, outSize }) {
-  const { process, width, height, channels } = createRawFfmpeg({ path, inWidth, inHeight, seekTo, streamIndex, oneFrameOnly: true, execaOpts: { encoding: null }, outSize });
-  return { process, width, height, channels };
-}
+function encodeLiveRawStream({ path, inWidth, inHeight, seekTo, streamIndex, fps = 25 }) {
+  const { newWidth, newHeight } = calcSize({ inWidth, inHeight, outSize: 320 });
+
+  const args = [
+    '-hide_banner', '-loglevel', 'panic',
 
-function encodeLiveRawStream({ path, inWidth, inHeight, seekTo, streamIndex }) {
-  const { process, width, height, channels } = createRawFfmpeg({ path, inWidth, inHeight, seekTo, streamIndex, execaOpts: { encoding: null, buffer: false } });
+    '-re',
+
+    '-ss', seekTo,
+
+    '-noautorotate',
+
+    '-i', path,
+
+    '-vf', `fps=${fps},scale=${newWidth}:${newHeight}:flags=lanczos`,
+    '-map', `0:${streamIndex}`,
+    '-vcodec', 'rawvideo',
+    '-pix_fmt', 'rgba',
+
+    '-f', 'image2pipe',
+    '-',
+  ];
+
+  // console.log(args);
 
   return {
-    process,
-    width,
-    height,
-    channels,
+    process: runFfmpegProcess(args, { encoding: null, buffer: false }, { logCli: true }),
+    width: newWidth,
+    height: newHeight,
+    channels: 4,
   };
 }
 

src/CanvasPlayer.js

@@ -5,7 +5,7 @@ const { command, abortAll } = remote.require('./canvasPlayer');
 export default ({ path, width: inWidth, height: inHeight, streamIndex, getCanvas }) => {
   let terminated;
 
-  function drawOnCanvas(rgbaImage, width, height) {
+  async function drawRawFrame(rgbaImage, width, height) {
     const canvas = getCanvas();
     if (!canvas || rgbaImage.length === 0) return;
 
@@ -18,16 +18,31 @@ export default ({ path, width: inWidth, height: inHeight, streamIndex, getCanvas
     ctx.putImageData(new ImageData(Uint8ClampedArray.from(rgbaImage), width, height), 0, 0);
   }
 
+  function drawJpegFrame(jpegImage, width, height) {
+    const canvas = getCanvas();
+    if (!canvas) return;
+
+    canvas.width = width;
+    canvas.height = height;
+
+    const ctx = canvas.getContext('2d');
+
+    const img = new Image();
+    img.onload = () => ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
+    img.onerror = (error) => console.error('Canvas JPEG image error', error);
+    img.src = `data:image/jpeg;base64,${jpegImage.toString('base64')}`;
+  }
+
   function pause(seekTo) {
     if (terminated) return;
     abortAll();
-    command({ path, inWidth, inHeight, streamIndex, seekTo, onFrame: drawOnCanvas, playing: false });
+    command({ path, inWidth, inHeight, streamIndex, seekTo, onJpegFrame: drawJpegFrame, onRawFrame: drawRawFrame, playing: false });
   }
 
   function play(playFrom) {
     if (terminated) return;
     abortAll();
-    command({ path, inWidth, inHeight, streamIndex, seekTo: playFrom, onFrame: drawOnCanvas, playing: true });
+    command({ path, inWidth, inHeight, streamIndex, seekTo: playFrom, onJpegFrame: drawJpegFrame, onRawFrame: drawRawFrame, playing: true });
   }
 
   function terminate() {