implement detect silence

closes #1334

also allow chnaging parameters of black detect #623

README.md

@@ -52,7 +52,7 @@ The main feature is lossless trimming and cutting of video and audio files, whic
 - MKV/MP4 embedded chapters marks editor
 - View subtitles
 - Customizable keyboard hotkeys
-- Black scene detection
+- Black scene detection and silent audio detection
 - Divide timeline into segments with length L or into N segments or even randomized segments!
 - [Basic CLI support](cli.md)
 

public/menu.js

@@ -299,6 +299,12 @@ module.exports = (app, mainWindow, newVersion) => {
             mainWindow.webContents.send('detectBlackScenes');
           },
         },
+        {
+          label: i18n.t('Detect silent scenes'),
+          click() {
+            mainWindow.webContents.send('detectSilentScenes');
+          },
+        },
         {
           label: i18n.t('Last ffmpeg commands'),
           click() { mainWindow.webContents.send('toggleLastCommands'); },

src/App.jsx

@@ -53,7 +53,7 @@ import {
   getStreamFps, isCuttingStart, isCuttingEnd,
   readFileMeta, getSmarterOutFormat, renderThumbnails as ffmpegRenderThumbnails,
   extractStreams, runStartupCheck, setCustomFfPath as ffmpegSetCustomFfPath,
-  isIphoneHevc, tryMapChaptersToEdl, blackDetect,
+  isIphoneHevc, tryMapChaptersToEdl, blackDetect, silenceDetect,
   getDuration, getTimecodeFromStreams, createChaptersFromSegments, extractSubtitleTrack,
   getFfmpegPath, RefuseOverwriteError,
 } from './ffmpeg';
@@ -69,7 +69,7 @@ import {
 } from './util';
 import { formatDuration } from './util/duration';
 import { adjustRate } from './util/rate-calculator';
-import { askForOutDir, askForInputDir, askForImportChapters, createNumSegments as createNumSegmentsDialog, createFixedDurationSegments as createFixedDurationSegmentsDialog, createRandomSegments as createRandomSegmentsDialog, promptTimeOffset, askForHtml5ifySpeed, askForFileOpenAction, confirmExtractAllStreamsDialog, showCleanupFilesDialog, showDiskFull, showCutFailedDialog, labelSegmentDialog, openYouTubeChaptersDialog, openAbout, showEditableJsonDialog, askForShiftSegments, selectSegmentsByLabelDialog, confirmExtractFramesAsImages, showRefuseToOverwrite } from './dialogs';
+import { askForOutDir, askForInputDir, askForImportChapters, createNumSegments as createNumSegmentsDialog, createFixedDurationSegments as createFixedDurationSegmentsDialog, createRandomSegments as createRandomSegmentsDialog, promptTimeOffset, askForHtml5ifySpeed, askForFileOpenAction, confirmExtractAllStreamsDialog, showCleanupFilesDialog, showDiskFull, showCutFailedDialog, labelSegmentDialog, openYouTubeChaptersDialog, openAbout, showEditableJsonDialog, askForShiftSegments, selectSegmentsByLabelDialog, confirmExtractFramesAsImages, showRefuseToOverwrite, showParametersDialog } from './dialogs';
 import { openSendReportDialog } from './reporting';
 import { fallbackLng } from './i18n';
 import { createSegment, getCleanCutSegments, getSegApparentStart, findSegmentsAtCursor, sortSegments, invertSegments, getSegmentTags, convertSegmentsToChapters, hasAnySegmentOverlap } from './segments';
@@ -1753,24 +1753,60 @@ const App = memo(() => {
     }
   }, [customOutDir, enableOverwriteOutput, filePath, mainStreams, setWorking]);
 
-  const detectBlackScenes = useCallback(async () => {
+  const detectScenes = useCallback(async ({ name, workingText, errorText, fn }) => {
     if (!filePath) return;
     if (workingRef.current) return;
     try {
-      setWorking(i18n.t('Detecting black scenes'));
+      setWorking(workingText);
       setCutProgress(0);
 
-      const blackSegments = await blackDetect({ filePath, duration, onProgress: setCutProgress, from: currentApparentCutSeg.start, to: currentApparentCutSeg.end });
-      console.log('blackSegments', blackSegments);
-      loadCutSegments(blackSegments.map(({ blackStart, blackEnd }) => ({ start: blackStart, end: blackEnd })), true);
+      const newSegments = await fn();
+      console.log(name, newSegments);
+      loadCutSegments(newSegments, true);
     } catch (err) {
-      errorToast(i18n.t('Failed to detect black scenes'));
-      console.error('Failed to detect black scenes', err);
+      errorToast(errorText);
+      console.error('Failed to detect scenes', name, err);
     } finally {
       setWorking();
       setCutProgress();
     }
-  }, [filePath, setWorking, duration, currentApparentCutSeg, loadCutSegments]);
+  }, [filePath, setWorking, loadCutSegments]);
+
+  const detectBlackScenes = useCallback(async () => {
+    const parameters = {
+      black_min_duration: {
+        value: '2.0',
+        hint: i18n.t('Set the minimum detected black duration expressed in seconds. It must be a non-negative floating point number.'),
+      },
+      picture_black_ratio_th: {
+        value: '0.98',
+        hint: i18n.t('Set the threshold for considering a picture "black".'),
+      },
+      pixel_black_th: {
+        value: '0.10',
+        hint: i18n.t('Set the threshold for considering a pixel "black".'),
+      },
+    };
+    const filterOptions = await showParametersDialog({ title: i18n.t('Enter parameters'), parameters, docUrl: 'https://ffmpeg.org/ffmpeg-filters.html#blackdetect' });
+    if (filterOptions == null) return;
+    await detectScenes({ name: 'blackScenes', workingText: i18n.t('Detecting black scenes'), errorText: i18n.t('Failed to detect black scenes'), fn: async () => blackDetect({ filePath, duration, filterOptions, onProgress: setCutProgress, from: currentApparentCutSeg.start, to: currentApparentCutSeg.end }) });
+  }, [currentApparentCutSeg.end, currentApparentCutSeg.start, detectScenes, duration, filePath]);
+
+  const detectSilentScenes = useCallback(async () => {
+    const parameters = {
+      noise: {
+        value: '-60dB',
+        hint: i18n.t('Set noise tolerance. Can be specified in dB (in case "dB" is appended to the specified value) or amplitude ratio. Default is -60dB, or 0.001.'),
+      },
+      duration: {
+        value: '2.0',
+        hint: i18n.t('Set silence duration until notification (default is 2 seconds).'),
+      },
+    };
+    const filterOptions = await showParametersDialog({ title: i18n.t('Enter parameters'), parameters, docUrl: 'https://ffmpeg.org/ffmpeg-filters.html#silencedetect' });
+    if (filterOptions == null) return;
+    await detectScenes({ name: 'silentScenes', workingText: i18n.t('Detecting silent scenes'), errorText: i18n.t('Failed to detect silent scenes'), fn: async () => silenceDetect({ filePath, duration, filterOptions, onProgress: setCutProgress, from: currentApparentCutSeg.start, to: currentApparentCutSeg.end }) });
+  }, [currentApparentCutSeg.end, currentApparentCutSeg.start, detectScenes, duration, filePath]);
 
   const userHtml5ifyCurrentFile = useCallback(async () => {
     if (!filePath) return;
@@ -2268,13 +2304,14 @@ const App = memo(() => {
       reorderSegsByStartTime,
       concatCurrentBatch,
       detectBlackScenes,
+      detectSilentScenes,
       shiftAllSegmentTimes,
     };
 
     const entries = Object.entries(action);
     entries.forEach(([key, value]) => electron.ipcRenderer.on(key, value));
     return () => entries.forEach(([key, value]) => electron.ipcRenderer.removeListener(key, value));
-  }, [apparentCutSegments, askSetStartTimeOffset, checkFileOpened, clearSegments, closeBatch, closeFileWithConfirm, concatCurrentBatch, createFixedDurationSegments, createNumSegments, createRandomSegments, customOutDir, cutSegments, detectBlackScenes, detectedFps, extractAllStreams, fileFormat, filePath, fillSegmentsGaps, getFrameCount, invertAllSegments, loadCutSegments, loadMedia, openFilesDialog, openSendReportDialogWithState, reorderSegsByStartTime, setWorking, shiftAllSegmentTimes, shuffleSegments, toggleKeyboardShortcuts, toggleLastCommands, toggleSettings, tryFixInvalidDuration, userHtml5ifyCurrentFile, userOpenFiles]);
+  }, [apparentCutSegments, askSetStartTimeOffset, checkFileOpened, clearSegments, closeBatch, closeFileWithConfirm, concatCurrentBatch, createFixedDurationSegments, createNumSegments, createRandomSegments, customOutDir, cutSegments, detectBlackScenes, detectSilentScenes, detectedFps, extractAllStreams, fileFormat, filePath, fillSegmentsGaps, getFrameCount, invertAllSegments, loadCutSegments, loadMedia, openFilesDialog, openSendReportDialogWithState, reorderSegsByStartTime, setWorking, shiftAllSegmentTimes, shuffleSegments, toggleKeyboardShortcuts, toggleLastCommands, toggleSettings, tryFixInvalidDuration, userHtml5ifyCurrentFile, userOpenFiles]);
 
   const showAddStreamSourceDialog = useCallback(async () => {
     try {

src/dialogs.jsx

@@ -1,5 +1,5 @@
 import React, { useState, useCallback } from 'react';
-import { Checkbox, RadioGroup, Paragraph } from 'evergreen-ui';
+import { Button, TextInputField, Checkbox, RadioGroup, Paragraph, LinkIcon } from 'evergreen-ui';
 import Swal from 'sweetalert2';
 import i18n from 'i18next';
 import { Trans } from 'react-i18next';
@@ -13,6 +13,7 @@ import { parseYouTube } from './edlFormats';
 import CopyClipboardButton from './components/CopyClipboardButton';
 
 const { dialog, app } = window.require('@electron/remote');
+const electron = window.require('electron');
 
 const ReactSwal = withReactContent(Swal);
 
@@ -403,6 +404,44 @@ export async function showCleanupFilesDialog(cleanupChoicesIn = {}) {
   return undefined;
 }
 
+const ParametersInput = ({ description, parameters: parametersIn, onChange: onChangeProp, docUrl }) => {
+  const [parameters, setParameters] = useState(parametersIn);
+
+  const getParameter = (key) => parameters[key]?.value;
+  const onChange = (key, value) => setParameters((existing) => {
+    const newParameters = { ...existing, [key]: { ...existing[key], value } };
+    onChangeProp(newParameters);
+    return newParameters;
+  });
+
+  return (
+    <div style={{ textAlign: 'left' }}>
+      {description && <p>{description}</p>}
+
+      {docUrl && <p><Button iconBefore={LinkIcon} onClick={() => electron.shell.openExternal(docUrl)}>Read more</Button></p>}
+
+      {Object.entries(parametersIn).map(([key, parameter]) => (
+        <TextInputField key={key} label={parameter.label || key} value={getParameter(key)} onChange={(e) => onChange(key, e.target.value)} hint={parameter.hint} />
+      ))}
+    </div>
+  );
+};
+
+export async function showParametersDialog({ title, description, parameters: parametersIn, docUrl }) {
+  let parameters = parametersIn;
+
+  const { value } = await ReactSwal.fire({
+    title,
+    html: <ParametersInput description={description} parameters={parameters} onChange={(newParameters) => { parameters = newParameters; }} docUrl={docUrl} />,
+    confirmButtonText: i18n.t('Confirm'),
+    showCancelButton: true,
+    cancelButtonText: i18n.t('Cancel'),
+  });
+
+  if (value) return Object.fromEntries(Object.entries(parameters).map(([key, parameter]) => [key, parameter.value]));
+  return undefined;
+}
+
 
 export async function createFixedDurationSegments(fileDuration) {
   const segmentDuration = await askForSegmentDuration(fileDuration);

src/ffmpeg.js

@@ -548,31 +548,64 @@ export async function renderWaveformPng({ filePath, aroundTime, window, color })
   }
 }
 
-export async function blackDetect({ filePath, duration, minInterval = 0.05, onProgress, from, to }) {
+export async function detectIntervals({ filePath, duration, customArgs, onProgress, from, to, matchLineTokens }) {
   const args = [
     '-hide_banner',
     ...(from != null ? ['-ss', from.toFixed(5)] : []),
     '-i', filePath,
     ...(to != null ? ['-t', (to - from).toFixed(5)] : []),
-    '-vf', `blackdetect=d=${minInterval}`, '-an', '-f', 'null', '-',
+    ...customArgs,
+    '-f', 'null', '-',
   ];
   const process = execa(getFfmpegPath(), args, { encoding: null, buffer: false });
 
-  const blackSegments = [];
+  const segments = [];
 
   function customMatcher(line) {
-    const match = line.match(/^[blackdetect @ 0x[0-9a-f]+] black_start:([\d\\.]+) black_end:([\d\\.]+) black_duration:[\d\\.]+/);
-    if (!match) return;
-    const blackStart = parseFloat(match[1]);
-    const blackEnd = parseFloat(match[2]);
-    if (Number.isNaN(blackStart) || Number.isNaN(blackEnd)) return;
-    blackSegments.push({ blackStart, blackEnd });
+    const { start: startStr, end: endStr } = matchLineTokens(line);
+    const start = parseFloat(startStr);
+    const end = parseFloat(endStr);
+    if (start == null || end == null || Number.isNaN(start) || Number.isNaN(end)) return;
+    segments.push({ start, end });
   }
   handleProgress(process, duration, onProgress, customMatcher);
 
   await process;
   const offset = from != null ? from : 0;
-  return blackSegments.map(({ blackStart, blackEnd }) => ({ blackStart: blackStart + offset, blackEnd: blackEnd + offset }));
+  return segments.map(({ start, end }) => ({ start: start + offset, end: end + offset }));
+}
+
+const mapFilterOptions = (options) => Object.entries(options).map(([key, value]) => `${key}=${value}`).join(':');
+
+export async function blackDetect({ filePath, duration, filterOptions, onProgress, from, to }) {
+  function matchLineTokens(line) {
+    const match = line.match(/^[blackdetect\s*@\s*0x[0-9a-f]+] black_start:([\d\\.]+) black_end:([\d\\.]+) black_duration:[\d\\.]+/);
+    if (!match) return {};
+    return {
+      start: parseFloat(match[1]),
+      end: parseFloat(match[2]),
+    };
+  }
+  const customArgs = ['-vf', `blackdetect=${mapFilterOptions(filterOptions)}`, '-an'];
+  return detectIntervals({ filePath, duration, onProgress, from, to, matchLineTokens, customArgs });
+}
+
+export async function silenceDetect({ filePath, duration, filterOptions, onProgress, from, to }) {
+  function matchLineTokens(line) {
+    const match = line.match(/^[silencedetect\s*@\s*0x[0-9a-f]+] silence_end: ([\d\\.]+)[|\s]+silence_duration: ([\d\\.]+)/);
+    if (!match) return {};
+    const end = parseFloat(match[1]);
+    const silenceDuration = parseFloat(match[2]);
+    if (Number.isNaN(end) || Number.isNaN(silenceDuration)) return {};
+    const start = end - silenceDuration;
+    if (start < 0 || end <= 0 || start >= end) return {};
+    return {
+      start,
+      end,
+    };
+  }
+  const customArgs = ['-af', `silencedetect=${mapFilterOptions(filterOptions)}`, '-vn'];
+  return detectIntervals({ filePath, duration, onProgress, from, to, matchLineTokens, customArgs });
 }
 
 export async function extractWaveform({ filePath, outPath }) {