From d4a64d4e722fc958cc987e5ae94cbf6fc4ce9ae1 Mon Sep 17 00:00:00 2001 From: Matt Jenkins Date: Mon, 6 Jul 2020 18:17:49 +0100 Subject: [PATCH] Automate gain profile generation --- .../audiobookrecorder/AudiobookRecorder.java | 15 ++ .../co/majenko/audiobookrecorder/Biquad.java | 15 ++ src/uk/co/majenko/audiobookrecorder/Book.java | 1 + .../co/majenko/audiobookrecorder/Options.java | 16 +- .../majenko/audiobookrecorder/Sentence.java | 170 ++++++++++++++++-- .../audiobookrecorder/VersionChecker.java | 1 + .../majenko/audiobookrecorder/Waveform.java | 11 ++ 7 files changed, 207 insertions(+), 22 deletions(-) diff --git a/src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java b/src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java index 4764c82..c752f35 100644 --- a/src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java +++ b/src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java @@ -185,6 +185,7 @@ public class AudiobookRecorder extends JFrame implements DocumentListener { JToggleButtonSpacePlay selectCutMode; JButtonSpacePlay doCutSplit; JToggleButtonSpacePlay editGainCurve; + JButtonSpacePlay autoCreatePoints; JButtonSpacePlay refreshSentence; @@ -449,9 +450,20 @@ public class AudiobookRecorder extends JFrame implements DocumentListener { } }); + autoCreatePoints = new JButtonSpacePlay(Icons.normalize, "Create peak points", new ActionListener() { + public void actionPerformed(ActionEvent e) { + Debug.trace(); + if (selectedSentence != null) { + selectedSentence.autoAddPeakGainPoints(); + updateWaveform(true); + } + } + }); + editGainCurve = new JToggleButtonSpacePlay(Icons.normalize, "Edit gain curve", new ActionListener() { public void actionPerformed(ActionEvent e) { Debug.trace(); + autoCreatePoints.setEnabled(editGainCurve.isSelected()); sampleWaveform.setDisplayGainCurve(editGainCurve.isSelected()); } }); @@ -635,6 +647,8 @@ public class AudiobookRecorder extends JFrame implements DocumentListener { } }); + controlsBottom.add(autoCreatePoints); + autoCreatePoints.setEnabled(false); controlsBottom.add(editGainCurve); controlsBottom.addSeparator(); controlsBottom.add(selectSplitMode); @@ -2458,6 +2472,7 @@ public class AudiobookRecorder extends JFrame implements DocumentListener { } else { lastGain = snt.normalize(lastGain - variance, lastGain + variance); } + snt.autoAddPeakGainPoints(); } dialog.closeDialog(); diff --git a/src/uk/co/majenko/audiobookrecorder/Biquad.java b/src/uk/co/majenko/audiobookrecorder/Biquad.java index dce80c2..e72499b 100644 --- a/src/uk/co/majenko/audiobookrecorder/Biquad.java +++ b/src/uk/co/majenko/audiobookrecorder/Biquad.java @@ -78,6 +78,21 @@ public class Biquad implements Effect { setPeakGain(peakGainDB); } + // Special single channel version for wave profile processing + public void process(double[] samples) { + Debug.trace(); + lz1 = 0d; + lz2 = 0d; + for (int i = 0; i < samples.length; i++) { + double lout = samples[i] * a0 + lz1; + + lz1 = samples[i] * a1 + lz2 - b1 * lout; + lz2 = samples[i] * a2 - b2 * lout; + + samples[i] = lout; + } + } + public void process(double[][] samples) { Debug.trace(); lz1 = 0d; diff --git a/src/uk/co/majenko/audiobookrecorder/Book.java b/src/uk/co/majenko/audiobookrecorder/Book.java index 98c811d..a1b179f 100644 --- a/src/uk/co/majenko/audiobookrecorder/Book.java +++ b/src/uk/co/majenko/audiobookrecorder/Book.java @@ -472,6 +472,7 @@ public class Book extends BookTreeNode { public void onSelect(BookTreeNode target) { Debug.trace(); AudiobookRecorder.setSelectedBook(this); + AudiobookRecorder.window.setTitle("AudioBook Recorder :: " + name); if (target == this) { AudiobookRecorder.setSelectedChapter(null); AudiobookRecorder.setSelectedSentence(null); diff --git a/src/uk/co/majenko/audiobookrecorder/Options.java b/src/uk/co/majenko/audiobookrecorder/Options.java index 45536fc..40a6e2b 100644 --- a/src/uk/co/majenko/audiobookrecorder/Options.java +++ b/src/uk/co/majenko/audiobookrecorder/Options.java @@ -60,7 +60,9 @@ public class Options extends JDialog { JSpinner shortSentenceGap; JSpinner postParagraphGap; JSpinner postSectionGap; - JSpinner maxGainVariance; + JSpinner rmsLow; + JSpinner rmsHigh; + JCheckBox autoNormalize; JTextField ffmpegLocation; JComboBox bitRate; JComboBox channels; @@ -358,7 +360,9 @@ public class Options extends JDialog { trimMethod = addDropdown(optionsPanel, "Auto-trim method:", getTrimMethods(), get("audio.recording.trim"), "None: don't auto-trim. FFT: Compare the FFT profile of blocks to the room noise profile and trim silent blocks, Peak: Look for the start and end rise and fall points"); fftThreshold = addSpinner(optionsPanel, "FFT threshold:", 0, 100, 1, getInteger("audio.recording.trim.fft"), "", "This specifies the difference (in hundredths) between the power of FFT buckets in a sample block compared to the overall power of the same FFT bucket in the room noise. Raising this number makes the FFT trimming less sensitive."); fftBlockSize = addDropdown(optionsPanel, "FFT Block size:", getFFTBlockSizes(), get("audio.recording.trim.blocksize"), "How large an FFT block should be when processing. Larger values increase sensitivity but at the epxense of resolution."); - maxGainVariance = addSpinner(optionsPanel, "Maximum gain variance:", 0, 100, 1, getInteger("audio.recording.variance"), "", "This is how much the gain is allowed to vary by from phrase to phrase when normalizing an entire chapter."); + rmsLow = addSpinner(optionsPanel, "Target RMS (low):", -100, 0, 1, getInteger("audio.recording.rms.low"), "", "When normalizing this is the lowest target average RMS to aim for"); + rmsHigh = addSpinner(optionsPanel, "Target RMS (high):", -100, 0, 1, getInteger("audio.recording.rms.high"), "", "When normalizing this is the highest target average RMS to aim for"); + autoNormalize = addCheckBox(optionsPanel, "Enable automatic normalization", getBoolean("process.normalize"), "This will automatically normalize each phrase after recording"); addSeparator(optionsPanel); @@ -599,7 +603,9 @@ public class Options extends JDialog { defaultPrefs.put("catenation.post-section", "3000"); defaultPrefs.put("audio.recording.trim.fft", "10"); - defaultPrefs.put("audio.recording.variance", "10"); + defaultPrefs.put("audio.recording.rms.low", "-22"); + defaultPrefs.put("audio.recording.rms.high", "-20"); + defaultPrefs.put("process.normalize", "true"); defaultPrefs.put("path.storage", (new File(System.getProperty("user.home"), "Recordings")).toString()); defaultPrefs.put("path.archive", (new File(new File(System.getProperty("user.home"), "Recordings"),"archive")).toString()); @@ -736,7 +742,9 @@ public class Options extends JDialog { set("editor.external", externalEditor.getText()); set("cache.size", cacheSize.getValue()); set("audio.recording.trim.fft", fftThreshold.getValue()); - set("audio.recording.variance", maxGainVariance.getValue()); + set("audio.recording.rms.low", rmsLow.getValue()); + set("audio.recording.rms.high", rmsHigh.getValue()); + set("process.normalize", autoNormalize.isSelected()); if (fftBlockSize.getSelectedItem() != null) set("audio.recording.trim.blocksize", ((KVPair)fftBlockSize.getSelectedItem()).key); if (playbackBlockSize.getSelectedItem() != null) set("audio.playback.blocksize", ((KVPair)playbackBlockSize.getSelectedItem()).key); diff --git a/src/uk/co/majenko/audiobookrecorder/Sentence.java b/src/uk/co/majenko/audiobookrecorder/Sentence.java index 63272b2..d970ad6 100644 --- a/src/uk/co/majenko/audiobookrecorder/Sentence.java +++ b/src/uk/co/majenko/audiobookrecorder/Sentence.java @@ -103,6 +103,8 @@ public class Sentence extends BookTreeNode implements Cacheable { double[][] processedAudio = null; double[] fftProfile = null; + + double[] waveProfile = null; TreeMap gainPoints = null; @@ -332,6 +334,7 @@ public class Sentence extends BookTreeNode implements Cacheable { public void run() { sentence.autoTrimSamplePeak(); AudiobookRecorder.window.updateWaveformMarkers(); + if (Options.getBoolean("process.normalize")) sentence.normalize(); } }); } else if (tm.equals("fft")) { @@ -339,6 +342,7 @@ public class Sentence extends BookTreeNode implements Cacheable { public void run() { sentence.autoTrimSampleFFT(); AudiobookRecorder.window.updateWaveformMarkers(); + if (Options.getBoolean("process.normalize")) sentence.normalize(); } }); } else { @@ -449,6 +453,7 @@ public class Sentence extends BookTreeNode implements Cacheable { updateCrossings(); intens = null; samples = null; + waveProfile = null; processed = true; reloadTree(); } @@ -791,6 +796,7 @@ public class Sentence extends BookTreeNode implements Cacheable { public void clearCache() { Debug.trace(); audioData = null; + waveProfile = null; processedAudio = null; storedFormat = null; } @@ -940,6 +946,10 @@ public class Sentence extends BookTreeNode implements Cacheable { } public void setGain(double g) { + setGain(g, false); + } + + public void setGain(double g, boolean batch) { Debug.trace(); if (g <= 0.0001d) g = 1.0d; if (g == gain) return; @@ -950,7 +960,7 @@ public class Sentence extends BookTreeNode implements Cacheable { if (gint != gainint) { refreshAllData(); peak = -1; - reloadTree(); + if (!batch) reloadTree(); } } @@ -962,27 +972,28 @@ public class Sentence extends BookTreeNode implements Cacheable { public double normalize(double low, double high) { Debug.trace(); if (locked) return gain; - double max = getPeakValue(false); - double d = 0.708 / max; - if (d < low) d = low; - if (d > high) d = high; - setGain(d); + + int targetLow = Options.getInteger("audio.recording.rms.low"); + int targetHigh = Options.getInteger("audio.recording.rms.high"); + + while ((int)getRMS() < targetLow) { + setGain(gain + 0.1); + if (gain >= 10.0d) break; + } + + while ((int)getRMS() > targetHigh) { + setGain(gain - 0.1); + } + + refreshAllData(); peak = -1; getPeak(); reloadTree(); - return d; + return gain; } public double normalize() { - Debug.trace(); - if (locked) return gain; - double max = getPeakValue(false); - double d = 0.708 / max; - setGain(d); - peak = -1; - getPeak(); - reloadTree(); - return d; + return normalize(0, 0); } class ExternalEditor implements Runnable { @@ -1832,6 +1843,7 @@ public class Sentence extends BookTreeNode implements Cacheable { peak = -1d; sampleSize = -1; audioData = null; + waveProfile = null; processedAudio = null; fftProfile = null; CacheManager.removeFromCache(this); @@ -1937,6 +1949,24 @@ public class Sentence extends BookTreeNode implements Cacheable { return rms; } + public boolean isClipping(int start, int end) { + + double[][] samples = getProcessedAudioData(); + if (samples == null) { + return false; + } + + for (int i = start; i <= end; i++) { + if (Math.abs(samples[LEFT][i]) > 0.708) { + return true; + } + if (Math.abs(samples[RIGHT][i]) > 0.708) { + return true; + } + } + return false; + } + public boolean isClipping() { if (clipping > 0) { if (clipping == 1) return false; @@ -1950,15 +1980,119 @@ public class Sentence extends BookTreeNode implements Cacheable { clipping = 1; for (int i = 0; i < samples[LEFT].length; i++) { - if (samples[LEFT][i] > 0.708) { + if (Math.abs(samples[LEFT][i]) > 0.708) { clipping = 2; return true; } - if (samples[RIGHT][i] > 0.708) { + if (Math.abs(samples[RIGHT][i]) > 0.708) { clipping = 2; return true; } } return false; } + + public double mix(double a, double b) { + return (a + b) / 2d; + } + + final int window = 500; + + public double[] getWaveProfile() { + if (waveProfile != null) return waveProfile; + double[][] samples = getProcessedAudioData(); + waveProfile = new double[samples[LEFT].length]; + + double rt = 0; + + int nsamp = samples[LEFT].length; + int nbuckets = nsamp / window; + + double[] buckets = new double[nbuckets + 1]; + + for (int i = 0; i < nsamp; i++) { + double sval = Math.abs(mix(samples[LEFT][i], samples[RIGHT][i])); + int bnum = i / window; + if (sval > buckets[bnum]) buckets[bnum] = sval; + } + + for (int i = 0; i < nsamp; i++) { + waveProfile[i] = buckets[i / window]; + } + + return waveProfile; + } + + int findBiggestPeak() { + double[][] samples = getProcessedAudioData(); + + int pos = 0; + + double peak = 0; + + for (int i = 0; i < samples[LEFT].length; i++) { + if (Math.abs(samples[LEFT][i]) > peak) { + peak = Math.abs(samples[LEFT][i]); + pos = i; + } + if (Math.abs(samples[RIGHT][i]) > peak) { + peak = Math.abs(samples[RIGHT][i]); + pos = i; + } + } + return pos; + } + + int findPreviousZero(int offset) { + double[] profile = getWaveProfile(); + + int pos = offset; + while (pos > 0) { + if (profile[pos] < 0.05) return pos - (window/2); + pos--; + } + return -1; + } + + int findNextZero(int offset) { + double[] profile = getWaveProfile(); + + int pos = offset; + while (pos < profile.length) { + if (profile[pos] < 0.05) return pos + (window / 2); + pos++; + } + return -1; + } + + public void autoAddPeakGainPoints() { + while (true) { + double[][] samples = getProcessedAudioData(); + int pos = findBiggestPeak(); + System.err.println("Biggest peak: " + pos); + if ((Math.abs(samples[LEFT][pos]) < 0.708) && (Math.abs(samples[RIGHT][pos]) < 0.708)) { + System.err.println("Not a peak!"); + return; + } + + int start = findPreviousZero(pos); + int end = findNextZero(pos); + if (start == -1) { + System.err.println("Unable to find previous zero"); + return; + } + if (end == -1) { + System.err.println("Unable to find next zero"); + return; + } + + addGainPoint(start, 1d); + addGainPoint(pos, 1d); + addGainPoint(end, 1d); + + while (isClipping(start, end)) { + adjustGainPoint(pos, -0.05); + } + } + } } diff --git a/src/uk/co/majenko/audiobookrecorder/VersionChecker.java b/src/uk/co/majenko/audiobookrecorder/VersionChecker.java index 4fddeec..9c48fd2 100644 --- a/src/uk/co/majenko/audiobookrecorder/VersionChecker.java +++ b/src/uk/co/majenko/audiobookrecorder/VersionChecker.java @@ -56,6 +56,7 @@ public class VersionChecker implements Runnable { if (Utils.s2i(installedVersion) >= Utils.s2i(availableVersion)) return; JButton upgrade = new JButton("A new version is available."); + upgrade.setFocusable(false); upgrade.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { Utils.browse(website); diff --git a/src/uk/co/majenko/audiobookrecorder/Waveform.java b/src/uk/co/majenko/audiobookrecorder/Waveform.java index c6ba163..767f7fc 100644 --- a/src/uk/co/majenko/audiobookrecorder/Waveform.java +++ b/src/uk/co/majenko/audiobookrecorder/Waveform.java @@ -99,6 +99,7 @@ public class Waveform extends JPanel implements MouseListener, MouseMotionListen if (sentence != null) { double[][] samples = sentence.getDoubleAudioData(true); + double[] waveProfile = sentence.getWaveProfile(); int num = samples[Sentence.LEFT].length; step = num / zoomFactor / w; @@ -117,9 +118,13 @@ public class Waveform extends JPanel implements MouseListener, MouseMotionListen double lave = 0; double lmax = 0; + double prof = 0; + for (int o = 0; o < step; o++) { if (offset + (n * step) + o >= samples[Sentence.LEFT].length) break; double sample = (samples[Sentence.LEFT][offset + (n * step) + o] + samples[Sentence.RIGHT][offset + (n * step) + o]) / 2d; + double asamp = waveProfile[offset + (n * step) + o]; + if (asamp > prof) prof = asamp; if (sample >= 0) { have += sample; hcnt++; @@ -158,6 +163,12 @@ public class Waveform extends JPanel implements MouseListener, MouseMotionListen g.drawLine(n, (int)(h/2 + lmax), n, (int)(h/2 - hmax)); g.setColor(new Color(0, 100, 255)); g.drawLine(n, (int)(h/2 + lave), n, (int)(h/2 - have)); + if (prof < 0.05d) { + g.setColor(new Color(0, 255, 0)); + } else { + g.setColor(new Color(0, 100, 0)); + } + g.drawLine(n, (int)(h/2 - prof * scale), n, (int)(h/2 - prof * scale)); } g.setColor(new Color(255, 0, 0, 64));