From 702160c1dfef5c428b494e7e7396257e9e35ea86 Mon Sep 17 00:00:00 2001 From: Matt Jenkins Date: Mon, 4 May 2020 16:45:58 +0100 Subject: [PATCH] Added sentence gain envelopes --- .../audiobookrecorder/AudiobookRecorder.java | 64 ++----- .../majenko/audiobookrecorder/Sentence.java | 77 ++++++++ .../majenko/audiobookrecorder/Waveform.java | 172 ++++++++++-------- 3 files changed, 190 insertions(+), 123 deletions(-) diff --git a/src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java b/src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java index 2bc6cf5..e3969dc 100644 --- a/src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java +++ b/src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java @@ -184,6 +184,7 @@ public class AudiobookRecorder extends JFrame implements DocumentListener { JToggleButtonSpacePlay selectSplitMode; JToggleButtonSpacePlay selectCutMode; JButtonSpacePlay doCutSplit; + JToggleButtonSpacePlay editGainCurve; JButtonSpacePlay refreshSentence; @@ -407,34 +408,6 @@ public class AudiobookRecorder extends JFrame implements DocumentListener { } }); - sampleWaveform.addMarkerDragListener(new MarkerDragListener() { - public void leftMarkerMoved(MarkerDragEvent e) { - Debug.trace(); - if (selectedSentence != null) { - if (!selectedSentence.isLocked()) { - selectedSentence.setStartOffset(e.getPosition()); - selectedSentence.updateCrossings(); - sampleWaveform.setAltMarkers(selectedSentence.getStartCrossing(), selectedSentence.getEndCrossing()); - } else { - sampleWaveform.setLeftMarker(selectedSentence.getStartOffset()); - } - } - } - - public void rightMarkerMoved(MarkerDragEvent e) { - Debug.trace(); - if (selectedSentence != null) { - if (!selectedSentence.isLocked()) { - selectedSentence.setEndOffset(e.getPosition()); - selectedSentence.updateCrossings(); - sampleWaveform.setAltMarkers(selectedSentence.getStartCrossing(), selectedSentence.getEndCrossing()); - } else { - sampleWaveform.setRightMarker(selectedSentence.getEndOffset()); - } - } - } - }); - sampleControl.add(sampleWaveform, BorderLayout.CENTER); reprocessAudioFFT = new JButtonSpacePlay(Icons.fft, "Autotrim Audio (FFT)", new ActionListener() { @@ -475,7 +448,14 @@ public class AudiobookRecorder extends JFrame implements DocumentListener { } } }); - + + editGainCurve = new JToggleButtonSpacePlay(Icons.normalize, "Edit gain curve", new ActionListener() { + public void actionPerformed(ActionEvent e) { + Debug.trace(); + sampleWaveform.setDisplayGainCurve(editGainCurve.isSelected()); + } + }); + selectSplitMode = new JToggleButtonSpacePlay(Icons.split, "Toggle split mode", new ActionListener() { public void actionPerformed(ActionEvent e) { Debug.trace(); @@ -655,6 +635,8 @@ public class AudiobookRecorder extends JFrame implements DocumentListener { } }); + controlsBottom.add(editGainCurve); + controlsBottom.addSeparator(); controlsBottom.add(selectSplitMode); controlsBottom.add(selectCutMode); controlsBottom.add(doCutSplit); @@ -814,10 +796,8 @@ public class AudiobookRecorder extends JFrame implements DocumentListener { if (n instanceof Sentence) { Sentence s = (Sentence)n; //selectedSentence = s; - sampleWaveform.setData(s.getDoubleAudioData(effectsEnabled)); - sampleWaveform.setMarkers(s.getStartOffset(), s.getEndOffset()); s.updateCrossings(); - sampleWaveform.setAltMarkers(s.getStartCrossing(), s.getEndCrossing()); + sampleWaveform.setSentence(s); postSentenceGap.setValue(s.getPostGap()); gainPercent.setValue((int)(s.getGain() * 100d)); locked.setSelected(s.isLocked()); @@ -836,7 +816,7 @@ public class AudiobookRecorder extends JFrame implements DocumentListener { selectSplitMode.setSelected(false); } else { //selectedSentence = null; - sampleWaveform.clearData(); + sampleWaveform.setSentence(null); postSentenceGap.setValue(0); gainPercent.setValue(100); locked.setSelected(false); @@ -2201,7 +2181,7 @@ public class AudiobookRecorder extends JFrame implements DocumentListener { bookTree.setSelectionPath(new TreePath(s.getPath())); bookTree.scrollPathToVisible(new TreePath(s.getPath())); } else { - sampleWaveform.clearData(); + sampleWaveform.setSentence(null); } // selectedSentence = s; saveBook(getBook()); @@ -3114,25 +3094,13 @@ public class AudiobookRecorder extends JFrame implements DocumentListener { Debug.trace(); SwingUtilities.invokeLater(new Runnable() { public void run() { - if (selectedSentence != null) { - if ((!force) && (sampleWaveform.getId() != null) && (sampleWaveform.getId().equals(selectedSentence.getId()))) return; - - sampleWaveform.setId(selectedSentence.getId()); - if (rawAudio.isSelected()) { - sampleWaveform.setData(selectedSentence.getRawAudioData()); - } else { - sampleWaveform.setData(selectedSentence.getDoubleAudioData(effectsEnabled)); - } - } + sampleWaveform.setSentence(selectedSentence); } }); } synchronized public void updateWaveformMarkers() { - if (selectedSentence != null) { - sampleWaveform.setMarkers(selectedSentence.getStartOffset(), selectedSentence.getEndOffset()); - sampleWaveform.setAltMarkers(selectedSentence.getStartCrossing(), selectedSentence.getEndCrossing()); - } + sampleWaveform.updateMarkers(); } public void updateEffectChains(TreeMap effs) { diff --git a/src/uk/co/majenko/audiobookrecorder/Sentence.java b/src/uk/co/majenko/audiobookrecorder/Sentence.java index 7afcb65..080a95e 100644 --- a/src/uk/co/majenko/audiobookrecorder/Sentence.java +++ b/src/uk/co/majenko/audiobookrecorder/Sentence.java @@ -20,6 +20,7 @@ import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import org.w3c.dom.Attr; import org.w3c.dom.Document; +import org.w3c.dom.NodeList; import org.w3c.dom.Element; import org.w3c.dom.Text; import javax.sound.sampled.TargetDataLine; @@ -100,6 +101,8 @@ public class Sentence extends BookTreeNode implements Cacheable { double[][] processedAudio = null; double[] fftProfile = null; + + TreeMap gainPoints = null; RecordingThread recordingThread; @@ -226,6 +229,19 @@ public class Sentence extends BookTreeNode implements Cacheable { peak = Utils.s2d(Book.getTextNode(root, "peak", "-1.000")); isDetected = Utils.s2b(Book.getTextNode(root, "detected")); + gainPoints = new TreeMap(); + Element gp = Book.getNode(root, "gainpoints"); + if (gp != null) { + NodeList points = gp.getElementsByTagName("gainpoint"); + + for (int i = 0; i < points.getLength(); i++) { + Element point = (Element)points.item(i); + int loc = Utils.s2i(point.getAttribute("location")); + double g = Utils.s2d(point.getAttribute("gain")); + gainPoints.put(loc, g); + } + } + if (text == null) text = id; if (text.equals("")) text = id; @@ -1481,6 +1497,12 @@ public class Sentence extends BookTreeNode implements Cacheable { } } + double[] gc = calculateGains(); + for (int i = 0; i < processedAudio[LEFT].length; i++) { + processedAudio[LEFT][i] *= gc[i]; + processedAudio[RIGHT][i] *= gc[i]; + } + return processedAudio; } @@ -1663,6 +1685,15 @@ public class Sentence extends BookTreeNode implements Cacheable { sentenceNode.appendChild(Book.makeTextNode(doc, "time", getLength())); sentenceNode.appendChild(Book.makeTextNode(doc, "peak", getPeak())); sentenceNode.appendChild(Book.makeTextNode(doc, "detected", beenDetected())); + Element gp = doc.createElement("gainpoints"); + for (Integer loc : gainPoints.keySet()) { + Double g = gainPoints.get(loc); + Element p = doc.createElement("gainpoint"); + p.setAttribute("location", String.format("%d", loc)); + p.setAttribute("gain", String.format("%.3g", g)); + gp.appendChild(p); + } + sentenceNode.appendChild(gp); return sentenceNode; } @@ -1802,4 +1833,50 @@ public class Sentence extends BookTreeNode implements Cacheable { getPeakDB(); reloadTree(); } + + public TreeMap getGainPoints() { + return gainPoints; + } + + public void addGainPoint(Integer loc, Double g) { + gainPoints.put(loc, g); + CacheManager.removeFromCache(this); + } + + public void removeGainPoint(Integer loc) { + gainPoints.remove(loc); + CacheManager.removeFromCache(this); + } + + public double[] calculateGains() { + double[] gains = new double[sampleSize]; + + double y = 1.0d; + int x1 = 0; + + if (gainPoints == null) { + for (int x = 0; x < sampleSize; x++) { + gains[x] = 1.0d; + } + return gains; + } + + for (Integer loc : gainPoints.keySet()) { + int x2 = loc; + double y2 = gainPoints.get(loc); + + int range = x2 - x1; + double diff = y2 - y; + double ystep = diff / (double)range; + for (int x = 0; x < range; x++) { + y += ystep; + gains[x1 + x] = y; + } + x1 = x2; + } + for (int x = x1; x < sampleSize; x++) { + gains[x] = y; + } + return gains; + } } diff --git a/src/uk/co/majenko/audiobookrecorder/Waveform.java b/src/uk/co/majenko/audiobookrecorder/Waveform.java index 8d10b0f..fb1995e 100644 --- a/src/uk/co/majenko/audiobookrecorder/Waveform.java +++ b/src/uk/co/majenko/audiobookrecorder/Waveform.java @@ -3,6 +3,7 @@ package uk.co.majenko.audiobookrecorder; import java.awt.event.MouseListener; import java.awt.event.MouseMotionListener; import javax.swing.JPanel; +import java.util.TreeMap; import java.util.ArrayList; import java.awt.Graphics; import java.awt.event.MouseEvent; @@ -12,7 +13,7 @@ import java.awt.Cursor; public class Waveform extends JPanel implements MouseListener, MouseMotionListener { - double[][] samples = null; + Sentence sentence = null; int leftMarker = 0; int rightMarker = 0; @@ -29,6 +30,8 @@ public class Waveform extends JPanel implements MouseListener, MouseMotionListen boolean displayCut = false; boolean displaySplit = false; + boolean displayGainCurve = false; + int dragging = 0; int step = 1; @@ -39,21 +42,30 @@ public class Waveform extends JPanel implements MouseListener, MouseMotionListen String loadedId = null; - ArrayList markerDragListeners; - public Waveform() { super(); addMouseListener(this); addMouseMotionListener(this); - markerDragListeners = new ArrayList(); } - public void addMarkerDragListener(MarkerDragListener l) { - if (markerDragListeners.indexOf(l) == -1) { - markerDragListeners.add(l); - } + public void setSentence(Sentence s) { + sentence = s; + playMarker = 0; + displayCut = false; + displaySplit = false; + updateMarkers(); } + public void updateMarkers() { + if (sentence != null) { + leftMarker = sentence.getStartOffset(); + rightMarker = sentence.getEndOffset(); + leftAltMarker = sentence.getStartCrossing(); + rightAltMarker = sentence.getEndCrossing(); + } + repaint(); + } + public void paintComponent(Graphics g) { Dimension size = getSize(); @@ -82,7 +94,8 @@ public class Waveform extends JPanel implements MouseListener, MouseMotionListen double scale = (h/2); - if (samples != null) { + if (sentence != null) { + double[][] samples = sentence.getDoubleAudioData(true); int num = samples[Sentence.LEFT].length; step = num / zoomFactor / w; @@ -173,51 +186,30 @@ public class Waveform extends JPanel implements MouseListener, MouseMotionListen for (int i = 0; i < h; i += 2) { g.drawLine((playMarker - offset) / step, i, (playMarker - offset) / step, i); } + + if (displayGainCurve) { + int x1 = 0; + double y1 = 1.0; + g.setColor(new Color(200, 200, 200)); + TreeMap points = sentence.getGainPoints(); + for (Integer loc : points.keySet()) { + int x2 = loc; + double y2 = points.get(loc); + + g.fillRect((x1 - offset) / step - 1, h - (int)((double)h / 2.0 * y1) - 1, 3, 3); + + g.drawLine((x1 - offset) / step, h - (int)((double)h / 2.0 * y1), (x2 - offset) / step, h - (int)((double)h / 2.0 * y2)); + x1 = x2; + y1 = y2; + } + g.fillRect((x1 - offset) / step - 1, h - (int)((double)h / 2.0 * y1) - 1, 3, 3); + g.drawLine((x1 - offset) / step, h - (int)((double)h / 2.0 * y1), (num - offset) / step, h - (int)((double)h / 2.0 * y1)); + } } } - public void setAltMarkers(int l, int r) { - leftAltMarker = l; - rightAltMarker = r; - repaint(); - } - - public void setMarkers(int l, int r) { - leftMarker = l; - rightMarker = r; - repaint(); - } - - public void setLeftAltMarker(int l) { - leftAltMarker = l; - repaint(); - } - - public void setRightAltMarker(int r) { - rightAltMarker = r; - repaint(); - } - - public void setLeftMarker(int l) { - leftMarker = l; - repaint(); - } - - public void setRightMarker(int r) { - rightMarker = r; - repaint(); - } - - public void clearData() { - samples = null; - repaint(); - } - - public void setData(double[][] s) { - samples = s; - playMarker = 0; - displayCut = false; - displaySplit = false; + public void setDisplayGainCurve(boolean b) { + displayGainCurve = b; repaint(); } @@ -264,46 +256,76 @@ public class Waveform extends JPanel implements MouseListener, MouseMotionListen public void mouseReleased(MouseEvent e) { if (dragging == 1) { - MarkerDragEvent evt = new MarkerDragEvent(this, leftMarker); - for (MarkerDragListener l : markerDragListeners) { - l.leftMarkerMoved(evt); - } + sentence.setStartOffset(leftMarker); + sentence.updateCrossings(); + updateMarkers(); } else if (dragging == 2) { - MarkerDragEvent evt = new MarkerDragEvent(this, rightMarker); - for (MarkerDragListener l : markerDragListeners) { - l.rightMarkerMoved(evt); - } + sentence.setEndOffset(rightMarker); + sentence.updateCrossings(); + updateMarkers(); } dragging = 0; } public void mouseClicked(MouseEvent e) { + if (displayGainCurve) { + if (e.getButton() == MouseEvent.BUTTON1) { + Dimension size = getSize(); + + int w = size.width; + int h = size.height; + + int x = e.getX() * step + offset; + double y = (double)(h - e.getY()) / (double)h * 2.0; + + sentence.addGainPoint(x, y); + repaint(); + } else if (e.getButton() == MouseEvent.BUTTON3) { + int x = e.getX() * step + offset; + int f = -1; + int diff = Integer.MAX_VALUE; + + TreeMap gc = sentence.getGainPoints(); + for (Integer loc : gc.keySet()) { + int d = Math.abs(loc - x); + if (d < diff) { + diff = d; + f = loc; + } + } + sentence.removeGainPoint(f); + repaint(); + } + } } public void mouseMoved(MouseEvent e) { int x = e.getX(); - if ((x >= ((leftMarker - offset)/step) - 10) && (x <= ((leftMarker - offset)/step) + 10)) { - setCursor(Cursor.getPredefinedCursor(Cursor.W_RESIZE_CURSOR)); - return; - } - if ((x >= ((rightMarker - offset)/step) - 10) && (x <= ((rightMarker - offset)/step) + 10)) { - setCursor(Cursor.getPredefinedCursor(Cursor.E_RESIZE_CURSOR)); - return; - } - if (displayCut || displaySplit) { - if ((x >= ((cutEntry - offset)/step) - 10) && (x <= ((cutEntry - offset)/step) + 10)) { - setCursor(Cursor.getPredefinedCursor(Cursor.TEXT_CURSOR)); - return; - } - } + int y = e.getY(); - if (displayCut) { + if (displayGainCurve) { + + } else if (displayCut) { if ((x >= ((cutExit - offset)/step) - 10) && (x <= ((cutExit - offset)/step) + 10)) { setCursor(Cursor.getPredefinedCursor(Cursor.TEXT_CURSOR)); return; } + } else { + if ((x >= ((leftMarker - offset)/step) - 10) && (x <= ((leftMarker - offset)/step) + 10)) { + setCursor(Cursor.getPredefinedCursor(Cursor.W_RESIZE_CURSOR)); + return; + } + if ((x >= ((rightMarker - offset)/step) - 10) && (x <= ((rightMarker - offset)/step) + 10)) { + setCursor(Cursor.getPredefinedCursor(Cursor.E_RESIZE_CURSOR)); + return; + } + if (displayCut || displaySplit) { + if ((x >= ((cutEntry - offset)/step) - 10) && (x <= ((cutEntry - offset)/step) + 10)) { + setCursor(Cursor.getPredefinedCursor(Cursor.TEXT_CURSOR)); + return; + } + } } - setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); }