From e46012c9884b23d99188777a7529f5e8f17a6d9c Mon Sep 17 00:00:00 2001 From: Matt Jenkins Date: Mon, 10 Sep 2018 13:37:48 +0100 Subject: [PATCH] Disabled speech temporarily. Using FFT for silence detection --- .../audiobookrecorder/AudiobookRecorder.java | 103 ++++++++++-- src/uk/co/majenko/audiobookrecorder/FFT.java | 130 +++++++++++++++ .../co/majenko/audiobookrecorder/Options.java | 28 +++- .../majenko/audiobookrecorder/Sentence.java | 153 +++++++++++++++++- 4 files changed, 402 insertions(+), 12 deletions(-) create mode 100644 src/uk/co/majenko/audiobookrecorder/FFT.java diff --git a/src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java b/src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java index 00ec870..569a1ed 100644 --- a/src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java +++ b/src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java @@ -13,6 +13,9 @@ import java.util.prefs.*; import java.io.*; import it.sauronsoftware.jave.*; import com.mpatric.mp3agic.*; +import edu.cmu.sphinx.api.*; +import edu.cmu.sphinx.decoder.adaptation.*; +import edu.cmu.sphinx.result.*; public class AudiobookRecorder extends JFrame { @@ -43,7 +46,7 @@ public class AudiobookRecorder extends JFrame { Book book = null; JTree bookTree; - DefaultTreeModel bookTreeModel; + public DefaultTreeModel bookTreeModel; Sentence recording = null; Sentence playing = null; @@ -61,8 +64,28 @@ public class AudiobookRecorder extends JFrame { Random rng = new Random(); + SourceDataLine play; + + public Configuration sphinxConfig; + public StreamSpeechRecognizer recognizer; + + public static AudiobookRecorder window; + void initSphinx() { + sphinxConfig = new Configuration(); + + sphinxConfig.setAcousticModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us"); + sphinxConfig.setDictionaryPath("resource:/edu/cmu/sphinx/models/en-us/cmudict-en-us.dict"); + sphinxConfig.setLanguageModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us.lm.bin"); + + try { + recognizer = new StreamSpeechRecognizer(sphinxConfig); + } catch (Exception e) { + e.printStackTrace(); + } + } + void buildToolbar(Container ob) { toolBar = new MainToolBar(this); toolBar.disableBook(); @@ -155,6 +178,8 @@ public class AudiobookRecorder extends JFrame { Icons.loadIcons(); +// initSphinx(); + try { // String clsname = "com.jtattoo.plaf.aluminium.AluminiumLookAndFeel"; @@ -374,12 +399,23 @@ public class AudiobookRecorder extends JFrame { centralPanel.getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW).put(KeyStroke.getKeyStroke("released D"), "deleteLast"); + centralPanel.getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW).put(KeyStroke.getKeyStroke("SPACE"), "startPlayback"); + + centralPanel.getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW).put(KeyStroke.getKeyStroke("E"), "startRerecord"); + centralPanel.getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW).put(KeyStroke.getKeyStroke("released E"), "stopRecord"); + centralPanel.getActionMap().put("startRecord", new AbstractAction() { public void actionPerformed(ActionEvent e) { if (bookTree.isEditing()) return; startRecording(); } }); + centralPanel.getActionMap().put("startRerecord", new AbstractAction() { + public void actionPerformed(ActionEvent e) { + if (bookTree.isEditing()) return; + startReRecording(); + } + }); centralPanel.getActionMap().put("stopRecord", new AbstractAction() { public void actionPerformed(ActionEvent e) { if (bookTree.isEditing()) return; @@ -393,6 +429,13 @@ public class AudiobookRecorder extends JFrame { } }); + centralPanel.getActionMap().put("startPlayback", new AbstractAction() { + public void actionPerformed(ActionEvent e) { + if (bookTree.isEditing()) return; + playSelectedSentence(); + } + }); + mainScroll = new JScrollPane(); centralPanel.add(mainScroll, BorderLayout.CENTER); @@ -408,11 +451,6 @@ public class AudiobookRecorder extends JFrame { int r = JOptionPane.showConfirmDialog(this, info, "New Book", JOptionPane.OK_CANCEL_OPTION); if (r != JOptionPane.OK_OPTION) return; - System.err.println("Name: " + info.getTitle()); - System.err.println("Author: " + info.getAuthor()); - System.err.println("Genre: " + info.getGenre()); - System.err.println("Comment: " + info.getComment()); - String name = info.getTitle(); File bookdir = new File(Options.get("path.storage"), name); @@ -507,6 +545,7 @@ public class AudiobookRecorder extends JFrame { JMenuObject o = (JMenuObject)e.getSource(); Sentence s = (Sentence)o.getObject(); s.autoTrimSample(); + s.recognise(); sampleWaveform.setData(s.getAudioData()); sampleWaveform.setMarkers(s.getStartOffset(), s.getEndOffset()); startOffset.setValue(s.getStartOffset()); @@ -543,6 +582,30 @@ public class AudiobookRecorder extends JFrame { } } + public void startReRecording() { + + if (recording != null) return; + if (book == null) return; + + toolBar.disableBook(); + toolBar.disableSentence(); + + DefaultMutableTreeNode selectedNode = (DefaultMutableTreeNode)bookTree.getLastSelectedPathComponent(); + + if (selectedNode == null) { + return; + } + + if (!(selectedNode instanceof Sentence)) { + return; + } + + if (((Sentence)selectedNode).startRecording()) { + recording = (Sentence)selectedNode; + centralPanel.setFlash(true); + } + } + public void startRecording() { if (recording != null) return; @@ -722,6 +785,10 @@ public class AudiobookRecorder extends JFrame { bookTree.setEditable(true); bookTree.setUI(new CustomTreeUI(mainScroll)); + + InputMap im = bookTree.getInputMap(JComponent.WHEN_FOCUSED); + im.put(KeyStroke.getKeyStroke(KeyEvent.VK_SPACE, 0), "startPlayback"); + roomNoise = new Sentence("room-noise", "Room Noise"); bookTree.addTreeSelectionListener(new TreeSelectionListener() { @@ -957,13 +1024,24 @@ public class AudiobookRecorder extends JFrame { export.mkdirs(); } - Encoder encoder = new Encoder(); + Encoder encoder; + + String ffloc = Options.get("path.ffmpeg"); + if (ffloc != null && !ffloc.equals("")) { + encoder = new Encoder(new FFMPEGLocator() { + public String getFFMPEGExecutablePath() { + return Options.get("path.ffmpeg"); + } + }); + } else { + encoder = new Encoder(); + } EncodingAttributes attributes = new EncodingAttributes(); AudioAttributes audioAttributes = new AudioAttributes(); audioAttributes.setCodec("libmp3lame"); - audioAttributes.setBitRate(new Integer(256000)); - audioAttributes.setSamplingRate(new Integer(44100)); + audioAttributes.setBitRate(Options.getInteger("audio.export.bitrate")); + audioAttributes.setSamplingRate(Options.getInteger("audio.export.samplerate")); audioAttributes.setChannels(new Integer(2)); attributes.setFormat("mp3"); @@ -1061,7 +1139,7 @@ public class AudiobookRecorder extends JFrame { try { AudioFormat format = s.getAudioFormat(); - SourceDataLine play = AudioSystem.getSourceDataLine(format, Options.getPlaybackMixer()); + play = AudioSystem.getSourceDataLine(format, Options.getPlaybackMixer()); play.open(format); play.start(); @@ -1097,6 +1175,7 @@ public class AudiobookRecorder extends JFrame { play.write(data, 0, data.length); } s = (Sentence)next; + bookTree.setSelectionPath(new TreePath(s.getPath())); } } catch (Exception e) { e.printStackTrace(); @@ -1138,6 +1217,10 @@ public class AudiobookRecorder extends JFrame { } public void stopPlaying() { + if (play != null) { + play.close(); + play = null; + } playing = null; } } diff --git a/src/uk/co/majenko/audiobookrecorder/FFT.java b/src/uk/co/majenko/audiobookrecorder/FFT.java new file mode 100644 index 0000000..1258b09 --- /dev/null +++ b/src/uk/co/majenko/audiobookrecorder/FFT.java @@ -0,0 +1,130 @@ +package uk.co.majenko.audiobookrecorder; + +/** +* @author Orlando Selenu +* +*/ +public class FFT { +/** + * The Fast Fourier Transform (generic version, with NO optimizations). + * + * @param inputReal + * an array of length n, the real part + * @param inputImag + * an array of length n, the imaginary part + * @param DIRECT + * TRUE = direct transform, FALSE = inverse transform + * @return a new array of length 2n + */ +public static double[] fft(final double[] inputReal, double[] inputImag, + boolean DIRECT) { + // - n is the dimension of the problem + // - nu is its logarithm in base e + int n = inputReal.length; + + // If n is a power of 2, then ld is an integer (_without_ decimals) + double ld = Math.log(n) / Math.log(2.0); + + // Here I check if n is a power of 2. If exist decimals in ld, I quit + // from the function returning null. + if (((int) ld) - ld != 0) { + System.out.println("The number of elements is not a power of 2."); + return null; + } + + // Declaration and initialization of the variables + // ld should be an integer, actually, so I don't lose any information in + // the cast + int nu = (int) ld; + int n2 = n / 2; + int nu1 = nu - 1; + double[] xReal = new double[n]; + double[] xImag = new double[n]; + double tReal, tImag, p, arg, c, s; + + // Here I check if I'm going to do the direct transform or the inverse + // transform. + double constant; + if (DIRECT) + constant = -2 * Math.PI; + else + constant = 2 * Math.PI; + + // I don't want to overwrite the input arrays, so here I copy them. This + // choice adds \Theta(2n) to the complexity. + for (int i = 0; i < n; i++) { + xReal[i] = inputReal[i]; + xImag[i] = inputImag[i]; + } + + // First phase - calculation + int k = 0; + for (int l = 1; l <= nu; l++) { + while (k < n) { + for (int i = 1; i <= n2; i++) { + p = bitreverseReference(k >> nu1, nu); + // direct FFT or inverse FFT + arg = constant * p / n; + c = Math.cos(arg); + s = Math.sin(arg); + tReal = xReal[k + n2] * c + xImag[k + n2] * s; + tImag = xImag[k + n2] * c - xReal[k + n2] * s; + xReal[k + n2] = xReal[k] - tReal; + xImag[k + n2] = xImag[k] - tImag; + xReal[k] += tReal; + xImag[k] += tImag; + k++; + } + k += n2; + } + k = 0; + nu1--; + n2 /= 2; + } + + // Second phase - recombination + k = 0; + int r; + while (k < n) { + r = bitreverseReference(k, nu); + if (r > k) { + tReal = xReal[k]; + tImag = xImag[k]; + xReal[k] = xReal[r]; + xImag[k] = xImag[r]; + xReal[r] = tReal; + xImag[r] = tImag; + } + k++; + } + + // Here I have to mix xReal and xImag to have an array (yes, it should + // be possible to do this stuff in the earlier parts of the code, but + // it's here to readibility). + double[] newArray = new double[xReal.length * 2]; + double radice = 1 / Math.sqrt(n); + for (int i = 0; i < newArray.length; i += 2) { + int i2 = i / 2; + // I used Stephen Wolfram's Mathematica as a reference so I'm going + // to normalize the output while I'm copying the elements. + newArray[i] = xReal[i2] * radice; + newArray[i + 1] = xImag[i2] * radice; + } + return newArray; +} + +/** + * The reference bitreverse function. + */ +private static int bitreverseReference(int j, int nu) { + int j2; + int j1 = j; + int k = 0; + for (int i = 1; i <= nu; i++) { + j2 = j1 / 2; + k = 2 * k + j1 - 2 * j2; + j1 = j2; + } + return k; + } +} diff --git a/src/uk/co/majenko/audiobookrecorder/Options.java b/src/uk/co/majenko/audiobookrecorder/Options.java index a6c2b0d..fc47c0a 100644 --- a/src/uk/co/majenko/audiobookrecorder/Options.java +++ b/src/uk/co/majenko/audiobookrecorder/Options.java @@ -21,6 +21,9 @@ public class Options extends JDialog { JSpinner preChapterGap; JSpinner postChapterGap; JSpinner postSentenceGap; + JTextField ffmpegLocation; + JComboBox bitRate; + JComboBox exportRate; static HashMap defaultPrefs; @@ -179,6 +182,12 @@ public class Options extends JDialog { addSeparator(); + ffmpegLocation = addFilePath("FFMPEG location:", get("path.ffmpeg")); + bitRate = addDropdown("Export bitrate:", getBitrates(), get("audio.export.bitrate")); + exportRate = addDropdown("Export sample rate:", getSampleRateList(), get("audio.export.samplerate")); + + + addSeparator(); @@ -259,11 +268,16 @@ public class Options extends JDialog { defaultPrefs.put("audio.recording.channels", "2"); defaultPrefs.put("audio.recording.samplerate", "48000"); defaultPrefs.put("audio.playback.device", mixers[0].key); - defaultPrefs.put("path.storage", (new File(System.getProperty("user.home"), "Recordings")).toString()); defaultPrefs.put("catenation.pre-chapter", "2000"); defaultPrefs.put("catenation.post-chapter", "2000"); defaultPrefs.put("catenation.post-sentence", "1000"); + + defaultPrefs.put("path.storage", (new File(System.getProperty("user.home"), "Recordings")).toString()); + defaultPrefs.put("path.ffmpeg", ""); + + defaultPrefs.put("audio.export.bitrate", "256000"); + defaultPrefs.put("audio.export.samplerate", "44100"); if (prefs == null) { prefs = Preferences.userNodeForPackage(AudiobookRecorder.class); @@ -307,9 +321,12 @@ public class Options extends JDialog { set("audio.recording.samplerate", ((KVPair)rateList.getSelectedItem()).key); set("audio.playback.device", ((KVPair)playbackList.getSelectedItem()).key); set("path.storage", storageFolder.getText()); + set("path.ffmpeg", ffmpegLocation.getText()); set("catenation.pre-chapter", "" + preChapterGap.getValue()); set("catenation.post-chapter", "" + postChapterGap.getValue()); set("catenation.post-sentence", "" + postSentenceGap.getValue()); + set("audio.export.bitrate", ((KVPair)bitRate.getSelectedItem()).key); + set("audio.export.samplerate", ((KVPair)exportRate.getSelectedItem()).key); savePreferences(); } @@ -356,4 +373,13 @@ public class Options extends JDialog { public static Mixer.Info getPlaybackMixer() { return getMixerByName(get("audio.playback.device")); } + + public static KVPair[] getBitrates() { + KVPair[] pairs = new KVPair[4]; + pairs[0] = new KVPair("128000", "128kbps"); + pairs[1] = new KVPair("192000", "192kbps"); + pairs[2] = new KVPair("256000", "256kbps"); + pairs[3] = new KVPair("320000", "320kbps"); + return pairs; + } } diff --git a/src/uk/co/majenko/audiobookrecorder/Sentence.java b/src/uk/co/majenko/audiobookrecorder/Sentence.java index 34cb3b1..8f903eb 100644 --- a/src/uk/co/majenko/audiobookrecorder/Sentence.java +++ b/src/uk/co/majenko/audiobookrecorder/Sentence.java @@ -9,6 +9,9 @@ import java.io.*; import java.nio.file.*; import javax.swing.tree.*; import javax.sound.sampled.*; +import edu.cmu.sphinx.api.*; +import edu.cmu.sphinx.decoder.adaptation.*; +import edu.cmu.sphinx.result.*; public class Sentence extends DefaultMutableTreeNode { @@ -29,7 +32,6 @@ public class Sentence extends DefaultMutableTreeNode { Thread recordingThread = null; - public Sentence() { super(""); id = UUID.randomUUID().toString(); @@ -118,13 +120,90 @@ public class Sentence extends DefaultMutableTreeNode { if (!id.equals("room-noise")) { autoTrimSample(); + recognise(); } } public void autoTrimSample() { int[] samples = getAudioData(); + if (samples == null) return; + int noiseFloor = AudiobookRecorder.window.getNoiseFloor(); + int blocks = samples.length / 4096 + 1; + + int[] intens = new int[blocks]; + int block = 0; + + for (int i = 0; i < samples.length; i+= 4096) { + double[] real = new double[4096]; + double[] imag = new double[4096]; + + for (int j = 0; j < 4096; j++) { + if (i + j < samples.length) { + real[j] = samples[i+j] / 32768d; + imag[j] = 0; + } else { + real[j] = 0; + imag[j] = 0; + } + } + + double[] buckets = FFT.fft(real, imag, true); + double av = 0; + for (int j = 1; j < 2048; j++) { + av += Math.abs(buckets[j]); + } + av /= 2047d; + + intens[block] = 0; + + for (int j = 1; j < 2048; j++) { + double d = Math.abs(av - buckets[j]); + if (d > 0.05) { + intens[block]++; + } + } + block++; + + } + + // Find first block with > 0 intensity and subtract one. + + int start = 0; + for (int i = 0; i < blocks; i++) { + if (intens[i] > 0) break; + start = i; + } + + if (start >= blocks) { + start = 0; + } + + startOffset = start * 4096; + if (startOffset < 0) startOffset = 0; + if (startOffset >= samples.length) startOffset = samples.length; + + int end = blocks - 1; + // And last block with > 0 intensity and add one. + for (int i = blocks-1; i >= 0; i--) { + if (intens[i] > 0) break; + end = i; + } + + if (end <= 0) { + end = blocks - 1; + } + + endOffset = end * 4096; + if (endOffset < 0) endOffset = 0; + if (endOffset >= samples.length) endOffset = samples.length; + +/* + + + + isSilence = false; // Find start for (int i = 0; i < samples.length; i++) { @@ -153,6 +232,7 @@ public class Sentence extends DefaultMutableTreeNode { isSilence = true; endOffset = samples.length-1; } +*/ } @@ -330,6 +410,7 @@ public class Sentence extends DefaultMutableTreeNode { play.write(buffer, 0, nr); }; + play.drain(); play.close(); } catch (Exception e) { e.printStackTrace(); @@ -356,4 +437,74 @@ public class Sentence extends DefaultMutableTreeNode { return null; } + public void recognise() { + + return; +/* + + Thread t = new Thread(new Runnable() { + + public void run() { + try { + + Configuration sphinxConfig = new Configuration(); + + sphinxConfig.setAcousticModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us"); + sphinxConfig.setDictionaryPath("resource:/edu/cmu/sphinx/models/en-us/cmudict-en-us.dict"); + sphinxConfig.setLanguageModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us.lm.bin"); + + + StreamSpeechRecognizer recognizer; + + recognizer = new StreamSpeechRecognizer(sphinxConfig); + + AudioInputStream s = AudioSystem.getAudioInputStream(getFile()); + AudioFormat format = s.getFormat(); + int frameSize = format.getFrameSize(); + int length = (int)s.getFrameLength(); + byte[] data = new byte[length * frameSize]; + + s.read(data); + + int channels = format.getChannels(); + int newLen = (length / 3); + byte[] decimated = new byte[newLen * 2]; + + for (int i = 0; i < newLen; i++) { + if (channels == 1) { + decimated[i * 2] = data[i * 6]; + decimated[i * 2 + 1] = data[i * 6 + 1]; + } else { + decimated[i * 2] = data[i * 12]; + decimated[i * 2 + 1] = data[i * 12 + 1]; + } + } + + + System.err.println("Decimated from " + length + " to " + newLen); + + ByteArrayInputStream bas = new ByteArrayInputStream(decimated); + recognizer.startRecognition(bas); + SpeechResult result; + String res = ""; + while ((result = recognizer.getResult()) != null) { + res += result.getHypothesis(); + res += " "; + } + recognizer.stopRecognition(); + + text = res; + + AudiobookRecorder.window.bookTreeModel.reload(Sentence.this); + + } catch (Exception e) { + e.printStackTrace(); + } + + } + }); + + t.start(); +*/ + } }