Disabled speech temporarily. Using FFT for silence detection

src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java

@@ -13,6 +13,9 @@ import java.util.prefs.*;
 import java.io.*;
 import it.sauronsoftware.jave.*;
 import com.mpatric.mp3agic.*;
+import edu.cmu.sphinx.api.*;
+import edu.cmu.sphinx.decoder.adaptation.*;
+import edu.cmu.sphinx.result.*;
 
 public class AudiobookRecorder extends JFrame {
 
@@ -43,7 +46,7 @@ public class AudiobookRecorder extends JFrame {
     Book book = null;
 
     JTree bookTree;
-    DefaultTreeModel bookTreeModel;
+    public DefaultTreeModel bookTreeModel;
 
     Sentence recording = null;
     Sentence playing = null;
@@ -61,8 +64,28 @@ public class AudiobookRecorder extends JFrame {
 
     Random rng = new Random();
 
+    SourceDataLine play;
+
+    public Configuration sphinxConfig;
+    public StreamSpeechRecognizer recognizer;
+
+
     public static AudiobookRecorder window;
 
+    void initSphinx() {
+        sphinxConfig = new Configuration();
+
+        sphinxConfig.setAcousticModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us");
+        sphinxConfig.setDictionaryPath("resource:/edu/cmu/sphinx/models/en-us/cmudict-en-us.dict");
+        sphinxConfig.setLanguageModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us.lm.bin");
+
+        try {
+            recognizer = new StreamSpeechRecognizer(sphinxConfig);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
     void buildToolbar(Container ob) {
         toolBar = new MainToolBar(this);
         toolBar.disableBook();
@@ -155,6 +178,8 @@ public class AudiobookRecorder extends JFrame {
 
         Icons.loadIcons();
 
+//        initSphinx();
+
 
         try {
 //            String clsname = "com.jtattoo.plaf.aluminium.AluminiumLookAndFeel";
@@ -374,12 +399,23 @@ public class AudiobookRecorder extends JFrame {
 
         centralPanel.getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW).put(KeyStroke.getKeyStroke("released D"), "deleteLast");
 
+        centralPanel.getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW).put(KeyStroke.getKeyStroke("SPACE"), "startPlayback");
+
+        centralPanel.getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW).put(KeyStroke.getKeyStroke("E"), "startRerecord");
+        centralPanel.getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW).put(KeyStroke.getKeyStroke("released E"), "stopRecord");
+
         centralPanel.getActionMap().put("startRecord", new AbstractAction() {
             public void actionPerformed(ActionEvent e) {
                 if (bookTree.isEditing()) return;
                 startRecording();
             }
         });
+        centralPanel.getActionMap().put("startRerecord", new AbstractAction() {
+            public void actionPerformed(ActionEvent e) {
+                if (bookTree.isEditing()) return;
+                startReRecording();
+            }
+        });
         centralPanel.getActionMap().put("stopRecord", new AbstractAction() {
             public void actionPerformed(ActionEvent e) {
                 if (bookTree.isEditing()) return;
@@ -393,6 +429,13 @@ public class AudiobookRecorder extends JFrame {
             }
         });
 
+        centralPanel.getActionMap().put("startPlayback", new AbstractAction() {
+            public void actionPerformed(ActionEvent e) {
+                if (bookTree.isEditing()) return;
+                playSelectedSentence();
+            }
+        });
+
         mainScroll = new JScrollPane();
         centralPanel.add(mainScroll, BorderLayout.CENTER);
 
@@ -408,11 +451,6 @@ public class AudiobookRecorder extends JFrame {
         int r = JOptionPane.showConfirmDialog(this, info, "New Book", JOptionPane.OK_CANCEL_OPTION);
         if (r != JOptionPane.OK_OPTION) return;
 
-        System.err.println("Name: " + info.getTitle());
-        System.err.println("Author: " + info.getAuthor());
-        System.err.println("Genre: " + info.getGenre());
-        System.err.println("Comment: " + info.getComment());
-        
         String name = info.getTitle();
 
         File bookdir = new File(Options.get("path.storage"), name);
@@ -507,6 +545,7 @@ public class AudiobookRecorder extends JFrame {
                         JMenuObject o = (JMenuObject)e.getSource();
                         Sentence s = (Sentence)o.getObject();
                         s.autoTrimSample();
+                        s.recognise();
                         sampleWaveform.setData(s.getAudioData());
                         sampleWaveform.setMarkers(s.getStartOffset(), s.getEndOffset());
                         startOffset.setValue(s.getStartOffset());
@@ -543,6 +582,30 @@ public class AudiobookRecorder extends JFrame {
         }
     }
 
+    public void startReRecording() {
+
+        if (recording != null) return;
+        if (book == null) return;
+
+        toolBar.disableBook();
+        toolBar.disableSentence();
+
+        DefaultMutableTreeNode selectedNode = (DefaultMutableTreeNode)bookTree.getLastSelectedPathComponent();
+
+        if (selectedNode == null) {
+            return;
+        }
+
+        if (!(selectedNode instanceof Sentence)) {
+            return;
+        }
+
+        if (((Sentence)selectedNode).startRecording()) {
+            recording = (Sentence)selectedNode;
+            centralPanel.setFlash(true);
+        }
+    }
+
     public void startRecording() {
 
         if (recording != null) return;
@@ -722,6 +785,10 @@ public class AudiobookRecorder extends JFrame {
         bookTree.setEditable(true);
         bookTree.setUI(new CustomTreeUI(mainScroll));
 
+
+        InputMap im = bookTree.getInputMap(JComponent.WHEN_FOCUSED);
+        im.put(KeyStroke.getKeyStroke(KeyEvent.VK_SPACE, 0), "startPlayback");
+
         roomNoise = new Sentence("room-noise", "Room Noise");
 
         bookTree.addTreeSelectionListener(new TreeSelectionListener() {
@@ -957,13 +1024,24 @@ public class AudiobookRecorder extends JFrame {
                 export.mkdirs();
             }
 
-            Encoder encoder = new Encoder();
+            Encoder encoder;
+
+            String ffloc = Options.get("path.ffmpeg");
+            if (ffloc != null && !ffloc.equals("")) {
+                encoder = new Encoder(new FFMPEGLocator() {
+                    public String getFFMPEGExecutablePath() {
+                        return Options.get("path.ffmpeg");
+                    }
+                });
+            } else {
+                encoder = new Encoder();
+            }
             EncodingAttributes attributes = new EncodingAttributes();
 
             AudioAttributes audioAttributes = new AudioAttributes();
             audioAttributes.setCodec("libmp3lame");
-            audioAttributes.setBitRate(new Integer(256000));
+            audioAttributes.setBitRate(Options.getInteger("audio.export.bitrate"));
-            audioAttributes.setSamplingRate(new Integer(44100));
+            audioAttributes.setSamplingRate(Options.getInteger("audio.export.samplerate"));
             audioAttributes.setChannels(new Integer(2));
             
             attributes.setFormat("mp3");
@@ -1061,7 +1139,7 @@ public class AudiobookRecorder extends JFrame {
                 try {
 
                     AudioFormat format = s.getAudioFormat();
-                    SourceDataLine play = AudioSystem.getSourceDataLine(format, Options.getPlaybackMixer());
+                    play = AudioSystem.getSourceDataLine(format, Options.getPlaybackMixer());
                     play.open(format);
                     play.start();
 
@@ -1097,6 +1175,7 @@ public class AudiobookRecorder extends JFrame {
                             play.write(data, 0, data.length);
                         }
                         s = (Sentence)next;
+                        bookTree.setSelectionPath(new TreePath(s.getPath()));
                     }
                 } catch (Exception e) {
                     e.printStackTrace();
@@ -1138,6 +1217,10 @@ public class AudiobookRecorder extends JFrame {
     }
 
     public void stopPlaying() {
+        if (play != null) {
+            play.close();
+            play = null;
+        }
         playing = null;
     }
 }

src/uk/co/majenko/audiobookrecorder/FFT.java

@@ -0,0 +1,130 @@
+package uk.co.majenko.audiobookrecorder;
+
+/**
+* @author Orlando Selenu
+*
+*/
+public class FFT {
+/**
+ * The Fast Fourier Transform (generic version, with NO optimizations).
+ *
+ * @param inputReal
+ *            an array of length n, the real part
+ * @param inputImag
+ *            an array of length n, the imaginary part
+ * @param DIRECT
+ *            TRUE = direct transform, FALSE = inverse transform
+ * @return a new array of length 2n
+ */
+public static double[] fft(final double[] inputReal, double[] inputImag,
+                           boolean DIRECT) {
+    // - n is the dimension of the problem
+    // - nu is its logarithm in base e
+    int n = inputReal.length;
+
+    // If n is a power of 2, then ld is an integer (_without_ decimals)
+    double ld = Math.log(n) / Math.log(2.0);
+
+    // Here I check if n is a power of 2. If exist decimals in ld, I quit
+    // from the function returning null.
+    if (((int) ld) - ld != 0) {
+        System.out.println("The number of elements is not a power of 2.");
+        return null;
+    }
+
+    // Declaration and initialization of the variables
+    // ld should be an integer, actually, so I don't lose any information in
+    // the cast
+    int nu = (int) ld;
+    int n2 = n / 2;
+    int nu1 = nu - 1;
+    double[] xReal = new double[n];
+    double[] xImag = new double[n];
+    double tReal, tImag, p, arg, c, s;
+
+    // Here I check if I'm going to do the direct transform or the inverse
+    // transform.
+    double constant;
+    if (DIRECT)
+        constant = -2 * Math.PI;
+    else
+        constant = 2 * Math.PI;
+
+    // I don't want to overwrite the input arrays, so here I copy them. This
+    // choice adds \Theta(2n) to the complexity.
+    for (int i = 0; i < n; i++) {
+        xReal[i] = inputReal[i];
+        xImag[i] = inputImag[i];
+    }
+
+    // First phase - calculation
+    int k = 0;
+    for (int l = 1; l <= nu; l++) {
+        while (k < n) {
+            for (int i = 1; i <= n2; i++) {
+                p = bitreverseReference(k >> nu1, nu);
+                // direct FFT or inverse FFT
+                arg = constant * p / n;
+                c = Math.cos(arg);
+                s = Math.sin(arg);
+                tReal = xReal[k + n2] * c + xImag[k + n2] * s;
+                tImag = xImag[k + n2] * c - xReal[k + n2] * s;
+                xReal[k + n2] = xReal[k] - tReal;
+                xImag[k + n2] = xImag[k] - tImag;
+                xReal[k] += tReal;
+                xImag[k] += tImag;
+                k++;
+            }
+            k += n2;
+        }
+        k = 0;
+        nu1--;
+        n2 /= 2;
+    }
+
+    // Second phase - recombination
+    k = 0;
+    int r;
+    while (k < n) {
+        r = bitreverseReference(k, nu);
+        if (r > k) {
+            tReal = xReal[k];
+            tImag = xImag[k];
+            xReal[k] = xReal[r];
+            xImag[k] = xImag[r];
+            xReal[r] = tReal;
+            xImag[r] = tImag;
+        }
+        k++;
+    }
+
+    // Here I have to mix xReal and xImag to have an array (yes, it should
+    // be possible to do this stuff in the earlier parts of the code, but
+    // it's here to readibility).
+    double[] newArray = new double[xReal.length * 2];
+    double radice = 1 / Math.sqrt(n);
+    for (int i = 0; i < newArray.length; i += 2) {
+        int i2 = i / 2;
+        // I used Stephen Wolfram's Mathematica as a reference so I'm going
+        // to normalize the output while I'm copying the elements.
+        newArray[i] = xReal[i2] * radice;
+        newArray[i + 1] = xImag[i2] * radice;
+    }
+    return newArray;
+}
+
+/**
+ * The reference bitreverse function.
+ */
+private static int bitreverseReference(int j, int nu) {
+    int j2;
+    int j1 = j;
+    int k = 0;
+    for (int i = 1; i <= nu; i++) {
+        j2 = j1 / 2;
+        k = 2 * k + j1 - 2 * j2;
+        j1 = j2;
+    }
+    return k;
+  }
+}

src/uk/co/majenko/audiobookrecorder/Options.java

@@ -21,6 +21,9 @@ public class Options extends JDialog {
     JSpinner preChapterGap;
     JSpinner postChapterGap;
     JSpinner postSentenceGap;
+    JTextField ffmpegLocation;
+    JComboBox<KVPair> bitRate;
+    JComboBox<KVPair> exportRate;
 
 
     static HashMap<String, String> defaultPrefs;
@@ -179,6 +182,12 @@ public class Options extends JDialog {
 
         addSeparator();
 
+        ffmpegLocation = addFilePath("FFMPEG location:", get("path.ffmpeg"));
+        bitRate = addDropdown("Export bitrate:", getBitrates(), get("audio.export.bitrate"));
+        exportRate = addDropdown("Export sample rate:", getSampleRateList(), get("audio.export.samplerate"));
+        
+
+        addSeparator();
 
 
 
@@ -259,11 +268,16 @@ public class Options extends JDialog {
         defaultPrefs.put("audio.recording.channels", "2");
         defaultPrefs.put("audio.recording.samplerate", "48000");
         defaultPrefs.put("audio.playback.device", mixers[0].key);
-        defaultPrefs.put("path.storage", (new File(System.getProperty("user.home"), "Recordings")).toString());
 
         defaultPrefs.put("catenation.pre-chapter", "2000");
         defaultPrefs.put("catenation.post-chapter", "2000");
         defaultPrefs.put("catenation.post-sentence", "1000");
+    
+        defaultPrefs.put("path.storage", (new File(System.getProperty("user.home"), "Recordings")).toString());
+        defaultPrefs.put("path.ffmpeg", "");
+
+        defaultPrefs.put("audio.export.bitrate", "256000");
+        defaultPrefs.put("audio.export.samplerate", "44100");
 
         if (prefs == null) {
             prefs = Preferences.userNodeForPackage(AudiobookRecorder.class);
@@ -307,9 +321,12 @@ public class Options extends JDialog {
         set("audio.recording.samplerate", ((KVPair)rateList.getSelectedItem()).key);
         set("audio.playback.device", ((KVPair)playbackList.getSelectedItem()).key);
         set("path.storage", storageFolder.getText());
+        set("path.ffmpeg", ffmpegLocation.getText());
         set("catenation.pre-chapter", "" + preChapterGap.getValue());
         set("catenation.post-chapter", "" + postChapterGap.getValue());
         set("catenation.post-sentence", "" + postSentenceGap.getValue());
+        set("audio.export.bitrate", ((KVPair)bitRate.getSelectedItem()).key);
+        set("audio.export.samplerate", ((KVPair)exportRate.getSelectedItem()).key);
 
         savePreferences();
     }
@@ -356,4 +373,13 @@ public class Options extends JDialog {
     public static Mixer.Info getPlaybackMixer() {
         return getMixerByName(get("audio.playback.device"));
     }
+
+    public static KVPair[] getBitrates() {
+        KVPair[] pairs = new KVPair[4];
+        pairs[0] = new KVPair("128000", "128kbps");
+        pairs[1] = new KVPair("192000", "192kbps");
+        pairs[2] = new KVPair("256000", "256kbps");
+        pairs[3] = new KVPair("320000", "320kbps");
+        return pairs;
+    }
 }

src/uk/co/majenko/audiobookrecorder/Sentence.java

@@ -9,6 +9,9 @@ import java.io.*;
 import java.nio.file.*;
 import javax.swing.tree.*;
 import javax.sound.sampled.*;
+import edu.cmu.sphinx.api.*;
+import edu.cmu.sphinx.decoder.adaptation.*;
+import edu.cmu.sphinx.result.*;
 
 public class Sentence extends DefaultMutableTreeNode {
     
@@ -29,7 +32,6 @@ public class Sentence extends DefaultMutableTreeNode {
 
     Thread recordingThread = null;
     
-
     public Sentence() {
         super("");
         id = UUID.randomUUID().toString();
@@ -118,13 +120,90 @@ public class Sentence extends DefaultMutableTreeNode {
 
         if (!id.equals("room-noise")) {
             autoTrimSample();
+            recognise();
         }
     }
 
     public void autoTrimSample() {
         int[] samples = getAudioData();
+        if (samples == null) return;
+
         int noiseFloor = AudiobookRecorder.window.getNoiseFloor();
 
+        int blocks = samples.length / 4096 + 1;
+
+        int[] intens = new int[blocks];
+        int block = 0;
+
+        for (int i = 0; i < samples.length; i+= 4096) {
+            double[] real = new double[4096];
+            double[] imag = new double[4096];
+
+            for (int j = 0; j < 4096; j++) {
+                if (i + j < samples.length) {
+                    real[j] = samples[i+j] / 32768d;
+                    imag[j] = 0;
+                } else {
+                    real[j] = 0;
+                    imag[j] = 0;
+                }
+            }
+
+            double[] buckets = FFT.fft(real, imag, true);
+            double av = 0;
+            for (int j = 1; j < 2048; j++) {
+                av += Math.abs(buckets[j]);
+            }
+            av /= 2047d;
+
+            intens[block] = 0;
+
+            for (int j = 1; j < 2048; j++) {
+                double d = Math.abs(av - buckets[j]);
+                if (d > 0.05) {
+                    intens[block]++;
+                }
+            }
+            block++;
+            
+        }
+
+        // Find first block with > 0 intensity and subtract one.
+
+        int start = 0;
+        for (int i = 0; i < blocks; i++) {
+            if (intens[i] > 0) break;
+            start = i;
+        }
+
+        if (start >= blocks) {
+            start = 0;
+        }
+
+        startOffset = start * 4096;
+        if (startOffset < 0) startOffset = 0;
+        if (startOffset >= samples.length) startOffset = samples.length;
+
+        int end = blocks - 1;
+        // And last block with > 0 intensity and add one.
+        for (int i = blocks-1; i >= 0; i--) {
+            if (intens[i] > 0) break;
+            end = i;
+        }
+
+        if (end <= 0) {
+            end = blocks - 1;
+        }
+
+        endOffset = end * 4096;
+        if (endOffset < 0) endOffset = 0;
+        if (endOffset >= samples.length) endOffset = samples.length;
+
+/*            
+
+        
+
+
         isSilence = false;
         // Find start
         for (int i = 0; i < samples.length; i++) {
@@ -153,6 +232,7 @@ public class Sentence extends DefaultMutableTreeNode {
             isSilence = true;
             endOffset = samples.length-1;
         }
+*/
         
     }
 
@@ -330,6 +410,7 @@ public class Sentence extends DefaultMutableTreeNode {
 
                 play.write(buffer, 0, nr);
             };
+            play.drain();
             play.close(); 
         } catch (Exception e) {
             e.printStackTrace();
@@ -356,4 +437,74 @@ public class Sentence extends DefaultMutableTreeNode {
         return null;
     }
 
+    public void recognise() {
+
+        return;
+/*
+
+        Thread t = new Thread(new Runnable() {
+
+            public void run() {
+                try {
+
+                    Configuration sphinxConfig = new Configuration();
+
+                    sphinxConfig.setAcousticModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us");
+                    sphinxConfig.setDictionaryPath("resource:/edu/cmu/sphinx/models/en-us/cmudict-en-us.dict");
+                    sphinxConfig.setLanguageModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us.lm.bin");
+
+
+                    StreamSpeechRecognizer recognizer;
+
+                    recognizer = new StreamSpeechRecognizer(sphinxConfig);
+
+                    AudioInputStream s = AudioSystem.getAudioInputStream(getFile());
+                    AudioFormat format = s.getFormat();
+                    int frameSize = format.getFrameSize();
+                    int length = (int)s.getFrameLength();
+                    byte[] data = new byte[length * frameSize];
+
+                    s.read(data);
+
+                    int channels = format.getChannels();
+                    int newLen = (length / 3);
+                    byte[] decimated = new byte[newLen * 2];
+
+                    for (int i = 0; i < newLen; i++) {
+                        if (channels == 1) {
+                            decimated[i * 2] = data[i * 6];
+                            decimated[i * 2 + 1] = data[i * 6 + 1];
+                        } else {
+                            decimated[i * 2] = data[i * 12];
+                            decimated[i * 2 + 1] = data[i * 12 + 1];
+                        }
+                    }
+
+
+                    System.err.println("Decimated from " + length + " to " + newLen);
+
+                    ByteArrayInputStream bas = new ByteArrayInputStream(decimated);
+                    recognizer.startRecognition(bas);
+                    SpeechResult result;
+                    String res = "";
+                    while ((result = recognizer.getResult()) != null) {
+                        res += result.getHypothesis();
+                        res += " ";
+                    }
+                    recognizer.stopRecognition(); 
+
+                    text = res;
+
+                    AudiobookRecorder.window.bookTreeModel.reload(Sentence.this);
+
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+
+            }
+        });
+
+        t.start();
+*/
+    }
 }