MajenkoProjects/AudiobookRecorder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Disabled speech temporarily. Using FFT for silence detection

Browse Source
This commit is contained in:
Matt Jenkins
2018-09-10 13:37:48 +01:00
parent ad0ad59d00
commit e46012c988
4 changed files with 402 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java
+93 -10
View File
@@ -13,6 +13,9 @@ import java.util.prefs.*;
import java.io.*;
import it.sauronsoftware.jave.*;
import com.mpatric.mp3agic.*;
import edu.cmu.sphinx.api.*;
import edu.cmu.sphinx.decoder.adaptation.*;
import edu.cmu.sphinx.result.*;
public class AudiobookRecorder extends JFrame {
@@ -43,7 +46,7 @@ public class AudiobookRecorder extends JFrame {
Book book = null;
JTree bookTree;
DefaultTreeModel bookTreeModel;
public DefaultTreeModel bookTreeModel;
Sentence recording = null;
Sentence playing = null;
@@ -61,8 +64,28 @@ public class AudiobookRecorder extends JFrame {
Random rng = new Random();
SourceDataLine play;
public Configuration sphinxConfig;
public StreamSpeechRecognizer recognizer;
public static AudiobookRecorder window;
void initSphinx() {
sphinxConfig = new Configuration();
sphinxConfig.setAcousticModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us");
sphinxConfig.setDictionaryPath("resource:/edu/cmu/sphinx/models/en-us/cmudict-en-us.dict");
sphinxConfig.setLanguageModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us.lm.bin");
try {
recognizer = new StreamSpeechRecognizer(sphinxConfig);
} catch (Exception e) {
e.printStackTrace();
}
}
void buildToolbar(Container ob) {
toolBar = new MainToolBar(this);
toolBar.disableBook();
@@ -155,6 +178,8 @@ public class AudiobookRecorder extends JFrame {
Icons.loadIcons();
// initSphinx();
try {
// String clsname = "com.jtattoo.plaf.aluminium.AluminiumLookAndFeel";
@@ -374,12 +399,23 @@ public class AudiobookRecorder extends JFrame {
centralPanel.getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW).put(KeyStroke.getKeyStroke("released D"), "deleteLast");
centralPanel.getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW).put(KeyStroke.getKeyStroke("SPACE"), "startPlayback");
centralPanel.getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW).put(KeyStroke.getKeyStroke("E"), "startRerecord");
centralPanel.getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW).put(KeyStroke.getKeyStroke("released E"), "stopRecord");
centralPanel.getActionMap().put("startRecord", new AbstractAction() {
public void actionPerformed(ActionEvent e) {
if (bookTree.isEditing()) return;
startRecording();
}
});
centralPanel.getActionMap().put("startRerecord", new AbstractAction() {
public void actionPerformed(ActionEvent e) {
if (bookTree.isEditing()) return;
startReRecording();
}
});
centralPanel.getActionMap().put("stopRecord", new AbstractAction() {
public void actionPerformed(ActionEvent e) {
if (bookTree.isEditing()) return;
@@ -393,6 +429,13 @@ public class AudiobookRecorder extends JFrame {
}
});
centralPanel.getActionMap().put("startPlayback", new AbstractAction() {
public void actionPerformed(ActionEvent e) {
if (bookTree.isEditing()) return;
playSelectedSentence();
}
});
mainScroll = new JScrollPane();
centralPanel.add(mainScroll, BorderLayout.CENTER);
@@ -408,11 +451,6 @@ public class AudiobookRecorder extends JFrame {
int r = JOptionPane.showConfirmDialog(this, info, "New Book", JOptionPane.OK_CANCEL_OPTION);
if (r != JOptionPane.OK_OPTION) return;
System.err.println("Name: " + info.getTitle());
System.err.println("Author: " + info.getAuthor());
System.err.println("Genre: " + info.getGenre());
System.err.println("Comment: " + info.getComment());
String name = info.getTitle();
File bookdir = new File(Options.get("path.storage"), name);
@@ -507,6 +545,7 @@ public class AudiobookRecorder extends JFrame {
JMenuObject o = (JMenuObject)e.getSource();
Sentence s = (Sentence)o.getObject();
s.autoTrimSample();
s.recognise();
sampleWaveform.setData(s.getAudioData());
sampleWaveform.setMarkers(s.getStartOffset(), s.getEndOffset());
startOffset.setValue(s.getStartOffset());
@@ -543,6 +582,30 @@ public class AudiobookRecorder extends JFrame {
}
}
public void startReRecording() {
if (recording != null) return;
if (book == null) return;
toolBar.disableBook();
toolBar.disableSentence();
DefaultMutableTreeNode selectedNode = (DefaultMutableTreeNode)bookTree.getLastSelectedPathComponent();
if (selectedNode == null) {
return;
}
if (!(selectedNode instanceof Sentence)) {
return;
}
if (((Sentence)selectedNode).startRecording()) {
recording = (Sentence)selectedNode;
centralPanel.setFlash(true);
}
}
public void startRecording() {
if (recording != null) return;
@@ -722,6 +785,10 @@ public class AudiobookRecorder extends JFrame {
bookTree.setEditable(true);
bookTree.setUI(new CustomTreeUI(mainScroll));
InputMap im = bookTree.getInputMap(JComponent.WHEN_FOCUSED);
im.put(KeyStroke.getKeyStroke(KeyEvent.VK_SPACE, 0), "startPlayback");
roomNoise = new Sentence("room-noise", "Room Noise");
bookTree.addTreeSelectionListener(new TreeSelectionListener() {
@@ -957,13 +1024,24 @@ public class AudiobookRecorder extends JFrame {
export.mkdirs();
}
Encoder encoder = new Encoder();
Encoder encoder;
String ffloc = Options.get("path.ffmpeg");
if (ffloc != null && !ffloc.equals("")) {
encoder = new Encoder(new FFMPEGLocator() {
public String getFFMPEGExecutablePath() {
return Options.get("path.ffmpeg");
}
});
} else {
encoder = new Encoder();
}
EncodingAttributes attributes = new EncodingAttributes();
AudioAttributes audioAttributes = new AudioAttributes();
audioAttributes.setCodec("libmp3lame");
audioAttributes.setBitRate(new Integer(256000));
audioAttributes.setSamplingRate(new Integer(44100));
audioAttributes.setBitRate(Options.getInteger("audio.export.bitrate"));
audioAttributes.setSamplingRate(Options.getInteger("audio.export.samplerate"));
audioAttributes.setChannels(new Integer(2));
attributes.setFormat("mp3");
@@ -1061,7 +1139,7 @@ public class AudiobookRecorder extends JFrame {
try {
AudioFormat format = s.getAudioFormat();
SourceDataLine play = AudioSystem.getSourceDataLine(format, Options.getPlaybackMixer());
play = AudioSystem.getSourceDataLine(format, Options.getPlaybackMixer());
play.open(format);
play.start();
@@ -1097,6 +1175,7 @@ public class AudiobookRecorder extends JFrame {
play.write(data, 0, data.length);
}
s = (Sentence)next;
bookTree.setSelectionPath(new TreePath(s.getPath()));
}
} catch (Exception e) {
e.printStackTrace();
@@ -1138,6 +1217,10 @@ public class AudiobookRecorder extends JFrame {
}
public void stopPlaying() {
if (play != null) {
play.close();
play = null;
}
playing = null;
}
}
src/uk/co/majenko/audiobookrecorder/FFT.java
+130
View File
@@ -0,0 +1,130 @@
package uk.co.majenko.audiobookrecorder;
/**
* @author Orlando Selenu
*
*/
public class FFT {
/**
* The Fast Fourier Transform (generic version, with NO optimizations).
*
* @param inputReal
* an array of length n, the real part
* @param inputImag
* an array of length n, the imaginary part
* @param DIRECT
* TRUE = direct transform, FALSE = inverse transform
* @return a new array of length 2n
*/
public static double[] fft(final double[] inputReal, double[] inputImag,
boolean DIRECT) {
// - n is the dimension of the problem
// - nu is its logarithm in base e
int n = inputReal.length;
// If n is a power of 2, then ld is an integer (_without_ decimals)
double ld = Math.log(n) / Math.log(2.0);
// Here I check if n is a power of 2. If exist decimals in ld, I quit
// from the function returning null.
if (((int) ld) - ld != 0) {
System.out.println("The number of elements is not a power of 2.");
return null;
}
// Declaration and initialization of the variables
// ld should be an integer, actually, so I don't lose any information in
// the cast
int nu = (int) ld;
int n2 = n / 2;
int nu1 = nu - 1;
double[] xReal = new double[n];
double[] xImag = new double[n];
double tReal, tImag, p, arg, c, s;
// Here I check if I'm going to do the direct transform or the inverse
// transform.
double constant;
if (DIRECT)
constant = -2 * Math.PI;
else
constant = 2 * Math.PI;
// I don't want to overwrite the input arrays, so here I copy them. This
// choice adds \Theta(2n) to the complexity.
for (int i = 0; i < n; i++) {
xReal[i] = inputReal[i];
xImag[i] = inputImag[i];
}
// First phase - calculation
int k = 0;
for (int l = 1; l <= nu; l++) {
while (k < n) {
for (int i = 1; i <= n2; i++) {
p = bitreverseReference(k >> nu1, nu);
// direct FFT or inverse FFT
arg = constant * p / n;
c = Math.cos(arg);
s = Math.sin(arg);
tReal = xReal[k + n2] * c + xImag[k + n2] * s;
tImag = xImag[k + n2] * c - xReal[k + n2] * s;
xReal[k + n2] = xReal[k] - tReal;
xImag[k + n2] = xImag[k] - tImag;
xReal[k] += tReal;
xImag[k] += tImag;
k++;
}
k += n2;
}
k = 0;
nu1--;
n2 /= 2;
}
// Second phase - recombination
k = 0;
int r;
while (k < n) {
r = bitreverseReference(k, nu);
if (r > k) {
tReal = xReal[k];
tImag = xImag[k];
xReal[k] = xReal[r];
xImag[k] = xImag[r];
xReal[r] = tReal;
xImag[r] = tImag;
}
k++;
}
// Here I have to mix xReal and xImag to have an array (yes, it should
// be possible to do this stuff in the earlier parts of the code, but
// it's here to readibility).
double[] newArray = new double[xReal.length * 2];
double radice = 1 / Math.sqrt(n);
for (int i = 0; i < newArray.length; i += 2) {
int i2 = i / 2;
// I used Stephen Wolfram's Mathematica as a reference so I'm going
// to normalize the output while I'm copying the elements.
newArray[i] = xReal[i2] * radice;
newArray[i + 1] = xImag[i2] * radice;
}
return newArray;
}
/**
* The reference bitreverse function.
*/
private static int bitreverseReference(int j, int nu) {
int j2;
int j1 = j;
int k = 0;
for (int i = 1; i <= nu; i++) {
j2 = j1 / 2;
k = 2 * k + j1 - 2 * j2;
j1 = j2;
}
return k;
}
}
src/uk/co/majenko/audiobookrecorder/Options.java
+27 -1
View File
@@ -21,6 +21,9 @@ public class Options extends JDialog {
JSpinner preChapterGap;
JSpinner postChapterGap;
JSpinner postSentenceGap;
JTextField ffmpegLocation;
JComboBox<KVPair> bitRate;
JComboBox<KVPair> exportRate;
static HashMap<String, String> defaultPrefs;
@@ -179,6 +182,12 @@ public class Options extends JDialog {
addSeparator();
ffmpegLocation = addFilePath("FFMPEG location:", get("path.ffmpeg"));
bitRate = addDropdown("Export bitrate:", getBitrates(), get("audio.export.bitrate"));
exportRate = addDropdown("Export sample rate:", getSampleRateList(), get("audio.export.samplerate"));
addSeparator();
@@ -259,11 +268,16 @@ public class Options extends JDialog {
defaultPrefs.put("audio.recording.channels", "2");
defaultPrefs.put("audio.recording.samplerate", "48000");
defaultPrefs.put("audio.playback.device", mixers[0].key);
defaultPrefs.put("path.storage", (new File(System.getProperty("user.home"), "Recordings")).toString());
defaultPrefs.put("catenation.pre-chapter", "2000");
defaultPrefs.put("catenation.post-chapter", "2000");
defaultPrefs.put("catenation.post-sentence", "1000");
defaultPrefs.put("path.storage", (new File(System.getProperty("user.home"), "Recordings")).toString());
defaultPrefs.put("path.ffmpeg", "");
defaultPrefs.put("audio.export.bitrate", "256000");
defaultPrefs.put("audio.export.samplerate", "44100");
if (prefs == null) {
prefs = Preferences.userNodeForPackage(AudiobookRecorder.class);
@@ -307,9 +321,12 @@ public class Options extends JDialog {
set("audio.recording.samplerate", ((KVPair)rateList.getSelectedItem()).key);
set("audio.playback.device", ((KVPair)playbackList.getSelectedItem()).key);
set("path.storage", storageFolder.getText());
set("path.ffmpeg", ffmpegLocation.getText());
set("catenation.pre-chapter", "" + preChapterGap.getValue());
set("catenation.post-chapter", "" + postChapterGap.getValue());
set("catenation.post-sentence", "" + postSentenceGap.getValue());
set("audio.export.bitrate", ((KVPair)bitRate.getSelectedItem()).key);
set("audio.export.samplerate", ((KVPair)exportRate.getSelectedItem()).key);
savePreferences();
}
@@ -356,4 +373,13 @@ public class Options extends JDialog {
public static Mixer.Info getPlaybackMixer() {
return getMixerByName(get("audio.playback.device"));
}
public static KVPair[] getBitrates() {
KVPair[] pairs = new KVPair[4];
pairs[0] = new KVPair("128000", "128kbps");
pairs[1] = new KVPair("192000", "192kbps");
pairs[2] = new KVPair("256000", "256kbps");
pairs[3] = new KVPair("320000", "320kbps");
return pairs;
}
}
src/uk/co/majenko/audiobookrecorder/Sentence.java
+152 -1
View File
@@ -9,6 +9,9 @@ import java.io.*;
import java.nio.file.*;
import javax.swing.tree.*;
import javax.sound.sampled.*;
import edu.cmu.sphinx.api.*;
import edu.cmu.sphinx.decoder.adaptation.*;
import edu.cmu.sphinx.result.*;
public class Sentence extends DefaultMutableTreeNode {
@@ -29,7 +32,6 @@ public class Sentence extends DefaultMutableTreeNode {
Thread recordingThread = null;
public Sentence() {
super("");
id = UUID.randomUUID().toString();
@@ -118,13 +120,90 @@ public class Sentence extends DefaultMutableTreeNode {
if (!id.equals("room-noise")) {
autoTrimSample();
recognise();
}
}
public void autoTrimSample() {
int[] samples = getAudioData();
if (samples == null) return;
int noiseFloor = AudiobookRecorder.window.getNoiseFloor();
int blocks = samples.length / 4096 + 1;
int[] intens = new int[blocks];
int block = 0;
for (int i = 0; i < samples.length; i+= 4096) {
double[] real = new double[4096];
double[] imag = new double[4096];
for (int j = 0; j < 4096; j++) {
if (i + j < samples.length) {
real[j] = samples[i+j] / 32768d;
imag[j] = 0;
} else {
real[j] = 0;
imag[j] = 0;
}
}
double[] buckets = FFT.fft(real, imag, true);
double av = 0;
for (int j = 1; j < 2048; j++) {
av += Math.abs(buckets[j]);
}
av /= 2047d;
intens[block] = 0;
for (int j = 1; j < 2048; j++) {
double d = Math.abs(av - buckets[j]);
if (d > 0.05) {
intens[block]++;
}
}
block++;
}
// Find first block with > 0 intensity and subtract one.
int start = 0;
for (int i = 0; i < blocks; i++) {
if (intens[i] > 0) break;
start = i;
}
if (start >= blocks) {
start = 0;
}
startOffset = start * 4096;
if (startOffset < 0) startOffset = 0;
if (startOffset >= samples.length) startOffset = samples.length;
int end = blocks - 1;
// And last block with > 0 intensity and add one.
for (int i = blocks-1; i >= 0; i--) {
if (intens[i] > 0) break;
end = i;
}
if (end <= 0) {
end = blocks - 1;
}
endOffset = end * 4096;
if (endOffset < 0) endOffset = 0;
if (endOffset >= samples.length) endOffset = samples.length;
/*
isSilence = false;
// Find start
for (int i = 0; i < samples.length; i++) {
@@ -153,6 +232,7 @@ public class Sentence extends DefaultMutableTreeNode {
isSilence = true;
endOffset = samples.length-1;
}
*/
}
@@ -330,6 +410,7 @@ public class Sentence extends DefaultMutableTreeNode {
play.write(buffer, 0, nr);
};
play.drain();
play.close();
} catch (Exception e) {
e.printStackTrace();
@@ -356,4 +437,74 @@ public class Sentence extends DefaultMutableTreeNode {
return null;
}
public void recognise() {
return;
/*
Thread t = new Thread(new Runnable() {
public void run() {
try {
Configuration sphinxConfig = new Configuration();
sphinxConfig.setAcousticModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us");
sphinxConfig.setDictionaryPath("resource:/edu/cmu/sphinx/models/en-us/cmudict-en-us.dict");
sphinxConfig.setLanguageModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us.lm.bin");
StreamSpeechRecognizer recognizer;
recognizer = new StreamSpeechRecognizer(sphinxConfig);
AudioInputStream s = AudioSystem.getAudioInputStream(getFile());
AudioFormat format = s.getFormat();
int frameSize = format.getFrameSize();
int length = (int)s.getFrameLength();
byte[] data = new byte[length * frameSize];
s.read(data);
int channels = format.getChannels();
int newLen = (length / 3);
byte[] decimated = new byte[newLen * 2];
for (int i = 0; i < newLen; i++) {
if (channels == 1) {
decimated[i * 2] = data[i * 6];
decimated[i * 2 + 1] = data[i * 6 + 1];
} else {
decimated[i * 2] = data[i * 12];
decimated[i * 2 + 1] = data[i * 12 + 1];
}
}
System.err.println("Decimated from " + length + " to " + newLen);
ByteArrayInputStream bas = new ByteArrayInputStream(decimated);
recognizer.startRecognition(bas);
SpeechResult result;
String res = "";
while ((result = recognizer.getResult()) != null) {
res += result.getHypothesis();
res += " ";
}
recognizer.stopRecognition();
text = res;
AudiobookRecorder.window.bookTreeModel.reload(Sentence.this);
} catch (Exception e) {
e.printStackTrace();
}
}
});
t.start();
*/
}
}