MajenkoProjects/AudiobookRecorder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Dumped Sphinx and instead using Haven OnDemand speech API

Browse Source
This commit is contained in:
Matt Jenkins
2018-10-21 13:52:20 +01:00
parent ff08db3e44
commit f5c0d5bbfc
22 changed files with 3317 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/uk/co/majenko/audiobookrecorder/AudiobookRecorder.java
+13 -59
View File
@@ -13,9 +13,6 @@ import java.util.prefs.*;
import java.io.*;
import it.sauronsoftware.jave.*;
import com.mpatric.mp3agic.*;
import edu.cmu.sphinx.api.*;
import edu.cmu.sphinx.decoder.adaptation.*;
import edu.cmu.sphinx.result.*;
import java.nio.file.Files;
import java.util.zip.*;
import javax.swing.filechooser.*;
@@ -90,30 +87,19 @@ public class AudiobookRecorder extends JFrame {
SourceDataLine play = null;
public HavenQueue havenQueue = new HavenQueue();
public TargetDataLine microphone = null;
public AudioInputStream microphoneStream = null;
public Configuration sphinxConfig;
public StreamSpeechRecognizer recognizer;
public static AudiobookRecorder window;
void initSphinx() {
sphinxConfig = new Configuration();
sphinxConfig.setAcousticModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us");
sphinxConfig.setDictionaryPath("resource:/edu/cmu/sphinx/models/en-us/cmudict-en-us.dict");
sphinxConfig.setLanguageModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us.lm.bin");
try {
recognizer = new StreamSpeechRecognizer(sphinxConfig);
} catch (Exception e) {
e.printStackTrace();
}
}
void buildToolbar(Container ob) {
toolBar = new MainToolBar(this);
toolBar.addSeparator();
toolBar.add(Box.createHorizontalGlue());
toolBar.add(havenQueue);
ob.add(toolBar, BorderLayout.NORTH);
}
@@ -715,41 +701,6 @@ public class AudiobookRecorder extends JFrame {
}
}
class BatchConversionThread implements Runnable {
Chapter chapter;
public BatchConversionThread(Chapter c) {
chapter = c;
}
public void run() {
try {
Configuration sphinxConfig = new Configuration();
sphinxConfig.setAcousticModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us");
sphinxConfig.setDictionaryPath("resource:/edu/cmu/sphinx/models/en-us/cmudict-en-us.dict");
sphinxConfig.setLanguageModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us.lm.bin");
sphinxConfig.setSampleRate((int)(book.getAudioFormat().getSampleRate() / 4f));
StreamSpeechRecognizer recognizer;
recognizer = new StreamSpeechRecognizer(sphinxConfig);
for (Enumeration s = chapter.children(); s.hasMoreElements();) {
Sentence snt = (Sentence)s.nextElement();
if (!snt.isLocked()) {
if (snt.getId().equals(snt.getText())) {
snt.doRecognition(recognizer);
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
@SuppressWarnings("unchecked")
void treePopup(MouseEvent e) {
@@ -771,7 +722,7 @@ public class AudiobookRecorder extends JFrame {
JMenuObject o = (JMenuObject)e.getSource();
Sentence s = (Sentence)o.getObject();
if (!s.isLocked()) {
s.recognise();
havenQueue.submit(s);
}
}
});
@@ -1031,9 +982,12 @@ public class AudiobookRecorder extends JFrame {
public void actionPerformed(ActionEvent e) {
JMenuObject o = (JMenuObject)e.getSource();
Chapter c = (Chapter)o.getObject();
BatchConversionThread r = new BatchConversionThread(c);
Thread t = new Thread(r);
t.start();
for (Enumeration s = c.children(); s.hasMoreElements();) {
Sentence snt = (Sentence)s.nextElement();
if (snt.getId().equals(snt.getText())) {
havenQueue.submit(snt);
}
}
}
});
src/uk/co/majenko/audiobookrecorder/BookTreeRenderer.java
+4
View File
@@ -11,6 +11,10 @@ public class BookTreeRenderer extends DefaultTreeCellRenderer {
if (value instanceof Sentence) {
Sentence s = (Sentence)value;
if (s.getOverrideText() != null) {
ret.setText(s.getOverrideText());
}
if (s.getAttentionFlag()) {
ret.setForeground(new Color(0xFF, 0xFF, 0x00));
ret.setIcon(Icons.attention);
src/uk/co/majenko/audiobookrecorder/HavenQueue.java
+83
View File
@@ -0,0 +1,83 @@
package uk.co.majenko.audiobookrecorder;
import java.util.*;
import java.util.concurrent.*;
import javax.swing.*;
import java.awt.*;
import java.util.Timer;
public class HavenQueue extends JPanel {
ConcurrentLinkedQueue<Sentence> sentenceList = new ConcurrentLinkedQueue<Sentence>();
Timer timer = new Timer();
Sentence currentSentence = null;
JLabel count;
public HavenQueue() {
timer.schedule(new TimerTask() { public void run() { processQueue(); }}, 30000);
count = new JLabel("Haven queue: 0");
setLayout(new BorderLayout());
add(count, BorderLayout.CENTER);
count.setOpaque(false);
setOpaque(false);
count.setForeground(Color.WHITE);
}
public void processQueue() {
count.setText("Haven queue: " + sentenceList.size());
if (currentSentence == null) {
// Grab a new sentence to process.
currentSentence = sentenceList.poll();
if (currentSentence != null) {
if (!currentSentence.postHavenData()) { // Failed. Add to the end of the queue and wait a bit
submit(currentSentence);
currentSentence = null;
timer.schedule(new TimerTask() { public void run() { processQueue(); }}, 30000);
return;
}
timer.schedule(new TimerTask() { public void run() { processQueue(); }}, 5000);
return;
}
timer.schedule(new TimerTask() { public void run() { processQueue(); }}, 5000);
return;
}
if (currentSentence != null) {
currentSentence.processPendingHaven();
int status = currentSentence.getHavenStatus();
switch (status) {
case 0: // Um... not running...?
currentSentence = null;
timer.schedule(new TimerTask() { public void run() { processQueue(); }}, 30000);
return;
case 1: // Still processing...
timer.schedule(new TimerTask() { public void run() { processQueue(); }}, 5000);
return;
case 2: // Finished
currentSentence = null;
timer.schedule(new TimerTask() { public void run() { processQueue(); }}, 30000);
return;
case 3: // Failed
currentSentence = null;
timer.schedule(new TimerTask() { public void run() { processQueue(); }}, 30000);
return;
}
}
timer.schedule(new TimerTask() { public void run() { processQueue(); }}, 30000);
}
public void submit(Sentence s) {
s.setOverrideText("[queued...]");
AudiobookRecorder.window.bookTreeModel.reload(s);
sentenceList.add(s);
count.setText("Haven queue: " + sentenceList.size());
}
}
src/uk/co/majenko/audiobookrecorder/Options.java
+27 -1
View File
@@ -34,6 +34,8 @@ public class Options extends JDialog {
JCheckBox enableParsing;
JSpinner cacheSize;
JTextField havenApiKey;
Equaliser equaliser;
JTextArea startupScript;
@@ -98,6 +100,27 @@ public class Options extends JDialog {
return o;
}
JTextField addTextField(JPanel panel, String label, String def) {
JLabel l = new JLabel(label);
constraint.gridx = 0;
constraint.gridwidth = 1;
constraint.gridheight = 1;
constraint.anchor = GridBagConstraints.LINE_START;
panel.add(l, constraint);
JTextField a = new JTextField(def);
constraint.gridx = 1;
constraint.fill = GridBagConstraints.HORIZONTAL;
panel.add(a, constraint);
constraint.fill = GridBagConstraints.NONE;
constraint.gridy++;
return a;
}
JTextField addFilePath(JPanel panel, String label, String path, boolean dironly) {
JLabel l = new JLabel(label);
constraint.gridx = 0;
@@ -276,7 +299,8 @@ public class Options extends JDialog {
addSeparator(optionsPanel);
enableParsing = addCheckBox(optionsPanel, "Enable automatic sphinx speech-to-text (**SLOW**)", getBoolean("process.sphinx"));
enableParsing = addCheckBox(optionsPanel, "Enable automatic speech-to-text submission", getBoolean("process.haven.auto"));
havenApiKey = addTextField(optionsPanel, "Haven OnDemand API Key", get("process.haven.apikey"));
addSeparator(optionsPanel);
@@ -463,6 +487,7 @@ public class Options extends JDialog {
defaultPrefs.put("audio.export.samplerate", "44100");
defaultPrefs.put("process.sphinx", "false");
defaultPrefs.put("process.haven.apikey", "");
defaultPrefs.put("cache.size", "100");
@@ -603,6 +628,7 @@ public class Options extends JDialog {
set("audio.export.bitrate", ((KVPair)bitRate.getSelectedItem()).key);
set("audio.export.samplerate", ((KVPair)exportRate.getSelectedItem()).key);
set("process.sphinx", enableParsing.isSelected());
set("process.haven.apikey", havenApiKey.getText());
set("cache.size", cacheSize.getValue());
for (int i = 0; i < 31; i++) {
src/uk/co/majenko/audiobookrecorder/Sentence.java
+162 -69
View File
@@ -9,12 +9,25 @@ import java.io.*;
import java.nio.file.*;
import javax.swing.tree.*;
import javax.sound.sampled.*;
import edu.cmu.sphinx.api.*;
import edu.cmu.sphinx.decoder.adaptation.*;
import edu.cmu.sphinx.result.*;
import davaguine.jeq.spi.EqualizerInputStream;
import davaguine.jeq.core.IIRControls;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.entity.mime.content.FileBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.json.*;
import java.util.Timer;
public class Sentence extends DefaultMutableTreeNode implements Cacheable {
String text;
@@ -34,6 +47,19 @@ public class Sentence extends DefaultMutableTreeNode implements Cacheable {
boolean inSample;
boolean attention = false;
String havenJobId = "";
// 0: Not processed
// 1: Submitted
// 2: Procesisng finished
// 3: Processing failed
int havenStatus = 0;
String overrideText = null;
public void setOverrideText(String s) { overrideText = s; }
public String getOverrideText() { return overrideText; }
TargetDataLine line;
AudioInputStream inputStream;
AudioFormat storedFormat = null;
@@ -160,7 +186,7 @@ public class Sentence extends DefaultMutableTreeNode implements Cacheable {
} else if (tm.equals("fft")) {
autoTrimSampleFFT();
}
if (Options.getBoolean("process.sphinx")) {
if (Options.getBoolean("process.haven.auto")) {
recognise();
}
}
@@ -330,6 +356,7 @@ public class Sentence extends DefaultMutableTreeNode implements Cacheable {
}
public void setText(String t) {
overrideText = null;
text = t;
}
@@ -575,72 +602,8 @@ public class Sentence extends DefaultMutableTreeNode implements Cacheable {
return null;
}
public void doRecognition(StreamSpeechRecognizer recognizer) {
try {
setText("[recognising...]");
AudiobookRecorder.window.bookTreeModel.reload(this);
AudioFormat format = getAudioFormat();
byte[] inData = getRawAudioData();
int inLength = inData.length;
int bps = format.getFrameSize();
int inSamples = inLength / bps;
int outSamples = inSamples / 4;
byte[] outData = new byte[outSamples * bps];
for (int i = 0; i < outSamples; i++) {
for (int j = 0; j < bps; j++) {
outData[i * bps + j] = inData[(i * 4) * bps + j];
}
}
ByteArrayInputStream bas = new ByteArrayInputStream(outData);
recognizer.startRecognition(bas);
SpeechResult result;
String res = "";
while ((result = recognizer.getResult()) != null) {
res += result.getHypothesis();
res += " ";
}
recognizer.stopRecognition();
text = res;
AudiobookRecorder.window.bookTreeModel.reload(Sentence.this);
} catch (Exception e) {
e.printStackTrace();
}
}
public void recognise() {
Thread t = new Thread(new Runnable() {
public void run() {
try {
Configuration sphinxConfig = new Configuration();
sphinxConfig.setAcousticModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us");
sphinxConfig.setDictionaryPath("resource:/edu/cmu/sphinx/models/en-us/cmudict-en-us.dict");
sphinxConfig.setLanguageModelPath("resource:/edu/cmu/sphinx/models/en-us/en-us.lm.bin");
AudioInputStream s = AudioSystem.getAudioInputStream(getFile());
AudioFormat format = getAudioFormat();
sphinxConfig.setSampleRate((int)(format.getSampleRate() / 4f));
StreamSpeechRecognizer recognizer;
recognizer = new StreamSpeechRecognizer(sphinxConfig);
doRecognition(recognizer);
} catch (Exception e) {
e.printStackTrace();
}
}
});
t.start();
AudiobookRecorder.window.havenQueue.submit(Sentence.this);
}
public void setLocked(boolean l) {
@@ -741,4 +704,134 @@ public class Sentence extends DefaultMutableTreeNode implements Cacheable {
public boolean getAttentionFlag() {
return attention;
}
public String getHavenJobId() {
return havenJobId;
}
public void setHavenJobId(String i) {
havenJobId = i;
}
public int getHavenStatus() {
return havenStatus;
}
public void setHavenStatus(int i) {
havenStatus = i;
}
public boolean postHavenData() {
String apiKey = Options.get("process.haven.apikey");
if (apiKey == null || apiKey.equals("")) return false;
CloseableHttpClient httpclient = HttpClients.createDefault();
setOverrideText("[submitting...]");
AudiobookRecorder.window.bookTreeModel.reload(this);
try {
HttpPost httppost = new HttpPost("https://api.havenondemand.com/1/api/async/recognizespeech/v2?apikey=" + apiKey);
FileBody bin = new FileBody(getFile());
StringBody language = new StringBody("en-GB");
HttpEntity reqEntity = MultipartEntityBuilder.create()
.addPart("language_model", language)
.addPart("file", bin)
.build();
httppost.setEntity(reqEntity);
CloseableHttpResponse response = httpclient.execute(httppost);
try {
if (response.getStatusLine().getStatusCode() != 200) {
System.err.println("Error posting data: " + response.getStatusLine().getStatusCode());
return false;
}
HttpEntity resEntity = response.getEntity();
if (resEntity != null) {
JSONObject obj = new JSONObject(EntityUtils.toString(resEntity));
havenJobId = obj.getString("jobID");
System.err.println("Submitted new Haven OnDemand job #" + havenJobId);
havenStatus = 1;
}
EntityUtils.consume(resEntity);
} catch (Exception ex) {
ex.printStackTrace();
return false;
}
} catch (Exception ex) {
ex.printStackTrace();
return false;
}
return true;
// eddec91c-6018-4dcd-bd8d-5e96b23e334c --form "language_model=en-US" --form "file=@3e67460c-f298-4e2c-a412-d375d489e1b3.wav"
}
public void processPendingHaven() {
if (havenStatus != 1) return;
String apiKey = Options.get("process.haven.apikey");
if (apiKey == null || apiKey.equals("")) return;
CloseableHttpClient httpclient = HttpClients.createDefault();
try {
HttpPost httppost = new HttpPost("https://api.havenondemand.com/1/job/status/" + havenJobId + "?apikey=" + apiKey);
HttpEntity reqEntity = MultipartEntityBuilder.create().build();
httppost.setEntity(reqEntity);
CloseableHttpResponse response = httpclient.execute(httppost);
try {
if (response.getStatusLine().getStatusCode() != 200) {
havenStatus = 3;
return;
}
HttpEntity resEntity = response.getEntity();
if (resEntity != null) {
JSONObject obj = new JSONObject(EntityUtils.toString(resEntity));
System.err.println(havenJobId + ": " + obj.getString("status"));
if (obj.getString("status").equals("finished")) {
havenStatus = 2;
JSONArray textItems = obj.getJSONArray("actions").getJSONObject(0).getJSONObject("result").getJSONArray("items");
StringBuilder out = new StringBuilder();
for (int i = 0; i < textItems.length(); i++) {
out.append(textItems.getJSONObject(i).getString("text"));
out.append(" ");
}
String result = out.toString();
setText(result.trim());
AudiobookRecorder.window.bookTreeModel.reload(Sentence.this);
System.err.println(result);
} else if (obj.getString("status").equals("queued")) {
havenStatus = 1;
setOverrideText("[processing...]");
AudiobookRecorder.window.bookTreeModel.reload(Sentence.this);
} else {
text = id;
AudiobookRecorder.window.bookTreeModel.reload(Sentence.this);
havenStatus = 3;
return;
}
}
EntityUtils.consume(resEntity);
} catch (Exception ex) {
ex.printStackTrace();
}
} catch (Exception ex) {
ex.printStackTrace();
}
}
}