package com.smartor.service.impl; import com.alibaba.nls.client.AccessToken; import com.alibaba.nls.client.protocol.InputFormatEnum; import com.alibaba.nls.client.protocol.NlsClient; import com.alibaba.nls.client.protocol.OutputFormatEnum; import com.alibaba.nls.client.protocol.SampleRateEnum; import com.alibaba.nls.client.protocol.asr.SpeechRecognizer; import com.alibaba.nls.client.protocol.asr.SpeechRecognizerListener; import com.alibaba.nls.client.protocol.asr.SpeechRecognizerResponse; import com.alibaba.nls.client.protocol.tts.SpeechSynthesizer; import com.alibaba.nls.client.protocol.tts.SpeechSynthesizerListener; import com.alibaba.nls.client.protocol.tts.SpeechSynthesizerResponse; import com.ruoyi.common.utils.StringUtils; import com.ruoyi.common.utils.uuid.IdUtils; import com.smartor.domain.*; import com.smartor.mapper.*; import com.smartor.service.PersonVoiceService; import lombok.extern.slf4j.Slf4j; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.*; import java.nio.ByteBuffer; import java.nio.file.Files; /** * 短信账号Service业务层处理 * * @author smartor * @date 2023-03-06 */ @Slf4j @Service public class PersonVoiceServiceImpl implements PersonVoiceService { @Value("${accessKeyId}") private String accessKeyId; @Value("${accessKeySecret}") private String accessKeySecret; @Autowired private SvyLibTemplateWjxMapper svyLibTitleMapper; @Autowired private SvyLibTemplateScriptWjxMapper svyLibTopicMapper; @Autowired private SvyLibTopicdirectionMapper svyLibTopicdirectionMapper; @Autowired private SvyLibTemplateTargetoptionWjxMapper svyLibTopicoptionMapper; // // @Autowired // private ISvyLibTitleService svyLibTitleService; private NlsClient client; private String appKey = "ZurNHpaQLq6P55YS"; private String returnResult = null; String url = System.getenv().getOrDefault("NLS_GATEWAY_URL", "wss://nls-gateway-cn-shanghai.aliyuncs.com/ws/v1"); @Override public String speechtotext(String filePath) { this.accessToken(); this.process(filePath, 16000); client.shutdown(); return returnResult; } @Override public String texttospeech(String textspeech, String filePath) { this.accessToken(); this.process2(textspeech, filePath); client.shutdown(); return returnResult; } @Override public Boolean explainHTML() { try { explainHTML2(); } catch (IOException e) { e.printStackTrace(); } return true; } public String explainHTML2() throws IOException { String folderPath = "E:\\pc2"; // 获取文件夹下的所有文件 File[] files = new File(folderPath).listFiles(); // 遍历文件夹下的所有文件 a: for (File file : files) { // 获取文件名 String fileName = file.getName(); System.out.println(fileName); // 获取文件内容 String fileContent = new String(Files.readAllBytes(file.toPath())); Document parse = Jsoup.parse(fileContent); Element body = parse.body(); Elements elementsByClass3 = body.getElementsByClass("topic__type-des"); String desc = elementsByClass3.text(); //保存title String title = parse.title(); SvyLibTemplateWjx svyLibTitle = new SvyLibTemplateWjx(); // svyLibTitle.setCategoryid(System.currentTimeMillis()); svyLibTitle.setSvyname(title); svyLibTitle.setDescription(desc); svyLibTitle.setDelFlag("0"); svyLibTitle.setIsupload(0L); svyLibTitleMapper.insertSvyLibTemplateWjx(svyLibTitle); Elements elementsByClass = body.getElementsByClass("topic__type-body"); Elements children = elementsByClass.get(0).children(); for (Element element : children) { int legend = element.select("legend").size(); if (legend == 0) { //跳过该文件 noExpiain(fileName.substring(0, fileName.length() - 5), fileContent); continue a; } if (!element.getElementsByTag("legend").attr("class").equals("topic__type-title")) { //跳过该文件 noExpiain(fileName.substring(0, fileName.length() - 5), fileContent); continue a; } } for (Element element : children) { //获取题目 String legend = element.getElementsByTag("legend").text(); System.out.println("题目： " + legend); //将题目保存到表中 SvyLibTemplateScriptWjx svyLibTopic = new SvyLibTemplateScriptWjx(); svyLibTopic.setSvyid(svyLibTitle.getSvyid()); //判断该题目下是否有选项 Elements elementsByClass1 = element.getElementsByClass("topic__type-dry"); if (elementsByClass1.size() != 0) { Elements spans = elementsByClass1.get(0).getElementsByTag("span"); if (spans.size() != 0) { Elements radio__type = spans.get(0).getElementsByClass("radio__type"); if (radio__type.size() != 0) { svyLibTopic.setScriptType("1"); } else { Elements checkbox__type = spans.get(0).getElementsByClass("checkbox__type"); if (checkbox__type.size() != 0) { svyLibTopic.setScriptType("2"); } } } svyLibTopic.setSeqno(IdUtils.simpleUUID()); svyLibTopic.setScriptContent(legend); svyLibTopic.setDelFlag("0"); svyLibTopic.setIsupload(0L); svyLibTopic.setOrgid(fileName); try { svyLibTopicMapper.insertSvyLibTemplateScriptWjx(svyLibTopic); } catch (Exception e) { noExpiain(fileName.substring(0, fileName.length() - 5) + "出异常了1", fileContent); continue a; } String item = ""; Long i = 1L; for (Element span : spans) { Elements labels = span.getElementsByTag("label"); for (Element label : labels) { String text = label.text(); //将题目存库 // item = item + "------" + text; SvyLibTemplateTargetoptionWjx svyLibTopicoption = new SvyLibTemplateTargetoptionWjx(); svyLibTopicoption.setScriptid(svyLibTopic.getId()); svyLibTopicoption.setTemplateID(svyLibTitle.getSvyid()); svyLibTopicoption.setSort(i); svyLibTopicoption.setGuid(IdUtils.simpleUUID()); svyLibTopicoption.setOptioncontent(text); svyLibTopicoption.setDelFlag("0"); svyLibTopicoption.setIsupload(0L); try { svyLibTopicoptionMapper.insertSvyLibTemplateTargetoptionWjx(svyLibTopicoption); } catch (Exception e) { noExpiain(fileName.substring(0, fileName.length() - 5) + "出异常了2", fileContent); continue a; } i++; } } System.out.println("选项： " + item); } else { //将题目保存到表中 Elements elementsByClass2 = element.getElementsByClass("ui-control-group ui-matrix"); if (elementsByClass2.size() > 0) { SvyLibTemplateScriptWjx svyLibTopic2 = new SvyLibTemplateScriptWjx(); svyLibTopic2.setSvyid(svyLibTitle.getSvyid()); svyLibTopic2.setSeqno(IdUtils.simpleUUID()); svyLibTopic2.setScriptContent(legend); svyLibTopic2.setDelFlag("0"); svyLibTopic.setIsupload(0L); svyLibTopic2.setScriptType("4"); svyLibTopic2.setOrgid(fileName); try { svyLibTopicMapper.insertSvyLibTemplateScriptWjx(svyLibTopic2); } catch (Exception e) { noExpiain(fileName.substring(0, fileName.length() - 5) + "出异常了22", fileContent); continue a; } Elements lis = elementsByClass2.get(0).getElementsByTag("li"); String transverse = ""; for (Element element1 : lis) { transverse = transverse + element1.text() + "☆"; } Elements title1 = elementsByClass2.get(0).getElementsByClass("title"); String direction = ""; for (Element ele : title1) { direction = direction + ele.text() + "☆"; } SvyLibTopicdirection svyLibTopicdirection = new SvyLibTopicdirection(); svyLibTopicdirection.setTopicid(svyLibTopic2.getId()); svyLibTopicdirection.setDirection(direction); svyLibTopicdirection.setTransverse(transverse); svyLibTopicdirectionMapper.insertSvyLibTopicdirection(svyLibTopicdirection); } else { //将题目保存到表中 SvyLibTemplateScriptWjx svyLibTopic3 = new SvyLibTemplateScriptWjx(); svyLibTopic3.setSvyid(svyLibTitle.getSvyid()); svyLibTopic3.setSeqno(IdUtils.simpleUUID()); svyLibTopic3.setScriptContent(legend); svyLibTopic3.setDelFlag("0"); svyLibTopic3.setIsupload(0L); svyLibTopic3.setScriptType("3"); svyLibTopic3.setOrgid(fileName); try { svyLibTopicMapper.insertSvyLibTemplateScriptWjx(svyLibTopic3); } catch (Exception e) { noExpiain(fileName.substring(0, fileName.length() - 5) + "出异常了33", fileContent); continue a; } } } } } return null; } private void noExpiain(String filename, String content) throws IOException { File file = new File("E:\\noexplain\\" + filename + ".html"); FileWriter fileWriter = new FileWriter(file); fileWriter.write(content); } public SpeechRecognizerListener getRecognizerListener(int myOrder, String userParam) { SpeechRecognizerListener speechRecognizerListener = new SpeechRecognizerListener() { @Override public void onRecognitionResultChanged(SpeechRecognizerResponse response) { //getName是获取事件名称，getStatus是获取状态码，getRecognizedText是语音识别文本。 System.out.println("name: " + response.getName() + ", status: " + response.getStatus() + ", result: " + response.getRecognizedText()); } //识别完毕 @Override public void onRecognitionCompleted(SpeechRecognizerResponse response) { //getName是获取事件名称，getStatus是获取状态码，getRecognizedText是语音识别文本。 returnResult = response.getRecognizedText(); System.out.println("name: " + response.getName() + ", status: " + response.getStatus() + ", result: " + response.getRecognizedText()); } @Override public void onStarted(SpeechRecognizerResponse response) { System.out.println("myOrder: " + myOrder + "; myParam: " + userParam + "; task_id: " + response.getTaskId()); } @Override public void onFail(SpeechRecognizerResponse response) { //task_id是调用方和服务端通信的唯一标识，当遇到问题时，需要提供此task_id。 System.out.println("task_id: " + response.getTaskId() + ", status: " + response.getStatus() + ", status_text: " + response.getStatusText()); } }; return speechRecognizerListener; } public void accessToken() { //应用全局创建一个NlsClient实例，默认服务地址为阿里云线上服务地址。 //获取Token，实际使用时注意在accessToken.getExpireTime()过期前再次获取。 AccessToken accessToken = new AccessToken(accessKeyId, accessKeySecret); try { accessToken.apply(); System.out.println("get token: " + accessToken.getToken() + ", expire time: " + accessToken.getExpireTime()); if (url.isEmpty()) { client = new NlsClient(accessToken.getToken()); } else { client = new NlsClient(url, accessToken.getToken()); } } catch (IOException e) { e.printStackTrace(); } } public void process(String filepath, int sampleRate) { SpeechRecognizer recognizer = null; try { //传递用户自定义参数 String myParam = "user-param"; int myOrder = 1234; SpeechRecognizerListener listener = getRecognizerListener(myOrder, myParam); recognizer = new SpeechRecognizer(client, listener); recognizer.setAppKey(appKey); //设置音频编码格式。如果是OPUS文件，请设置为InputFormatEnum.OPUS。 recognizer.setFormat(InputFormatEnum.PCM); //设置音频采样率 if (sampleRate == 16000) { recognizer.setSampleRate(SampleRateEnum.SAMPLE_RATE_16K); } else if (sampleRate == 8000) { recognizer.setSampleRate(SampleRateEnum.SAMPLE_RATE_8K); } //设置是否返回中间识别结果 recognizer.setEnableIntermediateResult(true); //设置是否打开语音检测（即vad） recognizer.addCustomedParam("enable_voice_detection", true); //此方法将以上参数设置序列化为JSON发送给服务端，并等待服务端确认。 long now = System.currentTimeMillis(); recognizer.start(); log.info("ASR start latency : " + (System.currentTimeMillis() - now) + " ms"); File file = new File(filepath); FileInputStream fis = new FileInputStream(file); byte[] b = new byte[3200]; int len; while ((len = fis.read(b)) > 0) { log.info("send data pack length: " + len); recognizer.send(b, len); //本案例用读取本地文件的形式模拟实时获取语音流，因为读取速度较快，这里需要设置sleep时长。 // 如果实时获取语音则无需设置sleep时长，如果是8k采样率语音第二个参数设置为8000。 int deltaSleep = getSleepDelta(len, sampleRate); // Thread.sleep(deltaSleep); } //通知服务端语音数据发送完毕，等待服务端处理完成。 now = System.currentTimeMillis(); //计算实际延迟，调用stop返回之后一般即是识别结果返回时间。 log.info("ASR wait for complete"); recognizer.stop(); log.info("ASR stop latency : " + (System.currentTimeMillis() - now) + " ms"); fis.close(); } catch (Exception e) { System.err.println(e.getMessage()); } finally { //关闭连接 if (null != recognizer) { recognizer.close(); } } } public int getSleepDelta(int dataSize, int sampleRate) { // 仅支持16位采样。 int sampleBytes = 16; // 仅支持单通道。 int soundChannel = 1; return (dataSize * 10 * 8000) / (160 * sampleRate); } private SpeechSynthesizerListener getSynthesizerListener(String filePath) { SpeechSynthesizerListener listener = null; try { if (StringUtils.isEmpty(filePath)) { filePath = "tts_test.wav"; } String finalFilePath = filePath; listener = new SpeechSynthesizerListener() { File f = new File(finalFilePath); FileOutputStream fout = new FileOutputStream(f); private boolean firstRecvBinary = true; //语音合成结束 @Override public void onComplete(SpeechSynthesizerResponse response) { //调用onComplete时表示所有TTS数据已接收完成，因此为整个合成数据的延迟。该延迟可能较大，不一定满足实时场景。 System.out.println("name: " + response.getName() + ", status: " + response.getStatus() + ", output file :" + f.getAbsolutePath()); } //语音合成的语音二进制数据 @Override public void onMessage(ByteBuffer message) { try { if (firstRecvBinary) { //计算首包语音流的延迟，收到第一包语音流时，即可以进行语音播放，以提升响应速度（特别是实时交互场景下）。 firstRecvBinary = false; long now = System.currentTimeMillis(); // logger.info("tts first latency : " + (now - SpeechSynthesizerDemo.startTime) + " ms"); } byte[] bytesArray = new byte[message.remaining()]; message.get(bytesArray, 0, bytesArray.length); fout.write(bytesArray); } catch (IOException e) { e.printStackTrace(); } } @Override public void onFail(SpeechSynthesizerResponse response) { //task_id是调用方和服务端通信的唯一标识，当遇到问题时需要提供task_id以便排查。 System.out.println("task_id: " + response.getTaskId() + //状态码 20000000 表示识别成功 ", status: " + response.getStatus() + //错误信息 ", status_text: " + response.getStatusText()); } }; } catch (Exception e) { e.printStackTrace(); } return listener; } public void process2(String text, String filePath) { SpeechSynthesizer synthesizer = null; try { //创建实例，建立连接。 synthesizer = new SpeechSynthesizer(client, getSynthesizerListener(filePath)); synthesizer.setAppKey(appKey); //设置返回音频的编码格式 synthesizer.setFormat(OutputFormatEnum.WAV); //设置返回音频的采样率 synthesizer.setSampleRate(SampleRateEnum.SAMPLE_RATE_16K); //发音人 synthesizer.setVoice("siyue"); //语调，范围是-500~500，可选，默认是0。 synthesizer.setPitchRate(100); //语速，范围是-500~500，默认是0。 synthesizer.setSpeechRate(100); //设置用于语音合成的文本 synthesizer.setText(text); // 是否开启字幕功能（返回相应文本的时间戳），默认不开启，需要注意并非所有发音人都支持该参数。 synthesizer.addCustomedParam("enable_subtitle", false); //此方法将以上参数设置序列化为JSON格式发送给服务端，并等待服务端确认。 long start = System.currentTimeMillis(); synthesizer.start(); // logger.info("tts start latency " + (System.currentTimeMillis() - start) + " ms"); // SpeechSynthesizerDemo.startTime = System.currentTimeMillis(); //等待语音合成结束 synthesizer.waitForComplete(); log.info("tts stop latency " + (System.currentTimeMillis() - start) + " ms"); } catch (Exception e) { e.printStackTrace(); } finally { //关闭连接 if (null != synthesizer) { synthesizer.close(); } } } }