From ac49141b47ca6880c9aaa6bbc062ddab5a29e589 Mon Sep 17 00:00:00 2001 From: yangyakun Date: Sun, 27 Apr 2025 15:51:22 +0800 Subject: [PATCH] =?UTF-8?q?[8.0.0]=20[fea]=20[=E7=A7=91=E5=A4=A7=E8=AE=AF?= =?UTF-8?q?=E9=A3=9E=20=E7=A6=BB=E7=BA=BF=E5=94=A4=E9=86=92=20+=20?= =?UTF-8?q?=E7=99=BE=E5=BA=A6=E5=AF=B9=E8=AF=9D]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../passenger/ui/aiview/AIViewModel.kt | 11 + .../passenger/ui/setting/VerticalSeekBar.kt | 2 + .../passenger/ui/statusview/StatusBarView.kt | 10 + .../main/res/layout/taxt_u_p_statusview.xml | 1 + libraries/mogo-speech/build.gradle | 1 + .../speech/iflytek/WakeManager.kt | 297 ++++++++++++++++++ .../speech/iflytek/WakeUpListener.kt | 5 + 7 files changed, 327 insertions(+) create mode 100644 libraries/mogo-speech/src/main/java/com/mogo/mgintelligent/speech/iflytek/WakeUpListener.kt diff --git a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AIViewModel.kt b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AIViewModel.kt index 8d919a61c5..8c26b92290 100644 --- a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AIViewModel.kt +++ b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AIViewModel.kt @@ -11,6 +11,8 @@ import com.mogo.mgintelligent.speech.AsrResult import com.mogo.mgintelligent.speech.AsrState import com.mogo.mgintelligent.speech.IWakeUpListener import com.mogo.mgintelligent.speech.MGSpeech +import com.mogo.mgintelligent.speech.iflytek.WakeManager +import com.mogo.mgintelligent.speech.iflytek.WakeUpListener import com.mogo.och.bridge.autopilot.location.OchLocationManager import com.mogo.och.data.taxi.BaseOrderBean import com.mogo.och.data.taxi.TaxiOrderStatusEnum @@ -81,6 +83,13 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList } } + private val wakeUpListener = object :WakeUpListener{ + override fun wakeupSuccess() { + onWakeUp() + } + + } + init { } @@ -89,6 +98,7 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList OrderModel.setOrderStatusCallback(TAG,orderListener) OchLocationManager.addGCJ02Listener(TAG,1,locationCallback) AIMessageManager.registerListener(this) + WakeManager.setWakeUpListener(wakeUpListener) mgSpeech.weakUpListener = this mgSpeech.startWeakUp() @@ -123,6 +133,7 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList override fun onCleared() { AIMessageManager.unregisterListener(this) mgSpeech.weakUpListener = null + WakeManager.setWakeUpListener(null) mgSpeech.stopWeakUp() llmResultJob?.cancel() OrderModel.setOrderStatusCallback(TAG,null) diff --git a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/setting/VerticalSeekBar.kt b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/setting/VerticalSeekBar.kt index eaa8246fbe..04b2b5d5cf 100644 --- a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/setting/VerticalSeekBar.kt +++ b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/setting/VerticalSeekBar.kt @@ -11,7 +11,9 @@ import androidx.constraintlayout.widget.ConstraintLayout import androidx.constraintlayout.widget.ConstraintSet import androidx.core.view.marginBottom import androidx.core.view.marginTop +import com.mogo.eagle.core.utilcode.kotlin.onClick import com.mogo.eagle.core.utilcode.mogo.logger.CallerLogger +import com.mogo.mgintelligent.speech.iflytek.WakeManager import com.mogo.och.common.module.utils.NumberFormatUtil import com.mogo.och.unmanned.taxi.passenger.R import kotlinx.android.synthetic.main.taxi_p_setting_light_view.view.tv_current_value diff --git a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/statusview/StatusBarView.kt b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/statusview/StatusBarView.kt index 5c6f33de8c..d4960b3e42 100644 --- a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/statusview/StatusBarView.kt +++ b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/statusview/StatusBarView.kt @@ -11,14 +11,18 @@ import com.mogo.eagle.core.function.call.devatools.CallerDevaToolsManager import com.mogo.eagle.core.function.call.hmi.CallerHmiViewControlListenerManager import com.mogo.eagle.core.function.call.setting.CallerSkinModeListenerManager import com.mogo.eagle.core.function.hmi.ui.setting.ToggleDebugView +import com.mogo.eagle.core.utilcode.kotlin.onClick import com.mogo.eagle.core.utilcode.util.ActivityUtils +import com.mogo.mgintelligent.speech.iflytek.WakeManager import com.mogo.och.bridge.autopilot.autopilot.IOchAutopilotStatusListener import com.mogo.och.bridge.autopilot.autopilot.OchAutoPilotStatusListenerManager import com.mogo.och.common.module.manager.debug.DebugViewWatchDogFragment import com.mogo.och.unmanned.passenger.ui.debug.DebugEvent import com.mogo.och.unmanned.taxi.passenger.R +import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.bcsv_status import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.vShowDebugView import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.iv_biz_icon +import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.textClockDate import me.jessyan.autosize.utils.AutoSizeUtils import org.greenrobot.eventbus.EventBus import java.lang.ref.WeakReference @@ -58,6 +62,12 @@ class StatusBarView @JvmOverloads constructor( LayoutInflater.from(context).inflate(R.layout.taxt_u_p_statusview, this, true) isClickable = true isFocusable = true + textClockDate.onClick { + WakeManager.startWakeUp() + } + bcsv_status.onClick { + WakeManager.stopWakeup() + } } override fun onAttachedToWindow() { diff --git a/OCH/taxi/unmanned-passenger/src/main/res/layout/taxt_u_p_statusview.xml b/OCH/taxi/unmanned-passenger/src/main/res/layout/taxt_u_p_statusview.xml index 12e6db3744..a42f2af9e3 100644 --- a/OCH/taxi/unmanned-passenger/src/main/res/layout/taxt_u_p_statusview.xml +++ b/OCH/taxi/unmanned-passenger/src/main/res/layout/taxt_u_p_statusview.xml @@ -61,6 +61,7 @@ Log.i(TAG, "core listener code:$code") @@ -39,6 +79,259 @@ object WakeManager { } + fun setWakeUpListener(wakeUpListener:WakeUpListener?){ + this._wakeUpListener = wakeUpListener + } + + fun startWakeUp(){ + ThreadUtils.getIoPool().submit { + // 检测权限 + val granted = ActivityCompat.checkSelfPermission(AbsMogoApplication.getApp(), + Manifest.permission.RECORD_AUDIO) + if(granted != PackageManager. PERMISSION_GRANTED){ + CallerRequestActivityHandleManager.requestPermission(TAG,Manifest.permission.RECORD_AUDIO) + return@submit + } + AiHelper.getInst().registerListener(ABILITYID, edListener) + CallerLogger.d(TAG,"START") + val ret = start() + if (ret == 0) { + createAudioRecord() //创建录音器 + isRecording.set(true) + audioRecord?.startRecording() //录音器启动录音 + + writeRecording(AiStatus.BEGIN) + } + } + } + + fun stopWakeup(){ + if(!isEnd.get()) { + writeRecording(AiStatus.END) + } + } + + /** + * 能力监听回调 + */ + private val edListener: AiListener = object : AiListener { + override fun onResult(handleID: Int, outputData: List, usrContext: Any?) { + if (null != outputData && outputData.size > 0) { + for (i in outputData.indices) { + CallerLogger.d(TAG, "onResult:handleID:" + handleID + ":" + outputData[i].key) + val key = outputData[i].key //引擎结果的key + val bytes = outputData[i].value //识别结果 + val result = String(bytes) + CallerLogger.d(TAG, "key=$key") + CallerLogger.d(TAG, "value=$result") + CallerLogger.d(TAG, "status=" + outputData[i].status) + if ((key == "func_wake_up" || key == "func_pre_wakeup")) { + CallerLogger.d(TAG,"$key: \n $result") + } + _wakeUpListener?.wakeupSuccess() + } + } + } + + override fun onEvent(i: Int, i1: Int, list: List, o: Any) { + CallerLogger.i(TAG, "onEvent:$i,event:$i1") + } + + override fun onError(i: Int, i1: Int, s: String, o: Any) { + CallerLogger.d(TAG,"错误通知，能力执行终止,Ability $i ERROR::$s,err code:$i1") + } + } + + private fun writeRecording(aiStatus:AiStatus){ + + /** + * 写入数据-录音方式 + */ + val data = ByteArray(BUFFER_SIZE) + val read = audioRecord!!.read(data, 0, BUFFER_SIZE) + + //平方和除以数据总长度，得到音量大小。 + val volume = calculateVolume(data) +// CallerLogger.d(TAG,"当前分贝:${ abs(volume.toDouble())}") + if (AudioRecord.ERROR_INVALID_OPERATION != read) { + //处理录音数据 + write(data, aiStatus) //送尾帧 + } + if (AiStatus.END == aiStatus) { + if ( audioRecord != null) { + audioRecord!!.stop() + isRecording.set(false) + end() + } + } else { + if (isRecording.get()) { + writeRecording(AiStatus.CONTINUE) //调用write方法送音频数据给引擎 + } + } + } + + /** + * 结束会话 + */ + private fun end() { + if (!isEnd.get()) { + val ret = AiHelper.getInst().end(aiHandle) + if (ret == 0) { + isEnd.set(true) + aiHandle = null + CallerLogger.d(TAG,"唤醒完成，end： $ret") + } else { + isEnd.set(false) + ThreadUtils.runOnUiThreadDelayed({ + end() + CallerLogger.d(TAG,"关闭失败2s后重新广播唤醒完成，end： $ret") + },2_000) + CallerLogger.d(TAG,"唤醒完成，end： $ret") + } + } + } + + /** + * 写入数据 + */ + private fun write(part: ByteArray, status: AiStatus) { + if (isEnd.get()) { + return + } + val dataBuilder = AiRequest.builder() + var ret = 0 + + /** + * 送入音频需要标识音频的状态，第一帧为起始帧，status要传AiStatus.BEGIN,最后一帧为结束帧，status要传AiStatus.END,其他为中间帧，status要传AiStatus.CONTINUE + * 音频要求16bit，16K，单声道的pcm音频。 + * 建议每次发送音频间隔40ms，每次发送音频字节数为一帧音频大小的整数倍。 + */ + val aiAudio = AiAudio.get("wav").data(part).status(status).valid() + dataBuilder.payload(aiAudio) + + ret = AiHelper.getInst().write(dataBuilder.build(), aiHandle) + if (ret != 0) { + CallerLogger.d(TAG,"write失败：$ret ") + } + } + + /** + * 根据录音数据计算音量 + */ + private fun calculateVolume(buffer: ByteArray): Int { + var sumVolume = 0.0 + var avgVolume = 0.0 + var volume = 0 + var i = 0 + while (i < buffer.size) { + val v1 = buffer[i].toInt() and 0xFF + val v2 = buffer[i + 1].toInt() and 0xFF + var temp = v1 + (v2 shl 8) // 小端 + if (temp >= 0x8000) { + temp = 0xffff - temp + } + sumVolume += abs(temp.toDouble()) + i += 2 + } + avgVolume = sumVolume / buffer.size / 2 + volume = (log10(1 + avgVolume) * 10).toInt() + return volume + } + + /** + * 创建录音器 + */ + @SuppressLint("MissingPermission") + private fun createAudioRecord() { + if (isRecording.get()) { + return + } + if (audioRecord == null) { + Log.d(TAG, "createAudioRecord") + audioRecord = AudioRecord( + MediaRecorder.AudioSource.MIC, + 16000, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + BUFFER_SIZE + ) + } + } + + /** + * 开始会话 + */ + private fun start(): Int { + if (!keyword2File()) { + CallerLogger.d(TAG,"唤醒词文件写入失败，请检查是否有读写权限。") + return -1 + } + val customBuilder = AiRequest.builder() + customBuilder.customText("key_word", getIvwDir() + "/keyword.txt", 0) + var ret = AiHelper.getInst().loadData(ABILITYID, customBuilder.build()) + if (ret != 0) { + CallerLogger.d(TAG,"open ivw loadData 失败：$ret") + return ret + } + CallerLogger.d(TAG,"open ivw loadData success：$ret") + val indexs = intArrayOf(0) + ret = AiHelper.getInst().specifyDataSet(ABILITYID, "key_word", indexs) //从缓存中把个性化资源设置到引擎中 + if (ret != 0) { + CallerLogger.d(TAG,"open ivw specifyDataSet 失败：$ret") + return ret + } + CallerLogger.d(TAG,"open ivw specifyDataSet success：$ret") + + val paramBuilder = AiRequest.builder() + paramBuilder.param("wdec_param_nCmThreshold", "0 0:800") + paramBuilder.param("gramLoad", true) + isEnd.set(false) + aiHandle = AiHelper.getInst().start(ABILITYID, paramBuilder.build(), null) + if (aiHandle?.code != 0) { + CallerLogger.d(TAG,"open ivw start失败：${aiHandle?.getCode()}") + return aiHandle?.code ?:0 + } + return 0 + } + + private fun keyword2File(): Boolean { + try { + val keywordFile: File = File(getIvwDir() + "/keyword.txt") + if (keywordFile.exists()) { + //强制清空内容 + keywordFile.delete() + } + val binFile: File = File(getIvwDir() + "/keyword.bin") + if (binFile.exists()) { + binFile.delete() + } + var temp: String =weakupKey + if (temp.isEmpty()) { + temp = "你好小智" + } + val str = temp.replace("，", ",") + val keywords = str.split(",".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + if (!keywordFile.exists()) { + keywordFile.createNewFile() + } + val writer = OutputStreamWriter( + FileOutputStream(keywordFile), + "UTF-8" + ) + val bufferedWriter = BufferedWriter(writer) + for (i in keywords.indices) { + bufferedWriter.write(keywords[i]) + bufferedWriter.write(";") + bufferedWriter.newLine() //写入换行 + } + bufferedWriter.close() + } catch (e: IOException) { + e.printStackTrace() + return false + } + return true + } + fun initWakeAi(context: Context) { try { @@ -100,4 +393,8 @@ object WakeManager { } throw IllegalArgumentException("找不到 Application") } + + fun getIvwDir(): String { + return getWorkDir()+File.separator+"ivw" + } } \ No newline at end of file diff --git a/libraries/mogo-speech/src/main/java/com/mogo/mgintelligent/speech/iflytek/WakeUpListener.kt b/libraries/mogo-speech/src/main/java/com/mogo/mgintelligent/speech/iflytek/WakeUpListener.kt new file mode 100644 index 0000000000..b48cda94aa --- /dev/null +++ b/libraries/mogo-speech/src/main/java/com/mogo/mgintelligent/speech/iflytek/WakeUpListener.kt @@ -0,0 +1,5 @@ +package com.mogo.mgintelligent.speech.iflytek + +interface WakeUpListener { + fun wakeupSuccess() +} \ No newline at end of file