diff --git a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/TaxiUnmannedPassengerProvider.kt b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/TaxiUnmannedPassengerProvider.kt index 6bb7c536ba..8cbc82d45a 100644 --- a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/TaxiUnmannedPassengerProvider.kt +++ b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/TaxiUnmannedPassengerProvider.kt @@ -7,6 +7,7 @@ import com.alibaba.android.arouter.facade.annotation.Route import com.mogo.eagle.core.utilcode.mogo.logger.CallerLogger.d import com.mogo.eagle.core.utilcode.mogo.logger.scene.SceneConstant import com.mogo.mgintelligent.speech.MGSpeech +import com.mogo.mgintelligent.speech.iflytek.WakeManager import com.mogo.och.common.module.constant.OchCommonConst import com.mogo.och.common.module.biz.provider.CommonServiceImpl import com.mogo.och.bridge.autopilot.autopilot.OchAutopilotAnalytics @@ -32,6 +33,7 @@ class TaxiUnmannedPassengerProvider : CommonServiceImpl() { d(SceneConstant.M_TAXI_P + tag, "init") statusBarView = StatusBarView(context) MGSpeech.init(context) + WakeManager.initWakeAi(context) } override fun getStatusBarView(context: Context): View { diff --git a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AIViewModel.kt b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AIViewModel.kt index 8d919a61c5..ee71cbe84c 100644 --- a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AIViewModel.kt +++ b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AIViewModel.kt @@ -11,6 +11,8 @@ import com.mogo.mgintelligent.speech.AsrResult import com.mogo.mgintelligent.speech.AsrState import com.mogo.mgintelligent.speech.IWakeUpListener import com.mogo.mgintelligent.speech.MGSpeech +import com.mogo.mgintelligent.speech.iflytek.WakeManager +import com.mogo.mgintelligent.speech.iflytek.WakeUpListener import com.mogo.och.bridge.autopilot.location.OchLocationManager import com.mogo.och.data.taxi.BaseOrderBean import com.mogo.och.data.taxi.TaxiOrderStatusEnum @@ -81,6 +83,14 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList } } + private val wakeUpListener = object :WakeUpListener{ + override fun wakeupSuccess() { + onWakeUp() + WakeManager.stopWakeup() + } + + } + init { } @@ -89,6 +99,7 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList OrderModel.setOrderStatusCallback(TAG,orderListener) OchLocationManager.addGCJ02Listener(TAG,1,locationCallback) AIMessageManager.registerListener(this) + WakeManager.setWakeUpListener(wakeUpListener) mgSpeech.weakUpListener = this mgSpeech.startWeakUp() @@ -123,6 +134,7 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList override fun onCleared() { AIMessageManager.unregisterListener(this) mgSpeech.weakUpListener = null + WakeManager.setWakeUpListener(null) mgSpeech.stopWeakUp() llmResultJob?.cancel() OrderModel.setOrderStatusCallback(TAG,null) @@ -228,6 +240,7 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList } AsrState.STATE_EXIT -> { + WakeManager.startWakeUp() _asrUIStateFlow.value = AsrUIState.Idle mgSpeech.speak(tipsExit) mgSpeech.isAssistantShow(false) diff --git a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AiView.kt b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AiView.kt index 708fbdf71a..9f7483d449 100644 --- a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AiView.kt +++ b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AiView.kt @@ -23,6 +23,7 @@ import com.mogo.eagle.core.function.call.autopilot.CallerPlanningActionsListener import com.mogo.eagle.core.utilcode.kotlin.onClick import com.mogo.eagle.core.utilcode.mogo.logger.CallerLogger import com.mogo.eagle.core.utilcode.util.UiThreadHandler +import com.mogo.mgintelligent.speech.iflytek.WakeManager import com.mogo.och.common.module.manager.logchainanalytic.OchChainLogManager import com.mogo.och.common.module.utils.BigFrameAnimatorContainer import com.mogo.och.common.module.utils.RxUtils @@ -168,6 +169,11 @@ class AiView @JvmOverloads constructor( super.onVisibilityAggregated(isVisible) CallerLogger.d(TAG,"是否展示中：${isVisible}") try { + if(isVisible){ + WakeManager.startWakeUp() + }else{ + WakeManager.stopWakeup() + } if(isVisible){ aiAnimator?.start() RxUtils.createSubscribe(3_000) { diff --git a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/setting/VerticalSeekBar.kt b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/setting/VerticalSeekBar.kt index eaa8246fbe..04b2b5d5cf 100644 --- a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/setting/VerticalSeekBar.kt +++ b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/setting/VerticalSeekBar.kt @@ -11,7 +11,9 @@ import androidx.constraintlayout.widget.ConstraintLayout import androidx.constraintlayout.widget.ConstraintSet import androidx.core.view.marginBottom import androidx.core.view.marginTop +import com.mogo.eagle.core.utilcode.kotlin.onClick import com.mogo.eagle.core.utilcode.mogo.logger.CallerLogger +import com.mogo.mgintelligent.speech.iflytek.WakeManager import com.mogo.och.common.module.utils.NumberFormatUtil import com.mogo.och.unmanned.taxi.passenger.R import kotlinx.android.synthetic.main.taxi_p_setting_light_view.view.tv_current_value diff --git a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/statusview/StatusBarView.kt b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/statusview/StatusBarView.kt index 5c6f33de8c..d4960b3e42 100644 --- a/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/statusview/StatusBarView.kt +++ b/OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/statusview/StatusBarView.kt @@ -11,14 +11,18 @@ import com.mogo.eagle.core.function.call.devatools.CallerDevaToolsManager import com.mogo.eagle.core.function.call.hmi.CallerHmiViewControlListenerManager import com.mogo.eagle.core.function.call.setting.CallerSkinModeListenerManager import com.mogo.eagle.core.function.hmi.ui.setting.ToggleDebugView +import com.mogo.eagle.core.utilcode.kotlin.onClick import com.mogo.eagle.core.utilcode.util.ActivityUtils +import com.mogo.mgintelligent.speech.iflytek.WakeManager import com.mogo.och.bridge.autopilot.autopilot.IOchAutopilotStatusListener import com.mogo.och.bridge.autopilot.autopilot.OchAutoPilotStatusListenerManager import com.mogo.och.common.module.manager.debug.DebugViewWatchDogFragment import com.mogo.och.unmanned.passenger.ui.debug.DebugEvent import com.mogo.och.unmanned.taxi.passenger.R +import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.bcsv_status import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.vShowDebugView import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.iv_biz_icon +import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.textClockDate import me.jessyan.autosize.utils.AutoSizeUtils import org.greenrobot.eventbus.EventBus import java.lang.ref.WeakReference @@ -58,6 +62,12 @@ class StatusBarView @JvmOverloads constructor( LayoutInflater.from(context).inflate(R.layout.taxt_u_p_statusview, this, true) isClickable = true isFocusable = true + textClockDate.onClick { + WakeManager.startWakeUp() + } + bcsv_status.onClick { + WakeManager.stopWakeup() + } } override fun onAttachedToWindow() { diff --git a/OCH/taxi/unmanned-passenger/src/main/res/layout/taxt_u_p_statusview.xml b/OCH/taxi/unmanned-passenger/src/main/res/layout/taxt_u_p_statusview.xml index 12e6db3744..a42f2af9e3 100644 --- a/OCH/taxi/unmanned-passenger/src/main/res/layout/taxt_u_p_statusview.xml +++ b/OCH/taxi/unmanned-passenger/src/main/res/layout/taxt_u_p_statusview.xml @@ -61,6 +61,7 @@ + Log.i(TAG, "core listener code:$code") + + when (type) { + ErrType.AUTH -> { + if (code == 0) { + Log.i(TAG, "SDK授权成功") + } else { + Log.i(TAG, "SDK授权失败，授权码为:$code") + } + + } + + ErrType.HTTP -> Log.i(TAG, "SDK状态：HTTP认证结果$code") + + else -> Log.i(TAG, "SDK状态：其他错误$code") + } + + } + + fun setWakeUpListener(wakeUpListener:WakeUpListener?){ + this._wakeUpListener = wakeUpListener + } + + fun startWakeUp(){ + ThreadUtils.getIoPool().submit { + // 检测权限 + val granted = ActivityCompat.checkSelfPermission(AbsMogoApplication.getApp(), + Manifest.permission.RECORD_AUDIO) + if(granted != PackageManager. PERMISSION_GRANTED){ + CallerRequestActivityHandleManager.requestPermission(TAG,Manifest.permission.RECORD_AUDIO) + return@submit + } + AiHelper.getInst().registerListener(ABILITYID, edListener) + CallerLogger.d(TAG,"START") + val ret = start() + if (ret == 0) { + createAudioRecord() //创建录音器 + isRecording.set(true) + audioRecord?.startRecording() //录音器启动录音 + + writeRecording(AiStatus.BEGIN) + } + } + } + + fun stopWakeup(){ + if(!isEnd.get()) { + writeRecording(AiStatus.END) + } + } + + /** + * 能力监听回调 + */ + private val edListener: AiListener = object : AiListener { + override fun onResult(handleID: Int, outputData: List, usrContext: Any?) { + if (null != outputData && outputData.size > 0) { + for (i in outputData.indices) { + CallerLogger.d(TAG, "onResult:handleID:" + handleID + ":" + outputData[i].key) + val key = outputData[i].key //引擎结果的key + val bytes = outputData[i].value //识别结果 + val result = String(bytes) + CallerLogger.d(TAG, "key=$key") + CallerLogger.d(TAG, "value=$result") + CallerLogger.d(TAG, "status=" + outputData[i].status) + if ((key == "func_wake_up" || key == "func_pre_wakeup")) { + CallerLogger.d(TAG,"$key: \n $result") + } + _wakeUpListener?.wakeupSuccess() + } + } + } + + override fun onEvent(i: Int, i1: Int, list: List, o: Any) { + CallerLogger.i(TAG, "onEvent:$i,event:$i1") + } + + override fun onError(i: Int, i1: Int, s: String, o: Any) { + CallerLogger.d(TAG,"错误通知，能力执行终止,Ability $i ERROR::$s,err code:$i1") + } + } + + private fun writeRecording(aiStatus:AiStatus){ + + /** + * 写入数据-录音方式 + */ + val data = ByteArray(BUFFER_SIZE) + val read = audioRecord!!.read(data, 0, BUFFER_SIZE) + + //平方和除以数据总长度，得到音量大小。 + val volume = calculateVolume(data) +// CallerLogger.d(TAG,"当前分贝:${ abs(volume.toDouble())}") + if (AudioRecord.ERROR_INVALID_OPERATION != read) { + //处理录音数据 + write(data, aiStatus) //送尾帧 + } + if (AiStatus.END == aiStatus) { + if ( audioRecord != null) { + audioRecord!!.stop() + isRecording.set(false) + end() + } + } else { + if (isRecording.get()) { + writeRecording(AiStatus.CONTINUE) //调用write方法送音频数据给引擎 + } + } + } + + /** + * 结束会话 + */ + private fun end() { + if (!isEnd.get()) { + val ret = AiHelper.getInst().end(aiHandle) + if (ret == 0) { + isEnd.set(true) + aiHandle = null + CallerLogger.d(TAG,"唤醒完成，end： $ret") + } else { + isEnd.set(false) + ThreadUtils.runOnUiThreadDelayed({ + end() + CallerLogger.d(TAG,"关闭失败2s后重新广播唤醒完成，end： $ret") + },2_000) + CallerLogger.d(TAG,"唤醒完成，end： $ret") + } + } + } + + /** + * 写入数据 + */ + private fun write(part: ByteArray, status: AiStatus) { + if (isEnd.get()) { + return + } + val dataBuilder = AiRequest.builder() + var ret = 0 + + /** + * 送入音频需要标识音频的状态，第一帧为起始帧，status要传AiStatus.BEGIN,最后一帧为结束帧，status要传AiStatus.END,其他为中间帧，status要传AiStatus.CONTINUE + * 音频要求16bit，16K，单声道的pcm音频。 + * 建议每次发送音频间隔40ms，每次发送音频字节数为一帧音频大小的整数倍。 + */ + val aiAudio = AiAudio.get("wav").data(part).status(status).valid() + dataBuilder.payload(aiAudio) + + ret = AiHelper.getInst().write(dataBuilder.build(), aiHandle) + if (ret != 0) { + CallerLogger.d(TAG,"write失败：$ret ") + } + } + + /** + * 根据录音数据计算音量 + */ + private fun calculateVolume(buffer: ByteArray): Int { + var sumVolume = 0.0 + var avgVolume = 0.0 + var volume = 0 + var i = 0 + while (i < buffer.size) { + val v1 = buffer[i].toInt() and 0xFF + val v2 = buffer[i + 1].toInt() and 0xFF + var temp = v1 + (v2 shl 8) // 小端 + if (temp >= 0x8000) { + temp = 0xffff - temp + } + sumVolume += abs(temp.toDouble()) + i += 2 + } + avgVolume = sumVolume / buffer.size / 2 + volume = (log10(1 + avgVolume) * 10).toInt() + return volume + } + + /** + * 创建录音器 + */ + @SuppressLint("MissingPermission") + private fun createAudioRecord() { + if (isRecording.get()) { + return + } + if (audioRecord == null) { + Log.d(TAG, "createAudioRecord") + audioRecord = AudioRecord( + MediaRecorder.AudioSource.MIC, + 16000, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + BUFFER_SIZE + ) + } + } + + /** + * 开始会话 + */ + private fun start(): Int { + if (!keyword2File()) { + CallerLogger.d(TAG,"唤醒词文件写入失败，请检查是否有读写权限。") + return -1 + } + val customBuilder = AiRequest.builder() + customBuilder.customText("key_word", getIvwDir() + "/keyword.txt", 0) + var ret = AiHelper.getInst().loadData(ABILITYID, customBuilder.build()) + if (ret != 0) { + CallerLogger.d(TAG,"open ivw loadData 失败：$ret") + return ret + } + CallerLogger.d(TAG,"open ivw loadData success：$ret") + val indexs = intArrayOf(0) + ret = AiHelper.getInst().specifyDataSet(ABILITYID, "key_word", indexs) //从缓存中把个性化资源设置到引擎中 + if (ret != 0) { + CallerLogger.d(TAG,"open ivw specifyDataSet 失败：$ret") + return ret + } + CallerLogger.d(TAG,"open ivw specifyDataSet success：$ret") + + val paramBuilder = AiRequest.builder() + paramBuilder.param("wdec_param_nCmThreshold", "0 0:800") + paramBuilder.param("gramLoad", true) + isEnd.set(false) + aiHandle = AiHelper.getInst().start(ABILITYID, paramBuilder.build(), null) + if (aiHandle?.code != 0) { + CallerLogger.d(TAG,"open ivw start失败：${aiHandle?.getCode()}") + return aiHandle?.code ?:0 + } + return 0 + } + + private fun keyword2File(): Boolean { + try { + val keywordFile: File = File(getIvwDir() + "/keyword.txt") + if (keywordFile.exists()) { + //强制清空内容 + keywordFile.delete() + } + val binFile: File = File(getIvwDir() + "/keyword.bin") + if (binFile.exists()) { + binFile.delete() + } + var temp: String =weakupKey + if (temp.isEmpty()) { + temp = "你好小智" + } + val str = temp.replace("，", ",") + val keywords = str.split(",".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + if (!keywordFile.exists()) { + keywordFile.createNewFile() + } + val writer = OutputStreamWriter( + FileOutputStream(keywordFile), + "UTF-8" + ) + val bufferedWriter = BufferedWriter(writer) + for (i in keywords.indices) { + bufferedWriter.write(keywords[i]) + bufferedWriter.write(";") + bufferedWriter.newLine() //写入换行 + } + bufferedWriter.close() + } catch (e: IOException) { + e.printStackTrace() + return false + } + return true + } + + fun initWakeAi(context: Context) { + + try { + val workDir = getWorkDir() + copyFile(workDir) + val file = File(workDir) + if(!file.exists()){ + file.mkdirs() + } + AiHelper.getInst().setLogInfo(LogLvl.VERBOSE, 1, "${workDir}aeeLog.txt") + + //设定初始化参数 + val params = BaseLibrary.Params.builder() + .appId("0c498b42") //您的应用ID，可从控制台查看 + .apiKey("8579f566eb7f3c4f4a07148ad9e2408c") //您的APIKEY，可从控制台查看 + .apiSecret("NTRmMmI5MWI4NzIzZTIxN2Q5N2FjMWVl") //您的APISECRET，可从控制台查看 + .workDir(workDir) //SDK的工作目录，需要确保有读写权限。一般用于存放离线能力资源，日志存放目录等使用。 + .build() + + AiHelper.getInst().registerListener(coreListener) // 注册SDK 初始化状态监听 + AiHelper.getInst().initEntry(context, params) + }catch (e:Exception){ + CallerLogger.d(TAG,"初始化失败:${e.message}") + } + } + + private fun copyFile(workDir: String) { + ThreadUtils.getIoPool().submit{ + val dir = "ivw" + val targetDir = workDir + File.separator + dir + if(checkCopied(dir,targetDir)) { + FileUtils.deleteAllInDir(targetDir) + ResourceUtils.copyFileFromAssets(dir, targetDir) + CallerLogger.d(TAG,"copy 完成") + }else{ + CallerLogger.d(TAG,"文件已经存在 不需要copy") + } + + } + } + + private fun checkCopied(dir: String, targetDir: String): Boolean { + val assets = Utils.getApp().assets.list(dir)?.toMutableList() + val tartgetFile = File(targetDir) + if(tartgetFile.isDirectory){ + tartgetFile.listFiles { dir, name -> + (assets?.contains(name)?:true).apply { + assets?.remove(name) + } + } + return assets?.isNotEmpty()?:false + } + return true + } + + fun getWorkDir():String{ + AbsMogoApplication.getApp()?.let { + return it.filesDir.path + File.separator + "iflytek" + File.separator + } + throw IllegalArgumentException("找不到 Application") + } + + fun getIvwDir(): String { + return getWorkDir()+File.separator+"ivw" + } +} \ No newline at end of file diff --git a/libraries/mogo-speech/src/main/java/com/mogo/mgintelligent/speech/iflytek/WakeUpListener.kt b/libraries/mogo-speech/src/main/java/com/mogo/mgintelligent/speech/iflytek/WakeUpListener.kt new file mode 100644 index 0000000000..b48cda94aa --- /dev/null +++ b/libraries/mogo-speech/src/main/java/com/mogo/mgintelligent/speech/iflytek/WakeUpListener.kt @@ -0,0 +1,5 @@ +package com.mogo.mgintelligent.speech.iflytek + +interface WakeUpListener { + fun wakeupSuccess() +} \ No newline at end of file