yangyakun921/MoGoEagleEye
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[8.0.0]

Browse Source 
[fea]
[科大讯飞 离线唤醒 + 百度对话]
This commit is contained in:
yangyakun
2025-04-27 15:51:22 +08:00
parent 2148212413
commit ac49141b47
7 changed files with 327 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AIViewModel.kt
View File

@@ -11,6 +11,8 @@ import com.mogo.mgintelligent.speech.AsrResult
import com.mogo.mgintelligent.speech.AsrState
import com.mogo.mgintelligent.speech.IWakeUpListener
import com.mogo.mgintelligent.speech.MGSpeech
import com.mogo.mgintelligent.speech.iflytek.WakeManager
import com.mogo.mgintelligent.speech.iflytek.WakeUpListener
import com.mogo.och.bridge.autopilot.location.OchLocationManager
import com.mogo.och.data.taxi.BaseOrderBean
import com.mogo.och.data.taxi.TaxiOrderStatusEnum
@@ -81,6 +83,13 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList
}
}
private val wakeUpListener = object :WakeUpListener{
override fun wakeupSuccess() {
onWakeUp()
}
}
init {
}
@@ -89,6 +98,7 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList
OrderModel.setOrderStatusCallback(TAG,orderListener)
OchLocationManager.addGCJ02Listener(TAG,1,locationCallback)
AIMessageManager.registerListener(this)
WakeManager.setWakeUpListener(wakeUpListener)
mgSpeech.weakUpListener = this
mgSpeech.startWeakUp()
@@ -123,6 +133,7 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList
override fun onCleared() {
AIMessageManager.unregisterListener(this)
mgSpeech.weakUpListener = null
WakeManager.setWakeUpListener(null)
mgSpeech.stopWeakUp()
llmResultJob?.cancel()
OrderModel.setOrderStatusCallback(TAG,null)

2
OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/setting/VerticalSeekBar.kt
View File

@@ -11,7 +11,9 @@ import androidx.constraintlayout.widget.ConstraintLayout
import androidx.constraintlayout.widget.ConstraintSet
import androidx.core.view.marginBottom
import androidx.core.view.marginTop
import com.mogo.eagle.core.utilcode.kotlin.onClick
import com.mogo.eagle.core.utilcode.mogo.logger.CallerLogger
import com.mogo.mgintelligent.speech.iflytek.WakeManager
import com.mogo.och.common.module.utils.NumberFormatUtil
import com.mogo.och.unmanned.taxi.passenger.R
import kotlinx.android.synthetic.main.taxi_p_setting_light_view.view.tv_current_value

10
OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/statusview/StatusBarView.kt
View File

@@ -11,14 +11,18 @@ import com.mogo.eagle.core.function.call.devatools.CallerDevaToolsManager
import com.mogo.eagle.core.function.call.hmi.CallerHmiViewControlListenerManager
import com.mogo.eagle.core.function.call.setting.CallerSkinModeListenerManager
import com.mogo.eagle.core.function.hmi.ui.setting.ToggleDebugView
import com.mogo.eagle.core.utilcode.kotlin.onClick
import com.mogo.eagle.core.utilcode.util.ActivityUtils
import com.mogo.mgintelligent.speech.iflytek.WakeManager
import com.mogo.och.bridge.autopilot.autopilot.IOchAutopilotStatusListener
import com.mogo.och.bridge.autopilot.autopilot.OchAutoPilotStatusListenerManager
import com.mogo.och.common.module.manager.debug.DebugViewWatchDogFragment
import com.mogo.och.unmanned.passenger.ui.debug.DebugEvent
import com.mogo.och.unmanned.taxi.passenger.R
import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.bcsv_status
import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.vShowDebugView
import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.iv_biz_icon
import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.textClockDate
import me.jessyan.autosize.utils.AutoSizeUtils
import org.greenrobot.eventbus.EventBus
import java.lang.ref.WeakReference
@@ -58,6 +62,12 @@ class StatusBarView @JvmOverloads constructor(
LayoutInflater.from(context).inflate(R.layout.taxt_u_p_statusview, this, true)
isClickable = true
isFocusable = true
textClockDate.onClick {
WakeManager.startWakeUp()
}
bcsv_status.onClick {
WakeManager.stopWakeup()
}
}
override fun onAttachedToWindow() {

1
OCH/taxi/unmanned-passenger/src/main/res/layout/taxt_u_p_statusview.xml
View File

@@ -61,6 +61,7 @@
<com.mogo.och.common.module.wigets.DriverConnectStatusView
android:id="@+id/bcsv_status"
app:layout_constraintTop_toTopOf="@+id/vShowDebugView"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toStartOf="@+id/blueToothView"

1
libraries/mogo-speech/build.gradle
View File

@@ -62,4 +62,5 @@ dependencies {
implementation project(':foudations:mogo-commons')
implementation project(':core:mogo-core-utils')
implementation project(':core:mogo-core-function-call')
}

297
libraries/mogo-speech/src/main/java/com/mogo/mgintelligent/speech/iflytek/WakeManager.kt
View File

@@ -1,24 +1,64 @@
package com.mogo.mgintelligent.speech.iflytek
import android.Manifest
import android.annotation.SuppressLint
import android.content.Context
import android.content.pm.PackageManager
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaRecorder
import android.util.Log
import androidx.core.app.ActivityCompat
import com.iflytek.aikit.core.AiAudio
import com.iflytek.aikit.core.AiHandle
import com.iflytek.aikit.core.AiHelper
import com.iflytek.aikit.core.AiListener
import com.iflytek.aikit.core.AiRequest
import com.iflytek.aikit.core.AiResponse
import com.iflytek.aikit.core.AiStatus
import com.iflytek.aikit.core.BaseLibrary
import com.iflytek.aikit.core.CoreListener
import com.iflytek.aikit.core.ErrType
import com.iflytek.aikit.core.LogLvl
import com.mogo.commons.AbsMogoApplication
import com.mogo.eagle.core.function.call.setting.CallerRequestActivityHandleManager
import com.mogo.eagle.core.utilcode.mogo.logger.CallerLogger
import com.mogo.eagle.core.utilcode.util.FileUtils
import com.mogo.eagle.core.utilcode.util.ResourceUtils
import com.mogo.eagle.core.utilcode.util.ThreadUtils
import com.mogo.eagle.core.utilcode.util.Utils
import java.io.BufferedWriter
import java.io.File
import java.io.FileOutputStream
import java.io.IOException
import java.io.OutputStreamWriter
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
import kotlin.math.abs
import kotlin.math.log10
object WakeManager {
private val TAG = "WakeManager"
private const val ABILITYID: String = "e867a88f2"
private val weakupKey = "你好小智,小智小智,小智你好"
private val isEnd = AtomicBoolean(true)
//录音是否在进行
private val isRecording = AtomicBoolean(false)
private var aiHandle: AiHandle? = null
private var audioRecord: AudioRecord? = null
private var _wakeUpListener:WakeUpListener?=null
//录音缓冲区大小
private const val BUFFER_SIZE = 1280
private val coreListener = CoreListener { type, code ->
Log.i(TAG, "core listener code:$code")
@@ -39,6 +79,259 @@ object WakeManager {
}
fun setWakeUpListener(wakeUpListener:WakeUpListener?){
this._wakeUpListener = wakeUpListener
}
fun startWakeUp(){
ThreadUtils.getIoPool().submit {
// 检测权限
val granted = ActivityCompat.checkSelfPermission(AbsMogoApplication.getApp(),
Manifest.permission.RECORD_AUDIO)
if(granted != PackageManager. PERMISSION_GRANTED){
CallerRequestActivityHandleManager.requestPermission(TAG,Manifest.permission.RECORD_AUDIO)
return@submit
}
AiHelper.getInst().registerListener(ABILITYID, edListener)
CallerLogger.d(TAG,"START")
val ret = start()
if (ret == 0) {
createAudioRecord() //创建录音器
isRecording.set(true)
audioRecord?.startRecording() //录音器启动录音
writeRecording(AiStatus.BEGIN)
}
}
}
fun stopWakeup(){
if(!isEnd.get()) {
writeRecording(AiStatus.END)
}
}
/**
* 能力监听回调
*/
private val edListener: AiListener = object : AiListener {
override fun onResult(handleID: Int, outputData: List<AiResponse>, usrContext: Any?) {
if (null != outputData && outputData.size > 0) {
for (i in outputData.indices) {
CallerLogger.d(TAG, "onResult:handleID:" + handleID + ":" + outputData[i].key)
val key = outputData[i].key //引擎结果的key
val bytes = outputData[i].value //识别结果
val result = String(bytes)
CallerLogger.d(TAG, "key=$key")
CallerLogger.d(TAG, "value=$result")
CallerLogger.d(TAG, "status=" + outputData[i].status)
if ((key == "func_wake_up" || key == "func_pre_wakeup")) {
CallerLogger.d(TAG,"$key: \n $result")
}
_wakeUpListener?.wakeupSuccess()
}
}
}
override fun onEvent(i: Int, i1: Int, list: List<AiResponse>, o: Any) {
CallerLogger.i(TAG, "onEvent:$i,event:$i1")
}
override fun onError(i: Int, i1: Int, s: String, o: Any) {
CallerLogger.d(TAG,"错误通知，能力执行终止,Ability $i ERROR::$s,err code:$i1")
}
}
private fun writeRecording(aiStatus:AiStatus){
/**
* 写入数据-录音方式
*/
val data = ByteArray(BUFFER_SIZE)
val read = audioRecord!!.read(data, 0, BUFFER_SIZE)
//平方和除以数据总长度，得到音量大小。
val volume = calculateVolume(data)
// CallerLogger.d(TAG,"当前分贝:${ abs(volume.toDouble())}")
if (AudioRecord.ERROR_INVALID_OPERATION != read) {
//处理录音数据
write(data, aiStatus) //送尾帧
}
if (AiStatus.END == aiStatus) {
if ( audioRecord != null) {
audioRecord!!.stop()
isRecording.set(false)
end()
}
} else {
if (isRecording.get()) {
writeRecording(AiStatus.CONTINUE) //调用write方法送音频数据给引擎
}
}
}
/**
* 结束会话
*/
private fun end() {
if (!isEnd.get()) {
val ret = AiHelper.getInst().end(aiHandle)
if (ret == 0) {
isEnd.set(true)
aiHandle = null
CallerLogger.d(TAG,"唤醒完成，end： $ret")
} else {
isEnd.set(false)
ThreadUtils.runOnUiThreadDelayed({
end()
CallerLogger.d(TAG,"关闭失败2s后重新广播唤醒完成，end： $ret")
},2_000)
CallerLogger.d(TAG,"唤醒完成，end： $ret")
}
}
}
/**
* 写入数据
*/
private fun write(part: ByteArray, status: AiStatus) {
if (isEnd.get()) {
return
}
val dataBuilder = AiRequest.builder()
var ret = 0
/**
* 送入音频需要标识音频的状态，第一帧为起始帧，status要传AiStatus.BEGIN,最后一帧为结束帧，status要传AiStatus.END,其他为中间帧，status要传AiStatus.CONTINUE
* 音频要求16bit，16K，单声道的pcm音频。
* 建议每次发送音频间隔40ms，每次发送音频字节数为一帧音频大小的整数倍。
*/
val aiAudio = AiAudio.get("wav").data(part).status(status).valid()
dataBuilder.payload(aiAudio)
ret = AiHelper.getInst().write(dataBuilder.build(), aiHandle)
if (ret != 0) {
CallerLogger.d(TAG,"write失败：$ret ")
}
}
/**
* 根据录音数据计算音量
*/
private fun calculateVolume(buffer: ByteArray): Int {
var sumVolume = 0.0
var avgVolume = 0.0
var volume = 0
var i = 0
while (i < buffer.size) {
val v1 = buffer[i].toInt() and 0xFF
val v2 = buffer[i + 1].toInt() and 0xFF
var temp = v1 + (v2 shl 8) // 小端
if (temp >= 0x8000) {
temp = 0xffff - temp
}
sumVolume += abs(temp.toDouble())
i += 2
}
avgVolume = sumVolume / buffer.size / 2
volume = (log10(1 + avgVolume) * 10).toInt()
return volume
}
/**
* 创建录音器
*/
@SuppressLint("MissingPermission")
private fun createAudioRecord() {
if (isRecording.get()) {
return
}
if (audioRecord == null) {
Log.d(TAG, "createAudioRecord")
audioRecord = AudioRecord(
MediaRecorder.AudioSource.MIC,
16000,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
BUFFER_SIZE
)
}
}
/**
* 开始会话
*/
private fun start(): Int {
if (!keyword2File()) {
CallerLogger.d(TAG,"唤醒词文件写入失败，请检查是否有读写权限。")
return -1
}
val customBuilder = AiRequest.builder()
customBuilder.customText("key_word", getIvwDir() + "/keyword.txt", 0)
var ret = AiHelper.getInst().loadData(ABILITYID, customBuilder.build())
if (ret != 0) {
CallerLogger.d(TAG,"open ivw loadData 失败：$ret")
return ret
}
CallerLogger.d(TAG,"open ivw loadData success：$ret")
val indexs = intArrayOf(0)
ret = AiHelper.getInst().specifyDataSet(ABILITYID, "key_word", indexs) //从缓存中把个性化资源设置到引擎中
if (ret != 0) {
CallerLogger.d(TAG,"open ivw specifyDataSet 失败：$ret")
return ret
}
CallerLogger.d(TAG,"open ivw specifyDataSet success：$ret")
val paramBuilder = AiRequest.builder()
paramBuilder.param("wdec_param_nCmThreshold", "0 0:800")
paramBuilder.param("gramLoad", true)
isEnd.set(false)
aiHandle = AiHelper.getInst().start(ABILITYID, paramBuilder.build(), null)
if (aiHandle?.code != 0) {
CallerLogger.d(TAG,"open ivw start失败：${aiHandle?.getCode()}")
return aiHandle?.code ?:0
}
return 0
}
private fun keyword2File(): Boolean {
try {
val keywordFile: File = File(getIvwDir() + "/keyword.txt")
if (keywordFile.exists()) {
//强制清空内容
keywordFile.delete()
}
val binFile: File = File(getIvwDir() + "/keyword.bin")
if (binFile.exists()) {
binFile.delete()
}
var temp: String =weakupKey
if (temp.isEmpty()) {
temp = "你好小智"
}
val str = temp.replace("，", ",")
val keywords = str.split(",".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray()
if (!keywordFile.exists()) {
keywordFile.createNewFile()
}
val writer = OutputStreamWriter(
FileOutputStream(keywordFile),
"UTF-8"
)
val bufferedWriter = BufferedWriter(writer)
for (i in keywords.indices) {
bufferedWriter.write(keywords[i])
bufferedWriter.write(";")
bufferedWriter.newLine() //写入换行
}
bufferedWriter.close()
} catch (e: IOException) {
e.printStackTrace()
return false
}
return true
}
fun initWakeAi(context: Context) {
try {
@@ -100,4 +393,8 @@ object WakeManager {
}
throw IllegalArgumentException("找不到 Application")
}
fun getIvwDir(): String {
return getWorkDir()+File.separator+"ivw"
}
}

5
libraries/mogo-speech/src/main/java/com/mogo/mgintelligent/speech/iflytek/WakeUpListener.kt Normal file
View File

@@ -0,0 +1,5 @@
package com.mogo.mgintelligent.speech.iflytek
interface WakeUpListener {
fun wakeupSuccess()
}