yangyakun921/MoGoEagleEye
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dev_robotaxi-d_250417_8.0.0_ai' into dev_robotaxi-d_250417_8.0.0

Browse Source
This commit is contained in:
yangyakun
2025-04-27 16:41:09 +08:00
parent e9f685e5e4 cfdcbbbf73
commit c05761374b
15 changed files with 446 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/TaxiUnmannedPassengerProvider.kt
View File

@@ -7,6 +7,7 @@ import com.alibaba.android.arouter.facade.annotation.Route
import com.mogo.eagle.core.utilcode.mogo.logger.CallerLogger.d
import com.mogo.eagle.core.utilcode.mogo.logger.scene.SceneConstant
import com.mogo.mgintelligent.speech.MGSpeech
import com.mogo.mgintelligent.speech.iflytek.WakeManager
import com.mogo.och.common.module.constant.OchCommonConst
import com.mogo.och.common.module.biz.provider.CommonServiceImpl
import com.mogo.och.bridge.autopilot.autopilot.OchAutopilotAnalytics
@@ -32,6 +33,7 @@ class TaxiUnmannedPassengerProvider : CommonServiceImpl() {
d(SceneConstant.M_TAXI_P + tag, "init")
statusBarView = StatusBarView(context)
MGSpeech.init(context)
WakeManager.initWakeAi(context)
}
override fun getStatusBarView(context: Context): View {

13
OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AIViewModel.kt
View File

@@ -11,6 +11,8 @@ import com.mogo.mgintelligent.speech.AsrResult
import com.mogo.mgintelligent.speech.AsrState
import com.mogo.mgintelligent.speech.IWakeUpListener
import com.mogo.mgintelligent.speech.MGSpeech
import com.mogo.mgintelligent.speech.iflytek.WakeManager
import com.mogo.mgintelligent.speech.iflytek.WakeUpListener
import com.mogo.och.bridge.autopilot.location.OchLocationManager
import com.mogo.och.data.taxi.BaseOrderBean
import com.mogo.och.data.taxi.TaxiOrderStatusEnum
@@ -81,6 +83,14 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList
}
}
private val wakeUpListener = object :WakeUpListener{
override fun wakeupSuccess() {
onWakeUp()
WakeManager.stopWakeup()
}
}
init {
}
@@ -89,6 +99,7 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList
OrderModel.setOrderStatusCallback(TAG,orderListener)
OchLocationManager.addGCJ02Listener(TAG,1,locationCallback)
AIMessageManager.registerListener(this)
WakeManager.setWakeUpListener(wakeUpListener)
mgSpeech.weakUpListener = this
mgSpeech.startWeakUp()
@@ -123,6 +134,7 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList
override fun onCleared() {
AIMessageManager.unregisterListener(this)
mgSpeech.weakUpListener = null
WakeManager.setWakeUpListener(null)
mgSpeech.stopWeakUp()
llmResultJob?.cancel()
OrderModel.setOrderStatusCallback(TAG,null)
@@ -228,6 +240,7 @@ class AIViewModel : ViewModel(), AIMessageManager.AIMessageListener, IWakeUpList
}
AsrState.STATE_EXIT -> {
WakeManager.startWakeUp()
_asrUIStateFlow.value = AsrUIState.Idle
mgSpeech.speak(tipsExit)
mgSpeech.isAssistantShow(false)

6
OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/aiview/AiView.kt
View File

@@ -23,6 +23,7 @@ import com.mogo.eagle.core.function.call.autopilot.CallerPlanningActionsListener
import com.mogo.eagle.core.utilcode.kotlin.onClick
import com.mogo.eagle.core.utilcode.mogo.logger.CallerLogger
import com.mogo.eagle.core.utilcode.util.UiThreadHandler
import com.mogo.mgintelligent.speech.iflytek.WakeManager
import com.mogo.och.common.module.manager.logchainanalytic.OchChainLogManager
import com.mogo.och.common.module.utils.BigFrameAnimatorContainer
import com.mogo.och.common.module.utils.RxUtils
@@ -168,6 +169,11 @@ class AiView @JvmOverloads constructor(
super.onVisibilityAggregated(isVisible)
CallerLogger.d(TAG,"是否展示中：${isVisible}")
try {
if(isVisible){
WakeManager.startWakeUp()
}else{
WakeManager.stopWakeup()
}
if(isVisible){
aiAnimator?.start()
RxUtils.createSubscribe(3_000) {

2
OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/setting/VerticalSeekBar.kt
View File

@@ -11,7 +11,9 @@ import androidx.constraintlayout.widget.ConstraintLayout
import androidx.constraintlayout.widget.ConstraintSet
import androidx.core.view.marginBottom
import androidx.core.view.marginTop
import com.mogo.eagle.core.utilcode.kotlin.onClick
import com.mogo.eagle.core.utilcode.mogo.logger.CallerLogger
import com.mogo.mgintelligent.speech.iflytek.WakeManager
import com.mogo.och.common.module.utils.NumberFormatUtil
import com.mogo.och.unmanned.taxi.passenger.R
import kotlinx.android.synthetic.main.taxi_p_setting_light_view.view.tv_current_value

10
OCH/taxi/unmanned-passenger/src/main/java/com/mogo/och/unmanned/passenger/ui/statusview/StatusBarView.kt
View File

@@ -11,14 +11,18 @@ import com.mogo.eagle.core.function.call.devatools.CallerDevaToolsManager
import com.mogo.eagle.core.function.call.hmi.CallerHmiViewControlListenerManager
import com.mogo.eagle.core.function.call.setting.CallerSkinModeListenerManager
import com.mogo.eagle.core.function.hmi.ui.setting.ToggleDebugView
import com.mogo.eagle.core.utilcode.kotlin.onClick
import com.mogo.eagle.core.utilcode.util.ActivityUtils
import com.mogo.mgintelligent.speech.iflytek.WakeManager
import com.mogo.och.bridge.autopilot.autopilot.IOchAutopilotStatusListener
import com.mogo.och.bridge.autopilot.autopilot.OchAutoPilotStatusListenerManager
import com.mogo.och.common.module.manager.debug.DebugViewWatchDogFragment
import com.mogo.och.unmanned.passenger.ui.debug.DebugEvent
import com.mogo.och.unmanned.taxi.passenger.R
import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.bcsv_status
import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.vShowDebugView
import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.iv_biz_icon
import kotlinx.android.synthetic.main.taxt_u_p_statusview.view.textClockDate
import me.jessyan.autosize.utils.AutoSizeUtils
import org.greenrobot.eventbus.EventBus
import java.lang.ref.WeakReference
@@ -58,6 +62,12 @@ class StatusBarView @JvmOverloads constructor(
LayoutInflater.from(context).inflate(R.layout.taxt_u_p_statusview, this, true)
isClickable = true
isFocusable = true
textClockDate.onClick {
WakeManager.startWakeUp()
}
bcsv_status.onClick {
WakeManager.stopWakeup()
}
}
override fun onAttachedToWindow() {

1
OCH/taxi/unmanned-passenger/src/main/res/layout/taxt_u_p_statusview.xml
View File

@@ -61,6 +61,7 @@
<com.mogo.och.common.module.wigets.DriverConnectStatusView
android:id="@+id/bcsv_status"
app:layout_constraintTop_toTopOf="@+id/vShowDebugView"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toStartOf="@+id/blueToothView"

6
libraries/mogo-speech/build.gradle
View File

@@ -57,4 +57,10 @@ dependencies {
implementation(files("libs/bdasr_V3_20210628_cfe8c44.jar"))
implementation(files("libs/com.baidu.tts_2.6.3.c2aaa9f_20220922113422.jar"))
implementation files('libs/AIKit.aar')
implementation project(':foudations:mogo-commons')
implementation project(':core:mogo-core-utils')
implementation project(':core:mogo-core-function-call')
}

BIN
libraries/mogo-speech/libs/AIKit.aar Normal file
View File

Binary file not shown.

BIN
libraries/mogo-speech/src/main/assets/ivw/IVW_FILLER_1 Normal file
View File

Binary file not shown.

BIN
libraries/mogo-speech/src/main/assets/ivw/IVW_GRAM_1 Normal file
View File

Binary file not shown.

BIN
libraries/mogo-speech/src/main/assets/ivw/IVW_KEYWORD_1 Normal file
View File

Binary file not shown.

BIN
libraries/mogo-speech/src/main/assets/ivw/IVW_MLP_1 Normal file
View File

Binary file not shown.

1
libraries/mogo-speech/src/main/assets/ivw/version_2 Normal file
View File

@@ -0,0 +1 @@
0

400
libraries/mogo-speech/src/main/java/com/mogo/mgintelligent/speech/iflytek/WakeManager.kt Normal file
View File

@@ -0,0 +1,400 @@
package com.mogo.mgintelligent.speech.iflytek
import android.Manifest
import android.annotation.SuppressLint
import android.content.Context
import android.content.pm.PackageManager
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaRecorder
import android.util.Log
import androidx.core.app.ActivityCompat
import com.iflytek.aikit.core.AiAudio
import com.iflytek.aikit.core.AiHandle
import com.iflytek.aikit.core.AiHelper
import com.iflytek.aikit.core.AiListener
import com.iflytek.aikit.core.AiRequest
import com.iflytek.aikit.core.AiResponse
import com.iflytek.aikit.core.AiStatus
import com.iflytek.aikit.core.BaseLibrary
import com.iflytek.aikit.core.CoreListener
import com.iflytek.aikit.core.ErrType
import com.iflytek.aikit.core.LogLvl
import com.mogo.commons.AbsMogoApplication
import com.mogo.eagle.core.function.call.setting.CallerRequestActivityHandleManager
import com.mogo.eagle.core.utilcode.mogo.logger.CallerLogger
import com.mogo.eagle.core.utilcode.util.FileUtils
import com.mogo.eagle.core.utilcode.util.ResourceUtils
import com.mogo.eagle.core.utilcode.util.ThreadUtils
import com.mogo.eagle.core.utilcode.util.Utils
import java.io.BufferedWriter
import java.io.File
import java.io.FileOutputStream
import java.io.IOException
import java.io.OutputStreamWriter
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
import kotlin.math.abs
import kotlin.math.log10
object WakeManager {
private val TAG = "WakeManager"
private const val ABILITYID: String = "e867a88f2"
private val weakupKey = "你好小智,小智小智,小智你好"
private val isEnd = AtomicBoolean(true)
//录音是否在进行
private val isRecording = AtomicBoolean(false)
private var aiHandle: AiHandle? = null
private var audioRecord: AudioRecord? = null
private var _wakeUpListener:WakeUpListener?=null
//录音缓冲区大小
private const val BUFFER_SIZE = 1280
private val coreListener = CoreListener { type, code ->
Log.i(TAG, "core listener code:$code")
when (type) {
ErrType.AUTH -> {
if (code == 0) {
Log.i(TAG, "SDK授权成功")
} else {
Log.i(TAG, "SDK授权失败，授权码为:$code")
}
}
ErrType.HTTP -> Log.i(TAG, "SDK状态：HTTP认证结果$code")
else -> Log.i(TAG, "SDK状态：其他错误$code")
}
}
fun setWakeUpListener(wakeUpListener:WakeUpListener?){
this._wakeUpListener = wakeUpListener
}
fun startWakeUp(){
ThreadUtils.getIoPool().submit {
// 检测权限
val granted = ActivityCompat.checkSelfPermission(AbsMogoApplication.getApp(),
Manifest.permission.RECORD_AUDIO)
if(granted != PackageManager. PERMISSION_GRANTED){
CallerRequestActivityHandleManager.requestPermission(TAG,Manifest.permission.RECORD_AUDIO)
return@submit
}
AiHelper.getInst().registerListener(ABILITYID, edListener)
CallerLogger.d(TAG,"START")
val ret = start()
if (ret == 0) {
createAudioRecord() //创建录音器
isRecording.set(true)
audioRecord?.startRecording() //录音器启动录音
writeRecording(AiStatus.BEGIN)
}
}
}
fun stopWakeup(){
if(!isEnd.get()) {
writeRecording(AiStatus.END)
}
}
/**
* 能力监听回调
*/
private val edListener: AiListener = object : AiListener {
override fun onResult(handleID: Int, outputData: List<AiResponse>, usrContext: Any?) {
if (null != outputData && outputData.size > 0) {
for (i in outputData.indices) {
CallerLogger.d(TAG, "onResult:handleID:" + handleID + ":" + outputData[i].key)
val key = outputData[i].key //引擎结果的key
val bytes = outputData[i].value //识别结果
val result = String(bytes)
CallerLogger.d(TAG, "key=$key")
CallerLogger.d(TAG, "value=$result")
CallerLogger.d(TAG, "status=" + outputData[i].status)
if ((key == "func_wake_up" || key == "func_pre_wakeup")) {
CallerLogger.d(TAG,"$key: \n $result")
}
_wakeUpListener?.wakeupSuccess()
}
}
}
override fun onEvent(i: Int, i1: Int, list: List<AiResponse>, o: Any) {
CallerLogger.i(TAG, "onEvent:$i,event:$i1")
}
override fun onError(i: Int, i1: Int, s: String, o: Any) {
CallerLogger.d(TAG,"错误通知，能力执行终止,Ability $i ERROR::$s,err code:$i1")
}
}
private fun writeRecording(aiStatus:AiStatus){
/**
* 写入数据-录音方式
*/
val data = ByteArray(BUFFER_SIZE)
val read = audioRecord!!.read(data, 0, BUFFER_SIZE)
//平方和除以数据总长度，得到音量大小。
val volume = calculateVolume(data)
// CallerLogger.d(TAG,"当前分贝:${ abs(volume.toDouble())}")
if (AudioRecord.ERROR_INVALID_OPERATION != read) {
//处理录音数据
write(data, aiStatus) //送尾帧
}
if (AiStatus.END == aiStatus) {
if ( audioRecord != null) {
audioRecord!!.stop()
isRecording.set(false)
end()
}
} else {
if (isRecording.get()) {
writeRecording(AiStatus.CONTINUE) //调用write方法送音频数据给引擎
}
}
}
/**
* 结束会话
*/
private fun end() {
if (!isEnd.get()) {
val ret = AiHelper.getInst().end(aiHandle)
if (ret == 0) {
isEnd.set(true)
aiHandle = null
CallerLogger.d(TAG,"唤醒完成，end： $ret")
} else {
isEnd.set(false)
ThreadUtils.runOnUiThreadDelayed({
end()
CallerLogger.d(TAG,"关闭失败2s后重新广播唤醒完成，end： $ret")
},2_000)
CallerLogger.d(TAG,"唤醒完成，end： $ret")
}
}
}
/**
* 写入数据
*/
private fun write(part: ByteArray, status: AiStatus) {
if (isEnd.get()) {
return
}
val dataBuilder = AiRequest.builder()
var ret = 0
/**
* 送入音频需要标识音频的状态，第一帧为起始帧，status要传AiStatus.BEGIN,最后一帧为结束帧，status要传AiStatus.END,其他为中间帧，status要传AiStatus.CONTINUE
* 音频要求16bit，16K，单声道的pcm音频。
* 建议每次发送音频间隔40ms，每次发送音频字节数为一帧音频大小的整数倍。
*/
val aiAudio = AiAudio.get("wav").data(part).status(status).valid()
dataBuilder.payload(aiAudio)
ret = AiHelper.getInst().write(dataBuilder.build(), aiHandle)
if (ret != 0) {
CallerLogger.d(TAG,"write失败：$ret ")
}
}
/**
* 根据录音数据计算音量
*/
private fun calculateVolume(buffer: ByteArray): Int {
var sumVolume = 0.0
var avgVolume = 0.0
var volume = 0
var i = 0
while (i < buffer.size) {
val v1 = buffer[i].toInt() and 0xFF
val v2 = buffer[i + 1].toInt() and 0xFF
var temp = v1 + (v2 shl 8) // 小端
if (temp >= 0x8000) {
temp = 0xffff - temp
}
sumVolume += abs(temp.toDouble())
i += 2
}
avgVolume = sumVolume / buffer.size / 2
volume = (log10(1 + avgVolume) * 10).toInt()
return volume
}
/**
* 创建录音器
*/
@SuppressLint("MissingPermission")
private fun createAudioRecord() {
if (isRecording.get()) {
return
}
if (audioRecord == null) {
Log.d(TAG, "createAudioRecord")
audioRecord = AudioRecord(
MediaRecorder.AudioSource.MIC,
16000,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
BUFFER_SIZE
)
}
}
/**
* 开始会话
*/
private fun start(): Int {
if (!keyword2File()) {
CallerLogger.d(TAG,"唤醒词文件写入失败，请检查是否有读写权限。")
return -1
}
val customBuilder = AiRequest.builder()
customBuilder.customText("key_word", getIvwDir() + "/keyword.txt", 0)
var ret = AiHelper.getInst().loadData(ABILITYID, customBuilder.build())
if (ret != 0) {
CallerLogger.d(TAG,"open ivw loadData 失败：$ret")
return ret
}
CallerLogger.d(TAG,"open ivw loadData success：$ret")
val indexs = intArrayOf(0)
ret = AiHelper.getInst().specifyDataSet(ABILITYID, "key_word", indexs) //从缓存中把个性化资源设置到引擎中
if (ret != 0) {
CallerLogger.d(TAG,"open ivw specifyDataSet 失败：$ret")
return ret
}
CallerLogger.d(TAG,"open ivw specifyDataSet success：$ret")
val paramBuilder = AiRequest.builder()
paramBuilder.param("wdec_param_nCmThreshold", "0 0:800")
paramBuilder.param("gramLoad", true)
isEnd.set(false)
aiHandle = AiHelper.getInst().start(ABILITYID, paramBuilder.build(), null)
if (aiHandle?.code != 0) {
CallerLogger.d(TAG,"open ivw start失败：${aiHandle?.getCode()}")
return aiHandle?.code ?:0
}
return 0
}
private fun keyword2File(): Boolean {
try {
val keywordFile: File = File(getIvwDir() + "/keyword.txt")
if (keywordFile.exists()) {
//强制清空内容
keywordFile.delete()
}
val binFile: File = File(getIvwDir() + "/keyword.bin")
if (binFile.exists()) {
binFile.delete()
}
var temp: String =weakupKey
if (temp.isEmpty()) {
temp = "你好小智"
}
val str = temp.replace("，", ",")
val keywords = str.split(",".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray()
if (!keywordFile.exists()) {
keywordFile.createNewFile()
}
val writer = OutputStreamWriter(
FileOutputStream(keywordFile),
"UTF-8"
)
val bufferedWriter = BufferedWriter(writer)
for (i in keywords.indices) {
bufferedWriter.write(keywords[i])
bufferedWriter.write(";")
bufferedWriter.newLine() //写入换行
}
bufferedWriter.close()
} catch (e: IOException) {
e.printStackTrace()
return false
}
return true
}
fun initWakeAi(context: Context) {
try {
val workDir = getWorkDir()
copyFile(workDir)
val file = File(workDir)
if(!file.exists()){
file.mkdirs()
}
AiHelper.getInst().setLogInfo(LogLvl.VERBOSE, 1, "${workDir}aeeLog.txt")
//设定初始化参数
val params = BaseLibrary.Params.builder()
.appId("0c498b42") //您的应用ID，可从控制台查看
.apiKey("8579f566eb7f3c4f4a07148ad9e2408c") //您的APIKEY，可从控制台查看
.apiSecret("NTRmMmI5MWI4NzIzZTIxN2Q5N2FjMWVl") //您的APISECRET，可从控制台查看
.workDir(workDir) //SDK的工作目录，需要确保有读写权限。一般用于存放离线能力资源，日志存放目录等使用。
.build()
AiHelper.getInst().registerListener(coreListener) // 注册SDK 初始化状态监听
AiHelper.getInst().initEntry(context, params)
}catch (e:Exception){
CallerLogger.d(TAG,"初始化失败:${e.message}")
}
}
private fun copyFile(workDir: String) {
ThreadUtils.getIoPool().submit{
val dir = "ivw"
val targetDir = workDir + File.separator + dir
if(checkCopied(dir,targetDir)) {
FileUtils.deleteAllInDir(targetDir)
ResourceUtils.copyFileFromAssets(dir, targetDir)
CallerLogger.d(TAG,"copy 完成")
}else{
CallerLogger.d(TAG,"文件已经存在 不需要copy")
}
}
}
private fun checkCopied(dir: String, targetDir: String): Boolean {
val assets = Utils.getApp().assets.list(dir)?.toMutableList()
val tartgetFile = File(targetDir)
if(tartgetFile.isDirectory){
tartgetFile.listFiles { dir, name ->
(assets?.contains(name)?:true).apply {
assets?.remove(name)
}
}
return assets?.isNotEmpty()?:false
}
return true
}
fun getWorkDir():String{
AbsMogoApplication.getApp()?.let {
return it.filesDir.path + File.separator + "iflytek" + File.separator
}
throw IllegalArgumentException("找不到 Application")
}
fun getIvwDir(): String {
return getWorkDir()+File.separator+"ivw"
}
}

5
libraries/mogo-speech/src/main/java/com/mogo/mgintelligent/speech/iflytek/WakeUpListener.kt Normal file
View File

@@ -0,0 +1,5 @@
package com.mogo.mgintelligent.speech.iflytek
interface WakeUpListener {
fun wakeupSuccess()
}