yangyakun921/MoGoEagleEye
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[8.1.0][adas] 新增VLLM 接口，新增VLLM图像接口，VLLM数据与VLLM图像创建同一个新线程

Browse Source
This commit is contained in:
xinfengkun
2025-06-19 11:01:29 +08:00
parent b63a24dc73
commit ac89e55d06
12 changed files with 288 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
core/function-impl/mogo-core-function-datacenter/src/main/java/com/mogo/eagle/core/function/datacenter/autopilot/adapter/MoGoAdasListenerImpl.kt
View File

@@ -86,6 +86,7 @@ import com.mogo.eagle.core.function.call.autopilot.CallerSweeperFutianCloudTaskL
import com.mogo.eagle.core.function.call.autopilot.CallerTakeoverListenerManager
import com.mogo.eagle.core.function.call.autopilot.CallerV2XListenerManager
import com.mogo.eagle.core.function.call.autopilot.CallerV2nNioEventListenerManager
import com.mogo.eagle.core.function.call.autopilot.CallerVlmManager
import com.mogo.eagle.core.function.call.devatools.CallerCaptureImgManager
import com.mogo.eagle.core.function.call.devatools.CallerDiskCopyManager
import com.mogo.eagle.core.function.call.devatools.CallerNDECloudManager
@@ -152,6 +153,7 @@ import prediction2025.Prediction2025
import record_cache.RecordPanelOuterClass
import system_master.SsmInfo
import system_master.SystemStatusInfo
import vllm.Vlm
import java.io.PrintWriter
import java.io.StringWriter
import kotlin.math.roundToInt
@@ -1612,6 +1614,26 @@ class MoGoAdasListenerImpl : OnAdasListener {
CallerNDECloudManager.onNdeCloudAdviceAvw(advicePojo,adviceAvwInfo)
}
/**
* 视觉语言模型
*
* @param header 头
* @param vllm 数据
*/
override fun onVllm(header: MessagePad.Header, vllm: Vlm.VLLMObject) {
CallerVlmManager.invokeVllm(header.sourceTimestamp, vllm)
}
/**
* 视觉语言模型图像
*
* @param header 头
* @param image 数据
*/
override fun onVllmImage(header: MessagePad.Header, image: ByteArray) {
CallerVlmManager.invokeVllmImage(header.sourceTimestamp, image)
}
/**
* 域控上报OBU开关状态响应
*

27
core/mogo-core-function-api/src/main/java/com/mogo/eagle/core/function/api/autopilot/IVlmListener.kt Normal file
View File

@@ -0,0 +1,27 @@
package com.mogo.eagle.core.function.api.autopilot
import vllm.Vlm
/**
* 视觉语言模型
*/
interface IVlmListener {
/**
* 视觉语言模型数据
*
* @param sourceTimestamp 数据源时间戳 vlm数据与vlm图像的数据源时间戳完全一致则表示为同一条数据
* @param vllm 数据
*/
fun onVllm(sourceTimestamp: Double, vllm: Vlm.VLLMObject)
/**
* 视觉语言模型图像
*
* @param sourceTimestamp 数据源时间戳 vlm数据与vlm图像的数据源时间戳完全一致则表示为同一条数据
* @param image 数据
*/
fun onVllmImage(sourceTimestamp: Double, image: ByteArray)
}

26
core/mogo-core-function-call/src/main/java/com/mogo/eagle/core/function/call/autopilot/CallerVlmManager.kt Normal file
View File

@@ -0,0 +1,26 @@
package com.mogo.eagle.core.function.call.autopilot
import com.mogo.eagle.core.function.api.autopilot.IVlmListener
import com.mogo.eagle.core.function.call.base.CallerBase
import vllm.Vlm
/**
* 视觉语言模型
*/
object CallerVlmManager : CallerBase<IVlmListener>() {
fun invokeVllm(sourceTimestamp: Double, vllm: Vlm.VLLMObject) {
M_LISTENERS.forEach {
val listener = it.value
listener.onVllm(sourceTimestamp, vllm)
}
}
fun invokeVllmImage(sourceTimestamp: Double, image: ByteArray) {
M_LISTENERS.forEach {
val listener = it.value
listener.onVllmImage(sourceTimestamp, image)
}
}
}

2
libraries/mogo-adas-data/src/main/java/com/zhjt/mogo/adas/common/MessageType.java
View File

@@ -34,6 +34,8 @@ public enum MessageType {
TYPE_RECEIVE_LOC_STATE(MessagePad.MessageType.MsgTypeLocState, "定位状态"),
TYPE_RECEIVE_CLOUD_REGULATORY_WARNING(MessagePad.MessageType.MsgTypeCloudRegulatoryWarning, "云控监管预警"),
TYPE_RECEIVE_PREDICTION_OBSTACLE_TRAJECTORY2025(MessagePad.MessageType.MsgTypePredictionObjects2025, "障碍物轨迹预测"),
TYPE_RECEIVE_VLLM(MessagePad.MessageType.MsgTypevllm, "视觉语言模型"),
TYPE_RECEIVE_VLLM_IMAGE(MessagePad.MessageType.MsgTypeVllmImage60, "视觉语言模型图像"),
TYPE_RECEIVE_BASIC_INFO_REQ(MessagePad.MessageType.MsgTypeBasicInfoReq, "自动驾驶设备基础信息请求"),
TYPE_SEND_BASIC_INFO_RESP(MessagePad.MessageType.MsgTypeBasicInfoResp, "自动驾驶设备基础信息应答"),

2
libraries/mogo-adas-data/src/main/proto/message_pad.proto
View File

@@ -48,6 +48,8 @@ enum MessageType
MsgTypeLocState = 0x10012; //定位呈现状态透传 用于pad图标显示 1hz 所有车型MAP440开始支持
MsgTypeCloudRegulatoryWarning = 0x10013; //云控监管预警信息上报
MsgTypePredictionObjects2025 = 0x10015; //自车他车轨迹预测2025 定频10hz
MsgTypevllm = 0x10016; //vllm 定频 5秒一条
MsgTypeVllmImage60 = 0x10017; //vllm图像 定频 5秒一条
//### 以下消息全部不定频 ###
MsgTypeBasicInfoReq = 0x10100; //自动驾驶设备基础信息请求

67
libraries/mogo-adas-data/src/main/proto/vlm.proto Normal file
View File

@@ -0,0 +1,67 @@
syntax = "proto2";
package vllm;
// common
import "header.proto";
message PathPoint {
optional double x = 1;
optional double y = 2;
optional double z = 3;
optional double yaw = 4;
}
message Trajectory {
optional double confidence = 1;
repeated PathPoint path_point = 2;
}
// vllm high command
message LonMetaAction {
enum Type {
COMMON = 0;
DECELERATE = 1;
}
optional Type type = 1 [default = COMMON];
}
message LatMetaAction {
enum Type{
KEEPLANE = 0;
LEFTTURN = 1;
RIGHTTURN = 2;
}
optional Type type = 1 [default = KEEPLANE];
}
message SceneLocation {
enum Type {
NONE = 0;
FRONTLEFT = 1;
AHEAD = 2;
FRONTRIGHT = 3;
}
optional Type type = 1 [default = NONE];
}
message WorkZone {
optional int32 isWorkZone = 1;
optional SceneLocation sceneLocation = 2;
optional LatMetaAction latMetaAction = 3;
optional LonMetaAction lonMetaAction = 4;
optional string sceneExplantion = 5;
optional int32 isWater = 6;
optional int32 isBranch = 7;
optional int32 isFence = 8;
optional int32 id = 9;
// repeated Trajectory trajectory = 6;
}
message VLLMObject {
optional common.Header header = 1;
optional WorkZone workZone = 2; // workZone
}

5
libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/AdasChannel.java
View File

@@ -321,6 +321,11 @@ public class AdasChannel implements IAdasNetCommApi, FpgaSocket.IWebSocketConnec
dispatchHandlers.put(MessagePad.MessageType.MsgTypeFSM2024State, new DispatchHandler(MessagePad.MessageType.MsgTypeFSM2024State, this));
//NED鄂州pad通信（不仅限于鄂州）
dispatchHandlers.put(MessagePad.MessageType.MsgTypeEzhouCloud, new DispatchHandler(MessagePad.MessageType.MsgTypeEzhouCloud, this));
//VLLM两个接口的线程 用同一个
DispatchHandler vllmDispatch = new DispatchHandler(MessagePad.MessageType.MsgTypevllm, this);
dispatchHandlers.put(MessagePad.MessageType.MsgTypevllm, vllmDispatch);
dispatchHandlers.put(MessagePad.MessageType.MsgTypeVllmImage60, vllmDispatch);
}
private void initSsmDispatch(AdasConstants.SsmSource source) {

17
libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/OnAdasListener.java
View File

@@ -57,6 +57,7 @@ import prediction2025.Prediction2025;
import record_cache.RecordPanelOuterClass;
import system_master.SsmInfo;
import system_master.SystemStatusInfo;
import vllm.Vlm;
/**
* @ProjectName: lib-adas-fpga
@@ -753,6 +754,22 @@ public interface OnAdasListener {
*/
void onNdeCloudAdviceAvw(@NonNull MessagePad.Header header, @NonNull AdvicePojo advicePojo, @NonNull AdviceAvwInfo adviceAvwInfo);
/**
* 视觉语言模型
*
* @param header 头
* @param vllm 数据
*/
void onVllm(@NonNull MessagePad.Header header, @NonNull Vlm.VLLMObject vllm);
/**
* 视觉语言模型图像
*
* @param header 头
* @param image 数据
*/
void onVllmImage(@NonNull MessagePad.Header header, @NonNull byte[] image);
/**
* 域控上报OBU开关状态响应
*

21
libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/MyMessageFactory.java
View File

@@ -60,6 +60,9 @@ public class MyMessageFactory implements IMyMessageFactory {
private IMsg copyBagMessage;//数据落盘
private IMsg cloudConfigMessage;//云端配置
private IMsg imgUploadCloudStatusRespMessage;//摄像头上传NDE云状态响应
private IMsg vlmMessage;//视觉语言模型
private IMsg vlmMessageImage;//视觉语言模型图像
private IMsg obuUploadStatusMessage;//域控上报OBU开关状态响应
private final AutopilotReview autopilotReview;
private final TurnLightState lightLeft = new TurnLightState();
@@ -352,6 +355,24 @@ public class MyMessageFactory implements IMyMessageFactory {
imgUploadCloudStatusRespMessage = new ImgUploadCloudStatusRespMessage();
}
return imgUploadCloudStatusRespMessage;
} else if (messageType == MessageType.TYPE_RECEIVE_VLLM.typeCode) {
//视觉语言模型
if (vlmMessage == null) {
vlmMessage = new VlmMessage();
}
return vlmMessage;
} else if (messageType == MessageType.TYPE_RECEIVE_VLLM_IMAGE.typeCode) {
//视觉语言模型
if (vlmMessageImage == null) {
vlmMessageImage = new VllmImageMessage();
}
return vlmMessageImage;
} else if (messageType == MessageType.TYPE_RECEIVE_OBU_UPLOAD_STATUS.typeCode) {
//域控上报OBU开关状态响应
if (obuUploadStatusMessage == null) {
obuUploadStatusMessage = new ObuUploadStatusMessage();
}
return obuUploadStatusMessage;
} else {
//MessageType.TYPE_DEFAULT.typeCode
return null;

34
libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/ObuUploadStatusMessage.java Normal file
View File

@@ -0,0 +1,34 @@
package com.zhidao.support.adas.high.msg;
import android.os.SystemClock;
import com.google.protobuf.InvalidProtocolBufferException;
import com.zhidao.support.adas.high.AdasChannel;
import com.zhidao.support.adas.high.OnAdasListener;
import com.zhidao.support.adas.high.common.CupidLogUtils;
import com.zhidao.support.adas.high.protocol.RawData;
import mogo.telematics.pad.MessagePad;
/**
* 域控上报OBU开关状态响应
*/
public class ObuUploadStatusMessage extends MyAbstractMessageHandler {
public ObuUploadStatusMessage() {
}
@Override
public void handlerMsg(RawData raw, OnAdasListener adasListener) throws InvalidProtocolBufferException {
MessagePad.SetEnableReq setEnableReq = MessagePad.SetEnableReq.parser().parseFrom(raw.originalData.toByteArray(), raw.getOffsetValue(), raw.getPackageLengthValue() - raw.getOffsetValue());
AdasChannel.calculateTimeConsumingOnDispatchRaw("域控上报OBU开关状态响应", raw.receiveTime);
long nowTime = 0;
if (CupidLogUtils.isEnableLog())
nowTime = SystemClock.elapsedRealtime();
if (adasListener != null) {
adasListener.onObuUploadStatus(raw.getHeader(), setEnableReq);
}
AdasChannel.calculateTimeConsumingBusiness("域控上报OBU开关状态响应", nowTime);
}
}

33
libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/VllmImageMessage.java Normal file
View File

@@ -0,0 +1,33 @@
package com.zhidao.support.adas.high.msg;
import android.os.SystemClock;
import com.google.protobuf.InvalidProtocolBufferException;
import com.zhidao.support.adas.high.AdasChannel;
import com.zhidao.support.adas.high.OnAdasListener;
import com.zhidao.support.adas.high.common.CupidLogUtils;
import com.zhidao.support.adas.high.protocol.RawData;
/**
* 视觉语言模型 图像接口
*/
public class VllmImageMessage extends MyAbstractMessageHandler {
@Override
public void handlerMsg(RawData raw, OnAdasListener adasListener) throws InvalidProtocolBufferException {
int len = raw.getPackageLengthValue() - raw.getOffsetValue();
byte[] data = new byte[len];
System.arraycopy(raw.originalData.toByteArray(), raw.getOffsetValue(), data, 0, len);
AdasChannel.calculateTimeConsumingOnDispatchRaw("视觉语言模型图像", raw.receiveTime);
long nowTime = 0;
if (CupidLogUtils.isEnableLog())
nowTime = SystemClock.elapsedRealtime();
if (adasListener != null) {
adasListener.onVllmImage(raw.getHeader(), data);
}
AdasChannel.calculateTimeConsumingBusiness("视觉语言模型图像", nowTime);
}
}

32
libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/VlmMessage.java Normal file
View File

@@ -0,0 +1,32 @@
package com.zhidao.support.adas.high.msg;
import android.os.SystemClock;
import com.google.protobuf.InvalidProtocolBufferException;
import com.zhidao.support.adas.high.AdasChannel;
import com.zhidao.support.adas.high.OnAdasListener;
import com.zhidao.support.adas.high.common.CupidLogUtils;
import com.zhidao.support.adas.high.protocol.RawData;
import vllm.Vlm;
/**
* 视觉语言模型
*/
public class VlmMessage extends MyAbstractMessageHandler {
@Override
public void handlerMsg(RawData raw, OnAdasListener adasListener) throws InvalidProtocolBufferException {
Vlm.VLLMObject vllm = Vlm.VLLMObject.parser().parseFrom(raw.originalData.toByteArray(), raw.getOffsetValue(), raw.getPackageLengthValue() - raw.getOffsetValue());
AdasChannel.calculateTimeConsumingOnDispatchRaw("视觉语言模型", raw.receiveTime);
long nowTime = 0;
if (CupidLogUtils.isEnableLog())
nowTime = SystemClock.elapsedRealtime();
if (adasListener != null) {
adasListener.onVllm(raw.getHeader(), vllm);
}
AdasChannel.calculateTimeConsumingBusiness("视觉语言模型", nowTime);
// CupidLogUtils.e("到站提醒--->" + arrivalNotification.toString());
}
}