From ac89e55d065230da822d298eeb786b3a7a0ee51b Mon Sep 17 00:00:00 2001 From: xinfengkun Date: Thu, 19 Jun 2025 11:01:29 +0800 Subject: [PATCH] =?UTF-8?q?[8.1.0][adas]=20=E6=96=B0=E5=A2=9EVLLM=20?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=EF=BC=8C=E6=96=B0=E5=A2=9EVLLM=E5=9B=BE?= =?UTF-8?q?=E5=83=8F=E6=8E=A5=E5=8F=A3=EF=BC=8CVLLM=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=B8=8EVLLM=E5=9B=BE=E5=83=8F=E5=88=9B=E5=BB=BA=E5=90=8C?= =?UTF-8?q?=E4=B8=80=E4=B8=AA=E6=96=B0=E7=BA=BF=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../autopilot/adapter/MoGoAdasListenerImpl.kt | 22 ++++++ .../function/api/autopilot/IVlmListener.kt | 27 ++++++++ .../call/autopilot/CallerVlmManager.kt | 26 +++++++ .../zhjt/mogo/adas/common/MessageType.java | 2 + .../src/main/proto/message_pad.proto | 2 + .../mogo-adas-data/src/main/proto/vlm.proto | 67 +++++++++++++++++++ .../zhidao/support/adas/high/AdasChannel.java | 5 ++ .../support/adas/high/OnAdasListener.java | 17 +++++ .../adas/high/msg/MyMessageFactory.java | 21 ++++++ .../adas/high/msg/ObuUploadStatusMessage.java | 34 ++++++++++ .../adas/high/msg/VllmImageMessage.java | 33 +++++++++ .../support/adas/high/msg/VlmMessage.java | 32 +++++++++ 12 files changed, 288 insertions(+) create mode 100644 core/mogo-core-function-api/src/main/java/com/mogo/eagle/core/function/api/autopilot/IVlmListener.kt create mode 100644 core/mogo-core-function-call/src/main/java/com/mogo/eagle/core/function/call/autopilot/CallerVlmManager.kt create mode 100644 libraries/mogo-adas-data/src/main/proto/vlm.proto create mode 100644 libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/ObuUploadStatusMessage.java create mode 100644 libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/VllmImageMessage.java create mode 100644 libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/VlmMessage.java diff --git a/core/function-impl/mogo-core-function-datacenter/src/main/java/com/mogo/eagle/core/function/datacenter/autopilot/adapter/MoGoAdasListenerImpl.kt b/core/function-impl/mogo-core-function-datacenter/src/main/java/com/mogo/eagle/core/function/datacenter/autopilot/adapter/MoGoAdasListenerImpl.kt index 05218db30e..60b686678b 100644 --- a/core/function-impl/mogo-core-function-datacenter/src/main/java/com/mogo/eagle/core/function/datacenter/autopilot/adapter/MoGoAdasListenerImpl.kt +++ b/core/function-impl/mogo-core-function-datacenter/src/main/java/com/mogo/eagle/core/function/datacenter/autopilot/adapter/MoGoAdasListenerImpl.kt @@ -86,6 +86,7 @@ import com.mogo.eagle.core.function.call.autopilot.CallerSweeperFutianCloudTaskL import com.mogo.eagle.core.function.call.autopilot.CallerTakeoverListenerManager import com.mogo.eagle.core.function.call.autopilot.CallerV2XListenerManager import com.mogo.eagle.core.function.call.autopilot.CallerV2nNioEventListenerManager +import com.mogo.eagle.core.function.call.autopilot.CallerVlmManager import com.mogo.eagle.core.function.call.devatools.CallerCaptureImgManager import com.mogo.eagle.core.function.call.devatools.CallerDiskCopyManager import com.mogo.eagle.core.function.call.devatools.CallerNDECloudManager @@ -152,6 +153,7 @@ import prediction2025.Prediction2025 import record_cache.RecordPanelOuterClass import system_master.SsmInfo import system_master.SystemStatusInfo +import vllm.Vlm import java.io.PrintWriter import java.io.StringWriter import kotlin.math.roundToInt @@ -1612,6 +1614,26 @@ class MoGoAdasListenerImpl : OnAdasListener { CallerNDECloudManager.onNdeCloudAdviceAvw(advicePojo,adviceAvwInfo) } + /** + * 视觉语言模型 + * + * @param header 头 + * @param vllm 数据 + */ + override fun onVllm(header: MessagePad.Header, vllm: Vlm.VLLMObject) { + CallerVlmManager.invokeVllm(header.sourceTimestamp, vllm) + } + + /** + * 视觉语言模型图像 + * + * @param header 头 + * @param image 数据 + */ + override fun onVllmImage(header: MessagePad.Header, image: ByteArray) { + CallerVlmManager.invokeVllmImage(header.sourceTimestamp, image) + } + /** * 域控上报OBU开关状态响应 * diff --git a/core/mogo-core-function-api/src/main/java/com/mogo/eagle/core/function/api/autopilot/IVlmListener.kt b/core/mogo-core-function-api/src/main/java/com/mogo/eagle/core/function/api/autopilot/IVlmListener.kt new file mode 100644 index 0000000000..883008890b --- /dev/null +++ b/core/mogo-core-function-api/src/main/java/com/mogo/eagle/core/function/api/autopilot/IVlmListener.kt @@ -0,0 +1,27 @@ +package com.mogo.eagle.core.function.api.autopilot + + +import vllm.Vlm + +/** + * 视觉语言模型 + */ +interface IVlmListener { + + + /** + * 视觉语言模型数据 + * + * @param sourceTimestamp 数据源时间戳 vlm数据与vlm图像的数据源时间戳完全一致则表示为同一条数据 + * @param vllm 数据 + */ + fun onVllm(sourceTimestamp: Double, vllm: Vlm.VLLMObject) + + /** + * 视觉语言模型图像 + * + * @param sourceTimestamp 数据源时间戳 vlm数据与vlm图像的数据源时间戳完全一致则表示为同一条数据 + * @param image 数据 + */ + fun onVllmImage(sourceTimestamp: Double, image: ByteArray) +} \ No newline at end of file diff --git a/core/mogo-core-function-call/src/main/java/com/mogo/eagle/core/function/call/autopilot/CallerVlmManager.kt b/core/mogo-core-function-call/src/main/java/com/mogo/eagle/core/function/call/autopilot/CallerVlmManager.kt new file mode 100644 index 0000000000..fcbc8241d5 --- /dev/null +++ b/core/mogo-core-function-call/src/main/java/com/mogo/eagle/core/function/call/autopilot/CallerVlmManager.kt @@ -0,0 +1,26 @@ +package com.mogo.eagle.core.function.call.autopilot + +import com.mogo.eagle.core.function.api.autopilot.IVlmListener +import com.mogo.eagle.core.function.call.base.CallerBase +import vllm.Vlm + +/** + * 视觉语言模型 + */ +object CallerVlmManager : CallerBase() { + + + fun invokeVllm(sourceTimestamp: Double, vllm: Vlm.VLLMObject) { + M_LISTENERS.forEach { + val listener = it.value + listener.onVllm(sourceTimestamp, vllm) + } + } + + fun invokeVllmImage(sourceTimestamp: Double, image: ByteArray) { + M_LISTENERS.forEach { + val listener = it.value + listener.onVllmImage(sourceTimestamp, image) + } + } +} \ No newline at end of file diff --git a/libraries/mogo-adas-data/src/main/java/com/zhjt/mogo/adas/common/MessageType.java b/libraries/mogo-adas-data/src/main/java/com/zhjt/mogo/adas/common/MessageType.java index 0325ea343f..3892c83e6b 100644 --- a/libraries/mogo-adas-data/src/main/java/com/zhjt/mogo/adas/common/MessageType.java +++ b/libraries/mogo-adas-data/src/main/java/com/zhjt/mogo/adas/common/MessageType.java @@ -34,6 +34,8 @@ public enum MessageType { TYPE_RECEIVE_LOC_STATE(MessagePad.MessageType.MsgTypeLocState, "定位状态"), TYPE_RECEIVE_CLOUD_REGULATORY_WARNING(MessagePad.MessageType.MsgTypeCloudRegulatoryWarning, "云控监管预警"), TYPE_RECEIVE_PREDICTION_OBSTACLE_TRAJECTORY2025(MessagePad.MessageType.MsgTypePredictionObjects2025, "障碍物轨迹预测"), + TYPE_RECEIVE_VLLM(MessagePad.MessageType.MsgTypevllm, "视觉语言模型"), + TYPE_RECEIVE_VLLM_IMAGE(MessagePad.MessageType.MsgTypeVllmImage60, "视觉语言模型图像"), TYPE_RECEIVE_BASIC_INFO_REQ(MessagePad.MessageType.MsgTypeBasicInfoReq, "自动驾驶设备基础信息请求"), TYPE_SEND_BASIC_INFO_RESP(MessagePad.MessageType.MsgTypeBasicInfoResp, "自动驾驶设备基础信息应答"), diff --git a/libraries/mogo-adas-data/src/main/proto/message_pad.proto b/libraries/mogo-adas-data/src/main/proto/message_pad.proto index 36b1a49fec..496ece746f 100644 --- a/libraries/mogo-adas-data/src/main/proto/message_pad.proto +++ b/libraries/mogo-adas-data/src/main/proto/message_pad.proto @@ -48,6 +48,8 @@ enum MessageType MsgTypeLocState = 0x10012; //定位呈现状态透传 用于pad图标显示 1hz 所有车型MAP440开始支持 MsgTypeCloudRegulatoryWarning = 0x10013; //云控监管预警信息上报 MsgTypePredictionObjects2025 = 0x10015; //自车他车轨迹预测2025 定频10hz + MsgTypevllm = 0x10016; //vllm 定频 5秒一条 + MsgTypeVllmImage60 = 0x10017; //vllm图像 定频 5秒一条 //### 以下消息全部不定频 ### MsgTypeBasicInfoReq = 0x10100; //自动驾驶设备基础信息请求 diff --git a/libraries/mogo-adas-data/src/main/proto/vlm.proto b/libraries/mogo-adas-data/src/main/proto/vlm.proto new file mode 100644 index 0000000000..25b68c8b6f --- /dev/null +++ b/libraries/mogo-adas-data/src/main/proto/vlm.proto @@ -0,0 +1,67 @@ +syntax = "proto2"; + +package vllm; + +// common +import "header.proto"; + + + +message PathPoint { + optional double x = 1; + optional double y = 2; + optional double z = 3; + optional double yaw = 4; +} + +message Trajectory { + optional double confidence = 1; + repeated PathPoint path_point = 2; +} + +// vllm high command +message LonMetaAction { + enum Type { + COMMON = 0; + DECELERATE = 1; + } + optional Type type = 1 [default = COMMON]; +} + +message LatMetaAction { + enum Type{ + KEEPLANE = 0; + LEFTTURN = 1; + RIGHTTURN = 2; + } + optional Type type = 1 [default = KEEPLANE]; +} + +message SceneLocation { + enum Type { + NONE = 0; + FRONTLEFT = 1; + AHEAD = 2; + FRONTRIGHT = 3; + } + optional Type type = 1 [default = NONE]; +} + + +message WorkZone { + optional int32 isWorkZone = 1; + optional SceneLocation sceneLocation = 2; + optional LatMetaAction latMetaAction = 3; + optional LonMetaAction lonMetaAction = 4; + optional string sceneExplantion = 5; + optional int32 isWater = 6; + optional int32 isBranch = 7; + optional int32 isFence = 8; + optional int32 id = 9; + // repeated Trajectory trajectory = 6; +} + +message VLLMObject { + optional common.Header header = 1; + optional WorkZone workZone = 2; // workZone +} diff --git a/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/AdasChannel.java b/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/AdasChannel.java index e13dde7037..f6db68ae6b 100644 --- a/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/AdasChannel.java +++ b/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/AdasChannel.java @@ -321,6 +321,11 @@ public class AdasChannel implements IAdasNetCommApi, FpgaSocket.IWebSocketConnec dispatchHandlers.put(MessagePad.MessageType.MsgTypeFSM2024State, new DispatchHandler(MessagePad.MessageType.MsgTypeFSM2024State, this)); //NED鄂州pad通信（不仅限于鄂州） dispatchHandlers.put(MessagePad.MessageType.MsgTypeEzhouCloud, new DispatchHandler(MessagePad.MessageType.MsgTypeEzhouCloud, this)); + + //VLLM两个接口的线程 用同一个 + DispatchHandler vllmDispatch = new DispatchHandler(MessagePad.MessageType.MsgTypevllm, this); + dispatchHandlers.put(MessagePad.MessageType.MsgTypevllm, vllmDispatch); + dispatchHandlers.put(MessagePad.MessageType.MsgTypeVllmImage60, vllmDispatch); } private void initSsmDispatch(AdasConstants.SsmSource source) { diff --git a/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/OnAdasListener.java b/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/OnAdasListener.java index 51ccb7fbdc..ce1670483c 100644 --- a/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/OnAdasListener.java +++ b/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/OnAdasListener.java @@ -57,6 +57,7 @@ import prediction2025.Prediction2025; import record_cache.RecordPanelOuterClass; import system_master.SsmInfo; import system_master.SystemStatusInfo; +import vllm.Vlm; /** * @ProjectName: lib-adas-fpga @@ -753,6 +754,22 @@ public interface OnAdasListener { */ void onNdeCloudAdviceAvw(@NonNull MessagePad.Header header, @NonNull AdvicePojo advicePojo, @NonNull AdviceAvwInfo adviceAvwInfo); + /** + * 视觉语言模型 + * + * @param header 头 + * @param vllm 数据 + */ + void onVllm(@NonNull MessagePad.Header header, @NonNull Vlm.VLLMObject vllm); + + /** + * 视觉语言模型图像 + * + * @param header 头 + * @param image 数据 + */ + void onVllmImage(@NonNull MessagePad.Header header, @NonNull byte[] image); + /** * 域控上报OBU开关状态响应 * diff --git a/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/MyMessageFactory.java b/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/MyMessageFactory.java index 2105635501..f2351c52ab 100644 --- a/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/MyMessageFactory.java +++ b/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/MyMessageFactory.java @@ -60,6 +60,9 @@ public class MyMessageFactory implements IMyMessageFactory { private IMsg copyBagMessage;//数据落盘 private IMsg cloudConfigMessage;//云端配置 private IMsg imgUploadCloudStatusRespMessage;//摄像头上传NDE云状态响应 + private IMsg vlmMessage;//视觉语言模型 + private IMsg vlmMessageImage;//视觉语言模型图像 + private IMsg obuUploadStatusMessage;//域控上报OBU开关状态响应 private final AutopilotReview autopilotReview; private final TurnLightState lightLeft = new TurnLightState(); @@ -352,6 +355,24 @@ public class MyMessageFactory implements IMyMessageFactory { imgUploadCloudStatusRespMessage = new ImgUploadCloudStatusRespMessage(); } return imgUploadCloudStatusRespMessage; + } else if (messageType == MessageType.TYPE_RECEIVE_VLLM.typeCode) { + //视觉语言模型 + if (vlmMessage == null) { + vlmMessage = new VlmMessage(); + } + return vlmMessage; + } else if (messageType == MessageType.TYPE_RECEIVE_VLLM_IMAGE.typeCode) { + //视觉语言模型 + if (vlmMessageImage == null) { + vlmMessageImage = new VllmImageMessage(); + } + return vlmMessageImage; + } else if (messageType == MessageType.TYPE_RECEIVE_OBU_UPLOAD_STATUS.typeCode) { + //域控上报OBU开关状态响应 + if (obuUploadStatusMessage == null) { + obuUploadStatusMessage = new ObuUploadStatusMessage(); + } + return obuUploadStatusMessage; } else { //MessageType.TYPE_DEFAULT.typeCode return null; diff --git a/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/ObuUploadStatusMessage.java b/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/ObuUploadStatusMessage.java new file mode 100644 index 0000000000..3935d97bba --- /dev/null +++ b/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/ObuUploadStatusMessage.java @@ -0,0 +1,34 @@ + +package com.zhidao.support.adas.high.msg; + +import android.os.SystemClock; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.zhidao.support.adas.high.AdasChannel; +import com.zhidao.support.adas.high.OnAdasListener; +import com.zhidao.support.adas.high.common.CupidLogUtils; +import com.zhidao.support.adas.high.protocol.RawData; + +import mogo.telematics.pad.MessagePad; + +/** + * 域控上报OBU开关状态响应 + */ +public class ObuUploadStatusMessage extends MyAbstractMessageHandler { + + public ObuUploadStatusMessage() { + } + + @Override + public void handlerMsg(RawData raw, OnAdasListener adasListener) throws InvalidProtocolBufferException { + MessagePad.SetEnableReq setEnableReq = MessagePad.SetEnableReq.parser().parseFrom(raw.originalData.toByteArray(), raw.getOffsetValue(), raw.getPackageLengthValue() - raw.getOffsetValue()); + AdasChannel.calculateTimeConsumingOnDispatchRaw("域控上报OBU开关状态响应", raw.receiveTime); + long nowTime = 0; + if (CupidLogUtils.isEnableLog()) + nowTime = SystemClock.elapsedRealtime(); + if (adasListener != null) { + adasListener.onObuUploadStatus(raw.getHeader(), setEnableReq); + } + AdasChannel.calculateTimeConsumingBusiness("域控上报OBU开关状态响应", nowTime); + } +} diff --git a/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/VllmImageMessage.java b/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/VllmImageMessage.java new file mode 100644 index 0000000000..2c239378a9 --- /dev/null +++ b/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/VllmImageMessage.java @@ -0,0 +1,33 @@ +package com.zhidao.support.adas.high.msg; + +import android.os.SystemClock; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.zhidao.support.adas.high.AdasChannel; +import com.zhidao.support.adas.high.OnAdasListener; +import com.zhidao.support.adas.high.common.CupidLogUtils; +import com.zhidao.support.adas.high.protocol.RawData; + +/** + * 视觉语言模型 图像接口 + */ +public class VllmImageMessage extends MyAbstractMessageHandler { + + + @Override + public void handlerMsg(RawData raw, OnAdasListener adasListener) throws InvalidProtocolBufferException { + int len = raw.getPackageLengthValue() - raw.getOffsetValue(); + byte[] data = new byte[len]; + System.arraycopy(raw.originalData.toByteArray(), raw.getOffsetValue(), data, 0, len); + AdasChannel.calculateTimeConsumingOnDispatchRaw("视觉语言模型图像", raw.receiveTime); + long nowTime = 0; + if (CupidLogUtils.isEnableLog()) + nowTime = SystemClock.elapsedRealtime(); + if (adasListener != null) { + + adasListener.onVllmImage(raw.getHeader(), data); + + } + AdasChannel.calculateTimeConsumingBusiness("视觉语言模型图像", nowTime); + } +} diff --git a/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/VlmMessage.java b/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/VlmMessage.java new file mode 100644 index 0000000000..257dcea1c0 --- /dev/null +++ b/libraries/mogo-adas/src/main/java/com/zhidao/support/adas/high/msg/VlmMessage.java @@ -0,0 +1,32 @@ +package com.zhidao.support.adas.high.msg; + +import android.os.SystemClock; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.zhidao.support.adas.high.AdasChannel; +import com.zhidao.support.adas.high.OnAdasListener; +import com.zhidao.support.adas.high.common.CupidLogUtils; +import com.zhidao.support.adas.high.protocol.RawData; + +import vllm.Vlm; + +/** + * 视觉语言模型 + */ +public class VlmMessage extends MyAbstractMessageHandler { + + @Override + public void handlerMsg(RawData raw, OnAdasListener adasListener) throws InvalidProtocolBufferException { + Vlm.VLLMObject vllm = Vlm.VLLMObject.parser().parseFrom(raw.originalData.toByteArray(), raw.getOffsetValue(), raw.getPackageLengthValue() - raw.getOffsetValue()); + AdasChannel.calculateTimeConsumingOnDispatchRaw("视觉语言模型", raw.receiveTime); + long nowTime = 0; + if (CupidLogUtils.isEnableLog()) + nowTime = SystemClock.elapsedRealtime(); + if (adasListener != null) { + adasListener.onVllm(raw.getHeader(), vllm); + } + AdasChannel.calculateTimeConsumingBusiness("视觉语言模型", nowTime); +// CupidLogUtils.e("到站提醒--->" + arrivalNotification.toString()); + } + +}