LMDrive大语言模型加持的自动驾驶闭环系统 原理与复现过程记录

LMdrive作为一个优秀的开源视觉语言驾驶模型,是未来技术的导向,官方:
https://github.com/opendilab/LMDrive?tab=readme-ov-file
论文:https://arxiv.org/abs/2312.07488

复现VLM

安装carla

本环境使用的carla_leaderboard 1.0 ，使用的carla版本是0.9.10.1
https://mirrors.sustech.edu.cn/carla/carla/

安装conda

安装conda时，先下载安装包
https://www.anaconda.com/download/success

直接下载x86的sh，然后运行
./Anaconda3-2024.06-1-Linux-x86_64.sh
安装完后创建一个虚拟环境，官方使用的3.8，支持torch cuda的版本
conda create -n python=3.8
克隆官方代码

git clone https://github.com/opendilab/LMDrive.git

首先是跟着官方的教程安装好vision_encoder等:

conda activate lmdrive

cd vision_encoder
pip3 install -r requirements.txt
python setup.py develop

cd ../LAVIS
pip3 install -r requirements.txt
python setup.py develop

pip install flash-attn --no-build-isolation

这两个库比较重要,所以优先安装好.

安装其他相关依赖，为了避免大家碰到问题，我这里直接将我的环境打包到requirement.txt中：
这里面有很多是ros的依赖项,所以不能直接去安装,请对照一下关键的python依赖就好

absl-py==2.1.0
accelerate==1.0.1
actionlib==1.14.0
addict==2.4.0
aiofiles==23.2.1
altair==5.4.1
angles==1.9.13
annotated-types==0.7.0
antlr4-python3-runtime==4.9.3
anyio==4.5.2
asttokens==3.0.0
attrs==25.1.0
backcall==0.2.0
base_local_planner==1.17.3
bleach==6.1.0
blinker==1.8.2
blis==0.7.11
bondpy==1.8.6
braceexpand==0.1.7
cachetools==5.5.2
camera-calibration==1.17.0
camera-calibration-parsers==1.12.0
catalogue==2.0.10
catkin==0.8.10
certifi==2025.1.31
cfgv==3.4.0
charset-normalizer==3.4.1
click==8.1.8
cmake==3.25.0
comm==0.2.2
confection==0.1.5
ConfigArgParse==1.7
contexttimer==0.3.3
contourpy==1.1.1
controller-manager==0.20.0
controller-manager-msgs==0.20.0
cv-bridge==1.16.2
cycler==0.12.1
cymem==2.0.11
dash==2.18.2
dash-core-components==2.0.0
dash-html-components==2.0.0
dash-table==5.0.0
decorator==5.2.1
decord==0.6.0
diagnostic-analysis==1.11.0
diagnostic-common-diagnostics==1.11.0
diagnostic-updater==1.11.0
dictor==0.1.12
diffusers==0.16.0
distlib==0.3.9
distro==1.9.0
docker-pycreds==0.4.0
dynamic-reconfigure==1.7.3
easydict==1.13
einops==0.8.1
elementpath==1.3.3
ephem==4.2
etils==1.3.0
eval_type_backport==0.2.2
exceptiongroup==1.2.2
executing==2.2.0
fairscale==0.4.4
Farama-Notifications==0.0.4
fastapi==0.115.11
fastjsonschema==2.21.1
ffmpy==0.5.0
filelock==3.13.1
Flask==3.0.3
fonttools==4.56.0
fsspec==2025.3.0
ftfy==6.2.3
future==1.0.0
gazebo_plugins==2.9.3
gazebo_ros==2.9.3
gencpp==0.7.0
geneus==3.0.0
genlisp==0.4.18
genmsg==0.6.0
gennodejs==2.0.2
genpy==0.6.15
gitdb==4.0.12
GitPython==3.1.44
glfw==2.8.0
gradio==4.44.1
gradio_client==1.3.0
gym==0.17.2
gym-notices==0.0.8
gymnasium==1.0.0
gymnasium-robotics==1.3.1
h11==0.14.0
h5py==2.10.0
httpcore==1.0.7
httpx==0.28.1
huggingface-hub==0.29.2
Hydra==2.5
identify==2.6.1
idna==3.10
image-geometry==1.16.2
imageio==2.35.1
importlib_metadata==8.5.0
importlib_resources==6.4.5
interactive-markers==1.12.0
iopath==0.1.10
ipython==8.12.3
ipywidgets==8.1.5
itsdangerous==2.2.0
jedi==0.19.2
Jinja2==3.1.4
jiter==0.8.2
joblib==1.4.2
joint-state-publisher==1.15.1
joint-state-publisher-gui==1.15.1
jsk-recognition-msgs==1.2.17
jsk-recognition-utils==1.2.17
jsk_rviz_plugins==2.1.10
jsk_topic_tools==2.2.15
jsonschema==4.23.0
jsonschema-specifications==2023.12.1
jupyter_core==5.7.2
jupyterlab_widgets==3.0.13
kaggle==1.6.17
kiwisolver==1.4.7
langcodes==3.4.1
language_data==1.3.0
laser_geometry==1.6.7
lazy_loader==0.4
lit==15.0.7
marisa-trie==1.2.1
markdown-it-py==3.0.0
MarkupSafe==2.1.5
matplotlib==3.5.3
matplotlib-inline==0.1.7
mdurl==0.1.2
message-filters==1.17.0
modelscope==1.23.2
mpmath==1.3.0
mujoco==3.1.6
murmurhash==1.0.12
narwhals==1.28.0
nbformat==5.10.4
nest-asyncio==1.6.0
networkx==3.1
nodeenv==1.9.1
numpy==1.21.0
nvidia-cublas-cu11==11.10.3.66
nvidia-cuda-cupti-cu11==11.7.101
nvidia-cuda-nvrtc-cu11==11.7.99
nvidia-cuda-runtime-cu11==11.7.99
nvidia-cudnn-cu11==8.5.0.96
nvidia-cufft-cu11==10.9.0.58
nvidia-curand-cu11==10.2.10.91
nvidia-cusolver-cu11==11.4.0.1
nvidia-cusparse-cu11==11.7.4.91
nvidia-nccl-cu11==2.14.3
nvidia-nvtx-cu11==11.7.91
omegaconf==2.3.0
open3d==0.19.0
openai==1.65.2
opencv-python==4.2.0.32
opencv-python-headless==4.5.5.64
opendatasets==0.1.22
orjson==3.10.15
packaging==24.2
pandas==2.0.3
parso==0.8.4
pathlib_abc==0.1.1
pathy==0.11.0
peft==0.13.2
pettingzoo==1.24.3
pexpect==4.9.0
pickleshare==0.7.5
pillow==10.4.0
pkgutil_resolve_name==1.3.10
platformdirs==4.3.6
plotly==6.0.0
portalocker==3.0.0
pre-commit==3.5.0
preshed==3.0.9
prompt_toolkit==3.0.50
protobuf==5.29.3
psutil==7.0.0
ptyprocess==0.7.0
pure_eval==0.2.3
py-cpuinfo==9.0.0
py-trees==0.8.3
pyarrow==17.0.0
pycocoevalcap==1.2
pycocotools==2.0.7
pydantic==2.10.6
pydantic_core==2.27.2
pydeck==0.9.1
pydot==3.0.4
pydub==0.25.1
pygame==2.4.0
pyglet==1.5.0
Pygments==2.19.1
pyparsing==3.1.4
pyquaternion==0.9.9
python-dateutil==2.9.0.post0
python-magic==0.4.27
python-multipart==0.0.20
python-qt-binding==0.4.4
python-slugify==8.0.4
python-tr==0.1.2
pytz==2025.1
PyWavelets==1.4.1
PyYAML==6.0.2
qt-dotgraph==0.4.2
qt-gui==0.4.2
qt-gui-cpp==0.4.2
qt-gui-py-common==0.4.2
referencing==0.35.1
regex==2024.11.6
requests==2.32.3
resource_retriever==1.12.8
retrying==1.3.4
rich==13.9.4
rosbag==1.17.0
rosboost-cfg==1.15.8
rosclean==1.15.8
roscreate==1.15.8
rosgraph==1.17.0
roslaunch==1.17.0
roslib==1.15.8
roslint==0.12.0
roslz4==1.17.0
rosmake==1.15.8
rosmaster==1.17.0
rosmsg==1.17.0
rosnode==1.17.0
rosparam==1.17.0
rospy==1.17.0
rospy_message_converter==0.5.9
rosservice==1.17.0
rostest==1.17.0
rostopic==1.17.0
rosunit==1.15.8
roswtf==1.17.0
rpds-py==0.20.1
rqt-console==0.4.12
rqt-image-view==0.4.17
rqt-logger-level==0.4.12
rqt-moveit==0.5.11
rqt-reconfigure==0.5.5
rqt-robot-dashboard==0.5.8
rqt-robot-monitor==0.5.15
rqt-runtime-monitor==0.5.10
rqt-rviz==0.7.0
rqt-tf-tree==0.6.4
rqt_action==0.4.9
rqt_bag==0.5.1
rqt_bag_plugins==0.5.1
rqt_dep==0.4.12
rqt_graph==0.4.14
rqt_gui==0.5.3
rqt_gui_py==0.5.3
rqt_launch==0.4.9
rqt_msg==0.4.10
rqt_nav_view==0.5.7
rqt_plot==0.4.13
rqt_pose_view==0.5.11
rqt_publisher==0.4.10
rqt_py_common==0.5.3
rqt_py_console==0.4.10
rqt_robot_steering==0.5.12
rqt_service_caller==0.4.10
rqt_shell==0.4.11
rqt_srv==0.4.9
rqt_top==0.4.10
rqt_topic==0.4.13
rqt_web==0.4.10
ruff==0.9.10
rviz==1.14.25
safetensors==0.5.3
# Editable install with no version control (salesforce-lavis==1.0.1)
# -e /home/cyun/Documents/LMDrive-main/LAVIS
scikit-image==0.21.0
scikit-learn==1.3.2
scipy==1.8.1
semantic-version==2.10.0
sensor-msgs==1.13.1
sentencepiece==0.2.0
sentry-sdk==2.22.0
setproctitle==1.3.5
shapely==2.0.7
shellingham==1.5.4
simple-watchdog-timer==0.1.1
six==1.17.0
smach==2.5.3
smach-ros==2.5.3
smart-open==6.4.0
smclib==1.8.6
smmap==5.0.2
sniffio==1.3.1
sound-play==0.3.18
spacy==3.5.1
spacy-legacy==3.0.12
spacy-loggers==1.0.5
srsly==2.4.8
stable_baselines3==2.4.1
stack-data==0.6.3
starlette==0.44.0
streamlit==1.40.1
sympy==1.13.1
tabulate==0.9.0
tenacity==9.0.0
text-unidecode==1.3
tf==1.13.2
tf-conversions==1.13.2
tf2-geometry-msgs==0.7.7
tf2-kdl==0.7.7
tf2-py==0.7.7
tf2-ros==0.7.7
thinc==8.1.12
threadpoolctl==3.5.0
tifffile==2023.7.10
# Editable install with no version control (timm==0.4.13)
# -e /home/cyun/Documents/LMDrive-main/vision_encoder
tokenizers==0.20.3
toml==0.10.2
tomlkit==0.12.0
topic-tools==1.17.0
torch==2.0.1+cu118
torch-scatter==2.1.2+pt20cu118
torchaudio==2.0.2+cu118
torchvision==0.15.2+cu118
tornado==6.4.2
tqdm==4.67.1
traitlets==5.14.3
transformers==4.46.3
triton==2.0.0
typer==0.15.2
typing_extensions==4.12.2
tzdata==2025.1
urdfdom-py==0.4.6
urllib3==2.2.3
uvicorn==0.33.0
virtualenv==20.29.2
wandb==0.19.8
wasabi==1.1.3
watchdog==4.0.2
wcwidth==0.2.13
webdataset==0.2.100
webencodings==0.5.1
websockets==12.0
Werkzeug==3.0.6
widgetsnbextension==4.0.13
xacro==1.14.19
xmlschema==1.0.18
zipp==3.20.2

{
对应一下我的依赖库的版本，这里面需要特别注意的内容：
这里需要注意的一些关键内容

pip install setuptools==58

pip install torch==2.0.1  torchvision==0.15.2  torchaudio==2.0.2 --index-url https://download.pytorch.org/whl/cu118

pip install torch_scatter==2.1.2+pt20cu118 -f https://data.pyg.org/whl/torch-2.0.1+cu118.html

pip install huggingface_hub==0.23 transformers==4.28.0 spacy==3.5.1 shapely==2.0.7 py_trees==0.8.3 tabulate==0.9.0 ephem==4.2 dictor==0.1.12 easydict==1.13 pygame opencv-python==4.2.0.32 torchvision==0.15.2

sudo apt-get install libomp5

}
准备好模型文件
这里需要提供三个模型,vision,llm和lmdrive模型

这两个模型和我的大模型

修改模型路径

我的路径是:

llm_model = '/home/cyun/Documents/LMDrive/data/llava-v1.5-7b'
preception_model = 'memfuser_baseline_e1d3_return_feature'
preception_model_ckpt = '/home/cyun/Documents/LMDrive/vlm1/vision-encoder-r50.pth.tar'
lmdrive_ckpt = '/home/cyun/Documents/LMDrive/vlm1/llava-v1.5-checkpoint.pth'

请自行下载模型,下载有问题的可以找我.

https://huggingface.co/OpenDILabCommunity/LMDrive-llava-v1.5-7b-v1.0
https://huggingface.co/OpenDILabCommunity/LMDrive-vision-encoder-r50-v1.0
https://huggingface.co/liuhaotian/llava-v1.5-7b

修改run_evaluation

最后修改启动的脚本,需要指定各种路径,我们运行sh必须在初始路径:

注意把carla文件夹放在里面:

此时,export CARLA_ROOT=carla才能链接到这里,所以最后的sh修改为

#!/bin/bash
# export PT=2000

export PT=$(($RANDOM % 1000 + 16000))
bash carla/CarlaUE4.sh --world-port=$PT &

sleep 4

export CARLA_ROOT=carla
export CARLA_SERVER=${CARLA_ROOT}/CarlaUE4.sh
export PYTHONPATH=$PYTHONPATH:${CARLA_ROOT}/PythonAPI
export PYTHONPATH=$PYTHONPATH:${CARLA_ROOT}/PythonAPI/carla
export PYTHONPATH=$PYTHONPATH:$CARLA_ROOT/PythonAPI/carla/dist/carla-0.9.10-py3.7-linux-x86_64.egg
export PYTHONPATH=$PYTHONPATH:leaderboard
export PYTHONPATH=$PYTHONPATH:leaderboard/team_code
export PYTHONPATH=$PYTHONPATH:scenario_runner

export LEADERBOARD_ROOT=leaderboard
export CHALLENGE_TRACK_CODENAME=SENSORS
export PORT=$PT # same as the carla server port
export TM_PORT=$(($PT+500)) # port for traffic manager, required when spawning multiple servers/clients
export DEBUG_CHALLENGE=0
export REPETITIONS=1 # multiple evaluation runs
export ROUTES=langauto/benchmark_long.xml
export TEAM_AGENT=leaderboard/team_code/lmdriver_agent.py # agent
export TEAM_CONFIG=leaderboard/team_code/lmdriver_config.py # model checkpoint, not required for expert
export CHECKPOINT_ENDPOINT=results/sample_result.json # results file
#export SCENARIOS=leaderboard/data/scenarios/no_scenarios.json #town05_all_scenarios.json
export SCENARIOS=leaderboard/data/official/all_towns_traffic_scenarios_public.json
export SAVE_PATH=data/eval # path for saving episodes while evaluating
export RESUME=True

echo ${LEADERBOARD_ROOT}/leaderboard/leaderboard_evaluator.py
python3 -u  ${LEADERBOARD_ROOT}/leaderboard/leaderboard_evaluator.py \
--scenarios=${SCENARIOS}  \
--routes=${ROUTES} \
--repetitions=${REPETITIONS} \
--track=${CHALLENGE_TRACK_CODENAME} \
--checkpoint=${CHECKPOINT_ENDPOINT} \
--agent=${TEAM_AGENT} \
--agent-config=${TEAM_CONFIG} \
--debug=${DEBUG_CHALLENGE} \
--record=${RECORD_PATH} \
--resume=${RESUME} \
--port=${PORT} \
--trafficManagerPort=${TM_PORT}

运行即可

CUDA_VISIBLE_DEVICES=0 ./leaderboard/scripts/run_evaluation.sh

效果:

理解原理

之后是原理的理解，待完善2025.3.17