diff --git a/docker/Dockerfile.user b/docker/Dockerfile.user
index 5591e923a1..21e07216f7 100644
--- a/docker/Dockerfile.user
+++ b/docker/Dockerfile.user
@@ -10,7 +10,7 @@ ARG GROUP_NAME=root
 
 RUN (getent group ${GROUP_ID} || groupadd --gid ${GROUP_ID} ${GROUP_NAME}) && \
     (getent passwd ${USER_ID} || useradd --gid ${GROUP_ID} --uid ${USER_ID} --create-home --no-log-init --shell /bin/bash ${USER_NAME}) && \
-    chown ${USER_NAME}:${GROUP_NAME} /app/tensorrt_llm && \
+    if [ -d /app/tensorrt_llm ]; then chown ${USER_NAME}:${GROUP_NAME} /app/tensorrt_llm; fi && \
     apt-get update && \
     apt-get install -y sudo && \
     adduser ${USER_NAME} sudo && \