feat(docker): Optimize GPU support in container startup scripts

WORKSPACE

@@ -75,12 +75,11 @@ http_archive(
 # grpc
 http_archive(
     name = "com_github_grpc_grpc",
-    sha256 = "419dba362eaf8f1d36849ceee17c3e2ff8ff12ac666b42d3ff02a164ebe090e9",
+    sha256 = "2378b608557a4331c6a6a97f89a9257aee2f8e56a095ce6619eea62e288fcfbe",
     patches = ["//third_party/absl:grpc.patch"],
     strip_prefix = "grpc-1.30.0",
     urls = [
-        "https://apollo-system.cdn.bcebos.com/archive/6.0/v1.30.0.tar.gz",
-        "https://github.com/grpc/grpc/archive/v1.30.0.tar.gz",
+        "https://apollo-system.cdn.bcebos.com/archive/8.0/v1.30.0-apollo.tar.gz",
     ],
 )
 http_archive(
@@ -100,4 +99,4 @@ grpc_deps()
 
 load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
 
-grpc_extra_deps()
+grpc_extra_deps()

cyber/message/protobuf_factory.cc

@@ -47,15 +47,28 @@ bool ProtobufFactory::RegisterMessage(const Descriptor& desc) {
 }
 
 bool ProtobufFactory::RegisterMessage(const ProtoDesc& proto_desc) {
+  FileDescriptorProto file_desc_proto;
+  file_desc_proto.ParseFromString(proto_desc.desc());
+
+  // If the message in this proto file has been registered, return true.
+  if (FindMessageTypeByFile(file_desc_proto)) {
+    return true;
+  }
   for (int i = 0; i < proto_desc.dependencies_size(); ++i) {
     auto dep = proto_desc.dependencies(i);
     if (!RegisterMessage(dep)) {
       return false;
     }
+    FileDescriptorProto dep_file_desc_proto;
+    dep_file_desc_proto.ParseFromString(dep.desc());
+    const google::protobuf::Descriptor* descriptor =
+        FindMessageTypeByFile(dep_file_desc_proto);
+
+    // If descriptor is found, replace the dependency with registered path.
+    if (descriptor != nullptr) {
+      file_desc_proto.set_dependency(i, descriptor->file()->name());
+    }
   }
-
-  FileDescriptorProto file_desc_proto;
-  file_desc_proto.ParseFromString(proto_desc.desc());
   return RegisterMessage(file_desc_proto);
 }
 
@@ -207,6 +220,18 @@ const google::protobuf::ServiceDescriptor* ProtobufFactory::FindServiceByName(
   return pool_->FindServiceByName(name);
 }
 
+const Descriptor* ProtobufFactory::FindMessageTypeByFile(
+    const FileDescriptorProto& file_desc_proto) {
+  const std::string& scope = file_desc_proto.package();
+  std::string type;
+  if (file_desc_proto.message_type_size()) {
+    type = scope + "." + file_desc_proto.message_type(0).name();
+  }
+  const google::protobuf::Descriptor* descriptor =
+      pool_->FindMessageTypeByName(type);
+  return descriptor;
+}
+
 void ErrorCollector::AddError(const std::string& filename,
                               const std::string& element_name,
                               const google::protobuf::Message* descriptor,

cyber/message/protobuf_factory.h

@@ -87,6 +87,10 @@ class ProtobufFactory {
   google::protobuf::Message* GenerateMessageByType(
       const std::string& type) const;
 
+  // Find a descriptor by FileDescriptorProto. Returns nullptr if not found.
+  const Descriptor* FindMessageTypeByFile(
+      const FileDescriptorProto& file_desc_proto);
+
   // Find a top-level message type by name. Returns nullptr if not found.
   const Descriptor* FindMessageTypeByName(const std::string& type) const;
 

cyber/setup.bash

@@ -27,7 +27,6 @@ for entry in "${mainboard_path}" \
 done
 
 pathprepend ${bazel_bin_path}/cyber/python/internal PYTHONPATH
-# todo(zero): The python version determines the path and needs to be optimized
 pathprepend "${PYTHON_INSTALL_PATH}/lib/python${PYTHON_VERSION}/site-packages" PYTHONPATH
 pathprepend "${PYTHON_INSTALL_PATH}/bin/" PATH
 

docker/scripts/dev_into.sh

@@ -16,10 +16,44 @@
 # limitations under the License.
 ###############################################################################
 DOCKER_USER="${USER}"
-DEV_CONTAINER="apollo_dev_${USER}"
+DEV_CONTAINER_PREFIX='apollo_dev_'
+DEV_CONTAINER="${DEV_CONTAINER_PREFIX}${USER}"
+
+function parse_arguments {
+    local container_name=''
+
+    while [ $# -gt 0 ]; do
+        local opt="$1"
+        shift
+        case "${opt}" in
+            -n | --name)
+                container_name="$1"
+                shift
+                ;;
+
+            --user)
+                export CUSTOM_USER="$1"
+                shift
+                ;;
+        esac
+    done
+
+    [[ ! -z "${container_name}" ]] && DEV_CONTAINER="${DEV_CONTAINER_PREFIX}${container_name}"
+    [[ ! -z "${CUSTOM_USER}" ]] && DOCKER_USER="${CUSTOM_USER}"
+}
+
+function restart_stopped_container {
+    if docker ps -f status=exited -f name="${DEV_CONTAINER}" | grep "${DEV_CONTAINER}"; then
+        docker start "${DEV_CONTAINER}"
+    fi
+}
 
 xhost +local:root 1>/dev/null 2>&1
 
+parse_arguments "$@"
+
+restart_stopped_container
+
 docker exec \
     -u "${DOCKER_USER}" \
     -e HISTFILE=/apollo/.dev_bash_hist \

docker/scripts/dev_start.sh

@@ -21,21 +21,23 @@ source "${CURR_DIR}/docker_base.sh"
 CACHE_ROOT_DIR="${APOLLO_ROOT_DIR}/.cache"
 
 DOCKER_REPO="apolloauto/apollo"
-DEV_CONTAINER="apollo_dev_${USER}"
+DEV_CONTAINER_PREFIX='apollo_dev_'
+DEV_CONTAINER="${DEV_CONTAINER_PREFIX}${USER}"
 DEV_INSIDE="in-dev-docker"
 
 SUPPORTED_ARCHS=(x86_64 aarch64)
 TARGET_ARCH="$(uname -m)"
 
 VERSION_X86_64="dev-x86_64-18.04-20221124_1708"
 TESTING_VERSION_X86_64="dev-x86_64-18.04-testing-20210112_0008"
-
-VERSION_AARCH64="dev-aarch64-18.04-20201218_0030"
+VERSION_AARCH64="dev-aarch64-20.04-20231024_1054"
 USER_VERSION_OPT=
-
-FAST_MODE="y"
+FAST_MODE="n"
 
 GEOLOC=
+TIMEZONE_CN=(
+  "Time zone: Asia/Shanghai (CST, +0800)"
+)
 
 USE_LOCAL_IMAGE=0
 CUSTOM_DIST=
@@ -47,16 +49,21 @@ USER_SPECIFIED_MAPS=
 MAP_VOLUMES_CONF=
 
 # Install python tools
-PYTHON_INSTALL_PATH="/opt/apollo/python_tools"
-PYTHON_VERSION=$(python3 -c 'import sys; print(".".join(map(str, sys.version_info[0:2])))')
+source docker/setup_host/host_env.sh
 DEFAULT_PYTHON_TOOLS=(
-  amodel
+  amodel~=0.1.0
 )
 
 # Model
 MODEL_REPOSITORY="https://apollo-pkg-beta.cdn.bcebos.com/perception_model"
 DEFAULT_INSTALL_MODEL=(
-  "${MODEL_REPOSITORY}/cnnseg128_caffe.zip"
+  "${MODEL_REPOSITORY}/tl_detection_caffe.zip"
+  "${MODEL_REPOSITORY}/horizontal_caffe.zip"
+  "${MODEL_REPOSITORY}/quadrate_caffe.zip"
+  "${MODEL_REPOSITORY}/vertical_caffe.zip"
+  "${MODEL_REPOSITORY}/darkSCNN_caffe.zip"
+  "${MODEL_REPOSITORY}/cnnseg16_caffe.zip"
+  "${MODEL_REPOSITORY}/3d-r4-half_caffe.zip"
 )
 
 # Map
@@ -65,7 +72,7 @@ DEFAULT_MAPS=(
     sunnyvale_loop
     sunnyvale_with_two_offices
     san_mateo
-    apollo_virutal_map
+    #apollo_virutal_map
 )
 
 DEFAULT_TEST_MAPS=(
@@ -206,6 +213,18 @@ function check_target_arch() {
     exit 1
 }
 
+function check_timezone_cn() {
+    # https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+    time_zone=$(timedatectl | grep "Time zone" | xargs)
+
+    for tz in "${TIMEZONE_CN[@]}"; do
+        if [[ "${time_zone}" == "${tz}" ]]; then
+            GEOLOC="cn"
+            return 0
+        fi
+    done
+}
+
 function setup_devices_and_mount_local_volumes() {
     local __retval="$1"
 
@@ -332,15 +351,20 @@ function install_python_tools() {
 
   for tool in ${DEFAULT_PYTHON_TOOLS[@]}; do
     info "Install python tool ${tool} ..."
-    pip3 install --user "${tool}"
+    # Use /usr/bin/pip3 because native python is used in the container.
+    /usr/bin/pip3 install --user "${tool}"
+    if [ $? -ne 0 ]; then
+        error "Failed to install ${tool}"
+        exit 1
+    fi
   done
 }
 
 function install_perception_models() {
   if [ "$FAST_MODE" == "n" ] || [ "$FAST_MODE" == "no" ]; then
     for model_url in ${DEFAULT_INSTALL_MODEL[@]}; do
         info "Install model ${model_url} ..."
-        amodel install "${model_url}"
+        amodel install "${model_url}" -s
     done
   else
     warning "Skip the model installation, if you need to run the perception module, you can manually install."
@@ -358,6 +382,8 @@ function main() {
     fi
 
     determine_dev_image "${USER_VERSION_OPT}"
+
+    [[ -z "${GEOLOC}" ]] && check_timezone_cn
     geo_specific_config "${GEOLOC}"
 
     if [[ "${USE_LOCAL_IMAGE}" -gt 0 ]]; then
@@ -381,11 +407,17 @@ function main() {
 
     mount_map_volumes
 
-    info "Installing python tools ..."
-    install_python_tools
+    if ! [ -x "$(command -v pip3)" ]; then
+      warning "Skip install perception models!!! " \
+          "Need pip3 to install Apollo model management tool!" \
+          "Try \"sudo apt install python3-pip\" "
+    else
+      info "Installing python tools ..."
+      install_python_tools
 
-    info "Installing perception models ..."
-    install_perception_models
+      info "Installing perception models ..."
+      install_perception_models
+    fi
 
     info "Starting Docker container \"${DEV_CONTAINER}\" ..."
 

docker/scripts/docker_base.sh

@@ -62,11 +62,15 @@ function determine_gpu_use_host() {
 
     local nv_docker_doc="https://github.com/NVIDIA/nvidia-docker/blob/master/README.md"
     if [[ "${USE_GPU_HOST}" -eq 1 ]]; then
-        if [[ -x "$(which nvidia-container-toolkit)" ]]; then
+        if [[ -x "$(which nvidia-container-toolkit)" || -x "$(which nvidia-container-runtime)" ]]; then
             local docker_version
             docker_version="$(docker version --format '{{.Server.Version}}')"
             if dpkg --compare-versions "${docker_version}" "ge" "19.03"; then
-                DOCKER_RUN_CMD="docker run --gpus all"
+                if [[ "${HOST_ARCH}" == "aarch64" ]]; then
+                    DOCKER_RUN_CMD="docker run --runtime nvidia"
+                else
+                    DOCKER_RUN_CMD="docker run --gpus all"
+                fi
             else
                 warning "Please upgrade to docker-ce 19.03+ to access GPU from container."
                 USE_GPU_HOST=0

docker/setup_host/host_env.sh

@@ -15,10 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ###############################################################################
-
 APOLLO_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
 
-PYTHON_INSTALL_PATH="/opt/apollo/python_tools"
+PYTHON_USER_BASE=$(python3 -c 'import site; print(site.USER_BASE)')
+PYTHON_INSTALL_PATH="${PYTHON_USER_BASE}/apollo/tools"
 PYTHON_VERSION=$(python3 -c 'import sys; print(".".join(map(str, sys.version_info[0:2])))')
 
 source ${APOLLO_ROOT_DIR}/scripts/common.bashrc

modules/audio/inference/direction_detection.cc

@@ -136,13 +136,30 @@ double DirectionDetection::GccPhat(const torch::Tensor& sig,
             n = n_sig + n_refsig;
   torch::Tensor psig = at::constant_pad_nd(sig, {0, n_refsig}, 0);
   torch::Tensor prefsig = at::constant_pad_nd(refsig, {0, n_sig}, 0);
+#if TORCH_VERSION_MINOR <= 7
   psig = at::rfft(psig, 1, false, true);
   prefsig = at::rfft(prefsig, 1, false, true);
+#else
+  auto psig_complex = at::fft_rfft(psig, c10::nullopt, -1, c10::nullopt);
+  psig = at::stack({torch::real(psig_complex), torch::imag(psig_complex)}, -1);
+
+  auto prefsig_complex = at::fft_rfft(prefsig, c10::nullopt, -1, c10::nullopt);
+  prefsig = at::stack(
+      {torch::real(prefsig_complex), torch::imag(prefsig_complex)}, -1);
+#endif
 
   ConjugateTensor(&prefsig);
   torch::Tensor r = ComplexMultiply(psig, prefsig);
+#if TORCH_VERSION_MINOR <= 7
   torch::Tensor cc =
       at::irfft(r / ComplexAbsolute(r), 1, false, true, {interp * n});
+#else
+  auto irfft_input_transpose = at::transpose(r / ComplexAbsolute(r), 0, 1);
+  auto irfft_complex =
+      torch::complex(irfft_input_transpose[0], irfft_input_transpose[1]);
+  torch::Tensor cc =
+      torch::real(torch::fft::irfft(irfft_complex, n, -1, c10::nullopt));
+#endif
   int max_shift = static_cast<int>(interp * n / 2);
   if (max_tau != 0)
     max_shift = std::min(static_cast<int>(interp * fs * max_tau), max_shift);

modules/bridge/common/bridge_header.cc

@@ -53,14 +53,19 @@ bool BridgeHeader::Diserialize(const char *buf, size_t buf_size) {
       i -= static_cast<int>(sizeof(HType) + sizeof(bsize) + size + 3);
       continue;
     }
-    size_t value_size = 0;
+
     for (int j = 0; j < Header_Tail; j++) {
       if (type == header_item[j]->GetType()) {
-        cursor = header_item[j]->DiserializeItem(cursor, &value_size);
+        size_t value_size = 0;
+        cursor = header_item[j]->DiserializeItem(cursor, static_cast<size_t>(i),
+                                                 &value_size);
+        i -= static_cast<int>(value_size);
+        if (cursor == nullptr) {
+          return false;
+        }
         break;
       }
     }
-    i -= static_cast<int>(value_size);
   }
   return true;
 }