diff --git a/build/Dockerfile.debug_env b/build/Dockerfile.debug_env new file mode 100644 index 0000000..112170c --- /dev/null +++ b/build/Dockerfile.debug_env @@ -0,0 +1,130 @@ +# build GCC +FROM debian:bullseye AS gcc-builder +ENV GCC_VERSION=9.3.0 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential wget \ + libgmp-dev libmpfr-dev libmpc-dev \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +WORKDIR /build +RUN wget https://mirrors.tuna.tsinghua.edu.cn/gnu/gcc/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.gz \ + && tar -xf gcc-${GCC_VERSION}.tar.gz \ + && cd gcc-${GCC_VERSION} \ + && mkdir build && cd build \ + && ../configure \ + --prefix=/usr/local/gcc-${GCC_VERSION} \ + --disable-multilib \ + --enable-languages=c,c++ \ + --disable-bootstrap \ + --disable-nls \ + --disable-libsanitizer \ + --disable-libvtv \ + --disable-libssp \ + --disable-libquadmath \ + --disable-libgomp \ + --disable-libada \ + --disable-libstdcxx-pch \ + && make -j $(nproc) \ + && make install-strip DESTDIR=/gcc-install \ + && cd /build && rm -rf * + +# build Bison +FROM debian:bullseye AS bison-builder +ENV BISON_VERSION=3.4.2 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential wget \ + flex \ + m4 \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +WORKDIR /build +RUN wget --no-check-certificate https://ftp.gnu.org/gnu/bison/bison-${BISON_VERSION}.tar.gz \ + && tar -xf bison-${BISON_VERSION}.tar.gz \ + && cd bison-${BISON_VERSION} \ + && ./configure --prefix=/usr/local \ + && make -j $(nproc) \ + && make install DESTDIR=/bison-install \ + && cd /build && rm -rf * + +# final stage +FROM debian:bullseye +COPY --from=gcc-builder /gcc-install/usr/local/gcc-9.3.0 /usr/local/gcc-9.3.0 +COPY --from=bison-builder /bison-install/usr/local/bin/bison /usr/local/bin/bison +COPY --from=bison-builder /bison-install/usr/local/lib/liby.* /usr/local/lib/ +COPY --from=bison-builder /bison-install/usr/local/share/bison /usr/local/share/bison + +# online required part +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + cmake \ + curl \ + python3.9 \ + python3.9-dev \ + python3.9-venv \ + libgmp10 \ + libmpfr6 \ + libmpc3 \ + m4 \ + pkg-config \ + git \ + git-lfs \ + libssl-dev \ + libreadline-dev \ + zlib1g-dev \ + libcurl4-openssl-dev \ + libldap2-dev \ + libsasl2-dev \ + libsasl2-modules-gssapi-mit \ + libkrb5-dev \ + libnuma-dev \ + libmecab-dev \ + libaio-dev \ + libncurses-dev \ + libtirpc-dev \ + gdb \ + gdbserver \ + openssh-server \ + net-tools \ + rsync \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# configure Python and other tools +RUN ln -sf /usr/local/gcc-9.3.0/bin/gcc /usr/local/bin/gcc \ + && ln -sf /usr/bin/python3.9 /usr/bin/python3 \ + && ln -sf /usr/bin/python3.9 /usr/bin/python \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3.9 get-pip.py \ + && rm get-pip.py + +# validation +RUN gcc --version | grep "9.3.0" \ + && bison --version \ + && curl --version | grep "curl" \ + && python3.9 --version | grep "3.9" \ + && cmake --version | grep "cmake version" \ + && gdb --version | grep "GNU gdb" \ + && rsync --version | grep "rsync" \ + && echo "All version checks passed!" + +ENV PATH=/usr/local/gcc-9.3.0/bin:/opt/tiger/typhoon-blade:/opt/common_tools:$PATH +ENV LD_LIBRARY_PATH=/usr/local/gcc-9.3.0/lib64 + +# SECURITY WARNING: +# The root password is set via the ROOT_PASSWORD build argument (default: 'root'). +# DO NOT use the default password in production or internet-accessible environments. +# Always override ROOT_PASSWORD at build time for any non-debug use: +# docker build --build-arg ROOT_PASSWORD=your_strong_password -f Dockerfile.debug_env . +ARG ROOT_PASSWORD=root +RUN mkdir -p /var/run/sshd && \ + echo "root:${ROOT_PASSWORD}" | chpasswd && \ + sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \ + sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config && \ + echo 'ClientAliveInterval 60' >> /etc/ssh/sshd_config + +# Application ports: +# 13308 - MySQL service +# 5001 - Videx service +# 1234 - GDB debug server +# 22 - SSH service +EXPOSE 13308 5001 1234 22 + +CMD ["/etc/init.d/ssh", "start"] diff --git a/build/debug.sh b/build/debug.sh new file mode 100755 index 0000000..df90553 --- /dev/null +++ b/build/debug.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPT_DIR}/config.sh" + +# Error handling +set -e # Exit on error +set -x # Print commands for debugging + +# Start SSH service +/etc/init.d/ssh restart + +# Copy videx to storage +if [ -d "$MYSQL_HOME/storage/videx" ]; then + echo "Videx directory already exists. Skipping copy." +else + echo "Copying videx to $MYSQL_HOME/storage..." + cp -r "$VIDEX_HOME/src/mysql/videx" "$MYSQL_HOME/storage" +fi + +BOOST_DIR=$MYSQL_HOME/boost + +# Create necessary directories +mkdir -p "$BOOST_DIR" +mkdir -p "$MYSQL_BUILD_DIR"/{etc,build,lib64,data,log} + +# Copy my.cnf to build directory +if [ -f "$SCRIPT_DIR/my.cnf" ]; then + cp "$SCRIPT_DIR/my.cnf" "$MYSQL_BUILD_DIR/etc/my.cnf" +else + echo "Warning: my.cnf not found!" + exit 1 +fi diff --git a/doc/debug_guide.md b/doc/debug_guide.md new file mode 100644 index 0000000..e2b145f --- /dev/null +++ b/doc/debug_guide.md @@ -0,0 +1,167 @@ +# CLion GDB Remote Debugging VIDEX Guide + +This guide shows how to mount the `MySQL8` and `videx` projects into a container using Docker, creating a debugging environment isolated from the host environment. The code in the container can be affected by local modifications in real-time, facilitating development and debugging. + +**Note: This guide is currently only applicable for macOS.** + +## 1. Preparation + +Please follow the instructions in **Section 1 Preparation** in the [installation guide](https://github.com/bytedance/videx/blob/main/doc/installation.md). + +## 2. Build and Start the Debug Container + +### 2.1 Build Docker Image + +```bash +cd $VIDEX_HOME +docker build -t videx_debug:latest -f build/Dockerfile.debug_env . +``` + +### 2.2 Start the Container and Mount the Project + +```bash +docker run -dit \ + --name clion-videx-gdb \ + -p 2222:22 -p 13308:13308 -p 5001:5001 -p 1234:1234 \ + -v $MySQL8_HOME:/root/mysql_server \ + -v $VIDEX_HOME:/root/videx_server \ + videx_debug:latest \ + sleep infinity +``` + +### 2.3 Run the Debug Script in the Container + +**Required for the First Time.** +Enter the container and execute the compile command: + +```bash +cd /root/videx_server/build +chmod +x *.sh +./debug.sh +``` + +![container bash](debug_img/container_debugsh.png) + +## 3. Open Project in CLion + +Open the local `mysql_server` project. + +## 4. Configure Remote Development Environment + +### 4.1 Configure SSH Connection +1. Go to `Settings` → `Tools` → `SSH Configurations` +2. Add a new configuration: + - Host: localhost + - Port: 2222 + - User name: root + - Password: root +3. Click `Test Connection` to verify the connection + +![ssh setting](debug_img/ssh_configuration.png) + +### 4.2 Configure Remote Toolchain +1. Go to `Settings` → `Build, Execution, Deployment` → `Toolchains` +2. Add a new Remote Host toolchain: + - Name: Remote Host + - C++ Compiler: /usr/local/gcc-9.3.0/bin/g++ + +![toolchain setting](debug_img/toolchain.png) + +## 5. Configure Remote Path Mapping + +1. Go to `Settings` → `Build, Execution, Deployment` → `Deployment` +2. Add a new deployment configuration: + - Type: SFTP + - SSH Configuration: Use the SSH configured in the previous step + - Root path: /root + - Select `Use rsync for download/...` +3. In the Mappings tab: + - Local path: Set to the mysql_server path on **your local host** + - Deployment path: /mysql_server + +![deployment setting](debug_img/deployment_setting.png) + +![mapping setting](debug_img/deployment_mapping.png) + +## 6. Configure CMake Profile +1. Go to `Settings` → `Build, Execution, Deployment` → `CMake` +2. Add a new CMake Profile: + - Build type: Debug + - Toolchain: Remote Host +3. CMake options: + +```bash +-DWITH_DEBUG=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_CONFIG=mysql_release -DFEATURE_SET=community -DCMAKE_INSTALL_PREFIX=. -DMYSQL_DATADIR=./data -DSYSCONFDIR=./etc -DWITH_BOOST=/root/mysql_server/boost -DDOWNLOAD_BOOST=ON -DWITH_ROCKSDB=OFF -DDOWNLOAD_BOOST_TIMEOUT=3600 -DWITH_VIDEX_STORAGE_ENGINE=1 +``` + +4. Build directory: mysql_build_output/build + +![cmake setting](debug_img/cmake_configuration.png) + +## 7. Configure and Start MySQL Service + +### 7.1 Initialize mysqld + +1. Go to `Run/Debug Configurations` +2. Search mysqld and add a new instance + - Name: mysqld-init + - Target: mysqld +3. Set the program arguments and working directory +4. Click ▶️ to run the mysqld-init + +```bash +--defaults-file=./etc/my.cnf --initialize-insecure --user=root --basedir=./ --datadir=./data +``` + +![mysqld-init](debug_img/mysqld-init.png) + +### 7.2 Start mysqld + +1. Go to `Run/Debug Configurations` +2. Search mysqld and add a new instance + - Name: mysqld-run + - Target: mysqld +3. Set the program arguments and working directory +4. Click ▶️ to run the mysqld-run + +```bash +--defaults-file=./etc/my.cnf --user=root --basedir=./ --datadir=./data --socket=./mysql_80.sock --port=13308 +``` + +![mysqld-run](debug_img/mysqld-run.png) + +## 8. Create Debug User + +Execute in the container terminal: + +```bash +/root/mysql_server/mysql_build_output/build/runtime_output_directory/mysql \ + -h127.0.0.1 -uroot -P13308 \ + -e "CREATE USER 'videx'@'%' IDENTIFIED WITH mysql_native_password BY 'password'; \ + GRANT ALL ON *.* TO 'videx'@'%'; FLUSH PRIVILEGES;" +``` + +Test in your local terminal: + +```bash +mysql -h127.0.0.1 -P13308 -uvidex -ppassword -e 'SELECT 1;' +``` + +You will get the following result if all goes well + +local terminal check + +## 9. Start Videx Server +```bash +cd /root/videx_server +python3.9 -m pip install -e . --use-pep517 + +cd src/sub_platforms/sql_opt/videx/scripts +python3.9 start_videx_server.py --port 5001 +``` + +## 10. Start GDB Debugging + +Switch to mysqld-run, and click debug + +![debug demo](debug_img/clion_debug.png) diff --git a/doc/debug_img/clion_debug.png b/doc/debug_img/clion_debug.png new file mode 100644 index 0000000..dae925c Binary files /dev/null and b/doc/debug_img/clion_debug.png differ diff --git a/doc/debug_img/cmake_configuration.png b/doc/debug_img/cmake_configuration.png new file mode 100644 index 0000000..6a955d9 Binary files /dev/null and b/doc/debug_img/cmake_configuration.png differ diff --git a/doc/debug_img/container_debugsh.png b/doc/debug_img/container_debugsh.png new file mode 100644 index 0000000..2cf32f3 Binary files /dev/null and b/doc/debug_img/container_debugsh.png differ diff --git a/doc/debug_img/deployment_mapping.png b/doc/debug_img/deployment_mapping.png new file mode 100644 index 0000000..3defd6b Binary files /dev/null and b/doc/debug_img/deployment_mapping.png differ diff --git a/doc/debug_img/deployment_setting.png b/doc/debug_img/deployment_setting.png new file mode 100644 index 0000000..2f57305 Binary files /dev/null and b/doc/debug_img/deployment_setting.png differ diff --git a/doc/debug_img/local_check.png b/doc/debug_img/local_check.png new file mode 100644 index 0000000..751ff10 Binary files /dev/null and b/doc/debug_img/local_check.png differ diff --git a/doc/debug_img/mysqld-init.png b/doc/debug_img/mysqld-init.png new file mode 100644 index 0000000..0971b47 Binary files /dev/null and b/doc/debug_img/mysqld-init.png differ diff --git a/doc/debug_img/mysqld-run.png b/doc/debug_img/mysqld-run.png new file mode 100644 index 0000000..8c5744a Binary files /dev/null and b/doc/debug_img/mysqld-run.png differ diff --git a/doc/debug_img/ssh_configuration.png b/doc/debug_img/ssh_configuration.png new file mode 100644 index 0000000..6f3e598 Binary files /dev/null and b/doc/debug_img/ssh_configuration.png differ diff --git a/doc/debug_img/toolchain.png b/doc/debug_img/toolchain.png new file mode 100644 index 0000000..083c9d2 Binary files /dev/null and b/doc/debug_img/toolchain.png differ