Merge pull request #3878 from sysown/v2.x-reproducibility-fixes

V2.x reproducibility fixes

Makefile

@@ -675,9 +675,9 @@ cleanall:
 	cd lib && ${MAKE} clean
 	cd src && ${MAKE} clean
 	cd test/tap && ${MAKE} clean
-	rm binaries/*deb || true
-	rm binaries/*rpm || true
-	rm binaries/*id-hash || true
+	rm -f binaries/*deb || true
+	rm -f binaries/*rpm || true
+	rm -f binaries/*id-hash || true
 
 .PHONY: cleanbuild
 cleanbuild:

deps/Makefile

@@ -45,7 +45,7 @@ endif
 
 
 libinjection/libinjection/src/libinjection.a:
-	cd libinjection && rm -rf libinjection-3.10.0 || true
+	cd libinjection && rm -rf libinjection-*/ || true
 	cd libinjection && tar -zxf libinjection-3.10.0.tar.gz
 ifneq ($(CENTOSVER),6)
 	cd libinjection/libinjection && patch -p1 < ../update-build-py3.diff
@@ -73,8 +73,7 @@ ifeq ($(MIN_VERSION),$(lastword $(sort $(GCC_VERSION) $(MIN_VERSION))))
 endif
 
 libhttpserver/libhttpserver/build/src/.libs/libhttpserver.a: libmicrohttpd/libmicrohttpd/src/microhttpd/.libs/libmicrohttpd.a
-	cd libhttpserver && rm -rf libhttpserver-master_20191121 || true
-	cd libhttpserver && rm -rf libhttpserver-0.18.1 || true
+	cd libhttpserver && rm -rf libhttpserver-*/ || true
 	cd libhttpserver && tar -zxf libhttpserver-0.18.1.tar.gz
 ifeq ($(REQUIRE_PATCH), true)
 	cd libhttpserver/libhttpserver && patch src/httpserver/basic_auth_fail_response.hpp < ../basic_auth_fail_response.hpp.patch
@@ -98,7 +97,7 @@ endif
 libhttpserver: libhttpserver/libhttpserver/build/src/.libs/libhttpserver.a
 
 libev/libev/.libs/libev.a:
-	cd libev && rm -rf libev-4.24 || true
+	cd libev && rm -rf libev-*/ || true
 	cd libev && tar -zxf libev-4.24.tar.gz
 	cd libev/libev && patch ev.c < ../ev.c-multiplication-overflow.patch
 	cd libev/libev  && ./configure
@@ -113,8 +112,7 @@ curl/curl/lib/.libs/libcurl.a: libssl/openssl/libssl.a
 curl: curl/curl/lib/.libs/libcurl.a
 
 libmicrohttpd/libmicrohttpd/src/microhttpd/.libs/libmicrohttpd.a:
-	cd libmicrohttpd && rm -rf libmicrohttpd-0.9.55 || true
-	cd libmicrohttpd && rm -rf libmicrohttpd-0.9.68 || true
+	cd libmicrohttpd && rm -rf libmicrohttpd-*/ || true
 	cd libmicrohttpd && rm -f libmicrohttpd || true
 ifeq ($(CENTOSVER),6)
 	cd libmicrohttpd && ln -s libmicrohttpd-0.9.55 libmicrohttpd
@@ -131,19 +129,19 @@ endif
 microhttpd: libmicrohttpd/libmicrohttpd/src/microhttpd/.libs/libmicrohttpd.a
 
 cityhash/cityhash/src/.libs/libcityhash.a:
-	cd cityhash && rm -rf cityhash || true
+	cd cityhash && rm -rf cityhash/ || true
 	cd cityhash && tar -zxf cityhash.tar.gz
 	cd cityhash/cityhash && cp ../config.guess . && chmod +x config.guess && ./configure && CC=${CC} CXX=${CXX} ${MAKE}
 cityhash: cityhash/cityhash/src/.libs/libcityhash.a
 
 lz4/lz4/liblz4.a:
-	cd lz4 && rm -rf lz4-1.7.5 || true
+	cd lz4 && rm -rf lz4-*/ || true
 	cd lz4 && tar -zxf lz4-1.7.5.tar.gz
 	cd lz4/lz4 && CC=${CC} CXX=${CXX} ${MAKE}
 lz4: lz4/lz4/liblz4.a
 
 clickhouse-cpp/clickhouse-cpp/clickhouse/libclickhouse-cpp-lib.a:
-	cd clickhouse-cpp && rm -rf clickhouse-cpp || true
+	cd clickhouse-cpp && rm -rf clickhouse-cpp/ || true
 	cd clickhouse-cpp && tar -zxf clickhouse-cpp.tar.gz
 ifeq ($(CXX),clang++)
 	cd clickhouse-cpp/clickhouse-cpp && sed -i 's/"-O2 -pthread -Wall -Wextra -Werror"/"-O2 -pthread -Wall -Wextra -Werror -Wno-deprecated-copy"/' CMakeLists.txt
@@ -156,15 +154,15 @@ clickhouse-cpp: clickhouse-cpp/clickhouse-cpp/clickhouse/libclickhouse-cpp-lib.a
 
 
 libdaemon/libdaemon/libdaemon/.libs/libdaemon.a:
-	cd libdaemon && rm -rf libdaemon-0.14
+	cd libdaemon && rm -rf libdaemon-*/ || true
 	cd libdaemon && tar -zxf libdaemon-0.14.tar.gz
 	cd libdaemon/libdaemon && cp ../config.guess . && chmod +x config.guess && ./configure --disable-examples
 	cd libdaemon/libdaemon && CC=${CC} CXX=${CXX} ${MAKE}
 
 libdaemon: libdaemon/libdaemon/libdaemon/.libs/libdaemon.a
 
 jemalloc/jemalloc/lib/libjemalloc.a:
-	cd jemalloc && rm -rf jemalloc-5.2.0
+	cd jemalloc && rm -rf jemalloc-*/ || true
 	cd jemalloc && tar --no-same-owner -jxf jemalloc-5.2.0.tar.bz2
 	cd jemalloc/jemalloc && patch src/jemalloc.c < ../issue823.520.patch
 	cd jemalloc/jemalloc && patch src/jemalloc.c < ../issue2358.patch
@@ -177,7 +175,7 @@ jemalloc: jemalloc/jemalloc/lib/libjemalloc.a
 WITHASAN := $(shell echo $(WITHASAN))
 
 mariadb-client-library/mariadb_client/libmariadb/libmariadbclient.a: libssl/openssl/libssl.a
-	cd mariadb-client-library && rm -rf mariadb-connector-c-3.1.9-src
+	cd mariadb-client-library && rm -rf mariadb-connector-c-*/ || true
 	cd mariadb-client-library && tar -zxf mariadb-connector-c-3.1.9-src.tar.gz
 	cd mariadb-client-library/mariadb_client && patch ./plugins/auth/CMakeLists.txt < ../plugin_auth_CMakeLists.txt.patch
 	cd mariadb-client-library/mariadb_client && patch -p0 < ../ConnectorName.patch
@@ -218,7 +216,7 @@ mariadb_client: mariadb-client-library/mariadb_client/libmariadb/libmariadbclien
 
 
 sqlite3/sqlite3/sqlite3.o:
-	cd sqlite3 && rm -rf sqlite-amalgamation-3190200
+	cd sqlite3 && rm -rf sqlite-amalgamation-*/ || true
 	cd sqlite3 && tar -zxf sqlite-amalgamation-3190200.tar.gz
 	cd sqlite3/sqlite3 && patch sqlite3.c < ../from_unixtime.patch
 	cd sqlite3/sqlite3 && patch sqlite3.c < ../sqlite3.c-multiplication-overflow.patch
@@ -228,15 +226,15 @@ sqlite3/sqlite3/sqlite3.o:
 sqlite3: sqlite3/sqlite3/sqlite3.o
 
 libconfig/libconfig/lib/.libs/libconfig++.a:
-	cd libconfig && rm -rf libconfig-1.7.2
+	cd libconfig && rm -rf libconfig-*/ || true
 	cd libconfig && tar -zxf libconfig-1.7.2.tar.gz
 	cd libconfig/libconfig && ./configure --disable-examples
 	cd libconfig/libconfig && CC=${CC} CXX=${CXX} ${MAKE}
 
 libconfig: libconfig/libconfig/lib/.libs/libconfig++.a
 
 prometheus-cpp/prometheus-cpp/lib/libprometheus-cpp-core.a:
-	cd prometheus-cpp && rm -rf prometheus-cpp-0.9.0
+	cd prometheus-cpp && rm -rf prometheus-cpp-*/ || true
 	cd prometheus-cpp && tar -zxf v0.9.0.tar.gz
 	cd prometheus-cpp && tar --strip-components=1 -zxf civetweb-v1.11.tar.gz -C prometheus-cpp/3rdparty/civetweb
 	cd prometheus-cpp/prometheus-cpp && patch -p1 < ../serial_exposer.patch
@@ -248,9 +246,7 @@ prometheus-cpp/prometheus-cpp/lib/libprometheus-cpp-core.a:
 prometheus-cpp: prometheus-cpp/prometheus-cpp/lib/libprometheus-cpp-core.a
 
 re2/re2/obj/libre2.a:
-	cd re2 && rm -rf re2-2018-07-01 || true
-	cd re2 && rm -rf re2-2020-07-06 || true
-#	cd re2 && tar -zxf re2-20140304.tgz
+	cd re2 && rm -rf re2-*/ || true
 	cd re2 && tar -zxf re2.tar.gz
 #	cd re2/re2 && sed -i -e 's/-O3 -g /-O3 -fPIC /' Makefile
 #	cd re2/re2 && patch util/mutex.h < ../mutex.h.patch
@@ -262,55 +258,38 @@ re2/re2/obj/libre2.a:
 re2: re2/re2/obj/libre2.a
 
 pcre/pcre/.libs/libpcre.a:
-	cd pcre && rm -rf pcre-8.39
-	cd pcre && rm -rf pcre-8.44
+	cd pcre && rm -rf pcre-*/ || true
 	cd pcre && tar -zxf pcre-8.44.tar.gz
 	cd pcre/pcre && patch pcretest.c < ../pcretest.c-multiplication-overflow.patch
 	cd pcre/pcre && ./configure
 	cd pcre/pcre && CC=${CC} CXX=${CXX} ${MAKE}
 pcre: pcre/pcre/.libs/libpcre.a
 
 cleanpart:
-	cd mariadb-client-library && rm -rf mariadb-connector-c-2.3.1
-	cd mariadb-client-library && rm -rf mariadb-connector-c-3.0.2-src
-	cd mariadb-client-library && rm -rf mariadb-connector-c-3.1.4-src
-	cd mariadb-client-library && rm -rf mariadb-connector-c-3.1.9-src
-	cd jemalloc && rm -rf jemalloc-4.2.1
-	cd sqlite3/sqlite3 && rm -rf *
+	cd mariadb-client-library && rm -rf mariadb-connector-c-*/ || true
+	cd jemalloc && rm -rf jemalloc-*/ || true
+	cd sqlite3 && rm -rf sqlite-amalgamation-*/ || true
 .PHONY: cleanpart
 
 cleanall:
-	cd libinjection && rm -rf libinjection-3.10.0 || true
-	cd libhttpserver && rm -rf libhttpserver-master_20191121 || true
-	cd libhttpserver && rm -rf libhttpserver-0.18.1 || true
-	cd libdaemon && rm -rf libdaemon-0.14
-	cd jemalloc && rm -rf jemalloc-4.3.1 || true
-	cd jemalloc && rm -rf jemalloc-5.2.0 || true
-	cd mariadb-client-library && rm -rf mariadb-connector-c-2.3.1
-	cd mariadb-client-library && rm -rf mariadb-connector-c-3.0.2-src
-	cd mariadb-client-library && rm -rf mariadb-connector-c-3.1.4-src
-	cd mariadb-client-library && rm -rf mariadb-connector-c-3.1.9-src
-	cd libconfig && rm -rf libconfig-1.4.9
-	cd re2 && rm -rf re2-2018-07-01 || true
-	cd re2 && rm -rf re2-2020-07-06 || true
-	cd pcre && rm -rf pcre-8.39 || true
-	cd pcre && rm -rf pcre-8.44 || true
-	cd sqlite3/sqlite3 && rm -rf * || true
-	cd clickhouse-cpp/clickhouse-cpp && rm -rf * || true
-	cd lz4 && rm -rf lz4-1.7.5 || true
+	cd libinjection && rm -rf libinjection-*/ || true
+	cd libhttpserver && rm -rf libhttpserver-*/ || true
+	cd libdaemon && rm -rf libdaemon-*/ || true
+	cd jemalloc && rm -rf jemalloc-*/ || true
+	cd mariadb-client-library && rm -rf mariadb-connector-c-*/ || true
+	cd libconfig && rm -rf libconfig-*/ || true
+	cd re2 && rm -rf re2-*/ || true
+	cd pcre && rm -rf pcre-*/ || true
+	cd sqlite3 && rm -rf sqlite-amalgamation-*/ || true
+	cd clickhouse-cpp/ && rm -rf clickhouse-cpp/ || true
+	cd lz4 && rm -rf lz4-*/ || true
+	cd libmicrohttpd && rm -rf libmicrohttpd-*/ || true
 	cd libmicrohttpd && rm -f libmicrohttpd || true
-	cd libmicrohttpd && rm -rf libmicrohttpd-0.9.55 || true
-	cd libmicrohttpd && rm -rf libmicrohttpd-0.9.68 || true
-	cd curl && rm -rf curl-7.57.0 || true
-	cd curl && rm -rf curl-7.77.0 || true
-	cd libev && rm -rf libev-4.24 || true
-	cd libssl && rm -rf openssl-1.1.0h || true
-	cd libssl && rm -rf openssl-1.1.1b || true
-	cd libssl && rm -rf openssl-1.1.1d || true
-	cd libssl && rm -rf openssl-1.1.1g || true
-	cd libssl && rm -rf openssl-1.1.1j || true
-	cd libssl && rm -rf openssl-openssl-3.0.0 || true
-	cd libconfig && rm -rf libconfig-1.7.2 || true
-	cd prometheus-cpp && rm -rf prometheus-cpp-0.9.0 || true
+	cd curl && rm -rf curl-*/ || true
+	cd libev && rm -rf libev-*/ || true
+	cd libssl && rm -rf openssl-openssl-*/ || true
+	cd libconfig && rm -rf libconfig-*/ || true
+	cd prometheus-cpp && rm -rf prometheus-cpp-*/ || true
+	cd cityhash && rm -rf cityhash/ || true
 .PHONY: cleanall
 

docker/images/proxysql/deb-compliant/entrypoint/entrypoint.bash

@@ -22,12 +22,12 @@ rm -f /opt/proxysql/proxysql.ctl /opt/proxysql/proxysql || true
 
 # Clean and build dependancies and source
 echo "==> Building"
-git config --global --add safe.directory '/opt/proxysql'
+git config --system --add safe.directory '/opt/proxysql'
 cd /opt/proxysql
 echo "==> ProxySQL '$(git describe --long --abbrev=7)'"
 export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)
 echo "==> Setting SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}"
-find /opt/proxysql -exec touch --date=@${SOURCE_DATE_EPOCH} {} \;
+find /opt/proxysql -not -path "/opt/proxysql/binaries/*" -exec touch -h --date=@${SOURCE_DATE_EPOCH} {} \;
 
 if [[ -z ${PROXYSQL_BUILD_TYPE:-} ]] ; then
 	deps_target="build_deps"

docker/images/proxysql/rhel-compliant/entrypoint/entrypoint.bash

@@ -22,12 +22,12 @@ rm -fr /root/.pki /root/rpmbuild/{BUILDROOT,RPMS,SRPMS,BUILD,SOURCES,tmp} /opt/p
 
 # Clean and build dependancies and source
 echo "==> Building"
-git config --global --add safe.directory '/opt/proxysql'
+git config --system --add safe.directory '/opt/proxysql'
 cd /opt/proxysql
 echo "==> ProxySQL '$(git describe --long --abbrev=7)'"
 export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)
 echo "==> Setting SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}"
-find /opt/proxysql -exec touch --date=@${SOURCE_DATE_EPOCH} {} \;
+find /opt/proxysql -not -path "/opt/proxysql/binaries/*" -exec touch -h --date=@${SOURCE_DATE_EPOCH} {} \;
 
 if [[ -z ${PROXYSQL_BUILD_TYPE:-} ]] ; then
 	deps_target="build_deps"

docker/images/proxysql/rhel-compliant/rhel6/entrypoint/entrypoint.bash

@@ -7,7 +7,7 @@ env
 ARCH=$(rpm --eval '%{_arch}')
 echo "==> '${ARCH}' architecture detected for package"
 
-DIST=$(source /etc/os-release; echo ${ID%%[-._ ]*}${VERSION%%[-._ ]*})
+DIST=$(cat /etc/redhat-release| sed 's/ .*//')
 echo "==> '${DIST}' distro detected for package"
 
 #echo -e "==> C compiler: ${CC} -> $(readlink -e $(which ${CC}))\n$(${CC} --version)"
@@ -22,12 +22,12 @@ rm -fr /root/.pki /root/rpmbuild/{BUILDROOT,RPMS,SRPMS,BUILD,SOURCES,tmp} /opt/p
 
 # Clean and build dependancies and source
 echo "==> Building"
-git config --global --add safe.directory '/opt/proxysql'
+git config --system --add safe.directory '/opt/proxysql'
 cd /opt/proxysql
 echo "==> ProxySQL '$(git describe --long --abbrev=7)'"
 export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)
 echo "==> Setting SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}"
-find /opt/proxysql -exec touch --date=@${SOURCE_DATE_EPOCH} {} \;
+find /opt/proxysql -not -path "/opt/proxysql/binaries/*" -exec touch -h --date=@${SOURCE_DATE_EPOCH} {} \;
 
 if [[ -z ${PROXYSQL_BUILD_TYPE:-} ]] ; then
 	deps_target="build_deps"

docker/images/proxysql/rhel-compliant/rhel7/entrypoint/entrypoint.bash

@@ -22,12 +22,12 @@ rm -fr /root/.pki /root/rpmbuild/{BUILDROOT,RPMS,SRPMS,BUILD,SOURCES,tmp} /opt/p
 
 # Clean and build dependancies and source
 echo "==> Building"
-git config --global --add safe.directory '/opt/proxysql'
+git config --system --add safe.directory '/opt/proxysql'
 cd /opt/proxysql
 echo "==> ProxySQL '$(git describe --long --abbrev=7)'"
 export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)
 echo "==> Setting SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}"
-find /opt/proxysql -exec touch --date=@${SOURCE_DATE_EPOCH} {} \;
+find /opt/proxysql -not -path "/opt/proxysql/binaries/*" -exec touch -h --date=@${SOURCE_DATE_EPOCH} {} \;
 
 if [[ -z ${PROXYSQL_BUILD_TYPE:-} ]] ; then
 	deps_target="build_deps"

docker/images/proxysql/suse-compliant/entrypoint/entrypoint.bash

@@ -22,12 +22,12 @@ rm -fr /root/.pki /root/rpmbuild/{BUILDROOT,RPMS,SRPMS,BUILD,SOURCES,tmp} /opt/p
 
 # Clean and build dependancies and source
 echo "==> Building"
-git config --global --add safe.directory '/opt/proxysql'
+git config --system --add safe.directory '/opt/proxysql'
 cd /opt/proxysql
 echo "==> ProxySQL '$(git describe --long --abbrev=7)'"
 export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)
 echo "==> Setting SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}"
-find /opt/proxysql -exec touch --date=@${SOURCE_DATE_EPOCH} {} \;
+find /opt/proxysql -not -path "/opt/proxysql/binaries/*" -exec touch -h --date=@${SOURCE_DATE_EPOCH} {} \;
 
 if [[ -z ${PROXYSQL_BUILD_TYPE:-} ]] ; then
 	deps_target="build_deps"

lib/Makefile

@@ -143,7 +143,7 @@ $(ODIR):
 
 
 clean:
-	rm -f *.pid $(ODIR)/*.oo $(ODIR)/*.o $(ODIR)/*.gcno $(ODIR)/*.gcda *.ko *.so *~ core libproxysql.a
+	rm -rf *.pid $(ODIR)/*.oo $(ODIR)/*.o $(ODIR)/*.gcno $(ODIR)/*.gcda *.ko *.so *~ core libproxysql.a $(ODIR)
 
 
 ## self note

src/Makefile

@@ -185,5 +185,5 @@ $(LIBPROXYSQLAR):
 default: $(EXECUTABLE)
 
 clean:
-	rm -f *.pid $(ODIR)/*.o $(ODIR)/*.gcno $(ODIR)/*.gcda *~ core perf.data* heaptrack.proxysql.* $(EXECUTABLE) $(EXECUTABLE).sha1
+	rm -rf *.pid $(ODIR)/*.o $(ODIR)/*.gcno $(ODIR)/*.gcda *~ core perf.data* heaptrack.proxysql.* $(EXECUTABLE) $(EXECUTABLE).sha1 $(ODIR)
 

test/afl_digest_test/Makefile

@@ -10,8 +10,8 @@ all: afl_test
 afl_test: $(OBJS)
 	$(CC) $(CFLAGS) $(OBJS) -I$(PROXYSQL_IDIR) afl_mysql_query_digest.cpp -o afl_test
 
-c_tokenizer.o: c_tokenizer.c c_tokenizer.h
-	$(CC) $(CFLAGS) -c c_tokenizer.c
+c_tokenizer.o: c_tokenizer.cpp c_tokenizer.h
+	$(CC) $(CFLAGS) -c c_tokenizer.cpp
 
 clean:
 	rm -f *~ *.o afl_test

test/afl_digest_test/c_tokenizer.cpp

@@ -0,0 +1 @@
+../../lib/c_tokenizer.cpp