From f524e9eb1e2d80f2dfd06b53a623ac2f615ec22f Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 20 May 2025 05:43:05 +0200 Subject: [PATCH 01/19] cleanup/native/cma->socketfiles --- .../postgresql-debug/configure.diff | 147 ---------- .../src-Makefile.shlib.diff | 49 ---- .../src-backend-commands-async.c.diff | 15 - .../src-backend-libpq-pqcomm.c.diff | 261 ------------------ .../src-bin-initdb-initdb.c.diff | 217 --------------- pglite-REL_17_4_WASM/build.sh | 13 +- pglite-REL_17_4_WASM/interactive_one.c | 97 ++++--- pglite-REL_17_4_WASM/native.sh | 189 +++++++++++++ pglite-REL_17_4_WASM/pg_main.c | 6 + pglite-REL_17_4_WASM/pglite-modpython.c | 218 +++++++++++++++ pglite-REL_17_4_WASM/repl.html | 3 +- src/backend/libpq/pqcomm.c | 56 ++-- src/backend/tcop/postgres.c | 1 + src/port/pqsignal.c | 10 +- 14 files changed, 531 insertions(+), 751 deletions(-) delete mode 100644 patches-REL_17_4_WASM/postgresql-debug/configure.diff delete mode 100644 patches-REL_17_4_WASM/postgresql-emscripten/src-Makefile.shlib.diff delete mode 100644 patches-REL_17_4_WASM/postgresql-emscripten/src-backend-commands-async.c.diff delete mode 100644 patches-REL_17_4_WASM/postgresql-pglite/src-backend-libpq-pqcomm.c.diff delete mode 100644 patches-REL_17_4_WASM/postgresql-pglite/src-bin-initdb-initdb.c.diff create mode 100755 pglite-REL_17_4_WASM/native.sh create mode 100644 pglite-REL_17_4_WASM/pglite-modpython.c diff --git a/patches-REL_17_4_WASM/postgresql-debug/configure.diff b/patches-REL_17_4_WASM/postgresql-debug/configure.diff deleted file mode 100644 index d2d620958ea73..0000000000000 --- a/patches-REL_17_4_WASM/postgresql-debug/configure.diff +++ /dev/null @@ -1,147 +0,0 @@ ---- REL_17_4/configure -+++ pglite-REL_17_4/configure -@@ -4328,7 +4328,7 @@ - ac_save_c_werror_flag=$ac_c_werror_flag - ac_c_werror_flag=yes - ac_cv_prog_cc_g=no -- CFLAGS="-g" -+ CFLAGS="-g2" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext - /* end confdefs.h. */ - -@@ -4359,7 +4359,7 @@ - - else - ac_c_werror_flag=$ac_save_c_werror_flag -- CFLAGS="-g" -+ CFLAGS="-g2" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext - /* end confdefs.h. */ - -@@ -4387,13 +4387,13 @@ - CFLAGS=$ac_save_CFLAGS - elif test $ac_cv_prog_cc_g = yes; then - if test "$GCC" = yes; then -- CFLAGS="-g -O2" -+ CFLAGS="-g2 -Os" - else -- CFLAGS="-g" -+ CFLAGS="-g2" - fi - else - if test "$GCC" = yes; then -- CFLAGS="-O2" -+ CFLAGS="-Os" - else - CFLAGS= - fi -@@ -4859,7 +4859,7 @@ - ac_save_cxx_werror_flag=$ac_cxx_werror_flag - ac_cxx_werror_flag=yes - ac_cv_prog_cxx_g=no -- CXXFLAGS="-g" -+ CXXFLAGS="-g2" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext - /* end confdefs.h. */ - -@@ -4890,7 +4890,7 @@ - - else - ac_cxx_werror_flag=$ac_save_cxx_werror_flag -- CXXFLAGS="-g" -+ CXXFLAGS="-g2" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext - /* end confdefs.h. */ - -@@ -4918,13 +4918,13 @@ - CXXFLAGS=$ac_save_CXXFLAGS - elif test $ac_cv_prog_cxx_g = yes; then - if test "$GXX" = yes; then -- CXXFLAGS="-g -O2" -+ CXXFLAGS="-g2 -Os" - else -- CXXFLAGS="-g" -+ CXXFLAGS="-g2" - fi - else - if test "$GXX" = yes; then -- CXXFLAGS="-O2" -+ CXXFLAGS="-Os" - else - CXXFLAGS= - fi -@@ -5284,11 +5284,11 @@ - elif test "$enable_coverage" = yes; then - : # no optimization by default - elif test "$GCC" = yes; then -- CFLAGS="-O2" -+ CFLAGS="-Os" - else - # if the user selected debug mode, don't use -O - if test "$enable_debug" != yes; then -- CFLAGS="-O" -+ CFLAGS="-Os" - fi - fi - -@@ -5299,11 +5299,11 @@ - elif test "$enable_coverage" = yes; then - : # no optimization by default - elif test "$GCC" = yes; then -- CXXFLAGS="-O2" -+ CXXFLAGS="-Os" - else - # if the user selected debug mode, don't use -O - if test "$enable_debug" != yes; then -- CXXFLAGS="-O" -+ CXXFLAGS="-Os" - fi - fi - -@@ -5316,12 +5316,12 @@ - if test "$ac_env_BITCODE_CFLAGS_set" = set; then - BITCODE_CFLAGS=$ac_env_BITCODE_CFLAGS_value - else -- BITCODE_CFLAGS="-O2 $BITCODE_CFLAGS" -+ BITCODE_CFLAGS="-Os $BITCODE_CFLAGS" - fi - if test "$ac_env_BITCODE_CXXFLAGS_set" = set; then - BITCODE_CXXFLAGS=$ac_env_BITCODE_CXXFLAGS_value - else -- BITCODE_CXXFLAGS="-O2 $BITCODE_CXXFLAGS" -+ BITCODE_CXXFLAGS="-Os $BITCODE_CXXFLAGS" - fi - - # C[XX]FLAGS we determined above will be added back at the end -@@ -7656,11 +7656,11 @@ - - # supply -g if --enable-debug - if test "$enable_debug" = yes && test "$ac_cv_prog_cc_g" = yes; then -- CFLAGS="$CFLAGS -g" -+ CFLAGS="$CFLAGS -g2" - fi - - if test "$enable_debug" = yes && test "$ac_cv_prog_cxx_g" = yes; then -- CXXFLAGS="$CXXFLAGS -g" -+ CXXFLAGS="$CXXFLAGS -g2" - fi - - # enable code coverage if --enable-coverage -@@ -13313,7 +13313,7 @@ - #ifdef __cplusplus - extern "C" - #endif --char uuid_export (); -+extern char uuid_export (); - int - main () - { -@@ -13336,7 +13336,7 @@ - if test "x$ac_cv_lib_uuid_uuid_export" = xyes; then : - UUID_LIBS="-luuid" - else -- as_fn_error $? "library 'ossp-uuid' or 'uuid' is required for OSSP UUID" "$LINENO" 5 -+ ac_cv_lib_ossp_uuid_uuid_export=yes - fi - - fi diff --git a/patches-REL_17_4_WASM/postgresql-emscripten/src-Makefile.shlib.diff b/patches-REL_17_4_WASM/postgresql-emscripten/src-Makefile.shlib.diff deleted file mode 100644 index 06697ff6ade24..0000000000000 --- a/patches-REL_17_4_WASM/postgresql-emscripten/src-Makefile.shlib.diff +++ /dev/null @@ -1,49 +0,0 @@ ---- REL_17_4/src/Makefile.shlib -+++ pglite-REL_17_4/src/Makefile.shlib -@@ -224,6 +224,33 @@ - override CXXFLAGS += $(CXXFLAGS_SL_MODULE) - endif - -+ifeq ($(PORTNAME), emscripten) -+ LINK.shared = emsdk-shared -+ ifdef soname -+ # emscripten uses unversioned shared libraries -+ shlib = $(shlib_bare) -+ soname = $(shlib_bare) -+ endif -+ BUILD.exports = ( echo '{ global:'; $(AWK) '/^[^\#]/ {printf "%s;\n",$$1}' $<; echo ' local: *; };' ) >$@ -+ exports_file = $(SHLIB_EXPORTS:%.txt=%.list) -+# ifneq (,$(exports_file)) -+# LINK.shared += -Wl,--version-script=$(exports_file) -+# endif -+endif -+ -+ifeq ($(PORTNAME), wasi) -+ LINK.shared = wasi-shared -+ ifdef soname -+ # emscripten uses unversioned shared libraries -+ shlib = $(shlib_bare) -+ soname = $(shlib_bare) -+ endif -+ BUILD.exports = ( echo '{ global:'; $(AWK) '/^[^\#]/ {printf "%s;\n",$$1}' $<; echo ' local: *; };' ) >$@ -+ exports_file = $(SHLIB_EXPORTS:%.txt=%.list) -+# ifneq (,$(exports_file)) -+# LINK.shared += -Wl,--version-script=$(exports_file) -+# endif -+endif - - ## - ## BUILD -@@ -239,8 +266,11 @@ - endif - - all-static-lib: $(stlib) -- -+ifdef wasi -+all-shared-lib: all-static-lib -+else - all-shared-lib: $(shlib) -+endif - - # In this rule, "touch $@" works around a problem on some platforms wherein - # ar updates the library file's mod time with a value calculated to diff --git a/patches-REL_17_4_WASM/postgresql-emscripten/src-backend-commands-async.c.diff b/patches-REL_17_4_WASM/postgresql-emscripten/src-backend-commands-async.c.diff deleted file mode 100644 index 0299b57028d65..0000000000000 --- a/patches-REL_17_4_WASM/postgresql-emscripten/src-backend-commands-async.c.diff +++ /dev/null @@ -1,15 +0,0 @@ ---- REL_17_4/src/backend/commands/async.c -+++ pglite-REL_17_4/src/backend/commands/async.c -@@ -1651,8 +1651,12 @@ - * NotifyQueueLock; which is unlikely but certainly possible. So we - * just log a low-level debug message if it happens. - */ -+#if defined(__EMSCRIPTEN__) || defined(__wasi__) -+ HandleNotifyInterrupt(); -+#else - if (SendProcSignal(pid, PROCSIG_NOTIFY_INTERRUPT, procnos[i]) < 0) - elog(DEBUG3, "could not signal backend with PID %d: %m", pid); -+#endif - } - - pfree(pids); diff --git a/patches-REL_17_4_WASM/postgresql-pglite/src-backend-libpq-pqcomm.c.diff b/patches-REL_17_4_WASM/postgresql-pglite/src-backend-libpq-pqcomm.c.diff deleted file mode 100644 index 68b1992aac4bc..0000000000000 --- a/patches-REL_17_4_WASM/postgresql-pglite/src-backend-libpq-pqcomm.c.diff +++ /dev/null @@ -1,261 +0,0 @@ ---- REL_17_4/src/backend/libpq/pqcomm.c -+++ pglite-REL_17_4/src/backend/libpq/pqcomm.c -@@ -122,10 +122,18 @@ - static int PqSendBufferSize; /* Size send buffer */ - static size_t PqSendPointer; /* Next index to store a byte in PqSendBuffer */ - static size_t PqSendStart; /* Next index to send a byte in PqSendBuffer */ -- -+#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) - static char PqRecvBuffer[PQ_RECV_BUFFER_SIZE]; - static int PqRecvPointer; /* Next index to read a byte from PqRecvBuffer */ - static int PqRecvLength; /* End of data available in PqRecvBuffer */ -+#else -+static char PqRecvBuffer_static[PQ_RECV_BUFFER_SIZE]; -+static char *PqRecvBuffer; -+static int PqRecvPointer; -+static int PqRecvLength; -+volatile int querylen = 0; -+volatile FILE* queryfp = NULL; -+#endif - - /* - * Message status -@@ -135,6 +143,7 @@ - - - /* Internal functions */ -+ - static void socket_comm_reset(void); - static void socket_close(int code, Datum arg); - static void socket_set_nonblocking(bool nonblocking); -@@ -148,9 +157,6 @@ - static pg_noinline int internal_flush_buffer(const char *buf, size_t *start, - size_t *end); - --static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); --static int Setup_AF_UNIX(const char *sock_path); -- - static const PQcommMethods PqCommSocketMethods = { - .comm_reset = socket_comm_reset, - .flush = socket_flush, -@@ -160,6 +166,10 @@ - .putmessage_noblock = socket_putmessage_noblock - }; - -+static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); -+static int Setup_AF_UNIX(const char *sock_path); -+ -+ - const PQcommMethods *PqCommMethods = &PqCommSocketMethods; - - WaitEventSet *FeBeWaitSet; -@@ -181,7 +191,7 @@ - port->sock = client_sock->sock; - memcpy(&port->raddr.addr, &client_sock->raddr.addr, client_sock->raddr.salen); - port->raddr.salen = client_sock->raddr.salen; -- -+#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) - /* fill in the server (local) address */ - port->laddr.salen = sizeof(port->laddr.addr); - if (getsockname(port->sock, -@@ -273,14 +283,15 @@ - (void) pq_setkeepalivescount(tcp_keepalives_count, port); - (void) pq_settcpusertimeout(tcp_user_timeout, port); - } -- -+#endif /* WASM */ -+PDEBUG("# 285:" __FILE__); - /* initialize state variables */ - PqSendBufferSize = PQ_SEND_BUFFER_SIZE; - PqSendBuffer = MemoryContextAlloc(TopMemoryContext, PqSendBufferSize); - PqSendPointer = PqSendStart = PqRecvPointer = PqRecvLength = 0; - PqCommBusy = false; - PqCommReadingMsg = false; -- -+#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) - /* set up process-exit hook to close the socket */ - on_proc_exit(socket_close, 0); - -@@ -310,7 +321,12 @@ - MyLatch, NULL); - AddWaitEventToSet(FeBeWaitSet, WL_POSTMASTER_DEATH, PGINVALID_SOCKET, - NULL, NULL); -- -+#else /* WASM */ -+ PDEBUG("# 323: FIXME: socketfile"); -+ #pragma message "FIXME: use socketfile when overflowing PqRecvBuffer_static" -+ /* because we fill before starting reading message */ -+ PqRecvBuffer = &PqRecvBuffer_static[0]; -+#endif /* WASM */ - /* - * The event positions match the order we added them, but let's sanity - * check them to be sure. -@@ -730,7 +746,7 @@ - Assert(Unix_socket_group); - if (Unix_socket_group[0] != '\0') - { --#ifdef WIN32 -+#if defined(WIN32) || defined(__wasi__) - elog(WARNING, "configuration item \"unix_socket_group\" is not supported on this platform"); - #else - char *endptr; -@@ -909,6 +925,20 @@ - else - PqRecvLength = PqRecvPointer = 0; - } -+#if defined(__EMSCRIPTEN__) || defined(__wasi__) -+ if (queryfp && querylen) { -+ int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE - PqRecvPointer, queryfp); -+ querylen -= got; -+ PqRecvLength += got; -+ if (querylen<=0) { -+ puts("# 931: could close fp early here " __FILE__); -+ queryfp = NULL; -+ } -+ if (got>0) -+ return 0; -+ } -+ return EOF; -+#endif - - /* Ensure that we're in blocking mode */ - socket_set_nonblocking(false); -@@ -1011,7 +1041,7 @@ - *c = PqRecvBuffer[PqRecvPointer++]; - return 1; - } -- -+puts("# 1028: pq_getbyte_if_available N/I in " __FILE__ ); abort(); - /* Put the socket into non-blocking mode */ - socket_set_nonblocking(true); - -@@ -1115,6 +1145,7 @@ - return 0; - } - -+ - /* -------------------------------- - * pq_buffer_remaining_data - return number of bytes in receive buffer - * -@@ -1136,6 +1167,26 @@ - * This must be called before any of the pq_get* functions. - * -------------------------------- - */ -+#if defined(__EMSCRIPTEN__) || defined(__wasi__) -+EMSCRIPTEN_KEEPALIVE void -+pq_recvbuf_fill(FILE* fp, int packetlen) { -+ if (packetlen>PQ_RECV_BUFFER_SIZE) { -+ int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE, fp); -+ queryfp = fp; -+ querylen = packetlen - got; -+ PqRecvLength = got; -+puts("# 1160: input overflow"); -+ } else { -+ fread( PqRecvBuffer, packetlen, 1, fp); -+ PqRecvLength = packetlen; -+ queryfp = NULL; -+ querylen = 0; -+ } -+ PqRecvPointer = 0; -+} -+#endif -+extern int cma_rsize; -+static char * PqSendBuffer_save; - void - pq_startmsgread(void) - { -@@ -1147,7 +1198,29 @@ - ereport(FATAL, - (errcode(ERRCODE_PROTOCOL_VIOLATION), - errmsg("terminating connection because protocol synchronization was lost"))); -+#if defined(__EMSCRIPTEN__) || defined(__wasi__) -+ if (!pq_buffer_remaining_data()) { -+ if (cma_rsize) { -+ PqRecvPointer = 0; -+ PqRecvLength = cma_rsize; -+ PqRecvBuffer = (char*)0x1; -+ -+ PqSendPointer = 0; -+ PqSendBuffer_save = PqSendBuffer; -+ PqSendBuffer = 2 + (char*)(cma_rsize); -+ PqSendBufferSize = (CMA_MB*1024*1024) - (int)(&PqSendBuffer[0]); -+ } else { -+ PqRecvBuffer = &PqRecvBuffer_static[0]; -+ if (PqSendBuffer_save) -+ PqSendBuffer=PqSendBuffer_save; -+ PqSendBufferSize = PQ_SEND_BUFFER_SIZE; -+ } -+ } -+#if PDEBUG -+ printf("# 1199: pq_startmsgread cma_rsize=%d PqRecvLength=%d buf=%p reply=%p\n", cma_rsize, PqRecvLength, &PqRecvBuffer[0], &PqSendBuffer[0]); -+#endif - -+#endif - PqCommReadingMsg = true; - } - -@@ -1270,7 +1343,55 @@ - - return 0; - } -+#if defined(__EMSCRIPTEN__) || defined(__wasi__) -+extern FILE* SOCKET_FILE; -+extern int SOCKET_DATA; -+static int -+internal_putbytes(const char *s, size_t len) { -+ if (PqSendPointer >= PqSendBufferSize) { -+ fprintf(stderr, "# 1329: overflow %zu >= %d cma_rsize=%d CMA=%d\n", PqSendPointer, PqSendBufferSize,cma_rsize, CMA_MB); -+ } -+ -+ if (!cma_rsize) { -+ int wc= fwrite(s, 1, len, SOCKET_FILE); -+ SOCKET_DATA+=wc; -+ } else { -+ size_t amount; -+ while (len > 0) { -+ /* If buffer is full, then flush it out */ -+ if (PqSendPointer >= PqSendBufferSize) { -+ socket_set_nonblocking(false); -+ if (internal_flush()) -+ return EOF; -+ } -+ amount = PqSendBufferSize - PqSendPointer; -+ if (amount > len) -+ amount = len; -+ memcpy(PqSendBuffer + PqSendPointer, s, amount); -+ PqSendPointer += amount; -+ s += amount; -+ len -= amount; -+ SOCKET_DATA+=amount; -+ } -+ } -+ return 0; -+} - -+static int -+socket_flush(void) { -+ return internal_flush(); -+} -+ -+static int -+internal_flush(void) { -+ /* no flush for raw wire */ -+ if (!cma_rsize) { -+ PqSendStart = PqSendPointer = 0; -+ } -+ return 0; -+} -+ -+#else - - static inline int - internal_putbytes(const char *s, size_t len) -@@ -1421,7 +1542,7 @@ - *start = *end = 0; - return 0; - } -- -+#endif /* wasm */ - /* -------------------------------- - * pq_flush_if_writable - flush pending output if writable without blocking - * diff --git a/patches-REL_17_4_WASM/postgresql-pglite/src-bin-initdb-initdb.c.diff b/patches-REL_17_4_WASM/postgresql-pglite/src-bin-initdb-initdb.c.diff deleted file mode 100644 index 3e110b9c43ff3..0000000000000 --- a/patches-REL_17_4_WASM/postgresql-pglite/src-bin-initdb-initdb.c.diff +++ /dev/null @@ -1,217 +0,0 @@ ---- REL_17_4/src/bin/initdb/initdb.c -+++ pglite-REL_17_4/src/bin/initdb/initdb.c -@@ -171,7 +171,11 @@ - - - /* internal vars */ -+#if !defined(PGL_MAIN) - static const char *progname; -+#else -+# define dynamic_shared_memory_type initdb_dynamic_shared_memory_type -+#endif - static int encodingid; - static char *bki_file; - static char *hba_file; -@@ -811,6 +815,7 @@ - static char * - get_id(void) - { -+#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) - const char *username; - - #ifndef WIN32 -@@ -825,6 +830,10 @@ - username = get_user_name_or_exit(progname); - - return pg_strdup(username); -+#else -+ setenv("PGUSER", WASM_USERNAME, 0); -+ return pg_strdup(getenv("PGUSER")); -+#endif /* wasm */ - } - - static char * -@@ -1070,6 +1079,9 @@ - static const char * - choose_dsm_implementation(void) - { -+#if defined(__wasi__) || defined(__EMSCRIPTEN__) -+ return "posix"; -+#endif - #if defined(HAVE_SHM_OPEN) && !defined(__sun__) - int ntries = 10; - pg_prng_state prng_state; -@@ -1608,10 +1620,9 @@ - } - - PG_CMD_CLOSE(); -- -- termPQExpBuffer(&cmd); -+ termPQExpBuffer(&cmd); - free(bki_lines); -- -+PDEBUG("# 1624: BOOT pipe complete"); - check_ok(); - } - -@@ -1711,16 +1722,16 @@ - setup_run_file(FILE *cmdfd, const char *filename) - { - char **lines; -- -+int count=0; - lines = readfile(filename); -- - for (char **line = lines; *line != NULL; line++) - { - PG_CMD_PUTS(*line); - free(*line); -+ count ++ ; - } -- - PG_CMD_PUTS("\n\n"); -+fprintf(stderr, "# 1733: --------------------------------- added %s, %d lines\n", filename, count); - - free(lines); - } -@@ -2636,8 +2647,13 @@ - strlcpy(full_path, progname, sizeof(full_path)); - - if (ret == -1) -+#if defined(__EMSCRIPTEN__) || defined(__wasi__) -+ printf("# WARNING: program \"%s\" is needed by %s but was not found in the same directory as \"%s\"\n", -+ "postgres", progname, full_path); -+#else - pg_fatal("program \"%s\" is needed by %s but was not found in the same directory as \"%s\"", - "postgres", progname, full_path); -+#endif // wasm - else - pg_fatal("program \"%s\" was found by \"%s\" but was not the same version as %s", - "postgres", full_path, progname); -@@ -3096,7 +3112,7 @@ - initPQExpBuffer(&cmd); - printfPQExpBuffer(&cmd, "\"%s\" %s %s template1 >%s", - backend_exec, backend_options, extra_options, DEVNULL); -- -+PDEBUG("# 3115: post-bootstrap sql begin"); - PG_CMD_OPEN(cmd.data); - - setup_auth(cmdfd); -@@ -3134,14 +3150,53 @@ - - PG_CMD_CLOSE(); - termPQExpBuffer(&cmd); -- -+PDEBUG("# 3115: post-bootstrap sql end"); - check_ok(); - } - -+/* pglite entry point */ -+#if defined(PGL_INITDB_MAIN) -+extern void MemoryContextInit(void); -+extern volatile char *PREFIX; -+extern volatile char *PGDATA; -+extern char tmpstr[]; -+char * strcat_alloc(const char *head, const char *tail); -+void strconcat(char*p, const char *head, const char *tail); -+ - - int --main(int argc, char *argv[]) --{ -+pgl_initdb_main() { -+ char *pwfile = NULL; -+ char *pgdata = NULL; -+ -+ strconcat(tmpstr, "--pwfile=", PREFIX); -+ pwfile = strcat_alloc(tmpstr, "/password"); -+ -+ -+ strconcat(tmpstr, "--pwfile=", PREFIX); -+ pgdata = strcat_alloc("--pgdata=", PGDATA); -+ -+ char *argv[] = { -+ strcat_alloc(PREFIX,"/bin/initdb"), -+ // "--no-clean", -+ "--wal-segsize=1", -+ "--allow-group-access", "--no-sync", -+ "-E", "UTF8", -+ "--locale=C.UTF-8", "--locale-provider=libc", -+// "--builtin-locale=en_US.UTF-8", "--locale-provider=builtin", -+// "--locale-provider=icu", "--icu-locale=en-US", "--locale-provider=icu", -+ "-U", WASM_USERNAME, pwfile, //"--pwfile=" WASM_PREFIX "/password", -+ pgdata, // "--pgdata=" WASM_PREFIX "/base", -+ NULL -+ }; -+ -+ int argc = sizeof(argv) / sizeof(char*) - 1; -+ -+ -+#else -+int -+main(int argc, char *argv[]) { -+#endif - static struct option long_options[] = { - {"pgdata", required_argument, NULL, 'D'}, - {"encoding", required_argument, NULL, 'E'}, -@@ -3201,9 +3256,15 @@ - * POSIX says we must do this before any other usage of these files. - */ - setvbuf(stdout, NULL, PG_IOLBF, 0); -- -+#if defined(PGL_INITDB_MAIN) -+ progname = get_progname(argv[0]); -+printf("# 3245:" __FILE__ " calling pg_initdb_main for %s\n", progname); -+ MemoryContextInit(); -+ pg_logging_init(progname); -+#else - pg_logging_init(argv[0]); - progname = get_progname(argv[0]); -+#endif - set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("initdb")); - - if (argc > 1) -@@ -3409,7 +3470,7 @@ - if (icu_rules && locale_provider != COLLPROVIDER_ICU) - pg_fatal("%s cannot be specified unless locale provider \"%s\" is chosen", - "--icu-rules", "icu"); -- -+PDEBUG("# 3463:"__FILE__ " TODO: atexit(cleanup_directories_atexit)"); - atexit(cleanup_directories_atexit); - - /* If we only need to sync, just do it and exit */ -@@ -3445,13 +3506,13 @@ - get_restricted_token(); - - setup_pgdata(); -- -+PDEBUG("# 3493:pgl_initdb_main " __FILE__); - setup_bin_paths(argv[0]); -- -+PDEBUG("# 3495:pgl_initdb_main " __FILE__); - effective_user = get_id(); - if (!username) - username = effective_user; -- -+PDEBUG("# 3514:pgl_initdb_main " __FILE__); - if (strncmp(username, "pg_", 3) == 0) - pg_fatal("superuser name \"%s\" is disallowed; role names cannot begin with \"pg_\"", username); - -@@ -3479,7 +3540,7 @@ - get_su_pwd(); - - printf("\n"); -- -+puts("# 3527:" __FILE__); - initialize_data_directory(); - - if (do_sync) -@@ -3535,7 +3596,7 @@ - destroyPQExpBuffer(start_db_cmd); - } - -- -+PDEBUG("# 3583"); - success = true; - return 0; - } diff --git a/pglite-REL_17_4_WASM/build.sh b/pglite-REL_17_4_WASM/build.sh index bc8f709f84eed..71b209c0521db 100755 --- a/pglite-REL_17_4_WASM/build.sh +++ b/pglite-REL_17_4_WASM/build.sh @@ -52,8 +52,7 @@ Folders : build : $BUILD_PATH target : $PGROOT retarget : ${PGL_DIST_C} - native : ${PGL_DIST_NATIVE} - + native : ${PGL_DIST_NATIVE} build $(arch) : ${NATIVE} CPOPTS : $COPTS DEBUG : $DEBUG @@ -108,6 +107,7 @@ ________________________________________________________ cat > ${PGL_BUILD_NATIVE}/pglite-native.sh < 0 cma addr +volatile int channel = 0; + /* TODO : prevent multiple write and write while reading ? */ volatile int cma_wsize = 0; -volatile int cma_rsize = 0; // defined in postgres.c +volatile int cma_rsize = 0; // also defined in postgres.c for pqcomm +volatile bool sockfiles = false; // also defined in postgres.c for pqcomm + +__attribute__((export_name("get_buffer_size"))) +int +get_buffer_size(int fd) { + return (CMA_MB * 1024 * 1024) / CMA_FD; +} + +// TODO add query size +__attribute__((export_name("get_buffer_addr"))) +int +get_buffer_addr(int fd) { + return 1 + ( get_buffer_size(fd) *fd); +} + +__attribute__((export_name("get_channel"))) +int +get_channel() { + return channel; +} __attribute__((export_name("interactive_read"))) @@ -32,6 +55,8 @@ extern void ClientAuthentication(Port *port); extern FILE* SOCKET_FILE; extern int SOCKET_DATA; + + /* init sequence ___________________________________ @@ -162,16 +187,12 @@ static void io_init(bool in_auth, bool out_auth) { MyProcPort->canAcceptConnections = CAC_OK; #endif ClientAuthInProgress = out_auth; - SOCKET_FILE = NULL; SOCKET_DATA = 0; PDEBUG("\n\n\n# 165: io_init --------- Ready for CLIENT ---------"); } - - -volatile bool sockfiles = false; volatile bool is_wire = true; extern char * cma_port; extern void pq_startmsgread(void); @@ -191,13 +212,13 @@ use_wire(int state) { #endif if (state>0) { #if PGDEBUG - printf("# 199: wire mode, repl off, echo %d\n", force_echo); + printf("\n\n# 194: PACKET START: wire mode, repl off, echo %d\n", force_echo); #endif is_wire = true; is_repl = false; } else { #if PGDEBUG - printf("# 205: repl mode, no wire, echo %d\n", force_echo); + printf("\n\n# 200: PACKET START: repl mode, no wire, echo %d\n", force_echo); #endif is_wire = false; is_repl = true; @@ -359,7 +380,7 @@ interactive_one() { bool had_notification = notifyInterruptPending; bool notified = false; - send_ready_for_query = false; + // send_ready_for_query = false; if (!MyProcPort) { PDEBUG("# 353: client created"); @@ -367,9 +388,9 @@ interactive_one() { } #if PGDEBUG - puts("\n\n# 377: interactive_one"); + puts("\n\n# 369: interactive_one"); if (notifyInterruptPending) - PDEBUG("# 359: has notification !"); + PDEBUG("# 371: has notification !"); #endif // this could be pg_flush in sync mode. @@ -380,14 +401,19 @@ interactive_one() { } if (!cma_rsize) { - puts("# 388: socketfiles"); // prepare reply queue + // no cma : reading from file. writing to file. if (!SOCKET_FILE) { SOCKET_FILE = fopen(PGS_OLOCK, "w") ; MyProcPort->sock = fileno(SOCKET_FILE); } + } else { + // prepare file reply queue, just in case of cma overflow + // if unused the file will be kept open till next query. + if (!SOCKET_FILE) { + SOCKET_FILE = fopen(PGS_OLOCK, "w") ; + } } - MemoryContextSwitchTo(MessageContext); MemoryContextResetAndDeleteChildren(MessageContext); @@ -400,10 +426,10 @@ interactive_one() { if (send_ready_for_query) { if (IsAbortedTransactionBlockState()) { - puts("@@@@ TODO 390: idle in transaction (aborted)"); + PDEBUG("@@@@ TODO 403: idle in transaction (aborted)"); } else if (IsTransactionOrTransactionBlock()) { - puts("@@@@ TODO 393: idle in transaction"); + PDEBUG("@@@@ TODO 406: idle in transaction"); } else { if (notifyInterruptPending) { ProcessNotifyInterrupt(false); @@ -425,7 +451,9 @@ interactive_one() { /* * in cma mode (cma_rsize>0), client call the wire loop itself waiting synchronously for the results * in socketfiles mode, the wire loop polls a pseudo socket made from incoming and outgoing files. - * in repl mode (cma_rsize==0) output is on stdout not cma/socketfiles wire. repl mode is default. + * in repl mode (cma_rsize==0) output is on stdout not cma/socketfiles wire. + * repl mode is the simpleset mode where stdin is just copied into input buffer (limited by CMA size). + * TODO: allow to redirect stdout for fully external repl. */ peek = IO[0]; @@ -436,15 +464,15 @@ interactive_one() { if (!is_repl) { whereToSendOutput = DestRemote; if (!is_wire) - PDEBUG("# 426: repl message in cma buffer !"); + PDEBUG("# 439: repl message in cma buffer !"); } else { if (is_wire) - PDEBUG("# 429: wire message in cma buffer for REPL !"); + PDEBUG("# 442: wire message in cma buffer for REPL !"); whereToSendOutput = DestDebug; } } else { fp = fopen(PGS_IN, "r"); -PDEBUG("# 451:" PGS_IN); +PDEBUG("# 452:" PGS_IN "\n"); // read file in socket buffer for SocketBackend to consumme. if (fp) { fseek(fp, 0L, SEEK_END); @@ -489,7 +517,7 @@ PDEBUG("# 451:" PGS_IN); if (packetlen) { // it was startup/auth , write and return fast. if (peek<0) { - PDEBUG("# 471: handshake/auth/pass skip"); + PDEBUG("# 492: handshake/auth/pass skip"); goto wire_flush; } @@ -504,7 +532,7 @@ PDEBUG("# 451:" PGS_IN); firstchar = peek; goto incoming; } // wire msg -PDEBUG("# 500: NO DATA:" PGS_IN ); +PDEBUG("# 507: NO DATA:" PGS_IN "\n"); } // fp data read // is it REPL in cma ? @@ -521,7 +549,7 @@ PDEBUG("# 500: NO DATA:" PGS_IN ); #if PGDEBUG if (packetlen) IO[packetlen]=0; // wire blocks are not zero terminated - printf("\n# 500: fd=%d is_embed=%d is_repl=%d is_wire=%d fd %s,len=%d cma=%d peek=%d [%s]\n", MyProcPort->sock, is_embed, is_repl, is_wire, PGS_OLOCK, packetlen,cma_rsize, peek, IO); + printf("\n# 524: fd=%d is_embed=%d is_repl=%d is_wire=%d fd %s,len=%d cma=%d peek=%d [%s]\n", MyProcPort->sock, is_embed, is_repl, is_wire, PGS_OLOCK, packetlen,cma_rsize, peek, IO); #endif resetStringInfo(inBuf); @@ -531,9 +559,9 @@ PDEBUG("# 500: NO DATA:" PGS_IN ); appendStringInfoChar(inBuf, IO[i]); } } - + if (packetlen<2) { - puts("# 512: WARNING: empty packet"); + puts("# 536: WARNING: empty packet"); //cma_rsize= 0; if (is_repl) pg_prompt(); @@ -601,7 +629,7 @@ PDEBUG("# 500: NO DATA:" PGS_IN ); } DoingCommandRead = false; -#if PGDEBUG +#if 0 // PGDEBUG if (!pipelining) { printf("# 573: wire=%d 1stchar=%c Q: %s\n", is_wire, firstchar, inBuf->data); force_echo = false; @@ -612,7 +640,7 @@ PDEBUG("# 500: NO DATA:" PGS_IN ); if (!ignore_till_sync) { /* initially, or after error */ - send_ready_for_query = true; + // send_ready_for_query = true; if (notifyInterruptPending) ProcessClientReadInterrupt(true); } else { @@ -657,34 +685,41 @@ puts("# 631: PIPELINING + rfq"); if (SOCKET_DATA>0) { if (sockfiles) { - if (cma_wsize) + channel = -1; + if (cma_wsize) { puts("ERROR: cma was not flushed before socketfile interface"); + } } else { /* wsize may have increased with previous rfq so assign here */ cma_wsize = SOCKET_DATA; + channel = cma_rsize + 2; } if (SOCKET_FILE) { int outb = SOCKET_DATA; fclose(SOCKET_FILE); SOCKET_FILE = NULL; SOCKET_DATA = 0; - if (cma_wsize) - PDEBUG("# 626: cma and sockfile ???\n"); + + if (cma_wsize) { + PDEBUG("# 672: cma and sockfile ???\n"); + } + if (sockfiles) { #if PGDEBUG - printf("# 629: client:ready -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", outb); + printf("# 675: client:ready -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", outb); #endif rename(PGS_OLOCK, PGS_OUT); } } else { #if PGDEBUG - printf("\n# 635: in[%d] out[%d] flushed\n", cma_rsize, cma_wsize); + printf("\n# 681: in[%d] out[%d] flushed\n", cma_rsize, cma_wsize); #endif SOCKET_DATA = 0; } } else { - cma_wsize = 0; PDEBUG("# 680: no socket data"); + cma_wsize = 0; + PDEBUG("# 698: no data, send empty ?"); } } else { pg_prompt(); diff --git a/pglite-REL_17_4_WASM/native.sh b/pglite-REL_17_4_WASM/native.sh new file mode 100755 index 0000000000000..6b3777ae22391 --- /dev/null +++ b/pglite-REL_17_4_WASM/native.sh @@ -0,0 +1,189 @@ +#!/bin/bash + +[ -f /alpine ] && /sbin/apk add clang gcc + +mkdir -p ${SDKROOT}/src ${SDKROOT}/native +if [ -d ${SDKROOT}/src/w2c2 ] +then + echo " using local w2c2" +else + pushd ${SDKROOT}/src + git clone https://github.com/pygame-web/w2c2 + cp -R w2c2/w2c2/w2c2_base.h ${SDKROOT}/native/w2c2/ + cp -R w2c2/wasi ${SDKROOT}/native/ + popd +fi + +mv ${PGROOT}/bin/pglite.wasi pglite.wasi +${SDKROOT}/native/w2c2/w2c2 -f 0 -t 1 -p pglite.wasi pglite.c +mv pglite.wasi ${PGL_DIST_NATIVE}/ + +cat > tmp.py <0: + try: + bodies.write( makefunc(l) ) + except Exception as e: + print("?", l, e) + continue + +print("="*70) +with open('${WORKSPACE}/pglite-${PG_BRANCH}/pglite-modpython.c','r') as source: + with open('tmp.c','w') as out: + out.write( source.read().replace('\${WASM2C}','${WASM2C}') ) +END + +python3 tmp.py + +echo " building loader ..." + + +PYLD="-lm" +PYEXT="" + +if echo $CC|grep gcc +then + COPTS="-O0 -g0" + CCOPTS="-Wno-attributes" +else + COPTS="-O0 -g0" + CCOPTS="-fbracket-depth=4096 -Wno-unknown-attributes" +fi + +COMPILE="$CC -fPIC $PYINC $COPTS $CCOPTS -I${SDKROOT}/native -I${SDKROOT}/native/w2c2 -o ${WASM2C}$PYEXT tmp.c ${SDKROOT}/native/wasi/libw2c2wasi.a $PYLD -lc" +echo $COMPILE + +PYVER=$($PYTHON -V|cut -d' ' -f2|cut -d. -f1-2) +PYVER=${PYVER}$(${PYTHON}-config --abiflags) + +PYINC="-D__PYDK__=1 -shared $(${PYTHON}-config --includes)" +PYEXT=$(${PYTHON}-config --extension-suffix) +PYLD="-lpython$PYVER $(${PYTHON}-config --ldflags)" + +echo " +======================================================================== +WASM2C=$WASM2C +COPTS=$COPTS + +PYVER=$PYVER +PYEXT=$PYEXT + +PYINC=$PYINC +PYLD=$PYLD + +PGL_BUILD_NATIVE=${PGL_BUILD_NATIVE} +PGL_DIST_NATIVE=${PGL_DIST_NATIVE} +TARGET: ${WASM2C}$PYEXT +======================================================================== +" + +$COMPILE + + +[ -f ${WASM2C}$PYEXT ] && rm ${WASM2C}$PYEXT + +# -I${SDKROOT}/src/w2c2/w2c2 + +COMPILE="$CC -fPIC -Os -g0 $PYINC $CCOPTS -I${SDKROOT}/native -I${SDKROOT}/native/w2c2 -o ${WASM2C}$PYEXT tmp.c ${SDKROOT}/native/wasi/libw2c2wasi.a $PYLD -lc" +echo $COMPILE + +time $COMPILE + +if [ -f ${WASM2C} ] +then + du -hs ${WASM2C} + # ./${WASM2C} $@ +else + echo build native ${WASM2C} failed +fi + +PY=$(command -v python${PYMAJOR}.${PYMINOR}) + +echo "__________________________________________" +echo $PY +echo "__________________________________________" + + + +if [ -f ${WASM2C}$PYEXT ] +then + env -i $PY <= PqSendBufferSize) { - fprintf(stderr, "# 1329: overflow %zu >= %d cma_rsize=%d CMA=%d\n", PqSendPointer, PqSendBufferSize,cma_rsize, CMA_MB); - } - - if (!cma_rsize) { - int wc= fwrite(s, 1, len, SOCKET_FILE); - SOCKET_DATA+=wc; - } else { - size_t amount; + size_t amount; + if (!sockfiles) { while (len > 0) { /* If buffer is full, then flush it out */ if (PqSendPointer >= PqSendBufferSize) { + fprintf(stderr, "# 1361: overflow %zu >= %d cma_rsize=%d CMA_MB=%d\n", PqSendPointer, PqSendBufferSize, cma_rsize, CMA_MB); + int redirected = fwrite(PqSendBuffer, 1, PqSendPointer, SOCKET_FILE); + fprintf(stderr, "# 1363: redirected %d/%zu from cma to file\n", redirected, PqSendPointer); + sockfiles = true; + break; +/* socket_set_nonblocking(false); if (internal_flush()) return EOF; +*/ } amount = PqSendBufferSize - PqSendPointer; if (amount > len) @@ -1376,6 +1379,11 @@ internal_putbytes(const char *s, size_t len) { SOCKET_DATA+=amount; } } + + if (sockfiles) { + int wc= fwrite(s, 1, len, SOCKET_FILE); + SOCKET_DATA+=wc; + } return 0; } @@ -1387,7 +1395,7 @@ socket_flush(void) { static int internal_flush(void) { /* no flush for raw wire */ - if (!cma_rsize) { + if (sockfiles) { PqSendStart = PqSendPointer = 0; } return 0; @@ -2204,4 +2212,4 @@ pq_check_connection(void) } return true; -} \ No newline at end of file +} diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 767cda0ceebcd..59e94bddaa79c 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -110,6 +110,7 @@ int restrict_nonsystem_relation_kind; #if (defined(__EMSCRIPTEN__) || defined(__wasi__)) #if !defined(PGL_MAIN) volatile int cma_rsize = 0; + volatile bool sockfiles = false; #endif // PGL_MAIN bool quote_all_identifiers = false; FILE* SOCKET_FILE = NULL; diff --git a/src/port/pqsignal.c b/src/port/pqsignal.c index 8ceb097ac45df..5eb4d98840af6 100644 --- a/src/port/pqsignal.c +++ b/src/port/pqsignal.c @@ -170,7 +170,13 @@ pqsignal(int signo, pqsigfunc func) #endif } -/* sneak stubs into libpgport */ +/* sneak emsdk or wasi wasm port support into libpgport */ #if defined(SDK_PORT) -# include "sdk_port.c" +# if defined(__wasi__) +# include "sdk_port-wasi.c" +# endif +# if defined(__EMSCRIPTEN__) +// # include "sdk_port-emscripten.c" +# endif #endif + From e13a5a03d97dd9a4b31fa4326908a4ba491f900d Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 20 May 2025 07:14:53 +0200 Subject: [PATCH 02/19] fence debug --- src/backend/libpq/pqcomm.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 61cacec408401..ae9daac93479a 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -1356,18 +1356,14 @@ internal_putbytes(const char *s, size_t len) { size_t amount; if (!sockfiles) { while (len > 0) { - /* If buffer is full, then flush it out */ + /* If buffer is full, then flush it out from cma to file and continue from there */ if (PqSendPointer >= PqSendBufferSize) { - fprintf(stderr, "# 1361: overflow %zu >= %d cma_rsize=%d CMA_MB=%d\n", PqSendPointer, PqSendBufferSize, cma_rsize, CMA_MB); int redirected = fwrite(PqSendBuffer, 1, PqSendPointer, SOCKET_FILE); - fprintf(stderr, "# 1363: redirected %d/%zu from cma to file\n", redirected, PqSendPointer); sockfiles = true; +#if PGDEBUG + fprintf(stderr, "# 1364: overflow %zu >= %d redirect=%d cma_rsize=%d CMA_MB=%d \n", PqSendPointer, PqSendBufferSize, redirected, cma_rsize, CMA_MB); +#endif break; -/* - socket_set_nonblocking(false); - if (internal_flush()) - return EOF; -*/ } amount = PqSendBufferSize - PqSendPointer; if (amount > len) From ca552c0e0c19420cc510e78a2efb80470ebde9b9 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 20 May 2025 15:36:02 +0200 Subject: [PATCH 03/19] cleanup --- pglite-wasm | 1 - 1 file changed, 1 deletion(-) delete mode 120000 pglite-wasm diff --git a/pglite-wasm b/pglite-wasm deleted file mode 120000 index e2e295215d816..0000000000000 --- a/pglite-wasm +++ /dev/null @@ -1 +0,0 @@ -./pglite-REL_17_4_WASM \ No newline at end of file From a5d93fbf1299aca8ed638939587a45a1fc376b16 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 20 May 2025 16:48:20 +0200 Subject: [PATCH 04/19] link mode 2 --- pglite-REL_17_4_WASM/build.sh | 3 +- pglite-REL_17_4_WASM/pg_main.c | 2 +- wasm-build.sh | 50 +++++++++++++++++++++++++++------- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/pglite-REL_17_4_WASM/build.sh b/pglite-REL_17_4_WASM/build.sh index 71b209c0521db..8d6387eda4367 100755 --- a/pglite-REL_17_4_WASM/build.sh +++ b/pglite-REL_17_4_WASM/build.sh @@ -163,8 +163,9 @@ else EXPORTED_FUNCTIONS="_main,_use_wire,_pgl_initdb,_pgl_backend,_pgl_shutdown,_interactive_write,_interactive_read,_interactive_one" + EXPORTED_FUNCTIONS="$EXPORTED_FUNCTIONS,_get_channel,_get_buffer_size,_get_buffer_addr" - EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,FS_mount,FS_syncfs,FS_analyzePath,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack" +# EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,FS_mount,FS_syncfs,FS_analyzePath,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack" EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack" diff --git a/pglite-REL_17_4_WASM/pg_main.c b/pglite-REL_17_4_WASM/pg_main.c index abfea9ea37ddd..0f7e31c49081e 100644 --- a/pglite-REL_17_4_WASM/pg_main.c +++ b/pglite-REL_17_4_WASM/pg_main.c @@ -1,5 +1,5 @@ -// for handling REVOKIE ex in initdb +// for handling REVOKE exception in initdb #if defined(__wasi__) #define FIXME 1 #else diff --git a/wasm-build.sh b/wasm-build.sh index 4e026147fb11f..9cfdb344d01db 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -1,5 +1,4 @@ #!/bin/bash - export PG_VERSION=${PG_VERSION:-17.4} #set -x; @@ -33,6 +32,7 @@ export PG_DIST_EXT="${PG_DIST}/extensions-emsdk" export PGL_DIST_JS="${PG_DIST}/pglite-js" export PGL_DIST_NATIVE="${PG_DIST}/pglite-native" +export PGL_DIST_C="${PG_DIST}/pglite-native" export PGL_DIST_WEB="${PG_DIST}/pglite-web" export DEBUG=${DEBUG:-true} @@ -47,6 +47,7 @@ export PGUSER=${PGUSER:-postgres} export WASI=${WASI:-false} export WASI_SDK=${WASI_SDK:-25.0} export PYBUILD=${PYBUILD:-3.13} +export NATIVE=${NATIVE:-false} if $WASI @@ -64,9 +65,17 @@ else BUILD=emscripten if $DEBUG then - # clang default to O0 but specifying -O0 may trigger memory start address bug in emsdk - export COPTS="-g3 --no-wasm-opt" - export LOPTS=${LOPTS:-"-g3 --no-wasm-opt -sASSERTIONS=1"} + # clang default to O0 but specifying -O0 may trigger align bug in emsdk + if [ -f /alpine ] + then + # dev debug + export COPTS="-O2 -g3 --no-wasm-opt" + export LOPTS=${LOPTS:-"-O2 -g3 --no-wasm-opt -sASSERTIONS=1"} + else + # docker debug ( exepected to be ide friendly ) + export COPTS="-g3 --no-wasm-opt" + export LOPTS=${LOPTS:-"-g3 --no-wasm-opt -sASSERTIONS=1"} + fi else # DO NOT CHANGE COPTS - optimized wasm corruption fix export COPTS="-O2 -g3 --no-wasm-opt" @@ -198,6 +207,21 @@ export CC_PGLITE="-DPYDK=1 -DPG_PREFIX=${PGROOT} -I${PGROOT}/include ${CC_PGLITE +echo " + ---------------------------------------- +" +env|grep PG |grep -v BUILD +env|grep BUILD|grep -v PG +env|grep WA +env|grep PY + +echo " + ---------------------------------------- +" + + + + # ========================= symbol extractor ============================ OBJDUMP=${OBJDUMP:-true} @@ -497,9 +521,9 @@ then " else - if echo " $*"|grep -q " extra" - then - for extra_ext in ${EXTRA_EXT:-"vector"} +# if echo " $*"|grep -q " extra" +# then + for extra_ext in vector pg_ivm do if $CI then @@ -524,7 +548,7 @@ then python3 ${PORTABLE}/pack_extension.py done - fi +# fi # this is for initial emscripten MEMFS export PGPRELOAD="\ @@ -565,8 +589,14 @@ then cp ${PGL_DIST_WEB}/pglite.* pglite/packages/pglite/release/ pushd pglite export HOME=$PG_BUILD - [ -f $HOME/.local/share/pnpm/pnpm ] || wget -qO- https://get.pnpm.io/install.sh | ENV="$HOME/.bashrc" SHELL="$(which bash)" bash - - . $HOME/.bashrc + if [ -f ${PG_BUILD}/share/pnpm/pnpm ] + then + echo "assuming pnpm install done" + source $PG_BUILD/.bashrc + else + [ -f \$HOME/.local/share/pnpm/pnpm ] || wget -qO- https://get.pnpm.io/install.sh | ENV="$PG_BUILD/.bashrc" SHELL="\$(which bash)" bash - + fi + export PATH=$PATH:${PG_BUILD}/share/pnpm:\$(pwd)/node_modules/.pnpm/node_modules/.bin pnpm install -g npm vitest pnpm install pnpm run ts:build From 857809383d2f9fd48a14d40fafd5744a89d9a486 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 20 May 2025 16:53:52 +0200 Subject: [PATCH 05/19] test --- extra/pg_ivm.sh | 38 +++++++++++++++++++++++++++++++++++ extra/vector.sh | 4 ++-- pglite-REL_17_4_WASM/build.sh | 2 +- 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100755 extra/pg_ivm.sh diff --git a/extra/pg_ivm.sh b/extra/pg_ivm.sh new file mode 100755 index 0000000000000..cf7a31b77a96c --- /dev/null +++ b/extra/pg_ivm.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +mkdir -p build src + +pushd build + if [ -d pg_ivm ] + then + echo using local pgpg_ivm + else + git clone --recursive --no-tags --depth 1 --single-branch --branch main https://github.com/sraoss/pg_ivm + fi +popd + + + +if which emcc +then + echo -n +else + reset; + . ${SDKROOT}/wasm32-bi-emscripten-shell.sh + export PGROOT=${PGROOT:-/tmp/pglite} + export PATH=${PGROOT}/bin:$PATH +fi + + + +pushd build/pg_ivm + # path for wasm-shared already set to (pwd:pg build dir)/bin + # OPTFLAGS="" turns off arch optim (sse/neon). + PG_CONFIG=${PGROOT}/bin/pg_config emmake make OPTFLAGS="" install || exit 33 + + #cp sql/pg_ivm--1.10.sql ${PGROOT}/share/postgresql/extension/ + #rm -f ${PGROOT}/share/postgresql/extension/pg_ivm--?.?.?--?.?.?.sql ${PGROOT}/share/postgresql/extension/pg_ivm.sql + +popd + + diff --git a/extra/vector.sh b/extra/vector.sh index 3e6d98538cd02..3ecb0f46a5812 100755 --- a/extra/vector.sh +++ b/extra/vector.sh @@ -22,7 +22,7 @@ then echo -n else reset; - . /opt/python-wasm-sdk/wasm32-bi-emscripten-shell.sh + . ${SDKROOT:-/tmp/sdk}/wasm32-bi-emscripten-shell.sh export PGROOT=${PGROOT:-/tmp/pglite} export PATH=${PGROOT}/bin:$PATH fi @@ -33,7 +33,7 @@ pushd build/vector # OPTFLAGS="" turns off arch optim (sse/neon). PG_CONFIG=${PGROOT}/bin/pg_config emmake make OPTFLAGS="" install || exit 33 - cp sql/vector.sql sql/vector--0.8.0.sql ${PGROOT}/share/postgresql/extension/ + cp sql/vector--0.8.0.sql ${PGROOT}/share/postgresql/extension/ rm ${PGROOT}/share/postgresql/extension/vector--?.?.?--?.?.?.sql ${PGROOT}/share/postgresql/extension/vector.sql popd diff --git a/pglite-REL_17_4_WASM/build.sh b/pglite-REL_17_4_WASM/build.sh index 8d6387eda4367..55fe47660509d 100755 --- a/pglite-REL_17_4_WASM/build.sh +++ b/pglite-REL_17_4_WASM/build.sh @@ -180,7 +180,7 @@ else # tailored LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@exports" -LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS}" +# LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS}" fi echo " From 206e4f64778f58e02542d21b5b6b236aef59ee5b Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 20 May 2025 17:14:04 +0200 Subject: [PATCH 06/19] git missing in docker img --- wasm-build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wasm-build.sh b/wasm-build.sh index 9cfdb344d01db..4c0a0a064f9d7 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -521,9 +521,9 @@ then " else -# if echo " $*"|grep -q " extra" -# then - for extra_ext in vector pg_ivm + if echo " $*"|grep -q " extra" + then + for extra_ext in vector do if $CI then @@ -548,7 +548,7 @@ then python3 ${PORTABLE}/pack_extension.py done -# fi + fi # this is for initial emscripten MEMFS export PGPRELOAD="\ From ecf6012285e18d297aa4252ea1e81eb6e7526935 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Thu, 22 May 2025 22:17:57 +0200 Subject: [PATCH 07/19] extra/*sh --- extra/pg_ivm.sh | 22 ++---- extra/vector.sh | 19 +----- pglite-REL_17_4_WASM/build.sh | 7 +- wasm-build.sh | 125 ++++++++++++++++++++++------------ wasm-build/extension.sh | 14 ++++ wasm-build/linkexport.sh | 46 ++++++------- wasm-build/pack_extension.py | 100 ++++++++++++++------------- 7 files changed, 185 insertions(+), 148 deletions(-) create mode 100755 wasm-build/extension.sh diff --git a/extra/pg_ivm.sh b/extra/pg_ivm.sh index cf7a31b77a96c..cbdf6ad6114c4 100755 --- a/extra/pg_ivm.sh +++ b/extra/pg_ivm.sh @@ -1,34 +1,22 @@ #!/bin/bash -mkdir -p build src +. wasm-build/extension.sh pushd build if [ -d pg_ivm ] then echo using local pgpg_ivm else - git clone --recursive --no-tags --depth 1 --single-branch --branch main https://github.com/sraoss/pg_ivm + wget https://github.com/sraoss/pg_ivm/archive/refs/tags/v1.10.tar.gz -O-|tar xfz - + mv pg_ivm-* pg_ivm + #git clone --recursive --no-tags --depth 1 --single-branch --branch main https://github.com/sraoss/pg_ivm fi popd - - -if which emcc -then - echo -n -else - reset; - . ${SDKROOT}/wasm32-bi-emscripten-shell.sh - export PGROOT=${PGROOT:-/tmp/pglite} - export PATH=${PGROOT}/bin:$PATH -fi - - - pushd build/pg_ivm # path for wasm-shared already set to (pwd:pg build dir)/bin # OPTFLAGS="" turns off arch optim (sse/neon). - PG_CONFIG=${PGROOT}/bin/pg_config emmake make OPTFLAGS="" install || exit 33 + PG_CONFIG=${PGROOT}/bin/pg_config emmake make OPTFLAGS="" install || exit 19 #cp sql/pg_ivm--1.10.sql ${PGROOT}/share/postgresql/extension/ #rm -f ${PGROOT}/share/postgresql/extension/pg_ivm--?.?.?--?.?.?.sql ${PGROOT}/share/postgresql/extension/pg_ivm.sql diff --git a/extra/vector.sh b/extra/vector.sh index 3ecb0f46a5812..d0575d351f240 100755 --- a/extra/vector.sh +++ b/extra/vector.sh @@ -1,6 +1,6 @@ #!/bin/bash -mkdir -p build src +. wasm-build/extension.sh pushd build # [ -d pgvector ] || git clone --no-tags --depth 1 --single-branch --branch master https://github.com/pgvector/pgvector @@ -15,26 +15,13 @@ pushd build fi popd - - -if which emcc -then - echo -n -else - reset; - . ${SDKROOT:-/tmp/sdk}/wasm32-bi-emscripten-shell.sh - export PGROOT=${PGROOT:-/tmp/pglite} - export PATH=${PGROOT}/bin:$PATH -fi - - pushd build/vector # path for wasm-shared already set to (pwd:pg build dir)/bin # OPTFLAGS="" turns off arch optim (sse/neon). - PG_CONFIG=${PGROOT}/bin/pg_config emmake make OPTFLAGS="" install || exit 33 + PG_CONFIG=${PGROOT}/bin/pg_config emmake make OPTFLAGS="" install || exit 21 cp sql/vector--0.8.0.sql ${PGROOT}/share/postgresql/extension/ - rm ${PGROOT}/share/postgresql/extension/vector--?.?.?--?.?.?.sql ${PGROOT}/share/postgresql/extension/vector.sql + rm ${PGROOT}/share/postgresql/extension/vector--?.?.?--?.?.?.sql popd diff --git a/pglite-REL_17_4_WASM/build.sh b/pglite-REL_17_4_WASM/build.sh index 55fe47660509d..e8160e5e26671 100755 --- a/pglite-REL_17_4_WASM/build.sh +++ b/pglite-REL_17_4_WASM/build.sh @@ -234,9 +234,9 @@ ________________________________________________________ then echo " * linking node raw version of pglite ${PG_BRANCH}" - COPTS="$LOPTS" ${CC} ${CC_PGLITE} ${PGINC} -o ${PGL_DIST_JS}/pglite-js.js \ + COPTS="-O2 -g3 --no-wasm-opt" ${CC} ${CC_PGLITE} ${PGINC} -o ${PGL_DIST_JS}/pglite-js.js \ -sGLOBAL_BASE=${CMA_MB}MB -ferror-limit=1 \ - -sFORCE_FILESYSTEM=1 $EMCC_NODE \ + -sFORCE_FILESYSTEM=1 $EMCC_NODE -sMAIN_MODULE=1 -sEXPORT_ALL -sASSERTIONS=0 \ -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS \ -sEXPORTED_RUNTIME_METHODS=${EXPORTED_RUNTIME_METHODS} \ ${BUILD_PATH}/pglite.o \ @@ -244,6 +244,9 @@ ________________________________________________________ $LINK_ICU \ -lnodefs.js -lidbfs.js -lxml2 -lz + ./wasm-build/linkexport.sh + + echo " * linking web version of pglite ( with .data initial filesystem, and html repl)" COPTS="$LOPTS" ${CC} ${CC_PGLITE} -o ${PGL_DIST_WEB}/pglite.html --shell-file ${WORKSPACE}/pglite-${PG_BRANCH}/repl.html \ $PGPRELOAD \ diff --git a/wasm-build.sh b/wasm-build.sh index 4c0a0a064f9d7..d28b580eb2863 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -30,11 +30,14 @@ export PG_DIST=${DIST:-/tmp/sdk/dist} export PG_DIST_EXT="${PG_DIST}/extensions-emsdk" export PGL_DIST_JS="${PG_DIST}/pglite-js" +export PGL_DIST_LINK="${PG_DIST}/pglite-link" export PGL_DIST_NATIVE="${PG_DIST}/pglite-native" export PGL_DIST_C="${PG_DIST}/pglite-native" export PGL_DIST_WEB="${PG_DIST}/pglite-web" +DIST_ALL="${PGROOT}/bin ${PG_DIST} ${PG_DIST_EXT} ${PGL_DIST_JS} ${PGL_DIST_LINK} ${PGL_DIST_WEB} ${PGL_BUILD_NATIVE}" + export DEBUG=${DEBUG:-true} export USE_ICU=${USE_ICU:-false} @@ -99,12 +102,12 @@ EOE=true # default to user writeable paths in /tmp/ . -if mkdir -p ${PGROOT} ${PG_DIST} ${PG_DIST_EXT} ${PGL_DIST_JS} ${PGL_DIST_WEB} +if mkdir -p $DIST_ALL then echo "checking for valid prefix ${PGROOT} ${PG_DIST}" else - sudo mkdir -p ${PGROOT} ${PGROOT}/bin ${PG_DIST} ${PG_DIST_EXT} ${PGL_DIST_WEB} - sudo chown $(whoami) -R ${PGROOT} ${PG_DIST} + sudo mkdir -p $DIST_ALL + sudo chown $(whoami) -R $DIST_ALL fi # TODO: also handle PGPASSFILE hostname:port:database:username:password @@ -245,10 +248,45 @@ then fi else echo " - ERROR: $(which wasm-objdump) is not working properly ( is wasmtime ok ? ) + WARNING: $(which wasm-objdump) not working properly, trying alternate syntax " - OBJDUMP=false + cat > $WRAPPER < /proc/self/fd/2 +$(which wasmtime) --env PYTHONDONTWRITEBYTECODE=1 --dir / \$WASM \$@ +END + chmod +x $WRAPPER + + if $WRAPPER -h $WASIFILE | grep -q 'file format wasm 0x1' + then + mkdir -p $PGROOT/bin/ + if cp -f $WRAPPER $WASIFILE $PGROOT/bin/ + then + echo "wasm-objdump fixed and working, copied to $PGROOT/bin/" + else + OBJDUMP=false + fi + else + echo " + ERROR: $(which wasm-objdump) not working properly ( is wasmtime ok ? ) + + " + exit 268 + OBJDUMP=false + fi fi fi else @@ -325,7 +363,7 @@ END [ -f $dest/pg_debug.h ] || cp ${PG_DEBUG_HEADER} $dest/ done - # store all pg options that have impact on cmd line initdb/boot + # store all options that have impact on cmd line initdb/boot compile+link cat > ${PGROOT}/pgopts.sh </dev/null +# chmod +x ./extra/*.sh +# fi +# fi + + for extra_ext in ./extra/*.sh do - if $CI - then - #if [ -d $PREFIX/include/X11 ] - if true - then - echo -n - else - # install EXTRA sdk - . /etc/lsb-release - DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}" - CIVER=${CIVER:-$DISTRIB} - SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4 - echo "Installing $SDK_URL" - curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null - chmod +x ./extra/*.sh - fi - fi - echo "======================= ${extra_ext} : $(pwd) ===================" + LOG=$PG_DIST_EXT/$(basename ${extra_ext}).log + echo "======================= ${extra_ext} : $LOG ===================" - ./extra/${extra_ext}.sh || exit 522 + ${extra_ext} > $LOG || exit 552 - python3 ${PORTABLE}/pack_extension.py + python3 wasm-build/pack_extension.py done fi @@ -564,13 +605,12 @@ then echo " * building + linking pglite-wasm (initdb/loop/transport/repl/backend) " - if ${WORKSPACE}/pglite-${PG_BRANCH}/build.sh + + if $WASI then - if $WASI - then - echo "TODO: wasi pack/tests" - else - cat > pglite-link.sh < pglite-link.sh < ${WORKSPACE}/build/sizes.log -du -hs postgres.wasm >> ${WORKSPACE}/build/sizes.log -echo >> ${WORKSPACE}/build/sizes.log +echo "FULL:" > ${PGL_DIST_LINK}/sizes.log +du -hs ${PGL_DIST_JS}/pglite-js.* >> ${PGL_DIST_LINK}/sizes.log +echo >> ${PGL_DIST_LINK}/sizes.log -echo "getting wasm exports lists" -wasm-objdump -x $(realpath postgres.wasm) > ${WORKSPACE}/patches/exports/pgcore.wasm-objdump +echo " + * getting wasm exports lists +" + +pushd $(realpath ${PGL_DIST_JS}) + wasmtime --dir / --dir $(pwd)::. -- $(which wasm-objdump).wasi -x pglite-js.wasm > ${PGL_DIST_LINK}/exports/pgcore.wasm-objdump +popd -pushd ${WORKSPACE} - echo "getting postgres exports lists" - cat $(find build/postgres -type f |grep /exports) \ +pushd ${PGL_DIST_LINK} + echo " + * getting postgres exports lists from ${BUILD_PATH} +" + cat $(find ${BUILD_PATH} -type f |grep /exports) \ | grep -v ^\ local \ | grep -v ^{\ global \ - | sort | uniq > ${WORKSPACE}/patches/exports/pgcore.exports + | sort | uniq > ${PGL_DIST_LINK}/exports/pgcore.exports echo " - Merging wasm pg core symbols and postgres exports lists - into ${WORKSPACE} patches/exports/pgcore + * Merging wasm pg core symbols and postgres exports lists + into ${PGL_DIST_LINK}/exports/pgcore " - OBJDUMP=patches/exports/pgcore.wasm-objdump \ - PGDUMP=patches/exports/pgcore.exports \ - python3 wasm-build/getsyms.py exports > patches/exports/pgcore + OBJDUMP=${PGL_DIST_LINK}/exports/pgcore.wasm-objdump \ + PGDUMP=${PGL_DIST_LINK}/exports/pgcore.exports \ + python3 ${WORKSPACE}/wasm-build/getsyms.py exports > ${PGL_DIST_LINK}/exports/pgcore popd echo "============= link export : end ===============" diff --git a/wasm-build/pack_extension.py b/wasm-build/pack_extension.py index 2084f3c3becc5..249a1d9c22753 100644 --- a/wasm-build/pack_extension.py +++ b/wasm-build/pack_extension.py @@ -2,7 +2,7 @@ # TODO: use this file for merging symbols too. -# use recorded file list in ${PGROOT}/pg.${BUILD}.installed +# use recorded file list in ${PGROOT}/pg.installed # get other files into a tarball, find a .so and named everything after it @@ -12,11 +12,9 @@ import sys from pathlib import Path - class Error(Exception): pass - def gather(root: Path, *kw): for current, dirnames, filenames in os.walk(root): @@ -25,33 +23,45 @@ def gather(root: Path, *kw): # print(rel, len(dirnames), len(filenames)) yield rel, filenames +DIRS=[] +def pushd(d): + global DIRS + DIRS.append(os.getcwd()) + os.chdir(d) + return 1 + +def popd(): + global DIRS + os.chdir( DIRS.pop() ) + -def is_extension(path: Path, fullpath: Path): - global EXTNAME, SYMBOLS, PGPATCH, PGROOT +def is_extension(path:Path, fullpath:Path): + global EXTNAME, SYMBOLS, PGL_DIST_LINK, PGROOT asp = path.as_posix() # check .so - if asp.startswith("/lib/postgresql/"): + if asp.startswith('/lib/postgresql/'): if path.suffix == ".so": EXTNAME = path.stem - if os.environ.get("OBJDUMP", ""): - os.system(f"wasm-objdump -x {fullpath} 2>/dev/null > {PGROOT}/dumps/dump.{EXTNAME}") - os.system( - f"OBJDUMP={PGROOT}/dumps/dump.{EXTNAME} python3 wasm-build/getsyms.py imports 2>/dev/null > {PGPATCH}/imports/{EXTNAME}" - ) - with open(f"{PGPATCH}/imports/{EXTNAME}", "r") as f: - SYMBOLS = f.readlines() + dumpcmd = f"{PGROOT}/bin/wasm-objdump -x {fullpath} > {PGL_DIST_LINK}/dumps/dump.{EXTNAME} 2>/dev/null " + os.system(dumpcmd) + + os.system(f"OBJDUMP={PGL_DIST_LINK}/dumps/dump.{EXTNAME} python3 wasm-build/getsyms.py imports > {PGL_DIST_LINK}/imports/{EXTNAME}") + with open(f"{PGL_DIST_LINK}/imports/{EXTNAME}","r") as f: + SYMBOLS=f.readlines() return True # rpath - if asp.startswith("/lib/"): + if asp.startswith('/lib/'): return True - if asp.startswith("/share/postgresql/extension"): + if asp.startswith('/share/postgresql/extension'): return True + + async def archive(target_folder): global INSTALLED, PACKLIST @@ -59,66 +69,63 @@ async def archive(target_folder): for folder, filenames in gather(target_folder): walked.append([folder, filenames]) + for folder, filenames in walked: for filename in filenames: test = Path(folder) / Path(filename) asp = test.as_posix() - if (PGROOT / test).is_symlink(): + if (PGROOT/test).is_symlink(): print("SYMLINK:", test) continue if asp not in INSTALLED: - if asp.startswith("/sdk/"): - continue - - if asp.startswith("/base/"): + if asp.startswith('/sdk/'): continue - if asp.startswith("/dumps/"): - continue - - if asp.startswith("/dump."): + if asp.startswith('/base/'): continue fp = PGROOT / asp[1:] if fp.is_symlink(): + print("SYMLINK:", fp) continue + if is_extension(test, fp): - # print(f"{EXTNAME=}", test ) - PACKLIST.append([fp, test]) + print(f"{EXTNAME=}", test, fp) + PACKLIST.append( [fp, test] ) else: print("custom:", test) DIST = Path(os.environ.get("PG_DIST_EXT", "/tmp/sdk/dist/extensions-emsdk")) -PGROOT = Path(os.environ.get("PGROOT", "/tmp/pglite")) -PGPATCH = Path(os.environ.get("PGPATCH", PGROOT)) +BUILD=os.environ.get('BUILD','emscripten') +PGROOT=Path(os.environ.get('PGROOT',"/tmp/pglite")) +PGL_DIST_LINK=Path(os.environ.get('PGL_DIST_LINK', "/tmp/sdk/dist/pglite-link")) INSTALLED = [] EXTNAME = "" PACKLIST = [] -SYMBOLS = [] +SYMBOLS=[] PREINST = "/plpgsql" IS_PREINST = PREINST in sys.argv -for line in open(PGROOT / f"pg.{os.environ.get('BUILD','emscripten')}.installed" ).readlines(): +for line in open(PGROOT / f"pg.{BUILD}.installed" ).readlines(): asp = Path(line[1:].strip()).as_posix() if IS_PREINST: - if asp.find(PREINST) > 0: + if asp.find(PREINST)>0: continue - INSTALLED.append(asp) + INSTALLED.append( asp ) -print("=" * 80) -asyncio.run(archive(PGROOT)) -print("=" * 80) +print("="*80) +asyncio.run( archive(PGROOT) ) +print("="*80) if not EXTNAME: print("MAYBE ERROR: no new installed extension found, is it builtin ?") sys.exit(0) -print( - f""" +print(f""" PG installed in : {PGROOT=} @@ -126,21 +133,20 @@ async def archive(target_folder): -""" -) +""") swd = os.getcwd() if (not IS_PREINST) and ("builtin" not in sys.argv): if len(PACKLIST): - os.chdir(PGROOT) - with tarfile.open(DIST / f"{EXTNAME}.tar", "w:") as tar: - for fp, fn in PACKLIST: - print(f"{EXTNAME} : {fp} => {fn}") - tar.add(fn.as_posix()[1:]) - # if "builtin" not in sys.argv: - os.remove(fp) - os.chdir(swd) + if pushd(PGROOT): + with tarfile.open(DIST / f"{EXTNAME}.tar", "w:") as tar: + for fp, fn in PACKLIST: + print(f"{EXTNAME} : {fp} => {fn}") + tar.add(fn.as_posix()[1:]) + # if "builtin" not in sys.argv: + os.remove(fp) + popd() else: print(f"Nothing found to pack for {EXTNAME}, did you 'make install' ?") else: From a5bb9bc827cfe831e27ad9cbe4d6d0c61ebc5bb1 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 27 May 2025 07:58:36 +0200 Subject: [PATCH 08/19] symbol list fix --- pglite-REL_17_4_WASM/build.sh | 57 ++++++++++++++++++------ portable/Dockerfile | 4 +- portable/portable.sh | 84 ++++++++++++++++++++++++++--------- wasm-build.sh | 73 ++++++++++++++++-------------- wasm-build/linkimports.sh | 75 +++++++++++++++++++++++++------ 5 files changed, 209 insertions(+), 84 deletions(-) diff --git a/pglite-REL_17_4_WASM/build.sh b/pglite-REL_17_4_WASM/build.sh index e8160e5e26671..a957ada026b2a 100755 --- a/pglite-REL_17_4_WASM/build.sh +++ b/pglite-REL_17_4_WASM/build.sh @@ -179,7 +179,10 @@ else # LINKER="-sMAIN_MODULE=2" # tailored - LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@exports" +# LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pgcore" + LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pglite" + + # LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS}" fi @@ -232,42 +235,70 @@ ________________________________________________________ if ${CC} ${CC_PGLITE} ${PGINC} -o ${BUILD_PATH}/pglite.o -c ${WORKSPACE}/pglite-${PG_BRANCH}/pg_main.c \ -Wno-incompatible-pointer-types-discards-qualifiers then - echo " * linking node raw version of pglite ${PG_BRANCH}" + echo " * linking node raw version of pglite ${PG_BRANCH} (with all symbols)" - COPTS="-O2 -g3 --no-wasm-opt" ${CC} ${CC_PGLITE} ${PGINC} -o ${PGL_DIST_JS}/pglite-js.js \ + if COPTS="-O2 -g3 --no-wasm-opt" ${CC} ${CC_PGLITE} ${PGINC} -o ${PGL_DIST_JS}/pglite-js.js \ -sGLOBAL_BASE=${CMA_MB}MB -ferror-limit=1 \ -sFORCE_FILESYSTEM=1 $EMCC_NODE -sMAIN_MODULE=1 -sEXPORT_ALL -sASSERTIONS=0 \ -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS \ -sEXPORTED_RUNTIME_METHODS=${EXPORTED_RUNTIME_METHODS} \ ${BUILD_PATH}/pglite.o \ - $LINKER $LIBPGCORE \ + $LIBPGCORE \ $LINK_ICU \ -lnodefs.js -lidbfs.js -lxml2 -lz + then + ./wasm-build/linkexport.sh + ./wasm-build/linkimports.sh + else + echo " + * linking node raw version of pglite failed"; exit 250 + + fi + + + echo " + + * linking version of pglite ( with .data initial filesystem, and html repl) (required symbols) - ./wasm-build/linkexport.sh + BUILD_PATH=${BUILD_PATH} + LINKER=$LINKER + LIBPGCORE=$LIBPGCORE - echo " * linking web version of pglite ( with .data initial filesystem, and html repl)" - COPTS="$LOPTS" ${CC} ${CC_PGLITE} -o ${PGL_DIST_WEB}/pglite.html --shell-file ${WORKSPACE}/pglite-${PG_BRANCH}/repl.html \ +" + +# function from : +# ${BUILD_PATH}/src/interfaces/libpq/libpq.a +# required are to be found in : +# pglite.o +# + + if COPTS="$LOPTS" ${CC} ${CC_PGLITE} -o ${PGL_DIST_WEB}/pglite.html --shell-file ${WORKSPACE}/pglite-${PG_BRANCH}/repl.html \ $PGPRELOAD \ -sGLOBAL_BASE=${CMA_MB}MB -ferror-limit=1 \ -sFORCE_FILESYSTEM=1 -sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web \ + $LINKER \ -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module \ - -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS \ + -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sWARN_ON_UNDEFINED_SYMBOLS=1 -sERROR_ON_UNDEFINED_SYMBOLS=0 \ -sEXPORTED_RUNTIME_METHODS=${EXPORTED_RUNTIME_METHODS} \ ${PGINC} ${BUILD_PATH}/pglite.o \ - $LINKER $LIBPGCORE \ + $LIBPGCORE \ $LINK_ICU \ -lnodefs.js -lidbfs.js -lxml2 -lz - + then + du -hs du -hs ${PG_DIST}/* + touch ${WORKSPACE}/done + else + echo " + * linking web version of pglite failed"; exit 272 + fi else - echo "compilation of libpglite ${PG_BRANCH} failed" - exit 220 + echo "compilation of libpglite ${PG_BRANCH} failed"; exit 275 fi fi -du -hs ${PG_DIST}/* + echo "pglite/build($BUILD): end" diff --git a/portable/Dockerfile b/portable/Dockerfile index 2f70f37e0d75d..c4b6f251867b2 100755 --- a/portable/Dockerfile +++ b/portable/Dockerfile @@ -6,10 +6,10 @@ ENV PIP_NO_CACHE_DIR 1 # nb: the python for build from sdk built with clang directly is used as SYS_PYTHON for emsdk RUN apk add --no-cache --virtual .build-deps \ - tar file lz4 \ + tar file lz4 xz \ git patch bison flex \ findutils binutils coreutils \ - libffi curl perl nodejs \ + libffi tzdata-utils curl perl nodejs \ make autoconf automake libtool pkgconfig \ ; diff --git a/portable/portable.sh b/portable/portable.sh index 557eda934ca25..23ca54cd2f7a5 100755 --- a/portable/portable.sh +++ b/portable/portable.sh @@ -1,11 +1,13 @@ #!/usr/bin/env bash -export PG_VERSION=${PG_BRANCH:-17.4} +export PG_VERSION=${PG_VERSION:-17.4} export PG_BRANCH=${PG_BRANCH:-REL_17_4_WASM} export PORTABLE=$(realpath $(dirname $0)) export ROOT=$(realpath $(pwd)) export SDKROOT=${SDKROOT:-/tmp/sdk} +export WASI=${WASI:-false} echo " + ================================================================================================== ================================================================================================== @@ -16,6 +18,7 @@ PG_BRANCH=$PG_BRANCH SDKROOT=$SDKROOT DEBUG=$DEBUG USE_ICU=$USE_ICU +WASI=$WASI ================================================================================================== ================================================================================================== @@ -24,15 +27,16 @@ USE_ICU=$USE_ICU - export PATH=$PORTABLE:$PATH export WORKDIR=${ROOT} export CONTAINER_PATH=${CONTAINER_PATH:-/tmp/fs} export HOME=/tmp -export PROOT=${PORTABLE}/proot +export PROOT=${PORTABLE}/proot.$(arch) +export ALPINEPROOT_NO_PULSE=true + -# git remove empty dirs -mkdir -p ${WORKDIR}/sdk/dist +# git would remove empty dirs +mkdir -p ${WORKDIR}/dist-${PG_BRANCH} ${WORKDIR}/build-${PG_BRANCH} # -------------------------------------------------------- # "docker emulation" @@ -191,7 +195,7 @@ __get_container_url() { } __start() { - proot -0 rm -rf $CONTAINER_PATH/proc + $PROOT -0 rm -rf $CONTAINER_PATH/proc mkdir $CONTAINER_PATH/proc # Proceed make fake /proc/version @@ -371,11 +375,13 @@ __start() { COMMANDS+=" -b /proc/self/fd/1:/dev/stdout" COMMANDS+=" -b /proc/self/fd/2:/dev/stderr" COMMANDS+=" -b ${WORKDIR}:/workspace" - COMMANDS+=" -b ${WORKDIR}/dist:/tmp/sdk/dist" + COMMANDS+=" -b ${WORKDIR}/build-${PG_BRANCH}:/tmp/sdk/build" + COMMANDS+=" -b ${WORKDIR}/dist-${PG_BRANCH}:/tmp/sdk/dist" for f in stat version loadavg vmstat uptime do [ -f "$CONTAINER_PATH/proc/.$f" ] && COMMANDS+=" -b $CONTAINER_PATH/proc/.$f:/proc/$f" done +# --change-id=uid:gid COMMANDS+=" -r $CONTAINER_PATH -0 -w /root" COMMANDS+=" -b $CONTAINER_PATH/root:/dev/shm" @@ -407,7 +413,10 @@ __start() { } -if git checkout ${PG_BRANCH}-pglite + + +# git checkout ${PG_BRANCH} +if cd ${WORKDIR}/postgresql-${PG_BRANCH} then if [ -f postgresql-${PG_BRANCH}.patched ] then @@ -417,12 +426,13 @@ then Patching branch ${PG_BRANCH} with : -$(find patches-${PG_BRANCH}/postgresql-*) +$(find ${WORKDIR}/patches-${PG_BRANCH}/postgresql-*) " - # these don't exist in a released postgres. + + # these initially don't exist in a released postgres. touch ./src/template/emscripten \ ./src/include/port/emscripten.h \ ./src/include/port/wasm_common.h \ @@ -434,9 +444,9 @@ $(find patches-${PG_BRANCH}/postgresql-*) postgresql-wasi \ postgresql-pglite do - if [ -d patches-${PG_BRANCH}/$patchdir ] + if [ -d ${WORKDIR}/patches-${PG_BRANCH}/$patchdir ] then - for one in patches-${PG_BRANCH}/$patchdir/*.diff + for one in ${WORKDIR}/patches-${PG_BRANCH}/$patchdir/*.diff do if cat $one | patch -p1 then @@ -446,7 +456,7 @@ $(find patches-${PG_BRANCH}/postgresql-*) Fatal: failed to apply patch : $one " - exit 366 + exit 457 fi done fi @@ -454,21 +464,53 @@ Fatal: failed to apply patch : $one touch postgresql-${PG_BRANCH}.patched fi + if [ -d $CONTAINER_PATH/${SDKROOT} ] then - echo using cached version + echo " + * using cached sdk version from $CONTAINER_PATH/${SDKROOT} +" else - SDK_URL=https://github.com/pygame-web/portable-sdk/releases/download/3.1.74.7bi/python3.13-wasm-sdk-alpine-3.21.tar.lz4 - echo "setting up sdk $SDK_URL" + # SDK_URL=https://github.com/pygame-web/portable-sdk/releases/download/3.1.74.7bi/python3.13-wasm-sdk-alpine-3.21.tar.lz4 + SDK_URL=https://github.com/pygame-web/portable-sdk/releases/download/3.1.61.8/python3.12-wasm-sdk-debian12-$(arch).tar.lz4 + echo " + + * Setting up emsdk+wasi sdk from $SDK_URL + +" pushd $CONTAINER_PATH - mkdir -p /tmp/sdk - tmpfile=/tmp/sdk/python3.13-wasm-sdk-alpine-3.21.tar.lz4 - [ -f /opt/python3.13-wasm-sdk-alpine-3.21.tar.lz4 ] && cp -f /opt/python3.13-wasm-sdk-alpine-3.21.tar.lz4 $tmpfile - [ -f /tmp/sdk/python3.13-wasm-sdk-alpine-3.21.tar.lz4 ] || wget -q $SDK_URL -O$tmpfile + mkdir -p tmp + tmpfile=tmp/python-wasm-sdk-alpine-3.21.tar.lz4 + # local cache + [ -f $PORTABLE/python-wasm-sdk-alpine-3.21.tar.lz4 ] && cp -f $PORTABLE/python-wasm-sdk-alpine-3.21.tar.lz4 $tmpfile + [ -f $tmpfile ] || wget -q $SDK_URL -O$tmpfile cat $tmpfile | tar x --use-compress-program=lz4 + + # unpack wasi sdk (common) + tar xf $PORTABLE/wasi-sdk-25.tar.xz + # unpack wasi sdk ( binary ) + tar xf $PORTABLE/wasi-sdk-25.0-$(arch)-linux.tar.xz + + # install arch binaries to sdk wasi sdk root + mv tmp/sdk/wasisdk/wasi-sdk-25.0-$(arch)-linux/* tmp/sdk/wasisdk/upstream/ + + if [ -f $CONTAINER_PATH/usr/bin/python3 ] + then + echo "system python found" + else + if [ -L $CONTAINER_PATH/usr/bin/python3 ] + then + echo "linked python for build found" + else + echo "Setting python for build as system python3" + mkdir -p $CONTAINER_PATH/usr/bin + ln -s $SDKROOT/devices/$(arch)/usr/bin/python3 $CONTAINER_PATH/usr/bin/python3 + fi + fi popd fi + # prevent erasing touch ${CONTAINER_PATH}${SDKROOT}/dev @@ -479,5 +521,5 @@ Fatal: failed to apply patch : $one alpineproot "apk add bash;/bin/bash --init-file /initrc" fi else - echo Error need PG_BRANCH=$PG_BRANCH set to a valid branch + echo "Error need PG_BRANCH=$PG_BRANCH set to a valid postgres-pglite WASM branch" fi diff --git a/wasm-build.sh b/wasm-build.sh index d28b580eb2863..0bd4b98f422cd 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -613,44 +613,48 @@ then cat > pglite-link.sh < patches/imports/pgcore + > ${PGL_DIST_LINK}/imports/pgcore for extra_pg_so in $(find $PGROOT/lib/postgresql/|grep \.so$) do SOBASE=patches/imports.pgcore/$(basename $extra_pg_so .so) wasm-objdump -x $(realpath $extra_pg_so) > $SOBASE.wasm-objdump OBJDUMP=$SOBASE.wasm-objdump \ - PGDUMP=patches/exports/pgcore.exports \ - python3 wasm-build/getsyms.py imports >> patches/imports/pgcore + PGDUMP=${PGL_DIST_LINK}/exports/pgcore.exports \ + python3 wasm-build/getsyms.py imports >> ${PGL_DIST_LINK}/imports/pgcore done popd @@ -26,13 +27,13 @@ popd #_emscripten_copy_to_end # copyFrom,copyTo,copyToEnd - cat ${WORKSPACE}/patches/imports/* | sort | uniq > /tmp/symbols + cat ${PGL_DIST_LINK}/imports/* | sort | uniq > /tmp/symbols echo "Requesting $(wc -l /tmp/symbols) symbols from pg core for PGlite extensions" - python3 < ${WORKSPACE}/patches/exports/pglite + python3 < ${PGL_DIST_LINK}/exports/pglite import sys import os @@ -41,7 +42,7 @@ def dbg(*argv, **kw): kw.setdefault('file',sys.stderr) return print(*argv,**kw) -with open("${WORKSPACE}/patches/exports/pgcore", "r") as file: +with open("${PGL_DIST_LINK}/exports/pgcore", "r") as file: exports = set(map(str.strip, file.readlines())) with open("/tmp/symbols", "r") as file: @@ -49,8 +50,25 @@ with open("/tmp/symbols", "r") as file: matches = list( imports.intersection(exports) ) + # ? for sym in """ + +_clear_error +_get_buffer_addr +_get_buffer_size +_get_channel +_interactive_one +_interactive_read +_interactive_write +_pgl_backend +_pgl_closed +_pgl_initdb +_pgl_shutdown +_use_wire + +_main + _ErrorContext _check_function_bodies _clock_gettime @@ -58,13 +76,8 @@ _CurrentMemoryContext ___cxa_throw _error_context_stack _getenv -_interactive_one -_interactive_read -_interactive_write _lowerstr -_main -_pg_initdb -_pg_shutdown + _readstoplist _searchstoplist _setenv @@ -78,10 +91,44 @@ _TopMemoryContext matches.sort() -for sym in matches: - print(sym) +for sym in """ +_PQcancelCreate +_PQcancelErrorMessage +_PQcancelFinish +_PQcancelPoll +_PQcancelSocket +_PQcancelStart +_PQclear +_PQcmdStatus +_PQconnectPoll +_PQconnectStartParams +_PQconnectionUsedPassword +_PQconsumeInput +_PQerrorMessage +_PQfinish +_PQgetResult +_PQgetisnull +_PQgetvalue +_PQisBusy +_PQnfields +_PQntuples +_PQresultErrorField +_PQresultStatus +_PQsendQuery +_PQserverVersion +_PQsetSingleRowMode +_PQsocket +_PQstatus +_pgresStatus +""".splitlines(): + if sym and sym in matches: + dbg(f"\t* Removed symbol '{sym}'") + matches.remove(sym) +# matches.append('') +for sym in matches: + print(sym) dbg(f""" exports {len(exports)} From 4632e29e227c9f6754e29088f131902313d333be Mon Sep 17 00:00:00 2001 From: pmp-p Date: Mon, 2 Jun 2025 10:36:21 +0200 Subject: [PATCH 09/19] dce fix --- pglite-REL_17_4_WASM/build.sh | 15 ++++--- pglite-REL_17_4_WASM/interactive_one.c | 13 ++++++ pglite-REL_17_4_WASM/pg_main.c | 6 +-- pglite-REL_17_4_WASM/repl.html | 55 ++++++++++++++++++++++---- wasm-build.sh | 43 +++++++++++++------- wasm-build/linkexport.sh | 2 +- wasm-build/linkimports.sh | 12 ++++++ 7 files changed, 115 insertions(+), 31 deletions(-) diff --git a/pglite-REL_17_4_WASM/build.sh b/pglite-REL_17_4_WASM/build.sh index a957ada026b2a..88b8cc523e5f6 100755 --- a/pglite-REL_17_4_WASM/build.sh +++ b/pglite-REL_17_4_WASM/build.sh @@ -174,16 +174,19 @@ else then # FULL LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS}" +# LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pglite" else # min # LINKER="-sMAIN_MODULE=2" + +# LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS}" +# LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pglite" # tailored -# LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pgcore" LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pglite" - -# LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS}" + # LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pglite" + #export LOPTS="-O2 -g2 --no-wasm-opt -sASSERTIONS=0" fi echo " @@ -273,13 +276,15 @@ ________________________________________________________ # pglite.o # - if COPTS="$LOPTS" ${CC} ${CC_PGLITE} -o ${PGL_DIST_WEB}/pglite.html --shell-file ${WORKSPACE}/pglite-${PG_BRANCH}/repl.html \ +# LOPTS="-Os -g0" +# + if EMCC_FORCE_STDLIBS=1 COPTS="$LOPTS" ${CC} ${CC_PGLITE} -o ${PGL_DIST_WEB}/pglite.html --shell-file ${WORKSPACE}/pglite-${PG_BRANCH}/repl.html \ $PGPRELOAD \ -sGLOBAL_BASE=${CMA_MB}MB -ferror-limit=1 \ -sFORCE_FILESYSTEM=1 -sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web \ $LINKER \ -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module \ - -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sWARN_ON_UNDEFINED_SYMBOLS=1 -sERROR_ON_UNDEFINED_SYMBOLS=0 \ + -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS=1 \ -sEXPORTED_RUNTIME_METHODS=${EXPORTED_RUNTIME_METHODS} \ ${PGINC} ${BUILD_PATH}/pglite.o \ $LIBPGCORE \ diff --git a/pglite-REL_17_4_WASM/interactive_one.c b/pglite-REL_17_4_WASM/interactive_one.c index fb8de8033c271..9357ec62acf80 100644 --- a/pglite-REL_17_4_WASM/interactive_one.c +++ b/pglite-REL_17_4_WASM/interactive_one.c @@ -720,6 +720,12 @@ puts("# 631: PIPELINING + rfq"); } else { cma_wsize = 0; PDEBUG("# 698: no data, send empty ?"); +// TODO: dedup 739 + if (sockfiles) { + fclose(SOCKET_FILE); + SOCKET_FILE = NULL; + rename(PGS_OLOCK, PGS_OUT); + } } } else { pg_prompt(); @@ -729,6 +735,13 @@ puts("# 631: PIPELINING + rfq"); puts("# 686: socket has data"); if (sockfiles) printf("# 688: socket file not flushed -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", SOCKET_DATA); + } else { +// TODO: dedup 723 + if (sockfiles) { + fclose(SOCKET_FILE); + SOCKET_FILE = NULL; + rename(PGS_OLOCK, PGS_OUT); + } } if (cma_wsize) puts("ERROR: cma was not flushed before socketfile interface"); diff --git a/pglite-REL_17_4_WASM/pg_main.c b/pglite-REL_17_4_WASM/pg_main.c index 0f7e31c49081e..66822227b7fd4 100644 --- a/pglite-REL_17_4_WASM/pg_main.c +++ b/pglite-REL_17_4_WASM/pg_main.c @@ -1,9 +1,9 @@ // for handling REVOKE exception in initdb -#if defined(__wasi__) -#define FIXME 1 +#if defined(__wasi__) || defined(__EMSCRIPTEN__) +# define FIXME 1 #else -#define FIXME 0 +# define FIXME 0 #endif #define PGL_MAIN diff --git a/pglite-REL_17_4_WASM/repl.html b/pglite-REL_17_4_WASM/repl.html index 22ff371f5d161..4c7f2b3a445b9 100644 --- a/pglite-REL_17_4_WASM/repl.html +++ b/pglite-REL_17_4_WASM/repl.html @@ -307,9 +307,51 @@ break case 1: - //data = "SELECT now(), current_database(), session_user, current_user;" - data = "SELECT REPEAT('0123456789abcdef', 64*1024), block FROM generate_series(1,8) AS block;" + data = "SELECT now(), current_database(), session_user, current_user;" +// data = "SELECT REPEAT('0123456789abcdef', 64*1024), block FROM generate_series(1,8) AS block;" + break + + case 2: + data = ` +CREATE TABLE IF NOT EXISTS test ( + id SERIAL PRIMARY KEY, + number INT + ); + INSERT INTO test (number) VALUES (42); +` break + + case 3: + data = ` +CREATE EXTENSION IF NOT EXISTS pg_ivm; +` + break + +// CREATE MATERIALIZED VIEW myview AS SELECT * FROM test; +// SELECT pgivm.create_immv('myview', 'SELECT * FROM test'); + case 4: + data = ` +SELECT pgivm.create_immv('myview', 'SELECT * FROM test'); +` + break + + case 5: + data = ` +INSERT INTO test VALUES (43); +SELECT * FROM myview; +INSERT INTO test VALUES (44); +` + break + + +//REFRESH MATERIALIZED VIEW myview; + case 6: + data = ` +SELECT * FROM myview; +` + break + + /* case 2: data = ` @@ -408,6 +450,7 @@ async function delayed_init() { await Module.load_package("vector", "vector.tar.gz") await Module.load_package("uuid-ossp", "uuid-ossp.tar.gz") + await Module.load_package("pg_ivm", "pg_ivm.tar.gz") // await Module.load_package("pg_stat_statements", "pg_stat_statements.tar.gz") // await Module.load_package("postgis", "postgis-3.tar.gz") @@ -711,8 +754,6 @@ }; - //jsimport("tinytar.min.js") - window.vm = Module const { WasmTerminal } = await /**/ import("https://pmp-p.ddns.net/pglite-web/vtx.js") @@ -771,15 +812,15 @@ Module._use_wire(1) encodedArray.set(data) } else { - console.log("string data, using repl/no wire") + // console.log("string data, using repl/no wire") Module._use_wire(0) const encoder = new TextEncoder(); encodedArray = encoder.encode(data); } // sf - send_to_pglite(encodedArray, FS, -1) +// send_to_pglite(encodedArray, FS, -1) // cma -// send_to_pglite(encodedArray, FS, Module.cma_port || 1) + send_to_pglite(encodedArray, FS, Module.cma_port || 1) } else { console.warn("BC(srv):", ev); } diff --git a/wasm-build.sh b/wasm-build.sh index 0bd4b98f422cd..3b0d600210bc5 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -65,6 +65,27 @@ then export LOPTS=${LOPTS:-"-Oz -g0"} fi else + if grep -q __emscripten_tempret_get ${SDKROOT}/emsdk/upstream/emscripten/src/library_dylink.js + then + echo -n + else + pushd ${SDKROOT}/emsdk + patch -p1 < { + resolved ||= resolveSymbol(prop); + if (!resolved) { ++ if (prop==='getTempRet0') ++ return __emscripten_tempret_get(...args); + throw new Error(\`Dynamic linking error: cannot resolve symbol \${prop}\`); + } + return resolved(...args); +END + popd + fi + BUILD=emscripten if $DEBUG then @@ -82,7 +103,7 @@ else else # DO NOT CHANGE COPTS - optimized wasm corruption fix export COPTS="-O2 -g3 --no-wasm-opt" - export LOPTS=${LOPTS:-"-Oz -g0 --closure=0 --closure-args=--externs=/tmp/externs.js -sASSERTIONS=0"} + export LOPTS=${LOPTS:-"-Os -g0 --closure=0 -sASSERTIONS=0"} fi fi @@ -197,15 +218,12 @@ else " - # custom code for node/web builds that modify pg main/tools behaviour - # this used by both node/linkweb build stages - # pass the "kernel" contiguous memory zone size to the C compiler. CC_PGLITE="-DCMA_MB=${CMA_MB}" fi -# also used for non make (linking and pgl_main) +# also used for not makefile (manual linking and pgl_main) export CC_PGLITE="-DPYDK=1 -DPG_PREFIX=${PGROOT} -I${PGROOT}/include ${CC_PGLITE}" @@ -467,9 +485,9 @@ export PATH=${WORKSPACE}/${BUILD_PATH}/bin:${PGROOT}/bin:$PATH # =========================================================================== # =========================================================================== -if echo " $*"|grep -q " contrib" +if true then - mkdir -p ${PGL_DIST_LINK}/dumps + mkdir -p ${PGL_DIST_LINK}/dumps ${PGL_DIST_LINK}/imports if $WASI then @@ -629,14 +647,9 @@ then cp ${PGL_DIST_WEB}/pglite.* pglite/packages/pglite/release/ pushd pglite export HOME=$PG_BUILD - if [ -f ${PG_BUILD}/share/pnpm/pnpm ] - then - echo "assuming pnpm install done" - source $PG_BUILD/.bashrc - else - [ -f \$HOME/.local/share/pnpm/pnpm ] || wget -qO- https://get.pnpm.io/install.sh | ENV="$PG_BUILD/.bashrc" SHELL="\$(which bash)" bash - - fi - export PATH=$PATH:${PG_BUILD}/share/pnpm:\$(pwd)/node_modules/.pnpm/node_modules/.bin + export PNPM_HOME=$PG_BUILD + export PATH=$(echo -n ${SDKROOT}/emsdk/node/*.*.*/bin):$PNPM_HOME:$PATH + which pnpm || npm install -g pnpm pnpm install -g npm vitest pnpm install pnpm run ts:build diff --git a/wasm-build/linkexport.sh b/wasm-build/linkexport.sh index 0bcb36d2b2988..a41f1320e2481 100755 --- a/wasm-build/linkexport.sh +++ b/wasm-build/linkexport.sh @@ -22,7 +22,7 @@ pushd ${PGL_DIST_LINK} echo " * getting postgres exports lists from ${BUILD_PATH} " - cat $(find ${BUILD_PATH} -type f |grep /exports) \ + cat $(find ${BUILD_PATH} -type f |grep /exports|grep -v /interfaces/libpq/) \ | grep -v ^\ local \ | grep -v ^{\ global \ | sort | uniq > ${PGL_DIST_LINK}/exports/pgcore.exports diff --git a/wasm-build/linkimports.sh b/wasm-build/linkimports.sh index d051a50564a4e..356548ca18466 100755 --- a/wasm-build/linkimports.sh +++ b/wasm-build/linkimports.sh @@ -120,6 +120,13 @@ _PQsetSingleRowMode _PQsocket _PQstatus _pgresStatus + +_PQbackendPID +_PQconninfo +_PQconninfoFree +_PQconninfoParse +_PQendcopy +_PQescapeIdentifier """.splitlines(): if sym and sym in matches: dbg(f"\t* Removed symbol '{sym}'") @@ -127,6 +134,11 @@ _pgresStatus # matches.append('') +if not '_getTempRet0' in matches: + matches.append('_getTempRet0') +if not 'getTempRet0' in matches: + matches.append('getTempRet0') + for sym in matches: print(sym) From b766a35a0df7a2c43cf89a7a56ce2cbe5ec374d9 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Wed, 4 Jun 2025 08:05:15 +0200 Subject: [PATCH 10/19] move sdk fix, add wasmtime check --- pglite-REL_17_4_WASM/repl.html | 94 +++++++++++++------ wasm-build.sh | 165 +++++++++++++++++---------------- wasm-build/build-pgcore.sh | 26 +++--- wasm-build/sdk.sh | 28 +++++- 4 files changed, 193 insertions(+), 120 deletions(-) diff --git a/pglite-REL_17_4_WASM/repl.html b/pglite-REL_17_4_WASM/repl.html index 4c7f2b3a445b9..e87f0e3c1c817 100644 --- a/pglite-REL_17_4_WASM/repl.html +++ b/pglite-REL_17_4_WASM/repl.html @@ -77,7 +77,7 @@ config sample expected in options.config : { - "shared_preload_libraries" : ["pg_stat_statements"], + "shared_preload_libraries" : ["'pg_stat_statements'"], "compute_query_id" : "on", "pg_stat_statements.max" : "10000", "pg_stat_statements.track": "all", @@ -87,23 +87,33 @@ async function pgl_conf(vm, kv) { - var lines = vm.os.codec.decode(vm.FS.readFile(vm.PGDATA + "/postgresql.conf")).split('\n'); + //var lines = vm.os.codec.decode(vm.FS.readFile(vm.PGDATA + "/postgresql.conf")).split('\n'); + var lines = [] + for (const line of vm.os.codec.decode(vm.FS.readFile(vm.PGDATA + "/postgresql.conf")).split('\n')) { + var testl = line.trimStart().split("#")[0].trimEnd() + if (testl.length<4) + continue + if (testl.startsWith("#")) + continue + lines.push(testl) + } var buf = [] var newlines = []; - //return; for (var k in kv) { var old = "" for (var line = 0; line < lines.length; line++) { - if (lines[line].startsWith(k)) { - old = lines[line]; - } else { - buf.push(lines[line]) + var testl = lines[line]; + if (testl.startsWith(k)) { + old = testl; + lines[line] = "# " + testl + break } } + var values = [] if (kv[k] instanceof Array) { // TODO extract array of , from old @@ -115,24 +125,21 @@ values.push(kv[k].toString()) } - - const newline = `\n${k} = ${values.join(',')}\n` - - if (!old.length) { - console.log("NEW :", newline) - } else { - if (old==newline) - console.log("no conf change :", newline) - else - console.error("EDIT :",old,'->', newline) - } + const newline = `${k} = ${values.join(',')}` newlines.push(newline) } + if (newlines.length) { - buf.push(...newlines) -console.log("BUF", buf) - vm.FS.writeFile( vm.PGDATA + "/postgresql.conf", buf.join("") ) - vm.FS.syncfs(false, (e) => { if (e) console.error("pgl_conf error:", e)}) + lines.push(...newlines) + for (const line of lines) { + console.log(line) + } + + vm.FS.unlink(vm.PGDATA + "/postgresql.conf") + vm.FS.writeFile( vm.PGDATA + "/postgresql.conf", lines.join("\n") ) + vm.FS.syncfs(false, (e) => { if (e) console.error("pgl_conf:", e)}) + + console.log("Config file written at :", vm.PGDATA + "/postgresql.conf" ) } } @@ -298,7 +305,7 @@ return text_codec.decode( new Uint8Array(ary) ) } window.b_utf8 = b_utf8 - +// /tmp/pglite/base/postgresql.conf function test_drive(step) { var data = null switch (test_step) { @@ -452,10 +459,10 @@ await Module.load_package("uuid-ossp", "uuid-ossp.tar.gz") await Module.load_package("pg_ivm", "pg_ivm.tar.gz") - // await Module.load_package("pg_stat_statements", "pg_stat_statements.tar.gz") + await Module.load_package("pg_stat_statements", "pg_stat_statements.tar.gz") + // await Module.load_package("postgis", "postgis-3.tar.gz") // await Module.load_package("arrays", "arrays.tar.gz") - // await Module.load_package("pg_stat_statements", "pg_stat_statements.tar.gz") // vm._setenv(vm.stringToUTF8OnStack("REPL"),vm.stringToUTF8OnStack("Y"),1) console.warn("calling initdb") @@ -531,8 +538,10 @@ vm.PGDATA = PGDATA if (editconf) { await pgl_conf(vm, { - "shared_preload_libraries" : ["pg_stat_statements"], + "shared_preload_libraries" : ["'pg_stat_statements'"], + "max_connections" : "5", "compute_query_id" : "on", + "shared_buffers" : "128kB", "pg_stat_statements.max" : "10000", "pg_stat_statements.track": "all", }) @@ -724,7 +733,38 @@ return } - if (line.startsWith('pg_dump ')) { + const elems = line.trimStart().trimEnd().split(" ") + cmd = elems[0] + arg = elems.at(-1) + console.log("CMD:", cmd, "ARG:", arg) + + switch (cmd) { + case "rx": + var filepath = arg + console.log('getting file:', filepath) + filename = filepath.split("/").at(-1) + const blob = new Blob([vm.FS.readFile(filepath)]) + const elem = window.document.createElement('a'); + elem.href = window.URL.createObjectURL(blob, { oneTimeOnly: true }); + elem.download = filename; + document.body.appendChild(elem); + elem.click(); + document.body.removeChild(elem); + return + + case "cat": + for (const line of os.codec.decode(vm.FS.readFile(arg)).split("\n")) { + vm.vt.xterm.write( line + "\r\n" ) + } + return + + } + + + + + + if (line.startsWith("pg_dump ")) { console.log("RL:", line) } else { diff --git a/wasm-build.sh b/wasm-build.sh index 3b0d600210bc5..4418fc26b04ad 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -65,27 +65,6 @@ then export LOPTS=${LOPTS:-"-Oz -g0"} fi else - if grep -q __emscripten_tempret_get ${SDKROOT}/emsdk/upstream/emscripten/src/library_dylink.js - then - echo -n - else - pushd ${SDKROOT}/emsdk - patch -p1 < { - resolved ||= resolveSymbol(prop); - if (!resolved) { -+ if (prop==='getTempRet0') -+ return __emscripten_tempret_get(...args); - throw new Error(\`Dynamic linking error: cannot resolve symbol \${prop}\`); - } - return resolved(...args); -END - popd - fi - BUILD=emscripten if $DEBUG then @@ -217,6 +196,19 @@ else " + pushd ${SDKROOT} + # always install wasmtime because wasm-objdump needs it. + if [ -f ${SDKROOT}/devices/$(arch)/usr/bin/wasmtime ] + then + echo "keeping installed wasmtime and wasi binaries" + else +# TODO: window only has a zip archive, better use wasmtime-py instead. + + wget https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.0/wasmtime-v33.0.0-$(arch)-${PLATFORM}.tar.xz \ + -O-|xzcat|tar xfv - + mv -vf $(find wasmtime*|grep /wasmtime$) ${SDKROOT}/devices/$(arch)/usr/bin + fi + popd # pass the "kernel" contiguous memory zone size to the C compiler. CC_PGLITE="-DCMA_MB=${CMA_MB}" @@ -232,12 +224,18 @@ echo " ---------------------------------------- " env|grep PG |grep -v BUILD +echo env|grep BUILD|grep -v PG +echo env|grep WA +echo env|grep PY echo " ---------------------------------------- +PATH=${PATH} +wasmtime=$(which wasmtime) + ---------------------------------------- " @@ -471,7 +469,7 @@ fi # put local zic in the path from build dir # put emsdk-shared and also pg_config from the install dir. -export PATH=${WORKSPACE}/${BUILD_PATH}/bin:${PGROOT}/bin:$PATH +export PATH=${WORKSPACE}/${BUILD_PATH}/bin:${PGROOT}/bin:${HOST_PREFIX}/bin:$PATH # At this stage, PG should be installed to PREFIX and ready for linking @@ -485,9 +483,16 @@ export PATH=${WORKSPACE}/${BUILD_PATH}/bin:${PGROOT}/bin:$PATH # =========================================================================== # =========================================================================== -if true +if $WASI then + echo " + + * WASI: skipping contrib extensions build + + " +else mkdir -p ${PGL_DIST_LINK}/dumps ${PGL_DIST_LINK}/imports + cd ${WORKSPACE} if $WASI then @@ -509,52 +514,53 @@ then ]" fi - for extdir in postgresql-${PG_BRANCH}/contrib/* - do - if [ -f ${PGL_DIST_LINK}/dumps/dump.vector ] - then - echo " + if [ -f ${PGL_DIST_LINK}/dumps/dump.vector ] + then + echo " - * NOT rebuilding extensions + * NOT rebuilding extensions ( found ${PGL_DIST_LINK}/dumps/dump.vector ) " - break - fi + else - if [ -d "$extdir" ] - then - ext=$(echo -n $extdir|cut -d/ -f3) - if echo -n $SKIP|grep -q "$ext " - then - echo skipping extension $ext - else - echo " + for extdir in postgresql-${PG_BRANCH}/contrib/* + do - Building contrib extension : $ext : begin -" - pushd ${BUILD_PATH}/contrib/$ext - if PATH=$PREFIX/bin:$PATH emmake make install 2>&1 >/dev/null + if [ -d "$extdir" ] + then + ext=$(echo -n $extdir|cut -d/ -f3) + if echo -n $SKIP|grep -q "$ext " then - echo " - Building contrib extension : $ext : end -" + echo skipping extension $ext else echo " - Extension $ext from $extdir failed to build + Building contrib extension : $ext : begin -" - exit 216 - fi - popd + " + pushd ${BUILD_PATH}/contrib/$ext + if PATH=$PREFIX/bin:$PATH emmake make install 2>&1 >/dev/null + then + echo " + Building contrib extension : $ext : end + " + else + echo " - python3 ${PORTABLE}/pack_extension.py 2>&1 >/dev/null + Extension $ext from $extdir failed to build - fi - fi - done + " + exit 216 + fi + popd + + python3 ${PORTABLE}/pack_extension.py 2>&1 >/dev/null + fi + fi + done + fi fi @@ -568,6 +574,7 @@ echo " # only build extra when targeting pglite-wasm . +rm -f pglite-link.sh if [ -f ${WORKSPACE}/pglite-${PG_BRANCH}/build.sh ] then @@ -578,36 +585,34 @@ then " else - if echo " $*"|grep -q " extra" - then -# if $CI +# if $CI +# then +# #if [ -d $PREFIX/include/X11 ] +# if true # then -# #if [ -d $PREFIX/include/X11 ] -# if true -# then -# echo -n -# else -# # install EXTRA sdk -# . /etc/lsb-release -# DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}" -# CIVER=${CIVER:-$DISTRIB} -# SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4 -# echo "Installing extra lib from $SDK_URL" -# curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null -# chmod +x ./extra/*.sh -# fi +# echo -n +# else +# # install EXTRA sdk +# . /etc/lsb-release +# DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}" +# CIVER=${CIVER:-$DISTRIB} +# SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4 +# echo "Installing extra lib from $SDK_URL" +# curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null +# chmod +x ./extra/*.sh # fi +# fi + + for extra_ext in ./extra/*.sh + do + LOG=$PG_DIST_EXT/$(basename ${extra_ext}).log + echo "==== ${extra_ext} : $LOG ====" - for extra_ext in ./extra/*.sh - do - LOG=$PG_DIST_EXT/$(basename ${extra_ext}).log - echo "======================= ${extra_ext} : $LOG ===================" + ${extra_ext} > $LOG || exit 613 - ${extra_ext} > $LOG || exit 552 + python3 wasm-build/pack_extension.py + done - python3 wasm-build/pack_extension.py - done - fi # this is for initial emscripten MEMFS export PGPRELOAD="\ diff --git a/wasm-build/build-pgcore.sh b/wasm-build/build-pgcore.sh index ace44bb3b1b9b..f395478088a1d 100755 --- a/wasm-build/build-pgcore.sh +++ b/wasm-build/build-pgcore.sh @@ -40,7 +40,7 @@ Fatal: failed to apply patch : $one fi done touch postgresql-${PG_BRANCH}.patched - popd + popd # postgresql-${PG_BRANCH} fi # either a submodule dir or a symlink. @@ -311,8 +311,12 @@ END # same script handle emcc and wasi - cat > pg-make.sh < pg-make.sh + cat /tmp/portable.opts >> pg-make.sh + cat >> pg-make.sh < ${PGROOT}/pg.${BUILD}.installed + find . -type f | grep -v plpgsql > ${PGROOT}/pg.${BUILD}.installed popd - goback=$(pwd) - popd - python3 wasm-build/pack_extension.py builtin - pushd $goback + pushd ${WORKSPACE} - pushd ${PGROOT} - find . -type f > ${PGROOT}/pg.${BUILD}.installed - popd + python3 wasm-build/pack_extension.py builtin + + pushd ${PGROOT} + find . -type f > ${PGROOT}/pg.${BUILD}.installed + popd else cat /tmp/install.log diff --git a/wasm-build/sdk.sh b/wasm-build/sdk.sh index 0401c546b3515..87cbc69f54098 100755 --- a/wasm-build/sdk.sh +++ b/wasm-build/sdk.sh @@ -1,5 +1,31 @@ #!/bin/bash -mkdir -p /tmp/sdk + +SDKROOT=${SDKROOT:-/tmp/sdk} +mkdir -p ${SDKROOT} + +if grep -q __emscripten_tempret_get ${SDKROOT}/emsdk/upstream/emscripten/src/library_dylink.js +then + echo -n +else + pushd ${SDKROOT}/emsdk + patch -p1 < { + resolved ||= resolveSymbol(prop); + if (!resolved) { ++ if (prop==='getTempRet0') ++ return __emscripten_tempret_get(...args); + throw new Error(\`Dynamic linking error: cannot resolve symbol \${prop}\`); + } + return resolved(...args); +END + popd +fi + + + if ${NO_SDK_CHECK:-false} then exit 0 From b5ca8718bac0577acd55373845600047742afb19 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Wed, 4 Jun 2025 09:39:46 +0200 Subject: [PATCH 11/19] extra build caching, pg_ivm git --- extra/pg_ivm.sh | 20 ++- extra/vector.sh | 8 +- wasm-build.sh | 308 ++++++++--------------------------- wasm-build/build-ext.sh | 117 +++++++++++++ wasm-build/extension.sh | 32 +++- wasm-build/linkexport.sh | 6 +- wasm-build/linkimports.sh | 2 +- wasm-build/pack_extension.py | 5 +- wasm-build/sdk.sh | 32 ++++ 9 files changed, 269 insertions(+), 261 deletions(-) create mode 100755 wasm-build/build-ext.sh diff --git a/extra/pg_ivm.sh b/extra/pg_ivm.sh index cbdf6ad6114c4..9dd4c2ced2c1e 100755 --- a/extra/pg_ivm.sh +++ b/extra/pg_ivm.sh @@ -2,18 +2,28 @@ . wasm-build/extension.sh -pushd build +pushd $PG_EXTRA if [ -d pg_ivm ] then echo using local pgpg_ivm else - wget https://github.com/sraoss/pg_ivm/archive/refs/tags/v1.10.tar.gz -O-|tar xfz - - mv pg_ivm-* pg_ivm - #git clone --recursive --no-tags --depth 1 --single-branch --branch main https://github.com/sraoss/pg_ivm +# wget https://github.com/sraoss/pg_ivm/archive/refs/tags/v1.10.tar.gz -O-|tar xfz - +# mv pg_ivm-* pg_ivm + + git clone --recursive --no-tags --depth 1 --single-branch --branch main https://github.com/sraoss/pg_ivm + + if $WASI + then + echo "no patching" + else + echo "PATCH?" + + fi + fi popd -pushd build/pg_ivm +pushd $PG_EXTRA/pg_ivm # path for wasm-shared already set to (pwd:pg build dir)/bin # OPTFLAGS="" turns off arch optim (sse/neon). PG_CONFIG=${PGROOT}/bin/pg_config emmake make OPTFLAGS="" install || exit 19 diff --git a/extra/vector.sh b/extra/vector.sh index d0575d351f240..4dd8666684c09 100755 --- a/extra/vector.sh +++ b/extra/vector.sh @@ -2,20 +2,20 @@ . wasm-build/extension.sh -pushd build +pushd $PG_EXTRA # [ -d pgvector ] || git clone --no-tags --depth 1 --single-branch --branch master https://github.com/pgvector/pgvector if [ -d vector ] then echo using local pgvector else - [ -f ../src/pgvector.tar.gz ] || wget -c -q https://github.com/pgvector/pgvector/archive/refs/tags/v0.8.0.tar.gz -O../src/pgvector.tar.gz - tar xvfz ../src/pgvector.tar.gz + wget -c -q https://github.com/pgvector/pgvector/archive/refs/tags/v0.8.0.tar.gz -Opgvector.tar.gz + tar xvfz pgvector.tar.gz mv pgvector-?.?.? vector fi popd -pushd build/vector +pushd $PG_EXTRA/vector # path for wasm-shared already set to (pwd:pg build dir)/bin # OPTFLAGS="" turns off arch optim (sse/neon). PG_CONFIG=${PGROOT}/bin/pg_config emmake make OPTFLAGS="" install || exit 21 diff --git a/wasm-build.sh b/wasm-build.sh index 4418fc26b04ad..98f5d10ae0144 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -24,19 +24,20 @@ export PGROOT=${PGROOT:-/tmp/pglite} export WEBROOT=${WEBROOT:-/tmp/web} export PG_BUILD=${BUILD:-/tmp/sdk/build} -export PGL_BUILD_NATIVE="${PG_BUILD}/pglite-native" +export PGL_BUILD_NATIVE=${PG_BUILD}/pglite-native +export PGL_BUILD_DUMPS=${PG_BUILD}/dumps + export PG_DIST=${DIST:-/tmp/sdk/dist} -export PG_DIST_EXT="${PG_DIST}/extensions-emsdk" +export PG_DIST_EXT=${PG_DIST}/extensions-emsdk -export PGL_DIST_JS="${PG_DIST}/pglite-js" -export PGL_DIST_LINK="${PG_DIST}/pglite-link" +export PGL_DIST_JS=${PG_DIST}/pglite-js +export PGL_DIST_LINK=${PG_DIST}/pglite-link -export PGL_DIST_NATIVE="${PG_DIST}/pglite-native" -export PGL_DIST_C="${PG_DIST}/pglite-native" -export PGL_DIST_WEB="${PG_DIST}/pglite-web" +export PGL_DIST_NATIVE=${PG_DIST}/pglite-native +export PGL_DIST_C=${PG_DIST}/pglite-native +export PGL_DIST_WEB=${PG_DIST}/pglite-web -DIST_ALL="${PGROOT}/bin ${PG_DIST} ${PG_DIST_EXT} ${PGL_DIST_JS} ${PGL_DIST_LINK} ${PGL_DIST_WEB} ${PGL_BUILD_NATIVE}" export DEBUG=${DEBUG:-true} @@ -89,19 +90,14 @@ fi export BUILD export BUILD_PATH=${PG_BUILD}/${BUILD} -export PGDATA=${PGROOT}/base -export PGPATCH=${WORKSPACE}/patches - -chmod +x ${PORTABLE}/*.sh -[ -d ${PORTABLE}/extra ] && ${PORTABLE}/extra/*.sh - - -# this was set to false on 16.4 to skip some harmless exceptions without messing with core code. -# exit on error -EOE=true +export PG_EXTRA=${PG_BUILD}/extra-${BUILD} # default to user writeable paths in /tmp/ . +DIST_ALL="${PGROOT}/bin ${PG_DIST} ${PG_DIST_EXT} ${PGL_DIST_JS} ${PGL_BUILD_DUMPS} ${PGL_BUILD_NATIVE}" +DIST_ALL="$DIST_ALL ${PGL_DIST_LINK} ${PGL_DIST_NATIVE} ${PGL_DIST_WEB} ${PGL_DIST_C}" +DIST_ALL="$DIST_ALL ${PG_EXTRA}" + if mkdir -p $DIST_ALL then echo "checking for valid prefix ${PGROOT} ${PG_DIST}" @@ -110,6 +106,22 @@ else sudo chown $(whoami) -R $DIST_ALL fi + + + + +export PGDATA=${PGROOT}/base + + +chmod +x ${PORTABLE}/*.sh +[ -d ${PORTABLE}/extra ] && ${PORTABLE}/extra/*.sh + + +# this was set to false on 16.4 to skip some harmless exceptions without messing with core code. +# exit on error +EOE=true + + # TODO: also handle PGPASSFILE hostname:port:database:username:password # https://www.postgresql.org/docs/devel/libpq-pgpass.html export CRED="-U $PGUSER --pwfile=${PGROOT}/password" @@ -144,31 +156,10 @@ System node/pnpm ( may interfer) : # setup compiler+node. emsdk provides node 20, recent enough for bun. # TODO: but may need to adjust $PATH with stock emsdk. +pushd ${SDKROOT} if ${WASI} then - pushd ${SDKROOT} - . wasisdk/wasisdk_env.sh - popd - - if [ -f ${WASI_SYSROOT}/extra ] - then - echo -n - else - pushd ${WASI_SYSROOT} - VMLABS="https://github.com/vmware-labs/webassembly-language-runtimes/releases/download" - wget -q "${VMLABS}/libs%2Flibpng%2F1.6.39%2B20230629-ccb4cb0/libpng-1.6.39-wasi-sdk-20.0.tar.gz" -O-| tar xfz - - wget -q "${VMLABS}/libs%2Fzlib%2F1.2.13%2B20230623-2993864/libz-1.2.13-wasi-sdk-20.0.tar.gz" -O-| tar xfz - - wget -q "${VMLABS}/libs%2Fsqlite%2F3.42.0%2B20230623-2993864/libsqlite-3.42.0-wasi-sdk-20.0.tar.gz" -O-| tar xfz - - wget -q "${VMLABS}/libs%2Flibxml2%2F2.11.4%2B20230623-2993864/libxml2-2.11.4-wasi-sdk-20.0.tar.gz" -O-| tar xfz - - wget -q "${VMLABS}/libs%2Fbzip2%2F1.0.8%2B20230623-2993864/libbzip2-1.0.8-wasi-sdk-20.0.tar.gz" -O-| tar xfz - - wget -q "${VMLABS}/libs%2Flibuuid%2F1.0.3%2B20230623-2993864/libuuid-1.0.3-wasi-sdk-20.0.tar.gz" -O-| tar xfz - - popd - touch ${WASI_SYSROOT}/extra - fi - - # wasi does not use -sGLOBAL_BASE - CC_PGLITE="-DCMA_MB=${CMA_MB}" - + . wasisdk/wasisdk_env.sh else if which emcc then @@ -196,27 +187,14 @@ else " - pushd ${SDKROOT} - # always install wasmtime because wasm-objdump needs it. - if [ -f ${SDKROOT}/devices/$(arch)/usr/bin/wasmtime ] - then - echo "keeping installed wasmtime and wasi binaries" - else -# TODO: window only has a zip archive, better use wasmtime-py instead. - - wget https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.0/wasmtime-v33.0.0-$(arch)-${PLATFORM}.tar.xz \ - -O-|xzcat|tar xfv - - mv -vf $(find wasmtime*|grep /wasmtime$) ${SDKROOT}/devices/$(arch)/usr/bin - fi - popd - - # pass the "kernel" contiguous memory zone size to the C compiler. - CC_PGLITE="-DCMA_MB=${CMA_MB}" - fi +popd -# also used for not makefile (manual linking and pgl_main) -export CC_PGLITE="-DPYDK=1 -DPG_PREFIX=${PGROOT} -I${PGROOT}/include ${CC_PGLITE}" +# used for not makefile (manual linking and pgl_main) +# pass the "kernel" contiguous memory zone size to the C compiler with CMA_MB which will be multiplied by 1024x1024 in +# preprocessed source. +# nb: wasi does not use -sGLOBAL_BASE +export CC_PGLITE="-DPYDK=1 -DPG_PREFIX=${PGROOT} -I${PGROOT}/include -DCMA_MB=${CMA_MB}" @@ -242,32 +220,27 @@ wasmtime=$(which wasmtime) # ========================= symbol extractor ============================ - -OBJDUMP=${OBJDUMP:-true} - -if $OBJDUMP +if [ -f $PGROOT/bin/wasm-objdump ] then - if [ -f $PGROOT/bin/wasm-objdump ] + echo "wasm-objdump found" +else + WRAPPER=$(which wasm-objdump) + WASIFILE=$(realpath ${WRAPPER}.wasi) + if $WRAPPER -h $WASIFILE | grep -q 'file format wasm 0x1' then - echo "wasm-objdump found" - else - WRAPPER=$(which wasm-objdump) - WASIFILE=$(realpath ${WRAPPER}.wasi) - if $WRAPPER -h $WASIFILE | grep -q 'file format wasm 0x1' + mkdir -p $PGROOT/bin/ + if cp -f $WRAPPER $WASIFILE $PGROOT/bin/ then - mkdir -p $PGROOT/bin/ - if cp -f $WRAPPER $WASIFILE $PGROOT/bin/ - then - echo "wasm-objdump found and working, and copied to $PGROOT/bin/" - else - OBJDUMP=false - fi + echo "wasm-objdump found and working, and copied to $PGROOT/bin/" else - echo " - WARNING: $(which wasm-objdump) not working properly, trying alternate syntax + OBJDUMP=false + fi + else + echo " + WARNING: $(which wasm-objdump) not working properly, trying alternate syntax - " - cat > $WRAPPER < $WRAPPER < /proc/self/fd/2 $(which wasmtime) --env PYTHONDONTWRITEBYTECODE=1 --dir / \$WASM \$@ END - chmod +x $WRAPPER + chmod +x $WRAPPER - if $WRAPPER -h $WASIFILE | grep -q 'file format wasm 0x1' + if $WRAPPER -h $WASIFILE | grep -q 'file format wasm 0x1' + then + mkdir -p $PGROOT/bin/ + if cp -f $WRAPPER $WASIFILE $PGROOT/bin/ then - mkdir -p $PGROOT/bin/ - if cp -f $WRAPPER $WASIFILE $PGROOT/bin/ - then - echo "wasm-objdump fixed and working, copied to $PGROOT/bin/" - else - OBJDUMP=false - fi - else - echo " - ERROR: $(which wasm-objdump) not working properly ( is wasmtime ok ? ) - - " - exit 268 - OBJDUMP=false + echo "wasm-objdump fixed and working, copied to $PGROOT/bin/" fi - fi - fi -else - echo " - - WARNING: OBJDUMP disabled, some newer or complex extensions may not load properly - - -" -fi - -if $OBJDUMP -then - mkdir -p patches/imports patches/imports.pgcore -else - mkdir -p patches/imports - touch patches/imports/plpgsql - echo " - - WARNING: wasm-objdump not found or OBJDUMP disabled, some extensions may not load properly - + else + echo " + ERROR: $(which wasm-objdump) not working properly ( is wasmtime ok ? ) " + exit 262 + fi + fi fi -export OBJDUMP - # ========================= pg core configuration ============================ @@ -480,97 +426,14 @@ export PATH=${WORKSPACE}/${BUILD_PATH}/bin:${PGROOT}/bin:${HOST_PREFIX}/bin:$PAT # =========================================================================== # =========================================================================== # EXTENSIONS -# =========================================================================== -# =========================================================================== -if $WASI -then - echo " +cd ${WORKSPACE} +. ./wasm-build/build-ext.sh - * WASI: skipping contrib extensions build - - " -else - mkdir -p ${PGL_DIST_LINK}/dumps ${PGL_DIST_LINK}/imports - cd ${WORKSPACE} - - if $WASI - then - echo " ========= TODO WASI openssl ============== " - SKIP="\ - [\ - sslinfo bool_plperl hstore_plperl hstore_plpython jsonb_plperl jsonb_plpython\ - ltree_plpython sepgsql bool_plperl start-scripts\ - pgcrypto uuid-ossp xml2\ - ]" - else - # TEMP FIX for SDK - SSL_INCDIR=$EMSDK/upstream/emscripten/cache/sysroot/include/openssl - [ -f $SSL_INCDIR/evp.h ] || ln -s $PREFIX/include/openssl $SSL_INCDIR - SKIP="\ - [\ - sslinfo bool_plperl hstore_plperl hstore_plpython jsonb_plperl jsonb_plpython\ - ltree_plpython sepgsql bool_plperl start-scripts\ - ]" - fi - - if [ -f ${PGL_DIST_LINK}/dumps/dump.vector ] - then - echo " - - * NOT rebuilding extensions ( found ${PGL_DIST_LINK}/dumps/dump.vector ) - -" - else - - for extdir in postgresql-${PG_BRANCH}/contrib/* - do - - if [ -d "$extdir" ] - then - ext=$(echo -n $extdir|cut -d/ -f3) - if echo -n $SKIP|grep -q "$ext " - then - echo skipping extension $ext - else - echo " - - Building contrib extension : $ext : begin - - " - pushd ${BUILD_PATH}/contrib/$ext - if PATH=$PREFIX/bin:$PATH emmake make install 2>&1 >/dev/null - then - echo " - Building contrib extension : $ext : end - " - else - echo " - - Extension $ext from $extdir failed to build - - " - exit 216 - fi - popd - - python3 ${PORTABLE}/pack_extension.py 2>&1 >/dev/null - - fi - fi - done - - fi -fi - - -echo " - - Extensions distribution folder : ${PG_DIST_EXT} +# =========================================================================== +# =========================================================================== -" - # only build extra when targeting pglite-wasm . @@ -581,39 +444,10 @@ then if $WASI then echo " - * WASI build : skipping extra extensions and FS + * WASI build : skipping FS building " else -# if $CI -# then -# #if [ -d $PREFIX/include/X11 ] -# if true -# then -# echo -n -# else -# # install EXTRA sdk -# . /etc/lsb-release -# DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}" -# CIVER=${CIVER:-$DISTRIB} -# SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4 -# echo "Installing extra lib from $SDK_URL" -# curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null -# chmod +x ./extra/*.sh -# fi -# fi - - for extra_ext in ./extra/*.sh - do - LOG=$PG_DIST_EXT/$(basename ${extra_ext}).log - echo "==== ${extra_ext} : $LOG ====" - - ${extra_ext} > $LOG || exit 613 - - python3 wasm-build/pack_extension.py - done - - # this is for initial emscripten MEMFS export PGPRELOAD="\ --preload-file ${PGROOT}/share/postgresql@${PGROOT}/share/postgresql \ diff --git a/wasm-build/build-ext.sh b/wasm-build/build-ext.sh new file mode 100755 index 0000000000000..588ba50967917 --- /dev/null +++ b/wasm-build/build-ext.sh @@ -0,0 +1,117 @@ +if $WASI +then + echo " + + * WASI: skipping contrib extensions build + + " +else + mkdir -p ${PGL_DIST_LINK}/exports ${PGL_DIST_LINK}/imports + cd ${WORKSPACE} + + if $WASI + then + echo " ========= TODO WASI openssl ============== " + SKIP="\ + [\ + sslinfo bool_plperl hstore_plperl hstore_plpython jsonb_plperl jsonb_plpython\ + ltree_plpython sepgsql bool_plperl start-scripts\ + pgcrypto uuid-ossp xml2\ + ]" + else + # TEMP FIX for SDK + SSL_INCDIR=$EMSDK/upstream/emscripten/cache/sysroot/include/openssl + [ -f $SSL_INCDIR/evp.h ] || ln -s $PREFIX/include/openssl $SSL_INCDIR + SKIP="\ + [\ + sslinfo bool_plperl hstore_plperl hstore_plpython jsonb_plperl jsonb_plpython\ + ltree_plpython sepgsql bool_plperl start-scripts\ + ]" + fi + + for extdir in postgresql-${PG_BRANCH}/contrib/* + do + + if [ -d "$extdir" ] + then + ext=$(echo -n $extdir|cut -d/ -f3) + if echo -n $SKIP|grep -q "$ext " + then + echo skipping extension $ext + else + echo " + + Building contrib extension : $ext : begin + + " + pushd ${BUILD_PATH}/contrib/$ext + if PATH=$PREFIX/bin:$PATH emmake make install 2>&1 >/dev/null + then + echo " + Building contrib extension : $ext : end + " + else + echo " + + Extension $ext from $extdir failed to build + + " + exit 216 + fi + popd + + python3 ${PORTABLE}/pack_extension.py 2>&1 >/dev/null + + fi + fi + done + +fi + + +echo " + + Extensions distribution folder : ${PG_DIST_EXT} + + +" + +# if $CI +# then +# #if [ -d $PREFIX/include/X11 ] +# if true +# then +# echo -n +# else +# # install EXTRA sdk +# . /etc/lsb-release +# DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}" +# CIVER=${CIVER:-$DISTRIB} +# SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4 +# echo "Installing extra lib from $SDK_URL" +# curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null +# chmod +x ./extra/*.sh +# fi +# fi + +if [ -f ${PG_BUILD_DUMPS}/dump.vector ] +then + echo " + + * NOT rebuilding extra extensions ( found ${PG_BUILD_DUMPS}/dump.vector ) + +" +else + + for extra_ext in ./extra/*.sh + do + LOG=$PG_DIST_EXT/$(basename ${extra_ext}).log + echo "==== ${extra_ext} : $LOG ====" + + ${extra_ext} > $LOG || exit 112 + + python3 wasm-build/pack_extension.py + done + + +fi diff --git a/wasm-build/extension.sh b/wasm-build/extension.sh index 6c9838768c0e3..db35aee72fa3d 100755 --- a/wasm-build/extension.sh +++ b/wasm-build/extension.sh @@ -1,14 +1,28 @@ # this is to be sourced by extra/*sh +export PG_EXTRA=${PG_EXTRA:-build/extra} +mkdir -p $PG_EXTRA -mkdir -p build src - -if which emcc +if $WASI then - echo -n + if which wasi-c + then + echo -n + else + reset + . ${SDKROOT:-/tmp/sdk}/wasm32-wasi-shell.sh + fi else - reset; - . ${SDKROOT:-/tmp/sdk}/wasm32-bi-emscripten-shell.sh - export PGROOT=${PGROOT:-/tmp/pglite} - export PATH=${PGROOT}/bin:$PATH - . ${PGROOT}/pgopts.sh + + if which emcc + then + echo -n + else + reset + . ${SDKROOT:-/tmp/sdk}/wasm32-bi-emscripten-shell.sh + fi fi + +export PGROOT=${PGROOT:-/tmp/pglite} +export PATH=${PGROOT}/bin:$PATH +. ${PGROOT}/pgopts.sh + diff --git a/wasm-build/linkexport.sh b/wasm-build/linkexport.sh index a41f1320e2481..50506fdf7b1a0 100755 --- a/wasm-build/linkexport.sh +++ b/wasm-build/linkexport.sh @@ -1,6 +1,6 @@ # this only runs when wasm-objdump is working and OBJDUMP not set to false -mkdir -p ${PGL_DIST_LINK}/exports ${PGL_DIST_LINK}/imports ${PGL_DIST_LINK}/dumps +mkdir -p ${PGL_DIST_LINK}/exports ${PGL_DIST_LINK}/imports ${PG_BUILD_DUMPS} echo "============= link export to ${PGL_DIST_LINK}/exports : begin ===============" @@ -14,7 +14,7 @@ echo " " pushd $(realpath ${PGL_DIST_JS}) - wasmtime --dir / --dir $(pwd)::. -- $(which wasm-objdump).wasi -x pglite-js.wasm > ${PGL_DIST_LINK}/exports/pgcore.wasm-objdump + wasmtime --dir / --dir $(pwd)::. -- $(which wasm-objdump).wasi -x pglite-js.wasm > ${PG_BUILD_DUMPS}/pgcore.wasm-objdump popd @@ -32,7 +32,7 @@ pushd ${PGL_DIST_LINK} into ${PGL_DIST_LINK}/exports/pgcore " - OBJDUMP=${PGL_DIST_LINK}/exports/pgcore.wasm-objdump \ + OBJDUMP=${PG_BUILD_DUMPS}/pgcore.wasm-objdump \ PGDUMP=${PGL_DIST_LINK}/exports/pgcore.exports \ python3 ${WORKSPACE}/wasm-build/getsyms.py exports > ${PGL_DIST_LINK}/exports/pgcore popd diff --git a/wasm-build/linkimports.sh b/wasm-build/linkimports.sh index 356548ca18466..4ab35b48d5a76 100755 --- a/wasm-build/linkimports.sh +++ b/wasm-build/linkimports.sh @@ -12,7 +12,7 @@ pushd ${WORKSPACE} for extra_pg_so in $(find $PGROOT/lib/postgresql/|grep \.so$) do - SOBASE=patches/imports.pgcore/$(basename $extra_pg_so .so) + SOBASE=${PG_BUILD_DUMPS}/$(basename $extra_pg_so .so) wasm-objdump -x $(realpath $extra_pg_so) > $SOBASE.wasm-objdump OBJDUMP=$SOBASE.wasm-objdump \ PGDUMP=${PGL_DIST_LINK}/exports/pgcore.exports \ diff --git a/wasm-build/pack_extension.py b/wasm-build/pack_extension.py index 249a1d9c22753..2a8d89a5be673 100644 --- a/wasm-build/pack_extension.py +++ b/wasm-build/pack_extension.py @@ -43,10 +43,10 @@ def is_extension(path:Path, fullpath:Path): if asp.startswith('/lib/postgresql/'): if path.suffix == ".so": EXTNAME = path.stem - dumpcmd = f"{PGROOT}/bin/wasm-objdump -x {fullpath} > {PGL_DIST_LINK}/dumps/dump.{EXTNAME} 2>/dev/null " + dumpcmd = f"{PGROOT}/bin/wasm-objdump -x {fullpath} > {PG_BUILD_DUMPS}/dump.{EXTNAME} 2>/dev/null " os.system(dumpcmd) - os.system(f"OBJDUMP={PGL_DIST_LINK}/dumps/dump.{EXTNAME} python3 wasm-build/getsyms.py imports > {PGL_DIST_LINK}/imports/{EXTNAME}") + os.system(f"OBJDUMP={PG_BUILD_DUMPS}/dump.{EXTNAME} python3 wasm-build/getsyms.py imports > {PGL_DIST_LINK}/imports/{EXTNAME}") with open(f"{PGL_DIST_LINK}/imports/{EXTNAME}","r") as f: SYMBOLS=f.readlines() @@ -100,6 +100,7 @@ async def archive(target_folder): BUILD=os.environ.get('BUILD','emscripten') PGROOT=Path(os.environ.get('PGROOT',"/tmp/pglite")) PGL_DIST_LINK=Path(os.environ.get('PGL_DIST_LINK', "/tmp/sdk/dist/pglite-link")) +PG_BUILD_DUMPS=Path(os.environ.get('PG_BUILD_DUMPS', "/tmp/sdk/build/dumps")) INSTALLED = [] diff --git a/wasm-build/sdk.sh b/wasm-build/sdk.sh index 87cbc69f54098..f4c7c15f70908 100755 --- a/wasm-build/sdk.sh +++ b/wasm-build/sdk.sh @@ -155,3 +155,35 @@ fi cat $SDKROOT/VERSION + +if $WASI +then + # always install wasmtime because wasm-objdump needs it. + if [ -f ${SDKROOT}/devices/$(arch)/usr/bin/wasmtime ] + then + echo "keeping installed wasmtime and wasi binaries" + else +# TODO: window only has a zip archive, better use wasmtime-py instead. + + wget https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.0/wasmtime-v33.0.0-$(arch)-${PLATFORM}.tar.xz \ + -O-|xzcat|tar xfv - + mv -vf $(find wasmtime*|grep /wasmtime$) ${SDKROOT}/devices/$(arch)/usr/bin + fi + + if [ -f ${WASI_SYSROOT}/extra ] + then + echo -n + else + pushd ${WASI_SYSROOT} + VMLABS="https://github.com/vmware-labs/webassembly-language-runtimes/releases/download" + wget -q "${VMLABS}/libs%2Flibpng%2F1.6.39%2B20230629-ccb4cb0/libpng-1.6.39-wasi-sdk-20.0.tar.gz" -O-| tar xfz - + wget -q "${VMLABS}/libs%2Fzlib%2F1.2.13%2B20230623-2993864/libz-1.2.13-wasi-sdk-20.0.tar.gz" -O-| tar xfz - + wget -q "${VMLABS}/libs%2Fsqlite%2F3.42.0%2B20230623-2993864/libsqlite-3.42.0-wasi-sdk-20.0.tar.gz" -O-| tar xfz - + wget -q "${VMLABS}/libs%2Flibxml2%2F2.11.4%2B20230623-2993864/libxml2-2.11.4-wasi-sdk-20.0.tar.gz" -O-| tar xfz - + wget -q "${VMLABS}/libs%2Fbzip2%2F1.0.8%2B20230623-2993864/libbzip2-1.0.8-wasi-sdk-20.0.tar.gz" -O-| tar xfz - + wget -q "${VMLABS}/libs%2Flibuuid%2F1.0.3%2B20230623-2993864/libuuid-1.0.3-wasi-sdk-20.0.tar.gz" -O-| tar xfz - + popd + touch ${WASI_SYSROOT}/extra + fi +fi + From 38ade01e0c8b1c930e4075ee1b3ade9c169cf798 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Wed, 4 Jun 2025 10:05:27 +0200 Subject: [PATCH 12/19] use pg_ivm 1.11 --- extra/pg_ivm.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extra/pg_ivm.sh b/extra/pg_ivm.sh index 9dd4c2ced2c1e..df3a6606b0b83 100755 --- a/extra/pg_ivm.sh +++ b/extra/pg_ivm.sh @@ -7,10 +7,10 @@ pushd $PG_EXTRA then echo using local pgpg_ivm else -# wget https://github.com/sraoss/pg_ivm/archive/refs/tags/v1.10.tar.gz -O-|tar xfz - -# mv pg_ivm-* pg_ivm + wget https://github.com/sraoss/pg_ivm/archive/refs/tags/v1.11.tar.gz -O-|tar xfz - + mv pg_ivm-* pg_ivm - git clone --recursive --no-tags --depth 1 --single-branch --branch main https://github.com/sraoss/pg_ivm +# git clone --recursive --no-tags --depth 1 --single-branch --branch main https://github.com/sraoss/pg_ivm if $WASI then From 4e1a0bacc446965704f78adb834a185a81c2d5f7 Mon Sep 17 00:00:00 2001 From: Tudor Zaharia <37596026+tdrz@users.noreply.github.com> Date: Wed, 4 Jun 2025 10:34:45 +0200 Subject: [PATCH 13/19] Tdrz/incremental builds (#39) * allow incremental builds by storing objs files on the host filesystem --- pglite-REL_17_4_WASM/build.sh | 2 +- tmp/.gitignore | 4 ++++ wasm-build/build-pgcore.sh | 3 +-- wasm-build/build-with-docker.sh | 5 +++++ 4 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 tmp/.gitignore diff --git a/pglite-REL_17_4_WASM/build.sh b/pglite-REL_17_4_WASM/build.sh index 88b8cc523e5f6..24d5abe638884 100755 --- a/pglite-REL_17_4_WASM/build.sh +++ b/pglite-REL_17_4_WASM/build.sh @@ -11,7 +11,7 @@ fi echo "pglite/build-$BUILD: begin target BUILD_PATH=$BUILD_PATH" WORKSPACE=$(pwd) -PGROOT=/tmp/pglite +PGROOT=${PGROOT:-/tmp/pglite} if [ -d ${WORKSPACE}/src/fe_utils ] then diff --git a/tmp/.gitignore b/tmp/.gitignore new file mode 100644 index 0000000000000..86d0cb2726c6c --- /dev/null +++ b/tmp/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore \ No newline at end of file diff --git a/wasm-build/build-pgcore.sh b/wasm-build/build-pgcore.sh index f395478088a1d..76d3f5a96018d 100755 --- a/wasm-build/build-pgcore.sh +++ b/wasm-build/build-pgcore.sh @@ -217,6 +217,7 @@ END echo configure failed exit 218 fi +fi echo " @@ -424,8 +425,6 @@ END echo "TODO: node/wasi cmdline initdb for PGDATA=${PGDATA} " popd -fi - echo "build-pgcore: end($BUILD) diff --git a/wasm-build/build-with-docker.sh b/wasm-build/build-with-docker.sh index fb90c67ac0b3c..b6b546ddffcc4 100755 --- a/wasm-build/build-with-docker.sh +++ b/wasm-build/build-with-docker.sh @@ -25,6 +25,9 @@ cat .buildconfig mkdir -p dist/pglite dist/extensions-emsdk +# needed to be able to do incremental builds with docker +mkdir -p ${WORKSPACE}/postgres-pglite/tmp/sdk/build ${WORKSPACE}/postgres-pglite/tmp/pglite + if echo -n $@|grep -q it$ then PROMPT="&& bash ) || bash" @@ -37,6 +40,8 @@ docker run $@ \ --env-file .buildconfig \ -e DEBUG=${DEBUG:-false} \ --workdir=${DOCKER_WORKSPACE} \ + -v ${WORKSPACE}/postgres-pglite/tmp/sdk/build:/tmp/sdk/build:rw \ + -v ${WORKSPACE}/postgres-pglite/tmp/pglite:/tmp/pglite:rw \ -v ${WORKSPACE}/postgres-pglite:${DOCKER_WORKSPACE}:rw \ -v ${WORKSPACE}/postgres-pglite/dist:/tmp/sdk/dist:rw \ $IMG_NAME:$IMG_TAG \ From 8501a3e6aa9c5a056cd4fe0debdd950ac7002503 Mon Sep 17 00:00:00 2001 From: Tudor Zaharia <37596026+tdrz@users.noreply.github.com> Date: Wed, 4 Jun 2025 11:26:58 +0200 Subject: [PATCH 14/19] Tdrz/incremental builds (#40) * allow incremental builds by storing objs files on the host filesystem From 787bdae98a697ecddbf050122ddf73575af2d7d9 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Wed, 4 Jun 2025 11:21:47 +0200 Subject: [PATCH 15/19] force sdk patching --- wasm-build.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/wasm-build.sh b/wasm-build.sh index 98f5d10ae0144..e8f21696a319d 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -165,16 +165,18 @@ else then echo "emcc found in PATH=$PATH" else - if ${PORTABLE}/sdk.sh - then - echo "$PORTABLE : sdk check passed (emscripten)" - else - echo emsdk failed - exit 150 - fi - . ${SDKROOT}/wasm32-bi-emscripten-shell.sh fi + + if ${PORTABLE}/sdk.sh + then + echo "$PORTABLE : sdk check passed (emscripten)" + else + echo emsdk failed + exit 150 + fi + + export PG_LINK=${PG_LINK:-$(which emcc)} echo " From e54f0eefb5e641269f16bfc9811ee52991f6b065 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Wed, 4 Jun 2025 13:19:52 +0200 Subject: [PATCH 16/19] sdk patch --- wasm-build.sh | 46 ++++++++++++++++----------------- wasm-build/build-pgcore.sh | 3 ++- wasm-build/build-with-docker.sh | 5 ---- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/wasm-build.sh b/wasm-build.sh index e8f21696a319d..f54cf7aa37941 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -157,39 +157,39 @@ System node/pnpm ( may interfer) : # TODO: but may need to adjust $PATH with stock emsdk. pushd ${SDKROOT} -if ${WASI} -then - . wasisdk/wasisdk_env.sh -else - if which emcc + if ${WASI} then - echo "emcc found in PATH=$PATH" + . wasisdk/wasisdk_env.sh else - . ${SDKROOT}/wasm32-bi-emscripten-shell.sh - fi + if which emcc + then + echo "emcc found in PATH=$PATH" + else + . ${SDKROOT}/wasm32-bi-emscripten-shell.sh + fi - if ${PORTABLE}/sdk.sh - then - echo "$PORTABLE : sdk check passed (emscripten)" - else - echo emsdk failed - exit 150 - fi + if ${PORTABLE}/sdk.sh + then + echo "$PORTABLE : sdk check passed (emscripten)" + else + echo emsdk failed + exit 176 + fi - export PG_LINK=${PG_LINK:-$(which emcc)} + export PG_LINK=${PG_LINK:-$(which emcc)} - echo " + echo " - Using provided emsdk from $(which emcc) - Using PG_LINK=$PG_LINK as linker + Using provided emsdk from $(which emcc) + Using PG_LINK=$PG_LINK as linker - node : $(which node) $($(which node) -v) - PNPM : $(which pnpm) + node : $(which node) $($(which node) -v) + PNPM : $(which pnpm) -" -fi + " + fi popd # used for not makefile (manual linking and pgl_main) diff --git a/wasm-build/build-pgcore.sh b/wasm-build/build-pgcore.sh index 76d3f5a96018d..f395478088a1d 100755 --- a/wasm-build/build-pgcore.sh +++ b/wasm-build/build-pgcore.sh @@ -217,7 +217,6 @@ END echo configure failed exit 218 fi -fi echo " @@ -425,6 +424,8 @@ END echo "TODO: node/wasi cmdline initdb for PGDATA=${PGDATA} " popd +fi + echo "build-pgcore: end($BUILD) diff --git a/wasm-build/build-with-docker.sh b/wasm-build/build-with-docker.sh index b6b546ddffcc4..fb90c67ac0b3c 100755 --- a/wasm-build/build-with-docker.sh +++ b/wasm-build/build-with-docker.sh @@ -25,9 +25,6 @@ cat .buildconfig mkdir -p dist/pglite dist/extensions-emsdk -# needed to be able to do incremental builds with docker -mkdir -p ${WORKSPACE}/postgres-pglite/tmp/sdk/build ${WORKSPACE}/postgres-pglite/tmp/pglite - if echo -n $@|grep -q it$ then PROMPT="&& bash ) || bash" @@ -40,8 +37,6 @@ docker run $@ \ --env-file .buildconfig \ -e DEBUG=${DEBUG:-false} \ --workdir=${DOCKER_WORKSPACE} \ - -v ${WORKSPACE}/postgres-pglite/tmp/sdk/build:/tmp/sdk/build:rw \ - -v ${WORKSPACE}/postgres-pglite/tmp/pglite:/tmp/pglite:rw \ -v ${WORKSPACE}/postgres-pglite:${DOCKER_WORKSPACE}:rw \ -v ${WORKSPACE}/postgres-pglite/dist:/tmp/sdk/dist:rw \ $IMG_NAME:$IMG_TAG \ From 04eb770cdea1b14bd8e96cd20a29a90cc8e980a3 Mon Sep 17 00:00:00 2001 From: tdrz Date: Wed, 4 Jun 2025 14:24:37 +0200 Subject: [PATCH 17/19] fix incremental build --- wasm-build/build-with-docker.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wasm-build/build-with-docker.sh b/wasm-build/build-with-docker.sh index fb90c67ac0b3c..f6b20e5839f5e 100755 --- a/wasm-build/build-with-docker.sh +++ b/wasm-build/build-with-docker.sh @@ -23,7 +23,7 @@ source .buildconfig cat .buildconfig -mkdir -p dist/pglite dist/extensions-emsdk +mkdir -p dist/pglite dist/extensions-emsdk ${WORKSPACE}/postgres-pglite/tmp/sdk/build ${WORKSPACE}/postgres-pglite/tmp/pglite if echo -n $@|grep -q it$ then @@ -37,6 +37,8 @@ docker run $@ \ --env-file .buildconfig \ -e DEBUG=${DEBUG:-false} \ --workdir=${DOCKER_WORKSPACE} \ + -v ${WORKSPACE}/postgres-pglite/tmp/pglite:/tmp/pglite:rw \ + -v ${WORKSPACE}/postgres-pglite/tmp/sdk/build:/tmp/sdk/build:rw \ -v ${WORKSPACE}/postgres-pglite:${DOCKER_WORKSPACE}:rw \ -v ${WORKSPACE}/postgres-pglite/dist:/tmp/sdk/dist:rw \ $IMG_NAME:$IMG_TAG \ From 180712612371dc518a0383c729bc32a9fafad169 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Sat, 7 Jun 2025 06:11:11 +0200 Subject: [PATCH 18/19] fix debug build --- pglite-REL_17_4_WASM/build.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pglite-REL_17_4_WASM/build.sh b/pglite-REL_17_4_WASM/build.sh index 24d5abe638884..591359a851c81 100755 --- a/pglite-REL_17_4_WASM/build.sh +++ b/pglite-REL_17_4_WASM/build.sh @@ -11,7 +11,7 @@ fi echo "pglite/build-$BUILD: begin target BUILD_PATH=$BUILD_PATH" WORKSPACE=$(pwd) -PGROOT=${PGROOT:-/tmp/pglite} +PGROOT=/tmp/pglite if [ -d ${WORKSPACE}/src/fe_utils ] then @@ -174,6 +174,7 @@ else then # FULL LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS}" + unset EMCC_FORCE_STDLIBS # LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pglite" else # min @@ -182,11 +183,11 @@ else # LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS}" # LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pglite" + # tailored LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pglite" + EMCC_FORCE_STDLIBS=1 - # LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pglite" - #export LOPTS="-O2 -g2 --no-wasm-opt -sASSERTIONS=0" fi echo " @@ -278,7 +279,7 @@ ________________________________________________________ # LOPTS="-Os -g0" # - if EMCC_FORCE_STDLIBS=1 COPTS="$LOPTS" ${CC} ${CC_PGLITE} -o ${PGL_DIST_WEB}/pglite.html --shell-file ${WORKSPACE}/pglite-${PG_BRANCH}/repl.html \ + if EMCC_FORCE_STDLIBS=$EMCC_FORCE_STDLIBS COPTS="$LOPTS" ${CC} ${CC_PGLITE} -o ${PGL_DIST_WEB}/pglite.html --shell-file ${WORKSPACE}/pglite-${PG_BRANCH}/repl.html \ $PGPRELOAD \ -sGLOBAL_BASE=${CMA_MB}MB -ferror-limit=1 \ -sFORCE_FILESYSTEM=1 -sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web \ From 4b6a71d391578f584cebccebe7bd78c249b4791f Mon Sep 17 00:00:00 2001 From: pmp-p Date: Sat, 7 Jun 2025 06:49:29 +0200 Subject: [PATCH 19/19] fix debug build --- pglite-REL_17_4_WASM/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pglite-REL_17_4_WASM/build.sh b/pglite-REL_17_4_WASM/build.sh index 591359a851c81..243ba44200cb5 100755 --- a/pglite-REL_17_4_WASM/build.sh +++ b/pglite-REL_17_4_WASM/build.sh @@ -186,7 +186,7 @@ else # tailored LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pglite" - EMCC_FORCE_STDLIBS=1 + export EMCC_FORCE_STDLIBS=1 fi @@ -279,7 +279,7 @@ ________________________________________________________ # LOPTS="-Os -g0" # - if EMCC_FORCE_STDLIBS=$EMCC_FORCE_STDLIBS COPTS="$LOPTS" ${CC} ${CC_PGLITE} -o ${PGL_DIST_WEB}/pglite.html --shell-file ${WORKSPACE}/pglite-${PG_BRANCH}/repl.html \ + if COPTS="$LOPTS" ${CC} ${CC_PGLITE} -o ${PGL_DIST_WEB}/pglite.html --shell-file ${WORKSPACE}/pglite-${PG_BRANCH}/repl.html \ $PGPRELOAD \ -sGLOBAL_BASE=${CMA_MB}MB -ferror-limit=1 \ -sFORCE_FILESYSTEM=1 -sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web \