diff --git a/extra/pg_ivm.sh b/extra/pg_ivm.sh new file mode 100755 index 0000000000000..df3a6606b0b83 --- /dev/null +++ b/extra/pg_ivm.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +. wasm-build/extension.sh + +pushd $PG_EXTRA + if [ -d pg_ivm ] + then + echo using local pgpg_ivm + else + wget https://github.com/sraoss/pg_ivm/archive/refs/tags/v1.11.tar.gz -O-|tar xfz - + mv pg_ivm-* pg_ivm + +# git clone --recursive --no-tags --depth 1 --single-branch --branch main https://github.com/sraoss/pg_ivm + + if $WASI + then + echo "no patching" + else + echo "PATCH?" + + fi + + fi +popd + +pushd $PG_EXTRA/pg_ivm + # path for wasm-shared already set to (pwd:pg build dir)/bin + # OPTFLAGS="" turns off arch optim (sse/neon). + PG_CONFIG=${PGROOT}/bin/pg_config emmake make OPTFLAGS="" install || exit 19 + + #cp sql/pg_ivm--1.10.sql ${PGROOT}/share/postgresql/extension/ + #rm -f ${PGROOT}/share/postgresql/extension/pg_ivm--?.?.?--?.?.?.sql ${PGROOT}/share/postgresql/extension/pg_ivm.sql + +popd + + diff --git a/extra/vector.sh b/extra/vector.sh index 3e6d98538cd02..4dd8666684c09 100755 --- a/extra/vector.sh +++ b/extra/vector.sh @@ -1,40 +1,27 @@ #!/bin/bash -mkdir -p build src +. wasm-build/extension.sh -pushd build +pushd $PG_EXTRA # [ -d pgvector ] || git clone --no-tags --depth 1 --single-branch --branch master https://github.com/pgvector/pgvector if [ -d vector ] then echo using local pgvector else - [ -f ../src/pgvector.tar.gz ] || wget -c -q https://github.com/pgvector/pgvector/archive/refs/tags/v0.8.0.tar.gz -O../src/pgvector.tar.gz - tar xvfz ../src/pgvector.tar.gz + wget -c -q https://github.com/pgvector/pgvector/archive/refs/tags/v0.8.0.tar.gz -Opgvector.tar.gz + tar xvfz pgvector.tar.gz mv pgvector-?.?.? vector fi popd - - -if which emcc -then - echo -n -else - reset; - . /opt/python-wasm-sdk/wasm32-bi-emscripten-shell.sh - export PGROOT=${PGROOT:-/tmp/pglite} - export PATH=${PGROOT}/bin:$PATH -fi - - -pushd build/vector +pushd $PG_EXTRA/vector # path for wasm-shared already set to (pwd:pg build dir)/bin # OPTFLAGS="" turns off arch optim (sse/neon). - PG_CONFIG=${PGROOT}/bin/pg_config emmake make OPTFLAGS="" install || exit 33 + PG_CONFIG=${PGROOT}/bin/pg_config emmake make OPTFLAGS="" install || exit 21 - cp sql/vector.sql sql/vector--0.8.0.sql ${PGROOT}/share/postgresql/extension/ - rm ${PGROOT}/share/postgresql/extension/vector--?.?.?--?.?.?.sql ${PGROOT}/share/postgresql/extension/vector.sql + cp sql/vector--0.8.0.sql ${PGROOT}/share/postgresql/extension/ + rm ${PGROOT}/share/postgresql/extension/vector--?.?.?--?.?.?.sql popd diff --git a/pglite-REL_17_4_WASM/build.sh b/pglite-REL_17_4_WASM/build.sh index 71b209c0521db..243ba44200cb5 100755 --- a/pglite-REL_17_4_WASM/build.sh +++ b/pglite-REL_17_4_WASM/build.sh @@ -163,8 +163,9 @@ else EXPORTED_FUNCTIONS="_main,_use_wire,_pgl_initdb,_pgl_backend,_pgl_shutdown,_interactive_write,_interactive_read,_interactive_one" + EXPORTED_FUNCTIONS="$EXPORTED_FUNCTIONS,_get_channel,_get_buffer_size,_get_buffer_addr" - EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,FS_mount,FS_syncfs,FS_analyzePath,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack" +# EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,FS_mount,FS_syncfs,FS_analyzePath,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack" EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack" @@ -173,13 +174,20 @@ else then # FULL LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS}" + unset EMCC_FORCE_STDLIBS +# LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pglite" else # min # LINKER="-sMAIN_MODULE=2" + +# LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS}" +# LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pglite" + # tailored - LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@exports" -LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS}" + LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@${PGL_DIST_LINK}/exports/pglite" + export EMCC_FORCE_STDLIBS=1 + fi echo " @@ -231,39 +239,72 @@ ________________________________________________________ if ${CC} ${CC_PGLITE} ${PGINC} -o ${BUILD_PATH}/pglite.o -c ${WORKSPACE}/pglite-${PG_BRANCH}/pg_main.c \ -Wno-incompatible-pointer-types-discards-qualifiers then - echo " * linking node raw version of pglite ${PG_BRANCH}" + echo " * linking node raw version of pglite ${PG_BRANCH} (with all symbols)" - COPTS="$LOPTS" ${CC} ${CC_PGLITE} ${PGINC} -o ${PGL_DIST_JS}/pglite-js.js \ + if COPTS="-O2 -g3 --no-wasm-opt" ${CC} ${CC_PGLITE} ${PGINC} -o ${PGL_DIST_JS}/pglite-js.js \ -sGLOBAL_BASE=${CMA_MB}MB -ferror-limit=1 \ - -sFORCE_FILESYSTEM=1 $EMCC_NODE \ + -sFORCE_FILESYSTEM=1 $EMCC_NODE -sMAIN_MODULE=1 -sEXPORT_ALL -sASSERTIONS=0 \ -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS \ -sEXPORTED_RUNTIME_METHODS=${EXPORTED_RUNTIME_METHODS} \ ${BUILD_PATH}/pglite.o \ - $LINKER $LIBPGCORE \ + $LIBPGCORE \ $LINK_ICU \ -lnodefs.js -lidbfs.js -lxml2 -lz + then + ./wasm-build/linkexport.sh + ./wasm-build/linkimports.sh + else + echo " + * linking node raw version of pglite failed"; exit 250 + + fi + + + echo " + + * linking version of pglite ( with .data initial filesystem, and html repl) (required symbols) + + BUILD_PATH=${BUILD_PATH} + LINKER=$LINKER + LIBPGCORE=$LIBPGCORE - echo " * linking web version of pglite ( with .data initial filesystem, and html repl)" - COPTS="$LOPTS" ${CC} ${CC_PGLITE} -o ${PGL_DIST_WEB}/pglite.html --shell-file ${WORKSPACE}/pglite-${PG_BRANCH}/repl.html \ + +" + +# function from : +# ${BUILD_PATH}/src/interfaces/libpq/libpq.a +# required are to be found in : +# pglite.o +# + +# LOPTS="-Os -g0" +# + if COPTS="$LOPTS" ${CC} ${CC_PGLITE} -o ${PGL_DIST_WEB}/pglite.html --shell-file ${WORKSPACE}/pglite-${PG_BRANCH}/repl.html \ $PGPRELOAD \ -sGLOBAL_BASE=${CMA_MB}MB -ferror-limit=1 \ -sFORCE_FILESYSTEM=1 -sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web \ + $LINKER \ -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module \ - -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS \ + -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS=1 \ -sEXPORTED_RUNTIME_METHODS=${EXPORTED_RUNTIME_METHODS} \ ${PGINC} ${BUILD_PATH}/pglite.o \ - $LINKER $LIBPGCORE \ + $LIBPGCORE \ $LINK_ICU \ -lnodefs.js -lidbfs.js -lxml2 -lz - + then + du -hs du -hs ${PG_DIST}/* + touch ${WORKSPACE}/done + else + echo " + * linking web version of pglite failed"; exit 272 + fi else - echo "compilation of libpglite ${PG_BRANCH} failed" - exit 220 + echo "compilation of libpglite ${PG_BRANCH} failed"; exit 275 fi fi -du -hs ${PG_DIST}/* + echo "pglite/build($BUILD): end" diff --git a/pglite-REL_17_4_WASM/interactive_one.c b/pglite-REL_17_4_WASM/interactive_one.c index fb8de8033c271..9357ec62acf80 100644 --- a/pglite-REL_17_4_WASM/interactive_one.c +++ b/pglite-REL_17_4_WASM/interactive_one.c @@ -720,6 +720,12 @@ puts("# 631: PIPELINING + rfq"); } else { cma_wsize = 0; PDEBUG("# 698: no data, send empty ?"); +// TODO: dedup 739 + if (sockfiles) { + fclose(SOCKET_FILE); + SOCKET_FILE = NULL; + rename(PGS_OLOCK, PGS_OUT); + } } } else { pg_prompt(); @@ -729,6 +735,13 @@ puts("# 631: PIPELINING + rfq"); puts("# 686: socket has data"); if (sockfiles) printf("# 688: socket file not flushed -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", SOCKET_DATA); + } else { +// TODO: dedup 723 + if (sockfiles) { + fclose(SOCKET_FILE); + SOCKET_FILE = NULL; + rename(PGS_OLOCK, PGS_OUT); + } } if (cma_wsize) puts("ERROR: cma was not flushed before socketfile interface"); diff --git a/pglite-REL_17_4_WASM/pg_main.c b/pglite-REL_17_4_WASM/pg_main.c index abfea9ea37ddd..66822227b7fd4 100644 --- a/pglite-REL_17_4_WASM/pg_main.c +++ b/pglite-REL_17_4_WASM/pg_main.c @@ -1,9 +1,9 @@ -// for handling REVOKIE ex in initdb -#if defined(__wasi__) -#define FIXME 1 +// for handling REVOKE exception in initdb +#if defined(__wasi__) || defined(__EMSCRIPTEN__) +# define FIXME 1 #else -#define FIXME 0 +# define FIXME 0 #endif #define PGL_MAIN diff --git a/pglite-REL_17_4_WASM/repl.html b/pglite-REL_17_4_WASM/repl.html index 22ff371f5d161..e87f0e3c1c817 100644 --- a/pglite-REL_17_4_WASM/repl.html +++ b/pglite-REL_17_4_WASM/repl.html @@ -77,7 +77,7 @@ config sample expected in options.config : { - "shared_preload_libraries" : ["pg_stat_statements"], + "shared_preload_libraries" : ["'pg_stat_statements'"], "compute_query_id" : "on", "pg_stat_statements.max" : "10000", "pg_stat_statements.track": "all", @@ -87,23 +87,33 @@ async function pgl_conf(vm, kv) { - var lines = vm.os.codec.decode(vm.FS.readFile(vm.PGDATA + "/postgresql.conf")).split('\n'); + //var lines = vm.os.codec.decode(vm.FS.readFile(vm.PGDATA + "/postgresql.conf")).split('\n'); + var lines = [] + for (const line of vm.os.codec.decode(vm.FS.readFile(vm.PGDATA + "/postgresql.conf")).split('\n')) { + var testl = line.trimStart().split("#")[0].trimEnd() + if (testl.length<4) + continue + if (testl.startsWith("#")) + continue + lines.push(testl) + } var buf = [] var newlines = []; - //return; for (var k in kv) { var old = "" for (var line = 0; line < lines.length; line++) { - if (lines[line].startsWith(k)) { - old = lines[line]; - } else { - buf.push(lines[line]) + var testl = lines[line]; + if (testl.startsWith(k)) { + old = testl; + lines[line] = "# " + testl + break } } + var values = [] if (kv[k] instanceof Array) { // TODO extract array of , from old @@ -115,24 +125,21 @@ values.push(kv[k].toString()) } - - const newline = `\n${k} = ${values.join(',')}\n` - - if (!old.length) { - console.log("NEW :", newline) - } else { - if (old==newline) - console.log("no conf change :", newline) - else - console.error("EDIT :",old,'->', newline) - } + const newline = `${k} = ${values.join(',')}` newlines.push(newline) } + if (newlines.length) { - buf.push(...newlines) -console.log("BUF", buf) - vm.FS.writeFile( vm.PGDATA + "/postgresql.conf", buf.join("") ) - vm.FS.syncfs(false, (e) => { if (e) console.error("pgl_conf error:", e)}) + lines.push(...newlines) + for (const line of lines) { + console.log(line) + } + + vm.FS.unlink(vm.PGDATA + "/postgresql.conf") + vm.FS.writeFile( vm.PGDATA + "/postgresql.conf", lines.join("\n") ) + vm.FS.syncfs(false, (e) => { if (e) console.error("pgl_conf:", e)}) + + console.log("Config file written at :", vm.PGDATA + "/postgresql.conf" ) } } @@ -298,7 +305,7 @@ return text_codec.decode( new Uint8Array(ary) ) } window.b_utf8 = b_utf8 - +// /tmp/pglite/base/postgresql.conf function test_drive(step) { var data = null switch (test_step) { @@ -307,9 +314,51 @@ break case 1: - //data = "SELECT now(), current_database(), session_user, current_user;" - data = "SELECT REPEAT('0123456789abcdef', 64*1024), block FROM generate_series(1,8) AS block;" + data = "SELECT now(), current_database(), session_user, current_user;" +// data = "SELECT REPEAT('0123456789abcdef', 64*1024), block FROM generate_series(1,8) AS block;" + break + + case 2: + data = ` +CREATE TABLE IF NOT EXISTS test ( + id SERIAL PRIMARY KEY, + number INT + ); + INSERT INTO test (number) VALUES (42); +` + break + + case 3: + data = ` +CREATE EXTENSION IF NOT EXISTS pg_ivm; +` + break + +// CREATE MATERIALIZED VIEW myview AS SELECT * FROM test; +// SELECT pgivm.create_immv('myview', 'SELECT * FROM test'); + case 4: + data = ` +SELECT pgivm.create_immv('myview', 'SELECT * FROM test'); +` + break + + case 5: + data = ` +INSERT INTO test VALUES (43); +SELECT * FROM myview; +INSERT INTO test VALUES (44); +` + break + + +//REFRESH MATERIALIZED VIEW myview; + case 6: + data = ` +SELECT * FROM myview; +` break + + /* case 2: data = ` @@ -408,11 +457,12 @@ async function delayed_init() { await Module.load_package("vector", "vector.tar.gz") await Module.load_package("uuid-ossp", "uuid-ossp.tar.gz") + await Module.load_package("pg_ivm", "pg_ivm.tar.gz") + + await Module.load_package("pg_stat_statements", "pg_stat_statements.tar.gz") - // await Module.load_package("pg_stat_statements", "pg_stat_statements.tar.gz") // await Module.load_package("postgis", "postgis-3.tar.gz") // await Module.load_package("arrays", "arrays.tar.gz") - // await Module.load_package("pg_stat_statements", "pg_stat_statements.tar.gz") // vm._setenv(vm.stringToUTF8OnStack("REPL"),vm.stringToUTF8OnStack("Y"),1) console.warn("calling initdb") @@ -488,8 +538,10 @@ vm.PGDATA = PGDATA if (editconf) { await pgl_conf(vm, { - "shared_preload_libraries" : ["pg_stat_statements"], + "shared_preload_libraries" : ["'pg_stat_statements'"], + "max_connections" : "5", "compute_query_id" : "on", + "shared_buffers" : "128kB", "pg_stat_statements.max" : "10000", "pg_stat_statements.track": "all", }) @@ -681,7 +733,38 @@ return } - if (line.startsWith('pg_dump ')) { + const elems = line.trimStart().trimEnd().split(" ") + cmd = elems[0] + arg = elems.at(-1) + console.log("CMD:", cmd, "ARG:", arg) + + switch (cmd) { + case "rx": + var filepath = arg + console.log('getting file:', filepath) + filename = filepath.split("/").at(-1) + const blob = new Blob([vm.FS.readFile(filepath)]) + const elem = window.document.createElement('a'); + elem.href = window.URL.createObjectURL(blob, { oneTimeOnly: true }); + elem.download = filename; + document.body.appendChild(elem); + elem.click(); + document.body.removeChild(elem); + return + + case "cat": + for (const line of os.codec.decode(vm.FS.readFile(arg)).split("\n")) { + vm.vt.xterm.write( line + "\r\n" ) + } + return + + } + + + + + + if (line.startsWith("pg_dump ")) { console.log("RL:", line) } else { @@ -711,8 +794,6 @@ }; - //jsimport("tinytar.min.js") - window.vm = Module const { WasmTerminal } = await /**/ import("https://pmp-p.ddns.net/pglite-web/vtx.js") @@ -771,15 +852,15 @@ Module._use_wire(1) encodedArray.set(data) } else { - console.log("string data, using repl/no wire") + // console.log("string data, using repl/no wire") Module._use_wire(0) const encoder = new TextEncoder(); encodedArray = encoder.encode(data); } // sf - send_to_pglite(encodedArray, FS, -1) +// send_to_pglite(encodedArray, FS, -1) // cma -// send_to_pglite(encodedArray, FS, Module.cma_port || 1) + send_to_pglite(encodedArray, FS, Module.cma_port || 1) } else { console.warn("BC(srv):", ev); } diff --git a/portable/Dockerfile b/portable/Dockerfile index 2f70f37e0d75d..c4b6f251867b2 100755 --- a/portable/Dockerfile +++ b/portable/Dockerfile @@ -6,10 +6,10 @@ ENV PIP_NO_CACHE_DIR 1 # nb: the python for build from sdk built with clang directly is used as SYS_PYTHON for emsdk RUN apk add --no-cache --virtual .build-deps \ - tar file lz4 \ + tar file lz4 xz \ git patch bison flex \ findutils binutils coreutils \ - libffi curl perl nodejs \ + libffi tzdata-utils curl perl nodejs \ make autoconf automake libtool pkgconfig \ ; diff --git a/portable/portable.sh b/portable/portable.sh index 557eda934ca25..23ca54cd2f7a5 100755 --- a/portable/portable.sh +++ b/portable/portable.sh @@ -1,11 +1,13 @@ #!/usr/bin/env bash -export PG_VERSION=${PG_BRANCH:-17.4} +export PG_VERSION=${PG_VERSION:-17.4} export PG_BRANCH=${PG_BRANCH:-REL_17_4_WASM} export PORTABLE=$(realpath $(dirname $0)) export ROOT=$(realpath $(pwd)) export SDKROOT=${SDKROOT:-/tmp/sdk} +export WASI=${WASI:-false} echo " + ================================================================================================== ================================================================================================== @@ -16,6 +18,7 @@ PG_BRANCH=$PG_BRANCH SDKROOT=$SDKROOT DEBUG=$DEBUG USE_ICU=$USE_ICU +WASI=$WASI ================================================================================================== ================================================================================================== @@ -24,15 +27,16 @@ USE_ICU=$USE_ICU - export PATH=$PORTABLE:$PATH export WORKDIR=${ROOT} export CONTAINER_PATH=${CONTAINER_PATH:-/tmp/fs} export HOME=/tmp -export PROOT=${PORTABLE}/proot +export PROOT=${PORTABLE}/proot.$(arch) +export ALPINEPROOT_NO_PULSE=true + -# git remove empty dirs -mkdir -p ${WORKDIR}/sdk/dist +# git would remove empty dirs +mkdir -p ${WORKDIR}/dist-${PG_BRANCH} ${WORKDIR}/build-${PG_BRANCH} # -------------------------------------------------------- # "docker emulation" @@ -191,7 +195,7 @@ __get_container_url() { } __start() { - proot -0 rm -rf $CONTAINER_PATH/proc + $PROOT -0 rm -rf $CONTAINER_PATH/proc mkdir $CONTAINER_PATH/proc # Proceed make fake /proc/version @@ -371,11 +375,13 @@ __start() { COMMANDS+=" -b /proc/self/fd/1:/dev/stdout" COMMANDS+=" -b /proc/self/fd/2:/dev/stderr" COMMANDS+=" -b ${WORKDIR}:/workspace" - COMMANDS+=" -b ${WORKDIR}/dist:/tmp/sdk/dist" + COMMANDS+=" -b ${WORKDIR}/build-${PG_BRANCH}:/tmp/sdk/build" + COMMANDS+=" -b ${WORKDIR}/dist-${PG_BRANCH}:/tmp/sdk/dist" for f in stat version loadavg vmstat uptime do [ -f "$CONTAINER_PATH/proc/.$f" ] && COMMANDS+=" -b $CONTAINER_PATH/proc/.$f:/proc/$f" done +# --change-id=uid:gid COMMANDS+=" -r $CONTAINER_PATH -0 -w /root" COMMANDS+=" -b $CONTAINER_PATH/root:/dev/shm" @@ -407,7 +413,10 @@ __start() { } -if git checkout ${PG_BRANCH}-pglite + + +# git checkout ${PG_BRANCH} +if cd ${WORKDIR}/postgresql-${PG_BRANCH} then if [ -f postgresql-${PG_BRANCH}.patched ] then @@ -417,12 +426,13 @@ then Patching branch ${PG_BRANCH} with : -$(find patches-${PG_BRANCH}/postgresql-*) +$(find ${WORKDIR}/patches-${PG_BRANCH}/postgresql-*) " - # these don't exist in a released postgres. + + # these initially don't exist in a released postgres. touch ./src/template/emscripten \ ./src/include/port/emscripten.h \ ./src/include/port/wasm_common.h \ @@ -434,9 +444,9 @@ $(find patches-${PG_BRANCH}/postgresql-*) postgresql-wasi \ postgresql-pglite do - if [ -d patches-${PG_BRANCH}/$patchdir ] + if [ -d ${WORKDIR}/patches-${PG_BRANCH}/$patchdir ] then - for one in patches-${PG_BRANCH}/$patchdir/*.diff + for one in ${WORKDIR}/patches-${PG_BRANCH}/$patchdir/*.diff do if cat $one | patch -p1 then @@ -446,7 +456,7 @@ $(find patches-${PG_BRANCH}/postgresql-*) Fatal: failed to apply patch : $one " - exit 366 + exit 457 fi done fi @@ -454,21 +464,53 @@ Fatal: failed to apply patch : $one touch postgresql-${PG_BRANCH}.patched fi + if [ -d $CONTAINER_PATH/${SDKROOT} ] then - echo using cached version + echo " + * using cached sdk version from $CONTAINER_PATH/${SDKROOT} +" else - SDK_URL=https://github.com/pygame-web/portable-sdk/releases/download/3.1.74.7bi/python3.13-wasm-sdk-alpine-3.21.tar.lz4 - echo "setting up sdk $SDK_URL" + # SDK_URL=https://github.com/pygame-web/portable-sdk/releases/download/3.1.74.7bi/python3.13-wasm-sdk-alpine-3.21.tar.lz4 + SDK_URL=https://github.com/pygame-web/portable-sdk/releases/download/3.1.61.8/python3.12-wasm-sdk-debian12-$(arch).tar.lz4 + echo " + + * Setting up emsdk+wasi sdk from $SDK_URL + +" pushd $CONTAINER_PATH - mkdir -p /tmp/sdk - tmpfile=/tmp/sdk/python3.13-wasm-sdk-alpine-3.21.tar.lz4 - [ -f /opt/python3.13-wasm-sdk-alpine-3.21.tar.lz4 ] && cp -f /opt/python3.13-wasm-sdk-alpine-3.21.tar.lz4 $tmpfile - [ -f /tmp/sdk/python3.13-wasm-sdk-alpine-3.21.tar.lz4 ] || wget -q $SDK_URL -O$tmpfile + mkdir -p tmp + tmpfile=tmp/python-wasm-sdk-alpine-3.21.tar.lz4 + # local cache + [ -f $PORTABLE/python-wasm-sdk-alpine-3.21.tar.lz4 ] && cp -f $PORTABLE/python-wasm-sdk-alpine-3.21.tar.lz4 $tmpfile + [ -f $tmpfile ] || wget -q $SDK_URL -O$tmpfile cat $tmpfile | tar x --use-compress-program=lz4 + + # unpack wasi sdk (common) + tar xf $PORTABLE/wasi-sdk-25.tar.xz + # unpack wasi sdk ( binary ) + tar xf $PORTABLE/wasi-sdk-25.0-$(arch)-linux.tar.xz + + # install arch binaries to sdk wasi sdk root + mv tmp/sdk/wasisdk/wasi-sdk-25.0-$(arch)-linux/* tmp/sdk/wasisdk/upstream/ + + if [ -f $CONTAINER_PATH/usr/bin/python3 ] + then + echo "system python found" + else + if [ -L $CONTAINER_PATH/usr/bin/python3 ] + then + echo "linked python for build found" + else + echo "Setting python for build as system python3" + mkdir -p $CONTAINER_PATH/usr/bin + ln -s $SDKROOT/devices/$(arch)/usr/bin/python3 $CONTAINER_PATH/usr/bin/python3 + fi + fi popd fi + # prevent erasing touch ${CONTAINER_PATH}${SDKROOT}/dev @@ -479,5 +521,5 @@ Fatal: failed to apply patch : $one alpineproot "apk add bash;/bin/bash --init-file /initrc" fi else - echo Error need PG_BRANCH=$PG_BRANCH set to a valid branch + echo "Error need PG_BRANCH=$PG_BRANCH set to a valid postgres-pglite WASM branch" fi diff --git a/tmp/.gitignore b/tmp/.gitignore new file mode 100644 index 0000000000000..86d0cb2726c6c --- /dev/null +++ b/tmp/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore \ No newline at end of file diff --git a/wasm-build.sh b/wasm-build.sh index 4e026147fb11f..f54cf7aa37941 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -1,5 +1,4 @@ #!/bin/bash - export PG_VERSION=${PG_VERSION:-17.4} #set -x; @@ -25,15 +24,20 @@ export PGROOT=${PGROOT:-/tmp/pglite} export WEBROOT=${WEBROOT:-/tmp/web} export PG_BUILD=${BUILD:-/tmp/sdk/build} -export PGL_BUILD_NATIVE="${PG_BUILD}/pglite-native" +export PGL_BUILD_NATIVE=${PG_BUILD}/pglite-native +export PGL_BUILD_DUMPS=${PG_BUILD}/dumps + export PG_DIST=${DIST:-/tmp/sdk/dist} -export PG_DIST_EXT="${PG_DIST}/extensions-emsdk" +export PG_DIST_EXT=${PG_DIST}/extensions-emsdk + +export PGL_DIST_JS=${PG_DIST}/pglite-js +export PGL_DIST_LINK=${PG_DIST}/pglite-link -export PGL_DIST_JS="${PG_DIST}/pglite-js" +export PGL_DIST_NATIVE=${PG_DIST}/pglite-native +export PGL_DIST_C=${PG_DIST}/pglite-native +export PGL_DIST_WEB=${PG_DIST}/pglite-web -export PGL_DIST_NATIVE="${PG_DIST}/pglite-native" -export PGL_DIST_WEB="${PG_DIST}/pglite-web" export DEBUG=${DEBUG:-true} @@ -47,6 +51,7 @@ export PGUSER=${PGUSER:-postgres} export WASI=${WASI:-false} export WASI_SDK=${WASI_SDK:-25.0} export PYBUILD=${PYBUILD:-3.13} +export NATIVE=${NATIVE:-false} if $WASI @@ -64,21 +69,49 @@ else BUILD=emscripten if $DEBUG then - # clang default to O0 but specifying -O0 may trigger memory start address bug in emsdk - export COPTS="-g3 --no-wasm-opt" - export LOPTS=${LOPTS:-"-g3 --no-wasm-opt -sASSERTIONS=1"} + # clang default to O0 but specifying -O0 may trigger align bug in emsdk + if [ -f /alpine ] + then + # dev debug + export COPTS="-O2 -g3 --no-wasm-opt" + export LOPTS=${LOPTS:-"-O2 -g3 --no-wasm-opt -sASSERTIONS=1"} + else + # docker debug ( exepected to be ide friendly ) + export COPTS="-g3 --no-wasm-opt" + export LOPTS=${LOPTS:-"-g3 --no-wasm-opt -sASSERTIONS=1"} + fi else # DO NOT CHANGE COPTS - optimized wasm corruption fix export COPTS="-O2 -g3 --no-wasm-opt" - export LOPTS=${LOPTS:-"-Oz -g0 --closure=0 --closure-args=--externs=/tmp/externs.js -sASSERTIONS=0"} + export LOPTS=${LOPTS:-"-Os -g0 --closure=0 -sASSERTIONS=0"} fi fi export BUILD export BUILD_PATH=${PG_BUILD}/${BUILD} +export PG_EXTRA=${PG_BUILD}/extra-${BUILD} + + +# default to user writeable paths in /tmp/ . +DIST_ALL="${PGROOT}/bin ${PG_DIST} ${PG_DIST_EXT} ${PGL_DIST_JS} ${PGL_BUILD_DUMPS} ${PGL_BUILD_NATIVE}" +DIST_ALL="$DIST_ALL ${PGL_DIST_LINK} ${PGL_DIST_NATIVE} ${PGL_DIST_WEB} ${PGL_DIST_C}" +DIST_ALL="$DIST_ALL ${PG_EXTRA}" + +if mkdir -p $DIST_ALL +then + echo "checking for valid prefix ${PGROOT} ${PG_DIST}" +else + sudo mkdir -p $DIST_ALL + sudo chown $(whoami) -R $DIST_ALL +fi + + + + + export PGDATA=${PGROOT}/base -export PGPATCH=${WORKSPACE}/patches + chmod +x ${PORTABLE}/*.sh [ -d ${PORTABLE}/extra ] && ${PORTABLE}/extra/*.sh @@ -89,15 +122,6 @@ chmod +x ${PORTABLE}/*.sh EOE=true -# default to user writeable paths in /tmp/ . -if mkdir -p ${PGROOT} ${PG_DIST} ${PG_DIST_EXT} ${PGL_DIST_JS} ${PGL_DIST_WEB} -then - echo "checking for valid prefix ${PGROOT} ${PG_DIST}" -else - sudo mkdir -p ${PGROOT} ${PGROOT}/bin ${PG_DIST} ${PG_DIST_EXT} ${PGL_DIST_WEB} - sudo chown $(whoami) -R ${PGROOT} ${PG_DIST} -fi - # TODO: also handle PGPASSFILE hostname:port:database:username:password # https://www.postgresql.org/docs/devel/libpq-pgpass.html export CRED="-U $PGUSER --pwfile=${PGROOT}/password" @@ -132,126 +156,128 @@ System node/pnpm ( may interfer) : # setup compiler+node. emsdk provides node 20, recent enough for bun. # TODO: but may need to adjust $PATH with stock emsdk. -if ${WASI} -then - pushd ${SDKROOT} - . wasisdk/wasisdk_env.sh - popd - - if [ -f ${WASI_SYSROOT}/extra ] +pushd ${SDKROOT} + if ${WASI} then - echo -n + . wasisdk/wasisdk_env.sh else - pushd ${WASI_SYSROOT} - VMLABS="https://github.com/vmware-labs/webassembly-language-runtimes/releases/download" - wget -q "${VMLABS}/libs%2Flibpng%2F1.6.39%2B20230629-ccb4cb0/libpng-1.6.39-wasi-sdk-20.0.tar.gz" -O-| tar xfz - - wget -q "${VMLABS}/libs%2Fzlib%2F1.2.13%2B20230623-2993864/libz-1.2.13-wasi-sdk-20.0.tar.gz" -O-| tar xfz - - wget -q "${VMLABS}/libs%2Fsqlite%2F3.42.0%2B20230623-2993864/libsqlite-3.42.0-wasi-sdk-20.0.tar.gz" -O-| tar xfz - - wget -q "${VMLABS}/libs%2Flibxml2%2F2.11.4%2B20230623-2993864/libxml2-2.11.4-wasi-sdk-20.0.tar.gz" -O-| tar xfz - - wget -q "${VMLABS}/libs%2Fbzip2%2F1.0.8%2B20230623-2993864/libbzip2-1.0.8-wasi-sdk-20.0.tar.gz" -O-| tar xfz - - wget -q "${VMLABS}/libs%2Flibuuid%2F1.0.3%2B20230623-2993864/libuuid-1.0.3-wasi-sdk-20.0.tar.gz" -O-| tar xfz - - popd - touch ${WASI_SYSROOT}/extra - fi - - # wasi does not use -sGLOBAL_BASE - CC_PGLITE="-DCMA_MB=${CMA_MB}" + if which emcc + then + echo "emcc found in PATH=$PATH" + else + . ${SDKROOT}/wasm32-bi-emscripten-shell.sh + fi -else - if which emcc - then - echo "emcc found in PATH=$PATH" - else if ${PORTABLE}/sdk.sh then echo "$PORTABLE : sdk check passed (emscripten)" else echo emsdk failed - exit 150 + exit 176 fi - . ${SDKROOT}/wasm32-bi-emscripten-shell.sh - fi - export PG_LINK=${PG_LINK:-$(which emcc)} - echo " + export PG_LINK=${PG_LINK:-$(which emcc)} - Using provided emsdk from $(which emcc) - Using PG_LINK=$PG_LINK as linker + echo " - node : $(which node) $($(which node) -v) - PNPM : $(which pnpm) + Using provided emsdk from $(which emcc) + Using PG_LINK=$PG_LINK as linker + node : $(which node) $($(which node) -v) + PNPM : $(which pnpm) -" - # custom code for node/web builds that modify pg main/tools behaviour - # this used by both node/linkweb build stages + " + fi +popd - # pass the "kernel" contiguous memory zone size to the C compiler. - CC_PGLITE="-DCMA_MB=${CMA_MB}" +# used for not makefile (manual linking and pgl_main) +# pass the "kernel" contiguous memory zone size to the C compiler with CMA_MB which will be multiplied by 1024x1024 in +# preprocessed source. +# nb: wasi does not use -sGLOBAL_BASE +export CC_PGLITE="-DPYDK=1 -DPG_PREFIX=${PGROOT} -I${PGROOT}/include -DCMA_MB=${CMA_MB}" -fi -# also used for non make (linking and pgl_main) -export CC_PGLITE="-DPYDK=1 -DPG_PREFIX=${PGROOT} -I${PGROOT}/include ${CC_PGLITE}" +echo " + ---------------------------------------- +" +env|grep PG |grep -v BUILD +echo +env|grep BUILD|grep -v PG +echo +env|grep WA +echo +env|grep PY +echo " + ---------------------------------------- +PATH=${PATH} +wasmtime=$(which wasmtime) + ---------------------------------------- +" -# ========================= symbol extractor ============================ -OBJDUMP=${OBJDUMP:-true} -if $OBJDUMP + +# ========================= symbol extractor ============================ +if [ -f $PGROOT/bin/wasm-objdump ] then - if [ -f $PGROOT/bin/wasm-objdump ] + echo "wasm-objdump found" +else + WRAPPER=$(which wasm-objdump) + WASIFILE=$(realpath ${WRAPPER}.wasi) + if $WRAPPER -h $WASIFILE | grep -q 'file format wasm 0x1' then - echo "wasm-objdump found" + mkdir -p $PGROOT/bin/ + if cp -f $WRAPPER $WASIFILE $PGROOT/bin/ + then + echo "wasm-objdump found and working, and copied to $PGROOT/bin/" + else + OBJDUMP=false + fi else - WRAPPER=$(which wasm-objdump) - WASIFILE=$(realpath ${WRAPPER}.wasi) + echo " + WARNING: $(which wasm-objdump) not working properly, trying alternate syntax + +" + cat > $WRAPPER < /proc/self/fd/2 +$(which wasmtime) --env PYTHONDONTWRITEBYTECODE=1 --dir / \$WASM \$@ +END + chmod +x $WRAPPER + if $WRAPPER -h $WASIFILE | grep -q 'file format wasm 0x1' then mkdir -p $PGROOT/bin/ if cp -f $WRAPPER $WASIFILE $PGROOT/bin/ then - echo "wasm-objdump found and working, and copied to $PGROOT/bin/" - else - OBJDUMP=false + echo "wasm-objdump fixed and working, copied to $PGROOT/bin/" fi else echo " - ERROR: $(which wasm-objdump) is not working properly ( is wasmtime ok ? ) + ERROR: $(which wasm-objdump) not working properly ( is wasmtime ok ? ) - " - OBJDUMP=false +" + exit 262 fi fi -else - echo " - - WARNING: OBJDUMP disabled, some newer or complex extensions may not load properly - - -" fi -if $OBJDUMP -then - mkdir -p patches/imports patches/imports.pgcore -else - mkdir -p patches/imports - touch patches/imports/plpgsql - echo " - - WARNING: wasm-objdump not found or OBJDUMP disabled, some extensions may not load properly - - -" -fi - -export OBJDUMP - # ========================= pg core configuration ============================ @@ -301,7 +327,7 @@ END [ -f $dest/pg_debug.h ] || cp ${PG_DEBUG_HEADER} $dest/ done - # store all pg options that have impact on cmd line initdb/boot + # store all options that have impact on cmd line initdb/boot compile+link cat > ${PGROOT}/pgopts.sh <&1 >/dev/null - then - echo " - Building contrib extension : $ext : end -" - else - echo " - - Extension $ext from $extdir failed to build - -" - exit 216 - fi - popd - - python3 ${PORTABLE}/pack_extension.py 2>&1 >/dev/null - - fi - fi - done - - -fi - -echo " - - Extensions distribution folder : ${PG_DIST_EXT} +cd ${WORKSPACE} +. ./wasm-build/build-ext.sh +# =========================================================================== +# =========================================================================== -" # only build extra when targeting pglite-wasm . +rm -f pglite-link.sh -if [ -f ${WORKSPACE}/pglite-${PG_BRANCH}/build.sh ] +if [ -f ${WORKSPACE}/pglite-${PG_BRANCH}/build.sh ] then if $WASI then echo " - * WASI build : skipping extra extensions and FS + * WASI build : skipping FS building " else - if echo " $*"|grep -q " extra" - then - for extra_ext in ${EXTRA_EXT:-"vector"} - do - if $CI - then - #if [ -d $PREFIX/include/X11 ] - if true - then - echo -n - else - # install EXTRA sdk - . /etc/lsb-release - DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}" - CIVER=${CIVER:-$DISTRIB} - SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4 - echo "Installing $SDK_URL" - curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null - chmod +x ./extra/*.sh - fi - fi - echo "======================= ${extra_ext} : $(pwd) ===================" - - ./extra/${extra_ext}.sh || exit 522 - - python3 ${PORTABLE}/pack_extension.py - done - fi - # this is for initial emscripten MEMFS export PGPRELOAD="\ --preload-file ${PGROOT}/share/postgresql@${PGROOT}/share/postgresql \ @@ -540,52 +464,57 @@ then echo " * building + linking pglite-wasm (initdb/loop/transport/repl/backend) " - if ${WORKSPACE}/pglite-${PG_BRANCH}/build.sh + + if $WASI then - if $WASI - then - echo "TODO: wasi pack/tests" - else - cat > pglite-link.sh < pglite-link.sh <&1 >/dev/null + then + echo " + Building contrib extension : $ext : end + " + else + echo " + + Extension $ext from $extdir failed to build + + " + exit 216 + fi + popd + + python3 ${PORTABLE}/pack_extension.py 2>&1 >/dev/null + + fi + fi + done + +fi + + +echo " + + Extensions distribution folder : ${PG_DIST_EXT} + + +" + +# if $CI +# then +# #if [ -d $PREFIX/include/X11 ] +# if true +# then +# echo -n +# else +# # install EXTRA sdk +# . /etc/lsb-release +# DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}" +# CIVER=${CIVER:-$DISTRIB} +# SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4 +# echo "Installing extra lib from $SDK_URL" +# curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null +# chmod +x ./extra/*.sh +# fi +# fi + +if [ -f ${PG_BUILD_DUMPS}/dump.vector ] +then + echo " + + * NOT rebuilding extra extensions ( found ${PG_BUILD_DUMPS}/dump.vector ) + +" +else + + for extra_ext in ./extra/*.sh + do + LOG=$PG_DIST_EXT/$(basename ${extra_ext}).log + echo "==== ${extra_ext} : $LOG ====" + + ${extra_ext} > $LOG || exit 112 + + python3 wasm-build/pack_extension.py + done + + +fi diff --git a/wasm-build/build-pgcore.sh b/wasm-build/build-pgcore.sh index ace44bb3b1b9b..f395478088a1d 100755 --- a/wasm-build/build-pgcore.sh +++ b/wasm-build/build-pgcore.sh @@ -40,7 +40,7 @@ Fatal: failed to apply patch : $one fi done touch postgresql-${PG_BRANCH}.patched - popd + popd # postgresql-${PG_BRANCH} fi # either a submodule dir or a symlink. @@ -311,8 +311,12 @@ END # same script handle emcc and wasi - cat > pg-make.sh < pg-make.sh + cat /tmp/portable.opts >> pg-make.sh + cat >> pg-make.sh < ${PGROOT}/pg.${BUILD}.installed + find . -type f | grep -v plpgsql > ${PGROOT}/pg.${BUILD}.installed popd - goback=$(pwd) - popd - python3 wasm-build/pack_extension.py builtin - pushd $goback + pushd ${WORKSPACE} - pushd ${PGROOT} - find . -type f > ${PGROOT}/pg.${BUILD}.installed - popd + python3 wasm-build/pack_extension.py builtin + + pushd ${PGROOT} + find . -type f > ${PGROOT}/pg.${BUILD}.installed + popd else cat /tmp/install.log diff --git a/wasm-build/build-with-docker.sh b/wasm-build/build-with-docker.sh index fb90c67ac0b3c..f6b20e5839f5e 100755 --- a/wasm-build/build-with-docker.sh +++ b/wasm-build/build-with-docker.sh @@ -23,7 +23,7 @@ source .buildconfig cat .buildconfig -mkdir -p dist/pglite dist/extensions-emsdk +mkdir -p dist/pglite dist/extensions-emsdk ${WORKSPACE}/postgres-pglite/tmp/sdk/build ${WORKSPACE}/postgres-pglite/tmp/pglite if echo -n $@|grep -q it$ then @@ -37,6 +37,8 @@ docker run $@ \ --env-file .buildconfig \ -e DEBUG=${DEBUG:-false} \ --workdir=${DOCKER_WORKSPACE} \ + -v ${WORKSPACE}/postgres-pglite/tmp/pglite:/tmp/pglite:rw \ + -v ${WORKSPACE}/postgres-pglite/tmp/sdk/build:/tmp/sdk/build:rw \ -v ${WORKSPACE}/postgres-pglite:${DOCKER_WORKSPACE}:rw \ -v ${WORKSPACE}/postgres-pglite/dist:/tmp/sdk/dist:rw \ $IMG_NAME:$IMG_TAG \ diff --git a/wasm-build/extension.sh b/wasm-build/extension.sh new file mode 100755 index 0000000000000..db35aee72fa3d --- /dev/null +++ b/wasm-build/extension.sh @@ -0,0 +1,28 @@ +# this is to be sourced by extra/*sh +export PG_EXTRA=${PG_EXTRA:-build/extra} +mkdir -p $PG_EXTRA + +if $WASI +then + if which wasi-c + then + echo -n + else + reset + . ${SDKROOT:-/tmp/sdk}/wasm32-wasi-shell.sh + fi +else + + if which emcc + then + echo -n + else + reset + . ${SDKROOT:-/tmp/sdk}/wasm32-bi-emscripten-shell.sh + fi +fi + +export PGROOT=${PGROOT:-/tmp/pglite} +export PATH=${PGROOT}/bin:$PATH +. ${PGROOT}/pgopts.sh + diff --git a/wasm-build/linkexport.sh b/wasm-build/linkexport.sh index c0e2769e155d7..50506fdf7b1a0 100755 --- a/wasm-build/linkexport.sh +++ b/wasm-build/linkexport.sh @@ -1,40 +1,40 @@ # this only runs when wasm-objdump is working and OBJDUMP not set to false -echo "============= link export : begin ===============" +mkdir -p ${PGL_DIST_LINK}/exports ${PGL_DIST_LINK}/imports ${PG_BUILD_DUMPS} -# -O0 -sEXPORT_ALL should make all symbols visible without any mangling. -COPT="-O0 -g3" \ - emcc \ - $EMCC_WEB -fPIC -sMAIN_MODULE=1 -sEXPORT_ALL -sERROR_ON_UNDEFINED_SYMBOLS -sASSERTIONS=0 \ - -DPREFIX=${PGROOT} -lnodefs.js -lidbfs.js \ - -sEXPORTED_RUNTIME_METHODS=FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,ccall,cwrap,callMain \ - $PGPRELOAD \ - -o postgres.html $PG_O $PG_L || exit 14 +echo "============= link export to ${PGL_DIST_LINK}/exports : begin ===============" -echo "FULL:" > ${WORKSPACE}/build/sizes.log -du -hs postgres.wasm >> ${WORKSPACE}/build/sizes.log -echo >> ${WORKSPACE}/build/sizes.log +echo "FULL:" > ${PGL_DIST_LINK}/sizes.log +du -hs ${PGL_DIST_JS}/pglite-js.* >> ${PGL_DIST_LINK}/sizes.log +echo >> ${PGL_DIST_LINK}/sizes.log -echo "getting wasm exports lists" -wasm-objdump -x $(realpath postgres.wasm) > ${WORKSPACE}/patches/exports/pgcore.wasm-objdump +echo " + * getting wasm exports lists +" + +pushd $(realpath ${PGL_DIST_JS}) + wasmtime --dir / --dir $(pwd)::. -- $(which wasm-objdump).wasi -x pglite-js.wasm > ${PG_BUILD_DUMPS}/pgcore.wasm-objdump +popd -pushd ${WORKSPACE} - echo "getting postgres exports lists" - cat $(find build/postgres -type f |grep /exports) \ +pushd ${PGL_DIST_LINK} + echo " + * getting postgres exports lists from ${BUILD_PATH} +" + cat $(find ${BUILD_PATH} -type f |grep /exports|grep -v /interfaces/libpq/) \ | grep -v ^\ local \ | grep -v ^{\ global \ - | sort | uniq > ${WORKSPACE}/patches/exports/pgcore.exports + | sort | uniq > ${PGL_DIST_LINK}/exports/pgcore.exports echo " - Merging wasm pg core symbols and postgres exports lists - into ${WORKSPACE} patches/exports/pgcore + * Merging wasm pg core symbols and postgres exports lists + into ${PGL_DIST_LINK}/exports/pgcore " - OBJDUMP=patches/exports/pgcore.wasm-objdump \ - PGDUMP=patches/exports/pgcore.exports \ - python3 wasm-build/getsyms.py exports > patches/exports/pgcore + OBJDUMP=${PG_BUILD_DUMPS}/pgcore.wasm-objdump \ + PGDUMP=${PGL_DIST_LINK}/exports/pgcore.exports \ + python3 ${WORKSPACE}/wasm-build/getsyms.py exports > ${PGL_DIST_LINK}/exports/pgcore popd echo "============= link export : end ===============" diff --git a/wasm-build/linkimports.sh b/wasm-build/linkimports.sh index 66200bcf00127..4ab35b48d5a76 100755 --- a/wasm-build/linkimports.sh +++ b/wasm-build/linkimports.sh @@ -5,17 +5,18 @@ echo "============= link imports : begin ===============" # extract own pg lib requirements +mkdir -p ${PGL_DIST_LINK}/imports pushd ${WORKSPACE} - > patches/imports/pgcore + > ${PGL_DIST_LINK}/imports/pgcore for extra_pg_so in $(find $PGROOT/lib/postgresql/|grep \.so$) do - SOBASE=patches/imports.pgcore/$(basename $extra_pg_so .so) + SOBASE=${PG_BUILD_DUMPS}/$(basename $extra_pg_so .so) wasm-objdump -x $(realpath $extra_pg_so) > $SOBASE.wasm-objdump OBJDUMP=$SOBASE.wasm-objdump \ - PGDUMP=patches/exports/pgcore.exports \ - python3 wasm-build/getsyms.py imports >> patches/imports/pgcore + PGDUMP=${PGL_DIST_LINK}/exports/pgcore.exports \ + python3 wasm-build/getsyms.py imports >> ${PGL_DIST_LINK}/imports/pgcore done popd @@ -26,13 +27,13 @@ popd #_emscripten_copy_to_end # copyFrom,copyTo,copyToEnd - cat ${WORKSPACE}/patches/imports/* | sort | uniq > /tmp/symbols + cat ${PGL_DIST_LINK}/imports/* | sort | uniq > /tmp/symbols echo "Requesting $(wc -l /tmp/symbols) symbols from pg core for PGlite extensions" - python3 < ${WORKSPACE}/patches/exports/pglite + python3 < ${PGL_DIST_LINK}/exports/pglite import sys import os @@ -41,7 +42,7 @@ def dbg(*argv, **kw): kw.setdefault('file',sys.stderr) return print(*argv,**kw) -with open("${WORKSPACE}/patches/exports/pgcore", "r") as file: +with open("${PGL_DIST_LINK}/exports/pgcore", "r") as file: exports = set(map(str.strip, file.readlines())) with open("/tmp/symbols", "r") as file: @@ -49,8 +50,25 @@ with open("/tmp/symbols", "r") as file: matches = list( imports.intersection(exports) ) + # ? for sym in """ + +_clear_error +_get_buffer_addr +_get_buffer_size +_get_channel +_interactive_one +_interactive_read +_interactive_write +_pgl_backend +_pgl_closed +_pgl_initdb +_pgl_shutdown +_use_wire + +_main + _ErrorContext _check_function_bodies _clock_gettime @@ -58,13 +76,8 @@ _CurrentMemoryContext ___cxa_throw _error_context_stack _getenv -_interactive_one -_interactive_read -_interactive_write _lowerstr -_main -_pg_initdb -_pg_shutdown + _readstoplist _searchstoplist _setenv @@ -78,10 +91,56 @@ _TopMemoryContext matches.sort() -for sym in matches: - print(sym) +for sym in """ +_PQcancelCreate +_PQcancelErrorMessage +_PQcancelFinish +_PQcancelPoll +_PQcancelSocket +_PQcancelStart +_PQclear +_PQcmdStatus +_PQconnectPoll +_PQconnectStartParams +_PQconnectionUsedPassword +_PQconsumeInput +_PQerrorMessage +_PQfinish +_PQgetResult +_PQgetisnull +_PQgetvalue +_PQisBusy +_PQnfields +_PQntuples +_PQresultErrorField +_PQresultStatus +_PQsendQuery +_PQserverVersion +_PQsetSingleRowMode +_PQsocket +_PQstatus +_pgresStatus + +_PQbackendPID +_PQconninfo +_PQconninfoFree +_PQconninfoParse +_PQendcopy +_PQescapeIdentifier +""".splitlines(): + if sym and sym in matches: + dbg(f"\t* Removed symbol '{sym}'") + matches.remove(sym) + +# matches.append('') +if not '_getTempRet0' in matches: + matches.append('_getTempRet0') +if not 'getTempRet0' in matches: + matches.append('getTempRet0') +for sym in matches: + print(sym) dbg(f""" exports {len(exports)} diff --git a/wasm-build/pack_extension.py b/wasm-build/pack_extension.py index 2084f3c3becc5..2a8d89a5be673 100644 --- a/wasm-build/pack_extension.py +++ b/wasm-build/pack_extension.py @@ -2,7 +2,7 @@ # TODO: use this file for merging symbols too. -# use recorded file list in ${PGROOT}/pg.${BUILD}.installed +# use recorded file list in ${PGROOT}/pg.installed # get other files into a tarball, find a .so and named everything after it @@ -12,11 +12,9 @@ import sys from pathlib import Path - class Error(Exception): pass - def gather(root: Path, *kw): for current, dirnames, filenames in os.walk(root): @@ -25,33 +23,45 @@ def gather(root: Path, *kw): # print(rel, len(dirnames), len(filenames)) yield rel, filenames +DIRS=[] +def pushd(d): + global DIRS + DIRS.append(os.getcwd()) + os.chdir(d) + return 1 + +def popd(): + global DIRS + os.chdir( DIRS.pop() ) + -def is_extension(path: Path, fullpath: Path): - global EXTNAME, SYMBOLS, PGPATCH, PGROOT +def is_extension(path:Path, fullpath:Path): + global EXTNAME, SYMBOLS, PGL_DIST_LINK, PGROOT asp = path.as_posix() # check .so - if asp.startswith("/lib/postgresql/"): + if asp.startswith('/lib/postgresql/'): if path.suffix == ".so": EXTNAME = path.stem - if os.environ.get("OBJDUMP", ""): - os.system(f"wasm-objdump -x {fullpath} 2>/dev/null > {PGROOT}/dumps/dump.{EXTNAME}") - os.system( - f"OBJDUMP={PGROOT}/dumps/dump.{EXTNAME} python3 wasm-build/getsyms.py imports 2>/dev/null > {PGPATCH}/imports/{EXTNAME}" - ) - with open(f"{PGPATCH}/imports/{EXTNAME}", "r") as f: - SYMBOLS = f.readlines() + dumpcmd = f"{PGROOT}/bin/wasm-objdump -x {fullpath} > {PG_BUILD_DUMPS}/dump.{EXTNAME} 2>/dev/null " + os.system(dumpcmd) + + os.system(f"OBJDUMP={PG_BUILD_DUMPS}/dump.{EXTNAME} python3 wasm-build/getsyms.py imports > {PGL_DIST_LINK}/imports/{EXTNAME}") + with open(f"{PGL_DIST_LINK}/imports/{EXTNAME}","r") as f: + SYMBOLS=f.readlines() return True # rpath - if asp.startswith("/lib/"): + if asp.startswith('/lib/'): return True - if asp.startswith("/share/postgresql/extension"): + if asp.startswith('/share/postgresql/extension'): return True + + async def archive(target_folder): global INSTALLED, PACKLIST @@ -59,66 +69,64 @@ async def archive(target_folder): for folder, filenames in gather(target_folder): walked.append([folder, filenames]) + for folder, filenames in walked: for filename in filenames: test = Path(folder) / Path(filename) asp = test.as_posix() - if (PGROOT / test).is_symlink(): + if (PGROOT/test).is_symlink(): print("SYMLINK:", test) continue if asp not in INSTALLED: - if asp.startswith("/sdk/"): - continue - - if asp.startswith("/base/"): + if asp.startswith('/sdk/'): continue - if asp.startswith("/dumps/"): - continue - - if asp.startswith("/dump."): + if asp.startswith('/base/'): continue fp = PGROOT / asp[1:] if fp.is_symlink(): + print("SYMLINK:", fp) continue + if is_extension(test, fp): - # print(f"{EXTNAME=}", test ) - PACKLIST.append([fp, test]) + print(f"{EXTNAME=}", test, fp) + PACKLIST.append( [fp, test] ) else: print("custom:", test) DIST = Path(os.environ.get("PG_DIST_EXT", "/tmp/sdk/dist/extensions-emsdk")) -PGROOT = Path(os.environ.get("PGROOT", "/tmp/pglite")) -PGPATCH = Path(os.environ.get("PGPATCH", PGROOT)) +BUILD=os.environ.get('BUILD','emscripten') +PGROOT=Path(os.environ.get('PGROOT',"/tmp/pglite")) +PGL_DIST_LINK=Path(os.environ.get('PGL_DIST_LINK', "/tmp/sdk/dist/pglite-link")) +PG_BUILD_DUMPS=Path(os.environ.get('PG_BUILD_DUMPS', "/tmp/sdk/build/dumps")) INSTALLED = [] EXTNAME = "" PACKLIST = [] -SYMBOLS = [] +SYMBOLS=[] PREINST = "/plpgsql" IS_PREINST = PREINST in sys.argv -for line in open(PGROOT / f"pg.{os.environ.get('BUILD','emscripten')}.installed" ).readlines(): +for line in open(PGROOT / f"pg.{BUILD}.installed" ).readlines(): asp = Path(line[1:].strip()).as_posix() if IS_PREINST: - if asp.find(PREINST) > 0: + if asp.find(PREINST)>0: continue - INSTALLED.append(asp) + INSTALLED.append( asp ) -print("=" * 80) -asyncio.run(archive(PGROOT)) -print("=" * 80) +print("="*80) +asyncio.run( archive(PGROOT) ) +print("="*80) if not EXTNAME: print("MAYBE ERROR: no new installed extension found, is it builtin ?") sys.exit(0) -print( - f""" +print(f""" PG installed in : {PGROOT=} @@ -126,21 +134,20 @@ async def archive(target_folder): -""" -) +""") swd = os.getcwd() if (not IS_PREINST) and ("builtin" not in sys.argv): if len(PACKLIST): - os.chdir(PGROOT) - with tarfile.open(DIST / f"{EXTNAME}.tar", "w:") as tar: - for fp, fn in PACKLIST: - print(f"{EXTNAME} : {fp} => {fn}") - tar.add(fn.as_posix()[1:]) - # if "builtin" not in sys.argv: - os.remove(fp) - os.chdir(swd) + if pushd(PGROOT): + with tarfile.open(DIST / f"{EXTNAME}.tar", "w:") as tar: + for fp, fn in PACKLIST: + print(f"{EXTNAME} : {fp} => {fn}") + tar.add(fn.as_posix()[1:]) + # if "builtin" not in sys.argv: + os.remove(fp) + popd() else: print(f"Nothing found to pack for {EXTNAME}, did you 'make install' ?") else: diff --git a/wasm-build/sdk.sh b/wasm-build/sdk.sh index 0401c546b3515..f4c7c15f70908 100755 --- a/wasm-build/sdk.sh +++ b/wasm-build/sdk.sh @@ -1,5 +1,31 @@ #!/bin/bash -mkdir -p /tmp/sdk + +SDKROOT=${SDKROOT:-/tmp/sdk} +mkdir -p ${SDKROOT} + +if grep -q __emscripten_tempret_get ${SDKROOT}/emsdk/upstream/emscripten/src/library_dylink.js +then + echo -n +else + pushd ${SDKROOT}/emsdk + patch -p1 < { + resolved ||= resolveSymbol(prop); + if (!resolved) { ++ if (prop==='getTempRet0') ++ return __emscripten_tempret_get(...args); + throw new Error(\`Dynamic linking error: cannot resolve symbol \${prop}\`); + } + return resolved(...args); +END + popd +fi + + + if ${NO_SDK_CHECK:-false} then exit 0 @@ -129,3 +155,35 @@ fi cat $SDKROOT/VERSION + +if $WASI +then + # always install wasmtime because wasm-objdump needs it. + if [ -f ${SDKROOT}/devices/$(arch)/usr/bin/wasmtime ] + then + echo "keeping installed wasmtime and wasi binaries" + else +# TODO: window only has a zip archive, better use wasmtime-py instead. + + wget https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.0/wasmtime-v33.0.0-$(arch)-${PLATFORM}.tar.xz \ + -O-|xzcat|tar xfv - + mv -vf $(find wasmtime*|grep /wasmtime$) ${SDKROOT}/devices/$(arch)/usr/bin + fi + + if [ -f ${WASI_SYSROOT}/extra ] + then + echo -n + else + pushd ${WASI_SYSROOT} + VMLABS="https://github.com/vmware-labs/webassembly-language-runtimes/releases/download" + wget -q "${VMLABS}/libs%2Flibpng%2F1.6.39%2B20230629-ccb4cb0/libpng-1.6.39-wasi-sdk-20.0.tar.gz" -O-| tar xfz - + wget -q "${VMLABS}/libs%2Fzlib%2F1.2.13%2B20230623-2993864/libz-1.2.13-wasi-sdk-20.0.tar.gz" -O-| tar xfz - + wget -q "${VMLABS}/libs%2Fsqlite%2F3.42.0%2B20230623-2993864/libsqlite-3.42.0-wasi-sdk-20.0.tar.gz" -O-| tar xfz - + wget -q "${VMLABS}/libs%2Flibxml2%2F2.11.4%2B20230623-2993864/libxml2-2.11.4-wasi-sdk-20.0.tar.gz" -O-| tar xfz - + wget -q "${VMLABS}/libs%2Fbzip2%2F1.0.8%2B20230623-2993864/libbzip2-1.0.8-wasi-sdk-20.0.tar.gz" -O-| tar xfz - + wget -q "${VMLABS}/libs%2Flibuuid%2F1.0.3%2B20230623-2993864/libuuid-1.0.3-wasi-sdk-20.0.tar.gz" -O-| tar xfz - + popd + touch ${WASI_SYSROOT}/extra + fi +fi +