freebsd · bapt · Nov 11, 2024 · Nov 11, 2024 · Nov 10, 2024 · Nov 11, 2024
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -0,0 +1,179 @@
+name: build
+# GitHub action to compile pkg on ubuntu-24.04, ubuntu-latest/22.04 (amd64) and macos-latest (arm64)
+#  * set-up prerequisites
+#  * configure && make && make check && make install
+#  * upload installed binaries as well as kyua reports as build artefact
+#
+# We run in a matrix with os/number of workers, to demonstrate the errors on MacOSX when running 
+# pkg repo multi-threaded.
+# Intended to increase awareness in issues with MacOS builds of pkg.
+
+
+on:
+    pull_request:
+        branches:
+            - main
+    push:
+    workflow_dispatch:
+
+permissions:
+    contents: read
+
+jobs:
+    build:
+        name: build ${{ matrix.workers-count }} ${{ matrix.build-os }} ${{ matrix.compiler }}
+        runs-on: "${{ matrix.build-os }}"
+        strategy:
+            fail-fast: false
+            matrix:
+                build-os:
+                - ubuntu-24.04
+                - ubuntu-latest
+                - macos-latest
+                workers-count:
+                - 0
+                - 1
+                exclude:
+                  - build-os: ubuntu-24.04
+                    workers-count: 1
+                  - build-os: ubuntu-latest
+                    workers-count: 1
+                include:
+                - build-os: macos-latest
+                  compiler: clang-19
+                  pkgs: 
+                  - pkg-config
+                  - libarchive 
+                  - llvm@19
+                  llvm-bindir: /opt/homebrew/opt/llvm@19/bin
+                - build-os: ubuntu-24.04
+                  compiler: clang-18
+                  pkgs: 
+                  - clang-18
+                  - libcurl4-openssl-dev
+                  - libsqlite3-dev
+                  - libbsd-dev
+                  - libarchive-tools
+                  - libarchive-dev
+                  - libssl-dev
+                  - liblzma-dev
+                  - liblua5.2-dev
+                  - liblzo2-dev
+                  - libattr1-dev
+                  - libacl1-dev
+                  - libatf-dev
+                  - kyua
+                  - atf-sh
+                  - build-essential
+                  - zlib1g-dev
+                  - libbz2-dev
+                  - python3
+                  - pkg-config
+                  - m4
+                  llvm-bindir: /usr/lib/llvm-18/bin
+
+                - build-os: ubuntu-latest 
+                  compiler: clang-15
+                  pkgs: 
+                  - clang-15
+                  - libcurl4-openssl-dev
+                  - libsqlite3-dev
+                  - libbsd-dev
+                  - libarchive-tools
+                  - libarchive-dev
+                  - libssl-dev
+                  - liblzma-dev
+                  - liblua5.2-dev
+                  - liblzo2-dev
+                  - libattr1-dev
+                  - libacl1-dev
+                  - libatf-dev
+                  - kyua
+                  - atf-sh
+                  - build-essential
+                  - zlib1g-dev
+                  - libbz2-dev
+                  - python3
+                  - pkg-config
+                  - m4
+                  llvm-bindir: /usr/lib/llvm-15/bin
+        steps:
+        - name: install packages (macOS)
+          if: runner.os == 'macOS'
+          run: |
+            # on MacOS we build with 
+            #   * latest clang@19 from brew (system provided clang lacks sanitizers)
+            #   * ld from system
+            #   * libarchive from brew (system provided libarchive has issues)
+            #   * openssl from brew
+            #   * libcurl from system
+            #
+
+            brew update --quiet || true
+            HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=Y brew install ${{ join(matrix.pkgs, ' ') }}
+
+            # kyua was kicked out of brew due to lack of activity
+            # we patch away the disabled line and install the last built binary version
+            curl https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/k/kyua.rb | sed 's/[[:space:]]*disable.*$//'  > kyua.rb
+            brew install --formula ./kyua.rb
+
+            echo PKG_CONFIG_PATH=$(brew --prefix libarchive)/lib/pkgconfig >> "${GITHUB_ENV}"
+        - name: install packages (Linux)
+          if: runner.os == 'Linux'
+          run: |
+            sudo apt-get update --quiet || true
+            sudo apt-get -yq --no-install-suggests --no-install-recommends install ${{ join(matrix.pkgs, ' ') }}
+        - uses: actions/checkout@v4
+          with:
+            path: src.pkg
+        - name: setup environment
+          run: |
+            echo "CC=${{ matrix.llvm-bindir }}/clang" >> "${GITHUB_ENV}"
+            echo "CXX=${{ matrix.llvm-bindir }}/clang++" >> "${GITHUB_ENV}"
+            echo "CPP=${{ matrix.llvm-bindir }}/clang-cpp" >> "${GITHUB_ENV}"
+            # this is miracolously not picked up by automake as the default
+            echo "CC_FOR_BUILD=${{ matrix.llvm-bindir }}/clang" >> "${GITHUB_ENV}"
+            echo "BUILD_PKG=${GITHUB_WORKSPACE}/build.pkg" >> "${GITHUB_ENV}"
+            echo "INST_PKG=${GITHUB_WORKSPACE}/inst.pkg" >> "${GITHUB_ENV}"
+
+        - name: build pkg
+          run: |
+            echo Building pkg
+            echo CC="${CC}"
+            echo CPP="${CPP}"
+            echo PKG_CONFIG_PATH="${PKG_CONFIG_PATH}"
+            echo BUILD_PKG="${BUILD_PKG}"
+            echo INST_PKG="${INST_PKG}"
+            mkdir -p "${BUILD_PKG}"
+            cd "${BUILD_PKG}"
+            ${GITHUB_WORKSPACE}/src.pkg/configure --prefix=${INST_PKG} --with-libarchive.pc --with-libcurl --with-openssl.pc
+            make
+
+        - name: test&instal pkg
+          run: |
+            echo Checking and  pkg
+            cd "${BUILD_PKG}"
+            if WORKERS_COUNT=${{ matrix.workers-count }} make check; then
+              echo "All mandatory checks passed"
+              kyua report
+              kyua report-html
+            else
+              kyua report --verbose
+              kyua report-html
+              exit 0
+            fi
+            make install
+
+        - name: tar build & test reports
+          run: |
+            test -d ${INST_PKG} && tar cvf pkg-${{ matrix.build-os }}-${{ matrix.compiler }}.tar -C ${INST_PKG} .
+            tar cvf pkg-${{ matrix.build-os }}-${{ matrix.compiler }}-report${{ matrix.workers-count }}.tar -C "${BUILD_PKG}/html" .
+
+        - name: archive build artefacts
+          uses: actions/upload-artifact@v4
+          with:
+              name: pkg-test${{ matrix.workers-count }}-${{ matrix.build-os }}-${{ matrix.compiler }}
+              path: pkg*.tar
+              compression-level: 0
+              retention-days: 10
+              overwrite: true
diff --git a/.gitignore b/.gitignore
@@ -14,6 +14,7 @@
 *.log
 *.gz
 *.pc
+*.dylib
 jimsh0
 tags
 TAGS

diff --git a/auto.def b/auto.def
@@ -101,6 +101,7 @@ cc-with { -libs { -llzma }} {
 	}
 }
 if {[string match *-freebsd* [get-define host]]} {
+	define pkg_freebsd
 	cc-with { -libs { -lmd }} {
 		if {![cc-check-functions SHA256_Data]} {
 			user-error "Unable to find libmd"
@@ -336,6 +337,9 @@ if {[pkg-config-init 0]} {
 		if {![pkg-config libcurl]} {
 			user-error "Unable to find libcurl"
 		}
+		user-notice LIBCURL_CFLAGS=[get-define PKG_LIBCURL_CFLAGS]
+		user-notice LIBCURL_LDLAGS=[get-define PKG_LIBCURL_LDFLAGS]
+		user-notice LIBCURL_LIBS=[get-define PKG_LIBCURL_LIBS]
 	}
 	if {[opt-bool with-libarchive.pc]} {
 		if {![pkg-config libarchive]} {
@@ -346,12 +350,18 @@ if {[pkg-config-init 0]} {
 				cc-check-functions archive_write_add_filter_zstd
 			}
 		}
+		user-notice LIBARCHIVE_CFLAGS=[get-define PKG_LIBARCHIVE_CFLAGS]
+		user-notice LIBARCHIVE_LDLAGS=[get-define PKG_LIBARCHIVE_LDFLAGS]
+		user-notice LIBARCHIVE_LIBS=[get-define PKG_LIBARCHIVE_LIBS]
 	}
 
 	if {[opt-bool with-openssl.pc]} {
 		if {![pkg-config openssl]} {
 			user-error "Unable to find openssl"
 		}
+		user-notice OPENSSL_CFLAGS=[get-define PKG_OPENSSL_CFLAGS]
+		user-notice OPENSSL_LDLAGS=[get-define PKG_OPENSSL_LDFLAGS]
+		user-notice OPENSSL_LIBS=[get-define PKG_OPENSSL_LIBS]
 	}
 }
 

diff --git a/libpkg/Makefile.autosetup b/libpkg/Makefile.autosetup
@@ -129,7 +129,12 @@ LOCAL_LDFLAGS+=	-L$(top_builddir)/external/libmachista -lmachista_pic \
 SRCS+=		pkg_macho.c
 @else
 SRCS+=		pkg_elf.c
-LOCAL_LDFLAGS+=	-Wl,--version-script=$(top_srcdir)/libpkg/libpkg.ver,--undefined-version \
+@if pkg_freebsd
+LOCAL_LDFLAGS+=	-Wl,--version-script=$(top_srcdir)/libpkg/libpkg.ver,--undefined-version 
+@else
+# --undefined-version is a FreeBSD ld option
+LOCAL_LDFLAGS+=	-Wl,--version-script=$(top_srcdir)/libpkg/libpkg.ver
+@endif
 @endif
 
 @if libelf-internal

diff --git a/libpkg/pkg_macho.c b/libpkg/pkg_macho.c
@@ -107,7 +107,7 @@ analyse_macho(struct pkg *pkg, const char *fpath,
 			xasprintf(&libname, "%s.%s", march->mat_install_name, ai->name);
 			pkg_addshlib_provided(pkg, libname);
 			is_shlib = true;
-		}
+
 
 		/* Now find all dependencies */
 		for (macho_loadcmd_t *cmd = march->mat_loadcmds; cmd != NULL; cmd = cmd->next) {
@@ -140,6 +140,7 @@ analyse_macho(struct pkg *pkg, const char *fpath,
 
 			action(actdata, pkg, fpath, libname, is_shlib);
 		}
+		}
 	}
 
 	return EPKG_OK;

diff --git a/libpkg/pkg_repo_create.c b/libpkg/pkg_repo_create.c
@@ -365,7 +365,11 @@ pkg_create_repo_thread(void *arg)
 			ucl_object_emit_streamline_add_object(te->ctx, o);
 			ucl_object_emit_fd(o, UCL_EMIT_JSON_COMPACT, te->mfd);
 			dprintf(te->mfd, "\n");
+#if defined(_DARWIN_C_SOURCE) || defined(__APPLE__)
+			fcntl(te->mfd, F_FULLFSYNC);
+#else
 			fdatasync(te->mfd);
+#endif
 			ucl_object_unref(o);
 
 			pthread_mutex_unlock(&te->flock);
@@ -397,7 +401,7 @@ pkg_create_repo_thread(void *arg)
 	return (NULL);
 }
 
-#ifdef __linux__
+#if defined (__linux__) || defined(_DARWIN_C_SOURCE) || defined (__APPLE__)
 typedef const FTSENT *FTSENTP;
 #else
 typedef const FTSENT *const FTSENTP;

diff --git a/src/Makefile.autosetup b/src/Makefile.autosetup
@@ -50,7 +50,7 @@ LIBPKGFLAT=	-L$(top_builddir)/libpkg -lpkg_flat
 OTHER_LIBS=	@EXTRA_LIBS@ -lm -pthread
 
 @if HAVE_PKG_LIBARCHIVE
-OTHER_LIBS+=	@PKG_LIBARCHIVE_LDFLAGS@ @PKG_LIBARCHIVE_LIBS_STATIC@
+OTHER_LIBS+=	@PKG_LIBARCHIVE_LDFLAGS@ @PKG_LIBARCHIVE_LIBS@
 LOCAL_CFLAGS+=	@PKG_LIBARCHIVE_CFLAGS@
 @else
 OTHER_LIBS+=	-larchive -lbz2 -lz -llzma @ZSTDLIB@
@@ -78,12 +78,17 @@ OTHER_LIBS+=	-lmd
 
 @if PKG_OPENSSL_LIBS
 LOCAL_CFLAGS+=	@PKG_OPENSSL_CFLAGS@
-LOCAL_LDFLAGS+=	@PKG_OPENSSL_LDFLAGS@
-OTHER_LIBS+=	@PKG_OPENSSL_LIBS@
+OTHER_LIBS+=	@PKG_OPENSSL_LDFLAGS@ @PKG_OPENSSL_LIBS@
 @else
 OTHER_LIBS+=	-lssl -lcrypto
 @endif
 
+@if PKG_LIBCURL_LIBS
+LOCAL_CFLAGS+=	@PKG_LIBCURL_CFLAGS@
+LOCAL_LDFLAGS+=	@PKG_LIBCURL_LDFLAGS@ 
+OTHER_LIBS+=	@PKG_LIBCURL_LIBS@
+@endif
+
 @if libmachista
 LOCAL_LDFLAGS=	$(LIBPKGFLAT) $(LIBS) $(OTHER_LIBS) -lresolv
 STATIC_LDFLAGS=	$(LIBPKGFLAT) $(LIBS) $(OTHER_LIBS) -lresolv

diff --git a/tests/Makefile.autosetup b/tests/Makefile.autosetup
@@ -119,7 +119,8 @@ LIBS=	\
 	@EXTRA_LIBS@
 
 @if HAVE_PKG_LIBARCHIVE
-LIBS+=	@PKG_LIBARCHIVE_LDFLAGS@ @PKG_LIBARCHIVE_LIBS_STATIC@
+CFLAGS+=	@PKG_LIBARCHIVE_CFLAGS@
+LIBS+=	@PKG_LIBARCHIVE_LDFLAGS@ @PKG_LIBARCHIVE_LIBS@
 @else
 LIBS+=	-L/usr/lib -larchive -lbz2 -lz -llzma
 @endif
@@ -136,6 +137,15 @@ OTHER_LIBS+=	-ldl
 @if HAVE_LIBRESOLV
 OTHER_LIBS+=	-lresolv
 @endif
+@if PKG_LIBCURL_LIBS
+CFLAGS+=	@PKG_LIBCURL_CFLAGS@
+LIBS+=	@PKG_LIBCURL_LDFLAGS@ @PKG_LIBCURL_LIBS@
+@endif
+@if PKG_OPENSSL_LIBS
+CFLAGS+=	@PKG_OPENSSL_CFLAGS@
+LIBS+=	@PKG_OPENSSL_LDFLAGS@ @PKG_OPENSSL_LIBS@
+@endif
+
 
 # Hack to determine we are on osx
 @if libmachista