feat: build using system deps (#33)

---------

Signed-off-by: Michele Dolfi <[email protected]>

.github/scripts/build_rhel.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e  # trigger failure on error - do not remove!
+set -x  # display command on output
+
+# Build the sdist
+poetry build -f sdist
+
+# Compile the wheel from sdist in centos stream
+
+docker build -f - . <<EOF
+    FROM quay.io/centos/centos:stream9
+    RUN dnf config-manager --set-enabled crb
+    # RUN dnf copr -y enable cheimes/deepsearch-glm rhel-9-x86_64
+    RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \
+       && dnf clean all
+    RUN dnf install -y --nodocs \
+            gcc gcc-c++ git make cmake pkgconfig glibc-devel \
+            python3.11 python3.11-pip python3.11-devel \
+            libjpeg-turbo-devel libpng-devel qpdf-devel json-devel utf8cpp-devel zlib-devel \
+            loguru-devel  \
+        && dnf clean all
+    
+    RUN mkdir /src
+    COPY ./dist/*.tar.gz /src/
+
+    RUN USE_SYSTEM_DEPS=ON pip3.11 install /src/docling_parse*.tar.gz \
+        && python3.11 -c 'from docling_parse.docling_parse import pdf_parser'
+    
+    COPY ./tests /src/tests
+    RUN cd /src \
+        && pip3.11 install pytest \
+        && pytest
+EOF

.github/workflows/ci.yml

@@ -21,4 +21,6 @@ jobs:
     uses: ./.github/workflows/checks.yml
   build-wheels:
     uses: ./.github/workflows/wheels.yml
+  rhel-build:
+    uses: ./.github/workflows/rhel.yml
 

.github/workflows/rhel.yml

@@ -0,0 +1,12 @@
+on:
+  workflow_call:
+
+jobs:
+  run-checks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-poetry
+      - name: Run build in docker
+        run: ./.github/scripts/build_rhel.sh
+        shell: bash

.gitignore

@@ -287,7 +287,6 @@ pyrightconfig.json
 [Ll]ib
 [Ll]ib64
 [Ll]ocal
-[Ss]cripts
 pyvenv.cfg
 pip-selfcheck.json
 

CMakeLists.txt

@@ -84,63 +84,73 @@ message(STATUS "cxx-flags   : " ${CMAKE_CXX_FLAGS})
 message(STATUS "cxx-standard: " ${CMAKE_CXX_STANDARD})
 
 # ************************
-# *** libraries        ***
+# *** dependencies     ***
 # ************************
 
-if(NOT EXISTS ${EXTERNALS_PREFIX_PATH})
-  execute_process(COMMAND mkdir ${EXTERNALS_PREFIX_PATH})
-  execute_process(COMMAND mkdir ${EXTERNALS_PREFIX_PATH}/bin)
-  execute_process(COMMAND mkdir ${EXTERNALS_PREFIX_PATH}/lib)
-  execute_process(COMMAND mkdir ${EXTERNALS_PREFIX_PATH}/include)
-  execute_process(COMMAND mkdir ${EXTERNALS_PREFIX_PATH}/resources)
+option(USE_SYSTEM_DEPS OFF "If enabled, the build will find and link to system dependencies, otherwise they are sourced from the original repos and compiled on the fly.")
+
+# prepare setup for building external packages
+if(NOT USE_SYSTEM_DEPS)
+    if(NOT EXISTS ${EXTERNALS_PREFIX_PATH})
+        file(MAKE_DIRECTORY ${EXTERNALS_PREFIX_PATH})
+        file(MAKE_DIRECTORY ${EXTERNALS_PREFIX_PATH}/bin)
+        file(MAKE_DIRECTORY ${EXTERNALS_PREFIX_PATH}/lib)
+        file(MAKE_DIRECTORY ${EXTERNALS_PREFIX_PATH}/include)
+        file(MAKE_DIRECTORY ${EXTERNALS_PREFIX_PATH}/resources)
+    endif()
 endif()
 
-include(cmake/subdirlist.cmake)
 
+# include dependencies
 include(cmake/extlib_json.cmake)
 include(cmake/extlib_utf8.git.cmake)
 include(cmake/extlib_jpeg.cmake)
-# include(cmake/extlib_qpdf_v10.cmake)
 include(cmake/extlib_qpdf_v11.cmake)
-# include(cmake/extlib_pybind11.cmake)
 
+# aggregate the targets created by the dependencies
+set(DEPENDENCIES qpdf jpeg utf8 json)
+
+
+# ************************
+# *** libraries        ***
+# ************************
+
+# define LIB_LINK and OS_DEPENDENCIES
 include(cmake/os_opts.cmake)
+list(APPEND DEPENDENCIES ${OS_DEPENDENCIES})
 
 # ***********************
 # ***  includes       ***
 # ***********************
 
-include_directories(${EXTERNALS_PREFIX_PATH}/include)
-
-subdirlist(EXTERNALS_INCLUDES ${EXTERNALS_PREFIX_PATH}/include)	
-include_directories(${EXTERNALS_INCLUDES})
-
-message(STATUS "subdirs: " ${SUBDIRS})
+# define subdirlist utility
+include(cmake/subdirlist.cmake)
 
 include_directories(${TOPLEVEL_PREFIX_PATH}/src)
 
 subdirlist(SUBDIRS ${TOPLEVEL_PREFIX_PATH}/src)
+message(STATUS "subdirs: " ${SUBDIRS})
 include_directories(${SUBDIRS})
 
 # **********************
 # ***  Executables   ***
 # **********************
 
 add_executable(parse.exe "${TOPLEVEL_PREFIX_PATH}/app/parse.cpp")
-
 set_property(TARGET parse.exe PROPERTY CXX_STANDARD 20)
-
-target_link_libraries(parse.exe ${LIB_LINK})
+add_dependencies(parse.exe ${DEPENDENCIES})
+target_include_directories(parse.exe INTERFACE ${DEPENDENCIES})
+target_link_libraries(parse.exe ${DEPENDENCIES} ${LIB_LINK})
 
 # **********************
 # ***  Libraries     ***
 # **********************
 
-add_library(parse STATIC "${TOPLEVEL_PREFIX_PATH}/app/parse.cpp")
-
-set_target_properties(parse PROPERTIES POSITION_INDEPENDENT_CODE ON)
-
-target_link_libraries(parse ${LIB_LINK})
+add_library(libparse STATIC "${TOPLEVEL_PREFIX_PATH}/app/parse.cpp")
+add_dependencies(libparse ${DEPENDENCIES})
+target_include_directories(libparse INTERFACE ${DEPENDENCIES})
+set_target_properties(libparse PROPERTIES POSITION_INDEPENDENT_CODE ON)
+target_link_libraries(libparse ${DEPENDENCIES} ${LIB_LINK})
 
 # ***************************
 # ***  Python-binding     ***
@@ -151,11 +161,10 @@ target_link_libraries(parse ${LIB_LINK})
 find_package(pybind11 CONFIG REQUIRED)
 
 pybind11_add_module(docling_parse "${TOPLEVEL_PREFIX_PATH}/app/pybind_parse.cpp")
-
-add_dependencies(docling_parse json utf8)  # we have to list the headers-only deps here
-target_compile_definitions(docling_parse PRIVATE VERSION_INFO=${EXAMPLE_VERSION_INFO})
-
-target_link_libraries(docling_parse PRIVATE parse)
+add_dependencies(docling_parse libparse)
+target_include_directories(docling_parse INTERFACE ${DEPENDENCIES})
+target_compile_definitions(docling_parse PRIVATE VERSION_INFO=${CMAKE_PROJECT_VERSION})
+target_link_libraries(docling_parse PRIVATE libparse)
 
 # *****************
 # ***  Install  ***

app/parse.cpp

@@ -12,7 +12,7 @@ Email: {taa, dol, cau, ttt}@zurich.ibm.com
 Date: 18/08/2017
 **********************************************************/
 
-#include <utf8/utf8.h>
+#include <utf8.h>
 
 #include "include_libs.h"
 

build.py

@@ -40,11 +40,15 @@ def run(cmd: List[str], cwd: str="./"):
 
 def build_local(num_threads: int):
 
+    USE_SYSTEM_DEPS = os.getenv("USE_SYSTEM_DEPS", "OFF")
+
+
     print("python prefix: ", sys.exec_prefix)
     print("python executable: ", sys.executable)
     config_cmd = [
         "cmake",
         "-B", f"{BUILD_DIR}",
+        f"-DUSE_SYSTEM_DEPS={USE_SYSTEM_DEPS}",
         f"-DPYTHON_EXECUTABLE={sys.executable}",
     ]
     config_cmd.extend(get_pybind11_cmake_args())

cmake/extlib_jpeg.cmake

@@ -1,36 +1,54 @@
 
 message(STATUS "entering in extlib_jpeg.cmake")
 
-include(ExternalProject)
-include(CMakeParseArguments)
+set(ext_name "jpeg")
 
-set(JPEG_URL https://github.com/libjpeg-turbo/libjpeg-turbo.git)
-set(JPEG_TAG 3.0.3)
+if(USE_SYSTEM_DEPS)
+    find_package(PkgConfig)
+    pkg_check_modules(libjpeg REQUIRED IMPORTED_TARGET libjpeg)
 
-ExternalProject_Add(extlib_jpeg
-    PREFIX extlib_jpeg
+    add_library(${ext_name} ALIAS PkgConfig::libjpeg)
+    #set_target_properties(${ext_name} PROPERTIES INTERFACE_LINK_LIBRARIES "${libjpeg_LIBRARIES}")
+    #set_target_properties(${ext_name} PROPERTIES INTERFACE_LINK_DIRECTORIES "${libjpeg_LIBRARY_DIRS}")
+    #set_target_properties(${ext_name} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${libjpeg_INCLUDEDIR}")
 
-    UPDATE_COMMAND ""
-    GIT_REPOSITORY ${JPEG_URL}
-    GIT_TAG ${JPEG_TAG}
+else()
+    include(ExternalProject)
+    include(CMakeParseArguments)
 
-    BUILD_ALWAYS OFF
+    set(JPEG_URL https://github.com/libjpeg-turbo/libjpeg-turbo.git)
+    set(JPEG_TAG 3.0.3)
 
-    INSTALL_DIR ${EXTERNALS_PREFIX_PATH}
+    ExternalProject_Add(extlib_jpeg
 
-    CMAKE_ARGS \\
-    -DCMAKE_POSITION_INDEPENDENT_CODE=ON \\
-    -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \\
-    -DCMAKE_C_FLAGS=${ENV_ARCHFLAGS} \\
-    -DARMV8_BUILD=ON \\
-    -DCMAKE_INSTALL_LIBDIR=${EXTERNALS_PREFIX_PATH}/lib \\
-    -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH}
+        PREFIX extlib_jpeg
 
-    BUILD_IN_SOURCE ON
-    LOG_DOWNLOAD ON
-    # LOG_BUILD ON
-)
+        UPDATE_COMMAND ""
+        GIT_REPOSITORY ${JPEG_URL}
+        GIT_TAG ${JPEG_TAG}
 
-add_library(jpeg STATIC IMPORTED)
-set_target_properties(jpeg PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libjpeg.a)
-add_dependencies(jpeg extlib_jpeg)
+        BUILD_ALWAYS OFF
+
+        INSTALL_DIR ${EXTERNALS_PREFIX_PATH}
+
+        CMAKE_ARGS \\
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON \\
+        -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \\
+        -DCMAKE_C_FLAGS=${ENV_ARCHFLAGS} \\
+        -DARMV8_BUILD=ON \\
+        -DCMAKE_INSTALL_LIBDIR=${EXTERNALS_PREFIX_PATH}/lib \\
+        -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH}
+
+        BUILD_IN_SOURCE ON
+        LOG_DOWNLOAD ON
+        # LOG_BUILD ON
+    )
+
+    add_library(${ext_name} STATIC IMPORTED)
+    add_dependencies(${ext_name} extlib_jpeg)
+    set_target_properties(${ext_name} PROPERTIES
+        IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libjpeg.a
+        INTERFACE_INCLUDE_DIRECTORIES ${EXTERNALS_PREFIX_PATH}/include
+    )
+
+endif()

cmake/extlib_json.cmake

@@ -1,28 +1,44 @@
 
 message(STATUS "entering in extlib_json.cmake")
 
-include(ExternalProject)
-include(CMakeParseArguments)
+set(ext_name "json")
 
-set(JSON_URL https://github.com/nlohmann/json.git)
-set(JSON_TAG v3.11.3)
+if(USE_SYSTEM_DEPS)
+    # this will define the nlohmann_json::nlohmann_json target
+    find_package(nlohmann_json REQUIRED)
 
-ExternalProject_Add(extlib_json
-    PREFIX extlib_json
+    add_library(${ext_name} INTERFACE IMPORTED)
+    add_dependencies(${ext_name} nlohmann_json::nlohmann_json)
 
-    GIT_REPOSITORY ${JSON_URL}
-    GIT_TAG ${JSON_TAG}
+else()
 
-    UPDATE_COMMAND ""
-    CONFIGURE_COMMAND ""
+    include(ExternalProject)
+    include(CMakeParseArguments)
 
-    BUILD_COMMAND ""
-    BUILD_ALWAYS OFF
+    set(JSON_URL https://github.com/nlohmann/json.git)
+    set(JSON_TAG v3.11.3)
 
-    INSTALL_DIR     ${EXTERNALS_PREFIX_PATH}
-    INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory <SOURCE_DIR>/include/ ${EXTERNALS_PREFIX_PATH}/include/
+    ExternalProject_Add(extlib_json
+
+        PREFIX extlib_json
+
+        GIT_REPOSITORY ${JSON_URL}
+        GIT_TAG ${JSON_TAG}
+
+        UPDATE_COMMAND ""
+        CONFIGURE_COMMAND ""
+
+        BUILD_COMMAND ""
+        BUILD_ALWAYS OFF
+
+        INSTALL_DIR     ${EXTERNALS_PREFIX_PATH}
+        INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory <SOURCE_DIR>/include/ ${EXTERNALS_PREFIX_PATH}/include/
     )
 
-add_library(json INTERFACE)
-add_custom_target(install_extlib_json DEPENDS extlib_json)
-add_dependencies(json install_extlib_json)
+    add_library(${ext_name} INTERFACE IMPORTED)
+    add_dependencies(${ext_name} extlib_json)
+    set_target_properties(${ext_name} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${EXTERNALS_PREFIX_PATH}/include)
+
+endif()
+
+