Single-package with IDL in wheel

Author: Thijs Miedema

.azure/templates/build-cyclone.yml

@@ -16,7 +16,7 @@
 
 steps:
   - bash: |
-      git clone https://github.com/eclipse-cyclonedds/cyclonedds.git
+      git clone https://github.com/eclipse-cyclonedds/cyclonedds.git cyclonedds_c
     name: clone_cyclone
     displayName: Clone the CycloneDDS repository
   - bash: |
@@ -26,7 +26,7 @@ steps:
     displayName: Check the build type.
   - task: Cache@2
     inputs:
-      key: cyclonedds_build | 4 | $(Agent.OS) | $(build_type) | cyclonedds/.git/refs/heads/master
+      key: cyclonedds_build | 4 | $(Agent.OS) | $(build_type) | cyclonedds_c/.git/refs/heads/master
       path: cyclone-$(Agent.OS)-$(build_type)
       cacheHitVar: CACHE_RESTORED
     name: cyclonedds_cache
@@ -52,8 +52,8 @@ steps:
     displayName: Setup for Windows builds
   - bash: |
       set -e -x
-      mkdir -p cyclonedds/build cyclone-$(Agent.OS)-$(build_type)
-      cd cyclonedds/build
+      mkdir -p cyclonedds_c/build cyclone-$(Agent.OS)-$(build_type)
+      cd cyclonedds_c/build
       cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
             -DCMAKE_INSTALL_PREFIX=../../cyclone-$(Agent.OS)-$(build_type) \
             -DENABLE_LIFESPAN=ON \

.azure/templates/build-test.yml

@@ -34,34 +34,21 @@ steps:
       set -e -x
       pip install --user --upgrade pip
       pip install --user --upgrade pytest pytest-azurepipelines pytest-cov pytest-mock flake8
-      pip install --user ./src/pycdr
-      pip install --user ./src/cyclonedds
+      pip install --user .
     name: install_cyclonedds_py
     displayName: Run installers
     env:
       cflags: -Werror
   - bash: |
-      python -m flake8 ./src/pycdr/pycdr --count --select=E9,F63,F7,F82 --show-source --statistics
-      python -m flake8 ./src/cyclonedds/cyclonedds --count --select=E9,F63,F7,F82 --show-source --statistics
-      python -m flake8 ./src/pycdr/pycdr --count --exit-zero --max-complexity=10 --max-line-length=127 --per-file-ignores="__init__.py:F401" --statistics
-      python -m flake8 ./src/cyclonedds/cyclonedds --count --exit-zero --max-complexity=10 --max-line-length=127 --per-file-ignores="__init__.py:F401" --statistics
+      python -m flake8 ./cyclonedds --count --select=E9,F63,F7,F82 --show-source --statistics
+      python -m flake8 ./cyclonedds --count --exit-zero --max-complexity=10 --max-line-length=127 --per-file-ignores="__init__.py:F401" --statistics
     name: flake8_lint
     displayName: Run Flake8 Linter
   - bash: |
-      python -m pytest ./src/pycdr -v --no-coverage-upload
-    name: test_pycdr
-    displayName: Run tests for PyCDR
-  - bash: |
-      python -m pytest ./src/cyclonedds -v --no-coverage-upload
+      cd tests
+      python -m pytest .. -v --no-coverage-upload
     name: test_cyclonedds_py
     displayName: Run tests for CycloneDDS
-  - bash: |
-      python -m coverage combine src/cyclonedds/.coverage src/pycdr/.coverage
-      python -m coverage xml
-    name: collect_test_coverage
-    displayName: Collect PyCDR and CycloneDDS coverage data
-  - task: PublishCodeCoverageResults@1
-    displayName: Publish Cobertura Coverage Report
-    inputs:
-      codeCoverageTool: Cobertura
-      summaryFileLocation: coverage.xml
+  - publish: tests/.coverage
+    artifact: coverage-$(Agent.JobName)
+    displayName: Publish coverage artifact

.azure/templates/build-wheel.yml

@@ -24,11 +24,11 @@ steps:
     displayName: Install dependencies
   - bash: |
       set -x -e
-      python -m cibuildwheel --output-dir wheelhouse ./src/cyclonedds
+      python -m cibuildwheel --output-dir wheelhouse .
     displayName: Build wheels (Linux/Macos)
     condition: ne(variables['Agent.OS'], 'Windows_NT')
   - pwsh: |
-      python -m cibuildwheel --output-dir wheelhouse ./src/cyclonedds
+      python -m cibuildwheel --output-dir wheelhouse .
     displayName: Build wheels (Windows)
     condition: eq(variables['Agent.OS'], 'Windows_NT')
   - publish: wheelhouse

.azure/templates/publish-package.yml

@@ -26,12 +26,6 @@ steps:
     name: install_twine
     displayName: Install Twine
   - bash: |
-      cd ./src/pycdr
-      python setup.py sdist
-    name: build_pycdr_src_dist
-    displayName: Build PyCDR source distribution
-  - bash: |
-      cd ./src/cyclonedds
       python setup.py sdist
     name: build_cyclonedds_src_dist
     displayName: Build CycloneDDS source distribution
@@ -40,12 +34,6 @@ steps:
     inputs:
       secureFile: pypi_credentials
   - bash: |
-      cd ./src/pycdr
-      python -m twine upload -r $(PYPI_REPOSITORY) --config-file $(pypi_credentials.secureFilePath) --skip-existing dist/*
-    name: publish_pycdr
-    displayName: Publish PyCDR
-  - bash: |
-      cd ./src/cyclonedds
       python -m twine upload -r $(PYPI_REPOSITORY) --config-file $(pypi_credentials.secureFilePath) --skip-existing dist/*
     name: publish_cyclonedds
     displayName: Publish CycloneDDS

src/cyclonedds/.coveragerc renamed to .coveragerc

@@ -1,17 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
 build/
-build-windows/
-**/build
-.vscode/*
-__pycache__
-docs/_build
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
 .coverage
-.env
-cov.xml
+.coverage.*
+.cache
+nosetests.xml
 coverage.xml
-*.egg-info
-.inputrc
-.python-version
-snippets.txt
-src/cyclonedds/dist/
-src/pycdr/dist/
-cdds-activate
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/

.gitignore

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.18)
 # CMAKE_PREFIX_PATH environment variable is available in CMake >= 3.18
-project(_clayer LANGUAGES C)
+project(clayer LANGUAGES C)
 find_package(PythonExtensions REQUIRED)
 find_package(CycloneDDS REQUIRED)
 
@@ -11,15 +11,21 @@ if(CMAKE_C_COMPILER_ID STREQUAL "Clang" OR
                       -Winfinite-recursion -Wassign-enum -Wcomma -Wshadow
                       -Wstrict-prototypes -Wconditional-uninitialized)
 elseif(CMAKE_C_COMPILER_ID STREQUAL "GNU")
-  add_compile_options(-Wall -Wextra -Wconversion -Wmissing-prototypes)
+  add_compile_options(-Wall -Wextra -Wconversion -Wmissing-prototypes -Wpedantic --pedantic-errors)
 elseif(CMAKE_C_COMPILER_ID STREQUAL "MSVC")
   add_compile_options(/W3)
 endif()
 
-add_library(_clayer MODULE clayer/src/cdrkeyvm.c clayer/src/pysertype.c)
+# Build python c layer
+add_library(_clayer MODULE clayer/cdrkeyvm.c clayer/pysertype.c)
 target_link_libraries(_clayer CycloneDDS::ddsc)
 python_extension_module(_clayer)
 install(
   TARGETS _clayer
-  LIBRARY DESTINATION cyclonedds
-  RUNTIME DESTINATION cyclonedds)
+  LIBRARY DESTINATION cyclonedds)
+
+
+# Build idl python backend
+if (TARGET CycloneDDS::idl)
+  add_subdirectory(idlpy)
+endif()

src/cyclonedds/CMakeLists.txt renamed to CMakeLists.txt

@@ -35,11 +35,8 @@ You can either install the source from the latest release from pypi:
 
 or you can download the code from this repository to get the bleeding edge and directly install from your local filesystem:
 
-    $ git clone https://github.com/eclipse-cyclonedds/cyclonedds-python
-    $ cd cyclonedds-python
     $ export CYCLONEDDS_HOME="/path/to/cyclone"
-    $ pip install ./src/pycdr
-    $ pip install ./src/cyclonedds
+    $ pip install https://github.com/eclipse-cyclonedds/cyclonedds-python
 
 If you get permission errors you are using your system python. This is not recommended, we recommend using [a virtual environment][2], [poetry][3], [pipenv][4] or [pyenv][5]. If you _just_ want to get going, you can add `--user` to your pip command to install for the current user. See the [Installing Python Modules][6] Python documentation.
 
@@ -51,19 +48,6 @@ If you get permission errors you are using your system python. This is not recom
 [6]: https://docs.python.org/3/installing/index.html
 
 
-# Installing the Python backend for the IDL compiler
-
-The code for the Python backend for the IDL compiler is contained in `src/idlpy` and builds like any other cmake project:
-
-    $ git clone https://github.com/eclipse-cyclonedds/cyclonedds-python
-    $ cd cyclonedds-python/src/idlpy
-    $ mkdir build
-    $ cmake -DCMAKE_INSTALL_PREFIX=<install-location> \
-            -DCMAKE_PREFIX_PATH="<cyclonedds-install-location>" \
-            ..
-    $ cmake --build .
-    $ cmake --build . --target install
-
 For more details on this process take a look at the [Eclipse Cyclone DDS C++ repository][7] which explains the cmake process in depth.
 
 [7]: https://github.com/eclipse-cyclonedds/cyclonedds-cxx/

README.md

@@ -56,6 +56,9 @@ jobs:
       'macOS 10.15 with Python 3.9':
         image: macOS-10.15
         python-version: 3.9
+      'Windows Server 2019 with Python 3.6':
+        image: windows-2019
+        python-version: 3.6
       'Windows Server 2019 with Python 3.9':
         image: windows-2019
         python-version: 3.9
@@ -72,3 +75,25 @@ jobs:
     name: set_cyclonedds_home
     displayName: Set CYCLONEDDS_HOME
   - template: /.azure/templates/build-test.yml
+- job: CoverageReport
+  dependsOn: AutomatedTests
+  pool:
+    vmImage: ubuntu-20.04
+  steps:
+  - bash: |
+      mkdir coverage
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      path: coverage
+    displayName: Download Coverage
+  - task: UsePythonVersion@0
+  - bash: |
+      ls coverage
+      pip install --user coverage
+      python -m coverage combine ./coverage/*/.coverage
+      python -m coverage xml
+  - task: PublishCodeCoverageResults@1
+    displayName: Publish Cobertura Coverage Report
+    inputs:
+      codeCoverageTool: Cobertura
+      summaryFileLocation: coverage.xml