Skip to content

Commit

Permalink
Build platform-specific wheels containing libmagic
Browse files Browse the repository at this point in the history
  • Loading branch information
ddelange committed Sep 7, 2023
1 parent 2a01b18 commit dc9c393
Show file tree
Hide file tree
Showing 4 changed files with 208 additions and 34 deletions.
125 changes: 125 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
name: GH

permissions:
contents: write

on:
pull_request:
push:
branches: master
release:
types: [released, prereleased]
workflow_dispatch: # allows running workflow manually from the Actions tab

jobs:

build-sdist:
runs-on: ubuntu-latest

env:
PIP_DISABLE_PIP_VERSION_CHECK: 1

steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.x'

- run: sudo apt-get install -y libmagic1

- name: Build source distribution
run: |
pip install -U setuptools wheel pip
python setup.py sdist
- uses: actions/upload-artifact@v3
with:
name: dist
path: dist/*.tar.*


build-wheels-matrix:
runs-on: ubuntu-latest
outputs:
include: ${{ steps.set-matrix.outputs.include }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- run: pip install cibuildwheel==2.15.0
- id: set-matrix
env:
CIBW_PROJECT_REQUIRES_PYTHON: '==3.8.*'
run: |
MATRIX_INCLUDE=$(
{
cibuildwheel --print-build-identifiers --platform linux --arch x86_64,aarch64 | grep cp | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \
&& cibuildwheel --print-build-identifiers --platform macos --arch x86_64 | grep cp | jq -nRc '{"only": inputs, "os": "macos-11"}' \
&& cibuildwheel --print-build-identifiers --platform macos --arch arm64 | grep cp | jq -nRc '{"only": inputs, "os": "macos-11"}' \
&& cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | grep cp | jq -nRc '{"only": inputs, "os": "windows-latest"}'
} | jq -sc
)
echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT
build-wheels:
needs: build-wheels-matrix
runs-on: ${{ matrix.os }}
name: Build ${{ matrix.only }}

strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }}

steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Set up QEMU
if: runner.os == 'Linux'
uses: docker/setup-qemu-action@v2

- uses: pypa/[email protected]
timeout-minutes: 10
with:
only: ${{ matrix.only }}
env:
CIBW_BUILD_VERBOSITY: 1
CIBW_BEFORE_BUILD: 'bash -c "make install_libmagic"'

- uses: actions/upload-artifact@v3
with:
name: dist
path: wheelhouse/*.whl


publish:
needs: [build-sdist, build-wheels]
if: github.event_name == 'release'
runs-on: ubuntu-latest

steps:
- uses: actions/download-artifact@v3
with:
name: dist
path: dist/

- run: ls -ltra dist/

- name: Upload release assets
uses: softprops/[email protected]
with:
files: dist/*

- name: Upload to PyPI
uses: pypa/[email protected]
with:
user: __token__
password: ${{ secrets.PYPI_TOKEN }}
19 changes: 19 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
SHELL := /bin/bash

.PHONY: install_libmagic
## Install libmagic
install_libmagic:
# Debian https://packages.ubuntu.com/libmagic1
# RHEL https://git.almalinux.org/rpms/file
# Mac https://formulae.brew.sh/formula/libmagic
# Windows https://github.com/julian-r/file-windows
( ( ( brew install libmagic || ( apt-get update && apt-get install -y libmagic1 ) ) || apk add --update libmagic ) || yum install file-libs ) || ( python -c 'import platform, sysconfig, io, zipfile, urllib.request; assert platform.system() == "Windows"; machine = "x86" if sysconfig.get_platform() == "win32" else "x64"; print(machine); zipfile.ZipFile(io.BytesIO(urllib.request.urlopen(f"https://github.com/julian-r/file-windows/releases/download/v5.44/file_5.44-build104-vs2022-{machine}.zip").read())).extractall(".")' && ls )
# on cibuildwheel, the lib needs to exist in the project before running setup.py
python -c "import subprocess; from magic.loader import load_lib; lib = load_lib()._name; print(f'linking {lib}'); subprocess.check_call(['cp', lib, 'magic'])"
ls magic

.DEFAULT_GOAL := help
.PHONY: help
## Print Makefile documentation
help:
@perl -0 -nle 'printf("\033[36m %-15s\033[0m %s\n", "$$2", "$$1") while m/^##\s*([^\r\n]+)\n^([\w.-]+):[^=]/gm' $(MAKEFILE_LIST) | sort
67 changes: 36 additions & 31 deletions magic/loader.py
Original file line number Diff line number Diff line change
@@ -1,50 +1,55 @@
from ctypes.util import find_library
import ctypes
import sys
import glob
import os.path
import subprocess
import sys

def _lib_candidates():

yield find_library('magic')
if sys.platform == 'darwin':

if sys.platform == 'darwin':
paths = [
'/opt/local/lib',
'/usr/local/lib',
'/opt/homebrew/lib',
] + glob.glob('/usr/local/Cellar/libmagic/*/lib')

paths = [
'/opt/local/lib',
'/usr/local/lib',
'/opt/homebrew/lib',
] + glob.glob('/usr/local/Cellar/libmagic/*/lib')
for i in paths:
yield os.path.join(i, 'libmagic.dylib')

for i in paths:
yield os.path.join(i, 'libmagic.dylib')
elif sys.platform in ('win32', 'cygwin'):

elif sys.platform in ('win32', 'cygwin'):
prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1']

prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1']
for i in prefixes:
# find_library searches in %PATH% but not the current directory,
# so look for both
yield os.path.join('.', '%s.dll' % i)
yield find_library(i)

for i in prefixes:
# find_library searches in %PATH% but not the current directory,
# so look for both
yield './%s.dll' % (i,)
yield find_library(i)
elif sys.platform == 'linux':
# on some linux systems (musl/alpine), find_library('magic') returns None
yield subprocess.check_output(
"( ldconfig -p | grep 'libmagic.so.1' | grep -o '/.*' ) || echo '/usr/lib/libmagic.so.1'",
shell=True,
universal_newlines=True,
).strip()

elif sys.platform == 'linux':
# This is necessary because alpine is bad
yield 'libmagic.so.1'
yield find_library('magic')


def load_lib():

for lib in _lib_candidates():
# find_library returns None when lib not found
if lib is None:
continue
try:
return ctypes.CDLL(lib)
except OSError:
pass
else:
# It is better to raise an ImportError since we are importing magic module
raise ImportError('failed to find libmagic. Check your installation')
for lib in _lib_candidates():
# find_library returns None when lib not found
if lib is None:
continue
try:
return ctypes.CDLL(lib)
except OSError as exc:
pass
else:
# It is better to raise an ImportError since we are importing magic module
raise ImportError('failed to find libmagic. Check your installation')

31 changes: 28 additions & 3 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,40 @@
import setuptools
import io
import os
import sys

# python packages should not install succesfully if libraries are missing
from magic.loader import load_lib
lib = load_lib()._name

def read(file_name):
"""Read a text file and return the content as a string."""
with io.open(os.path.join(os.path.dirname(__file__), file_name),
encoding='utf-8') as f:
return f.read()

def get_cmdclass():
"""Build a forward compatible ABI3 wheel when `setup.py bdist_wheel` is called."""
if sys.version_info[0] == 2:
return {}

try:
from wheel.bdist_wheel import bdist_wheel
except ImportError:
return {}

class bdist_wheel_abi3(bdist_wheel):
def get_tag(self):
python, abi, _ = super().get_tag()
# get the platform tag based on libmagic included in this wheel
self.root_is_pure = False
_, _, plat = super().get_tag()
return python, abi, plat

return {"bdist_wheel": bdist_wheel_abi3}

cmdclass = get_cmdclass()

setuptools.setup(
name='python-magic',
description='File type identification using libmagic',
Expand All @@ -22,9 +48,8 @@ def read(file_name):
long_description=read('README.md'),
long_description_content_type='text/markdown',
packages=['magic'],
package_data={
'magic': ['py.typed', '*.pyi', '**/*.pyi'],
},
package_data={'magic': ['py.typed', '*.pyi', '*.dylib*', '*.dll', '*.so*']},
cmdclass=cmdclass,
keywords="mime magic file",
license="MIT",
python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
Expand Down

0 comments on commit dc9c393

Please sign in to comment.