Skip to content

Commit

Permalink
Build platform-specific wheels containing libmagic
Browse files Browse the repository at this point in the history
  • Loading branch information
ddelange committed Sep 7, 2023
1 parent 2a01b18 commit ec952d7
Show file tree
Hide file tree
Showing 6 changed files with 255 additions and 45 deletions.
134 changes: 134 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
name: GH

permissions:
contents: write

on:
pull_request:
push:
branches: master
release:
types: [released, prereleased]
workflow_dispatch: # allows running workflow manually from the Actions tab

jobs:

build-sdist:
runs-on: ubuntu-latest

env:
PIP_DISABLE_PIP_VERSION_CHECK: 1

steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.x'

- run: sudo apt-get install -y libmagic1

- name: Build source distribution
run: |
pip install -U setuptools wheel pip
python setup.py sdist
- uses: actions/upload-artifact@v3
with:
name: dist
path: dist/*.tar.*


build-wheels-matrix:
runs-on: ubuntu-latest
outputs:
include: ${{ steps.set-matrix.outputs.include }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- run: pip install cibuildwheel==2.15.0
- id: set-matrix
env:
CIBW_PROJECT_REQUIRES_PYTHON: '==3.8.*'
run: |
MATRIX_INCLUDE=$(
{
cibuildwheel --print-build-identifiers --platform linux --arch x86_64,aarch64,i686 | grep cp | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \
&& cibuildwheel --print-build-identifiers --platform macos --arch x86_64,arm64 | grep cp | jq -nRc '{"only": inputs, "os": "macos-11"}' \
&& cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | grep cp | jq -nRc '{"only": inputs, "os": "windows-latest"}'
} | jq -sc
)
echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT
build-wheels:
needs: build-wheels-matrix
runs-on: ${{ matrix.os }}
name: Build ${{ matrix.only }}

strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }}

steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Set up QEMU
if: runner.os == 'Linux'
uses: docker/setup-qemu-action@v2

- uses: pypa/[email protected]
timeout-minutes: 10
with:
only: ${{ matrix.only }}
env:
CIBW_BUILD_VERBOSITY: 1
CIBW_BEFORE_BUILD: 'bash -c "make install_libmagic"'

- uses: actions/upload-artifact@v3
with:
name: dist
path: wheelhouse/*.whl


publish:
needs: [build-sdist, build-wheels]
if: github.event_name == 'release'
runs-on: ubuntu-latest

steps:
- uses: actions/setup-python@v4
with:
python-version: 3.x

- uses: actions/download-artifact@v3
with:
name: dist
path: dist/

- run: ls -ltra dist/

- run: pip install -U twine python-magic --find-links ./dist

- name: Smoketest
run: python -c "import magic; magic.Magic()"

- name: Upload release assets
uses: softprops/[email protected]
with:
files: dist/*

- name: Upload to PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
twine upload dist/*
20 changes: 20 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
SHELL := /bin/bash

.PHONY: install_libmagic
## Install libmagic
install_libmagic:
# Debian https://packages.ubuntu.com/libmagic1
# RHEL https://git.almalinux.org/rpms/file
# Mac https://formulae.brew.sh/formula/libmagic
# Windows https://github.com/julian-r/file-windows
( ( ( brew install libmagic || ( apt-get update && apt-get install -y libmagic1 ) ) || apk add --update libmagic ) || yum install file-libs ) || ( python -c 'import platform, sysconfig, io, zipfile, urllib.request; assert platform.system() == "Windows"; machine = "x86" if sysconfig.get_platform() == "win32" else "x64"; print(machine); zipfile.ZipFile(io.BytesIO(urllib.request.urlopen(f"https://github.com/julian-r/file-windows/releases/download/v5.44/file_5.44-build104-vs2022-{machine}.zip").read())).extractall(".")' && ls -ltra )
# on cibuildwheel, the lib needs to exist in the project before running setup.py
python -c "import subprocess; from magic.loader import load_lib; lib = load_lib()._name; print(f'linking {lib}'); subprocess.check_call(['cp', lib, 'magic'])"
cp /usr/share/misc/magic.mgc magic || true # only on linux
ls -ltra magic

.DEFAULT_GOAL := help
.PHONY: help
## Print Makefile documentation
help:
@perl -0 -nle 'printf("\033[36m %-15s\033[0m %s\n", "$$2", "$$1") while m/^##\s*([^\r\n]+)\n^([\w.-]+):[^=]/gm' $(MAKEFILE_LIST) | sort
37 changes: 25 additions & 12 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ will fail throw if this is attempted.
```python
>>> f = magic.Magic(uncompress=True)
>>> f.from_file('testdata/test.gz')
'ASCII text (gzip compressed data, was "test", last modified: Sat Jun 28
21:32:52 2008, from Unix)'
'ASCII text (gzip compressed data, was "test", last modified: Sat Jun 28 21:32:52 2008, from Unix)'
```

You can also combine the flag options:
Expand All @@ -53,26 +52,40 @@ Other sources:
- GitHub: https://github.com/ahupp/python-magic

This module is a simple wrapper around the libmagic C library, and
that must be installed as well:
comes bundled in the wheels on PyPI. For systems not supported by the wheels, libmagic
needs to be installed before installing this library:

### Debian/Ubuntu
### Linux

```
sudo apt-get install libmagic1
The Linux wheels should run on most systems out of the box.

Depending on your system and CPU architecture, there might be no compatible wheel uploaded. However, precompiled libmagic might still be available for your system:

```sh
# Debian/Ubuntu
apt-get update && apt-get install -y libmagic1
# Alpine
apk add --update libmagic
# RHEL
yum install file-libs
```

### Windows

You'll need DLLs for libmagic. @julian-r maintains a pypi package with the DLLs, you can fetch it with:
The DLLs that are bundled in the Windows wheels are compiled by @julian-r and hosted at https://github.com/julian-r/file-windows/releases.

```
pip install python-magic-bin
```
For ARM64 Windows, you'll need to compile libmagic from source.

### OSX

- When using Homebrew: `brew install libmagic`
- When using macports: `port install file`
The Mac wheels are compiled on GitHub Actions using `macos-11` runners. For older Macs, you'll need to install libmagic from source:

```sh
# homebrew
brew install libmagic
# macports
port install file
```

### Troubleshooting

Expand Down
7 changes: 6 additions & 1 deletion magic/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,12 @@ def magic_descriptor(cookie, fd):


def magic_load(cookie, filename):
return _magic_load(cookie, coerce_filename(filename))
try:
return _magic_load(cookie, coerce_filename(filename))
except MagicException:
# wheels package the mime database in this directory
filename = os.path.join(os.path.dirname(__file__), 'magic.mgc')
return _magic_load(cookie, coerce_filename(filename))


magic_setflags = libmagic.magic_setflags
Expand Down
73 changes: 42 additions & 31 deletions magic/loader.py
Original file line number Diff line number Diff line change
@@ -1,50 +1,61 @@
from ctypes.util import find_library
import ctypes
import sys
import glob
import os.path
import subprocess
import sys

def _lib_candidates():
here = os.path.dirname(__file__)

yield find_library('magic')
if sys.platform == 'darwin':

if sys.platform == 'darwin':
paths = [
here,
os.path.abspath("."),
'/opt/local/lib',
'/usr/local/lib',
'/opt/homebrew/lib',
] + glob.glob('/usr/local/Cellar/libmagic/*/lib')

paths = [
'/opt/local/lib',
'/usr/local/lib',
'/opt/homebrew/lib',
] + glob.glob('/usr/local/Cellar/libmagic/*/lib')
for i in paths:
yield os.path.join(i, 'libmagic.dylib')

for i in paths:
yield os.path.join(i, 'libmagic.dylib')
elif sys.platform in ('win32', 'cygwin'):

elif sys.platform in ('win32', 'cygwin'):
prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1']

prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1']
for i in prefixes:
# find_library searches in %PATH% but not the current directory,
# so look for both
yield os.path.join(here, '%s.dll' % i)
yield os.path.join(os.path.abspath("."), '%s.dll' % i)
yield find_library(i)

for i in prefixes:
# find_library searches in %PATH% but not the current directory,
# so look for both
yield './%s.dll' % (i,)
yield find_library(i)
elif sys.platform == 'linux':
# on some linux systems (musl/alpine), find_library('magic') returns None
yield subprocess.check_output(
"( ldconfig -p | grep 'libmagic.so.1' | grep -o '/.*' ) || echo '/usr/lib/libmagic.so.1'",
shell=True,
universal_newlines=True,
).strip()
yield os.path.join(here, 'libmagic.so.1')
yield os.path.join(os.path.abspath("."), 'libmagic.so.1')

elif sys.platform == 'linux':
# This is necessary because alpine is bad
yield 'libmagic.so.1'
yield find_library('magic')


def load_lib():

for lib in _lib_candidates():
# find_library returns None when lib not found
if lib is None:
continue
try:
return ctypes.CDLL(lib)
except OSError:
pass
else:
# It is better to raise an ImportError since we are importing magic module
raise ImportError('failed to find libmagic. Check your installation')
for lib in _lib_candidates():
# find_library returns None when lib not found
if lib is None:
continue
try:
return ctypes.CDLL(lib)
except OSError as exc:
pass
else:
# It is better to raise an ImportError since we are importing magic module
raise ImportError('failed to find libmagic. Check your installation')

29 changes: 28 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,40 @@
import setuptools
import io
import os
import sys

# python packages should not install succesfully if libraries are missing
from magic.loader import load_lib
lib = load_lib()._name

def read(file_name):
"""Read a text file and return the content as a string."""
with io.open(os.path.join(os.path.dirname(__file__), file_name),
encoding='utf-8') as f:
return f.read()

def get_cmdclass():
"""Build a forward compatible ABI3 wheel when `setup.py bdist_wheel` is called."""
if sys.version_info[0] == 2:
return {}

try:
from wheel.bdist_wheel import bdist_wheel
except ImportError:
return {}

class bdist_wheel_abi3(bdist_wheel):
def get_tag(self):
python, abi, _ = super().get_tag()
# get the platform tag based on libmagic included in this wheel
self.root_is_pure = False
_, _, plat = super().get_tag()
return python, abi, plat

return {"bdist_wheel": bdist_wheel_abi3}

cmdclass = get_cmdclass()

setuptools.setup(
name='python-magic',
description='File type identification using libmagic',
Expand All @@ -23,8 +49,9 @@ def read(file_name):
long_description_content_type='text/markdown',
packages=['magic'],
package_data={
'magic': ['py.typed', '*.pyi', '**/*.pyi'],
'magic': ['py.typed', '*.pyi', '*.dylib*', '*.dll', '*.so*', 'magic.mgc']
},
cmdclass=cmdclass,
keywords="mime magic file",
license="MIT",
python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
Expand Down

0 comments on commit ec952d7

Please sign in to comment.