diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..2390d8c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c4e4c9..1f0317d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,13 +19,11 @@ jobs: with: python-version: ${{ matrix.python-version }} allow-prereleases: true + - name: Copy libmagic into magic dir + run: ${{ ( startsWith( matrix.os, 'windows' ) && 'bash add_libmagic.sh' ) || 'sudo -E bash add_libmagic.sh' }} - run: pip install --upgrade pip - run: pip install --upgrade pytest - run: pip install --editable . - - if: runner.os == 'macOS' - run: brew install libmagic - - if: runner.os == 'Windows' - run: pip install python-magic-bin - run: LC_ALL=en_US.UTF-8 pytest shell: bash timeout-minutes: 15 # Limit Windows infinite loop. diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..cd3b48e --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,141 @@ +name: wheels + +on: + pull_request: + push: + branches: master + release: + types: [released, prereleased] + workflow_dispatch: # allows running workflow manually from the Actions tab + +jobs: + + build-sdist: + runs-on: ubuntu-latest + + env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - run: sudo apt-get install -y libmagic1 + + - name: Build source distribution + run: | + pip install --upgrade setuptools wheel pip + python setup.py sdist + + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/*.tar.* + + + build-wheels-matrix: + runs-on: ubuntu-latest + outputs: + include: ${{ steps.set-matrix.outputs.include }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + - run: pip install cibuildwheel==2.17.0 # sync version with pypa/cibuildwheel below + - id: set-matrix + env: + # only mention one (trivial) python version, as py2.py3 wheels only need to be build once per arch + CIBW_PROJECT_REQUIRES_PYTHON: '==3.12.*' + # skip PyPy wheels for now, and skip i686 wheels because pytest is failing + CIBW_SKIP: pp* *i686 + run: | + MATRIX_INCLUDE=$( + { + cibuildwheel --print-build-identifiers --platform linux --arch all | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ + && cibuildwheel --print-build-identifiers --platform macos --arch x86_64 | jq -nRc '{"only": inputs, "os": "macos-13"}' \ + && cibuildwheel --print-build-identifiers --platform macos --arch arm64 | jq -nRc '{"only": inputs, "os": "macos-14"}' \ + && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | jq -nRc '{"only": inputs, "os": "windows-latest"}' + } | jq -sc + ) + echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT + + + build-wheels: + name: build ${{ matrix.only }} + needs: build-wheels-matrix + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 + + - uses: pypa/cibuildwheel@v2.17.0 # sync version with pip install cibuildwheel above + timeout-minutes: 10 + with: + only: ${{ matrix.only }} + env: + CIBW_BUILD_VERBOSITY: 1 + # add compiled libmagic to the build directory (to include in the wheel) + CIBW_BEFORE_BUILD: ${{ ( startsWith( matrix.os, 'macos' ) && 'sudo -E bash add_libmagic.sh' ) || 'bash add_libmagic.sh' }} + # build macos wheels with maximum backwards compatibility (gcc -mmacosx-version-min flag) + MACOSX_DEPLOYMENT_TARGET: ${{ ( endsWith( matrix.only, 'arm64' ) && '11.0' ) || '10.9' }} + # simple smoke test run on each wheel: this is an HLS MP4 video, only recognised in recent versions of libmagic + CIBW_TEST_COMMAND: python -c "import magic; assert magic.Magic(mime=True).from_buffer(b'\x00\x00\x00\x1cftypiso5\x00\x00\x00\x01isomiso5hlsf\x00\x00') == 'video/mp4'" + + - uses: actions/upload-artifact@v4 + with: + name: dist-${{ matrix.only }} + path: wheelhouse/*.whl + + + publish: + if: github.event_name == 'release' + needs: [build-sdist, build-wheels] + runs-on: ubuntu-latest + + permissions: + contents: write # softprops/action-gh-release + id-token: write # pypa/gh-action-pypi-publish + + steps: + - uses: actions/setup-python@v5 + with: + python-version: 3.x + + - uses: actions/download-artifact@v4 + with: + path: dist/ + pattern: dist-* + merge-multiple: true + + - run: ls -ltra dist/ + + - run: pip install --upgrade python-magic --find-links ./dist + + - name: Smoketest + run: python -c "import magic; magic.Magic()" + + - name: Upload release assets + uses: softprops/action-gh-release@v0.1.15 + with: + files: dist/* + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@v1.8.14 diff --git a/CHANGELOG b/CHANGELOG index a8370c6..eb7cac9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,10 +1,14 @@ -Changes to 0.4.29: +Changes to 0.4.28: +- libmagic and magic.mgc now come bundled in the wheels on PyPI, and will be copied + into site-packages/magic along with the Python files of this library +- magic.loader.load_lib now first searches for libmagic in the same directory as the + Python files, then in the current working directory, and only then in standard paths +- magic.Magic(magic_file=...) and magic.compat.Magic.load(magic_file=...) will now + prefer "magic.mgc" in the same directory as the Python files, only if left + unspecified by the user (and the MAGIC env var is empty or not set) - support MAGIC_SYMLINK (via follow_symlink flag on Magic constructor) - correctly throw FileNotFoundException depending on flag - -Changes to 0.4.28: - - support "magic-1.dll" on Windows, which is produced by vcpkg - add python 3.10 to tox config - update test for upstream gzip extensions diff --git a/README.md b/README.md index 010cc8f..89c1807 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,11 @@ [![ci](https://github.com/ahupp/python-magic/actions/workflows/ci.yml/badge.svg)](https://github.com/ahupp/python-magic/actions/workflows/ci.yml) [![Join the chat at https://gitter.im/ahupp/python-magic](https://badges.gitter.im/ahupp/python-magic.svg)](https://gitter.im/ahupp/python-magic?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -python-magic is a Python interface to the libmagic file type -identification library. libmagic identifies file types by checking +[python-magic](https://github.com/ahupp/python-magic) is a Python interface to the libmagic file type +identification library. libmagic identifies file types by checking their headers according to a predefined list of file types. This functionality is exposed to the command line by the Unix command -`file`. +[`file`](https://www.darwinsys.com/file/). ## Usage @@ -31,8 +31,7 @@ will fail throw if this is attempted. ```python >>> f = magic.Magic(uncompress=True) >>> f.from_file('testdata/test.gz') -'ASCII text (gzip compressed data, was "test", last modified: Sat Jun 28 -21:32:52 2008, from Unix)' +'ASCII text (gzip compressed data, was "test", last modified: Sat Jun 28 21:32:52 2008, from Unix)' ``` You can also combine the flag options: @@ -45,27 +44,53 @@ You can also combine the flag options: ## Installation -The current stable version of python-magic is available on PyPI and -can be installed by running `pip install python-magic`. +This module is a simple [CDLL](https://docs.python.org/3/library/ctypes.html) wrapper around the libmagic C library. +The current stable version of python-magic is available on [PyPI](http://pypi.python.org/pypi/python-magic/) +and can be installed by running `pip install python-magic`. -Other sources: +Compiled libmagic and the magic database come bundled in the wheels on PyPI. +You can use your own `magic.mgc` database by setting the `MAGIC` +environment variable, or by using `magic.Magic(magic_file='path/to/magic.mgc')`. +If you want to compile your own libmagic, circumvent the wheels +by installing from source: `pip install python-magic --no-binary python-magic`. -- PyPI: http://pypi.python.org/pypi/python-magic/ -- GitHub: https://github.com/ahupp/python-magic +For systems not supported by the wheels, pip installs from source, +requiring libmagic to be available before installing python-magic: -This module is a simple wrapper around the libmagic C library, and -that must be installed as well: +### Linux -### Debian/Ubuntu +The Linux wheels should run on most systems out of the box. +Depending on your system and CPU architecture, there might be no compatible wheel uploaded. +However, precompiled libmagic might still be available for your system: + +```sh +# Debian/Ubuntu +apt-get update && apt-get install -y libmagic1 +# Alpine +apk add --update libmagic +# RHEL +dnf install file-libs ``` -sudo apt-get install libmagic1 -``` + +### Windows + +The DLLs that are bundled in the Windows wheels are compiled by @julian-r +and are hosted at https://github.com/julian-r/file-windows/releases. + +For ARM64 Windows, you'll need to compile libmagic from source. ### OSX -- When using Homebrew: `brew install libmagic` -- When using macports: `port install file` +The Mac wheels are compiled with maximum backward compatibility. +For older Macs, you'll need to install libmagic from source: + +```sh +# homebrew +brew install libmagic +# macports +port install file +``` If python-magic fails to load the library it may be in a non-standard location, in which case you can set the environment variable `DYLD_LIBRARY_PATH` to point to it. @@ -78,7 +103,7 @@ If python-magic fails to load the library it may be in a non-standard location, - 'MagicException: could not find any magic files!': some installations of libmagic do not correctly point to their magic database file. Try specifying the path to the file explicitly in the - constructor: `magic.Magic(magic_file="path_to_magic_file")`. + constructor: `magic.Magic(magic_file='path/to/magic.mgc')`. - 'WindowsError: [Error 193] %1 is not a valid Win32 application': Attempting to run the 32-bit libmagic DLL in a 64-bit build of @@ -88,7 +113,6 @@ If python-magic fails to load the library it may be in a non-standard location, - 'WindowsError: exception: access violation writing 0x00000000 ' This may indicate you are mixing Windows Python and Cygwin Python. Make sure your libmagic and python builds are consistent. - ## Bug Reports python-magic is a thin layer over the libmagic C library. diff --git a/add_libmagic.sh b/add_libmagic.sh new file mode 100755 index 0000000..ba90bdc --- /dev/null +++ b/add_libmagic.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +install_source() { + # install from source + # https://www.darwinsys.com/file/ + # https://github.com/file/file/blob/FILE5_45/INSTALL#L51 + ( + version="file-5.45" && + tmpfile="$(mktemp)" && + curl -sSLo "${tmpfile}" "https://astron.com/pub/file/${version}.tar.gz" && + tar xvf "${tmpfile}" && + cd "${version}" && + ./configure && + make && + make install && + make installcheck && + cd .. && + rm -r "${version}" + ) || ( cd .. && false ) +} + +install_precompiled() { + # Mac https://formulae.brew.sh/formula/libmagic + # Debian https://packages.ubuntu.com/libmagic1 + # Alpine https://pkgs.alpinelinux.org/package/libmagic + # RHEL https://git.almalinux.org/rpms/file + # Windows https://github.com/julian-r/file-windows + if [ -n "$(which brew)" ]; then + brew install libmagic + elif [ -n "$(which apt-get)" ]; then + apt-get update + apt-get install -y libmagic1 + elif [ -n "$(which apk)" ]; then + apk add --update libmagic + elif [ -n "$(which dnf)" ]; then + dnf --setopt install_weak_deps=false -y install file-libs + else + # windows (no install, just download into current working directory) + # could also consider install using `pacman`: https://packages.msys2.org/base/mingw-w64-file + # which would require an update of copy_libmagic below to account for new magic.mgc paths + python <