setupinfo.py

import sys
import io
import os
import os.path
from distutils.core import Extension
from distutils.errors import CompileError, DistutilsOptionError
from distutils.command.build_ext import build_ext as _build_ext
from versioninfo import get_base_dir

try:
    import Cython.Compiler.Version
    CYTHON_INSTALLED = True
except ImportError:
    CYTHON_INSTALLED = False

EXT_MODULES = ["lxml.etree", "lxml.objectify"]
COMPILED_MODULES = [
    "lxml.builder",
    "lxml._elementpath",
    "lxml.html.diff",
    "lxml.html.clean",
    "lxml.sax",
]
HEADER_FILES = ['etree.h', 'etree_api.h']

if hasattr(sys, 'pypy_version_info') or (
        getattr(sys, 'implementation', None) and sys.implementation.name != 'cpython'):
    # disable Cython compilation of Python modules in PyPy and other non-CPythons
    del COMPILED_MODULES[:]

SOURCE_PATH = "src"
INCLUDE_PACKAGE_PATH = os.path.join(SOURCE_PATH, 'lxml', 'includes')

if sys.version_info[0] >= 3:
    _system_encoding = sys.getdefaultencoding()
    if _system_encoding is None:
        _system_encoding = "iso-8859-1" # :-)
    def decode_input(data):
        if isinstance(data, str):
            return data
        return data.decode(_system_encoding)
else:
    def decode_input(data):
        return data

def env_var(name):
    value = os.getenv(name)
    if value:
        value = decode_input(value)
        if sys.platform == 'win32' and ';' in value:
            return value.split(';')
        else:
            return value.split()
    else:
        return []


def _prefer_reldirs(base_dir, dirs):
    return [
        os.path.relpath(path) if path.startswith(base_dir) else path
        for path in dirs
    ]

def ext_modules(static_include_dirs, static_library_dirs,
                static_cflags, static_binaries):
    global XML2_CONFIG, XSLT_CONFIG
    if OPTION_BUILD_LIBXML2XSLT:
        from buildlibxml import build_libxml2xslt, get_prebuilt_libxml2xslt
        if sys.platform.startswith('win'):
            get_prebuilt_libxml2xslt(
                OPTION_DOWNLOAD_DIR, static_include_dirs, static_library_dirs)
        else:
            XML2_CONFIG, XSLT_CONFIG = build_libxml2xslt(
                OPTION_DOWNLOAD_DIR, 'build/tmp',
                static_include_dirs, static_library_dirs,
                static_cflags, static_binaries,
                libiconv_version=OPTION_LIBICONV_VERSION,
                libxml2_version=OPTION_LIBXML2_VERSION,
                libxslt_version=OPTION_LIBXSLT_VERSION,
                zlib_version=OPTION_ZLIB_VERSION,
                multicore=OPTION_MULTICORE)

    modules = EXT_MODULES + COMPILED_MODULES
    if OPTION_WITHOUT_OBJECTIFY:
        modules = [entry for entry in modules if 'objectify' not in entry]

    module_files = list(os.path.join(SOURCE_PATH, *module.split('.')) for module in modules)
    c_files_exist = [os.path.exists(module + '.c') for module in module_files]

    use_cython = True
    if CYTHON_INSTALLED and (OPTION_WITH_CYTHON or not all(c_files_exist)):
        print("Building with Cython %s." % Cython.Compiler.Version.version)
        # generate module cleanup code
        from Cython.Compiler import Options
        Options.generate_cleanup_code = 3
        Options.clear_to_none = False
    elif not OPTION_WITHOUT_CYTHON and not all(c_files_exist):
        for exists, module in zip(c_files_exist, module_files):
            if not exists:
                raise RuntimeError(
                    "ERROR: Trying to build without Cython, but pre-generated '%s.c' "
                    "is not available (pass --without-cython to ignore this error)." % module)
    else:
        if not all(c_files_exist):
            for exists, module in zip(c_files_exist, module_files):
                if not exists:
                    print("WARNING: Trying to build without Cython, but pre-generated "
                          "'%s.c' is not available." % module)
        use_cython = False
        print("Building without Cython.")

    lib_versions = get_library_versions()
    versions_ok = True
    if lib_versions[0]:
        print("Using build configuration of libxml2 %s and libxslt %s" %
              lib_versions)
        versions_ok = check_min_version(lib_versions[0], (2, 7, 0), 'libxml2')
    else:
        print("Using build configuration of libxslt %s" %
              lib_versions[1])
    versions_ok |= check_min_version(lib_versions[1], (1, 1, 23), 'libxslt')
    if not versions_ok:
        raise RuntimeError("Dependency missing")

    base_dir = get_base_dir()
    _include_dirs = _prefer_reldirs(
        base_dir, include_dirs(static_include_dirs) + [
            SOURCE_PATH,
            INCLUDE_PACKAGE_PATH,
        ])
    _library_dirs = _prefer_reldirs(base_dir, library_dirs(static_library_dirs))
    _cflags = cflags(static_cflags)
    _ldflags = ['-isysroot', get_xcode_isysroot()] if sys.platform == 'darwin' else None
    _define_macros = define_macros()
    _libraries = libraries()

    if _library_dirs:
        message = "Building against libxml2/libxslt in "
        if len(_library_dirs) > 1:
            print(message + "one of the following directories:")
            for dir in _library_dirs:
                print("  " + dir)
        else:
            print(message + "the following directory: " +
                  _library_dirs[0])

    if OPTION_AUTO_RPATH:
        runtime_library_dirs = _library_dirs
    else:
        runtime_library_dirs = []

    if CYTHON_INSTALLED and OPTION_SHOW_WARNINGS:
        from Cython.Compiler import Errors
        Errors.LEVEL = 0

    cythonize_directives = {
        'binding': True,
    }
    if OPTION_WITH_COVERAGE:
        cythonize_directives['linetrace'] = True

    result = []
    for module, src_file in zip(modules, module_files):
        is_py = module in COMPILED_MODULES
        main_module_source = src_file + (
            '.c' if not use_cython else '.py' if is_py else '.pyx')
        result.append(
            Extension(
                module,
                sources = [main_module_source],
                depends = find_dependencies(module),
                extra_compile_args = _cflags,
                extra_link_args = None if is_py else _ldflags,
                extra_objects = None if is_py else static_binaries,
                define_macros = _define_macros,
                include_dirs = _include_dirs,
                library_dirs = None if is_py else _library_dirs,
                runtime_library_dirs = None if is_py else runtime_library_dirs,
                libraries = None if is_py else _libraries,
            ))
    if CYTHON_INSTALLED and OPTION_WITH_CYTHON_GDB:
        for ext in result:
            ext.cython_gdb = True

    if CYTHON_INSTALLED and use_cython:
        # build .c files right now and convert Extension() objects
        from Cython.Build import cythonize
        result = cythonize(result, compiler_directives=cythonize_directives)

    # for backwards compatibility reasons, provide "etree[_api].h" also as "lxml.etree[_api].h"
    for header_filename in HEADER_FILES:
        src_file = os.path.join(SOURCE_PATH, 'lxml', header_filename)
        dst_file = os.path.join(SOURCE_PATH, 'lxml', 'lxml.' + header_filename)
        if not os.path.exists(src_file):
            continue
        if os.path.exists(dst_file) and os.path.getmtime(dst_file) >= os.path.getmtime(src_file):
            continue

        with io.open(src_file, 'r', encoding='iso8859-1') as f:
            content = f.read()
        for filename in HEADER_FILES:
            content = content.replace('"%s"' % filename, '"lxml.%s"' % filename)
        with io.open(dst_file, 'w', encoding='iso8859-1') as f:
            f.write(content)

    return result


def find_dependencies(module):
    if not CYTHON_INSTALLED or 'lxml.html' in module:
        return []
    base_dir = get_base_dir()
    package_dir = os.path.join(base_dir, SOURCE_PATH, 'lxml')
    includes_dir = os.path.join(base_dir, INCLUDE_PACKAGE_PATH)

    pxd_files = [
        os.path.join(INCLUDE_PACKAGE_PATH, filename)
        for filename in os.listdir(includes_dir)
        if filename.endswith('.pxd')
    ]

    if module == 'lxml.etree':
        pxi_files = [
            os.path.join(SOURCE_PATH, 'lxml', filename)
            for filename in os.listdir(package_dir)
            if filename.endswith('.pxi') and 'objectpath' not in filename
        ]
        pxd_files = [
            filename for filename in pxd_files
            if 'etreepublic' not in filename
        ]
    elif module == 'lxml.objectify':
        pxi_files = [os.path.join(SOURCE_PATH, 'lxml', 'objectpath.pxi')]
    else:
        pxi_files = pxd_files = []

    return pxd_files + pxi_files


def extra_setup_args():
    class CheckLibxml2BuildExt(_build_ext):
        """Subclass to check whether libxml2 is really available if the build fails"""
        def run(self):
            try:
                _build_ext.run(self)  # old-style class in Py2
            except CompileError as e:
                print('Compile failed: %s' % e)
                if not seems_to_have_libxml2():
                    print_libxml_error()
                raise
    result = {'cmdclass': {'build_ext': CheckLibxml2BuildExt}}
    return result


def seems_to_have_libxml2():
    from distutils import ccompiler
    compiler = ccompiler.new_compiler()
    return compiler.has_function(
        'xmlXPathInit',
        include_dirs=include_dirs([]) + ['/usr/include/libxml2'],
        includes=['libxml/xpath.h'],
        library_dirs=library_dirs([]),
        libraries=['xml2'])


def print_libxml_error():
    print('*********************************************************************************')
    print('Could not find function xmlCheckVersion in library libxml2. Is libxml2 installed?')
    if sys.platform in ('darwin',):
        print('Perhaps try: xcode-select --install')
    print('*********************************************************************************')


def libraries():
    standard_libs = []
    if 'linux' in sys.platform:
        standard_libs.append('rt')
    if not OPTION_BUILD_LIBXML2XSLT:
        standard_libs.append('z')
    standard_libs.append('m')

    if sys.platform in ('win32',):
        libs = ['libxslt', 'libexslt', 'libxml2', 'iconv']
        if OPTION_STATIC:
            libs = ['%s_a' % lib for lib in libs]
        libs.extend(['zlib', 'WS2_32'])
    elif OPTION_STATIC:
        libs = standard_libs
    else:
        libs = ['xslt', 'exslt', 'xml2'] + standard_libs
    return libs

def library_dirs(static_library_dirs):
    if OPTION_STATIC:
        if not static_library_dirs:
            static_library_dirs = env_var('LIBRARY')
        assert static_library_dirs, "Static build not configured, see doc/build.txt"
        return static_library_dirs
    # filter them from xslt-config --libs
    result = []
    possible_library_dirs = flags('libs')
    for possible_library_dir in possible_library_dirs:
        if possible_library_dir.startswith('-L'):
            result.append(possible_library_dir[2:])
    return result

def include_dirs(static_include_dirs):
    if OPTION_STATIC:
        if not static_include_dirs:
            static_include_dirs = env_var('INCLUDE')
        return static_include_dirs
    # filter them from xslt-config --cflags
    result = []
    possible_include_dirs = flags('cflags')
    for possible_include_dir in possible_include_dirs:
        if possible_include_dir.startswith('-I'):
            result.append(possible_include_dir[2:])
    return result

def cflags(static_cflags):
    result = []
    if not OPTION_SHOW_WARNINGS:
        result.append('-w')
    if OPTION_DEBUG_GCC:
        result.append('-g2')

    if OPTION_STATIC:
        if not static_cflags:
            static_cflags = env_var('CFLAGS')
        result.extend(static_cflags)
    else:
        # anything from xslt-config --cflags that doesn't start with -I
        possible_cflags = flags('cflags')
        for possible_cflag in possible_cflags:
            if not possible_cflag.startswith('-I'):
                result.append(possible_cflag)

    if sys.platform in ('darwin',):
        for opt in result:
            if 'flat_namespace' in opt:
                break
        else:
            result.append('-flat_namespace')

    return result

def define_macros():
    macros = []
    if OPTION_WITHOUT_ASSERT:
        macros.append(('PYREX_WITHOUT_ASSERTIONS', None))
    if OPTION_WITHOUT_THREADING:
        macros.append(('WITHOUT_THREADING', None))
    if OPTION_WITH_REFNANNY:
        macros.append(('CYTHON_REFNANNY', None))
    if OPTION_WITH_UNICODE_STRINGS:
        macros.append(('LXML_UNICODE_STRINGS', '1'))
    if OPTION_WITH_COVERAGE:
        macros.append(('CYTHON_TRACE_NOGIL', '1'))
    # Disable showing C lines in tracebacks, unless explicitly requested.
    macros.append(('CYTHON_CLINE_IN_TRACEBACK', '1' if OPTION_WITH_CLINES else '0'))
    return macros

_ERROR_PRINTED = False

def run_command(cmd, *args):
    if not cmd:
        return ''
    if args:
        cmd = ' '.join((cmd,) + args)
    import subprocess
    p = subprocess.Popen(cmd, shell=True,
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout_data, errors = p.communicate()
    global _ERROR_PRINTED
    if errors and not _ERROR_PRINTED:
        _ERROR_PRINTED = True
        print("ERROR: %s" % errors)
        print("** make sure the development packages of libxml2 and libxslt are installed **\n")
    return decode_input(stdout_data).strip()


def check_min_version(version, min_version, error_name):
    if not version:
        # this is ok for targets like sdist etc.
        return True
    version = tuple(map(int, version.split('.')[:3]))
    min_version = tuple(min_version)
    if version < min_version:
        print("Minimum required version of %s is %s, found %s" % (
            error_name, '.'.join(map(str, version)), '.'.join(map(str, min_version))))
        return False
    return True


def get_library_version(config_tool):
    is_pkgconfig = "pkg-config" in config_tool
    return run_command(config_tool,
                       "--modversion" if is_pkgconfig else "--version")


def get_library_versions():
    xml2_version = get_library_version(find_xml2_config())
    xslt_version = get_library_version(find_xslt_config())
    return xml2_version, xslt_version


def flags(option):
    xml2_flags = run_command(find_xml2_config(), "--%s" % option)
    xslt_flags = run_command(find_xslt_config(), "--%s" % option)

    flag_list = xml2_flags.split()
    for flag in xslt_flags.split():
        if flag not in flag_list:
            flag_list.append(flag)
    return flag_list


def get_xcode_isysroot():
    return run_command('xcrun', '--show-sdk-path')

XSLT_CONFIG = None
XML2_CONFIG = None

def find_xml2_config():
    global XML2_CONFIG
    if XML2_CONFIG:
        return XML2_CONFIG
    option = '--with-xml2-config='
    for arg in sys.argv:
        if arg.startswith(option):
            sys.argv.remove(arg)
            XML2_CONFIG = arg[len(option):]
            return XML2_CONFIG
    else:
        # default: do nothing, rely only on xslt-config
        XML2_CONFIG = os.getenv('XML2_CONFIG', '')
    return XML2_CONFIG

def find_xslt_config():
    global XSLT_CONFIG
    if XSLT_CONFIG:
        return XSLT_CONFIG
    option = '--with-xslt-config='
    for arg in sys.argv:
        if arg.startswith(option):
            sys.argv.remove(arg)
            XSLT_CONFIG = arg[len(option):]
            return XSLT_CONFIG
    else:
        XSLT_CONFIG = os.getenv('XSLT_CONFIG', 'xslt-config')
    return XSLT_CONFIG

## Option handling:

def has_option(name):
    try:
        sys.argv.remove('--%s' % name)
        return True
    except ValueError:
        pass
    # allow passing all cmd line options also as environment variables
    env_val = os.getenv(name.upper().replace('-', '_'), 'false').lower()
    if env_val == "true":
        return True
    return False

def option_value(name):
    for index, option in enumerate(sys.argv):
        if option == '--' + name:
            if index+1 >= len(sys.argv):
                raise DistutilsOptionError(
                    'The option %s requires a value' % option)
            value = sys.argv[index+1]
            sys.argv[index:index+2] = []
            return value
        if option.startswith('--' + name + '='):
            value = option[len(name)+3:]
            sys.argv[index:index+1] = []
            return value
    env_val = os.getenv(name.upper().replace('-', '_'))
    return env_val

staticbuild = bool(os.environ.get('STATICBUILD', ''))
# pick up any commandline options and/or env variables
OPTION_WITHOUT_OBJECTIFY = has_option('without-objectify')
OPTION_WITH_UNICODE_STRINGS = has_option('with-unicode-strings')
OPTION_WITHOUT_ASSERT = has_option('without-assert')
OPTION_WITHOUT_THREADING = has_option('without-threading')
OPTION_WITHOUT_CYTHON = has_option('without-cython')
OPTION_WITH_CYTHON = has_option('with-cython')
OPTION_WITH_CYTHON_GDB = has_option('cython-gdb')
OPTION_WITH_REFNANNY = has_option('with-refnanny')
OPTION_WITH_COVERAGE = has_option('with-coverage')
OPTION_WITH_CLINES = has_option('with-clines')
if OPTION_WITHOUT_CYTHON:
    CYTHON_INSTALLED = False
OPTION_STATIC = staticbuild or has_option('static')
OPTION_DEBUG_GCC = has_option('debug-gcc')
OPTION_SHOW_WARNINGS = has_option('warnings')
OPTION_AUTO_RPATH = has_option('auto-rpath')
OPTION_BUILD_LIBXML2XSLT = staticbuild or has_option('static-deps')
if OPTION_BUILD_LIBXML2XSLT:
    OPTION_STATIC = True
OPTION_LIBXML2_VERSION = option_value('libxml2-version')
OPTION_LIBXSLT_VERSION = option_value('libxslt-version')
OPTION_LIBICONV_VERSION = option_value('libiconv-version')
OPTION_ZLIB_VERSION = option_value('zlib-version')
OPTION_MULTICORE = option_value('multicore')
OPTION_DOWNLOAD_DIR = option_value('download-dir')
if OPTION_DOWNLOAD_DIR is None:
    OPTION_DOWNLOAD_DIR = 'libs'