Skip to content

Commit

Permalink
Use precompiled extensions only
Browse files Browse the repository at this point in the history
  • Loading branch information
soltanmm-google committed Feb 12, 2016
1 parent 9b0f81b commit f747409
Show file tree
Hide file tree
Showing 5 changed files with 128 additions and 121 deletions.
1 change: 1 addition & 0 deletions PYTHON-MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ graft third_party/zlib
include src/python/grpcio/commands.py
include src/python/grpcio/grpc_version.py
include src/python/grpcio/grpc_core_dependencies.py
include src/python/grpcio/precompiled.py
include src/python/grpcio/support.py
include src/python/grpcio/README.rst
include requirements.txt
Expand Down
42 changes: 24 additions & 18 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@

# Break import-style to ensure we can actually find our in-repo dependencies.
import commands
import precompiled
import grpc_core_dependencies
import grpc_version

Expand Down Expand Up @@ -156,15 +157,14 @@ def cython_extensions(package_names, module_names, extra_sources, include_dirs,
) + INSTALL_REQUIRES

COMMAND_CLASS = {
'install': commands.Install,
'doc': commands.SphinxDocumentation,
'build_proto_modules': commands.BuildProtoModules,
'build_project_metadata': commands.BuildProjectMetadata,
'build_py': commands.BuildPy,
'build_ext': commands.BuildExt,
'build_tagged_ext': precompiled.BuildTaggedExt,
'gather': commands.Gather,
'run_interop': commands.RunInterop,
'bdist_wheel_grpc_custom': commands.BdistWheelCustomName,
}

# Ensure that package data is copied over before any commands have been run:
Expand Down Expand Up @@ -205,9 +205,12 @@ def cython_extensions(package_names, module_names, extra_sources, include_dirs,
'grpc._adapter': [
'credentials/roots.pem'
],
# Binaries that may or may not be present in the final installation, but are
# mentioned here for completeness.
'grpc._cython': [
'_windows/grpc_c.32.python',
'_windows/grpc_c.64.python',
'cygrpc.so',
],
}
if INSTALL_TESTS:
Expand All @@ -217,19 +220,22 @@ def cython_extensions(package_names, module_names, extra_sources, include_dirs,
PACKAGES = setuptools.find_packages(
PYTHON_STEM, exclude=['tests', 'tests.*'])

setuptools.setup(
name='grpcio',
version=grpc_version.VERSION,
license=LICENSE,
ext_modules=CYTHON_EXTENSION_MODULES,
packages=list(PACKAGES),
package_dir=PACKAGE_DIRECTORIES,
package_data=PACKAGE_DATA,
install_requires=INSTALL_REQUIRES,
setup_requires=SETUP_REQUIRES,
cmdclass=COMMAND_CLASS,
tests_require=TESTS_REQUIRE,
test_suite=TEST_SUITE,
test_loader=TEST_LOADER,
test_runner=TEST_RUNNER,
)
setup_arguments = {
'name': 'grpcio',
'version': grpc_version.VERSION,
'license': LICENSE,
'ext_modules': CYTHON_EXTENSION_MODULES,
'packages': list(PACKAGES),
'package_dir': PACKAGE_DIRECTORIES,
'package_data': PACKAGE_DATA,
'install_requires': INSTALL_REQUIRES,
'setup_requires': SETUP_REQUIRES,
'cmdclass': COMMAND_CLASS,
'tests_require': TESTS_REQUIRE,
'test_loader': TEST_LOADER,
'test_runner': TEST_RUNNER,
}

precompiled.update_setup_arguments(setup_arguments)

setuptools.setup(**setup_arguments)
102 changes: 0 additions & 102 deletions src/python/grpcio/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,21 +46,11 @@
from setuptools.command import easy_install
from setuptools.command import install
from setuptools.command import test
from wheel import bdist_wheel

import support

PYTHON_STEM = os.path.dirname(os.path.abspath(__file__))

BINARIES_REPOSITORY = os.environ.get(
'GRPC_PYTHON_BINARIES_REPOSITORY',
'https://storage.googleapis.com/grpc-precompiled-binaries/python')

USE_GRPC_CUSTOM_BDIST = bool(int(os.environ.get(
'GRPC_PYTHON_USE_CUSTOM_BDIST', '1')))

GRPC_CUSTOM_BDIST_EXT = '.whl'

CONF_PY_ADDENDUM = """
extensions.append('sphinx.ext.napoleon')
napoleon_google_docstring = True
Expand Down Expand Up @@ -111,98 +101,6 @@ def _get_grpc_custom_bdist(decorated_basename, target_bdist_basename):
return bdist_path


class WheelNameMixin(object):
"""Mixin for setuptools.Command classes to enable acquiring the bdist name."""

def wheel_custom_name(self):
base = self.wheel_name()
# Drop troublesome parts of the target tuple
base_split = base.split('-')
base = '-'.join(base_split[0:3] + base_split[4:])
flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4'
return '{base}-{flavor}'.format(base=base, flavor=flavor)

def wheel_name(self):
wheel_command = self.get_finalized_command('bdist_wheel')
return wheel_command.get_archive_basename()


class Install(install.install, WheelNameMixin):
"""Custom Install command for gRPC Python.
This is for bdist shims and whatever else we might need a custom install
command for.
"""

user_options = install.install.user_options + [
# TODO(atash): remove this once PyPI has better Linux bdist support. See
# https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
('use-grpc-custom-bdist', None,
'Whether to retrieve a binary from the gRPC binary repository instead '
'of building from source.'),
]

def initialize_options(self):
install.install.initialize_options(self)
self.use_grpc_custom_bdist = USE_GRPC_CUSTOM_BDIST

def finalize_options(self):
install.install.finalize_options(self)

def run(self):
if self.use_grpc_custom_bdist:
try:
try:
bdist_path = _get_grpc_custom_bdist(self.wheel_custom_name(),
self.wheel_name())
except CommandError as error:
sys.stderr.write(
'\nWARNING: Failed to acquire grpcio prebuilt binary:\n'
'{}.\n\n'.format(error.message))
raise
try:
self._run_bdist_retrieval_install(bdist_path)
except Exception as error:
# if anything else happens (and given how there's no way to really know
# what's happening in setuptools here, I mean *anything*), warn the user
# and fall back to building from source.
sys.stderr.write(
'{}\nWARNING: Failed to install grpcio prebuilt binary.\n\n'
.format(traceback.format_exc()))
raise
except Exception:
install.install.run(self)
else:
install.install.run(self)

# TODO(atash): Remove this once PyPI has better Linux bdist support. See
# https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
def _run_bdist_retrieval_install(self, bdist_path):
import pip
pip.main(['install', bdist_path])


class BdistWheelCustomName(bdist_wheel.bdist_wheel, WheelNameMixin):
"""Thin wrapper around the bdist command to build with our custom name."""

description = ("Create a gRPC custom-named wheel distribution. "
"Cannot be run with any other distribution-related command.")

def run(self):
# TODO(atash): if the hack we use to support Linux binaries becomes
# 'supported' (i.e.
# https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
# is not solved and we see users beginning to use this command, ill-advised
# as that may be) consider making the following capable of running with
# other distribution-related commands. Currently it depends on the (AFAIK
# undocumented, private) ordering of the distribution files.
bdist_wheel.bdist_wheel.run(self)
output = self.distribution.dist_files[-1][2]
target = os.path.join(
self.dist_dir, '{}.whl'.format(self.wheel_custom_name()))
shutil.move(output, target)


class SphinxDocumentation(setuptools.Command):
"""Command to generate documentation via sphinx."""

Expand Down
102 changes: 102 additions & 0 deletions src/python/grpcio/precompiled.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
# Copyright 2015-2016, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import platform
import shutil
import sys

import setuptools

import commands
import grpc_version

try:
from urllib2 import urlopen
except ImportError:
from urllib.request import urlopen

PYTHON_STEM = os.path.dirname(os.path.abspath(__file__))
BINARIES_REPOSITORY = os.environ.get(
'GRPC_PYTHON_BINARIES_REPOSITORY',
'https://storage.googleapis.com/grpc-precompiled-binaries/python')
USE_PRECOMPILED_BINARIES = bool(int(os.environ.get(
'GRPC_PYTHON_USE_PRECOMPILED_BINARIES', '1')))

def _tagged_ext_name(base):
uname = platform.uname()
tags = '-'.join((grpc_version.VERSION, uname[0], uname[4]))
flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4'
return '{base}-{tags}-{flavor}'.format(base=base, tags=tags, flavor=flavor)


class BuildTaggedExt(setuptools.Command):

description = 'build the gRPC tagged extensions'
user_options = []

def initialize_options(self):
# distutils requires this override.
pass

def finalize_options(self):
# distutils requires this override.
pass

def run(self):
if 'linux' in sys.platform:
self.run_command('build_ext')
try:
os.makedirs('dist/')
except OSError:
pass
shutil.copyfile(
os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so'),
'dist/{}.so'.format(_tagged_ext_name('cygrpc')))
else:
sys.stderr.write('nothing to do for build_tagged_ext\n')


def update_setup_arguments(setup_arguments):
url = '{}/{}.so'.format(BINARIES_REPOSITORY, _tagged_ext_name('cygrpc'))
target_path = os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so')
try:
extension = urlopen(url).read()
except:
sys.stderr.write(
'could not download precompiled extension: {}\n'.format(url))
return
try:
with open(target_path, 'w') as target:
target.write(extension)
setup_arguments['ext_modules'] = []
except:
sys.stderr.write(
'could not write precompiled extension to directory: {} -> {}\n'
.format(url, target_path))
2 changes: 1 addition & 1 deletion tools/run_tests/build_artifact_python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ fi
GRPC_PYTHON_USE_CUSTOM_BDIST=0 \
GRPC_PYTHON_BUILD_WITH_CYTHON=1 \
${SETARCH_CMD} python setup.py \
bdist_wheel_grpc_custom
build_tagged_ext

GRPC_PYTHON_USE_CUSTOM_BDIST=0 \
GRPC_PYTHON_BUILD_WITH_CYTHON=1 \
Expand Down

0 comments on commit f747409

Please sign in to comment.