Add Python3.5 artifact targets

PYTHON-MANIFEST.in

@@ -7,7 +7,7 @@ graft include/grpc
 graft third_party/boringssl
 graft third_party/nanopb
 graft third_party/zlib
-include src/python/grpcio/_unixccompiler_patch.py
+include src/python/grpcio/_spawn_patch.py
 include src/python/grpcio/commands.py
 include src/python/grpcio/grpc_version.py
 include src/python/grpcio/grpc_core_dependencies.py

setup.py

@@ -28,7 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 """A setup module for the GRPC Python package."""
-
+from distutils import cygwinccompiler
 from distutils import extension as _extension
 from distutils import util
 import os
@@ -58,14 +58,12 @@
 sys.path.insert(0, os.path.abspath(PYTHON_STEM))
 
 # Break import-style to ensure we can actually find our in-repo dependencies.
-import _unixccompiler_patch
+import _spawn_patch
 import commands
 import grpc_core_dependencies
 import grpc_version
 
-if 'win32' in sys.platform:
-  _unixccompiler_patch.monkeypatch_unix_compiler()
-
+_spawn_patch.monkeypatch_spawn()
 
 LICENSE = '3-clause BSD'
 
@@ -83,9 +81,38 @@
 # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
 # We use these environment variables to thus get around that without locking
 # ourselves in w.r.t. the multitude of operating systems this ought to build on.
-# By default we assume a GCC-like compiler.
-EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS', ''))
-EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS', ''))
+# We can also use these variables as a way to inject environment-specific
+# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a
+# reasonable default.
+EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None)
+EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None)
+if EXTRA_ENV_COMPILE_ARGS is None:
+  EXTRA_ENV_COMPILE_ARGS = ''
+  if 'win32' in sys.platform and sys.version_info < (3, 5):
+    # We use define flags here and don't directly add to DEFINE_MACROS below to
+    # ensure that the expert user/builder has a way of turning it off (via the
+    # envvars) without adding yet more GRPC-specific envvars.
+    # See https://sourceforge.net/p/mingw-w64/bugs/363/
+    if '32' in platform.architecture()[0]:
+      EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s'
+    else:
+      EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64'
+  elif "linux" in sys.platform or "darwin" in sys.platform:
+    EXTRA_ENV_COMPILE_ARGS += ' -fno-wrapv'
+if EXTRA_ENV_LINK_ARGS is None:
+  EXTRA_ENV_LINK_ARGS = ''
+  if "linux" in sys.platform or "darwin" in sys.platform:
+    EXTRA_ENV_LINK_ARGS += ' -lpthread'
+  elif "win32" in sys.platform and sys.version_info < (3, 5):
+    msvcr = cygwinccompiler.get_msvcr()[0]
+    # TODO(atash) sift through the GCC specs to see if libstdc++ can have any
+    # influence on the linkage outcome on MinGW for non-C++ programs.
+    EXTRA_ENV_LINK_ARGS += (
+        ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} '
+        '-static'.format(msvcr=msvcr))
+
+EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS)
+EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS)
 
 CYTHON_EXTENSION_PACKAGE_NAMES = ()
 
@@ -104,28 +131,29 @@
 if not "win32" in sys.platform:
   EXTENSION_LIBRARIES += ('m',)
 if "win32" in sys.platform:
-  EXTENSION_LIBRARIES += ('ws2_32',)
+  EXTENSION_LIBRARIES += ('advapi32', 'ws2_32',)
 
 DEFINE_MACROS = (
     ('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600),
     ('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),)
 if "win32" in sys.platform:
-  DEFINE_MACROS += (('OPENSSL_WINDOWS', 1), ('WIN32_LEAN_AND_MEAN', 1),)
+  DEFINE_MACROS += (('WIN32_LEAN_AND_MEAN', 1),)
   if '64bit' in platform.architecture()[0]:
     DEFINE_MACROS += (('MS_WIN64', 1),)
+  elif sys.version_info >= (3, 5):
+    # For some reason, this is needed to get access to inet_pton/inet_ntop
+    # on msvc, but only for 32 bits
+    DEFINE_MACROS += (('NTDDI_VERSION', 0x06000000),)
 
 LDFLAGS = tuple(EXTRA_LINK_ARGS)
 CFLAGS = tuple(EXTRA_COMPILE_ARGS)
 if "linux" in sys.platform:
   LDFLAGS += ('-Wl,-wrap,memcpy',)
 if "linux" in sys.platform or "darwin" in sys.platform:
-  CFLAGS += ('-fvisibility=hidden',)
-
-  pymodinit_type = 'PyObject*' if PY3 else 'void'
-
-  pymodinit = '__attribute__((visibility ("default"))) {}'.format(pymodinit_type)
-  DEFINE_MACROS += (('PyMODINIT_FUNC', pymodinit),)
-
+   CFLAGS += ('-fvisibility=hidden',)
+   pymodinit_type = 'PyObject*' if PY3 else 'void'
+   pymodinit = '__attribute__((visibility ("default"))) {}'.format(pymodinit_type)
+   DEFINE_MACROS += (('PyMODINIT_FUNC', pymodinit),)
 
 # By default, Python3 distutils enforces compatibility of
 # c plugins (.so files) with the OSX version Python3 was built with.

src/python/grpcio/_unixccompiler_patch.py → src/python/grpcio/_spawn_patch.py

@@ -27,51 +27,47 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-"""Covers inadequacies in distutils."""
+"""Patches the spawn() command for windows compilers.
+
+Windows has an 8191 character command line limit, but some compilers
+support an @command_file directive where command_file is a file
+containing the full command line.
+"""
 
 from distutils import ccompiler
-from distutils import errors
-from distutils import unixccompiler
 import os
 import os.path
-import shlex
 import shutil
 import sys
 import tempfile
 
-def _unix_commandfile_spawn(self, command):
-  """Wrapper around distutils.util.spawn that attempts to use command files.
+MAX_COMMAND_LENGTH = 8191
 
-  Meant to replace the CCompiler method `spawn` on UnixCCompiler and its
-  derivatives (e.g. the MinGW32 compiler).
+_classic_spawn = ccompiler.CCompiler.spawn
 
-  Some commands like `gcc` (and friends like `clang`) support command files to
-  work around shell command length limits.
-  """
-  # Sometimes distutils embeds the executables as full strings including some
-  # hard-coded flags rather than as lists.
-  command = list(shlex.split(command[0])) + list(command[1:])
-  command_base = os.path.basename(command[0].strip())
-  if command_base == 'ccache':
-    command_base = command[:2]
-    command_args = command[2:]
-  elif command_base.startswith('ccache') or command_base in ['gcc', 'clang', 'clang++', 'g++']:
-    command_base = command[:1]
-    command_args = command[1:]
+def _commandfile_spawn(self, command):
+  command_length = sum([len(arg) for arg in command])
+  if os.name == 'nt' and command_length > MAX_COMMAND_LENGTH:
+    # Even if this command doesn't support the @command_file, it will
+    # fail as is so we try blindly
+    print('Command line length exceeded, using command file')
+    print(' '.join(command))
+    temporary_directory = tempfile.mkdtemp()
+    command_filename = os.path.abspath(
+    os.path.join(temporary_directory, 'command'))
+    with open(command_filename, 'w') as command_file:
+      escaped_args = ['"' + arg.replace('\\', '\\\\') + '"' for arg in command[1:]]
+      command_file.write(' '.join(escaped_args))
+    modified_command = command[:1] + ['@{}'.format(command_filename)]
+    try:
+      _classic_spawn(self, modified_command)
+    finally:
+      shutil.rmtree(temporary_directory)
   else:
-    return ccompiler.CCompiler.spawn(self, command)
-  temporary_directory = tempfile.mkdtemp()
-  command_filename = os.path.abspath(os.path.join(temporary_directory, 'command'))
-  with open(command_filename, 'w') as command_file:
-    escaped_args = [arg.replace('\\', '\\\\') for arg in command_args]
-    command_file.write(' '.join(escaped_args))
-  modified_command = command_base + ['@{}'.format(command_filename)]
-  result = ccompiler.CCompiler.spawn(self, modified_command)
-  shutil.rmtree(temporary_directory)
-  return result
+    _classic_spawn(self, command)
 
 
-def monkeypatch_unix_compiler():
+def monkeypatch_spawn():
   """Monkeypatching is dumb, but it's either that or we become maintainers of
      something much, much bigger."""
-  unixccompiler.UnixCCompiler.spawn = _unix_commandfile_spawn
+  ccompiler.CCompiler.spawn = _commandfile_spawn

tools/distrib/python/grpcio_tools/setup.py

@@ -27,6 +27,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from distutils import cygwinccompiler
 from distutils import extension
 from distutils import util
 import errno
@@ -57,11 +58,38 @@
 # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
 # We use these environment variables to thus get around that without locking
 # ourselves in w.r.t. the multitude of operating systems this ought to build on.
-# By default we assume a GCC-like compiler.
-EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS',
-                                                '-fno-wrapv -frtti -std=c++11'))
-EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS',
-                                             '-lpthread'))
+# We can also use these variables as a way to inject environment-specific
+# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a
+# reasonable default.
+EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None)
+EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None)
+if EXTRA_ENV_COMPILE_ARGS is None:
+  EXTRA_ENV_COMPILE_ARGS = '-std=c++11'
+  if 'win32' in sys.platform and sys.version_info < (3, 5):
+    # We use define flags here and don't directly add to DEFINE_MACROS below to
+    # ensure that the expert user/builder has a way of turning it off (via the
+    # envvars) without adding yet more GRPC-specific envvars.
+    # See https://sourceforge.net/p/mingw-w64/bugs/363/
+    if '32' in platform.architecture()[0]:
+      EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s'
+    else:
+      EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64'
+  elif "linux" in sys.platform or "darwin" in sys.platform:
+    EXTRA_ENV_COMPILE_ARGS += ' -fno-wrapv -frtti'
+if EXTRA_ENV_LINK_ARGS is None:
+  EXTRA_ENV_LINK_ARGS = ''
+  if "linux" in sys.platform or "darwin" in sys.platform:
+    EXTRA_ENV_LINK_ARGS += ' -lpthread'
+  elif "win32" in sys.platform and sys.version_info < (3, 5):
+    msvcr = cygwinccompiler.get_msvcr()[0]
+    # TODO(atash) sift through the GCC specs to see if libstdc++ can have any
+    # influence on the linkage outcome on MinGW for non-C++ programs.
+    EXTRA_ENV_LINK_ARGS += (
+        ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} '
+        '-static'.format(msvcr=msvcr))
+
+EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS)
+EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS)
 
 CC_FILES = [
   os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES]
@@ -73,9 +101,13 @@
 GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools'
 GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto'
 
-DEFINE_MACROS = (('HAVE_PTHREAD', 1),)
-if "win32" in sys.platform and '64bit' in platform.architecture()[0]:
-  DEFINE_MACROS += (('MS_WIN64', 1),)
+DEFINE_MACROS = ()
+if "win32" in sys.platform:
+  DEFINE_MACROS += (('WIN32_LEAN_AND_MEAN', 1),)
+  if '64bit' in platform.architecture()[0]:
+    DEFINE_MACROS += (('MS_WIN64', 1),)
+elif "linux" in sys.platform or "darwin" in sys.platform:
+  DEFINE_MACROS += (('HAVE_PTHREAD', 1),)
 
 # By default, Python3 distutils enforces compatibility of
 # c plugins (.so files) with the OSX version Python3 was built with.

tools/distrib/python/make_grpcio_tools.py

@@ -88,22 +88,23 @@
     os.path.join(os.path.dirname(os.path.abspath(__file__)),
                  '..', '..', '..'))
 
-GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools/distrib/python/grpcio_tools')
+GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools', 'distrib',
+                                'python', 'grpcio_tools')
 
-GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = 'third_party/protobuf/src'
+GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = os.path.join('third_party', 'protobuf', 'src')
 GRPC_PROTOBUF = os.path.join(GRPC_ROOT, GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT)
-GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src/compiler')
-GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT,
-                                    'third_party/protobuf/src')
-GRPC_PYTHON_PROTOC_PLUGINS = os.path.join(GRPC_PYTHON_ROOT,
-                                          'grpc_root/src/compiler')
+GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src', 'compiler')
+GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT, 'third_party', 'protobuf',
+                                    'src')
+GRPC_PYTHON_PROTOC_PLUGINS = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root', 'src',
+                                          'compiler')
 GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join(GRPC_PYTHON_ROOT,
                                            'protoc_lib_deps.py')
 
 GRPC_INCLUDE = os.path.join(GRPC_ROOT, 'include')
-GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root/include')
+GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root', 'include')
 
-BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools/distrib/python/bazel_deps.sh')
+BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python', 'bazel_deps.sh')
 BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib'
 BAZEL_DEPS_COMMON_PROTOS_QUERY = '//:well_known_protos'
 
@@ -136,64 +137,6 @@ def long_path(path):
   else:
     return path
 
-def atomic_file_copy(src, dst):
-  """Based on the lock-free-whack-a-mole algorithm, depending on filesystem
-     renaming being atomic. Described at http://stackoverflow.com/a/28090883.
-  """
-  try:
-    if filecmp.cmp(src, dst):
-      return
-  except:
-    pass
-  dst_dir = os.path.abspath(os.path.dirname(dst))
-  dst_base = os.path.basename(dst)
-  this_id = str(uuid.uuid4()).replace('.', '-')
-  temporary_file = os.path.join(dst_dir, '{}.{}.tmp'.format(dst_base, this_id))
-  mole_file = os.path.join(dst_dir, '{}.{}.mole.tmp'.format(dst_base, this_id))
-  mole_pattern = os.path.join(dst_dir, '{}.*.mole.tmp'.format(dst_base))
-  src = long_path(src)
-  dst = long_path(dst)
-  temporary_file = long_path(temporary_file)
-  mole_file = long_path(mole_file)
-  mole_pattern = long_path(mole_pattern)
-  shutil.copy2(src, temporary_file)
-  try:
-    os.rename(temporary_file, mole_file)
-  except:
-    print('Error moving temporary file {} to {}'.format(temporary_file, mole_file), file=sys.stderr)
-    print('while trying to copy file {} to {}'.format(src, dst), file=sys.stderr)
-    raise
-  for other_file in glob.glob(mole_pattern):
-    other_id = other_file.split('.')[-3]
-    if this_id == other_id:
-      pass
-    elif this_id < other_id:
-      try:
-        os.remove(other_file)
-      except:
-        pass
-    else:
-      try:
-        os.remove(mole_file)
-      except:
-        pass
-      this_id = other_id
-      mole_file = other_file
-  try:
-    if filecmp.cmp(src, dst):
-      try:
-        os.remove(mole_file)
-      except:
-        pass
-      return
-  except:
-    pass
-  try:
-    os.rename(mole_file, dst)
-  except:
-    pass
-
-
 def main():
   os.chdir(GRPC_ROOT)
 
@@ -211,7 +154,7 @@ def main():
       for relative_file in files:
         source_file = os.path.abspath(os.path.join(source_dir, relative_file))
         target_file = os.path.abspath(os.path.join(target_dir, relative_file))
-        atomic_file_copy(source_file, target_file)
+        shutil.copyfile(source_file, target_file)
 
   try:
     protoc_lib_deps_content = get_deps()