diff --git a/.circleci/config.yml b/.circleci/config.yml index bca5712..8ea0052 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,80 +1,29 @@ version: 2 jobs: build: - working_directory: /tmp/vendored docker: - - image: nathanleclaire/curl:latest - steps: - - deploy: - name: Run lint job - command: | - set -x - # Only run if on the CircleCI cloud instance - if [ "$CIRCLE_BUILD_NUM" ]; then - # Gross method to have Circle trigger the job - # https://circleci.com/docs/2.0/defining-multiple-jobs/#triggering-jobs - curl -u ${CIRCLE_API_TOKEN}: \ - -d build_parameters[CIRCLE_JOB]=lint \ - -d revision=$CIRCLE_SHA1 \ - https://circleci.com/api/v1.1/project/github/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/tree/$CIRCLE_BRANCH - fi - - deploy: - name: Run spell check job - command: | - set -x - # Only run if on the CircleCI cloud instance - if [ "$CIRCLE_BUILD_NUM" ]; then - # Gross method to have Circle trigger the job - # https://circleci.com/docs/2.0/defining-multiple-jobs/#triggering-jobs - curl -u ${CIRCLE_API_TOKEN}: \ - -d build_parameters[CIRCLE_JOB]=spell-check \ - -d revision=$CIRCLE_SHA1 \ - https://circleci.com/api/v1.1/project/github/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/tree/$CIRCLE_BRANCH - fi - lint: - working_directory: /tmp/vendored - docker: - - image: clburlison/pylint:py2-wheezy - steps: - - checkout - - run: - name: Run tests - command: | - set -x - # Only run lint tests if a python file changed - changed_files="$(git --no-pager diff --name-only origin/master)" - if [[ $changed_files =~ .py ]]; then - echo "A python file was modified. Running python linter..." - flake8 --exclude=_src*,_patch*,payload,build . - fi - spell-check: - working_directory: /tmp/vendored - docker: - - image: clburlison/pylint:py2-wheezy + - image: circleci/python:2.7-jessie + working_directory: ~/repo steps: - checkout + # Download and cache dependencies + - restore_cache: + keys: + - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- - run: - name: Run spell check + name: install dependencies command: | - set -x - #set +e # Add this to override spelling errors failing CI tests. - # Then add an `exit 0` at the end of the run - pylint --disable=all --reports=n --enable=spelling \ - --spelling-dict=en_US --ignore-comments=no \ - --spelling-private-dict-file=tests/words \ - build.py build openssl python tests tlsssl vendir - add-words: - working_directory: /tmp/vendored - docker: - - image: clburlison/pylint:py2-wheezy - steps: - - checkout + virtualenv venv + . venv/bin/activate + pip install -r .circleci/requirements.txt + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - run: - name: Add unknown words to 'tests/words' + name: run linting command: | - set -x - pylint --disable=all --reports=n --enable=spelling \ - --spelling-dict=en_US --ignore-comments=no \ - --spelling-store-unknown-words=y \ - --spelling-private-dict-file=tests/words \ - build.py build openssl python tests tlsssl vendir + . venv/bin/activate + flake8 diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt new file mode 100644 index 0000000..2aa560f --- /dev/null +++ b/.circleci/requirements.txt @@ -0,0 +1,2 @@ +flake8==3.5.0 +flake8_docstrings==1.3.0 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..6827030 --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] +# ignore = F401,F405,E402,F403 +# max-line-length = 100 +exclude = venv/*, code/tlsssl/_diffs/*, code/tlsssl/build/*, code/tlsssl/_patch/*, code/tlsssl/_src/*, code/tlsssl/payload/* diff --git a/README.md b/README.md index 0929bf5..1ddac27 100644 --- a/README.md +++ b/README.md @@ -6,19 +6,23 @@ The goal of this repo is to make it easy to "vendor" your own frameworks and programming languages in an automated fashion. -Once this project is complete you will be able to have own version of python, ruby, pyojbc bridge, OpenSSL, and more all in one nice big package or multiple smaller packages for easy deployment. +Once this project is complete you will be able to have own version of python with custom pip modules included, ruby, OpenSSL, and more all in one nice big package or multiple smaller packages for easy deployment. To keep track of progress look at the [Master list](https://github.com/clburlison/vendored/issues/1) # Usage -Currently parts of this project are working. You can run `./build.py` to build and optionally package some of these pieces. Or `cd` into one of the subfolders and run `python setup.py` directly (the help is quite 'helpful'). This will give you the most control at this point until the build script matures and has more arguments added. +Currently parts of this project are working. Make sure you `cd code`. You can run `sudo ./build.py` to build and optionally package some of these pieces. Or `cd` into one of the sub-folders and run `sudo python setup.py` directly (the help is quite 'helpful'). This will give you the most control at this point until the build script matures and has more arguments added. + +It is recommended that you build this project in a clean environnement IE - a virtual machine. That will give you the best results and make sure your packages are clean. + +## Requires: -### Requires: * Apple Command Line Tools (installable with `xcode-select --install`) * Python 2 -### Override +## Override + vendored was created to be customizable. As such, it is possible to override almost every option. These overrides live in the `config.ini` file. The `config.ini` file contains two sections: @@ -31,14 +35,8 @@ A sample is shown below: pkgid: com.clburlison sign_cert_cn: Developer ID Installer: Clayton Burlison -## Creating a patch -Some of these tools require patch files for compiling. If you're unfamiliar with creating a patch file the basics look a little something like: - -```bash -diff -u hello.c hello_new.c > hello.c.patch -``` -# Working with the CI +## Working with the CI CircleCI is set to run tests on this repo. Coding should follow the [PEP 8](https://www.python.org/dev/peps/pep-0008/) style guide and is verified with [`flake8`](https://pypi.python.org/pypi/flake8). While correct spellings for code is verified with enchant using the python-enchant module. To run the CI tests locally the following tools must be installed: @@ -52,27 +50,11 @@ To run the CI tests locally the following tools must be installed: To run the lint job: ```bash -circleci build --job lint +circleci build ``` -## Spelling CI -As with all projects some words that are used will not be part of the standard 'en_US' dictionary and are added to a custom file located in `tests/words`. - -To run the spell-check job: - -```bash -circleci build --job spell-check -``` - -To run the add-words job: - -_Note:_ This will add new words to 'tests/words' so please run `spell-check` first - -```bash -circleci build --job add-words -``` +## Credits -# Credits Huge thanks to... * the [Google MacOps](https://github.com/google/macops/) team for open sourcing their solution * [@pudquick](https://github.com/pudquick) for his work on tlsssl so we can patch the native Python 2.7 that ships on macOS @@ -87,6 +69,6 @@ This project uses works from: [EmojiOne](http://emojione.com/) | [briefcase](https://github.com/Ranks/emojione/blob/master/assets/png_512x512/1f4bc.png?raw=true) [Ignace Mouzannar](http://ghantoos.org/) | [CI python spell checks post](http://ghantoos.org/2016/02/21/continuous-integration-python-comments-spellchecks-with-pylint-pyenchant-and-tox/) -# License +## License This project uses the MIT License. diff --git a/build.py b/code/build.py similarity index 81% rename from build.py rename to code/build.py index 8d16933..66d0f3b 100755 --- a/build.py +++ b/code/build.py @@ -1,10 +1,10 @@ #!/usr/bin/python -""" -build.py - main program for vendored -""" +"""Build vendored packages.""" import os +from vendir import root + CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -12,7 +12,7 @@ def build_openssl(*args): """Build the openssl project.""" openssl_dir = os.path.join(CURRENT_DIR, 'openssl') os.chdir(openssl_dir) - cmd = ['/usr/bin/python', 'setup.py', '-vv', '-p', '-b', '-s'] + cmd = ['/usr/bin/python', 'setup.py', '-vv', '-p', '-b', '-i'] os.system(' '.join(cmd)) @@ -33,10 +33,10 @@ def build_tlsssl(): def main(): - """Main routine""" + """Build our required packages.""" + root.root_check() build_openssl() build_python() - build_tlsssl() if __name__ == '__main__': diff --git a/build/distribution.plist b/code/build/distribution.plist similarity index 63% rename from build/distribution.plist rename to code/build/distribution.plist index 8ab263b..72d2402 100644 --- a/build/distribution.plist +++ b/code/build/distribution.plist @@ -19,18 +19,22 @@ - - + + openssl-1.0.2n.pkg + - openssl-1.0.2k.pkg - + + + python-2.7.14.pkg + - python-2.7.13.pkg - + + + tlsssl-1.1.0.pkg + - tlsssl-1.0.0.pkg - tlsssl-1.1.0.pkg + diff --git a/build/resources/logo.png b/code/build/resources/logo.png similarity index 100% rename from build/resources/logo.png rename to code/build/resources/logo.png diff --git a/build/resources/welcome.rtf b/code/build/resources/welcome.rtf similarity index 100% rename from build/resources/welcome.rtf rename to code/build/resources/welcome.rtf diff --git a/build/temp_build.sh b/code/build/temp_build.sh similarity index 66% rename from build/temp_build.sh rename to code/build/temp_build.sh index 31deeda..1a57ae4 100755 --- a/build/temp_build.sh +++ b/code/build/temp_build.sh @@ -1,8 +1,8 @@ #!/bin/bash # /usr/bin/productbuild --synthesize \ -# --package ../openssl/openssl-1.0.2k.pkg \ -# --package ../python/python-2.7.13.pkg \ -# --package ../tlsssl/tlsssl-1.0.0.pkg \ +# --package ../openssl/openssl-1.0.2n.pkg \ +# --package ../python/python-2.7.14.pkg \ +# --package ../tlsssl/tlsssl-1.1.0.pkg \ # distribution.plist diff --git a/config.ini b/code/config.ini similarity index 73% rename from config.ini rename to code/config.ini index decafa2..4b637ed 100644 --- a/config.ini +++ b/code/config.ini @@ -16,20 +16,17 @@ pth_fname: 000vendored.pth # the temporary build directory for openssl. Can be relative or absolute file paths openssl_build_dir: /tmp/build-openssl # the url path for openssl distribution, hash & version -openssl_dist: https://www.openssl.org/source/openssl-1.0.2k.tar.gz -openssl_dist_hash: 6b3977c61f2aedf0f96367dcfb5c6e578cf37e7b8d913b4ecb6643c3cb88d8c0 -openssl_version: 1.0.2k +openssl_dist: https://www.openssl.org/source/openssl-1.0.2n.tar.gz +openssl_dist_hash: 370babb75f278c39e0c50e8c4e7493bc0f18db6867478341a832a982fd15a8fe +openssl_version: 1.0.2n ############## python variables ############## -# the temporary build directory for openssl. Can be relative or absolute file paths +# the temporary build directory for python. Can be relative or absolute file paths python_build_dir: /tmp/build-python # the url path for python2 distribution, hash & version -python2_dist: https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tar.xz -python2_dist_hash: 35d543986882f78261f97787fd3e06274bfa6df29fac9b4a94f73930ff98f731 -python2_version: 2.7.13 -# the specific version of PyOjbC that you want pip to install. The latest can -# be found at https://pypi.python.org/pypi/pyobjc -python2_objc_version: 3.2.1 +python2_dist: https://www.python.org/ftp/python/2.7.14/Python-2.7.14.tar.xz +python2_dist_hash: 71ffb26e09e78650e424929b2b457b9c912ac216576e6bd9e7d204ed03296a66 +python2_version: 2.7.14 ############## tlsssl variables ############## # The install directory that tlsssl will be installed into diff --git a/openssl/setup.py b/code/openssl/setup.py similarity index 90% rename from openssl/setup.py rename to code/openssl/setup.py index 8b3b5f3..0ed181c 100644 --- a/openssl/setup.py +++ b/code/openssl/setup.py @@ -1,5 +1,6 @@ """ -Setup script to compile OpenSSL 1.0.2 for macOS +Setup script to compile OpenSSL 1.0.2 for macOS. + NOTE: OpenSSL 1.1 is not supported at this time due to large API changes. However OpenSSL version 1.0.2 is supported by the OpenSSL Software Foundation until 2019-12-31 (LTS). @@ -26,6 +27,7 @@ from vendir import log # noqa from vendir import package # noqa from vendir import runner # noqa +from vendir import root # noqa CONFIG = config.ConfigSectionMap() @@ -37,8 +39,7 @@ def download_and_extract_openssl(): - """Download openssl distribution from the internet and extract it to - openssl_build_dir.""" + """Download openssl distribution and extract it to OPENSSL_BUILD_DIR.""" if os.path.isdir(OPENSSL_BUILD_DIR): shutil.rmtree(OPENSSL_BUILD_DIR, ignore_errors=True) mkpath(OPENSSL_BUILD_DIR) @@ -54,7 +55,7 @@ def download_and_extract_openssl(): # We are calling os.system so we can get download progress live rc = runner.system(cmd) if rc == 0 or rc is True: - log.debug("OpenSSL download sucessful") + log.debug("OpenSSL download successfully") else: log.error("OpenSSL download failed with exit code: '{}'".format(rc)) sys.exit(1) @@ -68,7 +69,7 @@ def download_and_extract_openssl(): download_hash, config_hash)) sys.exit(1) else: - log.detail("Hash verification of OpenSSL sucessful") + log.detail("Hash verification of OpenSSL successfully") # Extract openssl to the openssl_build_dir log.info("Extracting OpenSSL...") @@ -76,14 +77,14 @@ def download_and_extract_openssl(): '--strip-components', '1'] out = runner.Popen(cmd) if out[2] == 0: - log.debug("Extraction completed sucessfully") + log.debug("Extraction completed successfullyly") else: log.error("Extraction has failed: {}".format(out[1])) os.remove(temp_filename) def build(): - """This is the main processing step that builds openssl from source""" + """Build OpenSSL from source.""" # Step 1: change into our build directory os.chdir(OPENSSL_BUILD_DIR) # Don't compile openssl if the skip option is passed @@ -155,8 +156,10 @@ def build(): sys.stdout.flush() # does this help? -def post_install(): +def current_certs(): """ + Include current SystemRoot certs with OpenSSL. + This helps work around a limitation with bundling your own version of OpenSSL. We copy the certs from Apple's 'SystemRootCertificates.keychain' into OPENSSL_BUILD_DIR/cert.pem @@ -178,7 +181,7 @@ def post_install(): def main(): - """Main routine""" + """Build and package OpenSSL.""" parser = argparse.ArgumentParser(prog='OpenSSL setup', description='This script will compile ' 'OpenSSL 1.0.1+ and optionally create ' @@ -190,6 +193,8 @@ def main(): 'for development purposes.') parser.add_argument('-p', '--pkg', action='store_true', help='Package the OpenSSL output directory.') + parser.add_argument('-i', '--install', action='store_true', + help='Install the OpenSSL package.') parser.add_argument('-v', '--verbose', action='count', default=1, help="Increase verbosity level. Repeatable up to " "2 times (-vv)") @@ -202,6 +207,8 @@ def main(): log.verbose = args.verbose skip = args.skip + root.root_check() + if args.build: log.info("Bulding OpenSSL...") check_dir = os.path.isdir(PKG_PAYLOAD_DIR) @@ -213,7 +220,7 @@ def main(): else: download_and_extract_openssl() build() - post_install() + current_certs() if args.pkg: log.info("Building a package for OpenSSL...") @@ -231,6 +238,13 @@ def main(): else: log.error("Looks like package creation failed") + if args.install: + log.info("Installing OpenSSL pacakge...") + os.chdir(CURRENT_DIR) + cmd = ['/usr/sbin/installer', '-pkg', + 'openssl-{}.pkg'.format(version), '-tgt', '/'] + runner.Popen(cmd) + if __name__ == '__main__': main() diff --git a/code/python/README.md b/code/python/README.md new file mode 100644 index 0000000..2990df7 --- /dev/null +++ b/code/python/README.md @@ -0,0 +1,26 @@ +Better docs required... + + +# Resources: +* https://github.com/Homebrew/homebrew-core/blob/master/Formula/python.rb +* https://github.com/Homebrew/homebrew-core/blob/master/Formula/readline.rb +* https://github.com/saltstack/salt/blob/develop/pkg/osx/ +* https://github.com/Homebrew/brew/blob/master/docs/Formula-Cookbook.md + + + diff --git a/code/python/requirements.txt b/code/python/requirements.txt new file mode 100644 index 0000000..9116c6b --- /dev/null +++ b/code/python/requirements.txt @@ -0,0 +1,5 @@ +# pyobjc==2.5.1 # Stock version on macOS 10.13 +pyobjc==4.1 # Latest version as of 2017-12-06 +xattr==0.6.4 +pyOpenSSL==17.5.0 +readline==6.2.4.1 diff --git a/python/setup.py b/code/python/setup.py similarity index 54% rename from python/setup.py rename to code/python/setup.py index 5d56873..dd95bcb 100644 --- a/python/setup.py +++ b/code/python/setup.py @@ -1,7 +1,4 @@ -""" -Setup script to compile Python for macOS, includes: -* PyObjC bridge -""" +"""Setup script to compile Python2 for macOS.""" # standard libs from distutils.dir_util import mkpath @@ -11,7 +8,6 @@ import inspect import tempfile import argparse -import urllib2 # our libs. kind of hacky since this isn't a valid python package. CURRENT_DIR = os.path.dirname( @@ -24,24 +20,20 @@ from vendir import log # noqa from vendir import package # noqa from vendir import runner # noqa +from vendir import root # noqa CONFIG = config.ConfigSectionMap() PYTHON_BUILD_DIR = os.path.abspath(CONFIG['python_build_dir']) BASE_INSTALL_PATH = CONFIG['base_install_path'] BASE_INSTALL_PATH_S = CONFIG['base_install_path'].lstrip('/') -PKG_PAYLOAD_DIR = os.path.join(PYTHON_BUILD_DIR, 'payload') PYTHON2_VERSION = CONFIG['python2_version'] -SRC_DIR = os.path.join(CURRENT_DIR, '_src') -PATCH_DIR = os.path.join(CURRENT_DIR, '_patch') -PYTHON2_INSTALL = os.path.join(PYTHON_BUILD_DIR, 'payload', - BASE_INSTALL_PATH_S, 'Python', '2.7') +PYTHON2_INSTALL = os.path.join(BASE_INSTALL_PATH, 'Python', '2.7') OPENSSL_INSTALL_PATH = os.path.join(CONFIG['base_install_path'], 'openssl') def dl_and_extract_python(): - """Download Python distribution from the internet and extract it to - PYTHON_BUILD_DIR.""" + """Download Python distribution and extract it to PYTHON_BUILD_DIR.""" if os.path.isdir(PYTHON_BUILD_DIR): shutil.rmtree(PYTHON_BUILD_DIR, ignore_errors=True) mkpath(PYTHON_BUILD_DIR) @@ -57,7 +49,7 @@ def dl_and_extract_python(): # We are calling os.system so we can get download progress live rc = runner.system(cmd) if rc == 0 or rc is True: - log.debug("Python download sucessful") + log.debug("Python download successful") else: log.error("Python download failed with exit code: '{}'".format(rc)) sys.exit(1) @@ -71,7 +63,7 @@ def dl_and_extract_python(): download_hash, config_hash)) sys.exit(1) else: - log.detail("Hash verification of Python sucessful") + log.detail("Hash verification of Python successful") # Extract Python to the PYTHON_BUILD_DIR log.info("Extracting Python...") @@ -79,72 +71,14 @@ def dl_and_extract_python(): '--strip-components', '1'] out = runner.Popen(cmd) if out[2] == 0: - log.debug("Extraction completed sucessfully") + log.debug("Extraction completed successfullyly") else: log.error("Extraction has failed: {}".format(out[0])) os.remove(temp_filename) -def dl_apple_patch_files(): - """ - Download patches from Apple Opensource page. - https://opensource.apple.com/source/python/python-97/2.7/fix/ - """ - log.info("Downloading and verifying python source files...") - if not os.path.exists(SRC_DIR): - log.debug("Creating _src directory...") - mkpath(SRC_DIR) - os.chdir(SRC_DIR) - os_url = ('https://opensource.apple.com/source/python/python-97/2.7/fix/') - configure_ed = ( - '3580144bc552fd9b70160b540b35af8ab18e15b592235e4c0731090c6dd98895') - setup_py_ed = ( - '9db0803df2d816facf03b7879a5f6cca425b3e9513b60023323676d8c612d93d') - readline_c_ed = ( - '96ff20308b223e22739f9942683bf8f36825e2bf0c426a2edcb6d741b56ff06f') - setup_py_patch = ( - 'c6bcd396cab445c3c7aed293720c7936ef773f0649849b658bba177046412f97') - # This ugly looking block of code is a pair that matches the filename, - # github url, and sha256 hash for each required python source file - fp = [ - ['configure.ed', '{}configure.ed'.format(os_url), configure_ed], - ['setup.py.ed', '{}setup.py.ed'.format(os_url), setup_py_ed], - ['readline.c.ed', '{}readline.c.ed'.format(os_url), readline_c_ed], - ['setup.py.patch', '{}setup.py.patch'.format(os_url), setup_py_patch], - ] - # Verify we have the correct python source files else download it - log.detail("Downloading & checking hash of python patch files...") - for fname, url, sha256 in fp: - # This is a dual check step for file existence and hash matching - log.debug("Checking source file: {}...".format(fname)) - if not os.path.isfile(fname) or ( - hash_helper.getsha256hash(fname) != sha256): - log.info("Downloading '{}' source file...".format(fname)) - log.debug("Download url: {}".format(url)) - try: - data = urllib2.urlopen(url) - f = open(fname, "w") - content = data.read() - f.write(content) - f.close() - # Verify the hash of the source file we just downloaded - download_file_hash = hash_helper.getsha256hash(fname) - if download_file_hash != sha256: - log.warn("The hash for '{}' does not match the expected " - "hash. The downloaded hash is '{}'".format( - fname, download_file_hash)) - else: - log.debug("The download file '{}' matches our expected " - "hash of '{}'".format(fname, sha256)) - except(urllib2.HTTPError, urllib2.URLError, - OSError, IOError) as err: - log.error("Unable to download '{}' " - "due to {}\n".format(fname, err)) - sys.exit(1) - - def build(skip): - """This is the main processing step that builds Python from source""" + """Build custom Python2 from source.""" # Step 1: change into our build directory os.chdir(PYTHON_BUILD_DIR) # Don't compile Python if the skip option is passed @@ -157,33 +91,6 @@ def build(skip): f.write("_ssl _ssl.c -DUSE_SSL " "-I{0}/include -I{0}/include/openssl -L{0}/lib " "-lssl -lcrypto".format(OPENSSL_INSTALL_PATH)) - # Step 1.8: Run a few patches so we can compile cleanly - dl_apple_patch_files() - log.debug("Patching files into source...") - cmd = ['/bin/ed', '-', os.path.join( - PYTHON_BUILD_DIR, 'configure'), '<', - os.path.join(SRC_DIR, 'configure.ed')] - out = runner.Popen(cmd) - runner.pprint(out) - - cmd = ['/bin/ed', '-', os.path.join( - PYTHON_BUILD_DIR, 'setup.py'), '<', - os.path.join(SRC_DIR, 'setup.py.ed')] - out = runner.Popen(cmd) - runner.pprint(out) - - cmd = ['/bin/ed', '-', os.path.join( - PYTHON_BUILD_DIR, 'Modules/readline.c'), '<', - os.path.join(SRC_DIR, 'readline.c.ed')] - out = runner.Popen(cmd) - runner.pprint(out) - - cmd = ['/usr/bin/patch', os.path.join( - PYTHON_BUILD_DIR, 'setup.py'), - os.path.join(SRC_DIR, 'setup.py.patch')] - out = runner.Popen(cmd) - runner.pprint(out) - # Step 2: Run the Configure setup of Python to set correct paths os.chdir(PYTHON_BUILD_DIR) if os.path.isdir(PYTHON2_INSTALL): @@ -202,52 +109,38 @@ def build(skip): '--datarootdir={}/share'.format(PYTHON2_INSTALL), '--datadir={}/share'.format(PYTHON2_INSTALL), ] - out = runner.Popen(cmd, stdout=sys.stdout) - # Step 2.5: Patch files - # FIXME: These patches fail and likely need to be recreated - source = os.path.join(PYTHON_BUILD_DIR, 'Lib/ssl.py') - patch = os.path.join(CURRENT_DIR, 'ssl.py.patch') - cmd = ['/usr/bin/patch', source, patch] - out = runner.Popen(cmd) - runner.pprint(out) - - source = os.path.join(PYTHON_BUILD_DIR, 'Lib/logging/handlers.py') - patch = os.path.join(CURRENT_DIR, 'handlers.py.patch') - cmd = ['/usr/bin/patch', source, patch] - out = runner.Popen(cmd) - runner.pprint(out) + runner.Popen(cmd, stdout=sys.stdout) # Step 3: compile Python. this will take a while. + # FIXME: We need to check return codes. log.info("Compiling Python. This will take a while time...") log.detail("Running Python make routine...") cmd = ['/usr/bin/make'] - out = runner.Popen(cmd, stdout=sys.stdout) + runner.Popen(cmd, stdout=sys.stdout) sys.stdout.flush() # does this help? log.debug("Create some temp files thats") log.detail("Running Python make install routine...") cmd = ['/usr/bin/make', 'install'] - out = runner.Popen(cmd, stdout=sys.stdout) + runner.Popen(cmd, stdout=sys.stdout) sys.stdout.flush() # does this help? - # Step 4: Install PyOjbC bridge + # Step 4: Install pip + requirements os.chdir(os.path.join(PYTHON2_INSTALL, 'bin')) # Update pip to latest log.info("Upgrading pip...") cmd = ['./pip', 'install', '--upgrade', 'pip'] runner.Popen(cmd, stdout=sys.stdout) - # Install a the specific version of PyObjc from config.ini - log.info("Install PyObjC...") - cmd = ['./python2.7', '-m', 'pip', 'install', '-U', - 'pyobjc=={}'.format(CONFIG['python2_objc_version'])] + # Install all pip modules from requirements.txt + log.info("Install requirements...") + requirements = os.path.join(CURRENT_DIR, 'requirements.txt') + cmd = ['./python2.7', '-m', 'pip', 'install', '-r', requirements] runner.Popen(cmd, stdout=sys.stdout) - # os.system("./python2.7 -m pip install -U pyobjc-core") # Not needed? - else: log.info("Python compile skipped due to -skip option") def main(): - """Main routine""" + """Build and package Python2.""" parser = argparse.ArgumentParser(prog='Python setup', description='This script will compile ' 'Python 1.0.1+ and optionally create ' @@ -271,12 +164,21 @@ def main(): log.verbose = args.verbose skip = args.skip + root.root_check() + + # Check for OpenSSL. If it isn't on disk in the proper location + # we can't link against it. + if not os.path.isdir(OPENSSL_INSTALL_PATH): + log.warn("OpenSSL must be installed to '{}' prior to compiling " + "Python.".format(OPENSSL_INSTALL_PATH)) + sys.exit(1) + if args.build: log.info("Bulding Python...") - check_dir = os.path.isdir(PKG_PAYLOAD_DIR) + # When the skip option is passed and the build directory exists, skip # download and compiling of Python. Note we still do linking. - if (skip and check_dir): + if skip: log.debug("Skip flag was provided. We will not compile Python " "on this run.") else: @@ -291,9 +193,10 @@ def main(): # Change back into our local directory so we can output our package # via relative paths os.chdir(CURRENT_DIR) - rc = package.pkg(root=PKG_PAYLOAD_DIR, + rc = package.pkg(root=PYTHON2_INSTALL, version=PYTHON2_VERSION, identifier="{}.python".format(CONFIG['pkgid']), + install_location=PYTHON2_INSTALL, output='python-{}.pkg'.format(PYTHON2_VERSION), ) if rc == 0: diff --git a/tests/version_tester.py b/code/tests/version_tester.py similarity index 85% rename from tests/version_tester.py rename to code/tests/version_tester.py index 430c4a7..a41bd54 100644 --- a/tests/version_tester.py +++ b/code/tests/version_tester.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- """ -This is a utility script designed to allow you to run either py2 or py3 via: - /some/path/to/python base_tester.py -in order to verify TLS version. You want TLS 1.2 as the output. +Utility to test TLS version. + +Designed to allow you to run either py2 or py3 via: + /some/path/to/python version_tester.py + +You want TLS 1.2 as the output. """ # standard libs diff --git a/tlsssl/.gitignore b/code/tlsssl/.gitignore similarity index 100% rename from tlsssl/.gitignore rename to code/tlsssl/.gitignore diff --git a/code/tlsssl/README.md b/code/tlsssl/README.md new file mode 100644 index 0000000..df25886 --- /dev/null +++ b/code/tlsssl/README.md @@ -0,0 +1,38 @@ +# NO LONGER USEFUL FOR 10.13 +This is based on the proof of concept work done by [@pudquick](https://github.com/pudquick) here https://github.com/pudquick/tlsssl + +This hack only useful for machines <10.13. With 10.13 Apple has linked python against LibreSSL so we gain TLS 1.2 support out of the box (see below). + + +# Sample TLS outputs + +## Stock 10.13 + +```bash +$ /usr/bin/python tests/version_tester.py +Our python is located: /usr/bin/python +Our python version: 2.7.10 +Our openssl is: LibreSSL 2.2.7 +------------------------------------------------------------------ +SUCCESS: Connection was made using TLS 1.2 +``` + +## Using a vendored Python version + +```bash +Our python is located: /Library/vendored/Python/2.7/bin/python +Our python version: 2.7.14 +Our openssl is: OpenSSL 1.0.2n 7 Dec 2017 +------------------------------------------------------------------ +SUCCESS: Connection was made using TLS 1.2 +``` + +## Using this tlsssl patch on 10.12.6 + +```bash +Our python is located: /usr/bin/python +Our python version: 2.7.10 +Our openssl is: OpenSSL 1.0.2k 26 Jan 2017 +------------------------------------------------------------------ +SUCCESS: Connection was made using TLS 1.2 +``` diff --git a/tlsssl/setup.py b/code/tlsssl/setup.py similarity index 96% rename from tlsssl/setup.py rename to code/tlsssl/setup.py index f66e775..89d8531 100644 --- a/tlsssl/setup.py +++ b/code/tlsssl/setup.py @@ -1,7 +1,4 @@ -""" -Setup script to compile tlsssl to run againt python 2.7. -Has a dependency on openssl package being installed on this local machine. -""" +"""Setup script to compile tlsssl to run against py2.7 on macOS.""" # standard libs from distutils.dir_util import mkpath @@ -37,8 +34,10 @@ def download_python_source_files(): - """Download CPython source files from Github. Verify the sha hash and - redownload if they do not match.""" + """Download CPython source files from Github. + + Verify the sha hash and redownload if they do not match. + """ log.info("Downloading and verifying python source files...") src_dir = os.path.join(CURRENT_DIR, '_src') if not os.path.exists(src_dir): @@ -90,7 +89,7 @@ def download_python_source_files(): def patch(): - """This step creates are patch source files for usage in the build phase""" + """Patch source files for the build phase.""" log.info("Creating our patch files for tlsssl...") patch_dir = os.path.join(CURRENT_DIR, '_patch') if not os.path.exists(patch_dir): @@ -134,7 +133,7 @@ def patch(): def build(): - """This is the main processing step that builds tlsssl from source""" + """Build our tlsssl patch.""" log.info("Building tlsssl...") patch_dir = os.path.join(CURRENT_DIR, '_patch') # Step 2: make sure the _ssl_data.h header has been generated @@ -231,7 +230,7 @@ def build(): "-c", "_patch/_ssl.c", "-o", "build/_ssl.o"] out = runner.Popen(cmd) if out[2] == 0: - log.debug("Build of '_ssl.o' completed sucessfully") + log.debug("Build of '_ssl.o' completed successfullyly") else: log.error("Build has failed: {}".format(out[1])) @@ -241,7 +240,7 @@ def build(): "build/_ssl.so"] out = runner.Popen(cmd) if out[2] == 0: - log.debug("Build of '_ssl.so' completed sucessfully") + log.debug("Build of '_ssl.so' completed successfullyly") else: log.error("Build has failed: {}".format(out[1])) @@ -250,7 +249,7 @@ def build(): def main(): - """Main routine""" + """Build and package the tlsssl patch.""" parser = argparse.ArgumentParser(prog='tlsssl setup', description='This script will compile ' 'tlsssl and optionally create ' diff --git a/code/vendir/__init__.py b/code/vendir/__init__.py new file mode 100644 index 0000000..f5a33df --- /dev/null +++ b/code/vendir/__init__.py @@ -0,0 +1 @@ +"""Required to make Python recognize the directory as a module package.""" diff --git a/vendir/config.py b/code/vendir/config.py similarity index 66% rename from vendir/config.py rename to code/vendir/config.py index 054ba71..ef8e920 100644 --- a/vendir/config.py +++ b/code/vendir/config.py @@ -1,6 +1,7 @@ """ -Function to read the configuration file and return our values. We only read -keys from the DEFAULT and override sections. +Function to read the configuration file and return our values. + +We only read keys from the DEFAULT and override sections. """ import ConfigParser @@ -14,8 +15,10 @@ def ConfigSectionMap(): """ - Return a dict from our config.ini file. All values in the override - section will override the default section. No other sections are looked at. + Return a dict from our config.ini file. + + All values in the override section will override the default section. + No other sections are looked at. """ opts = {} # Read all keys in the DEFAULT section to build our initial listing @@ -27,9 +30,7 @@ def ConfigSectionMap(): def ConfigPrint(): - """ - Prints the current configuration using pprint - """ + """Print the current configuration using pprint.""" pp = pprint.PrettyPrinter(indent=2) pp.pprint(ConfigSectionMap()) diff --git a/vendir/hash_helper.py b/code/vendir/hash_helper.py similarity index 76% rename from vendir/hash_helper.py rename to code/vendir/hash_helper.py index f1b1e8f..50426cd 100644 --- a/vendir/hash_helper.py +++ b/code/vendir/hash_helper.py @@ -1,6 +1,7 @@ """ -Functions for hashing a file. This has been completely lifted from -Munki3.munkilib.munkihash +Functions for hashing a file. + +This has been completely lifted from Munki3.munkilib.munkihash """ import hashlib @@ -9,7 +10,7 @@ def gethash(filename, hash_function): """ - Calculates the hashvalue of the given file with the given hash_function. + Calculate the hashvalue of the given file with the given hash_function. Args: filename: The file name to calculate the hash value of. @@ -18,6 +19,7 @@ def gethash(filename, hash_function): Returns: The hashvalue of the given file as hex string. + """ if not os.path.isfile(filename): return 'NOT A FILE' @@ -33,9 +35,7 @@ def gethash(filename, hash_function): def getsha256hash(filename): - """ - Returns the SHA-256 hash value of a file as a hex string. - """ + """Return the SHA-256 hash value of a file as a hex string.""" hash_function = hashlib.sha256() return gethash(filename, hash_function) diff --git a/vendir/log.py b/code/vendir/log.py similarity index 84% rename from vendir/log.py rename to code/vendir/log.py index 6b1c6ec..597252d 100644 --- a/vendir/log.py +++ b/code/vendir/log.py @@ -1,5 +1,6 @@ """ -Functions for logging +Functions for logging. + TODO: Update print statements to be py3 compatible Borrowed heavily from Munki3.munkilib.munkilog & Munki3.munkilib.display @@ -18,7 +19,7 @@ def _to_unicode(obj, encoding='UTF-8'): - """Coerces basestring obj to unicode""" + """Coerces basestring obj to unicode.""" if isinstance(obj, basestring): if not isinstance(obj, unicode): obj = unicode(obj, encoding) @@ -26,8 +27,12 @@ def _to_unicode(obj, encoding='UTF-8'): def _concat_message(msg, *args): - """Concatenates a string with any additional arguments, - making sure everything is unicode""" + """Concatenate a string with any additional arguments. + + Returns: + Unicode friendly string + + """ # coerce msg to unicode if it's not already msg = _to_unicode(msg) if args: @@ -47,9 +52,7 @@ def _concat_message(msg, *args): def info(msg, *args): - """ - Displays info messages. - """ + """Display info messages.""" msg = _concat_message(str(msg), *args) if verbose > 0: print ' %s' % msg.encode('UTF-8') @@ -58,7 +61,8 @@ def info(msg, *args): def detail(msg, *args): """ - Displays minor info messages. + Display minor info messages. + These are usually logged only, but can be printed to stdout if verbose is set greater than 1 """ @@ -69,9 +73,7 @@ def detail(msg, *args): def debug(msg, *args): - """ - Displays debug messages, formatting as needed. - """ + """Display debug messages, formatting as needed.""" msg = _concat_message(str(msg), *args) if verbose > 2: print ' %s' % msg.encode('UTF-8') @@ -79,9 +81,7 @@ def debug(msg, *args): def warn(msg, *args): - """ - Prints warning msgs to stderr and the log - """ + """Print warning msgs to stderr and the log.""" msg = _concat_message(msg, *args) warning = 'WARNING: %s' % msg if verbose > 0: @@ -89,9 +89,7 @@ def warn(msg, *args): def error(msg, *args): - """ - Prints msg to stderr and the log - """ + """Print msg to stderr and the log.""" msg = _concat_message(msg, *args) errmsg = 'ERROR: %s' % msg if verbose > 0: diff --git a/vendir/package.py b/code/vendir/package.py similarity index 87% rename from vendir/package.py rename to code/vendir/package.py index 4b19cca..147f05f 100644 --- a/vendir/package.py +++ b/code/vendir/package.py @@ -1,5 +1,6 @@ """ Functions for packaging. + TODO: Better doc string message as I was getting lazy on this one """ @@ -19,9 +20,10 @@ def pkg(root, ownership='recommended' ): """ - Function to create a package. Most of the input parameters should be - recognizable for most admins. `output` is the path so make sure - and attach the pkg extension. + Create a package. + + Most of the input parameters should be recognizable for most admins. + `output` is the path so make sure and attach the pkg extension. Return: The exit code from pkgbuild. If non-zero an error has occurred diff --git a/code/vendir/root.py b/code/vendir/root.py new file mode 100644 index 0000000..096efac --- /dev/null +++ b/code/vendir/root.py @@ -0,0 +1,15 @@ +"""Check for root.""" + +import sys +import os + + +def root_check(): + """Check for root access.""" + if not os.geteuid() == 0: + sys.stderr.write("You must run this as root!") + exit(1) + + +if __name__ == '__main__': + print 'This is a library of support tools' diff --git a/vendir/runner.py b/code/vendir/runner.py similarity index 70% rename from vendir/runner.py rename to code/vendir/runner.py index 529ebd5..e822e7f 100644 --- a/vendir/runner.py +++ b/code/vendir/runner.py @@ -1,6 +1,4 @@ -""" -Wrapper functions for os.system and subprocess.Popen -""" +"""Wrapper functions for os.system and subprocess.Popen.""" import subprocess import os @@ -10,9 +8,10 @@ def pprint(data, level='debug'): """ - Pretty print our tuple return message from Popen method. The default level - out output is 'debug' so we only display these messages on the maximum - level of verbosity. + Pretty print our tuple return message from Popen method. + + The default level out output is 'debug' so we only display these messages + on the maximum level of verbosity. """ if level is 'info': log.detail(data[0]) @@ -30,7 +29,9 @@ def pprint(data, level='debug'): def system(cmd): """ - Wrapper for os.system() + Wrap system calls. + + TODO: Stop using this and use https://github.com/facebook/IT-CPE/blob/new_autopkg_tools/autopkg_tools/shell_tools.py#L89-L117 # noqa Args: cmd: the command to run in list format @@ -45,7 +46,9 @@ def Popen(cmd, shell=False, bufsize=-1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE): """ - Runner for subprocess.Popen() + Wrap subprocess calls. + + TODO: Stop using this and use https://github.com/facebook/IT-CPE/blob/new_autopkg_tools/autopkg_tools/shell_tools.py#L57-L86 # noqa Args: these match the standard input arguments for subprocess.Popen diff --git a/python/README.md b/python/README.md deleted file mode 100644 index 4497686..0000000 --- a/python/README.md +++ /dev/null @@ -1,8 +0,0 @@ -Helpful notes - -https://github.com/python/cpython/commit/c2fc7c4f53069558b52d7a497fc195efebe8b4db - -cd /private/tmp/build-python/Lib/test -../../Library/ITOps/Python/bin/python ./ssltests.py - -git diff HEAD:Lib/ssl.py 1fae982b9b6fff5a987a69856b91339e5d023838:Lib/ssl.py diff --git a/python/_src/.gitignore b/python/_src/.gitignore deleted file mode 100644 index a0ce431..0000000 --- a/python/_src/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -# Ignore everything -* - -# But not these files... -!.gitignore -!README.md \ No newline at end of file diff --git a/python/handlers.py.patch b/python/handlers.py.patch deleted file mode 100644 index f952a39..0000000 --- a/python/handlers.py.patch +++ /dev/null @@ -1,10 +0,0 @@ -@@ -737,7 +737,8 @@ - "CRITICAL" : "critical" - } - -- def __init__(self, address=('localhost', SYSLOG_UDP_PORT), -+ # On Darwin, use /var/run/syslog by default -+ def __init__(self, address='/var/run/syslog', - facility=LOG_USER, socktype=None): - """ - Initialize a handler. diff --git a/python/ssl.py.patch b/python/ssl.py.patch deleted file mode 100644 index a20c694..0000000 --- a/python/ssl.py.patch +++ /dev/null @@ -1,41 +0,0 @@ -@@ -90,6 +90,7 @@ - import textwrap - import re - import sys -+import syslog - import os - from collections import namedtuple - from contextlib import closing -@@ -232,6 +233,20 @@ - pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) - return pat.match(hostname) - -+def log_cert_details(cert): -+ """Log certain certificate fields, such as the subject and issuer CN.""" -+ template = ("CN: %s, Org: %s, ValidFrom: %s, " -+ "ValidUntil: %s, Serial: %s, Issuer CN: %s") -+ -+ details = template % ( -+ cert['subject'][4][0][1], -+ cert['subject'][3][0][1], -+ cert['notBefore'], -+ cert['notAfter'], -+ cert['serialNumber'], -+ cert['issuer'][1][0][1]) -+ syslog.syslog( -+ syslog.LOG_INFO, 'Server certificate details: %s' % details) - - def match_hostname(cert, hostname): - """Verify that *cert* (in decoded format as returned by -@@ -245,6 +260,11 @@ - raise ValueError("empty or no certificate, match_hostname needs a " - "SSL socket or SSL context with either " - "CERT_OPTIONAL or CERT_REQUIRED") -+ try: -+ log_cert_details(cert) -+ except IndexError: -+ syslog.syslog(syslog.LOG_ERR, 'Could not read certificate details!') -+ - dnsnames = [] - san = cert.get('subjectAltName', ()) - for key, value in san: diff --git a/tests/words b/tests/words deleted file mode 100644 index 795c8d3..0000000 --- a/tests/words +++ /dev/null @@ -1,30 +0,0 @@ -dict -config -ini -pprint -munkilib -munkihash -hashvalue -hashlib -sha -msg -unicode -args -rstrip -todo -munkilog -basestring -stdout -msgs -stderr -admins -pkgbuild -os -cmd -returncode -vv -usr -py -vendored -openssl -tlsssl diff --git a/tlsssl/README.md b/tlsssl/README.md deleted file mode 100644 index 5cc54bd..0000000 --- a/tlsssl/README.md +++ /dev/null @@ -1 +0,0 @@ -This is based on the proof of concept work done by [@pudquick](https://github.com/pudquick) here https://github.com/pudquick/tlsssl diff --git a/vendir/__init__.py b/vendir/__init__.py deleted file mode 100644 index 35fb043..0000000 --- a/vendir/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# this is needed to make Python recognize the directory as a module package