Python release tool

.gitignore

@@ -100,3 +100,4 @@ Thumbs.db
 # server private key file
 server/rematch/.rematch_secret.key
 server/postgres-data/
+release/dist/

.travis.yml

@@ -25,9 +25,9 @@ matrix:
       services:
         - docker
     - python: "2.7"
-      env: PROJECT=setup.py
+      env: PROJECT=release
     - python: "3.6"
-      env: PROJECT=setup.py
+      env: PROJECT=release
     - python: "3.6"
       env: PROJECT=docs
     - python: "2.7"
@@ -98,8 +98,8 @@ script:
         py.test -rapP ./${PROJECT} ./tests/${PROJECT} --verbose --cov-report= --cov=${PROJECT} ;
       fi ;
     fi ;
-  - if [ "${PROJECT}" == "setup.py" ]; then python ./setup.py server install ; fi ;
-  - if [ "${PROJECT}" == "setup.py" ]; then python ./setup.py idaplugin install ; fi ;
+  - if [ "${PROJECT}" == "release" ]; then python ./release server install ; fi ;
+  - if [ "${PROJECT}" == "release" ]; then python ./release idaplugin install ; fi ;
   - if [ "${PROJECT}" == "docs" ]; then sphinx-build ./docs/ ./docs/_build -a -E -n -W ; fi ;
 
 after_script:

release/package.py

@@ -0,0 +1,89 @@
+import os
+import re
+
+from pkg_resources import parse_version
+from setuptools import setup, find_packages
+from twine.cli import dispatch as twine
+
+
+class Package(object):
+  def __init__(self, name, path, version_path, zip_safe, package_data=None,
+               classifiers=[]):
+    self.name = name
+    self.path = path
+    self.version_path = os.path.join(self.path, version_path, "version.py")
+    self.zip_safe = zip_safe
+    self.classifiers = [
+      "Development Status :: 3 - Alpha",
+      "License :: OSI Approved :: GNU General Public License v3 (GPLv3)"
+    ]
+    self.classifiers += classifiers
+    self.package_data = package_data or {}
+
+  def get_version(self):
+    version_content = open(self.version_path).read()
+    # grab version string from file, this excludes inline comments too
+    version_str = re.search(r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]',
+                            version_content).group(1)
+    return parse_version(version_str)
+
+  def get_released_version(self):
+    # TODO
+    pass
+
+  def generate_changelog(self):
+    # TODO
+    pass
+
+  def get_requirements(self, *parts):
+    fpath = os.path.join(*parts)
+    if not os.path.exists(fpath):
+      return []
+
+    with open(fpath) as fh:
+      return (l for l in fh.readlines() if not l.startswith('-r '))
+
+  def build(self, *script_args):
+    # generate install_requires based on requirements.txt
+    install_requires = self.get_requirements(self.path, "requirements.txt")
+
+    # include requirementst.txt as part of package
+    if install_requires:
+      if self.path not in self.package_data:
+        self.package_data[self.path] = []
+      self.package_data[self.path].append('requirements.txt')
+
+    extras_require = {'test': self.get_requirements("tests", self.path,
+                                                    "requirements.txt")}
+
+    with open("README.rst") as fh:
+      long_description = fh.read()
+
+    setup(
+      script_args=script_args + ('--dist-dir=./release/dist',),
+      name=self.name,
+      version=str(self.get_version()),
+      author="Nir Izraeli",
+      author_email="[email protected]",
+      description=("A IDA Pro plugin and server framework for binary "
+                   "function level diffing."),
+      keywords=["rematch", "ida", "idapro", "binary diffing",
+                "reverse engineering"],
+      url="https://www.github.com/nirizr/rematch/",
+      packages=find_packages(self.path),
+      package_dir={'': self.path},
+      package_data=self.package_data,
+      extras_require=extras_require,
+      install_requires=install_requires,
+      long_description=long_description,
+      classifiers=self.classifiers
+    )
+
+  def get_dist_file(self):
+    return './release/dist/{}-{}.zip'.format(self.name, self.get_version())
+
+  def upload(self, repo="pypi"):
+    twine(['upload', self.get_dist_file(), '-r', repo])
+
+  def __repr__(self):
+    return "<Package {}/{}>".format(self.name, self.get_version())

release/packages.py

@@ -0,0 +1,21 @@
+from .package import Package
+
+
+server = Package(name='rematch-server', path='server', version_path='./',
+                 classifiers=["Programming Language :: Python :: 2",
+                              "Programming Language :: Python :: 2.7",
+                              "Programming Language :: Python :: 3",
+                              "Programming Language :: Python :: 3.4",
+                              "Programming Language :: Python :: 3.5",
+                              "Programming Language :: Python :: 3.6",
+                              "Programming Language :: Python :: 3.7",
+                              "Environment :: Web Environment",
+                              "Framework :: Django"], zip_safe=True)
+
+ida = Package(name='rematch-idaplugin', path='idaplugin',
+              version_path='rematch', zip_safe=False,
+              package_data={'idaplugin/rematch': ['images/*']},
+              classifiers=["Programming Language :: Python :: 2",
+                           "Programming Language :: Python :: 2.7"])
+
+package_list = [server, ida]

release/release.py

@@ -0,0 +1,73 @@
+#!python
+
+import logging
+import subprocess
+import argparse
+
+from . import setup
+from .packages import package_list
+
+
+REMOTE="origin"
+BRANCH="master"
+# TODO: Remove next line just before merging
+BRANCH="nirizr/release_py"
+
+
+def sh_exec(*args):
+  logging.getLogger('sh_exec').info("Executing '%s'", args)
+  output = subprocess.check_output(args, shell=False).strip().decode()
+  logging.getLogger('sh_exec').info("Output '%s'", output)
+  return output
+
+
+def validate_git_state():
+  logging.info("Validating git state is clean")
+
+  if sh_exec("git", "rev-parse", "--abbrev-ref", "HEAD") != BRANCH:
+    raise RuntimeError("Current branch name doesn't match release branch.")
+
+  if "nothing to commit" not in sh_exec("git", "status", "-uno"):
+    raise RuntimeError("Local branch is dirty, can only release in clean "
+                       "workspaces.")
+
+  remote_branch = sh_exec("git", "ls-remote", REMOTE, "-h",
+                          "refs/heads/" + BRANCH)
+  remote_branch_hash = remote_branch.split()[0]
+  if sh_exec("git", "rev-parse", BRANCH) is not remote_branch_hash:
+    raise RuntimeError("Local and remote branches are out of sync, releases "
+                       "are only possible on up-to-date branch")
+
+
+def identify_new_packages():
+  new_packages = set()
+
+  for package in package_list:
+    print(package)
+    # if package.get_version()
+    new_packages.add(package)
+
+  return new_packages
+
+
+def main():
+  parser = argparse.ArgumentParser(description="Rematch release utility")
+  parser.add_argument('--verbose', '-v', action='count')
+  parser.add_argument('--skip-validation', '-sv', default=False, action='store_true')
+  args = parser.parse_args()
+
+  logging.basicConfig(level=logging.ERROR - args.verbose * 10)
+
+  if not args.skip_validation:
+    validate_git_state()
+
+  packages = identify_new_packages()
+  for package in packages:
+    package.generate_changelog()
+    package.build('sdist', '--formats=zip')
+    package.upload('test')
+    package.upload()
+
+
+if __name__ == '__main__':
+  main()