diff --git a/scripts/get_latest_pypi_version.py b/scripts/get_latest_pypi_version.py
new file mode 100644
index 0000000..e391318
--- /dev/null
+++ b/scripts/get_latest_pypi_version.py
@@ -0,0 +1,37 @@
+"""Given a PyPI package name, print the latest version number of the package.
+
+This uses the standard library instead of requests to avoid adding a dependency to the project.
+"""
+
+from __future__ import annotations
+
+import json
+import ssl
+import sys
+import urllib.request
+
+VERSION_INFO_URL = "https://pypi.org/pypi/{}/json"
+
+
+def get_latest_version(package_name: str) -> str | None:
+    """Get latest version of package_name from PyPI"""
+    try:
+        url = VERSION_INFO_URL.format(package_name)
+        ssl_context = ssl._create_unverified_context()
+        response = urllib.request.urlopen(url, context=ssl_context)
+        data = json.load(response)
+        return data["info"]["version"]
+    except Exception as e:
+        raise ValueError(f"Error retrieving version for {package_name}: {e}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} PACKAGE_NAME")
+        sys.exit(1)
+    package_name = sys.argv[1]
+    try:
+        print(get_latest_version(package_name))
+    except ValueError as e:
+        print(e)
+        sys.exit(1)
diff --git a/scripts/pyapp-runner.sh b/scripts/pyapp-runner.sh
index 72e4b1b..a643d7a 100755
--- a/scripts/pyapp-runner.sh
+++ b/scripts/pyapp-runner.sh
@@ -33,6 +33,18 @@ SERVER=$1
 PROJECT_NAME=$2
 PROJECT_VERSION=$3
 
+# verify PROJECT_VERSION is valid
+# PyApp will happily build with an invalid version number
+# get directory of this script
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PYPI_VERSION=$(python $DIR/get_latest_pypi_version.py $PROJECT_NAME)
+if [ "$PYPI_VERSION" != "$PROJECT_VERSION" ]; then
+    echo "Invalid version number: $PROJECT_VERSION"
+    echo "Latest version on PyPI: $PYPI_VERSION"
+    echo "Did you forget to run 'flit publish'?"
+    exit 1
+fi
+
 # define the remote server and user
 # assumes ssh keys are setup for the user
 # Set these directly or read from environment variables