Python build with a far more portable wheel (#396)

* Identified the appropriate build flags to get a working python build that doesn't rely on -march=native or -mtune=native.  We've run benchmarks on multiple computers that indicate the only important flag other than -mavx2 -msse2 -mfma is -funroll-loops.  Optimization levels such as -O1, -O2, or -O3 actually makes for less performant code. -Ofast is unavailble for use in Python, as it causes problems with floating point math in Python

* 1.22 was left in a comment despite 1.25 being the value specified

* Python 3.8 is not supported by numpy 1.25, so we're removing it.

Author: daxpryce

.github/workflows/build-python.yml

@@ -6,7 +6,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        cibw-identifier: ["cp38-manylinux_x86_64", "cp39-manylinux_x86_64", "cp310-manylinux_x86_64", "cp311-manylinux_x86_64"]
+        cibw-identifier: ["cp39-manylinux_x86_64", "cp310-manylinux_x86_64", "cp311-manylinux_x86_64"]
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -25,7 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        cibw-identifier: ["cp38-win_amd64", "cp39-win_amd64", "cp310-win_amd64", "cp311-win_amd64"]
+        cibw-identifier: ["cp39-win_amd64", "cp310-win_amd64", "cp311-win_amd64"]
     runs-on: windows-latest
     defaults:
       run:

CMakeLists.txt

@@ -279,17 +279,18 @@ if(MSVC)
 	set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${PROJECT_SOURCE_DIR}/x64/Release)
 	set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE ${PROJECT_SOURCE_DIR}/x64/Release)
 else()
-	set(ENV{TCMALLOC_LARGE_ALLOC_REPORT_THRESHOLD} 500000000000)
-    #	set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -DDEBUG -O0 -fsanitize=address -fsanitize=leak -fsanitize=undefined")
-    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -DDEBUG -Wall -Wextra")
+    set(ENV{TCMALLOC_LARGE_ALLOC_REPORT_THRESHOLD} 500000000000)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2 -mfma -msse2 -ftree-vectorize -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -fopenmp -fopenmp-simd -funroll-loops -Wfatal-errors -DUSE_AVX2")
+    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -DDEBUG")
     if (NOT PYBIND)
-	    set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Ofast -DNDEBUG -march=native -mtune=native -ftree-vectorize")
+        set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DNDEBUG -Ofast")
+        if (NOT PORTABLE)
+            set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -march=native -mtune=native")
+        endif()
     else()
-        #-Ofast is super problematic for python.  see: https://moyix.blogspot.com/2022/09/someones-been-messing-with-my-subnormals.html
-        set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DNDEBUG -march=native -mtune=native -ftree-vectorize")
-        add_compile_options(-fPIC)
+        # -Ofast is not supported in a python extension module
+        set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DNDEBUG -fPIC")
     endif()
-	add_compile_options(-march=native -Wall -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -fopenmp -fopenmp-simd -funroll-loops -Wfatal-errors -DUSE_AVX2)
 endif()
 
 add_subdirectory(src)

pyproject.toml

@@ -3,22 +3,22 @@ requires = [
     "setuptools>=59.6",
     "pybind11>=2.10.0",
     "cmake>=3.22",
-    "numpy>=1.21",
+    "numpy==1.25", # this is important to keep fixed. It also means anyone using something other than 1.25 won't be able to use this library
     "wheel",
     "ninja"
 ]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "diskannpy"
-version = "0.5.0.rc2"
+version = "0.5.0.rc4"
 
 description = "DiskANN Python extension module"
-# readme = "../README.md"
-requires-python = ">=3.8"
+readme = "README.md"
+requires-python = ">=3.9"
 license = {text = "MIT License"}
 dependencies = [
-    "numpy"
+    "numpy==1.25"
 ]
 authors = [
     {name = "Harsha Vardhan Simhadri", email = "[email protected]"},

python/README.md

@@ -0,0 +1,40 @@
+# diskannpy
+
+## Installation
+Packages published to PyPI will always be built using the latest numpy major.minor release (at this time, 1.25).
+
+Conda distributions for versions 1.19-1.25 will be completed as a future effort.  In the meantime, feel free to 
+clone this repository and build it yourself.
+
+## Local Build Instructions
+Please see the [Project README](https://github.com/microsoft/DiskANN/blob/main/README.md) for system dependencies and requirements.
+
+After ensuring you've followed the directions to build the project library and executables, you will be ready to also
+build `diskannpy` with these additional instructions.
+
+### Changing Numpy Version
+In the root folder of DiskANN, there is a file `pyproject.toml`. You will need to edit the version of numpy in both the
+`[build-system.requires]` section, as well as the `[project.dependencies]` section.  The version numbers must match.
+
+```bash
+python3.11 -m venv venv # versions from python3.8 and up should work. on windows, you might need to use py -3.11 -m venv venv
+source venv/bin/activate # linux
+# or
+venv\Scripts\Activate.{ps1, bat} # windows
+pip install build
+python -m build
+```
+
+The built wheel will be placed in the `dist` directory in your DiskANN root. Install it using `pip install dist/<wheel name>.whl`
+
+## Citations
+Please cite this software in your work as:
+```
+@misc{diskann-github,
+   author = {Simhadri, Harsha Vardhan and Krishnaswamy, Ravishankar and Srinivasa, Gopal and Subramanya, Suhas Jayaram and Antonijevic, Andrija and Pryce, Dax and Kaczynski, David and Williams, Shane and Gollapudi, Siddarth and Sivashankar, Varun and Karia, Neel and Singh, Aditi and Jaiswal, Shikhar and Mahapatro, Neelam and Adams, Philip and Tower, Bryan}},
+   title = {{DiskANN: Graph-structured Indices for Scalable, Fast, Fresh and Filtered Approximate Nearest Neighbor Search}},
+   url = {https://github.com/Microsoft/DiskANN},
+   version = {0.5},
+   year = {2023}
+}
+```