MP-Gadget · nianyic7 · Sep 20, 2024 · Sep 25, 2024 · Sep 25, 2024 · Sep 27, 2024
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -20,7 +20,8 @@ jobs:
 
     - name: Checkout source code
       uses: actions/checkout@v4
-
+    - name: Install MPI
+      run: sudo apt install -y -q build-essential mpich libmpich-dev python-is-python3 python3
     - name: Cache conda
       uses: actions/cache@v4
       env:
@@ -34,7 +35,7 @@ jobs:
     - name: Cache depends/
       uses: actions/cache@v4
       with:
-        key: ${{ runner.os }}-build-${{ hashFiles('depends/Makefile', 'depends/install_pfft.sh') }}
+        key: ${{ runner.os }}-build-${{ hashFiles('depends/Makefile') }}
         path: |
           ~/depends/install
           ~/depends/include

diff --git a/LICENSE b/LICENSE
@@ -14,6 +14,4 @@ Redistribution and use in source and binary forms, with or without modification,
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-MP-Gadget is also available under the terms of the GNU Public License v2 or later at the option of the user. MP-Gadget is a derived code of Gadget-2 and has been re-licensed under the above license with the permission of all original copyright holders, including Volker Springel, the author of Gadget-2. 
-
-Note that as MP-Gadget depends on PFFT and the GNU scientific library, a compiled version is implicitly distributed under the terms of the GNU General Public License version under which the version of the library llinked against is available.
+Shenqi is a derived code of Gadget-2 and has been re-licensed under the above license with the permission of all original copyright holders, including Volker Springel, the author of Gadget-2. 
diff --git a/Makefile b/Makefile
@@ -13,7 +13,6 @@ all: $(CONFIG)
 	cd libgenic; $(MAKE)
 	cd gadget; $(MAKE)
 	cd genic; $(MAKE)
-
 clean :
 	cd libgadget; $(MAKE) clean
 	cd libgenic; $(MAKE) clean
@@ -59,9 +58,7 @@ tag:
 sdist:
 	(git rev-parse --abbrev-ref HEAD | grep master )|| (echo "Must be on master" && exit 1);
 	git checkout -B "rc-$(VERSION)";
-	git add -f depends/pfft-1.0.8-alpha2-fftw3.tar.gz
 	git commit -m "rc-$(VERSION) packaging"
-	git rm --cached depends/pfft-1.0.8-alpha2-fftw3.tar.gz
 	git commit -m "rc-$(VERSION) cleanup"
 	bash maintainer/git-archive-all.sh --prefix MPGadget-$(VERSION)/ -- - | gzip -c > MPGadget-$(VERSION).tar.gz
 	git checkout master

diff --git a/Makefile.rules b/Makefile.rules
@@ -1,35 +1,57 @@
 # vim: set ft=make:
 #
 AR ?= ar
-MPICC ?= mpicc
+MPICC ?= mpic++
+NVOPTIMIZE ?=  
 LOW_PRECISION ?= double
 
 OPTIMIZE ?= -O2 -g -fopenmp -Wall
-GSL_INCL ?= $(shell pkg-config --cflags gsl)
-GSL_LIBS ?= $(shell pkg-config --libs gsl)
+
 ifneq ($(findstring -DUSE_CFITSIO, $(OPT)),)
     # If found, set FITSIO_INCL with the cfitsio flags
     FITSIO_INCL ?= $(shell pkg-config --cflags cfitsio)
     FITSIO_LIBS ?= $(shell pkg-config --libs cfitsio)
 endif
 
+ifneq ($(findstring -DUSE_CUDA, $(OPT)),)
+    CUDA_INCL ?= 
+    CUDA_LIBS ?= -lcudart
+    CUFFTMP_INCL ?= 
+    CUFFTMP_LIBS ?= -lcufftMp
+    NVSHMEM_INCL ?= 
+    NVSHMEM_LIBS ?= -lnvshmem_host
+    NVCC ?= nvcc
+    NVOPTIMIZE ?= -O3
+    MPI_INCL ?=
+endif
+
 OPTIONS = $(OPTIMIZE) $(OPT)
 GADGET_TESTDATA_ROOT = $(CURDIR)/../
 
-CFLAGS = $(OPTIONS) $(GSL_INCL) $(FITSIO_INCL)
+CFLAGS = $(OPTIONS) $(BOOST_INCL) $(FITSIO_INCL) $(CUDA_INCL) $(CUFFTMP_INCL) $(NVSHMEM_INCL)
 CFLAGS += -I../depends/include
 CFLAGS += -I../
 CFLAGS += "-DLOW_PRECISION=$(LOW_PRECISION)"
 #For tests
 TCFLAGS = $(CFLAGS) -DGADGET_TESTDATA_ROOT=\"$(GADGET_TESTDATA_ROOT)\"
 
-BUNDLEDLIBS = -lbigfile-mpi -lbigfile -lpfft_omp -lfftw3_mpi -lfftw3_omp -lfftw3
-LIBS  = -lm $(GSL_LIBS) $(FITSIO_LIBS)
+CUDAFLAGS = $(BOOST_INCL) $(CUDA_INCL) $(CUFFTMP_INCL) $(NVSHMEM_INCL) $(MPI_INCL)
+
+BUNDLEDLIBS = -lbigfile-mpi -lbigfile 
+LIBS  = -lm $(BOOST_LIBS) $(FITSIO_LIBS) $(CUDA_LIBS) $(CUFFTMP_LIBS) $(NVSHMEM_LIBS)
 LIBS += -L../depends/lib $(BUNDLEDLIBS)
 V ?= 0
 
-.objs/%.o: %.c Makefile $(CONFIG)
-	@cmd="$(MPICC) -MMD -c -o $@ $(CFLAGS) $<"; \
+.objs/%.o: %.c $(INCL) Makefile $(CONFIG)
+	@cmd="$(MPICC) -c -o $@ $(CFLAGS) $<"; \
 	if test "x$(V)" = "x1" ; then echo $$cmd; fi; \
 	mkdir -p `dirname $@`; \
 	echo Compiling $<; $$cmd
+
+
+.objs/%.o: %.cu Makefile $(CONFIG)
+	@cmd="$(NVCC) -MMD -c -o $@ $(CUDAFLAGS) $<"; \
+	if test "x$(V)" = "x1" ; then echo $$cmd; fi; \
+	echo Compiling $<; $$cmd
+
+
diff --git a/Options.mk.example b/Options.mk.example
@@ -1,7 +1,10 @@
 #These variables are set to useful defaults, but may be overriden if needed
-#MPICC=mpicc
-#GSL_LIBS=
-#GSL_INCL=
+#MPICC=mpic++
+#MPICCDEP=mpicc
+
+#NVCC=nvcc
+#NVOPTIMIZE = -O3 -arch=sm_61 # specify architecture according to you GPU model, sm_90 shall be used for Vista's H100
+
 #This is a good optimized build default for gcc
 OPTIMIZE =  -fopenmp -O3 -g -Wall -ffast-math
 #This is a good non-optimized default for debugging
@@ -12,7 +15,7 @@ OPTIMIZE =  -fopenmp -O3 -g -Wall -ffast-math
 #OPT += -DDEBUG      # print a lot of debugging messages
 #Disable openmp locking. This means no threading.
 #OPT += -DNO_OPENMP_SPINLOCK
-
+#OPT += -DUSE_CUDA  #Enable GPU-specific CUDA code
 #-----------
 #OPT += -DEXCUR_REION  # reionization with excursion set
 

diff --git a/README.rst b/README.rst
@@ -12,7 +12,7 @@ Description
 
 This version of Gadget is derived from main P-Gadget / Gadget-2, with the gravity solver algorithm from Gadget-4.
 It is the source code used to run the BlueTides and ASTRID simulations (http://bluetides-project.org).
-MP-Gadget requires GSL and a C compiler with OpenMP 4.5 support.
+MP-Gadget requires a C++ compiler with OpenMP 4.5 support.
 
 The infrastructure is heavily reworked. As a summary:
 
@@ -50,17 +50,6 @@ First time users:
     make -j
 
 The Makefile will automatically copy Options.mk.example to Options.mk. The default compile flags are appropriate for a linux using gcc, but may not be optimal.
-We will need gsl. On HPC systems with the modules command,
-usually it can be loaded with
-
-.. code:: bash
-
-    module load gsl
-
-    env | grep GSL  # check if GSL path is reasonable
-
-On a common PC/Linux system, refer to your package vendor how to
-install gsl and gsl-devel.
 
 If you wish to perform compile-time customisation (to, eg, change optimizations or use different compilers), you need an Options.mk file. The initial defaults are stored in Options.mk.example.
 
@@ -81,8 +70,6 @@ Compile-time options may be set in Options.mk. The remaining compile time option
 - EXCUR_REION enables the excursion set reionization model.
 - USE_CFITSIO enables the output of lenstools compatible potential planes using cfitsio,
 
-If compilation fails with errors related to the GSL, you may also need to set the GSL_INC or GSL_LIB variables in Options.mk to the filesystem path containing the GSL headers and libraries.
-
 To run a N-Body sim, use IC files with no gas particles.
 
 Now we are ready to build
@@ -140,15 +127,6 @@ Refer to https://github.com/rainwoodman/bigfile for usage.
 Otherwise directly open the blocks with Fortran or C, noting the data-type
 information and attributes in header and attrs files (in plain text)
 
-GLIBC 2.22
-----------
-
-Cray updated their GLIBC to 2.22+ recently.
-A good move but it happens to be a buggy version of GLIBC:
-https://sourceware.org/bugzilla/show_bug.cgi?id=19590
-causing non-existing symbols like `_ZGVcN4v___log_finite`.
-Adding `-lmvec -lmvec_nonshared` to GSL_LIBS works around the issue.
-
 Bigfile
 -------
 
@@ -192,7 +170,7 @@ For usage of the code, here is a DOI for this repository that you can cite
 Licence
 -------
 
-MP-Gadget is distributed under the terms of a 3-clause BSD license or the GNU General Public License v2 or later, at the option of the user. The use of PFFT and GSL libraries usually forces distribution under the terms of the GNU General Public License v3.
+MP-Gadget is distributed under the terms of a 3-clause BSD license or the GNU General Public License v2 or later, at the option of the user.
 
 Status
 ------

diff --git a/depends/Makefile b/depends/Makefile
@@ -2,34 +2,22 @@ CONFIG = ../Options.mk
 include $(CONFIG)
 
 .PHONY: depends
-.INTERMEDIATE: pfft
-MPICC ?= mpicc
+# MPICC ?= mpicc
+MPICCDEP ?= mpicc
 OPTIMIZE ?= -O2 -g -fopenmp -Wall
 LIBRARIES=lib/libbigfile-mpi.a
-FFTLIBRARIES=lib/libpfft_omp.a lib/libfftw3_mpi.a lib/libfftw3_omp.a
-depends: $(LIBRARIES) $(FFTLIBRARIES)
-$(FFTLIBRARIES): pfft
+depends: $(LIBRARIES)
 
 lib/libbigfile-mpi.a: bigfile/src/bigfile-mpi.c
 	mkdir -p lib; \
 	mkdir -p include; \
 	cd bigfile/src; \
-	make install PREFIX=$(PWD) CC="$(MPICC)" MPICC="$(MPICC)" CFLAGS="$(OPTIMIZE)" AR="$(AR)"
+	make install PREFIX=$(PWD) CC="$(MPICCDEP)" MPICC="$(MPICCDEP)" CFLAGS="$(OPTIMIZE)" AR="$(AR)"
 
-pfft: install_pfft.sh
-	mkdir -p lib; \
-	mkdir -p include; \
-	#Using -ipo causes icc to crash.
-	MPICC="$(MPICC)" CC="$(MPICC)" CFLAGS="$(filter-out -ipo,$(OPTIMIZE)) -I $(PWD)/include -L$(PWD)/lib" AR="$(AR)" RANLIB=$(RANLIB) \
-        sh $(PWD)/install_pfft.sh $(PWD)/
 
-clean: clean-fast clean-fft
+clean: clean-fast
 
 clean-fast:
 	rm -rf $(LIBRARIES)
 	cd bigfile/src; make clean
 
-clean-fft:
-	rm -rf $(FFTLIBRARIES)
-	rm -rf tmp-pfft-*/double
-	rm -rf tmp-pfft-*/single
diff --git a/depends/bigfile/.github/workflows/main.yaml b/depends/bigfile/.github/workflows/main.yaml
@@ -65,7 +65,7 @@ jobs:
                numpy=${{ matrix.numpy-version }} \
                nose cython mpi4py \
                compilers
-        conda install -q -y cmake gsl
+        conda install -q -y cmake
         conda install -q -y runtests
 
     - name: Build C

diff --git a/depends/bigfile/CMakeLists.txt b/depends/bigfile/CMakeLists.txt
@@ -4,7 +4,6 @@ project(bigfile)
 
 # Finding optional dependencies
 find_package(MPI)
-find_package(GSL)
 
 # Add library subdirectoy
 add_subdirectory(src)

diff --git a/depends/bigfile/utils/CMakeLists.txt b/depends/bigfile/utils/CMakeLists.txt
@@ -55,17 +55,6 @@ if(${MPI_C_FOUND})
 
     install(TARGETS bigfile-copy-mpi bigfile-iosim
             RUNTIME DESTINATION bin)
-
-    if(${GSL_FOUND})
-        include_directories(${GSL_INCLUDE_DIRS})
-
-        # bigfile-sample-mpi
-        add_executable(bigfile-sample-mpi bigfile-sample-mpi.c)
-        target_link_libraries(bigfile-sample-mpi bigfile-mpi bigfile ${GSL_LIBRARIES} ${MPI_C_LIBRARIES})
-
-        install(TARGETS bigfile-sample-mpi
-                RUNTIME DESTINATION bin)
-    endif()
 endif()
 
 # Install bash scripts

diff --git a/depends/bigfile/utils/Makefile b/depends/bigfile/utils/Makefile
@@ -4,7 +4,6 @@ all: \
 	bigfile-set-attr \
 	bigfile-copy \
 	bigfile-copy-mpi \
-	bigfile-sample-mpi \
 	bigfile-cat \
 	bigfile-create \
 	bigfile-ls \
@@ -19,8 +18,6 @@ bigfile-copy: bigfile-copy.c ../src/libbigfile.a
 	$(CC) -o $@ $< ../src/libbigfile.a -I../src
 bigfile-copy-mpi: bigfile-copy-mpi.c ../src/libbigfile.a ../src/libbigfile-mpi.a
 	$(CC) -o $@ $< ../src/libbigfile-mpi.a ../src/libbigfile.a -I../src
-bigfile-sample-mpi: bigfile-sample-mpi.c ../src/libbigfile.a ../src/libbigfile-mpi.a
-	$(CC) -o $@ $< ../src/libbigfile-mpi.a ../src/libbigfile.a -I../src -lgsl -lgslcblas -lm
 bigfile-cat: bigfile-cat.c ../src/libbigfile.a
 	$(CC) -o $@ $< ../src/libbigfile.a -I../src
 bigfile-create: bigfile-create.c ../src/libbigfile.a