GCC     = gfortran
oneAPI  = ifx

# These flags will be looped over to generate different datasets
FLAGS  := O0 O1 O2 O3 Ofast
# Also specify a default (note the conditional assignment operation, ?=)
OPTFLAGS ?= O3
# For the parallel runs, use all available processors
NPROC := $(shell nproc)

# Make some pretty colours
MAGENTA  := \033[1;35m
RESET  := \033[0m

# Define the start, stop, and step for "short" runs (including triple loops)
S_START:= 100
S_END  := 3500
S_STEP := 100

# And the step and end for "long" runs without triple loops.
# Automatically they start at S_END defined earlier
L_END  := 12000
L_STEP := 500


all:
	# This target simply compiles the binaries
	@mkdir -p bin/
	@echo "$(MAGENTA)Compiling serial and parallel binaries with $(OPTFLAGS)$(RESET)"
	$(GCC) matrixproduct.f90 -o bin/$(GCC).serial.out -$(OPTFLAGS) -fexternal-blas -lopenblas -march=native
	$(oneAPI) matrixproduct.f90 -o bin/$(oneAPI).serial.out -qmkl=sequential -$(OPTFLAGS) -heap-arrays -xHost
	$(GCC) matrixproduct.f90 -o bin/$(GCC).parallel.out -$(OPTFLAGS) -fexternal-blas -lopenblas -march=native -fopenmp
	$(oneAPI) matrixproduct.f90 -o bin/$(oneAPI).parallel.out -qmkl=parallel -$(OPTFLAGS) -heap-arrays -xHost -fopenmp

tests: clean
	# Run the parametric tests and sweep over compiler flags
	@mkdir -p results/
	@for opt in $(FLAGS); do \
                $(MAKE) all OPTFLAGS=$$opt; \
                echo "$(MAGENTA)Running serial runs with $$opt$(RESET)"; \
				$(MAKE) serial OPTFLAGS=$$opt; \
                echo "$(MAGENTA)Running parallel runs with $$opt$(RESET)"; \
				$(MAKE) parallel OPTFLAGS=$$opt; \
	done


## NOTE: For the below two, it is very important that lines have the '\' postfix
## By default, these are all executed on a different shell session, so to preserve
## the enivronment variables for OMP/MKL they need to be run as one "big" command
## serially. The backslash is the 'line continuation' operator that does this.

serial: ./bin/gfortran.serial.out ./bin/ifx.serial.out
	# run the serial tests. ensures that only one thread is used for blas and mkl
	export OMP_NUM_THREADS=1 && \
	export MKL_NUM_THREADS=1 && \
	./bin/$(GCC).serial.out $(S_START) $(S_END) $(S_STEP) yes | tee results/$(GCC)_short_serial_$(OPTFLAGS).out; \
	./bin/$(oneAPI).serial.out $(S_START) $(S_END) $(S_STEP) yes | tee results/$(oneAPI)_short_serial_$(OPTFLAGS).out; \
	./bin/$(GCC).serial.out $(S_END) $(L_END) $(L_STEP) no | tee results/$(GCC)_long_serial_$(OPTFLAGS).out; \
	./bin/$(oneAPI).serial.out $(S_END) $(L_END) $(L_STEP) no | tee results/$(oneAPI)_long_serial_$(OPTFLAGS).out; 

parallel: ./bin/gfortran.parallel.out ./bin/ifx.parallel.out	
	# run the parallel tests. ensures that all threads are used for the loops, blas, and mkl
	export OMP_NUM_THREADS=1 && \
	export OMP_NUM_THREADS=${NPROC} && \
	export MKL_NUM_THREADS=${NPROC} && \
	./bin/$(GCC).parallel.out $(S_START) $(S_END) $(S_STEP) yes | tee results/$(GCC)_short_parallel_$(OPTFLAGS)_$(NPROC).out; \
	./bin/$(oneAPI).parallel.out $(S_START) $(S_END) $(S_STEP) yes | tee results/$(oneAPI)_short_parallel_$(OPTFLAGS)_$(NPROC).out; \
	./bin/$(GCC).parallel.out $(S_END) $(L_END) $(L_STEP) no | tee results/$(GCC)_long_parallel_$(OPTFLAGS)_$(NPROC).out; \
	./bin/$(oneAPI).parallel.out $(S_END) $(L_END) $(L_STEP) no | tee results/$(oneAPI)_long_parallel_$(OPTFLAGS)_$(NPROC).out;


plots:
	# Deprecated. Plot some things in GNUPLOT. Currently unused.
	gnuplot -p plots.gnu

clean:
	# Clean up after ourselves
	@echo "$(MAGENTA)Cleaning bin/ and results/$(RESET)"
	@rm -rf bin/
	@rm -rf results/