##******************************************************************************
##  Copyright(C) 2006-2013 Intel Corporation. All Rights Reserved.
##  
##  The source code, information  and  material ("Material") contained herein is
##  owned  by Intel Corporation or its suppliers or licensors, and title to such
##  Material remains  with Intel Corporation  or its suppliers or licensors. The
##  Material  contains proprietary information  of  Intel or  its  suppliers and
##  licensors. The  Material is protected by worldwide copyright laws and treaty
##  provisions. No  part  of  the  Material  may  be  used,  copied, reproduced,
##  modified, published, uploaded, posted, transmitted, distributed or disclosed
##  in any way  without Intel's  prior  express written  permission. No  license
##  under  any patent, copyright  or  other intellectual property rights  in the
##  Material  is  granted  to  or  conferred  upon  you,  either  expressly,  by
##  implication, inducement,  estoppel or  otherwise.  Any  license  under  such
##  intellectual  property  rights must  be express  and  approved  by  Intel in
##  writing.
##  
##  *Third Party trademarks are the property of their respective owners.
##  
##  Unless otherwise  agreed  by Intel  in writing, you may not remove  or alter
##  this  notice or  any other notice embedded  in Materials by Intel or Intel's
##  suppliers or licensors in any way.
##
##******************************************************************************
##  Content:
##  Intel(R) Math Kernel Library
##  C-style Cluster DFT wrappers MPI FFTW examples creation and run
##******************************************************************************

help:
	@echo "Usage: make libia32|libintel64"
	@echo "		[mpi=intelmpi|mpich|mpich2|openmpi]"
	@echo "		[compiler=intel|gnu] [workdir=path] [mpidir=path]"
	@echo
	@echo "mpi=intelmpi - using Intel(R) MPI Library, default"
	@echo "mpi=mpich  - using MPICH"
	@echo "mpi=mpich2 - using MPICH2"
	@echo "mpi=openmpi - using Open MPI"
	@echo
	@echo "compiler=intel - using Intel(R) C Compiler, default"
	@echo "compiler=gnu   - using GNU C compiler"
	@echo
	@echo "workdir=path - path to work directory, which is accessible from any node."
	@echo "If parameter is omitted, executable files and results will be located in"
	@echo "current directory."
	@echo
	@echo "mpidir=path - path to MPI installation directory. MPI scripts are taken"
	@echo "from mpidir/bin (or mpidir/bin64 for Intel(R) MPI Library and Intel(R) 64 architecture). If this directory"
	@echo "is in PATH you can omit mpidir parameter. When using MPICH, make sure"
	@echo "that MPICH was compiled by same compiler as used to build examples."
	@echo
	@echo "Set environment variables LD_LIBRARY_PATH and etc properly before testing."

##*****************************************************************************
## examples of using:
##
## make libintel64
##              - compile and run wrappers examples
##                using Intel(R) MPI Library, Intel(R) C Compiler,
##                for double precision data for
##                Intel(R) 64 based applications
##
## make libia32 mpi=mpich mpidir=/opt/mpich compiler=gnu
##              - compile and run wrappers examples using
##                MPICH from /opt/mpich, GNU C compiler
##                for 32-bit applications
##*****************************************************************************

include cdft_wrappers.lst

example = $(WRAPPER)

ifndef mpi
   mpi = intelmpi
endif

ifndef compiler
   compiler = intel
endif

ifndef workdir
   workdir = .
endif

RES = $(addsuffix .res,$(example))

ifndef MKLROOT
   MKLROOT = ../..
endif
MKL_PATH = "$(MKLROOT)/lib/$(_IA)"
CMPLR_PATH = "$(MKLROOT)/../compiler/lib/$(_IA)"

INSTALL_DIR=$(PWD)

COPTS =

ifeq ($(_IA),ia32)
   IFACE_LIB=$(MKL_PATH)/libmkl_intel.$(EXT)
   BLACS_PART=
else
   IFACE_LIB=$(MKL_PATH)/libmkl_intel_lp64.$(EXT)
   BLACS_PART=_lp64
endif

THREADING_LIB=$(MKL_PATH)/libmkl_intel_thread.$(EXT)
OMP_LIB = -L$(CMPLR_PATH) -liomp5

CORE_LIB=$(MKL_PATH)/libmkl_core.$(EXT)

MKL_LIBS= -Wl,--start-group $(MKL_PATH)/libmkl_cdft_core.$(EXT) $(IFACE_LIB) $(THREADING_LIB) $(CORE_LIB) -lmkl_blacs$(Bs)$(BLACS_PART) -Wl,--end-group $(OMP_LIB)

ifeq ($(_IA),ia32)
   RES_DIR=_results/$(compiler)_$(_IA)_$(RES_EXT)$Z
else
   RES_DIR=_results/$(compiler)_$(_IA)_$(RES_EXT)$Z
endif

ifeq ($(mpi),mpich2)
   ifeq ($(compiler),intel)
      CS = mpicc -cc=icc
   endif
   ifeq ($(compiler),gnu)
      CS = mpicc -cc=gcc
   endif
   RS = mpiexec -n 2
   ifdef mpidir
      _CS = $(mpidir)/bin/$(CS)
      _RS = $(mpidir)/bin/$(RS)
   else
      _CS = $(CS)
      _RS = $(RS)
   endif
   Bs = _intelmpi
   export OMP_NUM_THREADS=1
endif

ifeq ($(mpi),intelmpi)
   ifeq ($(compiler),intel)
      CS = mpiicc -mt_mpi
   endif
   ifeq ($(compiler),gnu)
      CS = mpicc -mt_mpi
   endif
   RS = mpiexec -n 2
   ifdef mpidir
      _CS = $(mpidir)/$(_IA)/bin/$(CS)
      _RS = $(mpidir)/$(_IA)/bin/$(RS)
   else
      _CS = $(CS)
      _RS = $(RS)
   endif
   Bs = _intelmpi
endif

ifeq ($(mpi),mpich)
   ifeq ($(compiler),intel)
      CS = mpicc -cc=icc
   endif
   ifeq ($(compiler),gnu)
      CS = mpicc -cc=gcc
   endif
   RS = mpirun -np 2
   ifdef mpidir
      _CS = $(mpidir)/bin/$(CS)
      _RS = $(mpidir)/bin/$(RS)
   else
      _CS = $(CS)
      _RS = $(RS)
   endif
   DelF = rm $(RES_DIR)/*.o
   export OMP_NUM_THREADS=1
endif

ifeq ($(mpi),openmpi)
   ifeq ($(compiler),intel)
      CS = mpicc -cc=icc $(COPTS)
   endif
   ifeq ($(compiler),gnu)
      CS = mpicc -cc=gcc  $(COPTS)
   endif
   RS = mpiexec -np 2
   ifdef mpidir
      _CS = $(mpidir)/bin/$(CS)
      _RS = $(mpidir)/bin/$(RS)
   else
      _CS = $(CS)
      _RS = $(RS)
   endif
   Bs = _openmpi
   DelF = rm $(RES_DIR)/*.o
endif

#-------------------------------------------------------------------------------

vpath %.c source

libia32 lib32:
	$(MAKE) wrap_lib _IA=ia32
	$(MAKE) $(RES)   _IA=ia32    EXT=a RES_EXT=lib
libintel64 libem64t:
	$(MAKE) wrap_lib _IA=intel64
	$(MAKE) $(RES)   _IA=intel64 EXT=a RES_EXT=lib

wrap_lib:
	cd "$(MKLROOT)"/interfaces/fftw3x_cdft && $(MAKE) lib$(_IA) mpi=$(mpi) compiler=$(compiler) mpidir="$(mpidir)" INSTALL_DIR="$(INSTALL_DIR)/lib/$(_IA)" INSTALL_LIBNAME="libfftw3x_cdft"

$(RES): %.res: %.c
	mkdir -p $(RES_DIR)
	$(_CS) $(FL) $< -I $(MKLROOT)/include -I $(MKLROOT)/include/fftw -c -o $(RES_DIR)/$*.o
	$(_CS) $(FL) $(RES_DIR)/$*.o -O0 -L"$(INSTALL_DIR)/lib/$(_IA)" -lfftw3x_cdft -L $(MKLROOT)/lib/$(_IA) $(MKL_LIBS) -lpthread -lm -ldl -o $(RES_DIR)/$*.exe
	export LD_LIBRARY_PATH=$(MKL_PATH):"$(LD_LIBRARY_PATH)":$(CMPLR_PATH); $(_RS) $(RES_DIR)/$*.exe > $(RES_DIR)/$@;
	-$(DelF)
#-------------------------------------------------------------------------------
