##******************************************************************************
##  Copyright(C) 2005-2013 Intel Corporation. All Rights Reserved.
##  
##  The source code, information  and  material ("Material") contained herein is
##  owned  by Intel Corporation or its suppliers or licensors, and title to such
##  Material remains  with Intel Corporation  or its suppliers or licensors. The
##  Material  contains proprietary information  of  Intel or  its  suppliers and
##  licensors. The  Material is protected by worldwide copyright laws and treaty
##  provisions. No  part  of  the  Material  may  be  used,  copied, reproduced,
##  modified, published, uploaded, posted, transmitted, distributed or disclosed
##  in any way  without Intel's  prior  express written  permission. No  license
##  under  any patent, copyright  or  other intellectual property rights  in the
##  Material  is  granted  to  or  conferred  upon  you,  either  expressly,  by
##  implication, inducement,  estoppel or  otherwise.  Any  license  under  such
##  intellectual  property  rights must  be express  and  approved  by  Intel in
##  writing.
##  
##  *Third Party trademarks are the property of their respective owners.
##  
##  Unless otherwise  agreed  by Intel  in writing, you may not remove  or alter
##  this  notice or  any other notice embedded  in Materials by Intel or Intel's
##  suppliers or licensors in any way.
##
##******************************************************************************
##  Content:
##     Intel(R) Math Kernel Library Fortran-style Cluster DFT
##     examples creation and run
##******************************************************************************

help:
	@echo "Usage: make {soia32|sointel64|libia32|libintel64} [example=name]"
	@echo "[mpi=intelmpi3|intelmpi|mpich|mpich2|openmpi] [compiler=intel|gnu]"
	@echo "[interface=interface_name] [workdir=path] [mpidir=path]"
	@echo "[threading=threading_name]"
	@echo
	@echo "example=name - name of example. Please see names in cdftf.lst file."
	@echo "If parameter is omitted, all examples will be compiled."
	@echo
	@echo "mpi=intelmpi  - using Intel(R) MPI Library 4.x, default"
	@echo "mpi=intelmpi3  - using Intel(R) MPI Library 3.x"
	@echo "mpi=mpich   - using MPICH"
	@echo "mpi=mpich2  - using MPICH2"
	@echo "mpi=openmpi - using Open MPI"
	@echo
	@echo "compiler=intel - using Intel(R) Fortran Compiler, default"
	@echo "compiler=gnu   - using GNU gfortran compiler"
	@echo "interface_name - can be lp64 or ilp64 for intel64. Default value is lp64."
	@echo "threading_name - can be parallel or sequential. Default value is parallel."
	@echo
	@echo "workdir=path - path to work directory, which is accessible from any node."
	@echo "If parameter is omitted, executable files and results will be located in"
	@echo "current directory."
	@echo
	@echo "mpidir=path - path to MPI installation directory. MPI scripts are taken"
	@echo "from mpidir/bin (or mpidir/bin64 for Intel(R) MPI Library and Intel(R) 64 architecture). If this directory"
	@echo "is in PATH you can omit mpidir parameter."
	@echo "MPICH should be compiled by the compiler with same version as compiler used to build examples."
	@echo
	@echo "Set environment variables LD_LIBRARY_PATH and etc properly before testing."

##------------------------------------------------------------------------------
## examples of using:
##
## make libia32 mpi=intelmpi example=dm_complex_2d_double_ex1 workdir=/share
##                 - build and run DM_COMPLEX_2D_DOUBLE_EX1
##                   example for 32-bit applications in
##                   dir /share, static linking, using
##                   Intel(R) MPI Library 4.x, Intel(R) Fortran Compiler
##
## make libintel64 mpi=mpich compiler=intel mpidir=/opt/mpich
##                 - build and run all examples of CDFT for
##                   Intel(R) 64 processor family applications
##                   in current dir, static linking, using
##                   MPICH 1.2.x from /opt/mpich,
##                   Intel(R) Fortran Compiler
##------------------------------------------------------------------------------

include cdftf.lst

SHELL = /bin/sh
ENV = $(shell which env)

ifndef example
   example = $(CDFT)
endif

ifndef mpi
   mpi = intelmpi
endif

ifndef compiler
   compiler = intel
endif

ifndef workdir
   workdir = .
endif

ifndef interface
   interface=lp64
endif

ifndef threading
   threading=parallel
endif

RES = $(addsuffix .res,$(example))

ifndef MKLROOT
   MKLROOT = ../..
endif
MKL_PATH = "$(MKLROOT)/lib/$(_IA)"
CMPLR_PATH = "$(MKLROOT)/../compiler/lib/$(_IA)"

FOPTS =

ifeq ($(compiler),intel)
   IFACE_COMP_PART=intel
   IFACE_THREADING_PART=intel
   OMP_LIB = -openmp
   CPP_OPT = -cpp
   FOPTS += -warn all
# mpif.h file is in the fixed form and is included into source file
   MPI_FMT=-fixed
   MOD_DIR_OPTION=-module $(RES_DIR) -I.
endif

ifeq ($(compiler),gnu)
   IFACE_COMP_PART=gf
   IFACE_THREADING_PART=gnu
   OMP_LIB = -fopenmp
   CPP_OPT = -x f95-cpp-input
   FOPTS += -Wall
# mpif.h file is in the fixed form and is included into source file
   MPI_FMT=-ffixed-form
   MOD_DIR_OPTION=-J$(RES_DIR) -I.
endif

ifeq ($(_IA),ia32)
   ifeq ($(compiler),intel)
      SPEC_OPT+=-mia32
#This option is required by Intel(R) 11.0 compiler to produce workable binaries for Pentium(R) III.
#If you don't need it, you can remove this option.
   endif
   IFACE_LIB=$(MKL_PATH)/libmkl_$(IFACE_COMP_PART).$(EXT)
   FOPTS +=
   BLACS_PART=
else
   ifeq ($(interface),ilp64)
      ifeq ($(compiler),intel)
          FOPTS += -i8
      endif
      ifeq ($(compiler),gnu)
          FOPTS += -fdefault-integer-8
      endif
      IFACE_LIB=$(MKL_PATH)/libmkl_$(IFACE_COMP_PART)_ilp64.$(EXT)
      BLACS_PART=_ilp64
   else
      FOPTS +=
      IFACE_LIB=$(MKL_PATH)/libmkl_$(IFACE_COMP_PART)_lp64.$(EXT)
      BLACS_PART=_lp64
   endif
endif

ifeq ($(threading),sequential)
   THREADING_LIB=$(MKL_PATH)/libmkl_sequential.$(EXT)
else
   THREADING_LIB=$(MKL_PATH)/libmkl_$(IFACE_THREADING_PART)_thread.$(EXT)
endif

CORE_LIB=$(MKL_PATH)/libmkl_core.$(EXT)

ifeq ($(EXT),a)
#Required to import weak symbols if static version of the pthread library is linked
   OMP_LIB+= -Wl,--whole-archive -lpthread -Wl,--no-whole-archive
   # OpenMP lib in MKL_LIBS is set later
   MKL_LIBS= -Wl,--start-group $(MKL_PATH)/libmkl_cdft_core.$(EXT) $(IFACE_LIB) $(THREADING_LIB) $(CORE_LIB) $(BLACS_LIB) -Wl,--end-group
else
   OMP_LIB+= -lpthread
   # OpenMP lib in MKL_LIBS is set later
   MKL_LIBS=$(MKL_PATH)/libmkl_cdft_core.$(EXT) $(IFACE_LIB) $(THREADING_LIB) $(CORE_LIB) $(BLACS_LIB)
endif

ifeq ($(threading),parallel)
   MKL_LIBS += $(OMP_LIB)
endif

ifeq ($(_IA),ia32)
   RES_DIR=_results/$(compiler)_$(mpi)_$(_IA)_$(RES_EXT)_$(threading)$Z
else
   RES_DIR=_results/$(compiler)_$(mpi)_$(interface)_$(_IA)_$(RES_EXT)_$(threading)$Z
endif

libia32 lib32:
	$(MAKE) $(RES) _IA=ia32 Ibin=bin EXT=a RES_EXT=lib
libintel64 libem64t:
	$(MAKE) $(RES) _IA=intel64 Ibin=bin64 EXT=a RES_EXT=lib
soia32 so32:
	$(MAKE) $(RES) _IA=ia32 Ibin=bin EXT=so RES_EXT=so
sointel64 soem64t:
	$(MAKE) $(RES) _IA=intel64 Ibin=bin64 EXT=so RES_EXT=so

ifeq ($(mpi),mpich2)
   ifdef mpidir
      _BD = $(mpidir)/bin/
   endif
   ifeq ($(compiler),intel)
      CS = $(_BD)mpif90 -f90=ifort $(FOPTS)
   endif
   ifeq ($(compiler),gnu)
      CS = $(_BD)mpif90 -f90=gfortran $(FOPTS)
   endif
   MPITHR =#
   FOPTS += -DMPI_KIND_=4
   RS = $(_BD)mpiexec -n 2
   Bs = _intelmpi
   BLACS_LIB=$(MKL_PATH)/libmkl_blacs$(Bs)$(BLACS_PART).a
endif

ifeq ($(mpi),intelmpi3)
   ifdef mpidir
      _BD = $(mpidir)/$(Ibin)/
   endif
   ifeq ($(compiler),intel)
      CS = $(_BD)mpiifort $(FOPTS)
   endif
   ifeq ($(compiler),gnu)
      CS = $(_BD)mpif90 -f90=gfortran $(FOPTS)
   endif
   ifeq ($(threading),parallel)
      MPITHR = -mt_mpi
   else
      MPITHR =#
   endif
   FOPTS += -DMPI_KIND_=4
   RS = $(_BD)mpiexec -n 2
   Bs = _intelmpi
   BLACS_LIB=$(MKL_PATH)/libmkl_blacs$(Bs)$(BLACS_PART).$(EXT)
endif

ifeq ($(mpi),intelmpi)
   ifdef mpidir
      _BD = $(mpidir)/$(Ibin)/
   endif
   ifeq ($(compiler),intel)
      CS = $(_BD)mpiifort $(FOPTS)
   endif
   ifeq ($(compiler),gnu)
      CS = $(_BD)mpif90 -f90=gfortran $(FOPTS)
   endif
   ifeq ($(threading),parallel)
      MPITHR = -mt_mpi
   else
      MPITHR =#
   endif
   ifeq ($(_IA),ia32)
      FOPTS += -DMPI_KIND_=4
   else
      ifeq ($(interface),ilp64)
         ifeq ($(compiler),intel)
            # -i8 enables Intel MPI 4.x ilp64 interface
            FOPTS += -DMPI_KIND_=8
         else
            FOPTS += -DMPI_KIND_=4
         endif
      else
         FOPTS += -DMPI_KIND_=4
      endif
   endif
   RS = $(_BD)mpiexec -n 2
   Bs = _intelmpi
   BLACS_LIB=$(MKL_PATH)/libmkl_blacs$(Bs)$(BLACS_PART).$(EXT)
endif

ifeq ($(mpi),mpich)
   ifdef mpidir
      _BD = $(mpidir)/bin/
   endif
   ifeq ($(compiler),intel)
      CS = $(_BD)mpif90 -f90=ifort $(FOPTS)
   endif
   ifeq ($(compiler),gnu)
      CS = $(_BD)mpif90 -f90=gfortran $(FOPTS)
   endif
   MPITHR =#
   FOPTS += -DMPI_KIND_=4
   RS = $(_BD)mpirun -np 2
   Bs =#
   BLACS_LIB=$(MKL_PATH)/libmkl_blacs$(Bs)$(BLACS_PART).a
endif

ifeq ($(mpi),openmpi)
   ifdef mpidir
      _BD = $(mpidir)/bin/
   endif
   ifeq ($(compiler),intel)
      CS = $(_BD)mpif90 -f90=ifort $(FOPTS)
   endif
   ifeq ($(compiler),gnu)
      CS = $(_BD)mpif90 -f90=gfortran $(FOPTS)
   endif
   MPITHR =#
   FOPTS += -DMPI_KIND_=4
   RS = $(_BD)mpiexec -np 2
   Bs = _openmpi
   BLACS_LIB=$(MKL_PATH)/libmkl_blacs$(Bs)$(BLACS_PART).a
endif

COMMON = "$(MKLROOT)/include/mkl_cdft.f90" source/cdft_example_support.f90

#-------------------------------------------------------------------------------

vpath %.f90 source

$(RES): %.res: %.f90
	mkdir -p $(RES_DIR)
	$(CS) $(SPEC_OPT) $(CPP_OPT) "$(MKLROOT)/include/mkl_cdft.f90" $(MOD_DIR_OPTION) -c -o $(RES_DIR)/mkl_cdft.o
	$(CS) $(SPEC_OPT) $(CPP_OPT) $(MPI_FMT) source/cdft_example_support.f90 $(MOD_DIR_OPTION) -c -o $(RES_DIR)/cdft_example_support.o
	$(CS) $(SPEC_OPT) $(CPP_OPT) $(MPI_FMT) $< -c $(MOD_DIR_OPTION) -o $(RES_DIR)/$*.o
	$(CS) $(SPEC_OPT) $(RES_DIR)/$*.o $(RES_DIR)/cdft_example_support.o $(RES_DIR)/mkl_cdft.o -L$(MKL_PATH) $(MKL_LIBS) $(MPITHR) -ldl -lm -o $(RES_DIR)/$*.exe
	$(RS) $(ENV) LD_LIBRARY_PATH=$(MKL_PATH):$(LD_LIBRARY_PATH):$(CMPLR_PATH) $(RES_DIR)/$*.exe < data/$*.dat > $(RES_DIR)/$@
	-rm $(RES_DIR)/*.o $(RES_DIR)/*.mod
#-------------------------------------------------------------------------------
