##******************************************************************************
##  Copyright(C) 2005-2013 Intel Corporation. All Rights Reserved.
##  
##  The source code, information  and  material ("Material") contained herein is
##  owned  by Intel Corporation or its suppliers or licensors, and title to such
##  Material remains  with Intel Corporation  or its suppliers or licensors. The
##  Material  contains proprietary information  of  Intel or  its  suppliers and
##  licensors. The  Material is protected by worldwide copyright laws and treaty
##  provisions. No  part  of  the  Material  may  be  used,  copied, reproduced,
##  modified, published, uploaded, posted, transmitted, distributed or disclosed
##  in any way  without Intel's  prior  express written  permission. No  license
##  under  any patent, copyright  or  other intellectual property rights  in the
##  Material  is  granted  to  or  conferred  upon  you,  either  expressly,  by
##  implication, inducement,  estoppel or  otherwise.  Any  license  under  such
##  intellectual  property  rights must  be express  and  approved  by  Intel in
##  writing.
##  
##  *Third Party trademarks are the property of their respective owners.
##  
##  Unless otherwise  agreed  by Intel  in writing, you may not remove  or alter
##  this  notice or  any other notice embedded  in Materials by Intel or Intel's
##  suppliers or licensors in any way.
##
##******************************************************************************
##  Content:
##     Intel(R) Math Kernel Library C-style Cluster DFT
##     examples creation and run
##******************************************************************************

help:
	@echo "Usage: make {soia32|sointel64|libia32|libintel64|libmic} [example=name]"
	@echo "[mpi=intelmpi|mpich|mpich2|openmpi] [compiler=intel|gnu]"
	@echo "[interface=interface_name] [workdir=path] [mpidir=path]"
	@echo "[threading=threading_name]"
	@echo
	@echo "example=name - name of example. Please see names in cdftc.lst file."
	@echo "If parameter is omitted, all examples will be compiled."
	@echo
	@echo "mpi=intelmpi - using Intel(R) MPI Library, default"
	@echo "mpi=mpich    - using MPICH"
	@echo "mpi=mpich2   - using MPICH2"
	@echo "mpi=openmpi  - using Open MPI"
	@echo
	@echo "compiler=intel - using Intel(R) C Compiler, default"
	@echo "compiler=gnu   - using GNU C compiler"
	@echo "interface_name - can be lp64 or ilp64 for intel64. Default value is lp64."
	@echo "threading_name - can be parallel or sequential. Default value is parallel."
	@echo
	@echo "workdir=path - path to work directory, which is accessible from any node."
	@echo "If parameter is omitted, executable files and results will be located in"
	@echo "current directory."
	@echo
	@echo "mpidir=path - path to MPI installation directory. MPI scripts are taken"
	@echo "from mpidir/bin (or mpidir/bin64 for Intel(R) MPI Library and Intel(R) 64 architecture). If this directory"
	@echo "is in PATH you can omit mpidir parameter."
	@echo "MPICH should be compiled by the compiler with same version as compiler used to build examples."
	@echo
	@echo "Set environment variables LD_LIBRARY_PATH and etc properly before testing."

##------------------------------------------------------------------------------
## examples of using:
##
## make libia32 example=dm_complex_2d_double_ex1 workdir=/share
##                 - build and run DM_COMPLEX_2D_DOUBLE_EX1
##                   example for 32-bit applications in
##                   dir /share, static linking, using
##                   Intel(R) MPI Library, Intel(R) C Compiler
##
## make libintel64 mpi=mpich compiler=gnu mpidir=/opt/mpich
##                 - build and run all examples of CDFT for
##                   Intel(R) 64 processor family applications
##                   in current dir, static linking, using
##                   MPICH 1.2.x from /opt/mpich, GNU C compiler
##------------------------------------------------------------------------------

include cdftc.lst

SHELL = /bin/sh
ENV = $(shell which env)

ifndef example
   example = $(CDFT)
endif

ifndef mpi
   mpi = intelmpi
endif

ifndef compiler
   compiler = intel
endif

ifndef workdir
   workdir = .
endif

ifndef interface
   interface=lp64
endif

ifndef threading
   threading=parallel
endif

RES = $(addsuffix .res,$(example))

ifndef MKLROOT
   MKLROOT = ../..
endif
MKL_PATH = "$(MKLROOT)/lib/$(_IA)"
CMPLR_PATH = "$(MKLROOT)/../compiler/lib/$(_IA)"

COPTS = -Wall

ifeq ($(compiler),intel)
   IFACE_COMP_PART=intel
   IFACE_THREADING_PART=intel
   OMP_LIB = -openmp
endif
ifeq ($(compiler),gnu)
   IFACE_COMP_PART=intel
   IFACE_THREADING_PART=gnu
   OMP_LIB = -fopenmp
endif

ifeq ($(_IA),ia32)
   ifeq ($(compiler),intel)
      SPEC_OPT+=-mia32
#This option is required by Intel(R) 11.0 compiler to produce workable binaries for Pentium(R) III.
#If you don't need it, you can remove this option.
   endif
   IFACE_LIB=$(MKL_PATH)/libmkl_$(IFACE_COMP_PART).$(EXT)
   COPTS +=
   BLACS_PART=
endif
ifeq ($(_IA),intel64)
   ifeq ($(interface),ilp64)
      COPTS += -DMKL_ILP64
      IFACE_LIB=$(MKL_PATH)/libmkl_$(IFACE_COMP_PART)_ilp64.$(EXT)
      BLACS_PART=_ilp64
   else
      COPTS +=
      IFACE_LIB=$(MKL_PATH)/libmkl_$(IFACE_COMP_PART)_lp64.$(EXT)
      BLACS_PART=_lp64
   endif
endif
ifeq ($(_IA),mic)
   ifneq ($(mpi),intelmpi)
      $(error MIC is supported only with Intel MPI)
   endif
   ifneq ($(compiler),intel)
      $(error MIC is supported only with Intel Compiler)
   endif
   MIC_IP=mic0
   MICPUTCMD=scp
   MIC_RES_DIR=/tmp
   MICPUT_RES_DIR=$(MIC_IP):$(MIC_RES_DIR)
   ifeq ($(interface),ilp64)
      COPTS += -mmic -DMKL_ILP64
      IFACE_LIB=$(MKL_PATH)/libmkl_$(IFACE_COMP_PART)_ilp64.$(EXT)
      BLACS_PART=_ilp64
   else
      COPTS += -mmic
      IFACE_LIB=$(MKL_PATH)/libmkl_$(IFACE_COMP_PART)_lp64.$(EXT)
      BLACS_PART=_lp64
   endif
endif

ifeq ($(threading),sequential)
   THREADING_LIB=$(MKL_PATH)/libmkl_sequential.$(EXT)
else
   THREADING_LIB=$(MKL_PATH)/libmkl_$(IFACE_THREADING_PART)_thread.$(EXT)
endif

CORE_LIB=$(MKL_PATH)/libmkl_core.$(EXT)

ifeq ($(EXT),a)
   #Required to import weak symbols if static version of the pthread library is linked
   OMP_LIB+= -Wl,--whole-archive -lpthread -Wl,--no-whole-archive
   # OpenMP lib in MKL_LIBS is set later
   MKL_LIBS= -Wl,--start-group $(MKL_PATH)/libmkl_cdft_core.$(EXT) $(IFACE_LIB) $(THREADING_LIB) $(CORE_LIB) $(BLACS_LIB) -Wl,--end-group
else
   OMP_LIB+= -lpthread
   # OpenMP lib in MKL_LIBS is set later
   MKL_LIBS=$(MKL_PATH)/libmkl_cdft_core.$(EXT) $(IFACE_LIB) $(THREADING_LIB) $(CORE_LIB) $(BLACS_LIB)
endif

ifeq ($(threading),parallel)
   MKL_LIBS += $(OMP_LIB)
endif

ifeq ($(_IA),ia32)
   RES_DIR=_results/$(compiler)_$(mpi)_$(_IA)_$(RES_EXT)_$(threading)$Z
else
   RES_DIR=_results/$(compiler)_$(mpi)_$(interface)_$(_IA)_$(RES_EXT)_$(threading)$Z
endif

libia32 lib32:
	$(MAKE) $(RES) _IA=ia32 Ibin=bin EXT=a RES_EXT=lib
libintel64 libem64t:
	$(MAKE) $(RES) _IA=intel64 Ibin=bin64 EXT=a RES_EXT=lib
soia32 so32:
	$(MAKE) $(RES) _IA=ia32 Ibin=bin EXT=so RES_EXT=so
sointel64 soem64t:
	$(MAKE) $(RES) _IA=intel64 Ibin=bin64 EXT=so RES_EXT=so
libmic:
	$(MAKE) $(RES) _IA=mic Ibin=bin EXT=a RES_EXT=lib

ifeq ($(mpi),mpich2)
   ifdef mpidir
      _BD = $(mpidir)/bin/
   endif
   ifeq ($(compiler),intel)
      CS = $(_BD)mpicc -cc=icc $(COPTS)
   endif
   ifeq ($(compiler),gnu)
      CS = $(_BD)mpicc -cc=gcc $(COPTS)
   endif
   MPITHR =#
   RS = $(_BD)mpiexec -n 2
   Bs = _intelmpi
   BLACS_LIB=$(MKL_PATH)/libmkl_blacs$(Bs)$(BLACS_PART).a
endif

ifeq ($(mpi),intelmpi)
   ifdef mpidir
      _BD = $(mpidir)/$(Ibin)/
   endif
   ifeq ($(compiler),intel)
      CS = $(_BD)mpiicc $(COPTS)
   endif
   ifeq ($(compiler),gnu)
      CS = $(_BD)mpicc -cc=gcc $(COPTS)
   endif
   ifeq ($(threading),parallel)
      MPITHR = -mt_mpi
   else
      MPITHR =#
   endif
   ifeq ($(_IA),mic)
      RS = $(_BD)mpiexec -n 2 -wdir $(MIC_RES_DIR) -host $(MIC_IP)
   else
      RS = $(_BD)mpiexec -n 2
   endif
   Bs = _intelmpi
   BLACS_LIB=$(MKL_PATH)/libmkl_blacs$(Bs)$(BLACS_PART).$(EXT)
endif

ifeq ($(mpi),mpich)
   ifdef mpidir
      _BD = $(mpidir)/bin/
   endif
   ifeq ($(compiler),intel)
      CS = $(_BD)mpicc -cc=icc $(COPTS)
   endif
   ifeq ($(compiler),gnu)
      CS = $(_BD)mpicc -cc=gcc  $(COPTS)
   endif
   MPITHR =#
   RS = $(_BD)mpirun -np 2
   Bs =#
   BLACS_LIB=$(MKL_PATH)/libmkl_blacs$(Bs)$(BLACS_PART).a
endif

ifeq ($(mpi),openmpi)
   ifdef mpidir
      _BD = $(mpidir)/bin/
   endif
   COPTS += -DUSING_OPEN_MPI
   ifeq ($(compiler),intel)
      CS = $(_BD)mpicc -cc=icc $(COPTS)
   endif
   ifeq ($(compiler),gnu)
      CS = $(_BD)mpicc -cc=gcc  $(COPTS)
   endif
   MPITHR =#
   RS = $(_BD)mpiexec -np 2
   Bs = _openmpi
   BLACS_LIB=$(MKL_PATH)/libmkl_blacs$(Bs)$(BLACS_PART).a
endif

COMMON = source/cdft_example_support.c

#-------------------------------------------------------------------------------

vpath %.c source

$(RES): %.res: %.c
	mkdir -p $(RES_DIR)
	$(CS) $(SPEC_OPT) $(COMMON) -I "$(MKLROOT)/include" -c -o $(RES_DIR)/cdft_example_support.o
	$(CS) $(SPEC_OPT) $< -I "$(MKLROOT)/include" -c -o $(RES_DIR)/$*.o
	$(CS) $(SPEC_OPT) $(RES_DIR)/$*.o $(RES_DIR)/cdft_example_support.o -L $(MKL_PATH) $(MKL_LIBS) $(MPITHR) -ldl -lm -o $(RES_DIR)/$*.exe
ifeq ($(_IA),mic)
	$(MICPUTCMD) $(CMPLR_PATH)/libiomp5.so $(MICPUT_RES_DIR)/libiomp5.so
	$(MICPUTCMD) $(RES_DIR)/$*.exe $(MICPUT_RES_DIR)/$*.exe
	$(MICPUTCMD) data/$*.dat $(MICPUT_RES_DIR)/$*.dat
	$(RS) $(MIC_RES_DIR)/$*.exe $(MIC_RES_DIR)/$*.dat > $(RES_DIR)/$@
else
	$(RS) $(ENV) LD_LIBRARY_PATH=$(MKL_PATH):$(LD_LIBRARY_PATH):$(CMPLR_PATH) $(RES_DIR)/$*.exe data/$*.dat > $(RES_DIR)/$@
endif
	-rm $(RES_DIR)/*.o
#-------------------------------------------------------------------------------
