##*****************************************************************************
## Copyright(C) 2006-2013 Intel Corporation. All Rights Reserved.
##
## The source code, information  and  material ("Material") contained herein is
## owned  by Intel Corporation or its suppliers or licensors, and title to such
## Material remains  with Intel Corporation  or its suppliers or licensors. The
## Material  contains proprietary information  of  Intel or  its  suppliers and
## licensors. The  Material is protected by worldwide copyright laws and treaty
## provisions. No  part  of  the  Material  may  be  used,  copied, reproduced,
## modified, published, uploaded, posted, transmitted, distributed or disclosed
## in any way  without Intel's  prior  express written  permission. No  license
## under  any patent, copyright  or  other intellectual property rights  in the
## Material  is  granted  to  or  conferred  upon  you,  either  expressly,  by
## implication, inducement,  estoppel or  otherwise.  Any  license  under  such
## intellectual  property  rights must  be express  and  approved  by  Intel in
## writing.
##
## *Third Party trademarks are the property of their respective owners.
##
## Unless otherwise  agreed  by Intel  in writing, you may not remove  or alter
## this  notice or  any other notice embedded  in Materials by Intel or Intel's
## suppliers or licensors in any way.
##
##*****************************************************************************
## Content:
##      Intel(R) Math Kernel Library
##      C-style Cluster DFT wrappers MPI FFTW2 examples creation and run
##*****************************************************************************

help:
	@echo "Usage: make libia32|libintel64"
	@echo "            [mpi=intelmpi|mpich|mpich2|openmpi]"
	@echo "            [compiler=intel|gnu]"
	@echo "            [workdir=path]"
	@echo "            [mpidir=path]"
	@echo "            [precision=MKL_DOUBLE|MKL_SINGLE]"
	@echo
	@echo "mpi=intelmpi - using Intel(R) MPI Library, default"
	@echo "mpi=mpich    - using MPICH"
	@echo "mpi=mpich2   - using MPICH2"
	@echo "mpi=openmpi  - using Open MPI"
	@echo
	@echo "compiler=intel - using Intel(R) C Compiler, default"
	@echo "compiler=gnu   - using GNU C compiler"
	@echo
	@echo "workdir=path"
	@echo "  path to work directory, which is accessible from any node."
	@echo "  If parameter is omitted, executable files and results"
	@echo "  will be located in current directory."
	@echo
	@echo "precision=MKL_DOUBLE - for double precision data, default"
	@echo "precision=MKL_SINGLE - for single precision data"
	@echo
	@echo "mpidir=path"
	@echo "  path to MPI installation directory. MPI scripts are taken"
	@echo "  from mpidir/bin (or mpidir/bin64 for Intel(R) MPI Library"
	@echo "  and Intel(R) 64 architecture). If this directory"
	@echo "  is in PATH you can omit mpidir parameter."
	@echo "  When using MPICH, make sure that MPICH was compiled"
	@echo "  by same compiler as used to build examples."
	@echo
	@echo "MKLROOT=<path>"
	@echo "  Path to MKL root directory with header files and libraries."
	@echo "  Default value: ../.."
	@echo
	@echo "INSTALL_DIR=<path>"
	@echo "  Use <path> for installing of wraplib, if wraplib=yes"
	@echo "  Default value: . (that is, use current work directory)"
	@echo
	@echo "N.B. Make sure that environment variables such as LD_LIBRARY_PATH"
	@echo "     and etc are set properly before testing."
	@echo

##*****************************************************************************
## examples of using:
##
## make libintel64
##      - compile and run wrappers examples
##        using Intel(R) MPI Library, Intel(R) C Compiler,
##        for double precision data for
##        Intel(R) 64 based applications
##
## make libia32 precision=MKL_SINGLE mpi=mpich mpidir=/opt/mpich compiler=gnu
##      - compile and run wrappers examples using
##        MPICH from /opt/mpich, GNU C compiler
##        for single precision data for 32-bit applications
##*****************************************************************************

include cdft_wrappers.lst

ifndef INSTALL_DIR
  INSTALL_DIR = .
endif

ifndef RES_DIR
  RES_DIR = _results
endif

interface = lp64

ifeq ($(interface),ilp64)
  ILP_OPTS = -DMKL_ILP64
  ILP_EXT = _ilp64
else
  ILP_EXT = _lp64
endif

example = $(WRAPPER)

ifndef mpi
  mpi = intelmpi
endif

ifndef compiler
  compiler = intel
endif

ifndef workdir
  workdir = .
endif

ifneq (,$(filter MKL_SINGLE FFTW_ENABLE_FLOAT,$(precision)))
  FL = -DFFTW_ENABLE_FLOAT
  PR = _SINGLE
  PR_L = MKL_SINGLE
else
  FL =
  PR = _DOUBLE
  PR_L = MKL_DOUBLE
endif

RES = $(addsuffix .res,$(example))

ifndef MKLROOT
  MKLROOT = ../..
endif

MKL_PATH = $(MKLROOT)/lib/$(_IA)
CMPLR_PATH = $(MKLROOT)/../compiler/lib/$(_IA)

COPTS =

ifeq ($(_IA),ia32)
  IFACE_LIB = $(MKL_PATH)/libmkl_intel.$(EXT)
  BLACS_PART =
else
  IFACE_LIB = $(MKL_PATH)/libmkl_intel_lp64.$(EXT)
  BLACS_PART = _lp64
endif

THREADING_LIB=$(MKL_PATH)/libmkl_intel_thread.$(EXT)
OMP_LIB = -L$(CMPLR_PATH) -liomp5

CORE_LIB = $(MKL_PATH)/libmkl_core.$(EXT)

MKL_LIBS = -Wl,--start-group
MKL_LIBS += $(MKL_PATH)/libmkl_cdft_core.$(EXT)
MKL_LIBS += $(IFACE_LIB) $(THREADING_LIB) $(CORE_LIB)
MKL_LIBS += -lmkl_blacs$(Bs)$(BLACS_PART)
MKL_LIBS += -Wl,--end-group $(OMP_LIB)

ifeq ($(_IA),ia32)
  _resdir = $(RES_DIR)/$(compiler)_$(_IA)_$(RES_EXT)
  override interface =
else
  _resdir = $(RES_DIR)/$(compiler)_$(interface)_$(_IA)_$(RES_EXT)
endif

ifeq ($(mpi),mpich2)
  ifeq ($(compiler),intel)
    CS = mpicc -cc=icc
  endif
  ifeq ($(compiler),gnu)
    CS = mpicc -cc=gcc
  endif
  RS = mpiexec -n 2
  ifdef mpidir
    _CS = $(mpidir)/bin/$(CS)
    _RS = $(mpidir)/bin/$(RS)
  else
    _CS = $(CS)
    _RS = $(RS)
  endif
  Bs = _intelmpi
  export OMP_NUM_THREADS=1
endif

ifeq ($(mpi),intelmpi)
  ifeq ($(compiler),intel)
    CS = mpiicc -mt_mpi
  endif
  ifeq ($(compiler),gnu)
    CS = mpicc -mt_mpi
  endif
  RS = mpiexec -n 2
  ifdef mpidir
    _CS = $(mpidir)/$(_IA)/bin/$(CS)
    _RS = $(mpidir)/$(_IA)/bin/$(RS)
  else
    _CS = $(CS)
    _RS = $(RS)
  endif
  Bs = _intelmpi
endif

ifeq ($(mpi),mpich)
  ifeq ($(compiler),intel)
    CS = mpicc -cc=icc
  endif
  ifeq ($(compiler),gnu)
    CS = mpicc -cc=gcc
  endif
  RS = mpirun -np 2
  ifdef mpidir
    _CS = $(mpidir)/bin/$(CS)
    _RS = $(mpidir)/bin/$(RS)
  else
    _CS = $(CS)
    _RS = $(RS)
  endif
  DelF = rm -f $(_resdir)/*.o
  export OMP_NUM_THREADS=1
endif

ifeq ($(mpi),openmpi)
  ifeq ($(compiler),intel)
    CS = mpicc -cc=icc $(COPTS)
  endif
  ifeq ($(compiler),gnu)
    CS = mpicc -cc=gcc  $(COPTS)
  endif
  RS = mpiexec -np 2
  ifdef mpidir
    _CS = $(mpidir)/bin/$(CS)
    _RS = $(mpidir)/bin/$(RS)
  else
    _CS = $(CS)
    _RS = $(RS)
  endif
  Bs = _openmpi
  DelF = rm -f $(_resdir)/*.o
endif

wraplibdir = $(INSTALL_DIR)/wrap_lib/$(_IA)
wraplibname = fftw2x_cdft$(PR)$(ILP_EXT)

##-----------------------------------------------------------------------------
## Rules

.PHONY: libia32 lib32 libintel64 libem64t \
        wrap_lib mkresdir

vpath %.c source

.SUFFIXES:
.SUFFIXES: .c .res

libia32 lib32:
	$(MAKE) wrap_lib        _IA=ia32
	$(MAKE) mkresdir $(RES) _IA=ia32    EXT=a RES_EXT=lib

libintel64 libem64t:
	$(MAKE) wrap_lib        _IA=intel64
	$(MAKE) mkresdir $(RES) _IA=intel64 EXT=a RES_EXT=lib

wrap_lib: "$(wraplibdir)"/lib$(wraplibname).a

# We shall use absolute path for $(wraplibdir)
_install_to=$(shell mkdir -p "$(wraplibdir)"; cd "$(wraplibdir)" && pwd)

"$(wraplibdir)"/lib$(wraplibname).a:
	@echo
	# build $@
	cd $(MKLROOT)/interfaces/fftw2x_cdft \
	  && $(MAKE) lib$(_IA) mpi=$(mpi) \
	        interface=$(interface) compiler=$(compiler) mpidir="$(mpidir)" \
	        PRECISION=$(PR_L) INSTALL_DIR="$(_install_to)" INSTALL_LIBNAME="lib$(wraplibname)"

mkresdir:
	@echo
	@echo See results in $(_resdir)
	@echo
	test -d $(_resdir) || mkdir -p $(_resdir)

$(RES): %.res: %.c
	@echo
	# compile $*.c
	$(_CS) $(ILP_OPTS) $(FL) \
	  -I $(MKLROOT)/include -I $(MKLROOT)/include/fftw \
	  $< -c -o $(_resdir)/$*.o
	$(_CS) $(ILP_OPTS) $(FL) -O0 $(_resdir)/$*.o \
	  -L"$(wraplibdir)" -l$(wraplibname) \
	  -L$(MKLROOT)/lib/$(_IA) $(MKL_LIBS) \
	  -lpthread -lm -ldl -o $(_resdir)/$*.exe
	@echo
	# run $*.exe
	LD_LIBRARY_PATH=$(MKL_PATH):$(CMPLR_PATH):$(LD_LIBRARY_PATH) \
	  $(_RS) $(_resdir)/$*.exe > $(_resdir)/$@
	-$(DelF)

##-----------------------------------------------------------------------------
