##*****************************************************************************
## Copyright(C) 2006-2013 Intel Corporation. All Rights Reserved.
##
## The source code, information  and  material ("Material") contained herein is
## owned  by Intel Corporation or its suppliers or licensors, and title to such
## Material remains  with Intel Corporation  or its suppliers or licensors. The
## Material  contains proprietary information  of  Intel or  its  suppliers and
## licensors. The  Material is protected by worldwide copyright laws and treaty
## provisions. No  part  of  the  Material  may  be  used,  copied, reproduced,
## modified, published, uploaded, posted, transmitted, distributed or disclosed
## in any way  without Intel's  prior  express written  permission. No  license
## under  any patent, copyright  or  other intellectual property rights  in the
## Material  is  granted  to  or  conferred  upon  you,  either  expressly,  by
## implication, inducement,  estoppel or  otherwise.  Any  license  under  such
## intellectual  property  rights must  be express  and  approved  by  Intel in
## writing.
##
## *Third Party trademarks are the property of their respective owners.
##
## Unless otherwise  agreed  by Intel  in writing, you may not remove  or alter
## this  notice or  any other notice embedded  in Materials by Intel or Intel's
## suppliers or licensors in any way.
##
##*****************************************************************************
## Content:
##      Build and run examples using FFTW3 Fortran wrappers to Intel MKL.
##*****************************************************************************

## Usage examples:
##
## make libia32 compiler=gnu
##        Build and run all examples using MKL for 32-bit applications
##        and GNU Fortran compiler.
##
## make libintel64 function=complex_1d_double_ex1 threading=sequential
##        Build and run one example using MKL for Intel(R) 64
##        processor family applications with sequential layer.

help:
	@echo "Usage: make {libia32|soia32|libintel64|sointel64} [option...]"
	@echo
	@echo "Options:"
	@echo "  function=<name>"
	@echo "        Run only the specified example. Please see file"
	@echo "        fftw3xf.lst for the list of functions."
	@echo
	@echo "  compiler=gnu|pgi|intel"
	@echo "        Build examples using GNU gfortran, PGI pgf95, or"
	@echo "        Intel(R) Fortran compiler ifort."
	@echo "        Default value: intel"
	@echo "        Note: GNU g77 compiler is not supported in these examples."
	@echo
	@echo "  interface=ilp64|lp64"
	@echo "        For Intel(R) 64 only, use ILP64 MKL interface layer"
	@echo "        when INTEGER is INTEGER*8"
	@echo "        Default value: lp64"
	@echo
	@echo "  threading=sequential|parallel"
	@echo "        Build examples using parallel or sequential MKL."
	@echo "        Default value: parallel"
	@echo
	@echo "  parallel=gnu|pgi|intel"
	@echo "        For threading=parallel, select MKL threading layer."
	@echo "        Default value depends on <compiler>"
	@echo
	@echo "  omp=gomp|pgmp|iomp5"
	@echo "        For threading=parallel, select system OpenMP library:"
	@echo "          iomp5      (if compiler=intel)"
	@echo "          iomp5|gomp (if compiler=gnu)"
	@echo "          pgmp       (if compiler=pgi)"
	@echo "        Default value depends on <parallel>"
	@echo
	@echo "  wraplib=yes|no"
	@echo "        Build and use standalone FFTW3 wrappers library."
	@echo "        The library will be built with respective C compiler"
	@echo '        and put into $$(INSTALL_DIR)/wrap_lib_$$(IA)'
	@echo "        Default value: no (that is, use integrated wrappers)"
	@echo
	@echo "Additional macros:"
	@echo "  RES_DIR=<path>"
	@echo "        Use <path> for building the examples."
	@echo "        Default value: ./_results"
	@echo
	@echo "  MKLROOT=<path>"
	@echo "        Path to MKL root directory with header files and libraries."
	@echo "        Default value: ../.."
	@echo
	@echo "  INSTALL_DIR=<path>"
	@echo "        Use <path> for installing of wraplib, if wraplib=yes"
	@echo "        Default value: . (that is, use current work directory)"
	@echo
	@echo "  MKLRUNLIB_PATH - MKL lib directory to be used to run examples"
	@echo "        in case of dynamic linkage"
	@echo "        Default value is current MKL libs located at"
	@echo '          $$(MKLROOT)/lib/$$(IA)'
	@echo
	@echo "  LIB_PATH - If you are using gnu or pgi threading layer,"
	@echo "        specify path to libraries: libgomp, libpgmp"
	@echo "        or add needed path to LD_LIBRARY_PATH"
	@echo
	@echo "N.B. To run examples on some specific target architecture"
	@echo "  add needed options to make-arguments: TARGET_ARCH=..."
	@echo "  and refer to GNU, PGI and Intel compilers documentation"
	@echo "  for more details about target architecture specific options"
	@echo

##-----------------------------------------------------------------------------
## Default values

MY_MAKEFILE := $(MAKEFILE_LIST)

ifndef MKLROOT
  MKLROOT = ../..
endif

ifndef INSTALL_DIR
  INSTALL_DIR = .
endif

ifndef MKLRUNLIB_PATH
  MKLRUNLIB_PATH = $(MKLROOT)/lib/$(_IA)
endif

ifndef RES_DIR
  RES_DIR = _results
endif

FFTW3XF = $(MKLROOT)/examples/fftw3xf

include $(FFTW3XF)/fftw3xf.lst

function = $(DFT)
compiler = intel
interface = lp64
threading = parallel
wraplib = no

wraplibdir = $(INSTALL_DIR)/wrap_lib$(_ilp64)_$(_IA)

##-----------------------------------------------------------------------------
## Main targets

.PHONY: lib32 libia32 so32 soia32 \
        libem64t libintel64 soem64t sointel64 \
        report clean cleanup FORCE

libia32 lib32:
	$(MAKE) -f $(MY_MAKEFILE) run _IA=ia32 _LIB=lib

libintel64 libem64t:
	$(MAKE) -f $(MY_MAKEFILE) run _IA=intel64 _LIB=lib

soia32 so32:
	$(MAKE) -f $(MY_MAKEFILE) run _IA=ia32 _LIB=so

sointel64 soem64t:
	$(MAKE) -f $(MY_MAKEFILE) run _IA=intel64 _LIB=so

report: FORCE
	@echo
	@echo "--------------------------------------------------"
	@echo There are `ls $(FFTW3XF)/source/*.f* | wc -l` \
	  examples in $(FFTW3XF)/source
	@echo
	@for D in `ls -d $(RES_DIR)/* 2>/dev/null`; do \
	  echo -n "In   $$D	Total run examples:" \
	    "`ls $$D/*.res 2>/dev/null | wc -l`"; \
	  echo -n "	and Passed: "; \
	  cat /dev/null $$D/*.res 2>/dev/null | grep -c PASSED | cat; \
	done
	@echo "--------------------------------------------------"
	@echo

clean: FORCE
	-rm -rf $(RES_DIR) # clean all results

cleanup: clean
	-rm -rf $(INSTALL_DIR)/wrap_lib* # clean all built wrap_libs

FORCE: ;

ifdef VERBOSE
  $(info )
  $(info compiler=$(compiler) interface=$(interface) threading=$(threading) \
         parallel=$(parallel) omp=$(omp) wraplib=$(wraplib))
  $(info )
endif

ifdef _IA
##-----------------------------------------------------------------------------
## Supporting _macros

_FC_intel = ifort
_FC_gnu = gfortran
_FC_pgi = pgf95
FC = $(firstword $(_FC_$(compiler)) ifort)

ifneq ($(_IA),ia32)
  _ilp64 = _$(interface)
endif

# Define _fflags
_fflags = $(_fflags_$(compiler))
_fflags += $(_fflags_$(compiler)_$(_IA))
_fflags += $(_fflags_$(compiler)_$(interface))
_fflags += $(_fflags_$(compiler)_$(parallel))
_fflags += -I$(MKLROOT)/include/fftw
_fflags += $(FFLAGS) $(TARGET_ARCH)

_fflags_intel = -vec-report0
_fflags_intel_ilp64 = -i8
_fflags_intel_ia32 = -mia32

_fflags_gnu = -fcray-pointer
_fflags_gnu_ilp64 = -fdefault-integer-8
_fflags_gnu_ia32 = -m32
_fflags_gnu_intel64 = -m64

_fflags_pgi = -Mnokeepobj
_fflags_pgi_ilp64 = -i8
_fflags_pgi_pgi = -mp
_fflags_pgi_ia32 = -tp px
_fflags_pgi_intel64 = -tp x64

# Maybe define wraplib
ifeq ($(wraplib),yes)
  _maybe_wrap = _with_wraplib
  _maybe_wraplib = $(wraplibdir)/libfftw3xf.a
  _maybe_wraplib_libs = -L$(wraplibdir) -lfftw3xf
  _maybe_wraplib_makeflags_ilp64 = i8=yes
endif

# Pick MKL interface layer
_lib_iface = $(_lib_iface_$(compiler))$(_ilp64)
_lib_iface_intel = mkl_intel
_lib_iface_pgi = mkl_intel
_lib_iface_gnu = mkl_gf

# Pick MKL threading layer
ifeq ($(threading),parallel)
  parallel = $(_parallel_$(compiler))
  omp = $(_omp_$(parallel))

  _parallel_intel = intel
  _parallel_gnu = gnu
  _parallel_pgi = pgi

  _omp_intel = iomp5
  _omp_gnu = iomp5
  _omp_pgi = pgmp

  _lib_thread = mkl_$(parallel)_thread
  threadlayer = $(parallel)
else
  _lib_thread = mkl_sequential
  threadlayer = $(threading)
endif

ifeq ($(compiler)-$(parallel),pgi-pgi)
  override MKL_LIBS_LINK = mixed
endif

# Pick MKL core layer
_lib_core = mkl_core

_threading = $(threading)
# Maybe pick OpenMP library
ifeq ($(threading),parallel)
  ifneq ($(omp),)
    _maybe_omp = -L$(CMPLR_PATH) -l$(omp)
    _threading = $(threading)_$(omp)
  endif
endif


CMPLR_PATH = $(MKLROOT)/../compiler/lib/$(_IA)

# Finally compose the _libs
_libs = $(_maybe_wraplib_libs)
ifeq ($(_LIB),so)
  _libs += -L$(MKLROOT)/lib/$(_IA)
  ifneq ($(MKL_LIBS_LINK),mixed)
    _libs += -lmkl_rt
  else
    _libs += -l$(_lib_iface) -l$(_lib_thread) -l$(_lib_core)
  endif
else
  _libs += -Wl,--start-group
  _libs += $(MKLROOT)/lib/$(_IA)/lib$(_lib_iface).a
  _libs += $(MKLROOT)/lib/$(_IA)/lib$(_lib_thread).a
  _libs += $(MKLROOT)/lib/$(_IA)/lib$(_lib_core).a
  _libs += -Wl,--end-group
endif
_libs += $(_maybe_omp)
_libs += -lm -lpthread -ldl

ifneq ($(LIB_PATH),)
  RUNLIB_PATH = $(LIB_PATH):$(MKLRUNLIB_PATH):$(CMPLR_PATH)
else
  RUNLIB_PATH = $(MKLRUNLIB_PATH):$(CMPLR_PATH)
endif

ifeq ($(_LIB),so)
  RUNENV = LD_LIBRARY_PATH=$(RUNLIB_PATH):$(LD_LIBRARY_PATH)
  ifneq ($(MKL_LIBS_LINK),mixed)
        RUNENV += MKL_INTERFACE_LAYER=$(interface) MKL_THREADING_LAYER=$(threadlayer)
  endif
else
  RUNENV = LD_LIBRARY_PATH=$(CMPLR_PATH):$(LD_LIBRARY_PATH)
endif

_resdir = $(RES_DIR)/$(compiler)$(_ilp64)_$(_threading)_$(_LIB)$(_IA)$(_maybe_wrap)

##-----------------------------------------------------------------------------
## Rules

vpath %.f   $(FFTW3XF)/source
vpath %.f90 $(FFTW3XF)/source

.SUFFIXES:
.SUFFIXES: .f .f90 .o .res .out

.PHONY: run mkresdir wraplib

run: wraplib mkresdir $(function:%=$(_resdir)/%.res)

.PRECIOUS: $(_resdir)/%.out

$(_resdir)/%.res: $(_resdir)/%.out FORCE
	-rm -f $@ # remove $@ file before run
	@echo
	# run $*.out
	$(RUNENV) \
	  $< > $@

$(_resdir)/%.out: %.f90
	@echo
	# compile $*.f90
	$(FC) $(_fflags) $^ $(_libs) -o $@

mkresdir:
	@echo
	@echo See results in $(_resdir)
	@echo
	test -d $(_resdir) || mkdir -p $(_resdir)

wraplib: $(_maybe_wraplib)

ifeq ($(wraplib),yes)
# We shall use absolute path for install_to
_install_to=$(shell mkdir -p $(wraplibdir); cd $(wraplibdir) && pwd)
$(_maybe_wraplib):
	@echo
	# build $@
	$(MAKE) -C $(MKLROOT)/interfaces/fftw3xf lib$(_IA) \
	  INSTALL_DIR=$(_install_to) INSTALL_LIBNAME=$(@F) \
	  $(_maybe_wraplib_makeflags$(_ilp64))
endif

##-----------------------------------------------------------------------------
endif
