##*****************************************************************************
## Copyright(C) 2006-2013 Intel Corporation. All Rights Reserved.
##
## The source code, information  and  material ("Material") contained herein is
## owned  by Intel Corporation or its suppliers or licensors, and title to such
## Material remains  with Intel Corporation  or its suppliers or licensors. The
## Material  contains proprietary information  of  Intel or  its  suppliers and
## licensors. The  Material is protected by worldwide copyright laws and treaty
## provisions. No  part  of  the  Material  may  be  used,  copied, reproduced,
## modified, published, uploaded, posted, transmitted, distributed or disclosed
## in any way  without Intel's  prior  express written  permission. No  license
## under  any patent, copyright  or  other intellectual property rights  in the
## Material  is  granted  to  or  conferred  upon  you,  either  expressly,  by
## implication, inducement,  estoppel or  otherwise.  Any  license  under  such
## intellectual  property  rights must  be express  and  approved  by  Intel in
## writing.
##
## *Third Party trademarks are the property of their respective owners.
##
## Unless otherwise  agreed  by Intel  in writing, you may not remove  or alter
## this  notice or  any other notice embedded  in Materials by Intel or Intel's
## suppliers or licensors in any way.
##
##*****************************************************************************
## Content:
##      Build and run examples using FFTW3 C wrappers to Intel MKL.
##*****************************************************************************

## Usage examples:
##
## make libia32 compiler=gnu
##        Build and run all examples using MKL for 32-bit applications
##        and GNU C compiler.
##
## make libintel64 function=complex_1d_double_ex1 threading=sequential
##        Build and run one example using MKL for Intel(R) 64
##        processor family applications with sequential layer.

help:
	@echo "Usage: make {libia32|soia32|libintel64|sointel64} [option...]"
	@echo
	@echo "Options:"
	@echo "  function=<name>"
	@echo "        Run only the specified example. Please see file"
	@echo "        fftw3xc.lst for the list of functions."
	@echo
	@echo "  compiler=gnu|pgi|intel"
	@echo "        Build examples using GNU gcc, PGI pgcc, or"
	@echo "        Intel(R) C compiler icc."
	@echo "        Default value: intel"
	@echo
	@echo "  interface=ilp64|lp64"
	@echo "        For Intel(R) 64 only, use ILP64 MKL interface layer"
	@echo "        when MKL_INT is MKL_INT64"
	@echo "        Default value: lp64"
	@echo
	@echo "  threading=sequential|parallel"
	@echo "        Build examples using parallel or sequential MKL."
	@echo "        Default value: parallel"
	@echo
	@echo "  parallel=gnu|pgi|intel"
	@echo "        For threading=parallel, select MKL threading layer."
	@echo "        Default value depends on <compiler>"
	@echo
	@echo "  omp=gomp|pgmp|iomp5"
	@echo "        For threading=parallel, select system OpenMP library:"
	@echo "          iomp5      (if compiler=intel)"
	@echo "          iomp5|gomp (if compiler=gnu)"
	@echo "          pgmp       (if compiler=pgi)"
	@echo "        Default value depends on <parallel>"
	@echo
	@echo "  wraplib=yes|no"
	@echo "        Build and use standalone FFTW3 wrappers library."
	@echo "        The library will be built with respective C compiler"
	@echo '        and put into $$(INSTALL_DIR)/wrap_lib_$$(IA)'
	@echo "        Default value: no (that is, use integrated wrappers)"
	@echo
	@echo "Additional macros:"
	@echo "  RES_DIR=<path>"
	@echo "        Use <path> for building the examples."
	@echo "        Default value: ./_results"
	@echo
	@echo "  MKLROOT=<path>"
	@echo "        Path to MKL root directory with header files and libraries."
	@echo "        Default value: ../.."
	@echo
	@echo "  INSTALL_DIR=<path>"
	@echo "        Use <path> for installing of wraplib, if wraplib=yes"
	@echo "        Default value: . (that is, use current work directory)"
	@echo
	@echo "  MKLRUNLIB_PATH - MKL lib directory to be used to run examples"
	@echo "        in case of dynamic linkage"
	@echo "        Default value is current MKL libs located at"
	@echo '          $$(MKLROOT)/lib/$$(IA)'
	@echo
	@echo "  LIB_PATH - If you are using gnu or pgi threading layer,"
	@echo "        specify path to libraries: libgomp, libpgmp"
	@echo "        or add needed path to LD_LIBRARY_PATH"
	@echo
	@echo "N.B. To run examples on some specific target architecture"
	@echo "  add needed options to make-arguments: TARGET_ARCH=..."
	@echo "  and refer to GNU, PGI and Intel compilers documentation"
	@echo "  for more details about target architecture specific options"
	@echo

##-----------------------------------------------------------------------------
## Default values

MY_MAKEFILE := $(MAKEFILE_LIST)

ifndef MKLROOT
  MKLROOT = ../..
endif

ifndef INSTALL_DIR
  INSTALL_DIR = .
endif

ifndef MKLRUNLIB_PATH
  MKLRUNLIB_PATH = $(MKLROOT)/lib/$(_IA)
endif

ifndef RES_DIR
  RES_DIR = _results
endif

FFTW3XC = $(MKLROOT)/examples/fftw3xc

include $(FFTW3XC)/fftw3xc.lst

function = $(DFT)
compiler = intel
interface = lp64
threading = parallel
wraplib = no

wraplibdir = $(INSTALL_DIR)/wrap_lib$(_ilp64)_$(_IA)

##-----------------------------------------------------------------------------
## Main targets

.PHONY: lib32 libia32 so32 soia32 \
        libem64t libintel64 soem64t sointel64 \
        report clean cleanup FORCE

libia32 lib32:
	$(MAKE) -f $(MY_MAKEFILE) run _IA=ia32 _LIB=lib

libintel64 libem64t:
	$(MAKE) -f $(MY_MAKEFILE) run _IA=intel64 _LIB=lib

soia32 so32:
	$(MAKE) -f $(MY_MAKEFILE) run _IA=ia32 _LIB=so

sointel64 soem64t:
	$(MAKE) -f $(MY_MAKEFILE) run _IA=intel64 _LIB=so

report: FORCE
	@echo
	@echo "--------------------------------------------------"
	@echo There are `ls $(FFTW3XC)/source/*.c | wc -l` \
	  examples in $(FFTW3XC)/source
	@echo
	@for D in `ls -d $(RES_DIR)/* 2>/dev/null`; do \
	  echo -n "In   $$D	Total run examples:" \
	    "`ls $$D/*.res 2>/dev/null | wc -l`"; \
	  echo -n "	and Passed: "; \
	  cat /dev/null $$D/*.res 2>/dev/null | grep -c PASSED | cat; \
	done
	@echo "--------------------------------------------------"
	@echo

clean: FORCE
	-rm -rf $(RES_DIR) # clean all results

cleanup: clean
	-rm -rf $(INSTALL_DIR)/wrap_lib* # clean all built wrap_libs

FORCE: ;

ifdef VERBOSE
  $(info )
  $(info compiler=$(compiler) interface=$(interface) threading=$(threading) \
         parallel=$(parallel) omp=$(omp) wraplib=$(wraplib))
  $(info )
endif

ifdef _IA
##-----------------------------------------------------------------------------
## Supporting _macros

_CC_intel = icc
_CC_gnu = gcc
_CC_pgi = pgcc
CC = $(firstword $(_CC_$(compiler)) icc)

ifneq ($(_IA),ia32)
  _ilp64 = _$(interface)
endif

# Define _cflags
_cflags =
_cflags += $(_cflags_$(compiler))
_cflags += $(_cflags_$(compiler)_$(_IA))
_cflags += $(_cflags_$(compiler)_$(interface))
_cflags += $(_cflags_$(compiler)_$(parallel))
_cflags += $(CFLAGS) $(TARGET_ARCH)

_cflags_intel = -vec-report0
_cflags_intel_ia32 = -mia32

_cflags_pgi = -Mnokeepobj
_cflags_pgi_pgi = -mp -pgf90libs
_cflags_pgi_ia32 = -tp px
_cflags_pgi_intel64 = -tp x64

_cflags_gnu_ia32 = -m32
_cflags_gnu_intel64 = -m64

# Define preprocessor flags
_cppflags = -I$(MKLROOT)/include -I$(MKLROOT)/include/fftw $(CPPFLAGS)

# Maybe define wraplib
ifeq ($(wraplib),yes)
  _maybe_wrap = _with_wraplib
  _maybe_wraplib = $(wraplibdir)/libfftw3xc.a
  _maybe_wraplib_libs = -L$(wraplibdir) -lfftw3xc
  _maybe_wraplib_makeflags_ilp64 = CPPFLAGS=-DMKL_ILP64
endif

# Pick MKL interface layer
_lib_iface = $(_lib_iface_$(compiler))$(_ilp64)
_lib_iface_intel = mkl_intel
_lib_iface_pgi = mkl_intel
_lib_iface_gnu = mkl_gf

# Pick MKL threading layer
ifeq ($(threading),parallel)
  parallel = $(_parallel_$(compiler))
  omp = $(_omp_$(parallel))

  _parallel_intel = intel
  _parallel_gnu = gnu
  _parallel_pgi = pgi

  _omp_intel = iomp5
  _omp_gnu = iomp5
  _omp_pgi = pgmp

  _lib_thread = mkl_$(parallel)_thread
  threadlayer = $(parallel)
else
  _lib_thread = mkl_sequential
  threadlayer = $(threading)
endif

ifeq ($(compiler)-$(parallel),pgi-pgi)
  override MKL_LIBS_LINK = mixed
endif

# Pick MKL core layer
_lib_core = mkl_core

_threading = $(threading)
# Maybe pick OpenMP library
ifeq ($(threading),parallel)
  ifneq ($(omp),)
    _maybe_omp = -L$(CMPLR_PATH) -l$(omp)
    _threading = $(threading)_$(omp)
  endif
endif

CMPLR_PATH = $(MKLROOT)/../compiler/lib/$(_IA)

# Finally compose the _libs
_libs = $(_maybe_wraplib_libs)
ifeq ($(_LIB),so)
  _libs += -L$(MKLROOT)/lib/$(_IA)
  ifneq ($(MKL_LIBS_LINK),mixed)
    _libs += -lmkl_rt
  else
    _libs += -l$(_lib_iface) -l$(_lib_thread) -l$(_lib_core)
  endif
else
  _libs += -Wl,--start-group
  _libs += $(MKLROOT)/lib/$(_IA)/lib$(_lib_iface).a
  _libs += $(MKLROOT)/lib/$(_IA)/lib$(_lib_thread).a
  _libs += $(MKLROOT)/lib/$(_IA)/lib$(_lib_core).a
  _libs += -Wl,--end-group
endif
_libs += $(_maybe_omp)
_libs += -lm -lpthread -ldl 

ifneq ($(LIB_PATH),)
  RUNLIB_PATH = $(LIB_PATH):$(MKLRUNLIB_PATH):$(CMPLR_PATH)
else
  RUNLIB_PATH = $(MKLRUNLIB_PATH):$(CMPLR_PATH)
endif

ifeq ($(_LIB),so)
  RUNENV = LD_LIBRARY_PATH=$(RUNLIB_PATH):$(LD_LIBRARY_PATH)
  ifneq ($(MKL_LIBS_LINK),mixed)
        RUNENV += MKL_INTERFACE_LAYER=$(interface) MKL_THREADING_LAYER=$(threadlayer)
  endif
else
  RUNENV = LD_LIBRARY_PATH=$(CMPLR_PATH):$(LD_LIBRARY_PATH)
endif

_resdir = $(RES_DIR)/$(compiler)$(_ilp64)_$(_threading)_$(_LIB)$(_IA)$(_maybe_wrap)

##-----------------------------------------------------------------------------
## Rules

vpath %.c $(FFTW3XC)/source

.SUFFIXES:
.SUFFIXES: .c .o .res .out

.PHONY: run mkresdir wraplib

run: wraplib mkresdir $(function:%=$(_resdir)/%.res)

.PRECIOUS: $(_resdir)/%.out

$(_resdir)/%.res: $(_resdir)/%.out FORCE
	-rm -f $@ # remove $@ file before run
	@echo
	# run $*.out
	$(RUNENV) \
	  $< > $@

$(_resdir)/%.out: %.c
	@echo
	# compile $*.c
	$(CC) $(_cflags) $(_cppflags) $^ $(_libs) -o $@

$(_resdir)/%.o: %.c
	$(CC) $(_cflags) $(_cppflags) -c $< -o $@

mkresdir:
	@echo
	@echo See results in $(_resdir)
	@echo
	test -d $(_resdir) || mkdir -p $(_resdir)

wraplib: $(_maybe_wraplib)

ifeq ($(wraplib),yes)
# We shall use absolute path for install_to
_install_to=$(shell mkdir -p $(wraplibdir); cd $(wraplibdir) && pwd)
$(_maybe_wraplib):
	@echo
	# build $@
	$(MAKE) -C $(MKLROOT)/interfaces/fftw3xc lib$(_IA) \
	  INSTALL_DIR=$(_install_to) INSTALL_LIBNAME=$(@F) \
	  $(_maybe_wraplib_makeflags$(_ilp64))
endif

##-----------------------------------------------------------------------------
endif
