Source code for cplpy

from __future__ import print_function, division
from ctypes import c_int, c_double, c_bool, c_void_p, byref, POINTER, util
import ctypes
from mpi4py import MPI
import numpy as np
from numpy.ctypeslib import ndpointer, load_library
from functools import wraps
import os
import time
from subprocess import STDOUT, check_output, CalledProcessError
import shutil
import cPickle


__all__ = ["CPL", "cart_create", "run_test", "prepare_config"]


class OpenMPI_Not_Supported(Exception):
    pass

# TODO: Raise exception of library not loaded
_loaded = False

# All Python types except integers, strings, and unicode strings have to be
# wrapped in their corresponding ctypes type, so that they can be converted
# to the required C

_CPL_GET_VARS = {"icmin_olap": c_int, "jcmin_olap": c_int, "kcmin_olap": c_int,
                 "icmax_olap": c_int, "jcmax_olap": c_int, "kcmax_olap": c_int,
                 "icmin_cnst": c_int, "jcmin_cnst": c_int, "kcmin_cnst": c_int,
                 "icmax_cnst": c_int, "jcmax_cnst": c_int, "kcmax_cnst": c_int,
                 "ncx": c_int, "ncy": c_int, "ncz": c_int,
                 "npx_md": c_int, "npy_md": c_int, "npz_md": c_int,
                 "npx_cfd": c_int, "npy_cfd": c_int, "npz_cfd": c_int,
                 "overlap": c_int, "xl_md": c_double, "yl_md": c_double,
                 "zl_md": c_double, "xl_cfd": c_double, "yl_cfd": c_double,
                 "zl_cfd": c_double}

_CPL_SET_VARS = {"output_mode": c_int}

# Decorator to abort all processes if an exception is thrown. This
# avoids getting blocked when the exception do not occurs in every
# process.
def abortMPI(func):
    @wraps(func)
    def handleExcepts(self, *args, **kwargs):
        retval = None
        try:
            retval = func(self, *args, **kwargs)
        except Exception as e:
            print (e)
            # Dirty workaround to let the output be printed.
            time.sleep(2)
            MPI.COMM_WORLD.Abort(errorcode=1)
        else:
            return retval
    return handleExcepts


[docs]class CPL: # Shared attribute containing the library CFD_REALM = 1 MD_REALM = 2 GATHER_SCATTER = 1 SEND_RECEIVE = 2 NULL_REALM = 0 _libname = "libcpl" _lib_path = os.environ.get("CPL_LIBRARY_PATH") # Try using CPL_LIBRARY_PATH and if not look # in system path (or ctype path) try: _cpl_lib = load_library(_libname, _lib_path) except OSError as e: print("OSError: ", e) print("WARNING - " + _libname + " not found under path specified by \ CPL_LIBRARY_PATH variable: " + _lib_path) print("Attempting to find system version") _lib_path = util.find_library(_libname) # If not found, try to look around the current directory system # for the library else: try: _cpl_lib = load_library(_libname, _lib_path) except OSError: print("WARNING - " + _libname + " not found in system path or \ $CPL_LIBRARY_PATH: " + _lib_path) print("Attempting to search current folder system") trydir = '' found = False for level in range(10): if not found: trydir += "../" for root, dirs, files in os.walk(trydir): print(level, trydir, root) if "libcpl.so" in files: print("A version of " + _libname + " is found under \ path: " + root + ". Attempting to use this..") found = True _lib_path = root break if found: _cpl_lib = load_library(_libname, _lib_path) else: print(_libname + " not found, please ensure you have \ compiled correctly") print(" and set CPL_LIBRARY_PATH appropitely") time.sleep(2) MPI.COMM_WORLD.Abort(errorcode=1) def __init__(self): pass # py_test_python function py_test_python = _cpl_lib.CPLC_test_python py_test_python.argtypes = \ [c_int, c_double, c_bool, ndpointer(np.int32, ndim=2, flags='aligned, f_contiguous'), ndpointer(np.float64, ndim=2, flags='aligned, f_contiguous'), ndpointer(np.int32, shape=(2,), flags='aligned, f_contiguous'), ndpointer(np.int32, shape=(2,), flags='aligned, f_contiguous')] @abortMPI def test_python(self, int_p, doub_p, bool_p, int_pptr, doub_pptr): int_pptr_dims = np.array(int_pptr.shape, order='F', dtype=np.int32) doub_pptr_dims = np.array(doub_pptr.shape, order='F', dtype=np.int32) self.py_test_python(int_p, doub_p, bool_p, int_pptr, doub_pptr, int_pptr_dims, doub_pptr_dims) _py_init = _cpl_lib.CPLC_init #OpenMPI comm greater than c_int if MPI._sizeof(MPI.Comm) == ctypes.sizeof(c_int): _py_init.argtypes = [c_int, POINTER(c_int)] else: excptstr ="Problem is in create_comm wrapper, as the OpenMPI COMM handle is not " excptstr += "an integer, c_void_p should be used so C bindings needs something like **void" excptstr += "(No idea what to do in the Fortran code, maybe MPI_COMM_f2C required)" raise OpenMPI_Not_Supported(excptstr) _py_init.argtypes = [c_int, POINTER(c_void_p)] _py_init.argtypes = [c_int, POINTER(c_int)] @abortMPI def init(self, calling_realm): # Build a communicator mpi4py python object from the # handle returned by the CPL_init function. if MPI._sizeof(MPI.Comm) == ctypes.sizeof(c_int): MPI_Comm = c_int else: MPI_Comm = c_void_p # Call create comm returned_realm_comm = c_int() self._py_init(calling_realm, byref(returned_realm_comm)) # Use an intracomm object as the template and override value newcomm = MPI.Intracomm() newcomm_ptr = MPI._addressof(newcomm) comm_val = MPI_Comm.from_address(newcomm_ptr) comm_val.value = returned_realm_comm.value return newcomm py_setup_cfd = _cpl_lib.CPLC_setup_cfd py_setup_cfd.argtypes = \ [c_int, c_double, c_int, ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous'), ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous'), ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous'), c_double] @abortMPI
[docs] def setup_cfd(self, nsteps, dt, icomm_grid, xyzL, xyz_orig, ncxyz, density): """ Keyword arguments: real -- the real part (default 0.0) imag -- the imaginary part (default 0.0) """ self.py_setup_cfd(nsteps, dt, MPI._handleof(icomm_grid), xyzL, xyz_orig, ncxyz, density)
py_setup_md = _cpl_lib.CPLC_setup_md py_setup_md.argtypes = \ [POINTER(c_int), POINTER(c_int), c_double, c_int, ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous'), ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous'), c_double] @abortMPI
[docs] def setup_md(self, dt, icom_grid, xyzL, xyz_orig, density=1.0): """ setup_md(self, dt, icom_grid, xyzL, xyz_orig, density=1.0) Keyword arguments: real -- the real part (default 0.0) imag -- the imaginary part (default 0.0) """ nsteps = c_int() initialstep = c_int() self.py_setup_md(byref(nsteps), byref(initialstep), dt, MPI._handleof(icom_grid), xyzL, xyz_orig, density) return (nsteps.value, initialstep.value)
py_gather = _cpl_lib.CPLC_gather py_gather.argtypes = \ [ndpointer(np.float64, flags='aligned, f_contiguous'), ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous'), ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous'), ndpointer(np.float64, flags='aligned, f_contiguous'), ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous')] @abortMPI def gather(self, gather_array, limits, recv_array): gather_array = self._type_check(gather_array) recv_array = self._type_check(recv_array) gather_shape = np.array(gather_array.shape, order='F', dtype=np.int32) recv_shape = np.array(recv_array.shape, order='F', dtype=np.int32) self.py_gather(gather_array, gather_shape, limits, recv_array, recv_shape) return recv_array py_scatter = _cpl_lib.CPLC_scatter py_scatter.argtypes = \ [ndpointer(np.float64, flags='aligned, f_contiguous'), ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous'), ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous'), ndpointer(np.float64, flags='aligned, f_contiguous'), ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous')] @abortMPI def scatter(self, scatter_array, limits, recv_array): scatter_array = self._type_check(scatter_array) recv_array = self._type_check(recv_array) scatter_shape = np.array(scatter_array.shape, order='F', dtype=np.int32) recv_shape = np.array(recv_array.shape, order='F', dtype=np.int32) self.py_scatter(scatter_array, scatter_shape, limits, recv_array, recv_shape) return recv_array py_proc_extents = _cpl_lib.CPLC_proc_extents py_proc_extents.argtypes = \ [ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous'), c_int, ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')] @abortMPI def proc_extents(self, coord, realm): coord = self._type_check(coord) extents = np.zeros(6, order='F', dtype=np.int32) self.py_proc_extents(coord, realm, extents) return extents py_my_proc_extents = _cpl_lib.CPLC_my_proc_extents py_my_proc_extents.argtypes = \ [ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')] @abortMPI def my_proc_extents(self): extents = np.zeros(6, order='F', dtype=np.int32) self.py_my_proc_extents(extents) return extents py_proc_portion = _cpl_lib.CPLC_proc_portion py_proc_portion.argtypes = \ [ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous'), c_int, ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous'), ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')] @abortMPI def proc_portion(self, coord, realm, limits): coord = self._type_check(coord) limits = self._type_check(limits) portion = np.zeros(6, order='F', dtype=np.int32) self.py_proc_portion(coord, realm, limits, portion) return portion py_my_proc_portion = _cpl_lib.CPLC_my_proc_portion py_my_proc_portion.argtypes = \ [ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous'), ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')] @abortMPI def my_proc_portion(self, limits): limits = self._type_check(limits) portion = np.zeros(6, order='F', dtype=np.int32) self.py_my_proc_portion(limits, portion) return portion py_map_cfd2md_coord = _cpl_lib.CPLC_map_cfd2md_coord py_map_cfd2md_coord.argtypes = \ [ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous'), ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous')] @abortMPI def map_cfd2md_coord(self, coord_cfd): coord_cfd = self._type_check(coord_cfd) coord_md = np.zeros(3, order='F', dtype=np.float64) self.py_map_cfd2md_coord(coord_cfd, coord_md) return coord_md py_map_md2cfd_coord = _cpl_lib.CPLC_map_md2cfd_coord py_map_md2cfd_coord.argtypes = \ [ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous'), ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous')] @abortMPI def map_md2cfd_coord(self, coord_md): coord_md = self._type_check(coord_md) coord_cfd = np.zeros(3, order='F', dtype=np.float64) self.py_map_md2cfd_coord(coord_md, coord_cfd) return coord_cfd py_map_glob2loc_cell = _cpl_lib.CPLC_map_glob2loc_cell py_map_glob2loc_cell.argtypes = \ [ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous'), ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous'), ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous')] @abortMPI def map_glob2loc_cell(self, limits, glob_cell): limits = self._type_check(limits) glob_cell = self._type_check(glob_cell) loc_cell = np.zeros(3, order='F', dtype=np.int32) self.py_map_glob2loc_cell(limits, glob_cell, loc_cell) return loc_cell py_map_cell2coord = _cpl_lib.CPLC_map_cell2coord py_map_cell2coord.argtypes = \ [c_int, c_int, c_int, ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous')] @abortMPI def map_cell2coord(self, i, j, k): coord = np.zeros(3, order='F', dtype=np.float64) self.py_map_cell2coord(i, j, k, coord) return coord py_map_coord2cell = _cpl_lib.CPLC_map_coord2cell py_map_coord2cell.argtypes = \ [c_double, c_double, c_double, ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous')] @abortMPI def map_coord2cell(self, x, y, z): cell = np.zeros(3, order='F', dtype=np.int32) self.py_map_coord2cell(x, y, z, cell) return cell py_get_no_cells = _cpl_lib.CPLC_get_no_cells py_get_no_cells.argtypes = \ [ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous'), ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous')] @abortMPI def get_no_cells(self, limits): limits = self._type_check(limits) no_cells = np.zeros(3, order='F', dtype=np.int32) self.py_get_no_cells(limits, no_cells) return no_cells py_get_olap_limits = _cpl_lib.CPLC_get_olap_limits py_get_olap_limits.argtypes = \ [ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')] @abortMPI def get_olap_limits(self): limits = np.zeros(6, order='F', dtype=np.int32) self.py_get_olap_limits(limits) return limits py_get_cnst_limits = _cpl_lib.CPLC_get_cnst_limits py_get_cnst_limits.argtypes = \ [ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')] @abortMPI def get_cnst_limits(self): limits = np.zeros(6, order='F', dtype=np.int32) self.py_get_cnst_limits(limits) return limits py_send = _cpl_lib.CPLC_send py_send.argtypes = \ [ndpointer(np.float64, flags='aligned, f_contiguous'), ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous'), ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous'), POINTER(c_bool)] @abortMPI def send(self, asend, limits): asend = self._type_check(asend) asend_shape = np.array(asend.shape, order='F', dtype=np.int32) send_flag = c_bool() self.py_send(asend, asend_shape, limits, byref(send_flag)) return send_flag.value py_recv = _cpl_lib.CPLC_recv py_recv.argtypes = \ [ndpointer(np.float64, flags='aligned, f_contiguous'), ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous'), ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous'), POINTER(c_bool)] @abortMPI def recv(self, arecv, limits): arecv = self._type_check(arecv) arecv_shape = np.array(arecv.shape, order='F', dtype=np.int32) recv_flag = c_bool() self.py_recv(arecv, arecv_shape, limits, byref(recv_flag)) return arecv, recv_flag.value py_overlap = _cpl_lib.CPLC_overlap py_overlap.argtypes = [] @abortMPI def overlap(self): self.py_overlap.restype = c_bool return self.py_overlap() @abortMPI def get(self, var_name): try: var_type = _CPL_GET_VARS[var_name] fun = getattr(self._cpl_lib, "CPLC_" + var_name) except KeyError: print ("CPL-ERROR: CPL Library function '" + str(var_name) + "' not found!") raise KeyError else: fun.restype = var_type return fun() @abortMPI def set(self, var_name, value): try: var_type = _CPL_SET_VARS[var_name] fun = getattr(self._cpl_lib, "CPLC_set_" + var_name) except KeyError: print ("CPL-ERROR: CPL Library function '" + str(var_name) + "' not found!") raise KeyError else: fun.argtypes = [var_type] return fun(var_type(value)) @abortMPI def _type_check(self, A): if type(A) is list: ndtype = type(A[0]) if ndtype == float: ndtype = np.float64 elif ndtype == int: ndtype = np.int32 A = np.asfortranarray(A, dtype=ndtype) if not A.flags["F_CONTIGUOUS"]: A = np.require(A, requirements=['F']) if not A.flags["ALIGNED"]: A = np.require(A, requirements=['A']) return A
def cart_create(old_comm, dims, periods, coords): dummy_cart_comm = old_comm.Create_cart(dims, periods) temp_comm = old_comm.Split(0, dummy_cart_comm.Get_rank()) new_cart_comm = temp_comm.Create_cart(dims, periods) comm_coords = new_cart_comm.Get_coords(new_cart_comm.Get_rank()) if (not (coords == comm_coords).all()): print ("Not good!") exit() return new_cart_comm # -----------------------------TESTING ROUTINES------------------------------ # CONFIG_FILE = "COUPLER.in" TEST_DIR = os.path.dirname(os.path.realpath(__file__)) TEST_NAME = os.path.basename(os.path.realpath(__file__)) def parametrizeConfig(template_dir, params): # It assumes is in the temp directory with cpl/ folder accessible # from this level. with open(os.path.join(template_dir, CONFIG_FILE), "r+") as config_file: lines = config_file.readlines() for (k, v) in params.items(): lines = [l.replace("$[" + str(k) + "]", str(v)) for l in lines] with open(os.path.join("cpl/", CONFIG_FILE), "w") as config_file: config_file.writelines(lines) def prepare_config(tmpdir, test_dir, md_fname, cfd_fname): tmpdir.mkdir("cpl") shutil.copy(os.path.join(test_dir, md_fname), tmpdir.strpath) shutil.copy(os.path.join(test_dir, cfd_fname), tmpdir.strpath) os.chdir(tmpdir.strpath) def run_test(template_dir, config_params, md_exec, md_fname, cfd_exec, cfd_fname, md_params, cfd_params, err_msg, debug=False): parametrizeConfig(template_dir, config_params) cPickle.dump(md_params, open("md_params.dic", "wb")) cPickle.dump(cfd_params, open("cfd_params.dic", "wb")) try: mdprocs = md_params["npx"] * md_params["npy"] * md_params["npz"] cfdprocs = cfd_params["npx"] * cfd_params["npy"] * cfd_params["npz"] if os.path.isfile(md_fname) and os.path.isfile(cfd_fname): cmd = " ".join(["mpiexec", "-n", str(mdprocs), md_exec, md_fname, ":", "-n", str(cfdprocs), cfd_exec, cfd_fname]) if debug: print ("\nMPI run: " + cmd) check_output(cmd, stderr=STDOUT, shell=True) else: print (md_fname + " or " + cfd_fname + " are not found.") assert False return False except CalledProcessError as exc: print (exc.output) if err_msg != "": assert err_msg in exc.output else: assert exc.output == "" else: if err_msg != "": assert False else: assert True return True if __name__ == "__main__": lib = CPL()