Home Research Papers Posters Manuals Talks Download People Help Internal

libs/ck-libs/idxl/idxl.h

Go to the documentation of this file.

/*
IDXL--Index List communication library
C++ include file, for use by libraries.

This is a low-level bare-bones communication library.
The basic primitive is an "Index List", a list of user
array entries to send and receive.  This basic communication
primitive is enough to represent (for example) FEM shared
nodes or an FEM ghost layer; the ghost cells in a stencil-based
CFD computation, etc.

Orion Sky Lawlor, olawlor@acm.org, 1/3/2003
*/

#ifndef __CHARM_IDXL_H
#define __CHARM_IDXL_H

#include "mpi.h"
#include "pup.h"
#include "idxlc.h" /* C interface */
#include "idxl_comm.h" /* basic comm. data structures */
#include "idxl_layout.h" /* user-data description */
#include "tcharmc.h" /* for TCHARM_Get/Set_global */

void IDXL_Abort(const char *callingRoutine,const char *msg,int m0=0,int m1=0,int m2=0);

class IDXL_Comm {
public: //<- Sun CC demands these types be public for use from an inner class
    typedef enum { send_t=17,recv_t,sum_t} op_t;
    
    class sto_t { public:
        const IDXL_Side *idx; 
        const IDXL_Layout *dtype; 
        void *data; 
        op_t op; 
        
        sto_t(const IDXL_Side *idx_,const IDXL_Layout *dtype_,void *data_,op_t op_)
            :idx(idx_), dtype(dtype_), data(data_), op(op_) {}
        sto_t(void) {}
    };
    
    class msg_t { 
        CkVec<char> buf; /* Send/receive buffer */
    public:
        sto_t *sto; /* Indices to send/receive */
        int ll; /* Local processor to communicate with */
        void allocate(int len) {
            buf.resize(len);
        }
        void *getBuf(void) {return &buf[0];}
    };

private:
    CkVec<sto_t> sto;
    CkVec<msg_t *> msg; 
    int nMsgs;
    
    CkVec<MPI_Request> msgReq;
    CkVec<MPI_Status> msgSts;
    
    int tag; MPI_Comm comm;
    bool isPost; //If true, no more adds are allowed
    bool isDone; //If true, ready to reset
public:
    IDXL_Comm(int tag,int context);
    void reset(int tag,int context);
    ~IDXL_Comm();
    
    // prepare to write this field to the message:
    void send(const IDXL_Side *idx,const IDXL_Layout *dtype,const void *src);
    
    // prepare to recv and copy out this field
    void recv(const IDXL_Side *idx,const IDXL_Layout *dtype,void *dest);
    void sum(const IDXL_Side *idx,const IDXL_Layout *dtype,void *srcdest);
    
    // send off our added fields to their destinations & post receives
    void post(void);
    
    // wait until this communication is complete.
    void wait(void);
    
    bool isPosted(void) { return isPost; }
    bool isComplete(void) {return isDone;}
};

// This is our TCharm global ID:
enum {IDXL_globalID=32};

class IDXL_Chunk {
  // MPI Communicator to use by default
  int mpi_comm;
  
  // Lists of index lists: 
  
  CkVec<IDXL *> static_idxls;
  
  CkVec<IDXL *> dynamic_idxls;
  
  // Return the next free index in this table, or -1 if none:
  int storeToFreeIndex(CkVec<IDXL *> &inList,IDXL *store) {
    int i;
    for (i=0;i<inList.size();i++)
        if (inList[i]==NULL) {
            inList[i]=store;
            return i;
        }
    i=inList.size();
    inList.push_back(store);
    return i;
  }
  
  // Lists ongoing communications (FIXME: add list here)
  IDXL_Comm *currentComm; //The ongoing communicator
  
  void init(void);
public:
    IDXL_Chunk(int mpi_comm_);
    IDXL_Chunk(CkMigrateMessage *m);
    void pup(PUP::er &p);
    ~IDXL_Chunk();
    
  static IDXL_Chunk *getNULL(void) {
    return (IDXL_Chunk *)TCHARM_Get_global(IDXL_globalID);
  }
  static IDXL_Chunk *get(const char *callingRoutine);
    
// Manipulate index lists (IDXL's):
  IDXL_t addDynamic(void);
  IDXL_t addStatic(IDXL *idx,IDXL_t at=-1);
  
  void check(IDXL_t u,const char *callingRoutine="") const;
  
  IDXL &lookup(IDXL_t u,const char *callingRoutine="");
  const IDXL &lookup(IDXL_t u,const char *callingRoutine="") const;
  
  void destroy(IDXL_t t,const char *callingRoutine="");
  
// Manipulate user-data format descriptors:
  IDXL_Layout_List layouts;

// Manipulate ongoing communication:
  IDXL_Comm_t addComm(int tag,int context);
  IDXL_Comm *lookupComm(IDXL_Comm_t uc,const char *callingRoutine="");
  void waitComm(IDXL_Comm *comm);
};

#define IDXLAPI(routineName) TCHARM_API_TRACE(routineName,"IDXL");

#endif

---