Home Research Papers Posters Manuals Talks Download People Help Internal

ck-core/cklocation.h

Go to the documentation of this file.

/*
Location manager: keeps track of an indexed set of migratable
objects.  Used by the array manager to locate array elements,
interact with the load balancer, and perform migrations.

Does not handle reductions (see ckreduction.h), broadcasts,
array proxies, or the details of element creation (see ckarray.h).
*/
#ifndef __CKLOCATION_H
#define __CKLOCATION_H

#include <unordered_map>
struct IndexHasher {
  public:
    size_t operator()(const CkArrayIndex& idx) const {
      return std::hash<unsigned int>()(idx.hash());
    }
};

struct ArrayIDHasher {
  public:
    size_t operator()(const CkArrayID& aid) const {
      return std::hash<int>()(((CkGroupID)aid).idx);
    }
};

/*********************** Array Messages ************************/
class CkArrayMessage : public CkMessage {
public:
  //These routines are implementation utilities
  CmiUInt8 array_element_id(void);
  unsigned short &array_ep(void);
  unsigned short &array_ep_bcast(void);
  unsigned char &array_hops(void);
  unsigned int array_getSrcPe(void);
  unsigned int array_ifNotThere(void);
  void array_setIfNotThere(unsigned int);
  
  //This allows us to delete bare CkArrayMessages
  void operator delete(void *p){CkFreeMsg(p);}
};

/* Utility */
//#if CMK_LBDB_ON
#include "LBDatabase.h"
#include "MetaBalancer.h"
class LBDatabase;
//#endif

//Forward declarations
class CkArray;
class ArrayElement;
//What to do if an entry method is invoked on
// an array element that does not (yet) exist:
typedef enum : uint8_t {
    CkArray_IfNotThere_buffer=0, //Wait for it to be created
    CkArray_IfNotThere_createhere=1, //Make it on sending Pe
    CkArray_IfNotThere_createhome=2 //Make it on (a) home Pe
} CkArray_IfNotThere;

typedef enum : uint8_t {
    CkDeliver_queue=0, //Deliver via the scheduler's queue
    CkDeliver_inline=1  //Deliver via a regular call
} CkDeliver_t;
PUPbytes(CkDeliver_t)

class CkArrayOptions;
#include "CkLocation.decl.h"

/************************** Array Messages ****************************/
class CkArrayElementMigrateMessage : public CMessage_CkArrayElementMigrateMessage {
public:
  CkArrayElementMigrateMessage(CkArrayIndex idx_, CmiUInt8 id_, bool ignoreArrival_, int length_,
                               int nManagers_, bool bounced_)
    : idx(idx_), id(id_), ignoreArrival(ignoreArrival_), length(length_), nManagers(nManagers_), bounced(bounced_)
  { }

    CkArrayIndex idx; // Array index that is migrating
        CmiUInt8 id; // ID of the elements with this index in this collection
    bool ignoreArrival;   // if to inform LB of arrival
    int length;//Size in bytes of the packed data
    int nManagers; // Number of associated array managers
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
        CkGroupID gid; //gid of location manager
#endif
    bool bounced; // Fault evac related?
    char* packData;
};

/******************* Map object ******************/

extern CkGroupID _defaultArrayMapID;
extern CkGroupID _fastArrayMapID;
class CkLocMgr;
class CkArray;


#include "ckarrayoptions.h"

class CkArrayMap : public IrrGroup // : public CkGroupReadyCallback
{
public:
  CkArrayMap(void);
  CkArrayMap(CkMigrateMessage *m): IrrGroup(m) {}
  virtual ~CkArrayMap();
  virtual int registerArray(const CkArrayIndex& numElements, CkArrayID aid);
  virtual void unregisterArray(int idx);
  virtual void storeCkArrayOpts(CkArrayOptions options);
  virtual void populateInitial(int arrayHdl,CkArrayOptions& options,void *ctorMsg,CkArray *mgr);
  virtual int procNum(int arrayHdl,const CkArrayIndex &element) =0;
  virtual int homePe(int arrayHdl,const CkArrayIndex &element)
             { return procNum(arrayHdl, element); }

  virtual void pup(PUP::er &p);

  CkArrayOptions storeOpts;
  std::unordered_map<int, bool> dynamicIns;
};

extern int (*ArrayMapProcNumExtCallback)(int, int, const int *);

class ArrayMapExt: public CkArrayMap {
public:
  ArrayMapExt(void *impl_msg);

  static void __ArrayMapExt(void *impl_msg, void *impl_obj_void) {
    new (impl_obj_void) ArrayMapExt(impl_msg);
  }

  static void __entryMethod(void *impl_msg, void *impl_obj_void) {
    //fprintf(stderr, "ArrayMapExt:: entry method invoked\n");
    ArrayMapExt *obj = static_cast<ArrayMapExt *>(impl_obj_void);
    CkMarshallMsg *impl_msg_typed = (CkMarshallMsg *)impl_msg;
    char *impl_buf = impl_msg_typed->msgBuf;
    PUP::fromMem implP(impl_buf);
    int msgSize; implP|msgSize;
    int ep; implP|ep;
    int dcopy_start; implP|dcopy_start;
    GroupMsgRecvExtCallback(obj->thisgroup.idx, ep, msgSize, impl_buf+(3*sizeof(int)),
                            dcopy_start);
  }

  int procNum(int arrayHdl, const CkArrayIndex &element) {
    return ArrayMapProcNumExtCallback(thisgroup.idx, element.getDimension(), element.data());
    //fprintf(stderr, "[%d] ArrayMapExt - procNum is %d\n", CkMyPe(), pe);
  }
};

static inline CkGroupID CkCreatePropMap(void)
{
  return CProxy_PropMap::ckNew();
}

extern void _propMapInit(void);
extern void _CkMigratable_initInfoInit(void);

#include "cklocrec.h"

/*********************** CkMigratable ******************************/
#if CMK_OUT_OF_CORE
#  include "conv-ooc.h"
extern CooPrefetchManager CkArrayElementPrefetcher;
// If this flag is set, this creation/deletion is just 
//   a "fake" constructor/destructor call for prefetching.
CkpvExtern(int,CkSaveRestorePrefetch);
#endif

#include "ckmigratable.h"



/********************** CkLocMgr ********************/
class CkMagicNumber_impl {
 protected:
    int magic;
    void badMagicNumber(int expected,const char *file,int line,void *obj) const;
    CkMagicNumber_impl(int m);
};
template<class T>
class CkMagicNumber : public CkMagicNumber_impl {
    enum {good=sizeof(T)^0x7EDC0000};
 public:
    CkMagicNumber(void) :CkMagicNumber_impl(good) {}
    inline void check(const char *file,int line,void *obj) const {
        if (magic!=good) badMagicNumber(good,file,line,obj);
    }
#if CMK_ERROR_CHECKING
#   define CK_MAGICNUMBER_CHECK magic.check(__FILE__,__LINE__,this);
#else
#   define CK_MAGICNUMBER_CHECK /*empty, for speed*/
#endif
};

class CkLocation {
    CkLocMgr *mgr;
    CkLocRec *rec;
public:
    CkLocation(CkLocMgr *mgr_, CkLocRec *rec_);
    
    inline CkLocMgr *getManager(void) const {return mgr;}
    
    inline CkLocRec *getLocalRecord(void) const {return rec;}
    
    const CkArrayIndex &getIndex(void) const;
    CmiUInt8 getID() const;

        void destroyAll();

    void pup(PUP::er &p);
};

class CkLocIterator {
public:
    virtual ~CkLocIterator();
    
    virtual void addLocation(CkLocation &loc) =0;
};

enum CkElementCreation_t : uint8_t {
  CkElementCreation_migrate=2, // Create object for normal migration arrival
  CkElementCreation_resume=3,  // Create object after checkpoint
  CkElementCreation_restore=4  // Create object after checkpoint, skip listeners
};


#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
typedef void (*CkLocFn)(CkArray *,void *,CkLocRec *,CkArrayIndex *);
#endif

// Returns rank 0 for a pe for drone mode
#if CMK_DRONE_MODE
#define CMK_RANK_0(pe) (CkNodeOf(pe)*CkNodeSize(0))
#else
#define CMK_RANK_0(pe) pe
#endif

class CkLocMgr : public IrrGroup {
    CkMagicNumber<CkMigratable> magic; //To detect heap corruption

public:

typedef std::unordered_map<CkArrayID, CkArray*, ArrayIDHasher> ArrayIdMap;
typedef std::unordered_map<CmiUInt8, int> IdPeMap;
typedef std::unordered_map<CmiUInt8, std::vector<CkArrayMessage*> > MsgBuffer;
typedef std::unordered_map<CkArrayIndex, std::vector<CkArrayMessage *>, IndexHasher> IndexMsgBuffer;
typedef std::unordered_map<CkArrayIndex, std::vector<std::pair<int, bool> >, IndexHasher > LocationRequestBuffer;
typedef std::unordered_map<CkArrayIndex, CmiUInt8, IndexHasher> IdxIdMap;
typedef std::unordered_map<CmiUInt8, CkLocRec*> LocRecHash;

    CkLocMgr(CkArrayOptions opts);
    CkLocMgr(CkMigrateMessage *m);
        ~CkLocMgr();

    inline bool isLocMgr(void) { return true; }
    CkGroupID &getGroupID(void) {return thisgroup;}
    inline CProxy_CkLocMgr &getProxy(void) {return thisProxy;}
    inline CProxyElement_CkLocMgr &getLocalProxy(void) {return thislocalproxy;}

//Interface used by external users:
    inline int homePe(const CkArrayIndex &idx) const {return CMK_RANK_0(map->homePe(mapHandle,idx));}
        inline int homePe(const CmiUInt8 id) const {
          if (compressor)
            return CMK_RANK_0(homePe(compressor->decompress(id)));

          return CMK_RANK_0(id >> 24);
        }
    inline int procNum(const CkArrayIndex &idx) const {return CMK_RANK_0(map->procNum(mapHandle,idx));}
    inline bool isHome (const CkArrayIndex &idx) const {return (bool)(homePe(idx)==CkMyPe());}
  int whichPE(const CkArrayIndex &idx) const;
  int whichPE(const CmiUInt8 id) const;
    int lastKnown(const CkArrayIndex &idx);
    int lastKnown(CmiUInt8 id);

        inline void insertID(const CkArrayIndex& idx, const CmiUInt8 id) {
          if (compressor) return;
          idx2id[idx] = id;
        }

        inline CmiUInt8 lookupID(const CkArrayIndex &idx) const {
          if (compressor) {
            return compressor->compress(idx);
          } else {
            CkLocMgr::IdxIdMap::const_iterator itr = idx2id.find(idx);
            CkAssert(itr != idx2id.end());
            return itr->second;
          }
        }

        inline bool lookupID(const CkArrayIndex &idx, CmiUInt8& id) const {
          if (compressor) {
            id = compressor->compress(idx);
            return true;
          } else {
            CkLocMgr::IdxIdMap::const_iterator itr = idx2id.find(idx);
            if (itr == idx2id.end()) {
              return false;
            } else {
              id = itr->second;
              return true;
            }
          }
        }

        // Lookup CkArrayIndex for a CmiUInt8, used by BlockLB and OrbLB
        inline CkArrayIndex lookupIdx(const CmiUInt8 &id) const {
         if (compressor) {
           return compressor->decompress(id);
          } else {
           CkLocMgr::IdxIdMap::const_iterator itr;
            for ( itr = idx2id.begin(); itr != idx2id.end(); itr++ ) {
              if(itr->second == id)
                break;
            }
            CkAssert(itr != idx2id.end());
            return itr->first;
          }
        }

    //Look up array element in hash table.  Index out-of-bounds if not found.
    CkLocRec *elementRec(const CkArrayIndex &idx);
    //Look up array element in hash table.  Return NULL if not there.
    CkLocRec *elementNrec(const CmiUInt8 id);

    bool isRemote(const CkArrayIndex &idx,int *onPe) const;

#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
    //mark the duringMigration variable .. used for parallel restart
    void setDuringMigration(bool _duringMigration);
#endif

    void setDuringDestruction(bool _duringDestruction);

    void iterate(CkLocIterator &dest);

    void restore(const CkArrayIndex &idx, CmiUInt8 id, PUP::er &p);
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
    void resume(const CkArrayIndex &idx, CmiUInt8 id, PUP::er &p, bool create, int dummy=0);
#else
    void resume(const CkArrayIndex &idx, CmiUInt8 id, PUP::er &p, bool notify=true,bool=false);
#endif

//Interface used by array manager and proxies
    void addManager(CkArrayID aid,CkArray *mgr);
        void deleteManager(CkArrayID aid, CkArray *mgr);

    void populateInitial(CkArrayOptions& options,void *initMsg,CkArray *mgr)
    {
      map->storeCkArrayOpts(options);
      map->populateInitial(mapHandle,options,initMsg,mgr);
    }

    bool addElement(CkArrayID aid,const CkArrayIndex &idx, CkMigratable *elt,int ctorIdx,void *ctorMsg);

    void doneInserting(void);
    void startInserting(void);

    // How many elements of each associated array are local to this PE?
    // If this returns n, and there are k associated arrays, that
    // means k*n elements are living here
    unsigned int numLocalElements();

    //int deliverMsg(CkMessage *m, CkArrayID mgr, const CkArrayIndex &idx, CkDeliver_t type, int opts = 0);
    int deliverMsg(CkArrayMessage *m, CkArrayID mgr, CmiUInt8 id, const CkArrayIndex* idx, CkDeliver_t type, int opts = 0);

        void sendMsg(CkArrayMessage *msg, CkArrayID mgr, const CkArrayIndex &idx, CkDeliver_t type, int opts);

//Advisories:
    void inform(const CkArrayIndex &idx, CmiUInt8 id, int nowOnPe);
    void inform(CmiUInt8 id, int nowOnPe);

    void informHome(const CkArrayIndex &idx,int nowOnPe);

    void multiHop(CkArrayMessage *m);

//Interface used by CkLocRec

    //Migrate us to another processor
    void emigrate(CkLocRec *rec,int toPe);
    void informLBPeriod(CkLocRec *rec, int lb_ideal_period);
    void metaLBCallLB(CkLocRec *rec);

#if CMK_LBDB_ON
    LBDatabase *getLBDB(void) const { return the_lbdb; }
    MetaBalancer *getMetaBalancer(void) const { return the_metalb;}
    const LDOMHandle &getOMHandle(void) const { return myLBHandle; }
#endif

    //This index will no longer be used-- delete the associated elements
    void reclaim(CkLocRec* rec);

    bool demandCreateElement(CkArrayMessage *msg, const CkArrayIndex &idx, int onPe, CkDeliver_t type);
        void demandCreateElement(const CkArrayIndex &idx, int chareType, int onPe, CkArrayID mgr);

//Communication:
    void immigrate(CkArrayElementMigrateMessage *msg);
        void requestLocation(const CkArrayIndex &idx, int peToTell, bool suppressIfHere, int ifNonExistent, int chareType, CkArrayID mgr);
        void requestLocation(CmiUInt8 id, int peToTell, bool suppressIfHere);
        void updateLocation(const CkArrayIndex &idx, CmiUInt8 id, int nowOnPe);
        void updateLocation(CmiUInt8 id, int nowOnPe);
    void reclaimRemote(const CkArrayIndex &idx,int deletedOnPe);
    void dummyAtSync(void);

    void migratableList(CkLocRec *rec, std::vector<CkMigratable *> &list);

    void flushAllRecs(void);
    void flushLocalRecs(void);
    void pup(PUP::er &p);
    

private:
//Internal interface:
    //Add given element array record at idx, replacing the existing record
    void insertRec(CkLocRec *rec,const CmiUInt8 &id);

    //Remove this entry from the table (does not delete record)
    void removeFromTable(const CmiUInt8 id);

    friend class CkLocation; //so it can call pupElementsFor
    friend class ArrayElement;
    friend class MemElementPacker;
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
    void pupElementsFor(PUP::er &p,CkLocRec *rec,
        CkElementCreation_t type, bool create=true, int dummy=0);
#else
    void pupElementsFor(PUP::er &p,CkLocRec *rec,
        CkElementCreation_t type,bool rebuild = false);
#endif

    typedef void (CkMigratable::* CkMigratable_voidfn_t)(void);

    typedef void (CkMigratable::* CkMigratable_voidfn_arg_t)(void*);
    void callMethod(CkLocRec *rec,CkMigratable_voidfn_arg_t fn, void*);

    void deliverUnknown(CkArrayMessage *msg, const CkArrayIndex* idx, CkDeliver_t type, int opts);
    void deliverAnyBufferedMsgs(CmiUInt8, MsgBuffer &buffer);

#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
CkLocRec *createLocal(const CkArrayIndex &idx,
        bool forMigration, bool ignoreArrival,
        bool notifyHome,int dummy=0);
#else
    CkLocRec *createLocal(const CkArrayIndex &idx, 
        bool forMigration, bool ignoreArrival,
        bool notifyHome);
#endif

    LocationRequestBuffer bufferedLocationRequests;

public:
    void callMethod(CkLocRec *rec,CkMigratable_voidfn_t fn);

//Data Members:
    //Map array ID to manager and elements
    ArrayIdMap managers;
    // Map object ID to location
    IdPeMap id2pe;

    // Map array element index to object ID
    IdxIdMap idx2id;
    // Next ID to assign newly constructed array elements
    CmiUInt8 idCounter;
    CmiUInt8 getNewObjectID(const CkArrayIndex &idx);

    MsgBuffer bufferedMsgs;
    MsgBuffer bufferedRemoteMsgs;
    MsgBuffer bufferedShadowElemMsgs;

    IndexMsgBuffer bufferedIndexMsgs;

    bool addElementToRec(CkLocRec *rec,CkArray *m,
        CkMigratable *elt,int ctorIdx,void *ctorMsg);

    CProxy_CkLocMgr thisProxy;
    CProxyElement_CkLocMgr thislocalproxy;

    bool duringMigration;
    bool duringDestruction;

private:
    LocRecHash hash;
    CmiImmediateLockType hashImmLock;

    //Map object
    CkGroupID mapID;
    int mapHandle;
    CkArrayMap *map;

    CkGroupID lbdbID;
    CkGroupID metalbID;

    ck::ArrayIndexCompressor *compressor;
    CkArrayIndex bounds;
    void checkInBounds(const CkArrayIndex &idx);

#if CMK_LBDB_ON
    LBDatabase *the_lbdb;
  MetaBalancer *the_metalb;
    LDBarrierClient dummyBarrierHandle;
    static void staticDummyResumeFromSync(void* data);
    void dummyResumeFromSync(void);
    static void staticRecvAtSync(void* data);
    void recvAtSync(void);
    LDOMHandle myLBHandle;
        LDBarrierReceiver lbBarrierReceiver;
#endif
private:
    void initLB(CkGroupID lbdbID, CkGroupID metalbID);

#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
public:
    void callForAllRecords(CkLocFn,CkArray *,void *);
    int homeElementCount;
#endif

};





bool haveConfigurableRRMap();




#endif /*def(thisHeader)*/

---