Charm++: Charm Source Code Documentation

00001 /* -*- Mode: C; c-basic-offset:4 ; -*- */
00002 /* 
00003  *   Copyright (C) 2008 University of Chicago. 
00004  *   See COPYRIGHT notice in top-level directory.
00005  */
00006 
00007 #include "adio.h"
00008 #include "adio_extern.h"
00009 #ifdef AGGREGATION_PROFILE
00010 #include "mpe.h"
00011 #endif
00012 #include <assert.h>
00013 /* #define DEBUG */
00014 
00015 void ADIOI_Calc_file_realms_user_size (ADIO_File fd, int fr_size,
00016                        int nprocs_for_coll,
00017                        ADIO_Offset *file_realm_st_offs,
00018                        MPI_Datatype *file_realm_types);
00019 void ADIOI_Calc_file_realms_aar (ADIO_File fd, int nprocs_for_coll,
00020                  int pfr_enabled,
00021                  ADIO_Offset min_st_offset,
00022                  ADIO_Offset max_end_offset,
00023                  ADIO_Offset *file_realm_st_offs,
00024                  MPI_Datatype *file_realm_types);
00025 void ADIOI_Calc_file_realms_fsize (ADIO_File fd,
00026                    int nprocs_for_coll,
00027                    ADIO_Offset max_end_offset,
00028                    ADIO_Offset *file_realm_st_offs,
00029                    MPI_Datatype *file_realm_types);
00030 void ADIOI_Create_fr_simpletype (int size, int nprocs_for_coll,
00031                   MPI_Datatype *simpletype);
00032 static void align_fr (int fr_size, ADIO_Offset fr_off, int alignment,
00033            int *aligned_fr_size, ADIO_Offset *aligned_fr_off);
00034 void ADIOI_Verify_fr (int nprocs_for_coll, ADIO_Offset *file_realm_st_offs,
00035               MPI_Datatype *file_realm_types);
00036 
00037 void ADIOI_Calc_file_realms (ADIO_File fd, ADIO_Offset min_st_offset,
00038                  ADIO_Offset max_end_offset)
00039 {
00040     int nprocs_for_coll;
00041     int file_realm_calc_type;
00042     
00043     MPI_Datatype *file_realm_types = NULL;
00044     ADIO_Offset *file_realm_st_offs = NULL;
00045 
00046 #ifdef AGGREGATION_PROFILE
00047     MPE_Log_event (5004, 0, NULL);
00048 #endif
00049 #ifdef DEBUG
00050     printf ("ADIOI_Calc_file_realms\n");
00051 #endif
00052     
00053     nprocs_for_coll = fd->hints->cb_nodes;    
00054     file_realm_calc_type = fd->hints->cb_fr_type;
00055 
00056     /* If PFRs are disabled we know these pointers are not allocated */
00057     if (fd->hints->cb_pfr != ADIOI_HINT_ENABLE) {
00058     fd->file_realm_st_offs = NULL;
00059     fd->file_realm_types = NULL;
00060     }
00061 
00062     if (nprocs_for_coll == 1) {
00063     /* if there's only one aggregator, we can reset the file
00064      * realms every single time */
00065     if (fd->file_realm_st_offs == NULL)
00066     {
00067         file_realm_st_offs = (ADIO_Offset *)
00068         ADIOI_Malloc (sizeof(ADIO_Offset));
00069         file_realm_types   = (MPI_Datatype *)
00070         ADIOI_Malloc (sizeof(MPI_Datatype));
00071     }
00072     else
00073     {
00074         file_realm_st_offs = fd->file_realm_st_offs;
00075         file_realm_types   = fd->file_realm_types;
00076     }
00077     *file_realm_st_offs = min_st_offset;
00078     MPI_Type_contiguous ((max_end_offset - min_st_offset + 1), MPI_BYTE,
00079                  file_realm_types);
00080     MPI_Type_commit (file_realm_types);
00081     ADIOI_Add_contig_flattened (*file_realm_types);
00082     }
00083     else if (fd->file_realm_st_offs == NULL) {
00084     file_realm_st_offs = (ADIO_Offset *)
00085         ADIOI_Malloc (nprocs_for_coll * sizeof(ADIO_Offset));
00086     file_realm_types   = (MPI_Datatype *)
00087         ADIOI_Malloc (nprocs_for_coll * sizeof(MPI_Datatype));
00088     
00089     if (file_realm_calc_type == ADIOI_FR_AAR) {
00090         ADIOI_Calc_file_realms_aar (fd, nprocs_for_coll,
00091                     fd->hints->cb_pfr,
00092                     min_st_offset, max_end_offset,
00093                     file_realm_st_offs, file_realm_types);
00094         /* flatten file realm datatype for future use - only one
00095          * because all are the same*/
00096         ADIOI_Flatten_datatype (file_realm_types[0]);
00097     }
00098     else if (file_realm_calc_type == ADIOI_FR_FSZ) {
00099         ADIOI_Calc_file_realms_fsize (fd, nprocs_for_coll, max_end_offset,
00100                       file_realm_st_offs,
00101                       file_realm_types);
00102         /* flatten file realm datatype for future use - only one
00103          * because all are the same*/
00104         ADIOI_Flatten_datatype (file_realm_types[0]);
00105     }
00106     else if (file_realm_calc_type == ADIOI_FR_USR_REALMS) {
00107         /* copy user provided realm datatypes and realm offsets in
00108          * hints to file descriptor. may also want to verify that
00109          * the provided file realms are covering (for pfr at
00110          * least) and non-overlapping */
00111     }
00112     else if (file_realm_calc_type > 0) {
00113         ADIOI_Calc_file_realms_user_size (fd, file_realm_calc_type,
00114                           nprocs_for_coll,
00115                           file_realm_st_offs,
00116                           file_realm_types);
00117         /* flatten file realm datatype for future use - only one
00118          * because all are the same */
00119         ADIOI_Flatten_datatype (file_realm_types[0]);
00120     }
00121     }
00122     fd->file_realm_st_offs = file_realm_st_offs;
00123     fd->file_realm_types   = file_realm_types;
00124 #ifdef AGGREGATION_PROFILE
00125     MPE_Log_event (5005, 0, NULL);
00126 #endif
00127 }
00128 
00129 void ADIOI_Calc_file_realms_user_size (ADIO_File fd, int fr_size,
00130                        int nprocs_for_coll,
00131                        ADIO_Offset *file_realm_st_offs,
00132                        MPI_Datatype *file_realm_types)
00133 {
00134     int i;
00135     int aligned_fr_size;
00136     ADIO_Offset aligned_fr_off;
00137     MPI_Datatype simpletype;
00138 
00139     align_fr(fr_size, 0, fd->hints->cb_fr_alignment, &aligned_fr_size,
00140          &aligned_fr_off);
00141     fr_size = aligned_fr_size;
00142     ADIOI_Create_fr_simpletype (fr_size, nprocs_for_coll, &simpletype);
00143 
00144     if (fd->hints->cb_pfr == ADIOI_HINT_ENABLE)
00145     file_realm_st_offs[0] = 0;
00146     else
00147     file_realm_st_offs[0] = aligned_fr_off;
00148     file_realm_types[0]   = simpletype;
00149 #ifdef DEBUG
00150     printf ("file_realm[0] = (%lld, %d)\n", file_realm_st_offs[0],
00151         fr_size);
00152 #endif
00153 
00154     for (i=1; i < nprocs_for_coll; i++)
00155     {
00156     file_realm_st_offs[i] = file_realm_st_offs[i-1] + fr_size;
00157     file_realm_types[i]   = simpletype;
00158 #ifdef DEBUG
00159     printf ("file_realm[%d] = (%lld, %d)\n", i, file_realm_st_offs[i],
00160         aligned_fr_size);
00161 #endif
00162     }
00163 }
00164 
00165 /* takes an extra romio_cb_pfr param to decide whether file realms
00166  * should start at byte 0 of the file*/
00167 void ADIOI_Calc_file_realms_aar (ADIO_File fd, int nprocs_for_coll, int cb_pfr,
00168                  ADIO_Offset min_st_offset,
00169                  ADIO_Offset max_end_offset,
00170                  ADIO_Offset *file_realm_st_offs,
00171                  MPI_Datatype *file_realm_types)
00172 {
00173     int fr_size, aligned_fr_size, i;
00174     MPI_Datatype simpletype;
00175     ADIO_Offset aligned_start_off;
00176     char value[9];
00177 
00178     fr_size = (max_end_offset - min_st_offset + nprocs_for_coll) /
00179     nprocs_for_coll;
00180     align_fr(fr_size, min_st_offset, fd->hints->cb_fr_alignment,
00181          &aligned_fr_size, &aligned_start_off);
00182     fr_size = aligned_fr_size;
00183     ADIOI_Create_fr_simpletype (fr_size, nprocs_for_coll, &simpletype);
00184     if (cb_pfr == ADIOI_HINT_ENABLE)
00185     file_realm_st_offs[0] = 0;
00186     else
00187     file_realm_st_offs[0] = aligned_start_off;
00188     file_realm_types[0]   = simpletype;
00189 
00190 #ifdef DEBUG
00191     printf ("file_realm[0] = (%lld, %d)\n", file_realm_st_offs[0],
00192         fr_size);
00193 #endif
00194     for (i=1; i < nprocs_for_coll; i++)
00195     {
00196     file_realm_st_offs[i] = file_realm_st_offs[i-1] + fr_size;
00197     file_realm_types[i]   = simpletype;
00198 #ifdef DEBUG
00199     printf ("file_realm[%d] = (%lld, %d)\n", i, file_realm_st_offs[i],
00200         fr_size);
00201 #endif
00202     }
00203     if (fd->hints->cb_pfr == ADIOI_HINT_ENABLE) {
00204     sprintf (value, "%d", fr_size);
00205     ADIOI_Info_set (fd->info, "romio_cb_fr_type", value);
00206     }
00207 }
00208 
00209 void ADIOI_Calc_file_realms_fsize (ADIO_File fd, int nprocs_for_coll,
00210                    ADIO_Offset max_end_offset,
00211                    ADIO_Offset *file_realm_st_offs,
00212                    MPI_Datatype *file_realm_types)
00213 {
00214     int fr_size, aligned_fr_size, error_code, i;
00215     int fsize;
00216     ADIO_Offset aligned_fr_off;
00217     ADIO_Fcntl_t fcntl_struct;
00218     MPI_Datatype simpletype;
00219 
00220     ADIO_Fcntl (fd, ADIO_FCNTL_GET_FSIZE, &fcntl_struct, &error_code);
00221     
00222     /* use impending file size since a write call may lengthen the file */
00223     fsize = ADIOI_MAX (fcntl_struct.fsize, max_end_offset+1);
00224     fr_size = (fsize + nprocs_for_coll - 1) / nprocs_for_coll;
00225     align_fr(fr_size, 0, fd->hints->cb_fr_alignment,
00226          &aligned_fr_size, &aligned_fr_off);
00227     ADIOI_Create_fr_simpletype (fr_size, nprocs_for_coll, &simpletype);
00228 
00229     for (i=0; i < nprocs_for_coll; i++)
00230     {
00231     file_realm_st_offs[i] = fr_size * i;
00232     file_realm_types[i]   = simpletype;
00233     }
00234 }
00235 
00236 /* creates a datatype with an empty trailing edge */
00237 void ADIOI_Create_fr_simpletype (int size, int nprocs_for_coll,
00238                   MPI_Datatype *simpletype)
00239 {
00240     int count=2, blocklens[2];
00241     MPI_Aint indices[2];
00242     MPI_Datatype old_types[2];
00243 
00244     blocklens[0] = size;
00245     blocklens[1] = 1;
00246     indices[0]   = 0;
00247     indices[1]   = size*nprocs_for_coll;
00248     old_types[0] = MPI_BYTE;
00249     old_types[1] = MPI_UB;
00250 
00251     MPI_Type_struct (count, blocklens, indices, old_types, simpletype);
00252 
00253     MPI_Type_commit (simpletype);
00254 }
00255 
00256 /* Verify that file realms are covering (PFRs) and non-overlapping */
00257 void ADIOI_Verify_fr (int nprocs_for_coll, ADIO_Offset *file_realm_st_offs,
00258               MPI_Datatype *file_realm_types)
00259 {
00260 }
00261 
00262 int ADIOI_Agg_idx (int rank, ADIO_File fd) {
00263     int i, cb_nodes, *ranklist;
00264     cb_nodes = fd->hints->cb_nodes;
00265     ranklist = fd->hints->ranklist;
00266 
00267     for (i=0; i<cb_nodes; i++) {
00268     if (ranklist[i])
00269         return i;
00270     }
00271     return -1;
00272 }
00273 
00274 static void align_fr (int fr_size, ADIO_Offset fr_off, int alignment,
00275            int *aligned_fr_size, ADIO_Offset *aligned_fr_off) {
00276     *aligned_fr_off = fr_off - (fr_off % alignment);
00277     *aligned_fr_size = ((fr_off + fr_size) / alignment) * alignment - 
00278     *aligned_fr_off;
00279     if ((fr_off + fr_size) % alignment)
00280     *aligned_fr_size += alignment;
00281 
00282     assert(!((*aligned_fr_off % alignment) ||
00283          (*aligned_fr_size % alignment)));
00284 }
libs/ck-libs/ampi/romio/adio/common/ad_aggregate_new.c
