Charm++: Charm Source Code Documentation

00001 /* -*- Mode: C; c-basic-offset:4 ; -*- 
00002  * vim: ts=8 sts=4 sw=4 noexpandtab 
00003  * 
00004  *   Copyright (C) 2008 University of Chicago. 
00005  *   See COPYRIGHT notice in top-level directory.
00006  */
00007 
00008 #include "adio.h"
00009 #include "adio_extern.h"
00010 #include "ad_pvfs2.h"
00011 
00012 #include "ad_pvfs2_common.h"
00013 
00014 void ADIOI_PVFS2_OldReadStrided(ADIO_File fd, void *buf, int count,
00015                  MPI_Datatype datatype, int file_ptr_type,
00016                  ADIO_Offset offset, ADIO_Status *status, int
00017                  *error_code)
00018 {
00019     /* offset is in units of etype relative to the filetype. */
00020     ADIOI_Flatlist_node *flat_buf, *flat_file;
00021     int i, j, k,  brd_size, frd_size=0, st_index=0;
00022     int bufsize, sum, n_etypes_in_filetype, size_in_filetype;
00023     int n_filetypes, etype_in_filetype;
00024     ADIO_Offset abs_off_in_filetype=0;
00025     int filetype_size, etype_size, buftype_size;
00026     MPI_Aint filetype_extent, buftype_extent; 
00027     int buf_count, buftype_is_contig, filetype_is_contig;
00028     ADIO_Offset off, disp, start_off, initial_off;
00029     int flag, st_frd_size, st_n_filetypes;
00030 
00031     int mem_list_count, file_list_count;
00032     PVFS_size *mem_offsets;
00033     int64_t *file_offsets;
00034     int *mem_lengths;
00035     int32_t *file_lengths;
00036     int total_blks_to_read;
00037 
00038     int max_mem_list, max_file_list;
00039 
00040     int b_blks_read;
00041     int f_data_read;
00042     int size_read=0, n_read_lists, extra_blks;
00043 
00044     int end_brd_size, end_frd_size;
00045     int start_k, start_j, new_file_read, new_buffer_read;
00046     int start_mem_offset;
00047     PVFS_Request mem_req, file_req;
00048     ADIOI_PVFS2_fs * pvfs_fs;
00049     PVFS_sysresp_io resp_io;
00050     int err_flag=0;
00051     MPI_Offset total_bytes_read = 0;
00052     static char myname[] = "ADIOI_PVFS2_ReadStrided";
00053 
00054 #define MAX_ARRAY_SIZE 64
00055 
00056     *error_code = MPI_SUCCESS;  /* changed below if error */
00057 
00058     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
00059     ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
00060 
00061     /* the HDF5 tests showed a bug in this list processing code (see many many
00062      * lines down below).  We added a workaround, but common HDF5 file types
00063      * are actually contiguous and do not need the expensive workarond */
00064     if (!filetype_is_contig) {
00065     flat_file = ADIOI_Flatlist;
00066     while (flat_file->type != fd->filetype) flat_file = flat_file->next;
00067     if (flat_file->count == 1 && !buftype_is_contig)
00068         filetype_is_contig = 1;
00069     }
00070 
00071     MPI_Type_size(fd->filetype, &filetype_size);
00072     if ( ! filetype_size ) {
00073     *error_code = MPI_SUCCESS; 
00074     return;
00075     }
00076 
00077     MPI_Type_extent(fd->filetype, &filetype_extent);
00078     MPI_Type_size(datatype, &buftype_size);
00079     MPI_Type_extent(datatype, &buftype_extent);
00080     etype_size = fd->etype_size;
00081 
00082     bufsize = buftype_size * count;
00083     
00084     pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
00085 
00086     if (!buftype_is_contig && filetype_is_contig) {
00087 
00088 /* noncontiguous in memory, contiguous in file. */
00089         int64_t file_offsets;
00090     int32_t file_lengths;
00091 
00092     ADIOI_Flatten_datatype(datatype);
00093     flat_buf = ADIOI_Flatlist;
00094     while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00095 
00096     off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
00097         fd->disp + etype_size * offset;
00098 
00099     file_list_count = 1;
00100     file_offsets = off;
00101     file_lengths = 0;
00102     total_blks_to_read = count*flat_buf->count;
00103     b_blks_read = 0;
00104 
00105     /* allocate arrays according to max usage */
00106     if (total_blks_to_read > MAX_ARRAY_SIZE)
00107         mem_list_count = MAX_ARRAY_SIZE;
00108     else mem_list_count = total_blks_to_read;
00109     mem_offsets = (PVFS_size*)ADIOI_Malloc(mem_list_count*sizeof(PVFS_size));
00110     mem_lengths = (int*)ADIOI_Malloc(mem_list_count*sizeof(int));
00111 
00112     /* TODO: CHECK RESULTS OF MEMORY ALLOCATION */
00113 
00114     j = 0;
00115     /* step through each block in memory, filling memory arrays */
00116     while (b_blks_read < total_blks_to_read) {
00117         for (i=0; i<flat_buf->count; i++) {
00118         mem_offsets[b_blks_read % MAX_ARRAY_SIZE] = 
00119             /* TODO: fix this compiler warning */
00120             ((PVFS_size)buf + j*buftype_extent + flat_buf->indices[i]);
00121         mem_lengths[b_blks_read % MAX_ARRAY_SIZE] = 
00122             flat_buf->blocklens[i];
00123         file_lengths += flat_buf->blocklens[i];
00124         b_blks_read++;
00125         if (!(b_blks_read % MAX_ARRAY_SIZE) ||
00126             (b_blks_read == total_blks_to_read)) {
00127 
00128             /* in the case of the last read list call,
00129                adjust mem_list_count */
00130             if (b_blks_read == total_blks_to_read) {
00131                 mem_list_count = total_blks_to_read % MAX_ARRAY_SIZE;
00132             /* in case last read list call fills max arrays */
00133             if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
00134             }
00135             err_flag = PVFS_Request_hindexed(mem_list_count, 
00136                 mem_lengths, mem_offsets, PVFS_BYTE, &mem_req);
00137             if (err_flag < 0) break;
00138             err_flag = PVFS_Request_contiguous(file_lengths, 
00139                 PVFS_BYTE, &file_req);
00140             if (err_flag < 0) break;
00141 #ifdef ADIOI_MPE_LOGGING
00142                     MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
00143 #endif
00144             err_flag = PVFS_sys_read(pvfs_fs->object_ref, file_req, 
00145                 file_offsets, PVFS_BOTTOM, mem_req, 
00146                 &(pvfs_fs->credentials), &resp_io);
00147 #ifdef ADIOI_MPE_LOGGING
00148                     MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
00149 #endif
00150             /* --BEGIN ERROR HANDLING-- */
00151             if (err_flag != 0) {
00152             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00153                                MPIR_ERR_RECOVERABLE,
00154                                myname, __LINE__,
00155                                ADIOI_PVFS2_error_convert(err_flag),
00156                                "Error in PVFS_sys_read", 0);
00157             goto error_state;
00158             }
00159             PVFS_Request_free(&mem_req);
00160             PVFS_Request_free(&file_req);
00161             total_bytes_read += resp_io.total_completed;
00162             /* --END ERROR HANDLING-- */
00163           
00164             /* in the case of error or the last read list call, 
00165              * leave here */
00166             if (err_flag || b_blks_read == total_blks_to_read) break;
00167 
00168             file_offsets += file_lengths;
00169             file_lengths = 0;
00170         } 
00171         } /* for (i=0; i<flat_buf->count; i++) */
00172         j++;
00173     } /* while (b_blks_read < total_blks_to_read) */
00174     ADIOI_Free(mem_offsets);
00175     ADIOI_Free(mem_lengths);
00176 
00177         if (file_ptr_type == ADIO_INDIVIDUAL) 
00178         fd->fp_ind += total_bytes_read;
00179 
00180     fd->fp_sys_posn = -1;  /* set it to null. */
00181 
00182 #ifdef HAVE_STATUS_SET_BYTES
00183     MPIR_Status_set_bytes(status, datatype, bufsize);
00184     /* This isa temporary way of filling in status.  The right way is to
00185        keep tracke of how much data was actually read adn placed in buf
00186        by ADIOI_BUFFERED_READ. */
00187 #endif
00188     ADIOI_Delete_flattened(datatype);
00189 
00190     return;
00191     } /* if (!buftype_is_contig && filetype_is_contig) */
00192 
00193     /* know file is noncontiguous from above */
00194     /* noncontiguous in file */
00195 
00196     /* filetype already flattened in ADIO_Open */
00197     flat_file = ADIOI_Flatlist;
00198     while (flat_file->type != fd->filetype) flat_file = flat_file->next;
00199 
00200     disp = fd->disp;
00201     initial_off = offset;
00202 
00203 
00204     /* for each case - ADIO_Individual pointer or explicit, find the file
00205        offset in bytes (offset), n_filetypes (how many filetypes into
00206        file to start), frd_size (remaining amount of data in present
00207        file block), and st_index (start point in terms of blocks in
00208        starting filetype) */
00209     if (file_ptr_type == ADIO_INDIVIDUAL) {
00210         offset = fd->fp_ind; /* in bytes */
00211     n_filetypes = -1;
00212     flag = 0;
00213     while (!flag) {
00214         n_filetypes++;
00215         for (i=0; i<flat_file->count; i++) {
00216             if (disp + flat_file->indices[i] + 
00217             ((ADIO_Offset) n_filetypes)*filetype_extent +
00218             flat_file->blocklens[i]  >= offset) {
00219             st_index = i;
00220             frd_size = (int) (disp + flat_file->indices[i] + 
00221                     ((ADIO_Offset) n_filetypes)*filetype_extent
00222                       + flat_file->blocklens[i] - offset);
00223             flag = 1;
00224             break;
00225         }
00226         }
00227     } /* while (!flag) */
00228     } /* if (file_ptr_type == ADIO_INDIVIDUAL) */
00229     else {
00230         n_etypes_in_filetype = filetype_size/etype_size;
00231     n_filetypes = (int) (offset / n_etypes_in_filetype);
00232     etype_in_filetype = (int) (offset % n_etypes_in_filetype);
00233     size_in_filetype = etype_in_filetype * etype_size;
00234     
00235     sum = 0;
00236     for (i=0; i<flat_file->count; i++) {
00237         sum += flat_file->blocklens[i];
00238         if (sum > size_in_filetype) {
00239             st_index = i;
00240         frd_size = sum - size_in_filetype;
00241         abs_off_in_filetype = flat_file->indices[i] +
00242             size_in_filetype - (sum - flat_file->blocklens[i]);
00243         break;
00244         }
00245     }
00246     
00247     /* abs. offset in bytes in the file */
00248     offset = disp + ((ADIO_Offset) n_filetypes)*filetype_extent + 
00249         abs_off_in_filetype;
00250     } /* else [file_ptr_type != ADIO_INDIVIDUAL] */
00251 
00252     start_off = offset;
00253     st_frd_size = frd_size;
00254     st_n_filetypes = n_filetypes;
00255     
00256     if (buftype_is_contig && !filetype_is_contig) {
00257 
00258 /* contiguous in memory, noncontiguous in file. should be the most
00259    common case. */
00260 
00261         int mem_lengths;
00262     char *mem_offsets;
00263     
00264     i = 0;
00265     j = st_index;
00266     n_filetypes = st_n_filetypes;
00267     
00268     mem_list_count = 1;
00269     
00270     /* determine how many blocks in file to read */
00271     f_data_read = ADIOI_MIN(st_frd_size, bufsize);
00272     total_blks_to_read = 1;
00273     if (j < (flat_file->count-1)) j++;
00274     else {
00275         j = 0;
00276         n_filetypes++;
00277     }
00278     while (f_data_read < bufsize) {
00279         f_data_read += flat_file->blocklens[j];
00280         total_blks_to_read++;
00281         if (j<(flat_file->count-1)) j++;
00282         else j = 0; 
00283     }
00284       
00285     j = st_index;
00286     n_filetypes = st_n_filetypes;
00287     n_read_lists = total_blks_to_read/MAX_ARRAY_SIZE;
00288     extra_blks = total_blks_to_read%MAX_ARRAY_SIZE;
00289     
00290     mem_offsets = buf;
00291     mem_lengths = 0;
00292     
00293     /* if at least one full readlist, allocate file arrays
00294        at max array size and don't free until very end */
00295     if (n_read_lists) {
00296         file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
00297                           sizeof(int64_t));
00298         file_lengths = (int32_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
00299                           sizeof(int32_t));
00300     }
00301     /* if there's no full readlist allocate file arrays according
00302        to needed size (extra_blks) */
00303     else {
00304         file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
00305                           sizeof(int64_t));
00306         file_lengths = (int32_t*)ADIOI_Malloc(extra_blks*
00307                           sizeof(int32_t));
00308     }
00309     
00310     /* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
00311     for (i=0; i<n_read_lists; i++) {
00312         file_list_count = MAX_ARRAY_SIZE;
00313         if(!i) {
00314             file_offsets[0] = offset;
00315         file_lengths[0] = st_frd_size;
00316         mem_lengths = st_frd_size;
00317         }
00318         for (k=0; k<MAX_ARRAY_SIZE; k++) {
00319             if (i || k) {
00320             file_offsets[k] = disp + 
00321             ((ADIO_Offset)n_filetypes)*filetype_extent
00322               + flat_file->indices[j];
00323             file_lengths[k] = flat_file->blocklens[j];
00324             mem_lengths += file_lengths[k];
00325         }
00326         if (j<(flat_file->count - 1)) j++;
00327         else {
00328             j = 0;
00329             n_filetypes++;
00330         }
00331         } /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
00332         err_flag = PVFS_Request_contiguous(mem_lengths, 
00333                            PVFS_BYTE, &mem_req);
00334         /* --BEGIN ERROR HANDLING-- */
00335         if (err_flag != 0) {
00336         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00337                            MPIR_ERR_RECOVERABLE,
00338                            myname, __LINE__,
00339                            ADIOI_PVFS2_error_convert(err_flag),
00340                            "Error in PVFS_Request_contiguous (memory)", 0);
00341         goto error_state;
00342         }
00343         /* --END ERROR HANDLING-- */
00344 
00345         err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
00346                          file_offsets, PVFS_BYTE,
00347                          &file_req);
00348         /* --BEGIN ERROR HANDLING-- */
00349         if (err_flag != 0) {
00350         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00351                            MPIR_ERR_RECOVERABLE,
00352                            myname, __LINE__,
00353                            ADIOI_PVFS2_error_convert(err_flag),
00354                            "Error in PVFS_Request_hindexed (file)", 0);
00355         goto error_state;
00356         }
00357         /* --END ERROR HANDLING-- */
00358 
00359         /* PVFS_Request_hindexed already expresses the offsets into the
00360          * file, so we should not pass in an offset if we are using
00361          * hindexed for the file type */
00362 #ifdef ADIOI_MPE_LOGGING
00363             MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
00364 #endif
00365         err_flag = PVFS_sys_read(pvfs_fs->object_ref, file_req, 0, 
00366                      mem_offsets, mem_req,
00367                      &(pvfs_fs->credentials), &resp_io);
00368 #ifdef ADIOI_MPE_LOGGING
00369             MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
00370 #endif
00371         /* --BEGIN ERROR HANDLING-- */
00372         if (err_flag != 0) {
00373         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00374                            MPIR_ERR_RECOVERABLE,
00375                            myname, __LINE__,
00376                            ADIOI_PVFS2_error_convert(err_flag),
00377                            "Error in PVFS_sys_read", 0);
00378         goto error_state;
00379         }
00380         /* --END ERROR HANDING-- */
00381         PVFS_Request_free(&mem_req);
00382         PVFS_Request_free(&file_req);
00383 
00384         total_bytes_read += resp_io.total_completed;
00385 
00386         mem_offsets += mem_lengths;
00387         mem_lengths = 0;
00388     } /* for (i=0; i<n_read_lists; i++) */
00389 
00390     /* for file arrays smaller than MAX_ARRAY_SIZE (last read_list call) */
00391     if (extra_blks) {
00392         file_list_count = extra_blks;
00393         if(!i) {
00394             file_offsets[0] = offset;
00395         file_lengths[0] = ADIOI_MIN(st_frd_size, bufsize);
00396         }
00397         for (k=0; k<extra_blks; k++) {
00398             if(i || k) {
00399             file_offsets[k] = disp + 
00400             ((ADIO_Offset)n_filetypes)*filetype_extent +
00401             flat_file->indices[j];
00402             if (k == (extra_blks - 1)) {
00403                 file_lengths[k] = bufsize - (int32_t) mem_lengths
00404               - (int32_t) mem_offsets + (int32_t)  buf;
00405             }
00406             else file_lengths[k] = flat_file->blocklens[j];
00407         } /* if(i || k) */
00408         mem_lengths += file_lengths[k];
00409         if (j<(flat_file->count - 1)) j++;
00410         else {
00411             j = 0;
00412             n_filetypes++;
00413         }
00414         } /* for (k=0; k<extra_blks; k++) */
00415         err_flag = PVFS_Request_contiguous(mem_lengths,
00416                            PVFS_BYTE, &mem_req);
00417         /* --BEGIN ERROR HANDLING-- */
00418         if (err_flag != 0) {
00419         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00420                            MPIR_ERR_RECOVERABLE,
00421                            myname, __LINE__,
00422                            ADIOI_PVFS2_error_convert(err_flag),
00423                            "Error in PVFS_Request_contiguous (memory)", 0);
00424         goto error_state;
00425         }
00426         /* --END ERROR HANDLING-- */
00427 
00428         err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
00429             file_offsets, PVFS_BYTE, &file_req);
00430         /* --BEGIN ERROR HANDLING-- */
00431         if (err_flag != 0) {
00432         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00433                            MPIR_ERR_RECOVERABLE,
00434                            myname, __LINE__,
00435                            ADIOI_PVFS2_error_convert(err_flag),
00436                            "Error in PVFS_Request_hindexed (file)", 0);
00437         goto error_state;
00438         }
00439         /* --END ERROR HANDLING-- */
00440 
00441         /* as above, use 0 for 'offset' when using hindexed file type */
00442 #ifdef ADIOI_MPE_LOGGING
00443             MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
00444 #endif
00445         err_flag = PVFS_sys_read(pvfs_fs->object_ref, file_req, 0, 
00446             mem_offsets, mem_req, &(pvfs_fs->credentials), &resp_io);
00447 #ifdef ADIOI_MPE_LOGGING
00448             MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
00449 #endif
00450         /* --BEGIN ERROR HANDLING-- */
00451         if (err_flag != 0) {
00452         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00453                            MPIR_ERR_RECOVERABLE,
00454                            myname, __LINE__,
00455                            ADIOI_PVFS2_error_convert(err_flag),
00456                            "Error in PVFS_sys_read", 0);        
00457         goto error_state;
00458         }
00459         /* --END ERROR HANDLING-- */
00460         PVFS_Request_free(&mem_req);
00461         PVFS_Request_free(&file_req);
00462         total_bytes_read += resp_io.total_completed;
00463     }
00464     }
00465     else {
00466 /* noncontiguous in memory as well as in file */
00467       
00468         ADIOI_Flatten_datatype(datatype);
00469     flat_buf = ADIOI_Flatlist;
00470     while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00471 
00472     size_read = 0;
00473     n_filetypes = st_n_filetypes;
00474     frd_size = st_frd_size;
00475     brd_size = flat_buf->blocklens[0];
00476     buf_count = 0;
00477     start_mem_offset = 0;
00478     start_k = k = 0;
00479     start_j = st_index;
00480     max_mem_list = 0;
00481     max_file_list = 0;
00482 
00483     /* run through and file max_file_list and max_mem_list so that you 
00484        can allocate the file and memory arrays less than MAX_ARRAY_SIZE
00485        if possible */
00486 
00487     while (size_read < bufsize) {
00488         k = start_k;
00489         new_buffer_read = 0;
00490         mem_list_count = 0;
00491         while ((mem_list_count < MAX_ARRAY_SIZE) && 
00492            (new_buffer_read < bufsize-size_read)) {
00493             /* find mem_list_count and file_list_count such that both are
00494            less than MAX_ARRAY_SIZE, the sum of their lengths are
00495            equal, and the sum of all the data read and data to be
00496            read in the next immediate read list is less than
00497            bufsize */
00498             if(mem_list_count) {
00499             if((new_buffer_read + flat_buf->blocklens[k] + 
00500             size_read) > bufsize) {
00501                 end_brd_size = new_buffer_read + 
00502                 flat_buf->blocklens[k] - (bufsize - size_read);
00503             new_buffer_read = bufsize - size_read;
00504             }
00505             else {
00506                 new_buffer_read += flat_buf->blocklens[k];
00507             end_brd_size = flat_buf->blocklens[k];
00508             }
00509         }
00510         else {
00511             if (brd_size > (bufsize - size_read)) {
00512                 new_buffer_read = bufsize - size_read;
00513             brd_size = new_buffer_read;
00514             }
00515             else new_buffer_read = brd_size;
00516         }
00517         mem_list_count++;
00518         k = (k + 1)%flat_buf->count;
00519          } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00520            (new_buffer_read < bufsize-size_read)) */
00521         j = start_j;
00522         new_file_read = 0;
00523         file_list_count = 0;
00524         while ((file_list_count < MAX_ARRAY_SIZE) && 
00525            (new_file_read < new_buffer_read)) {
00526             if(file_list_count) {
00527             if((new_file_read + flat_file->blocklens[j]) > 
00528                new_buffer_read) {
00529                 end_frd_size = new_buffer_read - new_file_read;
00530             new_file_read = new_buffer_read;
00531             j--;
00532             }
00533             else {
00534                 new_file_read += flat_file->blocklens[j];
00535             end_frd_size = flat_file->blocklens[j];
00536             }
00537         }
00538         else {
00539             if (frd_size > new_buffer_read) {
00540                 new_file_read = new_buffer_read;
00541             frd_size = new_file_read;
00542             }
00543             else new_file_read = frd_size;
00544         }
00545         file_list_count++;
00546         if (j < (flat_file->count - 1)) j++;
00547         else j = 0;
00548         
00549         k = start_k;
00550         if ((new_file_read < new_buffer_read) && 
00551             (file_list_count == MAX_ARRAY_SIZE)) {
00552             new_buffer_read = 0;
00553             mem_list_count = 0;
00554             while (new_buffer_read < new_file_read) {
00555                 if(mem_list_count) {
00556                 if((new_buffer_read + flat_buf->blocklens[k]) >
00557                    new_file_read) {
00558                     end_brd_size = new_file_read - new_buffer_read;
00559                 new_buffer_read = new_file_read;
00560                 k--;
00561                 }
00562                 else {
00563                     new_buffer_read += flat_buf->blocklens[k];
00564                 end_brd_size = flat_buf->blocklens[k];
00565                 }
00566             }
00567             else {
00568                 new_buffer_read = brd_size;
00569                 if (brd_size > (bufsize - size_read)) {
00570                     new_buffer_read = bufsize - size_read;
00571                 brd_size = new_buffer_read;
00572                 }
00573             }
00574             mem_list_count++;
00575             k = (k + 1)%flat_buf->count;
00576             } /* while (new_buffer_read < new_file_read) */
00577         } /* if ((new_file_read < new_buffer_read) && (file_list_count
00578              == MAX_ARRAY_SIZE)) */
00579         } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00580          (new_buffer_read < bufsize-size_read)) */
00581 
00582         /*  fakes filling the readlist arrays of lengths found above  */
00583         k = start_k;
00584         j = start_j;
00585         for (i=0; i<mem_list_count; i++) {       
00586         if(i) {
00587             if (i == (mem_list_count - 1)) {
00588             if (flat_buf->blocklens[k] == end_brd_size)
00589                 brd_size = flat_buf->blocklens[(k+1)%
00590                               flat_buf->count];
00591             else {
00592                 brd_size = flat_buf->blocklens[k] - end_brd_size;
00593                 k--;
00594                 buf_count--;
00595             }
00596             }
00597         }
00598         buf_count++;
00599         k = (k + 1)%flat_buf->count;
00600         } /* for (i=0; i<mem_list_count; i++) */
00601         for (i=0; i<file_list_count; i++) {
00602         if (i) {
00603             if (i == (file_list_count - 1)) {
00604             if (flat_file->blocklens[j] == end_frd_size)
00605                 frd_size = flat_file->blocklens[(j+1)%
00606                               flat_file->count];   
00607             else {
00608                 frd_size = flat_file->blocklens[j] - end_frd_size;
00609                 j--;
00610             }
00611             }
00612         }
00613         if (j < flat_file->count - 1) j++;
00614         else {
00615             j = 0;
00616             n_filetypes++;
00617         }
00618         } /* for (i=0; i<file_list_count; i++) */
00619         size_read += new_buffer_read;
00620         start_k = k;
00621         start_j = j;
00622         if (max_mem_list < mem_list_count)
00623             max_mem_list = mem_list_count;
00624         if (max_file_list < file_list_count)
00625             max_file_list = file_list_count;
00626     } /* while (size_read < bufsize) */
00627 
00628     /* one last check before we actually carry out the operation:
00629      * this code has hard-to-fix bugs when a noncontiguous file type has
00630      * such large pieces that the sum of the lengths of the memory type is
00631      * not larger than one of those pieces (and vice versa for large memory
00632      * types and many pices of file types.  In these cases, give up and
00633      * fall back to naive reads and writes.  The testphdf5 test created a
00634      * type with two very large memory regions and 600 very small file
00635      * regions.  The same test also created a type with one very large file
00636      * region and many (700) very small memory regions.  both cases caused
00637      * problems for this code */
00638 
00639     if ( ( (file_list_count == 1) && 
00640             (new_file_read < flat_file->blocklens[0] ) ) ||
00641         ((mem_list_count == 1) && 
00642             (new_buffer_read < flat_buf->blocklens[0]) ) ||
00643         ((file_list_count == MAX_ARRAY_SIZE) && 
00644             (new_file_read < flat_buf->blocklens[0]) ) ||
00645         ( (mem_list_count == MAX_ARRAY_SIZE) &&
00646             (new_buffer_read < flat_file->blocklens[0])) )
00647     {
00648 
00649         ADIOI_Delete_flattened(datatype);
00650         ADIOI_GEN_ReadStrided_naive(fd, buf, count, datatype,
00651             file_ptr_type, initial_off, status, error_code);
00652         return;
00653     }
00654 
00655     mem_offsets = (PVFS_size*)ADIOI_Malloc(max_mem_list*sizeof(PVFS_size));
00656     mem_lengths = (int *)ADIOI_Malloc(max_mem_list*sizeof(int));
00657     file_offsets = (int64_t *)ADIOI_Malloc(max_file_list*sizeof(int64_t));
00658     file_lengths = (int32_t *)ADIOI_Malloc(max_file_list*sizeof(int32_t));
00659         
00660     size_read = 0;
00661     n_filetypes = st_n_filetypes;
00662     frd_size = st_frd_size;
00663     brd_size = flat_buf->blocklens[0];
00664     buf_count = 0;
00665     start_mem_offset = 0;
00666     start_k = k = 0;
00667     start_j = st_index;
00668 
00669     /*  this section calculates mem_list_count and file_list_count
00670         and also finds the possibly odd sized last array elements
00671         in new_frd_size and new_brd_size  */
00672     
00673     while (size_read < bufsize) {
00674         k = start_k;
00675         new_buffer_read = 0;
00676         mem_list_count = 0;
00677         while ((mem_list_count < MAX_ARRAY_SIZE) && 
00678            (new_buffer_read < bufsize-size_read)) {
00679             /* find mem_list_count and file_list_count such that both are
00680            less than MAX_ARRAY_SIZE, the sum of their lengths are
00681            equal, and the sum of all the data read and data to be
00682            read in the next immediate read list is less than
00683            bufsize */
00684             if(mem_list_count) {
00685             if((new_buffer_read + flat_buf->blocklens[k] + 
00686             size_read) > bufsize) {
00687                 end_brd_size = new_buffer_read + 
00688                 flat_buf->blocklens[k] - (bufsize - size_read);
00689             new_buffer_read = bufsize - size_read;
00690             }
00691             else {
00692                 new_buffer_read += flat_buf->blocklens[k];
00693             end_brd_size = flat_buf->blocklens[k];
00694             }
00695         }
00696         else {
00697             if (brd_size > (bufsize - size_read)) {
00698                 new_buffer_read = bufsize - size_read;
00699             brd_size = new_buffer_read;
00700             }
00701             else new_buffer_read = brd_size;
00702         }
00703         mem_list_count++;
00704         k = (k + 1)%flat_buf->count;
00705          } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00706            (new_buffer_read < bufsize-size_read)) */
00707         j = start_j;
00708         new_file_read = 0;
00709         file_list_count = 0;
00710         while ((file_list_count < MAX_ARRAY_SIZE) && 
00711            (new_file_read < new_buffer_read)) {
00712             if(file_list_count) {
00713             if((new_file_read + flat_file->blocklens[j]) > 
00714                new_buffer_read) {
00715                 end_frd_size = new_buffer_read - new_file_read;
00716             new_file_read = new_buffer_read;
00717             j--;
00718             }
00719             else {
00720                 new_file_read += flat_file->blocklens[j];
00721             end_frd_size = flat_file->blocklens[j];
00722             }
00723         }
00724         else {
00725             if (frd_size > new_buffer_read) {
00726                 new_file_read = new_buffer_read;
00727             frd_size = new_file_read;
00728             }
00729             else new_file_read = frd_size;
00730         }
00731         file_list_count++;
00732         if (j < (flat_file->count - 1)) j++;
00733         else j = 0;
00734         
00735         k = start_k;
00736         if ((new_file_read < new_buffer_read) && 
00737             (file_list_count == MAX_ARRAY_SIZE)) {
00738             new_buffer_read = 0;
00739             mem_list_count = 0;
00740             while (new_buffer_read < new_file_read) {
00741                 if(mem_list_count) {
00742                 if((new_buffer_read + flat_buf->blocklens[k]) >
00743                    new_file_read) {
00744                     end_brd_size = new_file_read - new_buffer_read;
00745                 new_buffer_read = new_file_read;
00746                 k--;
00747                 }
00748                 else {
00749                     new_buffer_read += flat_buf->blocklens[k];
00750                 end_brd_size = flat_buf->blocklens[k];
00751                 }
00752             }
00753             else {
00754                 new_buffer_read = brd_size;
00755                 if (brd_size > (bufsize - size_read)) {
00756                     new_buffer_read = bufsize - size_read;
00757                 brd_size = new_buffer_read;
00758                 }
00759             }
00760             mem_list_count++;
00761             k = (k + 1)%flat_buf->count;
00762             } /* while (new_buffer_read < new_file_read) */
00763         } /* if ((new_file_read < new_buffer_read) && (file_list_count
00764              == MAX_ARRAY_SIZE)) */
00765         } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00766          (new_buffer_read < bufsize-size_read)) */
00767 
00768         /*  fills the allocated readlist arrays  */
00769         k = start_k;
00770         j = start_j;
00771         for (i=0; i<mem_list_count; i++) {       
00772             mem_offsets[i] = ((PVFS_size)buf + buftype_extent*
00773                      (buf_count/flat_buf->count) +
00774                      (int)flat_buf->indices[k]);
00775         if(!i) {
00776             mem_lengths[0] = brd_size;
00777             mem_offsets[0] += flat_buf->blocklens[k] - brd_size;
00778         }
00779         else {
00780             if (i == (mem_list_count - 1)) {
00781                 mem_lengths[i] = end_brd_size;
00782             if (flat_buf->blocklens[k] == end_brd_size)
00783                 brd_size = flat_buf->blocklens[(k+1)%
00784                               flat_buf->count];
00785             else {
00786                 brd_size = flat_buf->blocklens[k] - end_brd_size;
00787                 k--;
00788                 buf_count--;
00789             }
00790             }
00791             else {
00792                 mem_lengths[i] = flat_buf->blocklens[k];
00793             }
00794         }
00795         buf_count++;
00796         k = (k + 1)%flat_buf->count;
00797         } /* for (i=0; i<mem_list_count; i++) */
00798         for (i=0; i<file_list_count; i++) {
00799             file_offsets[i] = disp + flat_file->indices[j] + 
00800             ((ADIO_Offset)n_filetypes) * filetype_extent;
00801             if (!i) {
00802             file_lengths[0] = frd_size;
00803             file_offsets[0] += flat_file->blocklens[j] - frd_size;
00804         }
00805         else {
00806             if (i == (file_list_count - 1)) {
00807                 file_lengths[i] = end_frd_size;
00808             if (flat_file->blocklens[j] == end_frd_size)
00809                 frd_size = flat_file->blocklens[(j+1)%
00810                               flat_file->count];   
00811             else {
00812                 frd_size = flat_file->blocklens[j] - end_frd_size;
00813                 j--;
00814             }
00815             }
00816             else file_lengths[i] = flat_file->blocklens[j];
00817         }
00818         if (j < flat_file->count - 1) j++;
00819         else {
00820             j = 0;
00821             n_filetypes++;
00822         }
00823         } /* for (i=0; i<file_list_count; i++) */
00824         err_flag = PVFS_Request_hindexed(mem_list_count, mem_lengths, 
00825             mem_offsets, PVFS_BYTE, &mem_req);
00826         /* --BEGIN ERROR HANDLING-- */
00827         if (err_flag != 0 ) {
00828         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00829                            MPIR_ERR_RECOVERABLE,
00830                            myname, __LINE__,
00831                            ADIOI_PVFS2_error_convert(err_flag),
00832                            "Error in PVFS_Request_hindexed (memory)", 0);
00833         goto error_state;
00834         }
00835         /* -- END ERROR HANDLING-- */
00836         err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
00837             file_offsets, PVFS_BYTE, &file_req);
00838         /* --BEGIN ERROR HANDLING-- */
00839         if (err_flag != 0) {
00840         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00841                            MPIR_ERR_RECOVERABLE,
00842                            myname, __LINE__,
00843                            ADIOI_PVFS2_error_convert(err_flag),
00844                            "Error in PVFS_Request_hindexed (file)", 0);
00845         goto error_state;
00846         }
00847         /* --END ERROR HANDLING-- */
00848 
00849         /* offset will be expressed in memory and file datatypes */
00850 #ifdef ADIOI_MPE_LOGGING
00851             MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
00852 #endif
00853         err_flag = PVFS_sys_read(pvfs_fs->object_ref, file_req, 0, 
00854             PVFS_BOTTOM, mem_req, &(pvfs_fs->credentials), &resp_io);
00855 #ifdef ADIOI_MPE_LOGGING
00856             MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
00857 #endif
00858         /* --BEGIN ERROR HANDLING-- */
00859         if (err_flag != 0) {
00860         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00861                            MPIR_ERR_RECOVERABLE,
00862                            myname, __LINE__,
00863                            ADIOI_PVFS2_error_convert(err_flag),
00864                            "Error in PVFS_sys_read", 0);
00865         }
00866         /* --END ERROR HANDLING-- */
00867         PVFS_Request_free(&mem_req);
00868         PVFS_Request_free(&file_req);
00869         total_bytes_read += resp_io.total_completed;
00870         size_read += new_buffer_read;
00871         start_k = k;
00872         start_j = j;
00873     } /* while (size_read < bufsize) */
00874     ADIOI_Free(mem_offsets);
00875     ADIOI_Free(mem_lengths);
00876     }
00877     /* Other ADIO routines will convert absolute bytes into counts of datatypes */
00878     /* when incrementing fp_ind, need to also take into account the file type:
00879      * consider an N-element 1-d subarray with a lb and ub: ( |---xxxxx-----|
00880      * if we wrote N elements, offset needs to point at beginning of type, not
00881      * at empty region at offset N+1) 
00882      *
00883      * As we discussed on mpich-discuss in may/june 2009, the code below might
00884      * look wierd, but by putting fp_ind at the last byte written, the next
00885      * time we run through the strided code we'll update the fp_ind to the
00886      * right location. */
00887     if (file_ptr_type == ADIO_INDIVIDUAL) {
00888     fd->fp_ind = file_offsets[file_list_count-1]+
00889         file_lengths[file_list_count-1];
00890     }
00891     
00892     ADIOI_Free(file_offsets);
00893     ADIOI_Free(file_lengths);
00894     
00895     if (err_flag == 0) *error_code = MPI_SUCCESS;
00896 
00897 error_state:
00898     fd->fp_sys_posn = -1;   /* set it to null. */
00899 
00900 #ifdef HAVE_STATUS_SET_BYTES
00901     MPIR_Status_set_bytes(status, datatype, bufsize);
00902     /* This is a temporary way of filling in status. The right way is to 
00903        keep track of how much data was actually read and placed in buf 
00904        by ADIOI_BUFFERED_READ. */
00905 #endif
00906     
00907     if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
00908 }
00909
libs/ck-libs/ampi/romio/adio/ad_pvfs2/ad_pvfs2_read_list_classic.c
