Charm++: Charm Source Code Documentation

00001 /* -*- Mode: C; c-basic-offset:4 ; -*- */
00002 /* 
00003  *
00004  *   Copyright (C) 1997 University of Chicago. 
00005  *   See COPYRIGHT notice in top-level directory.
00006  */
00007 
00008 #include <unistd.h>
00009 
00010 #include "adio.h"
00011 #include "adio_extern.h"
00012 
00013 
00014 /* #define IO_DEBUG 1 */
00015 void ADIOI_NOLOCK_WriteStrided(ADIO_File fd, void *buf, int count,
00016                  MPI_Datatype datatype, int file_ptr_type,
00017                  ADIO_Offset offset, ADIO_Status *status, int
00018                  *error_code)
00019 {
00020 /* borrowed from old-school PVFS (v1) code. A driver for file systems that
00021  * cannot or do not support client-side buffering
00022  * Does not do data sieving optimization
00023  * Does contain write-combining optimization for noncontig in memory, contig in
00024  * file 
00025  */
00026 
00027 /* offset is in units of etype relative to the filetype. */
00028 
00029     ADIOI_Flatlist_node *flat_buf, *flat_file;
00030     int j, k, err=-1, st_index=0;
00031     ADIO_Offset fwr_size=0, bwr_size, new_bwr_size, new_fwr_size, i_offset, num;
00032     unsigned bufsize; 
00033     int n_etypes_in_filetype;
00034     ADIO_Offset n_filetypes, etype_in_filetype, size, sum;
00035     ADIO_Offset abs_off_in_filetype=0, size_in_filetype;
00036     int filetype_size, etype_size, buftype_size;
00037     MPI_Aint filetype_extent, buftype_extent, indx;
00038     int buf_count, buftype_is_contig, filetype_is_contig;
00039     ADIO_Offset off, disp;
00040     int flag, err_flag=0;
00041     static char myname[] = "ADIOI_NOLOCK_WRITESTRIDED";
00042 #ifdef IO_DEBUG
00043     int rank,nprocs;
00044 #endif
00045 
00046     /* --BEGIN ERROR HANDLING-- */
00047     if (fd->atomicity) {
00048     *error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
00049                        myname, __LINE__,
00050                        MPI_ERR_INTERN,
00051                        "Atomic mode set in I/O function", 0);
00052     return;
00053     }
00054     /* --END ERROR HANDLING-- */
00055 
00056     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
00057     ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
00058 
00059     MPI_Type_size(fd->filetype, &filetype_size);
00060     if ( ! filetype_size ) {
00061     *error_code = MPI_SUCCESS; 
00062     return;
00063     }
00064 
00065 #ifdef IO_DEBUG
00066     MPI_Comm_rank(fd->comm, &rank);
00067     MPI_Comm_size(fd->comm, &nprocs);
00068 #endif
00069 
00070     MPI_Type_extent(fd->filetype, &filetype_extent);
00071     MPI_Type_size(datatype, &buftype_size);
00072     MPI_Type_extent(datatype, &buftype_extent);
00073     etype_size = fd->etype_size;
00074     
00075     ADIOI_Assert((buftype_size * count) == ((ADIO_Offset)(unsigned)buftype_size * (ADIO_Offset)count));
00076     bufsize = buftype_size * count;
00077 
00078     if (!buftype_is_contig && filetype_is_contig) {
00079     char *combine_buf, *combine_buf_ptr;
00080     ADIO_Offset combine_buf_remain;
00081 /* noncontiguous in memory, contiguous in file. use writev */
00082 
00083     ADIOI_Flatten_datatype(datatype);
00084     flat_buf = CtvAccess(ADIOI_Flatlist);
00085     while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00086 
00087     /* allocate our "combine buffer" to pack data into before writing */
00088     combine_buf = (char *) ADIOI_Malloc(fd->hints->ind_wr_buffer_size);
00089     combine_buf_ptr = combine_buf;
00090     combine_buf_remain = fd->hints->ind_wr_buffer_size;
00091 
00092     /* seek to the right spot in the file */
00093     if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
00094         off = fd->disp + etype_size * offset;
00095         lseek(fd->fd_sys, off, SEEK_SET);
00096     }
00097     else off = lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
00098 
00099     /* loop through all the flattened pieces.  combine into buffer until
00100      * no more will fit, then write.
00101      *
00102      * special case of a given piece being bigger than the combine buffer
00103      * is also handled.
00104      */
00105     for (j=0; j<count; j++) {
00106     int i;
00107         for (i=0; i<flat_buf->count; i++) {
00108         if (flat_buf->blocklens[i] > combine_buf_remain && combine_buf != combine_buf_ptr) {
00109             /* there is data in the buffer; write out the buffer so far */
00110 #ifdef IO_DEBUG
00111             printf("[%d/%d] nc mem c file (0) writing loc = %Ld sz = %Ld\n", 
00112                     rank, nprocs, off, 
00113                     fd->hints->ind_wr_buffer_size-combine_buf_remain);
00114 #endif
00115 #ifdef ADIOI_MPE_LOGGING
00116             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
00117 #endif
00118             err = write(fd->fd_sys,
00119                      combine_buf,
00120                      fd->hints->ind_wr_buffer_size - combine_buf_remain);
00121 #ifdef ADIOI_MPE_LOGGING
00122             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
00123 #endif
00124             if (err == -1) err_flag = 1;
00125 
00126             /* reset our buffer info */
00127             combine_buf_ptr = combine_buf;
00128             combine_buf_remain = fd->hints->ind_wr_buffer_size;
00129         }
00130 
00131         /* TODO: heuristic for when to not bother to use combine buffer? */
00132         if (flat_buf->blocklens[i] >= combine_buf_remain) {
00133             /* special case: blocklen is as big as or bigger than the combine buf;
00134              * write directly
00135              */
00136 #ifdef IO_DEBUG
00137             printf("[%d/%d] nc mem c file (1) writing loc = %Ld sz = %d\n", 
00138                     rank, nprocs, off, 
00139                     flat_buf->blocklens[i]);
00140 #endif
00141         ADIOI_Assert(flat_buf->blocklens[i] == (unsigned)flat_buf->blocklens[i]);
00142         ADIOI_Assert((((ADIO_Offset)(MPIR_Upint)buf) + (ADIO_Offset)j*(ADIO_Offset)buftype_extent + flat_buf->indices[i]) == (ADIO_Offset)((MPIR_Upint)buf + (ADIO_Offset)j*(ADIO_Offset)buftype_extent + flat_buf->indices[i]));
00143 #ifdef ADIOI_MPE_LOGGING
00144             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
00145 #endif
00146             err = write(fd->fd_sys,
00147                      ((char *) buf) + (ADIO_Offset)j*(ADIO_Offset)buftype_extent + flat_buf->indices[i],
00148                      (unsigned)flat_buf->blocklens[i]);
00149 #ifdef ADIOI_MPE_LOGGING
00150             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
00151 #endif
00152             if (err == -1) err_flag = 1;
00153             off += flat_buf->blocklens[i]; /* keep up with the final file offset too */
00154         }
00155         else {
00156             /* copy more data into combine buffer */
00157             memcpy(combine_buf_ptr,
00158                ((char *) buf) + j*buftype_extent + flat_buf->indices[i],
00159                flat_buf->blocklens[i]);
00160             combine_buf_ptr += flat_buf->blocklens[i];
00161             combine_buf_remain -= flat_buf->blocklens[i];
00162             off += flat_buf->blocklens[i]; /* keep up with the final file offset too */
00163         }
00164         }
00165     }
00166 
00167     if (combine_buf_ptr != combine_buf) {
00168         /* data left in buffer to write */
00169 #ifdef IO_DEBUG
00170         printf("[%d/%d] nc mem c file (2) writing loc = %Ld sz = %Ld\n", 
00171                 rank, nprocs, off, 
00172                  fd->hints->ind_wr_buffer_size-combine_buf_remain);
00173 #endif
00174 #ifdef ADIOI_MPE_LOGGING
00175         MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
00176 #endif
00177         err = write(fd->fd_sys,
00178                  combine_buf,
00179                  fd->hints->ind_wr_buffer_size - combine_buf_remain);
00180 #ifdef ADIOI_MPE_LOGGING
00181         MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
00182 #endif
00183         if (err == -1) err_flag = 1;
00184     }
00185 
00186     if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
00187 
00188     ADIOI_Free(combine_buf);
00189 
00190     if (err_flag) {
00191         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00192                            MPIR_ERR_RECOVERABLE, myname,
00193                            __LINE__, MPI_ERR_IO, "**io",
00194                            "**io %s", strerror(errno));
00195     }
00196     else *error_code = MPI_SUCCESS;
00197     } /* if (!buftype_is_contig && filetype_is_contig)  ... */
00198 
00199     else {  /* noncontiguous in file */
00200 
00201 /* split up into several contiguous writes */
00202 
00203 /* find starting location in the file */
00204 
00205 /* filetype already flattened in ADIO_Open */
00206     flat_file = CtvAccess(ADIOI_Flatlist);
00207     while (flat_file->type != fd->filetype) flat_file = flat_file->next;
00208         disp = fd->disp;
00209 
00210     if (file_ptr_type == ADIO_INDIVIDUAL) {
00211         offset = fd->fp_ind; /* in bytes */
00212             n_filetypes = -1;
00213             flag = 0;
00214             while (!flag) {
00215                 int i;
00216                 n_filetypes++;
00217                 for (i=0; i<flat_file->count; i++) {
00218                     if (disp + flat_file->indices[i] + 
00219                         n_filetypes*(ADIO_Offset)filetype_extent + flat_file->blocklens[i] 
00220                             >= offset) {
00221                         st_index = i;
00222                         fwr_size = disp + flat_file->indices[i] + 
00223                                 n_filetypes*(ADIO_Offset)filetype_extent
00224                                  + flat_file->blocklens[i] - offset;
00225                         flag = 1;
00226                         break;
00227                     }
00228                 }
00229             }
00230     }
00231     else {
00232             int i;
00233         n_etypes_in_filetype = filetype_size/etype_size;
00234         n_filetypes = offset / n_etypes_in_filetype;
00235         etype_in_filetype = offset % n_etypes_in_filetype;
00236         size_in_filetype = etype_in_filetype * etype_size;
00237  
00238         sum = 0;
00239         for (i=0; i<flat_file->count; i++) {
00240         sum += flat_file->blocklens[i];
00241         if (sum > size_in_filetype) {
00242             st_index = i;
00243             fwr_size = sum - size_in_filetype;
00244             abs_off_in_filetype = flat_file->indices[i] +
00245             size_in_filetype - (sum - flat_file->blocklens[i]);
00246             break;
00247         }
00248         }
00249 
00250         /* abs. offset in bytes in the file */
00251             offset = disp + n_filetypes*(ADIO_Offset)filetype_extent + abs_off_in_filetype;
00252     }
00253 
00254     if (buftype_is_contig && !filetype_is_contig) {
00255 
00256 /* contiguous in memory, noncontiguous in file. should be the most
00257    common case. */
00258 
00259         i_offset = 0;
00260         j = st_index;
00261         off = offset;
00262         fwr_size = ADIOI_MIN(fwr_size, bufsize);
00263         while (i_offset < bufsize) {
00264                 if (fwr_size) { 
00265                     /* TYPE_UB and TYPE_LB can result in 
00266                        fwr_size = 0. save system call in such cases */ 
00267 #ifdef ADIOI_MPE_LOGGING
00268             MPE_Log_event(ADIOI_MPE_lseek_a, 0, NULL);
00269 #endif
00270 #ifdef IO_DEBUG
00271             printf("[%d/%d] c mem nc file writing loc = %Ld sz = %d\n", 
00272                 rank, nprocs, off, fwr_size);
00273 #endif
00274             err = lseek(fd->fd_sys, off, SEEK_SET);
00275 #ifdef ADIOI_MPE_LOGGING
00276             MPE_Log_event(ADIOI_MPE_lseek_b, 0, NULL);
00277 #endif
00278             if (err == -1) err_flag = 1;
00279 #ifdef ADIOI_MPE_LOGGING
00280             MPE_Log_event(ADIOI_MPE_write_a, 0, NULL);
00281 #endif
00282             err = write(fd->fd_sys, ((char *) buf) + i_offset, fwr_size);
00283 #ifdef ADIOI_MPE_LOGGING
00284             MPE_Log_event(ADIOI_MPE_write_b, 0, NULL);
00285 #endif
00286             if (err == -1) err_flag = 1;
00287         }
00288         i_offset += fwr_size;
00289 
00290                 if (off + fwr_size < disp + flat_file->indices[j] +
00291                    flat_file->blocklens[j] + n_filetypes*(ADIO_Offset)filetype_extent)
00292                        off += fwr_size;
00293                 /* did not reach end of contiguous block in filetype.
00294                    no more I/O needed. off is incremented by fwr_size. */
00295                 else {
00296             if (j < (flat_file->count - 1)) j++;
00297             else {
00298             j = 0;
00299             n_filetypes++;
00300             }
00301             off = disp + flat_file->indices[j] + 
00302                                         n_filetypes*(ADIO_Offset)filetype_extent;
00303             fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i_offset);
00304         }
00305         }
00306     }
00307     else {
00308 /* noncontiguous in memory as well as in file */
00309 
00310         ADIOI_Flatten_datatype(datatype);
00311         flat_buf = CtvAccess(ADIOI_Flatlist);
00312         while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00313 
00314         k = num = buf_count = 0;
00315         indx = flat_buf->indices[0];
00316         j = st_index;
00317         off = offset;
00318         bwr_size = flat_buf->blocklens[0];
00319 
00320         while (num < bufsize) {
00321         size = ADIOI_MIN(fwr_size, bwr_size);
00322         if (size) {
00323 #ifdef IO_DEBUG
00324             printf("[%d/%d] nc mem nc file writing loc = %Ld sz = %d\n", 
00325                     rank, nprocs, off, size);
00326 #endif
00327 #ifdef ADIOI_MPE_LOGGING
00328             MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
00329 #endif
00330             lseek(fd->fd_sys, off, SEEK_SET);
00331 #ifdef ADIOI_MPE_LOGGING 
00332             MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
00333 #endif
00334             if (err == -1) err_flag = 1;
00335 #ifdef ADIOI_MPE_LOGGING 
00336             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
00337 #endif
00338                     ADIOI_Assert(size == (size_t) size);
00339                     ADIOI_Assert(off == (off_t) off);
00340             err = write(fd->fd_sys, ((char *) buf) + indx, size);
00341 #ifdef ADIOI_MPE_LOGGING
00342             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
00343 #endif
00344             if (err == -1) err_flag = 1;
00345         }
00346 
00347         new_fwr_size = fwr_size;
00348         new_bwr_size = bwr_size;
00349 
00350         if (size == fwr_size) {
00351 /* reached end of contiguous block in file */
00352                     if (j < (flat_file->count - 1)) j++;
00353                     else {
00354                         j = 0;
00355                         n_filetypes++;
00356                     }
00357 
00358                     off = disp + flat_file->indices[j] + 
00359                                    n_filetypes*(ADIO_Offset)filetype_extent;
00360 
00361             new_fwr_size = flat_file->blocklens[j];
00362             if (size != bwr_size) {
00363             indx += size;
00364             new_bwr_size -= size;
00365             }
00366         }
00367 
00368         if (size == bwr_size) {
00369 /* reached end of contiguous block in memory */
00370 
00371             k = (k + 1)%flat_buf->count;
00372             buf_count++;
00373             indx = buftype_extent*(buf_count/flat_buf->count) +
00374             flat_buf->indices[k]; 
00375             new_bwr_size = flat_buf->blocklens[k];
00376             if (size != fwr_size) {
00377             off += size;
00378             new_fwr_size -= size;
00379             }
00380         }
00381         num += size;
00382         fwr_size = new_fwr_size;
00383                 bwr_size = new_bwr_size;
00384         }
00385     }
00386 
00387         if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
00388     if (err_flag) {
00389         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00390                            MPIR_ERR_RECOVERABLE, myname,
00391                            __LINE__, MPI_ERR_IO, "**io",
00392                            "**io %s", strerror(errno));
00393     }
00394     else *error_code = MPI_SUCCESS;
00395     }
00396 
00397     fd->fp_sys_posn = -1;   /* set it to null. */
00398 
00399 #ifdef HAVE_STATUS_SET_BYTES
00400     MPIR_Status_set_bytes(status, datatype, bufsize);
00401 /* This is a temporary way of filling in status. The right way is to 
00402    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
00403 #endif
00404 
00405     if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
00406 }
libs/ck-libs/ampi/romio/adio/common/ad_write_nolock.c
