Actual source code: fretrieve.c


  2: /*
  3:       Code for opening and closing files.
  4: */
  5: #include <petscsys.h>
  6: #if defined(PETSC_HAVE_PWD_H)
  7: #include <pwd.h>
  8: #endif
  9: #include <ctype.h>
 10: #include <sys/stat.h>
 11: #if defined(PETSC_HAVE_UNISTD_H)
 12: #include <unistd.h>
 13: #endif
 14: #if defined(PETSC_HAVE_SYS_UTSNAME_H)
 15: #include <sys/utsname.h>
 16: #endif
 17: #include <fcntl.h>
 18: #include <time.h>
 19: #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H)
 20: #include <sys/systeminfo.h>
 21: #endif

 23: /*
 24:    Private routine to delete tmp/shared storage

 26:    This is called by MPI, not by users.

 28:    Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()

 30: */
 31: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelTmpShared(MPI_Comm comm,PetscMPIInt keyval,void *count_val,void *extra_state)
 32: {

 36:   PetscInfo1(NULL,"Deleting tmp/shared data in an MPI_Comm %ld\n",(long)comm);
 37:   PetscFree(count_val);
 38:   PetscFunctionReturn(MPI_SUCCESS);
 39: }

 41: /*@C
 42:    PetscGetTmp - Gets the name of the tmp directory

 44:    Collective

 46:    Input Parameters:
 47: +  comm - MPI_Communicator that may share /tmp
 48: -  len - length of string to hold name

 50:    Output Parameters:
 51: .  dir - directory name

 53:    Options Database Keys:
 54: +    -shared_tmp  - indicates the directory is shared among the MPI ranks
 55: .    -not_shared_tmp - indicates the directory is not shared among the MPI ranks
 56: -    -tmp tmpdir - name of the directory you wish to use as /tmp

 58:    Environmental Variables:
 59: +     PETSC_SHARED_TMP - indicates the directory is shared among the MPI ranks
 60: .     PETSC_NOT_SHARED_TMP - indicates the directory is not shared among the MPI ranks
 61: -     PETSC_TMP - name of the directory you wish to use as /tmp

 63:    Level: developer


 66: @*/
 67: PetscErrorCode  PetscGetTmp(MPI_Comm comm,char dir[],size_t len)
 68: {
 70:   PetscBool      flg;

 73:   PetscOptionsGetenv(comm,"PETSC_TMP",dir,len,&flg);
 74:   if (!flg) {
 75:     PetscStrncpy(dir,"/tmp",len);
 76:   }
 77:   return(0);
 78: }

 80: /*@C
 81:    PetscSharedTmp - Determines if all processors in a communicator share a
 82:          /tmp or have different ones.

 84:    Collective

 86:    Input Parameters:
 87: .  comm - MPI_Communicator that may share /tmp

 89:    Output Parameters:
 90: .  shared - PETSC_TRUE or PETSC_FALSE

 92:    Options Database Keys:
 93: +    -shared_tmp  - indicates the directory is shared among the MPI ranks
 94: .    -not_shared_tmp - indicates the directory is not shared among the MPI ranks
 95: -    -tmp tmpdir - name of the directory you wish to use as /tmp

 97:    Environmental Variables:
 98: +     PETSC_SHARED_TMP  - indicates the directory is shared among the MPI ranks
 99: .     PETSC_NOT_SHARED_TMP - indicates the directory is not shared among the MPI ranks
100: -     PETSC_TMP - name of the directory you wish to use as /tmp

102:    Level: developer

104:    Notes:
105:    Stores the status as a MPI attribute so it does not have
106:     to be redetermined each time.

108:       Assumes that all processors in a communicator either
109:        1) have a common /tmp or
110:        2) each has a separate /tmp
111:       eventually we can write a fancier one that determines which processors
112:       share a common /tmp.

114:    This will be very slow on runs with a large number of processors since
115:    it requires O(p*p) file opens.

117:    If the environmental variable PETSC_TMP is set it will use this directory
118:   as the "/tmp" directory.

120: @*/
121: PetscErrorCode  PetscSharedTmp(MPI_Comm comm,PetscBool  *shared)
122: {
123:   PetscErrorCode     ierr;
124:   PetscMPIInt        size,rank,*tagvalp,sum,cnt,i;
125:   PetscBool          flg,iflg;
126:   FILE               *fd;
127:   static PetscMPIInt Petsc_Tmp_keyval = MPI_KEYVAL_INVALID;
128:   int                err;

131:   MPI_Comm_size(comm,&size);
132:   if (size == 1) {
133:     *shared = PETSC_TRUE;
134:     return(0);
135:   }

137:   PetscOptionsGetenv(comm,"PETSC_SHARED_TMP",NULL,0,&flg);
138:   if (flg) {
139:     *shared = PETSC_TRUE;
140:     return(0);
141:   }

143:   PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_TMP",NULL,0,&flg);
144:   if (flg) {
145:     *shared = PETSC_FALSE;
146:     return(0);
147:   }

149:   if (Petsc_Tmp_keyval == MPI_KEYVAL_INVALID) {
150:     MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_DelTmpShared,&Petsc_Tmp_keyval,NULL);
151:   }

153:   MPI_Comm_get_attr(comm,Petsc_Tmp_keyval,(void**)&tagvalp,(int*)&iflg);
154:   if (!iflg) {
155:     char filename[PETSC_MAX_PATH_LEN],tmpname[PETSC_MAX_PATH_LEN];

157:     /* This communicator does not yet have a shared tmp attribute */
158:     PetscMalloc1(1,&tagvalp);
159:     MPI_Comm_set_attr(comm,Petsc_Tmp_keyval,tagvalp);

161:     PetscOptionsGetenv(comm,"PETSC_TMP",tmpname,238,&iflg);
162:     if (!iflg) {
163:       PetscStrcpy(filename,"/tmp");
164:     } else {
165:       PetscStrcpy(filename,tmpname);
166:     }

168:     PetscStrcat(filename,"/petsctestshared");
169:     MPI_Comm_rank(comm,&rank);

171:     /* each processor creates a /tmp file and all the later ones check */
172:     /* this makes sure no subset of processors is shared */
173:     *shared = PETSC_FALSE;
174:     for (i=0; i<size-1; i++) {
175:       if (rank == i) {
176:         fd = fopen(filename,"w");
177:         if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open test file %s",filename);
178:         err = fclose(fd);
179:         if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
180:       }
181:       MPI_Barrier(comm);
182:       if (rank >= i) {
183:         fd = fopen(filename,"r");
184:         if (fd) cnt = 1;
185:         else cnt = 0;
186:         if (fd) {
187:           err = fclose(fd);
188:           if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
189:         }
190:       } else cnt = 0;

192:       MPIU_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
193:       if (rank == i) unlink(filename);

195:       if (sum == size) {
196:         *shared = PETSC_TRUE;
197:         break;
198:       } else if (sum != 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Subset of processes share /tmp ");
199:     }
200:     *tagvalp = (int)*shared;
201:     PetscInfo2(NULL,"processors %s %s\n",(*shared) ? "share":"do NOT share",(iflg ? tmpname:"/tmp"));
202:   } else *shared = (PetscBool) *tagvalp;
203:   return(0);
204: }

206: /*@C
207:    PetscSharedWorkingDirectory - Determines if all processors in a communicator share a
208:          working directory or have different ones.

210:    Collective

212:    Input Parameters:
213: .  comm - MPI_Communicator that may share working directory

215:    Output Parameters:
216: .  shared - PETSC_TRUE or PETSC_FALSE

218:    Options Database Keys:
219: +    -shared_working_directory - indicates the directory is shared among the MPI ranks
220: -    -not_shared_working_directory - indicates the directory is shared among the MPI ranks

222:    Environmental Variables:
223: +     PETSC_SHARED_WORKING_DIRECTORY - indicates the directory is shared among the MPI ranks
224: -     PETSC_NOT_SHARED_WORKING_DIRECTORY - indicates the directory is shared among the MPI ranks

226:    Level: developer

228:    Notes:
229:    Stores the status as a MPI attribute so it does not have
230:     to be redetermined each time.

232:       Assumes that all processors in a communicator either
233:        1) have a common working directory or
234:        2) each has a separate working directory
235:       eventually we can write a fancier one that determines which processors
236:       share a common working directory.

238:    This will be very slow on runs with a large number of processors since
239:    it requires O(p*p) file opens.

241: @*/
242: PetscErrorCode  PetscSharedWorkingDirectory(MPI_Comm comm,PetscBool  *shared)
243: {
244:   PetscErrorCode     ierr;
245:   PetscMPIInt        size,rank,*tagvalp,sum,cnt,i;
246:   PetscBool          flg,iflg;
247:   FILE               *fd;
248:   static PetscMPIInt Petsc_WD_keyval = MPI_KEYVAL_INVALID;
249:   int                err;

252:   MPI_Comm_size(comm,&size);
253:   if (size == 1) {
254:     *shared = PETSC_TRUE;
255:     return(0);
256:   }

258:   PetscOptionsGetenv(comm,"PETSC_SHARED_WORKING_DIRECTORY",NULL,0,&flg);
259:   if (flg) {
260:     *shared = PETSC_TRUE;
261:     return(0);
262:   }

264:   PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_WORKING_DIRECTORY",NULL,0,&flg);
265:   if (flg) {
266:     *shared = PETSC_FALSE;
267:     return(0);
268:   }

270:   if (Petsc_WD_keyval == MPI_KEYVAL_INVALID) {
271:     MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_DelTmpShared,&Petsc_WD_keyval,NULL);
272:   }

274:   MPI_Comm_get_attr(comm,Petsc_WD_keyval,(void**)&tagvalp,(int*)&iflg);
275:   if (!iflg) {
276:     char filename[PETSC_MAX_PATH_LEN];

278:     /* This communicator does not yet have a shared  attribute */
279:     PetscMalloc1(1,&tagvalp);
280:     MPI_Comm_set_attr(comm,Petsc_WD_keyval,tagvalp);

282:     PetscGetWorkingDirectory(filename,240);
283:     PetscStrcat(filename,"/petsctestshared");
284:     MPI_Comm_rank(comm,&rank);

286:     /* each processor creates a  file and all the later ones check */
287:     /* this makes sure no subset of processors is shared */
288:     *shared = PETSC_FALSE;
289:     for (i=0; i<size-1; i++) {
290:       if (rank == i) {
291:         fd = fopen(filename,"w");
292:         if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open test file %s",filename);
293:         err = fclose(fd);
294:         if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
295:       }
296:       MPI_Barrier(comm);
297:       if (rank >= i) {
298:         fd = fopen(filename,"r");
299:         if (fd) cnt = 1;
300:         else cnt = 0;
301:         if (fd) {
302:           err = fclose(fd);
303:           if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
304:         }
305:       } else cnt = 0;

307:       MPIU_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
308:       if (rank == i) unlink(filename);

310:       if (sum == size) {
311:         *shared = PETSC_TRUE;
312:         break;
313:       } else if (sum != 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Subset of processes share working directory");
314:     }
315:     *tagvalp = (int)*shared;
316:   } else *shared = (PetscBool) *tagvalp;
317:   PetscInfo1(NULL,"processors %s working directory\n",(*shared) ? "shared" : "do NOT share");
318:   return(0);
319: }


322: /*@C
323:     PetscFileRetrieve - Obtains a file from a URL or compressed
324:         and copies into local disk space as uncompressed.

326:     Collective

328:     Input Parameter:
329: +   comm     - processors accessing the file
330: .   url      - name of file, including entire URL (with or without .gz)
331: -   llen     - length of localname

333:     Output Parameter:
334: +   localname - name of local copy of file - valid on only process zero
335: -   found - if found or retrieved the file - valid on all processes

337:     Notes:
338:     if the file already exists local this function just returns without downloading it.

340:     Level: intermediate
341: @*/
342: PetscErrorCode  PetscFileRetrieve(MPI_Comm comm,const char url[],char localname[],size_t llen,PetscBool  *found)
343: {
344:   char           buffer[PETSC_MAX_PATH_LEN],*par,*tlocalname,name[PETSC_MAX_PATH_LEN];
345:   FILE           *fp;
347:   PetscMPIInt    rank;
348:   size_t         len = 0;
349:   PetscBool      flg1,flg2,flg3,flg4,download,compressed = PETSC_FALSE;

352:   MPI_Comm_rank(comm,&rank);
353:   if (!rank) {
354:     *found = PETSC_FALSE;

356:     PetscStrstr(url,".gz",&par);
357:     if (par) {
358:       PetscStrlen(par,&len);
359:       if (len == 3) compressed = PETSC_TRUE;
360:     }

362:     PetscStrncmp(url,"ftp://",6,&flg1);
363:     PetscStrncmp(url,"http://",7,&flg2);
364:     PetscStrncmp(url,"file://",7,&flg3);
365:     PetscStrncmp(url,"https://",8,&flg4);
366:     download = (PetscBool) (flg1 || flg2 || flg3 || flg4);

368:     if (!download && !compressed) {
369:       PetscStrncpy(localname,url,llen);
370:       PetscTestFile(url,'r',found);
371:       if (*found) {
372:         PetscInfo1(NULL,"Found file %s\n",url);
373:       } else {
374:         PetscInfo1(NULL,"Did not find file %s\n",url);
375:       }
376:       goto done;
377:     }

379:     /* look for uncompressed file in requested directory */
380:     if (compressed) {
381:       PetscStrncpy(localname,url,llen);
382:       PetscStrstr(localname,".gz",&par);
383:       *par = 0; /* remove .gz extension */
384:       PetscTestFile(localname,'r',found);
385:       if (*found) goto done;
386:     }

388:     /* look for file in current directory */
389:     PetscStrrchr(url,'/',&tlocalname);
390:     PetscStrncpy(localname,tlocalname,llen);
391:     if (compressed) {
392:       PetscStrstr(localname,".gz",&par);
393:       *par = 0; /* remove .gz extension */
394:     }
395:     PetscTestFile(localname,'r',found);
396:     if (*found) goto done;

398:     if (download) {
399:       /* local file is not already here so use curl to get it */
400:       PetscStrncpy(localname,tlocalname,llen);
401:       PetscStrcpy(buffer,"curl --fail --silent --show-error ");
402:       PetscStrcat(buffer,url);
403:       PetscStrcat(buffer," > ");
404:       PetscStrcat(buffer,localname);
405: #if defined(PETSC_HAVE_POPEN)
406:       PetscPOpen(PETSC_COMM_SELF,NULL,buffer,"r",&fp);
407:       PetscPClose(PETSC_COMM_SELF,fp);
408: #else
409:       SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Cannot run external programs on this machine");
410: #endif
411:       PetscTestFile(localname,'r',found);
412:       if (*found) {
413:         FILE      *fd;
414:         char      buf[1024],*str,*substring;

416:         /* check if the file didn't exist so it downloaded an HTML message instead */
417:         fd = fopen(localname,"r");
418:         if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"PetscTestFile() indicates %s exists but fopen() cannot open it",localname);
419:         str = fgets(buf,sizeof(buf)-1,fd);
420:         while (str) {
421:           PetscStrstr(buf,"<!DOCTYPE html>",&substring);
422:           if (substring) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded",url);
423:           PetscStrstr(buf,"Not Found",&substring);
424:           if (substring) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded",url);
425:           str = fgets(buf,sizeof(buf)-1,fd);
426:         }
427:         fclose(fd);
428:       }
429:     } else if (compressed) {
430:       PetscTestFile(url,'r',found);
431:       if (!*found) goto done;
432:       PetscStrncpy(localname,url,llen);
433:     }
434:     if (compressed) {
435:       PetscStrrchr(localname,'/',&tlocalname);
436:       PetscStrncpy(name,tlocalname,PETSC_MAX_PATH_LEN);
437:       PetscStrstr(name,".gz",&par);
438:       *par = 0; /* remove .gz extension */
439:       /* uncompress file */
440:       PetscStrcpy(buffer,"gzip -c -d ");
441:       PetscStrcat(buffer,localname);
442:       PetscStrcat(buffer," > ");
443:       PetscStrcat(buffer,name);
444: #if defined(PETSC_HAVE_POPEN)
445:       PetscPOpen(PETSC_COMM_SELF,NULL,buffer,"r",&fp);
446:       PetscPClose(PETSC_COMM_SELF,fp);
447: #else
448:       SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Cannot run external programs on this machine");
449: #endif
450:       PetscStrncpy(localname,name,llen);
451:       PetscTestFile(localname,'r',found);
452:     }
453:   }
454:   done:
455:   MPI_Bcast(found,1,MPIU_BOOL,0,comm);
456:   MPI_Bcast(localname, llen, MPI_CHAR, 0, comm);
457:   return(0);
458: }