/*===================================================================== * DD SYSTEM base package * DD_Server library * ncfileop.c * V.6.7 * Last revision: * Oct 24, 2002 New calculation of the maximum record size V 4.1 * work with NC files is now in file ncfileop.c * Nov 19, 2004 Bug in the cach operation. * Aug 16, 2005 Cach improved * Mar 15, 2007 V.4.7 Some errors with cash * Jul 10, 2007 V.5.0 According to Server protocol 4.5.1 Call external database if need. * If "times" has "NODATA" string as name of the file, return "NOTIME" * Sep 22, 2007 V.6.0 New approach. It tries to get new data between two files * Sep 25, 2007 V.6.1 Variable Minimal Gap * May 06, 2008 V.6.2 Fedorov, Max Records number processing * May 12, 2008 V.6.3 Fedorov, In Max Records error in malloc sizeof(int *)!!! important for 64 * May 15, 2008 V.6.4 Fedorov, NoData interval processing. Just go to the next correct data interval. * Apr 08, 2009 V.6.5 Fedorov, Unziped files can be touched by groupe * Apr 14, 2009 V.6.6 Fedorov, nc_sync * Feb 02, 2010 V.6.7 Lena, wait to remove an old file in the cache *======================================================================*/ #include #include "DD.h" #include "DD_comm.h" /*------------------ GLOBAL VARIABLES --------------------------*/ extern int Verbose; /*============================================================= * CloseOldFile *===========================================================*/ int CloseOldFile(DD_Var_t *D) { size_t CashStart[2] = {0L,0L}; size_t CashCount[2] = {1L,MAXFILENAME}; int status; if(D->Maxnc_rec > 0) { status = nc_close(D->ncID); if(status < 0) { D->LastFileStatus = DATAFILEERR; if(Verbose) fprintf(stderr,"CloseOldFile: file %s, error while closed\n",&(D->Cash.names[D->CurrCushN][0]),D->CurrCushN); return DATAFILEERR; } if(Verbose) fprintf(stderr,"CloseOldFile: file %s, %d closed\n",&(D->Cash.names[D->CurrCushN][0]),D->CurrCushN); D->CurrCushN = -1; D->Maxnc_rec = 0; } else if(Verbose) fprintf(stderr,"CloseOldFile: Nothing To close\n"); return 1; } /*##############################################################*/ /*-----------------------SETNEWFILE-----------------------------*/ /*##############################################################*/ int SetNewFile(DD_Var_t *D, int N) /* * Function tries to get new data for the gap beetween files, or just * open a new data file according to offset N referring to current position in the times file. * The current position changed if file is succesefully open. * Return values: * OK * OUTOFTIME - next interval exceeeds GlobalStart/Stop * TIMEINEMPTY - next interval marked as "NODATA" * TRYAGAIN - the VI is blocked while new data is arriving * WAITEXTCALL - new data is requested * CACHTOOREC - now free space in the CACH * CHACHERR - unrecovable error in CACH * DATAFILEERR - unrecovable error in data file *---------------------------------------------------------------- * At the entry we are assuming that VI is not locked! */ { static char TimeDimName[] = "Time"; static size_t CashStart[2] = {0L,0L}; static size_t CashCount[2] = {1L,MAXFILENAME}; static size_t TimeCount[2] = {1,TIMELENGTH}; char Name[MAXFILENAME]; /* Name of data nc file */ char FullName[PATHLENGTH]; /* Full name of nc file */ char StartTime[TIMELENGTH]; char StopTime[TIMELENGTH]; static int NameID,TimeDimID; /* NC variables ID for Times file */ static size_t start[2] = {0,0}; /* NV start array */ static size_t FileCount[2] = {1,MAXFILENAME}; /* NC count array for Times file */ static size_t DimArray[NC_MAX_DIMS]; /* NC_MAX_DIMS is in netcdf.h */ static int DimIDArray[NC_MAX_DIMS]; size_t TestNumber; /* To test the new file number */ int FileNumber, Find, OldestNumber, NewAttempt; char command[300]; int dltt; int status; time_t CurrentTime; double FirstTime, LastTime; int More; /*--------------- Calculate the next file to open ---------------*/ TestNumber = D->TimeRecNumber + N; if(Verbose) fprintf(stderr,"SetNewFile(): Record %d\n",TestNumber); D->TimeRecNumber = TestNumber; start[0] = TestNumber; /*----------------------------------------------------------------* * Some test and call external archive we have to make only if N > 0 * Otherwise we just reopen existing file *----------------------------------------------------------------*/ if(N > 0) { if(TestNumber > D->RValidMax) /* Out of existing data files */ { if(Verbose) fprintf(stderr,"SetNewFile(): TestNumber %d exides D->RValidMax %d\n",TestNumber,D->RValidMax); if(D->ExtCallAllowed) { More = 1; while(More) { switch(status = ExtDataRequest(D, D->CDTime+D->MinGap)) { case OK: return WAITEXTCALL; case NODATAATTIME: /* The last files in the data base are only NODATA, let us try again with StopTime of TestNumber */ start[0] = TestNumber; status = nc_get_vara_text(D->tmID, D->StopID,start,TimeCount,StopTime); D->CDTime = DD_Time2Double(StopTime) + 0.001; if(Verbose) fprintf(stderr,"SetNewFile(): We found no data, Let us try another CDTime = %s\n",Double2DD_Time(D->CDTime)); More = 1; break; case GAPISSMALL: return TIMEINEMPTY; break; case OUTOFTIME: return OUTOFTIME; break; default: return DATAFILEERR; } } } else return OUTOFTIME; } } /*--------------- Get File Name and start Time ---------------------*/ status = nc_get_vara_text(D->tmID,D->NameID,start,FileCount,Name); status = nc_get_vara_text(D->tmID, D->StartID,start,TimeCount,StartTime); if(status != NC_NOERR) { if(Verbose) fprintf(stderr,"SetNewFile(): Get File Name and StartTime %s\n",nc_strerror(status)); D->LastFileStatus = DATAFILEERR; return DATAFILEERR; } FirstTime = DD_Time2Double(StartTime); if(N > 0) { if(strncmp(Name,"NODATA",6) == 0) /* Next record marked as NODATA */ { if(D->ExtCallAllowed && (FirstTime - D->CDTime > D->MinGap)) /* Try to fill the gap between the data * and NODATA segment if there is enough gap */ { if(Verbose) fprintf(stderr,"SetNewFile(): There is a gap %d\n",FirstTime - D->CDTime); switch(status = ExtDataRequest(D, D->CDTime+D->MinGap)) { case OK: return WAITEXTCALL; case NODATAATTIME: case GAPISSMALL: return TIMEINEMPTY; break; case OUTOFTIME: return OUTOFTIME; break; default: return DATAFILEERR; } } else /* Gap to NODATA interval is too small and we wiil try next time interval */ { if(Verbose) fprintf(stderr,"SetNewFile(): Next File is NODATA, try file %d\n",++TestNumber); status = nc_get_vara_text(D->tmID, D->StopID,start,TimeCount,StopTime); D->CDTime = DD_Time2Double(StopTime); switch( status = SetNewFile(D,1) ) { case OK: return OK; case WAITEXTCALL: return WAITEXTCALL; case TIMEINEMPTY: return TIMEINEMPTY; case OUTOFTIME: return OUTOFTIME; case CACHERR: return CACHERR; default: return DATAFILEERR; } } } else /* Next record is valid */ { // sleep(40); if(D->ExtCallAllowed && ((FirstTime - D->CDTime) > D->MinGap)) { if(Verbose) fprintf(stderr,"SetNewFile(): File OK, but there is a gap %d %d\n",FirstTime - D->CDTime, D->MinGap); switch(status = ExtDataRequest(D, D->CDTime+D->MinGap)) { case OK: return WAITEXTCALL; case NODATAATTIME: case GAPISSMALL: return TIMEINEMPTY; break; case OUTOFTIME: return OUTOFTIME; break; default: return DATAFILEERR; } } /* else just continue */ } } /*-------------- refresh the Cache table ---------------------------*/ status = nc_sync(D->Cash.ID); if(status != NC_NOERR) { if(Verbose) fprintf(stderr,"SetNewFile(): Cache Synchro: %s\n",nc_strerror(status)); return CACHERR; } for(CashStart[0] = 0; CashStart[0] < CASHLEN; CashStart[0]++) { status = nc_get_vara_text(D->Cash.ID, D->Cash.nameID,CashStart,CashCount,D->Cash.names[CashStart[0]]); if(status != NC_NOERR) { if(Verbose) fprintf(stderr,"SetNewFile(): Cache Names: %s\n",nc_strerror(status)); D->LastFileStatus = CACHERR; return CACHERR; } status = nc_get_vara_long(D->Cash.ID, D->Cash.timeID,CashStart,CashCount,&(D->Cash.times[CashStart[0]])); if(status != NC_NOERR) { if(Verbose) fprintf(stderr,"SetNewFile(): Cache Times %s\n",nc_strerror(status)); D->LastFileStatus = CACHERR; return CACHERR; } } /*----------- Close the old file ----------------------*/ if((status = CloseOldFile(D)) < 0) return DATAFILEERR; strcpy(FullName,D->path); strcpy(FullName+strlen(D->path), Name); // if(Verbose) fprintf(stderr,"SetNewFile(): New file to open: %s\n",FullName); NewAttempt = 0; while (NewAttempt < 3) { /*----- Search the new requested file in the cache and the oldest file ----------*/ FileNumber = 0; CurrentTime = time(NULL); Find = 0; OldestNumber = -1; while((FileNumber < CASHLEN) && // Cache is finished ((Find = strcmp(&(D->Cash.names[FileNumber][0]),Name)) != 0) && // File is already in the Cache (D->Cash.names[FileNumber][0] != ' ')) // There is empty space { if((D->Cash.times[FileNumber] < (CurrentTime - FILEACCMARG) ) && (OldestNumber == -1)) OldestNumber = FileNumber; dltt = (int)(D->Cash.times[OldestNumber] - D->Cash.times[FileNumber]); if(dltt > 0) OldestNumber = FileNumber; FileNumber++; } /* ==================================================== * Sometimes it is possible, that: * 1) No corresponding file in the cache (Find != 0) * 2) The oldest file is not found (OldestNumber == -1) * 3) cach is full (FileNumber == CASHLEN) * Then we need to repeat the search after one sec delay *==================================================== */ if((Find != 0) && (FileNumber == CASHLEN) && (OldestNumber == -1)) { NewAttempt++; sleep((unsigned )(FILEACCMARG)); if (Verbose) fprintf(stderr,"Waiting %d secs to get the Oldest File\n", FILEACCMARG); } else NewAttempt = 3; } //---------- Parsing resultat of search ------------- // if(Verbose) // { // if(OldestNumber > -1) fprintf(stderr,"SetNewFile(): Search: Is Oldest File %d %d %d\n", OldestNumber,D->Cash.times[OldestNumber],CurrentTime); // else fprintf(stderr,"SetNewFile(): Search: No Oldest File %d %d\n", OldestNumber,CurrentTime); // } if(Find != 0) /*---------------- No request file in the CACHE ----------------------*/ { if(FileNumber < CASHLEN) { if(D->Cash.names[FileNumber][0] == ' ') /* There is empty space in the table */ { strcpy(command, "gunzip -c "); strcat(command, FullName); strcat(command, ".gz > "); strcat(command, FullName); system(command); /* File is unzipped */ sprintf(command, "chmod g+w %s\0",FullName); system(command); /* File is unzipped */ strcpy(&(D->Cash.names[FileNumber][0]),Name); D->Cash.times[FileNumber] = CurrentTime; CashStart[0] = FileNumber; D->CurrCushN = FileNumber; } else { D->LastFileStatus = CACHERR; return CACHERR; /* Unrecoverable error */ } } else /* No empty space. It is necessury to remove one file */ { if (OldestNumber > -1) //------ Oldest file is found ------- { strcpy(command, "gunzip -c "); strcat(command, FullName); strcat(command, ".gz > "); strcat(command, FullName); system(command); /* File is unzipped */ sprintf(command, "chmod g+w %s\0",FullName); system(command); /* File is unzipped */ strcpy(command, "rm "); strcat(command,D->path); strcat(command,&(D->Cash.names[OldestNumber][0])); strcat(command, " &"); system(command); /* Old file removed */ strcpy(&(D->Cash.names[OldestNumber][0]),Name); D->Cash.times[OldestNumber] = CurrentTime; CashStart[0] = OldestNumber; D->CurrCushN = OldestNumber; } else { if(Verbose) fprintf(stderr,"SetNewFile(): return CACHTOOREC\n"); D->LastFileStatus = CACHTOOREC; D->TimeRecNumber -= N; return CACHTOOREC; //----- Say client that all files are too new } } /*------------ The place for new cache is found --------------*/ status = nc_put_vara_text(D->Cash.ID, D->Cash.nameID,CashStart,CashCount,&(D->Cash.names[CashStart[0]][0])); status = nc_put_vara_long(D->Cash.ID, D->Cash.timeID,CashStart,CashCount,&(D->Cash.times[CashStart[0]])); if(status != NC_NOERR) { if(Verbose) fprintf(stderr,"SetNewFile(): Write Name and Time to cash: %s\n",nc_strerror(status)); D->LastFileStatus = CACHERR; return CACHERR; } status = nc_sync(D->Cash.ID); } //------------------ There is FILE already in the CACHE -------------- else /*------- The requested file is already in the cache just refresh the time */ { CashStart[0] = FileNumber; D->Cash.times[FileNumber] = CurrentTime; /* refresh the time */ D->CurrCushN = FileNumber; status = nc_put_vara_long(D->Cash.ID, D->Cash.timeID,CashStart,CashCount,&(D->Cash.times[CashStart[0]])); if(status != NC_NOERR) { if(Verbose) fprintf(stderr,"SetNewFile(): File In Cache: %s\n",nc_strerror(status)); D->LastFileStatus = CACHERR; return CACHERR; } status = nc_sync(D->Cash.ID); } // if(Verbose) fprintf(stderr,"SetNewFile(): file %s, %d to be open\n",FullName,D->CurrCushN); /*--------------- CACHE refreshed and requested file is unzipped ------------------------*/ /*----------------- Open requested file -----------------------------------------------*/ status = nc_open(FullName,NC_NOWRITE,&(D->ncID)); if(status != NC_NOERR) { if(Verbose) fprintf(stderr,"SetNewFile(): Error while File Open: %s\n",nc_strerror(status)); D->LastFileStatus =DATAFILEERR ; return DATAFILEERR; } if(Verbose) fprintf(stderr,"SetNewFile(): %s is open\n",FullName); /* Recover time dimension */ status = nc_inq_dimid(D->ncID,TimeDimName,&TimeDimID); if(status != NC_NOERR) { if(Verbose) fprintf(stderr,"SetNewFile(): Get Time Dim: %s\n",nc_strerror(status)); D->LastFileStatus = DATAFILEERR; return DATAFILEERR; } status = nc_inq_dimlen(D->ncID,TimeDimID,&(D->Maxnc_rec)); if(status != NC_NOERR) { if(Verbose) fprintf(stderr,"SetNewFile: GetFile Length: %s\n",nc_strerror(status)); D->LastFileStatus = DATAFILEERR; return DATAFILEERR; } /* Setup current position in the new open file */ if(N > 0) D->nc_rec = 0; if(N < 0) D->nc_rec = D->Maxnc_rec-1; D->LastFileStatus = OK; return OK; } /*------------------------------------------------------------------------------------*/ /*----------------- VarSize -----------------------------------*/ int VarSize(nc_type type) { switch(type) { case NC_BYTE: return sizeof(char); case NC_CHAR: return sizeof(char); case NC_SHORT: return sizeof(short); case NC_INT: return sizeof(int); case NC_FLOAT: return sizeof(float); case NC_DOUBLE: return sizeof(double); default: return 1; } } /*##############################################################*/ /*----------------------- MaxRecord -----------------------------*/ /*##############################################################*/ size_t MaxRecord(int ncID) /* ncID - ID of already open NC file */ { int timedimid; /* ID for dimension "Time" */ int nvars; int ndims; int *ndims_arr; /* Array to keep number of dimensions of each variable */ size_t *dimleng_arr; /* Array to keep all length of dimensions */ nc_type *vartype_arr; /* Array to keep all types of variables */ int **dimid_arr; /* array of pointers to arrays of variable dimensions */ int status; /* Error indicator */ int iv, id; int cvarsize; /* size of variable */ int maxsize; /* maximal size */ int recordsize; /* size by units */ size_t RecordNumber; /*--- general information -------*/ status = nc_inq_ndims(ncID,&ndims); status = nc_inq_nvars(ncID,&nvars); status = nc_inq_unlimdim(ncID, &timedimid); /* What is unlimited dimension ID */ /*---- memory allocation --------*/ dimleng_arr = (size_t *)malloc(ndims * sizeof(size_t)); vartype_arr = (nc_type *)malloc(nvars * sizeof(nc_type)); dimid_arr = (int **)malloc(nvars * sizeof(int *)); ndims_arr = (int *)malloc(nvars * sizeof(int)); /* Length of each dimension */ for(id = 0; id < ndims; id++) { if(id == timedimid) dimleng_arr[id] = 1; else status = nc_inq_dimlen(ncID, id, &(dimleng_arr[id])); } /* type and dimension of each variable */ for(iv = 0; iv < nvars; iv++) { status = nc_inq_vartype(ncID,iv,&(vartype_arr[iv])); status = nc_inq_varndims(ncID,iv,&(ndims_arr[iv])); dimid_arr[iv] = (int *)malloc(ndims_arr[iv]*sizeof(int)); /* memory for dim ID array */ status = nc_inq_vardimid(ncID,iv,dimid_arr[iv]); } /* calculation of size of variable and maximum size of variables */ maxsize = 0; for(iv = 0; iv < nvars; iv++) { recordsize = 1; for(id = 0; id < ndims_arr[iv]; id++) recordsize *= dimleng_arr[(dimid_arr[iv])[id]]; cvarsize = recordsize * VarSize(vartype_arr[iv]); if(cvarsize > maxsize) maxsize = cvarsize; } /* calculation record number */ RecordNumber = (size_t)(MAXPACKSIZE / maxsize); if(RecordNumber == 0) { if(Verbose) fprintf(stderr,"MaxRecord(%d): Error in calculation of records number,MAXPACKSIZE = %d, maxsize = %d\n",ncID,MAXPACKSIZE,maxsize); } /* Free all */ for(iv = 0; iv < nvars; iv++) free(dimid_arr[iv]); free(ndims_arr); free(dimid_arr); free(vartype_arr); free(dimleng_arr); return RecordNumber; } /*----------------------------------------------------------------------*/