/*=====================================================================
 *                DD SYSTEM base package
 *                  DD_Server library
 *                    ncfileop.c
 *                      V.6.7
 * Last revision:
 *      Oct 24, 2002  New calculation of the maximum record size V 4.1
 *      work with NC files is now in file ncfileop.c
 *      Nov 19, 2004 Bug in the cach operation.
 *      Aug 16, 2005 Cach improved
 *      Mar 15, 2007 V.4.7 Some errors with cash
 *      Jul 10, 2007 V.5.0 According to Server protocol 4.5.1 Call external database if need.
 *                         If "times" has "NODATA" string as name of the file, return "NOTIME"
 *      Sep 22, 2007 V.6.0 New approach. It tries to get new data between two files
 *      Sep 25, 2007 V.6.1 Variable Minimal Gap
 *      May 06, 2008 V.6.2 Fedorov, Max Records number processing
 *      May 12, 2008 V.6.3 Fedorov, In Max Records error in malloc sizeof(int *)!!! important for 64
 *      May 15, 2008 V.6.4 Fedorov, NoData interval processing. Just go to the next correct data interval.
 *      Apr 08, 2009 V.6.5 Fedorov, Unziped files can be touched by groupe
 *      Apr 14, 2009 V.6.6 Fedorov, nc_sync
 *      Feb 02, 2010 V.6.7 Lena, wait to remove an old file in the cache
 *======================================================================*/
#include <netcdf.h>
#include "DD.h"
#include "DD_comm.h"

/*------------------ GLOBAL VARIABLES --------------------------*/
extern int Verbose;

/*=============================================================
 *                       CloseOldFile
 *===========================================================*/
int CloseOldFile(DD_Var_t *D)
{
    size_t CashStart[2] = {0L,0L};
    size_t CashCount[2] = {1L,MAXFILENAME};
    int status;

    if(D->Maxnc_rec > 0)
    {
       status = nc_close(D->ncID);
       if(status < 0) 
       {
          D->LastFileStatus = DATAFILEERR;
          if(Verbose) fprintf(stderr,"CloseOldFile: file %s, error while closed\n",&(D->Cash.names[D->CurrCushN][0]),D->CurrCushN);
          return DATAFILEERR;
       }
       if(Verbose) fprintf(stderr,"CloseOldFile: file %s, %d closed\n",&(D->Cash.names[D->CurrCushN][0]),D->CurrCushN);
       D->CurrCushN = -1;
       D->Maxnc_rec = 0;
    }
    else if(Verbose) fprintf(stderr,"CloseOldFile: Nothing To close\n");

    return 1;
}

/*##############################################################*/
/*-----------------------SETNEWFILE-----------------------------*/
/*##############################################################*/

int SetNewFile(DD_Var_t *D, int N)
/*
 * Function tries to get new data for the gap beetween files, or just 
 * open a new data file according to offset N referring to current position in the times file.
 * The current position changed if file is succesefully open.
 * Return values: 
 * OK 
 * OUTOFTIME - next interval exceeeds GlobalStart/Stop
 * TIMEINEMPTY - next interval marked as "NODATA"
 * TRYAGAIN - the VI is blocked while new data is arriving
 * WAITEXTCALL - new data is requested
 * CACHTOOREC - now free space in the CACH
 * CHACHERR - unrecovable error in CACH
 * DATAFILEERR - unrecovable error in data file
 *----------------------------------------------------------------
 * At the entry we are assuming that VI is not locked!
 */
{
  static char TimeDimName[] = "Time";
  static size_t CashStart[2] = {0L,0L};
  static size_t CashCount[2] = {1L,MAXFILENAME};
  static size_t TimeCount[2] = {1,TIMELENGTH};

  char Name[MAXFILENAME];               /* Name of data nc file */
  char FullName[PATHLENGTH];            /* Full name of nc file */
  char StartTime[TIMELENGTH];
  char StopTime[TIMELENGTH];
  static int NameID,TimeDimID;               /* NC variables ID for Times file */
  static size_t start[2] = {0,0};                   /* NV start array */
  static size_t FileCount[2] = {1,MAXFILENAME}; /* NC count array for Times file */
  static size_t DimArray[NC_MAX_DIMS];  /* NC_MAX_DIMS is in netcdf.h */
  static int DimIDArray[NC_MAX_DIMS];
  size_t TestNumber;                                /* To test the new file number  */
  int FileNumber, Find, OldestNumber, NewAttempt;
  char command[300];
  int dltt;
  int status;
  time_t CurrentTime;
  double FirstTime, LastTime;
  int More;

/*--------------- Calculate the next file to open ---------------*/
  TestNumber = D->TimeRecNumber + N;
  if(Verbose) fprintf(stderr,"SetNewFile(): Record %d\n",TestNumber);

  D->TimeRecNumber = TestNumber;
  start[0] = TestNumber;
  
/*----------------------------------------------------------------*
 *  Some test and call external archive we have to make only if  N > 0 
 *  Otherwise we just reopen existing file
 *----------------------------------------------------------------*/
  if(N > 0)
  {
     if(TestNumber > D->RValidMax) /* Out of existing data files */
     {
        if(Verbose) fprintf(stderr,"SetNewFile(): TestNumber %d exides D->RValidMax %d\n",TestNumber,D->RValidMax);

        if(D->ExtCallAllowed) 
        {
           More = 1;
           while(More)
           {
            switch(status = ExtDataRequest(D, D->CDTime+D->MinGap))
            {
                case OK: return WAITEXTCALL;
                case NODATAATTIME: /* The last files in the data base are only NODATA, let us try again with StopTime of  TestNumber */
                  start[0] = TestNumber;
                  status = nc_get_vara_text(D->tmID, D->StopID,start,TimeCount,StopTime);
                  D->CDTime = DD_Time2Double(StopTime) + 0.001;
                  if(Verbose) fprintf(stderr,"SetNewFile(): We found no data, Let us try another CDTime = %s\n",Double2DD_Time(D->CDTime));
                  More = 1;
                  break;
                case GAPISSMALL: return TIMEINEMPTY; break;
                case OUTOFTIME: return OUTOFTIME; break;
                default: return DATAFILEERR;
            }
           }
        } else return OUTOFTIME;
     }
  }
  
  /*--------------- Get File Name and start Time  ---------------------*/
  status = nc_get_vara_text(D->tmID,D->NameID,start,FileCount,Name);
  status = nc_get_vara_text(D->tmID, D->StartID,start,TimeCount,StartTime);
  if(status != NC_NOERR)
  {          
     if(Verbose) fprintf(stderr,"SetNewFile(): Get File Name and StartTime  %s\n",nc_strerror(status));
     D->LastFileStatus = DATAFILEERR;
     return DATAFILEERR; 
  }
  FirstTime = DD_Time2Double(StartTime);
  
  if(N > 0)
  {
     if(strncmp(Name,"NODATA",6) == 0) /* Next record marked as NODATA */
     {
        if(D->ExtCallAllowed && (FirstTime - D->CDTime > D->MinGap)) /* Try to fill the gap between the data 
                                                                      * and NODATA segment if there is enough gap
                                                                      */
        {
           if(Verbose) fprintf(stderr,"SetNewFile(): There is a gap  %d\n",FirstTime - D->CDTime);
           switch(status = ExtDataRequest(D, D->CDTime+D->MinGap))
           {
              case OK: return WAITEXTCALL;
              case NODATAATTIME:
              case GAPISSMALL: return TIMEINEMPTY; break;
              case OUTOFTIME: return OUTOFTIME; break;
              default: return DATAFILEERR;
           }
        } 
        else /* Gap to NODATA interval is too small and we wiil try next time interval */
        {
           if(Verbose) fprintf(stderr,"SetNewFile(): Next File is NODATA, try file %d\n",++TestNumber);
           status = nc_get_vara_text(D->tmID, D->StopID,start,TimeCount,StopTime);
           D->CDTime = DD_Time2Double(StopTime);

           switch( status = SetNewFile(D,1) )
           {
               case OK: return OK;
               case WAITEXTCALL: return WAITEXTCALL;
               case TIMEINEMPTY: return TIMEINEMPTY;
               case OUTOFTIME: return OUTOFTIME;
               case CACHERR: return CACHERR;
               default: return DATAFILEERR;
           }
        }
           
     }
     else   /* Next record is valid */
     {
//        sleep(40);
        if(D->ExtCallAllowed && ((FirstTime - D->CDTime) > D->MinGap))
        {
            if(Verbose) fprintf(stderr,"SetNewFile(): File OK, but there is a gap  %d %d\n",FirstTime - D->CDTime, D->MinGap);
           switch(status = ExtDataRequest(D, D->CDTime+D->MinGap))
           {
              case OK: return WAITEXTCALL;
              case NODATAATTIME:
              case GAPISSMALL: return TIMEINEMPTY; break;
              case OUTOFTIME: return OUTOFTIME; break;
              default: return DATAFILEERR;
           }
        }  /* else just continue */
     }
  }
  
/*-------------- refresh the Cache table ---------------------------*/
  status = nc_sync(D->Cash.ID);
  if(status != NC_NOERR) 
  {          
     if(Verbose) fprintf(stderr,"SetNewFile(): Cache Synchro: %s\n",nc_strerror(status));
     return CACHERR;
  }
  for(CashStart[0] = 0; CashStart[0] < CASHLEN; CashStart[0]++)
  {
    status = nc_get_vara_text(D->Cash.ID, D->Cash.nameID,CashStart,CashCount,D->Cash.names[CashStart[0]]);
    if(status != NC_NOERR) 
    {          
       if(Verbose) fprintf(stderr,"SetNewFile(): Cache Names: %s\n",nc_strerror(status));
       D->LastFileStatus = CACHERR;
       return CACHERR;
    }
    status = nc_get_vara_long(D->Cash.ID, D->Cash.timeID,CashStart,CashCount,&(D->Cash.times[CashStart[0]]));
    if(status != NC_NOERR)
    {          
       if(Verbose) fprintf(stderr,"SetNewFile(): Cache Times %s\n",nc_strerror(status));
       D->LastFileStatus = CACHERR;
       return CACHERR;
    }
  }

/*----------- Close the old file ----------------------*/
  if((status = CloseOldFile(D)) < 0) return DATAFILEERR;

  strcpy(FullName,D->path);
  strcpy(FullName+strlen(D->path), Name);
//  if(Verbose) fprintf(stderr,"SetNewFile(): New file to open: %s\n",FullName);

  NewAttempt = 0;  

  while (NewAttempt < 3) 
  {
      /*----- Search the new requested file in the cache and the oldest file ----------*/
      FileNumber = 0; 
      CurrentTime = time(NULL); Find = 0; OldestNumber = -1; 
 
      while((FileNumber < CASHLEN) &&                                   // Cache is finished
	((Find = strcmp(&(D->Cash.names[FileNumber][0]),Name)) != 0) && // File is already in the Cache
	(D->Cash.names[FileNumber][0] != ' '))                          // There is empty space
      {
	if((D->Cash.times[FileNumber] < (CurrentTime - FILEACCMARG) ) && (OldestNumber == -1)) OldestNumber = FileNumber;
	dltt = (int)(D->Cash.times[OldestNumber] - D->Cash.times[FileNumber]);
	if(dltt > 0) OldestNumber = FileNumber;
	FileNumber++;
      }

      /* ====================================================
       * Sometimes it is possible, that:
       * 1) No corresponding file in the cache (Find != 0)
       * 2) The oldest file is not found (OldestNumber == -1)
       * 3) cach is full (FileNumber == CASHLEN)
       * Then we need to repeat the search after one sec delay
       *==================================================== */
      if((Find != 0) && (FileNumber == CASHLEN) && (OldestNumber == -1))
      {
	NewAttempt++;
	sleep((unsigned )(FILEACCMARG));

        if (Verbose)
            fprintf(stderr,"Waiting %d secs to get the Oldest File\n", FILEACCMARG);

      } else NewAttempt = 3;
   }
  
//---------- Parsing resultat of search -------------
//  if(Verbose)
//  {
//    if(OldestNumber > -1) fprintf(stderr,"SetNewFile(): Search: Is Oldest File %d %d %d\n", OldestNumber,D->Cash.times[OldestNumber],CurrentTime);
//    else  fprintf(stderr,"SetNewFile(): Search: No Oldest File %d %d\n", OldestNumber,CurrentTime);
//  }
  
  if(Find != 0) 
/*---------------- No request file in the CACHE ----------------------*/
  {
     if(FileNumber < CASHLEN)
     {
        if(D->Cash.names[FileNumber][0] == ' ') /* There is empty space in the table */
        {
           strcpy(command, "gunzip -c ");
           strcat(command, FullName);
           strcat(command, ".gz > ");
           strcat(command, FullName);
           system(command);           /* File is unzipped */
           sprintf(command, "chmod g+w %s\0",FullName);
           system(command);           /* File is unzipped */

           strcpy(&(D->Cash.names[FileNumber][0]),Name);
           D->Cash.times[FileNumber] = CurrentTime;
           CashStart[0] = FileNumber;
           D->CurrCushN = FileNumber;
        }  
        else 
        {          
           D->LastFileStatus = CACHERR;
           return CACHERR;             /* Unrecoverable error */
        }
     }
     else  /* No empty space. It is necessury to remove one file */
     {
         if (OldestNumber > -1) //------ Oldest file is found -------
         {
            strcpy(command, "gunzip -c ");
            strcat(command, FullName);
            strcat(command, ".gz > ");
            strcat(command, FullName);
            system(command);           /* File is unzipped */
            sprintf(command, "chmod g+w %s\0",FullName);
            system(command);           /* File is unzipped */

            strcpy(command, "rm ");
            strcat(command,D->path);
            strcat(command,&(D->Cash.names[OldestNumber][0]));
            strcat(command, " &");
            system(command);           /* Old file removed */
            strcpy(&(D->Cash.names[OldestNumber][0]),Name);
            D->Cash.times[OldestNumber] = CurrentTime;
            CashStart[0] = OldestNumber;
            D->CurrCushN = OldestNumber;
         }
         else {    
           if(Verbose) fprintf(stderr,"SetNewFile(): return CACHTOOREC\n");      
           D->LastFileStatus = CACHTOOREC;
           D->TimeRecNumber -= N;
           return CACHTOOREC; //----- Say client that all files are too new
         }
      }
        /*------------ The place for new cache is found --------------*/
        status = nc_put_vara_text(D->Cash.ID, D->Cash.nameID,CashStart,CashCount,&(D->Cash.names[CashStart[0]][0]));
        status = nc_put_vara_long(D->Cash.ID, D->Cash.timeID,CashStart,CashCount,&(D->Cash.times[CashStart[0]]));
        if(status != NC_NOERR) 
        {
           if(Verbose) fprintf(stderr,"SetNewFile(): Write Name and Time to cash: %s\n",nc_strerror(status));
           D->LastFileStatus = CACHERR;
           return CACHERR;
        }
        status = nc_sync(D->Cash.ID);
  }
//------------------ There is FILE already in the CACHE --------------  
  else /*------- The requested file is already in the cache just refresh the time */
  {
        CashStart[0] = FileNumber;
        D->Cash.times[FileNumber] = CurrentTime;  /* refresh the time */
        D->CurrCushN = FileNumber;
        status = nc_put_vara_long(D->Cash.ID, D->Cash.timeID,CashStart,CashCount,&(D->Cash.times[CashStart[0]]));
        if(status != NC_NOERR) 
        {          
           if(Verbose) fprintf(stderr,"SetNewFile(): File In Cache: %s\n",nc_strerror(status));
           D->LastFileStatus = CACHERR;
           return CACHERR;
        }
        status = nc_sync(D->Cash.ID);
  }

//  if(Verbose) fprintf(stderr,"SetNewFile(): file %s, %d to be open\n",FullName,D->CurrCushN);


/*--------------- CACHE refreshed and requested file is unzipped ------------------------*/
/*----------------- Open requested file -----------------------------------------------*/

  status = nc_open(FullName,NC_NOWRITE,&(D->ncID));
  if(status != NC_NOERR)
  {
    if(Verbose) fprintf(stderr,"SetNewFile(): Error while File Open: %s\n",nc_strerror(status));
    D->LastFileStatus =DATAFILEERR ;
    return DATAFILEERR;
  }
  if(Verbose) fprintf(stderr,"SetNewFile(): %s is open\n",FullName);

/* Recover time  dimension */
  status = nc_inq_dimid(D->ncID,TimeDimName,&TimeDimID);
  if(status != NC_NOERR) 
  {
    if(Verbose) fprintf(stderr,"SetNewFile(): Get Time Dim: %s\n",nc_strerror(status));
    D->LastFileStatus = DATAFILEERR;
    return DATAFILEERR;
  }
  status = nc_inq_dimlen(D->ncID,TimeDimID,&(D->Maxnc_rec));
  if(status != NC_NOERR) 
  {          
    if(Verbose) fprintf(stderr,"SetNewFile: GetFile Length: %s\n",nc_strerror(status));
    D->LastFileStatus = DATAFILEERR;
    return DATAFILEERR;
  }

/* Setup current position in the new open file */
  if(N > 0) D->nc_rec = 0;
  if(N < 0) D->nc_rec = D->Maxnc_rec-1;
  
  D->LastFileStatus = OK;
  return OK;
}
/*------------------------------------------------------------------------------------*/

/*----------------- VarSize -----------------------------------*/
int  VarSize(nc_type type)
{
   switch(type)
   {
     case NC_BYTE: return sizeof(char);
     case NC_CHAR: return sizeof(char);
     case NC_SHORT: return sizeof(short);
     case NC_INT: return sizeof(int);
     case NC_FLOAT: return sizeof(float);
     case NC_DOUBLE: return sizeof(double);
     default: return 1;
   }
}

/*##############################################################*/
/*----------------------- MaxRecord -----------------------------*/
/*##############################################################*/

size_t MaxRecord(int ncID)
/* ncID - ID of already open NC file */
{
  int timedimid;   /* ID for dimension "Time" */

  int nvars;
  int ndims;
  int *ndims_arr;      /* Array to keep number of dimensions of each variable */
  size_t *dimleng_arr; /* Array to keep all length of dimensions */
  nc_type *vartype_arr; /* Array to keep all types of variables */
  int **dimid_arr; /* array of pointers to arrays of variable dimensions */

  int status;     /* Error indicator */
  int iv, id;
  int cvarsize;    /* size of variable */
  int maxsize;     /* maximal size */
  int recordsize;   /* size by units */
  size_t RecordNumber;

  /*--- general information -------*/
  status = nc_inq_ndims(ncID,&ndims);
  status = nc_inq_nvars(ncID,&nvars);
  status = nc_inq_unlimdim(ncID, &timedimid); /* What is unlimited dimension ID */

  /*---- memory allocation --------*/
  dimleng_arr = (size_t *)malloc(ndims * sizeof(size_t));
  vartype_arr = (nc_type *)malloc(nvars * sizeof(nc_type));
  dimid_arr = (int **)malloc(nvars * sizeof(int *));
  ndims_arr = (int *)malloc(nvars * sizeof(int));

  /* Length of each dimension */
  for(id = 0; id < ndims; id++)
  {
     if(id == timedimid) dimleng_arr[id] = 1;
     else status = nc_inq_dimlen(ncID, id, &(dimleng_arr[id]));
  }

  /* type and dimension of each variable */
  for(iv = 0; iv < nvars; iv++)
  {
      status = nc_inq_vartype(ncID,iv,&(vartype_arr[iv]));
      status = nc_inq_varndims(ncID,iv,&(ndims_arr[iv]));
      dimid_arr[iv] = (int *)malloc(ndims_arr[iv]*sizeof(int)); /* memory for dim ID array */
      status = nc_inq_vardimid(ncID,iv,dimid_arr[iv]);
  }

  /* calculation of size of variable and maximum size of variables */
  maxsize = 0;
  for(iv = 0; iv < nvars; iv++)
  {
     recordsize = 1;
     for(id = 0; id < ndims_arr[iv]; id++)
         recordsize *= dimleng_arr[(dimid_arr[iv])[id]];
     cvarsize  = recordsize * VarSize(vartype_arr[iv]);
     if(cvarsize > maxsize) maxsize = cvarsize;
  }

  /* calculation record number */
  RecordNumber = (size_t)(MAXPACKSIZE / maxsize);
  if(RecordNumber == 0)
  {
     if(Verbose) fprintf(stderr,"MaxRecord(%d): Error in calculation of records number,MAXPACKSIZE = %d, maxsize = %d\n",ncID,MAXPACKSIZE,maxsize);
  }

  /* Free all */
  for(iv = 0; iv < nvars; iv++) free(dimid_arr[iv]);
  free(ndims_arr);
  free(dimid_arr);
  free(vartype_arr);
  free(dimleng_arr);

  return RecordNumber;
}
/*----------------------------------------------------------------------*/