...

[phys/latlib.git] / configcache.cpp

diff --git a/configcache.cpp b/configcache.cpp
index fcfdd8eb2582503fd23defc645f135ea45eb2317..62519a4c7fea105d43fae0af6a448f1cf3cab7d0 100644 (file)
--- a/configcache.cpp
+++ b/configcache.cpp
@@ -5,15 +5,24 @@
 #include <time.h>
 #include <dirent.h>
 
 #include <time.h>
 #include <dirent.h>
 

+#define HEADER_READOK   0
+#define HEADER_READERR  1
+#define HEADER_READLAST 2
+

 configcache::configcache(const string& cacheid, const int& nequi, const int& nskip, const string& datadir, char **configmem, const int& configMemSize, const int& cachemode,
 configcache::configcache(const string& cacheid, const int& nequi, const int& nskip, const string& datadir, char **configmem, const int& configMemSize, const int& cachemode,

-                        writeout *out_a){
-  out = out_a;
+                        ostream *_log){
+  log = _log;

 
   NEQUI = nequi;
   NSKIP = nskip;
   DATADIR = datadir;
   CACHEID = cacheid;
 
 
   NEQUI = nequi;
   NSKIP = nskip;
   DATADIR = datadir;
   CACHEID = cacheid;
 

+  if ( cacheid.find("_") != -1 ) {
+    if(log) *log << "CCACHE: Invalid cacheid \"" << cacheid << "\" given. Cacheids must not contain underscores!" << endl << flush;
+    exit(1);
+  }
+

   configMem = (char*)malloc(configMemSize);
   tmpConfig = (char*)malloc(configMemSize);
 
   configMem = (char*)malloc(configMemSize);
   tmpConfig = (char*)malloc(configMemSize);
 


@@ -26,15 +35,13 @@ configcache::configcache(const string& cacheid, const int& nequi, const int& nsk
   MODE = cachemode;
 
   refetchDataFiles = false;
   MODE = cachemode;
 
   refetchDataFiles = false;

-
-  readHeaderData = NULL;

 }
 
 }
 

-string configcache::getFileId(const bool& shortid)
+string configcache::getFileId(int actnequi, const bool& shortid)

 {
   stringstream fileid;
 
 {
   stringstream fileid;
 

-  if(!shortid) fileid << CACHEID << "_" << NEQUI << "_" << NSKIP;
+  if(!shortid) fileid << CACHEID << "_" << actnequi << "_" << NSKIP;

   for(int ipara=0; ipara<Paras.size(); ipara++)
     fileid << "_" << Paras[ipara].id << Paras[ipara].val;
 
   for(int ipara=0; ipara<Paras.size(); ipara++)
     fileid << "_" << Paras[ipara].id << Paras[ipara].val;
 


@@ -65,7 +72,6 @@ bool configcache::isValidInFile(const string& infile, infiledesc *filedesc)
   string truncIn, truncOut;
 
   filedesc->filename = infile;
   string truncIn, truncOut;
 
   filedesc->filename = infile;

-  filedesc->doVirtualEquilibration = false;

 
   if( infile.size() < 4 ) return false;
 
 
   if( infile.size() < 4 ) return false;
 


@@ -78,7 +84,7 @@ bool configcache::isValidInFile(const string& infile, infiledesc *filedesc)
 
   inchar = new char [infile.size()+1];
   strcpy (inchar, infile.c_str());
 
   inchar = new char [infile.size()+1];
   strcpy (inchar, infile.c_str());

-  
+

   inParts = strtok( inchar, "_" );
   for(int iPart=0; inParts!=NULL; iPart++)
     {
   inParts = strtok( inchar, "_" );
   for(int iPart=0; inParts!=NULL; iPart++)
     {


@@ -86,12 +92,10 @@ bool configcache::isValidInFile(const string& infile, infiledesc *filedesc)
 
       switch(iPart)
        {
 
       switch(iPart)
        {

-       case 1: if(inParts != CACHEID) return false; break;
-       case 2: 
-         if (atoi(inParts) > NEQUI) 
-           return false; 
-         else if (atoi(inParts) < NEQUI)
-           filedesc->doVirtualEquilibration = true;
+       case 1: if(inParts != CACHEID)
+           return false;
+         break;
+       case 2:

          filedesc->nequi = atoi(inParts);
          break;
        case 3: 
          filedesc->nequi = atoi(inParts);
          break;
        case 3: 


@@ -106,44 +110,75 @@ bool configcache::isValidInFile(const string& infile, infiledesc *filedesc)
 
   delete[] inchar;
 
 
   delete[] inchar;
 

-  if( truncIn.find( getFileId(true) + "_" ) == string::npos ) return false;
+  if( truncIn.find( getFileId(NEQUI, true) + "_" ) == string::npos ) return false;

 
   return true;
 }
 
 
   return true;
 }
 

-bool configcache::readHeader()
+int configcache::readHeader()

 {
   long unsigned int headersize;
 {
   long unsigned int headersize;

+  
+  if( readDataToMem((char *)&headersize, sizeof(long unsigned int)) == sizeof(long unsigned int) && inFile.is_open() ) {
+    if ( headersize == 0 )
+      return HEADER_READLAST;

 
 

-  if( readDataToMem((char*)&headersize, sizeof(long unsigned int)) == sizeof(long unsigned int) && inFile.is_open() )
-    {
-      if( readHeaderData != NULL ) free(readHeaderData);
+    pair<unsigned long, void *> newHeader;

 
 

-      readHeaderData = (char*) malloc(headersize);
+    if( readDataToMem((char *)&newHeader.first, sizeof(unsigned long)) == sizeof(unsigned long) && inFile.is_open() ) {
+      newHeader.second = malloc(headersize);

 
 

-      if( readDataToMem(readHeaderData, headersize) == headersize && inFile.is_open() ) {
-       return true;
+      if( readDataToMem((char *)newHeader.second, headersize) == headersize && inFile.is_open() ) {
+       headerStore.push_back(newHeader);
+       return HEADER_READOK;

       }
       else {
       }
       else {

-       if(out) *out->log << "CCACHE: Could not read header! Closing dat-file: " << openFileDesc.filename << endl << flush;
+       if(log) *log << "CCACHE: Could not read heade-data! Closing dat-file: " << openFileDesc.filename << endl << flush;

        inFile.close();
        inFile.close();

-       return false;
+       return HEADER_READERR;

       }
     }
       }
     }

+    else {
+      if(log) *log << "CCACHE: Could not read headerid-hash! Closing dat-file: " << openFileDesc.filename << endl << flush;
+      inFile.close();
+      return HEADER_READERR;
+    }
+  }

   else {
   else {

-    if(out) *out->log << "CCACHE: Could not read header size. Closing dat-file: " << openFileDesc.filename << endl << flush;
+    if(log) *log << "CCACHE: Could not read header size. Closing dat-file: " << openFileDesc.filename << endl << flush;

     inFile.close();
     inFile.close();

-    return false;
+    return HEADER_READERR;

   }
 }
 
   }
 }
 

-void *configcache::getHeader() {
-  return readHeaderData;
+bool configcache::readAllHeaders()
+{
+  int readHeaderStatus;
+
+  deleteHeaderStore();
+  
+  do {
+    readHeaderStatus = readHeader();
+  }
+  while ( readHeaderStatus == HEADER_READOK );
+
+  if ( readHeaderStatus == HEADER_READLAST ) return true;
+  else if ( readHeaderStatus == HEADER_READERR ) return false;
+}
+
+void * configcache::getHeader(const string& headerid) {
+  for (vector< pair<unsigned long, void *> >::iterator headerStoreIt = headerStore.begin(); headerStoreIt != headerStore.end(); ++headerStoreIt)
+    if ( headerStoreIt->first == hash(headerid) )
+      return headerStoreIt->second;
+  
+  return NULL;

 }
 
 }
 

-bool configcache::readConfig()
+void configcache::readConfig(bool *readnewconfig, int *nequileft, vector<unsigned long> *excludeFileHashes)

 {
 {

-  if(DATADIR == "" || MODE == CACHE_MODE_DISABLED) return false;
+  *readnewconfig = false;
+
+  if(DATADIR == "" || MODE == CACHE_MODE_DISABLED) return;

 
   if(refetchDataFiles){
     refetchDataFiles = false;
 
   if(refetchDataFiles){
     refetchDataFiles = false;


@@ -152,15 +187,30 @@ bool configcache::readConfig()
 
   while(true)
     {
 
   while(true)
     {

-      if( (!inFile.is_open()) && inFiles.size() == 0 ) return false;
+      vector<infiledesc>::iterator inFileIt = getNextInfile(excludeFileHashes);
+      int iDidVirtualSkips;
+
+      if( (!inFile.is_open()) && inFileIt == inFiles.end() ) {
+       if (*readnewconfig)
+         *nequileft = nequileft_internal;
+       return;
+      }

 
       while( (!inFile.is_open()) && inFiles.size() > 0 ) {
 
       while( (!inFile.is_open()) && inFiles.size() > 0 ) {

-       if(out) *out->log << "CCACHE: Opening dat-file: " << inFiles.back().filename << endl << flush;
+       openFileDesc = *inFileIt;
+
+       if (openFileDesc.nequi < NEQUI)
+         doVirtualEquilibration = true;
+       else
+         doVirtualEquilibration = false;

 
 

-       openFileDesc = inFiles.back();
-       inFile.open( (DATADIR + "/" + inFiles.back().filename).c_str(), std::ios::binary );
-       inFiles.pop_back();
+       firstUsedConfig = true;

 
 

+       if(log) *log << "CCACHE: Opening dat-file: " << inFileIt->filename << endl << flush;
+       inFile.open( (DATADIR + "/" + inFileIt->filename).c_str(), std::ios::binary );
+       
+       inFiles.erase(inFileIt);
+       

        if( !inFile.is_open() ) continue;
 
        inBuffer = new boost::iostreams::filtering_istreambuf;
        if( !inFile.is_open() ) continue;
 
        inBuffer = new boost::iostreams::filtering_istreambuf;


@@ -170,69 +220,85 @@ bool configcache::readConfig()
 
       if( inFile.is_open() ) 
        {
 
       if( inFile.is_open() ) 
        {

-         if (openFileDesc.doVirtualEquilibration) {
-           if(out) *out->log << "CCACHE: Trying virtual equilibration." << endl << flush;
-           openFileDesc.doVirtualEquilibration = false;
-           for (int iskip=0; iskip < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iskip++) {
+         if (doVirtualEquilibration) {
+           if(log) *log << "CCACHE: Trying virtual equilibration." << endl << flush;
+           doVirtualEquilibration = false;
+           for (iDidVirtualSkips=0; iDidVirtualSkips < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iDidVirtualSkips++) {

              if( readFullBlock(tmpConfig, configSize) != configSize || ! inFile.is_open() )
                break;
              if( readFullBlock(tmpConfig, configSize) != configSize || ! inFile.is_open() )
                break;

+             else if ( (NEQUI-openFileDesc.nequi) - (iDidVirtualSkips+1)*openFileDesc.nskip < nequileft_internal ) {
+               memcpy(configMem, tmpConfig, configSize);
+               nequileft_internal = NEQUI - openFileDesc.nequi - (iDidVirtualSkips+1)*openFileDesc.nskip;
+               *readnewconfig = true;
+               firstUsedConfig = false;
+             }

            }
          }
 
          if( readFullBlock(tmpConfig, configSize) == configSize && inFile.is_open() )
            {
              memcpy(configMem, tmpConfig, configSize);
            }
          }
 
          if( readFullBlock(tmpConfig, configSize) == configSize && inFile.is_open() )
            {
              memcpy(configMem, tmpConfig, configSize);

-             return true;
+             *readnewconfig = true;
+             if (firstUsedConfig) {
+               firstUsedConfig = false;
+               if (openFileDesc.nequi < NEQUI)
+                 nequileft_internal = NEQUI - openFileDesc.nequi - iDidVirtualSkips*openFileDesc.nskip;
+               else
+                 nequileft_internal = NEQUI - openFileDesc.nequi;
+             }
+             nequileft_internal -= openFileDesc.nskip;
+             *nequileft = nequileft_internal;
+             return;

            }
          else {
            }
          else {

-           if(out) *out->log << "CCACHE: Could not read configuration. Closing dat-file: " << openFileDesc.filename << endl << flush;
+           if(log) *log << "CCACHE: Could not read configuration. Closing dat-file: " << openFileDesc.filename << endl << flush;

            inFile.close();
          }
        }
     }
 }
 
            inFile.close();
          }
        }
     }
 }
 

-void configcache::openOutFile()
+void configcache::openOutFile(int actnequi)

 {
   time_t secstamp = time(NULL);
 
   outFileName.str("");
 {
   time_t secstamp = time(NULL);
 
   outFileName.str("");

-  outFileName << DATADIR << "/" << secstamp << "_" << getFileId() << "_.edat.tmp";    
+  outFileName << DATADIR << "/" << secstamp << "_" << getFileId(actnequi) << "_.edat.tmp";
+

   outFile.open( outFileName.str().c_str(), std::ios::binary );
 
   outBuffer = new boost::iostreams::filtering_ostreambuf;
   outBuffer->push(boost::iostreams::bzip2_compressor());
   outBuffer->push(outFile);
   outFile.open( outFileName.str().c_str(), std::ios::binary );
 
   outBuffer = new boost::iostreams::filtering_ostreambuf;
   outBuffer->push(boost::iostreams::bzip2_compressor());
   outBuffer->push(outFile);

-
-  headerWritten = false;

 }
 
 }
 

-void configcache::writeHeader(char *header, long unsigned int size) {
+void configcache::writeHeader(const string& headerid, const char *header, long unsigned int size, int actnequi) {
+  unsigned long headeridhash;
+

   if( DATADIR == "" || MODE < 2 ) return;
 
   if(!outFile.is_open())
   if( DATADIR == "" || MODE < 2 ) return;
 
   if(!outFile.is_open())

-    openOutFile();
+    openOutFile(actnequi);
+
+  headeridhash = hash(headerid);

 
   boost::iostreams::write(*outBuffer, (char*)&size, sizeof(long unsigned int));
 
   boost::iostreams::write(*outBuffer, (char*)&size, sizeof(long unsigned int));

+  boost::iostreams::write(*outBuffer, (char*)&headeridhash, sizeof(unsigned long));

   boost::iostreams::write(*outBuffer, header, size);
   boost::iostreams::write(*outBuffer, header, size);

-
-  headerWritten = true;

 }
 
 }
 

-void configcache::writeConfig()
+void configcache::writeConfig(int actnequi)

 {
 {

+  long unsigned int zeroheader=0;
+

   if ( DATADIR == "" || MODE < 2 ) return;
 
   if ( ! outFile.is_open() )
   if ( DATADIR == "" || MODE < 2 ) return;
 
   if ( ! outFile.is_open() )

-    openOutFile();
+    openOutFile(actnequi);

   
   

-  if ( ! headerWritten ) {
-    long unsigned int zeroheader=0;
-    boost::iostreams::write(*outBuffer, (char*)&zeroheader, sizeof(long unsigned int));
-  }
+  boost::iostreams::write(*outBuffer, (char*)&zeroheader, sizeof(long unsigned int));

 
   boost::iostreams::write(*outBuffer, configMem, configSize);
 
   boost::iostreams::write(*outBuffer, configMem, configSize);

-  headerWritten = false;

 }
 
 void configcache::addPara(const string& parid, const double& val){
 }
 
 void configcache::addPara(const string& parid, const double& val){


@@ -255,6 +321,7 @@ void configcache::setPara(const string& parid, const double& value){
   inFiles.clear();
 
   refetchDataFiles = true;
   inFiles.clear();
 
   refetchDataFiles = true;

+  nequileft_internal = NEQUI;

 }
 
 configcache::~configcache()
 }
 
 configcache::~configcache()


@@ -283,7 +350,7 @@ int configcache::readFullBlock(char *tmpData, long unsigned int dataSize)
 {
   /* try to read header */
   if ( openFileDesc.extended )
 {
   /* try to read header */
   if ( openFileDesc.extended )

-    if ( ! readHeader() ) 
+    if ( ! readAllHeaders() ) 

       return -1;
 
   /* read data */
       return -1;
 
   /* read data */


@@ -298,17 +365,54 @@ int configcache::readDataToMem(char *tmpData, long unsigned int dataSize)
 
   try { readturn = boost::iostreams::read(*inBuffer, tmpData, dataSize); }
   catch(boost::iostreams::bzip2_error& error) { 
 
   try { readturn = boost::iostreams::read(*inBuffer, tmpData, dataSize); }
   catch(boost::iostreams::bzip2_error& error) { 

-    if(out) *out->log << "CCACHE: Caught bzip2 exception with error code: " << error.error() << endl << flush;
+    if(log) *log << "CCACHE: Caught bzip2 exception with error code: " << error.error() << endl << flush;

     inFile.close();
   } 
   catch (std::exception const& ex) {
     inFile.close();
   } 
   catch (std::exception const& ex) {

-    if(out) *out->log << "CCACHE: Caught exception: " << ex.what() << endl << flush;
+    if(log) *log << "CCACHE: Caught exception: " << ex.what() << endl << flush;

     inFile.close();
   }
   catch( ... ) {
     inFile.close();
   }
   catch( ... ) {

-    if(out) *out->log << "CCACHE: Caught unknown exception while reading." << endl << flush;
+    if(log) *log << "CCACHE: Caught unknown exception while reading." << endl << flush;

     inFile.close();
   }
 
   return readturn;
 }
     inFile.close();
   }
 
   return readturn;
 }

+
+unsigned long configcache::hash(const string& str)
+{
+  unsigned long hash = 5381;
+
+  for(string::const_iterator it=str.begin();it!=str.end();it++) 
+    hash = ((hash << 5) + hash) + *it; /* hash * 33 + character */
+
+  return hash;
+}
+
+void configcache::deleteHeaderStore()
+{
+  while ( headerStore.size() > 0 ) {
+    free(headerStore.back().second);
+    headerStore.pop_back();
+  }
+}
+
+vector<infiledesc>::iterator configcache::getNextInfile(vector<unsigned long> *excludeFileHashes) {
+  for (vector<infiledesc>::iterator init = inFiles.begin(); init != inFiles.end(); ++init) {
+    if (excludeFileHashes != NULL) {
+      bool excludethisfile = false;
+
+      for (vector<unsigned long>::iterator exit = excludeFileHashes->begin(); exit != excludeFileHashes->end(); ++exit)
+       if ( *exit == hash(init->filename) ) {
+         excludethisfile = true;
+         break;
+       }
+
+      if (excludethisfile)
+       continue;
+    }
+    return init;
+  }
+  return inFiles.end();
+}