Implemented startconfig.

[phys/latlib.git] / configcache.cpp

diff --git a/configcache.cpp b/configcache.cpp
index 3d160a98ad8f22863daae9f1c7ada67c48e2c5e2..86871231faed6472eb71dc2e19e63bb401a8f4e2 100644 (file)
--- a/configcache.cpp
+++ b/configcache.cpp
@@ -4,34 +4,74 @@
 #include <iostream>
 #include <time.h>
 #include <dirent.h>
 #include <iostream>
 #include <time.h>
 #include <dirent.h>

+#include <errno.h>
+#include <sys/stat.h>
+#include <fcntl.h>

 
 

-configcache::configcache(const string& cacheid, const int& nequi, const int& nskip, const string& datadir, char **configmem, const int& configMemSize, const int& cachemode){
+#include <boost/iostreams/filtering_streambuf.hpp>
+#include <boost/iostreams/stream.hpp>
+#include <boost/iostreams/filter/bzip2.hpp>
+#include <boost/iostreams/device/array.hpp>
+#include <boost/iostreams/copy.hpp>
+
+#define HEADER_READOK   0
+#define HEADER_READERR  1
+#define HEADER_READLAST 2
+
+struct configcache::iobuffers
+{
+  boost::iostreams::filtering_ostreambuf *out;
+};
+
+configcache::configcache(const string& cacheid, const int& nequi, const int& nskip, const string& datadir, char **configmem, const int& configMemSize, const int& cachemode,
+                        ostream *_log){
+  log = _log;
+
+  dataReader = new datread(configMemSize, log);
+  

   NEQUI = nequi;
   NSKIP = nskip;
   DATADIR = datadir;
   CACHEID = cacheid;
 
   NEQUI = nequi;
   NSKIP = nskip;
   DATADIR = datadir;
   CACHEID = cacheid;
 

+  if ( cacheid.find("_") != -1 ) {
+    if(log) *log << "CCACHE: Invalid cacheid \"" << cacheid << "\" given. Cacheids must not contain underscores!" << endl << flush;
+    exit(1);
+  }
+

   configMem = (char*)malloc(configMemSize);
   tmpConfig = (char*)malloc(configMemSize);
 
   *configmem = configMem;
   configSize = configMemSize;
 
   configMem = (char*)malloc(configMemSize);
   tmpConfig = (char*)malloc(configMemSize);
 
   *configmem = configMem;
   configSize = configMemSize;
 

-  outBuffer = NULL;
-  inBuffer = NULL;
+  ioBuffers = new iobuffers;
+  ioBuffers->out = NULL;

 
   MODE = cachemode;
 
   refetchDataFiles = false;
 }
 
 
   MODE = cachemode;
 
   refetchDataFiles = false;
 }
 

-string configcache::getFileId(const bool& shortid)
+string configcache::paraString() {
+  stringstream parastring;
+
+  for(int ipara=0; ipara<Paras.size(); ipara++)
+    parastring << "_" << Paras[ipara].id << Paras[ipara].val;
+  
+  return parastring.str();
+}
+
+string configcache::getFileId(int actnequi, const bool& superextended, const bool& shortid)

 {
   stringstream fileid;
 
 {
   stringstream fileid;
 

-  if(!shortid) fileid << CACHEID << "_" << NEQUI << "_" << NSKIP;
-  for(int ipara=0; ipara<Paras.size(); ipara++)
-    fileid << "_" << Paras[ipara].id << Paras[ipara].val;
+  if(!shortid) fileid << CACHEID << "_" << actnequi << "_" << NSKIP;
+
+  if( superextended )
+    fileid << "_" << hash( paraString() );
+  else
+    fileid << paraString();

 
   return fileid.str();
 }
 
   return fileid.str();
 }


@@ -40,31 +80,47 @@ void configcache::fetchDataFiles()
 {
   struct dirent *de=NULL;
   DIR *d=NULL;
 {
   struct dirent *de=NULL;
   DIR *d=NULL;

+  static infiledesc filedesc;

   
   d=opendir(DATADIR.c_str());
   if(d != NULL){
     while(de = readdir(d)){
       string filename = de->d_name;
   
   d=opendir(DATADIR.c_str());
   if(d != NULL){
     while(de = readdir(d)){
       string filename = de->d_name;

-      if(isValidInFile(filename)) 
+      if(isValidInFile(filename, &filedesc)) 

        {
        {

-         inFiles.push_back(filename);
+         inFiles.push_back(filedesc);

        }
     }
   }
 }
 
        }
     }
   }
 }
 

-bool configcache::isValidInFile(const string& infile)
+bool configcache::isValidInFile(const string& infile, infiledesc *filedesc)

 {
   char *inchar, *inParts;
   string truncIn, truncOut;
 
 {
   char *inchar, *inParts;
   string truncIn, truncOut;
 

+  filedesc->filename = infile;
+

   if( infile.size() < 4 ) return false;
 
   if( infile.size() < 4 ) return false;
 

-  if( infile.substr(infile.size()-3) != "dat" ) return false;
+  if( infile.substr(infile.size()-4) == ".dat" ) {
+    filedesc->extended = false;
+    filedesc->superextended = false;
+  }
+  else if( infile.substr(infile.size()-4) == "edat" ) {
+    filedesc->extended = true;
+    filedesc->superextended = false;
+  }
+  else if( infile.substr(infile.size()-4) == "sdat" ) {
+    filedesc->extended = true;
+    filedesc->superextended = true;
+  }
+  else
+    return false;

 
   inchar = new char [infile.size()+1];
   strcpy (inchar, infile.c_str());
 
   inchar = new char [infile.size()+1];
   strcpy (inchar, infile.c_str());

-  
+

   inParts = strtok( inchar, "_" );
   for(int iPart=0; inParts!=NULL; iPart++)
     {
   inParts = strtok( inchar, "_" );
   for(int iPart=0; inParts!=NULL; iPart++)
     {


@@ -72,9 +128,17 @@ bool configcache::isValidInFile(const string& infile)
 
       switch(iPart)
        {
 
       switch(iPart)
        {

-       case 1: if(inParts != CACHEID) return false; break;
-       case 2: if(atoi(inParts) != NEQUI) return false; break;
-       case 3: if(atoi(inParts) != NSKIP) return false; break;
+       case 1: if(inParts != CACHEID)
+           return false;
+         break;
+       case 2:
+         filedesc->nequi = atoi(inParts);
+         break;
+       case 3: 
+         if(atoi(inParts) != NSKIP) 
+           return false;
+         filedesc->nskip = atoi(inParts);
+         break;

        }
       inParts = strtok( NULL, "_");
     }
        }
       inParts = strtok( NULL, "_");
     }


@@ -82,14 +146,20 @@ bool configcache::isValidInFile(const string& infile)
 
   delete[] inchar;
 
 
   delete[] inchar;
 

-  if( truncIn.find( getFileId(true) + "_" ) == string::npos ) return false;
+  if( truncIn.find( getFileId(NEQUI, filedesc->superextended, true) + "_" ) == string::npos ) return false;

 
   return true;
 }
 
 
   return true;
 }
 

-bool configcache::readConfig()
+void * configcache::getHeader(const string& headerid) {
+  return dataReader->getHeader(headerid);
+}
+
+void configcache::readConfig(bool *readnewconfig, int *nequileft, vector<unsigned long> *excludeFileHashes)

 {
 {

-  if(DATADIR == "" || MODE == CACHE_MODE_DISABLED) return false;
+  *readnewconfig = false;
+
+  if( DATADIR == "" || !(MODE==CACHE_MODE_RO||MODE==CACHE_MODE_RW) ) return;

 
   if(refetchDataFiles){
     refetchDataFiles = false;
 
   if(refetchDataFiles){
     refetchDataFiles = false;


@@ -98,49 +168,138 @@ bool configcache::readConfig()
 
   while(true)
     {
 
   while(true)
     {

-      if( (!inFile.is_open()) && inFiles.size() == 0 ) return false;
+      vector<infiledesc>::iterator inFileIt = getNextInfile(excludeFileHashes);
+      int iDidVirtualSkips;

 
 

-      while( (!inFile.is_open()) && inFiles.size() > 0 )
-       {
-         inFile.open( (DATADIR + "/" + inFiles.back()).c_str(), std::ios::binary );
-         inFiles.pop_back();
+      if( (!dataReader->fisopen()) && inFileIt == inFiles.end() ) {
+       if (*readnewconfig)
+         *nequileft = nequileft_internal;
+       return;
+      }

 
 

-         if( !inFile.is_open() ) continue;
+      while( (!dataReader->fisopen()) && inFiles.size() > 0 ) {
+       string inFileParaString;

 
 

-         inBuffer = new boost::iostreams::filtering_istreambuf;
-         inBuffer->push( boost::iostreams::bzip2_decompressor() );
-         inBuffer->push(inFile);
+       openFileDesc = *inFileIt;
+
+       if (openFileDesc.nequi < NEQUI)
+         doVirtualEquilibration = true;
+       else
+         doVirtualEquilibration = false;
+
+       firstUsedConfig = true;
+
+       if(log) *log << "CCACHE: Opening dat-file: " << inFileIt->filename << endl << flush;
+
+       dataReader->openFile( DATADIR + "/" + inFileIt->filename );
+               
+       if( openFileDesc.superextended ) {
+         if( dataReader->getParaString() != paraString() ) {
+           if(log) *log << "CCACHE: Parastring does not match. Closing dat-file..." << endl << flush;
+           dataReader->closeFile();
+         }

        }
 
        }
 

-      if( inFile.is_open() ) 
+       inFiles.erase(inFileIt);
+       
+       if( !dataReader->fisopen() ) continue;
+      }
+
+      if( dataReader->fisopen() ) 

        {
        {

-         if( boost::iostreams::read(*inBuffer, tmpConfig, configSize) == configSize )
+         if (doVirtualEquilibration) {
+           if(log) *log << "CCACHE: Trying virtual equilibration." << endl << flush;
+           doVirtualEquilibration = false;
+           for (iDidVirtualSkips=0; iDidVirtualSkips < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iDidVirtualSkips++) {
+             if( dataReader->readFullBlock(tmpConfig) < 0 )
+               break;
+             else if ( (NEQUI-openFileDesc.nequi) - (iDidVirtualSkips+1)*openFileDesc.nskip < nequileft_internal ) {
+               memcpy(configMem, tmpConfig, configSize);
+               nequileft_internal = NEQUI - openFileDesc.nequi - (iDidVirtualSkips+1)*openFileDesc.nskip;
+               *readnewconfig = true;
+               firstUsedConfig = false;
+             }
+           }
+         }
+
+         if( dataReader->readFullBlock(tmpConfig) >= 0 )

            {
              memcpy(configMem, tmpConfig, configSize);
            {
              memcpy(configMem, tmpConfig, configSize);

-             return true;
+             *readnewconfig = true;
+             if (firstUsedConfig) {
+               firstUsedConfig = false;
+               if (openFileDesc.nequi < NEQUI)
+                 nequileft_internal = NEQUI - openFileDesc.nequi - iDidVirtualSkips*openFileDesc.nskip;
+               else
+                 nequileft_internal = NEQUI - openFileDesc.nequi;
+             }
+             nequileft_internal -= openFileDesc.nskip;
+             *nequileft = nequileft_internal;
+             return;

            }
            }

-         else inFile.close();

        }
     }
 }
 
        }
     }
 }
 

-void configcache::writeConfig()
-{
-  if( DATADIR == "" || MODE < 2 ) return;
+void configcache::openOutFile(int actnequi)
+{ 
+  time_t secstamp = time(NULL);
+  int iseq=0;
+  
+  while (true) {
+    outFileName.str("");
+    outFileName << DATADIR << "/" << secstamp << "." << iseq << "_" << getFileId(actnequi, true, false) << "_.sdat.tmp";

 
 

-  if(!outFile.is_open()){
-    time_t secstamp = time(NULL);
+    int tmpfd = open(outFileName.str().c_str(), O_CREAT | O_EXCL, 0644);

 
 

-    outFileName.str("");
-    outFileName << DATADIR << "/" << secstamp << "_" << getFileId() << "_.dat.tmp";    
-    outFile.open( outFileName.str().c_str(), std::ios::binary );
+    if ( tmpfd != -1 ) {
+      close(tmpfd);
+      break;
+    }
+    else if ( errno != EEXIST ) {
+      if(log) *log << "CCACHE: Could not create cachefile!" << endl << flush;
+      exit(1);
+    }

 
 

-    outBuffer = new boost::iostreams::filtering_ostreambuf;
-    outBuffer->push(boost::iostreams::bzip2_compressor());
-    outBuffer->push(outFile);
+    iseq++;

   }
   }

+  
+  outFile.open( outFileName.str().c_str(), std::ios::binary );
+
+  outFile << paraString() << endl;

 
 

-  boost::iostreams::write(*outBuffer, configMem, configSize);
+  ioBuffers->out = new boost::iostreams::filtering_ostreambuf;
+  ioBuffers->out->push(boost::iostreams::bzip2_compressor());
+  ioBuffers->out->push(outFile);
+}
+
+void configcache::writeHeader(const string& headerid, const char *header, long unsigned int size, int actnequi) {
+  unsigned long headeridhash;
+
+  if( DATADIR == "" || !(MODE==CACHE_MODE_WO||MODE==CACHE_MODE_RW) ) return;
+
+  if(!outFile.is_open())
+    openOutFile(actnequi);
+
+  headeridhash = hash(headerid);
+
+  boost::iostreams::write(*ioBuffers->out, (char*)&size, sizeof(long unsigned int));
+  boost::iostreams::write(*ioBuffers->out, (char*)&headeridhash, sizeof(unsigned long));
+  boost::iostreams::write(*ioBuffers->out, header, size);
+}
+
+void configcache::writeConfig(int actnequi)
+{
+  long unsigned int zeroheader=0;
+
+  if ( DATADIR == "" || !(MODE==CACHE_MODE_WO||MODE==CACHE_MODE_RW) ) return;
+
+  if ( ! outFile.is_open() )
+    openOutFile(actnequi);
+  
+  boost::iostreams::write(*ioBuffers->out, (char*)&zeroheader, sizeof(long unsigned int));
+
+  boost::iostreams::write(*ioBuffers->out, configMem, configSize);

 }
 
 void configcache::addPara(const string& parid, const double& val){
 }
 
 void configcache::addPara(const string& parid, const double& val){


@@ -157,27 +316,27 @@ int configcache::getParIndex(const string& parid){
 
 void configcache::setPara(const string& parid, const double& value){
   Paras[getParIndex(parid)].val = value;
 
 void configcache::setPara(const string& parid, const double& value){
   Paras[getParIndex(parid)].val = value;

+

   finishOutFile();
   finishOutFile();

-  if(inBuffer != NULL) { delete inBuffer; inBuffer=NULL; } 
-  inFile.close();
+  dataReader->closeFile();

   inFiles.clear();
 
   refetchDataFiles = true;
   inFiles.clear();
 
   refetchDataFiles = true;

+  nequileft_internal = NEQUI;

 }
 
 configcache::~configcache()
 {
   finishOutFile();
 }
 
 configcache::~configcache()
 {
   finishOutFile();

-  delete inBuffer;
-  inBuffer = NULL;
+  delete dataReader;

 }
 
 void configcache::finishOutFile()
 {
 }
 
 void configcache::finishOutFile()
 {

-  if( outBuffer != NULL )
+  if( ioBuffers->out != NULL )

     {
     {

-      delete outBuffer;
-      outBuffer = NULL;
+      delete ioBuffers->out;
+      ioBuffers->out = NULL;

     }
 
   if( outFile.is_open() )
     }
 
   if( outFile.is_open() )


@@ -186,3 +345,32 @@ void configcache::finishOutFile()
       rename( outFileName.str().c_str(), outFileName.str().substr(0, outFileName.str().size()-4).c_str() );
     }
 }
       rename( outFileName.str().c_str(), outFileName.str().substr(0, outFileName.str().size()-4).c_str() );
     }
 }

+
+unsigned long configcache::hash(const string& str)
+{
+  unsigned long hash = 5381;
+
+  for(string::const_iterator it=str.begin();it!=str.end();it++) 
+    hash = ((hash << 5) + hash) + *it; /* hash * 33 + character */
+
+  return hash;
+}
+
+vector<infiledesc>::iterator configcache::getNextInfile(vector<unsigned long> *excludeFileHashes) {
+  for (vector<infiledesc>::iterator init = inFiles.begin(); init != inFiles.end(); ++init) {
+    if (excludeFileHashes != NULL) {
+      bool excludethisfile = false;
+
+      for (vector<unsigned long>::iterator exit = excludeFileHashes->begin(); exit != excludeFileHashes->end(); ++exit)
+       if ( *exit == hash(init->filename) ) {
+         excludethisfile = true;
+         break;
+       }
+
+      if (excludethisfile)
+       continue;
+    }
+    return init;
+  }
+  return inFiles.end();
+}