X-Git-Url: http://git.treefish.org/~alex/phys/latlib.git/blobdiff_plain/836ca8cca791128595d9514b6d03a47dc32dd7a4..578c48ada63a6ea4f090728cf2b267fb3a731b78:/configcache.cpp

diff --git a/configcache.cpp b/configcache.cpp
index f00e187..c6f9082 100644
--- a/configcache.cpp
+++ b/configcache.cpp
@@ -5,34 +5,56 @@
 #include <time.h>
 #include <dirent.h>
 
+#include <boost/iostreams/filtering_streambuf.hpp>
+#include <boost/iostreams/stream.hpp>
+#include <boost/iostreams/filter/bzip2.hpp>
+#include <boost/iostreams/device/array.hpp>
+#include <boost/iostreams/copy.hpp>
+
+#define HEADER_READOK   0
+#define HEADER_READERR  1
+#define HEADER_READLAST 2
+
+struct configcache::iobuffers
+{
+  boost::iostreams::filtering_istreambuf *in;
+  boost::iostreams::filtering_ostreambuf *out;
+};
+
 configcache::configcache(const string& cacheid, const int& nequi, const int& nskip, const string& datadir, char **configmem, const int& configMemSize, const int& cachemode,
-			 writeout *out_a){
-  out = out_a;
+			 ostream *_log){
+  log = _log;
 
   NEQUI = nequi;
   NSKIP = nskip;
   DATADIR = datadir;
   CACHEID = cacheid;
 
+  if ( cacheid.find("_") != -1 ) {
+    if(log) *log << "CCACHE: Invalid cacheid \"" << cacheid << "\" given. Cacheids must not contain underscores!" << endl << flush;
+    exit(1);
+  }
+
   configMem = (char*)malloc(configMemSize);
   tmpConfig = (char*)malloc(configMemSize);
 
   *configmem = configMem;
   configSize = configMemSize;
 
-  outBuffer = NULL;
-  inBuffer = NULL;
+  ioBuffers = new iobuffers;
+  ioBuffers->in = NULL;
+  ioBuffers->out = NULL;
 
   MODE = cachemode;
 
   refetchDataFiles = false;
 }
 
-string configcache::getFileId(const bool& shortid)
+string configcache::getFileId(int actnequi, const bool& shortid)
 {
   stringstream fileid;
 
-  if(!shortid) fileid << CACHEID << "_" << NEQUI << "_" << NSKIP;
+  if(!shortid) fileid << CACHEID << "_" << actnequi << "_" << NSKIP;
   for(int ipara=0; ipara<Paras.size(); ipara++)
     fileid << "_" << Paras[ipara].id << Paras[ipara].val;
 
@@ -43,31 +65,39 @@ void configcache::fetchDataFiles()
 {
   struct dirent *de=NULL;
   DIR *d=NULL;
+  static infiledesc filedesc;
   
   d=opendir(DATADIR.c_str());
   if(d != NULL){
     while(de = readdir(d)){
       string filename = de->d_name;
-      if(isValidInFile(filename)) 
+      if(isValidInFile(filename, &filedesc)) 
 	{
-	  inFiles.push_back(filename);
+	  inFiles.push_back(filedesc);
 	}
     }
   }
 }
 
-bool configcache::isValidInFile(const string& infile)
+bool configcache::isValidInFile(const string& infile, infiledesc *filedesc)
 {
   char *inchar, *inParts;
   string truncIn, truncOut;
 
+  filedesc->filename = infile;
+
   if( infile.size() < 4 ) return false;
 
-  if( infile.substr(infile.size()-3) != "dat" ) return false;
+  if( infile.substr(infile.size()-4) == ".dat" )
+    filedesc->extended = false;
+  else if( infile.substr(infile.size()-4) == "edat" )
+    filedesc->extended = true;
+  else
+    return false;
 
   inchar = new char [infile.size()+1];
   strcpy (inchar, infile.c_str());
-  
+
   inParts = strtok( inchar, "_" );
   for(int iPart=0; inParts!=NULL; iPart++)
     {
@@ -75,9 +105,17 @@ bool configcache::isValidInFile(const string& infile)
 
       switch(iPart)
 	{
-	case 1: if(inParts != CACHEID) return false; break;
-	case 2: if(atoi(inParts) != NEQUI) return false; break;
-	case 3: if(atoi(inParts) != NSKIP) return false; break;
+	case 1: if(inParts != CACHEID)
+	    return false;
+	  break;
+	case 2:
+	  filedesc->nequi = atoi(inParts);
+	  break;
+	case 3: 
+	  if(atoi(inParts) != NSKIP) 
+	    return false;
+	  filedesc->nskip = atoi(inParts);
+	  break;
 	}
       inParts = strtok( NULL, "_");
     }
@@ -85,14 +123,75 @@ bool configcache::isValidInFile(const string& infile)
 
   delete[] inchar;
 
-  if( truncIn.find( getFileId(true) + "_" ) == string::npos ) return false;
+  if( truncIn.find( getFileId(NEQUI, true) + "_" ) == string::npos ) return false;
 
   return true;
 }
 
-bool configcache::readConfig()
+int configcache::readHeader()
+{
+  long unsigned int headersize;
+  
+  if( readDataToMem((char *)&headersize, sizeof(long unsigned int)) == sizeof(long unsigned int) && inFile.is_open() ) {
+    if ( headersize == 0 )
+      return HEADER_READLAST;
+
+    pair<unsigned long, void *> newHeader;
+
+    if( readDataToMem((char *)&newHeader.first, sizeof(unsigned long)) == sizeof(unsigned long) && inFile.is_open() ) {
+      newHeader.second = malloc(headersize);
+
+      if( readDataToMem((char *)newHeader.second, headersize) == headersize && inFile.is_open() ) {
+	headerStore.push_back(newHeader);
+	return HEADER_READOK;
+      }
+      else {
+	if(log) *log << "CCACHE: Could not read heade-data! Closing dat-file: " << openFileDesc.filename << endl << flush;
+	inFile.close();
+	return HEADER_READERR;
+      }
+    }
+    else {
+      if(log) *log << "CCACHE: Could not read headerid-hash! Closing dat-file: " << openFileDesc.filename << endl << flush;
+      inFile.close();
+      return HEADER_READERR;
+    }
+  }
+  else {
+    if(log) *log << "CCACHE: Could not read header size. Closing dat-file: " << openFileDesc.filename << endl << flush;
+    inFile.close();
+    return HEADER_READERR;
+  }
+}
+
+bool configcache::readAllHeaders()
+{
+  int readHeaderStatus;
+
+  deleteHeaderStore();
+  
+  do {
+    readHeaderStatus = readHeader();
+  }
+  while ( readHeaderStatus == HEADER_READOK );
+
+  if ( readHeaderStatus == HEADER_READLAST ) return true;
+  else if ( readHeaderStatus == HEADER_READERR ) return false;
+}
+
+void * configcache::getHeader(const string& headerid) {
+  for (vector< pair<unsigned long, void *> >::iterator headerStoreIt = headerStore.begin(); headerStoreIt != headerStore.end(); ++headerStoreIt)
+    if ( headerStoreIt->first == hash(headerid) )
+      return headerStoreIt->second;
+  
+  return NULL;
+}
+
+void configcache::readConfig(bool *readnewconfig, int *nequileft, vector<unsigned long> *excludeFileHashes)
 {
-  if(DATADIR == "" || MODE == CACHE_MODE_DISABLED) return false;
+  *readnewconfig = false;
+
+  if(DATADIR == "" || MODE == CACHE_MODE_DISABLED) return;
 
   if(refetchDataFiles){
     refetchDataFiles = false;
@@ -101,68 +200,118 @@ bool configcache::readConfig()
 
   while(true)
     {
-      if( (!inFile.is_open()) && inFiles.size() == 0 ) return false;
+      vector<infiledesc>::iterator inFileIt = getNextInfile(excludeFileHashes);
+      int iDidVirtualSkips;
 
-      while( (!inFile.is_open()) && inFiles.size() > 0 )
-	{
-	  if(out) *out->log << "CCACHE: Opening dat-file: " << inFiles.back() << endl << flush;
+      if( (!inFile.is_open()) && inFileIt == inFiles.end() ) {
+	if (*readnewconfig)
+	  *nequileft = nequileft_internal;
+	return;
+      }
 
-	  inFile.open( (DATADIR + "/" + inFiles.back()).c_str(), std::ios::binary );
-	  inFiles.pop_back();
+      while( (!inFile.is_open()) && inFiles.size() > 0 ) {
+	openFileDesc = *inFileIt;
 
-	  if( !inFile.is_open() ) continue;
+	if (openFileDesc.nequi < NEQUI)
+	  doVirtualEquilibration = true;
+	else
+	  doVirtualEquilibration = false;
 
-	  std::streampos fsize = inFile.tellg();
-          inFile.seekg( 0, std::ios::end );
-          fsize = inFile.tellg() - fsize;
-	  if( (int)fsize == 0 ) {
-	    *out->log << "CCACHE: dat-file has zero length! Skipping." << endl << flush;
-	    inFile.close();
-	    continue;
-	  }
-	  inFile.seekg(0);
+	firstUsedConfig = true;
 
-	  inBuffer = new boost::iostreams::filtering_istreambuf;
-	  inBuffer->push( boost::iostreams::bzip2_decompressor() );
-	  inBuffer->push(inFile);
-	}
+	if(log) *log << "CCACHE: Opening dat-file: " << inFileIt->filename << endl << flush;
+	inFile.open( (DATADIR + "/" + inFileIt->filename).c_str(), std::ios::binary );
+	
+	inFiles.erase(inFileIt);
+	
+	if( !inFile.is_open() ) continue;
+
+	ioBuffers->in = new boost::iostreams::filtering_istreambuf;
+	ioBuffers->in->push( boost::iostreams::bzip2_decompressor() );
+	ioBuffers->in->push(inFile);
+      }
 
       if( inFile.is_open() ) 
 	{
-	  int readturn = -1;
-	  try { readturn = boost::iostreams::read(*inBuffer, tmpConfig, configSize); }
-	  catch (std::exception const& ex) {
-	    if(out) *out->log << "CCACHE: Caught exception: " << ex.what() << endl << flush;
-	    inFile.close();
+	  if (doVirtualEquilibration) {
+	    if(log) *log << "CCACHE: Trying virtual equilibration." << endl << flush;
+	    doVirtualEquilibration = false;
+	    for (iDidVirtualSkips=0; iDidVirtualSkips < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iDidVirtualSkips++) {
+	      if( readFullBlock(tmpConfig, configSize) != configSize || ! inFile.is_open() )
+		break;
+	      else if ( (NEQUI-openFileDesc.nequi) - (iDidVirtualSkips+1)*openFileDesc.nskip < nequileft_internal ) {
+		memcpy(configMem, tmpConfig, configSize);
+		nequileft_internal = NEQUI - openFileDesc.nequi - (iDidVirtualSkips+1)*openFileDesc.nskip;
+		*readnewconfig = true;
+		firstUsedConfig = false;
+	      }
+	    }
 	  }
-	  
-	  if( readturn == configSize && inFile.is_open() )
+
+	  if( readFullBlock(tmpConfig, configSize) == configSize && inFile.is_open() )
 	    {
 	      memcpy(configMem, tmpConfig, configSize);
-	      return true;
+	      *readnewconfig = true;
+	      if (firstUsedConfig) {
+		firstUsedConfig = false;
+		if (openFileDesc.nequi < NEQUI)
+		  nequileft_internal = NEQUI - openFileDesc.nequi - iDidVirtualSkips*openFileDesc.nskip;
+		else
+		  nequileft_internal = NEQUI - openFileDesc.nequi;
+	      }
+	      nequileft_internal -= openFileDesc.nskip;
+	      *nequileft = nequileft_internal;
+	      return;
 	    }
-	  else inFile.close(); 
+	  else {
+	    if(log) *log << "CCACHE: Could not read configuration. Closing dat-file: " << openFileDesc.filename << endl << flush;
+	    inFile.close();
+	  }
 	}
     }
 }
 
-void configcache::writeConfig()
-{
+void configcache::openOutFile(int actnequi)
+{ 
+  time_t secstamp = time(NULL);
+
+  outFileName.str("");
+  outFileName << DATADIR << "/" << secstamp << "_" << getFileId(actnequi) << "_.edat.tmp";
+
+  outFile.open( outFileName.str().c_str(), std::ios::binary );
+
+  ioBuffers->out = new boost::iostreams::filtering_ostreambuf;
+  ioBuffers->out->push(boost::iostreams::bzip2_compressor());
+  ioBuffers->out->push(outFile);
+}
+
+void configcache::writeHeader(const string& headerid, const char *header, long unsigned int size, int actnequi) {
+  unsigned long headeridhash;
+
   if( DATADIR == "" || MODE < 2 ) return;
 
-  if(!outFile.is_open()){
-    time_t secstamp = time(NULL);
+  if(!outFile.is_open())
+    openOutFile(actnequi);
 
-    outFileName.str("");
-    outFileName << DATADIR << "/" << secstamp << "_" << getFileId() << "_.dat.tmp";    
-    outFile.open( outFileName.str().c_str(), std::ios::binary );
+  headeridhash = hash(headerid);
 
-    outBuffer = new boost::iostreams::filtering_ostreambuf;
-    outBuffer->push(boost::iostreams::bzip2_compressor());
-    outBuffer->push(outFile);
-  }
+  boost::iostreams::write(*ioBuffers->out, (char*)&size, sizeof(long unsigned int));
+  boost::iostreams::write(*ioBuffers->out, (char*)&headeridhash, sizeof(unsigned long));
+  boost::iostreams::write(*ioBuffers->out, header, size);
+}
+
+void configcache::writeConfig(int actnequi)
+{
+  long unsigned int zeroheader=0;
+
+  if ( DATADIR == "" || MODE < 2 ) return;
 
-  boost::iostreams::write(*outBuffer, configMem, configSize);
+  if ( ! outFile.is_open() )
+    openOutFile(actnequi);
+  
+  boost::iostreams::write(*ioBuffers->out, (char*)&zeroheader, sizeof(long unsigned int));
+
+  boost::iostreams::write(*ioBuffers->out, configMem, configSize);
 }
 
 void configcache::addPara(const string& parid, const double& val){
@@ -179,27 +328,29 @@ int configcache::getParIndex(const string& parid){
 
 void configcache::setPara(const string& parid, const double& value){
   Paras[getParIndex(parid)].val = value;
+
   finishOutFile();
-  if(inBuffer != NULL) { delete inBuffer; inBuffer=NULL; } 
+  if(ioBuffers->in != NULL) { delete ioBuffers->in; ioBuffers->in=NULL; } 
   inFile.close();
   inFiles.clear();
 
   refetchDataFiles = true;
+  nequileft_internal = NEQUI;
 }
 
 configcache::~configcache()
 {
   finishOutFile();
-  delete inBuffer;
-  inBuffer = NULL;
+  delete ioBuffers->in;
+  ioBuffers->in = NULL;
 }
 
 void configcache::finishOutFile()
 {
-  if( outBuffer != NULL )
+  if( ioBuffers->out != NULL )
     {
-      delete outBuffer;
-      outBuffer = NULL;
+      delete ioBuffers->out;
+      ioBuffers->out = NULL;
     }
 
   if( outFile.is_open() )
@@ -208,3 +359,74 @@ void configcache::finishOutFile()
       rename( outFileName.str().c_str(), outFileName.str().substr(0, outFileName.str().size()-4).c_str() );
     }
 }
+
+int configcache::readFullBlock(char *tmpData, long unsigned int dataSize)
+{
+  /* try to read header */
+  if ( openFileDesc.extended )
+    if ( ! readAllHeaders() ) 
+      return -1;
+
+  /* read data */
+  return readDataToMem(tmpData, dataSize);
+}
+
+int configcache::readDataToMem(char *tmpData, long unsigned int dataSize)
+{
+  int readturn = -1;
+
+  if ( dataSize == 0 ) return 0;
+
+  try { readturn = boost::iostreams::read(*ioBuffers->in, tmpData, dataSize); }
+  catch(boost::iostreams::bzip2_error& error) { 
+    if(log) *log << "CCACHE: Caught bzip2 exception with error code: " << error.error() << endl << flush;
+    inFile.close();
+  } 
+  catch (std::exception const& ex) {
+    if(log) *log << "CCACHE: Caught exception: " << ex.what() << endl << flush;
+    inFile.close();
+  }
+  catch( ... ) {
+    if(log) *log << "CCACHE: Caught unknown exception while reading." << endl << flush;
+    inFile.close();
+  }
+
+  return readturn;
+}
+
+unsigned long configcache::hash(const string& str)
+{
+  unsigned long hash = 5381;
+
+  for(string::const_iterator it=str.begin();it!=str.end();it++) 
+    hash = ((hash << 5) + hash) + *it; /* hash * 33 + character */
+
+  return hash;
+}
+
+void configcache::deleteHeaderStore()
+{
+  while ( headerStore.size() > 0 ) {
+    free(headerStore.back().second);
+    headerStore.pop_back();
+  }
+}
+
+vector<infiledesc>::iterator configcache::getNextInfile(vector<unsigned long> *excludeFileHashes) {
+  for (vector<infiledesc>::iterator init = inFiles.begin(); init != inFiles.end(); ++init) {
+    if (excludeFileHashes != NULL) {
+      bool excludethisfile = false;
+
+      for (vector<unsigned long>::iterator exit = excludeFileHashes->begin(); exit != excludeFileHashes->end(); ++exit)
+	if ( *exit == hash(init->filename) ) {
+	  excludethisfile = true;
+	  break;
+	}
+
+      if (excludethisfile)
+	continue;
+    }
+    return init;
+  }
+  return inFiles.end();
+}