only storing concurrent config file HASHES instead of full names.

[phys/latlib.git] / configcache.cpp

diff --git a/configcache.cpp b/configcache.cpp
index 1b227578bea85c73835bffce7c343674340d109c..1095775e8b097dd9afaf8a641d6a10f17630619c 100644 (file)
--- a/configcache.cpp
+++ b/configcache.cpp
@@ -5,6 +5,10 @@
 #include <time.h>
 #include <dirent.h>
 
 #include <time.h>
 #include <dirent.h>
 

+#define HEADER_READOK   0
+#define HEADER_READERR  1
+#define HEADER_READLAST 2
+

 configcache::configcache(const string& cacheid, const int& nequi, const int& nskip, const string& datadir, char **configmem, const int& configMemSize, const int& cachemode,
                         writeout *out_a){
   out = out_a;
 configcache::configcache(const string& cacheid, const int& nequi, const int& nskip, const string& datadir, char **configmem, const int& configMemSize, const int& cachemode,
                         writeout *out_a){
   out = out_a;


@@ -109,57 +113,67 @@ bool configcache::isValidInFile(const string& infile, infiledesc *filedesc)
   return true;
 }
 
   return true;
 }
 

-bool configcache::openInFile() {
-  while( (!inFile.is_open()) && inFiles.size() > 0 )
-    {
-      if(out) *out->log << "CCACHE: Opening dat-file: " << inFiles.back().filename << endl << flush;
-
-      openFileDesc = inFiles.back();
-      inFile.open( (DATADIR + "/" + inFiles.back().filename).c_str(), std::ios::binary );
-      inFiles.pop_back();
-
-      if( !inFile.is_open() ) continue;
-
-      inBuffer = new boost::iostreams::filtering_istreambuf;
-      inBuffer->push( boost::iostreams::bzip2_decompressor() );
-      inBuffer->push(inFile);
-
-      return true;
-    }
-
-  return false;
-}
-
-void configcache::readHeader()
+int configcache::readHeader()

 {
 {

-  int headersize;
+  long unsigned int headersize;

   
   

-  if( readDataToMem((char*)&headersize, sizeof(int)) == sizeof(int) && inFile.is_open() )
-    {
-      if( headerData != NULL ) delete headerData;
+  if( readDataToMem((char *)&headersize, sizeof(long unsigned int)) == sizeof(long unsigned int) && inFile.is_open() ) {
+    if ( headersize == 0 )
+      return HEADER_READLAST;
+
+    pair<unsigned long, void *> newHeader;
+
+    if( readDataToMem((char *)&newHeader.first, sizeof(unsigned long)) == sizeof(unsigned long) && inFile.is_open() ) {
+      newHeader.second = malloc(headersize);

 
 

-      headerData = new char(headersize);
-      
-      if( readDataToMem(headerData, headersize) == headersize && inFile.is_open() ) {
-       if(out) *out->log << "CCACHE: Read header information." << endl << flush;
+      if( readDataToMem((char *)newHeader.second, headersize) == headersize && inFile.is_open() ) {
+       headerStore.push_back(newHeader);
+       return HEADER_READOK;

       }
       else {
       }
       else {

-       if(out) *out->log << "CCACHE: Could not read header! Closing dat-file: " << openFileDesc.filename << endl << flush;
+       if(out) *out->log << "CCACHE: Could not read heade-data! Closing dat-file: " << openFileDesc.filename << endl << flush;

        inFile.close();
        inFile.close();

+       return HEADER_READERR;

       }
     }
       }
     }

+    else {
+      if(out) *out->log << "CCACHE: Could not read headerid-hash! Closing dat-file: " << openFileDesc.filename << endl << flush;
+      inFile.close();
+      return HEADER_READERR;
+    }
+  }

   else {
   else {

-    if(out) *out->log << "CCACHE: Could not read header size! Closing dat-file: " << openFileDesc.filename << endl << flush;
+    if(out) *out->log << "CCACHE: Could not read header size. Closing dat-file: " << openFileDesc.filename << endl << flush;

     inFile.close();
     inFile.close();

+    return HEADER_READERR;

   }
 }
 
   }
 }
 

-void *configcache::getHeader() {
-  return headerData;
+bool configcache::readAllHeaders()
+{
+  int readHeaderStatus;
+
+  deleteHeaderStore();
+  
+  do {
+    readHeaderStatus = readHeader();
+  }
+  while ( readHeaderStatus == HEADER_READOK );
+
+  if ( readHeaderStatus == HEADER_READLAST ) return true;
+  else if ( readHeaderStatus == HEADER_READERR ) return false;

 }
 
 }
 

-bool configcache::readConfig()
-{
+void * configcache::getHeader(const string& headerid) {
+  for (vector< pair<unsigned long, void *> >::iterator headerStoreIt = headerStore.begin(); headerStoreIt != headerStore.end(); ++headerStoreIt)
+    if ( headerStoreIt->first == hash(headerid) )
+      return headerStoreIt->second;
+  
+  return NULL;
+}
+
+bool configcache::readConfig(vector<unsigned long> *excludeFileHashes)
+{   

   if(DATADIR == "" || MODE == CACHE_MODE_DISABLED) return false;
 
   if(refetchDataFiles){
   if(DATADIR == "" || MODE == CACHE_MODE_DISABLED) return false;
 
   if(refetchDataFiles){


@@ -171,9 +185,33 @@ bool configcache::readConfig()
     {
       if( (!inFile.is_open()) && inFiles.size() == 0 ) return false;
 
     {
       if( (!inFile.is_open()) && inFiles.size() == 0 ) return false;
 

-      /* read header information */
-      if ( openInFile() && openFileDesc.extended )
-       readHeader();
+      while( (!inFile.is_open()) && inFiles.size() > 0 ) {
+       bool excludethisfile=false;
+
+       openFileDesc = inFiles.back();
+
+       if (excludeFileHashes != NULL)
+         for (vector<unsigned long>::iterator exit = excludeFileHashes->begin(); exit != excludeFileHashes->end(); ++exit)
+           if ( *exit == hash(inFiles.back().filename) ) {
+             excludethisfile = true;
+             break;
+           }
+             
+       if ( ! excludethisfile ) {
+         if(out) *out->log << "CCACHE: Opening dat-file: " << inFiles.back().filename << endl << flush;
+         inFile.open( (DATADIR + "/" + inFiles.back().filename).c_str(), std::ios::binary );
+       }
+       else
+         if(out) *out->log << "CCACHE: Excluded dat-file: " << inFiles.back().filename << endl << flush;
+       
+       inFiles.pop_back();
+       
+       if( !inFile.is_open() ) continue;
+
+       inBuffer = new boost::iostreams::filtering_istreambuf;
+       inBuffer->push( boost::iostreams::bzip2_decompressor() );
+       inBuffer->push(inFile);
+      }

 
       if( inFile.is_open() ) 
        {
 
       if( inFile.is_open() ) 
        {


@@ -181,12 +219,12 @@ bool configcache::readConfig()
            if(out) *out->log << "CCACHE: Trying virtual equilibration." << endl << flush;
            openFileDesc.doVirtualEquilibration = false;
            for (int iskip=0; iskip < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iskip++) {
            if(out) *out->log << "CCACHE: Trying virtual equilibration." << endl << flush;
            openFileDesc.doVirtualEquilibration = false;
            for (int iskip=0; iskip < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iskip++) {

-             if( readDataToMem(tmpConfig, configSize) != configSize || ! inFile.is_open() )
+             if( readFullBlock(tmpConfig, configSize) != configSize || ! inFile.is_open() )

                break;
            }
          }
 
                break;
            }
          }
 

-         if( readDataToMem(tmpConfig, configSize) == configSize && inFile.is_open() )
+         if( readFullBlock(tmpConfig, configSize) == configSize && inFile.is_open() )

            {
              memcpy(configMem, tmpConfig, configSize);
              return true;
            {
              memcpy(configMem, tmpConfig, configSize);
              return true;


@@ -204,7 +242,8 @@ void configcache::openOutFile()
   time_t secstamp = time(NULL);
 
   outFileName.str("");
   time_t secstamp = time(NULL);
 
   outFileName.str("");

-  outFileName << DATADIR << "/" << secstamp << "_" << getFileId() << "_.edat.tmp";    
+  outFileName << DATADIR << "/" << secstamp << "_" << getFileId() << "_.edat.tmp";
+

   outFile.open( outFileName.str().c_str(), std::ios::binary );
 
   outBuffer = new boost::iostreams::filtering_ostreambuf;
   outFile.open( outFileName.str().c_str(), std::ios::binary );
 
   outBuffer = new boost::iostreams::filtering_ostreambuf;


@@ -212,26 +251,31 @@ void configcache::openOutFile()
   outBuffer->push(outFile);
 }
 
   outBuffer->push(outFile);
 }
 

-void configcache::writeHeader(char *header, const int& size) {
+void configcache::writeHeader(const string& headerid, const char *header, long unsigned int size) {
+  unsigned long headeridhash;
+

   if( DATADIR == "" || MODE < 2 ) return;
 
   if( DATADIR == "" || MODE < 2 ) return;
 

-  if(!outFile.is_open()) {
+  if(!outFile.is_open())

     openOutFile();
     openOutFile();

-    boost::iostreams::write(*outBuffer, header, size);
-  }
-  else
-    if(out) *out->log << "CCACHE: Not writing header because outfile is already open!" << endl << flush;
+
+  headeridhash = hash(headerid);
+
+  boost::iostreams::write(*outBuffer, (char*)&size, sizeof(long unsigned int));
+  boost::iostreams::write(*outBuffer, (char*)&headeridhash, sizeof(unsigned long));
+  boost::iostreams::write(*outBuffer, header, size);

 }
 
 void configcache::writeConfig()
 {
 }
 
 void configcache::writeConfig()
 {

-  if( DATADIR == "" || MODE < 2 ) return;
+  long unsigned int zeroheader=0;

 
 

-  if(!outFile.is_open()){
-    int zeroheader=0;
+  if ( DATADIR == "" || MODE < 2 ) return;
+
+  if ( ! outFile.is_open() )

     openOutFile();
     openOutFile();

-    boost::iostreams::write(*outBuffer, (char*)&zeroheader, sizeof(int));
-  }
+  
+  boost::iostreams::write(*outBuffer, (char*)&zeroheader, sizeof(long unsigned int));

 
   boost::iostreams::write(*outBuffer, configMem, configSize);
 }
 
   boost::iostreams::write(*outBuffer, configMem, configSize);
 }


@@ -280,6 +324,17 @@ void configcache::finishOutFile()
     }
 }
 
     }
 }
 

+int configcache::readFullBlock(char *tmpData, long unsigned int dataSize)
+{
+  /* try to read header */
+  if ( openFileDesc.extended )
+    if ( ! readAllHeaders() ) 
+      return -1;
+
+  /* read data */
+  return readDataToMem(tmpData, dataSize);
+}
+

 int configcache::readDataToMem(char *tmpData, long unsigned int dataSize)
 {
   int readturn = -1;
 int configcache::readDataToMem(char *tmpData, long unsigned int dataSize)
 {
   int readturn = -1;


@@ -302,3 +357,21 @@ int configcache::readDataToMem(char *tmpData, long unsigned int dataSize)
 
   return readturn;
 }
 
   return readturn;
 }

+
+unsigned long configcache::hash(const string& str)
+{
+  unsigned long hash = 5381;
+
+  for(string::const_iterator it=str.begin();it!=str.end();it++) 
+    hash = ((hash << 5) + hash) + *it; /* hash * 33 + character */
+
+  return hash;
+}
+
+void configcache::deleteHeaderStore()
+{
+  while ( headerStore.size() > 0 ) {
+    free(headerStore.back().second);
+    headerStore.pop_back();
+  }
+}