X-Git-Url: http://git.treefish.org/~alex/phys/latlib.git/blobdiff_plain/a55a5830c5c98e950d58593697dd4ee5e4384716..a477390b3aa6eea3a7d822a0499816eb0e2bf499:/configcache.cpp?ds=sidebyside diff --git a/configcache.cpp b/configcache.cpp index fcfdd8e..4c7963d 100644 --- a/configcache.cpp +++ b/configcache.cpp @@ -5,15 +5,24 @@ #include #include +#define HEADER_READOK 0 +#define HEADER_READERR 1 +#define HEADER_READLAST 2 + configcache::configcache(const string& cacheid, const int& nequi, const int& nskip, const string& datadir, char **configmem, const int& configMemSize, const int& cachemode, - writeout *out_a){ - out = out_a; + ostream *_log){ + log = _log; NEQUI = nequi; NSKIP = nskip; DATADIR = datadir; CACHEID = cacheid; + if ( cacheid.find("_") != -1 ) { + if(log) *log << "CCACHE: Invalid cacheid \"" << cacheid << "\" given. Cacheids must not contain underscores!" << endl << flush; + exit(1); + } + configMem = (char*)malloc(configMemSize); tmpConfig = (char*)malloc(configMemSize); @@ -26,15 +35,13 @@ configcache::configcache(const string& cacheid, const int& nequi, const int& nsk MODE = cachemode; refetchDataFiles = false; - - readHeaderData = NULL; } -string configcache::getFileId(const bool& shortid) +string configcache::getFileId(int actnequi, const bool& shortid) { stringstream fileid; - if(!shortid) fileid << CACHEID << "_" << NEQUI << "_" << NSKIP; + if(!shortid) fileid << CACHEID << "_" << actnequi << "_" << NSKIP; for(int ipara=0; iparafilename = infile; - filedesc->doVirtualEquilibration = false; if( infile.size() < 4 ) return false; @@ -78,7 +84,7 @@ bool configcache::isValidInFile(const string& infile, infiledesc *filedesc) inchar = new char [infile.size()+1]; strcpy (inchar, infile.c_str()); - + inParts = strtok( inchar, "_" ); for(int iPart=0; inParts!=NULL; iPart++) { @@ -86,12 +92,10 @@ bool configcache::isValidInFile(const string& infile, infiledesc *filedesc) switch(iPart) { - case 1: if(inParts != CACHEID) return false; break; - case 2: - if (atoi(inParts) > NEQUI) - return false; - else if (atoi(inParts) < NEQUI) - filedesc->doVirtualEquilibration = true; + case 1: if(inParts != CACHEID) + return false; + break; + case 2: filedesc->nequi = atoi(inParts); break; case 3: @@ -106,44 +110,77 @@ bool configcache::isValidInFile(const string& infile, infiledesc *filedesc) delete[] inchar; - if( truncIn.find( getFileId(true) + "_" ) == string::npos ) return false; + if( truncIn.find( getFileId(NEQUI, true) + "_" ) == string::npos ) return false; return true; } -bool configcache::readHeader() +int configcache::readHeader() { long unsigned int headersize; + + if( readDataToMem((char *)&headersize, sizeof(long unsigned int)) == sizeof(long unsigned int) && inFile.is_open() ) { + if ( headersize == 0 ) + return HEADER_READLAST; - if( readDataToMem((char*)&headersize, sizeof(long unsigned int)) == sizeof(long unsigned int) && inFile.is_open() ) - { - if( readHeaderData != NULL ) free(readHeaderData); + pair newHeader; - readHeaderData = (char*) malloc(headersize); + if( readDataToMem((char *)&newHeader.first, sizeof(unsigned long)) == sizeof(unsigned long) && inFile.is_open() ) { + newHeader.second = malloc(headersize); - if( readDataToMem(readHeaderData, headersize) == headersize && inFile.is_open() ) { - return true; + if( readDataToMem((char *)newHeader.second, headersize) == headersize && inFile.is_open() ) { + headerStore.push_back(newHeader); + return HEADER_READOK; } else { - if(out) *out->log << "CCACHE: Could not read header! Closing dat-file: " << openFileDesc.filename << endl << flush; + if(log) *log << "CCACHE: Could not read heade-data! Closing dat-file: " << openFileDesc.filename << endl << flush; inFile.close(); - return false; + return HEADER_READERR; } } + else { + if(log) *log << "CCACHE: Could not read headerid-hash! Closing dat-file: " << openFileDesc.filename << endl << flush; + inFile.close(); + return HEADER_READERR; + } + } else { - if(out) *out->log << "CCACHE: Could not read header size. Closing dat-file: " << openFileDesc.filename << endl << flush; + if(log) *log << "CCACHE: Could not read header size. Closing dat-file: " << openFileDesc.filename << endl << flush; inFile.close(); - return false; + return HEADER_READERR; } } -void *configcache::getHeader() { - return readHeaderData; +bool configcache::readAllHeaders() +{ + int readHeaderStatus; + + deleteHeaderStore(); + + do { + readHeaderStatus = readHeader(); + } + while ( readHeaderStatus == HEADER_READOK ); + + if ( readHeaderStatus == HEADER_READLAST ) return true; + else if ( readHeaderStatus == HEADER_READERR ) return false; +} + +void * configcache::getHeader(const string& headerid) { + for (vector< pair >::iterator headerStoreIt = headerStore.begin(); headerStoreIt != headerStore.end(); ++headerStoreIt) + if ( headerStoreIt->first == hash(headerid) ) + return headerStoreIt->second; + + return NULL; } -bool configcache::readConfig() +/* returns number of equilibration-steps left with new read configuration + or zero if no new configuration was read */ +void configcache::readConfig(bool& readnewconfig, int& nequileft, vector *excludeFileHashes) { - if(DATADIR == "" || MODE == CACHE_MODE_DISABLED) return false; + readnewconfig = false; + + if(DATADIR == "" || MODE == CACHE_MODE_DISABLED) return; if(refetchDataFiles){ refetchDataFiles = false; @@ -152,15 +189,29 @@ bool configcache::readConfig() while(true) { - if( (!inFile.is_open()) && inFiles.size() == 0 ) return false; + vector::iterator inFileIt = getNextInfile(excludeFileHashes); + int iDidVirtualSkips; + + if( (!inFile.is_open()) && inFileIt == inFiles.end() ) { + nequileft = nequileft_internal; + return; + } while( (!inFile.is_open()) && inFiles.size() > 0 ) { - if(out) *out->log << "CCACHE: Opening dat-file: " << inFiles.back().filename << endl << flush; + openFileDesc = *inFileIt; + + if (openFileDesc.nequi < NEQUI) + doVirtualEquilibration = true; + else + doVirtualEquilibration = false; - openFileDesc = inFiles.back(); - inFile.open( (DATADIR + "/" + inFiles.back().filename).c_str(), std::ios::binary ); - inFiles.pop_back(); + firstUsedConfig = true; + if(log) *log << "CCACHE: Opening dat-file: " << inFileIt->filename << endl << flush; + inFile.open( (DATADIR + "/" + inFileIt->filename).c_str(), std::ios::binary ); + + inFiles.erase(inFileIt); + if( !inFile.is_open() ) continue; inBuffer = new boost::iostreams::filtering_istreambuf; @@ -170,69 +221,85 @@ bool configcache::readConfig() if( inFile.is_open() ) { - if (openFileDesc.doVirtualEquilibration) { - if(out) *out->log << "CCACHE: Trying virtual equilibration." << endl << flush; - openFileDesc.doVirtualEquilibration = false; - for (int iskip=0; iskip < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iskip++) { + if (doVirtualEquilibration) { + if(log) *log << "CCACHE: Trying virtual equilibration." << endl << flush; + doVirtualEquilibration = false; + for (iDidVirtualSkips=0; iDidVirtualSkips < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iDidVirtualSkips++) { if( readFullBlock(tmpConfig, configSize) != configSize || ! inFile.is_open() ) break; + else if ( (NEQUI-openFileDesc.nequi) - (iDidVirtualSkips+1)*openFileDesc.nskip < nequileft_internal ) { + memcpy(configMem, tmpConfig, configSize); + nequileft_internal = NEQUI - openFileDesc.nequi - (iDidVirtualSkips+1)*openFileDesc.nskip; + readnewconfig = true; + firstUsedConfig = false; + } } } if( readFullBlock(tmpConfig, configSize) == configSize && inFile.is_open() ) { memcpy(configMem, tmpConfig, configSize); - return true; + readnewconfig = true; + if (firstUsedConfig) { + firstUsedConfig = false; + if (openFileDesc.nequi < NEQUI) + nequileft_internal = NEQUI - openFileDesc.nequi - iDidVirtualSkips*openFileDesc.nskip; + else + nequileft_internal = NEQUI - openFileDesc.nequi; + } + nequileft_internal -= openFileDesc.nskip; + nequileft = nequileft_internal; + return; } else { - if(out) *out->log << "CCACHE: Could not read configuration. Closing dat-file: " << openFileDesc.filename << endl << flush; + if(log) *log << "CCACHE: Could not read configuration. Closing dat-file: " << openFileDesc.filename << endl << flush; inFile.close(); } } } } -void configcache::openOutFile() +void configcache::openOutFile(int actnequi) { time_t secstamp = time(NULL); outFileName.str(""); - outFileName << DATADIR << "/" << secstamp << "_" << getFileId() << "_.edat.tmp"; + outFileName << DATADIR << "/" << secstamp << "_" << getFileId(actnequi) << "_.edat.tmp"; + outFile.open( outFileName.str().c_str(), std::ios::binary ); outBuffer = new boost::iostreams::filtering_ostreambuf; outBuffer->push(boost::iostreams::bzip2_compressor()); outBuffer->push(outFile); - - headerWritten = false; } -void configcache::writeHeader(char *header, long unsigned int size) { +void configcache::writeHeader(const string& headerid, const char *header, long unsigned int size, int actnequi) { + unsigned long headeridhash; + if( DATADIR == "" || MODE < 2 ) return; if(!outFile.is_open()) - openOutFile(); + openOutFile(actnequi); + + headeridhash = hash(headerid); boost::iostreams::write(*outBuffer, (char*)&size, sizeof(long unsigned int)); + boost::iostreams::write(*outBuffer, (char*)&headeridhash, sizeof(unsigned long)); boost::iostreams::write(*outBuffer, header, size); - - headerWritten = true; } -void configcache::writeConfig() +void configcache::writeConfig(int actnequi) { + long unsigned int zeroheader=0; + if ( DATADIR == "" || MODE < 2 ) return; if ( ! outFile.is_open() ) - openOutFile(); + openOutFile(actnequi); - if ( ! headerWritten ) { - long unsigned int zeroheader=0; - boost::iostreams::write(*outBuffer, (char*)&zeroheader, sizeof(long unsigned int)); - } + boost::iostreams::write(*outBuffer, (char*)&zeroheader, sizeof(long unsigned int)); boost::iostreams::write(*outBuffer, configMem, configSize); - headerWritten = false; } void configcache::addPara(const string& parid, const double& val){ @@ -255,6 +322,7 @@ void configcache::setPara(const string& parid, const double& value){ inFiles.clear(); refetchDataFiles = true; + nequileft_internal = NEQUI; } configcache::~configcache() @@ -283,7 +351,7 @@ int configcache::readFullBlock(char *tmpData, long unsigned int dataSize) { /* try to read header */ if ( openFileDesc.extended ) - if ( ! readHeader() ) + if ( ! readAllHeaders() ) return -1; /* read data */ @@ -298,17 +366,54 @@ int configcache::readDataToMem(char *tmpData, long unsigned int dataSize) try { readturn = boost::iostreams::read(*inBuffer, tmpData, dataSize); } catch(boost::iostreams::bzip2_error& error) { - if(out) *out->log << "CCACHE: Caught bzip2 exception with error code: " << error.error() << endl << flush; + if(log) *log << "CCACHE: Caught bzip2 exception with error code: " << error.error() << endl << flush; inFile.close(); } catch (std::exception const& ex) { - if(out) *out->log << "CCACHE: Caught exception: " << ex.what() << endl << flush; + if(log) *log << "CCACHE: Caught exception: " << ex.what() << endl << flush; inFile.close(); } catch( ... ) { - if(out) *out->log << "CCACHE: Caught unknown exception while reading." << endl << flush; + if(log) *log << "CCACHE: Caught unknown exception while reading." << endl << flush; inFile.close(); } return readturn; } + +unsigned long configcache::hash(const string& str) +{ + unsigned long hash = 5381; + + for(string::const_iterator it=str.begin();it!=str.end();it++) + hash = ((hash << 5) + hash) + *it; /* hash * 33 + character */ + + return hash; +} + +void configcache::deleteHeaderStore() +{ + while ( headerStore.size() > 0 ) { + free(headerStore.back().second); + headerStore.pop_back(); + } +} + +vector::iterator configcache::getNextInfile(vector *excludeFileHashes) { + for (vector::iterator init = inFiles.begin(); init != inFiles.end(); ++init) { + if (excludeFileHashes != NULL) { + bool excludethisfile = false; + + for (vector::iterator exit = excludeFileHashes->begin(); exit != excludeFileHashes->end(); ++exit) + if ( *exit == hash(init->filename) ) { + excludethisfile = true; + break; + } + + if (excludethisfile) + continue; + } + return init; + } + return inFiles.end(); +}