X-Git-Url: http://git.treefish.org/~alex/phys/latlib.git/blobdiff_plain/b80a3fda9209144605a6306ba24ec2051d387f50..8d11db1cf57b103fcec701d71c9d620b1c381b73:/configcache.cpp?ds=inline diff --git a/configcache.cpp b/configcache.cpp index b375230..8687123 100644 --- a/configcache.cpp +++ b/configcache.cpp @@ -4,41 +4,74 @@ #include #include #include +#include +#include +#include + +#include +#include +#include +#include +#include #define HEADER_READOK 0 #define HEADER_READERR 1 #define HEADER_READLAST 2 +struct configcache::iobuffers +{ + boost::iostreams::filtering_ostreambuf *out; +}; + configcache::configcache(const string& cacheid, const int& nequi, const int& nskip, const string& datadir, char **configmem, const int& configMemSize, const int& cachemode, - writeout *out_a){ - out = out_a; + ostream *_log){ + log = _log; + dataReader = new datread(configMemSize, log); + NEQUI = nequi; NSKIP = nskip; DATADIR = datadir; CACHEID = cacheid; + if ( cacheid.find("_") != -1 ) { + if(log) *log << "CCACHE: Invalid cacheid \"" << cacheid << "\" given. Cacheids must not contain underscores!" << endl << flush; + exit(1); + } + configMem = (char*)malloc(configMemSize); tmpConfig = (char*)malloc(configMemSize); *configmem = configMem; configSize = configMemSize; - outBuffer = NULL; - inBuffer = NULL; + ioBuffers = new iobuffers; + ioBuffers->out = NULL; MODE = cachemode; refetchDataFiles = false; } -string configcache::getFileId(const bool& shortid) +string configcache::paraString() { + stringstream parastring; + + for(int ipara=0; iparafilename = infile; - filedesc->doVirtualEquilibration = false; if( infile.size() < 4 ) return false; - if( infile.substr(infile.size()-4) == ".dat" ) + if( infile.substr(infile.size()-4) == ".dat" ) { filedesc->extended = false; - else if( infile.substr(infile.size()-4) == "edat" ) + filedesc->superextended = false; + } + else if( infile.substr(infile.size()-4) == "edat" ) { + filedesc->extended = true; + filedesc->superextended = false; + } + else if( infile.substr(infile.size()-4) == "sdat" ) { filedesc->extended = true; + filedesc->superextended = true; + } else return false; inchar = new char [infile.size()+1]; strcpy (inchar, infile.c_str()); - + inParts = strtok( inchar, "_" ); for(int iPart=0; inParts!=NULL; iPart++) { @@ -88,12 +128,10 @@ bool configcache::isValidInFile(const string& infile, infiledesc *filedesc) switch(iPart) { - case 1: if(inParts != CACHEID) return false; break; - case 2: - if (atoi(inParts) > NEQUI) - return false; - else if (atoi(inParts) < NEQUI) - filedesc->doVirtualEquilibration = true; + case 1: if(inParts != CACHEID) + return false; + break; + case 2: filedesc->nequi = atoi(inParts); break; case 3: @@ -108,73 +146,20 @@ bool configcache::isValidInFile(const string& infile, infiledesc *filedesc) delete[] inchar; - if( truncIn.find( getFileId(true) + "_" ) == string::npos ) return false; + if( truncIn.find( getFileId(NEQUI, filedesc->superextended, true) + "_" ) == string::npos ) return false; return true; } -int configcache::readHeader() -{ - long unsigned int headersize; - - if( readDataToMem((char *)&headersize, sizeof(long unsigned int)) == sizeof(long unsigned int) && inFile.is_open() ) { - if ( headersize == 0 ) - return HEADER_READLAST; - - pair newHeader; - - if( readDataToMem((char *)&newHeader.first, sizeof(unsigned long)) == sizeof(unsigned long) && inFile.is_open() ) { - newHeader.second = malloc(headersize); - - if( readDataToMem((char *)newHeader.second, headersize) == headersize && inFile.is_open() ) { - headerStore.push_back(newHeader); - return HEADER_READOK; - } - else { - if(out) *out->log << "CCACHE: Could not read heade-data! Closing dat-file: " << openFileDesc.filename << endl << flush; - inFile.close(); - return HEADER_READERR; - } - } - else { - if(out) *out->log << "CCACHE: Could not read headerid-hash! Closing dat-file: " << openFileDesc.filename << endl << flush; - inFile.close(); - return HEADER_READERR; - } - } - else { - if(out) *out->log << "CCACHE: Could not read header size. Closing dat-file: " << openFileDesc.filename << endl << flush; - inFile.close(); - return HEADER_READERR; - } +void * configcache::getHeader(const string& headerid) { + return dataReader->getHeader(headerid); } -bool configcache::readAllHeaders() +void configcache::readConfig(bool *readnewconfig, int *nequileft, vector *excludeFileHashes) { - int readHeaderStatus; - - deleteHeaderStore(); - - do { - readHeaderStatus = readHeader(); - } - while ( readHeaderStatus == HEADER_READOK ); + *readnewconfig = false; - if ( readHeaderStatus == HEADER_READLAST ) return true; - else if ( readHeaderStatus == HEADER_READERR ) return false; -} - -void * configcache::getHeader(const string& headerid) { - for (vector< pair >::iterator headerStoreIt = headerStore.begin(); headerStoreIt != headerStore.end(); ++headerStoreIt) - if ( headerStoreIt->first == hash(headerid) ) - return headerStoreIt->second; - - return NULL; -} - -bool configcache::readConfig(vector *excludefiles) -{ - if(DATADIR == "" || MODE == CACHE_MODE_DISABLED) return false; + if( DATADIR == "" || !(MODE==CACHE_MODE_RO||MODE==CACHE_MODE_RW) ) return; if(refetchDataFiles){ refetchDataFiles = false; @@ -183,101 +168,138 @@ bool configcache::readConfig(vector *excludefiles) while(true) { - if( (!inFile.is_open()) && inFiles.size() == 0 ) return false; + vector::iterator inFileIt = getNextInfile(excludeFileHashes); + int iDidVirtualSkips; - while( (!inFile.is_open()) && inFiles.size() > 0 ) { - bool excludethisfile=false; + if( (!dataReader->fisopen()) && inFileIt == inFiles.end() ) { + if (*readnewconfig) + *nequileft = nequileft_internal; + return; + } - openFileDesc = inFiles.back(); + while( (!dataReader->fisopen()) && inFiles.size() > 0 ) { + string inFileParaString; - if (excludefiles != NULL) - for (vector::iterator exit = excludefiles->begin(); exit != excludefiles->end(); ++exit) - if ( *exit == inFiles.back().filename ) { - excludethisfile = true; - break; - } - - if ( ! excludethisfile ) { - if(out) *out->log << "CCACHE: Opening dat-file: " << inFiles.back().filename << endl << flush; - inFile.open( (DATADIR + "/" + inFiles.back().filename).c_str(), std::ios::binary ); - } + openFileDesc = *inFileIt; + + if (openFileDesc.nequi < NEQUI) + doVirtualEquilibration = true; else - if(out) *out->log << "CCACHE: Excluded dat-file: " << inFiles.back().filename << endl << flush; - - inFiles.pop_back(); - - if( !inFile.is_open() ) continue; + doVirtualEquilibration = false; + + firstUsedConfig = true; + + if(log) *log << "CCACHE: Opening dat-file: " << inFileIt->filename << endl << flush; - inBuffer = new boost::iostreams::filtering_istreambuf; - inBuffer->push( boost::iostreams::bzip2_decompressor() ); - inBuffer->push(inFile); + dataReader->openFile( DATADIR + "/" + inFileIt->filename ); + + if( openFileDesc.superextended ) { + if( dataReader->getParaString() != paraString() ) { + if(log) *log << "CCACHE: Parastring does not match. Closing dat-file..." << endl << flush; + dataReader->closeFile(); + } + } + + inFiles.erase(inFileIt); + + if( !dataReader->fisopen() ) continue; } - if( inFile.is_open() ) + if( dataReader->fisopen() ) { - if (openFileDesc.doVirtualEquilibration) { - if(out) *out->log << "CCACHE: Trying virtual equilibration." << endl << flush; - openFileDesc.doVirtualEquilibration = false; - for (int iskip=0; iskip < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iskip++) { - if( readFullBlock(tmpConfig, configSize) != configSize || ! inFile.is_open() ) + if (doVirtualEquilibration) { + if(log) *log << "CCACHE: Trying virtual equilibration." << endl << flush; + doVirtualEquilibration = false; + for (iDidVirtualSkips=0; iDidVirtualSkips < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iDidVirtualSkips++) { + if( dataReader->readFullBlock(tmpConfig) < 0 ) break; + else if ( (NEQUI-openFileDesc.nequi) - (iDidVirtualSkips+1)*openFileDesc.nskip < nequileft_internal ) { + memcpy(configMem, tmpConfig, configSize); + nequileft_internal = NEQUI - openFileDesc.nequi - (iDidVirtualSkips+1)*openFileDesc.nskip; + *readnewconfig = true; + firstUsedConfig = false; + } } } - if( readFullBlock(tmpConfig, configSize) == configSize && inFile.is_open() ) + if( dataReader->readFullBlock(tmpConfig) >= 0 ) { memcpy(configMem, tmpConfig, configSize); - return true; + *readnewconfig = true; + if (firstUsedConfig) { + firstUsedConfig = false; + if (openFileDesc.nequi < NEQUI) + nequileft_internal = NEQUI - openFileDesc.nequi - iDidVirtualSkips*openFileDesc.nskip; + else + nequileft_internal = NEQUI - openFileDesc.nequi; + } + nequileft_internal -= openFileDesc.nskip; + *nequileft = nequileft_internal; + return; } - else { - if(out) *out->log << "CCACHE: Could not read configuration. Closing dat-file: " << openFileDesc.filename << endl << flush; - inFile.close(); - } } } } -void configcache::openOutFile() -{ +void configcache::openOutFile(int actnequi) +{ time_t secstamp = time(NULL); + int iseq=0; + + while (true) { + outFileName.str(""); + outFileName << DATADIR << "/" << secstamp << "." << iseq << "_" << getFileId(actnequi, true, false) << "_.sdat.tmp"; + + int tmpfd = open(outFileName.str().c_str(), O_CREAT | O_EXCL, 0644); - outFileName.str(""); - outFileName << DATADIR << "/" << secstamp << "_" << getFileId() << "_.edat.tmp"; + if ( tmpfd != -1 ) { + close(tmpfd); + break; + } + else if ( errno != EEXIST ) { + if(log) *log << "CCACHE: Could not create cachefile!" << endl << flush; + exit(1); + } + iseq++; + } + outFile.open( outFileName.str().c_str(), std::ios::binary ); - outBuffer = new boost::iostreams::filtering_ostreambuf; - outBuffer->push(boost::iostreams::bzip2_compressor()); - outBuffer->push(outFile); + outFile << paraString() << endl; + + ioBuffers->out = new boost::iostreams::filtering_ostreambuf; + ioBuffers->out->push(boost::iostreams::bzip2_compressor()); + ioBuffers->out->push(outFile); } -void configcache::writeHeader(const string& headerid, const char *header, long unsigned int size) { +void configcache::writeHeader(const string& headerid, const char *header, long unsigned int size, int actnequi) { unsigned long headeridhash; - if( DATADIR == "" || MODE < 2 ) return; + if( DATADIR == "" || !(MODE==CACHE_MODE_WO||MODE==CACHE_MODE_RW) ) return; if(!outFile.is_open()) - openOutFile(); + openOutFile(actnequi); headeridhash = hash(headerid); - boost::iostreams::write(*outBuffer, (char*)&size, sizeof(long unsigned int)); - boost::iostreams::write(*outBuffer, (char*)&headeridhash, sizeof(unsigned long)); - boost::iostreams::write(*outBuffer, header, size); + boost::iostreams::write(*ioBuffers->out, (char*)&size, sizeof(long unsigned int)); + boost::iostreams::write(*ioBuffers->out, (char*)&headeridhash, sizeof(unsigned long)); + boost::iostreams::write(*ioBuffers->out, header, size); } -void configcache::writeConfig() +void configcache::writeConfig(int actnequi) { long unsigned int zeroheader=0; - if ( DATADIR == "" || MODE < 2 ) return; + if ( DATADIR == "" || !(MODE==CACHE_MODE_WO||MODE==CACHE_MODE_RW) ) return; if ( ! outFile.is_open() ) - openOutFile(); + openOutFile(actnequi); - boost::iostreams::write(*outBuffer, (char*)&zeroheader, sizeof(long unsigned int)); + boost::iostreams::write(*ioBuffers->out, (char*)&zeroheader, sizeof(long unsigned int)); - boost::iostreams::write(*outBuffer, configMem, configSize); + boost::iostreams::write(*ioBuffers->out, configMem, configSize); } void configcache::addPara(const string& parid, const double& val){ @@ -294,27 +316,27 @@ int configcache::getParIndex(const string& parid){ void configcache::setPara(const string& parid, const double& value){ Paras[getParIndex(parid)].val = value; + finishOutFile(); - if(inBuffer != NULL) { delete inBuffer; inBuffer=NULL; } - inFile.close(); + dataReader->closeFile(); inFiles.clear(); refetchDataFiles = true; + nequileft_internal = NEQUI; } configcache::~configcache() { finishOutFile(); - delete inBuffer; - inBuffer = NULL; + delete dataReader; } void configcache::finishOutFile() { - if( outBuffer != NULL ) + if( ioBuffers->out != NULL ) { - delete outBuffer; - outBuffer = NULL; + delete ioBuffers->out; + ioBuffers->out = NULL; } if( outFile.is_open() ) @@ -324,40 +346,6 @@ void configcache::finishOutFile() } } -int configcache::readFullBlock(char *tmpData, long unsigned int dataSize) -{ - /* try to read header */ - if ( openFileDesc.extended ) - if ( ! readAllHeaders() ) - return -1; - - /* read data */ - return readDataToMem(tmpData, dataSize); -} - -int configcache::readDataToMem(char *tmpData, long unsigned int dataSize) -{ - int readturn = -1; - - if ( dataSize == 0 ) return 0; - - try { readturn = boost::iostreams::read(*inBuffer, tmpData, dataSize); } - catch(boost::iostreams::bzip2_error& error) { - if(out) *out->log << "CCACHE: Caught bzip2 exception with error code: " << error.error() << endl << flush; - inFile.close(); - } - catch (std::exception const& ex) { - if(out) *out->log << "CCACHE: Caught exception: " << ex.what() << endl << flush; - inFile.close(); - } - catch( ... ) { - if(out) *out->log << "CCACHE: Caught unknown exception while reading." << endl << flush; - inFile.close(); - } - - return readturn; -} - unsigned long configcache::hash(const string& str) { unsigned long hash = 5381; @@ -368,10 +356,21 @@ unsigned long configcache::hash(const string& str) return hash; } -void configcache::deleteHeaderStore() -{ - while ( headerStore.size() > 0 ) { - free(headerStore.back().second); - headerStore.pop_back(); +vector::iterator configcache::getNextInfile(vector *excludeFileHashes) { + for (vector::iterator init = inFiles.begin(); init != inFiles.end(); ++init) { + if (excludeFileHashes != NULL) { + bool excludethisfile = false; + + for (vector::iterator exit = excludeFileHashes->begin(); exit != excludeFileHashes->end(); ++exit) + if ( *exit == hash(init->filename) ) { + excludethisfile = true; + break; + } + + if (excludethisfile) + continue; + } + return init; } + return inFiles.end(); }