X-Git-Url: http://git.treefish.org/~alex/phys/latlib.git/blobdiff_plain/e70a895119965e4cebd223da9aa7053acc36720c..2c7e8d0d8fdb233087376cc62752fdc45646c8c8:/configcache.cpp?ds=inline diff --git a/configcache.cpp b/configcache.cpp index 48c31f4..6cd46a1 100644 --- a/configcache.cpp +++ b/configcache.cpp @@ -5,7 +5,14 @@ #include #include -configcache::configcache(const string& cacheid, const int& nequi, const int& nskip, const string& datadir, char **configmem, const int& configMemSize){ +#define HEADER_READOK 0 +#define HEADER_READERR 1 +#define HEADER_READLAST 2 + +configcache::configcache(const string& cacheid, const int& nequi, const int& nskip, const string& datadir, char **configmem, const int& configMemSize, const int& cachemode, + ostream *_log){ + log = _log; + NEQUI = nequi; NSKIP = nskip; DATADIR = datadir; @@ -20,6 +27,8 @@ configcache::configcache(const string& cacheid, const int& nequi, const int& nsk outBuffer = NULL; inBuffer = NULL; + MODE = cachemode; + refetchDataFiles = false; } @@ -38,27 +47,36 @@ void configcache::fetchDataFiles() { struct dirent *de=NULL; DIR *d=NULL; + static infiledesc filedesc; d=opendir(DATADIR.c_str()); if(d != NULL){ while(de = readdir(d)){ string filename = de->d_name; - if(isValidInFile(filename)) + if(isValidInFile(filename, &filedesc)) { - inFiles.push_back(filename); + inFiles.push_back(filedesc); } } } } -bool configcache::isValidInFile(const string& infile) +bool configcache::isValidInFile(const string& infile, infiledesc *filedesc) { char *inchar, *inParts; string truncIn, truncOut; + filedesc->filename = infile; + filedesc->doVirtualEquilibration = false; + if( infile.size() < 4 ) return false; - if( infile.substr(infile.size()-3) != "dat" ) return false; + if( infile.substr(infile.size()-4) == ".dat" ) + filedesc->extended = false; + else if( infile.substr(infile.size()-4) == "edat" ) + filedesc->extended = true; + else + return false; inchar = new char [infile.size()+1]; strcpy (inchar, infile.c_str()); @@ -71,8 +89,18 @@ bool configcache::isValidInFile(const string& infile) switch(iPart) { case 1: if(inParts != CACHEID) return false; break; - case 2: if(atoi(inParts) < NEQUI) return false; break; - case 3: if(atoi(inParts) < NSKIP) return false; break; + case 2: + if (atoi(inParts) > NEQUI) + return false; + else if (atoi(inParts) < NEQUI) + filedesc->doVirtualEquilibration = true; + filedesc->nequi = atoi(inParts); + break; + case 3: + if(atoi(inParts) != NSKIP) + return false; + filedesc->nskip = atoi(inParts); + break; } inParts = strtok( NULL, "_"); } @@ -85,9 +113,71 @@ bool configcache::isValidInFile(const string& infile) return true; } -bool configcache::readConfig() +int configcache::readHeader() +{ + long unsigned int headersize; + + if( readDataToMem((char *)&headersize, sizeof(long unsigned int)) == sizeof(long unsigned int) && inFile.is_open() ) { + if ( headersize == 0 ) + return HEADER_READLAST; + + pair newHeader; + + if( readDataToMem((char *)&newHeader.first, sizeof(unsigned long)) == sizeof(unsigned long) && inFile.is_open() ) { + newHeader.second = malloc(headersize); + + if( readDataToMem((char *)newHeader.second, headersize) == headersize && inFile.is_open() ) { + headerStore.push_back(newHeader); + return HEADER_READOK; + } + else { + if(log) *log << "CCACHE: Could not read heade-data! Closing dat-file: " << openFileDesc.filename << endl << flush; + inFile.close(); + return HEADER_READERR; + } + } + else { + if(log) *log << "CCACHE: Could not read headerid-hash! Closing dat-file: " << openFileDesc.filename << endl << flush; + inFile.close(); + return HEADER_READERR; + } + } + else { + if(log) *log << "CCACHE: Could not read header size. Closing dat-file: " << openFileDesc.filename << endl << flush; + inFile.close(); + return HEADER_READERR; + } +} + +bool configcache::readAllHeaders() { - if(DATADIR == "") return false; + int readHeaderStatus; + + deleteHeaderStore(); + + do { + readHeaderStatus = readHeader(); + } + while ( readHeaderStatus == HEADER_READOK ); + + if ( readHeaderStatus == HEADER_READLAST ) return true; + else if ( readHeaderStatus == HEADER_READERR ) return false; +} + +void * configcache::getHeader(const string& headerid) { + for (vector< pair >::iterator headerStoreIt = headerStore.begin(); headerStoreIt != headerStore.end(); ++headerStoreIt) + if ( headerStoreIt->first == hash(headerid) ) + return headerStoreIt->second; + + return NULL; +} + +/* returns number of equilibration-steps left */ +int configcache::readConfig(vector *excludeFileHashes) +{ + int nequileft = NEQUI; + + if(DATADIR == "" || MODE == CACHE_MODE_DISABLED) return nequileft; if(refetchDataFiles){ refetchDataFiles = false; @@ -96,47 +186,92 @@ bool configcache::readConfig() while(true) { - if( (!inFile.is_open()) && inFiles.size() == 0 ) return false; + vector::iterator inFileIt = getNextInfile(excludeFileHashes); - while( (!inFile.is_open()) && inFiles.size() > 0 ) - { - inFile.open( (DATADIR + "/" + inFiles.back()).c_str(), std::ios::binary ); - inFiles.pop_back(); + if( (!inFile.is_open()) && inFileIt == inFiles.end() ) return nequileft; - if( !inFile.is_open() ) continue; + while( (!inFile.is_open()) && inFiles.size() > 0 ) { + openFileDesc = *inFileIt; - inBuffer = new boost::iostreams::filtering_istreambuf; - inBuffer->push( boost::iostreams::bzip2_decompressor() ); - inBuffer->push(inFile); - } + if(log) *log << "CCACHE: Opening dat-file: " << inFileIt->filename << endl << flush; + inFile.open( (DATADIR + "/" + inFileIt->filename).c_str(), std::ios::binary ); + + inFiles.erase(inFileIt); + + if( !inFile.is_open() ) continue; + + inBuffer = new boost::iostreams::filtering_istreambuf; + inBuffer->push( boost::iostreams::bzip2_decompressor() ); + inBuffer->push(inFile); + } if( inFile.is_open() ) { - if( boost::iostreams::read(*inBuffer, tmpConfig, configSize) == configSize ) + if (openFileDesc.doVirtualEquilibration) { + if(log) *log << "CCACHE: Trying virtual equilibration." << endl << flush; + openFileDesc.doVirtualEquilibration = false; + for (int iskip=0; iskip < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iskip++) { + if( readFullBlock(tmpConfig, configSize) != configSize || ! inFile.is_open() ) + break; + else if ( (NEQUI-openFileDesc.nequi) - (iskip+1)*openFileDesc.nskip < nequileft ) { + memcpy(configMem, tmpConfig, configSize); + nequileft = (NEQUI-openFileDesc.nequi) - (iskip+1)*openFileDesc.nskip; + } + } + } + + if( readFullBlock(tmpConfig, configSize) == configSize && inFile.is_open() ) { memcpy(configMem, tmpConfig, configSize); - return true; + return -1; } - else inFile.close(); + else { + if(log) *log << "CCACHE: Could not read configuration. Closing dat-file: " << openFileDesc.filename << endl << flush; + inFile.close(); + } } } } -void configcache::writeConfig() +void configcache::openOutFile() { - if( DATADIR == "") return; + time_t secstamp = time(NULL); - if(!outFile.is_open()){ - time_t secstamp = time(NULL); + outFileName.str(""); + outFileName << DATADIR << "/" << secstamp << "_" << getFileId() << "_.edat.tmp"; - outFileName.str(""); - outFileName << DATADIR << "/" << secstamp << "_" << getFileId() << "_.dat.tmp"; - outFile.open( outFileName.str().c_str(), std::ios::binary ); + outFile.open( outFileName.str().c_str(), std::ios::binary ); - outBuffer = new boost::iostreams::filtering_ostreambuf; - outBuffer->push(boost::iostreams::bzip2_compressor()); - outBuffer->push(outFile); - } + outBuffer = new boost::iostreams::filtering_ostreambuf; + outBuffer->push(boost::iostreams::bzip2_compressor()); + outBuffer->push(outFile); +} + +void configcache::writeHeader(const string& headerid, const char *header, long unsigned int size) { + unsigned long headeridhash; + + if( DATADIR == "" || MODE < 2 ) return; + + if(!outFile.is_open()) + openOutFile(); + + headeridhash = hash(headerid); + + boost::iostreams::write(*outBuffer, (char*)&size, sizeof(long unsigned int)); + boost::iostreams::write(*outBuffer, (char*)&headeridhash, sizeof(unsigned long)); + boost::iostreams::write(*outBuffer, header, size); +} + +void configcache::writeConfig() +{ + long unsigned int zeroheader=0; + + if ( DATADIR == "" || MODE < 2 ) return; + + if ( ! outFile.is_open() ) + openOutFile(); + + boost::iostreams::write(*outBuffer, (char*)&zeroheader, sizeof(long unsigned int)); boost::iostreams::write(*outBuffer, configMem, configSize); } @@ -184,3 +319,74 @@ void configcache::finishOutFile() rename( outFileName.str().c_str(), outFileName.str().substr(0, outFileName.str().size()-4).c_str() ); } } + +int configcache::readFullBlock(char *tmpData, long unsigned int dataSize) +{ + /* try to read header */ + if ( openFileDesc.extended ) + if ( ! readAllHeaders() ) + return -1; + + /* read data */ + return readDataToMem(tmpData, dataSize); +} + +int configcache::readDataToMem(char *tmpData, long unsigned int dataSize) +{ + int readturn = -1; + + if ( dataSize == 0 ) return 0; + + try { readturn = boost::iostreams::read(*inBuffer, tmpData, dataSize); } + catch(boost::iostreams::bzip2_error& error) { + if(log) *log << "CCACHE: Caught bzip2 exception with error code: " << error.error() << endl << flush; + inFile.close(); + } + catch (std::exception const& ex) { + if(log) *log << "CCACHE: Caught exception: " << ex.what() << endl << flush; + inFile.close(); + } + catch( ... ) { + if(log) *log << "CCACHE: Caught unknown exception while reading." << endl << flush; + inFile.close(); + } + + return readturn; +} + +unsigned long configcache::hash(const string& str) +{ + unsigned long hash = 5381; + + for(string::const_iterator it=str.begin();it!=str.end();it++) + hash = ((hash << 5) + hash) + *it; /* hash * 33 + character */ + + return hash; +} + +void configcache::deleteHeaderStore() +{ + while ( headerStore.size() > 0 ) { + free(headerStore.back().second); + headerStore.pop_back(); + } +} + +vector::iterator configcache::getNextInfile(vector *excludeFileHashes) { + for (vector::iterator init = inFiles.begin(); init != inFiles.end(); ++init) { + if (excludeFileHashes != NULL) { + bool excludethisfile = false; + + for (vector::iterator exit = excludeFileHashes->begin(); exit != excludeFileHashes->end(); ++exit) + if ( *exit == hash(init->filename) ) { + excludethisfile = true; + break; + } + + if (excludethisfile) + continue; + } + return init; + } + return inFiles.end(); +}