X-Git-Url: http://git.treefish.org/~alex/phys/latlib.git/blobdiff_plain/3bb55613d615dd93b662c4cd1a9c6198f7de53ff..7f5403eed95da4350dcc03d817eb8468dab07142:/configcache.cpp?ds=sidebyside diff --git a/configcache.cpp b/configcache.cpp index 1b22757..1095775 100644 --- a/configcache.cpp +++ b/configcache.cpp @@ -5,6 +5,10 @@ #include #include +#define HEADER_READOK 0 +#define HEADER_READERR 1 +#define HEADER_READLAST 2 + configcache::configcache(const string& cacheid, const int& nequi, const int& nskip, const string& datadir, char **configmem, const int& configMemSize, const int& cachemode, writeout *out_a){ out = out_a; @@ -109,57 +113,67 @@ bool configcache::isValidInFile(const string& infile, infiledesc *filedesc) return true; } -bool configcache::openInFile() { - while( (!inFile.is_open()) && inFiles.size() > 0 ) - { - if(out) *out->log << "CCACHE: Opening dat-file: " << inFiles.back().filename << endl << flush; - - openFileDesc = inFiles.back(); - inFile.open( (DATADIR + "/" + inFiles.back().filename).c_str(), std::ios::binary ); - inFiles.pop_back(); - - if( !inFile.is_open() ) continue; - - inBuffer = new boost::iostreams::filtering_istreambuf; - inBuffer->push( boost::iostreams::bzip2_decompressor() ); - inBuffer->push(inFile); - - return true; - } - - return false; -} - -void configcache::readHeader() +int configcache::readHeader() { - int headersize; + long unsigned int headersize; - if( readDataToMem((char*)&headersize, sizeof(int)) == sizeof(int) && inFile.is_open() ) - { - if( headerData != NULL ) delete headerData; + if( readDataToMem((char *)&headersize, sizeof(long unsigned int)) == sizeof(long unsigned int) && inFile.is_open() ) { + if ( headersize == 0 ) + return HEADER_READLAST; + + pair newHeader; + + if( readDataToMem((char *)&newHeader.first, sizeof(unsigned long)) == sizeof(unsigned long) && inFile.is_open() ) { + newHeader.second = malloc(headersize); - headerData = new char(headersize); - - if( readDataToMem(headerData, headersize) == headersize && inFile.is_open() ) { - if(out) *out->log << "CCACHE: Read header information." << endl << flush; + if( readDataToMem((char *)newHeader.second, headersize) == headersize && inFile.is_open() ) { + headerStore.push_back(newHeader); + return HEADER_READOK; } else { - if(out) *out->log << "CCACHE: Could not read header! Closing dat-file: " << openFileDesc.filename << endl << flush; + if(out) *out->log << "CCACHE: Could not read heade-data! Closing dat-file: " << openFileDesc.filename << endl << flush; inFile.close(); + return HEADER_READERR; } } + else { + if(out) *out->log << "CCACHE: Could not read headerid-hash! Closing dat-file: " << openFileDesc.filename << endl << flush; + inFile.close(); + return HEADER_READERR; + } + } else { - if(out) *out->log << "CCACHE: Could not read header size! Closing dat-file: " << openFileDesc.filename << endl << flush; + if(out) *out->log << "CCACHE: Could not read header size. Closing dat-file: " << openFileDesc.filename << endl << flush; inFile.close(); + return HEADER_READERR; } } -void *configcache::getHeader() { - return headerData; +bool configcache::readAllHeaders() +{ + int readHeaderStatus; + + deleteHeaderStore(); + + do { + readHeaderStatus = readHeader(); + } + while ( readHeaderStatus == HEADER_READOK ); + + if ( readHeaderStatus == HEADER_READLAST ) return true; + else if ( readHeaderStatus == HEADER_READERR ) return false; } -bool configcache::readConfig() -{ +void * configcache::getHeader(const string& headerid) { + for (vector< pair >::iterator headerStoreIt = headerStore.begin(); headerStoreIt != headerStore.end(); ++headerStoreIt) + if ( headerStoreIt->first == hash(headerid) ) + return headerStoreIt->second; + + return NULL; +} + +bool configcache::readConfig(vector *excludeFileHashes) +{ if(DATADIR == "" || MODE == CACHE_MODE_DISABLED) return false; if(refetchDataFiles){ @@ -171,9 +185,33 @@ bool configcache::readConfig() { if( (!inFile.is_open()) && inFiles.size() == 0 ) return false; - /* read header information */ - if ( openInFile() && openFileDesc.extended ) - readHeader(); + while( (!inFile.is_open()) && inFiles.size() > 0 ) { + bool excludethisfile=false; + + openFileDesc = inFiles.back(); + + if (excludeFileHashes != NULL) + for (vector::iterator exit = excludeFileHashes->begin(); exit != excludeFileHashes->end(); ++exit) + if ( *exit == hash(inFiles.back().filename) ) { + excludethisfile = true; + break; + } + + if ( ! excludethisfile ) { + if(out) *out->log << "CCACHE: Opening dat-file: " << inFiles.back().filename << endl << flush; + inFile.open( (DATADIR + "/" + inFiles.back().filename).c_str(), std::ios::binary ); + } + else + if(out) *out->log << "CCACHE: Excluded dat-file: " << inFiles.back().filename << endl << flush; + + inFiles.pop_back(); + + if( !inFile.is_open() ) continue; + + inBuffer = new boost::iostreams::filtering_istreambuf; + inBuffer->push( boost::iostreams::bzip2_decompressor() ); + inBuffer->push(inFile); + } if( inFile.is_open() ) { @@ -181,12 +219,12 @@ bool configcache::readConfig() if(out) *out->log << "CCACHE: Trying virtual equilibration." << endl << flush; openFileDesc.doVirtualEquilibration = false; for (int iskip=0; iskip < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iskip++) { - if( readDataToMem(tmpConfig, configSize) != configSize || ! inFile.is_open() ) + if( readFullBlock(tmpConfig, configSize) != configSize || ! inFile.is_open() ) break; } } - if( readDataToMem(tmpConfig, configSize) == configSize && inFile.is_open() ) + if( readFullBlock(tmpConfig, configSize) == configSize && inFile.is_open() ) { memcpy(configMem, tmpConfig, configSize); return true; @@ -204,7 +242,8 @@ void configcache::openOutFile() time_t secstamp = time(NULL); outFileName.str(""); - outFileName << DATADIR << "/" << secstamp << "_" << getFileId() << "_.edat.tmp"; + outFileName << DATADIR << "/" << secstamp << "_" << getFileId() << "_.edat.tmp"; + outFile.open( outFileName.str().c_str(), std::ios::binary ); outBuffer = new boost::iostreams::filtering_ostreambuf; @@ -212,26 +251,31 @@ void configcache::openOutFile() outBuffer->push(outFile); } -void configcache::writeHeader(char *header, const int& size) { +void configcache::writeHeader(const string& headerid, const char *header, long unsigned int size) { + unsigned long headeridhash; + if( DATADIR == "" || MODE < 2 ) return; - if(!outFile.is_open()) { + if(!outFile.is_open()) openOutFile(); - boost::iostreams::write(*outBuffer, header, size); - } - else - if(out) *out->log << "CCACHE: Not writing header because outfile is already open!" << endl << flush; + + headeridhash = hash(headerid); + + boost::iostreams::write(*outBuffer, (char*)&size, sizeof(long unsigned int)); + boost::iostreams::write(*outBuffer, (char*)&headeridhash, sizeof(unsigned long)); + boost::iostreams::write(*outBuffer, header, size); } void configcache::writeConfig() { - if( DATADIR == "" || MODE < 2 ) return; + long unsigned int zeroheader=0; - if(!outFile.is_open()){ - int zeroheader=0; + if ( DATADIR == "" || MODE < 2 ) return; + + if ( ! outFile.is_open() ) openOutFile(); - boost::iostreams::write(*outBuffer, (char*)&zeroheader, sizeof(int)); - } + + boost::iostreams::write(*outBuffer, (char*)&zeroheader, sizeof(long unsigned int)); boost::iostreams::write(*outBuffer, configMem, configSize); } @@ -280,6 +324,17 @@ void configcache::finishOutFile() } } +int configcache::readFullBlock(char *tmpData, long unsigned int dataSize) +{ + /* try to read header */ + if ( openFileDesc.extended ) + if ( ! readAllHeaders() ) + return -1; + + /* read data */ + return readDataToMem(tmpData, dataSize); +} + int configcache::readDataToMem(char *tmpData, long unsigned int dataSize) { int readturn = -1; @@ -302,3 +357,21 @@ int configcache::readDataToMem(char *tmpData, long unsigned int dataSize) return readturn; } + +unsigned long configcache::hash(const string& str) +{ + unsigned long hash = 5381; + + for(string::const_iterator it=str.begin();it!=str.end();it++) + hash = ((hash << 5) + hash) + *it; /* hash * 33 + character */ + + return hash; +} + +void configcache::deleteHeaderStore() +{ + while ( headerStore.size() > 0 ) { + free(headerStore.back().second); + headerStore.pop_back(); + } +}