From 061e6469f58f3e3e892f1cd1f6fdd2aad9c644e8 Mon Sep 17 00:00:00 2001 From: Regina Kleinhappel Date: Fri, 5 Dec 2014 01:48:25 +0100 Subject: [PATCH 1/1] ... --- CMakeLists.txt | 7 +- configcache.cpp | 141 +++++-------------------------------- configcache.h | 15 ++-- datread.cpp | 184 ++++++++++++++++++++++++++++++++++++++++++++++++ datread.h | 39 ++++++++++ 5 files changed, 251 insertions(+), 135 deletions(-) create mode 100644 datread.cpp create mode 100644 datread.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 39c0dc2..9558c25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,7 @@ project(latlib) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + #PROFILING #set(CMAKE_CXX_FLAGS -pg) #set(CMAKE_EXE_LINKER_FLAGS -pg) @@ -19,8 +21,11 @@ else() message("LATLIB: MPI support is disabled!") endif() +add_library(lat_datread datread.cpp) +target_link_libraries(lat_datread ${Boost_LIBRARIES}) + add_library(lat_configcache configcache.cpp) -target_link_libraries(lat_configcache ${Boost_LIBRARIES} lat_writeout) +target_link_libraries(lat_configcache ${Boost_LIBRARIES} lat_writeout lat_datread) add_library(lat_hypercache hypercache.cpp) target_link_libraries(lat_hypercache lat_configcache lat_writeout) diff --git a/configcache.cpp b/configcache.cpp index 59472e9..8687123 100644 --- a/configcache.cpp +++ b/configcache.cpp @@ -20,7 +20,6 @@ struct configcache::iobuffers { - boost::iostreams::filtering_istreambuf *in; boost::iostreams::filtering_ostreambuf *out; }; @@ -28,6 +27,8 @@ configcache::configcache(const string& cacheid, const int& nequi, const int& nsk ostream *_log){ log = _log; + dataReader = new datread(configMemSize, log); + NEQUI = nequi; NSKIP = nskip; DATADIR = datadir; @@ -45,7 +46,6 @@ configcache::configcache(const string& cacheid, const int& nequi, const int& nsk configSize = configMemSize; ioBuffers = new iobuffers; - ioBuffers->in = NULL; ioBuffers->out = NULL; MODE = cachemode; @@ -151,63 +151,8 @@ bool configcache::isValidInFile(const string& infile, infiledesc *filedesc) return true; } -int configcache::readHeader() -{ - long unsigned int headersize; - - if( readDataToMem((char *)&headersize, sizeof(long unsigned int)) == sizeof(long unsigned int) && inFile.is_open() ) { - if ( headersize == 0 ) - return HEADER_READLAST; - - pair newHeader; - - if( readDataToMem((char *)&newHeader.first, sizeof(unsigned long)) == sizeof(unsigned long) && inFile.is_open() ) { - newHeader.second = malloc(headersize); - - if( readDataToMem((char *)newHeader.second, headersize) == headersize && inFile.is_open() ) { - headerStore.push_back(newHeader); - return HEADER_READOK; - } - else { - if(log) *log << "CCACHE: Could not read heade-data! Closing dat-file: " << openFileDesc.filename << endl << flush; - inFile.close(); - return HEADER_READERR; - } - } - else { - if(log) *log << "CCACHE: Could not read headerid-hash! Closing dat-file: " << openFileDesc.filename << endl << flush; - inFile.close(); - return HEADER_READERR; - } - } - else { - if(log) *log << "CCACHE: Could not read header size. Closing dat-file: " << openFileDesc.filename << endl << flush; - inFile.close(); - return HEADER_READERR; - } -} - -bool configcache::readAllHeaders() -{ - int readHeaderStatus; - - deleteHeaderStore(); - - do { - readHeaderStatus = readHeader(); - } - while ( readHeaderStatus == HEADER_READOK ); - - if ( readHeaderStatus == HEADER_READLAST ) return true; - else if ( readHeaderStatus == HEADER_READERR ) return false; -} - void * configcache::getHeader(const string& headerid) { - for (vector< pair >::iterator headerStoreIt = headerStore.begin(); headerStoreIt != headerStore.end(); ++headerStoreIt) - if ( headerStoreIt->first == hash(headerid) ) - return headerStoreIt->second; - - return NULL; + return dataReader->getHeader(headerid); } void configcache::readConfig(bool *readnewconfig, int *nequileft, vector *excludeFileHashes) @@ -226,13 +171,13 @@ void configcache::readConfig(bool *readnewconfig, int *nequileft, vector::iterator inFileIt = getNextInfile(excludeFileHashes); int iDidVirtualSkips; - if( (!inFile.is_open()) && inFileIt == inFiles.end() ) { + if( (!dataReader->fisopen()) && inFileIt == inFiles.end() ) { if (*readnewconfig) *nequileft = nequileft_internal; return; } - while( (!inFile.is_open()) && inFiles.size() > 0 ) { + while( (!dataReader->fisopen()) && inFiles.size() > 0 ) { string inFileParaString; openFileDesc = *inFileIt; @@ -245,32 +190,28 @@ void configcache::readConfig(bool *readnewconfig, int *nequileft, vectorfilename << endl << flush; - inFile.open( (DATADIR + "/" + inFileIt->filename).c_str(), std::ios::binary ); - + + dataReader->openFile( DATADIR + "/" + inFileIt->filename ); + if( openFileDesc.superextended ) { - getline( inFile, inFileParaString ); - if( inFileParaString != paraString() ) { + if( dataReader->getParaString() != paraString() ) { if(log) *log << "CCACHE: Parastring does not match. Closing dat-file..." << endl << flush; - inFile.close(); + dataReader->closeFile(); } } - + inFiles.erase(inFileIt); - if( !inFile.is_open() ) continue; - - ioBuffers->in = new boost::iostreams::filtering_istreambuf; - ioBuffers->in->push( boost::iostreams::bzip2_decompressor() ); - ioBuffers->in->push(inFile); + if( !dataReader->fisopen() ) continue; } - if( inFile.is_open() ) + if( dataReader->fisopen() ) { if (doVirtualEquilibration) { if(log) *log << "CCACHE: Trying virtual equilibration." << endl << flush; doVirtualEquilibration = false; for (iDidVirtualSkips=0; iDidVirtualSkips < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iDidVirtualSkips++) { - if( readFullBlock(tmpConfig, configSize) != configSize || ! inFile.is_open() ) + if( dataReader->readFullBlock(tmpConfig) < 0 ) break; else if ( (NEQUI-openFileDesc.nequi) - (iDidVirtualSkips+1)*openFileDesc.nskip < nequileft_internal ) { memcpy(configMem, tmpConfig, configSize); @@ -281,7 +222,7 @@ void configcache::readConfig(bool *readnewconfig, int *nequileft, vectorreadFullBlock(tmpConfig) >= 0 ) { memcpy(configMem, tmpConfig, configSize); *readnewconfig = true; @@ -296,10 +237,6 @@ void configcache::readConfig(bool *readnewconfig, int *nequileft, vectorin != NULL) { delete ioBuffers->in; ioBuffers->in=NULL; } - inFile.close(); + dataReader->closeFile(); inFiles.clear(); refetchDataFiles = true; @@ -392,8 +328,7 @@ void configcache::setPara(const string& parid, const double& value){ configcache::~configcache() { finishOutFile(); - delete ioBuffers->in; - ioBuffers->in = NULL; + delete dataReader; } void configcache::finishOutFile() @@ -411,40 +346,6 @@ void configcache::finishOutFile() } } -int configcache::readFullBlock(char *tmpData, long unsigned int dataSize) -{ - /* try to read header */ - if ( openFileDesc.extended ) - if ( ! readAllHeaders() ) - return -1; - - /* read data */ - return readDataToMem(tmpData, dataSize); -} - -int configcache::readDataToMem(char *tmpData, long unsigned int dataSize) -{ - int readturn = -1; - - if ( dataSize == 0 ) return 0; - - try { readturn = boost::iostreams::read(*ioBuffers->in, tmpData, dataSize); } - catch(boost::iostreams::bzip2_error& error) { - if(log) *log << "CCACHE: Caught bzip2 exception with error code: " << error.error() << endl << flush; - inFile.close(); - } - catch (std::exception const& ex) { - if(log) *log << "CCACHE: Caught exception: " << ex.what() << endl << flush; - inFile.close(); - } - catch( ... ) { - if(log) *log << "CCACHE: Caught unknown exception while reading." << endl << flush; - inFile.close(); - } - - return readturn; -} - unsigned long configcache::hash(const string& str) { unsigned long hash = 5381; @@ -455,14 +356,6 @@ unsigned long configcache::hash(const string& str) return hash; } -void configcache::deleteHeaderStore() -{ - while ( headerStore.size() > 0 ) { - free(headerStore.back().second); - headerStore.pop_back(); - } -} - vector::iterator configcache::getNextInfile(vector *excludeFileHashes) { for (vector::iterator init = inFiles.begin(); init != inFiles.end(); ++init) { if (excludeFileHashes != NULL) { diff --git a/configcache.h b/configcache.h index d07aea4..74998c7 100644 --- a/configcache.h +++ b/configcache.h @@ -7,6 +7,8 @@ #include #include +#include "datread.h" + #define CACHE_MODE_OO 0 #define CACHE_MODE_RO 1 #define CACHE_MODE_WO 2 @@ -41,7 +43,7 @@ class configcache{ string getOutFileName() { return outFileName.str(); } string getInFileName() { return DATADIR + "/" + openFileDesc.filename; } static unsigned long hash(const string& str); - void closeInFile() { inFile.close(); } + void closeInFile() { dataReader->closeFile(); } int inFilesLeft() { return inFiles.size(); } char* getConfigMem() { return configMem; } int getConfigSize() { return configSize; } @@ -63,7 +65,6 @@ class configcache{ string getFileId(int actnequi, const bool& superextended=true, const bool& shortid=false); ofstream outFile; - ifstream inFile; stringstream outFileName; @@ -85,20 +86,12 @@ class configcache{ vector Paras; - int readDataToMem(char *tmpData, long unsigned int dataSize); - void openOutFile(int actnequi); int readHeader(); bool headerWritten; - int readFullBlock(char *tmpData, long unsigned int dataSize); - - vector< pair > headerStore; - - void deleteHeaderStore(); - bool readAllHeaders(); vector::iterator getNextInfile(vector *excludeFileHashes); @@ -108,6 +101,8 @@ class configcache{ bool doVirtualEquilibration, firstUsedConfig; string paraString(); + + datread *dataReader; }; #endif diff --git a/datread.cpp b/datread.cpp new file mode 100644 index 0000000..41d3724 --- /dev/null +++ b/datread.cpp @@ -0,0 +1,184 @@ +#include "datread.h" + +#define HEADER_READOK 0 +#define HEADER_READERR 1 +#define HEADER_READLAST 2 + +datread::datread (const unsigned int& _blocksize, ostream *_log) : blocksize(_blocksize), log(_log) +{ + inbuffer = NULL; +} + +int datread::openFile (const string& filename) +{ + infile.open(filename.c_str(), std::ios::binary); + + if ( ! infile.is_open() ) return -1; + + if( filename.substr(filename.size()-4) == ".dat" ) { + format = Format::DAT; + } + else if( filename.substr(filename.size()-4) == "edat" ) { + format = Format::EDAT; + } + else if( filename.substr(filename.size()-4) == "sdat" ) { + format = Format::SDAT; + getline(infile, parastring); + } + + if (inbuffer != NULL) + delete inbuffer; + + inbuffer = new boost::iostreams::filtering_istreambuf; + inbuffer->push( boost::iostreams::bzip2_decompressor() ); + inbuffer->push(infile); + + return 0; +} + +datread::~datread () +{ + if (inbuffer != NULL) + delete inbuffer; +} + +int datread::readDataToMem (char *tmpData, long unsigned int dataSize) +{ + int readturn = -1; + + if ( dataSize == 0 ) return 0; + + try { readturn = boost::iostreams::read(*inbuffer, tmpData, dataSize); } + catch(boost::iostreams::bzip2_error& error) { + if(log) *log << "DATREAD: Caught bzip2 exception with error code: " << error.error() << endl << flush; + infile.close(); + } + catch (std::exception const& ex) { + if(log) *log << "DATREAD: Caught exception: " << ex.what() << endl << flush; + infile.close(); + } + catch( ... ) { + if(log) *log << "DATREAD: Caught unknown exception while reading." << endl << flush; + infile.close(); + } + + return readturn; +} + +int datread::readHeader () +{ + long unsigned int headersize; + + if( readDataToMem((char *)&headersize, sizeof(long unsigned int)) == sizeof(long unsigned int) && infile.is_open() ) { + if ( headersize == 0 ) + return HEADER_READLAST; + + pair newHeader; + + if( readDataToMem((char *)&newHeader.first, sizeof(unsigned long)) == sizeof(unsigned long) && infile.is_open() ) { + newHeader.second = malloc(headersize); + + if( readDataToMem((char *)newHeader.second, headersize) == headersize && infile.is_open() ) { + headerStore.push_back(newHeader); + return HEADER_READOK; + } + else { + if(log) *log << "DATREAD: Could not read heade-data! Closing dat-file." << endl << flush; + infile.close(); + return HEADER_READERR; + } + } + else { + if(log) *log << "DATREAD: Could not read headerid-hash! Closing dat-file." << endl << flush; + infile.close(); + return HEADER_READERR; + } + } + else { + if(log) *log << "DATREAD: Could not read header size. Closing dat-file." << endl << flush; + infile.close(); + return HEADER_READERR; + } +} + +void datread::deleteHeaderStore () +{ + while ( headerStore.size() > 0 ) { + free(headerStore.back().second); + headerStore.pop_back(); + } +} + +bool datread::readAllHeaders () +{ + int readHeaderStatus; + + deleteHeaderStore(); + + do { + readHeaderStatus = readHeader(); + } + while ( readHeaderStatus == HEADER_READOK ); + + if ( readHeaderStatus == HEADER_READLAST ) return true; + else if ( readHeaderStatus == HEADER_READERR ) return false; +} + +int datread::readFullBlock (char *tmpData) +{ + if ( ! infile.is_open() ) + return -4; + + /* try to read header */ + if ( format == Format::EDAT || format == Format::SDAT ) + if ( ! readAllHeaders() ) { + infile.close(); + return -1; + } + + /* read data */ + if ( readDataToMem(tmpData, blocksize) != blocksize ) { + if(log) *log << "DATREAD: Could not read full datablock. Closing dat-file." << endl << flush; + infile.close(); + return -2; + } + + if ( ! infile.is_open() ) + return -3; + + return 0; +} + +void * datread::getHeader (const string& headerid) { + for (vector< pair >::iterator headerStoreIt = headerStore.begin(); headerStoreIt != headerStore.end(); ++headerStoreIt) + if ( headerStoreIt->first == hash(headerid) ) + return headerStoreIt->second; + + return NULL; +} + +bool datread::fisopen () +{ + return infile.is_open(); +} + +unsigned long datread::hash(const string& str) +{ + unsigned long hash = 5381; + + for(string::const_iterator it=str.begin();it!=str.end();it++) + hash = ((hash << 5) + hash) + *it; /* hash * 33 + character */ + + return hash; +} + +string datread::getParaString () +{ + return parastring; +} + +void datread::closeFile () +{ + if ( infile.is_open() ) + infile.close(); +} diff --git a/datread.h b/datread.h new file mode 100644 index 0000000..8ef4b15 --- /dev/null +++ b/datread.h @@ -0,0 +1,39 @@ +#ifndef DATREAD_H +#define DATREAD_H + +#include +#include + +#include +#include + +using namespace std; + +class datread { + public: + datread (const unsigned int& _blocksize, ostream *_log=NULL); + ~datread (); + int openFile (const string& filename); + enum class Format {DAT, EDAT, SDAT}; + void * getHeader (const string& headerid); + int readFullBlock (char *tmpData); + bool fisopen (); + string getParaString (); + void closeFile (); + private: + const unsigned int blocksize; + ifstream infile; + Format format; + string parastring; + boost::iostreams::filtering_istreambuf *inbuffer; + ostream * const log; + int readDataToMem (char *tmpData, long unsigned int dataSize); + string filename; + int readHeader (); + vector< pair > headerStore; + void deleteHeaderStore (); + bool readAllHeaders (); + static unsigned long hash(const string& str); +}; + +#endif -- 2.39.5