project(latlib)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+
#PROFILING
#set(CMAKE_CXX_FLAGS -pg)
#set(CMAKE_EXE_LINKER_FLAGS -pg)
message("LATLIB: MPI support is disabled!")
endif()
+add_library(lat_datread datread.cpp)
+target_link_libraries(lat_datread ${Boost_LIBRARIES})
+
add_library(lat_configcache configcache.cpp)
-target_link_libraries(lat_configcache ${Boost_LIBRARIES} lat_writeout)
+target_link_libraries(lat_configcache ${Boost_LIBRARIES} lat_writeout lat_datread)
add_library(lat_hypercache hypercache.cpp)
target_link_libraries(lat_hypercache lat_configcache lat_writeout)
struct configcache::iobuffers
{
- boost::iostreams::filtering_istreambuf *in;
boost::iostreams::filtering_ostreambuf *out;
};
ostream *_log){
log = _log;
+ dataReader = new datread(configMemSize, log);
+
NEQUI = nequi;
NSKIP = nskip;
DATADIR = datadir;
configSize = configMemSize;
ioBuffers = new iobuffers;
- ioBuffers->in = NULL;
ioBuffers->out = NULL;
MODE = cachemode;
return true;
}
-int configcache::readHeader()
-{
- long unsigned int headersize;
-
- if( readDataToMem((char *)&headersize, sizeof(long unsigned int)) == sizeof(long unsigned int) && inFile.is_open() ) {
- if ( headersize == 0 )
- return HEADER_READLAST;
-
- pair<unsigned long, void *> newHeader;
-
- if( readDataToMem((char *)&newHeader.first, sizeof(unsigned long)) == sizeof(unsigned long) && inFile.is_open() ) {
- newHeader.second = malloc(headersize);
-
- if( readDataToMem((char *)newHeader.second, headersize) == headersize && inFile.is_open() ) {
- headerStore.push_back(newHeader);
- return HEADER_READOK;
- }
- else {
- if(log) *log << "CCACHE: Could not read heade-data! Closing dat-file: " << openFileDesc.filename << endl << flush;
- inFile.close();
- return HEADER_READERR;
- }
- }
- else {
- if(log) *log << "CCACHE: Could not read headerid-hash! Closing dat-file: " << openFileDesc.filename << endl << flush;
- inFile.close();
- return HEADER_READERR;
- }
- }
- else {
- if(log) *log << "CCACHE: Could not read header size. Closing dat-file: " << openFileDesc.filename << endl << flush;
- inFile.close();
- return HEADER_READERR;
- }
-}
-
-bool configcache::readAllHeaders()
-{
- int readHeaderStatus;
-
- deleteHeaderStore();
-
- do {
- readHeaderStatus = readHeader();
- }
- while ( readHeaderStatus == HEADER_READOK );
-
- if ( readHeaderStatus == HEADER_READLAST ) return true;
- else if ( readHeaderStatus == HEADER_READERR ) return false;
-}
-
void * configcache::getHeader(const string& headerid) {
- for (vector< pair<unsigned long, void *> >::iterator headerStoreIt = headerStore.begin(); headerStoreIt != headerStore.end(); ++headerStoreIt)
- if ( headerStoreIt->first == hash(headerid) )
- return headerStoreIt->second;
-
- return NULL;
+ return dataReader->getHeader(headerid);
}
void configcache::readConfig(bool *readnewconfig, int *nequileft, vector<unsigned long> *excludeFileHashes)
vector<infiledesc>::iterator inFileIt = getNextInfile(excludeFileHashes);
int iDidVirtualSkips;
- if( (!inFile.is_open()) && inFileIt == inFiles.end() ) {
+ if( (!dataReader->fisopen()) && inFileIt == inFiles.end() ) {
if (*readnewconfig)
*nequileft = nequileft_internal;
return;
}
- while( (!inFile.is_open()) && inFiles.size() > 0 ) {
+ while( (!dataReader->fisopen()) && inFiles.size() > 0 ) {
string inFileParaString;
openFileDesc = *inFileIt;
firstUsedConfig = true;
if(log) *log << "CCACHE: Opening dat-file: " << inFileIt->filename << endl << flush;
- inFile.open( (DATADIR + "/" + inFileIt->filename).c_str(), std::ios::binary );
-
+
+ dataReader->openFile( DATADIR + "/" + inFileIt->filename );
+
if( openFileDesc.superextended ) {
- getline( inFile, inFileParaString );
- if( inFileParaString != paraString() ) {
+ if( dataReader->getParaString() != paraString() ) {
if(log) *log << "CCACHE: Parastring does not match. Closing dat-file..." << endl << flush;
- inFile.close();
+ dataReader->closeFile();
}
}
-
+
inFiles.erase(inFileIt);
- if( !inFile.is_open() ) continue;
-
- ioBuffers->in = new boost::iostreams::filtering_istreambuf;
- ioBuffers->in->push( boost::iostreams::bzip2_decompressor() );
- ioBuffers->in->push(inFile);
+ if( !dataReader->fisopen() ) continue;
}
- if( inFile.is_open() )
+ if( dataReader->fisopen() )
{
if (doVirtualEquilibration) {
if(log) *log << "CCACHE: Trying virtual equilibration." << endl << flush;
doVirtualEquilibration = false;
for (iDidVirtualSkips=0; iDidVirtualSkips < (NEQUI-openFileDesc.nequi)/openFileDesc.nskip; iDidVirtualSkips++) {
- if( readFullBlock(tmpConfig, configSize) != configSize || ! inFile.is_open() )
+ if( dataReader->readFullBlock(tmpConfig) < 0 )
break;
else if ( (NEQUI-openFileDesc.nequi) - (iDidVirtualSkips+1)*openFileDesc.nskip < nequileft_internal ) {
memcpy(configMem, tmpConfig, configSize);
}
}
- if( readFullBlock(tmpConfig, configSize) == configSize && inFile.is_open() )
+ if( dataReader->readFullBlock(tmpConfig) >= 0 )
{
memcpy(configMem, tmpConfig, configSize);
*readnewconfig = true;
*nequileft = nequileft_internal;
return;
}
- else {
- if(log) *log << "CCACHE: Could not read configuration. Closing dat-file: " << openFileDesc.filename << endl << flush;
- inFile.close();
- }
}
}
}
Paras[getParIndex(parid)].val = value;
finishOutFile();
- if(ioBuffers->in != NULL) { delete ioBuffers->in; ioBuffers->in=NULL; }
- inFile.close();
+ dataReader->closeFile();
inFiles.clear();
refetchDataFiles = true;
configcache::~configcache()
{
finishOutFile();
- delete ioBuffers->in;
- ioBuffers->in = NULL;
+ delete dataReader;
}
void configcache::finishOutFile()
}
}
-int configcache::readFullBlock(char *tmpData, long unsigned int dataSize)
-{
- /* try to read header */
- if ( openFileDesc.extended )
- if ( ! readAllHeaders() )
- return -1;
-
- /* read data */
- return readDataToMem(tmpData, dataSize);
-}
-
-int configcache::readDataToMem(char *tmpData, long unsigned int dataSize)
-{
- int readturn = -1;
-
- if ( dataSize == 0 ) return 0;
-
- try { readturn = boost::iostreams::read(*ioBuffers->in, tmpData, dataSize); }
- catch(boost::iostreams::bzip2_error& error) {
- if(log) *log << "CCACHE: Caught bzip2 exception with error code: " << error.error() << endl << flush;
- inFile.close();
- }
- catch (std::exception const& ex) {
- if(log) *log << "CCACHE: Caught exception: " << ex.what() << endl << flush;
- inFile.close();
- }
- catch( ... ) {
- if(log) *log << "CCACHE: Caught unknown exception while reading." << endl << flush;
- inFile.close();
- }
-
- return readturn;
-}
-
unsigned long configcache::hash(const string& str)
{
unsigned long hash = 5381;
return hash;
}
-void configcache::deleteHeaderStore()
-{
- while ( headerStore.size() > 0 ) {
- free(headerStore.back().second);
- headerStore.pop_back();
- }
-}
-
vector<infiledesc>::iterator configcache::getNextInfile(vector<unsigned long> *excludeFileHashes) {
for (vector<infiledesc>::iterator init = inFiles.begin(); init != inFiles.end(); ++init) {
if (excludeFileHashes != NULL) {
#include <sstream>
#include <ostream>
+#include "datread.h"
+
#define CACHE_MODE_OO 0
#define CACHE_MODE_RO 1
#define CACHE_MODE_WO 2
string getOutFileName() { return outFileName.str(); }
string getInFileName() { return DATADIR + "/" + openFileDesc.filename; }
static unsigned long hash(const string& str);
- void closeInFile() { inFile.close(); }
+ void closeInFile() { dataReader->closeFile(); }
int inFilesLeft() { return inFiles.size(); }
char* getConfigMem() { return configMem; }
int getConfigSize() { return configSize; }
string getFileId(int actnequi, const bool& superextended=true, const bool& shortid=false);
ofstream outFile;
- ifstream inFile;
stringstream outFileName;
vector<parameter> Paras;
- int readDataToMem(char *tmpData, long unsigned int dataSize);
-
void openOutFile(int actnequi);
int readHeader();
bool headerWritten;
- int readFullBlock(char *tmpData, long unsigned int dataSize);
-
- vector< pair<unsigned long, void *> > headerStore;
-
- void deleteHeaderStore();
-
bool readAllHeaders();
vector<infiledesc>::iterator getNextInfile(vector<unsigned long> *excludeFileHashes);
bool doVirtualEquilibration, firstUsedConfig;
string paraString();
+
+ datread *dataReader;
};
#endif
--- /dev/null
+#include "datread.h"
+
+#define HEADER_READOK 0
+#define HEADER_READERR 1
+#define HEADER_READLAST 2
+
+datread::datread (const unsigned int& _blocksize, ostream *_log) : blocksize(_blocksize), log(_log)
+{
+ inbuffer = NULL;
+}
+
+int datread::openFile (const string& filename)
+{
+ infile.open(filename.c_str(), std::ios::binary);
+
+ if ( ! infile.is_open() ) return -1;
+
+ if( filename.substr(filename.size()-4) == ".dat" ) {
+ format = Format::DAT;
+ }
+ else if( filename.substr(filename.size()-4) == "edat" ) {
+ format = Format::EDAT;
+ }
+ else if( filename.substr(filename.size()-4) == "sdat" ) {
+ format = Format::SDAT;
+ getline(infile, parastring);
+ }
+
+ if (inbuffer != NULL)
+ delete inbuffer;
+
+ inbuffer = new boost::iostreams::filtering_istreambuf;
+ inbuffer->push( boost::iostreams::bzip2_decompressor() );
+ inbuffer->push(infile);
+
+ return 0;
+}
+
+datread::~datread ()
+{
+ if (inbuffer != NULL)
+ delete inbuffer;
+}
+
+int datread::readDataToMem (char *tmpData, long unsigned int dataSize)
+{
+ int readturn = -1;
+
+ if ( dataSize == 0 ) return 0;
+
+ try { readturn = boost::iostreams::read(*inbuffer, tmpData, dataSize); }
+ catch(boost::iostreams::bzip2_error& error) {
+ if(log) *log << "DATREAD: Caught bzip2 exception with error code: " << error.error() << endl << flush;
+ infile.close();
+ }
+ catch (std::exception const& ex) {
+ if(log) *log << "DATREAD: Caught exception: " << ex.what() << endl << flush;
+ infile.close();
+ }
+ catch( ... ) {
+ if(log) *log << "DATREAD: Caught unknown exception while reading." << endl << flush;
+ infile.close();
+ }
+
+ return readturn;
+}
+
+int datread::readHeader ()
+{
+ long unsigned int headersize;
+
+ if( readDataToMem((char *)&headersize, sizeof(long unsigned int)) == sizeof(long unsigned int) && infile.is_open() ) {
+ if ( headersize == 0 )
+ return HEADER_READLAST;
+
+ pair<unsigned long, void *> newHeader;
+
+ if( readDataToMem((char *)&newHeader.first, sizeof(unsigned long)) == sizeof(unsigned long) && infile.is_open() ) {
+ newHeader.second = malloc(headersize);
+
+ if( readDataToMem((char *)newHeader.second, headersize) == headersize && infile.is_open() ) {
+ headerStore.push_back(newHeader);
+ return HEADER_READOK;
+ }
+ else {
+ if(log) *log << "DATREAD: Could not read heade-data! Closing dat-file." << endl << flush;
+ infile.close();
+ return HEADER_READERR;
+ }
+ }
+ else {
+ if(log) *log << "DATREAD: Could not read headerid-hash! Closing dat-file." << endl << flush;
+ infile.close();
+ return HEADER_READERR;
+ }
+ }
+ else {
+ if(log) *log << "DATREAD: Could not read header size. Closing dat-file." << endl << flush;
+ infile.close();
+ return HEADER_READERR;
+ }
+}
+
+void datread::deleteHeaderStore ()
+{
+ while ( headerStore.size() > 0 ) {
+ free(headerStore.back().second);
+ headerStore.pop_back();
+ }
+}
+
+bool datread::readAllHeaders ()
+{
+ int readHeaderStatus;
+
+ deleteHeaderStore();
+
+ do {
+ readHeaderStatus = readHeader();
+ }
+ while ( readHeaderStatus == HEADER_READOK );
+
+ if ( readHeaderStatus == HEADER_READLAST ) return true;
+ else if ( readHeaderStatus == HEADER_READERR ) return false;
+}
+
+int datread::readFullBlock (char *tmpData)
+{
+ if ( ! infile.is_open() )
+ return -4;
+
+ /* try to read header */
+ if ( format == Format::EDAT || format == Format::SDAT )
+ if ( ! readAllHeaders() ) {
+ infile.close();
+ return -1;
+ }
+
+ /* read data */
+ if ( readDataToMem(tmpData, blocksize) != blocksize ) {
+ if(log) *log << "DATREAD: Could not read full datablock. Closing dat-file." << endl << flush;
+ infile.close();
+ return -2;
+ }
+
+ if ( ! infile.is_open() )
+ return -3;
+
+ return 0;
+}
+
+void * datread::getHeader (const string& headerid) {
+ for (vector< pair<unsigned long, void *> >::iterator headerStoreIt = headerStore.begin(); headerStoreIt != headerStore.end(); ++headerStoreIt)
+ if ( headerStoreIt->first == hash(headerid) )
+ return headerStoreIt->second;
+
+ return NULL;
+}
+
+bool datread::fisopen ()
+{
+ return infile.is_open();
+}
+
+unsigned long datread::hash(const string& str)
+{
+ unsigned long hash = 5381;
+
+ for(string::const_iterator it=str.begin();it!=str.end();it++)
+ hash = ((hash << 5) + hash) + *it; /* hash * 33 + character */
+
+ return hash;
+}
+
+string datread::getParaString ()
+{
+ return parastring;
+}
+
+void datread::closeFile ()
+{
+ if ( infile.is_open() )
+ infile.close();
+}
--- /dev/null
+#ifndef DATREAD_H
+#define DATREAD_H
+
+#include <ostream>
+#include <fstream>
+
+#include <boost/iostreams/filtering_streambuf.hpp>
+#include <boost/iostreams/filter/bzip2.hpp>
+
+using namespace std;
+
+class datread {
+ public:
+ datread (const unsigned int& _blocksize, ostream *_log=NULL);
+ ~datread ();
+ int openFile (const string& filename);
+ enum class Format {DAT, EDAT, SDAT};
+ void * getHeader (const string& headerid);
+ int readFullBlock (char *tmpData);
+ bool fisopen ();
+ string getParaString ();
+ void closeFile ();
+ private:
+ const unsigned int blocksize;
+ ifstream infile;
+ Format format;
+ string parastring;
+ boost::iostreams::filtering_istreambuf *inbuffer;
+ ostream * const log;
+ int readDataToMem (char *tmpData, long unsigned int dataSize);
+ string filename;
+ int readHeader ();
+ vector< pair<unsigned long, void *> > headerStore;
+ void deleteHeaderStore ();
+ bool readAllHeaders ();
+ static unsigned long hash(const string& str);
+};
+
+#endif
projects / phys / latlib.git / commitdiff