areaDetector  3-5-0
EPICS areaDetector framework
NDFileHDF5.h
Go to the documentation of this file.
1 /* NDFileHDF5.h
2  * Writes NDArrays to HDF5 files.
3  *
4  * Ulrik Kofoed Pedersen
5  * March 20. 2011
6  */
7 #ifndef NDFileHDF5_H
8 #define NDFileHDF5_H
9 
10 #include <list>
11 #include <hdf5.h>
12 #include <asynDriver.h>
13 #include <NDPluginFile.h>
14 #include <NDArray.h>
15 #include "NDFileHDF5Layout.h"
16 #include "NDFileHDF5Dataset.h"
17 #include "NDFileHDF5LayoutXML.h"
19 #include "NDFileHDF5VersionCheck.h"
20 #include "Codec.h"
21 
22 #define MAXEXTRADIMS 10
23 
24 #define str_NDFileHDF5_nRowChunks "HDF5_nRowChunks"
25 #define str_NDFileHDF5_nColChunks "HDF5_nColChunks"
26 #define str_NDFileHDF5_nFramesChunks "HDF5_nFramesChunks"
27 #define str_NDFileHDF5_chunkBoundaryAlign "HDF5_chunkBoundaryAlign"
28 #define str_NDFileHDF5_chunkBoundaryThreshold "HDF5_chunkBoundaryThreshold"
29 #define str_NDFileHDF5_NDAttributeChunk "HDF5_NDAttributeChunk"
30 #define str_NDFileHDF5_nExtraDims "HDF5_nExtraDims"
31 #define str_NDFileHDF5_extraDimOffsetX "HDF5_extraDimOffsetX"
32 #define str_NDFileHDF5_extraDimOffsetY "HDF5_extraDimOffsetY"
33 #define str_NDFileHDF5_storeAttributes "HDF5_storeAttributes"
34 #define str_NDFileHDF5_storePerformance "HDF5_storePerformance"
35 #define str_NDFileHDF5_totalRuntime "HDF5_totalRuntime"
36 #define str_NDFileHDF5_totalIoSpeed "HDF5_totalIoSpeed"
37 #define str_NDFileHDF5_flushNthFrame "HDF5_flushNthFrame"
38 #define str_NDFileHDF5_compressionType "HDF5_compressionType"
39 #define str_NDFileHDF5_nbitsPrecision "HDF5_nbitsPrecision"
40 #define str_NDFileHDF5_nbitsOffset "HDF5_nbitsOffset"
41 #define str_NDFileHDF5_szipNumPixels "HDF5_szipNumPixels"
42 #define str_NDFileHDF5_zCompressLevel "HDF5_zCompressLevel"
43 #define str_NDFileHDF5_bloscShuffleType "HDF5_bloscShuffleType"
44 #define str_NDFileHDF5_bloscCompressor "HDF5_bloscCompressor"
45 #define str_NDFileHDF5_bloscCompressLevel "HDF5_bloscCompressLevel"
46 #define str_NDFileHDF5_dimAttDatasets "HDF5_dimAttDatasets"
47 #define str_NDFileHDF5_layoutErrorMsg "HDF5_layoutErrorMsg"
48 #define str_NDFileHDF5_layoutValid "HDF5_layoutValid"
49 #define str_NDFileHDF5_layoutFilename "HDF5_layoutFilename"
50 #define str_NDFileHDF5_posRunning "HDF5_posRunning"
51 #define str_NDFileHDF5_posNameDimN "HDF5_posNameDimN"
52 #define str_NDFileHDF5_posNameDimX "HDF5_posNameDimX"
53 #define str_NDFileHDF5_posNameDimY "HDF5_posNameDimY"
54 #define str_NDFileHDF5_posIndexDimN "HDF5_posIndexDimN"
55 #define str_NDFileHDF5_posIndexDimX "HDF5_posIndexDimX"
56 #define str_NDFileHDF5_posIndexDimY "HDF5_posIndexDimY"
57 #define str_NDFileHDF5_fillValue "HDF5_fillValue"
58 #define str_NDFileHDF5_SWMRFlushNow "HDF5_SWMRFlushNow"
59 #define str_NDFileHDF5_SWMRCbCounter "HDF5_SWMRCbCounter"
60 #define str_NDFileHDF5_SWMRSupported "HDF5_SWMRSupported"
61 #define str_NDFileHDF5_SWMRMode "HDF5_SWMRMode"
62 #define str_NDFileHDF5_SWMRRunning "HDF5_SWMRRunning"
63 
66 class epicsShareClass NDFileHDF5 : public NDPluginFile
67 {
68  public:
69  static const char *str_NDFileHDF5_extraDimSize[MAXEXTRADIMS];
70  static const char *str_NDFileHDF5_extraDimName[MAXEXTRADIMS];
71  static const char *str_NDFileHDF5_extraDimChunk[MAXEXTRADIMS];
72  static const char *str_NDFileHDF5_posName[MAXEXTRADIMS];
73  static const char *str_NDFileHDF5_posIndex[MAXEXTRADIMS];
74 
75  NDFileHDF5(const char *portName, int queueSize, int blockingCallbacks,
76  const char *NDArrayPort, int NDArrayAddr,
77  int priority, int stackSize);
78 
79  /* The methods that this class implements */
80  virtual asynStatus openFile(const char *fileName, NDFileOpenMode_t openMode, NDArray *pArray);
81  virtual asynStatus readFile(NDArray **pArray);
82  virtual asynStatus writeFile(NDArray *pArray);
83  virtual asynStatus closeFile();
84  virtual void report(FILE *fp, int details);
85  virtual asynStatus writeInt32(asynUser *pasynUser, epicsInt32 value);
86  virtual asynStatus writeOctet(asynUser *pasynUser, const char *value, size_t nChars, size_t *nActual);
87 
88  void flushTask();
89  asynStatus startSWMR();
90  asynStatus flushCallback();
91  asynStatus createXMLFileLayout();
92  asynStatus storeOnOpenAttributes();
93  asynStatus storeOnCloseAttributes();
94  asynStatus storeOnOpenCloseAttribute(hdf5::Element *element, bool open);
95  asynStatus createTree(hdf5::Group* root, hid_t h5handle);
96  asynStatus createHardLinks(hdf5::Group* root);
97 
98  hid_t writeHdfConstDataset( hid_t h5_handle, hdf5::Dataset* dset);
99  hid_t writeH5dsetStr(hid_t element, const std::string &name, const std::string &str_value) const;
100  hid_t writeH5dsetInt32(hid_t element, const std::string &name, const std::string &str_value) const;
101  hid_t writeH5dsetFloat64(hid_t element, const std::string &name, const std::string &str_value) const;
102 
103  void writeHdfAttributes( hid_t h5_handle, hdf5::Element* element);
104  hid_t createDataset(hid_t group, hdf5::Dataset *dset);
105  void writeH5attrStr(hid_t element,
106  const std::string &attr_name,
107  const std::string &str_attr_value) const;
108  void writeH5attrInt32(hid_t element, const std::string &attr_name, const std::string &str_attr_value) const;
109  void writeH5attrFloat64(hid_t element, const std::string &attr_name, const std::string &str_attr_value) const;
110  hid_t fromHdfToHidDatatype(hdf5::DataType_t in) const;
111  hid_t createDatasetDetector(hid_t group, hdf5::Dataset *dset);
112 
113  int fileExists(char *filename);
114  int verifyLayoutXMLFile();
115 
116  hsize_t getDim(int index);
117  hsize_t getMaxDim(int index);
118  hsize_t getChunkDim(int index);
119  hsize_t getOffset(int index);
120  hsize_t getVirtualDim(int index);
121 
122  std::map<std::string, NDFileHDF5Dataset *> detDataMap; // Map of handles to detector datasets, indexed by name
123  std::map<std::string, hid_t> attDataMap; // Map of handles to attribute datasets, indexed by name
124  std::string defDsetName; // Name of the default data set
125  std::string ndDsetName; // Name of NDAttribute that specifies the destination data set
126  std::map<std::string, hdf5::Element *> onOpenMap; // Map of handles to elements with onOpen ndattributes, indexed by fullname
127  std::map<std::string, hdf5::Element *> onCloseMap; // Map of handles to elements with onClose ndattributes, indexed by fullname
128  Codec_t codec; // Definition of codec used to compress the data.
129 
130  protected:
131  /* plugin parameters */
133  #define FIRST_NDFILE_HDF5_PARAM NDFileHDF5_nRowChunks
142  int NDFileHDF5_extraDimSize[MAXEXTRADIMS];
143  int NDFileHDF5_extraDimName[MAXEXTRADIMS];
144  int NDFileHDF5_extraDimChunk[MAXEXTRADIMS];
163  int NDFileHDF5_posName[MAXEXTRADIMS];
164  int NDFileHDF5_posIndex[MAXEXTRADIMS];
171 
172  asynStatus configureDims(NDArray *pArray);
173  void calcNumFrames();
174  void setMultiFrameFile(bool multi);
175 
176  private:
177  /* private helper functions */
178  inline bool IsPrime(int number)
179  {
180  if (((!(number & 1)) && number != 2) || (number < 2) || (number % 3 == 0 && number != 3)){
181  return false;
182  }
183 
184  for(int k = 1; 36*k*k-12*k < number;++k){
185  if ((number % (6*k+1) == 0) || (number % (6*k-1) == 0)){
186  return false;
187  }
188  }
189  return true;
190  };
191 
192  hid_t typeNd2Hdf(NDDataType_t datatype);
193  asynStatus configureDatasetDims(NDArray *pArray);
194  asynStatus configureDatasetCompression();
195  asynStatus configureCompression(NDArray *pArray);
196  char* getDimsReport();
197  asynStatus writeStringAttribute(hid_t element, const char* attrName, const char* attrStrValue);
198  asynStatus calculateAttributeChunking(int *chunking, int *mdim_chunking);
199  asynStatus writeAttributeDataset(hdf5::When_t whenToSave, int positionMode, hsize_t *offsets);
200  asynStatus closeAttributeDataset();
201  asynStatus configurePerformanceDataset();
202  asynStatus createPerformanceDataset();
203  asynStatus writePerformanceDataset();
204  unsigned int calcIstorek();
205  hsize_t calcChunkCacheBytes();
206  hsize_t calcChunkCacheSlots();
207 
208  void checkForOpenFile();
209  bool checkForSWMRMode();
210  bool checkForSWMRSupported();
211  void addDefaultAttributes(NDArray *pArray);
212  asynStatus writeDefaultDatasetAttributes(NDArray *pArray);
213  asynStatus createNewFile(const char *fileName);
214  asynStatus createFileLayout(NDArray *pArray);
215  asynStatus createAttributeDataset(NDArray *pArray);
216  int isAttributeIndex(const std::string& attName);
217  epicsInt32 findPositionIndex(NDArray *pArray, char *posName);
218 
219 
220  hdf5::LayoutXML layout;
221 
222  int arrayDataId;
223  int uniqueIdId;
224  int timeStampId;
225  int nextRecord;
226  hid_t h5type;
227  int *pAttributeId;
228  NDAttributeList *pFileAttributes;
229  epicsInt32 arrayDims[ND_ARRAY_MAX_DIMS];
230  bool multiFrameFile;
231  char *extraDimNameN;
232  char *extraDimNameX;
233  char *extraDimNameY;
234  char *extraDimName[MAXEXTRADIMS];
235  double *performanceBuf;
236  double *performancePtr;
237  epicsInt32 numPerformancePoints;
238  epicsTimeStamp prevts;
239  epicsTimeStamp opents;
240  epicsTimeStamp firstFrame;
241  double frameSize;
242  int bytesPerElement;
243  char *hostname;
244 
245  epicsEventId flushEventId;
246  epicsMutex flushLock;
247 
248  std::list<NDFileHDF5AttributeDataset*> attrList;
249 
250  /* HDF5 handles and references */
251  hid_t file;
252  hid_t dataspace;
253  hid_t datatype;
254  hid_t cparms;
255  void *ptrFillValue;
256  hid_t perf_dataset_id;
257 
258  /* dimension descriptors */
259  int rank;
260  int nvirtual;
261  hsize_t *dims;
262  hsize_t *maxdims;
263  hsize_t *chunkdims;
264  hsize_t *offset;
265  hsize_t *framesize;
266  hsize_t *virtualdims;
267  char *ptrDimensionNames[ND_ARRAY_MAX_DIMS + MAXEXTRADIMS];
269  char *dimsreport;
270 };
271 
272 #endif
273 
int NDFileHDF5_chunkBoundaryThreshold
Definition: NDFileHDF5.h:137
int NDFileHDF5_fillValue
Definition: NDFileHDF5.h:165
std::map< std::string, hid_t > attDataMap
Definition: NDFileHDF5.h:123
int NDFileHDF5_nFramesChunks
Definition: NDFileHDF5.h:135
int NDFileHDF5_zCompressLevel
Definition: NDFileHDF5.h:154
NDDataType_t
Enumeration of NDArray data types.
Definition: NDAttribute.h:27
virtual asynStatus closeFile()=0
Close the file opened with NDPluginFile::openFile; pure virtual function that must be implemented by ...
int NDFileHDF5_chunkBoundaryAlign
Definition: NDFileHDF5.h:136
int NDFileHDF5_nExtraDims
Definition: NDFileHDF5.h:139
int NDFileHDF5_SWMRSupported
Definition: NDFileHDF5.h:168
int NDFileHDF5_totalIoSpeed
Definition: NDFileHDF5.h:148
int NDFileHDF5_nColChunks
Definition: NDFileHDF5.h:134
virtual asynStatus openFile(const char *fileName, NDFileOpenMode_t openMode, NDArray *pArray)=0
Open a file; pure virtual function that must be implemented by derived classes.
#define ND_ARRAY_MAX_DIMS
NDArray.h.
Definition: NDArray.h:25
virtual void report(FILE *fp, int details)
Report status of the driver.
Definition: asynNDArrayDriver.cpp:679
Codec_t codec
Definition: NDFileHDF5.h:128
name
Definition: makeDbAndEdl.py:232
int NDFileHDF5_bloscShuffleType
Definition: NDFileHDF5.h:157
std::map< std::string, NDFileHDF5Dataset * > detDataMap
Definition: NDFileHDF5.h:122
int NDFileHDF5_layoutFilename
Definition: NDFileHDF5.h:161
virtual asynStatus writeFile(NDArray *pArray)=0
Write NDArray data to a file; pure virtual function that must be implemented by derived classes.
int NDFileHDF5_extraDimOffsetY
Definition: NDFileHDF5.h:141
int NDFileHDF5_szipNumPixels
Definition: NDFileHDF5.h:153
list value
Definition: makeDbAndEdl.py:234
std::string ndDsetName
Definition: NDFileHDF5.h:125
int NDFileHDF5_SWMRRunning
Definition: NDFileHDF5.h:170
int NDFileHDF5_nbitsOffset
Definition: NDFileHDF5.h:152
int NDFileHDF5_storePerformance
Definition: NDFileHDF5.h:146
int NDFileHDF5_nbitsPrecision
Definition: NDFileHDF5.h:151
DataType_t
Definition: NDFileHDF5Layout.h:33
int NDFileHDF5_storeAttributes
Definition: NDFileHDF5.h:145
Used to define layout of HDF5 file with NDFileHDF5 plugin.
Definition: NDFileHDF5LayoutXML.h:59
int NDFileHDF5_dimAttDatasets
Definition: NDFileHDF5.h:158
std::string defDsetName
Definition: NDFileHDF5.h:124
When_t
Definition: NDFileHDF5Layout.h:20
Class used for writing a DataSet with the NDFileHDF5 plugin.
Definition: NDFileHDF5Layout.h:146
int NDFileHDF5_posRunning
Definition: NDFileHDF5.h:162
int NDFileHDF5_layoutErrorMsg
Definition: NDFileHDF5.h:159
int NDFileHDF5_SWMRMode
Definition: NDFileHDF5.h:169
Definition: Codec.h:20
NDAttributeList.h.
Definition: NDAttributeList.h:21
virtual asynStatus readFile(NDArray **pArray)=0
Read NDArray data from a file; pure virtual function that must be implemented by derived classes.
int NDFileHDF5_extraDimOffsetX
Definition: NDFileHDF5.h:140
int NDFileHDF5_compressionType
Definition: NDFileHDF5.h:150
Writes NDArrays in the HDF5 file format; an XML file can control the structure of the HDF5 file.
Definition: NDFileHDF5.h:66
virtual asynStatus writeInt32(asynUser *pasynUser, epicsInt32 value)
Called when asyn clients call pasynInt32->write().
Definition: NDPluginFile.cpp:804
int NDFileHDF5_bloscCompressLevel
Definition: NDFileHDF5.h:156
std::map< std::string, hdf5::Element * > onCloseMap
Definition: NDFileHDF5.h:127
int NDFileHDF5_SWMRFlushNow
Definition: NDFileHDF5.h:166
std::map< std::string, hdf5::Element * > onOpenMap
Definition: NDFileHDF5.h:126
Describe a group element.
Definition: NDFileHDF5Layout.h:218
int NDFileHDF5_NDAttributeChunk
Definition: NDFileHDF5.h:138
int NDFileHDF5_totalRuntime
Definition: NDFileHDF5.h:147
N-dimensional array class; each array has a set of dimensions, a data type, pointer to data,...
Definition: NDArray.h:92
int NDFileOpenMode_t
Definition: NDPluginFile.h:17
Definition: NDFileHDF5Layout.h:42
int NDFileHDF5_nRowChunks
Definition: NDFileHDF5.h:132
Describe a generic structure element with the NDFileHDF5 plugin.
Definition: NDFileHDF5Layout.h:112
int NDFileHDF5_bloscCompressor
Definition: NDFileHDF5.h:155
virtual asynStatus writeOctet(asynUser *pasynUser, const char *value, size_t maxChars, size_t *nActual)
Called when asyn clients call pasynOctet->write().
Definition: NDPluginDriver.cpp:810
int NDFileHDF5_flushNthFrame
Definition: NDFileHDF5.h:149
int NDFileHDF5_layoutValid
Definition: NDFileHDF5.h:160
#define MAXEXTRADIMS
Definition: NDFileHDF5.h:22
Base class for NDArray file writing plugins; actual file writing plugins inherit from this class.
Definition: NDPluginFile.h:28
int NDFileHDF5_SWMRCbCounter
Definition: NDFileHDF5.h:167