11 #include <shogun/lib/config.h> 24 CHDF5File::CHDF5File()
32 CHDF5File::CHDF5File(
char* fname,
char rw, const
char* name) :
CFile()
35 H5Eset_auto2(H5E_DEFAULT, NULL, NULL);
38 set_variable_name(name);
43 h5file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
46 h5file = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
49 h5file = H5Fopen(fname, H5F_ACC_RDWR, H5P_DEFAULT);
51 h5file = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
58 SG_ERROR(
"Could not open file '%s'\n", fname)
61 CHDF5File::~CHDF5File()
66 #define GET_VECTOR(fname, sg_type, datatype) \ 67 void CHDF5File::fname(sg_type*& vec, int32_t& len) \ 70 SG_ERROR("File invalid.\n") \ 75 hid_t dataset = H5Dopen2(h5file, variable_name, H5P_DEFAULT); \ 77 SG_ERROR("Error opening data set\n") \ 78 hid_t dtype = H5Dget_type(dataset); \ 79 H5T_class_t t_class=H5Tget_class(dtype); \ 80 TSGDataType t datatype; hid_t h5_type=get_compatible_type(t_class, &t); \ 84 SG_INFO("No compatible datatype found\n") \ 86 get_dims(dataset, dims, ndims, nelements); \ 87 if (!((ndims==2 && dims[0]==nelements && dims[1]==1) || \ 88 (ndims==2 && dims[0]==1 && dims[1]==nelements) || \ 89 (ndims==1 && dims[0]==nelements))) \ 90 SG_ERROR("Error not a 1-dimensional vector (ndims=%d, dims[0]=%d)\n", ndims, dims[0]) \ 91 vec=SG_MALLOC(sg_type, nelements); \ 93 herr_t status = H5Dread(dataset, h5_type, H5S_ALL, \ 94 H5S_ALL, H5P_DEFAULT, vec); \ 101 SG_ERROR("Error reading dataset\n") \ 105 GET_VECTOR(get_vector,
bool, (CT_VECTOR, ST_NONE, PT_BOOL))
106 GET_VECTOR(get_vector, int8_t, (CT_VECTOR, ST_NONE, PT_INT8))
107 GET_VECTOR(get_vector, uint8_t, (CT_VECTOR, ST_NONE, PT_UINT8))
108 GET_VECTOR(get_vector,
char, (CT_VECTOR, ST_NONE, PT_CHAR))
109 GET_VECTOR(get_vector, int32_t, (CT_VECTOR, ST_NONE, PT_INT32))
110 GET_VECTOR(get_vector, uint32_t, (CT_VECTOR, ST_NONE, PT_UINT32))
114 GET_VECTOR(get_vector, int16_t, (CT_VECTOR, ST_NONE, PT_INT16))
115 GET_VECTOR(get_vector, uint16_t, (CT_VECTOR, ST_NONE, PT_INT16))
116 GET_VECTOR(get_vector, int64_t, (CT_VECTOR, ST_NONE, PT_INT64))
117 GET_VECTOR(get_vector, uint64_t, (CT_VECTOR, ST_NONE, PT_UINT64))
120 #define GET_MATRIX(fname, sg_type, datatype) \ 121 void CHDF5File::fname(sg_type*& matrix, int32_t& num_feat, int32_t& num_vec) \ 124 SG_ERROR("File invalid.\n") \ 129 hid_t dataset = H5Dopen2(h5file, variable_name, H5P_DEFAULT); \ 131 SG_ERROR("Error opening data set\n") \ 132 hid_t dtype = H5Dget_type(dataset); \ 133 H5T_class_t t_class=H5Tget_class(dtype); \ 134 TSGDataType t datatype; hid_t h5_type=get_compatible_type(t_class, &t); \ 138 SG_INFO("No compatible datatype found\n") \ 140 get_dims(dataset, dims, ndims, nelements); \ 142 SG_ERROR("Error not a 2-dimensional matrix\n") \ 143 matrix=SG_MALLOC(sg_type, nelements); \ 146 herr_t status = H5Dread(dataset, h5_type, H5S_ALL, \ 147 H5S_ALL, H5P_DEFAULT, matrix); \ 154 SG_ERROR("Error reading dataset\n") \ 158 GET_MATRIX(get_matrix,
bool, (CT_MATRIX, ST_NONE, PT_BOOL))
159 GET_MATRIX(get_matrix,
char, (CT_MATRIX, ST_NONE, PT_CHAR))
160 GET_MATRIX(get_matrix, uint8_t, (CT_MATRIX, ST_NONE, PT_UINT8))
161 GET_MATRIX(get_matrix, int32_t, (CT_MATRIX, ST_NONE, PT_INT32))
162 GET_MATRIX(get_matrix, uint32_t, (CT_MATRIX, ST_NONE, PT_INT32))
163 GET_MATRIX(get_matrix, int64_t, (CT_MATRIX, ST_NONE, PT_INT64))
164 GET_MATRIX(get_matrix, uint64_t, (CT_MATRIX, ST_NONE, PT_INT64))
165 GET_MATRIX(get_matrix, int16_t, (CT_MATRIX, ST_NONE, PT_INT16))
166 GET_MATRIX(get_matrix, uint16_t, (CT_MATRIX, ST_NONE, PT_INT16))
172 #define GET_SPARSEMATRIX(fname, sg_type, datatype) \ 173 void CHDF5File::fname(SGSparseVector<sg_type>*& matrix, int32_t& num_feat, int32_t& num_vec) \ 176 SG_ERROR("File invalid.\n") \ 191 #undef GET_SPARSEMATRIX 194 #define GET_STRING_LIST(fname, sg_type, datatype) \ 195 void CHDF5File::fname(SGString<sg_type>*& strings, int32_t& num_str, int32_t& max_string_len) \ 212 #undef GET_STRING_LIST 216 #define SET_VECTOR(fname, sg_type, dtype, h5type) \ 217 void CHDF5File::fname(const sg_type* vec, int32_t len) \ 219 if (h5file<0 || !vec) \ 220 SG_ERROR("File or vector invalid.\n") \ 222 create_group_hierarchy(); \ 224 hsize_t dims=(hsize_t) len; \ 225 hid_t dataspace, dataset, status; \ 226 dataspace=H5Screate_simple(1, &dims, NULL); \ 228 SG_ERROR("Could not create hdf5 dataspace\n") \ 229 dataset=H5Dcreate2(h5file, variable_name, h5type, dataspace, H5P_DEFAULT,\ 230 H5P_DEFAULT, H5P_DEFAULT); \ 233 SG_ERROR("Could not create hdf5 dataset - does" \ 234 " dataset '%s' already exist?\n", variable_name); \ 236 status=H5Dwrite(dataset, h5type, H5S_ALL, H5S_ALL, H5P_DEFAULT, vec); \ 238 SG_ERROR("Failed to write hdf5 dataset\n") \ 240 H5Sclose(dataspace); \ 242 SET_VECTOR(set_vector,
bool, DT_VECTOR_BOOL, boolean_type)
243 SET_VECTOR(set_vector, int8_t, DT_VECTOR_BYTE, H5T_NATIVE_INT8)
244 SET_VECTOR(set_vector, uint8_t, DT_VECTOR_BYTE, H5T_NATIVE_UINT8)
245 SET_VECTOR(set_vector,
char, DT_VECTOR_CHAR, H5T_NATIVE_CHAR)
246 SET_VECTOR(set_vector, int32_t, DT_VECTOR_INT, H5T_NATIVE_INT32)
247 SET_VECTOR(set_vector, uint32_t, DT_VECTOR_UINT, H5T_NATIVE_UINT32)
251 SET_VECTOR(set_vector, int16_t, DT_VECTOR_SHORT, H5T_NATIVE_INT16)
252 SET_VECTOR(set_vector, uint16_t, DT_VECTOR_WORD, H5T_NATIVE_UINT16)
253 SET_VECTOR(set_vector, int64_t, DT_VECTOR_LONG, H5T_NATIVE_LLONG)
254 SET_VECTOR(set_vector, uint64_t, DT_VECTOR_ULONG, H5T_NATIVE_ULLONG)
257 #define SET_MATRIX(fname, sg_type, dtype, h5type) \ 258 void CHDF5File::fname(const sg_type* matrix, int32_t num_feat, int32_t num_vec) \ 260 if (h5file<0 || !matrix) \ 261 SG_ERROR("File or matrix invalid.\n") \ 263 create_group_hierarchy(); \ 265 hsize_t dims[2]={(hsize_t) num_feat, (hsize_t) num_vec}; \ 266 hid_t dataspace, dataset, status; \ 267 dataspace=H5Screate_simple(2, dims, NULL); \ 269 SG_ERROR("Could not create hdf5 dataspace\n") \ 270 dataset=H5Dcreate2(h5file, variable_name, h5type, dataspace, H5P_DEFAULT, \ 271 H5P_DEFAULT, H5P_DEFAULT); \ 274 SG_ERROR("Could not create hdf5 dataset - does" \ 275 " dataset '%s' already exist?\n", variable_name); \ 277 status=H5Dwrite(dataset, h5type, H5S_ALL, H5S_ALL, H5P_DEFAULT, matrix); \ 279 SG_ERROR("Failed to write hdf5 dataset\n") \ 281 H5Sclose(dataspace); \ 283 SET_MATRIX(set_matrix,
bool, DT_DENSE_BOOL, boolean_type)
284 SET_MATRIX(set_matrix,
char, DT_DENSE_CHAR, H5T_NATIVE_CHAR)
285 SET_MATRIX(set_matrix, int8_t, DT_DENSE_BYTE, H5T_NATIVE_INT8)
286 SET_MATRIX(set_matrix, uint8_t, DT_DENSE_BYTE, H5T_NATIVE_UINT8)
287 SET_MATRIX(set_matrix, int32_t, DT_DENSE_INT, H5T_NATIVE_INT32)
288 SET_MATRIX(set_matrix, uint32_t, DT_DENSE_UINT, H5T_NATIVE_UINT32)
289 SET_MATRIX(set_matrix, int64_t, DT_DENSE_LONG, H5T_NATIVE_INT64)
290 SET_MATRIX(set_matrix, uint64_t, DT_DENSE_ULONG, H5T_NATIVE_UINT64)
291 SET_MATRIX(set_matrix, int16_t, DT_DENSE_SHORT, H5T_NATIVE_INT16)
292 SET_MATRIX(set_matrix, uint16_t, DT_DENSE_WORD, H5T_NATIVE_UINT16)
298 #define SET_SPARSEMATRIX(fname, sg_type, dtype) \ 299 void CHDF5File::fname(const SGSparseVector<sg_type>* matrix, \ 300 int32_t num_feat, int32_t num_vec) \ 302 if (!(file && matrix)) \ 303 SG_ERROR("File or matrix invalid.\n") \ 319 #undef SET_SPARSEMATRIX 321 #define SET_STRING_LIST(fname, sg_type, dtype) \ 322 void CHDF5File::fname(const SGString<sg_type>* strings, int32_t num_str) \ 324 if (!(file && strings)) \ 325 SG_ERROR("File or strings invalid.\n") \ 341 #undef SET_STRING_LIST 343 void CHDF5File::get_boolean_type()
345 boolean_type=H5T_NATIVE_UCHAR;
346 switch (
sizeof(
bool))
349 boolean_type = H5T_NATIVE_UCHAR;
352 boolean_type = H5T_NATIVE_UINT16;
355 boolean_type = H5T_NATIVE_UINT32;
358 boolean_type = H5T_NATIVE_UINT64;
361 SG_ERROR(
"Boolean type not supported on this platform\n")
365 hid_t CHDF5File::get_compatible_type(H5T_class_t t_class,
374 case PT_BOOL:
return boolean_type;
375 case PT_CHAR:
return H5T_NATIVE_CHAR;
376 case PT_INT8:
return H5T_NATIVE_INT8;
377 case PT_UINT8:
return H5T_NATIVE_UINT8;
378 case PT_INT16:
return H5T_NATIVE_INT16;
379 case PT_UINT16:
return H5T_NATIVE_UINT16;
380 case PT_INT32:
return H5T_NATIVE_INT32;
381 case PT_UINT32:
return H5T_NATIVE_UINT32;
382 case PT_INT64:
return H5T_NATIVE_INT64;
383 case PT_UINT64:
return H5T_NATIVE_UINT64;
384 case PT_FLOAT32:
return H5T_NATIVE_FLOAT;
385 case PT_FLOAT64:
return H5T_NATIVE_DOUBLE;
386 case PT_FLOATMAX:
return H5T_NATIVE_LDOUBLE;
388 SG_ERROR(
"complex128_t not compatible with HDF5File!");
392 SG_ERROR(
"Implementation error during writing " 400 SG_ERROR("Variable length containers currently not supported")
403 SG_ERROR("Array containers currently not supported")
411 void CHDF5File::get_dims(hid_t dataset, int32_t*& dims, int32_t& ndims, int64_t& total_elements)
413 hid_t dataspace = H5Dget_space(dataset);
415 SG_ERROR(
"Error obtaining hdf5 dataspace\n")
417 ndims = H5Sget_simple_extent_ndims(dataspace);
418 total_elements=H5Sget_simple_extent_npoints(dataspace);
419 hsize_t* dims_out=SG_MALLOC(hsize_t, ndims);
420 dims=SG_MALLOC(int32_t, ndims);
421 H5Sget_simple_extent_dims(dataspace, dims_out, NULL);
422 for (int32_t i=0; i<ndims; i++)
428 void CHDF5File::create_group_hierarchy()
430 char* vname=get_strdup(variable_name);
431 int32_t vlen=strlen(vname);
432 for (int32_t i=0; i<vlen; i++)
434 if (i!=0 && vname[i]==
'/')
437 hid_t g = H5Gopen2(h5file, vname, H5P_DEFAULT);
440 g=H5Gcreate2(h5file, vname, H5P_DEFAULT, H5P_DEFAULT,
443 SG_ERROR(
"Error creating group '%s'\n", vname)
#define GET_MATRIX(fname, sg_type, datatype)
#define GET_STRING_LIST(fname, sg_type, datatype)
#define SET_STRING_LIST(fname, sg_type, dtype)
#define SET_MATRIX(fname, sg_type, dtype)
Datatypes that shogun supports.
A File access base class.
#define SET_SPARSEMATRIX(fname, sg_type, dtype)
#define SET_VECTOR(fname, sg_type, dtype)
all of classes and functions are contained in the shogun namespace
#define GET_VECTOR(fname, sg_type, datatype)
#define SG_UNSTABLE(func,...)
#define GET_SPARSEMATRIX(fname, sg_type, datatype)