bes  Updated for version 3.20.6
h5gmcfdap.cc
Go to the documentation of this file.
1 // This file is part of hdf5_handler: an HDF5 file handler for the OPeNDAP
2 // data server.
3 
4 // Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
5 //
6 // This is free software; you can redistribute it and/or modify it under the
7 // terms of the GNU Lesser General Public License as published by the Free
8 // Software Foundation; either version 2.1 of the License, or (at your
9 // option) any later version.
10 //
11 // This software is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 // License for more details.
15 //
16 // You should have received a copy of the GNU Lesser General Public
17 // License along with this library; if not, write to the Free Software
18 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
22 // Suite 203, Champaign, IL 61820
23 
32 
33 #include <sys/types.h>
34 #include <sys/stat.h>
35 #include <fcntl.h>
36 #include <unistd.h>
37 #include <iostream>
38 #include <sstream>
39 
40 #include <BESDebug.h>
41 #include <InternalErr.h>
42 
43 #include "HDF5RequestHandler.h"
44 #include "h5cfdaputil.h"
45 #include "h5gmcfdap.h"
46 #include "HDF5CFByte.h"
47 #include "HDF5CFUInt16.h"
48 #include "HDF5CFInt16.h"
49 #include "HDF5CFUInt32.h"
50 #include "HDF5CFInt32.h"
51 #include "HDF5CFFloat32.h"
52 #include "HDF5CFFloat64.h"
53 #include "HDF5CFStr.h"
54 #include "HDF5CFArray.h"
55 #include "HDF5GMCFMissLLArray.h"
56 #include "HDF5GMCFFillIndexArray.h"
58 #include "HDF5GMCFSpecialCVArray.h"
59 #include "HDF5GMSPCFArray.h"
60 
61 using namespace std;
62 using namespace libdap;
63 using namespace HDF5CF;
64 
65 // Map general HDF5 products to DAP DDS
66 void map_gmh5_cfdds(DDS &dds, hid_t file_id, const string& filename){
67 
68  BESDEBUG("h5","Coming to GM products DDS mapping function map_gmh5_cfdds() "<<endl);
69 
70  H5GCFProduct product_type = check_product(file_id);
71 
72  GMPattern gproduct_pattern = OTHERGMS;
73 
74  GMFile * f = NULL;
75 
76  try {
77  f = new GMFile(filename.c_str(),file_id,product_type,gproduct_pattern);
78  }
79  catch(...) {
80  throw InternalErr(__FILE__,__LINE__,"Cannot allocate memory for GMFile ");
81  }
82  // Generally don't need to handle attributes when handling DDS.
83  bool include_attr = false;
84  try {
85  // Retrieve all HDF5 info(Not the values)
86  f->Retrieve_H5_Info(filename.c_str(),file_id,include_attr);
87 
88  // Update product type
89  // Newer version of a product may have different layout and the
90  // product type needs to be changed to reflect it. We also want
91  // to support the older version in case people still use them.
92  // This routine will check if newer layout can be applied. If yes,
93  // update the product type.
95 
97 
98  // Need to add dimension names.
99  f->Add_Dim_Name();
100 
101  // Handle coordinate variables
102  f->Handle_CVar();
103 #if 0
104  // We need to retrieve coordinate variable attributes for memory cache use.
105  //f->Retrieve_H5_CVar_Supported_Attr_Values();
106  //if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
107  // (HDF5RequestHandler::get_srdata_mem_cache() != NULL)){
108  // f->Retrieve_H5_Supported_Attr_Values();
109 #endif
110 
111 
112  // Handle special variables
113  f->Handle_SpVar();
114 
115  // When cv memory cache is on, the unit attributes are needed to
116  // distinguish whether this is lat/lon. Generally, memory cache
117  // is not used. This snipnet will not be accessed.
118  if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
119  (HDF5RequestHandler::get_srdata_mem_cache() != NULL)){
120 
121  // Handle unsupported datatypes including the attributes
122  f->Handle_Unsupported_Dtype(true);
123 
124  // Handle unsupported dataspaces including the attributes
125  f->Handle_Unsupported_Dspace(true);
126 
127  // We need to retrieve coordinate variable attributes for memory cache use.
129 
130  }
131  else {
132 
133  // Handle unsupported datatypes
134  f->Handle_Unsupported_Dtype(include_attr);
135 
136  // Handle unsupported dataspaces
137  f->Handle_Unsupported_Dspace(include_attr);
138 
139  }
140 
141  // Need to handle the "coordinate" attributes when memory cache is turned on.
142  if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
143  (HDF5RequestHandler::get_srdata_mem_cache() != NULL))
144  f->Add_Supplement_Attrs(HDF5RequestHandler::get_add_path_attrs());
145 
146  // Adjust object names(may remove redundant paths)
147  f->Adjust_Obj_Name();
148 
149  // Flatten the object names
150  f->Flatten_Obj_Name(include_attr);
151 
152  // Handle Object name clashings
153  // Only when the check_nameclashing key is turned on or
154  // general product.
155  if(General_Product == product_type ||
156  true == HDF5RequestHandler::get_check_name_clashing())
157  f->Handle_Obj_NameClashing(include_attr);
158 
159  // Adjust Dimension name
160  f->Adjust_Dim_Name();
161  if(General_Product == product_type ||
162  true == HDF5RequestHandler::get_check_name_clashing())
164 
165  f->Handle_Hybrid_EOS5();
166  if(true == f->Have_Grid_Mapping_Attrs())
168  // Need to handle the "coordinate" attributes when memory cache is turned on.
169  if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
170  (HDF5RequestHandler::get_srdata_mem_cache() != NULL))
171  f->Handle_Coor_Attr();
172 
175  }
176  catch (HDF5CF::Exception &e){
177  if (f != NULL)
178  delete f;
179  throw InternalErr(e.what());
180  }
181 
182  // generate DDS.
183  try {
184  gen_gmh5_cfdds(dds,f);
185  }
186  catch(...) {
187  if (f != NULL)
188  delete f;
189  throw;
190  }
191 
192  if (f != NULL)
193  delete f;
194 }
195 
196 // Map general HDF5 products to DAP DAS
197 void map_gmh5_cfdas(DAS &das, hid_t file_id, const string& filename){
198 
199  BESDEBUG("h5","Coming to GM products DAS mapping function map_gmh5_cfdas() "<<endl);
200 
201  H5GCFProduct product_type = check_product(file_id);
202  GMPattern gproduct_pattern = OTHERGMS;
203 
204  GMFile *f = NULL;
205 
206  try {
207  f = new GMFile(filename.c_str(),file_id,product_type,gproduct_pattern);
208  }
209  catch(...) {
210  throw InternalErr(__FILE__,__LINE__,"Cannot allocate memory for GMFile ");
211  }
212 
213  bool include_attr = true;
214  try {
215  f->Retrieve_H5_Info(filename.c_str(),file_id,include_attr);
216 
217  // Update product type(check comments of map_gmh5_cfdds)
218  f->Update_Product_Type();
219 
221 
222  f->Add_Dim_Name();
223  f->Handle_CVar();
224  f->Handle_SpVar();
225  f->Handle_Unsupported_Dtype(include_attr);
226 
227  // Remove unsupported dataspace
228  f->Handle_Unsupported_Dspace(include_attr);
229 
230  // Need to retrieve the attribute values to feed DAS
232 
233  // Handle other unsupported objects,
234  // currently it mainly generates the info. for the
235  // unsupported objects other than datatype, dataspace,links and named datatype
236  // One area is maybe very long string. So we retrieve the attribute
237  // values before calling this function.
238  f->Handle_Unsupported_Others(include_attr);
239 
240 
241  // Need to add original variable name and path
242  // and other special attributes
243  // Can be turned on/off by using the check_path_attrs keys.
244  f->Add_Supplement_Attrs(HDF5RequestHandler::get_add_path_attrs());
245  f->Adjust_Obj_Name();
246  f->Flatten_Obj_Name(include_attr);
247  if(General_Product == product_type ||
248  true == HDF5RequestHandler::get_check_name_clashing())
249  f->Handle_Obj_NameClashing(include_attr);
250  if(f->HaveUnlimitedDim() == true)
251  f->Adjust_Dim_Name();
252  // Handle the "coordinate" attributes.
253  f->Handle_Coor_Attr();
254 
255  f->Handle_Hybrid_EOS5();
256  if(true == f->Have_Grid_Mapping_Attrs())
258 
260 
262  }
263  catch (HDF5CF::Exception &e){
264  if (f!= NULL)
265  delete f;
266  throw InternalErr(e.what());
267  }
268 
269  // Generate the DAS attributes.
270  try {
271  gen_gmh5_cfdas(das,f);
272  }
273  catch (...) {
274  if (f!= NULL)
275  delete f;
276  throw;
277 
278  }
279 
280  if (f != NULL)
281  delete f;
282 }
283 
284 // Generate DDS mapped from general HDF5 products
285 void gen_gmh5_cfdds( DDS & dds, HDF5CF:: GMFile *f) {
286 
287  BESDEBUG("h5","Coming to GM DDS generation function gen_gmh5_cfdds() "<<endl);
288 
289  const vector<HDF5CF::Var *>& vars = f->getVars();
290  const vector<HDF5CF::GMCVar *>& cvars = f->getCVars();
291  const vector<HDF5CF::GMSPVar *>& spvars = f->getSPVars();
292  const string filename = f->getPath();
293  const hid_t fileid = f->getFileID();
294 
295  // Read Variable info.
296 
297  vector<HDF5CF::Var *>::const_iterator it_v;
298  vector<HDF5CF::GMCVar *>::const_iterator it_cv;
299  vector<HDF5CF::GMSPVar *>::const_iterator it_spv;
300 
301  for (it_v = vars.begin(); it_v !=vars.end();++it_v) {
302  BESDEBUG("h5","variable full path= "<< (*it_v)->getFullPath() <<endl);
303  // Handle 64-integer DAP4 CF mapping
304  if(need_attr_values_for_dap4(*it_v) == true)
305  f->Retrieve_H5_Var_Attr_Values(*it_v);
306  gen_dap_onevar_dds(dds,*it_v,fileid, filename);
307  }
308  for (it_cv = cvars.begin(); it_cv !=cvars.end();++it_cv) {
309  BESDEBUG("h5","variable full path= "<< (*it_cv)->getFullPath() <<endl);
310  gen_dap_onegmcvar_dds(dds,*it_cv,fileid, filename);
311  }
312 
313  for (it_spv = spvars.begin(); it_spv !=spvars.end();it_spv++) {
314  BESDEBUG("h5","variable full path= "<< (*it_spv)->getFullPath() <<endl);
315  gen_dap_onegmspvar_dds(dds,*it_spv,fileid, filename);
316  }
317 
318 }
319 
320 // Generate DAS mapped from general HDF5 products
321 void gen_gmh5_cfdas( DAS & das, HDF5CF:: GMFile *f) {
322 
323  BESDEBUG("h5","Coming to GM DAS generation function gen_gmh5_cfdas() "<<endl);
324 
325  // First check if this is for generating the ignored object info.
326  if(true == f->Get_IgnoredInfo_Flag()) {
327  gen_gmh5_cf_ignored_obj_info(das, f);
328  return;
329  }
330 
331  const vector<HDF5CF::Var *>& vars = f->getVars();
332  const vector<HDF5CF::GMCVar *>& cvars = f->getCVars();
333  const vector<HDF5CF::GMSPVar *>& spvars = f->getSPVars();
334  const vector<HDF5CF::Group *>& grps = f->getGroups();
335  const vector<HDF5CF::Attribute *>& root_attrs = f->getAttributes();
336 
337 
338  vector<HDF5CF::Var *>::const_iterator it_v;
339  vector<HDF5CF::GMCVar *>::const_iterator it_cv;
340  vector<HDF5CF::GMSPVar *>::const_iterator it_spv;
341  vector<HDF5CF::Group *>::const_iterator it_g;
342  vector<HDF5CF::Attribute *>::const_iterator it_ra;
343 
344  // Handling the file attributes(attributes under the root group)
345  // The table name is "HDF_GLOBAL".
346 
347  if (false == root_attrs.empty()) {
348 
349  AttrTable *at = das.get_table(FILE_ATTR_TABLE_NAME);
350  if (NULL == at)
351  at = das.add_table(FILE_ATTR_TABLE_NAME, new AttrTable);
352 
353  for (it_ra = root_attrs.begin(); it_ra != root_attrs.end(); ++it_ra) {
354  // Check and may update the 64-bit integer attributes in DAP4.
355  check_update_int64_attr("",*it_ra);
356  gen_dap_oneobj_das(at,*it_ra,NULL);
357  }
358  }
359 
360  if (false == grps.empty()) {
361  for (it_g = grps.begin();
362  it_g != grps.end(); ++it_g) {
363  AttrTable *at = das.get_table((*it_g)->getNewName());
364  if (NULL == at)
365  at = das.add_table((*it_g)->getNewName(), new AttrTable);
366 
367  for (it_ra = (*it_g)->getAttributes().begin();
368  it_ra != (*it_g)->getAttributes().end(); ++it_ra) {
369  check_update_int64_attr((*it_g)->getNewName(),*it_ra);
370  gen_dap_oneobj_das(at,*it_ra,NULL);
371  }
372  }
373  }
374 
375  for (it_v = vars.begin();
376  it_v != vars.end(); ++it_v) {
377  if (false == ((*it_v)->getAttributes().empty())) {
378 
379  // Skip the 64-bit integer variables. The attribute mapping of
380  // DAP4 CF 64-bit integer variable support
381  // has been taken care at the routine gen_dap_onevar_dds()
382  // defined at h5commoncfdap.cc
383  if(H5INT64 == (*it_v)->getType() || H5UINT64 == (*it_v)->getType()){
384  continue;
385  }
386 
387  AttrTable *at = das.get_table((*it_v)->getNewName());
388  if (NULL == at)
389  at = das.add_table((*it_v)->getNewName(), new AttrTable);
390 
391  for (it_ra = (*it_v)->getAttributes().begin();
392  it_ra != (*it_v)->getAttributes().end(); ++it_ra) {
393  gen_dap_oneobj_das(at,*it_ra,*it_v);
394  }
395  // TODO: If a var has integer-64 bit datatype attributes, maybe
396  // we can just keep that attributes(not consistent but
397  // easy to implement) or we have to duplicate all
398  // the var in dmr and delete this var from dds.
399 
400  }
401 
402  // GPM needs to be handled in a special way(mostly _FillValue)
403  if(GPMS_L3 == f->getProductType() || GPMM_L3 == f->getProductType()
404  || GPM_L1 == f->getProductType())
405  update_GPM_special_attrs(das,*it_v,false);
406 
407  }
408 
409  for (it_cv = cvars.begin();
410  it_cv != cvars.end(); ++it_cv) {
411  if (false == ((*it_cv)->getAttributes().empty())) {
412 
413  // TODO: Add 64-bit int support for coordinates, this has not been tackled.
414  if(H5INT64 == (*it_cv)->getType() || H5UINT64 == (*it_cv)->getType()){
415  continue;
416  }
417 
418  AttrTable *at = das.get_table((*it_cv)->getNewName());
419  if (NULL == at)
420  at = das.add_table((*it_cv)->getNewName(), new AttrTable);
421 
422  for (it_ra = (*it_cv)->getAttributes().begin();
423  it_ra != (*it_cv)->getAttributes().end(); ++it_ra){
424  gen_dap_oneobj_das(at,*it_ra,*it_cv);
425  }
426 
427  }
428  // Though CF doesn't allow _FillValue, still keep it to keep the original form.
429  if(GPMS_L3 == f->getProductType() || GPMM_L3 == f->getProductType()
430  || GPM_L1 == f->getProductType())
431  update_GPM_special_attrs(das,*it_cv,true);
432 
433  }
434 
435  // Currently the special variables are only limited to the ACOS/OCO2 64-bit integer variables
436  for (it_spv = spvars.begin();
437  it_spv != spvars.end(); ++it_spv) {
438  if (false == ((*it_spv)->getAttributes().empty())) {
439 
440  AttrTable *at = das.get_table((*it_spv)->getNewName());
441  if (NULL == at)
442  at = das.add_table((*it_spv)->getNewName(), new AttrTable);
443 #if 0
444  // cerr<<"spv coordinate variable name "<<(*it_spv)->getNewName() <<endl;
445 #endif
446 
447  for (it_ra = (*it_spv)->getAttributes().begin();
448  it_ra != (*it_spv)->getAttributes().end(); ++it_ra)
449  gen_dap_oneobj_das(at,*it_ra,*it_spv);
450  }
451  }
452 
453  // CHECK ALL UNLIMITED DIMENSIONS from the coordinate variables based on the names.
454  if(f->HaveUnlimitedDim() == true) {
455 
456  BESDEBUG("h5","Find unlimited dimension in the GM DAS generation function gen_gmh5_cfdas() "<<endl);
457 
458  // Currently there is no way for DAP to present the unlimited dimension info.
459  // when there are no dimension names. So don't create DODS_EXTRA even if
460  // there is an unlimited dimension in the file. KY 2016-02-18
461  if(cvars.size() >0){
462 
463  // First check if we do have unlimited dimension in the coordinate variables.
464  // Since unsupported fakedims are removed, we may not have unlimited dimensions.
465  bool still_has_unlimited = false;
466  for (it_cv = cvars.begin();
467  it_cv != cvars.end(); ++it_cv) {
468  // Check unlimited dimension names.
469  for (vector<Dimension*>::const_iterator ird = (*it_cv)->getDimensions().begin();
470  ird != (*it_cv)->getDimensions().end(); ++ird) {
471 
472  // Currently we only check one unlimited dimension, which is the most
473  // common case. When receiving the conventions from JG, will add
474  // the support of multi-unlimited dimension. KY 2016-02-09
475  if((*ird)->HaveUnlimitedDim() == true) {
476  still_has_unlimited = true;
477  break;
478  }// if((*ird) is HaveUnlimitedDim()
479  }// for (vector<Dimension*>::
480  if(true == still_has_unlimited)
481  break;
482  }// for (it_cv=cvars.begin();
483 
484  if(true == still_has_unlimited) {
485  AttrTable* at = das.get_table("DODS_EXTRA");
486  if (NULL == at)
487  at = das.add_table("DODS_EXTRA", new AttrTable);
488 
489  string unlimited_names;
490 
491  for (it_cv = cvars.begin();
492  it_cv != cvars.end(); ++it_cv) {
493 #if 0
494  bool has_unlimited_dim = false;
495 #endif
496  // Check unlimited dimension names.
497  for (vector<Dimension*>::const_iterator ird = (*it_cv)->getDimensions().begin();
498  ird != (*it_cv)->getDimensions().end(); ++ird) {
499 
500  // Currently we only check one unlimited dimension, which is the most
501  // common case. When receiving the conventions from JG, will add
502  // the support of multi-unlimited dimension. KY 2016-02-09
503  if((*ird)->HaveUnlimitedDim() == true) {
504  if(unlimited_names=="") {
505  unlimited_names = (*ird)->getNewName();
506  if(at !=NULL)
507  at->append_attr("Unlimited_Dimension","String",unlimited_names);
508  }
509  else {
510  if(unlimited_names.rfind((*ird)->getNewName()) == string::npos) {
511  unlimited_names = unlimited_names+" "+(*ird)->getNewName();
512  if(at !=NULL)
513  at->append_attr("Unlimited_Dimension","String",(*ird)->getNewName());
514  }
515  }
516  }// if((*ird)->HaveUnlimitedDim()
517  }// for (vector<Dimension*>::
518  }// for (it_cv=cvars.begin();
519  }// if(true == still_has_unlimited)
520 
521  }//if(cvars.size()>0)
522 #if 0
523  // The following line will generate the string like "Band1 str1 str2".
524  //if(unlimited_names!="")
525  // // at->append_attr("Unlimited_Dimension","String",unlimited_names);
526 #endif
527  }
528 }
529 
530 // Generate the ignored object info. for the CF option of the general products
531 void gen_gmh5_cf_ignored_obj_info(DAS &das, HDF5CF::GMFile *f) {
532 
533  BESDEBUG("h5","Coming to gen_gmh5_cf_ignored_obj_info() "<<endl);
534  AttrTable *at = das.get_table("Ignored_Object_Info");
535  if (NULL == at)
536  at = das.add_table("Ignored_Object_Info", new AttrTable);
537 
538  at->append_attr("Message","String",f->Get_Ignored_Msg());
539 
540 }
541 
542 // Generate the DDS for a coordinate variable of the General products
543 void gen_dap_onegmcvar_dds(DDS &dds,const HDF5CF::GMCVar* cvar, const hid_t file_id, const string & filename) {
544 
545  BESDEBUG("h5","Coming to gen_dap_onegmcvar_dds() "<<endl);
546 
547  if(cvar->getType() == H5INT64 || cvar->getType() == H5UINT64)
548  return;
549  BaseType *bt = NULL;
550 
551  switch(cvar->getType()) {
552 #define HANDLE_CASE(tid,type) \
553  case tid: \
554  bt = new (type)(cvar->getNewName(),cvar->getFullPath()); \
555  break;
556 
557  HANDLE_CASE(H5FLOAT32, HDF5CFFloat32);
558  HANDLE_CASE(H5FLOAT64, HDF5CFFloat64);
559  HANDLE_CASE(H5CHAR,HDF5CFInt16);
560  HANDLE_CASE(H5UCHAR, HDF5CFByte);
561  HANDLE_CASE(H5INT16, HDF5CFInt16);
562  HANDLE_CASE(H5UINT16, HDF5CFUInt16);
563  HANDLE_CASE(H5INT32, HDF5CFInt32);
564  HANDLE_CASE(H5UINT32, HDF5CFUInt32);
565  HANDLE_CASE(H5FSTRING, Str);
566  HANDLE_CASE(H5VSTRING, Str);
567 
568  default:
569  throw InternalErr(__FILE__,__LINE__,"unsupported data type.");
570 #undef HANDLE_CASE
571  }
572 
573  if (bt) {
574 
575  const vector<HDF5CF::Dimension *>& dims = cvar->getDimensions();
576  vector <HDF5CF::Dimension*>:: const_iterator it_d;
577  vector <size_t> dimsizes;
578  dimsizes.resize(cvar->getRank());
579  for(int i = 0; i <cvar->getRank();i++)
580  dimsizes[i] = (dims[i])->getSize();
581 
582 
583  if(dims.size() == 0)
584  throw InternalErr(__FILE__,__LINE__,"the coordinate variable cannot be a scalar");
585 
586  switch(cvar->getCVType()) {
587 
588  case CV_EXIST:
589  {
590  HDF5CFArray *ar = NULL;
591 
592  // Need to check if this CV is lat/lon. This is necessary when data memory cache is turned on.
593  bool is_latlon = cvar->isLatLon();
594 
595  try {
596  ar = new HDF5CFArray (
597  cvar->getRank(),
598  file_id,
599  filename,
600  cvar->getType(),
601  dimsizes,
602  cvar->getFullPath(),
603  cvar->getTotalElems(),
604  CV_EXIST,
605  is_latlon,
606  cvar->getCompRatio(),
607  cvar->getNewName(),
608  bt);
609  }
610  catch(...) {
611  delete bt;
612  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5CFArray. ");
613  }
614 
615  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
616  if (""==(*it_d)->getNewName())
617  ar->append_dim((*it_d)->getSize());
618  else
619  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
620  }
621 
622  dds.add_var(ar);
623  delete bt;
624  delete ar;
625  }
626  break;
627 
628  case CV_LAT_MISS:
629  case CV_LON_MISS:
630  {
631  // Using HDF5GMCFMissLLArray
632  HDF5GMCFMissLLArray *ar = NULL;
633  try {
634  ar = new HDF5GMCFMissLLArray (
635  cvar->getRank(),
636  filename,
637  file_id,
638  cvar->getType(),
639  cvar->getFullPath(),
640  cvar->getPtType(),
641  cvar->getCVType(),
642  cvar->getNewName(),
643  bt);
644  }
645  catch(...) {
646  delete bt;
647  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissLLArray. ");
648  }
649 
650 
651  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
652  if (""==(*it_d)->getNewName())
653  ar->append_dim((*it_d)->getSize());
654  else
655  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
656  }
657 
658  dds.add_var(ar);
659  delete bt;
660  delete ar;
661  }
662  break;
663 
664  case CV_NONLATLON_MISS:
665  {
666 
667  if (cvar->getRank() !=1) {
668  delete bt;
669  throw InternalErr(__FILE__, __LINE__, "The rank of missing Z dimension field must be 1");
670  }
671  int nelem = (cvar->getDimensions()[0])->getSize();
672 
673  HDF5GMCFMissNonLLCVArray *ar = NULL;
674 
675  try {
676  ar = new HDF5GMCFMissNonLLCVArray(
677  cvar->getRank(),
678  nelem,
679  cvar->getNewName(),
680  bt);
681  }
682  catch(...) {
683  delete bt;
684  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
685  }
686 
687 
688  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
689  if (""==(*it_d)->getNewName())
690  ar->append_dim((*it_d)->getSize());
691  else
692  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
693  }
694  dds.add_var(ar);
695  delete bt;
696  delete ar;
697  }
698  break;
699 
700  case CV_FILLINDEX:
701  {
702 
703  if (cvar->getRank() !=1) {
704  delete bt;
705  throw InternalErr(__FILE__, __LINE__, "The rank of missing Z dimension field must be 1");
706  }
707 
708  HDF5GMCFFillIndexArray *ar = NULL;
709 
710  try {
711  ar = new HDF5GMCFFillIndexArray(
712  cvar->getRank(),
713  cvar->getType(),
714  cvar->getNewName(),
715  bt);
716  }
717  catch(...) {
718  delete bt;
719  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
720  }
721 
722 
723  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
724  if (""==(*it_d)->getNewName())
725  ar->append_dim((*it_d)->getSize());
726  else
727  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
728  }
729  dds.add_var(ar);
730  delete bt;
731  delete ar;
732  }
733  break;
734 
735 
736  case CV_SPECIAL:
737  {
738  // Currently only handle 1-D special CV.
739  if (cvar->getRank() !=1) {
740  delete bt;
741  throw InternalErr(__FILE__, __LINE__, "The rank of special coordinate variable must be 1");
742  }
743  int nelem = (cvar->getDimensions()[0])->getSize();
744 
745  HDF5GMCFSpecialCVArray * ar = NULL;
746  ar = new HDF5GMCFSpecialCVArray(
747  cvar->getType(),
748  nelem,
749  cvar->getFullPath(),
750  cvar->getPtType(),
751  cvar->getNewName(),
752  bt);
753 
754  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
755  if (""==(*it_d)->getNewName())
756  ar->append_dim((*it_d)->getSize());
757  else
758  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
759  }
760 
761  dds.add_var(ar);
762  delete bt;
763  delete ar;
764 
765  }
766  break;
767  case CV_MODIFY:
768  default:
769  delete bt;
770  throw InternalErr(__FILE__,__LINE__,"Coordinate variable type is not supported.");
771  }
772  }
773 }
774 
775 // Generate DDS for special variable in a general product
776 void gen_dap_onegmspvar_dds(DDS &dds,const HDF5CF::GMSPVar* spvar, const hid_t fileid, const string & filename) {
777 
778  BESDEBUG("h5","Coming to gen_dap_onegmspvar_dds() "<<endl);
779  BaseType *bt = NULL;
780 
781  switch(spvar->getType()) {
782 #define HANDLE_CASE(tid,type) \
783  case tid: \
784  bt = new (type)(spvar->getNewName(),spvar->getFullPath()); \
785  break;
786 
787  HANDLE_CASE(H5FLOAT32, HDF5CFFloat32);
788  HANDLE_CASE(H5FLOAT64, HDF5CFFloat64);
789  HANDLE_CASE(H5CHAR,HDF5CFInt16);
790  HANDLE_CASE(H5UCHAR, HDF5CFByte);
791  HANDLE_CASE(H5INT16, HDF5CFInt16);
792  HANDLE_CASE(H5UINT16, HDF5CFUInt16);
793  HANDLE_CASE(H5INT32, HDF5CFInt32);
794  HANDLE_CASE(H5UINT32, HDF5CFUInt32);
795  HANDLE_CASE(H5FSTRING, Str);
796  HANDLE_CASE(H5VSTRING, Str);
797  default:
798  throw InternalErr(__FILE__,__LINE__,"unsupported data type.");
799 #undef HANDLE_CASE
800  }
801 
802  if (bt) {
803 
804  const vector<HDF5CF::Dimension *>& dims = spvar->getDimensions();
805  vector <HDF5CF::Dimension*>:: const_iterator it_d;
806 
807  if(dims.size() == 0)
808  throw InternalErr(__FILE__,__LINE__,"Currently don't support scalar special variables. ");
809 
810  HDF5GMSPCFArray *ar = NULL;
811 
812  try {
813  ar = new HDF5GMSPCFArray (
814  spvar->getRank(),
815  filename,
816  fileid,
817  spvar->getType(),
818  spvar->getFullPath(),
819  spvar->getOriginalType(),
820  spvar->getStartBit(),
821  spvar->getBitNum(),
822  spvar->getNewName(),
823  bt);
824  }
825  catch(...) {
826  delete bt;
827  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
828  }
829 
830 
831  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
832  if (""==(*it_d)->getNewName())
833  ar->append_dim((*it_d)->getSize());
834  else
835  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
836  }
837 
838  dds.add_var(ar);
839  delete bt;
840  delete ar;
841  }
842 
843 }
844 
845 // When we add floating point fill value at HDF5CF.cc, the value will be changed
846 // a little bit when it changes to string representation.
847 // For example, -9999.9 becomes -9999.9000123. To reduce the misunderstanding,we
848 // just add fillvalue in the string type here. KY 2014-04-02
849 void update_GPM_special_attrs(DAS& das, const HDF5CF::Var *var,bool is_cvar) {
850 
851  BESDEBUG("h5","Coming to update_GPM_special_attrs() "<<endl);
852  if(H5FLOAT64 == var->getType() ||
853  H5FLOAT32 == var->getType() ||
854  H5INT16 == var->getType() ||
855  H5CHAR == var->getType()) {
856 
857  AttrTable *at = das.get_table(var->getNewName());
858  if (NULL == at)
859  at = das.add_table(var->getNewName(), new AttrTable);
860  bool has_fillvalue = false;
861  AttrTable::Attr_iter it = at->attr_begin();
862  while (it!=at->attr_end() && false==has_fillvalue) {
863  if (at->get_name(it) =="_FillValue")
864  {
865  has_fillvalue = true;
866  string fillvalue ="";
867  if(H5FLOAT32 == var->getType()) {
868  const string cor_fill_value = "-9999.9";
869  fillvalue = (*at->get_attr_vector(it)->begin());
870  if((fillvalue.find(cor_fill_value) == 0) && (fillvalue!= cor_fill_value)) {
871  at->del_attr("_FillValue");
872  at->append_attr("_FillValue","Float32",cor_fill_value);
873  }
874  }
875  else if(H5FLOAT64 == var->getType()) {
876  const string cor_fill_value = "-9999.9";
877  const string exist_fill_value_substr = "-9999.8999";
878  fillvalue = (*at->get_attr_vector(it)->begin());
879  if((fillvalue.find(exist_fill_value_substr) == 0) && (fillvalue!= cor_fill_value)) {
880  at->del_attr("_FillValue");
881  at->append_attr("_FillValue","Float64",cor_fill_value);
882  }
883 
884  }
885  }
886  it++;
887  }
888 
889  // Add the fill value
890  if(false == is_cvar ) {
891 
892  // Current versions of GPM don't add fillvalues. We add the fillvalue according to the document.
893  if (has_fillvalue != true ) {
894 
895  if(H5FLOAT32 == var->getType())
896  at->append_attr("_FillValue","Float32","-9999.9");
897  else if(H5FLOAT64 == var->getType())
898  at->append_attr("_FillValue","Float64","-9999.9");
899  else if (H5INT16 == var->getType())
900  at->append_attr("_FillValue","Int16","-9999");
901  else if (H5CHAR == var->getType())// H5CHAR maps to DAP int16
902  at->append_attr("_FillValue","Int16","-99");
903 
904  }
905  }
906  }
907 }
908 
909 
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
This class provides a way to map HDF5 byte to DAP byte for the CF option.
This class provides a way to map HDF5 float to DAP float for the CF option.
This class provides a way to map HDF5 64-bit floating-point(double) to DAP 64-bit floating-point for ...
This class provides a way to map HDF5 int16 to DAP int16 for the CF option.
This class provides a way to map HDF5 32-bit integer to DAP Int32 for the CF option.
This class provides a way to map HDF5 Str to DAP Str for the CF option.
This class provides a way to map HDF5 unsigned 16-bit integer to DAP uint16 for the CF option.
This class provides a way to map HDF5 unsigned 32-bit integer to DAP uint32 for the CF option.
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
This class specifies the retrieval of the missing lat/lon values for general HDF5 products.
This class specifies the retrieval of the values of non-lat/lon coordinate variables for general HDF5...
This class specifies the retrieval of the missing lat/lon values for general HDF5 products.
This class specifies the retrieval of data values for special HDF5 products Currently this only appli...
include the entry functions to execute the handlers
CVType getCVType() const
Get the coordinate variable type of this variable.
Definition: HDF5CF.h:360
bool HaveUnlimitedDim() const
Has unlimited dimensions.
Definition: HDF5CF.h:676
virtual void Retrieve_H5_Var_Attr_Values(Var *var)
Retrieve attribute values for a variable.
Definition: HDF5CF.cc:741
const std::string & getPath() const
Obtain the path of the file.
Definition: HDF5CF.h:652
hid_t getFileID() const
Obtain the HDF5 file ID.
Definition: HDF5CF.h:646
const std::vector< Attribute * > & getAttributes() const
Public interface to obtain information of all attributes under the root group.
Definition: HDF5CF.h:664
const std::vector< Var * > & getVars() const
Public interface to obtain information of all variables.
Definition: HDF5CF.h:658
const std::vector< Group * > & getGroups() const
Public interface to obtain all the group info.
Definition: HDF5CF.h:670
This class is a derived class of CVar. It represents a coordinate variable for general HDF5 files.
Definition: HDF5CF.h:415
H5GCFProduct getPtType() const
Get the data type of this variable.
Definition: HDF5CF.h:427
This class is a derived class of File. It includes methods applied to general HDF5 files only.
Definition: HDF5CF.h:821
void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:384
void Adjust_Obj_Name()
Adjust object names based on different general NASA HDF5 products.
Definition: HDF5GMCF.cc:4730
void Retrieve_H5_CVar_Supported_Attr_Values()
Retrieve coordinate variable attributes.
Definition: HDF5GMCF.cc:317
bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition: HDF5GMCF.cc:6745
void Adjust_Dim_Name()
Adjust dimension name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5043
void Handle_Obj_NameClashing(bool)
Handle object name clashing for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4873
void Remove_Unused_FakeDimVars()
Unsupported datatype array may generate FakeDim. Remove them.
Definition: HDF5GMCF.cc:6753
void Update_Product_Type()
Update "product type" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:235
void Retrieve_H5_Info(const char *path, hid_t file_id, bool include_attr)
Retrieve DDS information from the HDF5 file; real implementation for general HDF5 products.
Definition: HDF5GMCF.cc:216
bool Get_IgnoredInfo_Flag()
Obtain ignored info. flag.
Definition: HDF5CF.h:916
const std::string & Get_Ignored_Msg()
Get the message that contains the ignored obj. info.
Definition: HDF5CF.h:922
void Handle_SpVar()
Handle special variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4640
void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for general HDF5 products.
Definition: HDF5GMCF.cc:577
void Handle_Unsupported_Others(bool)
Handle other unmapped objects/attributes for general HDF5 products.
Definition: HDF5GMCF.cc:674
void Remove_Unneeded_Objects()
Remove unneeded objects.
Definition: HDF5GMCF.cc:257
void Add_Dim_Name()
Add dimension name.
Definition: HDF5GMCF.cc:803
void Flatten_Obj_Name(bool include_attr)
Flatten the object name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4815
void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition: HDF5GMCF.cc:6749
void Handle_DimNameClashing()
Definition: HDF5GMCF.cc:4978
void Handle_Coor_Attr()
Handle "coordinates" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:5820
void Rename_NC4_NonCoordVars()
Remove the _nc4_non_coord from the variable new names.
Definition: HDF5GMCF.cc:6801
void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:332
void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5105
void Handle_CVar()
Handle coordinate variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:2768
This class is a derived class of Var. It represents a special general HDF5 product(currently ACOS and...
Definition: HDF5CF.h:380
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:259
int getRank() const
Get the dimension rank of this variable.
Definition: HDF5CF.h:295
const std::string & getFullPath() const
Get the full path of this variable.
Definition: HDF5CF.h:283
H5DataType getType() const
Get the data type of this variable(Not HDF5 datatype id)
Definition: HDF5CF.h:301
const std::vector< Dimension * > & getDimensions() const
Get the list of the dimensions.
Definition: HDF5CF.h:312
int getCompRatio() const
Get the compression ratio of this dataset.
Definition: HDF5CF.h:318
const std::string & getNewName() const
Get the new name of this variable.
Definition: HDF5CF.h:277
Helper functions for generating DAS attributes and a function to check BES Key.
Map and generate DDS and DAS for the CF option for generic HDF5 products.