Hyrax 2188

modules/dmrpp_module/get_dmrpp/unit-tests/Makefile.am

@@ -32,7 +32,9 @@ echo-variables:
 # allow writing to the source directory.
 #
 # I do not understand why the module *.so files are copied. jhrg 8/31/25
-#
+# Answer for the above comments: 
+# The generation of dmr with besstandalone needs to have these modules at this location.
+# KY 2026-06-08
 cp_build_script:
 	if test $(srcdir) != .; then \
 	cp -f `echo $(srcdir) | rev | cut -c 11- | rev`/gen_dmrpp_side_car $(builddir); \
@@ -46,6 +48,7 @@ cp_build_script:
 	cp -f $(abs_top_builddir)/modules/hdf5_handler/.libs/libhdf5_module.so $(builddir); \
 	cp -f $(abs_top_builddir)/modules/hdf4_handler/.libs/libhdf4_module.so $(builddir); \
 	cp -f $(abs_top_builddir)/modules/fileout_netcdf/.libs/libfonc_module.so $(builddir); \
+	cp -f $(abs_top_builddir)/modules/dmrpp_module/.libs/libdmrpp_module.so $(builddir); \
 	cp -f $(abs_top_builddir)/xmlcommand/.libs/libdap_xml_module.so $(builddir); \
 	cp -f $(abs_top_builddir)/dap/.libs/libdap_module.so $(builddir); \
 	else \
@@ -56,6 +59,7 @@ cp_build_script:
 	cp -f $(abs_top_builddir)/modules/hdf5_handler/.libs/libhdf5_module.so $(builddir); \
 	cp -f $(abs_top_builddir)/modules/hdf4_handler/.libs/libhdf4_module.so $(builddir); \
 	cp -f $(abs_top_builddir)/modules/fileout_netcdf/.libs/libfonc_module.so $(builddir); \
+	cp -f $(abs_top_builddir)/modules/dmrpp_module/.libs/libdmrpp_module.so $(builddir); \
 	cp -f $(abs_top_builddir)/xmlcommand/.libs/libdap_xml_module.so $(builddir); \
 	cp -f $(abs_top_builddir)/dap/.libs/libdap_module.so $(builddir); \
 	fi;
@@ -76,6 +80,7 @@ clean-local:
 	rm -f $(builddir)/libhdf5_module.so; \
 	rm -f $(builddir)/libhdf4_module.so; \
 	rm -f $(builddir)/libfonc_module.so; \
+	rm -f $(builddir)/libdmrpp_module.so; \
 	rm -f $(builddir)/libdap_xml_module.so; \
 	rm -f $(builddir)/libdap_module.so; \
 	rm -rf $(builddir)/__pycache__; \
@@ -84,4 +89,5 @@ clean-local:
 	rm -f $(builddir)/$(HDF5_FILES); \
 	rm -f $(builddir)/$(PYTHON_FILES); \
 	rm -f $(builddir)/$(BASELINE_FILES); \
+        rm -f $(builddir)/bes.log; \
 	fi;

modules/dmrpp_module/get_dmrpp/unit-tests/gen_bescmd_conf.py

@@ -0,0 +1,75 @@
+# This script that generates the bes configuration and xml files for testing the gen_dmrpp_side_car.
+# The functions defined in this script are used by the gen_dmrpp_side_car_test.py.
+import os
+import shutil
+import string
+import subprocess
+import sys
+
+# The function add_module_path() adds the module path
+def add_module_path(f_str, conf_mod_name, mod_path):
+    index = f_str.find(conf_mod_name) + len(conf_mod_name)
+    if (f_str[index] != '\n'):
+        print(conf_mod_name)
+        print(f_str[index])
+        print("The module path should be empty.")
+        sys.exit(1)
+    temp_text = f_str[:index] + mod_path + f_str[index:]
+    return temp_text
+
+# Generate the BES configuration file
+def generate_bes_conf():
+    bes_conf_name="bes.test.conf"
+    bes_conf_str=r'''BES.LogName=./bes.log
+BES.modules=dap,cmd,fonc,dmrpp
+BES.module.dap=
+BES.module.cmd=
+BES.module.dmrpp=
+BES.module.fonc=
+BES.Catalog.catalog.RootDirectory=
+BES.Data.RootDirectory=/dev/null
+BES.Catalog.catalog.TypeMatch+=dmrpp:.*\.(dmrpp)$;
+BES.Catalog.catalog.Include=;
+BES.Catalog.catalog.Exclude=^\..*;
+BES.FollowSymLinks=Yes
+BES.Catalog.catalog.FollowSymLinks=Yes
+BES.UncompressCache.dir=/tmp
+BES.UncompressCache.prefix=uncompress_cache
+BES.UncompressCache.size=500
+FONc.UseCompression=true
+FONc.ChunkSize=4096
+FONc.ClassicModel=false
+FONc.NoGlobalAttrs=true
+AllowedHosts+=^https?:\/\/'''
+    cur_dir_bs = os.getcwd() +'/'
+    bes_conf_str=add_module_path(bes_conf_str,"BES.module.dap=",cur_dir_bs+'libdap_module.so')
+    bes_conf_str=add_module_path(bes_conf_str,"BES.module.cmd=",cur_dir_bs+'libdap_xml_module.so')
+    bes_conf_str=add_module_path(bes_conf_str,"BES.module.dmrpp=",cur_dir_bs+'libdmrpp_module.so')
+    bes_conf_str=add_module_path(bes_conf_str,"BES.module.fonc=",cur_dir_bs+'libfonc_module.so')
+    bes_conf_str=add_module_path(bes_conf_str,"BES.Catalog.catalog.RootDirectory=",os.getcwd())
+    bc_fid=open(bes_conf_name, 'w')
+    bc_fid.write(bes_conf_str)
+    bc_fid.close()
+
+# Generate the xml file that tells the BES to return the netCDF-4 file.
+def generate_bes_cmd(dmrpp_name):
+    bescmd_str_p1 = '''<?xml version="1.0" encoding="UTF-8"?>
+<request reqID="some_unique_value" >
+    <setContext name="dap_format">dap4</setContext>
+    <setContainer name="data" space="catalog">'''
+    bescmd_str_p1 = bescmd_str_p1 + dmrpp_name + "</setContainer>"
+    bescmd_str_p2 = '''
+    <define name="d">
+        <container name="data" />'''
+    bescmd_str_p3 = '''
+    </define>
+    <get type="dap" definition="d" returnAs="netcdf-4"/>
+</request>'''
+    bescmd_str=bescmd_str_p1+bescmd_str_p2+bescmd_str_p3
+    bescmd_name=dmrpp_name+"_test.bescmd"
+    bc_fid=open(bescmd_name, 'w')
+    bc_fid.write(bescmd_str)
+    bc_fid.close()
+    return bescmd_name
+
+

modules/dmrpp_module/get_dmrpp/unit-tests/gen_dmrpp_side_car_test.py

@@ -3,6 +3,12 @@
 Test gen_dmrpp_side_car script.
 python3 -m unittest gen_dmrpp_side_car_test
 
+The testing procedure is:
+1) Generate the dmrpp file.
+2) Generate the netCDF-4 file based on the dmrpp and the original files.
+3) Use ncdump to dump the generated netCDF-4 files.
+4) Compare the dumped output with the baseline file.
+5) clean up the intermeidate testing files. 
 To persist generated assets (e.g. for debugging) set environment
 variable `PRESERVE_TEST_ASSETS` to any value:
 PRESERVE_TEST_ASSETS=yes python3 -m unittest gen_dmrpp_side_car_test
@@ -11,100 +17,101 @@
 import subprocess
 import filecmp
 import os
+import gen_bescmd_conf
 
 class TestSample(unittest.TestCase):
 
+    @classmethod
+    def setUpClass(self):
+        gen_bescmd_conf.generate_bes_conf()
+
     def test_gen_dmrpp_side_car(self):
 
         print("Testing grid_2_2d_ps.hdf")
-        subprocess.run(["./gen_dmrpp_side_car", "-i", "grid_2_2d_ps.hdf","-H"])
+        subprocess.run(["./gen_dmrpp_side_car", "-i", "grid_2_2d_ps.hdf", "-u", "grid_2_2d_ps.hdf", "-H"])
+        bescmd_name = gen_bescmd_conf.generate_bes_cmd("grid_2_2d_ps.hdf.dmrpp")
+        fonc_name=bescmd_name+"_fonc.nc4"
+        subprocess.run(["besstandalone", "-c", "bes.test.conf", "-i",bescmd_name , "-f", fonc_name])
+
+        # Here we use ncdump -h since the dmrpp sidecar data involves the floating-point data calculation 
+        # of complex formula. The output data may have a little difference in different platforms.
+        with open('grid_2_2d_ps.hdf.dmrpp_fonc.nc4.header','w') as nc_header_file:
+            subprocess.run(["ncdump", "-h", fonc_name],stdout=nc_header_file)
+        result = filecmp.cmp("grid_2_2d_ps.hdf.dmrpp_fonc.nc4.header","grid_2_2d_ps.hdf.dmrpp_fonc.nc4.header.baseline")
+        self.assertEqual(result ,True )
         if not os.environ.get('PRESERVE_TEST_ASSETS'):
             self.addCleanup(os.remove, "grid_2_2d_ps.hdf.dmrpp")
             self.addCleanup(os.remove, "grid_2_2d_ps.hdf_mvs.h5")
-
-        #result = filecmp.cmp("grid_2_2d_ps.hdf.dmrpp","grid_2_2d_ps.hdf.dmrpp.baseline")
-        #self.assertEqual(result ,True )
-
-        # Since we also add the dmrpp metadata generation information for the HDF4 files,
-        # we need to ignore that information when doing comparison.
-        with open('grid_2_2d_ps.hdf.dmrpp') as f:
-            dmrpp_lines_after_77 = f.readlines()[77:]
-        with open('grid_2_2d_ps.hdf.dmrpp.baseline') as f1:
-            baseline_lines_after_77 = f1.readlines()[77:]
-
-        # Hacky removal of lines that otherwise show spurious failure
-        # due to test brittleness. (Better fix would be to run same version
-        # stripping as on non-python tests)
-        # Although the above statement may be true, 
-        # the following two lines may cause IndexError,however, github macOS build keeps throwing errors.
-        dmrpp_lines_after_77.pop(221)
-        baseline_lines_after_77.pop(221)
-
-        self.assertEqual(dmrpp_lines_after_77 ,baseline_lines_after_77)
-
+            self.addCleanup(os.remove, fonc_name)
+            self.addCleanup(os.remove, bescmd_name)
+            self.addCleanup(os.remove, "grid_2_2d_ps.hdf.dmrpp_fonc.nc4.header")
+
 
     def test_gen_dmrpp_side_car2(self):
 
         print("Testing grid_2_2d_sin.h5")
-        subprocess.run(["./gen_dmrpp_side_car", "-i", "grid_2_2d_sin.h5"])
+        subprocess.run(["./gen_dmrpp_side_car", "-i", "grid_2_2d_sin.h5", "-u", "grid_2_2d_sin.h5"])
+        bescmd_name = gen_bescmd_conf.generate_bes_cmd("grid_2_2d_sin.h5.dmrpp")
+        fonc_name=bescmd_name+"_fonc.nc4"
+        subprocess.run(["besstandalone", "-c", "bes.test.conf", "-i",bescmd_name , "-f", fonc_name])
+
+        # For the use of ncdump -h, see comments of test_gen_dmrpp_side_car().
+        with open('grid_2_2d_sin.h5.dmrpp_fonc.nc4.header','w') as nc_header_file:
+            subprocess.run(["ncdump", "-h", fonc_name],stdout=nc_header_file)
+        result = filecmp.cmp("grid_2_2d_sin.h5.dmrpp_fonc.nc4.header","grid_2_2d_sin.h5.dmrpp_fonc.nc4.header.baseline")
+        self.assertEqual(result ,True )
         if not os.environ.get('PRESERVE_TEST_ASSETS'):
             self.addCleanup(os.remove, "grid_2_2d_sin.h5.dmrpp")
             self.addCleanup(os.remove, "grid_2_2d_sin.h5_mvs.h5")
-        with open('grid_2_2d_sin.h5.dmrpp') as f:
-            dmrpp_lines_after_19 = f.readlines()[19:]
-        with open('grid_2_2d_sin.h5.dmrpp.baseline') as f1:
-            baseline_lines_after_19 = f1.readlines()[19:]
-
-        # Hacky removal of lines that otherwise show spurious failure
-        # due to test brittleness. (Better fix would be to run same version
-        # stripping as on non-python tests)
-        # The following two lines may cause IndexError and they are not necessary since the testing file should not be changed.
-        # If the testing file is changed, the right way is to update the baseline file.
-        #dmrpp_lines_after_19.pop(221)
-        #baseline_lines_after_19.pop(221)
+            self.addCleanup(os.remove, fonc_name)
+            self.addCleanup(os.remove, bescmd_name)
+            self.addCleanup(os.remove, "grid_2_2d_sin.h5.dmrpp_fonc.nc4.header")
 
-        self.assertEqual(dmrpp_lines_after_19 ,baseline_lines_after_19)
 
     def test_gen_dmrpp_side_car_h4_nsc(self):
 
         print("Testing grid_1_2d.hdf :no side car file")
-        subprocess.run(["./gen_dmrpp_side_car", "-i", "grid_1_2d.hdf","-H","-D"])
+        subprocess.run(["./gen_dmrpp_side_car", "-i", "grid_1_2d.hdf","-H","-D","-u", "grid_1_2d.hdf"])
+        bescmd_name = gen_bescmd_conf.generate_bes_cmd("grid_1_2d.hdf.dmrpp")
+        fonc_name=bescmd_name+"_fonc.nc4"
+        subprocess.run(["besstandalone", "-c", "bes.test.conf", "-i",bescmd_name , "-f", fonc_name])
+        with open('grid_1_2d.hdf.dmrpp_fonc.nc4.data','w') as nc_data_file:
+            subprocess.run(["ncdump", fonc_name],stdout=nc_data_file)
+        result = filecmp.cmp("grid_1_2d.hdf.dmrpp_fonc.nc4.data","grid_1_2d.hdf.dmrpp_fonc.nc4.data.baseline")
+        self.assertEqual(result ,True )
         if not os.environ.get('PRESERVE_TEST_ASSETS'):
             self.addCleanup(os.remove, "grid_1_2d.hdf.dmrpp")
-
-        # Since we also add the dmrpp metadata generation information for the HDF4 files,
-        # we need to ignore those information when doing comparision.
-        with open('grid_1_2d.hdf.dmrpp') as f:
-            dmrpp_lines_after_54 = f.readlines()[54:]
-        with open('grid_1_2d.hdf.dmrpp.baseline') as f1:
-            baseline_lines_after_54 = f1.readlines()[54:]
-        self.assertEqual(dmrpp_lines_after_54 ,baseline_lines_after_54)
+            self.addCleanup(os.remove, fonc_name)
+            self.addCleanup(os.remove, bescmd_name)
+            self.addCleanup(os.remove, "grid_1_2d.hdf.dmrpp_fonc.nc4.data")
+
 
     def test_gen_dmrpp_side_car_h5_cf(self):
 
         print("Testing grid_1_2d.h5 :CF option")
-        subprocess.run(["./gen_dmrpp_side_car", "-i", "grid_1_2d.h5","-c"])
+        subprocess.run(["./gen_dmrpp_side_car", "-i", "grid_1_2d.h5","-c","-u", "grid_1_2d.h5"])
+        bescmd_name = gen_bescmd_conf.generate_bes_cmd("grid_1_2d.h5.dmrpp")
+        fonc_name=bescmd_name+"_fonc.nc4"
+        subprocess.run(["besstandalone", "-c", "bes.test.conf", "-i",bescmd_name , "-f", fonc_name])
+        with open('grid_1_2d.h5.dmrpp_fonc.nc4.data','w') as nc_data_file:
+            subprocess.run(["ncdump", fonc_name],stdout=nc_data_file)
+        result = filecmp.cmp("grid_1_2d.h5.dmrpp_fonc.nc4.data","grid_1_2d.h5.dmrpp_fonc.nc4.data.baseline")
+        self.assertEqual(result ,True )
+
         if not os.environ.get('PRESERVE_TEST_ASSETS'):
             self.addCleanup(os.remove, "grid_1_2d.h5.dmrpp")
-
-
-        # Since we also add the dmrpp metadata generation informatio for the HDF4 files,
-        # we need to ignore those information when doing comparision.
-        # We also ignore the first two lines to skip comparing dmrpp versions
-        with open('grid_1_2d.h5.dmrpp') as f:
-            dmrpp_minus_18_lines = f.readlines()[2:-18]
-        with open('grid_1_2d.h5.dmrpp.baseline') as f1:
-            baseline_minus_18_lines = f1.readlines()[2:-18]
-
-        # Hacky removal of lines that otherwise show spurious failure
-        # due to test brittleness. (Better fix would be to run same version
-        # stripping as on non-python tests)
-        # Although the above statement may be true, the following lines may cause IndexError.So comment them out.
-        #dmrpp_minus_18_lines.pop(62)
-        #baseline_minus_18_lines.pop(62)
-
-        self.assertEqual(dmrpp_minus_18_lines ,baseline_minus_18_lines)
-
+            self.addCleanup(os.remove, "grid_1_2d.h5_mvs.h5")
+            self.addCleanup(os.remove, fonc_name)
+            self.addCleanup(os.remove, bescmd_name)
+            self.addCleanup(os.remove, "grid_1_2d.h5.dmrpp_fonc.nc4.data")
+
+
+    @classmethod
+    def tearDownClass(cls):
+        if os.path.exists("bes.test.conf"):
+            os.remove("bes.test.conf")
+
+
 if __name__ == '__main__':
     unittest.main()
 

modules/dmrpp_module/get_dmrpp/unit-tests/grid_1_2d.h5.dmrpp.baseline

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="ISO-8859-1"?>
-<Dataset xmlns="http://xml.opendap.org/ns/DAP/4.0#" xmlns:dmrpp="http://xml.opendap.org/dap/dmrpp/1.0.0#" dapVersion="4.0" dmrVersion="1.0" name="grid_1_2d.h5" dmrpp:href="OPeNDAP_DMRpp_DATA_ACCESS_URL" dmrpp:version="3.21.1">
+<Dataset xmlns="http://xml.opendap.org/ns/DAP/4.0#" xmlns:dmrpp="http://xml.opendap.org/dap/dmrpp/1.0.0#" dapVersion="4.0" dmrVersion="removed" name="grid_1_2d.h5" dmrpp:href="grid_1_2d.h5" dmrpp:version="3.21.1">
     <Dimension name="lon" size="8"/>
     <Dimension name="lat" size="4"/>
     <Float32 name="lon">
@@ -70,7 +70,7 @@
     </Attribute>
     <Attribute name="build_dmrpp_metadata" type="Container">
         <Attribute name="created" type="String">
-            <Value>2026-04-29T17:58:18Z</Value>
+            <Value>2026-06-10T13:31:25Z</Value>
         </Attribute>
         <Attribute name="build_dmrpp" type="String">
             <Value>3.21.1</Value>
@@ -82,7 +82,7 @@
             <Value>libdap-3.21.1</Value>
         </Attribute>
         <Attribute name="invocation" type="String">
-            <Value>build_dmrpp -f /home/rocky/kent/hyrax-2121-test/bes/modules/dmrpp_module/get_dmrpp/unit-tests/grid_1_2d.h5 -r grid_1_2d.h5.dmr -u OPeNDAP_DMRpp_DATA_ACCESS_URL -M</Value>
+            <Value>/Users/myang6/work/opendap/hyrax-2180/bes/modules/dmrpp_module/.libs/build_dmrpp -f /Users/myang6/work/opendap/hyrax-2180/bes/modules/dmrpp_module/get_dmrpp/unit-tests/grid_1_2d.h5 -r grid_1_2d.h5.dmr -u grid_1_2d.h5 -M</Value>
         </Attribute>
     </Attribute>
 </Dataset>

modules/dmrpp_module/get_dmrpp/unit-tests/grid_1_2d.h5.dmrpp_fonc.nc4.data.baseline

@@ -0,0 +1,61 @@
+netcdf grid_1_2d.h5.dmrpp_test.bescmd_fonc {
+dimensions:
+	lon = 8 ;
+	lat = 4 ;
+	StructMetadata_0_len = 663 ;
+variables:
+	float lon(lon) ;
+		lon:units = "degrees_east" ;
+	float lat(lat) ;
+		lat:units = "degrees_north" ;
+	float temperature(lat, lon) ;
+		temperature:units = "K" ;
+		temperature:origname = "temperature" ;
+		temperature:fullnamepath = "/HDFEOS/GRIDS/GeoGrid/Data Fields/temperature" ;
+		temperature:orig_dimname_list = "YDim XDim" ;
+	char StructMetadata_0(StructMetadata_0_len) ;
+		StructMetadata_0:origname = "StructMetadata.0" ;
+		StructMetadata_0:fullnamepath = "/HDFEOS INFORMATION/StructMetadata.0" ;
+data:
+
+ lon = 0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5 ;
+
+ lat = 3.5, 2.5, 1.5, 0.5 ;
+
+ temperature =
+  10, 10, 10, 10, 10, 10, 10, 10,
+  11, 11, 11, 11, 11, 11, 11, 11,
+  12, 12, 12, 12, 12, 12, 12, 12,
+  13, 13, 13, 13, 13, 13, 13, 13 ;
+
+ StructMetadata_0 = "GROUP=SwathStructure\n",
+    "END_GROUP=SwathStructure\n",
+    "GROUP=GridStructure\n",
+    "\tGROUP=GRID_1\n",
+    "\t\tGridName=\"GeoGrid\"\n",
+    "\t\tXDim=8\n",
+    "\t\tYDim=4\n",
+    "\t\tUpperLeftPointMtrs=(0.000000,4000000.000000)\n",
+    "\t\tLowerRightMtrs=(8000000.000000,0.000000)\n",
+    "\t\tProjection=HE5_GCTP_GEO\n",
+    "\t\tGROUP=Dimension\n",
+    "\t\tEND_GROUP=Dimension\n",
+    "\t\tGROUP=DataField\n",
+    "\t\t\tOBJECT=DataField_1\n",
+    "\t\t\t\tDataFieldName=\"temperature\"\n",
+    "\t\t\t\tDataType=H5T_NATIVE_FLOAT\n",
+    "\t\t\t\tDimList=(\"YDim\",\"XDim\")\n",
+    "\t\t\t\tMaxdimList=(\"YDim\",\"XDim\")\n",
+    "\t\t\tEND_OBJECT=DataField_1\n",
+    "\t\tEND_GROUP=DataField\n",
+    "\t\tGROUP=MergedFields\n",
+    "\t\tEND_GROUP=MergedFields\n",
+    "\tEND_GROUP=GRID_1\n",
+    "END_GROUP=GridStructure\n",
+    "GROUP=PointStructure\n",
+    "END_GROUP=PointStructure\n",
+    "GROUP=ZaStructure\n",
+    "END_GROUP=ZaStructure\n",
+    "END\n",
+    "" ;
+}