SDAP-351: Fix reading of data with cftime.datetime.Gregorian type (#46)
* Generalized datetime fix so all calendars are supported
* Added unit tests for 4 calendar types (standard, missing attr, julian, gregorian
diff --git a/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py b/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
index dc28c19..c36b8d2 100644
--- a/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
+++ b/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
@@ -74,7 +74,7 @@
"Time slices must have length 1, but '{dim}' has length {dim_len}.".format(dim=self.time,
dim_len=time_slice_len))
- if isinstance(ds[self.time][time_slice.start].item(), cftime.DatetimeJulian):
+ if isinstance(ds[self.time][time_slice.start].item(), cftime.datetime):
ds[self.time] = ds.indexes[self.time].to_datetimeindex()
new_tile.time = int(ds[self.time][time_slice.start].item() / 1e9)
diff --git a/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py b/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
index 725ba35..97f282c 100644
--- a/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
+++ b/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
@@ -47,7 +47,7 @@
raise RuntimeError(
"Time slices must have length 1, but '{dim}' has length {dim_len}.".format(dim=self.time,
dim_len=time_slice_len))
- if isinstance(ds[self.time][time_slice.start].item(), cftime.DatetimeJulian):
+ if isinstance(ds[self.time][time_slice.start].item(), cftime.datetime):
ds[self.time] = ds.indexes[self.time].to_datetimeindex()
new_tile.time = int(ds[self.time][time_slice.start].item() / 1e9)
diff --git a/granule_ingester/tests/granules/20181231090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc b/granule_ingester/tests/granules/20181231090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc
new file mode 100644
index 0000000..31ca293
--- /dev/null
+++ b/granule_ingester/tests/granules/20181231090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc
Binary files differ
diff --git a/granule_ingester/tests/granules/20190630_d-ACRI-L4-CHL-MULTI_4KM-GLO-REP.nc b/granule_ingester/tests/granules/20190630_d-ACRI-L4-CHL-MULTI_4KM-GLO-REP.nc
new file mode 100644
index 0000000..6fd6cd3
--- /dev/null
+++ b/granule_ingester/tests/granules/20190630_d-ACRI-L4-CHL-MULTI_4KM-GLO-REP.nc
Binary files differ
diff --git a/granule_ingester/tests/granules/OISSS_L4_multimission_global_7d_v1.0_2021-03-12.nc b/granule_ingester/tests/granules/OISSS_L4_multimission_global_7d_v1.0_2021-03-12.nc
new file mode 100644
index 0000000..283bfe4
--- /dev/null
+++ b/granule_ingester/tests/granules/OISSS_L4_multimission_global_7d_v1.0_2021-03-12.nc
Binary files differ
diff --git a/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py b/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
index 607dd8b..3c47718 100644
--- a/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
+++ b/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
@@ -165,27 +165,71 @@
self.assertRaises(RuntimeError, GridMultiVariableReadingProcessor, [], 'lat', 'lon', time='time')
return
-class TestReadGPMData(unittest.TestCase):
- def test_generate_tile(self):
- granule_path = path.join(path.dirname(__file__), '../granules/3B-DAY-E.MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4')
+
+class TestCalendars(unittest.TestCase):
+ """
+ Test that various calendars can be ingested into SDAP without error.
+ """
+ def assert_time_data(self, granule_name, time_var, lat_var, lon_var, data_vars):
+ granule_path = path.join(
+ path.dirname(__file__),
+ '../granules/',
+ granule_name
+ )
dimensions_to_slices = {
- 'time': slice(0, 1),
- 'lat': slice(0, 30),
- 'lon': slice(0, 30)
+ time_var: slice(0, 1),
+ lat_var: slice(0, 30),
+ lon_var: slice(0, 30)
}
- tile = nexusproto.NexusTile()
+ tile = nexusproto.NexusTile()
with xr.open_dataset(granule_path, decode_cf=True) as ds:
reading_processor = GridMultiVariableReadingProcessor(
- ['HQprecipitation', 'HQprecipitation_cnt'],
- 'lat',
- 'lon',
- time='time'
+ data_vars,
+ lat_var,
+ lon_var,
+ time=time_var
)
tile = reading_processor._generate_tile(ds, dimensions_to_slices, tile)
assert tile.tile.grid_multi_variable_tile.time
+ def test_julian_calendar_tile(self):
+ self.assert_time_data(
+ granule_name='3B-DAY-E.MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4',
+ time_var='time',
+ lat_var='lat',
+ lon_var='lon',
+ data_vars=['HQprecipitation', 'HQprecipitation_cnt']
+ )
+
+ def test_gregorian_calendar_tile(self):
+ self.assert_time_data(
+ granule_name='20190630_d-ACRI-L4-CHL-MULTI_4KM-GLO-REP.nc',
+ time_var='time',
+ lat_var='lat',
+ lon_var='lon',
+ data_vars=['CHL', 'CHL_error']
+ )
+
+ def test_standard_calendar_tile(self):
+ self.assert_time_data(
+ granule_name='OISSS_L4_multimission_global_7d_v1.0_2021-03-12.nc',
+ time_var='time',
+ lat_var='latitude',
+ lon_var='longitude',
+ data_vars=['sss', 'sss_uncertainty']
+ )
+
+ def test_missing_calendar_tile(self):
+ self.assert_time_data(
+ granule_name='20181231090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc',
+ time_var='time',
+ lat_var='lat',
+ lon_var='lon',
+ data_vars=['analysed_sst', 'analysis_error']
+ )
+
if __name__ == '__main__':
unittest.main()
diff --git a/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
index 6cfa835..ef78394 100644
--- a/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
+++ b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
@@ -347,22 +347,71 @@
self.assertRaises(RuntimeError, GridReadingProcessor, [], 'lat', 'lon', time='time')
return
-class TestReadGPMData(unittest.TestCase):
- def test_generate_tile(self):
- granule_path = path.join(path.dirname(__file__), '../granules/3B-DAY-E.MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4')
+class TestCalendars(unittest.TestCase):
+ """
+ Test that various calendars can be ingested into SDAP without error.
+ """
+
+ def assert_time_data(self, granule_name, time_var, lat_var, lon_var, data_var):
+ granule_path = path.join(
+ path.dirname(__file__),
+ '../granules/',
+ granule_name
+ )
dimensions_to_slices = {
- 'time': slice(0, 1),
- 'lat': slice(0, 30),
- 'lon': slice(0, 30)
+ time_var: slice(0, 1),
+ lat_var: slice(0, 30),
+ lon_var: slice(0, 30)
}
- tile = nexusproto.NexusTile()
+ tile = nexusproto.NexusTile()
with xr.open_dataset(granule_path, decode_cf=True) as ds:
- reading_processor = GridReadingProcessor(['HQprecipitation'], 'lat', 'lon', time='time')
+ reading_processor = GridReadingProcessor(
+ [data_var],
+ lat_var,
+ lon_var,
+ time=time_var
+ )
tile = reading_processor._generate_tile(ds, dimensions_to_slices, tile)
assert tile.tile.grid_tile.time
+ def test_julian_calendar_tile(self):
+ self.assert_time_data(
+ granule_name='3B-DAY-E.MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4',
+ time_var='time',
+ lat_var='lat',
+ lon_var='lon',
+ data_var='HQprecipitation'
+ )
+
+ def test_gregorian_calendar_tile(self):
+ self.assert_time_data(
+ granule_name='20190630_d-ACRI-L4-CHL-MULTI_4KM-GLO-REP.nc',
+ time_var='time',
+ lat_var='lat',
+ lon_var='lon',
+ data_var='CHL'
+ )
+
+ def test_standard_calendar_tile(self):
+ self.assert_time_data(
+ granule_name='OISSS_L4_multimission_global_7d_v1.0_2021-03-12.nc',
+ time_var='time',
+ lat_var='latitude',
+ lon_var='longitude',
+ data_var='sss'
+ )
+
+ def test_missing_calendar_tile(self):
+ self.assert_time_data(
+ granule_name='20181231090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc',
+ time_var='time',
+ lat_var='lat',
+ lon_var='lon',
+ data_var='analysed_sst'
+ )
+
if __name__ == '__main__':
unittest.main()
