Merge pull request #73 from ec-jrc/DailyDecumulation

Daily decumulation and Gridding

Author: gnrgomes

.gitignore

@@ -53,6 +53,7 @@ tests/**/*aux.xml
 !src/lisfloodutilities/gridding/**/*.nc
 tests/data/gridding/meteo_out/**/*.nc
 tests/data/gridding/meteo_out/**/*.tiff
+tests/data/gridding/meteo_out/**/*.kiwis
 .settings
 .project
 .pydevproject

README.md

@@ -0,0 +1,14 @@
+#!python
+
+import os
+import sys
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+src_dir = os.path.normpath(os.path.join(current_dir, '../src/'))
+if os.path.exists(src_dir):
+    sys.path.append(src_dir)
+
+from lisfloodutilities.gridding.decumulate_daily_grids import main_script
+
+if __name__ == '__main__':
+    main_script()

bin/decumulate

@@ -16,7 +16,7 @@ more-itertools>=8.2.0
 netCDF4>=1.5.3,<=1.6.4
 nine>=1.1.0
 numexpr>=2.8.4
-numpy>=1.21.6
+numpy>=1.22.3
 packaging>=23.1
 pandas>=1.3.5
 partd>=1.4.0

requirements.txt

@@ -130,6 +130,7 @@ def run(self):
                 'compare: compare two set of netCDF files; '
                 'thresholds: compute discharge return period thresholds; '
                 'gridding: interpolate meteo variables observations; '
+                'decumulate: decumulate daily grids into 6 hourly grids in kiwis format; '
                 'ncextract: extract values from netCDF files; '
                 'catchstats: calculates catchment statistics; ',
     long_description=long_description,
@@ -143,12 +144,12 @@ def run(self):
                       # 'GDAL=={}'.format(gdal_version),
                       'netCDF4>=1.5.3', 'toolz', 'xarray>=0.15.1',
                       'dask', 'pandas>=0.25.1', 'nine', 'pyg2p'],
-    author="Valerio Lorini, Stefania Grimaldi, Carlo Russo, Domenico Nappo, Lorenzo Alfieri",
-    author_email="[email protected],[email protected],[email protected],[email protected],[email protected]",
+    author="Valerio Lorini, Stefania Grimaldi, Carlo Russo, Goncalo Gomes, Domenico Nappo, Lorenzo Alfieri",
+    author_email="[email protected],[email protected],[email protected],[email protected],[email protected],[email protected]",
     keywords=['netCDF4', 'PCRaster', 'mapstack', 'lisflood', 'efas', 'glofas', 'ecmwf', 'copernicus'],
     license='EUPL 1.2',
     url='https://github.com/ec-jrc/lisflood-utilities',
-    scripts=['bin/pcr2nc', 'bin/cutmaps', 'bin/compare', 'bin/nc2pcr', 'bin/thresholds', 'bin/gridding', 'bin/cddmap', 'bin/ncextract','bin/catchstats',],
+    scripts=['bin/pcr2nc', 'bin/cutmaps', 'bin/compare', 'bin/nc2pcr', 'bin/thresholds', 'bin/gridding', 'bin/decumulate', 'bin/cddmap', 'bin/ncextract','bin/catchstats',],
     zip_safe=True,
     classifiers=[
           # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers

setup.py

@@ -11,10 +11,19 @@ VALUE_OFFSET = 0.0
 DATA_TYPE_PACKED = i2
 STANDARD_NAME = precipitation_amount
 LONG_NAME = Daily Accumulated Precipitation
+
+# 1280 - IMGW
+# 1295 - MARS
+# 1302 - CarpatClim
 # 1303 - ERAinterim
+# 1304 - EURO4M-APGD
+# 1310 - HNMS
 # 1329 - ERA5-land
 KIWIS_FILTER_PLUGIN_CLASSES = {'ObservationsKiwisFilter': {'1303': 100.0, '1329': 100.0}}
 
+# 1310/HNMS until 31/12/2000, 1304/EURO4M-APGD until 31/12/2008, 1302/CarpatClim until 31/12/2010, 1280/IMGW until 31/12/2020 and 1295/MARS until the end of times
+KIWIS_FILTER_DECUMULATION_CONFIG = [{'START': '1989-12-31 00:00:00', 'END': '2000-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5, '1295': 1.5, '1302': 1.5, '1304': 1.5, '1310': 1.5}},{'START': '2001-01-01 00:00:00', 'END': '2008-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5, '1295': 1.5, '1302': 1.5, '1304': 1.5}},{'START': '2009-01-01 00:00:00', 'END': '2010-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5, '1295': 1.5, '1302': 1.5}},{'START': '2011-01-01 00:00:00', 'END': '2020-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1280': 1.5, '1295': 1.5}},{'START': '2021-01-01 00:00:00', 'END': '', 'PROVIDERS_RADIUS': {'1295': 1.5}}]
+
 [VAR_TIME]
 
 UNIT = days since 1990-01-02 06:00:00.0

src/lisfloodutilities/gridding/configuration/1arcmin/config_pr.txt

@@ -11,12 +11,15 @@ VALUE_OFFSET = 0.0
 DATA_TYPE_PACKED = i2
 STANDARD_NAME = precipitation_amount
 LONG_NAME = 6 Hourly Accumulated Precipitation
-# 1304 - EURO4M-APGD
-# 1302 - CarpatClim
+
+# 1280 - IMGW
 # 1295 - MARS
+# 1302 - CarpatClim
 # 1303 - ERAinterim
+# 1304 - EURO4M-APGD
+# 1310 - HNMS
 # 1329 - ERA5-land
-KIWIS_FILTER_PLUGIN_CLASSES = {'DowgradedObservationsKiwisFilter': {'1304': 1.0, '1302': 1.0, '1295': 1.0}, 'ObservationsKiwisFilter': {'1303': 100.0, '1329': 100.0}}
+# KIWIS_FILTER_PLUGIN_CLASSES = {'DowgradedObservationsKiwisFilter': {'1304': 1.0, '1302': 1.0, '1295': 1.0}, 'ObservationsKiwisFilter': {'1303': 100.0, '1329': 100.0}}
 
 
 [VAR_TIME]

src/lisfloodutilities/gridding/configuration/1arcmin/config_pr6.txt

@@ -12,6 +12,8 @@ DATA_TYPE_PACKED = i2
 STANDARD_NAME = integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air
 LONG_NAME = Daily Calculated Radiation
 
+KIWIS_FILTER_PLUGIN_CLASSES = {'SolarRadiationLimitsKiwisFilter': {'EXCLUDE_BELLOW_LATITUDE': 66.0, 'EXCLUDE_BELLOW_VALUE': 0.0}}
+
 [VAR_TIME]
 
 UNIT = days since 1990-01-02 00:00:00.0

src/lisfloodutilities/gridding/configuration/1arcmin/config_rg.txt

@@ -13,6 +13,16 @@ DATA_TYPE_PACKED = i2
 STANDARD_NAME = air_temperature
 LONG_NAME = 6 Hourly Average Temperature
 
+# 1280 - IMGW
+# 1295 - MARS
+# 1302 - CarpatClim
+# 1303 - ERAinterim
+# 1304 - EURO4M-APGD
+# 1310 - HNMS
+# 1323 - ICON
+# 1329 - ERA5-land
+KIWIS_FILTER_PLUGIN_CLASSES = {'ObservationsKiwisFilter': {'1329': 700.0}, 'ProvidersKiwisFilter': {'1323': [('2022-01-01 06:00:00', '2023-12-31 06:00:00')]}}
+
 [VAR_TIME]
 
 UNIT_PATTERN = hours since %%Y-%%m-%%d %%H:%%M:%%S.%%f

src/lisfloodutilities/gridding/configuration/1arcmin/config_ta6.txt

@@ -13,6 +13,16 @@ DATA_TYPE_PACKED = i2
 STANDARD_NAME = air_temperature
 LONG_NAME = Daily Minimum Temperature
 
+# 1280 - IMGW
+# 1295 - MARS
+# 1302 - CarpatClim
+# 1303 - ERAinterim
+# 1304 - EURO4M-APGD
+# 1310 - HNMS
+# 1323 - ICON
+# 1329 - ERA5-land
+KIWIS_FILTER_PLUGIN_CLASSES = {'ObservationsKiwisFilter': {'1329': 700.0}, 'ProvidersKiwisFilter': {'1323': [('2022-01-01 06:00:00', '2023-12-31 06:00:00')]}}
+
 [VAR_TIME]
 
 UNIT = days since 1990-01-01 06:00:00.0

src/lisfloodutilities/gridding/configuration/1arcmin/config_tn.txt

@@ -13,6 +13,16 @@ DATA_TYPE_PACKED = i2
 STANDARD_NAME = air_temperature
 LONG_NAME = Daily Maximum Temperature
 
+# 1280 - IMGW
+# 1295 - MARS
+# 1302 - CarpatClim
+# 1303 - ERAinterim
+# 1304 - EURO4M-APGD
+# 1310 - HNMS
+# 1323 - ICON
+# 1329 - ERA5-land
+KIWIS_FILTER_PLUGIN_CLASSES = {'ObservationsKiwisFilter': {'1329': 700.0}, 'ProvidersKiwisFilter': {'1323': [('2022-01-01 18:00:00', '2023-12-31 18:00:00')]}}
+
 [VAR_TIME]
 
 UNIT = days since 1990-01-01 18:00:00.0

src/lisfloodutilities/gridding/configuration/1arcmin/config_tx.txt

@@ -32,10 +32,21 @@ VALUE_OFFSET = 0.0
 DATA_TYPE_PACKED = i2
 STANDARD_NAME = DUMMY_STANDARD_NAME
 LONG_NAME = DUMMY LONG NAME
+
 KIWIS_FILTER_COLUMNS = {'COL_LAT': 'station_local_y', 'COL_LON': 'station_local_x', 'COL_IS_IN_DOMAIN': 'EFAS_ADDATTR_ISINNEWDOMAIN'}
 # KIWIS_FILTER_COLUMNS = {}
 KIWIS_FILTER_PLUGIN_CLASSES = {'KiwisFilter': {}}
 
+# 1280 - IMGW
+# 1295 - MARS
+# 1302 - CarpatClim
+# 1303 - ERAinterim
+# 1304 - EURO4M-APGD
+# 1310 - HNMS
+# 1329 - ERA5-land
+# KIWIS_FILTER_DECUMULATION_CONFIG = [{'START': '1989-12-31 00:00:00', 'END': '2000-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1295': 1.5, '1302': 1.5}}]
+KIWIS_FILTER_DECUMULATION_CONFIG = []
+
 [DIMENSION]
 
 TOP = 72.25000000000000000000

src/lisfloodutilities/gridding/configuration/1arcmin/default.txt

@@ -33,8 +33,18 @@ VALUE_OFFSET = 0.0
 DATA_TYPE_PACKED = i2
 STANDARD_NAME = DUMMY_STANDARD_NAME
 LONG_NAME = DUMMY LONG NAME
+
 KIWIS_FILTER_COLUMNS = {'COL_LAT': 'station_local_y', 'COL_LON': 'station_local_x', 'COL_IS_IN_DOMAIN': 'EFAS_ADDATTR_ISINNEWDOMAIN'}
 KIWIS_FILTER_PLUGIN_CLASSES = {'KiwisFilter': {}}
+# 1280 - IMGW
+# 1295 - MARS
+# 1302 - CarpatClim
+# 1303 - ERAinterim
+# 1304 - EURO4M-APGD
+# 1310 - HNMS
+# 1329 - ERA5-land
+# KIWIS_FILTER_DECUMULATION_CONFIG = [{'START': '1989-12-31 00:00:00', 'END': '2000-12-31 23:59:59', 'PROVIDERS_RADIUS': {'1295': 1.5, '1302': 1.5}}]
+KIWIS_FILTER_DECUMULATION_CONFIG = []
 
 [DIMENSION]