diff --git a/.gitignore b/.gitignore index 1f1c025f5..e8b16d9d4 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,6 @@ opendaTestReports/ **/build-test **/MANIFEST.MF **/tests/*.log +**/*_temp_build/ +**/castor_temp_src/ OpenDATimings_*.txt diff --git a/core/java/resources/openDA/core_castor.jar b/core/java/resources/openDA/core_castor.jar index 0a569c4ff..00e6a8f03 100644 Binary files a/core/java/resources/openDA/core_castor.jar and b/core/java/resources/openDA/core_castor.jar differ diff --git a/core/java/resources/openDA/core_castor_schemas.zip b/core/java/resources/openDA/core_castor_schemas.zip index a122eaed8..bdb47db18 100644 Binary files a/core/java/resources/openDA/core_castor_schemas.zip and b/core/java/resources/openDA/core_castor_schemas.zip differ diff --git a/core/java/resources/openDA/core_castor_src.zip b/core/java/resources/openDA/core_castor_src.zip index f97f7990f..94eb9cb2e 100644 Binary files a/core/java/resources/openDA/core_castor_src.zip and b/core/java/resources/openDA/core_castor_src.zip differ diff --git a/core/java/src/org/openda/exchange/GenericNetcdfArrayExchangeItem.java b/core/java/src/org/openda/exchange/GenericNetcdfArrayExchangeItem.java new file mode 100644 index 000000000..67c5ebe73 --- /dev/null +++ b/core/java/src/org/openda/exchange/GenericNetcdfArrayExchangeItem.java @@ -0,0 +1,64 @@ +package org.openda.exchange; + +import org.openda.exchange.dataobjects.GenericNetcdfDataObject; +import org.openda.interfaces.IExchangeItem; + +public class GenericNetcdfArrayExchangeItem extends ExchangeItem implements IExchangeItem { + + private final GenericNetcdfDataObject genericNetcdfDataObject; + + public GenericNetcdfArrayExchangeItem(String exchangeItemId, GenericNetcdfDataObject genericNetcdfDataObject){ + super(exchangeItemId); + this.genericNetcdfDataObject = genericNetcdfDataObject; + } + + @Override + public ValueType getValuesType() { + throw new RuntimeException("org.openda.exchange.GenericNetcdfArrayExchangeItem.getValuesType not implemented yet"); + } + + @Override + public Class getValueType() { + throw new RuntimeException("org.openda.exchange.GenericNetcdfArrayExchangeItem.getValueType not implemented yet"); + } + + @Override + public Object getValues() { + throw new RuntimeException("org.openda.exchange.GenericNetcdfArrayExchangeItem.getValues not implemented yet"); + } + + @Override + public double[] getValuesAsDoubles() { + return this.genericNetcdfDataObject.getArrayExchangeItemValues(this.getId()); + } + + @Override + public void axpyOnValues(double alpha, double[] axpyValues) { + double[] values = getValuesAsDoubles(); + for (int i = 0; i < values.length; i++) { + if (Double.isNaN(values[i])) { + System.out.println("OrgVal Nan at " + i); + } + values[i] += alpha * axpyValues[i]; + if (Double.isNaN(values[i])) { + System.out.println("ModVal Nan at " + i); + } + } + setValuesAsDoubles(values); + } + + @Override + public void multiplyValues(double[] multiplicationFactors) { + throw new RuntimeException("org.openda.exchange.GenericNetcdfArrayExchangeItem.multiplyValues not implemented yet"); + } + + @Override + public void setValues(Object values) { + throw new RuntimeException("org.openda.exchange.GenericNetcdfArrayExchangeItem.setValues not implemented yet"); + } + + @Override + public void setValuesAsDoubles(double[] values) { + this.genericNetcdfDataObject.setArrayExchangeItemValues(this.getId(), values); + } +} diff --git a/core/java/src/org/openda/exchange/dataobjects/GenericNetcdfArrayConfig.java b/core/java/src/org/openda/exchange/dataobjects/GenericNetcdfArrayConfig.java new file mode 100644 index 000000000..251cba779 --- /dev/null +++ b/core/java/src/org/openda/exchange/dataobjects/GenericNetcdfArrayConfig.java @@ -0,0 +1,33 @@ +package org.openda.exchange.dataobjects; + +import org.openda.utils.DimensionIndex; + +import java.util.HashMap; + +public class GenericNetcdfArrayConfig { + private final String exchangeitemId; + private final String netcdf_variable_name; + private final HashMap netcdf_dimension_var; + + public GenericNetcdfArrayConfig(String id, String variable_name, HashMap dimensions) { + exchangeitemId = id; + netcdf_variable_name = variable_name; + netcdf_dimension_var = dimensions; + } + + public String getExchangeitemId(){ + return exchangeitemId; + } + + public String getNetcdfVariableName(){ + return netcdf_variable_name; + } + + public String[] getNetcdfDimensionVariables(){ + return netcdf_dimension_var.keySet().toArray(new String[0]); + } + + public DimensionIndex getDimensionIndex(String dimension_variable_id){ + return netcdf_dimension_var.get(dimension_variable_id); + } +} diff --git a/core/java/src/org/openda/exchange/dataobjects/GenericNetcdfConfig.java b/core/java/src/org/openda/exchange/dataobjects/GenericNetcdfConfig.java new file mode 100644 index 000000000..806164ef7 --- /dev/null +++ b/core/java/src/org/openda/exchange/dataobjects/GenericNetcdfConfig.java @@ -0,0 +1,13 @@ +package org.openda.exchange.dataobjects; + +import java.util.ArrayList; +import java.util.HashMap; + +public class GenericNetcdfConfig { + +// ArrayList tsConfigs = new ArrayList<>(); +// ArrayList arrayConfigs = new ArrayList<>(); + + HashMap tsConfigs = new HashMap<>(); + HashMap arrayConfigs = new HashMap<>(); +} diff --git a/core/java/src/org/openda/exchange/dataobjects/GenericNetcdfConfigReader.java b/core/java/src/org/openda/exchange/dataobjects/GenericNetcdfConfigReader.java new file mode 100644 index 000000000..65e85c1b1 --- /dev/null +++ b/core/java/src/org/openda/exchange/dataobjects/GenericNetcdfConfigReader.java @@ -0,0 +1,64 @@ +package org.openda.exchange.dataobjects; + +import org.openda.core.io.castorgenerated.*; +import org.openda.utils.DimensionIndex; +import org.openda.utils.io.CastorUtils; + +import java.io.File; +import java.util.HashMap; + +public class GenericNetcdfConfigReader { + + private GenericNetcdfConfig genericNetcdfConfig; + + public GenericNetcdfConfigReader(File genericNetcdfConfigFile) { + TNC_dataObjectConfig tncDataObjectConfig = (TNC_dataObjectConfig) CastorUtils.parse(genericNetcdfConfigFile, TNC_dataObjectConfig.class); + genericNetcdfConfig = parseGenericNetcdfConfig(genericNetcdfConfigFile, tncDataObjectConfig); + } + + public GenericNetcdfConfig getGenericNetcdfConfig() { + return genericNetcdfConfig; + } + + public static GenericNetcdfConfig parseGenericNetcdfConfig(File genericNetcdfConfigFile, TNC_dataObjectConfig tncDataObjectConfig) { + + GenericNetcdfConfig netcdfConfig = new GenericNetcdfConfig(); + + org.openda.core.io.castorgenerated.TNC_arrayExchangeItem[] arrayExchangeItemsXML = tncDataObjectConfig.getArrayExchangeItem(); + for (org.openda.core.io.castorgenerated.TNC_arrayExchangeItem arrayExchangeItemXML: arrayExchangeItemsXML) { + String id = arrayExchangeItemXML.getId().getContent(); + String variable_name = arrayExchangeItemXML.getValues().getSelection().getVariable()[0].getName(); // only 1 variable is possible for array exchange item + int nDimension = arrayExchangeItemXML.getValues().getSelection().getVariable()[0].getDimensionCount(); + HashMap dimensions = new HashMap<>(); + if (nDimension>0){ + dimensions = new HashMap<>(); + for (int i=0; i exchangeItems = new HashMap<>(); + private GenericNetcdfConfig ncConfig; + private NetcdfFile netcdfFile; + private File ncFilePath; + + @Override + public void initialize(File workingDir, String[] arguments) { + ncFilePath = new File(workingDir, arguments[0]); + File xmlFilePath = new File(workingDir, arguments[1]); + GenericNetcdfConfigReader ncConfigReader = new GenericNetcdfConfigReader(xmlFilePath); + this.ncConfig = ncConfigReader.getGenericNetcdfConfig(); + // now create a handle to netCDF + try { + this.netcdfFile = NetcdfFile.open(ncFilePath.getAbsolutePath()); + } catch (IOException e) { + throw new RuntimeException("GenericNetcdfDataObject could not open netcdf file " + ncFilePath.getAbsolutePath()); + } + + // create array exchangeItems + if (this.ncConfig.arrayConfigs.size()>0){ + GenericNetcdfArrayConfig arrayConfig; + for (String keyset: this.ncConfig.arrayConfigs.keySet()){ + arrayConfig = ncConfig.arrayConfigs.get(keyset); + IExchangeItem arrayExchangeItem = new GenericNetcdfArrayExchangeItem(arrayConfig.getExchangeitemId(),this); + this.exchangeItems.put(arrayExchangeItem.getId(),arrayExchangeItem); + } + } + + // create time series exchangeItems + if (this.ncConfig.tsConfigs.size()>0){ + GenericNetcdfTSConfig tsConfig; + for (String keyset: this.ncConfig.tsConfigs.keySet()){ + tsConfig = ncConfig.tsConfigs.get(keyset); + String template = tsConfig.getTemplate(); + if (template != null){ + // create exchangeItemId's + String exchangeItemId; + String metaQuantity = ""; + if (template.toLowerCase().contains("${meta:quantity}")){ + metaQuantity = tsConfig.getMetaQuantity(); + } + String[] metaLocVarValuesXML = tsConfig.getMetaLocationVariableValues(); + String[] metaLocVarValuesNetCDF = null; + try { + Map temp = NetcdfUtils.readAndStoreStationIdsMap(netcdfFile, tsConfig.getMetaLocationVariableName()); + metaLocVarValuesNetCDF = new String[temp.size()]; + for (int i=0; i timeInfoMap = new HashMap<>(); + IArrayTimeInfo timeInfoCache = null; + timeInfoMap.put(timeVariable,timeInfoCache); + IArrayTimeInfo timeInfo = NetcdfUtils.createTimeInfo(timeVariable, netcdfFile, timeInfoMap); + + for (int iMetaLocVarValues=0; iMetaLocVarValues metaLocDimensions = metaLocationVariable.getDimensions(); + int dimensionIndex = -1; + String metaDimensionName; + for (int i = 0; i < metaLocDimensions.size(); i++) { + metaDimensionName = metaLocDimensions.get(i).getShortName(); + if (valuesVariable.findDimensionIndex(metaDimensionName) != -1) { + dimensionIndex = valuesVariable.findDimensionIndex(metaDimensionName); + } + } + if (dimensionIndex == -1) { + throw new RuntimeException("GenericNetcdfDataObject, TimeSeriesExchangeItem: no shared dimension is available between and ." + + "The specified cannot be used to select the "); + } + + origin[dimensionIndex] = Arrays.asList(metaLocVarValuesNetCDF).indexOf(metaLocVarValuesXML[iMetaLocVarValues]); + sizeArray[dimensionIndex] = 1; + + HashMap valuesDimension = tsConfig.getValuesVariableDimensions(); + int dimIndex; + for (String valuesDimensionName : valuesDimension.keySet()) { + dimIndex = valuesVariable.findDimensionIndex(valuesDimensionName); + if (dimIndex == -1) { + throw new RuntimeException("GenericNetcdfDataObject, TimeSeries, variable " + tsConfig.getValuesVariableName() + " doesn't have the following dimension-variable: " + valuesDimensionName); + } + DimensionIndex dimIndexDI = valuesDimension.get(valuesDimensionName); + int startIndex = dimIndexDI.getStart(); + int endIndex = dimIndexDI.getEnd(); + if (startIndex == -1) { + origin[dimIndex] = sizeArray[dimIndex] - 1; + sizeArray[dimIndex] = 1; + } else { + origin[dimIndex] = startIndex; + sizeArray[dimIndex] = endIndex - startIndex; + } + } + String metaLayerVariableName = tsConfig.getMetaLayerVariableName(); + double[] values = null; + if (metaLayerVariableName==null) { + + values = NetcdfUtils.readSelectedData(valuesVariable, origin, sizeArray, -1); + + if (values.length != timeInfo.getTimes().length) { + throw new RuntimeException("GenericNetcdfDataObject, TimeSeriesExchangeItem: the length of values and times are not identical. " + + "Please check the configuration of the GenericNetcdfDataObject. " + + "Include all selection variables in the template, e.g. ${meta:location}, ${meta:layer}, or ${meta:quantity}."); + } + + exchangeItemId = template.replace("${meta:location}", metaLocVarValuesXML[iMetaLocVarValues]); + exchangeItemId = exchangeItemId.replace("${meta:quantity}", metaQuantity); + + IExchangeItem tsExchangeItem = new GenericNetcdfTSExchangeItem(exchangeItemId, timeInfo, values); + this.exchangeItems.put(tsExchangeItem.getId(), tsExchangeItem); + } else { + Variable metaLayerVariable = netcdfFile.findVariable(metaLayerVariableName); + List metaLayDimensions = metaLayerVariable.getDimensions(); + dimensionIndex = -1; + for (int i = 0; i < metaLayDimensions.size(); i++) { + metaDimensionName = metaLayDimensions.get(i).getShortName(); + if (valuesVariable.findDimensionIndex(metaDimensionName) != -1) { + dimensionIndex = valuesVariable.findDimensionIndex(metaDimensionName); + } + } + if (dimensionIndex == -1) { + throw new RuntimeException("GenericNetcdfDataObject, TimeSeriesExchangeItem: no shared dimension is available between and ." + + "The specified cannot be used to select the "); + } + + int layerArraySize = sizeArray[dimensionIndex]; + sizeArray[dimensionIndex] = 1; + for (int iLayer=0; iLayer timeInfoMap = new HashMap<>(); + IArrayTimeInfo timeInfoCache = null; + timeInfoMap.put(timeVariable,timeInfoCache); + IArrayTimeInfo timeInfo = NetcdfUtils.createTimeInfo(timeVariable, netcdfFile, timeInfoMap); + + // get values: + Variable valuesVariable = netcdfFile.findVariable(tsConfig.getValuesVariableName()); + int[] origin = createOrigin(valuesVariable); + int[] sizeArray = valuesVariable.getShape(); + HashMap valuesDimension = tsConfig.getValuesVariableDimensions(); + int dimIndex; + for (String valuesDimensionName : valuesDimension.keySet()) { + dimIndex = valuesVariable.findDimensionIndex(valuesDimensionName); + if (dimIndex == -1) { + throw new RuntimeException("GenericNetcdfDataObject, TimeSeries, variable " + tsConfig.getValuesVariableName() + " doesn't have the following dimension-variable: " + valuesDimensionName); + } + DimensionIndex dimIndexDI = valuesDimension.get(valuesDimensionName); + int startIndex = dimIndexDI.getStart(); + int endIndex = dimIndexDI.getEnd(); + if (startIndex == -1) { + origin[dimIndex] = sizeArray[dimIndex] - 1; + sizeArray[dimIndex] = 1; + } else { + origin[dimIndex] = startIndex; + sizeArray[dimIndex] = endIndex - startIndex; + } + } + double[] values = NetcdfUtils.readSelectedData(valuesVariable, origin, sizeArray, -1); + IExchangeItem tsExchangeItem = new GenericNetcdfTSExchangeItem(id, timeInfo, values); + this.exchangeItems.put(tsExchangeItem.getId(), tsExchangeItem); + } + + } + } + } + + @Override + public String[] getExchangeItemIDs() { + return exchangeItems.keySet().toArray(new String[0]); + } + + @Override + public String[] getExchangeItemIDs(IPrevExchangeItem.Role role) { + return getExchangeItemIDs(); + } + + @Override + public IExchangeItem getDataObjectExchangeItem(String exchangeItemID) { + return exchangeItems.get(exchangeItemID); + } + + @Override + public void finish() { + for (String id : this.exchangeItems.keySet()){ + IExchangeItem exchangeItem = this.exchangeItems.get(id); + double[] values = exchangeItem.getValuesAsDoubles(); + writeArrayExchangeItemValues(id,values); + } + try { + netcdfFile.close(); + } catch (IOException e){ + e.printStackTrace(); + } + + } + + public static int[] createOrigin(Variable var) { + int dimensionCount = var.getDimensions().size(); + int[] origin = new int[dimensionCount]; + Arrays.fill(origin, 0); + return origin; + } + + public void writeArrayExchangeItemValues(String id, double[] values) { + GenericNetcdfArrayConfig arrayConfig = ncConfig.arrayConfigs.get(id); + String variableName = arrayConfig.getNetcdfVariableName(); + Variable variable = netcdfFile.findVariable(variableName); + int[] origin = createOrigin(variable); + int[] sizeArray = variable.getShape(); + // loop over dimension variables and their selection indices, and then redefine origin and sizeArray + for (int iDimVar=0; iDimVar valuesVariableDimensions; + + public GenericNetcdfTSConfig(String id, String template, TNC_timeSeriesExchangeItem timeseriesExchangeItemXML) { + this.id = id; + this.template = template; + if (timeseriesExchangeItemXML.getMeta() != null){ + if (timeseriesExchangeItemXML.getMeta().getLocation() == null){ + throw new RuntimeException("GenericNetcdfTSConfig: is missing in . Please check the configuration file."); + } + TNC_variable[] metaLocationVariablesXML = timeseriesExchangeItemXML.getMeta().getLocation().getSelection().getVariable(); + this.metaLocationVariableName = metaLocationVariablesXML[0].getName(); + this.metaLocationVariableValues = new String[metaLocationVariablesXML.length]; + for (int i=0; i(); + if (nDimension>0){ + for (int i=0; i getValuesVariableDimensions(){ + return this.valuesVariableDimensions; + } +} diff --git a/core/java/src/org/openda/exchange/dataobjects/GenericNetcdfTSExchangeItem.java b/core/java/src/org/openda/exchange/dataobjects/GenericNetcdfTSExchangeItem.java new file mode 100644 index 000000000..80d8608d0 --- /dev/null +++ b/core/java/src/org/openda/exchange/dataobjects/GenericNetcdfTSExchangeItem.java @@ -0,0 +1,102 @@ +package org.openda.exchange.dataobjects; + +import org.openda.interfaces.*; + +public class GenericNetcdfTSExchangeItem implements IExchangeItem { + private final String id; + private final IArrayTimeInfo timeInfo; + private final double[] values; + + public GenericNetcdfTSExchangeItem(String exchangeItemId, IArrayTimeInfo timeInfo, double[] values) { + this.id = exchangeItemId; + this.timeInfo = timeInfo; + this.values = values; + } + + @Override + public Role getRole() { + throw new RuntimeException("org.openda.exchange.dataobjects.GenericNetcdfTSExchangeItem.getRole not implemented yet"); + } + + @Override + public String getId() { + return this.id; + } + + @Override + public String getDescription() { + throw new RuntimeException("org.openda.exchange.dataobjects.GenericNetcdfTSExchangeItem.getDescription not implemented yet"); + } + + @Override + public Class getValueType() { + throw new RuntimeException("org.openda.exchange.dataobjects.GenericNetcdfTSExchangeItem.getValueType not implemented yet"); + } + + @Override + public void copyValuesFromItem(IExchangeItem sourceItem) { + throw new RuntimeException("org.openda.exchange.dataobjects.GenericNetcdfTSExchangeItem.copyValuesFromItem not implemented yet"); + } + + @Override + public ITimeInfo getTimeInfo() { + return this.timeInfo; + } + + @Override + public IQuantityInfo getQuantityInfo() { + throw new RuntimeException("org.openda.exchange.dataobjects.GenericNetcdfTSExchangeItem.getQuantityInfo not implemented yet"); + } + + @Override + public IGeometryInfo getGeometryInfo() { + throw new RuntimeException("org.openda.exchange.dataobjects.GenericNetcdfTSExchangeItem.getGeometryInfo not implemented yet"); + } + + @Override + public ValueType getValuesType() { + throw new RuntimeException("org.openda.exchange.dataobjects.GenericNetcdfTSExchangeItem.getValuesType not implemented yet"); + } + + @Override + public Object getValues() { + throw new RuntimeException("org.openda.exchange.dataobjects.GenericNetcdfTSExchangeItem.getValues not implemented yet"); + } + + @Override + public double[] getValuesAsDoubles() { + return this.values; + } + + @Override + public void axpyOnValues(double alpha, double[] axpyValues) { + for (int i=0; i + + + + + + + + + + + + + waterlevel + + + + + + + + + + + + + + + + + + + + + + + + + + waterlevel + + + + + + + + + + + + + + + + + + + + + + + + + + v_velocity + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + v_velocity + + + + + + + + + + + + + diff --git a/core/java/test/org/openda/exchange/dataobjects/testData/delft3d_map_nc.xml b/core/java/test/org/openda/exchange/dataobjects/testData/delft3d_map_nc.xml new file mode 100644 index 000000000..6882264f6 --- /dev/null +++ b/core/java/test/org/openda/exchange/dataobjects/testData/delft3d_map_nc.xml @@ -0,0 +1,43 @@ + + + + + Waterlevel.Grid + + + + + + + + + + + + + VELU.SurfaceLayer + + + + + + + + + + + + VELV.SurfaceLayerSection + + + + + + + + + + + + diff --git a/core/java/test/org/openda/exchange/dataobjects/testData/delft3d_timeseries_from_map_nc.xml b/core/java/test/org/openda/exchange/dataobjects/testData/delft3d_timeseries_from_map_nc.xml new file mode 100644 index 000000000..ad27d15a8 --- /dev/null +++ b/core/java/test/org/openda/exchange/dataobjects/testData/delft3d_timeseries_from_map_nc.xml @@ -0,0 +1,41 @@ + + + + + Waterlevel.Station_n10_m12 + + + + + + + + + + + + + + + + + + + V1_velocity.Station_n10_m12 + + + + + + + + + + + + + + + + + diff --git a/core/java/test/org/openda/exchange/dataobjects/testData/mapfile.nc b/core/java/test/org/openda/exchange/dataobjects/testData/mapfile.nc new file mode 100644 index 000000000..c2377b1f2 Binary files /dev/null and b/core/java/test/org/openda/exchange/dataobjects/testData/mapfile.nc differ diff --git a/core/java/test/org/openda/exchange/dataobjects/testData/netCDFDataObject.xsd b/core/java/test/org/openda/exchange/dataobjects/testData/netCDFDataObject.xsd new file mode 100644 index 000000000..d46059886 --- /dev/null +++ b/core/java/test/org/openda/exchange/dataobjects/testData/netCDFDataObject.xsd @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + index (e.g. 12) or index line (e.g. 11:18) in an array + + + + + + diff --git a/core/java/test/org/openda/exchange/dataobjects/testData/trih-cadagno_netcdf.nc b/core/java/test/org/openda/exchange/dataobjects/testData/trih-cadagno_netcdf.nc new file mode 100644 index 000000000..b1c315625 Binary files /dev/null and b/core/java/test/org/openda/exchange/dataobjects/testData/trih-cadagno_netcdf.nc differ diff --git a/core/xmlSchemas/genericNetcdfDataObject.xsd b/core/xmlSchemas/genericNetcdfDataObject.xsd new file mode 100644 index 000000000..3e1fc5bb4 --- /dev/null +++ b/core/xmlSchemas/genericNetcdfDataObject.xsd @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + index (e.g. 12) or index line (e.g. 11:18) in an array + + + + + + diff --git a/core/xmlSchemas/openDA.xsd b/core/xmlSchemas/openDA.xsd index 580b243f8..65fa8d3e2 100644 --- a/core/xmlSchemas/openDA.xsd +++ b/core/xmlSchemas/openDA.xsd @@ -11,4 +11,5 @@ +