From 4f68694801a7b6cd52edff03ce3b4d094c51b3ea Mon Sep 17 00:00:00 2001 From: pierrepetersmeier <155652256+pierrepetersmeier@users.noreply.github.com> Date: Wed, 28 May 2025 16:37:58 +0200 Subject: [PATCH 1/4] Add weather interpolation --- CHANGELOG.md | 1 + .../datamodel/io/source/WeatherSource.java | 245 ++++++++++++++---- .../datamodel/models/value/WeatherValue.java | 85 ++++-- 3 files changed, 250 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37d4cc181..947e8fdb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed handling of `CongestionResult.InputModelType` in `EntityProcessor` [#1325](https://github.com/ie3-institute/PowerSystemDataModel/issues/1325) +- Consider None-equivalent value for missing data points in weather [#1304](https://github.com/ie3-institute/PowerSystemDataModel/issues/1304) ### Changed - Updated dependabot workflow and added CODEOWNERS [#1328](https://github.com/ie3-institute/PowerSystemDataModel/issues/1328) diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index a7a056d85..1075c0438 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -2,7 +2,7 @@ * © 2021. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation -*/ + */ package edu.ie3.datamodel.io.source; import edu.ie3.datamodel.exceptions.SourceException; @@ -14,14 +14,17 @@ import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.datamodel.utils.Try; import edu.ie3.util.interval.ClosedInterval; +import java.time.Duration; import java.time.ZonedDateTime; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.measure.Quantity; import org.apache.commons.lang3.tuple.Pair; import org.locationtech.jts.geom.Point; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import tech.units.indriya.ComparableQuantity; /** Abstract class for WeatherSource by Csv and Sql Data */ public abstract class WeatherSource extends EntitySource { @@ -35,7 +38,7 @@ public abstract class WeatherSource extends EntitySource { protected static final String COORDINATE_ID = "coordinateid"; protected WeatherSource( - IdCoordinateSource idCoordinateSource, TimeBasedWeatherValueFactory weatherFactory) { + IdCoordinateSource idCoordinateSource, TimeBasedWeatherValueFactory weatherFactory) { this.idCoordinateSource = idCoordinateSource; this.weatherFactory = weatherFactory; } @@ -52,26 +55,162 @@ public void validate() throws ValidationException { validate(WeatherValue.class, this::getSourceFields, weatherFactory); } + /** + * Method for interpolating weather values. + * + * @return a new quantity + */ + protected List> interpolateMissingValues( + List> timeSeries) { + + List> result = new ArrayList<>(); + int i = 0; + while (i < timeSeries.size()) { + TimeBasedValue current = timeSeries.get(i); + + if (current.getValue() != null) { + result.add(current); + i++; + continue; + } + int prevIdx = i - 1; + int nextIdx = i + 1; + while (nextIdx < timeSeries.size() && timeSeries.get(nextIdx).getValue() == null) { + nextIdx++; + } + + if (prevIdx >= 0 && nextIdx < timeSeries.size()) { + TimeBasedValue prev = timeSeries.get(prevIdx); + TimeBasedValue next = timeSeries.get(nextIdx); + Duration total = Duration.between(prev.getTime(), next.getTime()); + for (int j = i; j < nextIdx; j++) { + TimeBasedValue missing = timeSeries.get(j); + Duration fromPrev = Duration.between(prev.getTime(), missing.getTime()); + double ratio = (double) fromPrev.toSeconds() / total.toSeconds(); + WeatherValue interpolated = + interpolateWeatherValue(prev.getValue(), next.getValue(), ratio); + result.add(new TimeBasedValue<>(missing.getTime(), interpolated)); + } + i = nextIdx; + } else { + result.add(current); + i++; + } + } + + return result; + } + + private WeatherValue interpolateWeatherValue(WeatherValue start, WeatherValue end, double ratio) { + var direct = interpolateOptional(start.getDirectIrradiance(), end.getDirectIrradiance(), ratio); + var diffuse = interpolateOptional(start.getDiffuseIrradiance(), end.getDiffuseIrradiance(), ratio); + + var temp = interpolateDirect(start.getTemperature(), end.getTemperature(), ratio); + var dir = interpolateDirect(start.getWindDirection(), end.getWindDirection(), ratio); + var vel = interpolateDirect(start.getWindVelocity(), end.getWindVelocity(), ratio); + + return new WeatherValue(start.getCoordinate(), direct, diffuse, temp, dir, vel); + } + + + private > ComparableQuantity interpolateOptional( + Optional> startOpt, + Optional> endOpt, + double ratio) { + return startOpt + .flatMap(startVal -> endOpt.map(endVal -> interpolateQuantity(startVal, endVal, ratio))) + .orElse(null); + } + + private > ComparableQuantity interpolateDirect( + ComparableQuantity start, ComparableQuantity end, double ratio) { + return interpolateQuantity(start, end, ratio); + } + + private > ComparableQuantity interpolateQuantity( + ComparableQuantity a, ComparableQuantity b, double ratio) { + return a.add(b.subtract(a).multiply(ratio)); + } + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- public abstract Map> getWeather( - ClosedInterval timeInterval) throws SourceException; + ClosedInterval timeInterval) throws SourceException; public abstract Map> getWeather( - ClosedInterval timeInterval, Collection coordinates) - throws SourceException; + ClosedInterval timeInterval, Collection coordinates) + throws SourceException; public abstract Optional> getWeather( - ZonedDateTime date, Point coordinate) throws SourceException; + ZonedDateTime date, Point coordinate) throws SourceException; + + public Optional getWeatherInterpolated( + ZonedDateTime date, Point coordinate, int plus, int minus) throws SourceException { + + ClosedInterval interpolationInterval = + new ClosedInterval<>(date.minusHours(minus), date.plusHours(plus)); + IndividualTimeSeries ts = + getWeather(interpolationInterval, List.of(coordinate)).get(coordinate); + + if (ts == null) { + log.warn("No time series available for coordinate {}", coordinate); + return Optional.empty(); + } + + Optional value = ts.getValue(date); + + if (value.isPresent() && value.get().isComplete()) { + return value; + } + + Optional> prevValue = ts.getPreviousTimeBasedValue(date); + Optional> nextValue = ts.getNextTimeBasedValue(date); + + if (prevValue.isEmpty() || nextValue.isEmpty()) { + log.warn( + "Not enough data to interpolate weather value at {} for coordinate {}", date, coordinate); + return Optional.empty(); + } + + TimeBasedValue prev = prevValue.get(); + TimeBasedValue next = nextValue.get(); + + Duration totalDuration = Duration.between(prev.getTime(), next.getTime()); + Duration partialDuration = Duration.between(prev.getTime(), date); + + if (totalDuration.isZero()) { + return Optional.of(prev.getValue()); + } + + double ratio = (double) partialDuration.toSeconds() / totalDuration.toSeconds(); + + WeatherValue interpolated = interpolateWeatherValue(prev.getValue(), next.getValue(), ratio); + + return Optional.of(interpolated); + } + public abstract Map> getTimeKeysAfter(ZonedDateTime time) - throws SourceException; + throws SourceException; public List getTimeKeysAfter(ZonedDateTime time, Point coordinate) - throws SourceException { + throws SourceException { return getTimeKeysAfter(time).getOrDefault(coordinate, Collections.emptyList()); } + protected Map> interpolateWeatherData( + Map> rawData) { + return rawData.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> + new IndividualTimeSeries<>( + new LinkedHashSet<>( + interpolateMissingValues( + new ArrayList<>(entry.getValue().getEntries())))))); + } + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- /** @@ -82,7 +221,7 @@ public List getTimeKeysAfter(ZonedDateTime time, Point coordinate * @return the TimeBasedWeatherValueData */ protected Optional toTimeBasedWeatherValueData( - Map fieldMap) { + Map fieldMap) { String coordinateValue = fieldMap.remove(COORDINATE_ID); fieldMap.putIfAbsent("uuid", UUID.randomUUID().toString()); int coordinateId = Integer.parseInt(coordinateValue); @@ -101,16 +240,16 @@ protected Optional toTimeBasedWeatherValueData( * @return a map of coordinate point to time series */ protected Map> mapWeatherValuesToPoints( - Collection> timeBasedValues) { + Collection> timeBasedValues) { Map>> coordinateToValues = - timeBasedValues.stream() - .collect( - Collectors.groupingBy( - timeBasedWeatherValue -> timeBasedWeatherValue.getValue().getCoordinate(), - Collectors.toSet())); + timeBasedValues.stream() + .collect( + Collectors.groupingBy( + timeBasedWeatherValue -> timeBasedWeatherValue.getValue().getCoordinate(), + Collectors.toSet())); Map> coordinateToTimeSeriesMap = new HashMap<>(); for (Map.Entry>> entry : - coordinateToValues.entrySet()) { + coordinateToValues.entrySet()) { Set> values = entry.getValue(); IndividualTimeSeries timeSeries = new IndividualTimeSeries<>(values); coordinateToTimeSeriesMap.put(entry.getKey(), timeSeries); @@ -119,34 +258,34 @@ protected Map> mapWeatherValuesToPoint } protected Map> toTimeKeys( - Stream> fieldMaps, TimeBasedWeatherValueFactory factory) { + Stream> fieldMaps, TimeBasedWeatherValueFactory factory) { return groupTime( - fieldMaps.map( - fieldMap -> { - String coordinateValue = fieldMap.get(COORDINATE_ID); - int coordinateId = Integer.parseInt(coordinateValue); - Optional coordinate = idCoordinateSource.getCoordinate(coordinateId); - ZonedDateTime time = factory.extractTime(fieldMap); - - if (coordinate.isEmpty()) { - log.warn("Unable to match coordinate ID {} to a point", coordinateId); - } - return Pair.of(coordinate, time); - })); + fieldMaps.map( + fieldMap -> { + String coordinateValue = fieldMap.get(COORDINATE_ID); + int coordinateId = Integer.parseInt(coordinateValue); + Optional coordinate = idCoordinateSource.getCoordinate(coordinateId); + ZonedDateTime time = factory.extractTime(fieldMap); + + if (coordinate.isEmpty()) { + log.warn("Unable to match coordinate ID {} to a point", coordinateId); + } + return Pair.of(coordinate, time); + })); } protected Map> groupTime( - Stream, ZonedDateTime>> values) { + Stream, ZonedDateTime>> values) { return values - .filter(pair -> pair.getKey().isPresent()) - .map(pair -> Pair.of(pair.getKey().get(), pair.getValue())) - .collect(Collectors.groupingBy(Pair::getKey, Collectors.toSet())) - .entrySet() - .stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, - e -> e.getValue().stream().map(Pair::getValue).sorted().toList())); + .filter(pair -> pair.getKey().isPresent()) + .map(pair -> Pair.of(pair.getKey().get(), pair.getValue())) + .collect(Collectors.groupingBy(Pair::getKey, Collectors.toSet())) + .entrySet() + .stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> e.getValue().stream().map(Pair::getValue).sorted().toList())); } // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- @@ -159,19 +298,19 @@ protected Map> groupTime( * @return a list of that TimeBasedValues */ protected List> buildTimeBasedValues( - TimeBasedWeatherValueFactory factory, Stream> inputStream) - throws SourceException { + TimeBasedWeatherValueFactory factory, Stream> inputStream) + throws SourceException { return Try.scanStream( - inputStream.map( - fieldsToAttributes -> { - fieldsToAttributes.remove("tid"); - Optional data = - toTimeBasedWeatherValueData(fieldsToAttributes); - return factory.get( - Try.from(data, () -> new SourceException("Missing data in: " + data))); - }), - "TimeBasedValue") - .transform(Stream::toList, SourceException::new) - .getOrThrow(); + inputStream.map( + fieldsToAttributes -> { + fieldsToAttributes.remove("tid"); + Optional data = + toTimeBasedWeatherValueData(fieldsToAttributes); + return factory.get( + Try.from(data, () -> new SourceException("Missing data in: " + data))); + }), + "TimeBasedValue") + .transform(Stream::toList, SourceException::new) + .getOrThrow(); } -} +} \ No newline at end of file diff --git a/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java b/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java index 09ec1787f..d9c413c80 100644 --- a/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java +++ b/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java @@ -2,11 +2,12 @@ * © 2021. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation -*/ + */ package edu.ie3.datamodel.models.value; import edu.ie3.util.quantities.interfaces.Irradiance; import java.util.Objects; +import java.util.Optional; import javax.measure.quantity.Angle; import javax.measure.quantity.Speed; import javax.measure.quantity.Temperature; @@ -32,10 +33,10 @@ public class WeatherValue implements Value { * @param wind values for this coordinate */ public WeatherValue( - Point coordinate, - SolarIrradianceValue solarIrradiance, - TemperatureValue temperature, - WindValue wind) { + Point coordinate, + SolarIrradianceValue solarIrradiance, + TemperatureValue temperature, + WindValue wind) { this.coordinate = coordinate; this.solarIrradiance = solarIrradiance; this.temperature = temperature; @@ -52,17 +53,17 @@ public WeatherValue( * @param velocity Wind velocity for this coordinate (typically in m/s) */ public WeatherValue( - Point coordinate, - ComparableQuantity directSolarIrradiance, - ComparableQuantity diffuseSolarIrradiance, - ComparableQuantity temperature, - ComparableQuantity direction, - ComparableQuantity velocity) { + Point coordinate, + ComparableQuantity directSolarIrradiance, + ComparableQuantity diffuseSolarIrradiance, + ComparableQuantity temperature, + ComparableQuantity direction, + ComparableQuantity velocity) { this( - coordinate, - new SolarIrradianceValue(directSolarIrradiance, diffuseSolarIrradiance), - new TemperatureValue(temperature), - new WindValue(direction, velocity)); + coordinate, + new SolarIrradianceValue(directSolarIrradiance, diffuseSolarIrradiance), + new TemperatureValue(temperature), + new WindValue(direction, velocity)); } public Point getCoordinate() { @@ -81,15 +82,43 @@ public WindValue getWind() { return wind; } + public Optional> getDirectIrradiance() { + return solarIrradiance.getDirectIrradiance(); + } + + public Optional> getDiffuseIrradiance() { + return solarIrradiance.getDiffuseIrradiance(); + } + + public ComparableQuantity getTemperature() { + return temperature.getTemperature(); + } + + public ComparableQuantity getWindDirection() { + return wind.getDirection(); + } + public ComparableQuantity getWindVelocity() { + return wind.getVelocity(); + } + + /** + * Checks if all mandatory values are present. + * + * @return true if all values are present, false otherwise + */ + public boolean isComplete() { + return solarIrradiance != null && temperature != null && wind != null; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; WeatherValue that = (WeatherValue) o; return coordinate.equals(that.coordinate) - && solarIrradiance.equals(that.solarIrradiance) - && temperature.equals(that.temperature) - && wind.equals(that.wind); + && solarIrradiance.equals(that.solarIrradiance) + && temperature.equals(that.temperature) + && wind.equals(that.wind); } @Override @@ -100,14 +129,14 @@ public int hashCode() { @Override public String toString() { return "WeatherValue{" - + "coordinate=" - + coordinate - + ", solarIrradiance=" - + solarIrradiance - + ", temperature=" - + temperature - + ", wind=" - + wind - + '}'; + + "coordinate=" + + coordinate + + ", solarIrradiance=" + + solarIrradiance + + ", temperature=" + + temperature + + ", wind=" + + wind + + '}'; } -} +} \ No newline at end of file From 607ac114e01ac56c3b6a72d2bdcf584b8bdf0b21 Mon Sep 17 00:00:00 2001 From: pierrepetersmeier <155652256+pierrepetersmeier@users.noreply.github.com> Date: Tue, 3 Jun 2025 15:26:58 +0200 Subject: [PATCH 2/4] Apply review suggestions and delete interpolateDirect --- CHANGELOG.md | 2 +- .../datamodel/io/source/WeatherSource.java | 203 ++++++------------ .../datamodel/models/value/WeatherValue.java | 67 +++--- 3 files changed, 104 insertions(+), 168 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9523f63c6..b9b493287 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed handling of `CongestionResult.InputModelType` in `EntityProcessor` [#1325](https://github.com/ie3-institute/PowerSystemDataModel/issues/1325) -- Consider None-equivalent value for missing data points in weather [#1304](https://github.com/ie3-institute/PowerSystemDataModel/issues/1304) +- Add interpolation functionality to estimate missing weather values [#1304](https://github.com/ie3-institute/PowerSystemDataModel/issues/1304) - -Fixed em fields in input models [#1331](https://github.com/ie3-institute/PowerSystemDataModel/issues/1331) - Consider None-equivalent value for missing data points in weather [#1304](https://github.com/ie3-institute/PowerSystemDataModel/issues/1304) diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index 1075c0438..5771fd202 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -2,7 +2,7 @@ * © 2021. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation - */ +*/ package edu.ie3.datamodel.io.source; import edu.ie3.datamodel.exceptions.SourceException; @@ -38,7 +38,7 @@ public abstract class WeatherSource extends EntitySource { protected static final String COORDINATE_ID = "coordinateid"; protected WeatherSource( - IdCoordinateSource idCoordinateSource, TimeBasedWeatherValueFactory weatherFactory) { + IdCoordinateSource idCoordinateSource, TimeBasedWeatherValueFactory weatherFactory) { this.idCoordinateSource = idCoordinateSource; this.weatherFactory = weatherFactory; } @@ -55,102 +55,51 @@ public void validate() throws ValidationException { validate(WeatherValue.class, this::getSourceFields, weatherFactory); } - /** - * Method for interpolating weather values. - * - * @return a new quantity - */ - protected List> interpolateMissingValues( - List> timeSeries) { - - List> result = new ArrayList<>(); - int i = 0; - while (i < timeSeries.size()) { - TimeBasedValue current = timeSeries.get(i); - - if (current.getValue() != null) { - result.add(current); - i++; - continue; - } - int prevIdx = i - 1; - int nextIdx = i + 1; - while (nextIdx < timeSeries.size() && timeSeries.get(nextIdx).getValue() == null) { - nextIdx++; - } - - if (prevIdx >= 0 && nextIdx < timeSeries.size()) { - TimeBasedValue prev = timeSeries.get(prevIdx); - TimeBasedValue next = timeSeries.get(nextIdx); - Duration total = Duration.between(prev.getTime(), next.getTime()); - for (int j = i; j < nextIdx; j++) { - TimeBasedValue missing = timeSeries.get(j); - Duration fromPrev = Duration.between(prev.getTime(), missing.getTime()); - double ratio = (double) fromPrev.toSeconds() / total.toSeconds(); - WeatherValue interpolated = - interpolateWeatherValue(prev.getValue(), next.getValue(), ratio); - result.add(new TimeBasedValue<>(missing.getTime(), interpolated)); - } - i = nextIdx; - } else { - result.add(current); - i++; - } - } - - return result; - } - private WeatherValue interpolateWeatherValue(WeatherValue start, WeatherValue end, double ratio) { var direct = interpolateOptional(start.getDirectIrradiance(), end.getDirectIrradiance(), ratio); - var diffuse = interpolateOptional(start.getDiffuseIrradiance(), end.getDiffuseIrradiance(), ratio); + var diffuse = + interpolateOptional(start.getDiffuseIrradiance(), end.getDiffuseIrradiance(), ratio); - var temp = interpolateDirect(start.getTemperature(), end.getTemperature(), ratio); - var dir = interpolateDirect(start.getWindDirection(), end.getWindDirection(), ratio); - var vel = interpolateDirect(start.getWindVelocity(), end.getWindVelocity(), ratio); + var temp = interpolateOptional(start.getTemperatureValue(), end.getTemperatureValue(), ratio); + var dir = interpolateOptional(start.getWindDirection(), end.getWindDirection(), ratio); + var vel = interpolateOptional(start.getWindVelocity(), end.getWindVelocity(), ratio); return new WeatherValue(start.getCoordinate(), direct, diffuse, temp, dir, vel); } - private > ComparableQuantity interpolateOptional( - Optional> startOpt, - Optional> endOpt, - double ratio) { + Optional> startOpt, + Optional> endOpt, + double ratio) { return startOpt - .flatMap(startVal -> endOpt.map(endVal -> interpolateQuantity(startVal, endVal, ratio))) - .orElse(null); - } - - private > ComparableQuantity interpolateDirect( - ComparableQuantity start, ComparableQuantity end, double ratio) { - return interpolateQuantity(start, end, ratio); + .flatMap(startVal -> endOpt.map(endVal -> interpolateQuantity(startVal, endVal, ratio))) + .orElse(null); } private > ComparableQuantity interpolateQuantity( - ComparableQuantity a, ComparableQuantity b, double ratio) { + ComparableQuantity a, ComparableQuantity b, double ratio) { return a.add(b.subtract(a).multiply(ratio)); } // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- public abstract Map> getWeather( - ClosedInterval timeInterval) throws SourceException; + ClosedInterval timeInterval) throws SourceException; public abstract Map> getWeather( - ClosedInterval timeInterval, Collection coordinates) - throws SourceException; + ClosedInterval timeInterval, Collection coordinates) + throws SourceException; public abstract Optional> getWeather( - ZonedDateTime date, Point coordinate) throws SourceException; + ZonedDateTime date, Point coordinate) throws SourceException; public Optional getWeatherInterpolated( - ZonedDateTime date, Point coordinate, int plus, int minus) throws SourceException { + ZonedDateTime date, Point coordinate, int plus, int minus) throws SourceException { ClosedInterval interpolationInterval = - new ClosedInterval<>(date.minusHours(minus), date.plusHours(plus)); + new ClosedInterval<>(date.minusHours(minus), date.plusHours(plus)); IndividualTimeSeries ts = - getWeather(interpolationInterval, List.of(coordinate)).get(coordinate); + getWeather(interpolationInterval, List.of(coordinate)).get(coordinate); if (ts == null) { log.warn("No time series available for coordinate {}", coordinate); @@ -159,7 +108,7 @@ public Optional getWeatherInterpolated( Optional value = ts.getValue(date); - if (value.isPresent() && value.get().isComplete()) { + if (value.isPresent() && value.get().hasPartialValues()) { return value; } @@ -168,7 +117,7 @@ public Optional getWeatherInterpolated( if (prevValue.isEmpty() || nextValue.isEmpty()) { log.warn( - "Not enough data to interpolate weather value at {} for coordinate {}", date, coordinate); + "Not enough data to interpolate weather value at {} for coordinate {}", date, coordinate); return Optional.empty(); } @@ -189,28 +138,14 @@ public Optional getWeatherInterpolated( return Optional.of(interpolated); } - public abstract Map> getTimeKeysAfter(ZonedDateTime time) - throws SourceException; + throws SourceException; public List getTimeKeysAfter(ZonedDateTime time, Point coordinate) - throws SourceException { + throws SourceException { return getTimeKeysAfter(time).getOrDefault(coordinate, Collections.emptyList()); } - protected Map> interpolateWeatherData( - Map> rawData) { - return rawData.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, - entry -> - new IndividualTimeSeries<>( - new LinkedHashSet<>( - interpolateMissingValues( - new ArrayList<>(entry.getValue().getEntries())))))); - } - // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- /** @@ -221,7 +156,7 @@ protected Map> interpolateWeatherData( * @return the TimeBasedWeatherValueData */ protected Optional toTimeBasedWeatherValueData( - Map fieldMap) { + Map fieldMap) { String coordinateValue = fieldMap.remove(COORDINATE_ID); fieldMap.putIfAbsent("uuid", UUID.randomUUID().toString()); int coordinateId = Integer.parseInt(coordinateValue); @@ -240,16 +175,16 @@ protected Optional toTimeBasedWeatherValueData( * @return a map of coordinate point to time series */ protected Map> mapWeatherValuesToPoints( - Collection> timeBasedValues) { + Collection> timeBasedValues) { Map>> coordinateToValues = - timeBasedValues.stream() - .collect( - Collectors.groupingBy( - timeBasedWeatherValue -> timeBasedWeatherValue.getValue().getCoordinate(), - Collectors.toSet())); + timeBasedValues.stream() + .collect( + Collectors.groupingBy( + timeBasedWeatherValue -> timeBasedWeatherValue.getValue().getCoordinate(), + Collectors.toSet())); Map> coordinateToTimeSeriesMap = new HashMap<>(); for (Map.Entry>> entry : - coordinateToValues.entrySet()) { + coordinateToValues.entrySet()) { Set> values = entry.getValue(); IndividualTimeSeries timeSeries = new IndividualTimeSeries<>(values); coordinateToTimeSeriesMap.put(entry.getKey(), timeSeries); @@ -258,34 +193,34 @@ protected Map> mapWeatherValuesToPoint } protected Map> toTimeKeys( - Stream> fieldMaps, TimeBasedWeatherValueFactory factory) { + Stream> fieldMaps, TimeBasedWeatherValueFactory factory) { return groupTime( - fieldMaps.map( - fieldMap -> { - String coordinateValue = fieldMap.get(COORDINATE_ID); - int coordinateId = Integer.parseInt(coordinateValue); - Optional coordinate = idCoordinateSource.getCoordinate(coordinateId); - ZonedDateTime time = factory.extractTime(fieldMap); - - if (coordinate.isEmpty()) { - log.warn("Unable to match coordinate ID {} to a point", coordinateId); - } - return Pair.of(coordinate, time); - })); + fieldMaps.map( + fieldMap -> { + String coordinateValue = fieldMap.get(COORDINATE_ID); + int coordinateId = Integer.parseInt(coordinateValue); + Optional coordinate = idCoordinateSource.getCoordinate(coordinateId); + ZonedDateTime time = factory.extractTime(fieldMap); + + if (coordinate.isEmpty()) { + log.warn("Unable to match coordinate ID {} to a point", coordinateId); + } + return Pair.of(coordinate, time); + })); } protected Map> groupTime( - Stream, ZonedDateTime>> values) { + Stream, ZonedDateTime>> values) { return values - .filter(pair -> pair.getKey().isPresent()) - .map(pair -> Pair.of(pair.getKey().get(), pair.getValue())) - .collect(Collectors.groupingBy(Pair::getKey, Collectors.toSet())) - .entrySet() - .stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, - e -> e.getValue().stream().map(Pair::getValue).sorted().toList())); + .filter(pair -> pair.getKey().isPresent()) + .map(pair -> Pair.of(pair.getKey().get(), pair.getValue())) + .collect(Collectors.groupingBy(Pair::getKey, Collectors.toSet())) + .entrySet() + .stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> e.getValue().stream().map(Pair::getValue).sorted().toList())); } // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- @@ -298,19 +233,19 @@ protected Map> groupTime( * @return a list of that TimeBasedValues */ protected List> buildTimeBasedValues( - TimeBasedWeatherValueFactory factory, Stream> inputStream) - throws SourceException { + TimeBasedWeatherValueFactory factory, Stream> inputStream) + throws SourceException { return Try.scanStream( - inputStream.map( - fieldsToAttributes -> { - fieldsToAttributes.remove("tid"); - Optional data = - toTimeBasedWeatherValueData(fieldsToAttributes); - return factory.get( - Try.from(data, () -> new SourceException("Missing data in: " + data))); - }), - "TimeBasedValue") - .transform(Stream::toList, SourceException::new) - .getOrThrow(); + inputStream.map( + fieldsToAttributes -> { + fieldsToAttributes.remove("tid"); + Optional data = + toTimeBasedWeatherValueData(fieldsToAttributes); + return factory.get( + Try.from(data, () -> new SourceException("Missing data in: " + data))); + }), + "TimeBasedValue") + .transform(Stream::toList, SourceException::new) + .getOrThrow(); } -} \ No newline at end of file +} diff --git a/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java b/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java index d9c413c80..92f9368e8 100644 --- a/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java +++ b/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java @@ -2,7 +2,7 @@ * © 2021. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation - */ +*/ package edu.ie3.datamodel.models.value; import edu.ie3.util.quantities.interfaces.Irradiance; @@ -33,10 +33,10 @@ public class WeatherValue implements Value { * @param wind values for this coordinate */ public WeatherValue( - Point coordinate, - SolarIrradianceValue solarIrradiance, - TemperatureValue temperature, - WindValue wind) { + Point coordinate, + SolarIrradianceValue solarIrradiance, + TemperatureValue temperature, + WindValue wind) { this.coordinate = coordinate; this.solarIrradiance = solarIrradiance; this.temperature = temperature; @@ -53,17 +53,17 @@ public WeatherValue( * @param velocity Wind velocity for this coordinate (typically in m/s) */ public WeatherValue( - Point coordinate, - ComparableQuantity directSolarIrradiance, - ComparableQuantity diffuseSolarIrradiance, - ComparableQuantity temperature, - ComparableQuantity direction, - ComparableQuantity velocity) { + Point coordinate, + ComparableQuantity directSolarIrradiance, + ComparableQuantity diffuseSolarIrradiance, + ComparableQuantity temperature, + ComparableQuantity direction, + ComparableQuantity velocity) { this( - coordinate, - new SolarIrradianceValue(directSolarIrradiance, diffuseSolarIrradiance), - new TemperatureValue(temperature), - new WindValue(direction, velocity)); + coordinate, + new SolarIrradianceValue(directSolarIrradiance, diffuseSolarIrradiance), + new TemperatureValue(temperature), + new WindValue(direction, velocity)); } public Point getCoordinate() { @@ -90,14 +90,15 @@ public Optional> getDiffuseIrradiance() { return solarIrradiance.getDiffuseIrradiance(); } - public ComparableQuantity getTemperature() { + public Optional> getTemperatureValue() { return temperature.getTemperature(); } - public ComparableQuantity getWindDirection() { + public Optional> getWindDirection() { return wind.getDirection(); } - public ComparableQuantity getWindVelocity() { + + public Optional> getWindVelocity() { return wind.getVelocity(); } @@ -106,8 +107,8 @@ public ComparableQuantity getWindVelocity() { * * @return true if all values are present, false otherwise */ - public boolean isComplete() { - return solarIrradiance != null && temperature != null && wind != null; + public boolean hasPartialValues() { + return solarIrradiance != null || temperature != null || wind != null; } @Override @@ -116,9 +117,9 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; WeatherValue that = (WeatherValue) o; return coordinate.equals(that.coordinate) - && solarIrradiance.equals(that.solarIrradiance) - && temperature.equals(that.temperature) - && wind.equals(that.wind); + && solarIrradiance.equals(that.solarIrradiance) + && temperature.equals(that.temperature) + && wind.equals(that.wind); } @Override @@ -129,14 +130,14 @@ public int hashCode() { @Override public String toString() { return "WeatherValue{" - + "coordinate=" - + coordinate - + ", solarIrradiance=" - + solarIrradiance - + ", temperature=" - + temperature - + ", wind=" - + wind - + '}'; + + "coordinate=" + + coordinate + + ", solarIrradiance=" + + solarIrradiance + + ", temperature=" + + temperature + + ", wind=" + + wind + + '}'; } -} \ No newline at end of file +} From e7a5e7d98fb97704153b5623a77626a2c74b8f5e Mon Sep 17 00:00:00 2001 From: pierrepetersmeier <155652256+pierrepetersmeier@users.noreply.github.com> Date: Tue, 3 Jun 2025 15:29:13 +0200 Subject: [PATCH 3/4] CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9b493287..997690c07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed handling of `CongestionResult.InputModelType` in `EntityProcessor` [#1325](https://github.com/ie3-institute/PowerSystemDataModel/issues/1325) -- Add interpolation functionality to estimate missing weather values [#1304](https://github.com/ie3-institute/PowerSystemDataModel/issues/1304) - -Fixed em fields in input models [#1331](https://github.com/ie3-institute/PowerSystemDataModel/issues/1331) -- Consider None-equivalent value for missing data points in weather [#1304](https://github.com/ie3-institute/PowerSystemDataModel/issues/1304) +- Add interpolation functionality to estimate missing weather values [#1304](https://github.com/ie3-institute/PowerSystemDataModel/issues/1304) + ### Changed - Updated dependabot workflow and added CODEOWNERS [#1328](https://github.com/ie3-institute/PowerSystemDataModel/issues/1328) From 9b91756c286a2e1bb191e3d59a964fa86f4c6011 Mon Sep 17 00:00:00 2001 From: pierrepetersmeier <155652256+pierrepetersmeier@users.noreply.github.com> Date: Tue, 3 Jun 2025 15:31:34 +0200 Subject: [PATCH 4/4] fmt --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 997690c07..aa59cb90a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - -Fixed em fields in input models [#1331](https://github.com/ie3-institute/PowerSystemDataModel/issues/1331) - Add interpolation functionality to estimate missing weather values [#1304](https://github.com/ie3-institute/PowerSystemDataModel/issues/1304) - ### Changed - Updated dependabot workflow and added CODEOWNERS [#1328](https://github.com/ie3-institute/PowerSystemDataModel/issues/1328) - Extend azimuth angle range to [-180°, 180°] for PV inputs [#1330](https://github.com/ie3-institute/PowerSystemDataModel/issues/1330)