Skip to content

explicit parsing of timestamps for Solcast API changes #2931

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lorenzo-solcast wants to merge 1 commit into
base: main
Choose a base branch
from
+80 −2
Open

explicit parsing of timestamps for Solcast API changes #2931

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion pvlib/iotools/solcast.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ def get_solcast_tmy(
data : pandas.DataFrame
containing the values for the parameters requested. The times
in the DataFrame index indicate the midpoint of each interval.
The index is timezone-aware, using the UTC offset returned by
the API, or UTC if the returned timestamps have mixed offsets.
metadata: dict
latitude and longitude of the request.

Expand Down Expand Up @@ -179,6 +181,8 @@ def get_solcast_historic(
data : pandas.DataFrame
containing the values for the parameters requested. The times
in the DataFrame index indicate the midpoint of each interval.
The index is timezone-aware, using the UTC offset returned by
the API, or UTC if the returned timestamps have mixed offsets.
metadata: dict
latitude and longitude of the request.

Expand Down Expand Up @@ -263,6 +267,8 @@ def get_solcast_forecast(
data : pandas.DataFrame
Contains the values for the parameters requested. The times
in the DataFrame index indicate the midpoint of each interval.
The index is timezone-aware, using the UTC offset returned by
the API, or UTC if the returned timestamps have mixed offsets.
metadata: dict
latitude and longitude of the request.

Expand Down Expand Up @@ -341,6 +347,8 @@ def get_solcast_live(
data : pandas.DataFrame
containing the values for the parameters requested. The times
in the DataFrame index indicate the midpoint of each interval.
The index is timezone-aware, using the UTC offset returned by
the API, or UTC if the returned timestamps have mixed offsets.
metadata: dict
latitude and longitude of the request.

Expand Down Expand Up @@ -400,6 +408,22 @@ def get_solcast_live(
return data, {"latitude": latitude, "longitude": longitude}


def _parse_period_end(period_end):
"""Parse Solcast ``period_end`` timestamps to a tz-aware DatetimeIndex.

Supports both the legacy format (``2025-04-16T06:00:00.0000000Z``) and
the ISO 8601 UTC offset format (``2025-04-16T14:00:00+01:30``).
If all timestamps share the same UTC offset, that offset is kept;
otherwise the timestamps are converted to UTC.
"""
period_end = pd.Series(period_end).astype(str)
times = pd.DatetimeIndex(pd.to_datetime(period_end, utc=True))
offsets = period_end.str.extract(r'(Z|[+-]\d{2}:?\d{2})$')[0]
if len(times) > 0 and offsets.nunique(dropna=False) == 1:
times = times.tz_convert(pd.Timestamp(period_end.iloc[0]).tz)
return times


def _solcast2pvlib(data):
"""Format the data from Solcast to pvlib's conventions.

Expand All @@ -414,7 +438,7 @@ def _solcast2pvlib(data):
"""
# move from period_end to period_middle as per pvlib convention

data["period_mid"] = pd.to_datetime(
data["period_mid"] = _parse_period_end(
data.period_end) - pd.to_timedelta(data.period.values) / 2
data = data.set_index("period_mid").drop(columns=["period_end", "period"])

Expand Down
56 changes: 55 additions & 1 deletion tests/iotools/test_solcast.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,11 +185,65 @@ def test_get_solcast_tmy(
['2023-09-20 01:45:00+00:00', '2023-09-20 01:15:00+00:00'],
dtype='datetime64[ns, UTC]', name='period_mid', freq=None)
)
)
),
(
# new standard API format: float air_temp, ISO 8601 UTC offset
pd.DataFrame(
[[942, 843, 1017.4, 30.2, 7.8, 316, 1010, -2, 4.6, 16.4,
'2023-09-20T12:00:00+10:00', 'PT30M', 90],
[936, 832, 1017.9, 29.8, 7.9, 316, 996, -14, 4.5, 16.3,
'2023-09-20T11:30:00+10:00', 'PT30M', 0]],
columns=[
'dni', 'ghi', 'surface_pressure', 'air_temp', 'wind_speed_10m',
'wind_direction_10m', 'gti', 'azimuth', 'dewpoint_temp',
'precipitable_water', 'period_end', 'period', 'zenith'],
index=pd.RangeIndex(start=0, stop=2, step=1)
),
pd.DataFrame(
[[9.4200e+02, 8.4300e+02, 1.0174e+05, 3.0200e+01, 7.8000e+00,
3.1600e+02, 1.0100e+03, 2.0000e+00, 4.6000e+00, 1.6400e+00, 90],
[9.3600e+02, 8.3200e+02, 1.0179e+05, 2.9800e+01, 7.9000e+00,
3.1600e+02, 9.9600e+02, 1.4000e+01, 4.5000e+00, 1.6300e+00, 0]],
columns=[
'dni', 'ghi', 'pressure', 'temp_air', 'wind_speed',
'wind_direction', 'poa_global', 'solar_azimuth',
'temp_dew', 'precipitable_water', 'solar_zenith'],
index=pd.DatetimeIndex(
['2023-09-20 11:45:00+10:00', '2023-09-20 11:15:00+10:00'],
name='period_mid', freq=None)
)
),
])
def test_solcast2pvlib(in_df, out_df):
df = pvlib.iotools.solcast._solcast2pvlib(in_df)
pd.testing.assert_frame_equal(df.astype(float), out_df.astype(float))
assert df.index.tz is not None


@pytest.mark.parametrize("period_end,expected", [
# legacy format, 7 fractional-second digits and Z suffix
(['2025-04-16T06:00:00.0000000Z', '2025-04-16T06:30:00.0000000Z'],
['2025-04-16T06:00:00+00:00', '2025-04-16T06:30:00+00:00']),
# new standard format, explicit UTC offset
(['2025-04-16T14:00:00+01:30', '2025-04-16T14:30:00+01:30'],
['2025-04-16T14:00:00+01:30', '2025-04-16T14:30:00+01:30']),
(['2025-04-16T06:00:00+00:00'], ['2025-04-16T06:00:00+00:00']),
])
def test_parse_period_end(period_end, expected):
out = pvlib.iotools.solcast._parse_period_end(pd.Series(period_end))
expected = pd.DatetimeIndex(pd.to_datetime(expected))
assert out.equals(expected)
assert str(out.tz) == str(expected.tz)


def test_parse_period_end_mixed_offsets():
# e.g. a local time series spanning a daylight saving transition
out = pvlib.iotools.solcast._parse_period_end(pd.Series(
['2025-04-06T01:00:00+11:00', '2025-04-06T03:00:00+10:00']))
expected = pd.DatetimeIndex(
['2025-04-05T14:00:00', '2025-04-05T17:00:00'], tz='UTC')
assert str(out.tz) == 'UTC'
assert out.equals(expected)


@pytest.mark.parametrize("map_variables", [True, False])
Expand Down
Loading