Dataset Open Access
A comprehensive dataset for the accelerated development and benchmarking of solar forecasting methods
Carreira Pedro, Hugo; Larson, David; Coimbra, Carlos
JSON Export
{
"files": [
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Folsom_irradiance.csv"
},
"checksum": "md5:f7deba7ccd089dbd3f52a46405a7dfc2",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Folsom_irradiance.csv",
"type": "csv",
"size": 76536976
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Folsom_NAM_lat38.579454_lon-121.260320.csv"
},
"checksum": "md5:3d917eeecdf967d1f90f803fad5e5467",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Folsom_NAM_lat38.579454_lon-121.260320.csv",
"type": "csv",
"size": 1599165
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Folsom_NAM_lat38.599891_lon-121.126680.csv"
},
"checksum": "md5:30024faae0123990cf29c81c281eaccc",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Folsom_NAM_lat38.599891_lon-121.126680.csv",
"type": "csv",
"size": 1593101
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Folsom_NAM_lat38.683880_lon-121.286556.csv"
},
"checksum": "md5:c0d6db7093b957603cb05c90fff23167",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Folsom_NAM_lat38.683880_lon-121.286556.csv",
"type": "csv",
"size": 1599189
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Folsom_NAM_lat38.704328_lon-121.152788.csv"
},
"checksum": "md5:792f830c261e2c041d35ebeb6eadbeac",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Folsom_NAM_lat38.704328_lon-121.152788.csv",
"type": "csv",
"size": 1590411
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Folsom_satellite.csv"
},
"checksum": "md5:f68086048ee5d764d1d992404147c421",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Folsom_satellite.csv",
"type": "csv",
"size": 15711562
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Folsom_sky_image_features.csv"
},
"checksum": "md5:86d58b6b84393399735a93ce1657cfab",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Folsom_sky_image_features.csv",
"type": "csv",
"size": 104681298
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Folsom_sky_images_2014.tar.bz2"
},
"checksum": "md5:fb2dee79429725ac91df539b310a9f98",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Folsom_sky_images_2014.tar.bz2",
"type": "bz2",
"size": 13759682249
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Folsom_sky_images_2015.tar.bz2"
},
"checksum": "md5:bce043f846a4dd01668a32943578b652",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Folsom_sky_images_2015.tar.bz2",
"type": "bz2",
"size": 16945355105
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Folsom_sky_images_2016.tar.bz2"
},
"checksum": "md5:af72cd28b398fb531ae1ab877c19eba0",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Folsom_sky_images_2016.tar.bz2",
"type": "bz2",
"size": 18616207524
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Folsom_weather.csv"
},
"checksum": "md5:b04e0dc7edf3513a769ea2c8c59beb27",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Folsom_weather.csv",
"type": "csv",
"size": 138793384
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Forecast_day-ahead.py"
},
"checksum": "md5:763f1666ff1485d631b7417cc8c4a5e8",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Forecast_day-ahead.py",
"type": "py",
"size": 5071
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Forecast_intra-day.py"
},
"checksum": "md5:6030752b33ce675859d131833a5e127d",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Forecast_intra-day.py",
"type": "py",
"size": 5112
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Forecast_intra-hour.py"
},
"checksum": "md5:7dd387b298e4c75f84a5fe7093bde2dd",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Forecast_intra-hour.py",
"type": "py",
"size": 5134
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Irradiance_features_day-ahead.csv"
},
"checksum": "md5:889efab48e0c0c690c45b11e641ba388",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Irradiance_features_day-ahead.csv",
"type": "csv",
"size": 725564
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Irradiance_features_intra-day.csv"
},
"checksum": "md5:971eee5f86677536b6238e73d923cedc",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Irradiance_features_intra-day.csv",
"type": "csv",
"size": 8266520
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Irradiance_features_intra-hour.csv"
},
"checksum": "md5:9e25e78b816e51b95d4349f304155f56",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Irradiance_features_intra-hour.csv",
"type": "csv",
"size": 49579520
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/NAM_nearest_node_day-ahead.csv"
},
"checksum": "md5:978905d0c0d1b1488325b33456446d23",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "NAM_nearest_node_day-ahead.csv",
"type": "csv",
"size": 519327
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Postprocess.py"
},
"checksum": "md5:73601ae78e2e49942673688650abfa3d",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Postprocess.py",
"type": "py",
"size": 4801
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Sat_image_features_intra-day.csv"
},
"checksum": "md5:8af401d02a090108b1863cb953ef64cf",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Sat_image_features_intra-day.csv",
"type": "csv",
"size": 20753628
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Sky_image_features_intra-hour.csv"
},
"checksum": "md5:a81c753c308213e2b506b94e0412403a",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Sky_image_features_intra-hour.csv",
"type": "csv",
"size": 23572470
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Target_day-ahead.csv"
},
"checksum": "md5:ed4959b21d282177cedcefe2e8e27f83",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Target_day-ahead.csv",
"type": "csv",
"size": 1158472
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Target_intra-day.csv"
},
"checksum": "md5:9d530ea7cbe0f122bc26041e9da74afd",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Target_intra-day.csv",
"type": "csv",
"size": 10663334
},
{
"links": {
"self": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed/Target_intra-hour.csv"
},
"checksum": "md5:ac6ebc385b6f6112c68ea967fc437c69",
"bucket": "68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"key": "Target_intra-hour.csv",
"type": "csv",
"size": 64523351
}
],
"owners": [
60965
],
"doi": "10.5281/zenodo.2826939",
"stats": {
"version_unique_downloads": 3126.0,
"unique_views": 987.0,
"views": 1103.0,
"version_views": 1103.0,
"unique_downloads": 3126.0,
"version_unique_views": 987.0,
"volume": 277085885021559.0,
"version_downloads": 18937.0,
"downloads": 18937.0,
"version_volume": 277085885021559.0
},
"links": {
"doi": "https://doi.org/10.5281/zenodo.2826939",
"conceptdoi": "https://doi.org/10.5281/zenodo.2826938",
"bucket": "https://zenodo.org/api/files/68c7feea-d2e8-4e9f-a55d-b50df76f91ed",
"conceptbadge": "https://zenodo.org/badge/doi/10.5281/zenodo.2826938.svg",
"html": "https://zenodo.org/record/2826939",
"latest_html": "https://zenodo.org/record/2826939",
"badge": "https://zenodo.org/badge/doi/10.5281/zenodo.2826939.svg",
"latest": "https://zenodo.org/api/records/2826939"
},
"conceptdoi": "10.5281/zenodo.2826938",
"created": "2019-06-25T19:47:27.652664+00:00",
"updated": "2020-01-24T19:25:10.862705+00:00",
"conceptrecid": "2826938",
"revision": 5,
"id": 2826939,
"metadata": {
"access_right_category": "success",
"doi": "10.5281/zenodo.2826939",
"description": "<p><strong>Description</strong><br>\nThis repository contains a comprehensive solar irradiance, imaging, and forecasting dataset. <br>\nThe goal with this release is to provide standardized solar and meteorological datasets to the research community for the accelerated development and benchmarking of forecasting methods. <br>\nThe data consist of three years (2014–2016) of quality-controlled, 1-min resolution global horizontal irradiance and direct normal irradiance ground measurements in California. <br>\nIn addition, we provide overlapping data from commonly used exogenous variables, including sky images, satellite imagery, Numerical Weather Prediction forecasts, and weather data. <br>\nWe also include sample codes of baseline models for benchmarking of more elaborated models.</p>\n\n<p><strong>Data usage</strong><br>\nThe usage of the datasets and sample codes presented here is intended for research and development purposes only and implies explicit reference to the paper:<br>\n<em>Pedro, H.T.C., Larson, D.P., Coimbra, C.F.M., 2019. A comprehensive dataset for the accelerated development and benchmarking of solar forecasting methods. Journal of Renewable and Sustainable Energy 11, 036102. https://doi.org/10.1063/1.5094494</em></p>\n\n<p>Although every effort was made to ensure the quality of the data, no guarantees or liabilities are implied by the authors or publishers of the data.</p>\n\n<p><strong>Sample code</strong><br>\nAs part of the data release, we are also including the sample code written in Python 3. <br>\nThe preprocessed data used in the scripts are also provided. <br>\nThe code can be used to reproduce the results presented in this work and as a starting point for future studies. <br>\nBesides the standard scientific Python packages (numpy, scipy, and matplotlib), the code depends on pandas for time-series operations, pvlib for common solar-related tasks, and scikit-learn for Machine Learning models. <br>\nAll required Python packages are readily available on Mac, Linux, and Windows and can be installed via, e.g., pip. </p>\n\n<p><strong>Units</strong><br>\nAll time stamps are in UTC (YYYY-MM-DD HH:MM:SS).<br>\nAll irradiance and weather data are in SI units.<br>\nSky image features are derived from 8-bit RGB (256 color levels) data.<br>\nSatellite images are derived from 8-bit gray-scale (256 color levels) data.</p>\n\n<p><strong>Missing data</strong><br>\nThe string "NAN" indicates missing data</p>\n\n<p><strong>File formats</strong><br>\nAll time series data files as in CSV (comma separated values)<br>\nImages are given in tar.bz2 files</p>\n\n<p><strong>Files </strong></p>\n\n<ul>\n\t<li><em>Folsom_irradiance.csv</em> Primary One-minute GHI, DNI, and DHI data.</li>\n\t<li><em>Folsom_weather.csv </em> Primary One-minute weather data.</li>\n\t<li><em>Folsom_sky_images_{YEAR}.tar.bz2</em> Primary Tar archives with daytime sky images captured at 1-min intervals for the years 2014, 2015, and 2016, compressed with bz2.</li>\n\t<li><em>Folsom_NAM_lat{LAT}_lon{LON}.csv </em> Primary NAM forecasts for the four nodes nearest the target location. {LAT} and {LON} are replaced by the node’s coordinates listed in Table I in the paper. </li>\n\t<li><em>Folsom_sky_image_features.csv </em> Secondary Features derived from the sky images.</li>\n\t<li><em>Folsom_satellite.csv </em> Secondary 10 pixel by 10 pixel GOES-15 images centered in the target location. </li>\n\t<li><em>Irradiance_features_{horizon}.csv</em> Secondary Irradiance features for the different forecasting horizons ({horizon} 1⁄4 {intra-hour, intra-day, day-ahead}). </li>\n\t<li><em>Sky_image_features_intra-hour.csv</em> Secondary Sky image features for the intra-hour forecasting issuing times. </li>\n\t<li><em>Sat_image_features_intra-day.csv</em> Secondary Satellite image features for the intra-day forecasting issuing times. </li>\n\t<li><em>NAM_nearest_node_day-ahead.csv </em> Secondary NAM forecasts (GHI, DNI computed with the DISC algorithm, and total cloud cover) for the nearest node to the target location prepared for day-ahead forecasting.</li>\n\t<li><em>Target_{horizon}.csv</em> Secondary Target data for the different forecasting horizons.</li>\n\t<li>F<em>orecast_{horizon}.py </em> Code Python script used to create the forecasts for the different horizons. </li>\n\t<li><em>Postprocess.py</em> Code Python script used to compute the error metric for all the forecasts.</li>\n</ul>\n\n<p> </p>",
"license": {
"id": "CC-BY-4.0"
},
"title": "A comprehensive dataset for the accelerated development and benchmarking of solar forecasting methods",
"relations": {
"version": [
{
"count": 1,
"index": 0,
"parent": {
"pid_type": "recid",
"pid_value": "2826938"
},
"is_last": true,
"last_child": {
"pid_type": "recid",
"pid_value": "2826939"
}
}
]
},
"version": "V1",
"references": [
"Pedro, H.T.C., Larson, D.P., Coimbra, C.F.M., 2019. A comprehensive dataset for the accelerated development and benchmarking of solar forecasting methods. Journal of Renewable and Sustainable Energy 11, 036102. https://doi.org/10.1063/1.5094494"
],
"keywords": [
"solar irradiance forecasting",
"sky images",
"satellite images",
"numerical weather prediction",
"forecast benchmarking"
],
"publication_date": "2019-06-24",
"creators": [
{
"affiliation": "University of California San Diego",
"name": "Carreira Pedro, Hugo"
},
{
"affiliation": "University of California San Diego",
"name": "Larson, David"
},
{
"affiliation": "University of California San Diego",
"name": "Coimbra, Carlos"
}
],
"access_right": "open",
"resource_type": {
"type": "dataset",
"title": "Dataset"
},
"related_identifiers": [
{
"scheme": "doi",
"identifier": "10.1063/1.5094494",
"relation": "isCompiledBy"
},
{
"scheme": "doi",
"identifier": "10.5281/zenodo.2826938",
"relation": "isVersionOf"
}
]
}
}
1,103
18,937
views
downloads
| All versions | This version | |
|---|---|---|
| Views | 1,103 | 1,103 |
| Downloads | 18,937 | 18,937 |
| Data volume | 277.1 TB | 277.1 TB |
| Unique views | 987 | 987 |
| Unique downloads | 3,126 | 3,126 |
Indexed in
- Publication date:
- June 24, 2019
- DOI:
-
- Keyword(s):
- solar irradiance forecasting sky images satellite images numerical weather prediction forecast benchmarking
- Related identifiers:
- Compiled by
- License (for files):
- Creative Commons Attribution 4.0 International