Dataset Open Access

A comprehensive dataset for the accelerated development and benchmarking of solar forecasting methods

Carreira Pedro, Hugo; Larson, David; Coimbra, Carlos


MARC21 XML Export

<?xml version='1.0' encoding='UTF-8'?>
<record xmlns="http://www.loc.gov/MARC21/slim">
  <leader>00000nmm##2200000uu#4500</leader>
  <datafield tag="999" ind1="C" ind2="5">
    <subfield code="x">Pedro, H.T.C., Larson, D.P., Coimbra, C.F.M., 2019. A comprehensive dataset for the accelerated development and benchmarking of solar forecasting methods.  Journal of Renewable and Sustainable Energy 11, 036102. https://doi.org/10.1063/1.5094494</subfield>
  </datafield>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">solar irradiance forecasting</subfield>
  </datafield>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">sky images</subfield>
  </datafield>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">satellite images</subfield>
  </datafield>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">numerical weather prediction</subfield>
  </datafield>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">forecast benchmarking</subfield>
  </datafield>
  <controlfield tag="005">20200124192510.0</controlfield>
  <controlfield tag="001">2826939</controlfield>
  <datafield tag="700" ind1=" " ind2=" ">
    <subfield code="u">University of California San Diego</subfield>
    <subfield code="a">Larson, David</subfield>
  </datafield>
  <datafield tag="700" ind1=" " ind2=" ">
    <subfield code="u">University of California San Diego</subfield>
    <subfield code="a">Coimbra, Carlos</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">76536976</subfield>
    <subfield code="z">md5:f7deba7ccd089dbd3f52a46405a7dfc2</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Folsom_irradiance.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">1599165</subfield>
    <subfield code="z">md5:3d917eeecdf967d1f90f803fad5e5467</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Folsom_NAM_lat38.579454_lon-121.260320.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">1593101</subfield>
    <subfield code="z">md5:30024faae0123990cf29c81c281eaccc</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Folsom_NAM_lat38.599891_lon-121.126680.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">1599189</subfield>
    <subfield code="z">md5:c0d6db7093b957603cb05c90fff23167</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Folsom_NAM_lat38.683880_lon-121.286556.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">1590411</subfield>
    <subfield code="z">md5:792f830c261e2c041d35ebeb6eadbeac</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Folsom_NAM_lat38.704328_lon-121.152788.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">15711562</subfield>
    <subfield code="z">md5:f68086048ee5d764d1d992404147c421</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Folsom_satellite.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">104681298</subfield>
    <subfield code="z">md5:86d58b6b84393399735a93ce1657cfab</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Folsom_sky_image_features.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">13759682249</subfield>
    <subfield code="z">md5:fb2dee79429725ac91df539b310a9f98</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Folsom_sky_images_2014.tar.bz2</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">16945355105</subfield>
    <subfield code="z">md5:bce043f846a4dd01668a32943578b652</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Folsom_sky_images_2015.tar.bz2</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">18616207524</subfield>
    <subfield code="z">md5:af72cd28b398fb531ae1ab877c19eba0</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Folsom_sky_images_2016.tar.bz2</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">138793384</subfield>
    <subfield code="z">md5:b04e0dc7edf3513a769ea2c8c59beb27</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Folsom_weather.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">5071</subfield>
    <subfield code="z">md5:763f1666ff1485d631b7417cc8c4a5e8</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Forecast_day-ahead.py</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">5112</subfield>
    <subfield code="z">md5:6030752b33ce675859d131833a5e127d</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Forecast_intra-day.py</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">5134</subfield>
    <subfield code="z">md5:7dd387b298e4c75f84a5fe7093bde2dd</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Forecast_intra-hour.py</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">725564</subfield>
    <subfield code="z">md5:889efab48e0c0c690c45b11e641ba388</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Irradiance_features_day-ahead.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">8266520</subfield>
    <subfield code="z">md5:971eee5f86677536b6238e73d923cedc</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Irradiance_features_intra-day.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">49579520</subfield>
    <subfield code="z">md5:9e25e78b816e51b95d4349f304155f56</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Irradiance_features_intra-hour.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">519327</subfield>
    <subfield code="z">md5:978905d0c0d1b1488325b33456446d23</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/NAM_nearest_node_day-ahead.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">4801</subfield>
    <subfield code="z">md5:73601ae78e2e49942673688650abfa3d</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Postprocess.py</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">20753628</subfield>
    <subfield code="z">md5:8af401d02a090108b1863cb953ef64cf</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Sat_image_features_intra-day.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">23572470</subfield>
    <subfield code="z">md5:a81c753c308213e2b506b94e0412403a</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Sky_image_features_intra-hour.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">1158472</subfield>
    <subfield code="z">md5:ed4959b21d282177cedcefe2e8e27f83</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Target_day-ahead.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">10663334</subfield>
    <subfield code="z">md5:9d530ea7cbe0f122bc26041e9da74afd</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Target_intra-day.csv</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">64523351</subfield>
    <subfield code="z">md5:ac6ebc385b6f6112c68ea967fc437c69</subfield>
    <subfield code="u">https://zenodo.org/record/2826939/files/Target_intra-hour.csv</subfield>
  </datafield>
  <datafield tag="542" ind1=" " ind2=" ">
    <subfield code="l">open</subfield>
  </datafield>
  <datafield tag="260" ind1=" " ind2=" ">
    <subfield code="c">2019-06-24</subfield>
  </datafield>
  <datafield tag="909" ind1="C" ind2="O">
    <subfield code="p">openaire_data</subfield>
    <subfield code="o">oai:zenodo.org:2826939</subfield>
  </datafield>
  <datafield tag="100" ind1=" " ind2=" ">
    <subfield code="u">University of California San Diego</subfield>
    <subfield code="a">Carreira Pedro, Hugo</subfield>
  </datafield>
  <datafield tag="245" ind1=" " ind2=" ">
    <subfield code="a">A comprehensive dataset for the accelerated development and benchmarking of solar forecasting methods</subfield>
  </datafield>
  <datafield tag="540" ind1=" " ind2=" ">
    <subfield code="u">https://creativecommons.org/licenses/by/4.0/legalcode</subfield>
    <subfield code="a">Creative Commons Attribution 4.0 International</subfield>
  </datafield>
  <datafield tag="650" ind1="1" ind2="7">
    <subfield code="a">cc-by</subfield>
    <subfield code="2">opendefinition.org</subfield>
  </datafield>
  <datafield tag="520" ind1=" " ind2=" ">
    <subfield code="a">&lt;p&gt;&lt;strong&gt;Description&lt;/strong&gt;&lt;br&gt;
This repository contains a comprehensive solar irradiance, imaging, and forecasting dataset.&amp;nbsp;&lt;br&gt;
The goal with this release is to provide standardized solar and meteorological datasets to the research community for the accelerated development and benchmarking of forecasting methods.&amp;nbsp;&lt;br&gt;
The data consist of three years (2014&amp;ndash;2016) of quality-controlled, 1-min resolution global horizontal irradiance and direct normal irradiance ground measurements in California.&amp;nbsp;&lt;br&gt;
In addition, we provide overlapping data from commonly used exogenous variables, including sky images, satellite imagery, Numerical Weather Prediction forecasts, and weather data.&amp;nbsp;&lt;br&gt;
We also include sample codes of baseline models for benchmarking of more elaborated models.&lt;/p&gt;

&lt;p&gt;&lt;strong&gt;Data usage&lt;/strong&gt;&lt;br&gt;
The usage of the datasets and sample codes presented here is intended for research and development purposes only and implies explicit reference to the paper:&lt;br&gt;
&lt;em&gt;Pedro, H.T.C., Larson, D.P., Coimbra, C.F.M., 2019. A comprehensive dataset for the accelerated development and benchmarking of solar forecasting methods.&amp;nbsp;Journal of Renewable and Sustainable Energy 11, 036102. https://doi.org/10.1063/1.5094494&lt;/em&gt;&lt;/p&gt;

&lt;p&gt;Although every effort was made to ensure the quality of the data, no guarantees or liabilities are implied by the authors or publishers of the data.&lt;/p&gt;

&lt;p&gt;&lt;strong&gt;Sample code&lt;/strong&gt;&lt;br&gt;
As part of the data release, we are also including the sample code written in Python 3.&amp;nbsp;&lt;br&gt;
The preprocessed data used in the scripts are also provided.&amp;nbsp;&lt;br&gt;
The code can be used to reproduce the results presented in this work and as a starting point for future studies.&amp;nbsp;&lt;br&gt;
Besides the standard scientific Python packages (numpy, scipy, and matplotlib), the code depends on pandas for time-series operations, pvlib for common solar-related tasks, and scikit-learn for Machine Learning models.&amp;nbsp;&lt;br&gt;
All required Python packages are readily available on Mac, Linux, and Windows and can be installed via, e.g., pip.&amp;nbsp;&lt;/p&gt;

&lt;p&gt;&lt;strong&gt;Units&lt;/strong&gt;&lt;br&gt;
All time stamps are in UTC (YYYY-MM-DD HH:MM:SS).&lt;br&gt;
All irradiance and weather data are in SI units.&lt;br&gt;
Sky image features are derived from 8-bit RGB (256 color levels) data.&lt;br&gt;
Satellite images are derived from 8-bit gray-scale (256 color levels) data.&lt;/p&gt;

&lt;p&gt;&lt;strong&gt;Missing data&lt;/strong&gt;&lt;br&gt;
The string &amp;quot;NAN&amp;quot; indicates missing data&lt;/p&gt;

&lt;p&gt;&lt;strong&gt;File formats&lt;/strong&gt;&lt;br&gt;
All time series data files as in CSV (comma separated values)&lt;br&gt;
Images are given in tar.bz2 files&lt;/p&gt;

&lt;p&gt;&lt;strong&gt;Files&amp;nbsp;&lt;/strong&gt;&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;&lt;em&gt;Folsom_irradiance.csv&lt;/em&gt;&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp;Primary&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp;One-minute GHI, DNI, and DHI data.&lt;/li&gt;
	&lt;li&gt;&lt;em&gt;Folsom_weather.csv&amp;nbsp;&lt;/em&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; Primary&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp;One-minute weather data.&lt;/li&gt;
	&lt;li&gt;&lt;em&gt;Folsom_sky_images_{YEAR}.tar.bz2&lt;/em&gt; &amp;nbsp; &amp;nbsp;Primary&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp;Tar archives with daytime sky images captured at 1-min intervals for the years 2014, 2015, and 2016, compressed with bz2.&lt;/li&gt;
	&lt;li&gt;&lt;em&gt;Folsom_NAM_lat{LAT}_lon{LON}.csv &lt;/em&gt;&amp;nbsp; &amp;nbsp;Primary&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp;NAM forecasts for the four nodes nearest the target location. {LAT} and {LON} are replaced by the node&amp;rsquo;s coordinates listed in Table I in the paper.&amp;nbsp;&lt;/li&gt;
	&lt;li&gt;&lt;em&gt;Folsom_sky_image_features.csv &lt;/em&gt;&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; Secondary&amp;nbsp; &amp;nbsp; Features derived from the sky images.&lt;/li&gt;
	&lt;li&gt;&lt;em&gt;Folsom_satellite.csv &lt;/em&gt;&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; Secondary &amp;nbsp; 10 pixel by 10 pixel GOES-15 images centered in the target location.&amp;nbsp;&lt;/li&gt;
	&lt;li&gt;&lt;em&gt;Irradiance_features_{horizon}.csv&lt;/em&gt;&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; Secondary &amp;nbsp; Irradiance features for the different forecasting horizons ({horizon} 1&amp;frasl;4 {intra-hour, intra-day, day-ahead}).&amp;nbsp;&lt;/li&gt;
	&lt;li&gt;&lt;em&gt;Sky_image_features_intra-hour.csv&lt;/em&gt;&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp;Secondary &amp;nbsp; Sky image features for the intra-hour forecasting issuing times.&amp;nbsp;&lt;/li&gt;
	&lt;li&gt;&lt;em&gt;Sat_image_features_intra-day.csv&lt;/em&gt;&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp;Secondary &amp;nbsp; Satellite image features for the intra-day forecasting issuing times.&amp;nbsp;&lt;/li&gt;
	&lt;li&gt;&lt;em&gt;NAM_nearest_node_day-ahead.csv &lt;/em&gt;&amp;nbsp; &amp;nbsp; &amp;nbsp;Secondary &amp;nbsp; NAM forecasts (GHI, DNI computed with the DISC algorithm, and total cloud cover) for the nearest node to the target location prepared for day-ahead forecasting.&lt;/li&gt;
	&lt;li&gt;&lt;em&gt;Target_{horizon}.csv&lt;/em&gt;&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; Secondary &amp;nbsp; Target data for the different forecasting horizons.&lt;/li&gt;
	&lt;li&gt;F&lt;em&gt;orecast_{horizon}.py &lt;/em&gt;&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp;Code&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; Python script used to create the forecasts for the different horizons.&amp;nbsp;&lt;/li&gt;
	&lt;li&gt;&lt;em&gt;Postprocess.py&lt;/em&gt;&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; Code&amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp;Python script used to compute the error metric for all the forecasts.&lt;/li&gt;
&lt;/ul&gt;

&lt;p&gt;&amp;nbsp;&lt;/p&gt;</subfield>
  </datafield>
  <datafield tag="773" ind1=" " ind2=" ">
    <subfield code="n">doi</subfield>
    <subfield code="i">isCompiledBy</subfield>
    <subfield code="a">10.1063/1.5094494</subfield>
  </datafield>
  <datafield tag="773" ind1=" " ind2=" ">
    <subfield code="n">doi</subfield>
    <subfield code="i">isVersionOf</subfield>
    <subfield code="a">10.5281/zenodo.2826938</subfield>
  </datafield>
  <datafield tag="024" ind1=" " ind2=" ">
    <subfield code="a">10.5281/zenodo.2826939</subfield>
    <subfield code="2">doi</subfield>
  </datafield>
  <datafield tag="980" ind1=" " ind2=" ">
    <subfield code="a">dataset</subfield>
  </datafield>
</record>
743
17,183
views
downloads
All versions This version
Views 743743
Downloads 17,18317,183
Data volume 253.9 TB253.9 TB
Unique views 678678
Unique downloads 2,7062,706

Share

Cite as