There is a newer version of this record available.

Preprint Open Access

Sharing interoperable workflow provenance: A review of best practices and their practical application in CWLProv

Farah Zaib Khan; Stian Soiland-Reyes; Richard O. Sinnott; Andrew Lonie; Carole Goble; Michael R. Crusoe


MARC21 XML Export

<?xml version='1.0' encoding='UTF-8'?>
<record xmlns="http://www.loc.gov/MARC21/slim">
  <leader>00000nam##2200000uu#4500</leader>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">Provenance</subfield>
  </datafield>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">Common Workflow Language</subfield>
  </datafield>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">CWL</subfield>
  </datafield>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">Research Object</subfield>
  </datafield>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">RO</subfield>
  </datafield>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">BagIt</subfield>
  </datafield>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">Interoperability</subfield>
  </datafield>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">Scientific Workflows</subfield>
  </datafield>
  <datafield tag="653" ind1=" " ind2=" ">
    <subfield code="a">Containers</subfield>
  </datafield>
  <controlfield tag="005">20200120173706.0</controlfield>
  <datafield tag="500" ind1=" " ind2=" ">
    <subfield code="a">Submitted to GigaScience (GIGA-D-18-00483)</subfield>
  </datafield>
  <controlfield tag="001">1966881</controlfield>
  <datafield tag="700" ind1=" " ind2=" ">
    <subfield code="u">The University of Manchester; Common Workflow Language project</subfield>
    <subfield code="0">(orcid)0000-0001-9842-9718</subfield>
    <subfield code="a">Stian Soiland-Reyes</subfield>
  </datafield>
  <datafield tag="700" ind1=" " ind2=" ">
    <subfield code="u">The University of Melbourne</subfield>
    <subfield code="0">(orcid)0000-0001-5998-222X</subfield>
    <subfield code="a">Richard O. Sinnott</subfield>
  </datafield>
  <datafield tag="700" ind1=" " ind2=" ">
    <subfield code="u">The University of Melbourne</subfield>
    <subfield code="0">(orcid)0000-0002-2006-3856</subfield>
    <subfield code="a">Andrew Lonie</subfield>
  </datafield>
  <datafield tag="700" ind1=" " ind2=" ">
    <subfield code="u">The University of Manchester</subfield>
    <subfield code="0">(orcid)0000-0003-1219-2137</subfield>
    <subfield code="a">Carole Goble</subfield>
  </datafield>
  <datafield tag="700" ind1=" " ind2=" ">
    <subfield code="u">Common Workflow Language project</subfield>
    <subfield code="0">(orcid)0000-0002-2961-9670</subfield>
    <subfield code="a">Michael R. Crusoe</subfield>
  </datafield>
  <datafield tag="700" ind1=" " ind2=" ">
    <subfield code="u">Curoverse; Common Workflow Language</subfield>
    <subfield code="0">(orcid)0000-0003-3566-7705</subfield>
    <subfield code="4">oth</subfield>
    <subfield code="a">Peter Amstutz</subfield>
  </datafield>
  <datafield tag="700" ind1=" " ind2=" ">
    <subfield code="u">EMBL-EBI</subfield>
    <subfield code="0">(orcid)0000-0002-1827-331X</subfield>
    <subfield code="4">oth</subfield>
    <subfield code="a">Pau Ruiz Safont</subfield>
  </datafield>
  <datafield tag="700" ind1=" " ind2=" ">
    <subfield code="0">(orcid)0000-0002-8021-9162</subfield>
    <subfield code="4">oth</subfield>
    <subfield code="a">Pjotr Prins</subfield>
  </datafield>
  <datafield tag="700" ind1=" " ind2=" ">
    <subfield code="u">Harvard School of Public Health</subfield>
    <subfield code="0">(orcid)0000-0002-3026-1856</subfield>
    <subfield code="4">oth</subfield>
    <subfield code="a">Brad Chapman</subfield>
  </datafield>
  <datafield tag="700" ind1=" " ind2=" ">
    <subfield code="u">RTI international</subfield>
    <subfield code="0">(orcid)0000-0003-3523-5312</subfield>
    <subfield code="4">oth</subfield>
    <subfield code="a">Christopher Ball</subfield>
  </datafield>
  <datafield tag="700" ind1=" " ind2=" ">
    <subfield code="u">University of California, Santa Cruz</subfield>
    <subfield code="0">(orcid)0000-0001-8352-873X</subfield>
    <subfield code="4">oth</subfield>
    <subfield code="a">Lon Blauvelt</subfield>
  </datafield>
  <datafield tag="856" ind1="4" ind2=" ">
    <subfield code="s">6214005</subfield>
    <subfield code="z">md5:a3f5abeeeff1a7c81b26edc54b009c0f</subfield>
    <subfield code="u">https://zenodo.org/record/1966881/files/CWLProv.pdf</subfield>
  </datafield>
  <datafield tag="542" ind1=" " ind2=" ">
    <subfield code="l">open</subfield>
  </datafield>
  <datafield tag="260" ind1=" " ind2=" ">
    <subfield code="c">2018-12-04</subfield>
  </datafield>
  <datafield tag="909" ind1="C" ind2="O">
    <subfield code="p">openaire</subfield>
    <subfield code="p">user-linkeddata</subfield>
    <subfield code="p">user-ro</subfield>
    <subfield code="o">oai:zenodo.org:1966881</subfield>
  </datafield>
  <datafield tag="100" ind1=" " ind2=" ">
    <subfield code="u">The University of Melbourne; Common Workflow Language project</subfield>
    <subfield code="0">(orcid)0000-0002-6337-3037</subfield>
    <subfield code="a">Farah Zaib Khan</subfield>
  </datafield>
  <datafield tag="245" ind1=" " ind2=" ">
    <subfield code="a">Sharing interoperable workflow provenance: A review of best practices and their practical application in CWLProv</subfield>
  </datafield>
  <datafield tag="980" ind1=" " ind2=" ">
    <subfield code="a">user-linkeddata</subfield>
  </datafield>
  <datafield tag="980" ind1=" " ind2=" ">
    <subfield code="a">user-ro</subfield>
  </datafield>
  <datafield tag="536" ind1=" " ind2=" ">
    <subfield code="c">730976</subfield>
    <subfield code="a">Industrial Biotechnology Innovation and Synthetic Biology Accelerator</subfield>
  </datafield>
  <datafield tag="536" ind1=" " ind2=" ">
    <subfield code="c">675728</subfield>
    <subfield code="a">Centre of Excellence for Biomolecular Research</subfield>
  </datafield>
  <datafield tag="540" ind1=" " ind2=" ">
    <subfield code="u">https://creativecommons.org/licenses/by/4.0/legalcode</subfield>
    <subfield code="a">Creative Commons Attribution 4.0 International</subfield>
  </datafield>
  <datafield tag="650" ind1="1" ind2="7">
    <subfield code="a">cc-by</subfield>
    <subfield code="2">opendefinition.org</subfield>
  </datafield>
  <datafield tag="520" ind1=" " ind2=" ">
    <subfield code="a">&lt;p&gt;&lt;strong&gt;Background:&lt;/strong&gt; The automation of data analysis in the form of scientific workflows has become a widely adopted practice in many fields of research. Computationally driven data-intensive experiments using workflows enable &lt;strong&gt;A&lt;/strong&gt;utomation, &lt;strong&gt;S&lt;/strong&gt;caling, &lt;strong&gt;A&lt;/strong&gt;daption and &lt;strong&gt;P&lt;/strong&gt;rovenance support (&lt;strong&gt;ASAP&lt;/strong&gt;). However, there are still several challenges associated with the effective sharing, publication and reproducibility of such workflows due to the incomplete capture of provenance and lack of interoperability between different technical (software) platforms.&lt;/p&gt;

&lt;p&gt;&lt;strong&gt;Results:&lt;/strong&gt; Based on best practice recommendations identified from literature on workflow design, sharing and publishing, we define a &lt;em&gt;hierarchical provenance framework&lt;/em&gt; to achieve uniformity in the provenance and support comprehensive and fully re-executable workflows equipped with domain-specific information. To realise this framework, we present &lt;a href="https://w3id.org/cwl/prov/"&gt;&lt;strong&gt;CWLProv&lt;/strong&gt;&lt;/a&gt;, a standard-based format to represent any workflow-based computational analysis to produce workflow output artefacts that satisfy the various levels of provenance. We utilise open source community-driven standards; interoperable workflow definitions in &lt;a href="https://www.commonwl.org/"&gt;Common Workflow Language&lt;/a&gt; (CWL), structured provenance representation using the &lt;a href="https://www.w3.org/TR/prov-overview/"&gt;W3C PROV&lt;/a&gt; model, and resource aggregation and sharing as workflow-centric &lt;a href="http://www.researchobject.org/"&gt;Research Objects&lt;/a&gt; (RO) generated along with the final outputs of a given workflow enactment. We demonstrate the utility of this approach through a practical implementation of &lt;em&gt;CWLProv&lt;/em&gt; and evaluation using real-life genomic workflows developed by independent groups.&lt;/p&gt;

&lt;p&gt;&lt;strong&gt;Conclusions: &lt;/strong&gt;The underlying principles of the standards utilised by &lt;em&gt;CWLProv&lt;/em&gt; enable semantically-rich and executable Research Objects that capture computational workflows with retrospective provenance such that any platform supporting CWL will be able to understand the analysis, re-use the methods for partial re-runs, or reproduce the analysis to validate the published findings.&lt;/p&gt;</subfield>
  </datafield>
  <datafield tag="773" ind1=" " ind2=" ">
    <subfield code="n">doi</subfield>
    <subfield code="i">cites</subfield>
    <subfield code="a">10.1109/BigData.2016.7840618</subfield>
  </datafield>
  <datafield tag="773" ind1=" " ind2=" ">
    <subfield code="n">doi</subfield>
    <subfield code="i">cites</subfield>
    <subfield code="a">10.5281/zenodo.592090</subfield>
  </datafield>
  <datafield tag="773" ind1=" " ind2=" ">
    <subfield code="n">doi</subfield>
    <subfield code="i">cites</subfield>
    <subfield code="a">10.5281/zenodo.51314</subfield>
  </datafield>
  <datafield tag="773" ind1=" " ind2=" ">
    <subfield code="n">url</subfield>
    <subfield code="i">isDocumentedBy</subfield>
    <subfield code="a">https://zenodo.org/record/1304969</subfield>
  </datafield>
  <datafield tag="773" ind1=" " ind2=" ">
    <subfield code="n">doi</subfield>
    <subfield code="i">isSupplementedBy</subfield>
    <subfield code="a">10.17632/xnwncxpw42.1</subfield>
  </datafield>
  <datafield tag="773" ind1=" " ind2=" ">
    <subfield code="n">doi</subfield>
    <subfield code="i">isSupplementedBy</subfield>
    <subfield code="a">10.17632/6wtpgr3kbj.1</subfield>
  </datafield>
  <datafield tag="773" ind1=" " ind2=" ">
    <subfield code="n">doi</subfield>
    <subfield code="i">isSupplementedBy</subfield>
    <subfield code="a">10.17632/97hj93mkfd.3</subfield>
  </datafield>
  <datafield tag="773" ind1=" " ind2=" ">
    <subfield code="n">doi</subfield>
    <subfield code="i">hasPart</subfield>
    <subfield code="a">10.5281/zenodo.1471376</subfield>
  </datafield>
  <datafield tag="773" ind1=" " ind2=" ">
    <subfield code="n">doi</subfield>
    <subfield code="i">hasPart</subfield>
    <subfield code="a">10.5281/zenodo.1471585</subfield>
  </datafield>
  <datafield tag="773" ind1=" " ind2=" ">
    <subfield code="n">doi</subfield>
    <subfield code="i">hasPart</subfield>
    <subfield code="a">10.5281/zenodo.1471589</subfield>
  </datafield>
  <datafield tag="773" ind1=" " ind2=" ">
    <subfield code="n">doi</subfield>
    <subfield code="i">isVersionOf</subfield>
    <subfield code="a">10.5281/zenodo.1208477</subfield>
  </datafield>
  <datafield tag="024" ind1=" " ind2=" ">
    <subfield code="a">10.5281/zenodo.1966881</subfield>
    <subfield code="2">doi</subfield>
  </datafield>
  <datafield tag="980" ind1=" " ind2=" ">
    <subfield code="a">publication</subfield>
    <subfield code="b">preprint</subfield>
  </datafield>
</record>
2,843
1,829
views
downloads
All versions This version
Views 2,843699
Downloads 1,829438
Data volume 7.2 GB2.7 GB
Unique views 2,286600
Unique downloads 1,379401

Share

Cite as