Presentation Open Access

Enabling reproducible ML and Systems research: the good, the bad, and the ugly

Grigori Fursin


DCAT Export

<?xml version='1.0' encoding='utf-8'?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:adms="http://www.w3.org/ns/adms#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:dct="http://purl.org/dc/terms/" xmlns:dctype="http://purl.org/dc/dcmitype/" xmlns:dcat="http://www.w3.org/ns/dcat#" xmlns:duv="http://www.w3.org/ns/duv#" xmlns:foaf="http://xmlns.com/foaf/0.1/" xmlns:frapo="http://purl.org/cerif/frapo/" xmlns:geo="http://www.w3.org/2003/01/geo/wgs84_pos#" xmlns:gsp="http://www.opengis.net/ont/geosparql#" xmlns:locn="http://www.w3.org/ns/locn#" xmlns:org="http://www.w3.org/ns/org#" xmlns:owl="http://www.w3.org/2002/07/owl#" xmlns:prov="http://www.w3.org/ns/prov#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xmlns:schema="http://schema.org/" xmlns:skos="http://www.w3.org/2004/02/skos/core#" xmlns:vcard="http://www.w3.org/2006/vcard/ns#" xmlns:wdrs="http://www.w3.org/2007/05/powder-s#">
  <rdf:Description rdf:about="https://doi.org/10.5281/zenodo.4005773">
    <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Dataset"/>
    <dct:type rdf:resource="http://purl.org/dc/dcmitype/Text"/>
    <dct:identifier rdf:datatype="http://www.w3.org/2001/XMLSchema#anyURI">https://doi.org/10.5281/zenodo.4005773</dct:identifier>
    <foaf:page rdf:resource="https://doi.org/10.5281/zenodo.4005773"/>
    <dct:creator>
      <rdf:Description rdf:about="http://orcid.org/0000-0001-7719-1624">
        <rdf:type rdf:resource="http://xmlns.com/foaf/0.1/Agent"/>
        <dct:identifier rdf:datatype="http://www.w3.org/2001/XMLSchema#string">0000-0001-7719-1624</dct:identifier>
        <foaf:name>Grigori Fursin</foaf:name>
        <org:memberOf>
          <foaf:Organization>
            <foaf:name>cTuning foundation, cKnowledge SAS</foaf:name>
          </foaf:Organization>
        </org:memberOf>
      </rdf:Description>
    </dct:creator>
    <dct:title>Enabling reproducible ML and Systems research: the good, the bad, and the ugly</dct:title>
    <dct:publisher>
      <foaf:Agent>
        <foaf:name>Zenodo</foaf:name>
      </foaf:Agent>
    </dct:publisher>
    <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#gYear">2020</dct:issued>
    <dcat:keyword>reproducibility</dcat:keyword>
    <dcat:keyword>reusability</dcat:keyword>
    <dcat:keyword>portability</dcat:keyword>
    <dcat:keyword>workflow</dcat:keyword>
    <dcat:keyword>automation</dcat:keyword>
    <dcat:keyword>machine learning</dcat:keyword>
    <dcat:keyword>systems</dcat:keyword>
    <dcat:keyword>benchmarking</dcat:keyword>
    <dcat:keyword>artifact evaluation</dcat:keyword>
    <dcat:keyword>fair principles</dcat:keyword>
    <dcat:keyword>open science</dcat:keyword>
    <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#date">2020-08-28</dct:issued>
    <dct:language rdf:resource="http://publications.europa.eu/resource/authority/language/ENG"/>
    <owl:sameAs rdf:resource="https://zenodo.org/record/4005773"/>
    <adms:identifier>
      <adms:Identifier>
        <skos:notation rdf:datatype="http://www.w3.org/2001/XMLSchema#anyURI">https://zenodo.org/record/4005773</skos:notation>
        <adms:schemeAgency>url</adms:schemeAgency>
      </adms:Identifier>
    </adms:identifier>
    <dct:isVersionOf rdf:resource="https://doi.org/10.5281/zenodo.4005587"/>
    <dct:isPartOf rdf:resource="https://zenodo.org/communities/ck"/>
    <dct:description>&lt;p&gt;&lt;strong&gt;Invited talk at FastPath 2020 (International Workshop on Performance Analysis of Machine Learning Systems) co-located with ISPASS 2020.&lt;/strong&gt;&lt;/p&gt; &lt;ul&gt; &lt;li&gt;&lt;strong&gt;Article: &lt;/strong&gt;&lt;a href="https://arxiv.org/pdf/2011.01149.pdf"&gt;arxiv.org/pdf/2011.01149.pdf&lt;/a&gt;&amp;nbsp; ( &lt;strong&gt;&lt;a href="https://github.com/ctuning/ck"&gt;code&lt;/a&gt; &lt;/strong&gt;and&lt;strong&gt; &lt;a href="https://github.com/ctuning/ai"&gt;data&lt;/a&gt;&lt;/strong&gt; )&lt;/li&gt; &lt;li&gt;&lt;strong&gt;Reproducibility initiative: &lt;/strong&gt;&lt;a href="https://cTuning.org/ae"&gt;systems and ML conferences&lt;/a&gt; ( &lt;a href="https://cKnowledge.io/reproduced-papers"&gt;reproduced papers&lt;/a&gt; and &lt;a href="https://cKnowledge.io/reproduced-results"&gt;results&lt;/a&gt; )&lt;/li&gt; &lt;li&gt;&lt;strong&gt;Workshop program: &lt;/strong&gt;&lt;a href="https://fastpath2020.github.io/Program"&gt;fastpath2020.github.io/Program&lt;/a&gt;&lt;/li&gt; &lt;li&gt;&lt;strong&gt;Author:&amp;nbsp;&lt;/strong&gt;&lt;a href="https://cKnowledge.io/@gfursin"&gt;Grigori Fursin&lt;/a&gt;&lt;/li&gt; &lt;/ul&gt; &lt;p&gt;&lt;strong&gt;Abstract:&lt;/strong&gt;&lt;/p&gt; &lt;p&gt;10 years ago we released our ML-based MILEPOST compiler with all related code and experimental data at &lt;a href="https://ctuning.org/wiki/index.php/CDatabase"&gt;cTuning.org&lt;/a&gt;. Unfortunately, this research quickly stalled after we struggled to reproduce performance results and predictive models shared by volunteers across rapidly changing systems.&lt;/p&gt; &lt;p&gt;In this talk, I will describe my 10-year effort to solve numerous reproducibility issues in ML&amp;amp;systems research. I will share my experience reproducing 150+ systems and ML papers during artifact evaluation at ASPLOS, MLSys, CGO, PPoPP and Supercomputing. This tedious experience motivated me to develop the &lt;a href="https://github.com/ctuning/ck"&gt;Collective Knowledge framework&lt;/a&gt; and the open &lt;a href="https://cKnowledge.io"&gt;cKnowledge.io&lt;/a&gt; portal to bring DevOps principles to our research. I will also present &lt;a href="https://cKnowledge.io/solutions"&gt;cKnowledge solutions&lt;/a&gt; - a new way to package and share research artifacts and results with common Python APIs, CLI actions, portable workflows and JSON meta descriptions. Such solutions can be used to automatically build, benchmark and validate ML&amp;amp;system experiments across continuously evolving platforms.&lt;/p&gt; &lt;p&gt;I will conclude with &lt;a href="https://cKnowledge.org/partners"&gt;several practical use-cases&lt;/a&gt; of our technology in collaboration with Arm, IBM, General Motors, the Raspberry Pi foundation and MLPerf. Our long-term goal is to help researchers share their new ML techniques as production-ready packages along with published papers and participate in collaborative and reproducible benchmarking, co-design and comparison of efficient ML/software/hardware stacks.&lt;/p&gt; &lt;p&gt;&amp;nbsp;&lt;/p&gt;</dct:description>
    <dct:description>{"references": ["Grigori Fursin (2020). The Collective Knowledge project: making ML models more portable and reproducible with open APIs, reusable best practices and MLOps. arXiv:2006.07161"]}</dct:description>
    <dct:accessRights rdf:resource="http://publications.europa.eu/resource/authority/access-right/PUBLIC"/>
    <dct:accessRights>
      <dct:RightsStatement rdf:about="info:eu-repo/semantics/openAccess">
        <rdfs:label>Open Access</rdfs:label>
      </dct:RightsStatement>
    </dct:accessRights>
    <dcat:distribution>
      <dcat:Distribution>
        <dct:license rdf:resource="https://creativecommons.org/licenses/by/4.0/legalcode"/>
        <dcat:accessURL rdf:resource="https://doi.org/10.5281/zenodo.4005773"/>
      </dcat:Distribution>
    </dcat:distribution>
    <dcat:distribution>
      <dcat:Distribution>
        <dcat:accessURL>https://doi.org/10.5281/zenodo.4005773</dcat:accessURL>
        <dcat:byteSize>4304641</dcat:byteSize>
        <dcat:downloadURL>https://zenodo.org/record/4005773/files/presentation-fastpath.pdf</dcat:downloadURL>
        <dcat:mediaType>application/pdf</dcat:mediaType>
      </dcat:Distribution>
    </dcat:distribution>
  </rdf:Description>
</rdf:RDF>
4,890
3,450
views
downloads
All versions This version
Views 4,8904,853
Downloads 3,4503,430
Data volume 14.8 GB14.8 GB
Unique views 4,1864,161
Unique downloads 3,1663,153

Share

Cite as