Conference paper Open Access

Learning Workflow Scheduling on Multi-Resource Clusters

Hu, Yang; de Laat, Cees; Zhao, Zhiming


JSON Export

{
  "files": [
    {
      "links": {
        "self": "https://zenodo.org/api/files/91ad735e-7d06-4339-b1cf-1f72675fa754/2019.8.conference.nas.camera.pdf"
      }, 
      "checksum": "md5:b9cc45d788adf288491267ffaf6bded9", 
      "bucket": "91ad735e-7d06-4339-b1cf-1f72675fa754", 
      "key": "2019.8.conference.nas.camera.pdf", 
      "type": "pdf", 
      "size": 390857
    }
  ], 
  "owners": [
    26570
  ], 
  "doi": "10.1109/NAS.2019.8834720", 
  "stats": {
    "version_unique_downloads": 40.0, 
    "unique_views": 43.0, 
    "views": 46.0, 
    "version_views": 46.0, 
    "unique_downloads": 40.0, 
    "version_unique_views": 43.0, 
    "volume": 16806851.0, 
    "version_downloads": 43.0, 
    "downloads": 43.0, 
    "version_volume": 16806851.0
  }, 
  "links": {
    "doi": "https://doi.org/10.1109/NAS.2019.8834720", 
    "latest_html": "https://zenodo.org/record/3466676", 
    "bucket": "https://zenodo.org/api/files/91ad735e-7d06-4339-b1cf-1f72675fa754", 
    "badge": "https://zenodo.org/badge/doi/10.1109/NAS.2019.8834720.svg", 
    "html": "https://zenodo.org/record/3466676", 
    "latest": "https://zenodo.org/api/records/3466676"
  }, 
  "created": "2019-10-01T10:10:43.235379+00:00", 
  "updated": "2020-01-20T17:28:44.452159+00:00", 
  "conceptrecid": "3466675", 
  "revision": 6, 
  "id": 3466676, 
  "metadata": {
    "access_right_category": "success", 
    "doi": "10.1109/NAS.2019.8834720", 
    "version": "Camera ready", 
    "license": {
      "id": "CC-BY-4.0"
    }, 
    "title": "Learning Workflow Scheduling on Multi-Resource Clusters", 
    "relations": {
      "version": [
        {
          "count": 1, 
          "index": 0, 
          "parent": {
            "pid_type": "recid", 
            "pid_value": "3466675"
          }, 
          "is_last": true, 
          "last_child": {
            "pid_type": "recid", 
            "pid_value": "3466676"
          }
        }
      ]
    }, 
    "grants": [
      {
        "code": "676247", 
        "links": {
          "self": "https://zenodo.org/api/grants/10.13039/501100000780::676247"
        }, 
        "title": "A Europe-wide Interoperable Virtual Research Environment to Empower Multidisciplinary Research Communities and Accelerate Innovation and Collaboration", 
        "acronym": "VRE4EIC", 
        "program": "H2020", 
        "funder": {
          "doi": "10.13039/501100000780", 
          "acronyms": [], 
          "name": "European Commission", 
          "links": {
            "self": "https://zenodo.org/api/funders/10.13039/501100000780"
          }
        }
      }, 
      {
        "code": "654182", 
        "links": {
          "self": "https://zenodo.org/api/grants/10.13039/501100000780::654182"
        }, 
        "title": "Environmental Research Infrastructures Providing Shared Solutions for Science and Society", 
        "acronym": "ENVRI PLUS", 
        "program": "H2020", 
        "funder": {
          "doi": "10.13039/501100000780", 
          "acronyms": [], 
          "name": "European Commission", 
          "links": {
            "self": "https://zenodo.org/api/funders/10.13039/501100000780"
          }
        }
      }, 
      {
        "code": "643963", 
        "links": {
          "self": "https://zenodo.org/api/grants/10.13039/501100000780::643963"
        }, 
        "title": "Software Workbench for Interactive, Time Critical and Highly self-adaptive cloud applications", 
        "acronym": "SWITCH", 
        "program": "H2020", 
        "funder": {
          "doi": "10.13039/501100000780", 
          "acronyms": [], 
          "name": "European Commission", 
          "links": {
            "self": "https://zenodo.org/api/funders/10.13039/501100000780"
          }
        }
      }, 
      {
        "code": "825134", 
        "links": {
          "self": "https://zenodo.org/api/grants/10.13039/501100000780::825134"
        }, 
        "title": "smART socIal media eCOsytstem in a blockchaiN Federated environment", 
        "acronym": "ARTICONF", 
        "program": "H2020", 
        "funder": {
          "doi": "10.13039/501100000780", 
          "acronyms": [], 
          "name": "European Commission", 
          "links": {
            "self": "https://zenodo.org/api/funders/10.13039/501100000780"
          }
        }
      }
    ], 
    "keywords": [
      "Workflow Scheduling", 
      "Multi-resource Clusters", 
      "Directed-Acyclic Graph (DAG)", 
      "Deep Reinforcement Learning"
    ], 
    "publication_date": "2019-08-16", 
    "creators": [
      {
        "affiliation": "University of Amsterdam", 
        "name": "Hu, Yang"
      }, 
      {
        "affiliation": "University of Amsterdam", 
        "name": "de Laat, Cees"
      }, 
      {
        "orcid": "0000-0002-6717-9418", 
        "affiliation": "University of Amsterdam", 
        "name": "Zhao, Zhiming"
      }
    ], 
    "meeting": {
      "acronym": "NAS", 
      "url": "http://www.nas-conference.org/NAS-2019/", 
      "dates": "15-17 August 2019", 
      "place": "Enshi, China", 
      "title": "The 14th International Conference on Networking, Architecture, and Storage (NAS 2019)"
    }, 
    "access_right": "open", 
    "resource_type": {
      "subtype": "conferencepaper", 
      "type": "publication", 
      "title": "Conference paper"
    }, 
    "description": "<p>Workflow scheduling is one of the key issues in</p>\n\n<p>the management of workflow execution. Typically, a workflow</p>\n\n<p>application can be modeled as a Directed-Acyclic Graph (DAG).</p>\n\n<p>In this paper, we present GoDAG, an approach that can learn</p>\n\n<p>to well schedule workflows on multi-resource clusters. GoDAG</p>\n\n<p>directly learns the scheduling policy from experience through</p>\n\n<p>deep reinforcement learning. In order to adapt deep reinforcement</p>\n\n<p>learning methods, we propose a novel state representation,</p>\n\n<p>a practical action space and a corresponding reward definition</p>\n\n<p>for workflow scheduling problem. We implement a GoDAG</p>\n\n<p>prototype and a simulator to simulate task running on multiresource</p>\n\n<p>clusters. In the evaluation, we compare the GoDAG with</p>\n\n<p>three state-of-the-art heuristics. The results show that GoDAG</p>\n\n<p>outperforms the baseline heuristics, leading to less average</p>\n\n<p>makespan to different workflow structures.</p>"
  }
}
46
43
views
downloads
Views 46
Downloads 43
Data volume 16.8 MB
Unique views 43
Unique downloads 40

Share

Cite as