Dataset Open Access

Natural Language-Guided Programming User Study

Heyman, Geert; Huysegems, Rafeal; Justen, Pascal; Van Cutsem, Tom


JSON Export

{
  "files": [
    {
      "links": {
        "self": "https://zenodo.org/api/files/30c22513-0bf3-4ee6-a71d-30ddc69eb860/benchmark.json"
      }, 
      "checksum": "md5:e1080d051298444befa1014d4fa7bda0", 
      "bucket": "30c22513-0bf3-4ee6-a71d-30ddc69eb860", 
      "key": "benchmark.json", 
      "type": "json", 
      "size": 4169673
    }, 
    {
      "links": {
        "self": "https://zenodo.org/api/files/30c22513-0bf3-4ee6-a71d-30ddc69eb860/eval-index.jsonl"
      }, 
      "checksum": "md5:39f504c8d64f261b7862e3917aa259f7", 
      "bucket": "30c22513-0bf3-4ee6-a71d-30ddc69eb860", 
      "key": "eval-index.jsonl", 
      "type": "jsonl", 
      "size": 632582
    }, 
    {
      "links": {
        "self": "https://zenodo.org/api/files/30c22513-0bf3-4ee6-a71d-30ddc69eb860/predictions-annotated.json"
      }, 
      "checksum": "md5:a50ad18a28bf8d1d750afba51118f00e", 
      "bucket": "30c22513-0bf3-4ee6-a71d-30ddc69eb860", 
      "key": "predictions-annotated.json", 
      "type": "json", 
      "size": 6392629
    }, 
    {
      "links": {
        "self": "https://zenodo.org/api/files/30c22513-0bf3-4ee6-a71d-30ddc69eb860/train-index.jsonl"
      }, 
      "checksum": "md5:cfd046716905235ee8f7139ee4ee240f", 
      "bucket": "30c22513-0bf3-4ee6-a71d-30ddc69eb860", 
      "key": "train-index.jsonl", 
      "type": "jsonl", 
      "size": 5692261
    }
  ], 
  "owners": [
    124380
  ], 
  "doi": "10.5281/zenodo.5384768", 
  "stats": {
    "version_unique_downloads": 1.0, 
    "unique_views": 54.0, 
    "views": 66.0, 
    "version_views": 66.0, 
    "unique_downloads": 1.0, 
    "version_unique_views": 54.0, 
    "volume": 16887145.0, 
    "version_downloads": 4.0, 
    "downloads": 4.0, 
    "version_volume": 16887145.0
  }, 
  "links": {
    "doi": "https://doi.org/10.5281/zenodo.5384768", 
    "conceptdoi": "https://doi.org/10.5281/zenodo.5384767", 
    "bucket": "https://zenodo.org/api/files/30c22513-0bf3-4ee6-a71d-30ddc69eb860", 
    "conceptbadge": "https://zenodo.org/badge/doi/10.5281/zenodo.5384767.svg", 
    "html": "https://zenodo.org/record/5384768", 
    "latest_html": "https://zenodo.org/record/5384768", 
    "badge": "https://zenodo.org/badge/doi/10.5281/zenodo.5384768.svg", 
    "latest": "https://zenodo.org/api/records/5384768"
  }, 
  "conceptdoi": "10.5281/zenodo.5384767", 
  "created": "2021-09-02T15:21:49.172864+00:00", 
  "updated": "2021-09-03T01:48:40.556163+00:00", 
  "conceptrecid": "5384767", 
  "revision": 3, 
  "id": 5384768, 
  "metadata": {
    "access_right_category": "success", 
    "doi": "10.5281/zenodo.5384768", 
    "description": "<p>In this dataset you find the&nbsp;user study data that was used in the <strong><em>Natural Language-Guided Programming</em></strong> paper, which is accepted for Onward! 2021. A preprint can be found here&nbsp;<a href=\"https://arxiv.org/pdf/2108.05198.pdf\">https://arxiv.org/pdf/2108.05198.pdf</a>. The dataset consists of the following files:</p>\n\n<ul>\n\t<li>\n\t<p>benchmark.json contains 201 test cases. Each test case consists of context, a natural language intent and target code. The test cases are intended to evaluate a model that can predict code giving a piece of context code and a natural language intent. The test cases were derived from Jupyter notebooks that were crawled from Github projects with permissive licenses. In the project_metadata field you find information about the original project such as its git url&nbsp;and&nbsp;license.</p>\n\t</li>\n\t<li>\n\t<p>predictions-annotated.json contains predictions of the three models used in the paper for 100 test cases in benchmark.json. Each prediction is accompanied with qualitive assesments from three annotators.</p>\n\t</li>\n\t<li>\n\t<p>train-index.jsonl is the list of github projects that were used for training the models.</p>\n\t</li>\n\t<li>\n\t<p>eval-index.jsonl is a list of github projects that we kept separate for evaluation. The benchmark.json was created from a random subset of the projects in this list.</p>\n\t</li>\n</ul>\n\n<p>For more details we refer to the paper.</p>", 
    "license": {
      "id": "BSD-3-Clause"
    }, 
    "title": "Natural Language-Guided Programming User Study", 
    "relations": {
      "version": [
        {
          "count": 1, 
          "index": 0, 
          "parent": {
            "pid_type": "recid", 
            "pid_value": "5384767"
          }, 
          "is_last": true, 
          "last_child": {
            "pid_type": "recid", 
            "pid_value": "5384768"
          }
        }
      ]
    }, 
    "version": "0.0.1", 
    "keywords": [
      "code completion", 
      "code prediction", 
      "natural language-guided programming", 
      "example-centric programming"
    ], 
    "publication_date": "2021-09-02", 
    "creators": [
      {
        "orcid": "0000-0001-6276-424X", 
        "affiliation": "Nokia Bell Labs", 
        "name": "Heyman, Geert"
      }, 
      {
        "orcid": "0000-0001-6244-9864", 
        "affiliation": "Nokia Bell Labs", 
        "name": "Huysegems, Rafeal"
      }, 
      {
        "affiliation": "Nokia Bell Labs", 
        "name": "Justen, Pascal"
      }, 
      {
        "orcid": "0000-0003-4116-4290", 
        "affiliation": "Nokia Bell Labs", 
        "name": "Van Cutsem, Tom"
      }
    ], 
    "access_right": "open", 
    "resource_type": {
      "type": "dataset", 
      "title": "Dataset"
    }, 
    "related_identifiers": [
      {
        "scheme": "doi", 
        "identifier": "10.5281/zenodo.5384767", 
        "relation": "isVersionOf"
      }
    ]
  }
}
66
4
views
downloads
All versions This version
Views 6666
Downloads 44
Data volume 16.9 MB16.9 MB
Unique views 5454
Unique downloads 11

Share

Cite as