{ "access": { "embargo": { "active": false, "reason": null }, "files": "public", "record": "public", "status": "open" }, "created": "2021-12-04T17:41:56.852363+00:00", "custom_fields": {}, "deletion_status": { "is_deleted": false, "status": "P" }, "files": { "count": 4, "enabled": true, "entries": { "hmdb_meta.Rds": { "access": { "hidden": false }, "checksum": "md5:0fa3c4f9ae6389e4712932f260b53318", "ext": "rds", "id": "9b3efee6-028b-41f8-86c0-857589e29890", "key": "hmdb_meta.Rds", "links": { "content": "https://zenodo.org/api/records/5758926/files/hmdb_meta.Rds/content", "self": "https://zenodo.org/api/records/5758926/files/hmdb_meta.Rds" }, "metadata": null, "mimetype": "application/octet-stream", "size": 15018581, "storage_class": "L" }, "hmdb_metabolites.Rds": { "access": { "hidden": false }, "checksum": "md5:71f927874f203f539c9d8a98de9466b0", "ext": "rds", "id": "5178f1cd-a5ec-4ccc-8577-ca09ba909199", "key": "hmdb_metabolites.Rds", "links": { "content": "https://zenodo.org/api/records/5758926/files/hmdb_metabolites.Rds/content", "self": "https://zenodo.org/api/records/5758926/files/hmdb_metabolites.Rds" }, "metadata": null, "mimetype": "application/octet-stream", "size": 2067123, "storage_class": "L" }, "hmdb_names.Rds": { "access": { "hidden": false }, "checksum": "md5:62502e08196e160355a36de476a10493", "ext": "rds", "id": "2cad5824-b3e7-44e1-af2e-e5a69c5a28eb", "key": "hmdb_names.Rds", "links": { "content": "https://zenodo.org/api/records/5758926/files/hmdb_names.Rds/content", "self": "https://zenodo.org/api/records/5758926/files/hmdb_names.Rds" }, "metadata": null, "mimetype": "application/octet-stream", "size": 6409585, "storage_class": "L" }, "hmdb_proteins.Rds": { "access": { "hidden": false }, "checksum": "md5:970ab6aa24db28c6e644473ddcd6e089", "ext": "rds", "id": "aa74d853-047a-4438-b8b7-c8af4c32634a", "key": "hmdb_proteins.Rds", "links": { "content": "https://zenodo.org/api/records/5758926/files/hmdb_proteins.Rds/content", "self": "https://zenodo.org/api/records/5758926/files/hmdb_proteins.Rds" }, "metadata": null, "mimetype": "application/octet-stream", "size": 12425546, "storage_class": "L" } }, "order": [], "total_bytes": 35920835 }, "id": "5758926", "is_draft": false, "is_published": true, "links": { "access": "https://zenodo.org/api/records/5758926/access", "access_grants": "https://zenodo.org/api/records/5758926/access/grants", "access_links": "https://zenodo.org/api/records/5758926/access/links", "access_request": "https://zenodo.org/api/records/5758926/access/request", "access_users": "https://zenodo.org/api/records/5758926/access/users", "archive": "https://zenodo.org/api/records/5758926/files-archive", "archive_media": "https://zenodo.org/api/records/5758926/media-files-archive", "communities": "https://zenodo.org/api/records/5758926/communities", "communities-suggestions": "https://zenodo.org/api/records/5758926/communities-suggestions", "doi": "https://doi.org/10.5281/zenodo.5758926", "draft": "https://zenodo.org/api/records/5758926/draft", "file_modification": "https://zenodo.org/api/records/5758926/file-modification", "files": "https://zenodo.org/api/records/5758926/files", "latest": "https://zenodo.org/api/records/5758926/versions/latest", "latest_html": "https://zenodo.org/records/5758926/latest", "media_files": "https://zenodo.org/api/records/5758926/media-files", "parent": "https://zenodo.org/api/records/5758925", "parent_doi": "https://doi.org/10.5281/zenodo.5758925", "parent_doi_html": "https://zenodo.org/doi/10.5281/zenodo.5758925", "parent_html": "https://zenodo.org/records/5758925", "preview_html": "https://zenodo.org/records/5758926?preview=1", "request_deletion": "https://zenodo.org/api/records/5758926/request-deletion", "requests": "https://zenodo.org/api/records/5758926/requests", "reserve_doi": "https://zenodo.org/api/records/5758926/draft/pids/doi", "self": "https://zenodo.org/api/records/5758926", "self_doi": "https://doi.org/10.5281/zenodo.5758926", "self_doi_html": "https://zenodo.org/doi/10.5281/zenodo.5758926", "self_html": "https://zenodo.org/records/5758926", "self_iiif_manifest": "https://zenodo.org/api/iiif/record:5758926/manifest", "self_iiif_sequence": "https://zenodo.org/api/iiif/record:5758926/sequence/default", "versions": "https://zenodo.org/api/records/5758926/versions" }, "media_files": { "count": 0, "enabled": false, "entries": {}, "order": [], "total_bytes": 0 }, "metadata": { "additional_descriptions": [ { "description": "#read current release information to set parameters for download\nsource(here::here(\"code\", \"current_release.R\"))\n\n#LOAD BIG data \ntemp <- tempfile()\ndownload.file(metabolites_url, temp)\nhmdb_full <- xmlToDataFrame(unzip(temp)) %>% #extracting XML took a long time for a 4GB XML file!\n janitor::clean_names()\nunlink(temp)\nunlink(\"hmdb_metabolites.xml\")\n\n#hmdb_full <- xmlToDataFrame(here::here(\"tmp\", \"hmdb_metabolites.xml\")) %>% janitor::clean_names()\n\n#for getting metabolite classes\nclass_finder <- function(string) {\n if(stringr::str_detect(string, \"class\\\\b\")) {\n class_compounds <- function(string) {\n tmp <- purrr::map(string, ~ unlist(str_split(., pattern = \"\\\\.\"))) %>% \n purrr::map(., ~keep(.x, ~ str_detect(.x, \"class\"))) %>% flatten_chr(.)\n class <- stringr::str_split(tmp, \"class of organic compounds known as \")\n return(stringr::str_to_sentence(class[[1]][[2]]))}\n tryCatch(class_compounds(string), \n error = function(x){\"Organic compound\"})\n } else if(stringr::str_detect(string, \"classified\\\\b\")) {\n classified_compounds <- function(string) {\n tmp <- purrr::map(string, ~ unlist(str_split(., pattern = \"\\\\.\"))) %>% \n purrr::map(., ~keep(.x, ~ str_detect(.x, \"classified\"))) %>% flatten_chr(.)\n class <- stringr::str_split(tmp, \"classified as a member of the \")\n return(stringr::str_to_sentence(class[[1]][[2]]))}\n tryCatch(classified_compounds(string), \n error = function(x){\"Organic compound\"})\n } else if(stringr::str_detect(string, \"steroid\")) {\n return(\"Steroid hormone\")\n } else if(stringr::str_detect(string, \"nucleoside\")) {\n return(\"Nucleoside\")\n } else if(stringr::str_detect(string, \"acid\")) {\n return(\"Organic acid\")\n } else {\n return(\"Organic compound\")\n }\n}\n\n#purrr::map(hmdb_class$description, ~ class_finder(.))\n\nhmdb_class <- \n hmdb_full %>% \n #slice(1:10) %>% #for testing\n dplyr::select(name, description) %>% \n dplyr::mutate(class = purrr::map(description, ~ class_finder(.))) %>% \n dplyr::select(-description)\n\nhmdb_names <-\n hmdb_full %>%\n #slice(1:10) %>% #for testing\n dplyr::select(name, synonyms, cid = pubchem_compound_id) %>% \n dplyr::left_join(hmdb_class, by = \"name\")\n\nhmdb_meta <-\n hmdb_full %>%\n dplyr::select(name, synonyms, cid = pubchem_compound_id, accession, description, chemical_formula, average_molecular_weight, wikipedia_id) %>% \n dplyr::left_join(hmdb_class, by = \"name\")\n\n\n\n\n\n\n\ntemp <- tempfile()\ndownload.file(metabolite_proteins_url, temp)\nhmdb_proteins_raw <- \n XML::xmlToDataFrame(unzip(temp)) %>% \n janitor::clean_names()\nunlink(temp)\nunlink(\"hmdb_proteins.xml\")\n\ncensor_proteins <- c(\"\")\ncensor_metabolites <- c(\"\",\"Water\", \"Hydrogen Ion\")\n\nhmdb_proteins_long <- \n hmdb_proteins_raw %>%\n #dplyr::slice(1:20) %>% #for testing\n dplyr::mutate(metabolite_associations = stringr::str_replace_all(metabolite_associations, \"HMDB\", \"\\\\.HMDB\")) %>% \n tidyr::separate_rows(metabolite_associations, sep = \"\\\\.\") %>% \n dplyr::filter(metabolite_associations != \"\") %>% \n tidyr::separate(metabolite_associations, \n into = c(\"metabolite_accession\", \"metabolite_name\"), \n sep = 11) %>% \n dplyr::select(gene_name, metabolite_name, gene_accession = accession, metabolite_accession) %>% \n dplyr::filter(!gene_name %in% censor_proteins, \n !metabolite_name %in% censor_metabolites, \n stringr::str_detect(metabolite_accession, \"HMDB\")) \n\n# tmp <- \"Water\"\n# tmp <- \"Glucose-6-Phosphate\"\n# tmp <- \"3-Carbamoyl-2-phenylpropionaldehyde\"\n# tmp <- \"-1(11),7,9-trien-11-ol\"\n# metabolite_string <- \"R-95913\"\n\n\n#collapse metabolite name if it's long\ncollapse_metabolites <- function(metabolite_string) {\n # #skip if missing\n # if(stringr::str_length(metabolite_string) == 0 | is.na(metabolite_string)) {\n # return(metabolite_string)\n # }\n #skip if too short\n if(stringr::str_length(metabolite_string) < 7) {\n return(metabolite_string)\n }\n #get first word\n new_metabolite_string <- \n stringr::str_extract(metabolite_string, \"[[:alpha:]]\\\\w+\") #alpha omits numbers and punct, w gets word character, + gets one or more\n #if new_string is NA b/c it has a bizarre name (like a drug name), code breaks; so return original string\n if(is.na(new_metabolite_string)) {\n return(metabolite_string)\n }\n #add plus to indicate that it got collapsed\n if(metabolite_string != new_metabolite_string) {\n new_metabolite_string <- glue::glue(\"{new_metabolite_string} +\")}\n return(new_metabolite_string)\n}\n# collapse_metabolites(tmp)\n\n# simplify_metabolites <- function(df) {\n# # if (nrow(df) < 20) {\n# # return(df)\n# # }\n# new_df <- \n# df %>% \n# mutate(metabolite_name_simple = map_chr(.x = df[[2]], .f = collapse_metabolites))\n# return(new_df)\n# }\n\nhmdb_proteins_full <-\n hmdb_proteins_long %>% \n dplyr::group_by(fav_gene = gene_name) %>% \n tidyr::nest() %>% \n dplyr::filter(!fav_gene %in% censor_proteins) %>%\n dplyr::mutate(original_num_rows = map_int(data, nrow)) \n\nhmdb_proteins <-\n hmdb_proteins_long %>% \n dplyr::mutate(metabolite_name_simple = map_chr(metabolite_name, collapse_metabolites)) %>% \n dplyr::distinct(gene_name, metabolite_name_simple, .keep_all = TRUE) %>% \n dplyr::group_by(fav_gene = gene_name) %>% \n tidyr::nest() %>% \n dplyr::filter(!fav_gene %in% censor_proteins) %>% \n dplyr::mutate(num_rows = map_int(data, nrow)) %>% \n dplyr::left_join(hmdb_proteins_full, by = \"fav_gene\", suffix = c(\"_collapsed\", \"_original\")) %>% \n dplyr::arrange(desc(num_rows))\n\n#colnames(hmdb_proteins) are fav_gene, data_collapsed, num_rows, data_original, original_num_rows\n\nhmdb_metabolites <-\n hmdb_proteins_long %>% \n #dplyr::mutate(metabolite_name = stringr::str_to_lower(metabolite_name)) %>% \n dplyr::group_by(fav_metabolite = metabolite_name) %>% \n tidyr::nest() %>% \n dplyr::filter(!fav_metabolite %in% censor_metabolites) %>% \n dplyr::mutate(num_rows = map_int(data, nrow)) %>% \n dplyr::arrange(desc(num_rows))\n\n#save files\n#saveRDS(hmdb_full, file = here::here(\"data\", paste0(release, \"_hmdb_full.Rds\")))\nsaveRDS(hmdb_names, file = here::here(\"data\", paste0(release, \"_hmdb_names.Rds\")))\nsaveRDS(hmdb_meta, file = here::here(\"data\", paste0(release, \"_hmdb_meta.Rds\")))\nsaveRDS(hmdb_proteins, file = here::here(\"data\", paste0(release, \"_hmdb_proteins.Rds\")))\nsaveRDS(hmdb_metabolites, file = here::here(\"data\", paste0(release, \"_hmdb_metabolites.Rds\")))", "type": { "id": "notes", "title": { "de": "Anmerkungen", "en": "Notes" } } } ], "creators": [ { "person_or_org": { "family_name": "Hirschey", "given_name": "Matthew", "name": "Hirschey, Matthew", "type": "personal" } } ], "description": "

This is a cleaned and condensed version of the HMDB XML metabolites file from https://hmdb.ca/downloads, current version 5.0. The original zipped file (~1GB) extracts to 6.49 GB xml file. The full is then cleaned and then only small sub-files are exported. This is too big to extract from the zipped file on the server, and gets clipped (to about 4GB). As a result, the XML to DF function fails because of missing data.

\n\n

This repo containing 4 files is an alternative pointer for only the data needed. Code to process it from the hmdb_full file is pasted in the notes below.

\n\n

These 4 files are used on www.datadrivenhypothesis.org

", "publication_date": "2021-12-03", "publisher": "Zenodo", "resource_type": { "id": "dataset", "title": { "de": "Datensatz", "en": "Dataset" } }, "rights": [ { "description": { "en": "The Creative Commons Attribution license allows re-distribution and re-use of a licensed work on the condition that the creator is appropriately credited." }, "icon": "cc-by-icon", "id": "cc-by-4.0", "props": { "scheme": "spdx", "url": "https://creativecommons.org/licenses/by/4.0/legalcode" }, "title": { "en": "Creative Commons Attribution 4.0 International" } } ], "title": "HMDB Clean", "version": "21Q4" }, "parent": { "access": { "owned_by": { "user": "119920" } }, "communities": {}, "id": "5758925", "pids": { "doi": { "client": "datacite", "identifier": "10.5281/zenodo.5758925", "provider": "datacite" } } }, "pids": { "doi": { "client": "datacite", "identifier": "10.5281/zenodo.5758926", "provider": "datacite" }, "oai": { "identifier": "oai:zenodo.org:5758926", "provider": "oai" } }, "revision_id": 2, "stats": { "all_versions": { "data_volume": 4123766029.0, "downloads": 462, "unique_downloads": 418, "unique_views": 631, "views": 653 }, "this_version": { "data_volume": 4123766029.0, "downloads": 462, "unique_downloads": 418, "unique_views": 630, "views": 652 } }, "status": "published", "swh": {}, "updated": "2021-12-05T13:48:44.757425+00:00", "versions": { "index": 1, "is_latest": true } }