mirror of
https://github.com/msberends/AMR.git
synced 2026-11-16 03:40:48 +01:00
(v3.0.1.9096) MO code registry: codes never change meaning, are never guessed, and are validated
- data-raw/microorganisms_files/: registry of all MO codes of every release since v2.0.0 (78,992 codes, built from the git tags), approved renames, and README with the rules; decision by Matthijs S. Berends: MO codes from before v2.0.0 are not supported - as.mo(): codes of earlier releases are translated deterministically via the internal MO_RETIRED_CODES (generated from the registry); unknown codes are no longer guessed (previously F_TRCHP, Trichophyton in v3.0.1, became Talaromyces chlamydosporus) - synonym_mo_to_accepted_mo(): a missing target of one source no longer overwrites a valid target of another source (e.g. Chaetomium abuense), and no 'renamed to itself' - taxonomy build: codes are reused and reserved from the registry instead of the development data (fixes 40 codes reused in the 2026 development data), also for phylum to family; synonym cycles are broken; dangling 'renamed to' identifiers are removed; the build stops on any broken integrity rule before saving - run_microorganisms_build.R: runs the build in chunks at its checkpoints in one R process, with progress logging and resuming from any checkpoint; prompt updated accordingly - tests: shared integrity rules (helper-microorganisms.R) used by build and testthat, registry tests over all codes, sentinel organisms, regression tests; current defects of the data on main are listed in microorganisms_known_defects.csv until the next rebuild Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ca5fa333ca
commit
d683b9e7c6
16 files changed
+80516
-246
No files matched your search
@@ -115,6 +115,9 @@ devtools::load_all(".") # to load the AMR package
|
||||
# keep the previous data set at hand, `AMR::microorganisms` will be replaced at the end of this script
|
||||
microorganisms_old <- AMR::microorganisms
|
||||
|
||||
# the integrity rules of the data set and the MO code registry, shared with the unit tests
|
||||
source("tests/testthat/helper-microorganisms.R")
|
||||
|
||||
|
||||
# Helper functions --------------------------------------------------------------------------------
|
||||
|
||||
@@ -143,6 +146,11 @@ review <- function(x, title = deparse(substitute(x))) {
|
||||
invisible(x)
|
||||
}
|
||||
|
||||
# taxonomic names in ASCII and without quotes
|
||||
ascii_names <- function(x) {
|
||||
gsub("[\"'`]", "", stringi::stri_trans_general(x, "Latin-ASCII"))
|
||||
}
|
||||
|
||||
# the priority of sources, used everywhere in this script
|
||||
source_priority <- c("LPSN" = 1L, "MycoBank" = 2L, "GBIF" = 3L, "manually added" = 4L, "inferred" = 5L)
|
||||
source_prio <- function(source) {
|
||||
@@ -1052,8 +1060,11 @@ saveRDS(taxonomy_mycobank, "data-raw/taxonomy_mycobank.rds", version = 2)
|
||||
|
||||
# Read GBIF data ----------------------------------------------------------------------------------
|
||||
|
||||
# @resume-block gbif_raw after=taxonomy_gbif
|
||||
# (needed by `current_gbif` when resuming from a checkpoint with run_microorganisms_build.R)
|
||||
taxonomy_gbif.bak <- vroom(file_gbif, guess_max = 5e5)
|
||||
colnames(taxonomy_gbif.bak) <- gsub(".*:(.*)", "\\1", colnames(taxonomy_gbif.bak))
|
||||
# @end-resume-block
|
||||
|
||||
# include all fungal orders from the mycobank db
|
||||
include_fungal_orders <- unique(taxonomy_mycobank$order[
|
||||
@@ -1282,10 +1293,7 @@ taxonomy_gbif <- readRDS("data-raw/taxonomy_gbif.rds")
|
||||
# Add full names ----------------------------------------------------------------------------------
|
||||
|
||||
# names must be ASCII before they are made unique, otherwise e.g. 'Fusarium aloës' and 'Fusarium aloes'
|
||||
# both end up as 'Fusarium aloes' in the final data set (which happened to 12 names in 2026)
|
||||
ascii_names <- function(x) {
|
||||
gsub("[\"'`]", "", stringi::stri_trans_general(x, "Latin-ASCII"))
|
||||
}
|
||||
# both end up as 'Fusarium aloes' in the final data set (which happened to 12 names in 2026), see ascii_names()
|
||||
taxonomy_gbif <- taxonomy_gbif %>%
|
||||
mutate(across(domain:subspecies, ascii_names))
|
||||
taxonomy_lpsn <- taxonomy_lpsn %>%
|
||||
@@ -1639,6 +1647,8 @@ saveRDS(taxonomy, "data-raw/taxonomy0.rds")
|
||||
# Ensure every referenced rank has its own row (domain through species).
|
||||
# Where possible, enrich with GBIF identifiers.
|
||||
|
||||
# @resume-block current_gbif after=taxonomy1
|
||||
# (needed by add_missing_parents() after filtering, when resuming from a checkpoint)
|
||||
current_gbif <- taxonomy_gbif.bak %>%
|
||||
filter(is.na(acceptedNameUsageID)) %>%
|
||||
mutate(
|
||||
@@ -1663,6 +1673,7 @@ current_gbif <- taxonomy_gbif.bak %>%
|
||||
phylum = if_else(taxonRank == "phylum", canonical, phylum)
|
||||
) %>%
|
||||
select(-canonical)
|
||||
# @end-resume-block
|
||||
|
||||
taxonomy <- add_missing_parents(taxonomy, current_gbif)
|
||||
|
||||
@@ -2035,6 +2046,8 @@ taxonomy$rank[is.na(taxonomy$rank)] <- "domain"
|
||||
# this part is required here, because it's needed for filtering on 'relevant' species to keep later on
|
||||
# (`pathogens` was read at the start of this script)
|
||||
|
||||
# @resume-block prevalence after=taxonomy2
|
||||
# (compute_prevalence() is used again later on, when resuming from a checkpoint)
|
||||
# get all established, both old and current taxonomic names
|
||||
established <- pathogens %>%
|
||||
filter(status == "established") %>%
|
||||
@@ -2147,6 +2160,7 @@ compute_prevalence <- function(taxonomy) {
|
||||
mutate(prevalence = if_else(rank == "genus", pmin(prevalence, gen_prevalence, na.rm = TRUE), prevalence)) %>%
|
||||
select(-gen_prevalence)
|
||||
}
|
||||
# @end-resume-block
|
||||
taxonomy <- compute_prevalence(taxonomy)
|
||||
|
||||
table(taxonomy$prevalence, useNA = "always")
|
||||
@@ -2513,6 +2527,77 @@ taxonomy <- taxonomy %>%
|
||||
distinct(fullname, .keep_all = TRUE)
|
||||
message("Removed ", n_before - nrow(taxonomy), " nonsense names")
|
||||
|
||||
# Break cycles of synonyms ------------------------------------------------------------------------
|
||||
|
||||
# Sources can contradict each other, e.g. in 2026 GBIF listed Capillidium denaeosporus as synonym of
|
||||
# Conidiobolus denaeosporum and also the other way around, so that the 'current name' depended on the starting
|
||||
# point. In a cycle, the statement of the most authoritative source counts: that record stays a synonym, and its
|
||||
# target becomes accepted.
|
||||
next_synonym_step <- function(df) {
|
||||
by_gbif <- df$fullname[match(df$gbif_renamed_to, df$gbif, incomparables = NA)]
|
||||
by_mycobank <- df$fullname[match(df$mycobank_renamed_to, df$mycobank, incomparables = NA)]
|
||||
by_lpsn <- df$fullname[match(df$lpsn_renamed_to, df$lpsn, incomparables = NA)]
|
||||
out <- coalesce(by_lpsn, by_mycobank, by_gbif)
|
||||
out[df$status != "synonym"] <- NA_character_
|
||||
setNames(out, df$fullname)
|
||||
}
|
||||
synonym_next <- next_synonym_step(taxonomy)
|
||||
synonym_cycles <- list()
|
||||
for (start in names(synonym_next)[!is.na(synonym_next)]) {
|
||||
path <- start
|
||||
repeat {
|
||||
nxt <- unname(synonym_next[path[length(path)]])
|
||||
if (is.na(nxt)) {
|
||||
break
|
||||
}
|
||||
if (nxt %in% path) {
|
||||
cycle <- sort(path[match(nxt, path):length(path)])
|
||||
synonym_cycles[[paste(cycle, collapse = " | ")]] <- cycle
|
||||
break
|
||||
}
|
||||
path <- c(path, nxt)
|
||||
if (length(path) > 25) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
# If multiple records in the cycle come from the most authoritative source, the sources contradict themselves and
|
||||
# the direction cannot be determined automatically (in 2026, even the authors in `ref` of these GBIF records were
|
||||
# unreliable). Then ALL records in the cycle become accepted, so that no wrong current name is given, and they are
|
||||
# shown for a human decision.
|
||||
cycle_fixes <- bind_rows(lapply(synonym_cycles, function(cycle_members) {
|
||||
# (not named `cycle`, since tibble() below would then use its own new column `cycle`)
|
||||
members <- taxonomy[match(cycle_members, taxonomy$fullname), , drop = FALSE]
|
||||
best <- members[source_prio(members$source) == min(source_prio(members$source)), , drop = FALSE]
|
||||
if (nrow(best) == 1) {
|
||||
tibble(
|
||||
cycle = paste(cycle_members, collapse = " | "),
|
||||
made_accepted = unname(synonym_next[best$fullname]),
|
||||
reason = paste("statement of", best$fullname, "from", best$source, "(most authoritative source)")
|
||||
)
|
||||
} else {
|
||||
tibble(
|
||||
cycle = paste(cycle_members, collapse = " | "),
|
||||
made_accepted = cycle_members,
|
||||
reason = paste("contradicting statements of", toString(unique(best$source)), "- all accepted, NEEDS A HUMAN DECISION")
|
||||
)
|
||||
}
|
||||
}))
|
||||
review(cycle_fixes, "Cycles of synonyms")
|
||||
if (nrow(cycle_fixes) > 0) {
|
||||
taxonomy <- taxonomy %>%
|
||||
mutate(
|
||||
fix = fullname %in% cycle_fixes$made_accepted,
|
||||
status = if_else(fix, "accepted", status),
|
||||
lpsn_renamed_to = if_else(fix, NA_character_, lpsn_renamed_to),
|
||||
mycobank_renamed_to = if_else(fix, NA_character_, mycobank_renamed_to),
|
||||
gbif_renamed_to = if_else(fix, NA_character_, gbif_renamed_to)
|
||||
) %>%
|
||||
select(-fix)
|
||||
}
|
||||
rm(synonym_next, synonym_cycles, cycle_fixes)
|
||||
|
||||
|
||||
# Fix genera that are synonyms while they contain accepted species --------------------------------
|
||||
|
||||
# e.g. in 2026, MycoBank listed the genus Blastomyces as a synonym, while Blastomyces dermatitidis was accepted
|
||||
@@ -2560,151 +2645,59 @@ mo_domain <- taxonomy %>%
|
||||
)
|
||||
mo_domain
|
||||
|
||||
# `domain` exists in the data set since 2026, so codes are kept per domain
|
||||
# (in the 2026 build, this was `AMR::microorganisms %>% rename(domain = kingdom)`, which would now fail)
|
||||
existing_mo_tbl <- microorganisms_old %>%
|
||||
mutate(mo = as.character(mo))
|
||||
|
||||
# @resume-block mo_registry after=taxonomy2c
|
||||
# (the MO code registry is used by the final checks, when resuming from a checkpoint)
|
||||
# parts of existing MO codes, split on the underscore (genus codes can contain digits or hyphens, such as
|
||||
# B_RBCLM1 or B_CND-P, so a regex with only [A-Z] does not work and caused repeated elements like
|
||||
# B_SCLLM_CNNM_LNSM_LNSM_LNSM in earlier versions)
|
||||
mo_part <- function(mo, i) {
|
||||
vapply(strsplit(as.character(mo), "_", fixed = TRUE), function(x) if (length(x) >= i) x[i] else NA_character_, character(1))
|
||||
}
|
||||
# parts of a name, e.g. the species of 'Escherichia coli'
|
||||
mo_part_of_name <- function(x, i) {
|
||||
vapply(strsplit(x, " ", fixed = TRUE), function(x) if (length(x) >= i) x[i] else "", character(1))
|
||||
}
|
||||
|
||||
# Existing codes come from the MO code registry (data-raw/microorganisms_files/), which contains every code of
|
||||
# every release since v2.0.0, and NOT from the previous data set: codes that only existed in a development
|
||||
# version carry no weight (decision by Matthijs S. Berends, 3 October 2026). A registered code is never given
|
||||
# to another taxon, also not when its taxon is not in the new data set anymore, since users store these codes.
|
||||
mo_registry <- read_mo_registry(".")
|
||||
mo_renames <- read_mo_renames(".")
|
||||
if (is.null(mo_registry)) {
|
||||
stop("MO code registry not found, see data-raw/microorganisms_files/README.md", call. = FALSE)
|
||||
}
|
||||
release_order <- function(release) {
|
||||
as.integer(factor(release, levels = unique(release[order(numeric_version(sub("^v", "", release)))])))
|
||||
}
|
||||
existing_mo_tbl <- mo_registry %>%
|
||||
# approved renames, e.g. a corrected spelling: the code continues with the new name
|
||||
left_join(mo_renames %>% select(mo, new_name), by = "mo") %>%
|
||||
mutate(
|
||||
fullname = coalesce(new_name, fullname),
|
||||
species = if_else(!is.na(new_name) & rank %in% c("species", "subspecies"), mo_part_of_name(new_name, 2), species),
|
||||
subspecies = if_else(!is.na(new_name) & rank == "subspecies", mo_part_of_name(new_name, 3), subspecies)
|
||||
) %>%
|
||||
select(-new_name) %>%
|
||||
# if a taxon had multiple codes over time, the code of the most recent release is used
|
||||
arrange(desc(release_order(last_release)), desc(release_order(first_release))) %>%
|
||||
distinct(domain, rank, fullname, .keep_all = TRUE)
|
||||
|
||||
# all codes that have ever been used must never be given to another taxon, also not when their taxon
|
||||
# is not in the new data set anymore (users may have stored old codes)
|
||||
reserved_genus_codes <- existing_mo_tbl %>%
|
||||
reserved_genus_codes <- mo_registry %>%
|
||||
filter(rank %in% c("genus", "species", "subspecies")) %>%
|
||||
transmute(domain, code = mo_part(mo, 2)) %>%
|
||||
distinct()
|
||||
reserved_species_codes <- existing_mo_tbl %>%
|
||||
reserved_species_codes <- mo_registry %>%
|
||||
filter(rank %in% c("species", "subspecies")) %>%
|
||||
transmute(domain, genus_code = mo_part(mo, 2), code = mo_part(mo, 3)) %>%
|
||||
distinct()
|
||||
# @end-resume-block
|
||||
|
||||
|
||||
# phylum until family are abbreviated with 8 characters and prefixed with their rank
|
||||
|
||||
# Phylum - keep old and fill up for new ones
|
||||
mo_phylum <- taxonomy %>%
|
||||
filter(rank == "phylum") %>%
|
||||
distinct(domain, phylum) %>%
|
||||
left_join(
|
||||
existing_mo_tbl %>%
|
||||
filter(rank == "phylum") %>%
|
||||
transmute(
|
||||
domain,
|
||||
phylum = fullname,
|
||||
mo_old = gsub("^[A-Z]{1,2}_", "", as.character(mo))
|
||||
),
|
||||
by = c("domain", "phylum")
|
||||
) %>%
|
||||
group_by(domain) %>%
|
||||
mutate(
|
||||
mo_phylum8 = AMR:::abbreviate_mo(phylum, minlength = 8, prefix = "[PHL]_"),
|
||||
mo_phylum9 = AMR:::abbreviate_mo(phylum, minlength = 9, prefix = "[PHL]_"),
|
||||
mo_phylum = if_else(!is.na(mo_old), mo_old, mo_phylum8),
|
||||
mo_duplicated = duplicated(mo_phylum),
|
||||
mo_phylum = if_else(mo_duplicated, mo_phylum9, mo_phylum),
|
||||
mo_duplicated = duplicated(mo_phylum)
|
||||
) %>%
|
||||
ungroup()
|
||||
if (any(mo_phylum$mo_duplicated, na.rm = TRUE)) {
|
||||
stop("Duplicate MO codes for phylum!")
|
||||
}
|
||||
mo_phylum <- mo_phylum %>%
|
||||
select(domain, phylum, mo_phylum)
|
||||
|
||||
# Class - keep old and fill up for new ones
|
||||
mo_class <- taxonomy %>%
|
||||
filter(rank == "class") %>%
|
||||
distinct(domain, class) %>%
|
||||
left_join(
|
||||
existing_mo_tbl %>%
|
||||
filter(rank == "class") %>%
|
||||
transmute(
|
||||
domain,
|
||||
class = fullname,
|
||||
mo_old = gsub("^[A-Z]{1,2}_", "", as.character(mo))
|
||||
),
|
||||
by = c("domain", "class")
|
||||
) %>%
|
||||
group_by(domain) %>%
|
||||
mutate(
|
||||
mo_class8 = AMR:::abbreviate_mo(class, minlength = 8, prefix = "[CLS]_"),
|
||||
mo_class9 = AMR:::abbreviate_mo(class, minlength = 9, prefix = "[CLS]_"),
|
||||
mo_class = if_else(!is.na(mo_old), mo_old, mo_class8),
|
||||
mo_duplicated = duplicated(mo_class),
|
||||
mo_class = if_else(mo_duplicated, mo_class9, mo_class),
|
||||
mo_duplicated = duplicated(mo_class)
|
||||
) %>%
|
||||
ungroup()
|
||||
if (any(mo_class$mo_duplicated, na.rm = TRUE)) {
|
||||
stop("Duplicate MO codes for class!")
|
||||
}
|
||||
mo_class <- mo_class %>%
|
||||
select(domain, class, mo_class)
|
||||
|
||||
# Order - keep old and fill up for new ones
|
||||
mo_order <- taxonomy %>%
|
||||
filter(rank == "order") %>%
|
||||
distinct(domain, order) %>%
|
||||
left_join(
|
||||
existing_mo_tbl %>%
|
||||
filter(rank == "order") %>%
|
||||
transmute(
|
||||
domain,
|
||||
order = fullname,
|
||||
mo_old = gsub("^[A-Z]{1,2}_", "", as.character(mo))
|
||||
),
|
||||
by = c("domain", "order")
|
||||
) %>%
|
||||
group_by(domain) %>%
|
||||
mutate(
|
||||
mo_order8 = AMR:::abbreviate_mo(order, minlength = 8, prefix = "[ORD]_"),
|
||||
mo_order9 = AMR:::abbreviate_mo(order, minlength = 9, prefix = "[ORD]_"),
|
||||
mo_order = if_else(!is.na(mo_old), mo_old, mo_order8),
|
||||
mo_duplicated = duplicated(mo_order),
|
||||
mo_order = if_else(mo_duplicated, mo_order9, mo_order),
|
||||
mo_duplicated = duplicated(mo_order)
|
||||
) %>%
|
||||
ungroup()
|
||||
if (any(mo_order$mo_duplicated, na.rm = TRUE)) {
|
||||
stop("Duplicate MO codes for order!")
|
||||
}
|
||||
mo_order <- mo_order %>%
|
||||
select(domain, order, mo_order)
|
||||
|
||||
# Family - keep old and fill up for new ones
|
||||
mo_family <- taxonomy %>%
|
||||
filter(rank == "family") %>%
|
||||
distinct(domain, family) %>%
|
||||
left_join(
|
||||
existing_mo_tbl %>%
|
||||
filter(rank == "family") %>%
|
||||
transmute(
|
||||
domain,
|
||||
family = fullname,
|
||||
mo_old = gsub("^[A-Z]{1,2}_", "", as.character(mo))
|
||||
),
|
||||
by = c("domain", "family")
|
||||
) %>%
|
||||
group_by(domain) %>%
|
||||
mutate(
|
||||
mo_family8 = AMR:::abbreviate_mo(family, minlength = 8, prefix = "[FAM]_"),
|
||||
mo_family9 = AMR:::abbreviate_mo(family, minlength = 9, prefix = "[FAM]_"),
|
||||
mo_family = if_else(!is.na(mo_old), mo_old, mo_family8),
|
||||
mo_duplicated = duplicated(mo_family),
|
||||
mo_family = if_else(mo_duplicated, mo_family9, mo_family),
|
||||
mo_duplicated = duplicated(mo_family)
|
||||
) %>%
|
||||
ungroup()
|
||||
if (any(mo_family$mo_duplicated, na.rm = TRUE)) {
|
||||
stop("Duplicate MO codes for family!")
|
||||
}
|
||||
mo_family <- mo_family %>%
|
||||
select(domain, family, mo_family)
|
||||
|
||||
# keep the old code where available, and give every new taxon the first candidate code that is not in use
|
||||
# and was never used before (`reserved`) - if all candidates are taken, a number is added to the first one
|
||||
assign_codes <- function(old, candidates, reserved = character(0)) {
|
||||
@@ -2726,6 +2719,47 @@ assign_codes <- function(old, candidates, reserved = character(0)) {
|
||||
out
|
||||
}
|
||||
|
||||
# Phylum until family - keep old (from the registry) and fill up for new ones, never with a registered code
|
||||
higher_rank_codes <- function(rank_name, tag) {
|
||||
reserved <- mo_registry %>%
|
||||
filter(rank == rank_name) %>%
|
||||
transmute(domain, code = sub("^[A-Z]{1,2}_", "", mo))
|
||||
out <- taxonomy %>%
|
||||
filter(rank == rank_name) %>%
|
||||
distinct(domain, name = .data[[rank_name]]) %>%
|
||||
left_join(
|
||||
existing_mo_tbl %>%
|
||||
filter(rank == rank_name) %>%
|
||||
transmute(domain, name = mo_name_without_suffix(fullname), mo_old = sub("^[A-Z]{1,2}_", "", mo)) %>%
|
||||
distinct(domain, name, .keep_all = TRUE),
|
||||
by = c("domain", "name")
|
||||
) %>%
|
||||
group_by(domain) %>%
|
||||
mutate(
|
||||
code = assign_codes(
|
||||
old = mo_old,
|
||||
candidates = Map(
|
||||
c,
|
||||
AMR:::abbreviate_mo(name, minlength = 8, prefix = paste0("[", tag, "]_")),
|
||||
AMR:::abbreviate_mo(name, minlength = 9, prefix = paste0("[", tag, "]_"))
|
||||
),
|
||||
reserved = reserved$code[reserved$domain == cur_group()$domain]
|
||||
)
|
||||
) %>%
|
||||
ungroup()
|
||||
if (anyDuplicated(paste(out$domain, out$code)) > 0 || anyNA(out$code)) {
|
||||
stop("Duplicate MO codes for ", rank_name, "!", call. = FALSE)
|
||||
}
|
||||
out %>%
|
||||
select(domain, name, code) %>%
|
||||
setNames(c("domain", rank_name, paste0("mo_", rank_name)))
|
||||
}
|
||||
mo_phylum <- higher_rank_codes("phylum", "PHL")
|
||||
mo_class <- higher_rank_codes("class", "CLS")
|
||||
mo_order <- higher_rank_codes("order", "ORD")
|
||||
mo_family <- higher_rank_codes("family", "FAM")
|
||||
|
||||
|
||||
# construct code part for genus - keep old code where available and generate new ones where needed
|
||||
mo_genus <- taxonomy %>%
|
||||
filter(rank == "genus") %>%
|
||||
@@ -2863,6 +2897,13 @@ mo_subspecies <- mo_subspecies %>%
|
||||
mo_unknown <- existing_mo_tbl %>%
|
||||
filter(fullname %like% "unknown") %>%
|
||||
transmute(fullname, mo_unknown = as.character(mo))
|
||||
# unknowns that were added after the last release keep their code too, as long as it is not registered
|
||||
mo_unknown <- mo_unknown %>%
|
||||
bind_rows(
|
||||
microorganisms_old %>%
|
||||
filter(fullname %like% "unknown", !fullname %in% mo_unknown$fullname, !mo %in% mo_registry$mo) %>%
|
||||
transmute(fullname, mo_unknown = as.character(mo))
|
||||
)
|
||||
|
||||
# apply the new codes!
|
||||
taxonomy1 <- taxonomy
|
||||
@@ -3588,23 +3629,33 @@ microorganisms_old %>%
|
||||
select(mo, fullname, rank, status, prevalence) %>%
|
||||
review("Previously manually added taxa that are not in the new data set")
|
||||
|
||||
# LPSN records must be prokaryotes and MycoBank records must be fungi, also in their MO code (issue #309)
|
||||
if (!all(taxonomy$domain[taxonomy$source == "LPSN"] %in% c("Bacteria", "Archaea")) ||
|
||||
!all(taxonomy$mo[taxonomy$source == "LPSN"] %like% "^(B|A)_") ||
|
||||
!all(taxonomy$domain[taxonomy$source == "MycoBank"] == "Fungi") ||
|
||||
!all(taxonomy$mo[taxonomy$source == "MycoBank"] %like% "^F_")) {
|
||||
warning("LPSN records outside prokaryotes or MycoBank records outside fungi, check 'Combine the datasets'", call. = FALSE)
|
||||
}
|
||||
|
||||
# these must never fail, as the package relies on them
|
||||
stopifnot(
|
||||
"MO codes must be unique" = !anyDuplicated(taxonomy$mo),
|
||||
"full names must be unique" = !anyDuplicated(taxonomy$fullname),
|
||||
"MO codes must not be empty" = !any(taxonomy$mo %in% c("", NA)),
|
||||
"prevalence must not be empty" = !anyNA(taxonomy$prevalence),
|
||||
"status must be accepted, synonym or unknown" = all(taxonomy$status %in% c("accepted", "synonym", "unknown")),
|
||||
"sources must be known" = all(taxonomy$source %in% c("LPSN", "MycoBank", "GBIF", "manually added"))
|
||||
# 'renamed to' identifiers of records that were removed after the earlier clean-up (e.g. by deduplication) lead
|
||||
# nowhere, so remove them again: these synonyms then have no current name, instead of a broken one
|
||||
n_dangling <- sum(
|
||||
(!is.na(taxonomy$lpsn_renamed_to) & !taxonomy$lpsn_renamed_to %in% taxonomy$lpsn) |
|
||||
(!is.na(taxonomy$mycobank_renamed_to) & !taxonomy$mycobank_renamed_to %in% taxonomy$mycobank) |
|
||||
(!is.na(taxonomy$gbif_renamed_to) & !taxonomy$gbif_renamed_to %in% taxonomy$gbif)
|
||||
)
|
||||
taxonomy <- taxonomy %>%
|
||||
mutate(
|
||||
lpsn_renamed_to = if_else(lpsn_renamed_to %in% lpsn, lpsn_renamed_to, NA_character_),
|
||||
mycobank_renamed_to = if_else(mycobank_renamed_to %in% mycobank, mycobank_renamed_to, NA_character_),
|
||||
gbif_renamed_to = if_else(gbif_renamed_to %in% gbif, gbif_renamed_to, NA_character_)
|
||||
)
|
||||
message("Removed dangling 'renamed to' identifiers of ", n_dangling, " records")
|
||||
|
||||
# synonyms without a current name are no error, but every one of them is a name that as.mo() cannot update
|
||||
review(mo_synonyms_without_current_name(taxonomy), "Synonyms without a current name")
|
||||
|
||||
# the integrity rules of the data set and the MO code registry (shared with the unit tests in
|
||||
# tests/testthat/helper-microorganisms.R): these must never fail, as the package and its users rely on them
|
||||
integrity_issues <- mo_integrity_issues(taxonomy, registry = mo_registry, renames = mo_renames)
|
||||
if (!is.null(mo_integrity_report(integrity_issues))) {
|
||||
stop("The new data set breaks integrity rules, fix the cause in this script:\n",
|
||||
mo_integrity_report(integrity_issues),
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
# Update other data sets --------------------------------------------------------------------------
|
||||
@@ -3676,11 +3727,13 @@ taxonomy <- taxonomy %>%
|
||||
AMR:::dataset_UTF8_to_ASCII()
|
||||
|
||||
# check again after the conversion to ASCII
|
||||
stopifnot(
|
||||
"full names must be unique after conversion to ASCII" = !anyDuplicated(taxonomy$fullname),
|
||||
"MO codes must be unique after conversion to ASCII" = !anyDuplicated(taxonomy$mo),
|
||||
"MO codes may only contain A-Z, 0-9, _, -, [ and ]" = all(taxonomy$mo %like_case% "^[][A-Z0-9_-]+$")
|
||||
)
|
||||
integrity_issues <- mo_integrity_issues(taxonomy, registry = mo_registry, renames = mo_renames)
|
||||
if (!is.null(mo_integrity_report(integrity_issues))) {
|
||||
stop("The data set breaks integrity rules after the conversion to ASCII:\n",
|
||||
mo_integrity_report(integrity_issues),
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
|
||||
microorganisms <- taxonomy
|
||||
|
||||
|
||||
@@ -65,20 +65,22 @@ These are needed for `TAXONOMY_VERSION` in `R/aa_globals.R` later. Write them do
|
||||
## Step 2: run the build
|
||||
|
||||
The script takes several hours (most of it is LPSN scraping, which is cached in
|
||||
`data-raw/lpsn_scrape_cache.rds`, so a restart is fast). Run it detached from your tool calls, so that
|
||||
it survives time limits, and write everything to a log:
|
||||
`data-raw/lpsn_scrape_cache.rds`, so a restart is fast). Never `source()` the whole script. Use the runner
|
||||
`data-raw/_reproduction_scripts/run_microorganisms_build.R`: it cuts the script into chunks at its
|
||||
checkpoints, runs them one by one in a single R process, logs each chunk with its duration in
|
||||
`<scratch>/logs/progress.tsv`, stops with the name of the failing chunk, and can resume from any checkpoint.
|
||||
See the chunks first with `--list`. Run it detached from your tool calls, so that it survives time limits:
|
||||
|
||||
```bash
|
||||
mkdir -p <scratch>/logs
|
||||
cd <repo root>
|
||||
nohup Rscript -e 'options(AMR_build_view = FALSE, AMR_build_print_rows = 200, AMR_build_review_dir = "<review dir>", warn = 1, width = 200); source("data-raw/_reproduction_scripts/reproduction_of_microorganisms.R", echo = TRUE, max.deparse.length = Inf, keep.source = TRUE)' > <scratch>/logs/build_1.log 2>&1 &
|
||||
nohup Rscript data-raw/_reproduction_scripts/run_microorganisms_build.R --review-dir=<review dir> --log-dir=<scratch>/logs > <scratch>/logs/build_1.log 2>&1 &
|
||||
```
|
||||
|
||||
Monitor the log at sensible intervals (every 10 to 20 minutes, not continuously). Read every
|
||||
`>>> REVIEW:` block as soon as it appears; you do not have to wait until the end to start thinking.
|
||||
With `AMR_build_review_dir` set, every review block is also saved in full as a CSV file in
|
||||
`<review dir>`, named after its title. These files are part of the PR. Set the same option in every
|
||||
resumed run.
|
||||
Monitor the log and `progress.tsv` at sensible intervals (every 10 to 20 minutes, not continuously). Read
|
||||
every `>>> REVIEW:` block as soon as it appears; you do not have to wait until the end to start thinking.
|
||||
Every review block is also saved in full as a CSV file in `<review dir>`, named after its title. These
|
||||
files are part of the PR.
|
||||
|
||||
### Intermediate results
|
||||
|
||||
@@ -105,24 +107,18 @@ and must not be committed.
|
||||
|
||||
### If the run fails
|
||||
|
||||
1. Read the error and the 100 lines before it. Find the root cause in the script, fix it, and
|
||||
check the syntax with `Rscript -e 'invisible(parse("<script>"))'`.
|
||||
2. Do not start from scratch if a checkpoint exists. Write a resume script in `<scratch>` (never in
|
||||
the repository) that contains, in this order:
|
||||
- the script from the top up to (not including) `# Read LPSN data ---`: setup, guard, helpers,
|
||||
clinically relevant genera;
|
||||
- `taxonomy_lpsn`, `taxonomy_mycobank` and `taxonomy_gbif` from their `.rds` files;
|
||||
- if resuming at or after `taxonomy0.rds`: the two lines that read the raw GBIF file
|
||||
(`taxonomy_gbif.bak <- vroom(...)` and the `colnames()` line after it) and the definition of
|
||||
`current_gbif` (needed by `add_missing_parents()`);
|
||||
- if resuming at or after `taxonomy2.rds`: the section `# Add prevalence ---` up to and including
|
||||
the definition of `compute_prevalence()` (needed later on);
|
||||
- `taxonomy <- readRDS("data-raw/<latest checkpoint>.rds")`;
|
||||
- the remainder of the script after that checkpoint's `saveRDS()` line.
|
||||
Number the logs (`build_2.log`, etc.).
|
||||
3. If the same section fails three times for different reasons, or the cause lies outside the
|
||||
script (e.g. the format of a source file changed fundamentally), stop and report (see "When to
|
||||
stop").
|
||||
1. Read the error in the log and the 100 lines before it, and `progress.tsv`. Find the root cause in
|
||||
the script, fix it, and check the syntax with `Rscript -e 'invisible(parse("<script>"))'`.
|
||||
2. Do not start from scratch if a checkpoint exists. The runner tells you where to resume; add
|
||||
`--resume-from=<checkpoint>` (e.g. `--resume-from=taxonomy1b`) and use the same `--review-dir` and
|
||||
`--log-dir`. It runs the setup chunk, loads the saved results and the parts of earlier chunks that
|
||||
later chunks need (the blocks marked with `# @resume-block` in the script), and continues after that
|
||||
checkpoint. Number the logs (`build_2.log`, etc.).
|
||||
3. If you add code to an earlier chunk that a later chunk needs (a function or an object), either move it
|
||||
to the setup part at the top of the script, or mark it as a `# @resume-block` with the right `after=`,
|
||||
otherwise resuming breaks. Check with `--list`.
|
||||
4. If the same chunk fails three times for different reasons, or the cause lies outside the script (e.g.
|
||||
the format of a source file changed fundamentally), stop and report (see "When to stop").
|
||||
|
||||
## Step 3: review as an expert
|
||||
|
||||
@@ -207,7 +203,19 @@ change of more than 5% in a domain needs an explanation. Also report:
|
||||
- the 30 most relevant (prevalence <= 1.25) new genera and species, and the 30 most relevant removed
|
||||
ones.
|
||||
|
||||
### 3d. Unit tests
|
||||
### 3d. Integrity rules and MO code registry
|
||||
|
||||
The script stops before saving if any rule in `tests/testthat/helper-microorganisms.R` is broken, including
|
||||
the MO code registry rules (see `data-raw/microorganisms_files/README.md`): a code from any release since
|
||||
v2.0.0 may never be given to another taxon. Never weaken a rule to get past this; fix the cause. A registered
|
||||
code may only get another name (e.g. a corrected spelling) if it is listed in
|
||||
`data-raw/microorganisms_files/mo_code_renames.csv`, and you may only add such rows with an empty
|
||||
`approved_by`: the build will then stop on it until the human approves it, so list every proposed rename under
|
||||
"Needs a human". After the build, `tests/testthat/microorganisms_known_defects.csv` (the known defects of the
|
||||
data set before this rebuild) must be emptied: delete the rows that are solved, and the file itself if none
|
||||
are left. Any remaining row must be explained under "Needs a human".
|
||||
|
||||
### 3e. Unit tests
|
||||
|
||||
After the data are saved and the package is reloaded, run the tests at the end of the script and then
|
||||
the full suite with `devtools::test()`. Analyse every failure: is the test outdated because of a
|
||||
@@ -242,7 +250,7 @@ human can review it before merging and next year's run can learn from it. Struct
|
||||
Every item states the check, the finding in numbers, your decision, and a link to the CSV file in
|
||||
`<review dir>` where relevant. There is one item for each review block of 3a, one for each sentinel
|
||||
organism of 3b that deviates (and one item for all sentinels that passed), one for each comparison
|
||||
of 3c, one for the unit tests (3d) and one for each changed test. Leave all boxes unticked: ticking
|
||||
of 3c, one for the integrity rules and registry (3d), one for the unit tests (3e) and one for each changed test. Leave all boxes unticked: ticking
|
||||
them is the human's job, it means "reviewed and agreed".
|
||||
5. **Needs a human**: also as checkboxes, everything you could not verify or decide, each with your
|
||||
recommendation. Be concrete, e.g. "COL lists *Enterobius vermicularis* as a synonym of ...;
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
# ==================================================================== #
|
||||
# TITLE: #
|
||||
# AMR: An R Package for Working with Antimicrobial Resistance Data #
|
||||
# #
|
||||
# SOURCE CODE: #
|
||||
# https://github.com/msberends/AMR #
|
||||
# #
|
||||
# PLEASE CITE THIS SOFTWARE AS: #
|
||||
# Berends MS, Luz CF, Friedrich AW, et al. (2022). #
|
||||
# AMR: An R Package for Working with Antimicrobial Resistance Data. #
|
||||
# Journal of Statistical Software, 104(3), 1-31. #
|
||||
# https://doi.org/10.18637/jss.v104.i03 #
|
||||
# #
|
||||
# Developed at the University of Groningen and the University Medical #
|
||||
# Center Groningen in the Netherlands, in collaboration with many #
|
||||
# colleagues from around the world, see our website. #
|
||||
# #
|
||||
# This R package is free software; you can freely use and distribute #
|
||||
# it for both personal and commercial purposes under the terms of the #
|
||||
# GNU General Public License version 2.0 (GNU GPL-2), as published by #
|
||||
# the Free Software Foundation. #
|
||||
# We created this package for both routine data analysis and academic #
|
||||
# research and it was publicly released in the hope that it will be #
|
||||
# useful, but it comes WITHOUT ANY WARRANTY OR LIABILITY. #
|
||||
# #
|
||||
# Visit our website for the full manual and a complete tutorial about #
|
||||
# how to conduct AMR data analysis: https://amr-for-r.org #
|
||||
# ==================================================================== #
|
||||
|
||||
# Runs reproduction_of_microorganisms.R non-interactively, in chunks, in one R process.
|
||||
#
|
||||
# The script itself stays one file (to be run line by line by a human as well). This runner cuts it into chunks
|
||||
# at its checkpoints (the saveRDS() lines of data-raw/taxonomy*.rds), runs these one by one, logs the start,
|
||||
# end and duration of each chunk in a progress file, and stops with the name of the failing chunk.
|
||||
# It can resume from any checkpoint: it then runs the setup chunk, loads the saved results and the parts of
|
||||
# earlier chunks that later chunks need (marked with '# @resume-block' in the script), and continues.
|
||||
#
|
||||
# Usage, from the root of the repository:
|
||||
# Rscript data-raw/_reproduction_scripts/run_microorganisms_build.R [options]
|
||||
# Options:
|
||||
# --review-dir=DIR folder for the CSV files of all reviews (git-tracked, for the human review)
|
||||
# --log-dir=DIR folder for the progress file (default: a temporary folder)
|
||||
# --resume-from=NAME continue after this checkpoint, e.g. --resume-from=taxonomy1b
|
||||
# --list only show the chunks and checkpoints, and stop
|
||||
|
||||
script_file <- "data-raw/_reproduction_scripts/reproduction_of_microorganisms.R"
|
||||
if (!file.exists(script_file)) {
|
||||
stop("Run this from the root of the AMR repository", call. = FALSE)
|
||||
}
|
||||
|
||||
# arguments
|
||||
args <- commandArgs(trailingOnly = TRUE)
|
||||
get_arg <- function(name, default = NULL) {
|
||||
hit <- args[startsWith(args, paste0("--", name, "="))]
|
||||
if (length(hit) == 0) {
|
||||
return(default)
|
||||
}
|
||||
sub("^[^=]+=", "", hit[1])
|
||||
}
|
||||
review_dir <- get_arg("review-dir")
|
||||
log_dir <- get_arg("log-dir", tempfile("microorganisms_build_"))
|
||||
resume_from <- get_arg("resume-from")
|
||||
dir.create(log_dir, recursive = TRUE, showWarnings = FALSE)
|
||||
progress_file <- file.path(log_dir, "progress.tsv")
|
||||
|
||||
# cut the script into chunks ----
|
||||
|
||||
lines <- readLines(script_file, encoding = "UTF-8")
|
||||
# the checkpoints in their order; taxonomy_lpsn_missing.rds is no checkpoint to resume from, as the rest of the
|
||||
# LPSN chunk is needed anyway (and is fast thanks to the LPSN cache)
|
||||
checkpoints <- c(
|
||||
"taxonomy_lpsn", "taxonomy_mycobank", "taxonomy_gbif",
|
||||
"taxonomy0", "taxonomy1", "taxonomy1b", "taxonomy1c",
|
||||
"taxonomy2", "taxonomy2b", "taxonomy2c", "taxonomy3", "taxonomy3b"
|
||||
)
|
||||
checkpoint_line <- vapply(checkpoints, function(cp) {
|
||||
hit <- which(startsWith(lines, "saveRDS(") & grepl(paste0("\"data-raw/", cp, ".rds\""), lines, fixed = TRUE))
|
||||
if (length(hit) != 1) {
|
||||
stop("Checkpoint ", cp, " must be saved exactly once in ", script_file, " (found ", length(hit), ")", call. = FALSE)
|
||||
}
|
||||
hit
|
||||
}, integer(1))
|
||||
if (is.unsorted(checkpoint_line)) {
|
||||
stop("The checkpoints in the script are not in the expected order", call. = FALSE)
|
||||
}
|
||||
setup_end <- which(startsWith(lines, "# Read LPSN data")) - 1
|
||||
stopifnot(length(setup_end) == 1)
|
||||
chunks <- data.frame(
|
||||
name = c("setup", checkpoints, "save_to_package"),
|
||||
from = c(1L, setup_end + 1L, checkpoint_line + 1L),
|
||||
to = c(setup_end, checkpoint_line, length(lines)),
|
||||
stringsAsFactors = FALSE
|
||||
)
|
||||
|
||||
# the blocks that are needed again when resuming
|
||||
block_start <- which(startsWith(lines, "# @resume-block "))
|
||||
block_end <- which(lines == "# @end-resume-block")
|
||||
stopifnot(length(block_start) == length(block_end), all(block_start < block_end))
|
||||
blocks <- data.frame(
|
||||
name = sub("^# @resume-block ([^ ]+) .*", "\\1", lines[block_start]),
|
||||
after = sub(".* after=([^ ]+).*", "\\1", lines[block_start]),
|
||||
from = block_start,
|
||||
to = block_end,
|
||||
stringsAsFactors = FALSE
|
||||
)
|
||||
stopifnot(all(blocks$after %in% checkpoints))
|
||||
|
||||
message("Chunks:")
|
||||
for (i in seq_len(nrow(chunks))) {
|
||||
message(sprintf(" %-17s lines %5d-%5d", chunks$name[i], chunks$from[i], chunks$to[i]))
|
||||
}
|
||||
message("Blocks needed when resuming:")
|
||||
for (i in seq_len(nrow(blocks))) {
|
||||
message(sprintf(" %-17s lines %5d-%5d, when resuming from %s or later", blocks$name[i], blocks$from[i], blocks$to[i], blocks$after[i]))
|
||||
}
|
||||
if ("--list" %in% args) {
|
||||
quit(save = "no", status = 0)
|
||||
}
|
||||
|
||||
# run ----
|
||||
|
||||
options(
|
||||
AMR_build_view = FALSE,
|
||||
AMR_build_print_rows = 200,
|
||||
warn = 1,
|
||||
width = 200
|
||||
)
|
||||
if (!is.null(review_dir)) {
|
||||
options(AMR_build_review_dir = review_dir)
|
||||
}
|
||||
if (!file.exists(progress_file)) {
|
||||
writeLines("chunk\tstarted\tfinished\tminutes\tstatus", progress_file)
|
||||
}
|
||||
log_progress <- function(chunk, started, status) {
|
||||
cat(
|
||||
paste(chunk, format(started, "%Y-%m-%d %H:%M:%S"), format(Sys.time(), "%Y-%m-%d %H:%M:%S"),
|
||||
round(as.numeric(difftime(Sys.time(), started, units = "mins")), 1), status,
|
||||
sep = "\t"
|
||||
), "\n",
|
||||
file = progress_file, append = TRUE, sep = ""
|
||||
)
|
||||
}
|
||||
run_lines <- function(name, from, to) {
|
||||
message("\n", strrep("=", 100), "\n=== CHUNK ", name, " (lines ", from, "-", to, ") started at ", format(Sys.time()), "\n", strrep("=", 100))
|
||||
started <- Sys.time()
|
||||
code <- paste(lines[from:to], collapse = "\n")
|
||||
ok <- tryCatch(
|
||||
{
|
||||
source(exprs = parse(text = code, keep.source = FALSE), local = globalenv(), echo = TRUE, max.deparse.length = 300)
|
||||
TRUE
|
||||
},
|
||||
error = function(e) {
|
||||
message("\n!!! CHUNK ", name, " FAILED: ", conditionMessage(e))
|
||||
log_progress(name, started, paste("FAILED:", gsub("[\t\n]", " ", conditionMessage(e))))
|
||||
FALSE
|
||||
}
|
||||
)
|
||||
if (!ok) {
|
||||
last_done <- which(chunks$name == name) - 2
|
||||
if (length(last_done) == 1 && last_done >= 1) {
|
||||
message("\nAfter fixing the cause, resume with: --resume-from=", checkpoints[last_done])
|
||||
} else {
|
||||
message("\nAfter fixing the cause, run again from the start")
|
||||
}
|
||||
quit(save = "no", status = 1)
|
||||
}
|
||||
log_progress(name, started, "OK")
|
||||
message("=== CHUNK ", name, " finished in ", round(as.numeric(difftime(Sys.time(), started, units = "mins")), 1), " minutes")
|
||||
}
|
||||
|
||||
run_lines("setup", chunks$from[1], chunks$to[1])
|
||||
|
||||
if (is.null(resume_from)) {
|
||||
first_chunk <- 2
|
||||
} else {
|
||||
if (!resume_from %in% checkpoints) {
|
||||
stop("--resume-from must be one of: ", paste(checkpoints, collapse = ", "), call. = FALSE)
|
||||
}
|
||||
k <- which(checkpoints == resume_from)
|
||||
message("\nResuming after checkpoint ", resume_from)
|
||||
started <- Sys.time()
|
||||
# the source taxonomies that were saved up to this checkpoint
|
||||
for (src in intersect(checkpoints[seq_len(k)], c("taxonomy_lpsn", "taxonomy_mycobank", "taxonomy_gbif"))) {
|
||||
assign(src, readRDS(file.path("data-raw", paste0(src, ".rds"))), envir = globalenv())
|
||||
}
|
||||
# the parts of earlier chunks that later chunks need
|
||||
for (i in which(match(blocks$after, checkpoints) <= k)) {
|
||||
run_lines(paste0("resume block ", blocks$name[i]), blocks$from[i], blocks$to[i])
|
||||
}
|
||||
if (!resume_from %in% c("taxonomy_lpsn", "taxonomy_mycobank", "taxonomy_gbif")) {
|
||||
assign("taxonomy", readRDS(file.path("data-raw", paste0(resume_from, ".rds"))), envir = globalenv())
|
||||
}
|
||||
log_progress(paste("resume from", resume_from), started, "OK")
|
||||
first_chunk <- which(chunks$name == resume_from) + 1
|
||||
}
|
||||
|
||||
for (i in seq(first_chunk, nrow(chunks))) {
|
||||
run_lines(chunks$name[i], chunks$from[i], chunks$to[i])
|
||||
}
|
||||
message("\nAll chunks finished. Progress file: ", progress_file)
|
||||
Reference in new issue
Block a user