mirror of
https://github.com/msberends/AMR.git
synced 2026-11-16 02:20:49 +01:00
(v3.0.1.9096) MO code registry: codes never change meaning, are never guessed, and are validated
- data-raw/microorganisms_files/: registry of all MO codes of every release since v2.0.0 (78,992 codes, built from the git tags), approved renames, and README with the rules; decision by Matthijs S. Berends: MO codes from before v2.0.0 are not supported - as.mo(): codes of earlier releases are translated deterministically via the internal MO_RETIRED_CODES (generated from the registry); unknown codes are no longer guessed (previously F_TRCHP, Trichophyton in v3.0.1, became Talaromyces chlamydosporus) - synonym_mo_to_accepted_mo(): a missing target of one source no longer overwrites a valid target of another source (e.g. Chaetomium abuense), and no 'renamed to itself' - taxonomy build: codes are reused and reserved from the registry instead of the development data (fixes 40 codes reused in the 2026 development data), also for phylum to family; synonym cycles are broken; dangling 'renamed to' identifiers are removed; the build stops on any broken integrity rule before saving - run_microorganisms_build.R: runs the build in chunks at its checkpoints in one R process, with progress logging and resuming from any checkpoint; prompt updated accordingly - tests: shared integrity rules (helper-microorganisms.R) used by build and testthat, registry tests over all codes, sentinel organisms, regression tests; current defects of the data on main are listed in microorganisms_known_defects.csv until the next rebuild Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ca5fa333ca
commit
d683b9e7c6
16 files changed
+80516
-246
No files matched your search
@@ -624,6 +624,28 @@ usethis::use_data(antimicrobials, overwrite = TRUE, version = 2, compress = "xz"
|
||||
pre_commit_lst$AB_LOOKUP <- create_AB_AV_lookup(antimicrobials)
|
||||
pre_commit_lst$AV_LOOKUP <- create_AB_AV_lookup(antivirals)
|
||||
|
||||
# MO codes of earlier releases that are not in the current data set anymore, with the current code of the same
|
||||
# taxon (see data-raw/microorganisms_files/README.md). as.mo() uses this to translate them, instead of guessing.
|
||||
source("tests/testthat/helper-microorganisms.R")
|
||||
mo_registry <- read_mo_registry(".")
|
||||
mo_renames <- read_mo_renames(".")
|
||||
retired <- mo_registry[!mo_registry$mo %in% as.character(microorganisms$mo), c("mo", "fullname", "rank"), drop = FALSE]
|
||||
renamed <- match(retired$mo, mo_renames$mo)
|
||||
retired$fullname[!is.na(renamed)] <- mo_renames$new_name[renamed[!is.na(renamed)]]
|
||||
# the same name, or else the same name without a rank suffix (such as "Kapabacteria {class}") and the same rank
|
||||
current_mo <- as.character(microorganisms$mo)[match(retired$fullname, microorganisms$fullname)]
|
||||
by_rank <- as.character(microorganisms$mo)[match(
|
||||
paste(mo_name_without_suffix(retired$fullname), retired$rank),
|
||||
paste(mo_name_without_suffix(microorganisms$fullname), microorganisms$rank)
|
||||
)]
|
||||
pre_commit_lst$MO_RETIRED_CODES <- data.frame(
|
||||
old_mo = retired$mo,
|
||||
mo = ifelse(is.na(current_mo), by_rank, current_mo),
|
||||
fullname = retired$fullname,
|
||||
stringsAsFactors = FALSE
|
||||
)
|
||||
rm(mo_registry, mo_renames, retired, renamed, current_mo, by_rank)
|
||||
|
||||
# Export to package as internal data ----
|
||||
# usethis::use_data() must receive unquoted object names, which is not flexible at all.
|
||||
# we'll use good old base::save() instead
|
||||
|
||||
@@ -115,6 +115,9 @@ devtools::load_all(".") # to load the AMR package
|
||||
# keep the previous data set at hand, `AMR::microorganisms` will be replaced at the end of this script
|
||||
microorganisms_old <- AMR::microorganisms
|
||||
|
||||
# the integrity rules of the data set and the MO code registry, shared with the unit tests
|
||||
source("tests/testthat/helper-microorganisms.R")
|
||||
|
||||
|
||||
# Helper functions --------------------------------------------------------------------------------
|
||||
|
||||
@@ -143,6 +146,11 @@ review <- function(x, title = deparse(substitute(x))) {
|
||||
invisible(x)
|
||||
}
|
||||
|
||||
# taxonomic names in ASCII and without quotes
|
||||
ascii_names <- function(x) {
|
||||
gsub("[\"'`]", "", stringi::stri_trans_general(x, "Latin-ASCII"))
|
||||
}
|
||||
|
||||
# the priority of sources, used everywhere in this script
|
||||
source_priority <- c("LPSN" = 1L, "MycoBank" = 2L, "GBIF" = 3L, "manually added" = 4L, "inferred" = 5L)
|
||||
source_prio <- function(source) {
|
||||
@@ -1052,8 +1060,11 @@ saveRDS(taxonomy_mycobank, "data-raw/taxonomy_mycobank.rds", version = 2)
|
||||
|
||||
# Read GBIF data ----------------------------------------------------------------------------------
|
||||
|
||||
# @resume-block gbif_raw after=taxonomy_gbif
|
||||
# (needed by `current_gbif` when resuming from a checkpoint with run_microorganisms_build.R)
|
||||
taxonomy_gbif.bak <- vroom(file_gbif, guess_max = 5e5)
|
||||
colnames(taxonomy_gbif.bak) <- gsub(".*:(.*)", "\\1", colnames(taxonomy_gbif.bak))
|
||||
# @end-resume-block
|
||||
|
||||
# include all fungal orders from the mycobank db
|
||||
include_fungal_orders <- unique(taxonomy_mycobank$order[
|
||||
@@ -1282,10 +1293,7 @@ taxonomy_gbif <- readRDS("data-raw/taxonomy_gbif.rds")
|
||||
# Add full names ----------------------------------------------------------------------------------
|
||||
|
||||
# names must be ASCII before they are made unique, otherwise e.g. 'Fusarium aloës' and 'Fusarium aloes'
|
||||
# both end up as 'Fusarium aloes' in the final data set (which happened to 12 names in 2026)
|
||||
ascii_names <- function(x) {
|
||||
gsub("[\"'`]", "", stringi::stri_trans_general(x, "Latin-ASCII"))
|
||||
}
|
||||
# both end up as 'Fusarium aloes' in the final data set (which happened to 12 names in 2026), see ascii_names()
|
||||
taxonomy_gbif <- taxonomy_gbif %>%
|
||||
mutate(across(domain:subspecies, ascii_names))
|
||||
taxonomy_lpsn <- taxonomy_lpsn %>%
|
||||
@@ -1639,6 +1647,8 @@ saveRDS(taxonomy, "data-raw/taxonomy0.rds")
|
||||
# Ensure every referenced rank has its own row (domain through species).
|
||||
# Where possible, enrich with GBIF identifiers.
|
||||
|
||||
# @resume-block current_gbif after=taxonomy1
|
||||
# (needed by add_missing_parents() after filtering, when resuming from a checkpoint)
|
||||
current_gbif <- taxonomy_gbif.bak %>%
|
||||
filter(is.na(acceptedNameUsageID)) %>%
|
||||
mutate(
|
||||
@@ -1663,6 +1673,7 @@ current_gbif <- taxonomy_gbif.bak %>%
|
||||
phylum = if_else(taxonRank == "phylum", canonical, phylum)
|
||||
) %>%
|
||||
select(-canonical)
|
||||
# @end-resume-block
|
||||
|
||||
taxonomy <- add_missing_parents(taxonomy, current_gbif)
|
||||
|
||||
@@ -2035,6 +2046,8 @@ taxonomy$rank[is.na(taxonomy$rank)] <- "domain"
|
||||
# this part is required here, because it's needed for filtering on 'relevant' species to keep later on
|
||||
# (`pathogens` was read at the start of this script)
|
||||
|
||||
# @resume-block prevalence after=taxonomy2
|
||||
# (compute_prevalence() is used again later on, when resuming from a checkpoint)
|
||||
# get all established, both old and current taxonomic names
|
||||
established <- pathogens %>%
|
||||
filter(status == "established") %>%
|
||||
@@ -2147,6 +2160,7 @@ compute_prevalence <- function(taxonomy) {
|
||||
mutate(prevalence = if_else(rank == "genus", pmin(prevalence, gen_prevalence, na.rm = TRUE), prevalence)) %>%
|
||||
select(-gen_prevalence)
|
||||
}
|
||||
# @end-resume-block
|
||||
taxonomy <- compute_prevalence(taxonomy)
|
||||
|
||||
table(taxonomy$prevalence, useNA = "always")
|
||||
@@ -2513,6 +2527,77 @@ taxonomy <- taxonomy %>%
|
||||
distinct(fullname, .keep_all = TRUE)
|
||||
message("Removed ", n_before - nrow(taxonomy), " nonsense names")
|
||||
|
||||
# Break cycles of synonyms ------------------------------------------------------------------------
|
||||
|
||||
# Sources can contradict each other, e.g. in 2026 GBIF listed Capillidium denaeosporus as synonym of
|
||||
# Conidiobolus denaeosporum and also the other way around, so that the 'current name' depended on the starting
|
||||
# point. In a cycle, the statement of the most authoritative source counts: that record stays a synonym, and its
|
||||
# target becomes accepted.
|
||||
next_synonym_step <- function(df) {
|
||||
by_gbif <- df$fullname[match(df$gbif_renamed_to, df$gbif, incomparables = NA)]
|
||||
by_mycobank <- df$fullname[match(df$mycobank_renamed_to, df$mycobank, incomparables = NA)]
|
||||
by_lpsn <- df$fullname[match(df$lpsn_renamed_to, df$lpsn, incomparables = NA)]
|
||||
out <- coalesce(by_lpsn, by_mycobank, by_gbif)
|
||||
out[df$status != "synonym"] <- NA_character_
|
||||
setNames(out, df$fullname)
|
||||
}
|
||||
synonym_next <- next_synonym_step(taxonomy)
|
||||
synonym_cycles <- list()
|
||||
for (start in names(synonym_next)[!is.na(synonym_next)]) {
|
||||
path <- start
|
||||
repeat {
|
||||
nxt <- unname(synonym_next[path[length(path)]])
|
||||
if (is.na(nxt)) {
|
||||
break
|
||||
}
|
||||
if (nxt %in% path) {
|
||||
cycle <- sort(path[match(nxt, path):length(path)])
|
||||
synonym_cycles[[paste(cycle, collapse = " | ")]] <- cycle
|
||||
break
|
||||
}
|
||||
path <- c(path, nxt)
|
||||
if (length(path) > 25) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
# If multiple records in the cycle come from the most authoritative source, the sources contradict themselves and
|
||||
# the direction cannot be determined automatically (in 2026, even the authors in `ref` of these GBIF records were
|
||||
# unreliable). Then ALL records in the cycle become accepted, so that no wrong current name is given, and they are
|
||||
# shown for a human decision.
|
||||
cycle_fixes <- bind_rows(lapply(synonym_cycles, function(cycle_members) {
|
||||
# (not named `cycle`, since tibble() below would then use its own new column `cycle`)
|
||||
members <- taxonomy[match(cycle_members, taxonomy$fullname), , drop = FALSE]
|
||||
best <- members[source_prio(members$source) == min(source_prio(members$source)), , drop = FALSE]
|
||||
if (nrow(best) == 1) {
|
||||
tibble(
|
||||
cycle = paste(cycle_members, collapse = " | "),
|
||||
made_accepted = unname(synonym_next[best$fullname]),
|
||||
reason = paste("statement of", best$fullname, "from", best$source, "(most authoritative source)")
|
||||
)
|
||||
} else {
|
||||
tibble(
|
||||
cycle = paste(cycle_members, collapse = " | "),
|
||||
made_accepted = cycle_members,
|
||||
reason = paste("contradicting statements of", toString(unique(best$source)), "- all accepted, NEEDS A HUMAN DECISION")
|
||||
)
|
||||
}
|
||||
}))
|
||||
review(cycle_fixes, "Cycles of synonyms")
|
||||
if (nrow(cycle_fixes) > 0) {
|
||||
taxonomy <- taxonomy %>%
|
||||
mutate(
|
||||
fix = fullname %in% cycle_fixes$made_accepted,
|
||||
status = if_else(fix, "accepted", status),
|
||||
lpsn_renamed_to = if_else(fix, NA_character_, lpsn_renamed_to),
|
||||
mycobank_renamed_to = if_else(fix, NA_character_, mycobank_renamed_to),
|
||||
gbif_renamed_to = if_else(fix, NA_character_, gbif_renamed_to)
|
||||
) %>%
|
||||
select(-fix)
|
||||
}
|
||||
rm(synonym_next, synonym_cycles, cycle_fixes)
|
||||
|
||||
|
||||
# Fix genera that are synonyms while they contain accepted species --------------------------------
|
||||
|
||||
# e.g. in 2026, MycoBank listed the genus Blastomyces as a synonym, while Blastomyces dermatitidis was accepted
|
||||
@@ -2560,151 +2645,59 @@ mo_domain <- taxonomy %>%
|
||||
)
|
||||
mo_domain
|
||||
|
||||
# `domain` exists in the data set since 2026, so codes are kept per domain
|
||||
# (in the 2026 build, this was `AMR::microorganisms %>% rename(domain = kingdom)`, which would now fail)
|
||||
existing_mo_tbl <- microorganisms_old %>%
|
||||
mutate(mo = as.character(mo))
|
||||
|
||||
# @resume-block mo_registry after=taxonomy2c
|
||||
# (the MO code registry is used by the final checks, when resuming from a checkpoint)
|
||||
# parts of existing MO codes, split on the underscore (genus codes can contain digits or hyphens, such as
|
||||
# B_RBCLM1 or B_CND-P, so a regex with only [A-Z] does not work and caused repeated elements like
|
||||
# B_SCLLM_CNNM_LNSM_LNSM_LNSM in earlier versions)
|
||||
mo_part <- function(mo, i) {
|
||||
vapply(strsplit(as.character(mo), "_", fixed = TRUE), function(x) if (length(x) >= i) x[i] else NA_character_, character(1))
|
||||
}
|
||||
# parts of a name, e.g. the species of 'Escherichia coli'
|
||||
mo_part_of_name <- function(x, i) {
|
||||
vapply(strsplit(x, " ", fixed = TRUE), function(x) if (length(x) >= i) x[i] else "", character(1))
|
||||
}
|
||||
|
||||
# Existing codes come from the MO code registry (data-raw/microorganisms_files/), which contains every code of
|
||||
# every release since v2.0.0, and NOT from the previous data set: codes that only existed in a development
|
||||
# version carry no weight (decision by Matthijs S. Berends, 3 October 2026). A registered code is never given
|
||||
# to another taxon, also not when its taxon is not in the new data set anymore, since users store these codes.
|
||||
mo_registry <- read_mo_registry(".")
|
||||
mo_renames <- read_mo_renames(".")
|
||||
if (is.null(mo_registry)) {
|
||||
stop("MO code registry not found, see data-raw/microorganisms_files/README.md", call. = FALSE)
|
||||
}
|
||||
release_order <- function(release) {
|
||||
as.integer(factor(release, levels = unique(release[order(numeric_version(sub("^v", "", release)))])))
|
||||
}
|
||||
existing_mo_tbl <- mo_registry %>%
|
||||
# approved renames, e.g. a corrected spelling: the code continues with the new name
|
||||
left_join(mo_renames %>% select(mo, new_name), by = "mo") %>%
|
||||
mutate(
|
||||
fullname = coalesce(new_name, fullname),
|
||||
species = if_else(!is.na(new_name) & rank %in% c("species", "subspecies"), mo_part_of_name(new_name, 2), species),
|
||||
subspecies = if_else(!is.na(new_name) & rank == "subspecies", mo_part_of_name(new_name, 3), subspecies)
|
||||
) %>%
|
||||
select(-new_name) %>%
|
||||
# if a taxon had multiple codes over time, the code of the most recent release is used
|
||||
arrange(desc(release_order(last_release)), desc(release_order(first_release))) %>%
|
||||
distinct(domain, rank, fullname, .keep_all = TRUE)
|
||||
|
||||
# all codes that have ever been used must never be given to another taxon, also not when their taxon
|
||||
# is not in the new data set anymore (users may have stored old codes)
|
||||
reserved_genus_codes <- existing_mo_tbl %>%
|
||||
reserved_genus_codes <- mo_registry %>%
|
||||
filter(rank %in% c("genus", "species", "subspecies")) %>%
|
||||
transmute(domain, code = mo_part(mo, 2)) %>%
|
||||
distinct()
|
||||
reserved_species_codes <- existing_mo_tbl %>%
|
||||
reserved_species_codes <- mo_registry %>%
|
||||
filter(rank %in% c("species", "subspecies")) %>%
|
||||
transmute(domain, genus_code = mo_part(mo, 2), code = mo_part(mo, 3)) %>%
|
||||
distinct()
|
||||
# @end-resume-block
|
||||
|
||||
|
||||
# phylum until family are abbreviated with 8 characters and prefixed with their rank
|
||||
|
||||
# Phylum - keep old and fill up for new ones
|
||||
mo_phylum <- taxonomy %>%
|
||||
filter(rank == "phylum") %>%
|
||||
distinct(domain, phylum) %>%
|
||||
left_join(
|
||||
existing_mo_tbl %>%
|
||||
filter(rank == "phylum") %>%
|
||||
transmute(
|
||||
domain,
|
||||
phylum = fullname,
|
||||
mo_old = gsub("^[A-Z]{1,2}_", "", as.character(mo))
|
||||
),
|
||||
by = c("domain", "phylum")
|
||||
) %>%
|
||||
group_by(domain) %>%
|
||||
mutate(
|
||||
mo_phylum8 = AMR:::abbreviate_mo(phylum, minlength = 8, prefix = "[PHL]_"),
|
||||
mo_phylum9 = AMR:::abbreviate_mo(phylum, minlength = 9, prefix = "[PHL]_"),
|
||||
mo_phylum = if_else(!is.na(mo_old), mo_old, mo_phylum8),
|
||||
mo_duplicated = duplicated(mo_phylum),
|
||||
mo_phylum = if_else(mo_duplicated, mo_phylum9, mo_phylum),
|
||||
mo_duplicated = duplicated(mo_phylum)
|
||||
) %>%
|
||||
ungroup()
|
||||
if (any(mo_phylum$mo_duplicated, na.rm = TRUE)) {
|
||||
stop("Duplicate MO codes for phylum!")
|
||||
}
|
||||
mo_phylum <- mo_phylum %>%
|
||||
select(domain, phylum, mo_phylum)
|
||||
|
||||
# Class - keep old and fill up for new ones
|
||||
mo_class <- taxonomy %>%
|
||||
filter(rank == "class") %>%
|
||||
distinct(domain, class) %>%
|
||||
left_join(
|
||||
existing_mo_tbl %>%
|
||||
filter(rank == "class") %>%
|
||||
transmute(
|
||||
domain,
|
||||
class = fullname,
|
||||
mo_old = gsub("^[A-Z]{1,2}_", "", as.character(mo))
|
||||
),
|
||||
by = c("domain", "class")
|
||||
) %>%
|
||||
group_by(domain) %>%
|
||||
mutate(
|
||||
mo_class8 = AMR:::abbreviate_mo(class, minlength = 8, prefix = "[CLS]_"),
|
||||
mo_class9 = AMR:::abbreviate_mo(class, minlength = 9, prefix = "[CLS]_"),
|
||||
mo_class = if_else(!is.na(mo_old), mo_old, mo_class8),
|
||||
mo_duplicated = duplicated(mo_class),
|
||||
mo_class = if_else(mo_duplicated, mo_class9, mo_class),
|
||||
mo_duplicated = duplicated(mo_class)
|
||||
) %>%
|
||||
ungroup()
|
||||
if (any(mo_class$mo_duplicated, na.rm = TRUE)) {
|
||||
stop("Duplicate MO codes for class!")
|
||||
}
|
||||
mo_class <- mo_class %>%
|
||||
select(domain, class, mo_class)
|
||||
|
||||
# Order - keep old and fill up for new ones
|
||||
mo_order <- taxonomy %>%
|
||||
filter(rank == "order") %>%
|
||||
distinct(domain, order) %>%
|
||||
left_join(
|
||||
existing_mo_tbl %>%
|
||||
filter(rank == "order") %>%
|
||||
transmute(
|
||||
domain,
|
||||
order = fullname,
|
||||
mo_old = gsub("^[A-Z]{1,2}_", "", as.character(mo))
|
||||
),
|
||||
by = c("domain", "order")
|
||||
) %>%
|
||||
group_by(domain) %>%
|
||||
mutate(
|
||||
mo_order8 = AMR:::abbreviate_mo(order, minlength = 8, prefix = "[ORD]_"),
|
||||
mo_order9 = AMR:::abbreviate_mo(order, minlength = 9, prefix = "[ORD]_"),
|
||||
mo_order = if_else(!is.na(mo_old), mo_old, mo_order8),
|
||||
mo_duplicated = duplicated(mo_order),
|
||||
mo_order = if_else(mo_duplicated, mo_order9, mo_order),
|
||||
mo_duplicated = duplicated(mo_order)
|
||||
) %>%
|
||||
ungroup()
|
||||
if (any(mo_order$mo_duplicated, na.rm = TRUE)) {
|
||||
stop("Duplicate MO codes for order!")
|
||||
}
|
||||
mo_order <- mo_order %>%
|
||||
select(domain, order, mo_order)
|
||||
|
||||
# Family - keep old and fill up for new ones
|
||||
mo_family <- taxonomy %>%
|
||||
filter(rank == "family") %>%
|
||||
distinct(domain, family) %>%
|
||||
left_join(
|
||||
existing_mo_tbl %>%
|
||||
filter(rank == "family") %>%
|
||||
transmute(
|
||||
domain,
|
||||
family = fullname,
|
||||
mo_old = gsub("^[A-Z]{1,2}_", "", as.character(mo))
|
||||
),
|
||||
by = c("domain", "family")
|
||||
) %>%
|
||||
group_by(domain) %>%
|
||||
mutate(
|
||||
mo_family8 = AMR:::abbreviate_mo(family, minlength = 8, prefix = "[FAM]_"),
|
||||
mo_family9 = AMR:::abbreviate_mo(family, minlength = 9, prefix = "[FAM]_"),
|
||||
mo_family = if_else(!is.na(mo_old), mo_old, mo_family8),
|
||||
mo_duplicated = duplicated(mo_family),
|
||||
mo_family = if_else(mo_duplicated, mo_family9, mo_family),
|
||||
mo_duplicated = duplicated(mo_family)
|
||||
) %>%
|
||||
ungroup()
|
||||
if (any(mo_family$mo_duplicated, na.rm = TRUE)) {
|
||||
stop("Duplicate MO codes for family!")
|
||||
}
|
||||
mo_family <- mo_family %>%
|
||||
select(domain, family, mo_family)
|
||||
|
||||
# keep the old code where available, and give every new taxon the first candidate code that is not in use
|
||||
# and was never used before (`reserved`) - if all candidates are taken, a number is added to the first one
|
||||
assign_codes <- function(old, candidates, reserved = character(0)) {
|
||||
@@ -2726,6 +2719,47 @@ assign_codes <- function(old, candidates, reserved = character(0)) {
|
||||
out
|
||||
}
|
||||
|
||||
# Phylum until family - keep old (from the registry) and fill up for new ones, never with a registered code
|
||||
higher_rank_codes <- function(rank_name, tag) {
|
||||
reserved <- mo_registry %>%
|
||||
filter(rank == rank_name) %>%
|
||||
transmute(domain, code = sub("^[A-Z]{1,2}_", "", mo))
|
||||
out <- taxonomy %>%
|
||||
filter(rank == rank_name) %>%
|
||||
distinct(domain, name = .data[[rank_name]]) %>%
|
||||
left_join(
|
||||
existing_mo_tbl %>%
|
||||
filter(rank == rank_name) %>%
|
||||
transmute(domain, name = mo_name_without_suffix(fullname), mo_old = sub("^[A-Z]{1,2}_", "", mo)) %>%
|
||||
distinct(domain, name, .keep_all = TRUE),
|
||||
by = c("domain", "name")
|
||||
) %>%
|
||||
group_by(domain) %>%
|
||||
mutate(
|
||||
code = assign_codes(
|
||||
old = mo_old,
|
||||
candidates = Map(
|
||||
c,
|
||||
AMR:::abbreviate_mo(name, minlength = 8, prefix = paste0("[", tag, "]_")),
|
||||
AMR:::abbreviate_mo(name, minlength = 9, prefix = paste0("[", tag, "]_"))
|
||||
),
|
||||
reserved = reserved$code[reserved$domain == cur_group()$domain]
|
||||
)
|
||||
) %>%
|
||||
ungroup()
|
||||
if (anyDuplicated(paste(out$domain, out$code)) > 0 || anyNA(out$code)) {
|
||||
stop("Duplicate MO codes for ", rank_name, "!", call. = FALSE)
|
||||
}
|
||||
out %>%
|
||||
select(domain, name, code) %>%
|
||||
setNames(c("domain", rank_name, paste0("mo_", rank_name)))
|
||||
}
|
||||
mo_phylum <- higher_rank_codes("phylum", "PHL")
|
||||
mo_class <- higher_rank_codes("class", "CLS")
|
||||
mo_order <- higher_rank_codes("order", "ORD")
|
||||
mo_family <- higher_rank_codes("family", "FAM")
|
||||
|
||||
|
||||
# construct code part for genus - keep old code where available and generate new ones where needed
|
||||
mo_genus <- taxonomy %>%
|
||||
filter(rank == "genus") %>%
|
||||
@@ -2863,6 +2897,13 @@ mo_subspecies <- mo_subspecies %>%
|
||||
mo_unknown <- existing_mo_tbl %>%
|
||||
filter(fullname %like% "unknown") %>%
|
||||
transmute(fullname, mo_unknown = as.character(mo))
|
||||
# unknowns that were added after the last release keep their code too, as long as it is not registered
|
||||
mo_unknown <- mo_unknown %>%
|
||||
bind_rows(
|
||||
microorganisms_old %>%
|
||||
filter(fullname %like% "unknown", !fullname %in% mo_unknown$fullname, !mo %in% mo_registry$mo) %>%
|
||||
transmute(fullname, mo_unknown = as.character(mo))
|
||||
)
|
||||
|
||||
# apply the new codes!
|
||||
taxonomy1 <- taxonomy
|
||||
@@ -3588,23 +3629,33 @@ microorganisms_old %>%
|
||||
select(mo, fullname, rank, status, prevalence) %>%
|
||||
review("Previously manually added taxa that are not in the new data set")
|
||||
|
||||
# LPSN records must be prokaryotes and MycoBank records must be fungi, also in their MO code (issue #309)
|
||||
if (!all(taxonomy$domain[taxonomy$source == "LPSN"] %in% c("Bacteria", "Archaea")) ||
|
||||
!all(taxonomy$mo[taxonomy$source == "LPSN"] %like% "^(B|A)_") ||
|
||||
!all(taxonomy$domain[taxonomy$source == "MycoBank"] == "Fungi") ||
|
||||
!all(taxonomy$mo[taxonomy$source == "MycoBank"] %like% "^F_")) {
|
||||
warning("LPSN records outside prokaryotes or MycoBank records outside fungi, check 'Combine the datasets'", call. = FALSE)
|
||||
}
|
||||
|
||||
# these must never fail, as the package relies on them
|
||||
stopifnot(
|
||||
"MO codes must be unique" = !anyDuplicated(taxonomy$mo),
|
||||
"full names must be unique" = !anyDuplicated(taxonomy$fullname),
|
||||
"MO codes must not be empty" = !any(taxonomy$mo %in% c("", NA)),
|
||||
"prevalence must not be empty" = !anyNA(taxonomy$prevalence),
|
||||
"status must be accepted, synonym or unknown" = all(taxonomy$status %in% c("accepted", "synonym", "unknown")),
|
||||
"sources must be known" = all(taxonomy$source %in% c("LPSN", "MycoBank", "GBIF", "manually added"))
|
||||
# 'renamed to' identifiers of records that were removed after the earlier clean-up (e.g. by deduplication) lead
|
||||
# nowhere, so remove them again: these synonyms then have no current name, instead of a broken one
|
||||
n_dangling <- sum(
|
||||
(!is.na(taxonomy$lpsn_renamed_to) & !taxonomy$lpsn_renamed_to %in% taxonomy$lpsn) |
|
||||
(!is.na(taxonomy$mycobank_renamed_to) & !taxonomy$mycobank_renamed_to %in% taxonomy$mycobank) |
|
||||
(!is.na(taxonomy$gbif_renamed_to) & !taxonomy$gbif_renamed_to %in% taxonomy$gbif)
|
||||
)
|
||||
taxonomy <- taxonomy %>%
|
||||
mutate(
|
||||
lpsn_renamed_to = if_else(lpsn_renamed_to %in% lpsn, lpsn_renamed_to, NA_character_),
|
||||
mycobank_renamed_to = if_else(mycobank_renamed_to %in% mycobank, mycobank_renamed_to, NA_character_),
|
||||
gbif_renamed_to = if_else(gbif_renamed_to %in% gbif, gbif_renamed_to, NA_character_)
|
||||
)
|
||||
message("Removed dangling 'renamed to' identifiers of ", n_dangling, " records")
|
||||
|
||||
# synonyms without a current name are no error, but every one of them is a name that as.mo() cannot update
|
||||
review(mo_synonyms_without_current_name(taxonomy), "Synonyms without a current name")
|
||||
|
||||
# the integrity rules of the data set and the MO code registry (shared with the unit tests in
|
||||
# tests/testthat/helper-microorganisms.R): these must never fail, as the package and its users rely on them
|
||||
integrity_issues <- mo_integrity_issues(taxonomy, registry = mo_registry, renames = mo_renames)
|
||||
if (!is.null(mo_integrity_report(integrity_issues))) {
|
||||
stop("The new data set breaks integrity rules, fix the cause in this script:\n",
|
||||
mo_integrity_report(integrity_issues),
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
# Update other data sets --------------------------------------------------------------------------
|
||||
@@ -3676,11 +3727,13 @@ taxonomy <- taxonomy %>%
|
||||
AMR:::dataset_UTF8_to_ASCII()
|
||||
|
||||
# check again after the conversion to ASCII
|
||||
stopifnot(
|
||||
"full names must be unique after conversion to ASCII" = !anyDuplicated(taxonomy$fullname),
|
||||
"MO codes must be unique after conversion to ASCII" = !anyDuplicated(taxonomy$mo),
|
||||
"MO codes may only contain A-Z, 0-9, _, -, [ and ]" = all(taxonomy$mo %like_case% "^[][A-Z0-9_-]+$")
|
||||
)
|
||||
integrity_issues <- mo_integrity_issues(taxonomy, registry = mo_registry, renames = mo_renames)
|
||||
if (!is.null(mo_integrity_report(integrity_issues))) {
|
||||
stop("The data set breaks integrity rules after the conversion to ASCII:\n",
|
||||
mo_integrity_report(integrity_issues),
|
||||
call. = FALSE
|
||||
)
|
||||
}
|
||||
|
||||
microorganisms <- taxonomy
|
||||
|
||||
|
||||
@@ -65,20 +65,22 @@ These are needed for `TAXONOMY_VERSION` in `R/aa_globals.R` later. Write them do
|
||||
## Step 2: run the build
|
||||
|
||||
The script takes several hours (most of it is LPSN scraping, which is cached in
|
||||
`data-raw/lpsn_scrape_cache.rds`, so a restart is fast). Run it detached from your tool calls, so that
|
||||
it survives time limits, and write everything to a log:
|
||||
`data-raw/lpsn_scrape_cache.rds`, so a restart is fast). Never `source()` the whole script. Use the runner
|
||||
`data-raw/_reproduction_scripts/run_microorganisms_build.R`: it cuts the script into chunks at its
|
||||
checkpoints, runs them one by one in a single R process, logs each chunk with its duration in
|
||||
`<scratch>/logs/progress.tsv`, stops with the name of the failing chunk, and can resume from any checkpoint.
|
||||
See the chunks first with `--list`. Run it detached from your tool calls, so that it survives time limits:
|
||||
|
||||
```bash
|
||||
mkdir -p <scratch>/logs
|
||||
cd <repo root>
|
||||
nohup Rscript -e 'options(AMR_build_view = FALSE, AMR_build_print_rows = 200, AMR_build_review_dir = "<review dir>", warn = 1, width = 200); source("data-raw/_reproduction_scripts/reproduction_of_microorganisms.R", echo = TRUE, max.deparse.length = Inf, keep.source = TRUE)' > <scratch>/logs/build_1.log 2>&1 &
|
||||
nohup Rscript data-raw/_reproduction_scripts/run_microorganisms_build.R --review-dir=<review dir> --log-dir=<scratch>/logs > <scratch>/logs/build_1.log 2>&1 &
|
||||
```
|
||||
|
||||
Monitor the log at sensible intervals (every 10 to 20 minutes, not continuously). Read every
|
||||
`>>> REVIEW:` block as soon as it appears; you do not have to wait until the end to start thinking.
|
||||
With `AMR_build_review_dir` set, every review block is also saved in full as a CSV file in
|
||||
`<review dir>`, named after its title. These files are part of the PR. Set the same option in every
|
||||
resumed run.
|
||||
Monitor the log and `progress.tsv` at sensible intervals (every 10 to 20 minutes, not continuously). Read
|
||||
every `>>> REVIEW:` block as soon as it appears; you do not have to wait until the end to start thinking.
|
||||
Every review block is also saved in full as a CSV file in `<review dir>`, named after its title. These
|
||||
files are part of the PR.
|
||||
|
||||
### Intermediate results
|
||||
|
||||
@@ -105,24 +107,18 @@ and must not be committed.
|
||||
|
||||
### If the run fails
|
||||
|
||||
1. Read the error and the 100 lines before it. Find the root cause in the script, fix it, and
|
||||
check the syntax with `Rscript -e 'invisible(parse("<script>"))'`.
|
||||
2. Do not start from scratch if a checkpoint exists. Write a resume script in `<scratch>` (never in
|
||||
the repository) that contains, in this order:
|
||||
- the script from the top up to (not including) `# Read LPSN data ---`: setup, guard, helpers,
|
||||
clinically relevant genera;
|
||||
- `taxonomy_lpsn`, `taxonomy_mycobank` and `taxonomy_gbif` from their `.rds` files;
|
||||
- if resuming at or after `taxonomy0.rds`: the two lines that read the raw GBIF file
|
||||
(`taxonomy_gbif.bak <- vroom(...)` and the `colnames()` line after it) and the definition of
|
||||
`current_gbif` (needed by `add_missing_parents()`);
|
||||
- if resuming at or after `taxonomy2.rds`: the section `# Add prevalence ---` up to and including
|
||||
the definition of `compute_prevalence()` (needed later on);
|
||||
- `taxonomy <- readRDS("data-raw/<latest checkpoint>.rds")`;
|
||||
- the remainder of the script after that checkpoint's `saveRDS()` line.
|
||||
Number the logs (`build_2.log`, etc.).
|
||||
3. If the same section fails three times for different reasons, or the cause lies outside the
|
||||
script (e.g. the format of a source file changed fundamentally), stop and report (see "When to
|
||||
stop").
|
||||
1. Read the error in the log and the 100 lines before it, and `progress.tsv`. Find the root cause in
|
||||
the script, fix it, and check the syntax with `Rscript -e 'invisible(parse("<script>"))'`.
|
||||
2. Do not start from scratch if a checkpoint exists. The runner tells you where to resume; add
|
||||
`--resume-from=<checkpoint>` (e.g. `--resume-from=taxonomy1b`) and use the same `--review-dir` and
|
||||
`--log-dir`. It runs the setup chunk, loads the saved results and the parts of earlier chunks that
|
||||
later chunks need (the blocks marked with `# @resume-block` in the script), and continues after that
|
||||
checkpoint. Number the logs (`build_2.log`, etc.).
|
||||
3. If you add code to an earlier chunk that a later chunk needs (a function or an object), either move it
|
||||
to the setup part at the top of the script, or mark it as a `# @resume-block` with the right `after=`,
|
||||
otherwise resuming breaks. Check with `--list`.
|
||||
4. If the same chunk fails three times for different reasons, or the cause lies outside the script (e.g.
|
||||
the format of a source file changed fundamentally), stop and report (see "When to stop").
|
||||
|
||||
## Step 3: review as an expert
|
||||
|
||||
@@ -207,7 +203,19 @@ change of more than 5% in a domain needs an explanation. Also report:
|
||||
- the 30 most relevant (prevalence <= 1.25) new genera and species, and the 30 most relevant removed
|
||||
ones.
|
||||
|
||||
### 3d. Unit tests
|
||||
### 3d. Integrity rules and MO code registry
|
||||
|
||||
The script stops before saving if any rule in `tests/testthat/helper-microorganisms.R` is broken, including
|
||||
the MO code registry rules (see `data-raw/microorganisms_files/README.md`): a code from any release since
|
||||
v2.0.0 may never be given to another taxon. Never weaken a rule to get past this; fix the cause. A registered
|
||||
code may only get another name (e.g. a corrected spelling) if it is listed in
|
||||
`data-raw/microorganisms_files/mo_code_renames.csv`, and you may only add such rows with an empty
|
||||
`approved_by`: the build will then stop on it until the human approves it, so list every proposed rename under
|
||||
"Needs a human". After the build, `tests/testthat/microorganisms_known_defects.csv` (the known defects of the
|
||||
data set before this rebuild) must be emptied: delete the rows that are solved, and the file itself if none
|
||||
are left. Any remaining row must be explained under "Needs a human".
|
||||
|
||||
### 3e. Unit tests
|
||||
|
||||
After the data are saved and the package is reloaded, run the tests at the end of the script and then
|
||||
the full suite with `devtools::test()`. Analyse every failure: is the test outdated because of a
|
||||
@@ -242,7 +250,7 @@ human can review it before merging and next year's run can learn from it. Struct
|
||||
Every item states the check, the finding in numbers, your decision, and a link to the CSV file in
|
||||
`<review dir>` where relevant. There is one item for each review block of 3a, one for each sentinel
|
||||
organism of 3b that deviates (and one item for all sentinels that passed), one for each comparison
|
||||
of 3c, one for the unit tests (3d) and one for each changed test. Leave all boxes unticked: ticking
|
||||
of 3c, one for the integrity rules and registry (3d), one for the unit tests (3e) and one for each changed test. Leave all boxes unticked: ticking
|
||||
them is the human's job, it means "reviewed and agreed".
|
||||
5. **Needs a human**: also as checkboxes, everything you could not verify or decide, each with your
|
||||
recommendation. Be concrete, e.g. "COL lists *Enterobius vermicularis* as a synonym of ...;
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
# ==================================================================== #
|
||||
# TITLE: #
|
||||
# AMR: An R Package for Working with Antimicrobial Resistance Data #
|
||||
# #
|
||||
# SOURCE CODE: #
|
||||
# https://github.com/msberends/AMR #
|
||||
# #
|
||||
# PLEASE CITE THIS SOFTWARE AS: #
|
||||
# Berends MS, Luz CF, Friedrich AW, et al. (2022). #
|
||||
# AMR: An R Package for Working with Antimicrobial Resistance Data. #
|
||||
# Journal of Statistical Software, 104(3), 1-31. #
|
||||
# https://doi.org/10.18637/jss.v104.i03 #
|
||||
# #
|
||||
# Developed at the University of Groningen and the University Medical #
|
||||
# Center Groningen in the Netherlands, in collaboration with many #
|
||||
# colleagues from around the world, see our website. #
|
||||
# #
|
||||
# This R package is free software; you can freely use and distribute #
|
||||
# it for both personal and commercial purposes under the terms of the #
|
||||
# GNU General Public License version 2.0 (GNU GPL-2), as published by #
|
||||
# the Free Software Foundation. #
|
||||
# We created this package for both routine data analysis and academic #
|
||||
# research and it was publicly released in the hope that it will be #
|
||||
# useful, but it comes WITHOUT ANY WARRANTY OR LIABILITY. #
|
||||
# #
|
||||
# Visit our website for the full manual and a complete tutorial about #
|
||||
# how to conduct AMR data analysis: https://amr-for-r.org #
|
||||
# ==================================================================== #
|
||||
|
||||
# Runs reproduction_of_microorganisms.R non-interactively, in chunks, in one R process.
|
||||
#
|
||||
# The script itself stays one file (to be run line by line by a human as well). This runner cuts it into chunks
|
||||
# at its checkpoints (the saveRDS() lines of data-raw/taxonomy*.rds), runs these one by one, logs the start,
|
||||
# end and duration of each chunk in a progress file, and stops with the name of the failing chunk.
|
||||
# It can resume from any checkpoint: it then runs the setup chunk, loads the saved results and the parts of
|
||||
# earlier chunks that later chunks need (marked with '# @resume-block' in the script), and continues.
|
||||
#
|
||||
# Usage, from the root of the repository:
|
||||
# Rscript data-raw/_reproduction_scripts/run_microorganisms_build.R [options]
|
||||
# Options:
|
||||
# --review-dir=DIR folder for the CSV files of all reviews (git-tracked, for the human review)
|
||||
# --log-dir=DIR folder for the progress file (default: a temporary folder)
|
||||
# --resume-from=NAME continue after this checkpoint, e.g. --resume-from=taxonomy1b
|
||||
# --list only show the chunks and checkpoints, and stop
|
||||
|
||||
script_file <- "data-raw/_reproduction_scripts/reproduction_of_microorganisms.R"
|
||||
if (!file.exists(script_file)) {
|
||||
stop("Run this from the root of the AMR repository", call. = FALSE)
|
||||
}
|
||||
|
||||
# arguments
|
||||
args <- commandArgs(trailingOnly = TRUE)
|
||||
get_arg <- function(name, default = NULL) {
|
||||
hit <- args[startsWith(args, paste0("--", name, "="))]
|
||||
if (length(hit) == 0) {
|
||||
return(default)
|
||||
}
|
||||
sub("^[^=]+=", "", hit[1])
|
||||
}
|
||||
review_dir <- get_arg("review-dir")
|
||||
log_dir <- get_arg("log-dir", tempfile("microorganisms_build_"))
|
||||
resume_from <- get_arg("resume-from")
|
||||
dir.create(log_dir, recursive = TRUE, showWarnings = FALSE)
|
||||
progress_file <- file.path(log_dir, "progress.tsv")
|
||||
|
||||
# cut the script into chunks ----
|
||||
|
||||
lines <- readLines(script_file, encoding = "UTF-8")
|
||||
# the checkpoints in their order; taxonomy_lpsn_missing.rds is no checkpoint to resume from, as the rest of the
|
||||
# LPSN chunk is needed anyway (and is fast thanks to the LPSN cache)
|
||||
checkpoints <- c(
|
||||
"taxonomy_lpsn", "taxonomy_mycobank", "taxonomy_gbif",
|
||||
"taxonomy0", "taxonomy1", "taxonomy1b", "taxonomy1c",
|
||||
"taxonomy2", "taxonomy2b", "taxonomy2c", "taxonomy3", "taxonomy3b"
|
||||
)
|
||||
checkpoint_line <- vapply(checkpoints, function(cp) {
|
||||
hit <- which(startsWith(lines, "saveRDS(") & grepl(paste0("\"data-raw/", cp, ".rds\""), lines, fixed = TRUE))
|
||||
if (length(hit) != 1) {
|
||||
stop("Checkpoint ", cp, " must be saved exactly once in ", script_file, " (found ", length(hit), ")", call. = FALSE)
|
||||
}
|
||||
hit
|
||||
}, integer(1))
|
||||
if (is.unsorted(checkpoint_line)) {
|
||||
stop("The checkpoints in the script are not in the expected order", call. = FALSE)
|
||||
}
|
||||
setup_end <- which(startsWith(lines, "# Read LPSN data")) - 1
|
||||
stopifnot(length(setup_end) == 1)
|
||||
chunks <- data.frame(
|
||||
name = c("setup", checkpoints, "save_to_package"),
|
||||
from = c(1L, setup_end + 1L, checkpoint_line + 1L),
|
||||
to = c(setup_end, checkpoint_line, length(lines)),
|
||||
stringsAsFactors = FALSE
|
||||
)
|
||||
|
||||
# the blocks that are needed again when resuming
|
||||
block_start <- which(startsWith(lines, "# @resume-block "))
|
||||
block_end <- which(lines == "# @end-resume-block")
|
||||
stopifnot(length(block_start) == length(block_end), all(block_start < block_end))
|
||||
blocks <- data.frame(
|
||||
name = sub("^# @resume-block ([^ ]+) .*", "\\1", lines[block_start]),
|
||||
after = sub(".* after=([^ ]+).*", "\\1", lines[block_start]),
|
||||
from = block_start,
|
||||
to = block_end,
|
||||
stringsAsFactors = FALSE
|
||||
)
|
||||
stopifnot(all(blocks$after %in% checkpoints))
|
||||
|
||||
message("Chunks:")
|
||||
for (i in seq_len(nrow(chunks))) {
|
||||
message(sprintf(" %-17s lines %5d-%5d", chunks$name[i], chunks$from[i], chunks$to[i]))
|
||||
}
|
||||
message("Blocks needed when resuming:")
|
||||
for (i in seq_len(nrow(blocks))) {
|
||||
message(sprintf(" %-17s lines %5d-%5d, when resuming from %s or later", blocks$name[i], blocks$from[i], blocks$to[i], blocks$after[i]))
|
||||
}
|
||||
if ("--list" %in% args) {
|
||||
quit(save = "no", status = 0)
|
||||
}
|
||||
|
||||
# run ----
|
||||
|
||||
options(
|
||||
AMR_build_view = FALSE,
|
||||
AMR_build_print_rows = 200,
|
||||
warn = 1,
|
||||
width = 200
|
||||
)
|
||||
if (!is.null(review_dir)) {
|
||||
options(AMR_build_review_dir = review_dir)
|
||||
}
|
||||
if (!file.exists(progress_file)) {
|
||||
writeLines("chunk\tstarted\tfinished\tminutes\tstatus", progress_file)
|
||||
}
|
||||
log_progress <- function(chunk, started, status) {
|
||||
cat(
|
||||
paste(chunk, format(started, "%Y-%m-%d %H:%M:%S"), format(Sys.time(), "%Y-%m-%d %H:%M:%S"),
|
||||
round(as.numeric(difftime(Sys.time(), started, units = "mins")), 1), status,
|
||||
sep = "\t"
|
||||
), "\n",
|
||||
file = progress_file, append = TRUE, sep = ""
|
||||
)
|
||||
}
|
||||
run_lines <- function(name, from, to) {
|
||||
message("\n", strrep("=", 100), "\n=== CHUNK ", name, " (lines ", from, "-", to, ") started at ", format(Sys.time()), "\n", strrep("=", 100))
|
||||
started <- Sys.time()
|
||||
code <- paste(lines[from:to], collapse = "\n")
|
||||
ok <- tryCatch(
|
||||
{
|
||||
source(exprs = parse(text = code, keep.source = FALSE), local = globalenv(), echo = TRUE, max.deparse.length = 300)
|
||||
TRUE
|
||||
},
|
||||
error = function(e) {
|
||||
message("\n!!! CHUNK ", name, " FAILED: ", conditionMessage(e))
|
||||
log_progress(name, started, paste("FAILED:", gsub("[\t\n]", " ", conditionMessage(e))))
|
||||
FALSE
|
||||
}
|
||||
)
|
||||
if (!ok) {
|
||||
last_done <- which(chunks$name == name) - 2
|
||||
if (length(last_done) == 1 && last_done >= 1) {
|
||||
message("\nAfter fixing the cause, resume with: --resume-from=", checkpoints[last_done])
|
||||
} else {
|
||||
message("\nAfter fixing the cause, run again from the start")
|
||||
}
|
||||
quit(save = "no", status = 1)
|
||||
}
|
||||
log_progress(name, started, "OK")
|
||||
message("=== CHUNK ", name, " finished in ", round(as.numeric(difftime(Sys.time(), started, units = "mins")), 1), " minutes")
|
||||
}
|
||||
|
||||
run_lines("setup", chunks$from[1], chunks$to[1])
|
||||
|
||||
if (is.null(resume_from)) {
|
||||
first_chunk <- 2
|
||||
} else {
|
||||
if (!resume_from %in% checkpoints) {
|
||||
stop("--resume-from must be one of: ", paste(checkpoints, collapse = ", "), call. = FALSE)
|
||||
}
|
||||
k <- which(checkpoints == resume_from)
|
||||
message("\nResuming after checkpoint ", resume_from)
|
||||
started <- Sys.time()
|
||||
# the source taxonomies that were saved up to this checkpoint
|
||||
for (src in intersect(checkpoints[seq_len(k)], c("taxonomy_lpsn", "taxonomy_mycobank", "taxonomy_gbif"))) {
|
||||
assign(src, readRDS(file.path("data-raw", paste0(src, ".rds"))), envir = globalenv())
|
||||
}
|
||||
# the parts of earlier chunks that later chunks need
|
||||
for (i in which(match(blocks$after, checkpoints) <= k)) {
|
||||
run_lines(paste0("resume block ", blocks$name[i]), blocks$from[i], blocks$to[i])
|
||||
}
|
||||
if (!resume_from %in% c("taxonomy_lpsn", "taxonomy_mycobank", "taxonomy_gbif")) {
|
||||
assign("taxonomy", readRDS(file.path("data-raw", paste0(resume_from, ".rds"))), envir = globalenv())
|
||||
}
|
||||
log_progress(paste("resume from", resume_from), started, "OK")
|
||||
first_chunk <- which(chunks$name == resume_from) + 1
|
||||
}
|
||||
|
||||
for (i in seq(first_chunk, nrow(chunks))) {
|
||||
run_lines(chunks$name[i], chunks$from[i], chunks$to[i])
|
||||
}
|
||||
message("\nAll chunks finished. Progress file: ", progress_file)
|
||||
@@ -0,0 +1,39 @@
|
||||
# MO code registry
|
||||
|
||||
MO codes (such as `B_ESCHR_COLI`) are stored by users in their own data, sometimes for years. A code must
|
||||
therefore **always mean the same taxon**, in every future version of the AMR package. This folder contains
|
||||
the registry that guarantees this.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Contents |
|
||||
|---|---|
|
||||
| `mo_code_registry.csv` | Every MO code that was ever part of a **released** version of the AMR package since v2.0.0, with the taxon it denotes (name, rank, domain, genus, species, subspecies), the first and last release it was in, and older names of the same code (e.g. a corrected spelling). |
|
||||
| `build_mo_code_registry.R` | Recreates `mo_code_registry.csv` from the git tags of all releases. Run it after every new release. |
|
||||
| `mo_code_renames.csv` | The only allowed exceptions: registered codes that get another name, because it is the same taxon (e.g. a corrected spelling or a renamed species group). Every row needs `approved_by` and `approved_date`. |
|
||||
|
||||
## Rules
|
||||
|
||||
1. **Only releases count.** Development versions (the main branch) are never part of the registry. Codes that
|
||||
only existed in a development version carry no weight and may be reassigned.
|
||||
2. **Codes from before v2.0.0 are not supported.** Decision by Matthijs S. Berends, 3 October 2026: MO codes
|
||||
from releases before v2.0.0 (12 March 2023) are not supported. Every code in all releases since v2.0.0 has
|
||||
always meant the same taxon (checked on 3 October 2026), so the registry starts without conflicts.
|
||||
3. **A registered code is never given to another taxon.** The taxonomy build
|
||||
(`data-raw/_reproduction_scripts/reproduction_of_microorganisms.R`) reuses the code of every taxon in the
|
||||
registry (most recent release first), never gives a registered code to a new taxon, and stops if a
|
||||
registered code would get another name than in the registry, unless that is listed and approved in
|
||||
`mo_code_renames.csv`.
|
||||
4. **A registered code that is no longer in the data still works.** `as.mo()` translates it to the current
|
||||
code of the same taxon (by its name, so also if the taxon moved to another domain), using an internal lookup
|
||||
table (`MO_RETIRED_CODES`) that is created from the registry in `data-raw/_pre_commit_checks.R`. If the
|
||||
taxon is no longer in the data at all, the result is `NA` with a warning. The package never guesses.
|
||||
5. **Tests enforce all of this.** `tests/testthat/test-data-microorganisms.R` checks the current data against
|
||||
the registry and the renames, and checks that every registered code resolves to its own taxon.
|
||||
|
||||
## After a release
|
||||
|
||||
1. Make sure the git tag of the release exists (e.g. `v3.1.0`).
|
||||
2. Run `source("data-raw/microorganisms_files/build_mo_code_registry.R")` from the root of the repository.
|
||||
3. Review the changes to `mo_code_registry.csv` (`git diff`): only new codes may be added, and existing codes
|
||||
may only get a later `last_release`. Commit the result.
|
||||
@@ -0,0 +1,113 @@
|
||||
# ==================================================================== #
|
||||
# TITLE: #
|
||||
# AMR: An R Package for Working with Antimicrobial Resistance Data #
|
||||
# #
|
||||
# SOURCE CODE: #
|
||||
# https://github.com/msberends/AMR #
|
||||
# #
|
||||
# PLEASE CITE THIS SOFTWARE AS: #
|
||||
# Berends MS, Luz CF, Friedrich AW, et al. (2022). #
|
||||
# AMR: An R Package for Working with Antimicrobial Resistance Data. #
|
||||
# Journal of Statistical Software, 104(3), 1-31. #
|
||||
# https://doi.org/10.18637/jss.v104.i03 #
|
||||
# #
|
||||
# Developed at the University of Groningen and the University Medical #
|
||||
# Center Groningen in the Netherlands, in collaboration with many #
|
||||
# colleagues from around the world, see our website. #
|
||||
# #
|
||||
# This R package is free software; you can freely use and distribute #
|
||||
# it for both personal and commercial purposes under the terms of the #
|
||||
# GNU General Public License version 2.0 (GNU GPL-2), as published by #
|
||||
# the Free Software Foundation. #
|
||||
# We created this package for both routine data analysis and academic #
|
||||
# research and it was publicly released in the hope that it will be #
|
||||
# useful, but it comes WITHOUT ANY WARRANTY OR LIABILITY. #
|
||||
# #
|
||||
# Visit our website for the full manual and a complete tutorial about #
|
||||
# how to conduct AMR data analysis: https://amr-for-r.org #
|
||||
# ==================================================================== #
|
||||
|
||||
# This script (re)creates data-raw/microorganisms_files/mo_code_registry.csv: the registry of all MO codes
|
||||
# that were ever part of a released version of the AMR package, since v2.0.0. See the README.md in this
|
||||
# folder for the rules.
|
||||
#
|
||||
# Run it from the root of the repository, after every new release (once its git tag exists):
|
||||
# source("data-raw/microorganisms_files/build_mo_code_registry.R")
|
||||
# It always rebuilds the whole registry from the git tags, so it is reproducible and can be run at any time.
|
||||
# Development versions (the main branch) are never part of the registry.
|
||||
|
||||
library(dplyr)
|
||||
devtools::load_all(".", quiet = TRUE) # for %like_case%
|
||||
|
||||
registry_file <- "data-raw/microorganisms_files/mo_code_registry.csv"
|
||||
|
||||
# Decision by Matthijs S. Berends, 3 October 2026: MO codes from releases before v2.0.0 (12 March 2023)
|
||||
# are not supported.
|
||||
first_supported_release <- "2.0.0"
|
||||
|
||||
# all release tags (vX.Y.Z, so no development versions) since the first supported release
|
||||
tags <- system2("git", "tag", stdout = TRUE)
|
||||
tags <- tags[tags %like_case% "^v[0-9]+[.][0-9]+[.][0-9]+$"]
|
||||
tags <- tags[numeric_version(sub("^v", "", tags)) >= numeric_version(first_supported_release)]
|
||||
tags <- tags[order(numeric_version(sub("^v", "", tags)))]
|
||||
message("Releases: ", toString(tags))
|
||||
|
||||
read_release <- function(tag) {
|
||||
tmp <- tempfile(fileext = ".rda")
|
||||
on.exit(unlink(tmp))
|
||||
status <- system2("git", c("show", paste0(tag, ":data/microorganisms.rda")), stdout = tmp)
|
||||
if (!identical(status, 0L)) {
|
||||
stop("Could not read data/microorganisms.rda of ", tag, call. = FALSE)
|
||||
}
|
||||
env <- new.env()
|
||||
load(tmp, envir = env)
|
||||
mo <- as.data.frame(env$microorganisms, stringsAsFactors = FALSE)
|
||||
# until v3.0.1, `kingdom` contained what is now called `domain`
|
||||
if (!"domain" %in% colnames(mo)) {
|
||||
mo$domain <- mo$kingdom
|
||||
mo$domain[mo$domain == "(unknown kingdom)"] <- "(unknown domain)"
|
||||
}
|
||||
data.frame(
|
||||
release = tag,
|
||||
mo = as.character(mo$mo),
|
||||
fullname = mo$fullname,
|
||||
rank = mo$rank,
|
||||
domain = mo$domain,
|
||||
genus = mo$genus,
|
||||
species = mo$species,
|
||||
subspecies = mo$subspecies,
|
||||
stringsAsFactors = FALSE
|
||||
)
|
||||
}
|
||||
|
||||
all_releases <- bind_rows(lapply(tags, read_release)) %>%
|
||||
mutate(release_order = match(release, tags))
|
||||
|
||||
# one row per MO code; the name, rank and taxonomy are taken from the most recent release that contained the code
|
||||
registry <- all_releases %>%
|
||||
group_by(mo) %>%
|
||||
arrange(desc(release_order), .by_group = TRUE) %>%
|
||||
summarise(
|
||||
# other names the code had in older releases (e.g. a corrected subspecies name), for transparency
|
||||
# (must come first, since `fullname` is overwritten below)
|
||||
previous_names = paste(setdiff(unique(fullname), first(fullname)), collapse = "; "),
|
||||
fullname = first(fullname),
|
||||
rank = first(rank),
|
||||
domain = first(domain),
|
||||
genus = first(genus),
|
||||
species = first(species),
|
||||
subspecies = first(subspecies),
|
||||
first_release = tags[min(release_order)],
|
||||
last_release = tags[max(release_order)],
|
||||
.groups = "drop"
|
||||
) %>%
|
||||
relocate(previous_names, .after = everything()) %>%
|
||||
arrange(mo)
|
||||
|
||||
message(
|
||||
nrow(registry), " MO codes, of which ", sum(registry$previous_names != ""),
|
||||
" had another name in an older release (check these when the registry changes)"
|
||||
)
|
||||
|
||||
write.csv(registry, registry_file, row.names = FALSE, na = "")
|
||||
message("Saved to ", registry_file)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,2 @@
|
||||
"mo","registry_name","new_name","reason","approved_by","approved_date"
|
||||
"B_MYCBC_AVIM-C","Mycobacterium avium-intracellulare complex","Mycobacterium avium complex","Species group renamed in the development version of 2026; same group of organisms","",""
|
||||
|
Reference in new issue
Block a user