mirror of
https://github.com/msberends/AMR.git
synced 2026-11-16 05:00:50 +01:00
(v3.0.1.9097) Never remove released taxa; link orthographic variants; registry line in NEWS
- taxonomy build: released taxa (since v2.0.0) that are missing are restored from their last release with their old code and last known status, and are exempt from later removals (decision by Matthijs S. Berends) - exceptions are retired on purpose in data-raw/microorganisms_files/mo_code_retirements.csv with a reason: other organisms with the same genus name (e.g. the Graphium butterflies, wrongly in the Fungi until v3.0.1) and taxa whose genus is now only known in another domain; as.mo() translates these codes to NA with that reason - synonyms without a current name are linked to an accepted name in the same genus that only differs by a Latin gender ending (e.g. Candida glabratus -> Nakaseomyces glabratus) - placeholders such as "(unknown class)" never become records; codes of manually added records are matched by domain and name - integrity rule released_taxon_missing; known defects of the data on main updated Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d683b9e7c6
commit
83a4853936
12 files changed
+5326
-23
No files matched your search
@@ -454,6 +454,8 @@ add_missing_parents <- function(taxonomy, current_gbif) {
|
||||
filter(.data[[rank_name]] != "") %>%
|
||||
distinct(across(all_of(unique(c("domain", rank_levels[seq_len(which(rank_levels == rank_name))]))))) %>%
|
||||
mutate(fullname = .data[[rank_name]], rank = rank_name) %>%
|
||||
# placeholders such as "(unknown class)" are no taxa
|
||||
filter(fullname %unlike% "^[(]unknown") %>%
|
||||
anti_join(
|
||||
taxonomy %>% filter(rank == rank_name),
|
||||
by = unique(c("domain", setNames(rank_name, rank_name), "rank"))
|
||||
@@ -483,7 +485,8 @@ add_missing_parents <- function(taxonomy, current_gbif) {
|
||||
}))
|
||||
# species implied by subspecies but missing as a species-rank row
|
||||
missing_species <- taxonomy %>%
|
||||
filter(species != "") %>%
|
||||
# placeholders such as "(unknown species)" are no taxa
|
||||
filter(species != "", genus %unlike% "^[(]unknown", species %unlike% "^[(]unknown") %>%
|
||||
distinct(domain, genus, species, .keep_all = TRUE) %>%
|
||||
select(domain:species) %>%
|
||||
mutate(fullname = paste(genus, species), rank = "species") %>%
|
||||
@@ -1538,6 +1541,16 @@ review(
|
||||
taxonomy %>% filter(.resolve == "remove") %>% count(genus, domain, best_domain, source, sort = TRUE),
|
||||
"Records removed as homonyms of a genus in another domain"
|
||||
)
|
||||
# these are other organisms with the same genus name, so they may never be restored as released taxa later on
|
||||
# (e.g. the Graphium butterflies, which were in the Fungi until v3.0.1), saved for 'Restore released taxa'
|
||||
saveRDS(
|
||||
taxonomy %>%
|
||||
filter(.resolve == "remove") %>%
|
||||
distinct(fullname, .keep_all = TRUE) %>%
|
||||
transmute(fullname, domain, genus, best_domain),
|
||||
"data-raw/taxonomy_homonyms_removed.rds",
|
||||
version = 2
|
||||
)
|
||||
taxonomy <- taxonomy %>%
|
||||
filter(.resolve != "remove") %>%
|
||||
mutate(
|
||||
@@ -2527,6 +2540,215 @@ taxonomy <- taxonomy %>%
|
||||
distinct(fullname, .keep_all = TRUE)
|
||||
message("Removed ", n_before - nrow(taxonomy), " nonsense names")
|
||||
|
||||
# Restore released taxa ---------------------------------------------------------------------------
|
||||
|
||||
# A taxon that was part of a release (since v2.0.0) is never removed, since users have stored its MO code
|
||||
# (decision by Matthijs S. Berends, 3 October 2026). Taxa that are missing now, e.g. because they are not
|
||||
# validly published or not clinically relevant, are restored from their last release, with their old code and
|
||||
# their last known status. The only exception: records that were removed as homonyms above, i.e. other organisms
|
||||
# with the same genus name (such as the Graphium butterflies, which were wrongly in the Fungi until v3.0.1).
|
||||
# Their codes are retired, with the reason, in data-raw/microorganisms_files/mo_code_retirements.csv, and are
|
||||
# translated to NA by as.mo() with that reason, so that they never lead to a wrong organism and are never reused.
|
||||
restore_registry <- read_mo_registry(".")
|
||||
restore_renames <- read_mo_renames(".")
|
||||
restore_registry$expected_name <- restore_registry$fullname
|
||||
renamed <- match(restore_registry$mo, restore_renames$mo)
|
||||
restore_registry$expected_name[!is.na(renamed)] <- restore_renames$new_name[renamed[!is.na(renamed)]]
|
||||
missing_released <- restore_registry %>%
|
||||
filter(
|
||||
!mo_name_without_suffix(expected_name) %in% mo_name_without_suffix(taxonomy$fullname),
|
||||
# species groups are added at the end of this script
|
||||
rank != "species group"
|
||||
)
|
||||
homonyms_removed <- if (file.exists("data-raw/taxonomy_homonyms_removed.rds")) {
|
||||
readRDS("data-raw/taxonomy_homonyms_removed.rds")
|
||||
} else {
|
||||
tibble(fullname = character(0), domain = character(0), genus = character(0), best_domain = character(0))
|
||||
}
|
||||
|
||||
# codes to retire: released taxa that are other organisms than the genus with their name
|
||||
retirements_file <- "data-raw/microorganisms_files/mo_code_retirements.csv"
|
||||
retirements <- utils::read.csv(retirements_file, colClasses = "character", na.strings = character(0))
|
||||
new_retirements <- missing_released %>%
|
||||
inner_join(
|
||||
homonyms_removed %>% select(fullname, removed_domain = domain, removed_genus = genus, best_domain),
|
||||
by = c("expected_name" = "fullname")
|
||||
) %>%
|
||||
filter(!mo %in% retirements$mo) %>%
|
||||
transmute(
|
||||
mo,
|
||||
fullname = expected_name,
|
||||
reason = paste0(
|
||||
"another organism (", removed_domain, ") with the same genus name as ", removed_genus, " (", best_domain, ")"
|
||||
),
|
||||
decided_by = "taxonomy build (automatic, homonym of a genus in another domain)",
|
||||
date = as.character(Sys.Date())
|
||||
)
|
||||
review(new_retirements, "Released MO codes that are retired, since their taxon is another organism with the same genus name")
|
||||
if (nrow(new_retirements) > 0) {
|
||||
retirements <- bind_rows(retirements, new_retirements) %>% arrange(mo)
|
||||
utils::write.csv(retirements, retirements_file, row.names = FALSE, na = "")
|
||||
}
|
||||
missing_released <- missing_released %>%
|
||||
filter(!mo %in% retirements$mo)
|
||||
|
||||
# get the full records from the releases in which these taxa were last present
|
||||
read_release_data <- function(tag) {
|
||||
tmp <- tempfile(fileext = ".rda")
|
||||
on.exit(unlink(tmp))
|
||||
if (!identical(system2("git", c("show", paste0(tag, ":data/microorganisms.rda")), stdout = tmp), 0L)) {
|
||||
stop("Could not read data/microorganisms.rda of release ", tag, call. = FALSE)
|
||||
}
|
||||
env <- new.env()
|
||||
load(tmp, envir = env)
|
||||
out <- as.data.frame(env$microorganisms, stringsAsFactors = FALSE)
|
||||
out$mo <- as.character(out$mo)
|
||||
# until v3.0.1, `kingdom` contained what is now called `domain`
|
||||
if (!"domain" %in% colnames(out)) {
|
||||
out$domain <- out$kingdom
|
||||
out$domain[out$domain == "(unknown kingdom)"] <- "(unknown domain)"
|
||||
out$kingdom <- NA_character_
|
||||
}
|
||||
out
|
||||
}
|
||||
released_records <- bind_rows(lapply(unique(missing_released$last_release), function(tag) {
|
||||
read_release_data(tag) %>%
|
||||
filter(mo %in% missing_released$mo[missing_released$last_release == tag]) %>%
|
||||
select(any_of(c(
|
||||
"mo", "fullname", "status", "domain", "kingdom", "phylum", "class", "order", "family", "genus", "species",
|
||||
"subspecies", "rank", "ref", "lpsn_renamed_to", "mycobank_renamed_to", "gbif_renamed_to", "prevalence"
|
||||
))) %>%
|
||||
mutate(across(c(fullname, domain:subspecies), ascii_names))
|
||||
}))
|
||||
|
||||
# a released taxon of which the genus now only exists in another domain cannot be restored consistently: it may be
|
||||
# the same organism in a new classification, or another organism (homonym), which cannot be decided automatically.
|
||||
# Its code is retired (as.mo() then gives NA with this reason, and translates it if the name exists elsewhere).
|
||||
genus_domains <- taxonomy %>%
|
||||
filter(rank == "genus") %>%
|
||||
distinct(genus, domain)
|
||||
conflicting <- released_records %>%
|
||||
filter(
|
||||
genus != "",
|
||||
!paste(domain, genus) %in% paste(genus_domains$domain, genus_domains$genus),
|
||||
genus %in% genus_domains$genus
|
||||
) %>%
|
||||
left_join(genus_domains %>% group_by(genus) %>% summarise(now_in = toString(domain)), by = "genus")
|
||||
conflict_retirements <- conflicting %>%
|
||||
transmute(
|
||||
mo,
|
||||
fullname,
|
||||
reason = paste0("its genus ", genus, " is now only known in another domain (", now_in, ") and could not be restored"),
|
||||
decided_by = "taxonomy build (automatic, genus in another domain), NEEDS REVIEW",
|
||||
date = as.character(Sys.Date())
|
||||
) %>%
|
||||
filter(!mo %in% read_mo_retirements(".")$mo)
|
||||
review(conflict_retirements, "Released MO codes that are retired, since their genus is now only known in another domain")
|
||||
if (nrow(conflict_retirements) > 0) {
|
||||
utils::write.csv(
|
||||
bind_rows(read_mo_retirements("."), conflict_retirements) %>% arrange(mo),
|
||||
retirements_file,
|
||||
row.names = FALSE, na = ""
|
||||
)
|
||||
}
|
||||
released_records <- released_records %>%
|
||||
filter(!mo %in% conflicting$mo)
|
||||
|
||||
# the current kingdom of prokaryotes (until v3.0.1, the kingdom was the domain) and the current higher taxonomy
|
||||
# of the genus if it still exists
|
||||
kingdom_of_phylum <- taxonomy %>%
|
||||
filter(domain %in% c("Bacteria", "Archaea"), phylum != "", kingdom != "") %>%
|
||||
count(domain, phylum, kingdom) %>%
|
||||
arrange(desc(n)) %>%
|
||||
distinct(domain, phylum, .keep_all = TRUE) %>%
|
||||
select(domain, phylum, new_kingdom = kingdom)
|
||||
lineage_of_genus <- taxonomy %>%
|
||||
filter(rank == "genus") %>%
|
||||
distinct(domain, genus, .keep_all = TRUE) %>%
|
||||
select(domain, genus, g_kingdom = kingdom, g_phylum = phylum, g_class = class, g_order = order, g_family = family)
|
||||
restored <- released_records %>%
|
||||
left_join(kingdom_of_phylum, by = c("domain", "phylum")) %>%
|
||||
left_join(lineage_of_genus, by = c("domain", "genus")) %>%
|
||||
mutate(
|
||||
kingdom = case_when(
|
||||
!is.na(g_kingdom) ~ g_kingdom,
|
||||
domain %in% c("Bacteria", "Archaea") & !is.na(new_kingdom) ~ new_kingdom,
|
||||
domain %in% c("Bacteria", "Archaea") ~ "(unknown kingdom)",
|
||||
TRUE ~ coalesce(kingdom, domain)
|
||||
),
|
||||
phylum = coalesce(g_phylum, phylum),
|
||||
class = coalesce(g_class, class),
|
||||
order = coalesce(g_order, order),
|
||||
family = coalesce(g_family, family),
|
||||
across(kingdom:subspecies, function(x) if_else(is.na(x), "", x)),
|
||||
# older releases used placeholders such as "(unknown class)", which would become records themselves
|
||||
across(phylum:family, function(x) if_else(x %like% "^[(]unknown", "", x)),
|
||||
# older releases also had other status values, such as "not validly published"
|
||||
status = if_else(status %in% c("accepted", "synonym", "unknown"), status, "unknown"),
|
||||
source = "manually added"
|
||||
) %>%
|
||||
select(-mo, -new_kingdom, -starts_with("g_"))
|
||||
review(
|
||||
restored %>% count(domain, rank, status, sort = TRUE),
|
||||
"Released taxa that were missing and are restored (by domain, rank and status)"
|
||||
)
|
||||
review(
|
||||
restored %>% filter(prevalence < 2) %>% select(fullname, domain, rank, status, ref, prevalence),
|
||||
"Restored released taxa that are clinically relevant"
|
||||
)
|
||||
n_before <- nrow(taxonomy)
|
||||
taxonomy <- taxonomy %>%
|
||||
bind_rows(restored) %>%
|
||||
add_missing_parents(current_gbif) %>%
|
||||
arrange(fullname, source_prio(source)) %>%
|
||||
distinct(fullname, .keep_all = TRUE) %>%
|
||||
compute_prevalence()
|
||||
message("Restored ", nrow(restored), " released taxa (", nrow(taxonomy) - n_before, " records including their parents)")
|
||||
rm(restore_registry, restore_renames, renamed, missing_released, homonyms_removed, retirements, new_retirements,
|
||||
released_records, genus_domains, conflicting, conflict_retirements, kingdom_of_phylum, lineage_of_genus, restored)
|
||||
|
||||
|
||||
# Link orthographic variants without a current name ----------------------------------------------
|
||||
|
||||
# A synonym without a current name of which the epithet only differs from an accepted name in the same genus by a
|
||||
# Latin gender ending (e.g. -us, -a, -um) is an orthographic variant of that name: under the nomenclatural codes,
|
||||
# such epithets are variants of one name, so two valid species in one genus cannot differ only in this way.
|
||||
# E.g. 'Nakaseomyces glabrata' -> 'Nakaseomyces glabratus' (from Candida glabrata, renamed in 2022).
|
||||
epithet_stem <- function(x) {
|
||||
sub("(us|um|a|is|e|er|ra|rum)$", "", x)
|
||||
}
|
||||
variant_candidates <- taxonomy %>%
|
||||
filter(
|
||||
status == "synonym",
|
||||
is.na(lpsn_renamed_to), is.na(mycobank_renamed_to), is.na(gbif_renamed_to),
|
||||
rank %in% c("species", "subspecies")
|
||||
) %>%
|
||||
mutate(stem = epithet_stem(if_else(rank == "species", species, subspecies)))
|
||||
accepted_variants <- taxonomy %>%
|
||||
filter(status == "accepted", rank %in% c("species", "subspecies")) %>%
|
||||
mutate(stem = epithet_stem(if_else(rank == "species", species, subspecies))) %>%
|
||||
select(domain, genus, rank, stem, species_acc = species, target = fullname, t_lpsn = lpsn, t_mycobank = mycobank, t_gbif = gbif)
|
||||
variant_links <- variant_candidates %>%
|
||||
inner_join(accepted_variants, by = c("domain", "genus", "rank", "stem")) %>%
|
||||
# for subspecies, the species must be the same variant too
|
||||
filter(rank == "species" | epithet_stem(species) == epithet_stem(species_acc)) %>%
|
||||
# only unambiguous links, and the target must have an identifier to link to
|
||||
group_by(fullname) %>%
|
||||
filter(n() == 1) %>%
|
||||
ungroup() %>%
|
||||
filter(!is.na(t_lpsn) | !is.na(t_mycobank) | !is.na(t_gbif))
|
||||
review(variant_links %>% select(fullname, target, source), "Orthographic variants linked to their accepted name")
|
||||
taxonomy <- taxonomy %>%
|
||||
left_join(variant_links %>% select(fullname, t_lpsn, t_mycobank, t_gbif), by = "fullname") %>%
|
||||
mutate(
|
||||
lpsn_renamed_to = coalesce(lpsn_renamed_to, t_lpsn),
|
||||
mycobank_renamed_to = coalesce(mycobank_renamed_to, t_mycobank),
|
||||
gbif_renamed_to = coalesce(gbif_renamed_to, t_gbif)
|
||||
) %>%
|
||||
select(-t_lpsn, -t_mycobank, -t_gbif)
|
||||
rm(variant_candidates, accepted_variants, variant_links)
|
||||
|
||||
|
||||
# Break cycles of synonyms ------------------------------------------------------------------------
|
||||
|
||||
# Sources can contradict each other, e.g. in 2026 GBIF listed Capillidium denaeosporus as synonym of
|
||||
@@ -2972,9 +3194,10 @@ taxonomy %>%
|
||||
# filter(mo != "")
|
||||
|
||||
# keep the codes from manually added ones
|
||||
# (by domain and name, so that a code can never end up in another domain)
|
||||
manual_mos <- as.character(existing_mo_tbl$mo)[match(
|
||||
taxonomy$fullname[taxonomy$source == "manually added"],
|
||||
existing_mo_tbl$fullname
|
||||
paste(taxonomy$domain, taxonomy$fullname)[taxonomy$source == "manually added"],
|
||||
paste(existing_mo_tbl$domain, existing_mo_tbl$fullname)
|
||||
)]
|
||||
taxonomy$mo[taxonomy$source == "manually added"][!is.na(manual_mos)] <- manual_mos[!is.na(manual_mos)]
|
||||
|
||||
@@ -3595,26 +3818,30 @@ taxonomy %>%
|
||||
|
||||
# Remove childless non-bacterial genera -----------------------------------------------------------
|
||||
|
||||
# this removes all genera that have no species, except for the domain of Bacteria.
|
||||
# this removes all genera that have no species, except for the domain of Bacteria, and except for released taxa
|
||||
# (a taxon that was part of a release is never removed, see 'Restore released taxa')
|
||||
released_names <- mo_name_without_suffix(read_mo_registry(".")$fullname)
|
||||
taxonomy <- taxonomy %>%
|
||||
filter(
|
||||
rank != "genus" |
|
||||
domain == "Bacteria" |
|
||||
genus %in% taxonomy$genus[taxonomy$rank == "species"]
|
||||
genus %in% taxonomy$genus[taxonomy$rank == "species"] |
|
||||
mo_name_without_suffix(fullname) %in% released_names
|
||||
)
|
||||
|
||||
# then remove all childless upper taxonomy caused by this
|
||||
# then remove all childless upper taxonomy caused by this (again except for released taxa)
|
||||
for (rank_name in c("family", "order", "class", "phylum", "kingdom")) {
|
||||
has_children <- taxonomy %>%
|
||||
filter(rank != rank_name, .data[[rank_name]] != "") %>%
|
||||
distinct(.data[[rank_name]]) %>%
|
||||
pull()
|
||||
|
||||
|
||||
n_before <- nrow(taxonomy)
|
||||
taxonomy <- taxonomy %>%
|
||||
filter(!(rank == rank_name & !fullname %in% has_children))
|
||||
filter(!(rank == rank_name & !fullname %in% has_children & !mo_name_without_suffix(fullname) %in% released_names))
|
||||
message("Removed ", n_before - nrow(taxonomy), " childless ", rank_name, " entries")
|
||||
}
|
||||
rm(released_names)
|
||||
|
||||
# records that were only inferred by this script get the same label as before
|
||||
taxonomy$source[taxonomy$source == "inferred"] <- "manually added"
|
||||
@@ -3649,7 +3876,7 @@ review(mo_synonyms_without_current_name(taxonomy), "Synonyms without a current n
|
||||
|
||||
# the integrity rules of the data set and the MO code registry (shared with the unit tests in
|
||||
# tests/testthat/helper-microorganisms.R): these must never fail, as the package and its users rely on them
|
||||
integrity_issues <- mo_integrity_issues(taxonomy, registry = mo_registry, renames = mo_renames)
|
||||
integrity_issues <- mo_integrity_issues(taxonomy, registry = mo_registry, renames = mo_renames, retirements = read_mo_retirements("."))
|
||||
if (!is.null(mo_integrity_report(integrity_issues))) {
|
||||
stop("The new data set breaks integrity rules, fix the cause in this script:\n",
|
||||
mo_integrity_report(integrity_issues),
|
||||
@@ -3727,7 +3954,7 @@ taxonomy <- taxonomy %>%
|
||||
AMR:::dataset_UTF8_to_ASCII()
|
||||
|
||||
# check again after the conversion to ASCII
|
||||
integrity_issues <- mo_integrity_issues(taxonomy, registry = mo_registry, renames = mo_renames)
|
||||
integrity_issues <- mo_integrity_issues(taxonomy, registry = mo_registry, renames = mo_renames, retirements = read_mo_retirements("."))
|
||||
if (!is.null(mo_integrity_report(integrity_issues))) {
|
||||
stop("The data set breaks integrity rules after the conversion to ASCII:\n",
|
||||
mo_integrity_report(integrity_issues),
|
||||
|
||||
@@ -211,7 +211,10 @@ v2.0.0 may never be given to another taxon. Never weaken a rule to get past this
|
||||
code may only get another name (e.g. a corrected spelling) if it is listed in
|
||||
`data-raw/microorganisms_files/mo_code_renames.csv`, and you may only add such rows with an empty
|
||||
`approved_by`: the build will then stop on it until the human approves it, so list every proposed rename under
|
||||
"Needs a human". After the build, `tests/testthat/microorganisms_known_defects.csv` (the known defects of the
|
||||
"Needs a human". The build never removes a released taxon: missing ones are restored, except codes it retires in
|
||||
`data-raw/microorganisms_files/mo_code_retirements.csv` (other organisms with the same genus name, or a genus that
|
||||
is now only known in another domain). Every new row in that file needs its own checkbox in the report, and rows
|
||||
marked `NEEDS REVIEW` also under "Needs a human". After the build, `tests/testthat/microorganisms_known_defects.csv` (the known defects of the
|
||||
data set before this rebuild) must be emptied: delete the rows that are solved, and the file itself if none
|
||||
are left. Any remaining row must be explained under "Needs a human".
|
||||
|
||||
|
||||
Reference in new issue
Block a user