(v3.0.1.9132) Make translating around 50 times faster and coercion of unknown input in as.ab() up to 70 times faster with identical results, use DOIs instead of PubMed URLs, shorten an example line, and reword cran-comments.md

This commit is contained in:
dr. M.S. (Matthijs) Berends committed 2026-10-14 20:17:28 +02:00
1 parent 4115db67fa
commit 1a1a3abed1
12 files changed
+164 -112

No files matched your search

+28 -2
View File
@@ -37,11 +37,15 @@
# 2. Coercion caches, `AMR_env$ab_previously_coerced` and `AMR_env$mo_previously_coerced`, that remember the
# outcome of earlier coercions. Both have the same structure: a `key` (the cleaned input together with the
# settings that affect the outcome), the original `input`, and the resulting code as `value`.
# 3. For microorganisms, the mapping from (outdated) codes to codes of currently accepted names, which is
# 3. Translated names of antimicrobials and microorganisms, built once per language, see get_ab_translated_index() and
# get_mo_translated_index().
# 4. For microorganisms, the mapping from (outdated) codes to codes of currently accepted names, which is
# memoised per unique code, see synonym_mo_to_accepted_mo().
#
# All of these are invalidated with reset_ab_cache() or reset_mo_cache(), which must be called whenever the
# lookup tables change, e.g. in add_custom_antimicrobials() and add_custom_microorganisms().
# The prepared translation tables of translate_into_language(), see get_translation_table(), are kept for the whole
# session, as the internal translations never change.
new_coercion_cache <- function() {
data.frame(
@@ -92,6 +96,7 @@ coercion_cache_add <- function(type, keys, inputs, values) {
reset_ab_cache <- function() {
AMR_env$AB_index <- NULL
AMR_env$AB_translated <- NULL
AMR_env$custom_ab_id_keys <- NULL
AMR_env$ab_previously_coerced <- new_coercion_cache()
}
@@ -111,16 +116,37 @@ get_ab_index <- function() {
atc_ab <- rep(ab, lengths(AMR_env$AB_lookup$atc))
synonyms <- tolower(unlist(AMR_env$AB_lookup$synonyms, use.names = FALSE))
synonyms_ab <- rep(ab, lengths(AMR_env$AB_lookup$synonyms))
# generalised LOINC codes, synonyms, and abbreviations as flat vectors, with the row number in `AMR_env$AB_lookup`
flat <- function(col) {
vals <- AMR_env$AB_lookup[[col]]
list(value = unlist(vals, use.names = FALSE), row = rep(seq_along(vals), lengths(vals)))
}
gen_syn <- flat("generalised_synonyms")
# synonyms of at least 5 characters, for the Levenshtein distance in as.ab()
gen_syn_long <- which(!is.na(gen_syn$value) & nchar(gen_syn$value) >= 5)
AMR_env$AB_index <- list(
atc = atc[!is.na(atc)],
atc_ab = atc_ab[!is.na(atc)],
synonyms = synonyms[!is.na(synonyms)],
synonyms_ab = synonyms_ab[!is.na(synonyms)]
synonyms_ab = synonyms_ab[!is.na(synonyms)],
generalised_loinc = flat("generalised_loinc"),
generalised_synonyms = gen_syn,
generalised_synonyms_long = list(value = gen_syn$value[gen_syn_long], row = gen_syn$row[gen_syn_long]),
generalised_abbreviations = flat("generalised_abbreviations")
)
}
AMR_env$AB_index
}
get_ab_translated_index <- function(language) {
# translated and generalised names of all antimicrobials, used by as.ab() for input that could not be matched otherwise
# this is built once per language
if (is.null(AMR_env$AB_translated[[language]])) {
AMR_env$AB_translated[[language]] <- generalise_antibiotic_name(translate_AMR(AMR_env$AB_lookup$name, language = language))
}
AMR_env$AB_translated[[language]]
}
get_mo_index <- function() {
add_MO_lookup_to_AMR_env()
if (is.null(AMR_env$MO_index)) {
+21 -38
View File
@@ -299,34 +299,25 @@ as.ab <- function(x, flag_multiple_results = TRUE, language = get_AMR_locale(),
}
# exact LOINC code
loinc_found <- unlist(lapply(
AMR_env$AB_lookup$generalised_loinc,
function(s) x[i] %in% s
))
found <- AMR_env$AB_lookup$ab[loinc_found == TRUE]
idx <- get_ab_index()$generalised_loinc
found <- AMR_env$AB_lookup$ab[sort(unique(idx$row[idx$value %in% x[i]]))]
if (length(found) > 0) {
x_new[i] <- note_if_more_than_one_found(found, i, from_text)
next
}
# exact synonym
synonym_found <- unlist(lapply(
AMR_env$AB_lookup$generalised_synonyms,
function(s) x[i] %in% s
))
found <- AMR_env$AB_lookup$ab[synonym_found == TRUE]
idx <- get_ab_index()$generalised_synonyms
found <- AMR_env$AB_lookup$ab[sort(unique(idx$row[idx$value %in% x[i]]))]
if (length(found) > 0) {
x_new[i] <- note_if_more_than_one_found(found, i, from_text)
next
}
# exact abbreviation
abbr_found <- unlist(lapply(
AMR_env$AB_lookup$generalised_abbreviations,
# require at least 2 characters for abbreviations
function(s) x[i] %in% s && nchar(x[i]) >= 2
))
found <- AMR_env$AB_lookup$ab[abbr_found == TRUE]
idx <- get_ab_index()$generalised_abbreviations
# require at least 2 characters for abbreviations
found <- AMR_env$AB_lookup$ab[sort(unique(idx$row[idx$value %in% x[i] & nchar(x[i]) >= 2]))]
if (length(found) > 0) {
x_new[i] <- note_if_more_than_one_found(found, i, from_text)
next
@@ -392,11 +383,8 @@ as.ab <- function(x, flag_multiple_results = TRUE, language = get_AMR_locale(),
}
# and try if any synonym starts with it
synonym_found <- unlist(lapply(
AMR_env$AB_lookup$generalised_synonyms,
function(s) any(s %like% paste0("^", x_spelling))
))
found <- AMR_env$AB_lookup$ab[synonym_found == TRUE]
idx <- get_ab_index()$generalised_synonyms
found <- AMR_env$AB_lookup$ab[sort(unique(idx$row[idx$value %like% paste0("^", x_spelling)]))]
if (length(found) > 0) {
x_new[i] <- note_if_more_than_one_found(found, i, from_text)
next
@@ -413,24 +401,19 @@ as.ab <- function(x, flag_multiple_results = TRUE, language = get_AMR_locale(),
costs = c(insertions = 1, deletions = 1, substitutions = 2),
counts = FALSE
))
ab_df$lev_syn <- vapply(
FUN.VALUE = double(1),
ab_df$generalised_synonyms,
function(y) {
ifelse(all(is.na(y)) || length(y[nchar(y) >= 5]) == 0,
999,
min(as.double(utils::adist(x[i], y[nchar(y) >= 5],
ignore.case = FALSE,
fixed = TRUE,
costs = c(insertions = 1, deletions = 1, substitutions = 2),
counts = FALSE
)), na.rm = TRUE)
)
},
USE.NAMES = FALSE
)
# lowest distance to the synonyms of at least 5 characters, 999 if there are none
idx <- get_ab_index()$generalised_synonyms_long
lev_syn <- as.double(utils::adist(x[i], idx$value,
ignore.case = FALSE,
fixed = TRUE,
costs = c(insertions = 1, deletions = 1, substitutions = 2),
counts = FALSE
))
ab_df$lev_syn <- 999
lev_syn_min <- vapply(FUN.VALUE = double(1), split(lev_syn, idx$row), min)
ab_df$lev_syn[as.integer(names(lev_syn_min))] <- lev_syn_min
if (!is.null(language) && language != "en") {
ab_df$trans <- generalise_antibiotic_name(translate_AMR(ab_df$name, language = language))
ab_df$trans <- get_ab_translated_index(language)
ab_df$lev_trans <- as.double(utils::adist(x[i], ab_df$trans,
ignore.case = FALSE,
fixed = TRUE,
+5 -3
View File
@@ -180,9 +180,11 @@ ab_group <- function(x, language = get_AMR_locale(), all_groups = FALSE, ...) {
# take the first match based on ABX_PRIORITY_LIST
grps[[i]] <- grps[[i]][1]
}
if (language != "en") {
grps[[i]] <- translate_into_language(grps[[i]], language = language, only_affect_ab_names = TRUE)
}
}
if (language != "en" && length(grps) > 0) {
# translate all groups at once
grps_translated <- translate_into_language(unlist(grps, use.names = FALSE), language = language, only_affect_ab_names = TRUE)
grps[] <- split(grps_translated, factor(rep(seq_along(grps), lengths(grps)), levels = seq_along(grps)))
}
names(grps) <- x
if (length(grps) == 1 || all_groups == FALSE) {
+2 -1
View File
@@ -49,7 +49,8 @@
#' @seealso [grepl()]
#' @examples
#' # data.table has a more limited version of %like%, so detach it (without unloading it, as data.table does not support being unloaded):
#' # data.table has a more limited version of %like%, so detach it
#' # (without unloading it, as data.table does not support being unloaded):
#' try(detach("package:data.table"), silent = TRUE)
#'
#' a <- "This is a test"
+1 -1
View File
@@ -50,7 +50,7 @@
#'
#' Determination of human pathogenicity ([mo_pathogenicity()]) is strongly based on Bartlett *et al.* (2022, \doi{10.1099/mic.0.001269}). This function returns a [factor] with the levels *Pathogenic*, *Potentially pathogenic*, *Non-pathogenic*, and *Unknown*.
#'
#' Determination of the Gram stain ([mo_gramstain()] is based on the taxonomic domain and phylum. Originally, Cavalier-Smith defined the so-called subkingdoms Negibacteria and Posibacteria (2002, [PMID 11837318](https://pubmed.ncbi.nlm.nih.gov/11837318/)), and only considered these phyla as Posibacteria: Actinobacteria, Chloroflexi, Firmicutes, and Tenericutes. These phyla were later renamed to Actinomycetota, Chloroflexota, Bacillota, and Mycoplasmatota (2021, [PMID 34694987](https://pubmed.ncbi.nlm.nih.gov/34694987/)). Bacteria in these phyla are considered Gram-positive in this `AMR` package, except for members of the class Negativicutes (within phylum Bacillota) which are Gram-negative. All other bacteria are considered Gram-negative. Species outside the kingdom of Bacteria will return a value `NA`. Functions [mo_is_gram_negative()] and [mo_is_gram_positive()] always return `TRUE` or `FALSE` (or `NA` when the input is `NA` or the MO code is `UNKNOWN`), thus always return `FALSE` for species outside the taxonomic kingdom of Bacteria.
#' Determination of the Gram stain ([mo_gramstain()] is based on the taxonomic domain and phylum. Originally, Cavalier-Smith defined the so-called subkingdoms Negibacteria and Posibacteria (2002, \doi{10.1099/00207713-52-1-7}), and only considered these phyla as Posibacteria: Actinobacteria, Chloroflexi, Firmicutes, and Tenericutes. These phyla were later renamed to Actinomycetota, Chloroflexota, Bacillota, and Mycoplasmatota (2021, \doi{10.1099/ijsem.0.005056}). Bacteria in these phyla are considered Gram-positive in this `AMR` package, except for members of the class Negativicutes (within phylum Bacillota) which are Gram-negative. All other bacteria are considered Gram-negative. Species outside the kingdom of Bacteria will return a value `NA`. Functions [mo_is_gram_negative()] and [mo_is_gram_positive()] always return `TRUE` or `FALSE` (or `NA` when the input is `NA` or the MO code is `UNKNOWN`), thus always return `FALSE` for species outside the taxonomic kingdom of Bacteria.
#'
#' Determination of yeasts ([mo_is_yeast()]) is based on the taxonomic domain and class. *Budding yeasts* are yeasts that reproduce asexually through a process called budding, where a new cell develops from a small protrusion on the parent cell. Taxonomically, these are members of the phylum Ascomycota, class Saccharomycetes (also called Hemiascomycetes) or Pichiomycetes. *True yeasts* quite specifically refers to yeasts in the underlying order Saccharomycetales (such as *Saccharomyces cerevisiae*). Thus, for all microorganisms that are member of the taxonomic class Saccharomycetes or Pichiomycetes, the function will return `TRUE`. It returns `FALSE` otherwise (or `NA` when the input is `NA` or the MO code is `UNKNOWN`).
#'
+92 -61
View File
@@ -200,9 +200,86 @@ translate_into_language <- function(from,
return(from)
}
df_trans <- TRANSLATIONS # internal data file
trns <- get_translation_table(lang, only_unknown = only_unknown, only_affect_ab_names = only_affect_ab_names, only_affect_mo_names = only_affect_mo_names)
if (is.null(trns)) {
return(from)
}
from.bak <- from
from_unique <- unique(from)
# check if text to look for is in one of the patterns
any_form_in_patterns <- tryCatch(
# the same as `from_unique %like% trns$any_pattern`, but without validating the long pattern on every call
any(grepl(trns$any_pattern, tolower(from_unique), fixed = trns$any_pattern_fixed, perl = !trns$any_pattern_fixed)),
error = function(e) {
warning_("Translation not possible. Please create an issue at {.url https://github.com/msberends/AMR/issues}. Many thanks!")
return(FALSE)
}
)
if (!any_form_in_patterns) {
return(from)
}
from_unique_translated <- from_unique
# combinations like "Ticarcillin/clavulanic acid" are translated per part, unless the combination itself is in the translations
is_combination <- grepl("/", from_unique, fixed = TRUE) & !from_unique %in% trns$pattern_all
if (any(is_combination)) {
# all parts of all combinations are translated at once
parts <- strsplit(from_unique[is_combination], "/", fixed = TRUE)
parts_translated <- translate_into_language(unlist(parts, use.names = FALSE), language = lang, only_unknown = only_unknown, only_affect_ab_names = only_affect_ab_names, only_affect_mo_names = only_affect_mo_names)
parts_translated <- split(parts_translated, factor(rep(seq_along(parts), lengths(parts)), levels = seq_along(parts)))
from_unique_translated[is_combination] <- vapply(FUN.VALUE = character(1), parts_translated, paste, collapse = "/", USE.NAMES = FALSE)
}
if (any(!is_combination)) {
# every pattern is applied to all strings at once, in order of the translation table
terms <- from_unique[!is_combination]
for (i in seq_along(trns$pattern)) {
terms <- gsub(
pattern = trns$pattern[i],
replacement = trns$replacement[i],
x = terms,
ignore.case = trns$ignore_case[i],
fixed = trns$fixed[i],
perl = !trns$fixed[i]
)
}
# replace the placeholders by their translations
has_placeholder <- grepl("\001", terms, fixed = TRUE)
if (any(has_placeholder)) {
found <- gregexpr("\001[^\001\002]{3}\002", terms[has_placeholder], perl = TRUE)
regmatches(terms[has_placeholder], found) <- lapply(regmatches(terms[has_placeholder], found), function(p) trns$literal[match(p, trns$placeholder)])
}
from_unique_translated[!is_combination] <- terms
}
# force UTF-8 for diacritics
from_unique_translated <- enc2utf8(from_unique_translated)
# a kind of left join to get all results back
out <- from_unique_translated[match(from.bak, from_unique)]
if (!identical(from.bak, out) && get_AMR_locale() == lang && is.null(getOption("AMR_locale", default = NULL)) && message_not_thrown_before("translation", entire_session = TRUE) && interactive()) {
message(font_blue(word_wrap(
"Assuming the ", LANGUAGES_SUPPORTED_NAMES[[lang]]$exonym, " language (",
LANGUAGES_SUPPORTED_NAMES[[lang]]$endonym, ") for the AMR package. See `set_AMR_locale()` to change this or to silence this once-per-session note.",
as_note = TRUE
)))
}
out
}
# the translation table for a language and the settings of translate_into_language(), built once per session as TRANSLATIONS is fixed
get_translation_table <- function(lang, only_unknown = FALSE, only_affect_ab_names = FALSE, only_affect_mo_names = FALSE) {
key <- paste(lang, only_unknown, only_affect_ab_names, only_affect_mo_names, sep = "|")
if (!key %in% names(AMR_env$translation_tables)) {
AMR_env$translation_tables[key] <- list(build_translation_table(lang, only_unknown, only_affect_ab_names, only_affect_mo_names))
}
AMR_env$translation_tables[[key]]
}
build_translation_table <- function(lang, only_unknown, only_affect_ab_names, only_affect_mo_names) {
df_trans <- TRANSLATIONS # internal data file
# only keep lines where translation is available for this language
df_trans <- df_trans[which(!is.na(df_trans[, lang, drop = TRUE])), , drop = FALSE]
# and where the original string is not equal to the string in the target language
@@ -217,7 +294,7 @@ translate_into_language <- function(from,
df_trans <- subset(df_trans, affect_mo_name == TRUE)
}
if (NROW(df_trans) == 0) {
return(from)
return(NULL)
}
# default: case sensitive if value if 'case_sensitive' is missing:
@@ -225,18 +302,8 @@ translate_into_language <- function(from,
# default: not using regular expressions if 'regular_expr' is missing:
df_trans$regular_expr[is.na(df_trans$regular_expr)] <- FALSE
# check if text to look for is in one of the patterns
any_form_in_patterns <- tryCatch(
any(from_unique %like% paste0("(", paste(gsub(" +\\(.*", "", df_trans$pattern), collapse = "|"), ")")),
error = function(e) {
warning_("Translation not possible. Please create an issue at {.url https://github.com/msberends/AMR/issues}. Many thanks!")
return(FALSE)
}
)
if (NROW(df_trans) == 0 | !any_form_in_patterns) {
return(from)
}
# to check if text to look for is in one of the patterns
any_pattern <- paste0("(", paste(gsub(" +\\(.*", "", df_trans$pattern), collapse = "|"), ")")
if (only_affect_ab_names == TRUE) {
df_trans$pattern[df_trans$regular_expr == TRUE] <- paste0(df_trans$pattern[df_trans$regular_expr == TRUE], "$")
@@ -257,52 +324,16 @@ translate_into_language <- function(from,
replacements,
vapply(FUN.VALUE = character(1), seq_along(replacements), function(i) sub(literals[i], placeholders[i], replacements[i], fixed = TRUE))
)
translate_exec <- function(term) {
for (i in seq_len(nrow(trns))) {
term <- gsub(
pattern = trns$pattern[i],
replacement = replacements_placeholder[i],
x = term,
ignore.case = !trns$case_sensitive[i] & trns$regular_expr[i],
fixed = !trns$regular_expr[i],
perl = trns$regular_expr[i]
)
}
for (i in which(vapply(FUN.VALUE = logical(1), placeholders, grepl, x = term, fixed = TRUE))) {
term <- gsub(placeholders[i], literals[i], term, fixed = TRUE)
}
term
}
from_unique_translated <- from_unique
# combinations like "Ticarcillin/clavulanic acid" are translated per part, unless the combination itself is in the translations
is_combination <- grepl("/", from_unique, fixed = TRUE) & !from_unique %in% df_trans$pattern
from_unique_translated[is_combination] <- vapply(
FUN.VALUE = character(1),
USE.NAMES = FALSE,
strsplit(from_unique[is_combination], "/", fixed = TRUE),
function(parts) paste(translate_into_language(parts, language = lang, only_unknown = only_unknown, only_affect_ab_names = only_affect_ab_names, only_affect_mo_names = only_affect_mo_names), collapse = "/")
list(
# all patterns, including combinations, to check whether a combination is translated as a whole
pattern_all = df_trans$pattern,
any_pattern = tolower(any_pattern),
any_pattern_fixed = !is_valid_regex(any_pattern),
pattern = trns$pattern,
replacement = replacements_placeholder,
ignore_case = !trns$case_sensitive & trns$regular_expr,
fixed = !trns$regular_expr,
placeholder = placeholders,
literal = literals
)
from_unique_translated[!is_combination] <- vapply(
FUN.VALUE = character(1),
USE.NAMES = FALSE,
from_unique[!is_combination],
translate_exec
)
# force UTF-8 for diacritics
from_unique_translated <- enc2utf8(from_unique_translated)
# a kind of left join to get all results back
out <- from_unique_translated[match(from.bak, from_unique)]
if (!identical(from.bak, out) && get_AMR_locale() == lang && is.null(getOption("AMR_locale", default = NULL)) && message_not_thrown_before("translation", entire_session = TRUE) && interactive()) {
message(font_blue(word_wrap(
"Assuming the ", LANGUAGES_SUPPORTED_NAMES[[lang]]$exonym, " language (",
LANGUAGES_SUPPORTED_NAMES[[lang]]$endonym, ") for the AMR package. See `set_AMR_locale()` to change this or to silence this once-per-session note.",
as_note = TRUE
)))
}
out
}