include IQR and MAD in freq

This commit is contained in:
dr. M.S. (Matthijs) Berends 2018-07-03 11:30:40 +02:00
parent 3d0d21f322
commit 10fce8382c
8 changed files with 110 additions and 163 deletions

View File

@ -1,5 +1,6 @@
# Generated by roxygen2: do not edit by hand
S3method(as.data.frame,frequency_tbl)
S3method(as.double,mic)
S3method(as.integer,mic)
S3method(as.numeric,mic)
@ -53,6 +54,7 @@ export(rsi_df)
export(rsi_predict)
export(semi_join_microorganisms)
export(top_freq)
exportMethods(as.data.frame.frequency_tbl)
exportMethods(as.double.mic)
exportMethods(as.integer.mic)
exportMethods(as.numeric.mic)
@ -115,8 +117,8 @@ importFrom(rvest,html_node)
importFrom(rvest,html_nodes)
importFrom(rvest,html_table)
importFrom(stats,fivenum)
importFrom(stats,mad)
importFrom(stats,pchisq)
importFrom(stats,quantile)
importFrom(stats,sd)
importFrom(tibble,tibble)
importFrom(utils,browseVignettes)

View File

@ -2,6 +2,7 @@
#### New
* Function `top_freq` function to get the top/below *n* items of frequency tables
* Vignette about frequency tables
* Header of frequency tables now also show MAD and IQR
* Possibility to globally set the default for the amount of items to print in frequency tables (`freq` function), with `options(max.print.freq = n)`
* Functions `clipboard_import` and `clipboard_export` as helper functions to quickly copy and paste from/to software like Excel and SPSS
* Function `g.test` to perform the Χ<sup>2</sup> distributed [*G*-test](https://en.wikipedia.org/wiki/G-test)

110
R/freq.R
View File

@ -19,40 +19,43 @@
#' Frequency table
#'
#' Create a frequency table of a vector with items or a data frame. Supports quasiquotation and markdown for reports. \code{top_freq} can be used to get the top/bottom \emph{n} items of a frequency table, with counts as names.
#' @param x vector with items, or \code{data.frame}
#' @param x vector with items, or a \code{data.frame}
#' @param ... up to nine different columns of \code{x} to calculate frequencies from, see Examples
#' @param sort.count sort on count, i.e. frequencies. Use \code{FALSE} to sort alphabetically on item.
#' @param nmax number of row to print. The default, \code{15}, uses \code{\link{getOption}("max.print.freq")}. Use \code{nmax = 0}, \code{nmax = NULL} or \code{nmax = NA} to print all rows.
#' @param na.rm a logical value indicating whether NA values should be removed from the frequency table. The header will always print the amount of \code{NA}s.
#' @param sort.count sort on count, i.e. frequencies. This will be \code{TRUE} at default for everything except for factors.
#' @param nmax number of row to print. The default, \code{15}, uses \code{\link{getOption}("max.print.freq")}. Use \code{nmax = 0}, \code{nmax = Inf}, \code{nmax = NULL} or \code{nmax = NA} to print all rows.
#' @param na.rm a logical value indicating whether \code{NA} values should be removed from the frequency table. The header will always print the amount of \code{NA}s.
#' @param row.names a logical value indicating whether row indices should be printed as \code{1:nrow(x)}
#' @param markdown print table in markdown format (this forces \code{nmax = NA})
#' @param digits how many significant digits are to be used for numeric values in the header (not for the items themselves, that depends on \code{\link{getOption}("digits")})
#' @param sep a character string to separate the terms when selecting multiple columns
#' @param f a frequency table
#' @param n number of top \emph{n} items to return, use -n for the bottom \emph{n} items. It will include more than \code{n} rows if there are ties.
#' @details This package also has a vignette available about this function, run: \code{browseVignettes("AMR")} to read it.
#' @details Frequency tables (or frequency distributions) are summaries of the distribution of values in a sample. With the `freq` function, you can create univariate frequency tables. Multiple variables will be pasted into one variable, so it forces a univariate distribution. This package also has a vignette available to explain the use of this function further, run \code{browseVignettes("AMR")} to read it.
#'
#' For numeric values of any class, these additional values will be calculated and shown into the header:
#' For numeric values of any class, these additional values will all be calculated with \code{na.rm = TRUE} and shown into the header:
#' \itemize{
#' \item{Mean, using \code{\link[base]{mean}}}
#' \item{Standard deviation, using \code{\link[stats]{sd}}}
#' \item{Five numbers of Tukey (min, Q1, median, Q3, max), using \code{\link[stats]{fivenum}}}
#' \item{Outliers (total count and unique count), using \code{\link{boxplot.stats}}}
#' \item{Coefficient of variation (CV), the standard deviation divided by the mean}
#' \item{Coefficient of quartile variation (CQV, sometimes called coefficient of dispersion), calculated as \code{(Q3 - Q1) / (Q3 + Q1)} using \code{\link{quantile}} with \code{type = 6} as quantile algorithm to comply with SPSS standards}
#' \item{Standard Deviation, using \code{\link[stats]{sd}}}
#' \item{Coefficient of Variation (CV), the standard deviation divided by the mean}
#' \item{Mean Absolute Deviation (MAD), using \code{\link[stats]{mad}}}
#' \item{Tukey Five-Number Summaries (minimum, Q1, median, Q3, maximum), using \code{\link[stats]{fivenum}}}
#' \item{Interquartile Range (IQR) calculated as \code{Q3 - Q1} using the Tukey Five-Number Summaries, i.e. \strong{not} using the \code{\link[stats]{quantile}} function}
#' \item{Coefficient of Quartile Variation (CQV, sometimes called coefficient of dispersion), calculated as \code{(Q3 - Q1) / (Q3 + Q1)} using the Tukey Five-Number Summaries}
#' \item{Outliers (total count and unique count), using \code{\link[grDevices]{boxplot.stats}}}
#' }
#'
#' For dates and times of any class, these additional values will be calculated and shown into the header:
#' For dates and times of any class, these additional values will be calculated with \code{na.rm = TRUE} and shown into the header:
#' \itemize{
#' \item{Oldest, using \code{\link[base]{min}}}
#' \item{Newest, using \code{\link[base]{max}}, with difference between newest and oldest}
#' \item{Median, using \code{\link[stats]{median}}, with percentage since oldest}
#' }
#'
#'
#' The function \code{top_freq} uses \code{\link[dplyr]{top_n}} internally and will include more than \code{n} rows if there are ties.
#' @importFrom stats fivenum sd quantile
#' @importFrom stats fivenum sd mad
#' @importFrom grDevices boxplot.stats
#' @importFrom dplyr %>% select pull n_distinct group_by arrange desc mutate summarise
#' @importFrom dplyr %>% select pull n_distinct group_by arrange desc mutate summarise n_distinct
#' @importFrom utils browseVignettes
#' @importFrom tibble tibble
#' @keywords summary summarise frequency freq
@ -88,17 +91,24 @@
#' filter(hospital_id == "A") %>%
#' freq(genus, species)
#'
#' # save frequency table to an object
#' years <- septic_patients %>%
#' mutate(year = format(date, "%Y")) %>%
#' freq(year)
#' years %>% pull(item)
#'
#' # get top 10 bugs of hospital A as a vector
#' septic_patients %>%
#' filter(hospital_id == "A") %>%
#' freq(bactid) %>%
#' top_freq(10)
#'
#' # save frequency table to an object
#' years <- septic_patients %>%
#' mutate(year = format(date, "%Y")) %>%
#' freq(year)
#'
#' # print only top 5
#' years %>% print(nmax = 5)
#'
#' # transform to plain data.frame
#' septic_patients %>%
#' freq(age) %>%
#' as.data.frame()
frequency_tbl <- function(x,
...,
sort.count = TRUE,
@ -135,11 +145,6 @@ frequency_tbl <- function(x,
mult.columns <- 0
if (NROW(x) == 0) {
cat('\nNo observations.\n')
return(invisible())
}
if (!is.null(ncol(x))) {
if (ncol(x) == 1 & any(class(x) == 'data.frame')) {
x <- x %>% pull(1)
@ -226,8 +231,8 @@ frequency_tbl <- function(x,
x <- x[!x %in% NAs]
}
if (missing(sort.count) & any(class(x) %in% c('double', 'integer', 'numeric', 'raw', 'single', 'factor'))) {
# sort on item/level at default when x is numeric or a factor and sort.count is not set
if (missing(sort.count) & 'factor' %in% class(x)) {
# sort on factor level at default when x is a factor and sort.count is not set
sort.count <- FALSE
}
@ -246,28 +251,30 @@ frequency_tbl <- function(x,
}
if (is.list(x) | is.matrix(x) | is.environment(x) | is.function(x)) {
cat(header, "\n")
stop('`freq()` does not support lists, matrices, environments or functions.', call. = FALSE)
stop('frequency tables do not support lists, matrices, environments and functions.', call. = FALSE)
}
header <- header %>% paste0(markdown_line, '\nLength: ', (NAs %>% length() + x %>% length()) %>% format(),
' (of which NA: ', NAs %>% length() %>% format(),
' = ', (NAs %>% length() / (NAs %>% length() + x %>% length())) %>% percent(force_zero = TRUE), ')')
' = ', (NAs %>% length() / (NAs %>% length() + x %>% length())) %>% percent(force_zero = TRUE) %>% sub('NaN', '0', ., fixed = TRUE), ')')
header <- header %>% paste0(markdown_line, '\nUnique: ', x %>% n_distinct() %>% format())
if (any(class(x) %in% c('double', 'integer', 'numeric', 'raw', 'single'))) {
if (NROW(x) > 0 & any(class(x) %in% c('double', 'integer', 'numeric', 'raw', 'single'))) {
# right align number
Tukey_five <- stats::fivenum(x, na.rm = TRUE)
x_align <- 'r'
header <- header %>% paste0('\n')
header <- header %>% paste(markdown_line, '\nMean: ', x %>% base::mean(na.rm = TRUE) %>% format(digits = digits))
header <- header %>% paste0(markdown_line, '\nStd. dev.: ', x %>% stats::sd(na.rm = TRUE) %>% format(digits = digits),
' (CV: ', x %>% cv(na.rm = TRUE) %>% format(digits = digits), ')')
header <- header %>% paste0(markdown_line, '\nFive-Num: ', x %>% stats::fivenum(na.rm = TRUE) %>% format(digits = digits) %>% trimws() %>% paste(collapse = ' | '),
' (CQV: ', x %>% cqv(na.rm = TRUE) %>% format(digits = digits), ')')
' (CV: ', x %>% cv(na.rm = TRUE) %>% format(digits = digits),
', MAD: ', x %>% stats::mad(na.rm = TRUE) %>% format(digits = digits), ')')
header <- header %>% paste0(markdown_line, '\nFive-Num: ', Tukey_five %>% format(digits = digits) %>% trimws() %>% paste(collapse = ' | '),
' (IQR: ', (Tukey_five[4] - Tukey_five[2]) %>% format(digits = digits),
', CQV: ', x %>% cqv(na.rm = TRUE) %>% format(digits = digits), ')')
outlier_length <- length(boxplot.stats(x)$out)
header <- header %>% paste0(markdown_line, '\nOutliers: ', outlier_length)
if (outlier_length > 0) {
header <- header %>% paste0(' (unique: ', boxplot.stats(x)$out %>% unique() %>% length(), ')')
header <- header %>% paste0(' (unique: ', boxplot.stats(x)$out %>% n_distinct(), ')')
}
}
@ -276,7 +283,7 @@ frequency_tbl <- function(x,
x <- x %>% as.POSIXlt()
formatdates <- "%H:%M:%S"
}
if (any(class(x) %in% c('Date', 'POSIXct', 'POSIXlt'))) {
if (NROW(x) > 0 & any(class(x) %in% c('Date', 'POSIXct', 'POSIXlt'))) {
header <- header %>% paste0('\n')
mindate <- x %>% min(na.rm = TRUE)
maxdate <- x %>% max(na.rm = TRUE)
@ -302,10 +309,9 @@ frequency_tbl <- function(x,
nmax <- length(x)
}
if (nmax == 0 | is.na(nmax) | is.null(nmax)) {
if (nmax %in% c(0, Inf, NA, NULL)) {
nmax <- length(x)
}
nmax.1 <- min(length(x), nmax + 1)
# create table with counts and percentages
column_names <- c('Item', 'Count', 'Percent', 'Cum. Count', 'Cum. Percent', '(Factor Level)')
@ -404,12 +410,12 @@ top_freq <- function(f, n) {
vect
}
#' @rdname print
#' @rdname freq
#' @exportMethod print.frequency_tbl
#' @importFrom knitr kable
#' @importFrom dplyr n_distinct
#' @export
print.frequency_tbl <- function(x, ...) {
print.frequency_tbl <- function(x, nmax = getOption("max.print.freq", default = 15), ...) {
opt <- attr(x, 'opt')
@ -423,7 +429,12 @@ print.frequency_tbl <- function(x, ...) {
title <- ""
}
cat("Frequency table", title, "\n\n")
if (!missing(nmax)) {
opt$nmax <- nmax
opt$nmax.set <- TRUE
}
cat("Frequency table", title, "\n")
if (!is.null(opt$header)) {
cat(opt$header)
@ -448,7 +459,13 @@ print.frequency_tbl <- function(x, ...) {
x.unprinted <- base::sum(x[(opt$nmax + 1):nrow(x), 'count'], na.rm = TRUE)
x.printed <- base::sum(x$count) - x.unprinted
x <- x[1:opt$nmax,]
if (opt$nmax.set == TRUE) {
nmax <- opt$nmax
} else {
nmax <- getOption("max.print.freq", default = 15)
}
x <- x[1:nmax,]
if (opt$nmax.set == TRUE) {
footer <- paste('[ reached `nmax = ', opt$nmax, '`', sep = '')
@ -496,3 +513,12 @@ print.frequency_tbl <- function(x, ...) {
}
#' @noRd
#' @exportMethod as.data.frame.frequency_tbl
#' @export
as.data.frame.frequency_tbl <- function(x, ...) {
attr(x, 'package') <- NULL
attr(x, 'package.version') <- NULL
attr(x, 'opt') <- NULL
as.data.frame.data.frame(x, ...)
}

View File

@ -91,10 +91,8 @@ cv <- function(x, na.rm = TRUE) {
# Coefficient of dispersion, or coefficient of quartile variation (CQV).
# (Bonett et al., 2006: Confidence interval for a coefficient of quartile variation).
cqv <- function(x, na.rm = TRUE) {
cqv.x <-
(stats::quantile(x, 0.75, na.rm = na.rm, type = 6) - stats::quantile(x, 0.25, na.rm = na.rm, type = 6)) /
(stats::quantile(x, 0.75, na.rm = na.rm, type = 6) + stats::quantile(x, 0.25, na.rm = na.rm, type = 6))
unname(cqv.x)
fives <- stats::fivenum(x, na.rm = na.rm)
(fives[4] - fives[2]) / (fives[4] + fives[2])
}
# show bytes as kB/MB/GB

View File

@ -124,8 +124,6 @@ prettyprint_df <- function(x,
if ('tbl_df' %in% class(x)) {
type <- 'tibble'
} else if ('frequency_tbl' %in% class(x)) {
type <- 'frequency table'
} else if ('data.table' %in% class(x)) {
type <- 'data.table'
} else {

View File

@ -4,6 +4,7 @@
\alias{freq}
\alias{frequency_tbl}
\alias{top_freq}
\alias{print.frequency_tbl}
\title{Frequency table}
\usage{
frequency_tbl(x, ..., sort.count = TRUE, nmax = getOption("max.print.freq"),
@ -15,17 +16,20 @@ freq(x, ..., sort.count = TRUE, nmax = getOption("max.print.freq"),
sep = " ")
top_freq(f, n)
\method{print}{frequency_tbl}(x, nmax = getOption("max.print.freq", default =
15), ...)
}
\arguments{
\item{x}{vector with items, or \code{data.frame}}
\item{x}{vector with items, or a \code{data.frame}}
\item{...}{up to nine different columns of \code{x} to calculate frequencies from, see Examples}
\item{sort.count}{sort on count, i.e. frequencies. Use \code{FALSE} to sort alphabetically on item.}
\item{sort.count}{sort on count, i.e. frequencies. This will be \code{TRUE} at default for everything except for factors.}
\item{nmax}{number of row to print. The default, \code{15}, uses \code{\link{getOption}("max.print.freq")}. Use \code{nmax = 0}, \code{nmax = NULL} or \code{nmax = NA} to print all rows.}
\item{nmax}{number of row to print. The default, \code{15}, uses \code{\link{getOption}("max.print.freq")}. Use \code{nmax = 0}, \code{nmax = Inf}, \code{nmax = NULL} or \code{nmax = NA} to print all rows.}
\item{na.rm}{a logical value indicating whether NA values should be removed from the frequency table. The header will always print the amount of \code{NA}s.}
\item{na.rm}{a logical value indicating whether \code{NA} values should be removed from the frequency table. The header will always print the amount of \code{NA}s.}
\item{row.names}{a logical value indicating whether row indices should be printed as \code{1:nrow(x)}}
@ -46,25 +50,28 @@ A \code{data.frame} with an additional class \code{"frequency_tbl"}
Create a frequency table of a vector with items or a data frame. Supports quasiquotation and markdown for reports. \code{top_freq} can be used to get the top/bottom \emph{n} items of a frequency table, with counts as names.
}
\details{
This package also has a vignette available about this function, run: \code{browseVignettes("AMR")} to read it.
Frequency tables (or frequency distributions) are summaries of the distribution of values in a sample. With the `freq` function, you can create univariate frequency tables. Multiple variables will be pasted into one variable, so it forces a univariate distribution. This package also has a vignette available to explain the use of this function further, run \code{browseVignettes("AMR")} to read it.
For numeric values of any class, these additional values will be calculated and shown into the header:
For numeric values of any class, these additional values will all be calculated with \code{na.rm = TRUE} and shown into the header:
\itemize{
\item{Mean, using \code{\link[base]{mean}}}
\item{Standard deviation, using \code{\link[stats]{sd}}}
\item{Five numbers of Tukey (min, Q1, median, Q3, max), using \code{\link[stats]{fivenum}}}
\item{Outliers (total count and unique count), using \code{\link{boxplot.stats}}}
\item{Coefficient of variation (CV), the standard deviation divided by the mean}
\item{Coefficient of quartile variation (CQV, sometimes called coefficient of dispersion), calculated as \code{(Q3 - Q1) / (Q3 + Q1)} using \code{\link{quantile}} with \code{type = 6} as quantile algorithm to comply with SPSS standards}
\item{Standard Deviation, using \code{\link[stats]{sd}}}
\item{Coefficient of Variation (CV), the standard deviation divided by the mean}
\item{Mean Absolute Deviation (MAD), using \code{\link[stats]{mad}}}
\item{Tukey Five-Number Summaries (minimum, Q1, median, Q3, maximum), using \code{\link[stats]{fivenum}}}
\item{Interquartile Range (IQR) calculated as \code{Q3 - Q1} using the Tukey Five-Number Summaries, i.e. \strong{not} using the \code{\link[stats]{quantile}} function}
\item{Coefficient of Quartile Variation (CQV, sometimes called coefficient of dispersion), calculated as \code{(Q3 - Q1) / (Q3 + Q1)} using the Tukey Five-Number Summaries}
\item{Outliers (total count and unique count), using \code{\link[grDevices]{boxplot.stats}}}
}
For dates and times of any class, these additional values will be calculated and shown into the header:
For dates and times of any class, these additional values will be calculated with \code{na.rm = TRUE} and shown into the header:
\itemize{
\item{Oldest, using \code{\link[base]{min}}}
\item{Newest, using \code{\link[base]{max}}, with difference between newest and oldest}
\item{Median, using \code{\link[stats]{median}}, with percentage since oldest}
}
The function \code{top_freq} uses \code{\link[dplyr]{top_n}} internally and will include more than \code{n} rows if there are ties.
}
\examples{
@ -95,17 +102,24 @@ septic_patients \%>\%
filter(hospital_id == "A") \%>\%
freq(genus, species)
# save frequency table to an object
years <- septic_patients \%>\%
mutate(year = format(date, "\%Y")) \%>\%
freq(year)
years \%>\% pull(item)
# get top 10 bugs of hospital A as a vector
septic_patients \%>\%
filter(hospital_id == "A") \%>\%
freq(bactid) \%>\%
top_freq(10)
# save frequency table to an object
years <- septic_patients \%>\%
mutate(year = format(date, "\%Y")) \%>\%
freq(year)
# print only top 5
years \%>\% print(nmax = 5)
# transform to plain data.frame
septic_patients \%>\%
freq(age) \%>\%
as.data.frame()
}
\keyword{freq}
\keyword{frequency}

View File

@ -1,15 +1,12 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/freq.R, R/print.R
\name{print.frequency_tbl}
\alias{print.frequency_tbl}
% Please edit documentation in R/print.R
\name{print}
\alias{print}
\alias{print.tbl_df}
\alias{print.tbl}
\alias{print.data.table}
\title{Printing Data Tables and Tibbles}
\usage{
\method{print}{frequency_tbl}(x, ...)
\method{print}{tbl_df}(x, nmax = 10, header = TRUE, row.names = TRUE,
right = FALSE, width = 1, na = "<NA>", ...)
@ -20,8 +17,6 @@
\arguments{
\item{x}{object of class \code{data.frame}.}
\item{...}{optional arguments to \code{print} or \code{plot} methods.}
\item{nmax}{amount of rows to print in total. When the total amount of rows exceeds this limit, the first and last \code{nmax / 2} rows will be printed. Use \code{nmax = NA} to print all rows.}
\item{header}{print header with information about data size and tibble grouping}
@ -36,6 +31,8 @@
\item{na}{value to print instead of NA}
\item{...}{optional arguments to \code{print} or \code{plot} methods.}
\item{print.keys}{print keys for \code{data.table}}
}
\description{

View File

@ -1,89 +0,0 @@
## ----setup, include = FALSE, results = 'markup'--------------------------
knitr::opts_chunk$set(
collapse = TRUE,
comment = "#"
)
library(dplyr)
library(AMR)
## ---- echo = TRUE, results = 'hide'--------------------------------------
# just using base R
freq(septic_patients$sex)
# using base R to select the variable and pass it on with a pipe from the dplyr package
septic_patients$sex %>% freq()
# do it all with pipes, using the `select` function from the dplyr package
septic_patients %>%
select(sex) %>%
freq()
# or the preferred way: using a pipe to pass the variable on to the freq function
septic_patients %>% freq(sex) # this also shows 'age' in the title
## ---- echo = TRUE--------------------------------------------------------
freq(septic_patients$sex)
## ---- echo = TRUE, results = 'hide'--------------------------------------
my_patients <- septic_patients %>% left_join_microorganisms()
## ---- echo = TRUE--------------------------------------------------------
colnames(microorganisms)
## ---- echo = TRUE--------------------------------------------------------
dim(septic_patients)
dim(my_patients)
## ---- echo = TRUE--------------------------------------------------------
my_patients %>% freq(genus, species)
## ---- echo = TRUE--------------------------------------------------------
# # get age distribution of unique patients
septic_patients %>%
distinct(patient_id, .keep_all = TRUE) %>%
freq(age, nmax = 5)
## ---- echo = TRUE--------------------------------------------------------
septic_patients %>%
freq(hospital_id)
## ---- echo = TRUE--------------------------------------------------------
septic_patients %>%
freq(hospital_id, sort.count = TRUE)
## ---- echo = TRUE--------------------------------------------------------
septic_patients %>%
select(amox) %>%
freq()
## ---- echo = TRUE--------------------------------------------------------
septic_patients %>%
select(date) %>%
freq(nmax = 5)
## ---- echo = TRUE--------------------------------------------------------
my_df <- septic_patients %>% freq(age)
class(my_df)
## ---- echo = TRUE--------------------------------------------------------
dim(my_df)
## ---- echo = TRUE--------------------------------------------------------
septic_patients %>%
freq(amox, na.rm = FALSE)
## ---- echo = TRUE--------------------------------------------------------
septic_patients %>%
freq(hospital_id, row.names = FALSE)
## ---- echo = TRUE--------------------------------------------------------
septic_patients %>%
freq(hospital_id, markdown = TRUE)
## ---- echo = FALSE-------------------------------------------------------
# this will print "2018" in 2018, and "2018-yyyy" after 2018.
yrs <- c(2018:format(Sys.Date(), "%Y"))
yrs <- c(min(yrs), max(yrs))
yrs <- paste(unique(yrs), collapse = "-")