69 lines
2.7 KiB
R
69 lines
2.7 KiB
R
source('interfaces.R')
|
|
source('createPatches.R')
|
|
source('stitchArchive.R')
|
|
|
|
dataPath = 'data'
|
|
inspectRead = 10
|
|
inspectReason = 'test selection of max IsolNr (only). mo and RIS can be NA'
|
|
|
|
reads = read_rds(file.path(dataPath, 'reads.rds'))
|
|
reads = reads %>% add_row(read = inspectRead, desription = inspectReason)
|
|
write_rds(reads, file.path(dataPath, 'reads.rds'))
|
|
|
|
tFiles = read_rds(file.path(dataPath, 'tFilesInfo.rds'))
|
|
tFileOvw = read_rds(file.path(dataPath, 'tFileOvw.rds'))
|
|
|
|
updateFileOvw = F
|
|
if (updateFileOvw) {
|
|
# on 2022-10-20 tFiles was up-to-date, due to the daily update.
|
|
tFiles = read_rds(file.path(dataPath, 'tFilesInfo.rds'))
|
|
# However, tFileOvw somehow wasn't
|
|
|
|
# get the last date that is present in the data sofar
|
|
maxDate = tFileOvw %>% pull(fileDate) %>% max(na.rm = T)
|
|
# catch up with today
|
|
nDaysToUpdate = difftime(today(), maxDate) %>% as.numeric()
|
|
daysToUpdate = (maxDate + 1:nDaysToUpdate)
|
|
|
|
tInfoDays = tFiles %>% filter(fileDate %in% daysToUpdate)
|
|
tFileOvwToday = deriveFileOverview(tInfoDays)
|
|
a = assert_that(nrow(tFileOvwToday) > 0,
|
|
msg = 'derivederiveFileOverview() failed')
|
|
|
|
tFileOvw = tFileOvw %>% ungroup() %>%
|
|
add_row(tFileOvwToday) %>%
|
|
filter(!is.na(fileDate))
|
|
|
|
# write_rds(tFiles, file.path(dataPath, 'tFilesInfo.rds')) Was up-to-date on 2022-10-20
|
|
write_rds(tFileOvw, file.path(dataPath, 'tFileOvw.rds'))
|
|
}
|
|
|
|
# define the patch periods
|
|
tPeriods = tribble(
|
|
~startDate, ~endDate,
|
|
ymd('2022-01-01'), ymd('2022-02-01') - 1,
|
|
ymd('2022-02-01'), ymd('2022-03-01') - 1
|
|
# ymd('2022-03-01'), ymd('2022-04-01') - 1,
|
|
# ymd('2022-04-01'), ymd('2022-05-01') - 1,
|
|
# ymd('2022-05-01'), ymd('2022-06-01') - 1,
|
|
# ymd('2022-06-01'), ymd('2022-07-01') - 1,
|
|
# ymd('2022-07-01'), ymd('2022-08-01') - 1,
|
|
# ymd('2022-08-01'), ymd('2022-09-01') - 1,
|
|
# ymd('2022-09-01'), ymd('2022-10-01') - 1,
|
|
# ymd('2022-10-01'), ymd('2022-10-20') - 1
|
|
# ymd('2022-09-01'), ymd('2022-10-20')
|
|
)
|
|
|
|
createPatches(tPeriods, dataPath, tFiles, tFileOvw)
|
|
# stitch together patches created by createPatches.R
|
|
# readAllPatches() assumes that all bk_*.rds files are in the 'data' path and
|
|
# that they all need to be read for stitching
|
|
tPatches = readAllPatches()# %>% filter(from >= '2022-01-01', to <= (ymd('2022-05-01') - 1))
|
|
minFrom = min(tPatches$from, na.rm = T)
|
|
maxTo = max(tPatches$to, na.rm = T)
|
|
cat(paste0('Stitching patches: ', minFrom, ' - ', maxTo, '\n'))
|
|
tStitched = stitchPatches(tPatches, dataPath = 'data', reqColumns = abAbbrevs)
|
|
tStitched %>% write_rds(file = paste0('data/read_', inspectRead, '_', minFrom, '_',
|
|
maxTo, '_x', today(), '.rds'),
|
|
compress = 'gz')
|