82 lines
2.7 KiB
R
82 lines
2.7 KiB
R
require(readr)
|
|
require(lubridate)
|
|
|
|
cat(paste0('\14\n', now(), '\n'))
|
|
|
|
oldDir = getwd()
|
|
basePath = '/share/mmbi/radar2'
|
|
setwd(basePath)
|
|
|
|
source('interfaces.R')
|
|
source('stitchArchive.R')
|
|
source('4D_archive.R')
|
|
|
|
|
|
dataPath = 'data'
|
|
archivePath = 'data/archive'
|
|
|
|
tFiles = read_rds(file.path(dataPath, 'tFilesInfo.rds'))
|
|
tFileOvw = read_rds(file.path(dataPath, 'tFileOvw.rds'))
|
|
|
|
if (today() >= '2022-09-30') {
|
|
require(assertthat, quietly = T)
|
|
|
|
# get the last date that is present in the data sofar
|
|
maxDate = tFiles %>% pull(fileDate) %>% max(na.rm = T)
|
|
# catch up with today
|
|
nDaysToUpdate = difftime(today(), maxDate) %>% as.numeric()
|
|
daysToUpdate = (maxDate + 1:nDaysToUpdate)
|
|
for (d in 1:nDaysToUpdate) {
|
|
currDate = daysToUpdate[d]
|
|
# aDate is converted to Excel style numeric format with origin '1970-01-01'
|
|
# currDate = as.Date(aDate, origin = lubridate::origin)
|
|
tFiles = tFiles %>%
|
|
filter(!is.na(fileDate)) %>%
|
|
addFileInfoOfDay(folders = folders,
|
|
ymdDate = currDate)
|
|
tInfoDay = tFiles %>% filter(fileDate == currDate)
|
|
a = assert_that(nrow(tInfoDay) > 0,
|
|
msg = paste0('tInfoDay has 0 rows: no files present for ',
|
|
currDate, '?'))
|
|
|
|
}
|
|
|
|
tInfoDays = tFiles %>% filter(fileDate %in% daysToUpdate)
|
|
tFileOvwToday = deriveFileOverview(tInfoDays)
|
|
a = assert_that(nrow(tFileOvwToday) > 0,
|
|
msg = 'derivederiveFileOverview() failed')
|
|
|
|
tFileOvw = tFileOvw %>% ungroup() %>%
|
|
add_row(tFileOvwToday) %>%
|
|
filter(!is.na(fileDate))
|
|
|
|
write_rds(tFiles, file.path(dataPath, 'tFilesInfo.rds'))
|
|
write_rds(tFileOvw, file.path(dataPath, 'tFileOvw.rds'))
|
|
|
|
# write GLIMMS data per date in 'daysToUpdate' and add it to the data sofar
|
|
for (d in 1:nDaysToUpdate) {
|
|
currDate = daysToUpdate[d]
|
|
tGlimmsOfDay = readArchiveOfDay(tFiles, tFileOvw, ymdDate = currDate)
|
|
a = assert_that(nrow(tGlimmsOfDay) > 0,
|
|
msg = paste0('readArchiveDay() yielded empty result for ',
|
|
currDate))
|
|
filename = paste0(dataPath,'/bk_', currDate, '_', currDate, '.rds')
|
|
write_rds(tGlimmsOfDay, filename, compress = 'gz')
|
|
|
|
filenameTotalSofar = paste0(dataPath, '/total_', currDate - 1, '.rds')
|
|
filenameTotal = paste0(dataPath, '/total_', currDate, '.rds')
|
|
filenameOfDay = filename
|
|
|
|
tTotal = appendDailyData(fnTotal = filenameTotal,
|
|
fnTotalSofar = filenameTotalSofar,
|
|
fnDay = filenameOfDay,
|
|
reqColumns = abAbbrevs)
|
|
# nNewRows = tTotal %>% filter(date == today()) %>% nrow()
|
|
|
|
# write_rds(tTotal, file = filenameTotal, compress = 'gz')
|
|
}
|
|
}
|
|
|
|
|
|
|
|
setwd(oldDir) |