require(readr)
require(lubridate)

cat(paste0('\14\n', now(), '\n'))

oldDir = getwd()
basePath = '/share/mmbi/radar2'
setwd(basePath)

source('interfaces.R')
source('stitchArchive.R')
source('4D_archive.R')


dataPath = 'data'
archivePath = 'data/archive'

tFiles = read_rds(file.path(dataPath, 'tFilesInfo.rds'))
tFileOvw = read_rds(file.path(dataPath, 'tFileOvw.rds'))

if (today() >= '2022-09-30') {
  require(assertthat, quietly = T)
  
  # get the last date that is present in the data sofar
  maxDate = tFiles %>% pull(fileDate) %>% max(na.rm = T)
  # catch up with today
  nDaysToUpdate = difftime(today(), maxDate) %>% as.numeric()
  daysToUpdate = (maxDate + 1:nDaysToUpdate)
  for (d in 1:nDaysToUpdate) {
    currDate = daysToUpdate[d]
    # aDate is converted to Excel style numeric format with origin '1970-01-01'
    # currDate = as.Date(aDate, origin = lubridate::origin)
    tFiles = tFiles %>% 
      filter(!is.na(fileDate)) %>% 
      addFileInfoOfDay(folders = folders, 
                       ymdDate = currDate)
    tInfoDay = tFiles %>% filter(fileDate == currDate)
    a = assert_that(nrow(tInfoDay) > 0, 
                    msg = paste0('tInfoDay has 0 rows: no files present for ', 
                                 currDate, '?'))
    
  }
  
  tInfoDays = tFiles %>% filter(fileDate %in% daysToUpdate)
  tFileOvwToday = deriveFileOverview(tInfoDays)
  a = assert_that(nrow(tFileOvwToday) > 0, 
                  msg = 'derivederiveFileOverview() failed')
  
  tFileOvw = tFileOvw %>% ungroup() %>% 
    add_row(tFileOvwToday) %>% 
    filter(!is.na(fileDate))
  
  write_rds(tFiles, file.path(dataPath, 'tFilesInfo.rds'))
  write_rds(tFileOvw, file.path(dataPath, 'tFileOvw.rds'))
  
  # write GLIMMS data per date in 'daysToUpdate' and add it to the data sofar
  for (d in 1:nDaysToUpdate) {
    currDate = daysToUpdate[d]
    tGlimmsOfDay = readArchiveOfDay(tFiles, tFileOvw, ymdDate = currDate)
    a = assert_that(nrow(tGlimmsOfDay) > 0, 
                    msg = paste0('readArchiveDay() yielded empty result for ', 
                                 currDate))
    filename = paste0(dataPath,'/bk_', currDate, '_', currDate, '.rds')
    write_rds(tGlimmsOfDay, filename, compress = 'gz')
    
    filenameTotalSofar = paste0(dataPath, '/total_', currDate - 1, '.rds')
    filenameTotal      = paste0(dataPath, '/total_', currDate, '.rds')
    filenameOfDay      = filename
    
    tTotal = appendDailyData(fnTotal      = filenameTotal, 
                             fnTotalSofar = filenameTotalSofar, 
                             fnDay        = filenameOfDay, 
                             reqColumns   = abAbbrevs)
    # nNewRows = tTotal %>% filter(date == today()) %>% nrow()
    
    # write_rds(tTotal, file = filenameTotal, compress = 'gz')
  }
}



setwd(oldDir)