##############################################################################################
#' @title Create publication table

#' @author
#' Cove Sturtevant \email{csturtevant@battelleecology.org}

#' @description
#' Definition function. 


#' @param data A list of Arrow datasets, each list element containing an arrow dataset as retrieved 
#' by arrow::open_dataset. These datasets contain time timeseries data from which the data for the 
#' publication table will be gathered from. Note that the datasets all must have the same number of rows.
#' @param pubWb A data from of the publication workbook, as generated by NEONprocIS.pub::def.read.pub.wb 
#' @param TablPub Optional. Character value indicating the publication table to produce. If not input or 
#' the value is NULL, the entire pub workbook will be created. Note that duplicated field names in the 
#' pubWb (after filtering for the table of interest) will be removed.
#' @param log A logger object as produced by NEONprocIS.base::def.log.init to produce structured log
#' output. Defaults to NULL, in which the logger will be created and used within the function.

#' @return A named list:
#' dataTabl: A data frame of the publication table. Note that the column order is determined by the 
#'           order of the rows in the publication workbook, not from the Rank field. This is intentional, 
#'           so as to avoid conflicts if there are duplicated field names or Rank values.
#' nameVarMtch: A character vector with the names of the variables in the pub table that were found in 
#'              the input datasets.

#' @references
#' License: (example) GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007

#' @keywords Currently none

#' @examples 
#' Not Run
#' FileData <- c('/path/to/file1.parquet','/path/to/file2.parquet') # Files must have same # of rows
#' data <- base::lapply(FileData,arrow::open_dataset)
#' FilePubWb <- 'path/to/publication/workbook.txt'
#' pubWb <- NEONprocIS.pub::def.read.pub.wb(NameFile=FilePubWb)
#' TablPub <- 'PARQL_30min'
#' def.pub.tabl.crea(data,pubWb,TablPub)



#' @seealso \link[arrow]{dataset}
#' @seealso \link[NEONprocIS.pub]{def.read.pub.wb}

#' @export

# changelog and author contributions / copyrights
#   Cove Sturtevant (2023-02-22)
#     original creation
##############################################################################################
def.pub.tabl.crea <- function(data,
                              pubWb,
                              TablPub=NULL,
                              log = NULL
  ) {
    
  library(dplyr)
  
  # Initialize log if not input
  if (is.null(log)) {
    log <- NEONprocIS.base::def.log.init()
  }

  # Constrain pub workbook to table of interest
  if(!base::is.null(TablPub)){
    pubWb <- pubWb[pubWb$table==TablPub,]
  }
  
  # Remove duplicated field names in the pub table (this can happen when multiple pub workbooks are combined prior to input).
  pubWb <- pubWb[!base::duplicated(pubWb$fieldName),]
  nameVarTabl <- pubWb$fieldName
  numCol <- base::length(nameVarTabl)
  
  # initialize the pub table output
  dataTabl <- base::data.frame(base::matrix(NA,nrow=data[[1]]$num_rows,ncol=numCol),stringsAsFactors=FALSE)
  base::names(dataTabl) <- nameVarTabl
  
  
  # Remove any overlap in variables across the datasets (same name across files). Effectively takes the first instance
  nameVarData <- base::lapply(data,FUN=function(dataIdx){dataIdx$schema$names})
  numData <- base::length(data)
  if(numData > 1){
    for(idxFile in 2:numData){
      nameVarData[[idxFile]] <- base::setdiff(nameVarData[[idxFile]],base::unlist(nameVarData[1:(idxFile-1)]))
    }
  }
  
  # Grab the data for this table
  dataMtch <- base::vector(numData,mode='list')
  for(idx in base::seq_len(numData)){
    dataIdx <- data[[idx]]
    nameVarMtch <- nameVarTabl[nameVarTabl %in% nameVarData[[idx]]]
    dataMtch[[idx]] <- dataIdx %>% 
                      dplyr::select(dplyr::all_of(nameVarMtch)) %>%
                      collect()
  }
  nameVarMtch <- base::unlist(base::lapply(dataMtch,base::names))
  base::names(nameVarMtch) <- NULL
  dataMtch <- base::do.call(base::cbind,dataMtch)
  base::names(dataMtch) <- nameVarMtch
  dataTabl[,nameVarMtch] <- dataMtch
  
  return(list(dataTabl=dataTabl,
              nameVarMtch=nameVarMtch))
}
