app.py

# Geodata-Harvester App
# Description: Streamlit app for the Geodata-Harvester package
# Author: Sebastian Haan

import streamlit as st
import sys
import os
import pandas as pd
import numpy as np
import yaml
import shutil
from types import SimpleNamespace
from PIL import Image
# Import the core package Geodata-Harvester
from geodata_harvester import harvest


## Limitations:
# The Google Earth Engine extension for Geodata-Harvester is not natively supported in Streamlit, 
# unless you run the app locally or provide GEE service account token.

FNAME_ZIP = 'harvest_collection.zip'
OUTPATH = 'harvest_collection'

# Link to settings templates
settings_template_link = 'https://huggingface.co/spaces/SIH/geodata-harvester-app/tree/main/data/templates_settings'
# link to data template
data_template_link = 'https://huggingface.co/spaces/SIH/geodata-harvester-app/tree/main/data'
# link to github page
link_to_githubpage = 'https://sydney-informatics-hub.github.io/geodata-harvester/'
# link to settings overview
link_to_settings_overview = 'https://github.com/Sydney-Informatics-Hub/geodata-harvester/blob/main/quarto/docs/Settings_Overview.md'

menu_dict = {
    "About": "https://sydney-informatics-hub.github.io/geodata-harvester/",
    "Get help": "https://github.com/Sydney-Informatics-Hub/geodata-harvester/"}

def init_header():
    # Add a title
    st.title('Geodata-Harvester App')
    st.subheader("Jumpstart your geospatial analysis.")
    image = Image.open('assets/dataharvester_streamlit_logo_blue.jpg')
    st.image(image)
    st.write(f"The Geodata-Harvester automates geodata download and spatio-temporal processing from a large range of datasources into ready-made datasets.\
        This application is a lightweight Streamlit wrapper crafted to work seamlessly with the Geodata-Harvester package.\
        For full accessibility to a comprehensive set of features and options, please visit the [Geodata-Harvester project]({link_to_githubpage}).")
    #st.divider()

def init_howto():
    #with workpanel.container():
    st.subheader('How to')
    st.write(f'1) Prepare a settings file that lists all requested layers and spatio-temporal settings (see [templates]({settings_template_link})). For more info please see section below.')
    st.write('2) Upload the settings file in the sidebar.')
    st.write('3) Optional: Select csv file including Latitude and Longitude points. This will enable the Geodata-Harvester to automatically populate the table with extracted data for all given locations.')
    st.write('4) Then click `Run Geodata-Harvester` on the top of the sidebar.')
    st.write('5) Download and processing of all data might take a couple of minutes. Once completed, you will be able to download all generated results as zip.')

def print_attr():
    st.subheader('Attributions and Acknowledgements')
    st.write('This software was developed by the Sydney Informatics Hub, a core research facility of the University of Sydney.')
    st.write('Acknowledgments are an important way for us to demonstrate the value we bring to your research. Your research outcomes are vital for ongoing funding of the Sydney Informatics Hub.')
    st.write('If you make use of this software for your research project, please include the following acknowledgment:')
    st.write('“This research was supported by the Sydney Informatics Hub, a Core Research Facility of the University of Sydney."')

def load_settings(settings_file):
    if settings_file is not None:
        fname_settings = settings_file#.getvalue()
        settings = yaml.load(settings_file, Loader=yaml.FullLoader)
        # Parse settings dictionary as namespace
        settings = SimpleNamespace(**settings)
        settings.date_min = str(settings.date_min)
        settings.date_max = str(settings.date_max)
        if st.session_state['data_file'] is not None:
            settings.infile = st.session_state['data_file']
        settings.outpath = OUTPATH
        return settings

def print_settings(settings):
    if settings:
        #with workpanel.container():
        st.header("Settings loaded:")
        for key in settings.__dict__:
            if key == "target_sources":
                st.write(f"settings.{key}:")
                for source in settings.target_sources:
                    st.write(f"   '{source}': {settings.target_sources[source]}")
            else:
                st.write(f"settings.{key} : {settings.__dict__[key]}")


def print_info():
    st.subheader('Settings Overview')
    st.write('The Geodata-Harvester is controlled by a settings file in YAML format, \
        which includes all user-defined settings and data layers for extraction and processing.')
    st.write('Example settings file can be found in the [templates]({settings_template_link}).')
    st.write('The settings file includes the following sections:')
    st.markdown("""yaml

    #Bounding Box as (lng_min, lat_min, lng_max, lat_max):
    #If not provided, the Geodata-Harvester will try to infer bbox from points in Locations file
    target_bbox: [149, -30, 149.9, -29]

    #Select start date:
    date_min: : 2023-01-01

    #Select end date:
    date_max: : 2023-02-01

    #Spatial Resolution [in arcsec]:
    target_res: 100.0

    # Number of time interval slices in given date range
    temp_intervals: 1 
    #Headername of Latitude in input file (only relevant if data file provided):
    colname_lat: Lat

    #Headername of Longitude in input file (only relevant if data file provided):
    colname_lng: Long

    target_sources:
    #Satellite data from Digital Earth Australia (optional)
    DEA:
    - landsat_barest_earth

    #National Digital Elevation Model (DEM) 1 Second (optional)
    DEM:
    - DEM
    - Slope
    - Aspect
    
    #Landscape Data (optional)
    Landscape:
    - Relief_300m

    #Radiometric Data (optional)
    Radiometric:
    - radmap2019_grid_dose_terr_awags_rad_2019
    - radmap2019_grid_dose_terr_filtered_awags_rad_2019

    # SILO Climate Data (optional)
    # temporal aggregation options: 'mean', 'median', 'sum', 'std', 'perc95', 'perc5', 'max', 'min'
    SILO:
        max_temp:
        - Median
        min_temp:
        - Median

    #Soil data from SLGA (optional)
    SLGA:
    Bulk_Density:
        - 0-5cm
    Clay:
        - 0-5cm
    """)
    st.subheader('Data Overview')
    # Data Overview list
    st.markdown("""
    The following data sources are currently supported by the Geodata-Harvester Streamlit app:
    - Soil Data 3D SLGA (Australia)
    - SILO Climate Database (Australia)
    - National Digital Elevation Model incl. Slope, Aspect (Australia)
    - Digital Earth Australia Geoscience Earth Observations (incl. Sentinel, Landsat, MODIS for Australia)
    - Radiometric Data (Australia)
    - Landscape Data (Australia)
    For more information, please visit the [Geodata-Harvester Data Overview]({https://github.com/Sydney-Informatics-Hub/geodata-harvester/blob/main/quarto/docs/Data_Overview.md}).
    Note that Google Earth Engine is not supported in streamlit app unless you have a GEE token or run the app locally.
    """)

def load_data(data_file):
    # load data into pandas dataframe from csv file
    if data_file is not None:
        #st.sidebar.write(f"Loading data from {data_file}")
        df = pd.read_csv(data_file)
        return df

def show_results():
    df_results = pd.read_csv(st.session_state['fname_results'])
    st.write('Extracted Data Table:')
    st.write(df_results)

def zip_folder(folder_path):
    filename = FNAME_ZIP.split('.')[0]
    directory = folder_path
    shutil.make_archive(filename, "zip", folder_path)

def gee_init():
    # initialize earth engine (experimental)
    st.sidebar.header('Google Earth Engine (Optional)')
    st.sidebar.write(f'Please copy your Google Earth Engine authentication token in the field below.')
    os.environ['EARTHENGINE_TOKEN'] = st.sidebar.text_input(
            label="Enter Google Earth Engine Token:",
            type="password"
        ).strip()

def open_sidebar():
    # Add run 
    st.sidebar.header('Run Geodata-Harvester')
    st.sidebar.write(f'This will harvest geodata as specified in the settings file below.')
    button_run = st.sidebar.button('Run Geodata-Harvester')

    st.sidebar.divider()

    # Add yaml file settings file upload
    st.sidebar.header('Settings File')
    settings_file = st.sidebar.file_uploader(f"Choose a yaml settings file (see [templates]({settings_template_link}))")
    button_show_settings = st.sidebar.button('Show Settings')
    st.session_state['settings_file'] = settings_file

    st.sidebar.divider()

    # Add data file uploader
    st.sidebar.header('Locations File')
    data_file = st.sidebar.file_uploader(f"Upload a CSV file with Latitudes and Longitudes (see [example data csv]({data_template_link}))")
    button_show_data = st.sidebar.button('Show Input Data')
    st.session_state['data_file'] = data_file

    st.sidebar.divider()

    # optional
    gee_init()
    
    return button_show_settings, button_show_data, button_run

def run_harvester():
    if ('settings_file' in st.session_state) & (st.session_state['settings_file'] is not None):
        fname_settings = st.session_state['settings_file']
        settings = load_settings(fname_settings)
        if ('data_file2' in st.session_state) & (st.session_state['data_file2'] is not None):
            settings.infile = st.session_state['data_file2'] #.name
            #st.write(f"settings.infile: {settings.infile}")
        #outpath = settings.outpath
        os.makedirs(name=OUTPATH, exist_ok=True)
        # write settings to disk
        with open(os.path.join(OUTPATH,'settings.yaml'), 'w') as file:
            yaml.dump(settings.__dict__, file)
        with st.spinner('Running Geodata-Harvester...'):
            df = harvest.run(os.path.join(OUTPATH,'settings.yaml'), return_df = True)
        st.success('Harvest complete!', icon="✅")
        return df 
    else:
        st.error('Please upload settings file.')
        return None



##########   Main  ###########

def main():
    #st.set_page_config(layout="wide")
    st.set_page_config(page_title="Geodata-Harvester App",
        layout="centered",
        menu_items=menu_dict)
    #st.markdown(footer, unsafe_allow_html=True)
    init_header()
    with st.expander("How to use this app"):
        init_howto()
    with st.expander("Settings and Data Overview"):
        print_info()
    with st.expander("Attributions and Acknowledgements"):
        print_attr()
    #st.divider()
    workpanel = st.empty()
    button_show_settings, button_show_data, button_run = open_sidebar()

    if button_show_settings:
        settings = load_settings(st.session_state['settings_file'])
        print_settings(settings)

    if button_show_data:
        data_file = st.session_state['data_file']
        df = load_data(data_file)
        st.header("Data Table:")
        st.write(df)

    if button_run:
        if ('data_file' in st.session_state) & (st.session_state['data_file'] is not None):
            # save data so it can be read by geopandas later
            df = load_data(st.session_state['data_file'])
            df.to_csv('temp_data.csv')
            st.session_state['data_file2'] = 'temp_data.csv'
        else:
            st.session_state['data_file2'] = None
        if ('settings_file' in st.session_state) & (st.session_state['settings_file'] is not None):
            df_results = run_harvester()
            # zip result folder in python and make available for download
            with st.spinner(f'Generating {FNAME_ZIP} file for download...'):
                zip_folder(OUTPATH)
                st.session_state['fname_results_zip'] = FNAME_ZIP
                st.success('Zip file generated!', icon="✅")
            # delete temp data file
            if os.path.exists('temp_data.csv'):
                os.remove('temp_data.csv')
            #shutil.rmtree(outpath)
            # show results
            st.session_state['fname_results'] = os.path.join(OUTPATH,'results.csv')
        else:
            #workpanel.empty()
            #with workpanel.container():
            st.text('No settings file provided. Please add settings file.')

    # Show options and buttons after run
    if 'fname_results_zip' in st.session_state:
        st.write('Harvester Results:')
        with open(FNAME_ZIP, 'rb') as f:
            st.download_button('Download Results (zip)', f, file_name=FNAME_ZIP)
        if os.path.exists(st.session_state['fname_results']):
            button_show_results = st.button('Show Result Table')
            if button_show_results:
                show_results()


if __name__ == "__main__":
    main()


#### Debugging options
#st.write('Streamlit version:', st.__version__)
#st.write('Session state:', st.session_state)
#for key in st.session_state:
#    st.write(f"{key}: {st.session_state[key]}")