git renew

.gitignore

@@ -1,6 +1,4 @@
-test_run.py
-.DS_Store
-analyzer.py
-main.py
-results/*
-inputtexts/*
+ECCE_analysis
+ECCE_texts
+results
+inputtexts

analyzer.py

@@ -0,0 +1,146 @@
+import re
+import os
+import spacy_streamlit
+from collections import Counter
+import glob
+
+import spacy
+from spacy.tokens import Doc
+from spacy.cli._util import import_code
+
+from utils.visualize import visualize_spans
+from utils.util import preprocess, delete_overlapping_span, cleanup_justify
+
+from resources.text_list import TEXT_LIST
+from resources.template_list import TPL_SPAN, TPL_SPAN_SLICE, TPL_SPAN_START
+from resources.colors import COLORS_1
+
+
+from skbio import diversity as dv
+
+from pipeline.post_processors import simple_table, const_table, ngrammar, diversity_values
+import pandas as pd
+
+# from pipeline.custom_functions import custom_functions
+SPAN_ATTRS = ["text", "label_", "start", "end"]
+CATEGORIES = ['ATTRIBUTION', "CITATION", "COUNTER", "DENY", "ENDOPHORIC", "ENTERTAIN", "JUSTIFYING", "MONOGLOSS", "PROCLAIM", "SOURCES"]
+
+
+# spacy.prefer_gpu()
+
+def load_model(spacy_model):
+    # source = spacy.blank("en")
+    nlp = spacy.load(spacy_model)  # , vocab=nlp_to_copy.vocab
+    nlp.add_pipe('sentencizer')
+    return (nlp)
+
+# source = spacy.blank("en")
+
+# modelname = "en_engagement_LSTM_f5"
+# modelname = "en_engagement_LSTM_f5"
+modelname = "en_engagement_Dual_RoBERTa_acad3_f4"
+
+os.makedirs(os.path.join("/Users/masakieguchi/Dropbox/0_Projects/0_basenlp/SFLAnalyzer/engagement-analyzer-demo/results", modelname))
+
+import_code("pipeline/custom_functions.py")
+
+# nlp = spacy.load("en_engagement_three_RoBERTa_base_LSTM384")
+nlp = spacy.load(modelname)
+# doc = nlp(preprocess(TEXT_LIST[0]))
+
+# cleanup_justify(doc, doc.spans["sc"])
+# delete_overlapping_span(doc.spans['sc'])
+
+# data, cols = const_table(doc, spans_key='sc', attrs=SPAN_ATTRS)
+# seq = [s for s in doc.spans["sc"]]
+# span_ngrams = ngrammar(seq=seq, n=3)
+
+# df = pd.DataFrame(data, columns=cols)
+
+# constant_value = 42
+# new_col = pd.Series([constant_value] * df.shape[0], name='new_col')
+
+# doclen = len(doc)
+# doc_len = pd.Series([doclen] * df.shape[0], name='nwords')
+
+# df.insert(0, "new", new_col, True)
+# df.insert(1, "nwords", doc_len, True)
+
+# df.to_csv("results/test.csv")
+
+
+
+inputfiles = glob.glob("ECCE_texts/preprocessed/*.txt")
+savedir = "ECCE_analysis"
+storeall = True
+storage = []
+os.makedirs(os.path.join("ECCE_analysis", modelname))
+
+
+doc_level_storage = []
+
+for file in inputfiles:
+
+    filename = os.path.split(file)[-1]
+
+    with open(file, "r") as f:
+        text = f.read()
+
+    text = preprocess(text)
+    doc = nlp(text)
+    cleanup_justify(doc, doc.spans["sc"])
+    delete_overlapping_span(doc.spans['sc'])
+
+    data, cols = const_table(doc, spans_key='sc', attrs=SPAN_ATTRS)
+    seq = [s for s in doc.spans["sc"]]
+    span_ngrams = ngrammar(seq=seq, n=3)
+
+
+    ### Make it a dataset
+    df = pd.DataFrame(data, columns=cols)
+    df = df.astype({"start": int, "end": int}) #convert col type
+    df = df.sort_values(by= ['start']) #and sort by start
+    # constant_value = 42
+    new_col = pd.Series([filename] * df.shape[0], name='filename')
+
+    doclen = len(doc)
+    doc_len = pd.Series([doclen] * df.shape[0], name='nwords')
+
+    df.insert(0, "filename", new_col, True)
+    df.insert(1, "nwords", doc_len, True)
+    df.to_csv(f"{savedir}/{modelname}/{filename}.csv")
+
+    sequences = list(df['label_'])
+    # Engagement ngrams
+    span_bigrams = ngrammar(seq=seq, n=2)
+    bidf = pd.DataFrame(span_bigrams)
+
+    # constant_value = 42
+    new_col = pd.Series([filename] * bidf.shape[0], name='filename')
+    bidf = bidf.insert(0, "filename", new_col, True)
+
+
+    ## Document level 
+    doc_level = {}
+    counts = df['label_'].value_counts().reindex(CATEGORIES, fill_value=0)
+    div = diversity_values(list(counts))
+
+    div_data = pd.DataFrame.from_dict(div, orient='index')
+    
+    doc_data = pd.concat([counts, div_data], axis = 0).T
+    doc_data.insert(0, "filename", filename, True)
+    doc_data.insert(1, "nwords", doc_len, True)
+    doc_data.to_csv(f"{savedir}/{modelname}/ddata_{filename}.csv")
+
+    if storeall:
+        storage.append(df)
+        doc_level_storage.append(doc_data)
+
+
+alldf = pd.concat(storage)
+
+alldf.to_csv(f"{savedir}/0_{modelname}_20230426.csv")
+
+
+alldoc = pd.concat(doc_level_storage)
+alldoc.to_csv(f"{savedir}/1_{modelname}_doc_20230426.csv")

main.py

@@ -0,0 +1,217 @@
+import re
+import spacy_streamlit
+from spacy_streamlit import visualize_parser
+from collections import Counter
+
+import spacy
+import streamlit as st
+
+# try:
+#     from .scripts.custom_functions import build_mean_max_reducer1, build_mean_max_reducer2, build_mean_max_reducer3
+# except ImportError:
+#     from pipeline.custom_functions import build_mean_max_reducer1, build_mean_max_reducer2, build_mean_max_reducer3
+from spacy.tokens import Doc
+from spacy.cli._util import import_code
+
+from utils.visualize import visualize_spans
+from utils.util import preprocess, delete_overlapping_span, cleanup_justify
+from resources.text_list import TEXT_LIST
+from resources.text_list_BAWE import TEXT_LIST_BAWE
+from resources.template_list import TPL_SPAN, TPL_SPAN_SLICE, TPL_SPAN_START
+from resources.colors import COLORS_1
+
+import_code("pipeline/custom_functions.py")
+st.set_page_config(page_title='Engagement model comparaer', layout="wide")
+
+# spacy.prefer_gpu()
+
+MODEL_LIST =['en_engagement_LSTM', 'en_engagement_LSTM']
+
+# MODEL_LIST = [
+#     'en_engagement_three_RoBERTa_base_LSTM384-0.9.2/en_engagement_three_RoBERTa_base_LSTM384/en_engagement_three_RoBERTa_base_LSTM384-0.9.2',
+#     'en_engagement_three_RoBERTa_acad3_db-0.9.2/en_engagement_three_RoBERTa_acad3_db/en_engagement_three_RoBERTa_acad3_db-0.9.2',
+#     'silver-sweep-34/model-best',
+#     'expert-sweep-4/model-best',
+#     'confused-sweep-6/model-best',
+#     'warm-sweep-20/model-best',
+#     "en_engagement_three_RoBERTa_base-1.10.0/en_engagement_three_RoBERTa_base/en_engagement_three_RoBERTa_base-1.10.0",
+#     "en_engagement_three_RoBERTa_acad_db-1.10.0/en_engagement_three_RoBERTa_acad_db/en_engagement_three_RoBERTa_acad_db-1.10.0",
+#     "en_engagement_para_RoBERTa_acad_db3-0.9.0/en_engagement_para_RoBERTa_acad_db3/en_engagement_para_RoBERTa_acad_db3-0.9.0",
+#     "en_engagement_para_RoBERTa_acad_LSTM2-0.9.0/en_engagement_para_RoBERTa_acad_LSTM2/en_engagement_para_RoBERTa_acad_LSTM2-0.9.0",
+#     "en_engagement_three_RoBERTa_acad_db3-0.9.1/en_engagement_three_RoBERTa_acad_db3/en_engagement_three_RoBERTa_acad_db3-0.9.1",
+#     "en_engagement_three_RoBERTa_acad_LSTM2-0.9.1/en_engagement_three_RoBERTa_acad_LSTM2/en_engagement_three_RoBERTa_acad_LSTM2-0.9.1",
+#     "en_engagement_three_RoBERTa_acad_db3-0.9.2/en_engagement_three_RoBERTa_acad_db3/en_engagement_three_RoBERTa_acad_db3-0.9.2",
+#     'en_engagement_spl_RoBERTa_acad_db-0.7.4/en_engagement_spl_RoBERTa_acad_db/en_engagement_spl_RoBERTa_acad_db-0.7.4',
+#     'en_engagement_spl_RoBERTa_acad_db3-0.9.0/en_engagement_spl_RoBERTa_acad_db3/en_engagement_spl_RoBERTa_acad_db3-0.9.0',
+#     'en_engagement_spl_RoBERTa_acad_LSTM-0.7.2/en_engagement_spl_RoBERTa_acad_LSTM/en_engagement_spl_RoBERTa_acad_LSTM-0.7.2',
+#     'en_engagement_spl_RoBERTa_acad_512',
+#     'en_engagement_spl_RoBERTa_acad',
+#     'en_engagement_spl_RoBERTa_exp-0.6.5/en_engagement_spl_RoBERTa_exp/en_engagement_spl_RoBERTa_exp-0.6.5',
+#     # 'en_engagement_spl_RoBERTa_acad-0.3.4.1221/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.3.4.1221',
+#     # 'en_engagement_spl_RoBERTa_acad-0.2.2.1228/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.2.2.1228',
+#     # 'en_engagement_spl_RoBERTa_acad-0.2.1.1228/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.2.1.1228',
+#     # 'en_engagement_spl_RoBERTa_acad-0.2.2.1220/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.2.2.1220',
+#     # 'en_engagement_spl_RoBERTa2-0.2.2.1210/en_engagement_spl_RoBERTa2/en_engagement_spl_RoBERTa2-0.2.2.1210',
+#     # 'en_engagement_spl_RoBERTa-0.2.2.1210/en_engagement_spl_RoBERTa/en_engagement_spl_RoBERTa-0.2.2.1210',
+#     # 'en_engagement_spl_RoBERTa_acad_max1_do02',
+#     # 'en_engagement_spl_RoBERTa2-0.2.2.1210/en_engagement_spl_RoBERTa2/en_engagement_spl_RoBERTa2-0.2.2.1210',
+#     # 'en_engagement_spl_RoBERTa_acad-0.2.3.1210/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.2.3.1210',
+#     # 'en_engagement_spl_RoBERTa_acad_max1_do02',
+#     # 'en_engagement_spl_RoBERTa_sqbatch_RAdam-20221202_0.1.5/en_engagement_spl_RoBERTa_sqbatch_RAdam/en_engagement_spl_RoBERTa_sqbatch_RAdam-20221202_0.1.5',
+#     # 'en_engagement_spl_RoBERTa_context_flz-20221130_0.1.4/en_engagement_spl_RoBERTa_context_flz/en_engagement_spl_RoBERTa_context_flz-20221130_0.1.4',
+#     # 'en_engagement_spl_RoBERTa_cx_max1_do2-20221202_0.1.5/en_engagement_spl_RoBERTa_cx_max1_do2/en_engagement_spl_RoBERTa_cx_max1_do2-20221202_0.1.5',
+#     # 'en_engagement_spl_RoBERTa_context_flz-20221125_0.1.4/en_engagement_spl_RoBERTa_context_flz/en_engagement_spl_RoBERTa_context_flz-20221125_0.1.4',
+#     # 'en_engagement_RoBERTa_context_flz-20221125_0.1.4/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-20221125_0.1.4',
+#     # 'en_engagement_RoBERTa_context_flz-20221117_0.1.3/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-20221117_0.1.3',
+#     # 'en_engagement_spl_RoBERTa_acad_context_flz-20221117_0.1.3/en_engagement_spl_RoBERTa_acad_context_flz/en_engagement_spl_RoBERTa_acad_context_flz-20221117_0.1.3',
+#     # 'en_engagement_RoBERTa_context_flz-Batch2_0.1.1/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-Batch2_0.1.1',
+#     # 'en_engagement_RoBERTa_context_flz-20221113_0.1.3/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-20221113_0.1.3',
+#     # 'en_engagement_RoBERTa_context_flz-20221113_0.1.1/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-20221113_0.1.1',
+#     # 'en_engagement_RoBERTa-0.0.2/en_engagement_RoBERTa/en_engagement_RoBERTa-0.0.2',
+#     # 'en_engagement_RoBERTa_combined-Batch2Eng_0.2/en_engagement_RoBERTa_combined/en_engagement_RoBERTa_combined-Batch2Eng_0.2',
+#     # 'en_engagement_RoBERTa_acad-0.2.1/en_engagement_RoBERTa_acad/en_engagement_RoBERTa_acad-0.2.1',
+#     # # 'en_engagement_BERT-0.0.2/en_engagement_BERT/en_engagement_BERT-0.0.2',
+#     # # 'en_engagement_BERT_acad-0.0.2/en_engagement_BERT_acad/en_engagement_BERT_acad-0.0.2',
+#     # # 'en_engagement_RoBERTa_acad-0.0.2/en_engagement_RoBERTa_acad/en_engagement_RoBERTa_acad-0.0.2',
+#     # 'en_engagement_RoBERTa-0.0.1/en_engagement_RoBERTa/en_engagement_RoBERTa-0.0.1',
+#     # # ' en_engagement_RoBERTa_sent-0.0.1_null/en_engagement_RoBERTa_sent/en_engagement_RoBERTa_sent-0.0.1_null',
+#     # # 'en_engagement_RoBERTa_combined-0.0.1/en_engagement_RoBERTa_combined/en_engagement_RoBERTa_combined-0.0.1',
+#     # 'en_engagement_RoBERTa-ME_AtoE/en_engagement_RoBERTa/en_engagement_RoBERTa-ME_AtoE',
+#     # 'en_engagement_RoBERTa-AtoI_0.0.3/en_engagement_RoBERTa/en_engagement_RoBERTa-AtoI_0.0.3',
+#     # 'en_engagement_RoBERTa-AtoI_0.0.3/en_engagement_RoBERTa/en_engagement_RoBERTa-AtoI_0.0.2'
+# ]
+
+multicol = st.checkbox("Compare two models", value=False, key=None, help=None)
+
+model1 = st.selectbox('Select model option 1', MODEL_LIST, index=0)
+model2 = st.selectbox('Select model option 2', MODEL_LIST, index=1)
+
+if '/' in model1:
+    model1 = "packages/" + model1
+
+if '/' in model2:
+    model2 = "packages/" + model2
+
+
+@st.cache(allow_output_mutation=True)
+def load_model(spacy_model):
+    # source = spacy.blank("en")
+    nlp = spacy.load(spacy_model) #, vocab=nlp_to_copy.vocab
+    nlp.add_pipe('sentencizer')
+    return (nlp)
+
+# source = spacy.blank("en")
+nlp = load_model(model1)
+
+if multicol:
+    nlp2 = load_model(model2)
+
+
+text = st.selectbox('select sent to debug', TEXT_LIST_BAWE)
+
+input_text = st.text_area("", height=200)
+
+# Dependency parsing
+st.header("Text", "text")
+if len(input_text.split(" ")) > 1:
+    doc = nlp(preprocess(input_text))
+    if multicol:
+        doc2 = nlp2(preprocess(input_text))
+    # st.markdown("> " + input_text)
+else:
+    doc = nlp(preprocess(text))
+    if multicol:
+        doc2 = nlp2(preprocess(text))
+    # st.markdown("> " + text)
+
+clearjustify = st.checkbox(
+    "Clear problematic JUSTIFYING spans", value=True, key=None, help=None)
+
+delete_overlaps = st.checkbox(
+    "Delete overlaps", value=True, key=None, help=None)
+
+# combine = st.checkbox(
+#     "Combine", value=False, key=None, help=None)
+
+# import copy
+# def combine_spangroups(doc1, doc2):
+#     # new_doc = Doc.from_docs([doc1, doc2], ensure_whitespace=True)
+#     new_doc = copy.deepcopy(doc1)
+#     # type()
+#     new_doc.spans['sc'].extend(doc2.spans['sc'])
+
+#     return new_doc
+
+
+# if combine:
+#     new_doc = combine_spangroups(doc, doc2)
+#     visualize_spans(new_doc,
+#                     spans_key="sc",
+#                     title='Combined spans:',
+#                     displacy_options={
+#                         'template': {
+#                               "span": TPL_SPAN,
+#                             'slice': TPL_SPAN_SLICE,
+#                             'start': TPL_SPAN_START,
+#                         },
+#                         "colors": COLORS_1,
+#                     },
+#                     simple=False)
+
+if clearjustify:
+    cleanup_justify(doc, doc.spans['sc'])
+
+if delete_overlaps:
+    delete_overlapping_span(doc.spans['sc'])
+    if multicol:
+        delete_overlapping_span(doc2.spans['sc'])
+
+if not multicol:
+    visualize_spans(doc,
+                    spans_key="sc",
+                    title='Engagement Span Anotations 1',
+                    displacy_options={
+                        'template': {
+                              "span": TPL_SPAN,
+                            'slice': TPL_SPAN_SLICE,
+                            'start': TPL_SPAN_START,
+                        },
+                        "colors": COLORS_1,
+                    },
+                    simple=False)
+
+
+else:
+    col1, col2 = st.columns(2)
+
+    with col1:
+        visualize_spans(doc,
+                        spans_key="sc",
+                        title='Engagement Span Anotations 1',
+                        displacy_options={
+                            'template': {
+                                "span": TPL_SPAN,
+                                'slice': TPL_SPAN_SLICE,
+                                'start': TPL_SPAN_START,
+                            },
+                            "colors": COLORS_1,
+                        },
+                        simple=False)
+
+    with col2:
+        visualize_spans(doc2,
+                        spans_key="sc",
+                        title='Engagement Span Anotations 2',
+                        displacy_options={
+                            'template': {
+                                "span": TPL_SPAN,
+                                'slice': TPL_SPAN_SLICE,
+                                'start': TPL_SPAN_START,
+                            },
+                            "colors": COLORS_1,
+                        },
+                        simple=False)
+
+
+dep_options = {"fine_grained": True, "distance": 120}
+visualize_parser(doc, displacy_options=dep_options)