Spaces:
Sleeping
Sleeping
File size: 6,201 Bytes
f93a065 e73f8ea f93a065 e73f8ea f93a065 e73f8ea f93a065 e73f8ea f93a065 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 |
from haystack.document_stores import FAISSDocumentStore
from haystack.nodes import DensePassageRetriever, FARMReader
from haystack.pipelines import ExtractiveQAPipeline, DocumentSearchPipeline
from haystack.utils import clean_wiki_text, print_answers
from sentence_transformers import SentenceTransformer
from haystack.nodes import EmbeddingRetriever
import streamlit as st
import os
import pandas as pd
from datetime import datetime
import json
import re
def intializeFAISS(docs):
try:
db_file_name='faiss_document_store.db'
faiss_index_file='faiss_index'
if 'reader' not in st.session_state:
st.session_state.reader=FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=False)
if os.path.exists(faiss_index_file):
document_store=FAISSDocumentStore(
sql_url=f"sqlite:///{db_file_name}",
index=faiss_index_file,
embedding_dim=384
)
else:
document_store = FAISSDocumentStore(embedding_dim=384, faiss_index_factory_str="Flat")
if 'retriever' not in st.session_state:
st.session_state.retriever = EmbeddingRetriever(
document_store=document_store,
embedding_model="sentence-transformers/all-MiniLM-L6-v2",
model_format="sentence_transformers",
use_gpu=False
)
document_store.delete_all_documents()
document_store.write_documents(docs)
document_store.update_embeddings(st.session_state.retriever)
document_store.save(index_path=faiss_index_file)
except Exception as ex:
print('--error--')
print('--intialization method--')
print(ex)
def dummy(r):
return{
}
def prepare_doc(r):
_content=f'''{r['Salutation']} {r['Initial']} {r['Name']}, Employee Id is {r['Employee Id']} and user id is {r['User Id']}, is from country {r['Country']}, Supervisor / Manager name is {r['Supervisor / Manager']} and Contracting Company is {r['Contracting Company']}, Primary Industry is {r['Primary Industry']} and Secondary Industry is {r['Secondary Industry']}, there Sector is {r['Sector']}, they are expertise in {r['Expertise']}, there role is {r['Industry Role']}. There last Last Promotion Date is {r['Last Promotion Date']} and {r['Last Promotional Level']}, There Job Title is {r['Job Title']}, they are working here since {r['Professional Since']}, there hired date is {r['Hired Date']}, there Relevant is {r['Relevant']}, Employee Sponser is {r['Employee Sponser']}, Job Description is {r['Job Description']}, Emergency Contact Name is {r['Emergency Contact Name']} and Emergency Contact Number is {r['Emergency Contact Number']}, Regional Supervisor is {r['Regional Supervisor']} Office Supervisor is {r['Office Supervisor']} Engagement Supervisor is {r['Engagement Supervisor']} '''
return {
'content':_content,
'meta':{
"Salutation":r["Salutation"],
"Initial":r["Initial"],
"User Id" :r["User Id"],
"Name" :r["Name"],
"Employee Id" :r["Employee Id"],
"Country":r["Country"],
"Supervisor / Manager" :r["Supervisor / Manager"],
"Contracting Company" :r["Contracting Company"],
"Primary Industry":r["Primary Industry"],
"Secondary Industry" :r["Secondary Industry"],
"Sector" :r["Sector"],
"Expertise":r["Expertise"],
"Industry Role":r["Industry Role"],
"Designation" :r["Designation"],
"Grade" :r["Grade"],
"Target Chargeability %" :r["Target Chargeability %"],
"Charge Out Rate":r["Charge Out Rate"],
"Last Promotion Date" :r["Last Promotion Date"],
"Last Promotional Level" :r["Last Promotional Level"],
"Job Title":r["Job Title"],
"Professional Since" :r["Professional Since"],
"Hired Date" :r["Hired Date"],
"Relevant":r["Relevant"],
"Employee Sponser":r["Employee Sponser"],
"Job Description" :r["Job Description"],
"Emergency Contact Name" :r["Emergency Contact Name"],
"Emergency Contact Number":r["Emergency Contact Number"],
"Regional Supervisor" :r["Regional Supervisor"],
"Office Supervisor" :r["Office Supervisor"],
"Engagement Supervisor":r["Engagement Supervisor"]
}
}
def on_submission(question):
try:
print('reader')
print(st.session_state.reader)
print('retriever')
print(st.session_state.retriever)
qa_pipeline=ExtractiveQAPipeline(reader=st.session_state.reader,retriever=st.session_state.retriever)
prediction=qa_pipeline.run(query=question,params={
"Retriever":{"top_k":5},
"Reader":{"top_k":1}
})
if prediction:
text=str(prediction['answers'][0])
except Exception as ex:
print('--error--')
print('--on_submission method--')
print(ex)
uploaded_file=st.file_uploader(label='please upload your file',type=['csv'])
if uploaded_file is not None:
try:
df=pd.read_csv(uploaded_file)
employee_list=df.to_json(orient='records')
doc=[]
json_emp=json.loads(employee_list)
for e in json_emp:
doc.append(prepare_doc(e))
intializeFAISS(docs=doc)
except Exception as ex:
print('--uploaded_file method--')
print(ex)
#intialize form
with st.form(key='workforce_management'):
st.title('workforce management')
question=st.text_input(label='please ask your question.')
submitted=st.form_submit_button(label='submit')
if submitted and question:
on_submission(question)
|