provide documentation

main.py

@@ -1,3 +1,12 @@
+# Author: Caitlin Blackmore
+# Project: Pathfinder
+# Project Description: This is a web application designed to facilitate job-mobility. 
+# It uses NLP to help job seekers find jobs that match their skills and interests.
+# Date: 2023-02-03
+# File Description: This is the main file, containing the FastAPI app and all the endpoints.
+# License: MIT License
+
+# IMPORTS
 from fastapi import FastAPI, Request, Form, File, UploadFile
 from fastapi.templating import Jinja2Templates
 from fastapi.staticfiles import StaticFiles
@@ -18,46 +27,94 @@ from nltk.tokenize import SpaceTokenizer
 import nltk
 from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
 from dotenv import load_dotenv
+
+# LOAD ENVIRONMENT VARIABLES
 load_dotenv()
 
+# SSL CERTIFICATE FIX
 try:
     _create_unverified_https_context = ssl._create_unverified_context
 except AttributeError:
     pass
 else:
     ssl._create_default_https_context = _create_unverified_https_context
+
+# DOWNLOAD NLTK DATA IF NOT ALREADY DOWNLOADED
 if os.path.isdir('nltk_data')==False:
     nltk.download('stopwords', quiet=True)
 
+# APP SETUP
 app = FastAPI()
 app.mount("/static", StaticFiles(directory='static'), name="static")
 templates = Jinja2Templates(directory="templates/")
 
+# LOAD DATA
 onet = pd.read_csv('static/ONET_JobTitles.csv')
 simdat = pd.read_csv('static/cohere_embeddings.csv')
-coheredat = pd.read_csv("static/cohere_tSNE_dat.csv")
 
+# LOAD FINE-TUNED MODEL 
+# (see https://huggingface.co/celise88/distilbert-base-uncased-finetuned-binary-classifier)
 model = AutoModelForSequenceClassification.from_pretrained('static/model_shards', low_cpu_mem_usage=True)
 tokenizer = AutoTokenizer.from_pretrained('static/tokenizer_shards', low_cpu_mem_usage=True)
 classifier = pipeline('text-classification', model = model, tokenizer = tokenizer)
 
-### job information center ###
-# get
+# UTILITY FUNCTIONS
+def clean_my_text(text):
+    clean_text = ' '.join(text.splitlines())
+    clean_text = clean_text.replace('-', " ").replace("/"," ")
+    clean_text = clean(clean_text.translate(str.maketrans('', '', string.punctuation)))
+    return clean_text
+
+def remove_new_line(value):
+        return ''.join(value.splitlines())
+
+def coSkillEmbed(text):
+    try:
+        co = cohere.Client(os.getenv("COHERE_TOKEN"))
+        response = co.embed(
+            model='large',
+            texts=[text])
+        return response.embeddings
+    except CohereError as e:
+        return e
+
+def skillNER(resume):
+    resume = clean_my_text(resume)
+    stops = set(nltk.corpus.stopwords.words('english'))
+    stops = stops.union({'eg', 'ie', 'etc', 'experience', 'experiences', 'experienced', 'experiencing', 'knowledge', 
+    'ability', 'abilities', 'skill', 'skills', 'skilled', 'including', 'includes', 'included', 'include'
+    'education', 'follow', 'following', 'follows', 'followed', 'make', 'made', 'makes', 'making', 'maker',
+    'available', 'large', 'larger', 'largescale', 'client', 'clients', 'responsible', 'x', 'many', 'team', 'teams'})
+    resume = [word for word in SpaceTokenizer().tokenize(resume) if word not in stops]
+    resume = [word for word in resume if ")" not in word]
+    resume = [word for word in resume if "(" not in word]
+            
+    labels = []
+    for i in range(len(resume)):
+        classification = classifier(resume[i])[0]['label']
+        if classification == 'LABEL_1':
+            labels.append("Skill")
+        else:
+            labels.append("Not Skill")
+        labels_dict = dict(zip(resume, labels))
+    return labels_dict
+
+def cosine(A, B):
+    return np.dot(A,B)/(norm(A)*norm(B))
+
+### JOB INFORMATION CENTER ###
+# GET
 @app.get("/")
 def render_job_list(request: Request):
     joblist = onet['JobTitle']
     return templates.TemplateResponse('job_list.html', context={'request': request, 'joblist': joblist})
 
-# post
+# POST
 @app.post("/")
 def render_job_info(request: Request, jobtitle: str = Form(enum=[x for x in onet['JobTitle']])):
-    
-    def remove_new_line(value):
-        return ''.join(value.splitlines())
-
     joblist = onet['JobTitle']
-
     if jobtitle: 
+        # SCRAPE ONET TO GET JOB DESCRIPTION, TASKS, ETC.
         onetCode = onet.loc[onet['JobTitle'] == jobtitle, 'onetCode']
         onetCode = onetCode.reindex().tolist()[0]
         headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}
@@ -76,6 +133,7 @@ def render_job_info(request: Request, jobtitle: str = Form(enum=[x for x in onet
         tasks = remove_new_line(tasks).replace("related occupations", " ").replace("core", " - ").replace(" )importance category task", "").replace(" find ", "")
         tasks = tasks.split(". ")
         tasks = [''.join(map(lambda c: '' if c in '0123456789-' else c, task)) for task in tasks]
+        
         return templates.TemplateResponse('job_list.html', context={
             'request': request, 
             'joblist': joblist, 
@@ -83,20 +141,22 @@ def render_job_info(request: Request, jobtitle: str = Form(enum=[x for x in onet
             'jobdescription': jobdescription, 
             'tasks': tasks})
 
-### job neighborhoods ###
+### JOB NEIGHBORHOODS ###
 @app.get("/explore-job-neighborhoods/", response_class=HTMLResponse)
 def render_job_neighborhoods(request: Request):
     return templates.TemplateResponse('job_neighborhoods.html', context={'request': request})
 
-### find my match ###
-# get
+### FIND-MY-MATCH ###
+# GET
 @app.get("/find-my-match/", response_class=HTMLResponse)
-async def match_page(request: Request):
+def match_page(request: Request):
     return templates.TemplateResponse('find_my_match.html', context={'request': request})
 
-# post
+# POST
 @app.post('/find-my-match/', response_class=HTMLResponse)
-def get_resume(request: Request, resume: UploadFile = File(...)):
+async def get_resume(request: Request, resume: UploadFile = File(...)):
+    
+    # READ AND PERFORM BASIC CLEANING ON RESUME
     path = f"static/{resume.filename}"
     with open(path, 'wb') as buffer:
         buffer.write(resume.file.read())
@@ -106,34 +166,13 @@ def get_resume(request: Request, resume: UploadFile = File(...)):
         text.append(para.text)
     resume = "\n".join(text)
 
-    def clean_my_text(text):
-        clean_text = ' '.join(text.splitlines())
-        clean_text = clean_text.replace('-', " ").replace("/"," ")
-        clean_text = clean(clean_text.translate(str.maketrans('', '', string.punctuation)))
-        return clean_text
-
-    def coSkillEmbed(text):
-        try:
-            co = cohere.Client(os.getenv("COHERE_TOKEN"))
-            response = co.embed(
-                model='large',
-                texts=[text])
-            return response.embeddings
-        except CohereError as e:
-            return e
-    
-    def cosine(A, B):
-        return np.dot(A,B)/(norm(A)*norm(B))
-
     # GET RESUME EMBEDDINGS AND JOB SIMILARITY SCORES
     embeds = coSkillEmbed(resume)
     simResults = []
-
     for i in range(len(simdat)):
         simResults.append(cosine(np.array(embeds), np.array(simdat.iloc[i,1:])))
     simResults = pd.DataFrame(simResults)
     simResults['JobTitle'] = simdat['Title']
-
     simResults = simResults.iloc[:,[1,0]]
     simResults.columns = ['JobTitle', 'Similarity']
     simResults = simResults.sort_values(by = "Similarity", ascending = False)
@@ -144,25 +183,6 @@ def get_resume(request: Request, resume: UploadFile = File(...)):
         simResults.iloc[x,1] = "{:0.2f}".format(simResults.iloc[x,1])
         
     # EXTRACT SKILLS FROM RESUME 
-    def skillNER(resume):
-        resume = clean_my_text(resume)
-        stops = set(nltk.corpus.stopwords.words('english'))
-        stops = stops.union({'eg', 'ie', 'etc', 'experience', 'experiences', 'experienced', 'experiencing', 'knowledge', 
-        'ability', 'abilities', 'skill', 'skills', 'skilled', 'including', 'includes', 'included', 'include'
-        'education', 'follow', 'following', 'follows', 'followed', 'make', 'made', 'makes', 'making', 'maker',
-        'available', 'large', 'larger', 'largescale', 'client', 'clients', 'responsible', 'x', 'many', 'team', 'teams'})
-        resume = [word for word in SpaceTokenizer().tokenize(resume) if word not in stops]
-        resume = [word for word in resume if ")" not in word]
-        resume = [word for word in resume if "(" not in word]
-            
-        labels = []
-        for i in range(len(resume)):
-            classification = classifier(resume[i])[0]['label']
-            if classification == 'LABEL_1':
-                labels.append("Skill")
-            else:
-                labels.append("Not Skill")
-            labels_dict = dict(zip(resume, labels))
-        return labels_dict
     skills = skillNER(resume)
+
     return templates.TemplateResponse('find_my_match.html', context={'request': request, 'resume': resume, 'skills': skills, 'simResults': simResults})

static/cohere_tSNE_dat.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ac3dbbea21867638654b3c399b988ca95c5573cc602383d8835cffe36952a7cb
-size 1858107