add job posting scrape capability to find my match page

main.py

@@ -17,7 +17,7 @@ from uuid import uuid1
 from localStoragePy import localStoragePy
 localStorage = localStoragePy('pathfinder', 'text')
 
-from scrape_onet import get_onet_code, get_onet_description, get_onet_tasks
+from scrape_onet import get_onet_code, get_onet_description, get_onet_tasks, get_job_postings
 from match_utils import neighborhoods, get_resume, skillNER, sim_result_loop, get_links, coSkillEmbed, sim_result_loop_jobFinder, sim_result_loop_candFinder
 from user_utils import Hash
 
@@ -143,6 +143,12 @@ def get_matches(request: Request):
 @app.post('/find-my-match/', response_class=HTMLResponse)
 async def post_matches(request: Request, bt: BackgroundTasks, resume: UploadFile = File(...)):
     
+    statelist = [ 'AK', 'AL', 'AR', 'AZ', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA',
+           'HI', 'IA', 'ID', 'IL', 'IN', 'KS', 'KY', 'LA', 'MA', 'MD', 'ME',
+           'MI', 'MN', 'MO', 'MS', 'MT', 'NC', 'ND', 'NE', 'NH', 'NJ', 'NM',
+           'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', 'TN', 'TX',
+           'UT', 'VA', 'VT', 'WA', 'WI', 'WV', 'WY']
+
     username = localStorage.getItem('username')
 
     def add_data_to_db(resume):
@@ -164,13 +170,17 @@ async def post_matches(request: Request, bt: BackgroundTasks, resume: UploadFile
         bt.add_task(add_data_to_db, resume)
         bt.add_task(get_jobs_from_db, resume)
 
-    return templates.TemplateResponse('find_my_match.html', context={'request': request, 'resume': resume, 'skills': skills, 'simResults': simResults[0], 'links': links})
+    return templates.TemplateResponse('find_my_match.html', context={'request': request, 'resume': resume, 'skills': skills, 'simResults': simResults[0], 'links': links, 'statelist': statelist})
 
 @app.get("/find-match/", response_class=HTMLResponse)
 def find_match(request: Request):
-    jobselection = str(request.url).split("=")[1].replace('HTTP/1.1', '').replace("-", " ").replace("%2C", "")
-    print(jobselection)
-    return templates.TemplateResponse('find_match.html', context={'request': request, 'jobselection': jobselection})
+    jobtitle = str(request.url).split("=")[1].replace('HTTP/1.1', '').replace("-", " ").replace("%2C", "").replace('&state', '')
+    state = str(request.url).split("=")[2]
+    onetCode = get_onet_code(jobtitle)
+    postings = get_job_postings(onetCode, state)
+    jobpostings = postings[0]
+    linklist = postings[1]
+    return templates.TemplateResponse('find_match.html', context={'request': request, 'jobpostings': jobpostings, 'linklist': linklist, 'jobtitle': jobtitle, 'state': state})
 
 @app.get("/find-my-hire/", response_class=HTMLResponse)
 def get_hires(request: Request):

scrape_onet.py

@@ -2,6 +2,7 @@ import requests
 from bs4 import BeautifulSoup
 from cleantext import clean
 import pandas as pd
+import numpy as np
 
 onet = pd.read_csv('static/ONET_JobTitles.csv')
 headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}
@@ -33,4 +34,23 @@ def get_onet_tasks(onetCode):
     tasks = remove_new_line(tasks).replace("related occupations", " ").replace("core", " - ").replace(" )importance category task", "").replace(" find ", "")
     tasks = tasks.split(". ")
     tasks = [''.join(map(lambda c: '' if c in '0123456789-' else c, task)) for task in tasks]
-    return tasks
+    return tasks
+
+def get_job_postings(onetCode, state):
+    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}
+    url = "https://www.onetonline.org/link/localjobs/" + onetCode + "?st=" + state
+    response = requests.get(url, headers=headers)
+    soup = BeautifulSoup(response.text, 'html.parser')
+    jobs = str(soup.get_text("tbody")).split('PostedtbodyTitle and CompanytbodyLocation')[1].split('Sources:')[0].split("tbody")
+    jobs = jobs[5:45]
+    starts = np.linspace(start=0, stop=len(jobs)-4,num= 10)
+    stops = np.linspace(start=3, stop=len(jobs)-1, num= 10)
+    jobpostings = []
+    for i in range(0,10):
+        jobpostings.append(str([' '.join(jobs[int(starts[i]):int(stops[i])])]).replace("['", '').replace("']", ''))
+    links = str(soup.find_all('a', href=True)).split("</small>")[1].split(', <a href="https://www.careeronestop.org/"')[0].split(' data-bs-toggle="modal" ')
+    linklist = []
+    for i in range(1, len(links)):
+        links[i] = "https://www.onetonline.org" + str(links[i]).split(' role="button">')[0].replace("href=", "")
+        linklist.append(links[i].replace('"', ''))
+    return jobpostings, linklist

templates/find_match.html

@@ -23,14 +23,25 @@
     </header>
     <main class="main">
         <h1 class="pagetitle">Matching Jobs</h1>
+        {% if jobpostings %}
+        <h2 class="pagesubtitle">Here are the top 10 {{ jobtitle }} jobs in {{ state }} that are a great match for your skillset and interests!</h2>
+        <ul class="sectionlist">
+            {% for n in range(10) %}
+            <li class="sectionlist__item"><a href={{ linklist[n] }}>{{ jobpostings[n] }}</a></li>
+            {% endfor %}
+        </ul>
+        {% else %}
         <h2 class="pagesubtitle">We're sorry! This page is currently under construction.</h2>
         <h2 class="pagesubtitle">Please check back soon to get {{ jobselection }} jobs that are a great match for your skillset and interests!</h2>
+        {% endif %}
         <br>
-        <br>    </main>
+        <br>    
+    </main>
     <footer class="footer">
         <ul class="footer__text">
             <li class="footer__text-item">© 2023 Pathfinder</li>
-        </ul>
+            <li class="footer__text-item">Job postings courtesy of <a class="footer__text-link" href="https://www.onetonline.org">onetonline.org</li>
+            </ul>
     </footer>
 </body>
 </html>

templates/find_my_match.html

@@ -46,7 +46,7 @@
             <article class="output__section">
                 <h2 class="output__subtitle">Job Matches</h3>
                 <p class="alert">(Note: You can click on the links to find out more.)</p>
-                <p class="alert">Instructions: When you have decided which job title is most closely aligned with your skills and interests, choose the corresponding radio button and click "Submit."</p>
+                <p class="alert">Instructions: Choose the radio button corresponding to the job title that is most closely aligned with your skills and interests"</p>
                 <form action="/find-match/" class="selection__form" method="GET">
                     <table>
                         <thead class="output__list">
@@ -71,9 +71,15 @@
                     </table>
                     <br>
                     <br>
-                    <div class="radio__submit">
-                        <button type="submit" class="radio__submit">Submit</button>
-                    </div>
+                    <label for="state" class="form__label">Select the state in which you would like to work:</label>
+                    <select name="state" class="form__dropdown" id="state">
+                        {% for state in statelist %}
+                        <option value="{{ state }}">{{ state }}</option>
+                        {% endfor %}
+                    </select>
+                    <br>
+                    <br>
+                    <button type="submit" class="radio__submit">Submit</button>
                 </form>
             </article>
             {% endif %}

templates/job_list.html

@@ -47,11 +47,11 @@
         {% if tasks %}
         <section>
             <h1 class="sectiontitle">Work Tasks:</h1>
-            <ui class="sectionlist"></ui>
+            <ul class="sectionlist">
                 {% for task in tasks %}
                 <li class="sectionlist__item">{{ task }}</li>
                 {% endfor %}
-            </ui>
+            </ul>
         </section>
         {% endif %}
         {% if activities %}