<!DOCTYPE html>

<html xmlns="http://www.w3.org/1999/xhtml">

<head>

<meta charset="utf-8" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="pandoc" />

<meta name="viewport" content="width=device-width, initial-scale=1">



<title>Scraping LinkedIn Job Ads with LinkedInJobsScrapeR</title>



<style type="text/css">code{white-space: pre;}</style>
<style type="text/css" data-origin="pandoc">
a.sourceLine { display: inline-block; line-height: 1.25; }
a.sourceLine { pointer-events: none; color: inherit; text-decoration: inherit; }
a.sourceLine:empty { height: 1.2em; }
.sourceCode { overflow: visible; }
code.sourceCode { white-space: pre; position: relative; }
div.sourceCode { margin: 1em 0; }
pre.sourceCode { margin: 0; }
@media screen {
div.sourceCode { overflow: auto; }
}
@media print {
code.sourceCode { white-space: pre-wrap; }
a.sourceLine { text-indent: -1em; padding-left: 1em; }
}
pre.numberSource a.sourceLine
  { position: relative; left: -4em; }
pre.numberSource a.sourceLine::before
  { content: attr(title);
    position: relative; left: -1em; text-align: right; vertical-align: baseline;
    border: none; pointer-events: all; display: inline-block;
    -webkit-touch-callout: none; -webkit-user-select: none;
    -khtml-user-select: none; -moz-user-select: none;
    -ms-user-select: none; user-select: none;
    padding: 0 4px; width: 4em;
    color: #aaaaaa;
  }
pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa;  padding-left: 4px; }
div.sourceCode
  {  }
@media screen {
a.sourceLine::before { text-decoration: underline; }
}
code span.al { color: #ff0000; font-weight: bold; } /* Alert */
code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */
code span.at { color: #7d9029; } /* Attribute */
code span.bn { color: #40a070; } /* BaseN */
code span.bu { } /* BuiltIn */
code span.cf { color: #007020; font-weight: bold; } /* ControlFlow */
code span.ch { color: #4070a0; } /* Char */
code span.cn { color: #880000; } /* Constant */
code span.co { color: #60a0b0; font-style: italic; } /* Comment */
code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */
code span.do { color: #ba2121; font-style: italic; } /* Documentation */
code span.dt { color: #902000; } /* DataType */
code span.dv { color: #40a070; } /* DecVal */
code span.er { color: #ff0000; font-weight: bold; } /* Error */
code span.ex { } /* Extension */
code span.fl { color: #40a070; } /* Float */
code span.fu { color: #06287e; } /* Function */
code span.im { } /* Import */
code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */
code span.kw { color: #007020; font-weight: bold; } /* Keyword */
code span.op { color: #666666; } /* Operator */
code span.ot { color: #007020; } /* Other */
code span.pp { color: #bc7a00; } /* Preprocessor */
code span.sc { color: #4070a0; } /* SpecialChar */
code span.ss { color: #bb6688; } /* SpecialString */
code span.st { color: #4070a0; } /* String */
code span.va { color: #19177c; } /* Variable */
code span.vs { color: #4070a0; } /* VerbatimString */
code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */

</style>
<script>
// apply pandoc div.sourceCode style to pre.sourceCode instead
(function() {
  var sheets = document.styleSheets;
  for (var i = 0; i < sheets.length; i++) {
    if (sheets[i].ownerNode.dataset["origin"] !== "pandoc") continue;
    try { var rules = sheets[i].cssRules; } catch (e) { continue; }
    for (var j = 0; j < rules.length; j++) {
      var rule = rules[j];
      // check if there is a div.sourceCode rule
      if (rule.type !== rule.STYLE_RULE || rule.selectorText !== "div.sourceCode") continue;
      var style = rule.style.cssText;
      // check if color or background-color is set
      if (rule.style.color === '' || rule.style.backgroundColor === '') continue;
      // replace div.sourceCode by a pre.sourceCode rule
      sheets[i].deleteRule(j);
      sheets[i].insertRule('pre.sourceCode{' + style + '}', j);
    }
  }
})();
</script>



<style type="text/css">body {
background-color: #fff;
margin: 1em auto;
max-width: 700px;
overflow: visible;
padding-left: 2em;
padding-right: 2em;
font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
font-size: 14px;
line-height: 1.35;
}
#header {
text-align: center;
}
#TOC {
clear: both;
margin: 0 0 10px 10px;
padding: 4px;
width: 400px;
border: 1px solid #CCCCCC;
border-radius: 5px;
background-color: #f6f6f6;
font-size: 13px;
line-height: 1.3;
}
#TOC .toctitle {
font-weight: bold;
font-size: 15px;
margin-left: 5px;
}
#TOC ul {
padding-left: 40px;
margin-left: -1.5em;
margin-top: 5px;
margin-bottom: 5px;
}
#TOC ul ul {
margin-left: -2em;
}
#TOC li {
line-height: 16px;
}
table {
margin: 1em auto;
border-width: 1px;
border-color: #DDDDDD;
border-style: outset;
border-collapse: collapse;
}
table th {
border-width: 2px;
padding: 5px;
border-style: inset;
}
table td {
border-width: 1px;
border-style: inset;
line-height: 18px;
padding: 5px 5px;
}
table, table th, table td {
border-left-style: none;
border-right-style: none;
}
table thead, table tr.even {
background-color: #f7f7f7;
}
p {
margin: 0.5em 0;
}
blockquote {
background-color: #f6f6f6;
padding: 0.25em 0.75em;
}
hr {
border-style: solid;
border: none;
border-top: 1px solid #777;
margin: 28px 0;
}
dl {
margin-left: 0;
}
dl dd {
margin-bottom: 13px;
margin-left: 13px;
}
dl dt {
font-weight: bold;
}
ul {
margin-top: 0;
}
ul li {
list-style: circle outside;
}
ul ul {
margin-bottom: 0;
}
pre, code {
background-color: #f7f7f7;
border-radius: 3px;
color: #333;
white-space: pre-wrap; 
}
pre {
border-radius: 3px;
margin: 5px 0px 10px 0px;
padding: 10px;
}
pre:not([class]) {
background-color: #f7f7f7;
}
code {
font-family: Consolas, Monaco, 'Courier New', monospace;
font-size: 85%;
}
p > code, li > code {
padding: 2px 0px;
}
div.figure {
text-align: center;
}
img {
background-color: #FFFFFF;
padding: 2px;
border: 1px solid #DDDDDD;
border-radius: 3px;
border: 1px solid #CCCCCC;
margin: 0 5px;
}
h1 {
margin-top: 0;
font-size: 35px;
line-height: 40px;
}
h2 {
border-bottom: 4px solid #f7f7f7;
padding-top: 10px;
padding-bottom: 2px;
font-size: 145%;
}
h3 {
border-bottom: 2px solid #f7f7f7;
padding-top: 10px;
font-size: 120%;
}
h4 {
border-bottom: 1px solid #f7f7f7;
margin-left: 8px;
font-size: 105%;
}
h5, h6 {
border-bottom: 1px solid #ccc;
font-size: 105%;
}
a {
color: #0033dd;
text-decoration: none;
}
a:hover {
color: #6666ff; }
a:visited {
color: #800080; }
a:visited:hover {
color: #BB00BB; }
a[href^="http:"] {
text-decoration: underline; }
a[href^="https:"] {
text-decoration: underline; }

code > span.kw { color: #555; font-weight: bold; } 
code > span.dt { color: #902000; } 
code > span.dv { color: #40a070; } 
code > span.bn { color: #d14; } 
code > span.fl { color: #d14; } 
code > span.ch { color: #d14; } 
code > span.st { color: #d14; } 
code > span.co { color: #888888; font-style: italic; } 
code > span.ot { color: #007020; } 
code > span.al { color: #ff0000; font-weight: bold; } 
code > span.fu { color: #900; font-weight: bold; }  code > span.er { color: #a61717; background-color: #e3d2d2; } 
</style>




</head>

<body>




<h1 class="title toc-ignore">Scraping LinkedIn Job Ads with <code>LinkedInJobsScrapeR</code></h1>



<p>In this vignette, I will demonstrate how to use <code>LinkedInJobsScrapeR</code> for scraping LinkedIn job data. We will specify the job titles, locations, and experience levels that we’re interested in scraping data from.</p>
<div id="defining-the-parameters" class="section level2">
<h2>Defining the parameters</h2>
<p>Let’s say we’re interested in scraping all of the LinkedIn job ads for Entry and Associate level positions in “data scientist” and “data engineer” positions, in the greater New York City and San Francisco areas.</p>
<p>First we define the job parameters by creating three data frames 1. <code>job_titles</code> - defines the job titles to scrape 2. <code>locations</code> - defines the locations to scrape 3. <code>experience_levels</code> - defines the levels of the positions (e.g., entry / associate)</p>
<p>The definitions below are what we’re after.</p>
<div class="sourceCode" id="cb1"><pre class="sourceCode r"><code class="sourceCode r"><a class="sourceLine" id="cb1-1" title="1"><span class="co"># Job titles to scrape job listings for</span></a>
<a class="sourceLine" id="cb1-2" title="2">job_titles &lt;-<span class="st"> </span><span class="kw">c</span>(<span class="st">&#39;data scientist&#39;</span>, <span class="st">&#39;data engineer&#39;</span>)</a>
<a class="sourceLine" id="cb1-3" title="3"></a>
<a class="sourceLine" id="cb1-4" title="4"><span class="co"># Locations to scrape jobs for</span></a>
<a class="sourceLine" id="cb1-5" title="5"><span class="co"># The first element in each item is the search query</span></a>
<a class="sourceLine" id="cb1-6" title="6"><span class="co"># The second element in each item is the abbreviation</span></a>
<a class="sourceLine" id="cb1-7" title="7">locations &lt;-<span class="st"> </span><span class="kw">list</span>(</a>
<a class="sourceLine" id="cb1-8" title="8">  <span class="kw">c</span>(<span class="st">&#39;New York City Metropolitan Area&#39;</span>, <span class="st">&#39;NYC&#39;</span>),</a>
<a class="sourceLine" id="cb1-9" title="9">  <span class="kw">c</span>(<span class="st">&#39;San Francisco Bay Area&#39;</span>, <span class="st">&#39;SF&#39;</span>)</a>
<a class="sourceLine" id="cb1-10" title="10">)</a>
<a class="sourceLine" id="cb1-11" title="11"></a>
<a class="sourceLine" id="cb1-12" title="12"><span class="co"># Experience levels to scrape jobs for</span></a>
<a class="sourceLine" id="cb1-13" title="13"><span class="co">#   1 = Intern</span></a>
<a class="sourceLine" id="cb1-14" title="14"><span class="co">#   2 = Entry</span></a>
<a class="sourceLine" id="cb1-15" title="15"><span class="co">#   3 = Associate</span></a>
<a class="sourceLine" id="cb1-16" title="16"><span class="co">#   4 = Mid-senior</span></a>
<a class="sourceLine" id="cb1-17" title="17">experience_levels &lt;-<span class="st"> </span><span class="kw">c</span>(<span class="dv">2</span>)</a></code></pre></div>
</div>
<div id="scraping-one-at-a-time" class="section level2">
<h2>Scraping – one at a time?</h2>
<p>Now, we could scrape each of the job titles, locations, and experience levels that we’re after one at a time.</p>
<p>The <code>scrape_job()</code> function is designed to accept indices for each of the dataframes we just created. So if we wanted to scrape the job at index 1 of each of the dataframes defined above (“data scientist”, “NYC”, and experience level 1), we could call the function like this:</p>
<div class="sourceCode" id="cb2"><pre class="sourceCode r"><code class="sourceCode r"><a class="sourceLine" id="cb2-1" title="1"><span class="kw">scrape_job</span>(<span class="dt">job_titles_index =</span> <span class="dv">1</span>,</a>
<a class="sourceLine" id="cb2-2" title="2">           <span class="dt">locations_index =</span> <span class="dv">1</span>,</a>
<a class="sourceLine" id="cb2-3" title="3">           <span class="dt">experience_level_index =</span> <span class="dv">1</span>)</a></code></pre></div>
<p>When we call this function, it will create a directory in our working directory called <code>data</code> with the following structure:</p>
<pre><code>data/
└── job title/
    └── experience level/
        └── location/
            └── file</code></pre>
<p>So after running the function, we can go there to find the scraped ads.</p>
<p>But there’s a more efficient way than running one at a time!</p>
</div>
<div id="scraping-all-the-things" class="section level2">
<h2>Scraping – all the things</h2>
<p>Instead of running one at a time, we can run them all at once using nested for-loops. The code below will also allow you to re-run in the event of an error or if you have to stop and return later, because at each iteration, before running it checks for the existence of saved ad text files.</p>
<div class="sourceCode" id="cb4"><pre class="sourceCode r"><code class="sourceCode r"><a class="sourceLine" id="cb4-1" title="1"><span class="co"># For the jobs to scrape, loop through all of the...</span></a>
<a class="sourceLine" id="cb4-2" title="2"><span class="co">#   i = locations</span></a>
<a class="sourceLine" id="cb4-3" title="3"><span class="co">#   k = experience levels</span></a>
<a class="sourceLine" id="cb4-4" title="4"><span class="co">#   j = job titles</span></a>
<a class="sourceLine" id="cb4-5" title="5"><span class="cf">for</span>(i <span class="cf">in</span> <span class="dv">1</span><span class="op">:</span><span class="kw">length</span>(locations)){</a>
<a class="sourceLine" id="cb4-6" title="6">  <span class="cf">for</span>(k <span class="cf">in</span> <span class="dv">1</span><span class="op">:</span><span class="kw">length</span>(experience_levels)){</a>
<a class="sourceLine" id="cb4-7" title="7">    <span class="cf">for</span> (j <span class="cf">in</span> <span class="dv">1</span><span class="op">:</span><span class="kw">length</span>(job_titles)){</a>
<a class="sourceLine" id="cb4-8" title="8">      </a>
<a class="sourceLine" id="cb4-9" title="9">      <span class="kw">print</span>(<span class="kw">paste0</span>(<span class="st">&quot;CURRENT JOB: &quot;</span>, </a>
<a class="sourceLine" id="cb4-10" title="10">                    job_titles[j], <span class="st">&quot;: &quot;</span>, </a>
<a class="sourceLine" id="cb4-11" title="11">                    experience_levels[k], <span class="st">&quot;: &quot;</span>, </a>
<a class="sourceLine" id="cb4-12" title="12">                    locations[[i]][<span class="dv">1</span>]))</a>
<a class="sourceLine" id="cb4-13" title="13">      </a>
<a class="sourceLine" id="cb4-14" title="14">      job_title_no_space &lt;-<span class="st"> </span><span class="kw">gsub</span>(<span class="st">&quot;</span><span class="ch">\\</span><span class="st">s&quot;</span>, <span class="st">&quot;&quot;</span>, job_titles[j])</a>
<a class="sourceLine" id="cb4-15" title="15">      </a>
<a class="sourceLine" id="cb4-16" title="16">      <span class="co"># Check if files exist in the directory</span></a>
<a class="sourceLine" id="cb4-17" title="17">      <span class="co">#   and skip if they do</span></a>
<a class="sourceLine" id="cb4-18" title="18">      files &lt;-<span class="st"> </span><span class="kw">list.files</span>(<span class="kw">paste0</span>(<span class="st">&#39;data/&#39;</span>,</a>
<a class="sourceLine" id="cb4-19" title="19">                                job_title_no_space, <span class="st">&#39;/&#39;</span>,</a>
<a class="sourceLine" id="cb4-20" title="20">                                experience_levels[k], <span class="st">&#39;/&#39;</span>,</a>
<a class="sourceLine" id="cb4-21" title="21">                                locations[[i]][<span class="dv">2</span>]))</a>
<a class="sourceLine" id="cb4-22" title="22">      <span class="cf">if</span>(<span class="kw">length</span>(files) <span class="op">&gt;</span><span class="st"> </span><span class="dv">0</span>) <span class="cf">next</span></a>
<a class="sourceLine" id="cb4-23" title="23">      </a>
<a class="sourceLine" id="cb4-24" title="24">      <span class="kw">scrape_job</span>(<span class="dt">locations_index =</span> i,</a>
<a class="sourceLine" id="cb4-25" title="25">                       <span class="dt">experience_level_index =</span> k,</a>
<a class="sourceLine" id="cb4-26" title="26">                       <span class="dt">job_titles_index =</span> j)  </a>
<a class="sourceLine" id="cb4-27" title="27">    }</a>
<a class="sourceLine" id="cb4-28" title="28">  }</a>
<a class="sourceLine" id="cb4-29" title="29">}</a></code></pre></div>
</div>
<div id="end" class="section level2">
<h2>End</h2>
<p>After running the above, we should have a <code>data</code> folder populated with job ads. On to data extraction!</p>
</div>



<!-- dynamically load mathjax for compatibility with self-contained -->
<script>
  (function () {
    var script = document.createElement("script");
    script.type = "text/javascript";
    script.src  = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";
    document.getElementsByTagName("head")[0].appendChild(script);
  })();
</script>

</body>
</html>
