Fix CompanyScraper

scrape_linkedin/Company.py

@@ -1,14 +1,60 @@
 import logging
 import re
+from typing import Optional
 
 from bs4 import BeautifulSoup
 
 from .ResultsObject import ResultsObject
-from .utils import AnyEC, all_or_default, get_info, one_or_default
+from .utils import all_or_default, get_info, one_or_default, text_or_default
+
+RE_DUPLICATE_WHITESPACE = re.compile(r"[\s]{2,}")
+COMPANY_SIZE_KEY = 'company_size'
 
 logger = logging.getLogger(__name__)
 
 
+def get_company_metadata(about_section):
+ """
+ Takes a Company's 'About' section, and returns a dict mapping metadata keys
+ to metadata values. Keys can be somewhat arbitrary, but common ones include
+ Company size, industry, website, specialties, headquarters, etc.
+
+ Note that this section container 'titles' and 'values' all at the same level
+ of nesting. It looks something like:
+ <dl>
+ <dt>Heading 1</dt>
+ <dd>Some value for heading 1</dd>
+ <dd>Another value for heading1</dd>
+ <dt>Heading 2</dt>
+ ...
+ </dl>
+ """
+ curr_header = None
+ results = {}
+ for child in all_or_default(about_section, "dl > *"):
+ # We've hit a new heading.
+ if child.name == 'dt':
+ curr_header = child.get_text().lower().strip().replace(" ", "_")
+ results[curr_header] = []
+ # We've hit content for the most recent heading.
+ elif child.name == 'dd':
+ content = child.get_text().strip()
+ results[curr_header].append(
+ RE_DUPLICATE_WHITESPACE.sub(" ", content)) # strip redundant whitespace
+
+ for r in results:
+ results[r] = '\n'.join(results[r])
+ return results
+
+
+def get_employee_count(s: str) -> Optional[int]:
+ """Extracts employee count from a string."""
+ employee_count_match = re.search(r'([\d,]+) on LinkedIn', s)
+ if employee_count_match:
+ return int(employee_count_match.group(1).replace(",", ""))
+ return None
+
+
 class Company(ResultsObject):
  """Linkedin User Profile Object"""
 
@@ -27,55 +73,34 @@ def __init__(self, overview, jobs, life, insights):
  def overview(self):
  """Return dict of the overview section of the Linkedin Page"""
 
+ overview = {
+ "description": None,
+ "image": None,
+ "name": None,
+ "num_employees": None,
+ "metadata": None
+ }
+
  # Banner containing company Name + Location
  banner = one_or_default(
  self.overview_soup, '.org-top-card')
 
  # Main container with company overview info
- container = one_or_default(
- self.overview_soup, '.org-grid__core-rail--wide')
-
- overview = {}
- overview['description'] = container.select_one(
- 'section > p').get_text().strip()
-
- metadata_keys = container.select('.org-page-details__definition-term')
- print(metadata_keys)
- metadata_keys = [
- x for x in metadata_keys if "Company size" not in x.get_text()]
- print(metadata_keys)
- metadata_values = container.select(
- '.org-page-details__definition-text')
- overview.update(
- get_info(banner, {'name': '.org-top-card-summary__title'})) # A fix to the name selector
- overview.update(
- get_info(container, {'company_size': '.org-about-company-module__company-size-definition-text'})) # Manually added Company size
-
- for key, val in zip(metadata_keys, metadata_values):
- dict_key = key.get_text().strip().lower().replace(" ", "_")
- dict_val = val.get_text().strip()
- if "company_size" not in dict_key:
- overview[dict_key] = dict_val
- print(overview)
-
- all_employees_links = all_or_default(
- banner, '.mt2 > a > span') # A fix to locate "See all ### employees on LinkedIn"
-
- if all_employees_links:
- all_employees_text = all_employees_links[-1].text
- else:
- all_employees_text = ''
-
- match = re.search(r'((\d+?,?)+)', all_employees_text)
- if match:
- overview['num_employees'] = int(match.group(1).replace(',', ''))
- else:
- overview['num_employees'] = None
+ container = one_or_default(self.overview_soup,
+ '.org-grid__content-height-enforcer')
+
+ overview["name"] = text_or_default(self.overview_soup, "#main h1")
+ overview['description'] = text_or_default(container, 'section > p')
 
  logo_image_tag = one_or_default(
  banner, '.org-top-card-primary-content__logo')
  overview['image'] = logo_image_tag['src'] if logo_image_tag else ''
 
+ company_metadata = get_company_metadata(container)
+ overview["metadata"] = company_metadata
+ overview["num_employees"] = get_employee_count(company_metadata.get(
+ COMPANY_SIZE_KEY, ""))
+
  return overview
 
  @property

scrape_linkedin/CompanyScraper.py

@@ -14,26 +14,38 @@
 
 class CompanyScraper(Scraper):
  def scrape(self, company, overview=True, jobs=False, life=False, insights=False):
- # Get Overview
- self.load_initial(company)
+ self.url = 'https://www.linkedin.com/company/{}'.format(company)
+ self.company = company
+
+ self.load_initial()
 
  jobs_html = life_html = insights_html = overview_html = ''
 
  if overview:
- overview_html = self.get_overview()
+ overview_html = self.fetch_page_html('about')
  if life:
- life_html = self.get_life()
+ life_html = self.fetch_page_html('life')
  if jobs:
- jobs_html = self.get_jobs()
+ jobs_html = self.fetch_page_html('jobs')
  if insights:
- insights_html = self.get_insights()
- #print("JOBS", jobs_html, "\n\n\n\n\nLIFE", life_html)
+ insights_html = self.fetch_page_html('insights')
  return Company(overview_html, jobs_html, life_html, insights_html)
 
- def load_initial(self, company):
- url = 'https://www.linkedin.com/company/{}'.format(company)
+ def fetch_page_html(self, page):
+ """
+ Navigates to a company subpage and returns the entire HTML contents of the page.
+ """
+ try:
+ self.driver.get(f"{self.url}/{page}")
+ return self.driver.find_element_by_css_selector(
+ '.organization-outlet').get_attribute('outerHTML')
+ except Exception as e:
+ logger.warn(
+ f"Unable to fetch '{page}' page for {self.company}: {e}")
+ return ''
 
- self.driver.get(url)
+ def load_initial(self):
+ self.driver.get(self.url)
  try:
  myElem = WebDriverWait(self.driver, self.timeout).until(AnyEC(
  EC.presence_of_element_located(
@@ -52,48 +64,3 @@ def load_initial(self, company):
  except:
  raise ValueError(
  'Company Unavailable: Company link does not match any companies on LinkedIn')
-
- def get_overview(self):
- try:
- tab_link = self.driver.find_element_by_css_selector(
- 'a[data-control-name="page_member_main_nav_about_tab"]')
- tab_link.click()
- self.wait_for_el(
- 'a[data-control-name="page_member_main_nav_about_tab"].active')
- return self.driver.find_element_by_css_selector(
- '.organization-outlet').get_attribute('outerHTML')
- except:
- return ''
-
- def get_life(self):
- try:
- tab_link = self.driver.find_element_by_css_selector(
- 'a[data-control-name="page_member_main_nav_life_tab"]')
- tab_link.click()
- self.wait_for_el(
- 'a[data-control-name="page_member_main_nav_life_tab"].active')
- return self.driver.find_element_by_css_selector('.org-life').get_attribute('outerHTML')
- except:
- return ''
-
- def get_jobs(self):
- try:
- tab_link = self.driver.find_element_by_css_selector(
- 'a[data-control-name="page_member_main_nav_jobs_tab"]')
- tab_link.click()
- self.wait_for_el(
- 'a[data-control-name="page_member_main_nav_jobs_tab"].active')
- return self.driver.find_element_by_css_selector('.org-jobs-container').get_attribute('outerHTML')
- except:
- return ''
-
- def get_insights(self):
- try:
- tab_link = self.driver.find_element_by_css_selector(
- 'a[data-control-name="page_member_main_nav_insights_tab"]')
- tab_link.click()
- self.wait_for_el(
- 'a[data-control-name="page_member_main_nav_insights_tab"].active')
- return self.driver.find_element_by_css_selector('.org-premium-insights-module').get_attribute('outerHTML')
- except:
- return ''