Merge pull request #4 from testdrivenio/base-update

update base

Author: mjhea0

project/scrapers/scraper.py

@@ -1,82 +1,83 @@
-import requests
+import csv
+from pathlib import Path
 
+import requests
+from bs4 import BeautifulSoup
 from selenium import webdriver
 from selenium.webdriver.common.by import By
-from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
-from bs4 import BeautifulSoup
+from selenium.webdriver.support.ui import WebDriverWait
+
+BASE_DIR = Path(__file__).resolve(strict=True).parent.parent
 
 
 def get_driver():
- # initialize options
  options = webdriver.ChromeOptions()
- # pass in headless argument to options
- options.add_argument('--headless')
+ options.add_argument("--headless")
+
  # initialize driver
  driver = webdriver.Chrome(chrome_options=options)
  return driver
 
 
-def connect_to_base(browser, page_number):
- base_url = f'https://news.ycombinator.com/news?p={page_number}'
+def connect_to_base(browser):
+ base_url = "https://en.wikipedia.org/wiki/Special:Random"
  connection_attempts = 0
  while connection_attempts < 3:
  try:
  browser.get(base_url)
- # wait for table element with id = 'hnmain' to load
+ # wait for table element with id = 'content' to load
  # before returning True
  WebDriverWait(browser, 5).until(
- EC.presence_of_element_located((By.ID, 'hnmain'))
+ EC.presence_of_element_located((By.ID, "content"))
  )
  return True
- except Exception as ex:
+ except Exception as e:
+ print(e)
  connection_attempts += 1
- print(f'Error connecting to {base_url}.')
- print(f'Attempt #{connection_attempts}.')
+ print(f"Error connecting to {base_url}.")
+ print(f"Attempt #{connection_attempts}.")
  return False
 
 
 def parse_html(html):
  # create soup object
- soup = BeautifulSoup(html, 'html.parser')
+ soup = BeautifulSoup(html, "html.parser")
  output_list = []
- # parse soup object to get article id, rank, score, and title
- tr_blocks = soup.find_all('tr', class_='athing')
- article = 0
- for tr in tr_blocks:
- article_id = tr.get('id')
- article_url = tr.find_all('a')[1]['href']
- # check if article is a hacker news article
- if 'item?id=' in article_url:
- article_url = f'https://news.ycombinator.com/{article_url}'
- load_time = get_load_time(article_url)
- try:
- score = soup.find(id=f'score_{article_id}').string
- except Exception as ex:
- score = '0 points'
- article_info = {
- 'id': article_id,
- 'load_time': load_time,
- 'rank': tr.span.string,
- 'score': score,
- 'title': tr.find(class_='storylink').string,
- 'url': article_url
- }
- # appends article_info to output_list
- output_list.append(article_info)
- article += 1
+ # parse soup object to get wikipedia article url, title, and last modified date
+ article_url = soup.find("link", {"rel": "canonical"})["href"]
+ article_title = soup.find("h1", {"id": "firstHeading"}).text
+ article_last_modified = soup.find("li", {"id": "footer-info-lastmod"}).text
+ article_info = {
+ "url": article_url,
+ "title": article_title,
+ "last_modified": article_last_modified,
+ }
+ output_list.append(article_info)
  return output_list
 
 
 def get_load_time(article_url):
  try:
  # set headers
- headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
+ headers = {
+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36"
+ }
  # make get request to article_url
  response = requests.get(
- article_url, headers=headers, stream=True, timeout=3.000)
+ article_url, headers=headers, stream=True, timeout=3.000
+ )
  # get page load time
  load_time = response.elapsed.total_seconds()
- except Exception as ex:
- load_time = 'Loading Error'
+ except Exception as e:
+ print(e)
+ load_time = "Loading Error"
  return load_time
+
+
+def write_to_file(output_list, filename):
+ for row in output_list:
+ with open(Path(BASE_DIR).joinpath(filename), "a") as csvfile:
+ fieldnames = ["url", "title", "last_modified"]
+ writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+ writer.writerow(row)

project/script.py

@@ -4,18 +4,21 @@
 from scrapers.scraper import get_driver, connect_to_base, parse_html
 
 
-def run_process(browser, page_number=1):
- if connect_to_base(browser, page_number):
- print(f'Scraping page {page_number}...')
+
+def run_process(rowser):
+ if connect_to_base(browser):
+ print(f'Scraping random Wikipedia page...')
  sleep(2)
  html = browser.page_source
  return parse_html(html)
  else:
+ print("Error connecting to Wikipedia")
  return False
 
 
 if __name__ == '__main__':
  browser = get_driver()
- data = run_process(browser, sys.argv[1])
+ data = run_process(browser)
+ print(data)
  browser.quit()
- print(f'Finished page {sys.argv[1]}')
+ print(f'Finished!')

requirements.txt

@@ -1,3 +1,3 @@
-beautifulsoup4==4.9.3
-requests==2.25.1
-selenium==3.141.0
+beautifulsoup4==4.10.0
+requests==2.27.1
+selenium==4.1.3