updates

Author: mjhea0

docker-compose.yml

@@ -0,0 +1,33 @@
+version: '3.7'
+
+services:
+
+ hub:
+ image: selenium/hub:3.141.59
+ ports:
+ - 4444:4444
+ deploy:
+ mode: replicated
+ replicas: 1
+ placement:
+ constraints:
+ - node.role == worker
+
+ chrome:
+ image: selenium/node-chrome:3.141.59
+ volumes:
+ - /dev/urandom:/dev/random
+ depends_on:
+ - hub
+ environment:
+ - HUB_PORT_4444_TCP_ADDR=hub
+ - HUB_PORT_4444_TCP_PORT=4444
+ - NODE_MAX_SESSION=1
+ entrypoint: bash -c 'SE_OPTS="-host $$HOSTNAME -port 5555" /opt/bin/entry_point.sh'
+ ports:
+ - 5555:5555
+ deploy:
+ replicas: 1
+ placement:
+ constraints:
+ - node.role == worker

project/create.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+
+echo "Spinning up four droplets..."
+
+for i in 1 2 3 4; do
+ docker-machine create \
+ --driver digitalocean \
+ --digitalocean-access-token $DIGITAL_OCEAN_ACCESS_TOKEN \
+ node-$i;
+done
+
+
+echo "Initializing Swarm mode..."
+
+docker-machine ssh node-1 -- docker swarm init --advertise-addr $(docker-machine ip node-1)
+
+
+echo "Adding the nodes to the Swarm..."
+
+TOKEN=`docker-machine ssh node-1 docker swarm join-token worker | grep token | awk '{ print $5 }'`
+
+docker-machine ssh node-2 "docker swarm join --token ${TOKEN} $(docker-machine ip node-1):2377"
+docker-machine ssh node-3 "docker swarm join --token ${TOKEN} $(docker-machine ip node-1):2377"
+docker-machine ssh node-4 "docker swarm join --token ${TOKEN} $(docker-machine ip node-1):2377"
+
+
+echo "Deploying Selenium Grid to http://$(docker-machine ip node-1):4444"
+
+eval $(docker-machine env node-1)
+docker stack deploy --compose-file=docker-compose.yml selenium
+docker service scale selenium_chrome=5

project/destroy.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+
+echo "Bringing down the services"
+
+docker service rm selenium_chrome
+docker service rm selenium_hub
+
+
+echo "Bringing down the droplets"
+
+docker-machine rm node-1 node-2 node-3 node-4 -y

project/scrapers/scraper.py

@@ -4,16 +4,19 @@
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from bs4 import BeautifulSoup
 
 
-def get_driver():
+def get_driver(address):
  # initialize options
  options = webdriver.ChromeOptions()
  # pass in headless argument to options
  options.add_argument('--headless')
  # initialize driver
- driver = webdriver.Chrome(chrome_options=options)
+ driver = webdriver.Remote(
+ command_executor=f'http://{address}:4444/wd/hub',
+ desired_capabilities=DesiredCapabilities.CHROME)
  return driver
 
 

project/script.py

@@ -15,7 +15,7 @@ def run_process(browser, page_number=1):
 
 
 if __name__ == '__main__':
- browser = get_driver()
+ browser = get_driver(sys.argv[2])
  data = run_process(browser, sys.argv[1])
  browser.quit()
  print(f'Finished page {sys.argv[1]}')