Added indeed scraper

Author: avidLearnerInProgress

indeed-scraper/indeed-scraper.py

@@ -0,0 +1,92 @@
+''' Written By : Parvez Alam '''
+
+import requests
+import bs4
+from bs4 import BeautifulSoup
+import pandas as pd
+import time
+
+max_results_per_city = 100
+
+city_set = ['mumbai','bangalore','hyderabad','pune']
+
+job_title_set = ['full+stack+developer','front+end+developer','back+end+developer','software+engineer','data+scientist','machine+learning+engineer','android+developer','ios+developer']
+
+columns = ["job_title","company_name","summary","city","location","salary","date"]
+
+sample_df = pd.DataFrame(columns = columns)
+for job_title in job_title_set:
+for city in city_set:
+for start in range(0,max_results_per_city,10):
+url = "http://www.indeed.co.in/jobs?q=%s&l=%s&start=%s"%(job_title,city,start,)
+page = requests.get(url)
+print(url)
+time.sleep(1)
+soup = BeautifulSoup(page.text,"lxml",from_encoding="utf-8")
+for div in soup.find_all(name="div",attrs={"class":"row"}):
+#specifying row num for index of job posting in dataframe
+print(job_title+' '+city+' '+str(start))
+print(len(sample_df))
+num = (len(sample_df) + 1)
+#an empty list to hold the data for each posting
+job_post = []
+
+#grabbing job title
+try:
+for a in div.find_all(name="a",attrs={"data-tn-element":"jobTitle"}):
+job_post.append(a["title"])
+except:
+job_post.append("Not Available")
+
+#grabbing company_name
+try:
+company = div.find_all(name="span",attrs={"class":"company"})
+if len(company) > 0:
+for b in company:
+job_post.append(b.text.strip())
+else:
+sec_try = div.find_all(name="span",attrs={"class":"result-link-source"})
+for span in sec_try:
+job_post.append(span.text)
+except:
+job_post.append("Not Available")
+
+#grabbing summary text
+try:
+d = div.findAll('span',attrs={'class':'summary'})
+for span in d:
+job_post.append(span.text.strip())
+except:
+job_post.append("Not Available")
+
+#append city name
+job_post.append(city)
+
+#grabbing location name
+try:
+c = div.find_all('span',attrs={'class':'location'})
+for span in c:
+job_post.append(span.text)
+except:
+job_post.append("Not Available")
+
+#grabbing salary
+try:
+#salary = div.find(name="span",attrs={"class":"no-wrap"})
+job_post.append(div.find(name="span",attrs={"class":"no-wrap"}).text)
+except:
+job_post.append("Not Available")
+
+#grabbing salary
+try:
+#salary = div.find(name="span",attrs={"class":"no-wrap"})
+job_post.append(div.find(name="span",attrs={"class":"date"}).text)
+except:
+job_post.append("Not Available")
+
+#appending list of job post info to dataframe at index num
+#print(job_post)
+sample_df.loc[num] = job_post
+
+
+sample_df.to_csv("job_listing.csv",encoding="utf-8")