一小时入门网络爬虫系列代码

Author: Jack-Cherish

one_hour_spider/biqukan.py

@@ -0,0 +1,91 @@
+# -*- coding:UTF-8 -*-
+from bs4 import BeautifulSoup
+import requests, sys
+
+"""
+类说明:下载《笔趣看》网小说《一念永恒》
+Parameters:
+无
+Returns:
+无
+Modify:
+2017-09-13
+"""
+class downloader(object):
+
+def __init__(self):
+self.server = 'http://www.biqukan.com/'
+self.target = 'http://www.biqukan.com/1_1094/'
+self.names = []#存放章节名
+self.urls = []#存放章节链接
+self.nums = 0#章节数
+
+"""
+函数说明:获取下载链接
+Parameters:
+无
+Returns:
+无
+Modify:
+2017-09-13
+"""
+def get_download_url(self):
+ req = requests.get(url = self.target)
+ html = req.text
+ div_bf = BeautifulSoup(html)
+ div = div_bf.find_all('div', class_ = 'listmain')
+ a_bf = BeautifulSoup(str(div[0]))
+ a = a_bf.find_all('a')
+ self.nums = len(a[15:])#剔除不必要的章节，并统计章节数
+ for each in a[15:]:
+ self.names.append(each.string)
+ self.urls.append(self.server + each.get('href'))
+
+"""
+函数说明:获取章节内容
+Parameters:
+target - 下载连接(string)
+Returns:
+texts - 章节内容(string)
+Modify:
+2017-09-13
+"""
+def get_contents(self, target):
+req = requests.get(url = target)
+html = req.text
+bf = BeautifulSoup(html)
+texts = bf.find_all('div', class_ = 'showtxt')
+texts = texts[0].text.replace('\xa0'*8,'\n\n')
+return texts
+
+"""
+函数说明:将爬取的文章内容写入文件
+Parameters:
+name - 章节名称(string)
+path - 当前路径下,小说保存名称(string)
+text - 章节内容(string)
+Returns:
+无
+Modify:
+2017-09-13
+"""
+def writer(self, name, path, text):
+write_flag = True
+with open(path, 'a', encoding='utf-8') as f:
+f.write(name + '\n')
+f.writelines(text)
+f.write('\n\n')
+
+if __name__ == "__main__":
+# dl = downloader()
+# dl.get_download_url()
+# print('《一年永恒》开始下载：')
+# for i in range(dl.nums):
+#	dl.writer(dl.names[i], '一念永恒.txt', dl.get_contents(dl.urls[i]))
+#	sys.stdout.write(" 已下载:%.3f%%" % float(i/dl.nums) + '\r')
+#	sys.stdout.flush()
+# print('《一年永恒》下载完成')
+url = 'https://unsplash.com/'
+headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36'}
+req = requests.get(url = url, headers = headers)
+print(req.text)

one_hour_spider/unsplash.py

@@ -0,0 +1,64 @@
+# -*- coding:UTF-8 -*-
+import requests, json, time, sys
+from contextlib import closing
+
+class get_photos(object):
+
+def __init__(self):
+self.photos_id = []
+self.download_server = 'https://unsplash.com/photos/xxx/download?force=trues'
+self.target = 'http://unsplash.com/napi/feeds/home'
+self.headers = {'authorization':'Client-ID c94869b36aa272dd62dfaeefed769d4115fb3189a9d1ec88ed457207747be626'}
+
+"""
+函数说明:获取图片ID
+Parameters:
+无
+Returns:
+无
+Modify:
+2017-09-13
+"""
+def get_ids(self):
+req = requests.get(url=self.target, headers=self.headers, verify=False)
+html = json.loads(req.text)
+next_page = html['next_page']
+for each in html['photos']:
+self.photos_id.append(each['id'])
+time.sleep(1)
+for i in range(5):
+req = requests.get(url=next_page, headers=self.headers, verify=False)
+html = json.loads(req.text)
+next_page = html['next_page']
+for each in html['photos']:
+self.photos_id.append(each['id'])
+time.sleep(1)
+
+
+"""
+函数说明:图片下载
+Parameters:
+无
+Returns:
+无
+Modify:
+2017-09-13
+"""
+def download(self, photo_id, filename):
+headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36'}
+target = self.download_server.replace('xxx', photo_id)
+with closing(requests.get(url=target, stream=True, verify = False, headers = self.headers)) as r:
+with open('%d.jpg' % filename, 'ab+') as f:
+for chunk in r.iter_content(chunk_size = 1024):
+if chunk:
+f.write(chunk)
+f.flush()
+
+if __name__ == '__main__':
+gp = get_photos()
+print('获取图片连接中:')
+gp.get_ids()
+print('图片下载中:')
+for i in range(len(gp.photos_id)):
+print(' 正在下载第%d张图片' % (i+1))
+gp.download(gp.photos_id[i], (i+1))

one_hour_spider/vidoe_downloader.py

@@ -0,0 +1,89 @@
+#-*- coding:UTF-8 -*-
+import requests,re, json, sys
+from bs4 import BeautifulSoup
+from urllib import request
+
+class video_downloader():
+def __init__(self, url):
+self.server = 'http://api.xfsub.com'
+self.api = 'http://api.xfsub.com/xfsub_api/?url='
+self.get_url_api = 'http://api.xfsub.com/xfsub_api/url.php'
+self.url = url.split('#')[0]
+self.target = self.api + self.url
+self.s = requests.session()
+
+"""
+函数说明:获取key、time、url等参数
+Parameters:
+无
+Returns:
+无
+Modify:
+2017-09-18
+"""
+def get_key(self):
+req = self.s.get(url=self.target)
+req.encoding = 'utf-8'
+self.info = json.loads(re.findall('"url.php",\ (.+),', req.text)[0])#使用正则表达式匹配结果，将匹配的结果存入info变量中
+
+"""
+函数说明:获取视频地址
+Parameters:
+无
+Returns:
+video_url - 视频存放地址
+Modify:
+2017-09-18
+"""
+def get_url(self):
+data = {'time':self.info['time'],
+'key':self.info['key'],
+'url':self.info['url'],
+'type':''}
+req = self.s.post(url=self.get_url_api,data=data)
+url = self.server + json.loads(req.text)['url']
+req = self.s.get(url)
+bf = BeautifulSoup(req.text,'xml')#因为文件是xml格式的，所以要进行xml解析。
+video_url = bf.find('file').string#匹配到视频地址
+return video_url
+
+"""
+函数说明:回调函数，打印下载进度
+Parameters:
+a b c - 返回信息
+Returns:
+无
+Modify:
+2017-09-18
+"""
+def Schedule(self, a, b, c):
+per = 100.0*a*b/c
+if per > 100 :
+per = 1
+sys.stdout.write(" " + "%.2f%% 已经下载的大小:%ld 文件大小:%ld" % (per,a*b,c) + '\r')
+sys.stdout.flush()
+
+"""
+函数说明:视频下载
+Parameters:
+url - 视频地址
+filename - 视频名字
+Returns:
+无
+Modify:
+2017-09-18
+"""
+def video_download(self, url, filename):
+request.urlretrieve(url=url,filename=filename,reporthook=self.Schedule)
+
+
+if __name__ == '__main__':
+url = 'http://www.iqiyi.com/v_19rr7qhfg0.html#vfrm=19-9-0-1'
+vd = video_downloader(url)
+filename = '加勒比海盗5'
+print('%s下载中:' % filename)
+vd.get_key()
+video_url = vd.get_url()
+print(' 获取地址成功:%s' % video_url)
+vd.video_download(video_url, filename+'.mp4')
+print('\n下载完成！')