Python爬蟲案例：爬取百度圖片

糾結於爬取百度圖片，竟然花費了一天的時間才讓程式順利跑起來。其中踩坑無數。而且還發現公司電腦實在是比較差勁。。。

1 import requests
2 import urllib  
3 import os , re
4 from os.path import join 
5 import time

 1 def getPages(keyword,pages=5):  
 2     params = []  
 3     for i in range(30, 30*pages+30, 30):  
 4         # 通過網上資料，可以使用 requests.get() 解析 json 數據，能夠得到對應 url
 5         # 其中一個坑是，原來並不是以下的每個數據都是需要的，某些不要也可以！
 6         # Python學習交流群125240963每天更新資料視頻，大牛指導
 7         params.append({  
 8                       'tn': 'resultjson_com',  
 9                       'ipn': 'rj',  
10                       'ct': 201326592,  
11                       'is': '',  
12                       'fp': 'result',  
13                       'queryWord': keyword,  
14                       'cl': 2,  
15                       'lm': -1,  
16                       'ie': 'utf-8',  
17                       'oe': 'utf-8',  
18                       'adpicid': '',  
19                       'st': -1,  
20                       'z': '',  
21                       'ic': 0,  
22                       'word': keyword,  
23                       's': '',  
24                       'se': '',  
25                       'tab': '',  
26                       'width': '',  
27                       'height': '',  
28                       'face': 0,  
29                       'istype': 2,  
30                       'qc': '',  
31                       'nc': 1,  
32                       'fr': '', 
33                       'pn': i,  
34                       'rn': 30,  
35                       #'gsm': '1e',  
36                       #'1488942260214': '' 
37                   })  
38     url = 'https://image.baidu.com/search/acjson'  
39     urls = []  
40     for param in params: 
41         # url 與 param 合成完整 url  
42         urls.append(requests.get(url,param,headers = headers,timeout = 3).url)     # 
43     #print (urls)
44     return urls  

 1 def get_Img_url(keyword,pages=5):
 2     # 每頁的 URL 集合
 3     pageUrls = getPages(keyword,pages)             
 4     # 圖片url : "thumbURL":"https://ss0.bdstatic.com/70cFuHSh_Q1YnxGkpoWK1HF6hhy/it/u=1789805203,3542215675&fm=27&gp=0.jpg"
 5     # 正則寫的很差！
 6     exp = re.compile(r'"thumbURL":"[\:\,\_\w\d\=\.\+\s\/\%\$\&]+\.jpg')
 7     imgUrls = []
 8     for url in pageUrls:
 9         # 逐個讀取每頁 URL
10         # Python學習交流群125240963 每天更新資料大牛指導
11         try:
12             with urllib.request.urlopen(url,timeout = 3) as pageUrl:
13                 imgUrl = pageUrl.read().decode('utf-8') 
14                 urls = re.findall(exp,imgUrl)
15                 for url in urls:
16                     # 除去 thumbURL":"
17                     imgUrls.append(url.replace('"thumbURL":"',''))
18         # 正則提取 ImgUrl
19         except:
20             print('SomePage is not opened!')           
21             continue
22     # 所有照片的 urls
23     return imgUrls

 1 def getImg(urlList,localPath):  
 2     if not os.path.exists(localPath):  
 3         os.makedirs(localPath)  
 4     x = 1 
 5     for url in urlList:
 6         # 將 for 迴圈寫在 try 外面
 7         try: 
 8             # 什麼時候應該轉義？這點還沒弄明白
 9             # 沒有打開特定文件夾！
10             with open(keyword+str(x)+'.jpg','wb') as f:        # 原本寫 ‘\.jpg’ 會出錯，列印 \\.jpg
11                 img = urllib.request.urlopen(url,timeout = 3).read()
12                 f.write(img)
13             print('%d.jpg is downloaded!' % x)
14             x += 1
15         except Exception:  
16             print("\n  Failed downloading NO. %d image\n" % x)

1 if __name__ == '__main__':
2     keyword = '美女'
3     pages = 500
4     localPath = 'F:/pythonCode/day1001/'
5     headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'}
6     urlList = get_Img_url(keyword,pages)
7     getImg(urlList,localPath)