使用python批量爬取主流搜索引擎图片

admin python 2021年1月5日

1.75W 0 12

最近在做一个口罩识别的应用，需要很多戴口罩的人的图片作为数据训练模型，因公司没有提供数据，只能我们自己用python爬虫爬取各主流网站的戴口罩的图片，我们主要爬取了必应、360、搜狗的图片(百度的有点杂，不如这三家个)，代码如下（仅供学习参考）：

必应搜索

from bs4 import BeautifulSoupimport urllib.requestimport requestsimport timeimport jsonimport sysimport reimport os
#爬取目标网站url
CRAWL_TARGET_URL = 'https://cn.bing.com/images/async?q=%s&first=%d&count=%d&relp=%d&lostate=r&mmasync=1'#每次抓取图片数量(35是此网页每次翻页请求数量)NUMS_PER_CRAWL = 35#抓取图片最小大小(单位字节)，小于此值抛弃
MIN_IMAGE_SIZE = 102400def get_image(url, path, count): try: u = urllib.request.urlopen(url, timeout=9) t = u.read() if sys.getsizeof(t) < MIN_IMAGE_SIZE: return -1 except Exception as e: print(url, e) return -2 #提取图片格式 frmt = url[url.rfind('.'):] p = re.compile("^\\.[a-zA-Z]+") m = p.match(frmt) frmt = m.group(0) try: if not os.path.exists(path): os.mkdir(path) f = open(os.path.join(path, str(count)+frmt), 'wb') f.write(t) f.close() except Exception as e: print(os.path.join(path, str(count)+frmt), e) return -3 return 0def crawl_data(info, num): first = 0 count = 0 #创建一个会话 s = requests.Session() #创建文件路径 path="./"+info if not os.path.exists(path): os.mkdir(path) index=len(os.listdir(path))#文件中原有图片数 while(count < num): u = CRAWL_TARGET_URL%(info, first, NUMS_PER_CRAWL, NUMS_PER_CRAWL) #5.05s为发送超时时间，10s为接收到数据超时时间 req = s.get(url =u, timeout=(5.05, 10)) bf = BeautifulSoup(req.text, "html.parser") imgtags = bf.find_all("a", class_ = "iusc") for e in imgtags: if count == num: return False urldict = json.loads(e.get('m')) if get_image(urldict["murl"], path, index) < 0: continue print("Downloaded %d picture"%(count+1)) sys.stdout.flush() count =count+1 index=index+1 time.sleep(0.09) first = first + NUMS_PER_CRAWL time.sleep(0.9) return Trueif __name__ == '__main__': # 关键词，可设置为多个 key_words=['戴口罩',] # 下载的图片数量 picture_num = 500 for i in range(len(key_words)): word=key_words[i] print(word) if crawl_data(word, picture_num): i=i+1

360搜索

import jsonimport osimport requests
# 路径
BASE_URL = './戴口罩'# 关键词
NAME = '戴口罩'class PictureDownload(object): def __init__(self, q=None, sn=100): self.url = 'https://m.image.so.com/j?q={}&src=srp&pn=100&sn={}&kn=0&gn=0&cn=0' self.headers = { 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1' } self.q = q self.sn = sn self.num = 0 self.total = 2 def makedir(self): if not os.path.exists(os.path.join(BASE_URL, self.q)): os.makedirs(os.path.join(BASE_URL, self.q)) def parse_url(self): response = requests.get(self.url.format(self.q, self.num), headers=self.headers) return response.content.decode() def parse_image_list(self, html_json_str): image_list = json.loads(html_json_str)['list'] total = json.loads(html_json_str)['total'] return image_list, total def save_image(self, image_list): for item in image_list: response = requests.get(item['thumb'], headers=self.headers) with open(os.path.join(BASE_URL, '%s\%s.jpg' % (self.q, item['index'])), 'wb') as f: f.write(response.content) def run(self): self.makedir() while self.num < self.total: html_json_str = self.parse_url() image_list, self.total = self.parse_image_list(html_json_str) self.save_image(image_list) self.num += 100 print(self.num)if __name__ == '__main__': xxx = PictureDownload(NAME) xxx.run()

搜狗搜索

import requestsimport jsonimport urllib
# 三个参数，你要获取整个图片集的名字，你要获取多少张，获取过来的放在哪里
def getSogoulmag(category,length,path): n=length cate=category # 获取的是图片所有信息 imgs=requests.get('http://pic.sogou.com/pics/channel/getAllRecomPicByTag.jsp?category='+cate+ '&tag=%E5%85%A8%E9%83%A8&start=0&len='+str(n)) # 转换成为json格式 jd=json.loads(imgs.text) # all_items所有的图片 jd=jd['all_items'] imgs_t=[] for j in jd: # 通过定位bthumbUrl获取图片 imgs_t.append(j['bthumbUrl']) m=0 for img in imgs_t: # 打印某一张图片正在下载 print(str(m)+'.jpg'+'Downlod......') # 用来把远程数据下载到本地 urllib.request.urlretrieve(img,path+str(m)+'.jpg') m=m+1 print('Complete!')
# 调用整个处理逻辑
# 三个参数，你要获取整个图片集的名字，你要获取多少张，获取过来的放在哪里getSogoulmag('戴口罩',10,'./sougoutupian/')