{"id":798,"date":"2021-01-05T20:42:03","date_gmt":"2021-01-05T12:42:03","guid":{"rendered":"https:\/\/byy3.com\/?p=798"},"modified":"2021-01-09T10:08:02","modified_gmt":"2021-01-09T02:08:02","slug":"%e4%bd%bf%e7%94%a8python%e6%89%b9%e9%87%8f%e7%88%ac%e5%8f%96%e4%b8%bb%e6%b5%81%e6%90%9c%e7%b4%a2%e5%bc%95%e6%93%8e%e5%9b%be%e7%89%87","status":"publish","type":"post","link":"https:\/\/byy3.com\/?p=798","title":{"rendered":"\u4f7f\u7528python\u6279\u91cf\u722c\u53d6\u4e3b\u6d41\u641c\u7d22\u5f15\u64ce\u56fe\u7247"},"content":{"rendered":"

\u6700\u8fd1\u5728\u505a\u4e00\u4e2a\u53e3\u7f69\u8bc6\u522b\u7684\u5e94\u7528\uff0c\u9700\u8981\u5f88\u591a\u6234\u53e3\u7f69\u7684\u4eba\u7684\u56fe\u7247\u4f5c\u4e3a\u6570\u636e\u8bad\u7ec3\u6a21\u578b\uff0c\u56e0\u516c\u53f8\u6ca1\u6709\u63d0\u4f9b\u6570\u636e\uff0c\u53ea\u80fd\u6211\u4eec\u81ea\u5df1\u7528python\u722c\u866b\u722c\u53d6\u5404\u4e3b\u6d41\u7f51\u7ad9\u7684\u6234\u53e3\u7f69\u7684\u56fe\u7247\uff0c\u6211\u4eec\u4e3b\u8981\u722c\u53d6\u4e86\u5fc5\u5e94\u3001360\u3001\u641c\u72d7\u7684\u56fe\u7247(\u767e\u5ea6\u7684\u6709\u70b9\u6742\uff0c\u4e0d\u5982\u8fd9\u4e09\u5bb6\u4e2a)\uff0c\u4ee3\u7801\u5982\u4e0b\uff08\u4ec5\u4f9b\u5b66\u4e60\u53c2\u8003\uff09\uff1a<\/p>\n

\u5fc5\u5e94\u641c\u7d22<\/strong><\/p>\n

from<\/span> bs4 import<\/span> BeautifulSoup\r\nimport<\/span> urllib.<\/span>request\r\nimport<\/span> requests\r\nimport<\/span> time\r\nimport<\/span> json\r\nimport<\/span> sys\r\nimport<\/span> re\r\nimport<\/span> os\r\n\r\n#\u722c\u53d6\u76ee\u6807\u7f51\u7ad9url\r\nCRAWL_TARGET_URL =<\/span> 'https:\/\/cn.bing.com\/images\/async?q=%s&first=%d&count=%d&relp=%d&lostate=r&mmasync=1'<\/span>\r\n#\u6bcf\u6b21\u6293\u53d6\u56fe\u7247\u6570\u91cf<\/span>(<\/span>35<\/span>\u662f\u6b64\u7f51\u9875\u6bcf\u6b21\u7ffb\u9875\u8bf7\u6c42\u6570\u91cf)<\/span>\r\nNUMS_PER_CRAWL =<\/span> 35<\/span>\r\n#\u6293\u53d6\u56fe\u7247\u6700\u5c0f\u5927\u5c0f<\/span>(<\/span>\u5355\u4f4d\u5b57\u8282)<\/span>\uff0c\u5c0f\u4e8e\u6b64\u503c\u629b\u5f03\r\nMIN_IMAGE_SIZE =<\/span> 102400<\/span>\r\n\r\n\r\ndef get_image<\/span>(<\/span>url,<\/span> path,<\/span> count)<\/span>:<\/span>\r\n try<\/span>:<\/span>\r\n u =<\/span> urllib.<\/span>request.<\/span>urlopen<\/span>(<\/span>url,<\/span> timeout=9<\/span>)<\/span>\r\n t =<\/span> u.<\/span>read<\/span>(<\/span>)<\/span>\r\n if<\/span> sys.<\/span>getsizeof<\/span>(<\/span>t)<\/span> <<\/span> MIN_IMAGE_SIZE:<\/span>\r\n return<\/span> -<\/span>1<\/span>\r\n except Exception as<\/span> e:<\/span>\r\n print<\/span>(<\/span>url,<\/span> e)<\/span>\r\n return<\/span> -<\/span>2<\/span>\r\n #\u63d0\u53d6\u56fe\u7247\u683c\u5f0f\r\n frmt =<\/span> url[<\/span>url.<\/span>rfind<\/span>(<\/span>'.'<\/span>)<\/span>:<\/span>]<\/span>\r\n p =<\/span> re.<\/span>compile<\/span>(<\/span>\"^\\\\.[a-zA-Z]+\"<\/span>)<\/span>\r\n m =<\/span> p.<\/span>match<\/span>(<\/span>frmt)<\/span>\r\n frmt =<\/span> m.<\/span>group<\/span>(<\/span>0<\/span>)<\/span>\r\n try<\/span>:<\/span>\r\n if<\/span> not os.<\/span>path.<\/span>exists<\/span>(<\/span>path)<\/span>:<\/span>\r\n os.<\/span>mkdir<\/span>(<\/span>path)<\/span>\r\n f =<\/span> open<\/span>(<\/span>os.<\/span>path.<\/span>join<\/span>(<\/span>path,<\/span> str<\/span>(<\/span>count)<\/span>+<\/span>frmt)<\/span>,<\/span> 'wb'<\/span>)<\/span>\r\n f.<\/span>write<\/span>(<\/span>t)<\/span>\r\n f.<\/span>close<\/span>(<\/span>)<\/span>\r\n except Exception as<\/span> e:<\/span>\r\n print<\/span>(<\/span>os.<\/span>path.<\/span>join<\/span>(<\/span>path,<\/span> str<\/span>(<\/span>count)<\/span>+<\/span>frmt)<\/span>,<\/span> e)<\/span>\r\n return<\/span> -<\/span>3<\/span>\r\n return<\/span> 0<\/span>\r\n\r\n\r\ndef crawl_data<\/span>(<\/span>info,<\/span> num)<\/span>:<\/span>\r\n first =<\/span> 0<\/span>\r\n count =<\/span> 0<\/span>\r\n #\u521b\u5efa\u4e00\u4e2a\u4f1a\u8bdd\r\n s =<\/span> requests.<\/span>Session<\/span>(<\/span>)<\/span>\r\n #\u521b\u5efa\u6587\u4ef6\u8def\u5f84\r\n path=<\/span>\".\/\"<\/span>+<\/span>info\r\n if<\/span> not os.<\/span>path.<\/span>exists<\/span>(<\/span>path)<\/span>:<\/span>\r\n os.<\/span>mkdir<\/span>(<\/span>path)<\/span>\r\n index=<\/span>len<\/span>(<\/span>os.<\/span>listdir<\/span>(<\/span>path)<\/span>)<\/span>#\u6587\u4ef6\u4e2d\u539f\u6709\u56fe\u7247\u6570\r\n while<\/span>(<\/span>count <<\/span> num)<\/span>:<\/span>\r\n u =<\/span> CRAWL_TARGET_URL%<\/span>(<\/span>info,<\/span> first,<\/span> NUMS_PER_CRAWL,<\/span> NUMS_PER_CRAWL)<\/span>\r\n #5.<\/span>05s\u4e3a\u53d1\u9001\u8d85\u65f6\u65f6\u95f4\uff0c10s\u4e3a\u63a5\u6536\u5230\u6570\u636e\u8d85\u65f6\u65f6\u95f4\r\n req =<\/span> s.<\/span>get<\/span>(<\/span>url =<\/span>u,<\/span> timeout=<\/span>(5<\/span>.05<\/span>,<\/span> 10<\/span>)<\/span>)<\/span>\r\n bf =<\/span> BeautifulSoup<\/span>(<\/span>req.<\/span>text,<\/span> \"html.parser\"<\/span>)<\/span>\r\n imgtags =<\/span> bf.<\/span>find_all<\/span>(<\/span>\"a\"<\/span>,<\/span> class_ =<\/span> \"iusc\"<\/span>)<\/span>\r\n for<\/span> e in<\/span> imgtags:<\/span>\r\n if<\/span> count ==<\/span> num:<\/span>\r\n return<\/span> False\r\n urldict =<\/span> json.<\/span>loads<\/span>(<\/span>e.<\/span>get<\/span>(<\/span>'m'<\/span>)<\/span>)<\/span>\r\n if<\/span> get_image<\/span>(<\/span>urldict[<\/span>\"murl\"<\/span>]<\/span>,<\/span> path,<\/span> index)<\/span> <<\/span> 0<\/span>:<\/span>\r\n continue<\/span>\r\n print<\/span>(<\/span>\"Downloaded %d picture\"<\/span>%<\/span>(<\/span>count+<\/span>1<\/span>)<\/span>)<\/span>\r\n sys.<\/span>stdout.<\/span>flush<\/span>(<\/span>)<\/span>\r\n count =<\/span>count+<\/span>1<\/span>\r\n index=<\/span>index+<\/span>1<\/span>\r\n time.<\/span>sleep<\/span>(<\/span>0.09<\/span>)<\/span>\r\n first =<\/span> first +<\/span> NUMS_PER_CRAWL\r\n time.<\/span>sleep<\/span>(<\/span>0.9<\/span>)<\/span>\r\n\r\n return<\/span> True\r\n\r\nif<\/span> __name__ ==<\/span> '__main__'<\/span>:<\/span>\r\n \r\n # \u5173\u952e\u8bcd\uff0c\u53ef\u8bbe\u7f6e\u4e3a\u591a\u4e2a\r\n key_words=<\/span>[<\/span>'\u6234\u53e3\u7f69'<\/span>,<\/span>]<\/span>\r\n # \u4e0b\u8f7d\u7684\u56fe\u7247\u6570\u91cf\r\n picture_num =<\/span> 500<\/span>\r\n\r\n for<\/span> i in<\/span> range<\/span>(<\/span>len<\/span>(<\/span>key_words)<\/span>)<\/span>:<\/span>\r\n word=<\/span>key_words[<\/span>i]<\/span>\r\n print<\/span>(<\/span>word)<\/span>\r\n if<\/span> crawl_data<\/span>(<\/span>word,<\/span> picture_num)<\/span>:<\/span>\r\n i=<\/span>i+<\/span>1<\/span>\r\n<\/pre>\n
360\u641c\u7d22<\/strong><\/p>\n
import<\/span> json\r\nimport<\/span> os\r\nimport<\/span> requests\r\n\r\n# \u8def\u5f84\r\nBASE_URL =<\/span> '.\/\u6234\u53e3\u7f69'<\/span>\r\n# \u5173\u952e\u8bcd\r\nNAME =<\/span> '\u6234\u53e3\u7f69'<\/span>\r\n\r\nclass<\/span> PictureDownload<\/span>(<\/span>object)<\/span>:<\/span>\r\n def __init__<\/span>(<\/span>self,<\/span> q=<\/span>None,<\/span> sn=<\/span>100<\/span>)<\/span>:<\/span>\r\n self.<\/span>url =<\/span> 'https:\/\/m.image.so.com\/j?q={}&src=srp&pn=100&sn={}&kn=0&gn=0&cn=0'<\/span>\r\n self.<\/span>headers =<\/span> {<\/span>\r\n 'User-Agent'<\/span>:<\/span> 'Mozilla\/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit\/604.1.38 (KHTML, like Gecko) Version\/11.0 Mobile\/15A372 Safari\/604.1'<\/span>\r\n }<\/span>\r\n self.<\/span>q =<\/span> q\r\n self.<\/span>sn =<\/span> sn\r\n self.<\/span>num =<\/span> 0<\/span>\r\n self.<\/span>total =<\/span> 2<\/span>\r\n\r\n def makedir<\/span>(<\/span>self)<\/span>:<\/span>\r\n if<\/span> not os.<\/span>path.<\/span>exists<\/span>(<\/span>os.<\/span>path.<\/span>join<\/span>(<\/span>BASE_URL,<\/span> self.<\/span>q)<\/span>)<\/span>:<\/span>\r\n os.<\/span>makedirs<\/span>(<\/span>os.<\/span>path.<\/span>join<\/span>(<\/span>BASE_URL,<\/span> self.<\/span>q)<\/span>)<\/span>\r\n\r\n def parse_url<\/span>(<\/span>self)<\/span>:<\/span>\r\n response =<\/span> requests.<\/span>get<\/span>(<\/span>self.<\/span>url.<\/span>format<\/span>(<\/span>self.<\/span>q,<\/span> self.<\/span>num)<\/span>,<\/span> headers=<\/span>self.<\/span>headers)<\/span>\r\n return<\/span> response.<\/span>content.<\/span>decode<\/span>(<\/span>)<\/span>\r\n\r\n def parse_image_list<\/span>(<\/span>self,<\/span> html_json_str)<\/span>:<\/span>\r\n image_list =<\/span> json.<\/span>loads<\/span>(<\/span>html_json_str)<\/span>[<\/span>'list'<\/span>]<\/span>\r\n total =<\/span> json.<\/span>loads<\/span>(<\/span>html_json_str)<\/span>[<\/span>'total'<\/span>]<\/span>\r\n return<\/span> image_list,<\/span> total\r\n\r\n def save_image<\/span>(<\/span>self,<\/span> image_list)<\/span>:<\/span>\r\n\r\n for<\/span> item in<\/span> image_list:<\/span>\r\n response =<\/span> requests.<\/span>get<\/span>(<\/span>item[<\/span>'thumb'<\/span>]<\/span>,<\/span> headers=<\/span>self.<\/span>headers)<\/span>\r\n with<\/span> open<\/span>(<\/span>os.<\/span>path.<\/span>join<\/span>(<\/span>BASE_URL,<\/span> '%s\\%s.jpg'<\/span> %<\/span> (<\/span>self.<\/span>q,<\/span> item[<\/span>'index'<\/span>]<\/span>)<\/span>)<\/span>,<\/span> 'wb'<\/span>)<\/span> as<\/span> f:<\/span>\r\n f.<\/span>write<\/span>(<\/span>response.<\/span>content)<\/span>\r\n\r\n def run<\/span>(<\/span>self)<\/span>:<\/span>\r\n self.<\/span>makedir<\/span>(<\/span>)<\/span>\r\n while<\/span> self.<\/span>num <<\/span> self.<\/span>total:<\/span>\r\n html_json_str =<\/span> self.<\/span>parse_url<\/span>(<\/span>)<\/span>\r\n image_list,<\/span> self.<\/span>total =<\/span> self.<\/span>parse_image_list<\/span>(<\/span>html_json_str)<\/span>\r\n self.<\/span>save_image<\/span>(<\/span>image_list)<\/span>\r\n self.<\/span>num +=<\/span> 100<\/span>\r\n print<\/span>(<\/span>self.<\/span>num)<\/span>\r\n\r\n\r\nif<\/span> __name__ ==<\/span> '__main__'<\/span>:<\/span>\r\n xxx =<\/span> PictureDownload<\/span>(<\/span>NAME)<\/span>\r\n xxx.<\/span>run<\/span>(<\/span>)<\/span>\r\n\r\n<\/pre>\n
\u641c\u72d7\u641c\u7d22<\/strong><\/p>\n
import<\/span> requests\r\nimport<\/span> json\r\nimport<\/span> urllib\r\n\r\n# \u4e09\u4e2a\u53c2\u6570\uff0c\u4f60\u8981\u83b7\u53d6\u6574\u4e2a\u56fe\u7247\u96c6\u7684\u540d\u5b57\uff0c\u4f60\u8981\u83b7\u53d6\u591a\u5c11\u5f20\uff0c\u83b7\u53d6\u8fc7\u6765\u7684\u653e\u5728\u54ea\u91cc\r\ndef getSogoulmag<\/span>(<\/span>category,<\/span>length,<\/span>path)<\/span>:<\/span>\r\n n=<\/span>length\r\n cate=<\/span>category\r\n # \u83b7\u53d6\u7684\u662f\u56fe\u7247\u6240\u6709\u4fe1\u606f\r\n imgs=<\/span>requests.<\/span>get<\/span>(<\/span>'http:\/\/pic.sogou.com\/pics\/channel\/getAllRecomPicByTag.jsp?category='<\/span>+<\/span>cate+<\/span>\r\n '&tag=%E5%85%A8%E9%83%A8&start=0&len='<\/span>+<\/span>str<\/span>(<\/span>n)<\/span>)<\/span>\r\n\r\n # \u8f6c\u6362\u6210\u4e3ajson\u683c\u5f0f\r\n jd=<\/span>json.<\/span>loads<\/span>(<\/span>imgs.<\/span>text)<\/span>\r\n # all_items\u6240\u6709\u7684\u56fe\u7247\r\n jd=<\/span>jd[<\/span>'all_items'<\/span>]<\/span>\r\n\r\n imgs_t=<\/span>[<\/span>]<\/span>\r\n for<\/span> j in<\/span> jd:<\/span>\r\n # \u901a\u8fc7\u5b9a\u4f4dbthumbUrl\u83b7\u53d6\u56fe\u7247\r\n imgs_t.<\/span>append<\/span>(<\/span>j[<\/span>'bthumbUrl'<\/span>]<\/span>)<\/span>\r\n m=<\/span>0<\/span>\r\n for<\/span> img in<\/span> imgs_t:<\/span>\r\n # \u6253\u5370\u67d0\u4e00\u5f20\u56fe\u7247\u6b63\u5728\u4e0b\u8f7d\r\n print<\/span>(<\/span>str<\/span>(<\/span>m)<\/span>+<\/span>'.jpg'<\/span>+<\/span>'Downlod......'<\/span>)<\/span>\r\n # \u7528\u6765\u628a\u8fdc\u7a0b\u6570\u636e\u4e0b\u8f7d\u5230\u672c\u5730\r\n urllib.<\/span>request.<\/span>urlretrieve<\/span>(<\/span>img,<\/span>path+<\/span>str<\/span>(<\/span>m)<\/span>+<\/span>'.jpg'<\/span>)<\/span>\r\n m=<\/span>m+<\/span>1<\/span>\r\n print<\/span>(<\/span>'Complete!'<\/span>)<\/span>\r\n \r\n# \u8c03\u7528\u6574\u4e2a\u5904\u7406\u903b\u8f91\r\n# \u4e09\u4e2a\u53c2\u6570\uff0c\u4f60\u8981\u83b7\u53d6\u6574\u4e2a\u56fe\u7247\u96c6\u7684\u540d\u5b57\uff0c\u4f60\u8981\u83b7\u53d6\u591a\u5c11\u5f20\uff0c\u83b7\u53d6\u8fc7\u6765\u7684\u653e\u5728\u54ea\u91cc\r\ngetSogoulmag<\/span>(<\/span>'\u6234\u53e3\u7f69'<\/span>,<\/span>10<\/span>,<\/span>'.\/sougoutupian\/'<\/span>)<\/span><\/pre>\n
\u722c\u53d6360\u56fe\u7247\u7684\u8fc7\u7a0b\u5982\u56fe1\u6240\u793a\uff1a<\/strong><\/p>\n
\n
\"\u4f7f\u7528python\u6279\u91cf\u722c\u53d6\u4e3b\u6d41\u641c\u7d22\u5f15\u64ce\u56fe\u7247\u63d2\u56fe\"<\/div>\n<\/figure>\n
\u56fe1 \u722c\u53d6360\u56fe\u7247\u5168\u8fc7\u7a0b<\/p>\n
\u6211\u4eec\u53ef\u4ee5\u770b\u5230\uff0c\u4f7f\u7528 pycharm\u8fd0\u884c\u7a0b\u5e8f\u540e\uff0c\u56fe\u7247\u9646\u7eed\u5f00\u59cb\u4e0b\u8f7d\uff0c\u5f53\u7136\uff0c\u6709\u4e9b\u56fe\u7247\u662f\u5e72\u6270\u6570\u636e\uff0c\u9700\u8981\u624b\u52a8\u6e05\u7406\u6389\uff0c\u76f8\u6bd4\u4e8e\u4e00\u5f20\u5f20\u4e0b\u8f7d\uff0c\u8fd8\u662f\u65b9\u4fbf\u5f88\u591a\u7684\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"
\u6700\u8fd1\u5728\u505a\u4e00\u4e2a\u53e3\u7f69\u8bc6\u522b\u7684\u5e94\u7528\uff0c\u9700\u8981\u5f88\u591a\u6234\u53e3\u7f69\u7684\u4eba\u7684\u56fe\u7247\u4f5c\u4e3a\u6570\u636e\u8bad\u7ec3\u6a21\u578b\uff0c\u56e0\u516c\u53f8\u6ca1\u6709\u63d0\u4f9b\u6570\u636e\uff0c\u53ea\u80fd\u6211\u4eec\u81ea\u5df1\u7528pyt […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[20],"tags":[33,352,61,317,51],"class_list":["post-798","post","type-post","status-publish","format-standard","hentry","category-python","tag-python"],"_links":{"self":[{"href":"https:\/\/byy3.com\/index.php?rest_route=\/wp\/v2\/posts\/798","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/byy3.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/byy3.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/byy3.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/byy3.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=798"}],"version-history":[{"count":0,"href":"https:\/\/byy3.com\/index.php?rest_route=\/wp\/v2\/posts\/798\/revisions"}],"wp:attachment":[{"href":"https:\/\/byy3.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=798"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/byy3.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=798"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/byy3.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=798"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}