{"id":1100,"date":"2021-11-30T07:00:43","date_gmt":"2021-11-29T23:00:43","guid":{"rendered":"https:\/\/byy3.com\/?p=1100"},"modified":"2021-11-30T07:02:18","modified_gmt":"2021-11-29T23:02:18","slug":"python%e4%bb%a3%e7%90%86%e8%ae%bf%e9%97%ae%e7%bd%91%e9%a1%b5","status":"publish","type":"post","link":"https:\/\/byy3.com\/?p=1100","title":{"rendered":"python\u4ee3\u7406\u8bbf\u95ee\u7f51\u9875"},"content":{"rendered":"<pre>\u4f7f\u7528\u968f\u673a\u4ee3\u7406\u8bbf\u95ee\u9700\u8981\u8bbf\u95ee\u7684\u7f51\u9875 \u6682\u65f6\u8fd8\u6ca1\u89e3\u51b3http\u534f\u8bae\u548chttps\u534f\u8bae\u95ee\u9898\r\nimport requests\r\nimport time\r\ntime1 = time.time()\r\n\r\nproxypool_url = 'http:\/\/129.151.235.55:5555\/random'\r\ntarget_url = 'https:\/\/byy3.com'\r\n\r\ndef get_random_proxy():\r\n \"\"\"\r\n get random proxy from proxypool\r\n :return: proxy\r\n \"\"\"\r\n return requests.get(proxypool_url).text.strip()\r\n\r\ndef crawl(url, proxy, headers):\r\n \"\"\"\r\n use proxy to crawl page\r\n :param url: page url\r\n :param proxy: proxy, such as 8.8.8.8:8888\r\n :return: html\r\n \"\"\"\r\n proxies = {'https': 'https:\/\/' + proxy}\r\n headers = headers\r\n return requests.get(url, proxies=proxies).text\r\n\r\n\r\ndef main():\r\n \"\"\"\r\n main method, entry point\r\n :return: none\r\n \"\"\"\r\n headers = {\r\n 'accept': 'text\/html,application\/xhtml+xml,application\/xml;q=0.9,image\/webp,image\/apng,*\/*;q=0.8',\r\n 'User-Agent':'Mozilla\/5.0 (Windows NT 10.0; WOW64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/68.0.3440.106 Safari\/537.36',\r\n }\r\n crawl.request_timeout = 15\r\n proxy = get_random_proxy()\r\n print('get random proxy', proxy)\r\n html = crawl(target_url, proxy,headers)\r\n print(html)\r\n\r\nif __name__ == '__main__':\r\n\u00a0 \u00a0 \u00a0main()<\/pre>\n","protected":false},"excerpt":{"rendered":"<p>\u4f7f\u7528\u968f\u673a\u4ee3\u7406\u8bbf\u95ee\u9700\u8981\u8bbf\u95ee\u7684\u7f51\u9875 \u6682\u65f6\u8fd8\u6ca1\u89e3\u51b3http\u534f\u8bae\u548chttps\u534f\u8bae\u95ee\u9898 import requests [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[20],"tags":[702,701,703,700],"class_list":["post-1100","post","type-post","status-publish","format-standard","hentry","category-python","tag-python","tag-pythonweb"],"_links":{"self":[{"href":"https:\/\/byy3.com\/index.php?rest_route=\/wp\/v2\/posts\/1100","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/byy3.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/byy3.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/byy3.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/byy3.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1100"}],"version-history":[{"count":0,"href":"https:\/\/byy3.com\/index.php?rest_route=\/wp\/v2\/posts\/1100\/revisions"}],"wp:attachment":[{"href":"https:\/\/byy3.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1100"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/byy3.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1100"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/byy3.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1100"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}