摘要：爬蟲(chóng)練手，主要運(yùn)用，由于要對(duì)內(nèi)部進(jìn)行分析，所以就直接用了正則匹配，平時(shí)也可以用用，應(yīng)該更加方便思路讀取首頁(yè)，就是如為了全部抓取，我們這里都是結(jié)尾找到頁(yè)面里面的標(biāo)簽里面的，這個(gè)這個(gè)地址就是頁(yè)面自動(dòng)播放的下個(gè)頁(yè)面同時(shí)讀出即圖片地址，然后保存我這
爬蟲(chóng)練手，主要運(yùn)用requests，由于要對(duì)script內(nèi)部進(jìn)行分析，所以就直接用了 re 正則匹配，平時(shí)也可以用用Beautifulsoup， 應(yīng)該更加方便
思路：
讀取首頁(yè)，就是如 http://newcar.xcar.com.cn/2674/2015/detail/1.htm；為了全部抓取，我們這里都是 1.htm 結(jié)尾
找到頁(yè)面里面的 script 標(biāo)簽里面的 " var nextUrl " ，這個(gè)這個(gè)地址就是頁(yè)面自動(dòng)播放的下個(gè)頁(yè)面;同時(shí)讀出 img src 即 圖片地址，然后保存（我這里是把 說(shuō)明 等內(nèi)容全部放進(jìn) 目標(biāo)文件的名稱(chēng)中了，其實(shí)大家只要圖片的話完全不需要）
遞歸抓取 全部頁(yè)面
這里另外做的一點(diǎn)實(shí)際 弄了個(gè)xcar_lst 記錄所有頁(yè)面、圖片等信息，只是留作記錄，暫時(shí)沒(méi)用
上代碼：
python
# coding:utf-8
__author__ = "BONFY CHEN"
import requests
import re
PROXIES = None
HEADERS = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36"
, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
, "Accept-Encoding": "gzip,deflate,sdch"
, "Accept-Language": "zh-CN,zh;q=0.8"
}
BASE_FOLDER = "D:/xxx_folder/"
class xcarDown(object):
_base_folder = None
_proxies = None
_headers = None
_website = "http://newcar.xcar.com.cn"
_xcar_lst = []
def set_base_folder(self, base_folder):
self._base_folder = base_folder
def set_headers(self, headers):
self._headers = headers
def set_proxies(self, proxies):
self._proxies = proxies
def __init__(self, base_folder=BASE_FOLDER, proxies=PROXIES, headers=HEADERS):
self.set_base_folder(base_folder)
self.set_headers(headers)
self.set_proxies(proxies)
def download_image_from_url(self, url, name=None):
"""
download_image_from_url
:param url: the resource image url
:param name: he destination file name
:return:
"""
local_filename = name + "_" + url.split("/")[-1]
r = requests.get(url, proxies=self._proxies, headers=self._headers, stream=True)
with open(self._base_folder + local_filename, "wb") as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
f.flush()
f.close()
return local_filename
def download_xcar(self, url):
"""
:param url: the source url in xcar.com.cn
http://newcar.xcar.com.cn/2674/2015/detail/1.htm
:return:
"""
r = requests.get(url, proxies=self._proxies, headers=self._headers)
# print r.encoding
r.encoding = "gbk"
m1 = re.search(r"var nextUrl = "(?P.*.htm)"", r.text)
next_url = m1.groupdict()["n_url"] if m1 else None
m2 = re.search(r")
(?P
.*)</div>", r.text)
title = m3.groupdict()["title"] if m3 else ""
m4 = re.search(r"<div id="6a22guqa" class="zs_c">(?P<cont>.*)</div>", r.text)
cont = m4.groupdict()["cont"] if m4 else ""
m5 = re.search(r"<title>(?P<model>.*)", r.text)
model = m5.groupdict()["model"] if m5 else ""
if pic_url:
try:
self.download_image_from_url(pic_url, name="_".join([model, title, cont]))
print "download complete: pic from {} ".format(pic_url)
except IOError:
print "file name IOERROR"
self.download_image_from_url(pic_url, name=model)
print "download complete: pic from {} ".format(pic_url)
except Exception as e:
print e
dct = dict(pic_url=pic_url, next_url=next_url, title=title, cont=cont, model=model)
self._xcar_lst.append(dct)
if next_url[-4:] == ".htm":
self.download_xcar(self._website + next_url)
if __name__ == "__main__":
print("Welcome to the Pic Download for xcar.com")
print("Downloaded files in the folder: " + BASE_FOLDER )
print("---------------------------------------")
id_modell = int(input("Please enter the modell id(eg.2674): "))
year = int(input("Please enter the year (eg.2015): "))
url = "http://newcar.xcar.com.cn/{}/{}/detail/1.htm".format(id_modell, year)
xcar = xcarDown()
xcar.download_xcar(url)
文章版權(quán)歸作者所有，未經(jīng)允許請(qǐng)勿轉(zhuǎn)載,若此文章存在違規(guī)行為，您可以聯(lián)系管理員刪除。
轉(zhuǎn)載請(qǐng)注明本文地址：http://m.hztianpu.com/yun/37570.html
