想做一个有趣的项目,首先整理一下思路,如何快速爬取关键信息。并且实现自动翻页功能。
想了想用最常规的requests加上re正则表达式,BeautifulSoup用于批量爬取
import requests
import re
from bs4 import BeautifulSoup
import pyMySQL
然后引入链接,注意这里有反爬虫机制,第一页必须为https://tianjin.anjuke.com/sale/,后面页必须为’https://tianjin.anjuke.com/sale/p%d/#filtersort’%page,不然会被机制检测到为爬虫,无法实现爬取。这里实现了翻页功能。
while page < 11:
# brower.get("https://tianjin.anjuke.com/sale/p%d/#filtersort"%page)
# time.sleep(1)
print ("这是第"+str(page) +"页")
# proxy=requests.get(pool_url).text
# proxies={
# 'http': 'http://' + proxy
# }
if page==1:
url='https://tianjin.anjuke.com/sale/'
headers={
'referer': 'https://tianjin.anjuke.com/sale/',
'user-agent': 'Mozilla/5.0 (windows NT 10.0; Win64; x64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
}
else:
url='https://tianjin.anjuke.com/sale/p%d/#filtersort'%page
headers={
'referer': 'https://tianjin.anjuke.com/sale/p%d/#filtersort'%page,
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
}
# html=requests.get(url,allow_redirects=False,headers=headers,proxies=proxies)
html = requests.get(url, headers=headers)
第二步自然是分析网页以及如何实现自动翻页,首先找到图片
正则表达式走起!
#图片地址
myjpg=r'<img src="(.*?)" width="180" height="135" />'
jpg=re.findall(myjpg,html.text)
照片信息已经完成爬取,接下来依葫芦画瓢,把其它信息页也迅速爬取!
#描述
mytail=r'<a data-from="" data-company="" title="(.*?)" href'
tail=re.findall(mytail,html.text)
# 获取总价
totalprice=r'<span class="price-det"><strong>(.*?)</strong>'
mytotal=re.findall(totalprice,html.text)
#单价
simpleprice=r'<span class="unit-price">(.*?)</span> '
simple=re.findall(simpleprice,html.text)
接下来实现用beauitfulsoup实现关键字标签取值!解析器我这里用lxml,速度比较快,当然也可以用html.parser
soup=BeautifulSoup(html.content,'lxml')
看图,这里用了很多换行符,并且span标签没有命名,所以请上我们的嘉宾bs4
这里使用了循环,因为是一次性爬取,一个300条信息,由于一页图片只有60张,所以将其5个一组进行划分,re.sub目的为了将其中的非字符信息替换为空以便存入数据库
#获取房子信息
itemdetail=soup.select(".details-item span")
# print(len(itemdetail))
you=[]
my=[]
for i in itemdetail:
# print(i.get_text())
you.append(i.get_text())
k = 0
while k < 60:
my.append([you[5 * k], you[5 * k + 1], you[5 * k + 2], you[5 * k + 3],re.sub(r's', "", you[5 * k + 4])])
k = k + 1
# print(my)
# print(len(my))
接下来存入数据库!
db = pymysql.connect("localhost", "root", "" ,"anjuke")
conn = db.cursor()
print(len(jpg))
for i in range(0,len(tail)):
jpgs = jpg[i]
scripts = tail[i]
localroom = my[i][0]
localarea=my[i][1]
localhigh=my[i][2]
localtimes=my[i][3]
local=my[i][4]
total = mytotal[i]
oneprice=simple[i]
sql = "insert into shanghai_admin value('%s','%s','%s','%s','%s','%s','%s','%s','%s')" %
(jpgs,scripts,local,total,oneprice,localroom,localarea,localhigh,localtimes)
conn.execute(sql)
db.commit()
db.close()
大功告成!来看看效果!
以下为完整代码:
# from selenium import webdriver
import requests
import re
from bs4 import BeautifulSoup
import pymysql
# import time
# chrome_driver=r"C:Users秦QQAppDataLocalProgramsPythonPython38-32Libsite-packagesselenium-3.141.0-py3.8.eggseleniumwebdriverchromechromedriver.exe"
# brower=webdriver.Chrome(executable_path=chrome_driver)
# pool_url='http://localhost:5555/random'
page=1
while page < 11:
# brower.get("https://tianjin.anjuke.com/sale/p%d/#filtersort"%page)
# time.sleep(1)
print ("这是第"+str(page) +"页")
# proxy=requests.get(pool_url).text
# proxies={
# 'http': 'http://' + proxy
# }
if page==1:
url='https://tianjin.anjuke.com/sale/'
headers={
'referer': 'https://tianjin.anjuke.com/sale/',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
}
else:
url='https://tianjin.anjuke.com/sale/p%d/#filtersort'%page
headers={
'referer': 'https://tianjin.anjuke.com/sale/p%d/#filtersort'%page,
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
}
# html=requests.get(url,allow_redirects=False,headers=headers,proxies=proxies)
html = requests.get(url, headers=headers)
soup=BeautifulSoup(html.content,'lxml')
#图片地址
myjpg=r'<img src="(.*?)" width="180" height="135" />'
jpg=re.findall(myjpg,html.text)
#描述
mytail=r'<a data-from="" data-company="" title="(.*?)" href'
tail=re.findall(mytail,html.text)
#获取房子信息
itemdetail=soup.select(".details-item span")
# print(len(itemdetail))
you=[]
my=[]
for i in itemdetail:
# print(i.get_text())
you.append(i.get_text())
k = 0
while k < 60:
my.append([you[5 * k], you[5 * k + 1], you[5 * k + 2], you[5 * k + 3],re.sub(r's', "", you[5 * k + 4])])
k = k + 1
# print(my)
# print(len(my))
# 获取总价
totalprice=r'<span class="price-det"><strong>(.*?)</strong>'
mytotal=re.findall(totalprice,html.text)
#单价
simpleprice=r'<span class="unit-price">(.*?)</span> '
simple=re.findall(simpleprice,html.text)
db = pymysql.connect("localhost", "root", "" ,"anjuke")
conn = db.cursor()
print(len(jpg))
for i in range(0,len(tail)):
jpgs = jpg[i]
scripts = tail[i]
localroom = my[i][0]
localarea=my[i][1]
localhigh=my[i][2]
localtimes=my[i][3]
local=my[i][4]
total = mytotal[i]
oneprice=simple[i]
sql = "insert into shanghai_admin value('%s','%s','%s','%s','%s','%s','%s','%s','%s')" %
(jpgs,scripts,local,total,oneprice,localroom,localarea,localhigh,localtimes)
conn.execute(sql)
db.commit()
db.close()
# button=brower.find_element_by_class_name('aNxt')
# button.click()
# time.sleep(1)
page=page+1
# brower.close()