mirror of
https://github.com/Kr1s77/awesome-python-login-model.git
synced 2026-10-22 07:06:13 +08:00
remove lianja spider.
This commit is contained in:
CriseLYJ
@@ -1,7 +0,0 @@
|
||||
一个基于分页、线程池、代理池的链家网快速爬虫项目，速度可达 10000 条/5 分钟，严禁将所得数据商用！
|
||||
|
||||
同时对数据进行了清洗、分析、可视化。
|
||||
|
||||
- lianjia:只爬取
|
||||
- pic：图表
|
||||
- 链家来自：https://github.com/Python3Spiders/LianJiaSpider
|
||||
@@ -1,189 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import requests
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from pyquery import PyQuery as pq
|
||||
|
||||
import json
|
||||
|
||||
import threading
|
||||
|
||||
import time
|
||||
|
||||
"""
|
||||
info:
|
||||
author:CriseLYJ
|
||||
github:https://github.com/CriseLYJ/
|
||||
update_time:2019-3-7
|
||||
"""
|
||||
|
||||
|
||||
def get_list_page_url(city):
|
||||
start_url = "https://{}.lianjia.com/ershoufang".format(city)
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
|
||||
}
|
||||
try:
|
||||
response = requests.get(start_url, headers=headers)
|
||||
# print(response.status_code, response.text)
|
||||
doc = pq(response.text)
|
||||
total_num = int(doc(".resultDes .total span").text())
|
||||
total_page = total_num // 30 + 1
|
||||
# 只能访问到前一百页
|
||||
if total_page > 100:
|
||||
total_page = 100
|
||||
|
||||
page_url_list = list()
|
||||
|
||||
for i in range(total_page):
|
||||
url = start_url + "/pg" + str(i + 1) + "/"
|
||||
page_url_list.append(url)
|
||||
# print(url)
|
||||
return page_url_list
|
||||
|
||||
except:
|
||||
print("获取总套数出错,请确认起始URL是否正确")
|
||||
return None
|
||||
|
||||
|
||||
detail_list = list()
|
||||
|
||||
|
||||
def get_valid_ip():
|
||||
url = "http://localhost:5000/get"
|
||||
try:
|
||||
ip = requests.get(url).text
|
||||
return ip
|
||||
except:
|
||||
print("请先运行代理池")
|
||||
|
||||
|
||||
def get_detail_page_url(page_url):
|
||||
global detail_list
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
|
||||
'Referer': 'https://bj.lianjia.com/ershoufang'
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(page_url, headers=headers, timeout=3)
|
||||
doc = pq(response.text)
|
||||
# broswer.get(page_url)
|
||||
# print(page_url)
|
||||
# doc = pq(broswer.page_source)
|
||||
i = 0
|
||||
detail_urls = list()
|
||||
for item in doc(".sellListContent li").items():
|
||||
i += 1
|
||||
if i == 31:
|
||||
break
|
||||
child_item = item(".noresultRecommend")
|
||||
if child_item == None:
|
||||
i -= 1
|
||||
detail_url = child_item.attr("href")
|
||||
detail_urls.append(detail_url)
|
||||
return detail_urls
|
||||
except:
|
||||
print("获取列表页" + page_url + "出错")
|
||||
|
||||
|
||||
lock = threading.Lock()
|
||||
|
||||
|
||||
def detail_page_parser(res):
|
||||
global detail_list
|
||||
detail_urls = res.result()
|
||||
if not detail_urls:
|
||||
print("detail url 为空")
|
||||
return None
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
|
||||
'Referer': 'https://bj.lianjia.com/ershoufang'
|
||||
}
|
||||
for detail_url in detail_urls:
|
||||
try:
|
||||
response = requests.get(url=detail_url, headers=headers, timeout=3)
|
||||
# print(response.status_code)
|
||||
detail_dict = dict()
|
||||
doc = pq(response.text)
|
||||
unit_price = doc(".unitPriceValue").text()
|
||||
unit_price = unit_price[0:unit_price.index("元")]
|
||||
title = doc("h1").text()
|
||||
area = doc(".areaName .info a").eq(0).text().strip()
|
||||
url = detail_url
|
||||
detail_dict["title"] = title
|
||||
detail_dict["area"] = area
|
||||
detail_dict["price"] = unit_price
|
||||
detail_dict["url"] = url
|
||||
|
||||
detail_list.append(detail_dict)
|
||||
|
||||
print(unit_price, title, area)
|
||||
|
||||
except:
|
||||
print("获取详情页出错,换ip重试")
|
||||
proxies = {
|
||||
"http": "http://" + get_valid_ip(),
|
||||
}
|
||||
try:
|
||||
response = requests.get(url=detail_url, headers=headers, proxies=proxies)
|
||||
# print(response.status_code)
|
||||
detail_dict = dict()
|
||||
doc = pq(response.text)
|
||||
unit_price = doc(".unitPriceValue").text()
|
||||
unit_price = unit_price[0:unit_price.index("元")]
|
||||
title = doc("h1").text()
|
||||
area = doc(".areaName .info a").eq(0).text().strip()
|
||||
url = detail_url
|
||||
# 已下架的还会爬取，但是没有价格
|
||||
if len(unit_price) > 0:
|
||||
detail_dict["title"] = title
|
||||
detail_dict["area"] = area
|
||||
detail_dict["price"] = unit_price
|
||||
detail_dict["url"] = url
|
||||
|
||||
detail_list.append(detail_dict)
|
||||
|
||||
print(unit_price, title, area)
|
||||
except:
|
||||
print("重试失败...")
|
||||
|
||||
|
||||
def save_data(data, filename):
|
||||
with open(filename + ".json", 'w', encoding="utf-8") as f:
|
||||
f.write(json.dumps(data, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
def main():
|
||||
# cq,cs,nj,dl,wh,cc
|
||||
city_list = ['nj']
|
||||
for city in city_list:
|
||||
page_url_list = get_list_page_url(city)
|
||||
|
||||
# pool = threadpool.ThreadPool(20)
|
||||
# requests = threadpool.makeRequests(page_and_detail_parser, page_url_list)
|
||||
# [pool.putRequest(req) for req in requests]
|
||||
# pool.wait()
|
||||
|
||||
p = ThreadPoolExecutor(30)
|
||||
|
||||
for page_url in page_url_list:
|
||||
p.submit(get_detail_page_url, page_url).add_done_callback(detail_page_parser)
|
||||
# 这里的回调函数拿到的是一个对象。
|
||||
# 先把返回的res得到一个结果。即在前面加上一个res.result(),这个结果就是get_detail_page_url的返回
|
||||
p.shutdown()
|
||||
|
||||
print(detail_list)
|
||||
|
||||
save_data(detail_list, city)
|
||||
|
||||
detail_list.clear()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
old = time.time()
|
||||
main()
|
||||
new = time.time()
|
||||
delta_time = new - old
|
||||
print("程序共运行{}s".format(delta_time))
|
||||
Reference in New Issue
Block a user