mirror of
https://github.com/Kr1s77/awesome-python-login-model.git
synced 2026-10-29 03:16:31 +08:00
增加新浪微博爬虫
This commit is contained in:
criseLYJ
@@ -1,4 +1,4 @@
|
||||
<h2 align="center"><code>Website_login_mode</code></h2>
|
||||
<h2 align="center"><code>🐍Website_login_mode</code></h2>
|
||||
|
||||
<br>
|
||||
<p align="center">
|
||||
@@ -57,26 +57,48 @@
|
||||
|
||||
## show
|
||||
|
||||
- bilibili自动登录测试正常，成功率98%
|
||||
### Bilibili自动登录测试正常，成功率98%
|
||||
|
||||
|
||||
|
||||
- web微信
|
||||
### web微信
|
||||
|
||||
|
||||
|
||||
|
||||
- 图虫爬虫
|
||||
### 图虫爬虫
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
- guoke.spider使用需谨慎，下载的比较快！10秒能下载一堆，截图我就不展示了，已经删除☺
|
||||
### 淘宝web
|
||||
- taobao.py为模拟登录
|
||||
- 剩下的文件为爬虫
|
||||
|
||||
```
|
||||
1. 爬取淘宝各子标签，按销量排名商品信息，按分类保存至MongoDB
|
||||
2. 通过pandas进行数据分析
|
||||
3 .将商品在各省分布、销量排行、地图分布等通过matplotlib绘图显示
|
||||
```
|
||||
|
||||
### guoke.spider使用需谨慎，下载的比较快！10秒能下载一堆，截图我就不展示了，已经删除,东西太多了😝
|
||||
|
||||
### 微博
|
||||
- sina.py为模拟登录
|
||||
- spider文件夹中为爬虫
|
||||
|
||||
```
|
||||
1. 输入要爬取的博主ID，获取ajax请求
|
||||
2. 解析json数据，爬取博主所有微博，保存至MySQL
|
||||
|
||||
```
|
||||
|
||||
|
||||
|
||||
## tips of pull request
|
||||
|
||||
- 欢迎大家一起来 pull request
|
||||
- 欢迎大家一起来 pull request 💗
|
||||
|
||||
## Problems
|
||||
|
||||
|
||||
Vendored
BIN
Binary file not shown.
@@ -0,0 +1,148 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from urllib.parse import urlencode
|
||||
import requests,pymysql
|
||||
from pyquery import PyQuery as pq
|
||||
from selenium import webdriver
|
||||
from time import sleep
|
||||
|
||||
# 连接数据库
|
||||
connection = pymysql.connect(host='localhost',
|
||||
port=3306,
|
||||
user='root',
|
||||
passwd='zkyr1006',
|
||||
db='python',
|
||||
charset='utf8')
|
||||
|
||||
cursor = connection.cursor()
|
||||
sql = "USE python;"
|
||||
cursor.execute(sql)
|
||||
connection.commit()
|
||||
|
||||
base_url='https://m.weibo.cn/api/container/getIndex?'
|
||||
headers = {
|
||||
'Host': 'm.weibo.cn',
|
||||
'Referer': 'https://m.weibo.cn/u/2145291155',
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
}
|
||||
|
||||
def create_sheet(bozhu):
|
||||
try:
|
||||
weibo = '''
|
||||
CREATE TABLE weibo(
|
||||
ID VARCHAR (255) NOT NULL PRIMARY KEY,
|
||||
text VARCHAR (255),
|
||||
attitudes VARCHAR (255),
|
||||
comments VARCHAR (255),
|
||||
reposts VARCHAR (255)
|
||||
)
|
||||
'''
|
||||
# 序号 INT NOT NULL PRIMARY KEY AUTO_INCREMENT,
|
||||
cursor.execute(weibo)
|
||||
connection.commit()
|
||||
except:
|
||||
pass
|
||||
|
||||
def url_get():
|
||||
# # 自动保持cookie,不需要自己维护cookie内容
|
||||
# cookies = {}
|
||||
# s = requests.session()
|
||||
# with open('E:\example\豆瓣读书爬虫\cookie.txt')as file:
|
||||
# raw_cookies = file.read()
|
||||
# for line in raw_cookies.split(';'):
|
||||
# key, value = line.split('=', 1)
|
||||
# cookies[key] = value
|
||||
# # 完善header
|
||||
# header = {'Upgrade-Insecure-Requests': '1',
|
||||
# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0',
|
||||
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
# 'Accept-Encoding': 'gzip, deflate, br',
|
||||
# 'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
|
||||
# }
|
||||
# # get请求，应答解码
|
||||
# response = s.get(url=xl_url, headers=header,cookies=cookies)
|
||||
browser=webdriver.PhantomJS()
|
||||
browser.get(url='https://m.weibo.cn/')
|
||||
wb_name=browser.find_element_by_class_name("W_input")
|
||||
wb_name.send_keys(input('输入博主ID：'))
|
||||
sleep(10)
|
||||
search=browser.find_element_by_class_name('W_ficon ficon_search S_ficon')
|
||||
search.click()
|
||||
sleep(5)
|
||||
bz_num=browser.find_element_by_class_name('name_txt')
|
||||
bz_num.click()
|
||||
sleep(5)
|
||||
# 开启了一个新页面，需要跳转到新页面
|
||||
handles = browser.window_handles
|
||||
browser.switch_to_window(handles[1])
|
||||
|
||||
|
||||
|
||||
|
||||
# https://m.weibo.cn/api/container/getIndex?type=uid&value=2145291155&containerid=1076032145291155
|
||||
# 拼接url
|
||||
def get_page(page):
|
||||
# 查询字符串
|
||||
params = {
|
||||
'type': 'uid',
|
||||
'value': '2145291155',
|
||||
'containerid': '1076032145291155',
|
||||
'page': page
|
||||
}
|
||||
# 调用urlencode() 方法将params参数转化为 URL 的 GET请求参数
|
||||
url = base_url + urlencode(params)
|
||||
try:
|
||||
response = requests.get(url, headers=headers)
|
||||
if response.status_code == 200:
|
||||
# print(response.json())
|
||||
return response.json()
|
||||
except requests.ConnectionError as e:
|
||||
print('Error', e.args)
|
||||
|
||||
|
||||
# 存储数据，存储到数据库
|
||||
def parse_page(json):
|
||||
if json:
|
||||
items = json.get('data').get('cards')
|
||||
for index,item in enumerate(items):
|
||||
if page == 1 and index == 1:
|
||||
continue
|
||||
else:
|
||||
item = item.get('mblog')
|
||||
# weibo = {}
|
||||
# weibo['id'] = item.get('id')
|
||||
# weibo['text'] =
|
||||
# weibo['attitudes'] = item.get('attitudes_count')
|
||||
# weibo['comments'] = item.get('comments_count')
|
||||
# weibo['reposts'] = item.get('reposts_count')
|
||||
weibo = []
|
||||
weibo.append(item.get('id'))
|
||||
weibo.append(pq(item.get('text')).text())
|
||||
weibo.append(item.get('attitudes_count'))
|
||||
weibo.append(item.get('comments_count'))
|
||||
weibo.append(item.get('reposts_count'))
|
||||
# 遇见重复数据，pass，是根据主键来判断，如果是重复数据，忽略，但是报警告
|
||||
try:
|
||||
sql = '''INSERT INTO weibo (ID,text,attitudes,comments,reposts)
|
||||
VALUES (%s,%s,%s,%s,%s) '''
|
||||
cursor.execute(sql, weibo)
|
||||
connection.commit()
|
||||
except:
|
||||
pass
|
||||
yield weibo
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
for page in range(1, 17):
|
||||
json = get_page(page)
|
||||
results = parse_page(json)
|
||||
for result in results:
|
||||
print(result)
|
||||
|
||||
|
||||
|
||||
|
||||
cursor.close()
|
||||
|
||||
# 可以爬任意指定博主所有微博，以博主名建立表，分别储存信息
|
||||
# 使用selenium+PhantomJS抓取对应博主主页链接
|
||||
@@ -0,0 +1,62 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from urllib.parse import urlencode
|
||||
import requests,pymysql
|
||||
from pyquery import PyQuery as pq
|
||||
from selenium import webdriver
|
||||
from time import sleep
|
||||
|
||||
|
||||
cookies={}
|
||||
# with open('E:\Spider\Ajax_微博\cookie.txt')as file:
|
||||
# raw_cookies=file.read()
|
||||
# for line in raw_cookies.split(';'):
|
||||
# key,value=line.split('=',1)
|
||||
# cookies[key]=value
|
||||
# # print(cookies)
|
||||
|
||||
webdriver.DesiredCapabilities.FIREFOX['firefox.page.settings.userAgent'] = "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0"
|
||||
browser = webdriver.Firefox()
|
||||
browser.get(url='https://m.weibo.cn/')
|
||||
sleep(2)
|
||||
browser.delete_all_cookies()
|
||||
browser.add_cookie({'name': '_T_WM', 'value': '52b0151e9490855bae540e40f881320f', 'path': '/', 'expiry': 1530671807, 'secure': False, 'httpOnly': True})
|
||||
browser.add_cookie({'name': 'SUB', 'value': '_2A252ENWfDeRhGeVH6FQX9SzKyD6IHXVV-vvXrDV6PUJbkdANLWHmkW1NT0iViQc71Gn2lMbDYa_ioQFHVnRmZaqg', 'path': '/', 'expiry': 1559615823, 'secure': False, 'httpOnly': True})
|
||||
browser.add_cookie({'name': 'SUHB', 'value': '0LcSnbMGfYxAbx', 'path': '/', 'expiry': 1559615823, 'secure': False, 'httpOnly': False})
|
||||
browser.add_cookie({'name': 'SCF', 'value': 'AlYBAKMIZxonUEvbmR6JgqyYWHL1yaDreGv8vXPl1FRCO7xm1IoOxBvQbDb-ZqzXTud9qRC3AnLVu7nFx_MvHdc.', 'path': '/', 'expiry': 1843439823, 'secure': False, 'httpOnly': True})
|
||||
browser.add_cookie({'name': 'SSOLoginState', 'value': '1528079823', 'path': '/','expiry': None, 'secure': False, 'httpOnly': False})
|
||||
browser.add_cookie({'name': 'MLOGIN', 'value': '1', 'path': '/', 'expiry': 1528083425, 'secure': False, 'httpOnly': False})
|
||||
browser.add_cookie({'name': 'M_WEIBOCN_PARAMS', 'value': 'uicode%3D20000174%26featurecode%3D20000320%26fid%3Dhotword', 'path': '/', 'expiry': 1528080425, 'secure': False, 'httpOnly': True})
|
||||
|
||||
sleep(2)
|
||||
browser.get(url='https://m.weibo.cn/')
|
||||
sleep(40)
|
||||
print(browser.get_cookies())
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# elem=browser.find_element_by_class_name('btn btnWhite')
|
||||
# elem.click()
|
||||
# username=browser.find_element_by_id('loginName')
|
||||
# username.send_keys('18401570769')
|
||||
|
||||
|
||||
# browser.refresh()
|
||||
# print(browser.page_source)
|
||||
# if browser.find_element_by_class_name()
|
||||
|
||||
|
||||
|
||||
# wb_name = browser.find_element_by_class_name("W_input")
|
||||
# wb_name.send_keys(input('输入博主ID：'))
|
||||
# sleep(10)
|
||||
# search = browser.find_element_by_class_name('W_ficon ficon_search S_ficon')
|
||||
# search.click()
|
||||
# sleep(5)
|
||||
# bz_num = browser.find_element_by_class_name('name_txt')
|
||||
# bz_num.click()
|
||||
# sleep(5)
|
||||
# # 开启了一个新页面，需要跳转到新页面
|
||||
# handles = browser.window_handles
|
||||
# browser.switch_to_window(handles[1])
|
||||
Reference in New Issue
Block a user