From 29c7956ccd37a4293ceff835927ed2e498aceff4 Mon Sep 17 00:00:00 2001 From: criseLYJ <1148995537@qq.com> Date: Thu, 7 Mar 2019 02:33:01 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=96=B0=E6=B5=AA=E5=BE=AE?= =?UTF-8?q?=E5=8D=9A=E7=88=AC=E8=99=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .DS_Store | Bin 10244 -> 10244 bytes README.md | 34 ++++++-- sina/.DS_Store | Bin 6148 -> 6148 bytes sina/spider/Ajax_weibo.py | 148 +++++++++++++++++++++++++++++++++++ sina/spider/selenium_test.py | 62 +++++++++++++++ 5 files changed, 238 insertions(+), 6 deletions(-) create mode 100644 sina/spider/Ajax_weibo.py create mode 100644 sina/spider/selenium_test.py diff --git a/.DS_Store b/.DS_Store index 6be2cfc4a52b7cb14cbd359ac340b11f9f8f90a6..040977c2a7f9db10eafa8bea467e2a1e87791fb8 100644 GIT binary patch delta 84 zcmZn(XbG6$&*;1{U^hRb^JX4_;~b(2|G|I(1Xvh~88R927!oJfiB4oPHk&Lc<|Yak aEDJ8m%gN762P$OTd`H5NZ8N*VUv>bl+!|8= delta 42 vcmZn(XbG6$&*-=@U^hRb<7OU#;~bnE|G|KPVe>UfL$-+xzc#Zg{AC9Kbh;8k diff --git a/README.md b/README.md index 3ffb061..de7f04b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -

Website_login_mode

+

🐍Website_login_mode


@@ -57,26 +57,48 @@ ## show -- bilibili自动登录测试正常，成功率98% +### Bilibili自动登录测试正常，成功率98% ![](./image/bilibili.jpg) -- web微信 +### web微信 ![Alt text](./image/weixin.jpg) -- 图虫爬虫 +### 图虫爬虫 ![](./image/Jietu20190306-232224.jpg) ![](./image/Jietu20190306-232303.jpg) -- guoke.spider使用需谨慎，下载的比较快！10秒能下载一堆，截图我就不展示了，已经删除☺ +### 淘宝web +- taobao.py为模拟登录 +- 剩下的文件为爬虫 + +``` +1. 爬取淘宝各子标签，按销量排名商品信息，按分类保存至MongoDB +2. 通过pandas进行数据分析 +3 .将商品在各省分布、销量排行、地图分布等通过matplotlib绘图显示 +``` + +### guoke.spider使用需谨慎，下载的比较快！10秒能下载一堆，截图我就不展示了，已经删除,东西太多了😝 + +### 微博 +- sina.py为模拟登录 +- spider文件夹中为爬虫 + +``` +1. 输入要爬取的博主ID，获取ajax请求 +2. 解析json数据，爬取博主所有微博，保存至MySQL + +``` + + ## tips of pull request -- 欢迎大家一起来 pull request +- 欢迎大家一起来 pull request 💗 ## Problems diff --git a/sina/.DS_Store b/sina/.DS_Store index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..46362762bb79e57030c18ed3ca8e4becc98a644f 100644 GIT binary patch delta 300 zcmZoMXfc=|#>B`mF;Q%yo}wrl0|Nsi1A_nqgC0WxL#1a4w3{`MCvP zQy>Ie46DT;llesyS($*QDoi#M5#H<}vWFP}a6?ED delta 69 zcmZoMXfc=|#>AjHu~2NHo+1YW5HK<@2yFhyD8{ylX%^#Vb`E|Hpgd6EJM(0I5k*d* SG(!SN-DDFU<;^i7E0_TqCl31n diff --git a/sina/spider/Ajax_weibo.py b/sina/spider/Ajax_weibo.py new file mode 100644 index 0000000..9d6c9bb --- /dev/null +++ b/sina/spider/Ajax_weibo.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +from urllib.parse import urlencode +import requests,pymysql +from pyquery import PyQuery as pq +from selenium import webdriver +from time import sleep + +# 连接数据库 +connection = pymysql.connect(host='localhost', + port=3306, + user='root', + passwd='zkyr1006', + db='python', + charset='utf8') + +cursor = connection.cursor() +sql = "USE python;" +cursor.execute(sql) +connection.commit() + +base_url='https://m.weibo.cn/api/container/getIndex?' +headers = { + 'Host': 'm.weibo.cn', + 'Referer': 'https://m.weibo.cn/u/2145291155', + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', + 'X-Requested-With': 'XMLHttpRequest', +} + +def create_sheet(bozhu): + try: + weibo = ''' + CREATE TABLE weibo( + ID VARCHAR (255) NOT NULL PRIMARY KEY, + text VARCHAR (255), + attitudes VARCHAR (255), + comments VARCHAR (255), + reposts VARCHAR (255) + ) + ''' + # 序号 INT NOT NULL PRIMARY KEY AUTO_INCREMENT, + cursor.execute(weibo) + connection.commit() + except: + pass + +def url_get(): + # # 自动保持cookie,不需要自己维护cookie内容 + # cookies = {} + # s = requests.session() + # with open('E:\example\豆瓣读书爬虫\cookie.txt')as file: + # raw_cookies = file.read() + # for line in raw_cookies.split(';'): + # key, value = line.split('=', 1) + # cookies[key] = value + # # 完善header + # header = {'Upgrade-Insecure-Requests': '1', + # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0', + # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + # 'Accept-Encoding': 'gzip, deflate, br', + # 'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2', + # } + # # get请求，应答解码 + # response = s.get(url=xl_url, headers=header,cookies=cookies) + browser=webdriver.PhantomJS() + browser.get(url='https://m.weibo.cn/') + wb_name=browser.find_element_by_class_name("W_input") + wb_name.send_keys(input('输入博主ID：')) + sleep(10) + search=browser.find_element_by_class_name('W_ficon ficon_search S_ficon') + search.click() + sleep(5) + bz_num=browser.find_element_by_class_name('name_txt') + bz_num.click() + sleep(5) + # 开启了一个新页面，需要跳转到新页面 + handles = browser.window_handles + browser.switch_to_window(handles[1]) + + + + +# https://m.weibo.cn/api/container/getIndex?type=uid&value=2145291155&containerid=1076032145291155 +# 拼接url +def get_page(page): + # 查询字符串 + params = { + 'type': 'uid', + 'value': '2145291155', + 'containerid': '1076032145291155', + 'page': page + } + # 调用urlencode() 方法将params参数转化为 URL 的 GET请求参数 + url = base_url + urlencode(params) + try: + response = requests.get(url, headers=headers) + if response.status_code == 200: + # print(response.json()) + return response.json() + except requests.ConnectionError as e: + print('Error', e.args) + + +# 存储数据，存储到数据库 +def parse_page(json): + if json: + items = json.get('data').get('cards') + for index,item in enumerate(items): + if page == 1 and index == 1: + continue + else: + item = item.get('mblog') + # weibo = {} + # weibo['id'] = item.get('id') + # weibo['text'] = + # weibo['attitudes'] = item.get('attitudes_count') + # weibo['comments'] = item.get('comments_count') + # weibo['reposts'] = item.get('reposts_count') + weibo = [] + weibo.append(item.get('id')) + weibo.append(pq(item.get('text')).text()) + weibo.append(item.get('attitudes_count')) + weibo.append(item.get('comments_count')) + weibo.append(item.get('reposts_count')) + # 遇见重复数据，pass，是根据主键来判断，如果是重复数据，忽略，但是报警告 + try: + sql = '''INSERT INTO weibo (ID,text,attitudes,comments,reposts) + VALUES (%s,%s,%s,%s,%s) ''' + cursor.execute(sql, weibo) + connection.commit() + except: + pass + yield weibo + + +if __name__ == '__main__': + for page in range(1, 17): + json = get_page(page) + results = parse_page(json) + for result in results: + print(result) + + + + +cursor.close() + +# 可以爬任意指定博主所有微博，以博主名建立表，分别储存信息 +# 使用selenium+PhantomJS抓取对应博主主页链接 \ No newline at end of file diff --git a/sina/spider/selenium_test.py b/sina/spider/selenium_test.py new file mode 100644 index 0000000..df1e118 --- /dev/null +++ b/sina/spider/selenium_test.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +from urllib.parse import urlencode +import requests,pymysql +from pyquery import PyQuery as pq +from selenium import webdriver +from time import sleep + + +cookies={} +# with open('E:\Spider\Ajax_微博\cookie.txt')as file: +# raw_cookies=file.read() +# for line in raw_cookies.split(';'): +# key,value=line.split('=',1) +# cookies[key]=value +# # print(cookies) + +webdriver.DesiredCapabilities.FIREFOX['firefox.page.settings.userAgent'] = "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0" +browser = webdriver.Firefox() +browser.get(url='https://m.weibo.cn/') +sleep(2) +browser.delete_all_cookies() +browser.add_cookie({'name': '_T_WM', 'value': '52b0151e9490855bae540e40f881320f', 'path': '/', 'expiry': 1530671807, 'secure': False, 'httpOnly': True}) +browser.add_cookie({'name': 'SUB', 'value': '_2A252ENWfDeRhGeVH6FQX9SzKyD6IHXVV-vvXrDV6PUJbkdANLWHmkW1NT0iViQc71Gn2lMbDYa_ioQFHVnRmZaqg', 'path': '/', 'expiry': 1559615823, 'secure': False, 'httpOnly': True}) +browser.add_cookie({'name': 'SUHB', 'value': '0LcSnbMGfYxAbx', 'path': '/', 'expiry': 1559615823, 'secure': False, 'httpOnly': False}) +browser.add_cookie({'name': 'SCF', 'value': 'AlYBAKMIZxonUEvbmR6JgqyYWHL1yaDreGv8vXPl1FRCO7xm1IoOxBvQbDb-ZqzXTud9qRC3AnLVu7nFx_MvHdc.', 'path': '/', 'expiry': 1843439823, 'secure': False, 'httpOnly': True}) +browser.add_cookie({'name': 'SSOLoginState', 'value': '1528079823', 'path': '/','expiry': None, 'secure': False, 'httpOnly': False}) +browser.add_cookie({'name': 'MLOGIN', 'value': '1', 'path': '/', 'expiry': 1528083425, 'secure': False, 'httpOnly': False}) +browser.add_cookie({'name': 'M_WEIBOCN_PARAMS', 'value': 'uicode%3D20000174%26featurecode%3D20000320%26fid%3Dhotword', 'path': '/', 'expiry': 1528080425, 'secure': False, 'httpOnly': True}) + +sleep(2) +browser.get(url='https://m.weibo.cn/') +sleep(40) +print(browser.get_cookies()) + + + + + +# elem=browser.find_element_by_class_name('btn btnWhite') +# elem.click() +# username=browser.find_element_by_id('loginName') + # username.send_keys('18401570769') + + +# browser.refresh() +# print(browser.page_source) +# if browser.find_element_by_class_name() + + + +# wb_name = browser.find_element_by_class_name("W_input") +# wb_name.send_keys(input('输入博主ID：')) +# sleep(10) +# search = browser.find_element_by_class_name('W_ficon ficon_search S_ficon') +# search.click() +# sleep(5) +# bz_num = browser.find_element_by_class_name('name_txt') +# bz_num.click() +# sleep(5) +# # 开启了一个新页面，需要跳转到新页面 +# handles = browser.window_handles +# browser.switch_to_window(handles[1]) \ No newline at end of file