Website_login_mode🐍Website_login_mode@@ -57,26 +57,48 @@ ## show -- bilibili自动登录测试正常，成功率98% +### Bilibili自动登录测试正常，成功率98%  -- web微信 +### web微信  -- 图虫爬虫 +### 图虫爬虫   -- guoke.spider使用需谨慎，下载的比较快！10秒能下载一堆，截图我就不展示了，已经删除☺ +### 淘宝web +- taobao.py为模拟登录 +- 剩下的文件为爬虫 + +``` +1. 爬取淘宝各子标签，按销量排名商品信息，按分类保存至MongoDB +2. 通过pandas进行数据分析 +3 .将商品在各省分布、销量排行、地图分布等通过matplotlib绘图显示 +``` + +### guoke.spider使用需谨慎，下载的比较快！10秒能下载一堆，截图我就不展示了，已经删除,东西太多了😝 + +### 微博 +- sina.py为模拟登录 +- spider文件夹中为爬虫 + +``` +1. 输入要爬取的博主ID，获取ajax请求 +2. 解析json数据，爬取博主所有微博，保存至MySQL + +``` + + ## tips of pull request -- 欢迎大家一起来 pull request +- 欢迎大家一起来 pull request 💗 ## Problems diff --git a/sina/.DS_Store b/sina/.DS_Store index 5008ddf..4636276 100644 Binary files a/sina/.DS_Store and b/sina/.DS_Store differ diff --git a/sina/spider/Ajax_weibo.py b/sina/spider/Ajax_weibo.py new file mode 100644 index 0000000..9d6c9bb --- /dev/null +++ b/sina/spider/Ajax_weibo.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +from urllib.parse import urlencode +import requests,pymysql +from pyquery import PyQuery as pq +from selenium import webdriver +from time import sleep + +# 连接数据库 +connection = pymysql.connect(host='localhost', + port=3306, + user='root', + passwd='zkyr1006', + db='python', + charset='utf8') + +cursor = connection.cursor() +sql = "USE python;" +cursor.execute(sql) +connection.commit() + +base_url='https://m.weibo.cn/api/container/getIndex?' +headers = { + 'Host': 'm.weibo.cn', + 'Referer': 'https://m.weibo.cn/u/2145291155', + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', + 'X-Requested-With': 'XMLHttpRequest', +} + +def create_sheet(bozhu): + try: + weibo = ''' + CREATE TABLE weibo( + ID VARCHAR (255) NOT NULL PRIMARY KEY, + text VARCHAR (255), + attitudes VARCHAR (255), + comments VARCHAR (255), + reposts VARCHAR (255) + ) + ''' + # 序号 INT NOT NULL PRIMARY KEY AUTO_INCREMENT, + cursor.execute(weibo) + connection.commit() + except: + pass + +def url_get(): + # # 自动保持cookie,不需要自己维护cookie内容 + # cookies = {} + # s = requests.session() + # with open('E:\example\豆瓣读书爬虫\cookie.txt')as file: + # raw_cookies = file.read() + # for line in raw_cookies.split(';'): + # key, value = line.split('=', 1) + # cookies[key] = value + # # 完善header + # header = {'Upgrade-Insecure-Requests': '1', + # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0', + # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + # 'Accept-Encoding': 'gzip, deflate, br', + # 'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2', + # } + # # get请求，应答解码 + # response = s.get(url=xl_url, headers=header,cookies=cookies) + browser=webdriver.PhantomJS() + browser.get(url='https://m.weibo.cn/') + wb_name=browser.find_element_by_class_name("W_input") + wb_name.send_keys(input('输入博主ID：')) + sleep(10) + search=browser.find_element_by_class_name('W_ficon ficon_search S_ficon') + search.click() + sleep(5) + bz_num=browser.find_element_by_class_name('name_txt') + bz_num.click() + sleep(5) + # 开启了一个新页面，需要跳转到新页面 + handles = browser.window_handles + browser.switch_to_window(handles[1]) + + + + +# https://m.weibo.cn/api/container/getIndex?type=uid&value=2145291155&containerid=1076032145291155 +# 拼接url +def get_page(page): + # 查询字符串 + params = { + 'type': 'uid', + 'value': '2145291155', + 'containerid': '1076032145291155', + 'page': page + } + # 调用urlencode() 方法将params参数转化为 URL 的 GET请求参数 + url = base_url + urlencode(params) + try: + response = requests.get(url, headers=headers) + if response.status_code == 200: + # print(response.json()) + return response.json() + except requests.ConnectionError as e: + print('Error', e.args) + + +# 存储数据，存储到数据库 +def parse_page(json): + if json: + items = json.get('data').get('cards') + for index,item in enumerate(items): + if page == 1 and index == 1: + continue + else: + item = item.get('mblog') + # weibo = {} + # weibo['id'] = item.get('id') + # weibo['text'] = + # weibo['attitudes'] = item.get('attitudes_count') + # weibo['comments'] = item.get('comments_count') + # weibo['reposts'] = item.get('reposts_count') + weibo = [] + weibo.append(item.get('id')) + weibo.append(pq(item.get('text')).text()) + weibo.append(item.get('attitudes_count')) + weibo.append(item.get('comments_count')) + weibo.append(item.get('reposts_count')) + # 遇见重复数据，pass，是根据主键来判断，如果是重复数据，忽略，但是报警告 + try: + sql = '''INSERT INTO weibo (ID,text,attitudes,comments,reposts) + VALUES (%s,%s,%s,%s,%s) ''' + cursor.execute(sql, weibo) + connection.commit() + except: + pass + yield weibo + + +if __name__ == '__main__': + for page in range(1, 17): + json = get_page(page) + results = parse_page(json) + for result in results: + print(result) + + + + +cursor.close() + +# 可以爬任意指定博主所有微博，以博主名建立表，分别储存信息 +# 使用selenium+PhantomJS抓取对应博主主页链接 \ No newline at end of file diff --git a/sina/spider/selenium_test.py b/sina/spider/selenium_test.py new file mode 100644 index 0000000..df1e118 --- /dev/null +++ b/sina/spider/selenium_test.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +from urllib.parse import urlencode +import requests,pymysql +from pyquery import PyQuery as pq +from selenium import webdriver +from time import sleep + + +cookies={} +# with open('E:\Spider\Ajax_微博\cookie.txt')as file: +# raw_cookies=file.read() +# for line in raw_cookies.split(';'): +# key,value=line.split('=',1) +# cookies[key]=value +# # print(cookies) + +webdriver.DesiredCapabilities.FIREFOX['firefox.page.settings.userAgent'] = "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0" +browser = webdriver.Firefox() +browser.get(url='https://m.weibo.cn/') +sleep(2) +browser.delete_all_cookies() +browser.add_cookie({'name': '_T_WM', 'value': '52b0151e9490855bae540e40f881320f', 'path': '/', 'expiry': 1530671807, 'secure': False, 'httpOnly': True}) +browser.add_cookie({'name': 'SUB', 'value': '_2A252ENWfDeRhGeVH6FQX9SzKyD6IHXVV-vvXrDV6PUJbkdANLWHmkW1NT0iViQc71Gn2lMbDYa_ioQFHVnRmZaqg', 'path': '/', 'expiry': 1559615823, 'secure': False, 'httpOnly': True}) +browser.add_cookie({'name': 'SUHB', 'value': '0LcSnbMGfYxAbx', 'path': '/', 'expiry': 1559615823, 'secure': False, 'httpOnly': False}) +browser.add_cookie({'name': 'SCF', 'value': 'AlYBAKMIZxonUEvbmR6JgqyYWHL1yaDreGv8vXPl1FRCO7xm1IoOxBvQbDb-ZqzXTud9qRC3AnLVu7nFx_MvHdc.', 'path': '/', 'expiry': 1843439823, 'secure': False, 'httpOnly': True}) +browser.add_cookie({'name': 'SSOLoginState', 'value': '1528079823', 'path': '/','expiry': None, 'secure': False, 'httpOnly': False}) +browser.add_cookie({'name': 'MLOGIN', 'value': '1', 'path': '/', 'expiry': 1528083425, 'secure': False, 'httpOnly': False}) +browser.add_cookie({'name': 'M_WEIBOCN_PARAMS', 'value': 'uicode%3D20000174%26featurecode%3D20000320%26fid%3Dhotword', 'path': '/', 'expiry': 1528080425, 'secure': False, 'httpOnly': True}) + +sleep(2) +browser.get(url='https://m.weibo.cn/') +sleep(40) +print(browser.get_cookies()) + + + + + +# elem=browser.find_element_by_class_name('btn btnWhite') +# elem.click() +# username=browser.find_element_by_id('loginName') + # username.send_keys('18401570769') + + +# browser.refresh() +# print(browser.page_source) +# if browser.find_element_by_class_name() + + + +# wb_name = browser.find_element_by_class_name("W_input") +# wb_name.send_keys(input('输入博主ID：')) +# sleep(10) +# search = browser.find_element_by_class_name('W_ficon ficon_search S_ficon') +# search.click() +# sleep(5) +# bz_num = browser.find_element_by_class_name('name_txt') +# bz_num.click() +# sleep(5) +# # 开启了一个新页面，需要跳转到新页面 +# handles = browser.window_handles +# browser.switch_to_window(handles[1]) \ No newline at end of file