GithubStar/GHunt
1
0
Fork 0
mirror of https://github.com/mxrch/GHunt.git synced 2026-10-29 12:41:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed gmaps scraping issue (reviews not extracted anymore, because google changed something)

Browse Source
This commit is contained in:
mxrch
2021-06-11 01:00:35 +02:00
parent 2f8fdbce62
commit eb25fde2d8
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/gmaps.py
+3 -3
View File
@@ -79,9 +79,9 @@ def scrape(gaiaID, client, cookies, config, headers, regex_rev_by_id, is_headles
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'div.section-scrollbox')))
scrollbox = driver.find_element_by_css_selector('div.section-scrollbox')
tab_info = driver.find_elements_by_css_selector('div.section-tab-info')
tab_info = scrollbox.find_element_by_tag_name("div")
if tab_info:
scroll_max = sum([int(x) for x in tab_info[0].text.split() if x.isdigit()])
scroll_max = sum([int(x) for x in tab_info.text.split() if x.isdigit()])
else:
return False
@@ -112,7 +112,7 @@ def scrape(gaiaID, client, cookies, config, headers, regex_rev_by_id, is_headles
for nb, review in enumerate(reviews_elements):
id = review.get_attribute("data-review-id")
location = re.compile(regex_rev_by_id.format(id)).findall(data)[0]
date = get_datetime(review.find_element_by_css_selector('span.section-review-publish-date').text)
date = get_datetime(review.find_element_by_tag_name('div').text.split("\n")[2])
reviews.append({"location": location, "date": date})
tmprinter.out(f"Fetching reviews location... ({nb + 1}/{len(reviews_elements)})")