lywsvip/EasySpider
1
0
Fork 0
mirror of https://github.com/NaiboWang/EasySpider.git synced 2026-05-06 22:23:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Scroll Exception Handle

Browse Source
This commit is contained in:
naibo
2023-12-22 14:35:22 +08:00
parent 498c3bda29
commit c5da52787d
1 changed files with 32 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
ExecuteStage/easyspider_executestage.py
View File

@@ -651,8 +651,8 @@ class BrowserThread(Thread):
time.sleep(param["scrollWaitTime"]) # 下拉完等待
except:
pass
except:
self.print_and_log('Time out after set seconds when scrolling. ')
except Exception as e:
self.print_and_log("滚动屏幕时出错|Error scrolling screen:", e)
try:
self.browser.execute_script('window.stop()')
except:
@@ -667,6 +667,36 @@ class BrowserThread(Thread):
body.send_keys(Keys.PGDN)
elif scrollType == 2:
body.send_keys(Keys.END)
elif scrollType == 3:
bodyText = ""
i = 0
while True:
newBodyText = self.browser.find_element(
By.CSS_SELECTOR, "body", iframe=False).text
if param["iframe"]: # 如果标记了iframe
iframes = self.browser.find_elements(
By.CSS_SELECTOR, "iframe", iframe=False)
for iframe in iframes:
self.browser.switch_to.default_content()
self.browser.switch_to.frame(iframe)
iframe_text = super(self.browser.__class__, self.browser).find_element(
By.CSS_SELECTOR, "body").text # 用super调用父类的方法
newBodyText += iframe_text
self.browser.switch_to.default_content()
if newBodyText == bodyText:
self.print_and_log("页面已检测不到新内容，停止滚动。")
self.print_and_log(
"No new content detected on the page, stop scrolling.")
break
else:
bodyText = newBodyText
body = self.browser.find_element(
By.CSS_SELECTOR, "body", iframe=param["iframe"])
body.send_keys(Keys.END)
self.print_and_log("滚动到底部，第", i + 1, "次。")
self.print_and_log(
"Scroll to the bottom, the", i + 1, "time.")
i = i + 1
try:
time.sleep(param["scrollWaitTime"]) # 下拉完等待
except: