当前位置：首页 > news >正文

爬虫获取数据：selenium的应用

news 来源：原创 2025/6/22 6:05:09

当我们在爬取数据时，如：古诗网。有时会出现输出内容不全的情况，针对这种问题如何解决？

个人思路：在遍历网页内容时，如果未发现显示全部内容字样，说明该条数据内容完整，则立即输出。若识别到显示全部内容，则表示内容不全。需要click点击事件，跳转成功后获取该页面中我们主要的数据，例如古诗名、作者、古诗内容等。获取全部数据后返回初始页面，继续寻找显示全部内容。如此循环，直到遍历完成。

此处默认已掌握全部相关知识点，只提供代码。欢迎大家探讨。

import time
import random
import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.edge.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
options = webdriver.EdgeOptions()
options.add_experimental_option('excludeSwitches', ['enable-automation'])
options.add_experimental_option('useAutomationExtension', False)
options = webdriver.EdgeOptions()
options.add_experimental_option('excludeSwitches', ['enable-automation'])
options.add_experimental_option('useAutomationExtension', False)# 加载当前浏览器驱动.exe.sh
edgeDriver = Service("../drivera/msedgedriver.exe")# 实例化当前的浏览器对象
browser = webdriver.Edge(service=edgeDriver, options=options)# 设置浏览器超时
wait = WebDriverWait(browser, 10)browser.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {'source': 'Object.defineProperty(navigator, "webdriver", {get: () => undefined})'
})
# 数据容器
poem_data = []
count = 1url = "https://www.shicimingju.com/chaxun/zuozhe/46.html"
browser.get(url)
time.sleep(2)# 获取所有诗词链接
more_links = browser.find_elements(By.CLASS_NAME, 'more')
total_links = len(more_links)
print(f"第一页找到 {total_links} 个诗词链接")# 循环处理每个链接
for i, link in enumerate(more_links):try:print(f"\n--- 处理第 {count}/{total_links} 个链接 ---")# 滚动并点击链接browser.execute_script("arguments[0].scrollIntoView();", link)time.sleep(1)link.click()time.sleep(2)title = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "h1"))).text.strip()author_elem = browser.find_elements(By.CSS_SELECTOR, "div.source, .name")author = "未知作者"if author_elem:author = author_elem[0].text.strip()author = author.split("·")[-1] if "·" in author else author# 提取内容content_elem = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.CLASS_NAME, "text")))soup = BeautifulSoup(content_elem.get_attribute('innerHTML'), "html.parser")content = soup.get_text().strip()# 清理内容exclude = ["注释", "赏析", "作品赏析"]content = "\n".join([line for line in content.split("\n") if not any(x in line for x in exclude)])# 保存数据poem_data.append({"序号": count,"标题": title,"作者": author,"内容": content})print("=" * 50)print(f"【{title}】 - {author} \n{content}")count += 1# 返回上一页browser.back()time.sleep(1)finally:pass
#
# # 保存到Excel
# if poem_data:
#     pd.DataFrame(poem_data).to_excel("陆游诗词_简化版.xlsx", index=False)
#     print(f"\n成功保存 {len(poem_data)} 条数据")
#
# browser.quit()