爬虫定位找到的信息不全,有隐藏元素,使用 JavaScript 来直接提取隐藏的文本内容
·
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# 初始化 WebDriver
driver = webdriver.Chrome()
driver.get("你的目标网页链接")
# 定义等待对象
wait = WebDriverWait(driver, 20)
# 提取表头数据
def extract_header_data():
headers = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.el-table__header-wrapper thead')))
header_cols = headers.find_elements(By.TAG_NAME, 'th')
columns = [col.text.strip() for col in header_cols if col.text.strip()]
return columns
# 提取表体数据(包括隐藏元素的内容)
def extract_table_data():
body = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.el-table__body-wrapper tbody')))
rows = body.find_elements(By.TAG_NAME, 'tr')
data = [] # 存储表格数据
for row in rows:
row_data = []
cols = row.find_elements(By.TAG_NAME, 'td')
for col in cols:
# 检查是否为隐藏内容
try:
# 尝试提取隐藏内容
hidden_text = driver.execute_script("return arguments[0].textContent", col.find_element(By.XPATH, './/div'))
row_data.append(hidden_text.strip())
except Exception:
# 如果无法提取隐藏内容,则使用可见内容
row_data.append(col.text.strip())
data.append(row_data)
return data
# 提取并打印表头和表体数据
try:
headers = extract_header_data()
print("表头:", headers)
table_data = extract_table_data()
for row in table_data:
print("表体行:", row)
finally:
# 关闭浏览器
driver.quit()
更多推荐
所有评论(0)