当前位置：首页 > news >正文

Python爬虫技术：招标信息抓取与关键词过滤 (1)

news 2025/9/29 8:15:15

一、技术架构概述

招标信息抓取系统主要包含三个核心模块：

网络请求模块：负责向目标网站发送HTTP请求并获取网页内容
数据解析模块：从网页HTML中提取结构化招标信息
关键词过滤模块：根据预设关键词对招标信息进行筛选

二、实现步骤详解

1. 环境准备

首先安装必要的Python库：

requests：用于发送HTTP请求
beautifulsoup4：用于解析HTML文档
pandas：用于数据处理和存储

2. 网页内容获取

使用requests库向目标招标网站发送GET请求，获取网页HTML内容：

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import reclass BidInfoSpider:def __init__(self):self.session = requests.Session()# 设置请求头，模拟浏览器行为self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}self.session.headers.update(self.headers)def get_page_content(self, url, params=None):"""获取网页内容:param url: 目标URL:param params: 请求参数:return: 网页HTML内容"""try:response = self.session.get(url, params=params, timeout=10)response.encoding = 'utf-8'if response.status_code == 200:return response.textelse:print(f"请求失败，状态码：{response.status_code}")return Noneexcept Exception as e:print(f"请求异常：{str(e)}")return None

3. 数据解析与提取

使用BeautifulSoup解析HTML，提取招标信息的关键字段：

def parse_bid_info(self, html):"""解析招标信息:param html: 网页HTML内容:return: 招标信息列表"""if not html:return []soup = BeautifulSoup(html, 'lxml')bid_list = []# 根据实际网站结构调整选择器# 这里以常见的招标信息列表结构为例items = soup.select('.bid-item, .tender-item, .zbxx-item')for item in items:try:bid_info = {}# 提取标题title_elem = item.select_one('.title, .tit, a')bid_info['title'] = title_elem.get_text().strip() if title_elem else ''# 提取链接if title_elem and title_elem.get('href'):link = title_elem['href']if not link.startswith('http'):# 处理相对链接base_url = 'http://www.example.com'  # 替换为实际基础URLlink = base_url + linkbid_info['link'] = linkelse:bid_info['link'] = ''# 提取发布时间time_elem = item.select_one('.time, .date, .publish-time')bid_info['publish_time'] = time_elem.get_text().strip() if time_elem else ''# 提取招标方owner_elem = item.select_one('.owner, .tenderer, .zbunit')bid_info['owner'] = owner_elem.get_text().strip() if owner_elem else ''# 提取项目预算（如果有）budget_elem = item.select_one('.budget, .amount, .money')bid_info['budget'] = budget_elem.get_text().strip() if budget_elem else ''if bid_info['title']:  # 只有包含标题的信息才保存bid_list.append(bid_info)except Exception as e:print(f"解析招标信息异常：{str(e)}")continuereturn bid_list

4. 关键词过滤系统

实现基于关键词的智能过滤机制：

class KeywordFilter:def __init__(self):# 定义关键词分类self.keyword_categories = {'industry': ['IT', '软件', '系统集成', '信息化', '智能化', '网络', '数据中心'],'technology': ['Python', 'Java', '云计算', '大数据', '人工智能', '物联网', '区块链'],'project_type': ['开发', '实施', '运维', '咨询', '设计', '培训'],'exclude': ['废标', '终止', '流标', '更正']  # 排除关键词}# 构建正则表达式模式self.patterns = {}for category, keywords in self.keyword_categories.items():# 将关键词用|连接，创建正则表达式pattern = '|'.join(keywords)self.patterns[category] = re.compile(pattern)def filter_by_keywords(self, bid_info, min_matches=2):"""根据关键词过滤招标信息:param bid_info: 招标信息字典:param min_matches: 最小匹配关键词数量:return: 是否通过过滤"""title = bid_info.get('title', '')content = bid_info.get('content', '')  # 如果有详细内容# 排除包含排除关键词的信息exclude_pattern = self.patterns['exclude']if exclude_pattern.search(title) or exclude_pattern.search(content):return False# 计算匹配的关键词数量match_count = 0for category in ['industry', 'technology', 'project_type']:pattern = self.patterns[category]if pattern.search(title) or pattern.search(content):match_count += 1return match_count >= min_matchesdef get_matched_keywords(self, text):"""获取文本中匹配的关键词:param text: 待匹配文本:return: 匹配的关键词列表"""matched_keywords = []for category in ['industry', 'technology', 'project_type']:pattern = self.patterns[category]matches = pattern.findall(text)matched_keywords.extend(matches)return list(set(matched_keywords))  # 去重

5. 完整爬虫实现

整合各模块，实现完整的招标信息抓取流程：

class BidInformationCrawler:def __init__(self):self.spider = BidInfoSpider()self.filter = KeywordFilter()self.all_bid_info = []def crawl_multiple_pages(self, base_url, pages=5):"""爬取多页招标信息:param base_url: 基础URL:param pages: 爬取页数"""print("开始爬取招标信息...")for page in range(1, pages + 1):print(f"正在爬取第 {page} 页...")# 构造每页的URL参数（根据实际网站调整）params = {'page': page,'size': 20  # 每页显示数量}html = self.spider.get_page_content(base_url, params)if html:bid_list = self.spider.parse_bid_info(html)filtered_bids = self.filter_bid_info(bid_list)self.all_bid_info.extend(filtered_bids)# 添加延迟，避免请求过于频繁time.sleep(2)print(f"爬取完成，共获取 {len(self.all_bid_info)} 条符合条件的招标信息")def filter_bid_info(self, bid_list):"""过滤招标信息:param bid_list: 原始招标信息列表:return: 过滤后的招标信息列表"""filtered_bids = []for bid_info in bid_list:# 获取详细内容（如果需要）if bid_info.get('link'):detail_html = self.spider.get_page_content(bid_info['link'])if detail_html:# 解析详细内容（根据实际网站结构实现）bid_info['content'] = self.extract_detail_content(detail_html)# 关键词过滤if self.filter.filter_by_keywords(bid_info):# 添加匹配的关键词title = bid_info.get('title', '')content = bid_info.get('content', '')all_text = title + ' ' + contentbid_info['matched_keywords'] = self.filter.get_matched_keywords(all_text)filtered_bids.append(bid_info)return filtered_bidsdef extract_detail_content(self, html):"""提取招标详细信息内容:param html: 详情页HTML:return: 纯文本内容"""soup = BeautifulSoup(html, 'lxml')# 根据实际网站结构调整选择器content_elem = soup.select_one('.content, .detail, .article-content')if content_elem:# 移除脚本和样式标签for script in content_elem(["script", "style"]):script.decompose()return content_elem.get_text().strip()return ''def save_to_excel(self, filename='bid_information.xlsx'):"""保存招标信息到Excel文件:param filename: 文件名"""if not self.all_bid_info:print("没有数据可保存")returndf = pd.DataFrame(self.all_bid_info)# 选择要保存的列columns_to_save = ['title', 'owner', 'publish_time', 'budget', 'link', 'matched_keywords']existing_columns = [col for col in columns_to_save if col in df.columns]df = df[existing_columns]df.to_excel(filename, index=False, engine='openpyxl')print(f"数据已保存到 {filename}")def display_results(self):"""显示筛选结果"""print("\n=== 符合条件的招标信息 ===")for i, bid_info in enumerate(self.all_bid_info, 1):print(f"\n{i}. 标题: {bid_info.get('title', '')}")print(f"   招标方: {bid_info.get('owner', '')}")print(f"   发布时间: {bid_info.get('publish_time', '')}")print(f"   预算: {bid_info.get('budget', '')}")print(f"   匹配关键词: {', '.join(bid_info.get('matched_keywords', []))}")print(f"   链接: {bid_info.get('link', '')}")# 使用示例
if __name__ == "__main__":# 替换为实际的招标网站URLbase_url = "http://www.example-bid-website.com/bid-list"crawler = BidInformationCrawler()crawler.crawl_multiple_pages(base_url, pages=3)crawler.display_results()crawler.save_to_excel()