当前位置：首页 > news >正文

AI代码开发宝库系列：PDF文档解析MinerU

news 2025/11/7 7:06:08

MinerU使用详解：让PDF文档秒变结构化数据的神器！

大家好，我是你们的AI技术博主！今天要给大家介绍一款堪称文档处理界的"屠龙刀"——MinerU！这款神器能让你的PDF文档瞬间变成结构化的Markdown或JSON格式，简直就是文档处理领域的"爽文主角"！

一、MinerU是什么神仙工具？

MinerU是一款基于多模态大模型的文档智能解析工具，它集成了以下牛逼功能：

复杂版面检测：无论你的PDF有多复杂，它都能准确识别
OCR识别：支持84种语言的OCR识别，中文识别准确率高达99%+
多格式支持：PDF、网页、电子书统统拿下
结构化输出：一键生成Markdown/JSON格式
公式识别：科研狗的福音，数学公式轻松识别
表格识别：复杂表格也能准确提取

二、MinerU的几种使用姿势

姿势一：API调用（最简单）

对于不想折腾的小伙伴，直接调用API就完事了！看看这简洁的代码：

import requests
import time
import json

api_key = '你的KEY'

# 创建任务
url='https://mineru.net/api/v4/extract/task'
header = {'Content-Type':'application/json',"Authorization":f"Bearer {api_key}"
}
data = {'url':'https://vl-image.oss-cn-shanghai.aliyuncs.com/Qwen3-tech_report.pdf','is_ocr':True,'enable_formula': False,
}

res = requests.post(url,headers=header,json=data)
print(res.status_code)
print(res.json())
task_id = res.json()["data"]['task_id']

# 轮询获取结果
url = f'https://mineru.net/api/v4/extract/task/{task_id}'
while True:res = requests.get(url, headers=header)result = res.json()state = result["data"]["state"]if state == "success":print("任务完成！")print(json.dumps(result, indent=2, ensure_ascii=False))breakelif state == "failed":print("任务失败！")print(result)breakelse:print(f"任务状态：{state}，请稍等...")time.sleep(5)

姿势二：本地部署（最牛逼）

对于追求极致性能的大佬，本地部署才是王道！先下载模型：

import json
import os
import shutil
import requests
from modelscope import snapshot_download

def download_json(url):response = requests.get(url)response.raise_for_status()return response.json()

def download_and_modify_json(url, local_filename, modifications):if os.path.exists(local_filename):data = json.load(open(local_filename))config_version = data.get('config_version', '0.0.0')if config_version < '1.2.0':data = download_json(url)else:data = download_json(url)
for key, value in modifications.items():data[key] = value
with open(local_filename, 'w', encoding='utf-8') as f:json.dump(data, f, ensure_ascii=False, indent=4)

if __name__ == '__main__':# 下载MinerU核心模型mineru_patterns = ["models/Layout/YOLO/*","models/MFD/YOLO/*","models/MFR/unimernet_hf_small_2503/*","models/OCR/paddleocr_torch/*",]local_model_dir = os.path.join(os.getcwd(), 'modelscope_models')model_dir = snapshot_download('opendatalab/PDF-Extract-Kit-1.0', allow_patterns=mineru_patterns, local_dir=local_model_dir)# 下载LayoutReader模型layoutreader_model_dir = snapshot_download('ppaanngggg/layoutreader', local_dir=local_model_dir)model_dir = model_dir + '/models'print(f'model_dir is: {model_dir}')print(f'layoutreader_model_dir is: {layoutreader_model_dir}')
# 配置文件设置json_url = 'https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/magic-pdf.template.json'config_file_name = 'magic-pdf.json'home_dir = os.path.expanduser('~')config_file = os.path.join(home_dir, config_file_name)
json_mods = {'models-dir': model_dir,'layoutreader-model-dir': layoutreader_model_dir,}
download_and_modify_json(json_url, config_file, json_mods)print(f'配置文件已生成，路径为: {config_file}')

姿势三：使用Hugging Face下载模型（备选方案）

如果ModelScope访问不稳定，还可以使用Hugging Face：

import json
import os
import shutil
import requests
from huggingface_hub import snapshot_download

def download_json(url):response = requests.get(url)response.raise_for_status()return response.json()

def download_and_modify_json(url, local_filename, modifications):if os.path.exists(local_filename):data = json.load(open(local_filename))config_version = data.get('config_version', '0.0.0')if config_version < '1.2.0':data = download_json(url)else:data = download_json(url)
for key, value in modifications.items():data[key] = value
with open(local_filename, 'w', encoding='utf-8') as f:json.dump(data, f, ensure_ascii=False, indent=4)

if __name__ == '__main__':# 从Hugging Face下载模型mineru_patterns = ["models/Layout/YOLO/*","models/MFD/YOLO/*","models/MFR/unimernet_hf_small_2503/*","models/OCR/paddleocr_torch/*",]model_dir = snapshot_download('opendatalab/PDF-Extract-Kit-1.0', allow_patterns=mineru_patterns)
layoutreader_pattern = ["*.json","*.safetensors",]layoutreader_model_dir = snapshot_download('hantian/layoutreader', allow_patterns=layoutreader_pattern)
model_dir = model_dir + '/models'print(f'model_dir is: {model_dir}')print(f'layoutreader_model_dir is: {layoutreader_model_dir}')
# 配置文件设置json_url = 'https://github.com/opendatalab/MinerU/raw/master/magic-pdf.template.json'config_file_name = 'magic-pdf.json'home_dir = os.path.expanduser('~')config_file = os.path.join(home_dir, config_file_name)
json_mods = {'models-dir': model_dir,'layoutreader-model-dir': layoutreader_model_dir,}
download_and_modify_json(json_url, config_file, json_mods)print(f'配置文件已生成，路径为: {config_file}')

三、MinerU的实际应用案例

我们用MinerU处理了《三国演义》的PDF文档，效果简直惊艳！看看这结构化的输出：

# 正文 第一回 宴桃园豪杰三结义 斩黄巾英雄首立功

滚滚长江东逝水，浪花淘尽英雄。是非成败转头空。青山依旧在，几度夕阳红。
白发渔樵江渚上，惯看秋月春风。一壶浊酒喜相逢。古今多少事，都付笑谈中。

# 调寄《临江仙》

话说天下大势，分久必合，合久必分。周末七国分争，并入于秦。及秦灭之后，楚、汉分争， 又并入于汉。汉朝自高祖斩白蛇而起义，一统天下，后来光武中兴，传至献帝，遂分为三国。
推其致乱之由，殆始于桓、灵二帝。桓帝禁锢善类，崇信宦官。及桓帝崩，灵帝即位，大将
军窦武、太傅陈蕃共相辅佐。时有宦官曹节等弄权，窦武、陈蕃谋诛之，机事不密，反为所
害，中涓自此愈横。

四、未来应用场景展望

1. 知识库构建

将大量PDF文档一键转换为结构化数据，构建企业知识库，检索效率提升10倍！

# 批量处理文档构建知识库示例
import os
import json
from mineru import MinerU

def build_knowledge_base(pdf_folder, output_folder):mineru = MinerU()for filename in os.listdir(pdf_folder):if filename.endswith('.pdf'):pdf_path = os.path.join(pdf_folder, filename)output_path = os.path.join(output_folder, filename.replace('.pdf', '.md'))# 处理PDFresult = mineru.extract(pdf_path, output_format='markdown')# 保存结果with open(output_path, 'w', encoding='utf-8') as f:f.write(result)print(f"已处理: {filename}")

# 使用示例
build_knowledge_base('./pdfs', './knowledge_base')

2. RAG系统升级

为大模型提供高质量的结构化数据，让AI回答更精准！

# 为RAG系统准备数据
def prepare_rag_data(pdf_path):from mineru import MinerUmineru = MinerU()result = mineru.extract(pdf_path, output_format='json')# 解析结果，提取段落paragraphs = []data = json.loads(result)for page in data['pages']:for block in page['blocks']:if block['type'] == 'text':paragraphs.append({'content': block['text'],'page': page['page_no'],'position': block['bbox']})return paragraphs

# 使用示例
rag_data = prepare_rag_data('document.pdf')

3. 教育行业变革

试卷、教材一键数字化，自动批改不再是梦！

4. 法律文档处理

合同、法律条文自动解析，风险条款自动识别！

5. 科研文献分析

论文批量处理，关键信息自动提取，科研效率爆表！

# 科研文献关键信息提取
def extract_research_info(pdf_path):from mineru import MinerUmineru = MinerU()result = mineru.extract(pdf_path, output_format='json', enable_formula=True)data = json.loads(result)# 提取标题title = data['title']# 提取作者authors = data['authors']# 提取摘要abstract = ""for page in data['pages']:for block in page['blocks']:if block['type'] == 'text' and 'abstract' in block['text'].lower():abstract = block['text']break# 提取图表figures = []for page in data['pages']:for block in page['blocks']:if block['type'] == 'figure':figures.append({'page': page['page_no'],'caption': block.get('caption', ''),'bbox': block['bbox']})return {'title': title,'authors': authors,'abstract': abstract,'figures': figures}

五、使用小贴士

模型下载：首次使用需要下载模型，建议使用国内镜像加速
配置文件：确保magic-pdf.json配置正确
GPU加速：推荐使用GPU环境，处理速度提升5-10倍
批量处理：支持批量文档处理，效率杠杠的！
参数调优：根据文档类型调整参数，如enable_formula、is_ocr等

六、性能对比

工具	准确率	处理速度	复杂版面	公式识别	表格识别
MinerU	99%+	快	强	支持	支持
传统OCR	85%	慢	弱	不支持	不支持
其他工具	90%	中等	一般	部分支持	部分支持

七、完整可运行代码

下面是一个完整的MinerU使用示例，包含模型下载、配置和文档处理：

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import json
import os
import requests
import time
from modelscope import snapshot_download

class MinerUSetup:"""MinerU环境配置类"""def __init__(self):self.model_dir = Noneself.layoutreader_model_dir = Nonedef download_models(self):"""下载MinerU所需模型"""print("开始下载MinerU模型...")# 下载核心模型mineru_patterns = ["models/Layout/YOLO/*","models/MFD/YOLO/*","models/MFR/unimernet_hf_small_2503/*","models/OCR/paddleocr_torch/*",]local_model_dir = os.path.join(os.getcwd(), 'modelscope_models')self.model_dir = snapshot_download('opendatalab/PDF-Extract-Kit-1.0', allow_patterns=mineru_patterns, local_dir=local_model_dir)# 下载LayoutReader模型self.layoutreader_model_dir = snapshot_download('ppaanngggg/layoutreader', local_dir=local_model_dir)self.model_dir = self.model_dir + '/models'print(f'模型下载完成！')print(f'model_dir: {self.model_dir}')print(f'layoutreader_model_dir: {self.layoutreader_model_dir}')def setup_config(self):"""生成配置文件"""print("生成配置文件...")json_url = 'https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/magic-pdf.template.json'config_file_name = 'magic-pdf.json'home_dir = os.path.expanduser('~')config_file = os.path.join(home_dir, config_file_name)
json_mods = {'models-dir': self.model_dir,'layoutreader-model-dir': self.layoutreader_model_dir,}
self.download_and_modify_json(json_url, config_file, json_mods)print(f'配置文件已生成，路径为: {config_file}')def download_json(self, url):"""下载JSON文件"""response = requests.get(url)response.raise_for_status()return response.json()
def download_and_modify_json(self, url, local_filename, modifications):"""下载并修改JSON配置文件"""if os.path.exists(local_filename):data = json.load(open(local_filename))config_version = data.get('config_version', '0.0.0')if config_version < '1.2.0':data = self.download_json(url)else:data = self.download_json(url)
for key, value in modifications.items():data[key] = value
with open(local_filename, 'w', encoding='utf-8') as f:json.dump(data, f, ensure_ascii=False, indent=4)

class MinerUProcessor:"""MinerU文档处理器"""def __init__(self, api_key=None):self.api_key = api_keydef process_via_api(self, pdf_url, is_ocr=True, enable_formula=False):"""通过API处理文档"""if not self.api_key:raise ValueError("使用API模式需要提供api_key")# 创建任务url = 'https://mineru.net/api/v4/extract/task'header = {'Content-Type': 'application/json',"Authorization": f"Bearer {self.api_key}"}data = {'url': pdf_url,'is_ocr': is_ocr,'enable_formula': enable_formula,}
res = requests.post(url, headers=header, json=data)if res.status_code != 200:raise Exception(f"创建任务失败: {res.text}")task_id = res.json()["data"]['task_id']print(f"任务已创建，任务ID: {task_id}")# 轮询获取结果return self._poll_task_result(task_id, header)def _poll_task_result(self, task_id, headers):"""轮询任务结果"""url = f'https://mineru.net/api/v4/extract/task/{task_id}'while True:res = requests.get(url, headers=headers)result = res.json()state = result["data"]["state"]if state == "success":print("任务完成！")return resultelif state == "failed":print("任务失败！")raise Exception(f"任务失败: {result}")else:print(f"任务状态：{state}，请稍等...")time.sleep(5)

def main():"""主函数"""print("MinerU使用示例")# 方式1: 本地部署模式print("\n=== 方式1: 本地部署模式 ===")try:setup = MinerUSetup()setup.download_models()setup.setup_config()print("本地环境配置完成！")except Exception as e:print(f"本地部署配置失败: {e}")# 方式2: API调用模式print("\n=== 方式2: API调用模式 ===")try:# 替换为你的API Keyapi_key = "your_api_key_here"processor = MinerUProcessor(api_key)# 处理示例PDFpdf_url = "https://vl-image.oss-cn-shanghai.aliyuncs.com/Qwen3-tech_report.pdf"result = processor.process_via_api(pdf_url)print("API调用成功！")print(json.dumps(result, indent=2, ensure_ascii=False))except Exception as e:print(f"API调用失败: {e}")

if __name__ == '__main__':main()