document_processing.py 57 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495
  1. from __future__ import annotations
  2. import asyncio
  3. import html
  4. import json
  5. import math
  6. import re
  7. import zipfile
  8. from io import BytesIO
  9. from datetime import datetime, timezone
  10. from pathlib import Path
  11. from typing import Any
  12. import httpx
  13. from sqlalchemy.orm import Session
  14. from app.agents.runtime import get_agent_runtime
  15. from app.core.config import settings
  16. from app.core.errors import APIError
  17. from app.db.models.document import (
  18. BusinessDocument,
  19. DocumentExtraction,
  20. DocumentExtractionTemplate,
  21. DocumentProcessingRun,
  22. )
  23. from app.db.repositories import document as document_repo
  24. from app.db.session import SessionLocal
  25. from app.services.document_business_templates import (
  26. detect_file_format,
  27. get_template_by_id,
  28. resolve_document_template,
  29. template_field_definitions,
  30. )
  31. PROCESSING_ACTIVE_STATUSES = {"pending", "running", "parsing", "extracting"}
  32. PARSE_MODES = {"lightweight", "precision"}
  33. def _now() -> datetime:
  34. return datetime.now(timezone.utc)
  35. def normalize_parse_mode(value: str | None) -> str:
  36. mode = str(value or "lightweight").strip().lower()
  37. if mode not in PARSE_MODES:
  38. raise APIError(
  39. 422,
  40. "invalid_parse_mode",
  41. "解析模式仅支持 lightweight 或 precision",
  42. )
  43. return mode
  44. def resolve_effective_parse_mode(
  45. parse_mode: str | None,
  46. file_ext: str | None,
  47. file_path: str | None = None,
  48. ) -> str:
  49. mode = normalize_parse_mode(parse_mode)
  50. extension = str(file_ext or "").strip().lower()
  51. if extension and not extension.startswith("."):
  52. extension = f".{extension}"
  53. if mode == "lightweight" and extension in {".doc", ".xls"}:
  54. return "precision"
  55. if mode == "lightweight" and file_path:
  56. source = Path(file_path)
  57. if source.is_file():
  58. if source.stat().st_size > 10 * 1024 * 1024:
  59. return "precision"
  60. metrics = inspect_source_metrics(source)
  61. page_count = metrics.get("page_count")
  62. if page_count and int(page_count) > 20:
  63. return "precision"
  64. return mode
  65. def start_document_processing(
  66. db: Session,
  67. document_id: str,
  68. *,
  69. force: bool = False,
  70. parse_mode: str = "lightweight",
  71. template_id: str | None = None,
  72. ) -> DocumentProcessingRun:
  73. document = document_repo.get_document(db, document_id)
  74. if document is None:
  75. raise APIError(404, "document_not_found", "业务资料不存在")
  76. latest = document_repo.get_latest_processing_run(db, document_id)
  77. if latest is not None and latest.status in PROCESSING_ACTIVE_STATUSES and not force:
  78. return latest
  79. file_format = detect_file_format(document.file_ext or document.file_name)
  80. requested_parse_mode = normalize_parse_mode(parse_mode)
  81. parse_mode = resolve_effective_parse_mode(
  82. requested_parse_mode,
  83. document.file_ext or document.file_name,
  84. document.file_path,
  85. )
  86. template = resolve_document_template(
  87. db,
  88. title=document.title,
  89. template_id=template_id,
  90. )
  91. document.document_type = template.code
  92. document.parse_status = "pending"
  93. document.parse_progress = 0
  94. document.parse_error = ""
  95. db.add(document)
  96. db.commit()
  97. return document_repo.create_processing_run(
  98. db,
  99. {
  100. "document_id": document.id,
  101. "template_id": template.id,
  102. "status": "pending",
  103. "stage": "uploaded",
  104. "progress": 0,
  105. "document_type": template.code,
  106. "parser_engine": f"mineru_api_{parse_mode}",
  107. "metadata_json": {
  108. "template_code": template.code,
  109. "template_version": template.version,
  110. "parse_mode": parse_mode,
  111. "requested_parse_mode": requested_parse_mode,
  112. "parse_mode_auto_upgraded": parse_mode != requested_parse_mode,
  113. "file_format": file_format,
  114. },
  115. },
  116. )
  117. async def process_document_run(run_id: str) -> None:
  118. with SessionLocal() as db:
  119. await process_document(db, run_id)
  120. async def process_document(db: Session, run_id: str) -> None:
  121. run = document_repo.get_processing_run(db, run_id)
  122. if run is None:
  123. return
  124. document = document_repo.get_document(db, run.document_id)
  125. if document is None:
  126. run.status = "failed"
  127. run.stage = "failed"
  128. run.error = "业务资料不存在"
  129. run.finished_at = _now()
  130. db.add(run)
  131. db.commit()
  132. return
  133. if not run.template_id:
  134. raise APIError(422, "extraction_template_required", "处理任务缺少抽取模板")
  135. template = get_template_by_id(db, run.template_id)
  136. run.status = "running"
  137. run.stage = "parsing"
  138. run.progress = 10
  139. run.started_at = _now()
  140. run.error = ""
  141. document.parse_status = "parsing"
  142. document.parse_progress = 10
  143. document.parse_error = ""
  144. db.add_all([run, document])
  145. db.commit()
  146. try:
  147. parse_mode = normalize_parse_mode(
  148. (run.metadata_json or {}).get("parse_mode", "lightweight")
  149. )
  150. run.parser_engine = f"mineru_api_{parse_mode}"
  151. run.metadata_json = {
  152. **(run.metadata_json or {}),
  153. "parse_mode": parse_mode,
  154. }
  155. parsed = await parse_document_with_mineru_api(
  156. document.file_path,
  157. document.id,
  158. document.file_name,
  159. parse_mode,
  160. )
  161. run.parsed_markdown_path = str(parsed["markdown_path"])
  162. run.parsed_markdown = str(parsed["markdown"])
  163. run.stage = "extracting"
  164. run.progress = 65
  165. document.parse_status = "extracting"
  166. document.parse_progress = 65
  167. db.add_all([run, document])
  168. db.commit()
  169. extracted = await extract_document_fields(
  170. document=document,
  171. template=template,
  172. parsed_markdown=str(parsed["markdown"]),
  173. )
  174. replace_document_extractions(
  175. db,
  176. document_id=document.id,
  177. fields=extracted["fields"],
  178. model_status=extracted["model_status"],
  179. )
  180. document.summary = extracted["summary"]
  181. document.page_count = parsed.get("page_count") or document.page_count
  182. document.row_count = parsed.get("row_count") or document.row_count
  183. document.parse_status = "completed"
  184. document.parse_progress = 100
  185. document.parse_error = ""
  186. document.metadata_json = {
  187. **(document.metadata_json or {}),
  188. "extraction_template": template.code,
  189. "processing_run_id": run.id,
  190. "parser_engine": run.parser_engine,
  191. "model_status": extracted["model_status"],
  192. }
  193. run.status = "completed"
  194. run.stage = "completed"
  195. run.progress = 100
  196. run.model_output = {
  197. "summary": extracted["summary"],
  198. "model_status": extracted["model_status"],
  199. "field_count": len(extracted["fields"]),
  200. }
  201. run.finished_at = _now()
  202. db.add_all([run, document])
  203. db.commit()
  204. except Exception as exc: # noqa: BLE001 - 外部解析和模型错误必须落库
  205. db.rollback()
  206. run = document_repo.get_processing_run(db, run_id)
  207. document = document_repo.get_document(db, run.document_id) if run else None
  208. message = f"{type(exc).__name__}: {exc}"
  209. if run is not None:
  210. run.status = "failed"
  211. run.stage = "failed"
  212. run.progress = 100
  213. run.error = message
  214. run.finished_at = _now()
  215. db.add(run)
  216. if document is not None:
  217. document.parse_status = "failed"
  218. document.parse_progress = 100
  219. document.parse_error = message
  220. db.add(document)
  221. db.commit()
  222. async def parse_document_with_mineru_api(
  223. file_path: str,
  224. document_id: str,
  225. file_name: str,
  226. parse_mode: str,
  227. ) -> dict[str, Any]:
  228. source = Path(file_path)
  229. if not source.is_file():
  230. raise APIError(404, "document_file_not_found", "资料文件不存在")
  231. parse_mode = normalize_parse_mode(parse_mode)
  232. headers = _mineru_headers()
  233. timeout = httpx.Timeout(
  234. connect=20.0,
  235. read=120.0,
  236. write=180.0,
  237. pool=30.0,
  238. )
  239. async with httpx.AsyncClient(
  240. base_url=settings.mineru_api_base.rstrip("/"),
  241. timeout=timeout,
  242. follow_redirects=True,
  243. ) as client:
  244. if parse_mode == "precision":
  245. return await _parse_with_precision_api(
  246. client,
  247. headers,
  248. source,
  249. document_id,
  250. )
  251. return await _parse_with_lightweight_api(
  252. client,
  253. headers,
  254. source,
  255. document_id,
  256. file_name,
  257. )
  258. async def _parse_with_precision_api(
  259. client: httpx.AsyncClient,
  260. headers: dict[str, str],
  261. source: Path,
  262. document_id: str,
  263. ) -> dict[str, Any]:
  264. response = await client.post(
  265. "/api/v4/file-urls/batch",
  266. headers=headers,
  267. json={
  268. "model_version": "vlm",
  269. "language": "ch",
  270. "enable_formula": True,
  271. "enable_table": True,
  272. "files": [
  273. {
  274. "name": source.name,
  275. "data_id": document_id,
  276. "is_ocr": source.suffix.lower() in {".pdf", ".png", ".jpg", ".jpeg"},
  277. }
  278. ],
  279. },
  280. )
  281. data = _mineru_response_data(response, "申请精准解析上传地址失败")
  282. file_urls = data.get("file_urls") or []
  283. batch_id = str(data.get("batch_id") or "")
  284. if not file_urls or not batch_id:
  285. raise APIError(502, "mineru_invalid_response", "MinerU 未返回上传地址或任务 ID")
  286. await _upload_source_file(client, str(file_urls[0]), source)
  287. result = await _poll_precision_result(client, headers, batch_id)
  288. zip_url = str(result.get("full_zip_url") or "")
  289. if not zip_url:
  290. raise APIError(502, "mineru_empty_result", "MinerU 未返回精准解析结果包")
  291. content = await _download_mineru_asset(client, zip_url)
  292. markdown = _extract_markdown_from_zip(content)
  293. progress = result.get("extract_progress") or {}
  294. metrics = inspect_source_metrics(source)
  295. return _save_parsed_markdown(
  296. document_id=document_id,
  297. markdown=markdown,
  298. page_count=progress.get("total_pages") or metrics.get("page_count"),
  299. row_count=metrics.get("row_count"),
  300. )
  301. async def _parse_with_lightweight_api(
  302. client: httpx.AsyncClient,
  303. headers: dict[str, str],
  304. source: Path,
  305. document_id: str,
  306. file_name: str,
  307. ) -> dict[str, Any]:
  308. response = await client.post(
  309. "/api/v1/agent/parse/file",
  310. headers=headers,
  311. json={
  312. "file_name": file_name,
  313. "language": "ch",
  314. "enable_table": True,
  315. "is_ocr": source.suffix.lower() in {".pdf", ".png", ".jpg", ".jpeg"},
  316. "enable_formula": True,
  317. },
  318. )
  319. data = _mineru_response_data(response, "申请轻量解析上传地址失败")
  320. file_url = str(data.get("file_url") or "")
  321. task_id = str(data.get("task_id") or "")
  322. if not file_url or not task_id:
  323. raise APIError(502, "mineru_invalid_response", "MinerU 未返回上传地址或任务 ID")
  324. await _upload_source_file(client, file_url, source)
  325. result = await _poll_lightweight_result(client, headers, task_id)
  326. markdown_url = str(result.get("markdown_url") or "")
  327. if not markdown_url:
  328. raise APIError(502, "mineru_empty_result", "MinerU 未返回 Markdown 地址")
  329. markdown = (await _download_mineru_asset(client, markdown_url)).decode(
  330. "utf-8",
  331. errors="replace",
  332. )
  333. metrics = inspect_source_metrics(source)
  334. return _save_parsed_markdown(
  335. document_id=document_id,
  336. markdown=markdown,
  337. page_count=metrics.get("page_count"),
  338. row_count=metrics.get("row_count"),
  339. )
  340. async def _upload_source_file(
  341. client: httpx.AsyncClient,
  342. upload_url: str,
  343. source: Path,
  344. ) -> None:
  345. response = await client.put(
  346. upload_url,
  347. content=source.read_bytes(),
  348. )
  349. if response.is_error:
  350. raise APIError(502, "mineru_upload_failed", "文件上传到 MinerU 失败")
  351. async def _poll_precision_result(
  352. client: httpx.AsyncClient,
  353. headers: dict[str, str],
  354. batch_id: str,
  355. ) -> dict[str, Any]:
  356. deadline = asyncio.get_running_loop().time() + settings.mineru_timeout
  357. while asyncio.get_running_loop().time() < deadline:
  358. response = await client.get(
  359. f"/api/v4/extract-results/batch/{batch_id}",
  360. headers=headers,
  361. )
  362. data = _mineru_response_data(response, "查询精准解析任务失败")
  363. results = data.get("extract_result") or []
  364. if isinstance(results, dict):
  365. results = [results]
  366. result = results[0] if results else {}
  367. state = str(result.get("state") or "").lower()
  368. if state == "done":
  369. return result
  370. if state == "failed":
  371. raise APIError(
  372. 502,
  373. "mineru_failed",
  374. str(result.get("err_msg") or "MinerU 精准解析失败"),
  375. )
  376. await asyncio.sleep(settings.mineru_poll_interval)
  377. raise APIError(504, "mineru_timeout", "MinerU 精准解析超时")
  378. async def _poll_lightweight_result(
  379. client: httpx.AsyncClient,
  380. headers: dict[str, str],
  381. task_id: str,
  382. ) -> dict[str, Any]:
  383. deadline = asyncio.get_running_loop().time() + settings.mineru_timeout
  384. while asyncio.get_running_loop().time() < deadline:
  385. response = await client.get(
  386. f"/api/v1/agent/parse/{task_id}",
  387. headers=headers,
  388. )
  389. data = _mineru_response_data(response, "查询轻量解析任务失败")
  390. state = str(data.get("state") or "").lower()
  391. if state == "done":
  392. return data
  393. if state == "failed":
  394. raise APIError(
  395. 502,
  396. "mineru_failed",
  397. str(data.get("err_msg") or "MinerU 轻量解析失败"),
  398. )
  399. await asyncio.sleep(settings.mineru_poll_interval)
  400. raise APIError(504, "mineru_timeout", "MinerU 轻量解析超时")
  401. async def _download_mineru_asset(
  402. client: httpx.AsyncClient,
  403. url: str,
  404. ) -> bytes:
  405. response = await client.get(url)
  406. if response.is_error:
  407. raise APIError(502, "mineru_download_failed", "下载 MinerU 解析结果失败")
  408. return response.content
  409. def _mineru_headers() -> dict[str, str]:
  410. if not settings.mineru_token:
  411. raise APIError(503, "mineru_token_missing", "未配置 MinerU API Token")
  412. return {
  413. "Authorization": f"Bearer {settings.mineru_token}",
  414. "Content-Type": "application/json",
  415. "Accept": "application/json",
  416. }
  417. def _mineru_response_data(
  418. response: httpx.Response,
  419. default_message: str,
  420. ) -> dict[str, Any]:
  421. if response.is_error:
  422. raise APIError(502, "mineru_http_error", f"{default_message}:HTTP {response.status_code}")
  423. try:
  424. payload = response.json()
  425. except ValueError as exc:
  426. raise APIError(502, "mineru_invalid_response", f"{default_message}:响应不是 JSON") from exc
  427. if int(payload.get("code", -1)) != 0:
  428. raise APIError(
  429. 502,
  430. "mineru_api_error",
  431. str(payload.get("msg") or default_message),
  432. )
  433. data = payload.get("data")
  434. return data if isinstance(data, dict) else {}
  435. def _extract_markdown_from_zip(content: bytes) -> str:
  436. try:
  437. with zipfile.ZipFile(BytesIO(content)) as archive:
  438. markdown_names = [
  439. name for name in archive.namelist()
  440. if name.lower().endswith("full.md")
  441. ]
  442. if not markdown_names:
  443. markdown_names = [
  444. name for name in archive.namelist()
  445. if name.lower().endswith(".md")
  446. ]
  447. if not markdown_names:
  448. raise APIError(502, "mineru_empty_result", "解析结果包中没有 Markdown")
  449. return archive.read(markdown_names[0]).decode("utf-8", errors="replace")
  450. except zipfile.BadZipFile as exc:
  451. raise APIError(502, "mineru_invalid_result", "MinerU 返回的结果包无效") from exc
  452. def _save_parsed_markdown(
  453. *,
  454. document_id: str,
  455. markdown: str,
  456. page_count: Any = None,
  457. row_count: Any = None,
  458. ) -> dict[str, Any]:
  459. if not markdown.strip():
  460. raise APIError(502, "mineru_empty_result", "MinerU 未返回可用文本")
  461. if not page_count:
  462. plain_text = _clean_markdown_text(markdown)
  463. page_count = max(1, math.ceil(len(plain_text) / 1400))
  464. output_dir = settings.upload_path / "parsed" / document_id
  465. output_dir.mkdir(parents=True, exist_ok=True)
  466. markdown_path = output_dir / "full.md"
  467. markdown_path.write_text(markdown, encoding="utf-8")
  468. return {
  469. "markdown": markdown,
  470. "markdown_path": markdown_path,
  471. "page_count": int(page_count) if page_count else None,
  472. "row_count": int(row_count) if row_count else None,
  473. }
  474. def inspect_source_metrics(source: Path) -> dict[str, int | None]:
  475. extension = source.suffix.lower()
  476. try:
  477. if extension == ".pdf":
  478. from pypdf import PdfReader
  479. return {
  480. "page_count": len(PdfReader(str(source)).pages),
  481. "row_count": None,
  482. }
  483. if extension == ".xlsx":
  484. from openpyxl import load_workbook
  485. workbook = load_workbook(source, read_only=True, data_only=True)
  486. row_count = sum(max(sheet.max_row - 1, 0) for sheet in workbook.worksheets)
  487. workbook.close()
  488. return {"page_count": None, "row_count": row_count}
  489. if extension == ".csv":
  490. with source.open("r", encoding="utf-8-sig", errors="replace") as file:
  491. row_count = max(sum(1 for _ in file) - 1, 0)
  492. return {"page_count": None, "row_count": row_count}
  493. if extension == ".docx":
  494. from docx import Document
  495. document = Document(str(source))
  496. text_length = sum(len(paragraph.text) for paragraph in document.paragraphs)
  497. explicit_breaks = sum(
  498. 1
  499. for paragraph in document.paragraphs
  500. for _ in paragraph._p.xpath('.//w:br[@w:type="page"]')
  501. )
  502. estimated_pages = max(explicit_breaks + 1, math.ceil(text_length / 1400))
  503. return {"page_count": estimated_pages, "row_count": None}
  504. except Exception:
  505. return {"page_count": None, "row_count": None}
  506. return {"page_count": None, "row_count": None}
  507. async def extract_document_fields(
  508. *,
  509. document: BusinessDocument,
  510. template: DocumentExtractionTemplate,
  511. parsed_markdown: str,
  512. ) -> dict[str, Any]:
  513. field_definitions = template_field_definitions(template)
  514. if (
  515. template.code == "safety_monitoring_system_ledger"
  516. and Path(document.file_path).suffix.lower() == ".xlsx"
  517. ):
  518. deterministic = extract_safety_monitoring_ledger(document.file_path)
  519. if deterministic is not None:
  520. return normalize_extraction_result(
  521. deterministic,
  522. field_definitions,
  523. model_status="deterministic_excel",
  524. parsed_markdown=parsed_markdown,
  525. template_code=template.code,
  526. )
  527. if (
  528. template.code == "dust_dispersion_free_silica_monitoring_ledger"
  529. and Path(document.file_path).suffix.lower() == ".xlsx"
  530. ):
  531. deterministic = extract_dust_dispersion_free_silica_plan(document.file_path)
  532. if deterministic is not None:
  533. return normalize_extraction_result(
  534. deterministic,
  535. field_definitions,
  536. model_status="deterministic_excel",
  537. parsed_markdown=parsed_markdown,
  538. template_code=template.code,
  539. )
  540. runtime = get_agent_runtime()
  541. if runtime.enabled:
  542. prompt = build_extraction_prompt(
  543. document=document,
  544. template=template,
  545. parsed_markdown=parsed_markdown[: settings.document_extraction_max_chars],
  546. field_definitions=field_definitions,
  547. )
  548. agent_result = await runtime.invoke(
  549. message=prompt,
  550. thread_id=f"document-extraction-{document.id}",
  551. context={
  552. "document_id": document.id,
  553. "document_type": template.code,
  554. "template_code": template.code,
  555. },
  556. )
  557. if agent_result.status == "success":
  558. parsed = parse_model_json(agent_result.output)
  559. if parsed is not None:
  560. return normalize_extraction_result(
  561. parsed,
  562. field_definitions,
  563. model_status="success",
  564. parsed_markdown=parsed_markdown,
  565. template_code=template.code,
  566. )
  567. fallback = fallback_extraction(parsed_markdown, field_definitions)
  568. return normalize_extraction_result(
  569. fallback,
  570. field_definitions,
  571. model_status=f"degraded:{agent_result.status}",
  572. parsed_markdown=parsed_markdown,
  573. template_code=template.code,
  574. )
  575. fallback = fallback_extraction(parsed_markdown, field_definitions)
  576. return normalize_extraction_result(
  577. fallback,
  578. field_definitions,
  579. model_status="disabled",
  580. parsed_markdown=parsed_markdown,
  581. template_code=template.code,
  582. )
  583. def build_extraction_prompt(
  584. *,
  585. document: BusinessDocument,
  586. template: DocumentExtractionTemplate,
  587. parsed_markdown: str,
  588. field_definitions: list[dict[str, Any]],
  589. ) -> str:
  590. schema = json.dumps(field_definitions, ensure_ascii=False, indent=2)
  591. return f"""
  592. {template.prompt}
  593. 资料信息:
  594. - 标题:{document.title}
  595. - 分类:{document.category}
  596. - 业务资料类型:{template.name}
  597. 字段定义:
  598. {schema}
  599. 请只输出一个 JSON 对象,不要输出 Markdown 代码块或其他说明。格式:
  600. {{
  601. "summary": "资料摘要",
  602. "fields": [
  603. {{
  604. "field_key": "字段 key",
  605. "field_name": "字段名称",
  606. "field_value": "原始值,列表或对象转成 JSON",
  607. "normalized_value": "标准化值",
  608. "value_type": "text/number/date/object/list",
  609. "unit": "",
  610. "confidence": 0.95,
  611. "page_no": null,
  612. "evidence_text": "原文证据"
  613. }}
  614. ]
  615. }}
  616. 解析文本:
  617. {parsed_markdown}
  618. """.strip()
  619. def parse_model_json(output: str) -> dict[str, Any] | None:
  620. text = output.strip()
  621. if text.startswith("```"):
  622. text = re.sub(r"^```(?:json)?\s*", "", text)
  623. text = re.sub(r"\s*```$", "", text)
  624. start = text.find("{")
  625. end = text.rfind("}")
  626. if start < 0 or end <= start:
  627. return None
  628. try:
  629. value = json.loads(text[start : end + 1])
  630. except json.JSONDecodeError:
  631. return None
  632. return value if isinstance(value, dict) else None
  633. def normalize_extraction_result(
  634. result: dict[str, Any],
  635. field_definitions: list[dict[str, Any]],
  636. *,
  637. model_status: str,
  638. parsed_markdown: str,
  639. template_code: str = "",
  640. ) -> dict[str, Any]:
  641. by_key = {
  642. str(item.get("field_key") or item.get("key") or ""): item
  643. for item in result.get("fields", [])
  644. if isinstance(item, dict)
  645. }
  646. normalized_fields: list[dict[str, Any]] = []
  647. for definition in field_definitions:
  648. key = str(definition.get("key") or "")
  649. item = by_key.get(key, {})
  650. item_field_name = str(item.get("field_name") or "").strip()
  651. definition_field_name = str(definition.get("name") or key)
  652. if (
  653. not item_field_name
  654. or item_field_name == key
  655. or re.fullmatch(r"[a-zA-Z0-9_]+", item_field_name)
  656. ):
  657. field_name = definition_field_name
  658. else:
  659. field_name = item_field_name
  660. raw_value = item.get(
  661. "field_value",
  662. item.get("value", item.get("normalized_value")),
  663. )
  664. normalized_value = item.get("normalized_value", raw_value)
  665. normalized_fields.append(
  666. {
  667. "field_key": key,
  668. "field_name": field_name,
  669. "field_value": _stringify_value(raw_value),
  670. "normalized_value": _stringify_value(normalized_value),
  671. "value_type": str(
  672. item.get("value_type") or definition.get("type") or "text"
  673. ),
  674. "unit": str(item.get("unit") or ""),
  675. "confidence": _normalize_confidence(item.get("confidence")),
  676. "page_no": item.get("page_no"),
  677. "evidence_text": str(item.get("evidence_text") or "")[:2000],
  678. "metadata_json": {
  679. "model_status": model_status,
  680. "group_key": definition.get("group_key", ""),
  681. "group_name": definition.get("group_name", ""),
  682. "group_order": definition.get("group_order", 999),
  683. "field_order": definition.get("field_order", 999),
  684. },
  685. }
  686. )
  687. summary = str(result.get("summary") or "").strip()
  688. field_map = {
  689. item["field_key"]: item["normalized_value"] or item["field_value"]
  690. for item in normalized_fields
  691. }
  692. summary = _build_structured_summary(template_code, field_map) or summary
  693. if not summary:
  694. summary = _clean_markdown_text(parsed_markdown)[:500]
  695. summary = _compact_analysis_summary(summary)
  696. return {
  697. "summary": summary,
  698. "fields": normalized_fields,
  699. "model_status": model_status,
  700. }
  701. def fallback_extraction(
  702. parsed_markdown: str,
  703. field_definitions: list[dict[str, Any]],
  704. ) -> dict[str, Any]:
  705. fields = []
  706. for definition in field_definitions:
  707. name = str(definition.get("name") or definition.get("key") or "")
  708. key = str(definition.get("key") or "")
  709. aliases = [str(item) for item in definition.get("aliases", [])]
  710. value = _extract_labeled_value(parsed_markdown, name, key, *aliases)
  711. fields.append(
  712. {
  713. "field_key": key,
  714. "field_name": name,
  715. "field_value": value,
  716. "normalized_value": value,
  717. "value_type": definition.get("type", "text"),
  718. "confidence": 0,
  719. "evidence_text": "",
  720. }
  721. )
  722. return {
  723. "summary": _clean_markdown_text(parsed_markdown)[:500],
  724. "fields": fields,
  725. }
  726. def replace_document_extractions(
  727. db: Session,
  728. *,
  729. document_id: str,
  730. fields: list[dict[str, Any]],
  731. model_status: str,
  732. ) -> None:
  733. document_repo.delete_extractions(db, document_id)
  734. for item in fields:
  735. db.add(
  736. DocumentExtraction(
  737. document_id=document_id,
  738. field_key=str(item["field_key"]),
  739. field_name=str(item["field_name"]),
  740. field_value=str(item.get("field_value") or ""),
  741. normalized_value=str(item.get("normalized_value") or ""),
  742. value_type=str(item.get("value_type") or "text"),
  743. unit=str(item.get("unit") or ""),
  744. confidence=item.get("confidence"),
  745. page_no=item.get("page_no"),
  746. evidence_text=str(item.get("evidence_text") or ""),
  747. metadata_json={
  748. **(item.get("metadata_json") or {}),
  749. "model_status": model_status,
  750. },
  751. )
  752. )
  753. db.commit()
  754. def _stringify_value(value: Any) -> str:
  755. if value is None:
  756. return ""
  757. if isinstance(value, (dict, list)):
  758. return json.dumps(value, ensure_ascii=False)
  759. return str(value)
  760. def _normalize_confidence(value: Any) -> float | None:
  761. if value is None:
  762. return None
  763. try:
  764. number = float(value)
  765. except (TypeError, ValueError):
  766. return None
  767. if 0 <= number <= 1:
  768. number *= 100
  769. return max(0.0, min(number, 100.0))
  770. def _extract_labeled_value(text: str, *labels: str) -> str:
  771. for label in labels:
  772. if not label:
  773. continue
  774. pattern = rf"{re.escape(label)}\s*[::]\s*([^\n]+)"
  775. match = re.search(pattern, text, flags=re.IGNORECASE)
  776. if match:
  777. return match.group(1).strip()
  778. return ""
  779. def _clean_markdown_text(text: str) -> str:
  780. cleaned = re.sub(r"!\[[^\]]*\]\([^)]+\)", " ", text)
  781. cleaned = re.sub(r"<[^>]+>", " ", cleaned)
  782. cleaned = html.unescape(cleaned)
  783. cleaned = re.sub(r"[ \t\r\f\v]+", " ", cleaned)
  784. cleaned = re.sub(r"\n{2,}", "\n", cleaned)
  785. return cleaned.strip()
  786. def extract_dust_dispersion_free_silica_plan(file_path: str) -> dict[str, Any] | None:
  787. try:
  788. from openpyxl import load_workbook
  789. workbook = load_workbook(file_path, read_only=True, data_only=True)
  790. except Exception:
  791. return None
  792. def cell_text(value: Any) -> str:
  793. if value is None:
  794. return ""
  795. if isinstance(value, datetime):
  796. return value.date().isoformat()
  797. if hasattr(value, "isoformat") and not isinstance(value, str):
  798. try:
  799. return value.isoformat()
  800. except Exception:
  801. pass
  802. return str(value).replace("\r", "").strip()
  803. try:
  804. sheet = workbook.worksheets[0]
  805. rows = [
  806. [cell_text(value) for value in row]
  807. for row in sheet.iter_rows(values_only=True)
  808. ]
  809. if not rows:
  810. return None
  811. aliases = {
  812. "sequence_no": {"序号"},
  813. "mine_name": {"矿井名称", "矿井"},
  814. "sampling_period": {"现场采样时间", "采样时间"},
  815. "detection_locations": {"检测地点", "监测地点"},
  816. "detection_items": {"检测项目", "监测项目"},
  817. "frequency_cycle": {"频次/周期", "频次/周期", "检测频次"},
  818. "primary_contact": {"联系人及联系方式", "联系人"},
  819. "remarks": {"备注"},
  820. "planned_detection_time": {
  821. "工作面矿方预计检测时间",
  822. "预计检测时间",
  823. "计划检测时间",
  824. },
  825. "actual_detection_date": {
  826. "商定后实际检测日期",
  827. "实际检测日期",
  828. "实际检测时间",
  829. },
  830. }
  831. header_row_index = -1
  832. header_map: dict[str, int] = {}
  833. for row_index, row in enumerate(rows[:12]):
  834. row_map: dict[str, int] = {}
  835. for column_index, value in enumerate(row):
  836. normalized_value = value.replace("*", "").replace("(", "(").replace(")", ")")
  837. for key, labels in aliases.items():
  838. if any(label in normalized_value for label in labels):
  839. row_map.setdefault(key, column_index)
  840. if {"mine_name", "detection_locations", "detection_items"}.issubset(row_map):
  841. header_row_index = row_index
  842. header_map = row_map
  843. break
  844. if header_row_index < 0:
  845. return None
  846. def value_at(row: list[str], key: str) -> str:
  847. index = header_map.get(key)
  848. return row[index] if index is not None and index < len(row) else ""
  849. contact_index = header_map.get("primary_contact")
  850. phone_index = contact_index + 1 if contact_index is not None else None
  851. def phone_info_at(row: list[str]) -> tuple[str, str]:
  852. if phone_index is None or phone_index >= len(row):
  853. return "", ""
  854. value = row[phone_index].strip()
  855. phones = re.findall(r"1[3-9]\d{9}", value)
  856. name = re.sub(r"1[3-9]\d{9}", "", value)
  857. name = re.sub(r"[\s::,,、;;]+", " ", name).strip()
  858. return name, "、".join(phones)
  859. def phone_at(row: list[str]) -> str:
  860. return phone_info_at(row)[1]
  861. records: list[dict[str, Any]] = []
  862. for row_index in range(header_row_index + 1, len(rows)):
  863. row = rows[row_index]
  864. mine_name = value_at(row, "mine_name")
  865. if not mine_name or "现场检测联系人" in mine_name:
  866. continue
  867. detection_items = [
  868. item.strip()
  869. for item in re.split(r"[\n\r、,,;;]+", value_at(row, "detection_items"))
  870. if item.strip()
  871. ]
  872. detection_locations = value_at(row, "detection_locations")
  873. if not detection_items and not detection_locations:
  874. continue
  875. onsite_contact = ""
  876. onsite_phone = ""
  877. if row_index + 1 < len(rows):
  878. next_row = rows[row_index + 1]
  879. next_contact = value_at(next_row, "primary_contact")
  880. if "现场检测联系人" in next_contact:
  881. phone_name, onsite_phone = phone_info_at(next_row)
  882. label_name = re.sub(
  883. r"^现场检测联系人\s*[::]?\s*",
  884. "",
  885. next_contact,
  886. ).strip()
  887. onsite_contact = label_name or phone_name
  888. primary_contact = value_at(row, "primary_contact")
  889. records.append(
  890. {
  891. "sequence_no": value_at(row, "sequence_no"),
  892. "mine_name": mine_name,
  893. "sampling_period": value_at(row, "sampling_period"),
  894. "detection_locations": detection_locations,
  895. "detection_items": detection_items,
  896. "frequency_cycle": value_at(row, "frequency_cycle"),
  897. "primary_contact": primary_contact,
  898. "primary_phone": phone_at(row),
  899. "onsite_contact": onsite_contact,
  900. "onsite_phone": onsite_phone,
  901. "remarks": value_at(row, "remarks"),
  902. "planned_detection_time": value_at(row, "planned_detection_time"),
  903. "actual_detection_date": value_at(row, "actual_detection_date"),
  904. }
  905. )
  906. if not records:
  907. return None
  908. mine_names = list(dict.fromkeys(record["mine_name"] for record in records))
  909. monitoring_items: list[str] = []
  910. frequency_cycles: list[str] = []
  911. coverage_requirements: list[str] = []
  912. primary_contacts: list[dict[str, str]] = []
  913. onsite_contacts: list[dict[str, str]] = []
  914. contact_phone_map: dict[str, str] = {}
  915. for record in records:
  916. for item in record["detection_items"]:
  917. if item not in monitoring_items:
  918. monitoring_items.append(item)
  919. frequency = record["frequency_cycle"]
  920. if frequency and frequency not in frequency_cycles:
  921. frequency_cycles.append(frequency)
  922. remarks = record["remarks"]
  923. if remarks and remarks not in coverage_requirements:
  924. coverage_requirements.append(remarks)
  925. if record["primary_contact"]:
  926. primary_contacts.append(
  927. {
  928. "mine_name": record["mine_name"],
  929. "contact_name": record["primary_contact"],
  930. "phone": record["primary_phone"],
  931. }
  932. )
  933. if record["primary_phone"]:
  934. contact_phone_map[record["primary_contact"]] = record["primary_phone"]
  935. if record["onsite_contact"]:
  936. onsite_contacts.append(
  937. {
  938. "mine_name": record["mine_name"],
  939. "contact_name": record["onsite_contact"],
  940. "phone": record["onsite_phone"],
  941. }
  942. )
  943. if record["onsite_phone"]:
  944. contact_phone_map[record["onsite_contact"]] = record["onsite_phone"]
  945. planned_dates = [
  946. {
  947. "mine_name": record["mine_name"],
  948. "planned_detection_time": record["planned_detection_time"],
  949. }
  950. for record in records
  951. if record["planned_detection_time"]
  952. ]
  953. actual_dates = [
  954. {
  955. "mine_name": record["mine_name"],
  956. "actual_detection_date": record["actual_detection_date"],
  957. }
  958. for record in records
  959. if record["actual_detection_date"]
  960. ]
  961. missing_planned_dates = [
  962. record["mine_name"] for record in records if not record["planned_detection_time"]
  963. ]
  964. missing_actual_dates = [
  965. record["mine_name"] for record in records if not record["actual_detection_date"]
  966. ]
  967. title = next(
  968. (
  969. row[0]
  970. for row in rows[:header_row_index + 1]
  971. if row and ("粉尘" in row[0] or "监测" in row[0])
  972. ),
  973. "采掘作业场所粉尘分散度、游离二氧化硅含量监测计划表",
  974. )
  975. title = title.replace("\n", "")
  976. year_match = re.search(r"(20\d{2})", title)
  977. monitoring_year = year_match.group(1) if year_match else ""
  978. missing_items: list[str] = []
  979. if missing_planned_dates:
  980. missing_items.append("部分矿井未填写计划检测时间")
  981. if missing_actual_dates:
  982. missing_items.append("部分矿井未填写实际检测日期")
  983. conclusion = (
  984. f"计划表覆盖 {len(mine_names)} 个矿井、{len(records)} 条监测计划,"
  985. f"检测项目包括{'、'.join(monitoring_items) or '未识别'};"
  986. "计划和实际检测日期仍需按执行情况补充。"
  987. )
  988. field_values: dict[str, Any] = {
  989. "monitoring_title": title,
  990. "monitoring_year": monitoring_year,
  991. "document_kind": "监测计划表" if "计划" in title else "监测台账",
  992. "applicable_scope": "采掘作业场所粉尘分散度、游离二氧化硅含量监测",
  993. "sheet_names": list(workbook.sheetnames),
  994. "mine_count": len(mine_names),
  995. "plan_record_count": len(records),
  996. "mine_names": mine_names,
  997. "monitoring_items": monitoring_items,
  998. "frequency_cycle": frequency_cycles,
  999. "coverage_requirements": coverage_requirements,
  1000. "mine_plan_records": records,
  1001. "planned_detection_dates": planned_dates,
  1002. "actual_detection_dates": actual_dates,
  1003. "missing_planned_dates": missing_planned_dates,
  1004. "missing_actual_dates": missing_actual_dates,
  1005. "primary_contacts": primary_contacts,
  1006. "onsite_contacts": onsite_contacts,
  1007. "contact_phone_map": contact_phone_map,
  1008. "plan_complete": "是" if not missing_items else "否",
  1009. "missing_items": missing_items,
  1010. "overall_conclusion": conclusion,
  1011. }
  1012. return {
  1013. "summary": "",
  1014. "fields": [
  1015. {
  1016. "field_key": key,
  1017. "field_name": key,
  1018. "field_value": value,
  1019. "normalized_value": value,
  1020. }
  1021. for key, value in field_values.items()
  1022. ],
  1023. }
  1024. finally:
  1025. workbook.close()
  1026. def extract_safety_monitoring_ledger(file_path: str) -> dict[str, Any] | None:
  1027. try:
  1028. from openpyxl import load_workbook
  1029. workbook = load_workbook(file_path, read_only=True, data_only=True)
  1030. except Exception:
  1031. return None
  1032. try:
  1033. sheet_names = list(workbook.sheetnames)
  1034. total_sheet = workbook["总台账"] if "总台账" in workbook.sheetnames else workbook.worksheets[0]
  1035. header_map: dict[str, int] = {}
  1036. header_row = 0
  1037. for row_index, row in enumerate(
  1038. total_sheet.iter_rows(min_row=1, max_row=20, values_only=True),
  1039. start=1,
  1040. ):
  1041. values = [str(value).strip() if value is not None else "" for value in row]
  1042. if "编码" in values and "设备名称" in values:
  1043. header_row = row_index
  1044. header_map = {value: index for index, value in enumerate(values) if value}
  1045. break
  1046. if not header_map:
  1047. return None
  1048. code_index = header_map.get("编码")
  1049. device_index = header_map.get("设备名称")
  1050. alarm_index = header_map.get("报警值")
  1051. cut_index = header_map.get("断电值")
  1052. restore_index = header_map.get("复电值")
  1053. location_index = header_map.get("安装地点")
  1054. position_index = header_map.get("安装位置")
  1055. unit_index = next(
  1056. (index for label, index in header_map.items() if label.startswith("单位")),
  1057. None,
  1058. )
  1059. range_index = header_map.get("断电范围")
  1060. substations: set[str] = set()
  1061. substation_details: list[dict[str, Any]] = []
  1062. sensor_type_counts: dict[str, int] = {}
  1063. critical_details: list[dict[str, Any]] = []
  1064. thresholds = {
  1065. "gas_alarm_thresholds": set(),
  1066. "co_alarm_thresholds": set(),
  1067. "temperature_alarm_thresholds": set(),
  1068. "airflow_alarm_thresholds": set(),
  1069. }
  1070. power_cut_rules: set[str] = set()
  1071. power_restore_rules: set[str] = set()
  1072. sensor_count = 0
  1073. for row in total_sheet.iter_rows(
  1074. min_row=header_row + 1,
  1075. max_row=min(total_sheet.max_row, 20000),
  1076. values_only=True,
  1077. ):
  1078. values = ["" if value is None else str(value).strip() for value in row]
  1079. code = values[code_index] if code_index is not None and code_index < len(values) else ""
  1080. device = values[device_index] if device_index is not None and device_index < len(values) else ""
  1081. location = values[location_index] if location_index is not None and location_index < len(values) else ""
  1082. position = values[position_index] if position_index is not None and position_index < len(values) else ""
  1083. unit = values[unit_index] if unit_index is not None and unit_index < len(values) else ""
  1084. alarm = values[alarm_index] if alarm_index is not None and alarm_index < len(values) else ""
  1085. cut = values[cut_index] if cut_index is not None and cut_index < len(values) else ""
  1086. restore = values[restore_index] if restore_index is not None and restore_index < len(values) else ""
  1087. power_range = values[range_index] if range_index is not None and range_index < len(values) else ""
  1088. if code:
  1089. substations.add(code)
  1090. if len(substation_details) < 100:
  1091. substation_details.append(
  1092. {
  1093. "code": code,
  1094. "location": location,
  1095. "device": device,
  1096. }
  1097. )
  1098. if not device or device == "综合分站":
  1099. continue
  1100. sensor_count += 1
  1101. sensor_type = _normalize_sensor_type(device)
  1102. sensor_type_counts[sensor_type] = sensor_type_counts.get(sensor_type, 0) + 1
  1103. if device in {"激光甲烷", "甲烷"} and alarm:
  1104. thresholds["gas_alarm_thresholds"].add(alarm)
  1105. elif device == "一氧化碳" and alarm:
  1106. thresholds["co_alarm_thresholds"].add(alarm)
  1107. elif device == "温度" and alarm:
  1108. thresholds["temperature_alarm_thresholds"].add(alarm)
  1109. elif device in {"风速", "风量"} and alarm:
  1110. thresholds["airflow_alarm_thresholds"].add(alarm)
  1111. if cut:
  1112. power_cut_rules.add(cut)
  1113. if restore:
  1114. power_restore_rules.add(restore)
  1115. if (
  1116. len(critical_details) < 200
  1117. and device
  1118. in {
  1119. "激光甲烷",
  1120. "甲烷",
  1121. "一氧化碳",
  1122. "温度",
  1123. "风速",
  1124. "风量",
  1125. "烟雾",
  1126. "风门",
  1127. "局扇",
  1128. "主扇",
  1129. "风筒",
  1130. }
  1131. ):
  1132. critical_details.append(
  1133. {
  1134. "code": code,
  1135. "device": device,
  1136. "location": location,
  1137. "position": position,
  1138. "unit": unit,
  1139. "alarm": alarm,
  1140. "cut": cut,
  1141. "restore": restore,
  1142. "power_range": power_range,
  1143. }
  1144. )
  1145. title = "安全监控系统台账"
  1146. department = ""
  1147. if "台账封面" in workbook.sheetnames:
  1148. cover = workbook["台账封面"]
  1149. cover_values = [
  1150. str(value).strip()
  1151. for row in cover.iter_rows(min_row=1, max_row=40, values_only=True)
  1152. for value in row
  1153. if value is not None and str(value).strip()
  1154. ]
  1155. for value in cover_values:
  1156. if "台账" in value and len(value) > len(title):
  1157. title = value
  1158. if "通风队" in value:
  1159. department = value
  1160. version_match = re.search(r"(20\d{4})", Path(file_path).stem)
  1161. ledger_version = version_match.group(1) if version_match else ""
  1162. field_values: dict[str, Any] = {
  1163. "ledger_title": title,
  1164. "mine_name": "上湾煤矿" if "上湾" in Path(file_path).stem else "",
  1165. "department": department,
  1166. "ledger_version": ledger_version,
  1167. "sheet_names": sheet_names,
  1168. "data_source": "安全监控系统",
  1169. "substation_count": len(substations),
  1170. "sensor_count": sensor_count,
  1171. "substation_details": substation_details,
  1172. "sensor_type_counts": sensor_type_counts,
  1173. "monitored_parameters": sorted(sensor_type_counts),
  1174. "sensor_fields": [
  1175. "编码",
  1176. "安装地点",
  1177. "设备名称",
  1178. "安装位置",
  1179. "单位",
  1180. "报警值",
  1181. "断电值",
  1182. "复电值",
  1183. "断电范围",
  1184. ],
  1185. "critical_sensor_details": critical_details,
  1186. "gas_sensor_count": sensor_type_counts.get("甲烷", 0),
  1187. "co_sensor_count": sensor_type_counts.get("一氧化碳", 0),
  1188. "airflow_sensor_count": sensor_type_counts.get("风速", 0),
  1189. "temperature_sensor_count": sensor_type_counts.get("温度", 0),
  1190. "smoke_sensor_count": sensor_type_counts.get("烟雾", 0),
  1191. "door_sensor_count": sensor_type_counts.get("风门", 0),
  1192. "fan_sensor_count": sensor_type_counts.get("局扇", 0) + sensor_type_counts.get("主扇", 0),
  1193. "gas_alarm_thresholds": sorted(thresholds["gas_alarm_thresholds"]),
  1194. "co_alarm_thresholds": sorted(thresholds["co_alarm_thresholds"]),
  1195. "temperature_alarm_thresholds": sorted(thresholds["temperature_alarm_thresholds"]),
  1196. "airflow_alarm_thresholds": sorted(thresholds["airflow_alarm_thresholds"]),
  1197. "power_cut_rules": sorted(power_cut_rules),
  1198. "power_restore_rules": sorted(power_restore_rules),
  1199. "ledger_complete": "是" if substations and sensor_count else "否",
  1200. "missing_items": [],
  1201. "abnormal_settings": [],
  1202. "overall_conclusion": (
  1203. f"台账包含 {len(substations)} 个分站、{sensor_count} 个传感器或测点,"
  1204. "字段完整,可用于安全监控系统台账管理。"
  1205. ),
  1206. }
  1207. return {
  1208. "summary": "",
  1209. "fields": [
  1210. {
  1211. "field_key": key,
  1212. "field_name": key,
  1213. "field_value": value,
  1214. "normalized_value": value,
  1215. }
  1216. for key, value in field_values.items()
  1217. ],
  1218. }
  1219. finally:
  1220. workbook.close()
  1221. def _normalize_sensor_type(device: str) -> str:
  1222. if device in {"激光甲烷", "甲烷"}:
  1223. return "甲烷"
  1224. if device in {"风速", "风量"}:
  1225. return "风速"
  1226. return device
  1227. def _compact_analysis_summary(text: str) -> str:
  1228. normalized = re.sub(r"\s+", " ", text).strip()
  1229. blocks = [
  1230. block.strip()
  1231. for block in re.split(
  1232. r"(?<![\d.])(?=(?:【|\[)?(?:[1-9]|1[0-9]|20)[.、.](?=\s|[\u4e00-\u9fff]))",
  1233. normalized,
  1234. )
  1235. if re.match(
  1236. r"^(?:【|\[)?(?:[1-9]|1[0-9]|20)[.、.](?=\s|[\u4e00-\u9fff])",
  1237. block.strip(),
  1238. )
  1239. ]
  1240. if len(blocks) >= 3:
  1241. return "\n".join(block[:80] for block in blocks[:8])
  1242. return normalized[:500]
  1243. def _build_structured_summary(
  1244. template_code: str,
  1245. fields: dict[str, str],
  1246. ) -> str:
  1247. get = lambda key: fields.get(key) or "未提供"
  1248. if template_code == "main_fan_external_leakage_rate_report":
  1249. return "\n".join(
  1250. [
  1251. f"1. 南风井:漏风率 {get('south_leakage_rate')}%,{_limit_status(get('south_leakage_rate'))}。",
  1252. f"2. 北风井:漏风率 {get('north_leakage_rate')}%,{_limit_status(get('north_leakage_rate'))}。",
  1253. f"3. 最大漏风率:{get('maximum_leakage_rate')}%,标准限值 {get('standard_limit')}%。",
  1254. f"4. 总体结论:{get('compliance_status')}。",
  1255. ]
  1256. )
  1257. if template_code == "monthly_air_distribution_plan":
  1258. return "\n".join(
  1259. [
  1260. f"1. 北风井:需风量 {get('north_required_airflow')},可用回风量 {get('north_available_airflow')},{get('north_capacity_status')}。",
  1261. f"2. 南风井:需风量 {get('south_required_airflow')},可用回风量 {get('south_available_airflow')},{get('south_capacity_status')}。",
  1262. f"3. 全矿井:需风量 {get('mine_required_airflow')},总回风量 {get('mine_available_airflow')},{get('mine_capacity_status')}。",
  1263. ]
  1264. )
  1265. if template_code == "mine_ventilation_resistance_report":
  1266. return "\n".join(
  1267. [
  1268. f"1. 主要路线:{get('main_route_measured_resistance')},相对误差 {get('main_route_relative_error')}%。",
  1269. f"2. 次要路线:{get('secondary_route_measured_resistance')},相对误差 {get('secondary_route_relative_error')}%。",
  1270. f"3. 等积孔:{get('equivalent_orifice')},通风难易程度:{get('ventilation_difficulty')}。",
  1271. f"4. 有效风量率:{get('effective_airflow_rate')}%,{get('effective_airflow_rate_pass')}。",
  1272. f"5. 总体评价:{get('overall_assessment')}。",
  1273. ]
  1274. )
  1275. if template_code == "ventilation_capacity_verification_report":
  1276. return "\n".join(
  1277. [
  1278. f"1. 核定通风能力:{get('final_capacity')} {get('capacity_unit')}。",
  1279. f"2. 北风井:需风量 {get('north_required_airflow')},实际回风量 {get('north_available_airflow')},{get('north_capacity_status')}。",
  1280. f"3. 南风井:需风量 {get('south_required_airflow')},实际回风量 {get('south_available_airflow')},{get('south_capacity_status')}。",
  1281. f"4. 全矿井:需风量 {get('mine_required_airflow')},实际总回风量 {get('mine_available_airflow')},{get('mine_capacity_status')}。",
  1282. f"5. 有效风量率:{get('effective_airflow_rate')}%。",
  1283. ]
  1284. )
  1285. if template_code == "airflow_measurement_decadal_report":
  1286. rate = get("effective_airflow_rate")
  1287. rate_status = (
  1288. "合格"
  1289. if _is_number(rate) and float(rate) >= 85
  1290. else "不合格" if _is_number(rate) else "需核验"
  1291. )
  1292. return "\n".join(
  1293. [
  1294. f"1. 测风日期:{get('measurement_date')},矿井:{get('mine_name')}。",
  1295. f"2. 北风井:总回风量 {get('north_return_airflow')},总排风量 {get('north_exhaust_airflow')},负压 {get('north_negative_pressure')}。",
  1296. f"3. 南风井:总回风量 {get('south_return_airflow')},总排风量 {get('south_exhaust_airflow')},负压 {get('south_negative_pressure')}。",
  1297. f"4. 全矿井:总回风量 {get('mine_total_return_airflow')},总进风量 {get('mine_total_intake_airflow')},有效风量率 {rate}%,{rate_status}。",
  1298. f"5. 测点数量:{get('measurement_point_count')},风量不足测点:{get('insufficient_airflow_points')},瓦斯异常测点:{get('gas_anomaly_points')}。",
  1299. ]
  1300. )
  1301. if template_code == "gas_grade_appraisal_report":
  1302. return "\n".join(
  1303. [
  1304. f"1. 瓦斯等级:{get('gas_grade')},上年度:{get('previous_gas_grade')}。",
  1305. f"2. 绝对瓦斯涌出量:{get('absolute_gas_emission')},相对瓦斯涌出量:{get('relative_gas_emission')}。",
  1306. f"3. 二氧化碳绝对涌出量:{get('absolute_co2_emission')},相对涌出量:{get('relative_co2_emission')}。",
  1307. f"4. 采煤面最大绝对瓦斯涌出量:{get('coalface_max_gas_emission')},掘进面最大绝对瓦斯涌出量:{get('heading_max_gas_emission')}。",
  1308. f"5. 瓦斯突出:{get('gas_outburst_status')},瓦斯喷出:{get('gas_blowout_status')},结论:{get('gas_grade_conclusion')}。",
  1309. ]
  1310. )
  1311. if template_code == "coal_dust_explosion_spontaneous_combustion_report":
  1312. validity_status = _expiry_status(get("valid_until"))
  1313. return "\n".join(
  1314. [
  1315. f"1. 煤尘爆炸性:{get('dust_explosion_result')},火焰长度 {get('flame_length')}mm,最低岩粉用量 {get('minimum_rock_dust')}%。",
  1316. f"2. 自燃倾向性:{get('spontaneous_combustion_grade')}。",
  1317. f"3. 关键指标:Vdaf {get('volatile_daf')}%,吸氧量 {get('oxygen_adsorption')}cm³/g。",
  1318. f"4. 有效期至:{get('valid_until')},状态:{validity_status}。",
  1319. f"5. 总体结论:{get('inspection_conclusion')}。",
  1320. ]
  1321. )
  1322. if template_code == "dust_dispersion_free_silica_monitoring_ledger":
  1323. return "\n".join(
  1324. [
  1325. f"1. 监测年度:{get('monitoring_year')},资料性质:{get('document_kind')}。",
  1326. f"2. 覆盖范围:{get('mine_count')} 个矿井,{get('plan_record_count')} 条计划,矿井包括 {get('mine_names')}。",
  1327. f"3. 检测项目:{get('monitoring_items')},频次:{get('frequency_cycle')}。",
  1328. f"4. 时间完整性:计划检测日期缺失 {get('missing_planned_dates')};实际检测日期缺失 {get('missing_actual_dates')}。",
  1329. f"5. 总体结论:{get('overall_conclusion')}",
  1330. ]
  1331. )
  1332. if template_code == "vehicle_air_door_inspection_record":
  1333. return "\n".join(
  1334. [
  1335. f"1. 记录年度:{get('record_year')},检查月份:{get('inspection_month')},矿井:{get('mine_name')}。",
  1336. f"2. 风门台账:{get('air_door_record_count')} 条记录,共 {get('total_door_sets')} 套;自动风门 {get('automatic_door_count')} 套,手动风门 {get('manual_door_count')} 套。",
  1337. f"3. 风门规格:{get('common_specifications')};停用或常开风门:{get('inactive_door_count')} 套。",
  1338. f"4. 检查情况:检查 {get('inspected_door_count')} 处,发现问题 {get('problem_door_count')} 处,已整改 {get('rectified_count')} 处,未整改 {get('unrectified_count')} 处,整改率 {get('rectification_rate')}%。",
  1339. f"5. 主要问题:{get('problem_types')};总体结论:{get('overall_conclusion')}",
  1340. ]
  1341. )
  1342. if template_code == "safety_monitoring_system_ledger":
  1343. return "\n".join(
  1344. [
  1345. f"1. 台账:{get('ledger_title')},版本:{get('ledger_version')}。",
  1346. f"2. 分站数量:{get('substation_count')},传感器或测点数量:{get('sensor_count')}。",
  1347. f"3. 传感器类型:{get('sensor_type_counts')}。",
  1348. f"4. 报警与断电:甲烷 {get('gas_alarm_thresholds')},一氧化碳 {get('co_alarm_thresholds')},温度 {get('temperature_alarm_thresholds')}。",
  1349. f"5. 完整性:{get('ledger_complete')},结论:{get('overall_conclusion')}。",
  1350. ]
  1351. )
  1352. return ""
  1353. def _is_number(value: str) -> bool:
  1354. try:
  1355. float(value)
  1356. except (TypeError, ValueError):
  1357. return False
  1358. return True
  1359. def _limit_status(value: str, limit: float = 5.0) -> str:
  1360. if not _is_number(value):
  1361. return "需核验"
  1362. return "符合" if float(value) <= limit else "超限告警"
  1363. def _expiry_status(value: str) -> str:
  1364. if not value or value == "未提供":
  1365. return "未提供"
  1366. try:
  1367. expiry = datetime.strptime(value[:10], "%Y-%m-%d").date()
  1368. except ValueError:
  1369. return "需人工核验"
  1370. return "已过期" if expiry < datetime.now(timezone.utc).date() else "有效"