feat(data-process): 清理PDF文档级噪声
This commit is contained in:
caoxiaozhu
@@ -37,6 +37,7 @@ from app.modules.data_process.algorithms import (
|
||||
desensitize_pii,
|
||||
desensitize_structured_record,
|
||||
detect_document_structure,
|
||||
detect_pdf_document_noise,
|
||||
estimate_token_count,
|
||||
extract_pdf_page_texts,
|
||||
generate_standard_records,
|
||||
@@ -44,6 +45,7 @@ from app.modules.data_process.algorithms import (
|
||||
near_duplicate_fingerprint,
|
||||
parse_text_content,
|
||||
preprocess_structured_records,
|
||||
remove_document_noise,
|
||||
score_quality,
|
||||
)
|
||||
from app.modules.data_process.generation import generate_model_records
|
||||
@@ -453,13 +455,25 @@ def _build_preview_items(
|
||||
for source in source_files:
|
||||
parsed = _parse_stored_source(source)
|
||||
if process_type == "unstructured":
|
||||
document_noise_spans = tuple(source.get("document_noise_spans") or ())
|
||||
chunks = _chunk_source_text(parsed.text, config, preprocess_options)
|
||||
for chunk, heading_path in chunks:
|
||||
content = (
|
||||
remove_document_noise(
|
||||
chunk.content,
|
||||
document_noise_spans,
|
||||
source_offset=chunk.start,
|
||||
)
|
||||
if should_clean_invalid and document_noise_spans
|
||||
else chunk.content
|
||||
)
|
||||
preprocess_flags = content_quality_flags(
|
||||
chunk.content,
|
||||
content,
|
||||
min_chars=0,
|
||||
min_tokens=0,
|
||||
)
|
||||
if content != chunk.content:
|
||||
preprocess_flags = (*preprocess_flags, "document_noise_removed")
|
||||
flag_set = set(preprocess_flags)
|
||||
if "clean_invalid_content" in preprocess_options and flag_set & {
|
||||
"empty_content",
|
||||
@@ -475,20 +489,19 @@ def _build_preview_items(
|
||||
}:
|
||||
continue
|
||||
if "deduplicate_content" in preprocess_options:
|
||||
band_keys = _near_duplicate_band_keys(chunk.content)
|
||||
band_keys = _near_duplicate_band_keys(content)
|
||||
candidates = {
|
||||
previous
|
||||
for key in band_keys
|
||||
for previous in seen_near_duplicate_bands.get(key, ())
|
||||
}
|
||||
if any(
|
||||
_safe_near_duplicate(chunk.content, previous)
|
||||
_safe_near_duplicate(content, previous)
|
||||
for previous in candidates
|
||||
):
|
||||
continue
|
||||
for key in band_keys:
|
||||
seen_near_duplicate_bands.setdefault(key, []).append(chunk.content)
|
||||
content = chunk.content
|
||||
seen_near_duplicate_bands.setdefault(key, []).append(content)
|
||||
pii_counts: dict[str, int] = {}
|
||||
if should_desensitize:
|
||||
content, pii_counts = desensitize_pii(content)
|
||||
@@ -512,7 +525,7 @@ def _build_preview_items(
|
||||
"source_end": chunk.end,
|
||||
"source_start_line": chunk.start_line,
|
||||
"source_end_line": chunk.end_line,
|
||||
"token_count": chunk.token_count,
|
||||
"token_count": estimate_token_count(content),
|
||||
"status": "modified" if content != chunk.content else "original",
|
||||
"quality_score": quality,
|
||||
}
|
||||
@@ -1234,6 +1247,7 @@ def pull_external_source(
|
||||
def _prepare_preview_items(
|
||||
task_id: str,
|
||||
store: DataProcessStore,
|
||||
storage: LocalDataProcessStorage,
|
||||
source_file_ids: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
task = store.get_task(task_id)
|
||||
@@ -1251,6 +1265,48 @@ def _prepare_preview_items(
|
||||
]
|
||||
if not sources:
|
||||
raise InvalidStateError("at least one source file is required")
|
||||
preprocess_options = _preprocess_options(task.get("config") or {})
|
||||
if (
|
||||
task.get("process_type") == "unstructured"
|
||||
and preprocess_options & {"clean_invalid", "clean_invalid_content"}
|
||||
):
|
||||
for index, source in enumerate(sources):
|
||||
if str(source.get("file_format") or "").lower() != "pdf":
|
||||
continue
|
||||
storage_object_id = str(source.get("storage_object_id") or "")
|
||||
actual_size = storage.file_size(
|
||||
storage_object_id,
|
||||
expected_task_id=task_id,
|
||||
expected_source_file_id=str(source["id"]),
|
||||
)
|
||||
if actual_size is None:
|
||||
logger.info(
|
||||
"skip PDF document noise detection for unavailable legacy source %s",
|
||||
source["id"],
|
||||
)
|
||||
continue
|
||||
expected_size = int(source.get("size_bytes") or 0)
|
||||
if expected_size and actual_size != expected_size:
|
||||
raise ValueError("source object size does not match metadata")
|
||||
raw = b"".join(
|
||||
storage.iter_bytes(
|
||||
storage_object_id,
|
||||
expected_task_id=task_id,
|
||||
expected_source_file_id=str(source["id"]),
|
||||
expected_size=actual_size,
|
||||
)
|
||||
)
|
||||
pages = extract_pdf_page_texts(raw)
|
||||
extracted_text = "\n\n".join(page.text for page in pages if page.text)
|
||||
if extracted_text != str(source.get("content") or ""):
|
||||
logger.warning(
|
||||
"skip PDF document noise detection because stored offsets differ for %s",
|
||||
source["id"],
|
||||
)
|
||||
continue
|
||||
enriched = dict(source)
|
||||
enriched["document_noise_spans"] = detect_pdf_document_noise(pages)
|
||||
sources[index] = enriched
|
||||
items = _build_preview_items(task, sources)
|
||||
if not items and source_file_ids is None:
|
||||
raise InvalidStateError("source files did not produce preview items")
|
||||
@@ -1262,10 +1318,11 @@ def build_preview(
|
||||
task_id: str,
|
||||
payload: PreviewBuildRequest = Body(default_factory=PreviewBuildRequest),
|
||||
store: DataProcessStore = Depends(get_data_process_store),
|
||||
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
||||
) -> dict[str, Any]:
|
||||
with api_errors():
|
||||
selected_ids = payload.source_file_ids
|
||||
items = _prepare_preview_items(task_id, store, selected_ids)
|
||||
items = _prepare_preview_items(task_id, store, storage, selected_ids)
|
||||
created = store.replace_preview_items(
|
||||
task_id,
|
||||
items,
|
||||
@@ -1405,9 +1462,10 @@ def start(
|
||||
background_tasks: BackgroundTasks,
|
||||
payload: GenerateRequest = Body(default_factory=GenerateRequest),
|
||||
store: DataProcessStore = Depends(get_data_process_store),
|
||||
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
||||
) -> dict[str, Any]:
|
||||
with api_errors():
|
||||
items = _prepare_preview_items(task_id, store)
|
||||
items = _prepare_preview_items(task_id, store, storage)
|
||||
store.replace_preview_items(task_id, items)
|
||||
return _start_generation(task_id, payload, background_tasks, store)
|
||||
|
||||
|
||||
@@ -175,6 +175,15 @@ class PdfPageText:
|
||||
source_end: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DocumentNoiseSpan:
|
||||
"""PDF 中可安全从展示内容移除的文本范围。"""
|
||||
|
||||
start: int
|
||||
end: int
|
||||
kind: Literal["page_number", "repeated_margin", "table_of_contents"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TextChunk:
|
||||
"""带有可追溯来源位置的非结构化文本切片。"""
|
||||
@@ -496,6 +505,240 @@ def extract_pdf_page_texts(raw: bytes) -> tuple[PdfPageText, ...]:
|
||||
return tuple(pages)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PdfLine:
|
||||
text: str
|
||||
start: int
|
||||
end: int
|
||||
|
||||
|
||||
_PDF_PAGE_NUMBER_LINE_PATTERN = re.compile(
|
||||
r"^(?:页次\s*)?(?:第\s*)?(?P<page>\d+)\s*页\s*"
|
||||
r"(?:(?:[/／]\s*)?共\s*(?P<total>\d+)\s*页)?$"
|
||||
)
|
||||
_PDF_FRACTION_PAGE_LINE_PATTERN = re.compile(
|
||||
r"^[—–-]?\s*(?P<page>\d+)\s*[/／]\s*(?P<total>\d+)\s*[—–-]?$"
|
||||
)
|
||||
_PDF_CLASSIFICATION_LABEL_PATTERN = re.compile(
|
||||
r"^(?:(?:秘密等级|密级)\s*)?(?:商密|秘密|机密|绝密)"
|
||||
r"\s*(?:[【\[(（][^】\])）]{1,8}[】\])）])?$"
|
||||
)
|
||||
_TOC_TITLE_PATTERN = re.compile(r"^(?:目\s*录|contents)$", re.IGNORECASE)
|
||||
_TOC_LEADER_ENTRY_PATTERN = re.compile(
|
||||
r"(?:[.．…·•]\s*){3,}\s*\d{1,4}\s*$"
|
||||
)
|
||||
_TOC_NUMBERED_ENTRY_PATTERN = re.compile(
|
||||
r"^(?:第[\u3400-\u4dbf\u4e00-\u9fff]{1,12}章|附表\s*\d+|\d+(?:\.\d+)+)"
|
||||
r"\s+.+\s+\d{1,4}\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_MARGIN_TEMPLATE_KEYWORDS = (
|
||||
"页",
|
||||
"页次",
|
||||
"版本",
|
||||
"文件编码",
|
||||
"秘密等级",
|
||||
"密级",
|
||||
"商密",
|
||||
"confidential",
|
||||
)
|
||||
|
||||
|
||||
def _pdf_page_lines(page: PdfPageText) -> tuple[_PdfLine, ...]:
|
||||
lines: list[_PdfLine] = []
|
||||
local_offset = 0
|
||||
for raw_line in page.text.splitlines(keepends=True):
|
||||
content = raw_line.rstrip("\r\n")
|
||||
leading = len(content) - len(content.lstrip())
|
||||
trailing = len(content.rstrip())
|
||||
text = content.strip()
|
||||
if text:
|
||||
lines.append(
|
||||
_PdfLine(
|
||||
text=text,
|
||||
start=page.source_start + local_offset + leading,
|
||||
end=page.source_start + local_offset + trailing,
|
||||
)
|
||||
)
|
||||
local_offset += len(raw_line)
|
||||
return tuple(lines)
|
||||
|
||||
|
||||
def _is_standalone_page_number(
|
||||
text: str,
|
||||
*,
|
||||
physical_page: int,
|
||||
page_count: int,
|
||||
) -> bool:
|
||||
normalized = unicodedata.normalize("NFKC", text).strip()
|
||||
match = _PDF_PAGE_NUMBER_LINE_PATTERN.fullmatch(normalized)
|
||||
if match is None:
|
||||
match = _PDF_FRACTION_PAGE_LINE_PATTERN.fullmatch(normalized)
|
||||
if match is None or int(match.group("page")) != physical_page:
|
||||
return False
|
||||
total = match.groupdict().get("total")
|
||||
return total is None or int(total) == page_count
|
||||
|
||||
|
||||
def _margin_signature(text: str) -> str:
|
||||
normalized = unicodedata.normalize("NFKC", text).casefold()
|
||||
normalized = re.sub(r"\s+", " ", normalized).strip()
|
||||
if any(keyword in normalized for keyword in _MARGIN_TEMPLATE_KEYWORDS):
|
||||
normalized = re.sub(r"\d+", "#", normalized)
|
||||
return normalized
|
||||
|
||||
|
||||
def _has_margin_metadata_keyword(text: str) -> bool:
|
||||
normalized = unicodedata.normalize("NFKC", text).casefold()
|
||||
return any(keyword in normalized for keyword in _MARGIN_TEMPLATE_KEYWORDS)
|
||||
|
||||
|
||||
def _has_meaningful_margin_signature(signature: str) -> bool:
|
||||
return len(re.sub(r"[#\W_]+", "", signature, flags=re.UNICODE)) >= 2
|
||||
|
||||
|
||||
def _is_toc_leader_entry(text: str) -> bool:
|
||||
return bool(_TOC_LEADER_ENTRY_PATTERN.search(text))
|
||||
|
||||
|
||||
def _is_toc_numbered_entry(text: str) -> bool:
|
||||
return bool(_TOC_NUMBERED_ENTRY_PATTERN.fullmatch(text))
|
||||
|
||||
|
||||
def detect_pdf_document_noise(
|
||||
pages: Sequence[PdfPageText],
|
||||
) -> tuple[DocumentNoiseSpan, ...]:
|
||||
"""识别 PDF 中的独立页码、重复页边内容和高置信目录。
|
||||
|
||||
规则只查看每页顶部 5 行和底部 3 行来推断页眉页脚；目录必须有
|
||||
明显的点引导线密度，避免仅因正文中出现“目录”或章节标题而误删。
|
||||
"""
|
||||
|
||||
page_lines = tuple(_pdf_page_lines(page) for page in pages)
|
||||
detected: dict[tuple[int, int], DocumentNoiseSpan] = {}
|
||||
|
||||
def mark(
|
||||
line: _PdfLine,
|
||||
kind: Literal["page_number", "repeated_margin", "table_of_contents"],
|
||||
) -> None:
|
||||
detected.setdefault(
|
||||
(line.start, line.end),
|
||||
DocumentNoiseSpan(
|
||||
start=line.start,
|
||||
end=line.end,
|
||||
kind=kind,
|
||||
),
|
||||
)
|
||||
|
||||
for page, lines in zip(pages, page_lines, strict=True):
|
||||
for line in lines:
|
||||
if _is_standalone_page_number(
|
||||
line.text,
|
||||
physical_page=page.page_number,
|
||||
page_count=len(pages),
|
||||
):
|
||||
mark(line, "page_number")
|
||||
outer_margin_lines = (*lines[:2], *lines[-2:])
|
||||
for line in outer_margin_lines:
|
||||
if _PDF_CLASSIFICATION_LABEL_PATTERN.fullmatch(line.text):
|
||||
mark(line, "repeated_margin")
|
||||
|
||||
# 只在三页及以上文档中推断通用页眉页脚，避免短文档误删。
|
||||
if len(pages) >= 3:
|
||||
signature_pages: dict[str, set[int]] = {}
|
||||
candidate_lines: list[tuple[int, _PdfLine, str]] = []
|
||||
for page_index, lines in enumerate(page_lines):
|
||||
boundary_lines = (
|
||||
*((line, index < 2) for index, line in enumerate(lines[:5])),
|
||||
*((line, index < 2) for index, line in enumerate(reversed(lines[-3:]))),
|
||||
)
|
||||
seen_ranges: set[tuple[int, int]] = set()
|
||||
for line, is_outer_margin in boundary_lines:
|
||||
line_range = (line.start, line.end)
|
||||
if (
|
||||
line_range in seen_ranges
|
||||
or line_range in detected
|
||||
or len(line.text) > 160
|
||||
):
|
||||
continue
|
||||
seen_ranges.add(line_range)
|
||||
if not is_outer_margin and not _has_margin_metadata_keyword(line.text):
|
||||
continue
|
||||
signature = _margin_signature(line.text)
|
||||
if not _has_meaningful_margin_signature(signature):
|
||||
continue
|
||||
signature_pages.setdefault(signature, set()).add(page_index)
|
||||
candidate_lines.append((page_index, line, signature))
|
||||
minimum_pages = max(3, math.ceil(len(pages) * 0.3))
|
||||
repeated_signatures = {
|
||||
signature
|
||||
for signature, matching_pages in signature_pages.items()
|
||||
if len(matching_pages) >= minimum_pages
|
||||
}
|
||||
for _, line, signature in candidate_lines:
|
||||
if signature in repeated_signatures:
|
||||
mark(line, "repeated_margin")
|
||||
|
||||
# 先依据强证据判定目录页，再补充删除少量不带点引导线的编号目录项。
|
||||
toc_active = False
|
||||
for lines in page_lines:
|
||||
content_lines = [
|
||||
line for line in lines if (line.start, line.end) not in detected
|
||||
]
|
||||
leader_entries = [line for line in content_lines if _is_toc_leader_entry(line.text)]
|
||||
titles = [line for line in content_lines if _TOC_TITLE_PATTERN.fullmatch(line.text)]
|
||||
starts_toc = bool(titles and len(leader_entries) >= 2)
|
||||
is_toc_dense = bool(
|
||||
len(leader_entries) >= 3
|
||||
and len(leader_entries) / max(1, len(content_lines)) >= 0.5
|
||||
)
|
||||
if not (starts_toc or (toc_active and is_toc_dense)):
|
||||
toc_active = False
|
||||
continue
|
||||
toc_active = True
|
||||
for line in content_lines:
|
||||
if (
|
||||
line in titles
|
||||
or _is_toc_leader_entry(line.text)
|
||||
or _is_toc_numbered_entry(line.text)
|
||||
):
|
||||
mark(line, "table_of_contents")
|
||||
|
||||
return tuple(sorted(detected.values(), key=lambda span: (span.start, span.end)))
|
||||
|
||||
|
||||
def remove_document_noise(
|
||||
text: str,
|
||||
spans: Sequence[DocumentNoiseSpan],
|
||||
*,
|
||||
source_offset: int = 0,
|
||||
) -> str:
|
||||
"""按原文绝对偏移移除噪声，不改动调用方保留的原文及偏移。"""
|
||||
|
||||
text_end = source_offset + len(text)
|
||||
intersections = sorted(
|
||||
(
|
||||
max(0, span.start - source_offset),
|
||||
min(len(text), span.end - source_offset),
|
||||
)
|
||||
for span in spans
|
||||
if span.start < text_end and span.end > source_offset
|
||||
)
|
||||
if not intersections:
|
||||
return text
|
||||
parts: list[str] = []
|
||||
cursor = 0
|
||||
for start, end in intersections:
|
||||
if end <= cursor:
|
||||
continue
|
||||
if start > cursor:
|
||||
parts.append(text[cursor:start])
|
||||
cursor = end
|
||||
parts.append(text[cursor:])
|
||||
cleaned = normalize_text("".join(parts))
|
||||
return re.sub(r"\n{3,}", "\n\n", cleaned)
|
||||
|
||||
|
||||
def _extract_pdf_text(raw: bytes) -> str:
|
||||
return "\n\n".join(page.text for page in extract_pdf_page_texts(raw) if page.text)
|
||||
|
||||
@@ -2737,6 +2980,7 @@ __all__ = [
|
||||
"ChunkMethod",
|
||||
"DatasetSplit",
|
||||
"DocumentHeading",
|
||||
"DocumentNoiseSpan",
|
||||
"DocumentStructure",
|
||||
"ParsedText",
|
||||
"PdfPageText",
|
||||
@@ -2752,6 +2996,7 @@ __all__ = [
|
||||
"desensitize_pii",
|
||||
"desensitize_structured_record",
|
||||
"detect_document_structure",
|
||||
"detect_pdf_document_noise",
|
||||
"detect_text_format",
|
||||
"deduplicate_structured_records",
|
||||
"estimate_token_count",
|
||||
@@ -2773,6 +3018,7 @@ __all__ = [
|
||||
"preprocess_structured_records",
|
||||
"protected_context_ranges",
|
||||
"record_fingerprint",
|
||||
"remove_document_noise",
|
||||
"score_quality",
|
||||
"stable_split",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user