#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ 优化版JSON文件合并脚本 根据字段英文名匹配逻辑模型表、物理模型表和元素治理模板表的数据 """ import json import os from collections import defaultdict from typing import Dict, List, Any def load_json_file(file_path: str) -> List[Dict[str, Any]]: """加载JSON文件""" try: with open(file_path, 'r', encoding='utf-8') as f: data = json.load(f) print(f"[OK] 加载文件: {os.path.basename(file_path)} - {len(data)} 条记录") return data except Exception as e: print(f"[ERROR] 加载文件失败 {file_path}: {e}") return [] def build_index(records: List[Dict], field_name: str) -> Dict[str, List[Dict]]: """为记录列表建立索引，加速查找""" index = defaultdict(list) for record in records: field_value = record.get(field_name) if field_value: index[field_value].append(record) print(f"[INFO] 建立索引完成: {len(index)} 个唯一字段值") return index def merge_records_optimized(logical_index: Dict, physical_index: Dict, element_records: List[Dict]) -> List[Dict]: """ 使用索引优化合并三个表的记录 """ merged_data = [] processed_fields = set() # 遍历元素治理表 print(f"\n[INFO] 开始合并数据...") for i, element_record in enumerate(element_records): if i % 5000 == 0: print(f" 处理进度: {i}/{len(element_records)}") field_english_name = element_record.get('字段英文名') if not field_english_name or field_english_name in processed_fields: continue processed_fields.add(field_english_name) # 创建合并记录 merged_record = {} # 添加元素治理模板表的数据 for key, value in element_record.items(): if key != '表名': merged_record[key] = value # 查找逻辑模型表中的匹配记录 logical_matches = logical_index.get(field_english_name, []) # 查找物理模型表中的匹配记录 physical_matches = physical_index.get(field_english_name, []) # 添加逻辑模型表的数据（添加前缀避免冲突） if logical_matches: for logical_match in logical_matches: for key, value in logical_match.items(): if key not in ['表名', '字段英文名']: new_key = f"逻辑模型_{key}" merged_record[new_key] = value # 只有当有匹配数据时才添加表名信息 merged_record['逻辑模型表_表名'] = '远光数据架构逻辑模型表' # 添加物理模型表的数据（添加前缀避免冲突） if physical_matches: for physical_match in physical_matches: for key, value in physical_match.items(): if key not in ['表名', '字段英文名']: new_key = f"物理模型_{key}" merged_record[new_key] = value # 只有当有匹配数据时才添加表名信息 merged_record['物理模型表_表名'] = '远光数据架构物理模型表' # 添加元素治理表名（始终存在） merged_record['元素治理表_表名'] = '远光数据架构元素治理模板表' merged_data.append(merged_record) print(f" 完成合并: {len(merged_data)} 条记录") return merged_data def add_unmatched_records_optimized(merged_data: List[Dict], logical_index: Dict, physical_index: Dict) -> List[Dict]: """ 添加未匹配的记录 """ print(f"\n[INFO] 处理未匹配的记录...") # 获取所有已处理的字段英文名 processed_fields = {record.get('字段英文名') for record in merged_data if record.get('字段英文名')} # 添加逻辑模型表中未匹配的记录 logical_unmatched = len(logical_index) - len([f for f in logical_index if f in processed_fields]) print(f" 逻辑模型表未匹配: {logical_unmatched} 条") for field_name, logical_matches in logical_index.items(): if field_name not in processed_fields: for logical_match in logical_matches: merged_record = {'字段英文名': field_name} for key, value in logical_match.items(): if key not in ['表名', '字段英文名']: merged_record[f"逻辑模型_{key}"] = value merged_record['逻辑模型表_表名'] = '远光数据架构逻辑模型表' merged_record['物理模型表_表名'] = None merged_record['元素治理表_表名'] = None merged_data.append(merged_record) # 添加物理模型表中未匹配的记录 physical_unmatched = len(physical_index) - len([f for f in physical_index if f in processed_fields]) print(f" 物理模型表未匹配: {physical_unmatched} 条") for field_name, physical_matches in physical_index.items(): if field_name not in processed_fields: # 检查是否已经添加过（通过逻辑模型表） already_added = any(r.get('字段英文名') == field_name for r in merged_data) if not already_added: for physical_match in physical_matches: merged_record = {'字段英文名': field_name} for key, value in physical_match.items(): if key not in ['表名', '字段英文名']: merged_record[f"物理模型_{key}"] = value merged_record['逻辑模型表_表名'] = None merged_record['物理模型表_表名'] = '远光数据架构物理模型表' merged_record['元素治理表_表名'] = None merged_data.append(merged_record) return merged_data def main(): """主函数""" print("="*60) print("优化版JSON文件合并工具") print("="*60) # 文件路径 logical_json_path = "Data_Export_Json/远光数据架构逻辑模型表.json" physical_json_path = "Data_Export_Json/远光数据架构物理模型表.json" element_json_path = "Data_Export_Json/远光数据架构元素治理模板表.json" output_path = "Data_Export_Json/final.json" # 加载JSON文件 print("\n[INFO] 加载JSON文件...") logical_records = load_json_file(logical_json_path) physical_records = load_json_file(physical_json_path) element_records = load_json_file(element_json_path) if not (logical_records and physical_records and element_records): print("\n[ERROR] 无法加载所有JSON文件") return # 建立索引 print(f"\n[INFO] 建立索引加速查找...") logical_index = build_index(logical_records, '字段英文名') physical_index = build_index(physical_records, '字段英文名') # 合并数据（只处理元素治理表中存在的字段） merged_data = merge_records_optimized(logical_index, physical_index, element_records) # 不再添加未匹配的记录，因为用户只关心元素治理表中的字段 # 保存合并后的数据 try: print(f"\n[INFO] 保存合并数据到 {output_path}...") with open(output_path, 'w', encoding='utf-8') as f: json.dump(merged_data, f, ensure_ascii=False, indent=2) file_size = os.path.getsize(output_path) / 1024 # KB print(f"\n[OK] 合并完成!") print(f" 输出文件: {output_path}") print(f" 合并记录: {len(merged_data)} 条") print(f" 文件大小: {file_size:.1f} KB") # 显示统计信息 three_table_match = sum(1 for r in merged_data if r.get('元素治理表_表名') and r.get('逻辑模型表_表名') and r.get('物理模型表_表名')) element_logical_match = sum(1 for r in merged_data if r.get('元素治理表_表名') and r.get('逻辑模型表_表名') and not r.get('物理模型表_表名')) element_physical_match = sum(1 for r in merged_data if r.get('元素治理表_表名') and r.get('物理模型表_表名') and not r.get('逻辑模型表_表名')) element_only_match = sum(1 for r in merged_data if r.get('元素治理表_表名') and not r.get('逻辑模型表_表名') and not r.get('物理模型表_表名')) logical_only_count = sum(1 for r in merged_data if r.get('逻辑模型表_表名') and not r.get('元素治理表_表名')) physical_only_count = sum(1 for r in merged_data if r.get('物理模型表_表名') and not r.get('元素治理表_表名')) print(f"\n[INFO] 统计信息:") print(f" 三表匹配: {three_table_match} 条") print(f" 元素治理+逻辑模型: {element_logical_match} 条") print(f" 元素治理+物理模型: {element_physical_match} 条") print(f" 仅元素治理: {element_only_match} 条") print(f" 仅逻辑模型: {logical_only_count} 条") print(f" 仅物理模型: {physical_only_count} 条") # 显示前3条记录的字段名 if merged_data: print(f"\n[INFO] 合并记录示例:") sample_record = merged_data[0] print(f" 字段数量: {len(sample_record)}") print(f" 字段名: {list(sample_record.keys())[:10]}...") # 只显示前10个字段 except Exception as e: print(f"\n[ERROR] 保存文件失败: {e}") import traceback traceback.print_exc() if __name__ == "__main__": main()