209 lines
6.8 KiB
Python
209 lines
6.8 KiB
Python
|
"""
|
|||
|
|
VLM 客户端 - 用于调用 VLM 模型进行文档理解
|
|||
|
|
"""
|
|||
|
|
import logging
|
|||
|
|
import base64
|
|||
|
|
import requests
|
|||
|
|
from typing import Optional, Dict, Any
|
|||
|
|
|
|||
|
|
logger = logging.getLogger(__name__)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class VLMClient:
|
|||
|
|
"""VLM 客户端，支持多种提供商"""
|
|||
|
|
|
|||
|
|
def __init__(self, config: Dict[str, Any]):
|
|||
|
|
"""
|
|||
|
|
初始化 VLM 客户端
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
config: VLM 配置，包含 provider, model, api_key, base_url, prompt 等
|
|||
|
|
"""
|
|||
|
|
self.config = config
|
|||
|
|
self.provider = config.get("provider", "openai")
|
|||
|
|
self.model = config.get("model", "gpt-4o")
|
|||
|
|
self.api_key = config.get("api_key", "")
|
|||
|
|
self.base_url = config.get("base_url", "")
|
|||
|
|
self.prompt = config.get("prompt", "") or self._default_prompt()
|
|||
|
|
|
|||
|
|
logger.info(f"VLMClient initialized: provider={self.provider}, model={self.model}")
|
|||
|
|
|
|||
|
|
def _default_prompt(self) -> str:
|
|||
|
|
"""默认提示词"""
|
|||
|
|
return """请分析这张图片中的文档内容，并将其转换为 Markdown 格式。
|
|||
|
|
要求：
|
|||
|
|
1. 保持原文的格式和结构
|
|||
|
|
2. 表格用 Markdown 表格格式
|
|||
|
|
3. 标题用 # ## ### 标记
|
|||
|
|
4. 代码块用 ``` 标记
|
|||
|
|
5. 尽量保留原文的所有信息"""
|
|||
|
|
|
|||
|
|
def analyze_image(self, image_data: bytes, mime_type: str = "image/png") -> Dict[str, Any]:
|
|||
|
|
"""
|
|||
|
|
使用 VLM 分析图片
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
image_data: 图片二进制数据
|
|||
|
|
mime_type: 图片 MIME 类型
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
包含分析结果的字典
|
|||
|
|
"""
|
|||
|
|
if self.provider == "openai":
|
|||
|
|
return self._call_openai(image_data, mime_type)
|
|||
|
|
elif self.provider == "anthropic":
|
|||
|
|
return self._call_anthropic(image_data, mime_type)
|
|||
|
|
elif self.provider == "qwen":
|
|||
|
|
return self._call_qwen(image_data, mime_type)
|
|||
|
|
else:
|
|||
|
|
return {
|
|||
|
|
"success": False,
|
|||
|
|
"content": "",
|
|||
|
|
"error": f"Unsupported provider: {self.provider}"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def _call_openai(self, image_data: bytes, mime_type: str) -> Dict[str, Any]:
|
|||
|
|
"""调用 OpenAI GPT-4o API"""
|
|||
|
|
try:
|
|||
|
|
url = (self.base_url or "https://api.openai.com/v1") + "/chat/completions"
|
|||
|
|
|
|||
|
|
# Base64 编码图片
|
|||
|
|
image_base64 = base64.b64encode(image_data).decode("utf-8")
|
|||
|
|
data_url = f"data:{mime_type};base64,{image_base64}"
|
|||
|
|
|
|||
|
|
headers = {
|
|||
|
|
"Authorization": f"Bearer {self.api_key}",
|
|||
|
|
"Content-Type": "application/json"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
payload = {
|
|||
|
|
"model": self.model,
|
|||
|
|
"messages": [
|
|||
|
|
{
|
|||
|
|
"role": "user",
|
|||
|
|
"content": [
|
|||
|
|
{"type": "text", "text": self.prompt},
|
|||
|
|
{"type": "image_url", "image_url": {"url": data_url}}
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"max_tokens": 4096
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
response = requests.post(url, headers=headers, json=payload, timeout=120)
|
|||
|
|
response.raise_for_status()
|
|||
|
|
|
|||
|
|
result = response.json()
|
|||
|
|
content = result["choices"][0]["message"]["content"]
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"success": True,
|
|||
|
|
"content": content,
|
|||
|
|
"usage": result.get("usage", {})
|
|||
|
|
}
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"OpenAI API error: {e}")
|
|||
|
|
return {
|
|||
|
|
"success": False,
|
|||
|
|
"content": "",
|
|||
|
|
"error": str(e)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def _call_anthropic(self, image_data: bytes, mime_type: str) -> Dict[str, Any]:
|
|||
|
|
"""调用 Anthropic Claude API"""
|
|||
|
|
try:
|
|||
|
|
url = (self.base_url or "https://api.anthropic.com/v1") + "/messages"
|
|||
|
|
|
|||
|
|
image_base64 = base64.b64encode(image_data).decode("utf-8")
|
|||
|
|
|
|||
|
|
headers = {
|
|||
|
|
"x-api-key": self.api_key,
|
|||
|
|
"anthropic-version": "2023-06-01",
|
|||
|
|
"Content-Type": "application/json"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Anthropic 支持 image 类型
|
|||
|
|
payload = {
|
|||
|
|
"model": self.model,
|
|||
|
|
"max_tokens": 4096,
|
|||
|
|
"messages": [
|
|||
|
|
{
|
|||
|
|
"role": "user",
|
|||
|
|
"content": [
|
|||
|
|
{"type": "text", "text": self.prompt},
|
|||
|
|
{
|
|||
|
|
"type": "image",
|
|||
|
|
"source": {
|
|||
|
|
"type": "base64",
|
|||
|
|
"media_type": mime_type,
|
|||
|
|
"data": image_base64
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
response = requests.post(url, headers=headers, json=payload, timeout=120)
|
|||
|
|
response.raise_for_status()
|
|||
|
|
|
|||
|
|
result = response.json()
|
|||
|
|
content = result["content"][0]["text"]
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"success": True,
|
|||
|
|
"content": content,
|
|||
|
|
"usage": result.get("usage", {})
|
|||
|
|
}
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"Anthropic API error: {e}")
|
|||
|
|
return {
|
|||
|
|
"success": False,
|
|||
|
|
"content": "",
|
|||
|
|
"error": str(e)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def _call_qwen(self, image_data: bytes, mime_type: str) -> Dict[str, Any]:
|
|||
|
|
"""调用阿里 Qwen VL API"""
|
|||
|
|
try:
|
|||
|
|
url = (self.base_url or "https://dashscope.aliyuncs.com/compatible-mode/v1") + "/chat/completions"
|
|||
|
|
|
|||
|
|
image_base64 = base64.b64encode(image_data).decode("utf-8")
|
|||
|
|
|
|||
|
|
headers = {
|
|||
|
|
"Authorization": f"Bearer {self.api_key}",
|
|||
|
|
"Content-Type": "application/json"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Qwen 格式
|
|||
|
|
payload = {
|
|||
|
|
"model": self.model,
|
|||
|
|
"messages": [
|
|||
|
|
{
|
|||
|
|
"role": "user",
|
|||
|
|
"content": [
|
|||
|
|
{"type": "text", "text": self.prompt},
|
|||
|
|
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{image_base64}"}}
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
response = requests.post(url, headers=headers, json=payload, timeout=120)
|
|||
|
|
response.raise_for_status()
|
|||
|
|
|
|||
|
|
result = response.json()
|
|||
|
|
content = result["choices"][0]["message"]["content"]
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"success": True,
|
|||
|
|
"content": content,
|
|||
|
|
"usage": {}
|
|||
|
|
}
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"Qwen API error: {e}")
|
|||
|
|
return {
|
|||
|
|
"success": False,
|
|||
|
|
"content": "",
|
|||
|
|
"error": str(e)
|
|||
|
|
}
|