caoxiaozhu/X-Agents
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 完善知识库数据模型和服务

Browse Source 
- 添加知识库更多字段配置
- 优化知识库服务逻辑
- 添加文档解析相关接口

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
DESKTOP-72TV0V4\caoxiaozhu
2026-03-08 23:02:09 +08:00
parent 7a2f6dd30a
commit bb04c4afd0
4 changed files with 137 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/cmd/api/main.go
View File

@@ -98,7 +98,7 @@ func main() {
knowledgeHandler := handler.NewKnowledgeHandler(knowledgeService)
var uploadHandler *handler.UploadHandler
if uploadService != nil {
uploadHandler = handler.NewUploadHandler(uploadService)
uploadHandler = handler.NewUploadHandler(uploadService, knowledgeRepo)
}
// 7. 设置路由
@@ -216,6 +216,8 @@ func main() {
// 上传路由
r.POST("/api/file_upload", uploadHandler.Upload)
r.DELETE("/api/file_upload/:filename", uploadHandler.Delete)
// 文件代理路由（解决 MinIO 内网和 HTTPS 问题）
r.GET("/api/file_proxy", uploadHandler.ProxyFile)
}
// 8. 启动服务

4
server/internal/handler/knowledge_handler.go
View File

@@ -147,6 +147,10 @@ func (h *KnowledgeHandler) UploadDocument(c *gin.Context) {
c.JSON(http.StatusInternalServerError, gin.H{"success": false, "message": err.Error()})
return
}
if doc == nil {
c.JSON(http.StatusInternalServerError, gin.H{"success": false, "message": "Upload failed"})
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,

74
server/internal/model/knowledge_info.go
View File

@@ -32,19 +32,47 @@ func (p ParsingConfig) Value() (driver.Value, error) {
return json.Marshal(p)
}
// StorageConfig 存储配置
type StorageConfig struct {
Type string `json:"type"` // local / minio / s3
Endpoint string `json:"endpoint"` // MinIO/S3 endpoint
Bucket string `json:"bucket"` // MinIO/S3 bucket
AccessKeyID string `json:"access_key_id"` // MinIO/S3 access key
SecretAccessKey string `json:"secret_access_key"` // MinIO/S3 secret key
UseSSL bool `json:"use_ssl"` // MinIO/S3 use SSL
}
// Scan 实现 sql.Scanner 接口
func (s *StorageConfig) Scan(value interface{}) error {
if value == nil {
return nil
}
bytes, ok := value.([]byte)
if !ok {
return errors.New("type assertion to []byte failed")
}
return json.Unmarshal(bytes, s)
}
// Value 实现 driver.Valuer 接口
func (s StorageConfig) Value() (driver.Value, error) {
return json.Marshal(s)
}
// KnowledgeBase 知识库
type KnowledgeBase struct {
ID string `json:"id" gorm:"primaryKey;type:varchar(36)"`
Name string `json:"name" gorm:"type:varchar(255);not null"`
Description string `json:"description" gorm:"type:text"`
LLMModelID string `json:"llm_model_id" gorm:"type:varchar(36);not null"`
EmbeddingModelID string `json:"embedding_model_id" gorm:"type:varchar(36);not null"`
ParsingConfig ParsingConfig `json:"parsing_config" gorm:"type:json"`
Status string `json:"status" gorm:"type:varchar(20);default:active"` // active / inactive
DocumentCount int `json:"document_count" gorm:"default:0"`
ChunkCount int `json:"chunk_count" gorm:"default:0"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ID string `json:"id" gorm:"primaryKey;type:varchar(36)"`
Name string `json:"name" gorm:"type:varchar(255);not null"`
Description string `json:"description" gorm:"type:text"`
LLMModelID string `json:"llm_model_id" gorm:"type:varchar(36);not null"`
EmbeddingModelID string `json:"embedding_model_id" gorm:"type:varchar(36);not null"`
ParsingConfig ParsingConfig `json:"parsing_config" gorm:"type:json"`
StorageConfig StorageConfig `json:"storage_config" gorm:"type:json"` // 存储配置
Status string `json:"status" gorm:"type:varchar(20);default:active"` // active / inactive
DocumentCount int `json:"document_count" gorm:"default:0"`
ChunkCount int `json:"chunk_count" gorm:"default:0"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
func (KnowledgeBase) TableName() string {
@@ -72,21 +100,23 @@ func (KnowledgeDocument) TableName() string {
// CreateKnowledgeRequest 创建知识库请求
type CreateKnowledgeRequest struct {
Name string `json:"name" binding:"required"`
Description string `json:"description"`
LLMModelID string `json:"llm_model_id" binding:"required"`
EmbeddingModelID string `json:"embedding_model_id" binding:"required"`
ParsingConfig ParsingConfig `json:"parsing_config" binding:"required"`
Name string `json:"name" binding:"required"`
Description string `json:"description"`
LLMModelID string `json:"llm_model_id" binding:"required"`
EmbeddingModelID string `json:"embedding_model_id" binding:"required"`
ParsingConfig ParsingConfig `json:"parsing_config" binding:"required"`
StorageConfig StorageConfig `json:"storage_config"` // 存储配置，不传则使用全局配置
}
// UpdateKnowledgeRequest 更新知识库请求
type UpdateKnowledgeRequest struct {
Name string `json:"name"`
Description string `json:"description"`
LLMModelID string `json:"llm_model_id"`
EmbeddingModelID string `json:"embedding_model_id"`
ParsingConfig ParsingConfig `json:"parsing_config"`
Status string `json:"status"`
Name string `json:"name"`
Description string `json:"description"`
LLMModelID string `json:"llm_model_id"`
EmbeddingModelID string `json:"embedding_model_id"`
ParsingConfig ParsingConfig `json:"parsing_config"`
StorageConfig StorageConfig `json:"storage_config"`
Status string `json:"status"`
}
// KnowledgeListResponse 知识库列表响应

82
server/internal/service/knowledge_service.go
View File

@@ -3,8 +3,11 @@ package service
import (
"bytes"
"encoding/json"
"log"
"mime/multipart"
"net/http"
"os"
"strings"
"time"
"github.com/google/uuid"
@@ -12,6 +15,14 @@ import (
"x-agents/server/internal/repository"
)
// debugLog 专用调试日志
var knowledgeDebugLog *log.Logger
func init() {
debugFile, _ := os.OpenFile("logs/debug.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
knowledgeDebugLog = log.New(debugFile, "", log.Ldate|log.Ltime)
}
type KnowledgeService struct {
repo *repository.KnowledgeRepository
modelRepo *repository.ModelRepository
@@ -46,6 +57,7 @@ func (s *KnowledgeService) Create(req model.CreateKnowledgeRequest) (*model.Know
LLMModelID: req.LLMModelID,
EmbeddingModelID: req.EmbeddingModelID,
ParsingConfig: req.ParsingConfig,
StorageConfig: req.StorageConfig,
Status: "active",
DocumentCount: 0,
ChunkCount: 0,
@@ -104,6 +116,9 @@ func (s *KnowledgeService) Update(id string, req model.UpdateKnowledgeRequest) e
if req.ParsingConfig.Engine != "" {
updates["parsing_config"] = req.ParsingConfig
}
if req.StorageConfig.Type != "" {
updates["storage_config"] = req.StorageConfig
}
if req.Status != "" {
updates["status"] = req.Status
}
@@ -122,29 +137,72 @@ func (s *KnowledgeService) Delete(id string) error {
// ListDocuments 获取知识库下的文档列表
func (s *KnowledgeService) ListDocuments(kbID string, status string) ([]model.KnowledgeDocument, error) {
return s.repo.FindDocumentsByKBID(kbID, status)
docs, err := s.repo.FindDocumentsByKBID(kbID, status)
if err != nil {
knowledgeDebugLog.Printf("[Knowledge ListDocuments] 错误: %v", err)
return nil, err
}
knowledgeDebugLog.Printf("[Knowledge ListDocuments] kbID=%s, count=%d", kbID, len(docs))
for i, doc := range docs {
knowledgeDebugLog.Printf("[Knowledge ListDocuments] doc[%d]: id=%s, name=%s, file_url=%q", i, doc.ID, doc.Name, doc.FileURL)
// 如果是 MinIO 内网地址，转换为代理 URL
if doc.FileURL != "" && (strings.Contains(doc.FileURL, "10.10.10.189") || strings.Contains(doc.FileURL, ":9768")) {
// 从 URL 中提取 fileKey（包含扩展名）
parts := strings.Split(doc.FileURL, "/")
if len(parts) > 0 {
fileName := parts[len(parts)-1]
fileKey := strings.Split(fileName, "?")[0]
doc.FileURL = "/api/file_proxy?kb_id=" + kbID + "&key=" + fileKey
}
}
}
return docs, nil
}
// UploadDocument 上传文档到知识库
func (s *KnowledgeService) UploadDocument(kbID string, file *multipart.FileHeader) (*model.KnowledgeDocument, string, error) {
knowledgeDebugLog.Printf("[Knowledge Upload] 开始上传文件: kbID=%s, filename=%s, size=%d", kbID, file.Filename, file.Size)
// 验证知识库存在
kb, err := s.repo.FindByID(kbID)
if err != nil {
knowledgeDebugLog.Printf("[Knowledge Upload] 错误: 知识库不存在, kbID=%s, err=%v", kbID, err)
return nil, "", err
}
// 上传文件
result, err := s.uploadService.Upload(file)
knowledgeDebugLog.Printf("[Knowledge Upload] 知识库配置: kbID=%s, storage_config.Type=%q, storage_config=%+v",
kbID, kb.StorageConfig.Type, kb.StorageConfig)
// 上传文件（根据知识库的 storage_config 选择存储方式）
var result *UploadResponse
if kb.StorageConfig.Type != "" {
// 使用知识库的存储配置
knowledgeDebugLog.Printf("[Knowledge Upload] 使用知识库存储配置: type=%s, endpoint=%s, bucket=%s",
kb.StorageConfig.Type, kb.StorageConfig.Endpoint, kb.StorageConfig.Bucket)
result, err = s.uploadService.UploadWithConfig(file, kb.StorageConfig)
} else {
// 使用全局配置
knowledgeDebugLog.Printf("[Knowledge Upload] 使用全局存储配置")
result, err = s.uploadService.Upload(file)
}
if err != nil {
knowledgeDebugLog.Printf("[Knowledge Upload] 错误: 上传失败, err=%v", err)
return nil, "", err
}
knowledgeDebugLog.Printf("[Knowledge Upload] 上传结果: success=%v, url=%s, message=%s",
result.Success, result.URL, result.Message)
if !result.Success {
knowledgeDebugLog.Printf("[Knowledge Upload] 错误: 上传返回失败, message=%s", result.Message)
return nil, "", nil
}
// 获取文件扩展名
ext := getFileExt(file.Filename)
knowledgeDebugLog.Printf("[Knowledge Upload] 准备创建文档记录: result.URL=%q, fileKey=%s, ext=%s",
result.URL, result.FileKey, ext)
// 创建文档记录
doc := &model.KnowledgeDocument{
ID: uuid.New().String(),
@@ -157,6 +215,8 @@ func (s *KnowledgeService) UploadDocument(kbID string, file *multipart.FileHeade
UploadedAt: time.Now(),
}
knowledgeDebugLog.Printf("[Knowledge Upload] 文档创建完成: doc.FileURL=%q", doc.FileURL)
if err := s.repo.CreateDocument(doc); err != nil {
return nil, "", err
}
@@ -235,9 +295,23 @@ func (s *KnowledgeService) DeleteDocument(kbID, docID string) error {
return nil
}
// 获取知识库配置
kb, err := s.repo.FindByID(kbID)
if err != nil {
return err
}
// 删除文件
if doc.FileKey != "" {
s.uploadService.DeleteFile(doc.FileKey)
knowledgeDebugLog.Printf("[Knowledge DeleteDocument] 删除文件: kbID=%s, docID=%s, fileKey=%s, storageType=%s",
kbID, docID, doc.FileKey, kb.StorageConfig.Type)
if kb.StorageConfig.Type != "" {
// 使用知识库的存储配置删除
s.uploadService.DeleteFileWithConfig(doc.FileKey, kb.StorageConfig)
} else {
// 使用全局配置删除
s.uploadService.DeleteFile(doc.FileKey)
}
}
// 删除文档记录