From b4af042c640cc0e710f7c4121b6375c50210d54e Mon Sep 17 00:00:00 2001 From: xiaoxin <718949661@qq.com> Date: Thu, 15 Aug 2024 11:28:02 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E3=80=90=E6=96=B0=E5=A2=9E=E3=80=91AI=20?= =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../module/ai/enums/ErrorCodeConstants.java | 4 ++ .../knowledge/AiKnowledgeController.java | 39 ++++++++++ .../vo/AiKnowledgeCreateMyReqVO.java | 31 ++++++++ .../vo/AiKnowledgeUpdateMyReqVO.java | 36 ++++++++++ .../knowledge/AiKnowledgeBaseDO.java | 56 +++++++++++++++ .../knowledge/AiKnowledgeDocumentDO.java | 55 ++++++++++++++ .../knowledge/AiKnowledgeSegmentDO.java | 48 +++++++++++++ .../knowledge/AiKnowledgeBaseMapper.java | 12 ++++ .../knowledge/AiKnowledgeDocumentMapper.java | 12 ++++ .../knowledge/AiKnowledgeSegmentMapper.java | 12 ++++ .../service/knowledge/AiEmbeddingService.java | 26 +++++++ ...eImpl.java => AiEmbeddingServiceImpl.java} | 12 ++-- .../knowledge/AiKnowledgeBaseService.java | 30 ++++++++ .../knowledge/AiKnowledgeBaseServiceImpl.java | 72 +++++++++++++++++++ .../knowledge/AiKnowledgeDocumentService.java | 10 +++ .../AiKnowledgeDocumentServiceImpl.java | 16 +++++ .../knowledge/AiKnowledgeSegmentService.java | 11 +++ .../AiKnowledgeSegmentServiceImpl.java | 16 +++++ .../ai/service/knowledge/DocService.java | 15 ---- .../yudao-spring-boot-starter-ai/pom.xml | 6 +- .../RedisVectorStoreAutoConfiguration.java | 1 + 21 files changed, 497 insertions(+), 23 deletions(-) create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeUpdateMyReqVO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java rename yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/{DocServiceImpl.java => AiEmbeddingServiceImpl.java} (76%) create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java delete mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java diff --git a/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java index ddfb489f3..b68591796 100644 --- a/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java +++ b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java @@ -50,4 +50,8 @@ public interface ErrorCodeConstants { ErrorCode WRITE_NOT_EXISTS = new ErrorCode(1_022_007_000, "作文不存在!"); ErrorCode WRITE_STREAM_ERROR = new ErrorCode(1_022_07_001, "写作生成异常!"); + + // ========== API 知识库 1-022-008-000 ========== + ErrorCode KNOWLEDGE_NOT_EXISTS = new ErrorCode(1_022_008_000, "知识库不存在!"); + } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java new file mode 100644 index 000000000..9d9c99a9a --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java @@ -0,0 +1,39 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge; + +import cn.iocoder.yudao.framework.common.pojo.CommonResult; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; +import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeBaseService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.annotation.Resource; +import jakarta.validation.Valid; +import org.springframework.web.bind.annotation.*; + +import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success; +import static cn.iocoder.yudao.framework.security.core.util.SecurityFrameworkUtils.getLoginUserId; + +@Tag(name = "管理后台 - AI 知识库") +@RestController +@RequestMapping("/ai/knowledge") +public class AiKnowledgeController { + + @Resource + private AiKnowledgeBaseService knowledgeBaseService; + + @PostMapping("/create-my") + @Operation(summary = "创建【我的】知识库") + public CommonResult createKnowledgeMy(@RequestBody @Valid AiKnowledgeCreateMyReqVO createReqVO) { + return success(knowledgeBaseService.createKnowledgeMy(createReqVO, getLoginUserId())); + } + + + @PutMapping("/update-my") + @Operation(summary = "更新【我的】知识库") + public CommonResult updateKnowledgeMy(@RequestBody @Valid AiKnowledgeUpdateMyReqVO updateReqVO) { + knowledgeBaseService.updateKnowledgeMy(updateReqVO, getLoginUserId()); + return success(true); + } + + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java new file mode 100644 index 000000000..817fcaabb --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java @@ -0,0 +1,31 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +import java.util.List; + +/** + * @author xiaoxin + */ +@Schema(description = "管理后台 - AI 知识库创建【我的】 Request VO") +@Data +public class AiKnowledgeCreateMyReqVO { + + @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "") + @NotBlank(message = "知识库名称不能为空") + private String name; + + @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "") + private String description; + + @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]") + private List visibilityPermissions; + + @Schema(description = "嵌入模型 id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") + @NotNull(message = "嵌入模型不能为空") + private Long modelId; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeUpdateMyReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeUpdateMyReqVO.java new file mode 100644 index 000000000..2bc39d5db --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeUpdateMyReqVO.java @@ -0,0 +1,36 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +import java.util.List; + +/** + * @author xiaoxin + */ +@Schema(description = "管理后台 - AI 知识库创建【我的】 Request VO") +@Data +public class AiKnowledgeUpdateMyReqVO { + + + @Schema(description = "对话编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204") + @NotNull(message = "知识库编号不能为空") + private Long id; + + @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "") + @NotBlank(message = "知识库名称不能为空") + private String name; + + @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "") + private String description; + + @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]") + private List visibilityPermissions; + + @Schema(description = "嵌入模型 id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") + @NotNull(message = "嵌入模型不能为空") + private Long modelId; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java new file mode 100644 index 000000000..3bf45d348 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java @@ -0,0 +1,56 @@ +package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; + + +import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.util.List; + +/** + * AI 知识库 DO + * + * @author xiaoxin + */ +@TableName(value = "ai_knowledge_base") +@Data +public class AiKnowledgeBaseDO extends BaseDO { + + /** + * 编号 + */ + @TableId(type = IdType.AUTO) + private Long id; + /** + * 用户编号 + *

+ * 关联 AdminUserDO 的 userId 字段 + */ + private Long userId; + /** + * 知识库名称 + */ + private String name; + /** + * 知识库描述 + */ + private String description; + /** + * 可见权限,只能选择哪些人可见 + */ + private List visibilityPermissions; + /** + * 嵌入模型编号，高质量模式时维护 + */ + private Long modelId; + /** + * 模型标识 + */ + private String model; + /** + * 是否启用 + */ + private Boolean status; +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java new file mode 100644 index 000000000..8de3697a7 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java @@ -0,0 +1,55 @@ +package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; + +import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +/** + * AI 知识库-文档 DO + * + * @author xiaoxin + */ +@TableName(value = "ai_knowledge_document") +@Data +public class AiKnowledgeDocumentDO extends BaseDO { + + /** + * 编号 + */ + @TableId(type = IdType.AUTO) + private Long id; + /** + * 知识库编号 + */ + private Long knowledgeId; + /** + * 文件名称 + */ + private String name; + /** + * 内容 + */ + private String content; + /** + * 文件 URL + */ + private String url; + /** + * token数量 + */ + private Integer tokens; + /** + * 字符数 + */ + private Integer wordCount; + /** + * 切片状态 + */ + private Integer sliceStatus; + /** + * 是否启用 + */ + private Boolean status; +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java new file mode 100644 index 000000000..4ce3bb4ee --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java @@ -0,0 +1,48 @@ +package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; + +import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +/** + * AI 知识库-文档分段 DO + * + * @author xiaoxin + */ +@TableName(value = "ai_knowledge_segment") +@Data +public class AiKnowledgeSegmentDO extends BaseDO { + + /** + * 编号 + */ + @TableId(type = IdType.AUTO) + private Long id; + /** + * 向量库的id + */ + private String vectorId; + /** + * 文档编号 + */ + private Long documentId; + /** + * 切片内容 + */ + private String content; + /** + * 字符数 + */ + private Integer wordCount; + /** + * token数量 + */ + private Integer tokens; + /** + * 是否启用 + */ + private Boolean status; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java new file mode 100644 index 000000000..3a23aa55d --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java @@ -0,0 +1,12 @@ +package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; + +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO; + +/** + * AI 知识库基础信息 Mapper + * + * @author xiaoxin + */ +public interface AiKnowledgeBaseMapper extends BaseMapperX { +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java new file mode 100644 index 000000000..35532956f --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java @@ -0,0 +1,12 @@ +package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; + +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; + +/** + * AI 知识库-文档 Mapper + * + * @author xiaoxin + */ +public interface AiKnowledgeDocumentMapper extends BaseMapperX { +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java new file mode 100644 index 000000000..c3cbca2c1 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java @@ -0,0 +1,12 @@ +package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; + +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; + +/** + * AI 知识库-分片 Mapper + * + * @author xiaoxin + */ +public interface AiKnowledgeSegmentMapper extends BaseMapperX { +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java new file mode 100644 index 000000000..38ff0d022 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java @@ -0,0 +1,26 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import org.springframework.ai.document.Document; + +import java.util.List; + +/** + * AI 嵌入 Service 接口 + * + * @author xiaoxin + */ +public interface AiEmbeddingService { + + /** + * 向量化文档 + */ + void embeddingDoc(); + + + /** + * 相似查询 + * + * @param content 查询内容 + */ + List similaritySearch(String content); +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java similarity index 76% rename from yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java rename to yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java index b0f4afaf8..37c414428 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java @@ -1,24 +1,23 @@ package cn.iocoder.yudao.module.ai.service.knowledge; import jakarta.annotation.Resource; -import lombok.extern.slf4j.Slf4j; import org.springframework.ai.document.Document; import org.springframework.ai.reader.tika.TikaDocumentReader; import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.ai.vectorstore.RedisVectorStore; +import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.util.List; /** - * AI 知识库 Service 实现类 + * AI 嵌入 Service 实现类 * * @author xiaoxin */ @Service -@Slf4j -public class DocServiceImpl implements DocService { +public class AiEmbeddingServiceImpl implements AiEmbeddingService { @Resource private RedisVectorStore vectorStore; @@ -40,4 +39,9 @@ public class DocServiceImpl implements DocService { vectorStore.add(segments); } + @Override + public List similaritySearch(String content) { + SearchRequest request = SearchRequest.query(content); + return vectorStore.similaritySearch(request); + } } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java new file mode 100644 index 000000000..7657ab748 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java @@ -0,0 +1,30 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; + +/** + * AI 知识库-基础信息 Service 接口 + * + * @author xiaoxin + */ +public interface AiKnowledgeBaseService { + + + /** + * 创建【我的】知识库 + * + * @param createReqVO 创建信息 + * @param userId 用户编号 + * @return 编号 + */ + Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId); + + + /** + * 创建【我的】知识库 + * + * @param updateReqVO 更新信息 + * @param userId 用户编号 + */ + void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId); +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java new file mode 100644 index 000000000..4f28726bd --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java @@ -0,0 +1,72 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import cn.hutool.core.lang.Assert; +import cn.hutool.core.util.ObjUtil; +import cn.iocoder.yudao.framework.common.util.object.BeanUtils; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO; +import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeBaseMapper; +import cn.iocoder.yudao.module.ai.service.model.AiChatModelService; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_EXISTS; + +/** + * AI 知识库-基础信息 Service 实现类 + * + * @author xiaoxin + */ +@Service +@Slf4j +public class AiKnowledgeBaseServiceImpl implements AiKnowledgeBaseService { + + @Resource + private AiKnowledgeBaseMapper knowledgeBaseMapper; + @Resource + private AiChatModelService chatModalService; + + @Override + public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) { + AiChatModelDO model = validateChatModel(createReqVO.getModelId()); + + AiKnowledgeBaseDO knowledgeBaseDO = BeanUtils.toBean(createReqVO, AiKnowledgeBaseDO.class); + knowledgeBaseDO.setModel(model.getModel()).setUserId(userId); + + knowledgeBaseMapper.insert(knowledgeBaseDO); + return knowledgeBaseDO.getId(); + } + + @Override + public void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId) { + + AiKnowledgeBaseDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId()); + if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) { + throw exception(KNOWLEDGE_NOT_EXISTS); + } + AiChatModelDO model = validateChatModel(updateReqVO.getModelId()); + AiKnowledgeBaseDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeBaseDO.class); + updateDO.setModel(model.getModel()); + + knowledgeBaseMapper.updateById(updateDO); + } + + + private AiChatModelDO validateChatModel(Long id) { + AiChatModelDO model = chatModalService.validateChatModel(id); + Assert.notNull(model, "未找到对应嵌入模型"); + return model; + } + + public AiKnowledgeBaseDO validateKnowledgeExists(Long id) { + AiKnowledgeBaseDO knowledgeBase = knowledgeBaseMapper.selectById(id); + if (knowledgeBase == null) { + throw exception(KNOWLEDGE_NOT_EXISTS); + } + return knowledgeBase; + } +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java new file mode 100644 index 000000000..5af45e5e8 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java @@ -0,0 +1,10 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +/** + * AI 知识库-文档 Service 接口 + * + * @author xiaoxin + */ +public interface AiKnowledgeDocumentService { + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java new file mode 100644 index 000000000..84ebb617e --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java @@ -0,0 +1,16 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +/** + * AI 知识库-文档 Service 实现类 + * + * @author xiaoxin + */ +@Service +@Slf4j +public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentService { + + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java new file mode 100644 index 000000000..003ce5c96 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java @@ -0,0 +1,11 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +/** + * AI 知识库-分片 Service 接口 + * + * @author xiaoxin + */ +public interface AiKnowledgeSegmentService { + + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java new file mode 100644 index 000000000..aa5facc36 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java @@ -0,0 +1,16 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +/** + * AI 知识库-基础信息 Service 实现类 + * + * @author xiaoxin + */ +@Service +@Slf4j +public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService { + + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java deleted file mode 100644 index 47905d4b1..000000000 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java +++ /dev/null @@ -1,15 +0,0 @@ -package cn.iocoder.yudao.module.ai.service.knowledge; - -/** - * AI 知识库 Service 接口 - * - * @author xiaoxin - */ -public interface DocService { - - /** - * 向量化文档 - */ - void embeddingDoc(); - -} diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml b/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml index ae1f37948..4585311c7 100644 --- a/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml @@ -57,11 +57,9 @@ ${spring-ai.version} - - org.springframework.data - spring-data-redis - true + cn.iocoder.boot + yudao-spring-boot-starter-redis diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java index 61c38dd1d..615b05f78 100644 --- a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java @@ -31,6 +31,7 @@ import redis.clients.jedis.JedisPooled; * TODO @xin 先拿 spring-ai 最新代码覆盖，1.0.0-M1 跟 redis 自动配置会冲突 * * TODO 这个官方，有说啥时候 fix 哇？ + * TODO 看着是列在1.0.0-M2版本 * * @author Christian Tzolov * @author Eddú Meléndez From 8e54eef8af762e77084c4d7742307afa2cdf212a Mon Sep 17 00:00:00 2001 From: xiaoxin <718949661@qq.com> Date: Thu, 15 Aug 2024 15:57:03 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E3=80=90=E6=96=B0=E5=A2=9E=E3=80=91AI=20?= =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93=EF=BC=9A=E6=96=87=E6=A1=A3=E5=88=87?= =?UTF-8?q?=E7=89=87=E5=90=91=E9=87=8F=E5=8C=96=E5=85=A5=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AiKnowledgeDocumentStatusEnum.java | 39 +++++++++++ .../vo/AiKnowledgeCreateMyReqVO.java | 4 +- .../vo/AiKnowledgeDocumentCreateReqVO.java | 27 ++++++++ .../knowledge/AiKnowledgeBaseDO.java | 12 +++- .../knowledge/AiKnowledgeDocumentDO.java | 11 ++- .../knowledge/AiKnowledgeSegmentDO.java | 7 +- .../knowledge/AiKnowledgeBaseMapper.java | 2 + .../knowledge/AiKnowledgeDocumentMapper.java | 2 + .../knowledge/AiKnowledgeSegmentMapper.java | 2 + .../service/knowledge/AiEmbeddingService.java | 7 +- .../knowledge/AiEmbeddingServiceImpl.java | 23 ++----- .../knowledge/AiKnowledgeBaseServiceImpl.java | 9 ++- .../knowledge/AiKnowledgeDocumentService.java | 11 +++ .../AiKnowledgeDocumentServiceImpl.java | 68 +++++++++++++++++++ 14 files changed, 190 insertions(+), 34 deletions(-) create mode 100644 yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeDocumentCreateReqVO.java diff --git a/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java new file mode 100644 index 000000000..a37fa8643 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java @@ -0,0 +1,39 @@ +package cn.iocoder.yudao.module.ai.enums.knowledge; + +import cn.iocoder.yudao.framework.common.core.IntArrayValuable; +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.Arrays; + +/** + * AI 知识库-文档状态的枚举 + * + * @author xiaoxin + */ +@AllArgsConstructor +@Getter +public enum AiKnowledgeDocumentStatusEnum implements IntArrayValuable { + + IN_PROGRESS(10, "索引中"), + SUCCESS(20, "可用"), + FAIL(30, "失败"); + + /** + * 状态 + */ + private final Integer status; + + /** + * 状态名 + */ + private final String name; + + public static final int[] ARRAYS = Arrays.stream(values()).mapToInt(AiKnowledgeDocumentStatusEnum::getStatus).toArray(); + + @Override + public int[] array() { + return ARRAYS; + } + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java index 817fcaabb..fa9161eba 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java @@ -14,11 +14,11 @@ import java.util.List; @Data public class AiKnowledgeCreateMyReqVO { - @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "") + @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "ruoyi-vue-pro 用户指南") @NotBlank(message = "知识库名称不能为空") private String name; - @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "") + @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "存储 ruoyi-vue-pro 操作文档") private String description; @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]") diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeDocumentCreateReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeDocumentCreateReqVO.java new file mode 100644 index 000000000..fa24eef72 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeDocumentCreateReqVO.java @@ -0,0 +1,27 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +/** + * @author xiaoxin + */ +@Schema(description = "管理后台 - AI 知识库【创建文档】 Request VO") +@Data +public class AiKnowledgeDocumentCreateReqVO { + + + @Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204") + @NotNull(message = "知识库编号不能为空") + private Long knowledgeId; + + @Schema(description = "文档名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "三方登陆") + @NotBlank(message = "文档名称不能为空") + private String name; + + @Schema(description = "文档 url", requiredMode = Schema.RequiredMode.REQUIRED, example = "https://doc.iocoder.cn") + private String url; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java index 3bf45d348..81cbf3ac9 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java @@ -1,10 +1,13 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler; import lombok.Data; import java.util.List; @@ -40,7 +43,8 @@ public class AiKnowledgeBaseDO extends BaseDO { /** * 可见权限,只能选择哪些人可见 */ - private List visibilityPermissions; + @TableField(typeHandler = JacksonTypeHandler.class) + private List visibilityPermissions; /** * 嵌入模型编号，高质量模式时维护 */ @@ -50,7 +54,9 @@ public class AiKnowledgeBaseDO extends BaseDO { */ private String model; /** - * 是否启用 + * 状态 + *

+ * 枚举 {@link CommonStatusEnum} */ - private Boolean status; + private Integer status; } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java index 8de3697a7..75d927cf0 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java @@ -1,6 +1,8 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; +import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; @@ -46,10 +48,15 @@ public class AiKnowledgeDocumentDO extends BaseDO { private Integer wordCount; /** * 切片状态 + *

+ * 枚举 {@link AiKnowledgeDocumentStatusEnum} */ private Integer sliceStatus; + /** - * 是否启用 + * 状态 + *

+ * 枚举 {@link CommonStatusEnum} */ - private Boolean status; + private Integer status; } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java index 4ce3bb4ee..657a3739b 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java @@ -1,5 +1,6 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; @@ -41,8 +42,10 @@ public class AiKnowledgeSegmentDO extends BaseDO { */ private Integer tokens; /** - * 是否启用 + * 状态 + *

+ * 枚举 {@link CommonStatusEnum} */ - private Boolean status; + private Integer status; } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java index 3a23aa55d..cad90fcfe 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java @@ -2,11 +2,13 @@ package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO; +import org.apache.ibatis.annotations.Mapper; /** * AI 知识库基础信息 Mapper * * @author xiaoxin */ +@Mapper public interface AiKnowledgeBaseMapper extends BaseMapperX { } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java index 35532956f..af55f545a 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java @@ -2,11 +2,13 @@ package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; +import org.apache.ibatis.annotations.Mapper; /** * AI 知识库-文档 Mapper * * @author xiaoxin */ +@Mapper public interface AiKnowledgeDocumentMapper extends BaseMapperX { } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java index c3cbca2c1..5043ee0ca 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java @@ -2,11 +2,13 @@ package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; +import org.apache.ibatis.annotations.Mapper; /** * AI 知识库-分片 Mapper * * @author xiaoxin */ +@Mapper public interface AiKnowledgeSegmentMapper extends BaseMapperX { } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java index 38ff0d022..9055cdc18 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java @@ -1,6 +1,7 @@ package cn.iocoder.yudao.module.ai.service.knowledge; import org.springframework.ai.document.Document; +import org.springframework.ai.vectorstore.SearchRequest; import java.util.List; @@ -12,9 +13,9 @@ import java.util.List; public interface AiEmbeddingService { /** - * 向量化文档 + * 向量化文档并存储 */ - void embeddingDoc(); + void add(List documents); /** @@ -22,5 +23,5 @@ public interface AiEmbeddingService { * * @param content 查询内容 */ - List similaritySearch(String content); + List similaritySearch(SearchRequest request); } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java index 37c414428..a2c3e819d 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java @@ -2,11 +2,9 @@ package cn.iocoder.yudao.module.ai.service.knowledge; import jakarta.annotation.Resource; import org.springframework.ai.document.Document; -import org.springframework.ai.reader.tika.TikaDocumentReader; -import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.ai.vectorstore.RedisVectorStore; import org.springframework.ai.vectorstore.SearchRequest; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import java.util.List; @@ -21,27 +19,14 @@ public class AiEmbeddingServiceImpl implements AiEmbeddingService { @Resource private RedisVectorStore vectorStore; - @Resource - private TokenTextSplitter tokenTextSplitter; - - // TODO @xin 临时测试用，后续删 - @Value("classpath:/webapp/test/Fel.pdf") - private org.springframework.core.io.Resource data; @Override - public void embeddingDoc() { - // 读取文件 - TikaDocumentReader loader = new TikaDocumentReader(data); - List documents = loader.get(); - // 文档分段 - List segments = tokenTextSplitter.apply(documents); - // 向量化并存储 - vectorStore.add(segments); + public void add(List documents) { + vectorStore.add(documents); } @Override - public List similaritySearch(String content) { - SearchRequest request = SearchRequest.query(content); + public List similaritySearch(SearchRequest request) { return vectorStore.similaritySearch(request); } } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java index 4f28726bd..63f4f53db 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java @@ -2,6 +2,7 @@ package cn.iocoder.yudao.module.ai.service.knowledge; import cn.hutool.core.lang.Assert; import cn.hutool.core.util.ObjUtil; +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; import cn.iocoder.yudao.framework.common.util.object.BeanUtils; import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; @@ -25,17 +26,19 @@ import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_ @Slf4j public class AiKnowledgeBaseServiceImpl implements AiKnowledgeBaseService { - @Resource - private AiKnowledgeBaseMapper knowledgeBaseMapper; @Resource private AiChatModelService chatModalService; + @Resource + private AiKnowledgeBaseMapper knowledgeBaseMapper; + + @Override public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) { AiChatModelDO model = validateChatModel(createReqVO.getModelId()); AiKnowledgeBaseDO knowledgeBaseDO = BeanUtils.toBean(createReqVO, AiKnowledgeBaseDO.class); - knowledgeBaseDO.setModel(model.getModel()).setUserId(userId); + knowledgeBaseDO.setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus()); knowledgeBaseMapper.insert(knowledgeBaseDO); return knowledgeBaseDO.getId(); diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java index 5af45e5e8..52c62abf7 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java @@ -1,5 +1,7 @@ package cn.iocoder.yudao.module.ai.service.knowledge; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeDocumentCreateReqVO; + /** * AI 知识库-文档 Service 接口 * @@ -7,4 +9,13 @@ package cn.iocoder.yudao.module.ai.service.knowledge; */ public interface AiKnowledgeDocumentService { + + /** + * 创建文档 + * + * @param createReqVO 文档创建 Request VO + * @return 文档编号 + */ + Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO); + } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java index 84ebb617e..caef4b802 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java @@ -1,7 +1,25 @@ package cn.iocoder.yudao.module.ai.service.knowledge; +import cn.hutool.core.collection.CollUtil; +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils; +import cn.iocoder.yudao.framework.common.util.object.BeanUtils; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeDocumentCreateReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper; +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper; +import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum; +import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.document.Document; +import org.springframework.ai.reader.tika.TikaDocumentReader; +import org.springframework.ai.transformer.splitter.TokenTextSplitter; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; /** * AI 知识库-文档 Service 实现类 @@ -12,5 +30,55 @@ import org.springframework.stereotype.Service; @Slf4j public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentService { + @Resource + private AiKnowledgeDocumentMapper documentMapper; + @Resource + private AiKnowledgeSegmentMapper segmentMapper; + @Resource + private TokenTextSplitter tokenTextSplitter; + + @Resource + private AiEmbeddingService embeddingService; + + // TODO @xin 临时测试用，后续删 + @Value("classpath:/webapp/test/Fel.pdf") + private org.springframework.core.io.Resource data; + + + @Override + @Transactional(rollbackFor = Exception.class) + public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) { + AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class); + documentDO + //todo + .setTokens(0).setWordCount(0) + .setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus()); + documentMapper.insert(documentDO); + + TikaDocumentReader loader = new TikaDocumentReader(data); + List documents = loader.get(); + Long documentId = documentDO.getId(); + if (CollUtil.isEmpty(documents)) { + log.info("文档内容为空"); + return documentId; + } + + // 文档分段 + List segments = tokenTextSplitter.apply(documents); + + List segmentDOList = CollectionUtils.convertList(segments, + segment -> new AiKnowledgeSegmentDO().setContent(segment.getContent()).setDocumentId(documentId) + //todo + .setTokens(0).setWordCount(0) + .setStatus(CommonStatusEnum.ENABLE.getStatus())); + + // 分段内容入库 + segmentMapper.insertBatch(segmentDOList); + + //向量化并存储 + embeddingService.add(segments); + + return documentId; + } } From 238f603f690ef7e747b177729237af1a008b3056 Mon Sep 17 00:00:00 2001 From: xiaoxin <718949661@qq.com> Date: Thu, 15 Aug 2024 16:50:16 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E3=80=90=E6=96=B0=E5=A2=9E=E3=80=91AI=20?= =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93=EF=BC=9A=E6=96=87=E6=A1=A3=20token?= =?UTF-8?q?=E3=80=81=E5=AD=97=E7=AC=A6=E6=95=B0=E8=AE=A1=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/knowledge/AiEmbeddingService.java | 2 +- .../knowledge/AiEmbeddingServiceImpl.java | 3 +- .../AiKnowledgeDocumentServiceImpl.java | 29 ++++++++++++------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java index 9055cdc18..eee2f8044 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java @@ -21,7 +21,7 @@ public interface AiEmbeddingService { /** * 相似查询 * - * @param content 查询内容 + * @param request 查询实体 */ List similaritySearch(SearchRequest request); } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java index a2c3e819d..2a6e75722 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java @@ -4,7 +4,6 @@ import jakarta.annotation.Resource; import org.springframework.ai.document.Document; import org.springframework.ai.vectorstore.RedisVectorStore; import org.springframework.ai.vectorstore.SearchRequest; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import java.util.List; @@ -21,6 +20,8 @@ public class AiEmbeddingServiceImpl implements AiEmbeddingService { private RedisVectorStore vectorStore; @Override +// @Async + // TODO xiaoxin 报错先注释 public void add(List documents) { vectorStore.add(documents); } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java index caef4b802..9ee5c4eed 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java @@ -14,12 +14,14 @@ import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.document.Document; import org.springframework.ai.reader.tika.TikaDocumentReader; +import org.springframework.ai.tokenizer.JTokkitTokenCountEstimator; import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.util.List; +import java.util.Objects; /** * AI 知识库-文档 Service 实现类 @@ -41,7 +43,9 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic @Resource private AiEmbeddingService embeddingService; - // TODO @xin 临时测试用，后续删 + private static final JTokkitTokenCountEstimator TOKEN_COUNT_ESTIMATOR = new JTokkitTokenCountEstimator(); + + // TODO xiaoxin 临时测试用，后续删 @Value("classpath:/webapp/test/Fel.pdf") private org.springframework.core.io.Resource data; @@ -49,18 +53,23 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic @Override @Transactional(rollbackFor = Exception.class) public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) { - AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class); - documentDO - //todo - .setTokens(0).setWordCount(0) - .setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus()); - documentMapper.insert(documentDO); + // TODO xiaoxin 后续从 url 加载 TikaDocumentReader loader = new TikaDocumentReader(data); + // 加载文档 List documents = loader.get(); + Document document = CollUtil.getFirst(documents); + // TODO 芋艿 文档层面有没有可能会比较大，这两个字段是否可以从分段表计算得出？ + Integer tokens = Objects.nonNull(document) ? TOKEN_COUNT_ESTIMATOR.estimate(document.getContent()) : 0; + Integer wordCount = Objects.nonNull(document) ? document.getContent().length() : 0; + + AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class); + documentDO.setTokens(tokens).setWordCount(wordCount) + .setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus()); + // 文档记录入库 + documentMapper.insert(documentDO); Long documentId = documentDO.getId(); if (CollUtil.isEmpty(documents)) { - log.info("文档内容为空"); return documentId; } @@ -69,10 +78,8 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic List segmentDOList = CollectionUtils.convertList(segments, segment -> new AiKnowledgeSegmentDO().setContent(segment.getContent()).setDocumentId(documentId) - //todo - .setTokens(0).setWordCount(0) + .setTokens(TOKEN_COUNT_ESTIMATOR.estimate(segment.getContent())).setWordCount(segment.getContent().length()) .setStatus(CommonStatusEnum.ENABLE.getStatus())); - // 分段内容入库 segmentMapper.insertBatch(segmentDOList);