From ed2296e4c772724004d92b0b0dc3b209cd70a78b Mon Sep 17 00:00:00 2001 From: xiaoxin <718949661@qq.com> Date: Wed, 28 Aug 2024 16:01:10 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90=E8=A7=A3=E5=86=B3todo=E3=80=91AI=20?= =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../knowledge/AiKnowledgeController.java | 4 +-- ...nowledgeBaseDO.java => AiKnowledgeDO.java} | 5 ++- .../knowledge/AiKnowledgeDocumentDO.java | 2 +- .../knowledge/AiKnowledgeBaseMapper.java | 4 +-- .../service/knowledge/AiEmbeddingService.java | 27 -------------- .../knowledge/AiEmbeddingServiceImpl.java | 35 ------------------- .../AiKnowledgeDocumentServiceImpl.java | 24 ++++++------- ...seService.java => AiKnowledgeService.java} | 2 +- ...eImpl.java => AiKnowledgeServiceImpl.java} | 31 ++++++---------- .../ai/config/YudaoAiAutoConfiguration.java | 10 +++++- 10 files changed, 39 insertions(+), 105 deletions(-) rename yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/{AiKnowledgeBaseDO.java => AiKnowledgeDO.java} (81%) delete mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java delete mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java rename yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/{AiKnowledgeBaseService.java => AiKnowledgeService.java} (94%) rename yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/{AiKnowledgeBaseServiceImpl.java => AiKnowledgeServiceImpl.java} (66%) diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java index 9d9c99a9a..9eae6b70c 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java @@ -3,7 +3,7 @@ package cn.iocoder.yudao.module.ai.controller.admin.knowledge; import cn.iocoder.yudao.framework.common.pojo.CommonResult; import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; -import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeBaseService; +import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.annotation.Resource; @@ -19,7 +19,7 @@ import static cn.iocoder.yudao.framework.security.core.util.SecurityFrameworkUti public class AiKnowledgeController { @Resource - private AiKnowledgeBaseService knowledgeBaseService; + private AiKnowledgeService knowledgeBaseService; @PostMapping("/create-my") @Operation(summary = "创建【我的】知识库") diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java similarity index 81% rename from yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java rename to yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java index d33114f2d..89e7486dc 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java @@ -10,15 +10,14 @@ import lombok.Data; import java.util.List; -// TODO @xin：要不把 AiKnowledgeBaseDO 改成 AiKnowledgeDO。感觉 base 后缀，感觉有点奇怪（让人以为是基类）。然后，我们很多地方的外键编号，都是 knowledgeId /** * AI 知识库 DO * * @author xiaoxin */ -@TableName(value = "ai_knowledge_base", autoResultMap = true) +@TableName(value = "ai_knowledge", autoResultMap = true) @Data -public class AiKnowledgeBaseDO extends BaseDO { +public class AiKnowledgeDO extends BaseDO { /** * 编号 diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java index 486602509..c5e526cce 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java @@ -24,7 +24,7 @@ public class AiKnowledgeDocumentDO extends BaseDO { /** * 知识库编号 * - * 关联 {@link AiKnowledgeBaseDO#getId()} + * 关联 {@link AiKnowledgeDO#getId()} */ private Long knowledgeId; /** diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java index cad90fcfe..710b65429 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java @@ -1,7 +1,7 @@ package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; -import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; import org.apache.ibatis.annotations.Mapper; /** @@ -10,5 +10,5 @@ import org.apache.ibatis.annotations.Mapper; * @author xiaoxin */ @Mapper -public interface AiKnowledgeBaseMapper extends BaseMapperX { +public interface AiKnowledgeBaseMapper extends BaseMapperX { } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java deleted file mode 100644 index ee4b3d03c..000000000 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java +++ /dev/null @@ -1,27 +0,0 @@ -package cn.iocoder.yudao.module.ai.service.knowledge; - -import org.springframework.ai.document.Document; -import org.springframework.ai.vectorstore.SearchRequest; - -import java.util.List; - -/** - * AI 嵌入 Service 接口 - * - * @author xiaoxin - */ -public interface AiEmbeddingService { - - /** - * 向量化文档并存储 - */ - void add(List documents); - - /** - * 相似查询 - * - * @param request 查询实体 - */ - List similaritySearch(SearchRequest request); - -} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java deleted file mode 100644 index 689ccea03..000000000 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java +++ /dev/null @@ -1,35 +0,0 @@ -package cn.iocoder.yudao.module.ai.service.knowledge; - -import jakarta.annotation.Resource; -import org.springframework.ai.document.Document; -import org.springframework.ai.vectorstore.RedisVectorStore; -import org.springframework.ai.vectorstore.SearchRequest; -import org.springframework.stereotype.Service; - -import java.util.List; - -// TODO @xin：是不是不用 AiEmbeddingServiceImpl，直接 vectorStore 注入到需要的地方就好啦。通过 KnowledgeDocumentService 返回就好。 -/** - * AI 嵌入 Service 实现类 - * - * @author xiaoxin - */ -@Service -public class AiEmbeddingServiceImpl implements AiEmbeddingService { - - @Resource - private RedisVectorStore vectorStore; - - @Override -// @Async - // TODO xiaoxin 报错先注释 - public void add(List documents) { - vectorStore.add(documents); - } - - @Override - public List similaritySearch(SearchRequest request) { - return vectorStore.similaritySearch(request); - } - -} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java index 537033015..69a73d6f7 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java @@ -14,8 +14,9 @@ import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.document.Document; import org.springframework.ai.reader.tika.TikaDocumentReader; -import org.springframework.ai.tokenizer.JTokkitTokenCountEstimator; +import org.springframework.ai.tokenizer.TokenCountEstimator; import org.springframework.ai.transformer.splitter.TokenTextSplitter; +import org.springframework.ai.vectorstore.RedisVectorStore; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -39,52 +40,49 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic @Resource private TokenTextSplitter tokenTextSplitter; - @Resource - private AiEmbeddingService embeddingService; + private TokenCountEstimator TOKEN_COUNT_ESTIMATOR; + @Resource + private RedisVectorStore vectorStore; - // TODO @xin：@Resource 注入 - private static final JTokkitTokenCountEstimator TOKEN_COUNT_ESTIMATOR = new JTokkitTokenCountEstimator(); // TODO xiaoxin 临时测试用，后续删 @Value("classpath:/webapp/test/Fel.pdf") private org.springframework.core.io.Resource data; // TODO 芋艿：需要 review 下，代码格式； - // TODO @xin：最好有 1、/2、/3 这种，让代码更有层次感 @Override @Transactional(rollbackFor = Exception.class) public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) { // TODO xiaoxin 后续从 url 加载 TikaDocumentReader loader = new TikaDocumentReader(data); - // 加载文档 + // 1.1 加载文档 List documents = loader.get(); Document document = CollUtil.getFirst(documents); // TODO @xin：是不是不存在，就抛出异常呀；厚泽 return 呀； - // TODO 芋艿 文档层面有没有可能会比较大，这两个字段是否可以从分段表计算得出？回复：先直接算； Integer tokens = Objects.nonNull(document) ? TOKEN_COUNT_ESTIMATOR.estimate(document.getContent()) : 0; Integer wordCount = Objects.nonNull(document) ? document.getContent().length() : 0; AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class) .setTokens(tokens).setWordCount(wordCount) .setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus()); - // 文档记录入库 + // 1.2 文档记录入库 documentMapper.insert(documentDO); Long documentId = documentDO.getId(); if (CollUtil.isEmpty(documents)) { return documentId; } - // 文档分段 + // 2.1 文档分段 List segments = tokenTextSplitter.apply(documents); - // 分段内容入库 + // 2.2 分段内容入库 List segmentDOList = CollectionUtils.convertList(segments, segment -> new AiKnowledgeSegmentDO().setContent(segment.getContent()).setDocumentId(documentId) .setTokens(TOKEN_COUNT_ESTIMATOR.estimate(segment.getContent())).setWordCount(segment.getContent().length()) .setStatus(CommonStatusEnum.ENABLE.getStatus())); segmentMapper.insertBatch(segmentDOList); - // 向量化并存储 - embeddingService.add(segments); + // 3 向量化并存储 + vectorStore.add(segments); return documentId; } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java similarity index 94% rename from yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java rename to yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java index be96b0918..91b0c9b3e 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java @@ -7,7 +7,7 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdat * * @author xiaoxin */ -public interface AiKnowledgeBaseService { +public interface AiKnowledgeService { /** * 创建【我的】知识库 diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java similarity index 66% rename from yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java rename to yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java index c208c92ba..a981b877f 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java @@ -1,12 +1,11 @@ package cn.iocoder.yudao.module.ai.service.knowledge; -import cn.hutool.core.lang.Assert; import cn.hutool.core.util.ObjUtil; import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; import cn.iocoder.yudao.framework.common.util.object.BeanUtils; import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; -import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeBaseMapper; import cn.iocoder.yudao.module.ai.service.model.AiChatModelService; @@ -24,7 +23,7 @@ import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_ */ @Service @Slf4j -public class AiKnowledgeBaseServiceImpl implements AiKnowledgeBaseService { +public class AiKnowledgeServiceImpl implements AiKnowledgeService { @Resource private AiChatModelService chatModalService; @@ -34,42 +33,34 @@ public class AiKnowledgeBaseServiceImpl implements AiKnowledgeBaseService { @Override public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) { - // TODO @xin：貌似直接调用 chatModalService.validateChatModel(id) 完事，不用搞个方法 // 1. 校验模型配置 - AiChatModelDO model = validateChatModel(createReqVO.getModelId()); + AiChatModelDO model = chatModalService.validateChatModel(createReqVO.getModelId()); // 2. 插入知识库 - // TODO @xin：不用 DO 结尾 - AiKnowledgeBaseDO knowledgeBaseDO = BeanUtils.toBean(createReqVO, AiKnowledgeBaseDO.class) + AiKnowledgeDO knowledgeBase = BeanUtils.toBean(createReqVO, AiKnowledgeDO.class) .setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus()); - knowledgeBaseMapper.insert(knowledgeBaseDO); - return knowledgeBaseDO.getId(); + knowledgeBaseMapper.insert(knowledgeBase); + return knowledgeBase.getId(); } @Override public void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId) { // 1.1 校验知识库存在 - AiKnowledgeBaseDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId()); + AiKnowledgeDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId()); if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) { throw exception(KNOWLEDGE_NOT_EXISTS); } // 1.2 校验模型配置 - AiChatModelDO model = validateChatModel(updateReqVO.getModelId()); + AiChatModelDO model = chatModalService.validateChatModel(updateReqVO.getModelId()); // 2. 更新知识库 - AiKnowledgeBaseDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeBaseDO.class); + AiKnowledgeDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeDO.class); updateDO.setModel(model.getModel()); knowledgeBaseMapper.updateById(updateDO); } - private AiChatModelDO validateChatModel(Long id) { - AiChatModelDO model = chatModalService.validateChatModel(id); - Assert.notNull(model, "未找到对应嵌入模型"); - return model; - } - - public AiKnowledgeBaseDO validateKnowledgeExists(Long id) { - AiKnowledgeBaseDO knowledgeBase = knowledgeBaseMapper.selectById(id); + public AiKnowledgeDO validateKnowledgeExists(Long id) { + AiKnowledgeDO knowledgeBase = knowledgeBaseMapper.selectById(id); if (knowledgeBase == null) { throw exception(KNOWLEDGE_NOT_EXISTS); } diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java index 543444fdd..8566a0941 100644 --- a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java @@ -13,6 +13,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.ai.autoconfigure.vectorstore.redis.RedisVectorStoreProperties; import org.springframework.ai.document.MetadataMode; import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.tokenizer.JTokkitTokenCountEstimator; +import org.springframework.ai.tokenizer.TokenCountEstimator; import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.ai.transformers.TransformersEmbeddingModel; import org.springframework.ai.vectorstore.RedisVectorStore; @@ -90,7 +92,7 @@ public class YudaoAiAutoConfiguration { } /** - * 我们启动有加载很多 Embedding 模型，不晓得取哪个好，先 new 个 TransformersEmbeddingModel 跑 + * TODO @xin 抽离出去，根据具体模型走 */ @Bean @Lazy // TODO 芋艿：临时注释，避免无法启动 @@ -114,4 +116,10 @@ public class YudaoAiAutoConfiguration { return new TokenTextSplitter(500, 100, 5, 10000, true); } + @Bean + @Lazy // TODO 芋艿：临时注释，避免无法启动 + public TokenCountEstimator tokenCountEstimator() { + return new JTokkitTokenCountEstimator(); + } + } \ No newline at end of file