huangge1199/vue-pro
1
0
Fork 0
mirror of https://gitee.com/huangge1199_admin/vue-pro.git synced 2024-11-22 23:31:52 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

!18 【新增】AI 知识库：文档切片向量化入库

Browse Source 
Merge pull request !18 from 小新/master-jdk21-ai
This commit is contained in:
芋道源码 2024-08-19 05:10:13 +00:00 committed by Gitee
commit 0db165f6d9
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
24 changed files with 700 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java
View File

@ -52,4 +52,8 @@ public interface ErrorCodeConstants {
// ========== API 思维导图 1-040-008-000 ==========
ErrorCode MIND_MAP_NOT_EXISTS = new ErrorCode(1_040_008_000, "思维导图不存在!");
// ========== API 知识库 1-022-008-000 ==========
ErrorCode KNOWLEDGE_NOT_EXISTS = new ErrorCode(1_022_008_000, "知识库不存在!");
}

39
yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java Normal file
View File

@ -0,0 +1,39 @@
package cn.iocoder.yudao.module.ai.enums.knowledge;
import cn.iocoder.yudao.framework.common.core.IntArrayValuable;
import lombok.AllArgsConstructor;
import lombok.Getter;
import java.util.Arrays;
/**
* AI 知识库-文档状态的枚举
*
* @author xiaoxin
*/
@AllArgsConstructor
@Getter
public enum AiKnowledgeDocumentStatusEnum implements IntArrayValuable {
IN_PROGRESS(10, "索引中"),
SUCCESS(20, "可用"),
FAIL(30, "失败");
/**
* 状态
*/
private final Integer status;
/**
* 状态名
*/
private final String name;
public static final int[] ARRAYS = Arrays.stream(values()).mapToInt(AiKnowledgeDocumentStatusEnum::getStatus).toArray();
@Override
public int[] array() {
return ARRAYS;
}
}

39
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java Normal file
View File

@ -0,0 +1,39 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge;
import cn.iocoder.yudao.framework.common.pojo.CommonResult;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO;
import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeBaseService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.annotation.Resource;
import jakarta.validation.Valid;
import org.springframework.web.bind.annotation.*;
import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success;
import static cn.iocoder.yudao.framework.security.core.util.SecurityFrameworkUtils.getLoginUserId;
@Tag(name = "管理后台 - AI 知识库")
@RestController
@RequestMapping("/ai/knowledge")
public class AiKnowledgeController {
@Resource
private AiKnowledgeBaseService knowledgeBaseService;
@PostMapping("/create-my")
@Operation(summary = "创建【我的】知识库")
public CommonResult<Long> createKnowledgeMy(@RequestBody @Valid AiKnowledgeCreateMyReqVO createReqVO) {
return success(knowledgeBaseService.createKnowledgeMy(createReqVO, getLoginUserId()));
}
@PutMapping("/update-my")
@Operation(summary = "更新【我的】知识库")
public CommonResult<Boolean> updateKnowledgeMy(@RequestBody @Valid AiKnowledgeUpdateMyReqVO updateReqVO) {
knowledgeBaseService.updateKnowledgeMy(updateReqVO, getLoginUserId());
return success(true);
}
}

31
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java Normal file
View File

@ -0,0 +1,31 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
/**
* @author xiaoxin
*/
@Schema(description = "管理后台 - AI 知识库创建【我的】 Request VO")
@Data
public class AiKnowledgeCreateMyReqVO {
@Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "ruoyi-vue-pro 用户指南")
@NotBlank(message = "知识库名称不能为空")
private String name;
@Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "存储 ruoyi-vue-pro 操作文档")
private String description;
@Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]")
private List<Long> visibilityPermissions;
@Schema(description = "嵌入模型 id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
@NotNull(message = "嵌入模型不能为空")
private Long modelId;
}

27
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeDocumentCreateReqVO.java Normal file
View File

@ -0,0 +1,27 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
/**
* @author xiaoxin
*/
@Schema(description = "管理后台 - AI 知识库【创建文档】 Request VO")
@Data
public class AiKnowledgeDocumentCreateReqVO {
@Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204")
@NotNull(message = "知识库编号不能为空")
private Long knowledgeId;
@Schema(description = "文档名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "三方登陆")
@NotBlank(message = "文档名称不能为空")
private String name;
@Schema(description = "文档 url", requiredMode = Schema.RequiredMode.REQUIRED, example = "https://doc.iocoder.cn")
private String url;
}

36
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeUpdateMyReqVO.java Normal file
View File

@ -0,0 +1,36 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
/**
* @author xiaoxin
*/
@Schema(description = "管理后台 - AI 知识库创建【我的】 Request VO")
@Data
public class AiKnowledgeUpdateMyReqVO {
@Schema(description = "对话编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204")
@NotNull(message = "知识库编号不能为空")
private Long id;
@Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "")
@NotBlank(message = "知识库名称不能为空")
private String name;
@Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "")
private String description;
@Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]")
private List<Long> visibilityPermissions;
@Schema(description = "嵌入模型 id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
@NotNull(message = "嵌入模型不能为空")
private Long modelId;
}

62
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java Normal file
View File

@ -0,0 +1,62 @@
package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
import lombok.Data;
import java.util.List;
/**
* AI 知识库 DO
*
* @author xiaoxin
*/
@TableName(value = "ai_knowledge_base")
@Data
public class AiKnowledgeBaseDO extends BaseDO {
/**
* 编号
*/
@TableId(type = IdType.AUTO)
private Long id;
/**
* 用户编号
* <p>
* 关联 AdminUserDO 的 userId 字段
*/
private Long userId;
/**
* 知识库名称
*/
private String name;
/**
* 知识库描述
*/
private String description;
/**
* 可见权限,只能选择哪些人可见
*/
@TableField(typeHandler = JacksonTypeHandler.class)
private List<Long> visibilityPermissions;
/**
* 嵌入模型编号，高质量模式时维护
*/
private Long modelId;
/**
* 模型标识
*/
private String model;
/**
* 状态
* <p>
* 枚举 {@link CommonStatusEnum}
*/
private Integer status;
}

62
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java Normal file
View File

@ -0,0 +1,62 @@
package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
/**
* AI 知识库-文档 DO
*
* @author xiaoxin
*/
@TableName(value = "ai_knowledge_document")
@Data
public class AiKnowledgeDocumentDO extends BaseDO {
/**
* 编号
*/
@TableId(type = IdType.AUTO)
private Long id;
/**
* 知识库编号
*/
private Long knowledgeId;
/**
* 文件名称
*/
private String name;
/**
* 内容
*/
private String content;
/**
* 文件 URL
*/
private String url;
/**
* token数量
*/
private Integer tokens;
/**
* 字符数
*/
private Integer wordCount;
/**
* 切片状态
* <p>
* 枚举 {@link AiKnowledgeDocumentStatusEnum}
*/
private Integer sliceStatus;
/**
* 状态
* <p>
* 枚举 {@link CommonStatusEnum}
*/
private Integer status;
}

51
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java Normal file
View File

@ -0,0 +1,51 @@
package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
/**
* AI 知识库-文档分段 DO
*
* @author xiaoxin
*/
@TableName(value = "ai_knowledge_segment")
@Data
public class AiKnowledgeSegmentDO extends BaseDO {
/**
* 编号
*/
@TableId(type = IdType.AUTO)
private Long id;
/**
* 向量库的id
*/
private String vectorId;
/**
* 文档编号
*/
private Long documentId;
/**
* 切片内容
*/
private String content;
/**
* 字符数
*/
private Integer wordCount;
/**
* token数量
*/
private Integer tokens;
/**
* 状态
* <p>
* 枚举 {@link CommonStatusEnum}
*/
private Integer status;
}

14
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java Normal file
View File

@ -0,0 +1,14 @@
package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO;
import org.apache.ibatis.annotations.Mapper;
/**
* AI 知识库基础信息 Mapper
*
* @author xiaoxin
*/
@Mapper
public interface AiKnowledgeBaseMapper extends BaseMapperX<AiKnowledgeBaseDO> {
}

14
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java Normal file
View File

@ -0,0 +1,14 @@
package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
import org.apache.ibatis.annotations.Mapper;
/**
* AI 知识库-文档 Mapper
*
* @author xiaoxin
*/
@Mapper
public interface AiKnowledgeDocumentMapper extends BaseMapperX<AiKnowledgeDocumentDO> {
}

14
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java Normal file
View File

@ -0,0 +1,14 @@
package cn.iocoder.yudao.module.ai.dal.mysql.knowledge;
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
import org.apache.ibatis.annotations.Mapper;
/**
* AI 知识库-分片 Mapper
*
* @author xiaoxin
*/
@Mapper
public interface AiKnowledgeSegmentMapper extends BaseMapperX<AiKnowledgeSegmentDO> {
}

27
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java Normal file
View File

@ -0,0 +1,27 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import org.springframework.ai.document.Document;
import org.springframework.ai.vectorstore.SearchRequest;
import java.util.List;
/**
* AI 嵌入 Service 接口
*
* @author xiaoxin
*/
public interface AiEmbeddingService {
/**
* 向量化文档并存储
*/
void add(List<Document> documents);
/**
* 相似查询
*
* @param request 查询实体
*/
List<Document> similaritySearch(SearchRequest request);
}

33
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java Normal file
View File

@ -0,0 +1,33 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import jakarta.annotation.Resource;
import org.springframework.ai.document.Document;
import org.springframework.ai.vectorstore.RedisVectorStore;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* AI 嵌入 Service 实现类
*
* @author xiaoxin
*/
@Service
public class AiEmbeddingServiceImpl implements AiEmbeddingService {
@Resource
private RedisVectorStore vectorStore;
@Override
// @Async
// TODO xiaoxin 报错先注释
public void add(List<Document> documents) {
vectorStore.add(documents);
}
@Override
public List<Document> similaritySearch(SearchRequest request) {
return vectorStore.similaritySearch(request);
}
}

30
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java Normal file
View File

@ -0,0 +1,30 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO;
/**
* AI 知识库-基础信息 Service 接口
*
* @author xiaoxin
*/
public interface AiKnowledgeBaseService {
/**
* 创建【我的】知识库
*
* @param createReqVO 创建信息
* @param userId 用户编号
* @return 编号
*/
Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId);
/**
* 创建【我的】知识库
*
* @param updateReqVO 更新信息
* @param userId 用户编号
*/
void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId);
}

75
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java Normal file
View File

@ -0,0 +1,75 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import cn.hutool.core.lang.Assert;
import cn.hutool.core.util.ObjUtil;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO;
import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeBaseMapper;
import cn.iocoder.yudao.module.ai.service.model.AiChatModelService;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_EXISTS;
/**
* AI 知识库-基础信息 Service 实现类
*
* @author xiaoxin
*/
@Service
@Slf4j
public class AiKnowledgeBaseServiceImpl implements AiKnowledgeBaseService {
@Resource
private AiChatModelService chatModalService;
@Resource
private AiKnowledgeBaseMapper knowledgeBaseMapper;
@Override
public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) {
AiChatModelDO model = validateChatModel(createReqVO.getModelId());
AiKnowledgeBaseDO knowledgeBaseDO = BeanUtils.toBean(createReqVO, AiKnowledgeBaseDO.class);
knowledgeBaseDO.setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus());
knowledgeBaseMapper.insert(knowledgeBaseDO);
return knowledgeBaseDO.getId();
}
@Override
public void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId) {
AiKnowledgeBaseDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId());
if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) {
throw exception(KNOWLEDGE_NOT_EXISTS);
}
AiChatModelDO model = validateChatModel(updateReqVO.getModelId());
AiKnowledgeBaseDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeBaseDO.class);
updateDO.setModel(model.getModel());
knowledgeBaseMapper.updateById(updateDO);
}
private AiChatModelDO validateChatModel(Long id) {
AiChatModelDO model = chatModalService.validateChatModel(id);
Assert.notNull(model, "未找到对应嵌入模型");
return model;
}
public AiKnowledgeBaseDO validateKnowledgeExists(Long id) {
AiKnowledgeBaseDO knowledgeBase = knowledgeBaseMapper.selectById(id);
if (knowledgeBase == null) {
throw exception(KNOWLEDGE_NOT_EXISTS);
}
return knowledgeBase;
}
}

21
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java Normal file
View File

@ -0,0 +1,21 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeDocumentCreateReqVO;
/**
* AI 知识库-文档 Service 接口
*
* @author xiaoxin
*/
public interface AiKnowledgeDocumentService {
/**
* 创建文档
*
* @param createReqVO 文档创建 Request VO
* @return 文档编号
*/
Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO);
}

91
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java Normal file
View File

@ -0,0 +1,91 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import cn.hutool.core.collection.CollUtil;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeDocumentCreateReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper;
import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper;
import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.document.Document;
import org.springframework.ai.reader.tika.TikaDocumentReader;
import org.springframework.ai.tokenizer.JTokkitTokenCountEstimator;
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
import java.util.Objects;
/**
* AI 知识库-文档 Service 实现类
*
* @author xiaoxin
*/
@Service
@Slf4j
public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentService {
@Resource
private AiKnowledgeDocumentMapper documentMapper;
@Resource
private AiKnowledgeSegmentMapper segmentMapper;
@Resource
private TokenTextSplitter tokenTextSplitter;
@Resource
private AiEmbeddingService embeddingService;
private static final JTokkitTokenCountEstimator TOKEN_COUNT_ESTIMATOR = new JTokkitTokenCountEstimator();
// TODO xiaoxin 临时测试用，后续删
@Value("classpath:/webapp/test/Fel.pdf")
private org.springframework.core.io.Resource data;
@Override
@Transactional(rollbackFor = Exception.class)
public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) {
// TODO xiaoxin 后续从 url 加载
TikaDocumentReader loader = new TikaDocumentReader(data);
// 加载文档
List<Document> documents = loader.get();
Document document = CollUtil.getFirst(documents);
// TODO 芋艿 文档层面有没有可能会比较大，这两个字段是否可以从分段表计算得出？
Integer tokens = Objects.nonNull(document) ? TOKEN_COUNT_ESTIMATOR.estimate(document.getContent()) : 0;
Integer wordCount = Objects.nonNull(document) ? document.getContent().length() : 0;
AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class);
documentDO.setTokens(tokens).setWordCount(wordCount)
.setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus());
// 文档记录入库
documentMapper.insert(documentDO);
Long documentId = documentDO.getId();
if (CollUtil.isEmpty(documents)) {
return documentId;
}
// 文档分段
List<Document> segments = tokenTextSplitter.apply(documents);
List<AiKnowledgeSegmentDO> segmentDOList = CollectionUtils.convertList(segments,
segment -> new AiKnowledgeSegmentDO().setContent(segment.getContent()).setDocumentId(documentId)
.setTokens(TOKEN_COUNT_ESTIMATOR.estimate(segment.getContent())).setWordCount(segment.getContent().length())
.setStatus(CommonStatusEnum.ENABLE.getStatus()));
// 分段内容入库
segmentMapper.insertBatch(segmentDOList);
//向量化并存储
embeddingService.add(segments);
return documentId;
}
}

11
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java Normal file
View File

@ -0,0 +1,11 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
/**
* AI 知识库-分片 Service 接口
*
* @author xiaoxin
*/
public interface AiKnowledgeSegmentService {
}

16
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java Normal file
View File

@ -0,0 +1,16 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
/**
* AI 知识库-基础信息 Service 实现类
*
* @author xiaoxin
*/
@Service
@Slf4j
public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService {
}

15
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java
View File

@ -1,15 +0,0 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
/**
* AI 知识库 Service 接口
*
* @author xiaoxin
*/
public interface DocService {
/**
* 向量化文档
*/
void embeddingDoc();
}

42
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java
View File

@ -1,42 +0,0 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.document.Document;
import org.springframework.ai.reader.tika.TikaDocumentReader;
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.ai.vectorstore.RedisVectorStore;
import org.springframework.beans.factory.annotation.Value;
import java.util.List;
/**
* AI 知识库 Service 实现类
*
* @author xiaoxin
*/
//@Service // TODO 芋艿：临时注释，避免无法启动
@Slf4j
public class DocServiceImpl implements DocService {
@Resource
private RedisVectorStore vectorStore;
@Resource
private TokenTextSplitter tokenTextSplitter;
// TODO @xin 临时测试用，后续删
@Value("classpath:/webapp/test/Fel.pdf")
private org.springframework.core.io.Resource data;
@Override
public void embeddingDoc() {
// 读取文件
TikaDocumentReader loader = new TikaDocumentReader(data);
List<Document> documents = loader.get();
// 文档分段
List<Document> segments = tokenTextSplitter.apply(documents);
// 向量化并存储
vectorStore.add(segments);
}
}

6
yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml
View File

@ -61,11 +61,9 @@
<version>${spring-ai.version}</version>
</dependency>
<!-- TODO @xin：引入我们项目的 starter -->
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-redis</artifactId>
<optional>true</optional>
<groupId>cn.iocoder.boot</groupId>
<artifactId>yudao-spring-boot-starter-redis</artifactId>
</dependency>
<dependency>

1
yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java
View File

@ -31,6 +31,7 @@ import redis.clients.jedis.JedisPooled;
* TODO @xin 先拿 spring-ai 最新代码覆盖，1.0.0-M1 跟 redis 自动配置会冲突
*
* TODO 这个官方，有说啥时候 fix 哇？
* TODO 看着是列在1.0.0-M2版本
*
* @author Christian Tzolov
* @author Eddú Meléndez