diff --git a/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java index 50934e7a0..714d49adb 100644 --- a/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java +++ b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java @@ -52,4 +52,8 @@ public interface ErrorCodeConstants { // ========== API 思维导图 1-040-008-000 ========== ErrorCode MIND_MAP_NOT_EXISTS = new ErrorCode(1_040_008_000, "思维导图不存在!"); + + // ========== API 知识库 1-022-008-000 ========== + ErrorCode KNOWLEDGE_NOT_EXISTS = new ErrorCode(1_022_008_000, "知识库不存在!"); + } diff --git a/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java new file mode 100644 index 000000000..a37fa8643 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java @@ -0,0 +1,39 @@ +package cn.iocoder.yudao.module.ai.enums.knowledge; + +import cn.iocoder.yudao.framework.common.core.IntArrayValuable; +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.Arrays; + +/** + * AI 知识库-文档状态的枚举 + * + * @author xiaoxin + */ +@AllArgsConstructor +@Getter +public enum AiKnowledgeDocumentStatusEnum implements IntArrayValuable { + + IN_PROGRESS(10, "索引中"), + SUCCESS(20, "可用"), + FAIL(30, "失败"); + + /** + * 状态 + */ + private final Integer status; + + /** + * 状态名 + */ + private final String name; + + public static final int[] ARRAYS = Arrays.stream(values()).mapToInt(AiKnowledgeDocumentStatusEnum::getStatus).toArray(); + + @Override + public int[] array() { + return ARRAYS; + } + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java new file mode 100644 index 000000000..9d9c99a9a --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java @@ -0,0 +1,39 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge; + +import cn.iocoder.yudao.framework.common.pojo.CommonResult; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; +import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeBaseService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.annotation.Resource; +import jakarta.validation.Valid; +import org.springframework.web.bind.annotation.*; + +import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success; +import static cn.iocoder.yudao.framework.security.core.util.SecurityFrameworkUtils.getLoginUserId; + +@Tag(name = "管理后台 - AI 知识库") +@RestController +@RequestMapping("/ai/knowledge") +public class AiKnowledgeController { + + @Resource + private AiKnowledgeBaseService knowledgeBaseService; + + @PostMapping("/create-my") + @Operation(summary = "创建【我的】知识库") + public CommonResult createKnowledgeMy(@RequestBody @Valid AiKnowledgeCreateMyReqVO createReqVO) { + return success(knowledgeBaseService.createKnowledgeMy(createReqVO, getLoginUserId())); + } + + + @PutMapping("/update-my") + @Operation(summary = "更新【我的】知识库") + public CommonResult updateKnowledgeMy(@RequestBody @Valid AiKnowledgeUpdateMyReqVO updateReqVO) { + knowledgeBaseService.updateKnowledgeMy(updateReqVO, getLoginUserId()); + return success(true); + } + + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java new file mode 100644 index 000000000..fa9161eba --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeCreateMyReqVO.java @@ -0,0 +1,31 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +import java.util.List; + +/** + * @author xiaoxin + */ +@Schema(description = "管理后台 - AI 知识库创建【我的】 Request VO") +@Data +public class AiKnowledgeCreateMyReqVO { + + @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "ruoyi-vue-pro 用户指南") + @NotBlank(message = "知识库名称不能为空") + private String name; + + @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "存储 ruoyi-vue-pro 操作文档") + private String description; + + @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]") + private List visibilityPermissions; + + @Schema(description = "嵌入模型 id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") + @NotNull(message = "嵌入模型不能为空") + private Long modelId; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeDocumentCreateReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeDocumentCreateReqVO.java new file mode 100644 index 000000000..fa24eef72 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeDocumentCreateReqVO.java @@ -0,0 +1,27 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +/** + * @author xiaoxin + */ +@Schema(description = "管理后台 - AI 知识库【创建文档】 Request VO") +@Data +public class AiKnowledgeDocumentCreateReqVO { + + + @Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204") + @NotNull(message = "知识库编号不能为空") + private Long knowledgeId; + + @Schema(description = "文档名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "三方登陆") + @NotBlank(message = "文档名称不能为空") + private String name; + + @Schema(description = "文档 url", requiredMode = Schema.RequiredMode.REQUIRED, example = "https://doc.iocoder.cn") + private String url; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeUpdateMyReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeUpdateMyReqVO.java new file mode 100644 index 000000000..2bc39d5db --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/AiKnowledgeUpdateMyReqVO.java @@ -0,0 +1,36 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +import java.util.List; + +/** + * @author xiaoxin + */ +@Schema(description = "管理后台 - AI 知识库创建【我的】 Request VO") +@Data +public class AiKnowledgeUpdateMyReqVO { + + + @Schema(description = "对话编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204") + @NotNull(message = "知识库编号不能为空") + private Long id; + + @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "") + @NotBlank(message = "知识库名称不能为空") + private String name; + + @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "") + private String description; + + @Schema(description = "可见权限,只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]") + private List visibilityPermissions; + + @Schema(description = "嵌入模型 id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") + @NotNull(message = "嵌入模型不能为空") + private Long modelId; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java new file mode 100644 index 000000000..81cbf3ac9 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeBaseDO.java @@ -0,0 +1,62 @@ +package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; + + +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler; +import lombok.Data; + +import java.util.List; + +/** + * AI 知识库 DO + * + * @author xiaoxin + */ +@TableName(value = "ai_knowledge_base") +@Data +public class AiKnowledgeBaseDO extends BaseDO { + + /** + * 编号 + */ + @TableId(type = IdType.AUTO) + private Long id; + /** + * 用户编号 + *

+ * 关联 AdminUserDO 的 userId 字段 + */ + private Long userId; + /** + * 知识库名称 + */ + private String name; + /** + * 知识库描述 + */ + private String description; + /** + * 可见权限,只能选择哪些人可见 + */ + @TableField(typeHandler = JacksonTypeHandler.class) + private List visibilityPermissions; + /** + * 嵌入模型编号,高质量模式时维护 + */ + private Long modelId; + /** + * 模型标识 + */ + private String model; + /** + * 状态 + *

+ * 枚举 {@link CommonStatusEnum} + */ + private Integer status; +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java new file mode 100644 index 000000000..75d927cf0 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java @@ -0,0 +1,62 @@ +package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; + +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; +import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +/** + * AI 知识库-文档 DO + * + * @author xiaoxin + */ +@TableName(value = "ai_knowledge_document") +@Data +public class AiKnowledgeDocumentDO extends BaseDO { + + /** + * 编号 + */ + @TableId(type = IdType.AUTO) + private Long id; + /** + * 知识库编号 + */ + private Long knowledgeId; + /** + * 文件名称 + */ + private String name; + /** + * 内容 + */ + private String content; + /** + * 文件 URL + */ + private String url; + /** + * token数量 + */ + private Integer tokens; + /** + * 字符数 + */ + private Integer wordCount; + /** + * 切片状态 + *

+ * 枚举 {@link AiKnowledgeDocumentStatusEnum} + */ + private Integer sliceStatus; + + /** + * 状态 + *

+ * 枚举 {@link CommonStatusEnum} + */ + private Integer status; +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java new file mode 100644 index 000000000..657a3739b --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java @@ -0,0 +1,51 @@ +package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; + +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +/** + * AI 知识库-文档分段 DO + * + * @author xiaoxin + */ +@TableName(value = "ai_knowledge_segment") +@Data +public class AiKnowledgeSegmentDO extends BaseDO { + + /** + * 编号 + */ + @TableId(type = IdType.AUTO) + private Long id; + /** + * 向量库的id + */ + private String vectorId; + /** + * 文档编号 + */ + private Long documentId; + /** + * 切片内容 + */ + private String content; + /** + * 字符数 + */ + private Integer wordCount; + /** + * token数量 + */ + private Integer tokens; + /** + * 状态 + *

+ * 枚举 {@link CommonStatusEnum} + */ + private Integer status; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java new file mode 100644 index 000000000..cad90fcfe --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeBaseMapper.java @@ -0,0 +1,14 @@ +package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; + +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO; +import org.apache.ibatis.annotations.Mapper; + +/** + * AI 知识库基础信息 Mapper + * + * @author xiaoxin + */ +@Mapper +public interface AiKnowledgeBaseMapper extends BaseMapperX { +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java new file mode 100644 index 000000000..af55f545a --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java @@ -0,0 +1,14 @@ +package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; + +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; +import org.apache.ibatis.annotations.Mapper; + +/** + * AI 知识库-文档 Mapper + * + * @author xiaoxin + */ +@Mapper +public interface AiKnowledgeDocumentMapper extends BaseMapperX { +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java new file mode 100644 index 000000000..5043ee0ca --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java @@ -0,0 +1,14 @@ +package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; + +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; +import org.apache.ibatis.annotations.Mapper; + +/** + * AI 知识库-分片 Mapper + * + * @author xiaoxin + */ +@Mapper +public interface AiKnowledgeSegmentMapper extends BaseMapperX { +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java new file mode 100644 index 000000000..eee2f8044 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingService.java @@ -0,0 +1,27 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import org.springframework.ai.document.Document; +import org.springframework.ai.vectorstore.SearchRequest; + +import java.util.List; + +/** + * AI 嵌入 Service 接口 + * + * @author xiaoxin + */ +public interface AiEmbeddingService { + + /** + * 向量化文档并存储 + */ + void add(List documents); + + + /** + * 相似查询 + * + * @param request 查询实体 + */ + List similaritySearch(SearchRequest request); +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java new file mode 100644 index 000000000..2a6e75722 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiEmbeddingServiceImpl.java @@ -0,0 +1,33 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import jakarta.annotation.Resource; +import org.springframework.ai.document.Document; +import org.springframework.ai.vectorstore.RedisVectorStore; +import org.springframework.ai.vectorstore.SearchRequest; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * AI 嵌入 Service 实现类 + * + * @author xiaoxin + */ +@Service +public class AiEmbeddingServiceImpl implements AiEmbeddingService { + + @Resource + private RedisVectorStore vectorStore; + + @Override +// @Async + // TODO xiaoxin 报错先注释 + public void add(List documents) { + vectorStore.add(documents); + } + + @Override + public List similaritySearch(SearchRequest request) { + return vectorStore.similaritySearch(request); + } +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java new file mode 100644 index 000000000..7657ab748 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseService.java @@ -0,0 +1,30 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; + +/** + * AI 知识库-基础信息 Service 接口 + * + * @author xiaoxin + */ +public interface AiKnowledgeBaseService { + + + /** + * 创建【我的】知识库 + * + * @param createReqVO 创建信息 + * @param userId 用户编号 + * @return 编号 + */ + Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId); + + + /** + * 创建【我的】知识库 + * + * @param updateReqVO 更新信息 + * @param userId 用户编号 + */ + void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId); +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java new file mode 100644 index 000000000..63f4f53db --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeBaseServiceImpl.java @@ -0,0 +1,75 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import cn.hutool.core.lang.Assert; +import cn.hutool.core.util.ObjUtil; +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.common.util.object.BeanUtils; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeCreateMyReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeUpdateMyReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeBaseDO; +import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeBaseMapper; +import cn.iocoder.yudao.module.ai.service.model.AiChatModelService; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_EXISTS; + +/** + * AI 知识库-基础信息 Service 实现类 + * + * @author xiaoxin + */ +@Service +@Slf4j +public class AiKnowledgeBaseServiceImpl implements AiKnowledgeBaseService { + + @Resource + private AiChatModelService chatModalService; + + @Resource + private AiKnowledgeBaseMapper knowledgeBaseMapper; + + + @Override + public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) { + AiChatModelDO model = validateChatModel(createReqVO.getModelId()); + + AiKnowledgeBaseDO knowledgeBaseDO = BeanUtils.toBean(createReqVO, AiKnowledgeBaseDO.class); + knowledgeBaseDO.setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus()); + + knowledgeBaseMapper.insert(knowledgeBaseDO); + return knowledgeBaseDO.getId(); + } + + @Override + public void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId) { + + AiKnowledgeBaseDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId()); + if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) { + throw exception(KNOWLEDGE_NOT_EXISTS); + } + AiChatModelDO model = validateChatModel(updateReqVO.getModelId()); + AiKnowledgeBaseDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeBaseDO.class); + updateDO.setModel(model.getModel()); + + knowledgeBaseMapper.updateById(updateDO); + } + + + private AiChatModelDO validateChatModel(Long id) { + AiChatModelDO model = chatModalService.validateChatModel(id); + Assert.notNull(model, "未找到对应嵌入模型"); + return model; + } + + public AiKnowledgeBaseDO validateKnowledgeExists(Long id) { + AiKnowledgeBaseDO knowledgeBase = knowledgeBaseMapper.selectById(id); + if (knowledgeBase == null) { + throw exception(KNOWLEDGE_NOT_EXISTS); + } + return knowledgeBase; + } +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java new file mode 100644 index 000000000..52c62abf7 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java @@ -0,0 +1,21 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeDocumentCreateReqVO; + +/** + * AI 知识库-文档 Service 接口 + * + * @author xiaoxin + */ +public interface AiKnowledgeDocumentService { + + + /** + * 创建文档 + * + * @param createReqVO 文档创建 Request VO + * @return 文档编号 + */ + Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO); + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java new file mode 100644 index 000000000..9ee5c4eed --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java @@ -0,0 +1,91 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import cn.hutool.core.collection.CollUtil; +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils; +import cn.iocoder.yudao.framework.common.util.object.BeanUtils; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.AiKnowledgeDocumentCreateReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper; +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper; +import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.document.Document; +import org.springframework.ai.reader.tika.TikaDocumentReader; +import org.springframework.ai.tokenizer.JTokkitTokenCountEstimator; +import org.springframework.ai.transformer.splitter.TokenTextSplitter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; +import java.util.Objects; + +/** + * AI 知识库-文档 Service 实现类 + * + * @author xiaoxin + */ +@Service +@Slf4j +public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentService { + + @Resource + private AiKnowledgeDocumentMapper documentMapper; + @Resource + private AiKnowledgeSegmentMapper segmentMapper; + + @Resource + private TokenTextSplitter tokenTextSplitter; + + @Resource + private AiEmbeddingService embeddingService; + + private static final JTokkitTokenCountEstimator TOKEN_COUNT_ESTIMATOR = new JTokkitTokenCountEstimator(); + + // TODO xiaoxin 临时测试用,后续删 + @Value("classpath:/webapp/test/Fel.pdf") + private org.springframework.core.io.Resource data; + + + @Override + @Transactional(rollbackFor = Exception.class) + public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) { + + // TODO xiaoxin 后续从 url 加载 + TikaDocumentReader loader = new TikaDocumentReader(data); + // 加载文档 + List documents = loader.get(); + Document document = CollUtil.getFirst(documents); + // TODO 芋艿 文档层面有没有可能会比较大,这两个字段是否可以从分段表计算得出? + Integer tokens = Objects.nonNull(document) ? TOKEN_COUNT_ESTIMATOR.estimate(document.getContent()) : 0; + Integer wordCount = Objects.nonNull(document) ? document.getContent().length() : 0; + + AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class); + documentDO.setTokens(tokens).setWordCount(wordCount) + .setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus()); + // 文档记录入库 + documentMapper.insert(documentDO); + Long documentId = documentDO.getId(); + if (CollUtil.isEmpty(documents)) { + return documentId; + } + + // 文档分段 + List segments = tokenTextSplitter.apply(documents); + + List segmentDOList = CollectionUtils.convertList(segments, + segment -> new AiKnowledgeSegmentDO().setContent(segment.getContent()).setDocumentId(documentId) + .setTokens(TOKEN_COUNT_ESTIMATOR.estimate(segment.getContent())).setWordCount(segment.getContent().length()) + .setStatus(CommonStatusEnum.ENABLE.getStatus())); + // 分段内容入库 + segmentMapper.insertBatch(segmentDOList); + + //向量化并存储 + embeddingService.add(segments); + + return documentId; + } +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java new file mode 100644 index 000000000..003ce5c96 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java @@ -0,0 +1,11 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +/** + * AI 知识库-分片 Service 接口 + * + * @author xiaoxin + */ +public interface AiKnowledgeSegmentService { + + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java new file mode 100644 index 000000000..aa5facc36 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java @@ -0,0 +1,16 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +/** + * AI 知识库-基础信息 Service 实现类 + * + * @author xiaoxin + */ +@Service +@Slf4j +public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService { + + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java deleted file mode 100644 index 47905d4b1..000000000 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java +++ /dev/null @@ -1,15 +0,0 @@ -package cn.iocoder.yudao.module.ai.service.knowledge; - -/** - * AI 知识库 Service 接口 - * - * @author xiaoxin - */ -public interface DocService { - - /** - * 向量化文档 - */ - void embeddingDoc(); - -} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java deleted file mode 100644 index 7ba5018bd..000000000 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java +++ /dev/null @@ -1,42 +0,0 @@ -package cn.iocoder.yudao.module.ai.service.knowledge; - -import jakarta.annotation.Resource; -import lombok.extern.slf4j.Slf4j; -import org.springframework.ai.document.Document; -import org.springframework.ai.reader.tika.TikaDocumentReader; -import org.springframework.ai.transformer.splitter.TokenTextSplitter; -import org.springframework.ai.vectorstore.RedisVectorStore; -import org.springframework.beans.factory.annotation.Value; - -import java.util.List; - -/** - * AI 知识库 Service 实现类 - * - * @author xiaoxin - */ -//@Service // TODO 芋艿:临时注释,避免无法启动 -@Slf4j -public class DocServiceImpl implements DocService { - - @Resource - private RedisVectorStore vectorStore; - @Resource - private TokenTextSplitter tokenTextSplitter; - - // TODO @xin 临时测试用,后续删 - @Value("classpath:/webapp/test/Fel.pdf") - private org.springframework.core.io.Resource data; - - @Override - public void embeddingDoc() { - // 读取文件 - TikaDocumentReader loader = new TikaDocumentReader(data); - List documents = loader.get(); - // 文档分段 - List segments = tokenTextSplitter.apply(documents); - // 向量化并存储 - vectorStore.add(segments); - } - -} diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml b/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml index d8caea0df..69a3cdbda 100644 --- a/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml @@ -61,11 +61,9 @@ ${spring-ai.version} - - org.springframework.data - spring-data-redis - true + cn.iocoder.boot + yudao-spring-boot-starter-redis diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java index 61c38dd1d..615b05f78 100644 --- a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java @@ -31,6 +31,7 @@ import redis.clients.jedis.JedisPooled; * TODO @xin 先拿 spring-ai 最新代码覆盖,1.0.0-M1 跟 redis 自动配置会冲突 * * TODO 这个官方,有说啥时候 fix 哇? + * TODO 看着是列在1.0.0-M2版本 * * @author Christian Tzolov * @author Eddú Meléndez