From d698ef70b5e8a0a5ca038b4fb7b881dedfe48902 Mon Sep 17 00:00:00 2001 From: xiaoxin <718949661@qq.com> Date: Mon, 5 Aug 2024 14:00:35 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90=E6=96=B0=E5=A2=9E=E3=80=91AI=20?= =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93=EF=BC=9A=E6=94=AF=E6=8C=81=E8=AF=BB?= =?UTF-8?q?=E5=8F=96=20PDF=E3=80=81DOC/DOCX=E3=80=81PPT/PPTX=20=E5=92=8C?= =?UTF-8?q?=20HTML=E7=AD=89=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ai/service/knowledge/DocServiceImpl.java | 4 +-- .../yudao-spring-boot-starter-ai/pom.xml | 6 ++++ .../ai/config/YudaoAiAutoConfiguration.java | 36 +++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java index 76fa1e530..eeffebf44 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java @@ -3,7 +3,7 @@ package cn.iocoder.yudao.module.ai.service.knowledge; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.document.Document; -import org.springframework.ai.reader.TextReader; +import org.springframework.ai.reader.tika.TikaDocumentReader; import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.ai.vectorstore.RedisVectorStore; import org.springframework.beans.factory.annotation.Value; @@ -34,7 +34,7 @@ public class DocServiceImpl implements DocService { public void embeddingDoc() { // 读取文件 org.springframework.core.io.Resource file = data; - TextReader loader = new TextReader(file); + TikaDocumentReader loader = new TikaDocumentReader(file); List documents = loader.get(); // 文档分段 List segments = tokenTextSplitter.apply(documents); diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml b/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml index f015a643b..95895e9b0 100644 --- a/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml @@ -39,11 +39,17 @@ spring-ai-stability-ai-spring-boot-starter ${spring-ai.version} + org.springframework.ai spring-ai-transformers-spring-boot-starter ${spring-ai.version} + + org.springframework.ai + spring-ai-tika-document-reader + ${spring-ai.version} + org.springframework.ai spring-ai-redis-store diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java index 05a317294..58340d45d 100644 --- a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java @@ -10,11 +10,18 @@ import cn.iocoder.yudao.framework.ai.core.model.xinghuo.XingHuoChatModel; import cn.iocoder.yudao.framework.ai.core.model.xinghuo.XingHuoChatOptions; import com.alibaba.cloud.ai.tongyi.TongYiAutoConfiguration; import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.autoconfigure.vectorstore.redis.RedisVectorStoreProperties; +import org.springframework.ai.document.MetadataMode; +import org.springframework.ai.transformer.splitter.TokenTextSplitter; +import org.springframework.ai.transformers.TransformersEmbeddingModel; +import org.springframework.ai.vectorstore.RedisVectorStore; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.data.redis.RedisProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; +import redis.clients.jedis.JedisPooled; /** * 芋道 AI 自动配置 @@ -73,4 +80,33 @@ public class YudaoAiAutoConfiguration { return new SunoApi(yudaoAiProperties.getSuno().getBaseUrl()); } + // ========== rag 相关 ========== + @Bean + public TransformersEmbeddingModel transformersEmbeddingClient() { + return new TransformersEmbeddingModel(MetadataMode.EMBED); + } + + /** + * 我们启动有加载很多 Embedding 模型，不晓得取哪个好，先 new 个 TransformersEmbeddingModel 跑 + */ + @Bean + public RedisVectorStore vectorStore(TransformersEmbeddingModel transformersEmbeddingModel, RedisVectorStoreProperties properties, + RedisProperties redisProperties) { + var config = RedisVectorStore.RedisVectorStoreConfig.builder() + .withIndexName(properties.getIndex()) + .withPrefix(properties.getPrefix()) + .build(); + + RedisVectorStore redisVectorStore = new RedisVectorStore(config, transformersEmbeddingModel, + new JedisPooled(redisProperties.getHost(), redisProperties.getPort()), + properties.isInitializeSchema()); + redisVectorStore.afterPropertiesSet(); + return redisVectorStore; + } + + @Bean + public TokenTextSplitter tokenTextSplitter() { + return new TokenTextSplitter(500, 100, 5, 10000, true); + } + } \ No newline at end of file