【移除】敏感词的管理,简化项目的复杂度

This commit is contained in:
YunaiV 2024-04-22 19:53:01 +08:00
parent 8093ef3b96
commit 9a31613e5b
17 changed files with 0 additions and 1204 deletions

View File

@ -1,30 +0,0 @@
package cn.iocoder.yudao.module.system.api.sensitiveword;
import java.util.List;
/**
* 敏感词 API 接口
*
* @author 永不言败
*/
public interface SensitiveWordApi {
/**
* 获得文本所包含的不合法的敏感词数组
*
* @param text 文本
* @param tags 标签数组
* @return 不合法的敏感词数组
*/
List<String> validateText(String text, List<String> tags);
/**
* 判断文本是否包含敏感词
*
* @param text 文本
* @param tags 表述数组
* @return 是否包含
*/
boolean isTextValid(String text, List<String> tags);
}

View File

@ -115,10 +115,6 @@ public interface ErrorCodeConstants {
ErrorCode TENANT_PACKAGE_USED = new ErrorCode(1_002_016_001, "租户正在使用该套餐,请给租户重新设置套餐后再尝试删除");
ErrorCode TENANT_PACKAGE_DISABLE = new ErrorCode(1_002_016_002, "名字为【{}】的租户套餐已被禁用");
// ========== 错误码模块 1-002-017-000 ==========
ErrorCode ERROR_CODE_NOT_EXISTS = new ErrorCode(1_002_017_000, "错误码不存在");
ErrorCode ERROR_CODE_DUPLICATE = new ErrorCode(1_002_017_001, "已经存在编码为【{}】的错误码");
// ========== 社交用户 1-002-018-000 ==========
ErrorCode SOCIAL_USER_AUTH_FAILURE = new ErrorCode(1_002_018_000, "社交授权失败,原因是:{}");
ErrorCode SOCIAL_USER_NOT_FOUND = new ErrorCode(1_002_018_001, "社交授权失败,找不到对应的用户");
@ -127,10 +123,6 @@ public interface ErrorCodeConstants {
ErrorCode SOCIAL_CLIENT_NOT_EXISTS = new ErrorCode(1_002_018_201, "社交客户端不存在");
ErrorCode SOCIAL_CLIENT_UNIQUE = new ErrorCode(1_002_018_202, "社交客户端已存在配置");
// ========== 系统敏感词 1-002-019-000 =========
ErrorCode SENSITIVE_WORD_NOT_EXISTS = new ErrorCode(1_002_019_000, "系统敏感词在所有标签中都不存在");
ErrorCode SENSITIVE_WORD_EXISTS = new ErrorCode(1_002_019_001, "系统敏感词已在标签中存在");
// ========== OAuth2 客户端 1-002-020-000 =========
ErrorCode OAUTH2_CLIENT_NOT_EXISTS = new ErrorCode(1_002_020_000, "OAuth2 客户端不存在");
ErrorCode OAUTH2_CLIENT_EXISTS = new ErrorCode(1_002_020_001, "OAuth2 客户端编号已存在");

View File

@ -1,29 +0,0 @@
package cn.iocoder.yudao.module.system.api.sensitiveword;
import cn.iocoder.yudao.module.system.service.sensitiveword.SensitiveWordService;
import org.springframework.stereotype.Service;
import jakarta.annotation.Resource;
import java.util.List;
/**
* 敏感词 API 实现类
*
* @author 永不言败
*/
@Service
public class SensitiveWordApiImpl implements SensitiveWordApi {
@Resource
private SensitiveWordService sensitiveWordService;
@Override
public List<String> validateText(String text, List<String> tags) {
return sensitiveWordService.validateText(text, tags);
}
@Override
public boolean isTextValid(String text, List<String> tags) {
return sensitiveWordService.isTextValid(text, tags);
}
}

View File

@ -1,4 +0,0 @@
### 请求 /system/sensitive-word/validate-text 接口 => 成功
GET {{baseUrl}}/system/sensitive-word/validate-text?text=XXX&tags=短信&tags=蔬菜
Authorization: Bearer {{token}}
tenant-id: {{adminTenentId}}

View File

@ -1,108 +0,0 @@
package cn.iocoder.yudao.module.system.controller.admin.sensitiveword;
import cn.iocoder.yudao.framework.apilog.core.annotation.ApiAccessLog;
import cn.iocoder.yudao.framework.common.pojo.CommonResult;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.framework.excel.core.util.ExcelUtils;
import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordPageReqVO;
import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordRespVO;
import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordSaveVO;
import cn.iocoder.yudao.module.system.dal.dataobject.sensitiveword.SensitiveWordDO;
import cn.iocoder.yudao.module.system.service.sensitiveword.SensitiveWordService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.annotation.Resource;
import jakarta.servlet.http.HttpServletResponse;
import jakarta.validation.Valid;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import java.io.IOException;
import java.util.List;
import java.util.Set;
import static cn.iocoder.yudao.framework.apilog.core.enums.OperateTypeEnum.EXPORT;
import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success;
@Tag(name = "管理后台 - 敏感词")
@RestController
@RequestMapping("/system/sensitive-word")
@Validated
public class SensitiveWordController {
@Resource
private SensitiveWordService sensitiveWordService;
@PostMapping("/create")
@Operation(summary = "创建敏感词")
@PreAuthorize("@ss.hasPermission('system:sensitive-word:create')")
public CommonResult<Long> createSensitiveWord(@Valid @RequestBody SensitiveWordSaveVO createReqVO) {
return success(sensitiveWordService.createSensitiveWord(createReqVO));
}
@PutMapping("/update")
@Operation(summary = "更新敏感词")
@PreAuthorize("@ss.hasPermission('system:sensitive-word:update')")
public CommonResult<Boolean> updateSensitiveWord(@Valid @RequestBody SensitiveWordSaveVO updateReqVO) {
sensitiveWordService.updateSensitiveWord(updateReqVO);
return success(true);
}
@DeleteMapping("/delete")
@Operation(summary = "删除敏感词")
@Parameter(name = "id", description = "编号", required = true)
@PreAuthorize("@ss.hasPermission('system:sensitive-word:delete')")
public CommonResult<Boolean> deleteSensitiveWord(@RequestParam("id") Long id) {
sensitiveWordService.deleteSensitiveWord(id);
return success(true);
}
@GetMapping("/get")
@Operation(summary = "获得敏感词")
@Parameter(name = "id", description = "编号", required = true, example = "1024")
@PreAuthorize("@ss.hasPermission('system:sensitive-word:query')")
public CommonResult<SensitiveWordRespVO> getSensitiveWord(@RequestParam("id") Long id) {
SensitiveWordDO sensitiveWord = sensitiveWordService.getSensitiveWord(id);
return success(BeanUtils.toBean(sensitiveWord, SensitiveWordRespVO.class));
}
@GetMapping("/page")
@Operation(summary = "获得敏感词分页")
@PreAuthorize("@ss.hasPermission('system:sensitive-word:query')")
public CommonResult<PageResult<SensitiveWordRespVO>> getSensitiveWordPage(@Valid SensitiveWordPageReqVO pageVO) {
PageResult<SensitiveWordDO> pageResult = sensitiveWordService.getSensitiveWordPage(pageVO);
return success(BeanUtils.toBean(pageResult, SensitiveWordRespVO.class));
}
@GetMapping("/export-excel")
@Operation(summary = "导出敏感词 Excel")
@PreAuthorize("@ss.hasPermission('system:sensitive-word:export')")
@ApiAccessLog(operateType = EXPORT)
public void exportSensitiveWordExcel(@Valid SensitiveWordPageReqVO exportReqVO,
HttpServletResponse response) throws IOException {
exportReqVO.setPageSize(PageParam.PAGE_SIZE_NONE);
List<SensitiveWordDO> list = sensitiveWordService.getSensitiveWordPage(exportReqVO).getList();
// 导出 Excel
ExcelUtils.write(response, "敏感词.xls", "数据", SensitiveWordRespVO.class,
BeanUtils.toBean(list, SensitiveWordRespVO.class));
}
@GetMapping("/get-tags")
@Operation(summary = "获取所有敏感词的标签数组")
@PreAuthorize("@ss.hasPermission('system:sensitive-word:query')")
public CommonResult<Set<String>> getSensitiveWordTagSet() {
return success(sensitiveWordService.getSensitiveWordTagSet());
}
@GetMapping("/validate-text")
@Operation(summary = "获得文本所包含的不合法的敏感词数组")
public CommonResult<List<String>> validateText(@RequestParam("text") String text,
@RequestParam(value = "tags", required = false) List<String> tags) {
return success(sensitiveWordService.validateText(text, tags));
}
}

View File

@ -1,33 +0,0 @@
package cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import org.springframework.format.annotation.DateTimeFormat;
import java.time.LocalDateTime;
import static cn.iocoder.yudao.framework.common.util.date.DateUtils.FORMAT_YEAR_MONTH_DAY_HOUR_MINUTE_SECOND;
@Schema(description = "管理后台 - 敏感词分页 Request VO")
@Data
@EqualsAndHashCode(callSuper = true)
@ToString(callSuper = true)
public class SensitiveWordPageReqVO extends PageParam {
@Schema(description = "敏感词", example = "敏感词")
private String name;
@Schema(description = "标签", example = "短信,评论")
private String tag;
@Schema(description = "状态,参见 CommonStatusEnum 枚举类", example = "1")
private Integer status;
@DateTimeFormat(pattern = FORMAT_YEAR_MONTH_DAY_HOUR_MINUTE_SECOND)
@Schema(description = "创建时间")
private LocalDateTime[] createTime;
}

View File

@ -1,45 +0,0 @@
package cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo;
import cn.iocoder.yudao.framework.excel.core.annotations.DictFormat;
import cn.iocoder.yudao.framework.excel.core.convert.DictConvert;
import cn.iocoder.yudao.framework.excel.core.convert.JsonConvert;
import cn.iocoder.yudao.module.system.enums.DictTypeConstants;
import com.alibaba.excel.annotation.ExcelProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import java.time.LocalDateTime;
import java.util.List;
@Schema(description = "管理后台 - 敏感词 Response VO")
@Data
public class SensitiveWordRespVO {
@Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
@ExcelProperty("编号")
private Long id;
@Schema(description = "敏感词", requiredMode = Schema.RequiredMode.REQUIRED, example = "敏感词")
@ExcelProperty("敏感词")
private String name;
@Schema(description = "标签", requiredMode = Schema.RequiredMode.REQUIRED, example = "短信,评论")
@ExcelProperty(value = "标签", converter = JsonConvert.class)
private List<String> tags;
@Schema(description = "状态,参见 CommonStatusEnum 枚举类", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
@ExcelProperty(value = "状态", converter = DictConvert.class)
@DictFormat(DictTypeConstants.COMMON_STATUS)
private Integer status;
@Schema(description = "描述", example = "污言秽语")
@ExcelProperty("描述")
private String description;
@Schema(description = "创建时间", requiredMode = Schema.RequiredMode.REQUIRED)
@ExcelProperty("创建时间")
private LocalDateTime createTime;
}

View File

@ -1,31 +0,0 @@
package cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import jakarta.validation.constraints.NotNull;
import java.util.List;
@Schema(description = "管理后台 - 敏感词创建/修改 Request VO")
@Data
public class SensitiveWordSaveVO {
@Schema(description = "编号", example = "1")
private Long id;
@Schema(description = "敏感词", requiredMode = Schema.RequiredMode.REQUIRED, example = "敏感词")
@NotNull(message = "敏感词不能为空")
private String name;
@Schema(description = "标签", requiredMode = Schema.RequiredMode.REQUIRED, example = "短信,评论")
@NotNull(message = "标签不能为空")
private List<String> tags;
@Schema(description = "状态,参见 CommonStatusEnum 枚举类", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
@NotNull(message = "状态不能为空")
private Integer status;
@Schema(description = "描述", example = "污言秽语")
private String description;
}

View File

@ -1,58 +0,0 @@
package cn.iocoder.yudao.module.system.dal.dataobject.sensitiveword;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
import cn.iocoder.yudao.framework.mybatis.core.type.StringListTypeHandler;
import com.baomidou.mybatisplus.annotation.KeySequence;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.*;
import java.util.List;
/**
* 敏感词 DO
*
* @author 永不言败
*/
@TableName(value = "system_sensitive_word", autoResultMap = true)
@KeySequence("system_sensitive_word_seq") // 用于 OraclePostgreSQLKingbaseDB2H2 数据库的主键自增如果是 MySQL 等数据库可不写
@Data
@EqualsAndHashCode(callSuper = true)
@ToString(callSuper = true)
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class SensitiveWordDO extends BaseDO {
/**
* 编号
*/
@TableId
private Long id;
/**
* 敏感词
*/
private String name;
/**
* 描述
*/
private String description;
/**
* 标签数组
*
* 用于实现不同的业务场景下需要使用不同标签的敏感词
* 例如说tag 有短信论坛两种敏感词 "推广" 在短信下是敏感词在论坛下不是敏感词
* 此时我们会存储一条敏感词记录它的 name "推广"tag 为短信
*/
@TableField(typeHandler = StringListTypeHandler.class)
private List<String> tags;
/**
* 状态
*
* 枚举 {@link CommonStatusEnum}
*/
private Integer status;
}

View File

@ -1,36 +0,0 @@
package cn.iocoder.yudao.module.system.dal.mysql.sensitiveword;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordPageReqVO;
import cn.iocoder.yudao.module.system.dal.dataobject.sensitiveword.SensitiveWordDO;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Select;
import java.time.LocalDateTime;
/**
* 敏感词 Mapper
*
* @author 永不言败
*/
@Mapper
public interface SensitiveWordMapper extends BaseMapperX<SensitiveWordDO> {
default PageResult<SensitiveWordDO> selectPage(SensitiveWordPageReqVO reqVO) {
return selectPage(reqVO, new LambdaQueryWrapperX<SensitiveWordDO>()
.likeIfPresent(SensitiveWordDO::getName, reqVO.getName())
.likeIfPresent(SensitiveWordDO::getTags, reqVO.getTag())
.eqIfPresent(SensitiveWordDO::getStatus, reqVO.getStatus())
.betweenIfPresent(SensitiveWordDO::getCreateTime, reqVO.getCreateTime())
.orderByDesc(SensitiveWordDO::getId));
}
default SensitiveWordDO selectByName(String name) {
return selectOne(SensitiveWordDO::getName, name);
}
@Select("SELECT COUNT(*) FROM system_sensitive_word WHERE update_time > #{maxUpdateTime}")
Long selectCountByUpdateTimeGt(LocalDateTime maxTime);
}

View File

@ -1,89 +0,0 @@
package cn.iocoder.yudao.module.system.service.sensitiveword;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordPageReqVO;
import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordSaveVO;
import cn.iocoder.yudao.module.system.dal.dataobject.sensitiveword.SensitiveWordDO;
import jakarta.validation.Valid;
import java.util.List;
import java.util.Set;
/**
* 敏感词 Service 接口
*
* @author 永不言败
*/
public interface SensitiveWordService {
/**
* 创建敏感词
*
* @param createReqVO 创建信息
* @return 编号
*/
Long createSensitiveWord(@Valid SensitiveWordSaveVO createReqVO);
/**
* 更新敏感词
*
* @param updateReqVO 更新信息
*/
void updateSensitiveWord(@Valid SensitiveWordSaveVO updateReqVO);
/**
* 删除敏感词
*
* @param id 编号
*/
void deleteSensitiveWord(Long id);
/**
* 获得敏感词
*
* @param id 编号
* @return 敏感词
*/
SensitiveWordDO getSensitiveWord(Long id);
/**
* 获得敏感词列表
*
* @return 敏感词列表
*/
List<SensitiveWordDO> getSensitiveWordList();
/**
* 获得敏感词分页
*
* @param pageReqVO 分页查询
* @return 敏感词分页
*/
PageResult<SensitiveWordDO> getSensitiveWordPage(SensitiveWordPageReqVO pageReqVO);
/**
* 获得所有敏感词的标签数组
*
* @return 标签数组
*/
Set<String> getSensitiveWordTagSet();
/**
* 获得文本所包含的不合法的敏感词数组
*
* @param text 文本
* @param tags 标签数组
* @return 不合法的敏感词数组
*/
List<String> validateText(String text, List<String> tags);
/**
* 判断文本是否包含敏感词
*
* @param text 文本
* @param tags 标签数组
* @return 是否包含敏感词
*/
boolean isTextValid(String text, List<String> tags);
}

View File

@ -1,262 +0,0 @@
package cn.iocoder.yudao.module.system.service.sensitiveword;
import cn.hutool.core.collection.CollUtil;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordPageReqVO;
import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordSaveVO;
import cn.iocoder.yudao.module.system.dal.dataobject.sensitiveword.SensitiveWordDO;
import cn.iocoder.yudao.module.system.dal.mysql.sensitiveword.SensitiveWordMapper;
import cn.iocoder.yudao.module.system.util.collection.SimpleTrie;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.util.Assert;
import org.springframework.validation.annotation.Validated;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.Resource;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.TimeUnit;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.filterList;
import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.getMaxValue;
import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SENSITIVE_WORD_EXISTS;
import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SENSITIVE_WORD_NOT_EXISTS;
/**
* 敏感词 Service 实现类
*
* @author 永不言败
*/
@Service
@Slf4j
@Validated
public class SensitiveWordServiceImpl implements SensitiveWordService {
/**
* 是否开启敏感词功能
*/
public static Boolean ENABLED = false;
/**
* 敏感词列表缓存
*/
@Getter
private volatile List<SensitiveWordDO> sensitiveWordCache = Collections.emptyList();
/**
* 敏感词标签缓存
* key敏感词编号 {@link SensitiveWordDO#getId()}
* <p>
* 这里声明 volatile 修饰的原因是每次刷新时直接修改指向
*/
@Getter
private volatile Set<String> sensitiveWordTagsCache = Collections.emptySet();
@Resource
private SensitiveWordMapper sensitiveWordMapper;
/**
* 默认的敏感词的字典树包含所有敏感词
*/
@Getter
private volatile SimpleTrie defaultSensitiveWordTrie = new SimpleTrie(Collections.emptySet());
/**
* 标签与敏感词的字段数的映射
*/
@Getter
private volatile Map<String, SimpleTrie> tagSensitiveWordTries = Collections.emptyMap();
/**
* 初始化缓存
*/
@PostConstruct
public void initLocalCache() {
if (!ENABLED) {
return;
}
// 第一步查询数据
List<SensitiveWordDO> sensitiveWords = sensitiveWordMapper.selectList();
log.info("[initLocalCache][缓存敏感词,数量为:{}]", sensitiveWords.size());
// 第二步构建缓存
// 写入 sensitiveWordTagsCache 缓存
Set<String> tags = new HashSet<>();
sensitiveWords.forEach(word -> tags.addAll(word.getTags()));
sensitiveWordTagsCache = tags;
sensitiveWordCache = sensitiveWords;
// 写入 defaultSensitiveWordTrietagSensitiveWordTries 缓存
initSensitiveWordTrie(sensitiveWords);
}
private void initSensitiveWordTrie(List<SensitiveWordDO> wordDOs) {
// 过滤禁用的敏感词
wordDOs = filterList(wordDOs, word -> word.getStatus().equals(CommonStatusEnum.ENABLE.getStatus()));
// 初始化默认的 defaultSensitiveWordTrie
this.defaultSensitiveWordTrie = new SimpleTrie(CollectionUtils.convertList(wordDOs, SensitiveWordDO::getName));
// 初始化 tagSensitiveWordTries
Multimap<String, String> tagWords = HashMultimap.create();
for (SensitiveWordDO word : wordDOs) {
if (CollUtil.isEmpty(word.getTags())) {
continue;
}
word.getTags().forEach(tag -> tagWords.put(tag, word.getName()));
}
// 添加到 tagSensitiveWordTries
Map<String, SimpleTrie> tagSensitiveWordTries = new HashMap<>();
tagWords.asMap().forEach((tag, words) -> tagSensitiveWordTries.put(tag, new SimpleTrie(words)));
this.tagSensitiveWordTries = tagSensitiveWordTries;
}
/**
* 通过定时任务轮询刷新缓存
*
* 目的多节点部署时通过轮询通知所有节点进行刷新
*/
@Scheduled(initialDelay = 60, fixedRate = 60, timeUnit = TimeUnit.SECONDS)
public void refreshLocalCache() {
// 情况一如果缓存里没有数据则直接刷新缓存
if (CollUtil.isEmpty(sensitiveWordCache)) {
initLocalCache();
return;
}
// 情况二如果缓存里数据则通过 updateTime 判断是否有数据变更有变更则刷新缓存
LocalDateTime maxTime = getMaxValue(sensitiveWordCache, SensitiveWordDO::getUpdateTime);
if (sensitiveWordMapper.selectCountByUpdateTimeGt(maxTime) > 0) {
initLocalCache();
}
}
@Override
public Long createSensitiveWord(SensitiveWordSaveVO createReqVO) {
// 校验唯一性
validateSensitiveWordNameUnique(null, createReqVO.getName());
// 插入
SensitiveWordDO sensitiveWord = BeanUtils.toBean(createReqVO, SensitiveWordDO.class);
sensitiveWordMapper.insert(sensitiveWord);
// 刷新缓存
initLocalCache();
return sensitiveWord.getId();
}
@Override
public void updateSensitiveWord(SensitiveWordSaveVO updateReqVO) {
// 校验唯一性
validateSensitiveWordExists(updateReqVO.getId());
validateSensitiveWordNameUnique(updateReqVO.getId(), updateReqVO.getName());
// 更新
SensitiveWordDO updateObj = BeanUtils.toBean(updateReqVO, SensitiveWordDO.class);
sensitiveWordMapper.updateById(updateObj);
// 刷新缓存
initLocalCache();
}
@Override
public void deleteSensitiveWord(Long id) {
// 校验存在
validateSensitiveWordExists(id);
// 删除
sensitiveWordMapper.deleteById(id);
// 刷新缓存
initLocalCache();
}
private void validateSensitiveWordNameUnique(Long id, String name) {
SensitiveWordDO word = sensitiveWordMapper.selectByName(name);
if (word == null) {
return;
}
// 如果 id 为空说明不用比较是否为相同 id 的敏感词
if (id == null) {
throw exception(SENSITIVE_WORD_EXISTS);
}
if (!word.getId().equals(id)) {
throw exception(SENSITIVE_WORD_EXISTS);
}
}
private void validateSensitiveWordExists(Long id) {
if (sensitiveWordMapper.selectById(id) == null) {
throw exception(SENSITIVE_WORD_NOT_EXISTS);
}
}
@Override
public SensitiveWordDO getSensitiveWord(Long id) {
return sensitiveWordMapper.selectById(id);
}
@Override
public List<SensitiveWordDO> getSensitiveWordList() {
return sensitiveWordMapper.selectList();
}
@Override
public PageResult<SensitiveWordDO> getSensitiveWordPage(SensitiveWordPageReqVO pageReqVO) {
return sensitiveWordMapper.selectPage(pageReqVO);
}
@Override
public Set<String> getSensitiveWordTagSet() {
return sensitiveWordTagsCache;
}
@Override
public List<String> validateText(String text, List<String> tags) {
Assert.isTrue(ENABLED, "敏感词功能未开启,请将 ENABLED 设置为 true");
// 无标签时默认所有
if (CollUtil.isEmpty(tags)) {
return defaultSensitiveWordTrie.validate(text);
}
// 有标签的情况
Set<String> result = new HashSet<>();
tags.forEach(tag -> {
SimpleTrie trie = tagSensitiveWordTries.get(tag);
if (trie == null) {
return;
}
result.addAll(trie.validate(text));
});
return new ArrayList<>(result);
}
@Override
public boolean isTextValid(String text, List<String> tags) {
Assert.isTrue(ENABLED, "敏感词功能未开启,请将 ENABLED 设置为 true");
// 无标签时默认所有
if (CollUtil.isEmpty(tags)) {
return defaultSensitiveWordTrie.isValid(text);
}
// 有标签的情况
for (String tag : tags) {
SimpleTrie trie = tagSensitiveWordTries.get(tag);
if (trie == null) {
continue;
}
// 如果有一个标签不合法则返回 false 不合法
if (!trie.isValid(text)) {
return false;
}
}
return true;
}
}

View File

@ -1,152 +0,0 @@
package cn.iocoder.yudao.module.system.util.collection;
import cn.hutool.core.collection.CollUtil;
import java.util.*;
/**
* 基于前缀树实现敏感词的校验
* <p>
* 相比 Apache Common 提供的 PatriciaTrie 来说性能可能会更加好一些
*
* @author 芋道源码
*/
@SuppressWarnings("unchecked")
public class SimpleTrie {
/**
* 一个敏感词结束后对应的 key
*/
private static final Character CHARACTER_END = '\0';
/**
* 使用敏感词构建的前缀树
*/
private final Map<Character, Object> children;
/**
* 基于字符串构建前缀树
*
* @param strs 字符串数组
*/
public SimpleTrie(Collection<String> strs) {
// 排序优先使用较短的前缀
strs = CollUtil.sort(strs, String::compareTo);
// 构建树
children = new HashMap<>();
for (String str : strs) {
Map<Character, Object> child = children;
// 遍历每个字符
for (Character c : str.toCharArray()) {
// 如果已经到达结束就没必要在添加更长的敏感词
// 例如说有两个敏感词是吃饭啊吃饭输入一句话是 我要吃饭啊则只要匹配到 吃饭 这个敏感词即可
if (child.containsKey(CHARACTER_END)) {
break;
}
if (!child.containsKey(c)) {
child.put(c, new HashMap<>());
}
child = (Map<Character, Object>) child.get(c);
}
// 结束
child.put(CHARACTER_END, null);
}
}
/**
* 验证文本是否合法即不包含敏感词
*
* @param text 文本
* @return 是否 true-合法 false-不合法
*/
public boolean isValid(String text) {
// 遍历 text使用每一个 [i, n) 段的字符串使用 children 前缀树匹配是否包含敏感词
for (int i = 0; i < text.length(); i++) {
Map<Character, Object> child = (Map<Character, Object>) children.get(text.charAt(i));
if (child == null) {
continue;
}
boolean ok = recursion(text, i + 1, child);
if (!ok) {
return false;
}
}
return true;
}
/**
* 验证文本从指定位置开始是否不包含某个敏感词
*
* @param text 文本
* @param index 开始位置
* @param child 节点当前遍历到的
* @return 是否不包含 true-不包含 false-包含
*/
private boolean recursion(String text, int index, Map<Character, Object> child) {
if (child.containsKey(CHARACTER_END)) {
return false;
}
if (index == text.length()) {
return true;
}
child = (Map<Character, Object>) child.get(text.charAt(index));
return child == null || !child.containsKey(CHARACTER_END) && recursion(text, ++index, child);
}
/**
* 获得文本所包含的不合法的敏感词
*
* 注意才当即最短匹配原则例如说当敏感词存在 煞笔煞笔二货 只会返回 煞笔
*
* @param text 文本
* @return 匹配的敏感词
*/
public List<String> validate(String text) {
Set<String> results = new HashSet<>();
for (int i = 0; i < text.length(); i++) {
Character c = text.charAt(i);
Map<Character, Object> child = (Map<Character, Object>) children.get(c);
if (child == null) {
continue;
}
StringBuilder result = new StringBuilder().append(c);
boolean ok = recursionWithResult(text, i + 1, child, result);
if (!ok) {
results.add(result.toString());
}
}
return new ArrayList<>(results);
}
/**
* 返回文本从 index 开始的敏感词并使用 StringBuilder 参数进行返回
*
* 逻辑和 {@link #recursion(String, int, Map)} 是一致只是多了 result 返回结果
*
* @param text 文本
* @param index 开始未知
* @param child 节点当前遍历到的
* @param result 返回敏感词
* @return 是否有敏感词
*/
@SuppressWarnings("unchecked")
private static boolean recursionWithResult(String text, int index, Map<Character, Object> child, StringBuilder result) {
if (child.containsKey(CHARACTER_END)) {
return false;
}
if (index == text.length()) {
return true;
}
Character c = text.charAt(index);
child = (Map<Character, Object>) child.get(c);
if (child == null) {
return true;
}
if (child.containsKey(CHARACTER_END)) {
result.append(c);
return false;
}
return recursionWithResult(text, ++index, child, result.append(c));
}
}

View File

@ -1,303 +0,0 @@
package cn.iocoder.yudao.module.system.service.sensitiveword;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.collection.SetUtils;
import cn.iocoder.yudao.framework.common.util.date.LocalDateTimeUtils;
import cn.iocoder.yudao.framework.test.core.ut.BaseDbUnitTest;
import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordPageReqVO;
import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordSaveVO;
import cn.iocoder.yudao.module.system.dal.dataobject.sensitiveword.SensitiveWordDO;
import cn.iocoder.yudao.module.system.dal.mysql.sensitiveword.SensitiveWordMapper;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.context.annotation.Import;
import jakarta.annotation.Resource;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.List;
import static cn.iocoder.yudao.framework.common.util.date.LocalDateTimeUtils.buildBetweenTime;
import static cn.iocoder.yudao.framework.common.util.date.LocalDateTimeUtils.buildTime;
import static cn.iocoder.yudao.framework.common.util.object.ObjectUtils.cloneIgnoreId;
import static cn.iocoder.yudao.framework.test.core.util.AssertUtils.assertPojoEquals;
import static cn.iocoder.yudao.framework.test.core.util.AssertUtils.assertServiceException;
import static cn.iocoder.yudao.framework.test.core.util.RandomUtils.randomLongId;
import static cn.iocoder.yudao.framework.test.core.util.RandomUtils.randomPojo;
import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SENSITIVE_WORD_NOT_EXISTS;
import static java.util.Collections.singletonList;
import static org.junit.jupiter.api.Assertions.*;
/**
* {@link SensitiveWordServiceImpl} 的单元测试类
*
* @author 永不言败
*/
@Import(SensitiveWordServiceImpl.class)
public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
@Resource
private SensitiveWordServiceImpl sensitiveWordService;
@Resource
private SensitiveWordMapper sensitiveWordMapper;
@BeforeEach
public void setUp() {
SensitiveWordServiceImpl.ENABLED = true;
}
@Test
public void testInitLocalCache() {
SensitiveWordDO wordDO1 = randomPojo(SensitiveWordDO.class, o -> o.setName("傻瓜")
.setTags(singletonList("论坛")).setStatus(CommonStatusEnum.ENABLE.getStatus()));
sensitiveWordMapper.insert(wordDO1);
SensitiveWordDO wordDO2 = randomPojo(SensitiveWordDO.class, o -> o.setName("笨蛋")
.setTags(singletonList("蔬菜")).setStatus(CommonStatusEnum.ENABLE.getStatus()));
sensitiveWordMapper.insert(wordDO2);
SensitiveWordDO wordDO3 = randomPojo(SensitiveWordDO.class, o -> o.setName("")
.setTags(singletonList("测试")).setStatus(CommonStatusEnum.ENABLE.getStatus()));
sensitiveWordMapper.insert(wordDO3);
SensitiveWordDO wordDO4 = randomPojo(SensitiveWordDO.class, o -> o.setName("白痴")
.setTags(singletonList("测试")).setStatus(CommonStatusEnum.ENABLE.getStatus()));
sensitiveWordMapper.insert(wordDO4);
// 调用
sensitiveWordService.initLocalCache();
// 断言 sensitiveWordTagsCache 缓存
assertEquals(SetUtils.asSet("论坛", "蔬菜", "测试"), sensitiveWordService.getSensitiveWordTagSet());
// 断言 sensitiveWordCache
assertEquals(4, sensitiveWordService.getSensitiveWordCache().size());
assertPojoEquals(wordDO1, sensitiveWordService.getSensitiveWordCache().get(0));
assertPojoEquals(wordDO2, sensitiveWordService.getSensitiveWordCache().get(1));
assertPojoEquals(wordDO3, sensitiveWordService.getSensitiveWordCache().get(2));
// 断言 tagSensitiveWordTries 缓存
assertNotNull(sensitiveWordService.getDefaultSensitiveWordTrie());
assertEquals(3, sensitiveWordService.getTagSensitiveWordTries().size());
assertNotNull(sensitiveWordService.getTagSensitiveWordTries().get("论坛"));
assertNotNull(sensitiveWordService.getTagSensitiveWordTries().get("蔬菜"));
assertNotNull(sensitiveWordService.getTagSensitiveWordTries().get("测试"));
}
@Test
public void testRefreshLocalCache() {
// mock 数据
SensitiveWordDO wordDO1 = randomPojo(SensitiveWordDO.class, o -> o.setName("傻瓜")
.setTags(singletonList("论坛")).setStatus(CommonStatusEnum.ENABLE.getStatus()));
wordDO1.setUpdateTime(LocalDateTime.now());
sensitiveWordMapper.insert(wordDO1);
sensitiveWordService.initLocalCache();
// mock 数据
SensitiveWordDO wordDO2 = randomPojo(SensitiveWordDO.class, o -> o.setName("笨蛋")
.setTags(singletonList("蔬菜")).setStatus(CommonStatusEnum.ENABLE.getStatus()));
wordDO2.setUpdateTime(LocalDateTimeUtils.addTime(Duration.ofMinutes(1))); // 避免时间相同
sensitiveWordMapper.insert(wordDO2);
// 调用
sensitiveWordService.refreshLocalCache();
// 断言 sensitiveWordTagsCache 缓存
assertEquals(SetUtils.asSet("论坛", "蔬菜"), sensitiveWordService.getSensitiveWordTagSet());
// 断言 sensitiveWordCache
assertEquals(2, sensitiveWordService.getSensitiveWordCache().size());
assertPojoEquals(wordDO1, sensitiveWordService.getSensitiveWordCache().get(0));
assertPojoEquals(wordDO2, sensitiveWordService.getSensitiveWordCache().get(1));
// 断言 tagSensitiveWordTries 缓存
assertNotNull(sensitiveWordService.getDefaultSensitiveWordTrie());
assertEquals(2, sensitiveWordService.getTagSensitiveWordTries().size());
assertNotNull(sensitiveWordService.getTagSensitiveWordTries().get("论坛"));
assertNotNull(sensitiveWordService.getTagSensitiveWordTries().get("蔬菜"));
}
@Test
public void testCreateSensitiveWord_success() {
// 准备参数
SensitiveWordSaveVO reqVO = randomPojo(SensitiveWordSaveVO.class)
.setId(null); // 防止 id 被赋值
// 调用
Long sensitiveWordId = sensitiveWordService.createSensitiveWord(reqVO);
// 断言
assertNotNull(sensitiveWordId);
// 校验记录的属性是否正确
SensitiveWordDO sensitiveWord = sensitiveWordMapper.selectById(sensitiveWordId);
assertPojoEquals(reqVO, sensitiveWord, "id");
}
@Test
public void testUpdateSensitiveWord_success() {
// mock 数据
SensitiveWordDO dbSensitiveWord = randomPojo(SensitiveWordDO.class);
sensitiveWordMapper.insert(dbSensitiveWord);// @Sql: 先插入出一条存在的数据
// 准备参数
SensitiveWordSaveVO reqVO = randomPojo(SensitiveWordSaveVO.class, o -> {
o.setId(dbSensitiveWord.getId()); // 设置更新的 ID
});
// 调用
sensitiveWordService.updateSensitiveWord(reqVO);
// 校验是否更新正确
SensitiveWordDO sensitiveWord = sensitiveWordMapper.selectById(reqVO.getId()); // 获取最新的
assertPojoEquals(reqVO, sensitiveWord);
}
@Test
public void testUpdateSensitiveWord_notExists() {
// 准备参数
SensitiveWordSaveVO reqVO = randomPojo(SensitiveWordSaveVO.class);
// 调用, 并断言异常
assertServiceException(() -> sensitiveWordService.updateSensitiveWord(reqVO), SENSITIVE_WORD_NOT_EXISTS);
}
@Test
public void testDeleteSensitiveWord_success() {
// mock 数据
SensitiveWordDO dbSensitiveWord = randomPojo(SensitiveWordDO.class);
sensitiveWordMapper.insert(dbSensitiveWord);// @Sql: 先插入出一条存在的数据
// 准备参数
Long id = dbSensitiveWord.getId();
// 调用
sensitiveWordService.deleteSensitiveWord(id);
// 校验数据不存在了
assertNull(sensitiveWordMapper.selectById(id));
}
@Test
public void testDeleteSensitiveWord_notExists() {
// 准备参数
Long id = randomLongId();
// 调用, 并断言异常
assertServiceException(() -> sensitiveWordService.deleteSensitiveWord(id), SENSITIVE_WORD_NOT_EXISTS);
}
@Test
public void testGetSensitiveWord() {
// mock 数据
SensitiveWordDO sensitiveWord = randomPojo(SensitiveWordDO.class);
sensitiveWordMapper.insert(sensitiveWord);
// 准备参数
Long id = sensitiveWord.getId();
// 调用
SensitiveWordDO dbSensitiveWord = sensitiveWordService.getSensitiveWord(id);
// 断言
assertPojoEquals(sensitiveWord, dbSensitiveWord);
}
@Test
public void testGetSensitiveWordList() {
// mock 数据
SensitiveWordDO sensitiveWord01 = randomPojo(SensitiveWordDO.class);
sensitiveWordMapper.insert(sensitiveWord01);
SensitiveWordDO sensitiveWord02 = randomPojo(SensitiveWordDO.class);
sensitiveWordMapper.insert(sensitiveWord02);
// 调用
List<SensitiveWordDO> list = sensitiveWordService.getSensitiveWordList();
// 断言
assertEquals(2, list.size());
assertEquals(sensitiveWord01, list.get(0));
assertEquals(sensitiveWord02, list.get(1));
}
@Test
public void testGetSensitiveWordPage() {
// mock 数据
SensitiveWordDO dbSensitiveWord = randomPojo(SensitiveWordDO.class, o -> { // 等会查询到
o.setName("笨蛋");
o.setTags(Arrays.asList("论坛", "蔬菜"));
o.setStatus(CommonStatusEnum.ENABLE.getStatus());
o.setCreateTime(buildTime(2022, 2, 8));
});
sensitiveWordMapper.insert(dbSensitiveWord);
// 测试 name 不匹配
sensitiveWordMapper.insert(cloneIgnoreId(dbSensitiveWord, o -> o.setName("傻瓜")));
// 测试 tags 不匹配
sensitiveWordMapper.insert(cloneIgnoreId(dbSensitiveWord, o -> o.setTags(Arrays.asList("短信", "日用品"))));
// 测试 createTime 不匹配
sensitiveWordMapper.insert(cloneIgnoreId(dbSensitiveWord, o -> o.setCreateTime(buildTime(2022, 2, 16))));
// 准备参数
SensitiveWordPageReqVO reqVO = new SensitiveWordPageReqVO();
reqVO.setName("");
reqVO.setTag("论坛");
reqVO.setStatus(CommonStatusEnum.ENABLE.getStatus());
reqVO.setCreateTime(buildBetweenTime(2022, 2, 1, 2022, 2, 12));
// 调用
PageResult<SensitiveWordDO> pageResult = sensitiveWordService.getSensitiveWordPage(reqVO);
// 断言
assertEquals(1, pageResult.getTotal());
assertEquals(1, pageResult.getList().size());
assertPojoEquals(dbSensitiveWord, pageResult.getList().get(0));
}
@Test
public void testValidateText_noTag() {
testInitLocalCache();
// 准备参数
String text = "你是傻瓜,你是笨蛋";
// 调用
List<String> result = sensitiveWordService.validateText(text, null);
// 断言
assertEquals(Arrays.asList("傻瓜", "笨蛋"), result);
// 准备参数
String text2 = "你是傻瓜,你是笨蛋,你是白";
// 调用
List<String> result2 = sensitiveWordService.validateText(text2, null);
// 断言
assertEquals(Arrays.asList("傻瓜", "笨蛋",""), result2);
}
@Test
public void testValidateText_hasTag() {
testInitLocalCache();
// 准备参数
String text = "你是傻瓜,你是笨蛋";
// 调用
List<String> result = sensitiveWordService.validateText(text, singletonList("论坛"));
// 断言
assertEquals(singletonList("傻瓜"), result);
// 准备参数
String text2 = "你是白";
// 调用
List<String> result2 = sensitiveWordService.validateText(text2, singletonList("测试"));
// 断言
assertEquals(singletonList(""), result2);
}
@Test
public void testIsTestValid_noTag() {
testInitLocalCache();
// 准备参数
String text = "你是傻瓜,你是笨蛋";
// 调用断言
assertFalse(sensitiveWordService.isTextValid(text, null));
// 准备参数
String text2 = "你是白";
// 调用断言
assertFalse(sensitiveWordService.isTextValid(text2, null));
}
@Test
public void testIsTestValid_hasTag() {
testInitLocalCache();
// 准备参数
String text = "你是傻瓜,你是笨蛋";
// 调用断言
assertFalse(sensitiveWordService.isTextValid(text, singletonList("论坛")));
// 准备参数
String text2 = "你是白";
// 调用断言
assertFalse(sensitiveWordService.isTextValid(text2, singletonList("测试")));
}
}

View File

@ -21,7 +21,6 @@ DELETE FROM "system_social_user";
DELETE FROM "system_social_user_bind";
DELETE FROM "system_tenant";
DELETE FROM "system_tenant_package";
DELETE FROM "system_sensitive_word";
DELETE FROM "system_oauth2_client";
DELETE FROM "system_oauth2_approve";
DELETE FROM "system_oauth2_access_token";

View File

@ -416,20 +416,6 @@ CREATE TABLE IF NOT EXISTS "system_tenant_package" (
PRIMARY KEY ("id")
) COMMENT '租户套餐表';
CREATE TABLE IF NOT EXISTS "system_sensitive_word" (
"id" bigint NOT NULL GENERATED BY DEFAULT AS IDENTITY,
"name" varchar(255) NOT NULL,
"tags" varchar(1024) NOT NULL,
"status" bit NOT NULL DEFAULT FALSE,
"description" varchar(512),
"creator" varchar(64) DEFAULT '',
"create_time" datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updater" varchar(64) DEFAULT '',
"update_time" datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
"deleted" bit NOT NULL DEFAULT FALSE,
PRIMARY KEY ("id")
) COMMENT '系统敏感词';
CREATE TABLE IF NOT EXISTS "system_oauth2_client" (
"id" bigint NOT NULL GENERATED BY DEFAULT AS IDENTITY,
"client_id" varchar NOT NULL,

View File

@ -170,7 +170,6 @@ logging:
cn.iocoder.yudao.module.pay.dal.mysql: debug
cn.iocoder.yudao.module.pay.dal.mysql.notify.PayNotifyTaskMapper: INFO # 配置 JobLogMapper 的日志级别为 info
cn.iocoder.yudao.module.system.dal.mysql: debug
cn.iocoder.yudao.module.system.dal.mysql.sensitiveword.SensitiveWordMapper: INFO # 配置 SensitiveWordMapper 的日志级别为 info
cn.iocoder.yudao.module.system.dal.mysql.sms.SmsChannelMapper: INFO # 配置 SmsChannelMapper 的日志级别为 info
cn.iocoder.yudao.module.tool.dal.mysql: debug
cn.iocoder.yudao.module.member.dal.mysql: debug