DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

数据过滤工具类

Browse Source
This commit is contained in:
zhangtao 2025-01-05 20:01:18 +08:00
parent 47c88e3ebc
commit ea506b84db
2 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/service/async/AsyncDataProcessService.java
View File

@ -1,6 +1,5 @@
package cn.iocoder.yudao.module.llm.service.async;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
@ -125,7 +124,7 @@ public class AsyncDataProcessService {
// 繁体转简体 繁体转简体，如“不經意，妳的笑容”清洗成“不经意，你的笑容”
JSONObject a_4 = a.getJSONObject("a_4");
if (a_4.getBool("is_on")) {
result = DataProcessUtil.TraditionalToSimplified(result);
result = DataProcessUtil.traditionalToSimplified(result);
}
// 去除网页标识符 移除文档中的html标签，如<html>,<dev><p>等
JSONObject a_5 = a.getJSONObject("a_5");

10
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/utils/DataProcessUtil.java
View File

@ -77,7 +77,7 @@ public class DataProcessUtil {
* @param input
* @return
*/
public static String TraditionalToSimplified(String input) {
public static String traditionalToSimplified(String input) {
return ZhConverterUtil.toSimple(input);
}
@ -467,11 +467,15 @@ public class DataProcessUtil {
public static void main(String[] args) {
String textWithIdentifiers = "Here are some identifiers: 123-456-7890, 1234567812345678, a1b2c3d4e5f6a1b2c3d4e5f6, 2023, and 987654.";
// 去除标识符
String textWithoutIdentifiers = removeIdentifiers(textWithIdentifiers);
// 打印结果
System.out.println(textWithoutIdentifiers);
String traditionalText = "不經意，妳的笑容";
String simplifiedText = traditionalToSimplified(traditionalText);
System.out.println("繁体文本: [" + traditionalText + "]");
System.out.println("简体文本: [" + simplifiedText + "]");
}
}