DATA_MIDDLE_PLATFORM/Data_Middle_Platform_Server
3
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

数据过滤工具类

Browse Source
This commit is contained in:
zhangtao 2025-01-05 19:58:51 +08:00
parent 9e0a17c996
commit 47c88e3ebc
1 changed files with 20 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/utils/DataProcessUtil.java
View File

@ -400,15 +400,16 @@ public class DataProcessUtil {
text = hashMatcher.replaceAll("");
// 使用StringBuilder和StringBuilder的replace方法去除其他数字，但跳过年份和简单数字
StringBuilder sb = new StringBuilder(text);
int index = 0;
while ((index = findNextNumberToReplace(sb.toString())) != -1) {
String number = sb.substring(index, findEndOfNumber(sb.toString(), index));
if (!isYear(number) && !isSimpleNumber(number)) {
sb.replace(index, index + number.length(), "");
}
}
return sb.toString();
// TODO: 这里目前有bug，先注释掉了。
// StringBuilder sb = new StringBuilder(text);
// int index = 0;
// while ((index = findNextNumberToReplace(sb.toString())) != -1) {
// String number = sb.substring(index, findEndOfNumber(sb.toString(), index));
// if (!isYear(number) && !isSimpleNumber(number)) {
// sb.replace(index, index + number.length(), "");
// }
// }
return text;
}
// 查找下一个要替换的数字的起始索引
@ -463,4 +464,14 @@ public class DataProcessUtil {
return false;
}
}
public static void main(String[] args) {
String textWithIdentifiers = "Here are some identifiers: 123-456-7890, 1234567812345678, a1b2c3d4e5f6a1b2c3d4e5f6, 2023, and 987654.";
// 去除标识符
String textWithoutIdentifiers = removeIdentifiers(textWithIdentifiers);
// 打印结果
System.out.println(textWithoutIdentifiers);
}
}