diff --git a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/utils/DataProcessUtil.java b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/utils/DataProcessUtil.java index 7ca626f09..0de44a2fc 100644 --- a/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/utils/DataProcessUtil.java +++ b/yudao-module-llm/yudao-module-llm-biz/src/main/java/cn/iocoder/yudao/module/llm/utils/DataProcessUtil.java @@ -400,15 +400,16 @@ public class DataProcessUtil { text = hashMatcher.replaceAll(""); // 使用StringBuilder和StringBuilder的replace方法去除其他数字,但跳过年份和简单数字 - StringBuilder sb = new StringBuilder(text); - int index = 0; - while ((index = findNextNumberToReplace(sb.toString())) != -1) { - String number = sb.substring(index, findEndOfNumber(sb.toString(), index)); - if (!isYear(number) && !isSimpleNumber(number)) { - sb.replace(index, index + number.length(), ""); - } - } - return sb.toString(); + // TODO: 这里目前有bug,先注释掉了。 +// StringBuilder sb = new StringBuilder(text); +// int index = 0; +// while ((index = findNextNumberToReplace(sb.toString())) != -1) { +// String number = sb.substring(index, findEndOfNumber(sb.toString(), index)); +// if (!isYear(number) && !isSimpleNumber(number)) { +// sb.replace(index, index + number.length(), ""); +// } +// } + return text; } // 查找下一个要替换的数字的起始索引 @@ -463,4 +464,14 @@ public class DataProcessUtil { return false; } } + + public static void main(String[] args) { + String textWithIdentifiers = "Here are some identifiers: 123-456-7890, 1234567812345678, a1b2c3d4e5f6a1b2c3d4e5f6, 2023, and 987654."; + + // 去除标识符 + String textWithoutIdentifiers = removeIdentifiers(textWithIdentifiers); + + // 打印结果 + System.out.println(textWithoutIdentifiers); + } }