做网站怎么调用栏目,纪念册设计制作公司,银川做网站推广,电商网站设计公司有哪些项目需要对用户提交的附件、文章、搜索框内容等做热词分析。如下图#xff1a; 公司有大数据团队。本着不麻烦别人就不麻烦别人的原则#xff0c;写了一款简易的记录工具#xff0c;原理也简单#xff0c;手工在业务插入锚点#xff0c;用分词器分好词#xff0c;排掉字…项目需要对用户提交的附件、文章、搜索框内容等做热词分析。如下图 公司有大数据团队。本着不麻烦别人就不麻烦别人的原则写了一款简易的记录工具原理也简单手工在业务插入锚点用分词器分好词排掉字母、数字、符号、敏感词。将词汇按年度累加记录到数据库中即可。代码如下:
Component
public class HotWordHelper {private static HotWordMapper hotWordMapper;static ListCharacter FILTER_CHARS new ArrayList();static {String number 123456789abcdefghijklnmopqrstuvwxyzABCDEFGHIJKLNMOPQRSTUVWXYZ;char[] chars number.toCharArray();for (char aChar : chars) {FILTER_CHARS.add(aChar);}}Autowiredpublic ZYHotWordHelper(HotWordMapper hotWordMapper) {ZYHotWordHelper.hotWordMapper hotWordMapper;}public static ListHotWord loaderHotWordTen(String moduleCode) {LocalDate now LocalDate.now();int year now.getYear();return loaderHotWord(year, 10, moduleCode);}public static ListHotWord loaderHotWord(int top, String moduleCode) {LocalDate now LocalDate.now();int year now.getYear();return loaderHotWord(year, top, moduleCode);}public static ListHotWord loaderHotWord(int year, int top, String moduleCode) {LambdaQueryWrapperHotWord wrapper Wrappers.lambdaQuery();wrapper.eq(HotWord::getRecordYear, year);wrapper.eq(HotWord::getModuleCode, moduleCode);return hotWordMapper.selectTop(wrapper, HotWord::getAppearTimes, top);}// 直接词汇如字典之类的。public static void putDirectHotWord(String text, String moduleCode) {// 为不影响主业务速度改成异常Runnable runnable () - putHotWord(true, text, moduleCode);AsyncExecutor.execute(runnable);}// 分析词汇public static void putAnalyzeHotWord(String text, String moduleCode) {// 为不影响主业务速度改成异常Runnable runnable () - putHotWord(false, text, moduleCode);AsyncExecutor.execute(runnable);}// 附件public static void putAttachmentAsync(StringsField attachmentIds, String moduleCode) {if (ZYListUtils.isEmptyList(attachmentIds)) {return;}Runnable runnable () - doPutAttachmentAsync(attachmentIds, moduleCode);AsyncExecutor.execute(runnable);}// 解析附件private static void doPutAttachmentAsync(StringsField attachmentIds, String moduleCode) {FileInfoMapper fileInfoMapper SpringUtils.getBean(FileInfoMapper.class);ListFileInfo fileInfos fileInfoMapper.selectBatchIds(attachmentIds);if (ZYListUtils.isEmptyList(fileInfos)) {return;}FileStoreService storeService ZYSpringUtils.getBean(FileStoreService.class);ListFileWrapper fileWrappers ZYListUtils.list2list(fileInfos, FileInfo::toFileWrapper);for (FileWrapper fileWrapper : fileWrappers) {try (InputStream objectStream storeService.getObjectStream(fileWrapper)) {String text IOUtils.toString(objectStream, StandardCharsets.UTF_8);putAnalyzeHotWord(text, moduleCode);} catch (Exception e) {return;}}}private static void putHotWord(boolean isDirect, String text, String moduleCode) {if (ZYStrUtils.isAnyNull(text, moduleCode)) {return;}ListString words analyzerWords(isDirect, text);if (ZYListUtils.isEmptyList(words)) {return;}// ListString smallWordCompare new ArrayList(words);words.removeIf(w - {if (!matchLength(w)) {return true;}char[] chars w.toCharArray();for (char aChar : chars) {// 不要数字字母if (FILTER_CHARS.contains(aChar)) {return true;}}// 存在误判还是不用这段代码/* for (String compareWord : smallWordCompare) {if (!w.equals(compareWord) compareWord.contains(w)) {return true;}}
*/return false;});MapString, Integer wordCount ZYMapUtils.countField(words, w - w);LocalDate now LocalDate.now();int year now.getYear();LambdaQueryWrapperHotWord wrapper Wrappers.lambdaQuery();wrapper.in(HotWord::getHotWord, words);wrapper.eq(HotWord::getRecordYear, year);wrapper.eq(HotWord::getModuleCode, moduleCode);ListHotWord existsWords hotWordMapper.selectList(wrapper);MapString, HotWord wordIdContainer ZYListUtils.groupModel(existsWords, HotWord::getHotWord);ListHotWord addHotWords new ArrayList();ListHotWord editHotWords new ArrayList();wordCount.forEach((w, times) - {HotWord hotWord wordIdContainer.get(w);if (null ! hotWord) {Integer appearTimes hotWord.getAppearTimes();appearTimes times;hotWord.setAppearTimes(appearTimes);editHotWords.add(hotWord);} else {HotWord newHotWord new HotWord();newHotWord.setRecordYear(year);newHotWord.setAppearTimes(times);newHotWord.setHotWord(w);newHotWord.setModuleCode(moduleCode);addHotWords.add(newHotWord);}});if (ZYListUtils.isNotEmptyList(addHotWords)) {hotWordMapper.insertBatch(addHotWords);}if (ZYListUtils.isNotEmptyList(editHotWords)) {for (HotWord editHotWord : editHotWords) {hotWordMapper.updateById(editHotWord);}}}private static ListString analyzerWords(boolean isDirect, String text) {ListString words;if (isDirect) {words Collections.singletonList(text);} else {words ZYDirtyWordHelper.analyze(text);if (matchLength(text) !words.contains(text)) {words.add(text);}}return words;}private static boolean matchLength(String text) {int length text.length();return length 1 length 6;}}记录表中效果图,实际效果还阔以实时性和记录速度都非常快。
