private TelNoNormalizer create() { try { Trie<String, Integer> trie = new PatriciaTrie<>(); trie.putAll(parse("soumu/000124070.xls")); trie.putAll(parse("soumu/000124071.xls")); trie.putAll(parse("soumu/000124072.xls")); trie.putAll(parse("soumu/000124073.xls")); trie.putAll(parse("soumu/000124074.xls")); trie.putAll(parse("soumu/000124075.xls")); trie.putAll(parse("soumu/000124076.xls")); trie.putAll(parse("soumu/000124077.xls")); TelNoNormalizerImpl impl = new TelNoNormalizerImpl(); impl.setAreaCodeTable(trie); return impl; } catch (InvalidFormatException | IOException ex) { throw new IllegalStateException(ex); } }
@Test public void test() throws Exception { List<Resource> resources = new ArrayList<>(9); resources.add(new ClassPathResource("cherry/goods/telno/soumu/000124070.xls")); resources.add(new ClassPathResource("cherry/goods/telno/soumu/000124071.xls")); resources.add(new ClassPathResource("cherry/goods/telno/soumu/000124072.xls")); resources.add(new ClassPathResource("cherry/goods/telno/soumu/000124073.xls")); resources.add(new ClassPathResource("cherry/goods/telno/soumu/000124074.xls")); resources.add(new ClassPathResource("cherry/goods/telno/soumu/000124075.xls")); resources.add(new ClassPathResource("cherry/goods/telno/soumu/000124076.xls")); resources.add(new ClassPathResource("cherry/goods/telno/soumu/000124077.xls")); resources.add(new ClassPathResource("cherry/goods/telno/soumu/000124078.xls")); AreaCodeTableFactory factory = new AreaCodeTableFactory(); factory.setSoumuExcelParser(new SoumuExcelParser()); factory.setResources(resources); Trie<String, Integer> trie = factory.getObject(); SortedMap<String, Integer> map = trie.prefixMap("042"); assertEquals(790, map.size()); assertEquals(new TreeSet<Integer>(asList(2, 3, 4)), new TreeSet<>(map.values())); assertEquals(4554 + 5683 + 3400 + 5343 + 5307 + 3000 + 5548 + 4526 + 5330, trie.size()); assertEquals(Trie.class, factory.getObjectType()); assertFalse(factory.isSingleton()); }
private void processContainers(List<ContainerInfo> containersToProcess, List<ContainerInfo> containersToRemove, Map<String, ContainerInfo> seenContainerPathToContainerInfo, Trie<String, MetadataNode> sanitisedRootSearchIndex) { // Lets remove references to files that are no longer present in classpath containersToRemove.forEach( container -> removeReferences(seenContainerPathToContainerInfo, sanitisedRootSearchIndex, container)); for (ContainerInfo containerInfo : containersToProcess) { // lets remove existing references from search index, as these files are modified, so that we can rebuild index if (seenContainerPathToContainerInfo.containsKey(containerInfo.getContainerPath())) { removeReferences(seenContainerPathToContainerInfo, sanitisedRootSearchIndex, containerInfo); } String metadataFilePath = containerInfo.getPath(); try (InputStream inputStream = containerInfo.getMetadataFile().getInputStream()) { SpringConfigurationMetadata springConfigurationMetadata = new Gson() .fromJson(new BufferedReader(new InputStreamReader(inputStream)), SpringConfigurationMetadata.class); buildMetadataHierarchy(sanitisedRootSearchIndex, containerInfo, springConfigurationMetadata); seenContainerPathToContainerInfo.put(containerInfo.getContainerPath(), containerInfo); } catch (IOException e) { log.error("Exception encountered while processing metadata file: " + metadataFilePath, e); removeReferences(seenContainerPathToContainerInfo, sanitisedRootSearchIndex, containerInfo); } } }
private void reindexModule(List<ContainerInfo> newProjectSourcesToProcess, List<ContainerInfo> projectContainersToRemove, Module module) { Map<String, ContainerInfo> moduleSeenContainerPathToSeenContainerInfo = moduleNameToSeenContainerPathToContainerInfo .computeIfAbsent(module.getName(), k -> new HashMap<>()); Trie<String, MetadataNode> moduleSanitisedRootSearchIndex = moduleNameToSanitisedRootSearchIndex.get(module.getName()); if (moduleSanitisedRootSearchIndex == null) { moduleSanitisedRootSearchIndex = new PatriciaTrie<>(); moduleNameToSanitisedRootSearchIndex.put(module.getName(), moduleSanitisedRootSearchIndex); } OrderEnumerator moduleOrderEnumerator = OrderEnumerator.orderEntries(module); List<ContainerInfo> newModuleContainersToProcess = computeNewContainersToProcess(moduleOrderEnumerator, moduleSeenContainerPathToSeenContainerInfo); newModuleContainersToProcess.addAll(newProjectSourcesToProcess); List<ContainerInfo> moduleContainersToRemove = computeContainersToRemove(moduleOrderEnumerator, moduleSeenContainerPathToSeenContainerInfo); moduleContainersToRemove.addAll(projectContainersToRemove); processContainers(newModuleContainersToProcess, moduleContainersToRemove, moduleSeenContainerPathToSeenContainerInfo, moduleSanitisedRootSearchIndex); }
private void removeReferences(Map<String, ContainerInfo> containerPathToContainerInfo, Trie<String, MetadataNode> sanitisedRootSearchIndex, ContainerInfo containerInfo) { debug(() -> log.debug("Removing references to " + containerInfo)); String containerPath = containerInfo.getContainerPath(); containerPathToContainerInfo.remove(containerPath); Iterator<String> searchIndexIterator = sanitisedRootSearchIndex.keySet().iterator(); while (searchIndexIterator.hasNext()) { MetadataNode root = sanitisedRootSearchIndex.get(searchIndexIterator.next()); boolean removeTree = root.removeRef(containerInfo.getContainerPath()); if (removeTree) { searchIndexIterator.remove(); } } }
/** * Constructor that wraps (not copies). * * @param trie the trie to decorate, must not be null * @throws NullPointerException if trie is null */ public UnmodifiableTrie(final Trie<K, ? extends V> trie) { if (trie == null) { throw new NullPointerException("Trie must not be null"); } @SuppressWarnings("unchecked") // safe to upcast final Trie<K, V> tmpTrie = (Trie<K, V>) trie; this.delegate = tmpTrie; }
public void testDecorateFactory() { fr.inria.diversify.testamplification.logger.Logger.writeTestStart(Thread.currentThread(),this, "testDecorateFactory"); final Trie<java.lang.String, V> trie = makeFullMap(); fr.inria.diversify.testamplification.logger.Logger.logAssertArgument(Thread.currentThread(),6355,trie); fr.inria.diversify.testamplification.logger.Logger.logAssertArgument(Thread.currentThread(),6357,null,6356,org.apache.commons.collections4.trie.UnmodifiableTrie.unmodifiableTrie(trie)); try { org.apache.commons.collections4.trie.UnmodifiableTrie.unmodifiableTrie(null); } catch (final IllegalArgumentException ex) { } fr.inria.diversify.testamplification.logger.Logger.writeTestFinish(Thread.currentThread()); }
@org.junit.Test(timeout = 1000) public void testDecorateFactory_add1999() { fr.inria.diversify.testamplification.logger.Logger.writeTestStart(Thread.currentThread(),this, "testDecorateFactory_add1999"); final Trie<java.lang.String, V> trie = makeFullMap(); fr.inria.diversify.testamplification.logger.Logger.logAssertArgument(Thread.currentThread(),6355,trie); fr.inria.diversify.testamplification.logger.Logger.logAssertArgument(Thread.currentThread(),6357,null,6356,org.apache.commons.collections4.trie.UnmodifiableTrie.unmodifiableTrie(trie)); try { org.apache.commons.collections4.trie.UnmodifiableTrie.unmodifiableTrie(null); org.apache.commons.collections4.trie.UnmodifiableTrie.unmodifiableTrie(null); } catch (final IllegalArgumentException ex) { } fr.inria.diversify.testamplification.logger.Logger.writeTestFinish(Thread.currentThread()); }
/** * 局番割当データ (キー「局番 (6桁)」、値「市外局番の長さ」のマップ) を取得する。 * * @return 局番割当データ (キー「局番 (6桁)」、値「市外局番の長さ」のマップ)。 * @throws InvalidFormatException 局番割当ファイルの形式が不正。 * @throws IOException 局番割当ファイルの読込み異常。 */ @Override public Trie<String, Integer> getObject() throws InvalidFormatException, IOException { Trie<String, Integer> trie = new PatriciaTrie<>(); for (Resource r : resources) { try (InputStream in = r.getInputStream()) { Map<String, Pair<String, String>> map = soumuExcelParser.parse(in); for (Map.Entry<String, Pair<String, String>> entry : map.entrySet()) { trie.put(entry.getKey(), entry.getValue().getLeft().length()); } } } return trie; }
@Override public boolean canProvideSuggestions(Project project, Module module) { Trie<String, MetadataNode> sanitisedRootSearchIndex = moduleNameToSanitisedRootSearchIndex.get(module.getName()); return sanitisedRootSearchIndex != null && sanitisedRootSearchIndex.size() != 0; }
private List<LookupElementBuilder> computeSuggestions( Trie<String, MetadataNode> sanitisedRootSearchIndex, ClassLoader classLoader, @Nullable List<String> ancestralKeys, String queryString) { debug(() -> log.debug("Search requested for " + queryString)); StopWatch timer = new StopWatch(); timer.start(); try { String sanitizedQueryString = MetadataNode.sanitize(queryString); String[] querySegments = toPathSegments(sanitizedQueryString); Set<Suggestion> suggestions = null; if (ancestralKeys != null) { String[] pathSegments = ancestralKeys.stream().flatMap(element -> stream(toPathSegments(element))) .toArray(String[]::new); MetadataNode searchStartNode = sanitisedRootSearchIndex.get(MetadataNode.sanitize(pathSegments[0])); if (searchStartNode != null) { if (pathSegments.length > 1) { searchStartNode = searchStartNode.findDeepestMatch(pathSegments, 1, true); } if (searchStartNode != null) { if (!searchStartNode.isLeaf()) { suggestions = searchStartNode.findChildSuggestions(querySegments, 0, 0, classLoader, false); // since we don't have any matches at the root level, may be a subset of intermediary nodes might match the entered string if (suggestions == null) { suggestions = searchStartNode.findChildSuggestions(querySegments, 0, 0, classLoader, true); } } else { // if the start node is a leaf, this means, the user is looking for values for the given key, lets find the suggestions for values suggestions = searchStartNode.getSuggestionValues(classLoader); } } } } else { String sanitisedQuerySegment = MetadataNode.sanitize(querySegments[0]); SortedMap<String, MetadataNode> topLevelQueryResults = sanitisedRootSearchIndex.prefixMap(sanitisedQuerySegment); Collection<MetadataNode> childNodes = topLevelQueryResults.values(); suggestions = getSuggestions(classLoader, querySegments, childNodes, 1, 1, false); // since we don't have any matches at the root level, may be a subset of intermediary nodes might match the entered string if (suggestions == null) { Collection<MetadataNode> nodesToSearchWithin = sanitisedRootSearchIndex.values(); suggestions = getSuggestions(classLoader, querySegments, nodesToSearchWithin, 1, 0, true); } } if (suggestions != null) { return toLookupElementBuilders(suggestions, classLoader); } return null; } finally { timer.stop(); debug(() -> log.debug("Search took " + timer.toString())); } }
private void buildMetadataHierarchy(Trie<String, MetadataNode> sanitisedRootSearchIndex, ContainerInfo containerInfo, SpringConfigurationMetadata springConfigurationMetadata) { debug(() -> log.debug("Adding container to index " + containerInfo)); String containerPath = containerInfo.getContainerPath(); // populate groups List<SpringConfigurationMetadataGroup> groups = springConfigurationMetadata.getGroups(); if (groups != null) { groups.sort(comparing(SpringConfigurationMetadataGroup::getName)); for (SpringConfigurationMetadataGroup group : groups) { String[] pathSegments = toPathSegments(group.getName()); MetadataNode closestMetadata = findDeepestMatch(sanitisedRootSearchIndex, pathSegments, false); if (closestMetadata == null) { String firstSegment = pathSegments[0]; closestMetadata = MetadataNode.newInstance(firstSegment, null, containerPath); boolean noMoreSegmentsLeft = pathSegments.length == 1; if (noMoreSegmentsLeft) { closestMetadata.setGroup(group); } String sanitizedFirstSegment = MetadataNode.sanitize(firstSegment); sanitisedRootSearchIndex.put(sanitizedFirstSegment, closestMetadata); } closestMetadata.addChildren(group, pathSegments, containerPath); } } // populate properties List<SpringConfigurationMetadataProperty> properties = springConfigurationMetadata.getProperties(); properties.sort(comparing(SpringConfigurationMetadataProperty::getName)); for (SpringConfigurationMetadataProperty property : properties) { String[] pathSegments = toPathSegments(property.getName()); MetadataNode closestMetadata = findDeepestMatch(sanitisedRootSearchIndex, pathSegments, false); if (closestMetadata == null) { String firstSegment = pathSegments[0]; closestMetadata = MetadataNode.newInstance(firstSegment, null, containerPath); boolean noMoreSegmentsLeft = pathSegments.length == 1; if (noMoreSegmentsLeft) { closestMetadata.setProperty(property); } String sanitizedFirstSegment = MetadataNode.sanitize(firstSegment); sanitisedRootSearchIndex.put(sanitizedFirstSegment, closestMetadata); } closestMetadata.addChildren(property, pathSegments, containerPath); } // update hints List<SpringConfigurationMetadataHint> hints = springConfigurationMetadata.getHints(); if (hints != null) { hints.sort(comparing(SpringConfigurationMetadataHint::getName)); for (SpringConfigurationMetadataHint hint : hints) { String[] pathSegments = toPathSegments(hint.getName()); MetadataNode closestMetadata = findDeepestMatch(sanitisedRootSearchIndex, pathSegments, true); if (closestMetadata != null && closestMetadata.getDepth() == pathSegments.length) { assert closestMetadata.getProperty() != null; closestMetadata.getProperty().setHint(hint); } } } }
@Override public Trie<java.lang.String, V> makeObject() { return org.apache.commons.collections4.trie.UnmodifiableTrie.unmodifiableTrie(new PatriciaTrie<V>()); }
@Override public Trie<java.lang.String, V> makeFullMap() { final Trie<java.lang.String, V> m = new PatriciaTrie<V>(); addSampleMappings(m); return org.apache.commons.collections4.trie.UnmodifiableTrie.unmodifiableTrie(m); }
public void setAreaCodeTable(Trie<String, Integer> areaCodeTable) { this.areaCodeTable = areaCodeTable; }
@Override public Class<?> getObjectType() { return Trie.class; }
public static void main(String[] args) throws Exception { Checker c = new Checker(); c.initialize(); Trie<String, InflectedFormType> reverse = new PatriciaTrie<>(); for (Entry<String, Checker.InflectedFormType> entry: Checker.formsDictionary.entrySet()) { //using a StringBuilder so that no entry is placed in the jvm string pool String key = new StringBuilder(entry.getKey()).reverse().toString(); reverse.put(key, entry.getValue()); } FileOutputStream fos = new FileOutputStream("c:/var/echos.txt"); OutputStreamWriter out = new OutputStreamWriter(fos, "utf-8"); for (String form : c.formsDictionary.keySet()) { if (form.length() > 2) { String reversedForm = StringUtils.reverse(form); Set<String> echoesReversed = reverse.prefixMap(reversedForm).keySet(); StringBuilder sb = new StringBuilder(); String delim = ""; for (String echoReversed : echoesReversed) { String echo = StringUtils.reverse(echoReversed); // exclude the same word and any word that is formed directly from it and another word or common prefix String diff = echo.replace(form, ""); if (diff.length() == 1) { diff = ""; // ignore 1-letter diffs } if (form.equals("античен")) { diff = ""; } if (!echo.equals(form) && !c.formsDictionary.containsKey(diff) && !commonPrefixes.contains(diff)) { sb.append(delim + echo); delim = ", "; } } if (sb.length() > 0) { sb.insert(0, form + ": "); sb.append("\r\n"); } out.write(sb.toString()); } } out.close(); }
/** * Factory method to create a unmodifiable trie. * * @param <K> the key type * @param <V> the value type * @param trie the trie to decorate, must not be null * @return a new unmodifiable trie * @throws NullPointerException if trie is null */ public static <K, V> Trie<K, V> unmodifiableTrie(final Trie<K, ? extends V> trie) { if (trie instanceof Unmodifiable) { @SuppressWarnings("unchecked") // safe to upcast final Trie<K, V> tmpTrie = (Trie<K, V>) trie; return tmpTrie; } return new UnmodifiableTrie<K, V>(trie); }