private static void lookup(AnalyzingInfixSuggester suggester, String keyword, String region) throws IOException { HashSet<BytesRef> contexts = new HashSet<BytesRef>(); contexts.add(new BytesRef(region.getBytes("UTF8"))); //先以contexts为过滤条件进行过滤,再以keyword为关键字进行筛选,根据weight值排序返回前2条 //第3个布尔值即是否每个Term都要匹配,第4个参数表示是否需要关键字高亮 List<Lookup.LookupResult> results = suggester.lookup(keyword, 2, true, false); System.out.println("-- \"" + keyword + "\" (" + region + "):"); for (Lookup.LookupResult result : results) { System.out.println(result.key); //从payload中反序列化出Product对象 BytesRef bytesRef = result.payload; InputStream is = Tools.bytes2InputStream(bytesRef.bytes); Product product = (Product) Tools.deSerialize(is); System.out.println("product-Name:" + product.getName()); System.out.println("product-regions:" + product.getRegions()); System.out.println("product-image:" + product.getImage()); System.out.println("product-numberSold:" + product.getNumberSold()); } System.out.println(); }
/** TODO add PathParam version of this */ @GET @Path("/suggest") @Produces({MediaType.APPLICATION_JSON}) @ApiOperation("Provides search query suggestions given a partially complete input query") public Response suggest(@QueryParam("q") String q) { if (q == null || (q = q.trim()).isEmpty() || q.length() > SuggestLengthMax) return Response.noContent().build(); String x = q; return Response.ok((StreamingOutput) os -> { List<Lookup.LookupResult> x1 = db.suggest(x, SuggestionResultsMax); JSON.toJSON(Lists.transform(x1, y -> y.key), os); }).build(); }
@Override public Lookup create(NamedList params, SolrCore core) { Object fieldTypeName = params.get(QUERY_ANALYZER); if (fieldTypeName == null) { throw new IllegalArgumentException("Error in configuration: " + QUERY_ANALYZER + " parameter is mandatory"); } FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString()); if (ft == null) { throw new IllegalArgumentException("Error in configuration: " + fieldTypeName.toString() + " is not defined in the schema"); } Analyzer indexAnalyzer = ft.getIndexAnalyzer(); Analyzer queryAnalyzer = ft.getQueryAnalyzer(); int grams = (params.get(NGRAMS) != null) ? Integer.parseInt(params.get(NGRAMS).toString()) : FreeTextSuggester.DEFAULT_GRAMS; byte separator = (params.get(SEPARATOR) != null) ? params.get(SEPARATOR).toString().getBytes(StandardCharsets.UTF_8)[0] : FreeTextSuggester.DEFAULT_SEPARATOR; return new FreeTextSuggester(indexAnalyzer, queryAnalyzer, grams, separator); }
/** * Test construction time. */ public void testConstructionTime() throws Exception { System.err.println("-- construction time"); for (final Class<? extends Lookup> cls : benchmarkClasses) { BenchmarkResult result = measure(new Callable<Integer>() { @Override public Integer call() throws Exception { final Lookup lookup = buildLookup(cls, dictionaryInput); return lookup.hashCode(); } }); System.err.println( String.format(Locale.ROOT, "%-15s input: %d, time[ms]: %s", cls.getSimpleName(), dictionaryInput.length, result.average.toString())); } }
/** * Test memory required for the storage. */ public void testStorageNeeds() throws Exception { System.err.println("-- RAM consumption"); for (Class<? extends Lookup> cls : benchmarkClasses) { Lookup lookup = buildLookup(cls, dictionaryInput); long sizeInBytes; if (lookup instanceof AnalyzingSuggester) { // Just get size of FST: else we are also measuring // size of MockAnalyzer which is non-trivial and // varies depending on test seed: sizeInBytes = ((AnalyzingSuggester) lookup).sizeInBytes(); } else { sizeInBytes = RamUsageEstimator.sizeOf(lookup); } System.err.println( String.format(Locale.ROOT, "%-15s size[B]:%,13d", lookup.getClass().getSimpleName(), sizeInBytes)); } }
@Override public Lookup create(NamedList params, SolrCore core) { Object fieldTypeName = params.get(QUERY_ANALYZER); if (fieldTypeName == null) { throw new IllegalArgumentException("Error in configuration: " + QUERY_ANALYZER + " parameter is mandatory"); } FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString()); if (ft == null) { throw new IllegalArgumentException("Error in configuration: " + fieldTypeName.toString() + " is not defined in the schema"); } Analyzer indexAnalyzer = ft.getAnalyzer(); Analyzer queryAnalyzer = ft.getQueryAnalyzer(); int grams = (params.get(NGRAMS) != null) ? Integer.parseInt(params.get(NGRAMS).toString()) : FreeTextSuggester.DEFAULT_GRAMS; byte separator = (params.get(SEPARATOR) != null) ? params.get(SEPARATOR).toString().getBytes(IOUtils.CHARSET_UTF_8)[0] : FreeTextSuggester.DEFAULT_SEPARATOR; return new FreeTextSuggester(indexAnalyzer, queryAnalyzer, grams, separator); }
@Override protected boolean lessThan(SuggestDoc a, SuggestDoc b) { if (a.score == b.score) { int cmp = Lookup.CHARSEQUENCE_COMPARATOR.compare(a.key, b.key); if (cmp == 0) { // prefer smaller doc id, in case of a tie return a.doc > b.doc; } else { return cmp > 0; } } return a.score < b.score; }
@Override protected boolean lessThan(Entry.Option a, Entry.Option b) { int cmp = comparator.compare(a, b); if (cmp != 0) { return cmp > 0; } return Lookup.CHARSEQUENCE_COMPARATOR.compare(a.getText().string(), b.getText().string()) > 0; }
/** * Provide highlighted suggestion for titles. * * @param query The current user input. * @return A list of highlighted (with HTML mark-ups) titles. * @throws IOException */ public List<String> suggest(String query) throws IOException { List<Lookup.LookupResult> results = suggester.lookup(query, suggestionCount, false, true); List<String> suggestions = new ArrayList<String>(); for (Lookup.LookupResult result : results) { if (result.highlightKey instanceof String) { suggestions.add((String) result.highlightKey); } else { suggestions.add(result.key.toString()); } } return suggestions; }
@NotNull public List<Lookup.LookupResult> suggest(String qText, int count) throws IOException { Lookup s = suggester(); if (s == null) return Collections.EMPTY_LIST; else return s.lookup(qText, false, count); }
/** * Add an element to the tree respecting a size limit * * @param results the tree to add in * @param result the result we try to add * @param num size limit */ private static void boundedTreeAdd(TreeSet<Lookup.LookupResult> results, Lookup.LookupResult result, int num) { if (results.size() >= num) { if (results.first().value < result.value) { results.pollFirst(); } else { return; } } results.add(result); }
@Override public int compare(Lookup.LookupResult o1, Lookup.LookupResult o2) { // order on weight if (o1.value > o2.value) { return 1; } else if (o1.value < o2.value) { return -1; } // otherwise on alphabetic order return CHARSEQUENCE_COMPARATOR.compare(o1.key, o2.key); }
public void /*testT*/rying() throws IOException { BytesRef lake = new BytesRef("lake"); BytesRef star = new BytesRef("star"); BytesRef ret = new BytesRef("ret"); Input keys[] = new Input[]{ new Input("top of the lake", 15, lake), new Input("star wars: episode v - the empire strikes back", 12, star), new Input("the returned", 10, ret), }; File tempDir = createTempDir("BlendedInfixSuggesterTest"); Analyzer a = new StandardAnalyzer(CharArraySet.EMPTY_SET); // if factor is small, we don't get the expected element BlendedInfixSuggester suggester = new BlendedInfixSuggester(TEST_VERSION_CURRENT, newFSDirectory(tempDir), a, a, AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS, BlendedInfixSuggester.BlenderType.POSITION_RECIPROCAL, BlendedInfixSuggester.DEFAULT_NUM_FACTOR); suggester.build(new InputArrayIterator(keys)); List<Lookup.LookupResult> responses = suggester.lookup("the", null, 4, true, false); for (Lookup.LookupResult response : responses) { System.out.println(response); } suggester.close(); }
private static long getInResults(BlendedInfixSuggester suggester, String prefix, BytesRef payload, int num) throws IOException { List<Lookup.LookupResult> responses = suggester.lookup(prefix, null, num, true, false); for (Lookup.LookupResult response : responses) { if (response.payload.equals(payload)) { return response.value; } } return -1; }
@Override public Lookup create(NamedList params, SolrCore core) { boolean exactMatchFirst = params.get(EXACT_MATCH_FIRST) != null ? Boolean.valueOf(params.get(EXACT_MATCH_FIRST).toString()) : true; return new WFSTCompletionLookup(exactMatchFirst); }
@Override public Lookup create(NamedList params, SolrCore core) { int buckets = params.get(WEIGHT_BUCKETS) != null ? Integer.parseInt(params.get(WEIGHT_BUCKETS).toString()) : 10; boolean exactMatchFirst = params.get(EXACT_MATCH_FIRST) != null ? Boolean.valueOf(params.get(EXACT_MATCH_FIRST).toString()) : true; return new FSTCompletionLookup(buckets, exactMatchFirst); }
private static Lookup setupSuggester_Analyzing() { setupIndexReader(); final Lookup suggester[] = new AnalyzingSuggester[1]; Display.getDefault().syncExec(new Runnable() { @Override public void run() { BusyIndicator.showWhile(Display.getDefault(), new Runnable() { @Override public void run() { try { final Analyzer queryAnalyzer = new StandardAnalyzer(new CharArraySet(0, true)); final InputIterator termIterator = createTermIterator(); suggester[0] = new AnalyzingSuggester(queryAnalyzer); suggester[0].build(termIterator); } catch (final Exception e) { StatusUtil.showStatus(e); } } }); } }); return suggester[0]; }
@Override public Lookup create(@SuppressWarnings("rawtypes") NamedList params, SolrCore core) { // mandatory parameter Object fieldTypeName = params.get(QUERY_ANALYZER); if (fieldTypeName == null) { throw new IllegalArgumentException("Error in configuration: " + QUERY_ANALYZER + " parameter is mandatory"); } FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString()); if (ft == null) { throw new IllegalArgumentException("Error in configuration: " + fieldTypeName.toString() + " is not defined in the schema"); } Analyzer indexAnalyzer = ft.getIndexAnalyzer(); Analyzer queryAnalyzer = ft.getQueryAnalyzer(); // optional parameters String indexPath = params.get(INDEX_PATH) != null ? params.get(INDEX_PATH).toString() : DEFAULT_INDEX_PATH; int minPrefixChars = params.get(MIN_PREFIX_CHARS) != null ? Integer.parseInt(params.get(MIN_PREFIX_CHARS).toString()) : AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS; Boolean highlight = params.getBooleanArg(HIGHLIGHT); if (highlight == null) { highlight = DEFAULT_HIGHLIGHT; } try { return new SafariInfixSuggester(core.getSolrConfig().luceneMatchVersion, FSDirectory.open(new File(indexPath)), indexAnalyzer, queryAnalyzer, minPrefixChars, highlight); } catch (IOException e) { throw new SolrException(ErrorCode.SERVER_ERROR, e); } }
private void runTest(Class<? extends Lookup> lookupClass, boolean supportsExactWeights) throws Exception { // Add all input keys. Lookup lookup = lookupClass.newInstance(); TermFreq[] keys = new TermFreq[this.keys.length]; for (int i = 0; i < keys.length; i++) keys[i] = new TermFreq(this.keys[i], i); lookup.build(new TermFreqArrayIterator(keys)); // Store the suggester. File storeDir = TEMP_DIR; lookup.store(new FileOutputStream(new File(storeDir, "lookup.dat"))); // Re-read it from disk. lookup = lookupClass.newInstance(); lookup.load(new FileInputStream(new File(storeDir, "lookup.dat"))); // Assert validity. Random random = random(); long previous = Long.MIN_VALUE; for (TermFreq k : keys) { List<LookupResult> list = lookup.lookup(_TestUtil.bytesToCharSequence(k.term, random), false, 1); assertEquals(1, list.size()); LookupResult lookupResult = list.get(0); assertNotNull(k.term.utf8ToString(), lookupResult.key); if (supportsExactWeights) { assertEquals(k.term.utf8ToString(), k.v, lookupResult.value); } else { assertTrue(lookupResult.value + ">=" + previous, lookupResult.value >= previous); previous = lookupResult.value; } } }
/** * Create {@link Lookup} instance and populate it. */ private Lookup buildLookup(Class<? extends Lookup> cls, TermFreq[] input) throws Exception { Lookup lookup = null; try { lookup = cls.newInstance(); } catch (InstantiationException e) { Constructor<? extends Lookup> ctor = cls.getConstructor(Analyzer.class); lookup = ctor.newInstance(new MockAnalyzer(random, MockTokenizer.KEYWORD, false)); } lookup.build(new TermFreqArrayIterator(input)); return lookup; }
@Override public Lookup create(NamedList params, SolrCore core) { // mandatory parameter Object fieldTypeName = params.get(QUERY_ANALYZER); if (fieldTypeName == null) { throw new IllegalArgumentException("Error in configuration: " + QUERY_ANALYZER + " parameter is mandatory"); } FieldType ft = core.getSchema().getFieldTypeByName(fieldTypeName.toString()); Analyzer indexAnalyzer = ft.getAnalyzer(); Analyzer queryAnalyzer = ft.getQueryAnalyzer(); // optional parameters boolean exactMatchFirst = params.get(EXACT_MATCH_FIRST) != null ? Boolean.valueOf(params.get(EXACT_MATCH_FIRST).toString()) : true; boolean preserveSep = params.get(PRESERVE_SEP) != null ? Boolean.valueOf(params.get(PRESERVE_SEP).toString()) : true; int flags = 0; if (exactMatchFirst) { flags |= AnalyzingSuggester.EXACT_FIRST; } if (preserveSep) { flags |= AnalyzingSuggester.PRESERVE_SEP; } int maxSurfaceFormsPerAnalyzedForm = params.get(MAX_SURFACE_FORMS) != null ? Integer.parseInt(params.get(MAX_SURFACE_FORMS).toString()) : 256; int maxGraphExpansions = params.get(MAX_EXPANSIONS) != null ? Integer.parseInt(params.get(MAX_EXPANSIONS).toString()) : -1; return new AnalyzingSuggester(indexAnalyzer, queryAnalyzer, flags, maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions); }
@Override public Lookup create(NamedList params, SolrCore core) { // mandatory parameter Object fieldTypeName = params.get(QUERY_ANALYZER); if (fieldTypeName == null) { throw new IllegalArgumentException("Error in configuration: " + QUERY_ANALYZER + " parameter is mandatory"); } FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString()); if (ft == null) { throw new IllegalArgumentException("Error in configuration: " + fieldTypeName.toString() + " is not defined in the schema"); } Analyzer indexAnalyzer = ft.getAnalyzer(); Analyzer queryAnalyzer = ft.getQueryAnalyzer(); // optional parameters String indexPath = params.get(INDEX_PATH) != null ? params.get(INDEX_PATH).toString() : DEFAULT_INDEX_PATH; int minPrefixChars = params.get(MIN_PREFIX_CHARS) != null ? Integer.parseInt(params.get(MIN_PREFIX_CHARS).toString()) : AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS; BlenderType blenderType = getBlenderType(params.get(BLENDER_TYPE)); int numFactor = params.get(NUM_FACTOR) != null ? Integer.parseInt(params.get(NUM_FACTOR).toString()) : BlendedInfixSuggester.DEFAULT_NUM_FACTOR; try { return new BlendedInfixSuggester(core.getSolrConfig().luceneMatchVersion, new File(indexPath), indexAnalyzer, queryAnalyzer, minPrefixChars, blenderType, numFactor); } catch (IOException e) { throw new RuntimeException(); } }
@Override public Lookup create(NamedList params, SolrCore core) { // mandatory parameter Object fieldTypeName = params.get(QUERY_ANALYZER); if (fieldTypeName == null) { throw new IllegalArgumentException("Error in configuration: " + QUERY_ANALYZER + " parameter is mandatory"); } FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString()); if (ft == null) { throw new IllegalArgumentException("Error in configuration: " + fieldTypeName.toString() + " is not defined in the schema"); } Analyzer indexAnalyzer = ft.getAnalyzer(); Analyzer queryAnalyzer = ft.getQueryAnalyzer(); // optional parameters String indexPath = params.get(INDEX_PATH) != null ? params.get(INDEX_PATH).toString() : DEFAULT_INDEX_PATH; int minPrefixChars = params.get(MIN_PREFIX_CHARS) != null ? Integer.parseInt(params.get(MIN_PREFIX_CHARS).toString()) : AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS; try { return new AnalyzingInfixSuggester(core.getSolrConfig().luceneMatchVersion, new File(indexPath), indexAnalyzer, queryAnalyzer, minPrefixChars); } catch (IOException e) { throw new RuntimeException(); } }