@Override public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { return new RandomAccessWeight(this) { @Override protected Bits getMatchingDocs(final LeafReaderContext context) throws IOException { final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), getField()); return new Bits() { @Override public boolean get(int doc) { values.setDocument(doc); for (int i = 0; i < values.count(); i++) { return contains(BitMixer.mix(values.valueAt(i))); } return contains(0); } @Override public int length() { return context.reader().maxDoc(); } }; } }; }
public void testLong() throws Exception { IndexService indexService = createIndex("index", Settings.EMPTY, "type", "long", "type=long"); client().prepareIndex("index", "type", "1") .setSource("long", 42) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L); ValuesSourceConfig<ValuesSource.Numeric> config = ValuesSourceConfig.resolve( context, null, "long", null, null, null, null); ValuesSource.Numeric valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); values.setDocument(0); assertEquals(1, values.count()); assertEquals(42, values.valueAt(0)); } }
public void testUnmappedLong() throws Exception { IndexService indexService = createIndex("index", Settings.EMPTY, "type"); client().prepareIndex("index", "type", "1") .setSource() .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L); ValuesSourceConfig<ValuesSource.Numeric> config = ValuesSourceConfig.resolve( context, ValueType.NUMBER, "long", null, null, null, null); ValuesSource.Numeric valuesSource = config.toValuesSource(context); assertNull(valuesSource); config = ValuesSourceConfig.resolve( context, ValueType.NUMBER, "long", null, 42, null, null); valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); values.setDocument(0); assertEquals(1, values.count()); assertEquals(42, values.valueAt(0)); } }
public void testBoolean() throws Exception { IndexService indexService = createIndex("index", Settings.EMPTY, "type", "bool", "type=boolean"); client().prepareIndex("index", "type", "1") .setSource("bool", true) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L); ValuesSourceConfig<ValuesSource.Numeric> config = ValuesSourceConfig.resolve( context, null, "bool", null, null, null, null); ValuesSource.Numeric valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); values.setDocument(0); assertEquals(1, values.count()); assertEquals(1, values.valueAt(0)); } }
public void testUnmappedBoolean() throws Exception { IndexService indexService = createIndex("index", Settings.EMPTY, "type"); client().prepareIndex("index", "type", "1") .setSource() .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L); ValuesSourceConfig<ValuesSource.Numeric> config = ValuesSourceConfig.resolve( context, ValueType.BOOLEAN, "bool", null, null, null, null); ValuesSource.Numeric valuesSource = config.toValuesSource(context); assertNull(valuesSource); config = ValuesSourceConfig.resolve( context, ValueType.BOOLEAN, "bool", null, true, null, null); valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); values.setDocument(0); assertEquals(1, values.count()); assertEquals(1, values.valueAt(0)); } }
public void testSingleValuedLongs() throws Exception { final int numDocs = scaledRandomIntBetween(1, 100); final long[] array = new long[numDocs]; final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs); for (int i = 0; i < array.length; ++i) { if (randomBoolean()) { array[i] = randomLong(); if (docsWithValue != null) { docsWithValue.set(i); } } else if (docsWithValue != null && randomBoolean()) { docsWithValue.set(i); } } final NumericDocValues singleValues = new NumericDocValues() { @Override public long get(int docID) { return array[docID]; } }; final SortedNumericDocValues multiValues = DocValues.singleton(singleValues, docsWithValue); verify(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); verify(multiValues, numDocs, rootDocs, innerDocs); }
private Longs wrap(long[][] values) { return new Longs(new SortedNumericDocValues() { long[] current; @Override public void setDocument(int doc) { current = values[doc]; } @Override public int count() { return current.length; } @Override public long valueAt(int index) { return current[index]; } }); }
private Dates wrap(long[][] values) { return new Dates(new SortedNumericDocValues() { long[] current; @Override public void setDocument(int doc) { current = values[doc]; } @Override public int count() { return current.length; } @Override public long valueAt(int index) { return current[index]; } }); }
/** * Return a {@link MurmurHash3Values} instance that returns each value as its hash. */ public static MurmurHash3Values cast(final SortedNumericDocValues values) { return new MurmurHash3Values() { @Override public void setDocument(int docId) { values.setDocument(docId); } @Override public int count() { return values.count(); } @Override public long valueAt(int index) { return values.valueAt(index); } }; }
@Override public SortedNumericDocValues getLongValues() { try { return DocValues.getSortedNumeric(reader, field); } catch (IOException e) { throw new IllegalStateException("Cannot load doc values", e); } }
@Override public SortedNumericDoubleValues getDoubleValues() { try { SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field); NumericDocValues single = DocValues.unwrapSingleton(raw); if (single != null) { return FieldData.singleton(new SingleHalfFloatValues(single), DocValues.unwrapSingletonBits(raw)); } else { return new MultiHalfFloatValues(raw); } } catch (IOException e) { throw new IllegalStateException("Cannot load doc values", e); } }
@Override public SortedNumericDoubleValues getDoubleValues() { try { SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field); NumericDocValues single = DocValues.unwrapSingleton(raw); if (single != null) { return FieldData.singleton(new SingleFloatValues(single), DocValues.unwrapSingletonBits(raw)); } else { return new MultiFloatValues(raw); } } catch (IOException e) { throw new IllegalStateException("Cannot load doc values", e); } }
@Override public SortedNumericDoubleValues getDoubleValues() { try { SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field); return FieldData.sortableLongBitsToDoubles(raw); } catch (IOException e) { throw new IllegalStateException("Cannot load doc values", e); } }
@Override public SortedNumericDoubleValues getDoubleValues() { final SortedNumericDocValues values = scaledFieldData.getLongValues(); final NumericDocValues singleValues = DocValues.unwrapSingleton(values); if (singleValues != null) { return FieldData.singleton(new NumericDoubleValues() { @Override public double get(int docID) { return singleValues.get(docID) * scalingFactorInverse; } }, DocValues.unwrapSingletonBits(values)); } else { return new SortedNumericDoubleValues() { @Override public double valueAt(int index) { return values.valueAt(index) * scalingFactorInverse; } @Override public void setDocument(int doc) { values.setDocument(doc); } @Override public int count() { return values.count(); } }; } }
public void testEmptyLong() throws Exception { IndexService indexService = createIndex("index", Settings.EMPTY, "type", "long", "type=long"); client().prepareIndex("index", "type", "1") .setSource() .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L); ValuesSourceConfig<ValuesSource.Numeric> config = ValuesSourceConfig.resolve( context, null, "long", null, null, null, null); ValuesSource.Numeric valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); values.setDocument(0); assertEquals(0, values.count()); config = ValuesSourceConfig.resolve( context, null, "long", null, 42, null, null); valuesSource = config.toValuesSource(context); values = valuesSource.longValues(ctx); values.setDocument(0); assertEquals(1, values.count()); assertEquals(42, values.valueAt(0)); } }
public void testEmptyBoolean() throws Exception { IndexService indexService = createIndex("index", Settings.EMPTY, "type", "bool", "type=boolean"); client().prepareIndex("index", "type", "1") .setSource() .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L); ValuesSourceConfig<ValuesSource.Numeric> config = ValuesSourceConfig.resolve( context, null, "bool", null, null, null, null); ValuesSource.Numeric valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); values.setDocument(0); assertEquals(0, values.count()); config = ValuesSourceConfig.resolve( context, null, "bool", null, true, null, null); valuesSource = config.toValuesSource(context); values = valuesSource.longValues(ctx); values.setDocument(0); assertEquals(1, values.count()); assertEquals(1, values.valueAt(0)); } }
private void verify(SortedNumericDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) { final NumericDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { final long actual = selected.get(root); long expected = 0; if (mode == MultiValueMode.MAX) { expected = Long.MIN_VALUE; } else if (mode == MultiValueMode.MIN) { expected = Long.MAX_VALUE; } int numValues = 0; for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { values.setDocument(child); for (int j = 0; j < values.count(); ++j) { if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { expected += values.valueAt(j); } else if (mode == MultiValueMode.MIN) { expected = Math.min(expected, values.valueAt(j)); } else if (mode == MultiValueMode.MAX) { expected = Math.max(expected, values.valueAt(j)); } ++numValues; } } if (numValues == 0) { expected = missingValue; } else if (mode == MultiValueMode.AVG) { expected = numValues > 1 ? Math.round((double) expected / (double) numValues) : expected; } assertEquals(mode.toString() + " docId=" + root, expected, actual); prevRoot = root; } } } }
public void testSortableLongBitsToDoubles() { final double value = randomDouble(); final long valueBits = NumericUtils.doubleToSortableLong(value); NumericDocValues values = new NumericDocValues() { @Override public long get(int docID) { return valueBits; } }; SortedNumericDoubleValues asMultiDoubles = FieldData.sortableLongBitsToDoubles(DocValues.singleton(values, null)); NumericDoubleValues asDoubles = FieldData.unwrapSingleton(asMultiDoubles); assertNotNull(asDoubles); assertEquals(value, asDoubles.get(0), 0); NumericDocValues backToLongs = DocValues.unwrapSingleton(FieldData.toSortableLongBits(asMultiDoubles)); assertSame(values, backToLongs); SortedNumericDocValues multiValues = new SortedNumericDocValues() { @Override public long valueAt(int index) { return valueBits; } @Override public void setDocument(int doc) { } @Override public int count() { return 1; } }; asMultiDoubles = FieldData.sortableLongBitsToDoubles(multiValues); assertEquals(value, asMultiDoubles.valueAt(0), 0); assertSame(multiValues, FieldData.toSortableLongBits(asMultiDoubles)); }
public void testDoublesToSortableLongBits() { final double value = randomDouble(); final long valueBits = NumericUtils.doubleToSortableLong(value); NumericDoubleValues values = new NumericDoubleValues() { @Override public double get(int docID) { return value; } }; SortedNumericDocValues asMultiLongs = FieldData.toSortableLongBits(FieldData.singleton(values, null)); NumericDocValues asLongs = DocValues.unwrapSingleton(asMultiLongs); assertNotNull(asLongs); assertEquals(valueBits, asLongs.get(0)); SortedNumericDoubleValues multiValues = new SortedNumericDoubleValues() { @Override public double valueAt(int index) { return value; } @Override public void setDocument(int doc) { } @Override public int count() { return 1; } }; asMultiLongs = FieldData.toSortableLongBits(multiValues); assertEquals(valueBits, asMultiLongs.valueAt(0)); assertSame(multiValues, FieldData.sortableLongBitsToDoubles(asMultiLongs)); }
public void testDefaults() throws IOException { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field").field("type", "boolean").endObject().endObject() .endObject().endObject().string(); DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping)); ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() .startObject() .field("field", true) .endObject() .bytes()); try (Directory dir = new RAMDirectory(); IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())))) { w.addDocuments(doc.docs()); try (DirectoryReader reader = DirectoryReader.open(w)) { final LeafReader leaf = reader.leaves().get(0).reader(); // boolean fields are indexed and have doc values by default assertEquals(new BytesRef("T"), leaf.terms("field").iterator().next()); SortedNumericDocValues values = leaf.getSortedNumericDocValues("field"); assertNotNull(values); values.setDocument(0); assertEquals(1, values.count()); assertEquals(1, values.valueAt(0)); } } }
@Override public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { SortedSetEntry ss = sortedNumerics.get(field.number); NumericEntry numericEntry = numerics.get(field.number); final LongValues values = getNumeric(numericEntry); if (ss.format == SORTED_SINGLE_VALUED) { final Bits docsWithField = getMissingBits(numericEntry.missingOffset); return DocValues.singleton(values, docsWithField); } else if (ss.format == SORTED_WITH_ADDRESSES) { final IndexInput data = this.data.clone(); final MonotonicBlockPackedReader ordIndex = getOrdIndexInstance(data, field, ordIndexes.get(field.number)); return new SortedNumericDocValues() { long startOffset; long endOffset; @Override public void setDocument(int doc) { startOffset = ordIndex.get(doc); endOffset = ordIndex.get(doc+1L); } @Override public long valueAt(int index) { return values.get(startOffset + index); } @Override public int count() { return (int) (endOffset - startOffset); } }; } else { throw new AssertionError(); } }
@Override public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { SortedSetEntry ss = sortedNumerics.get(field.number); NumericEntry numericEntry = numerics.get(field.number); final LongValues values = getNumeric(numericEntry); if (ss.format == SORTED_SINGLE_VALUED) { final Bits docsWithField = getMissingBits(numericEntry.missingOffset); return DocValues.singleton(values, docsWithField); } else if (ss.format == SORTED_WITH_ADDRESSES) { final MonotonicBlockPackedReader ordIndex = getOrdIndexInstance(field, ordIndexes.get(field.number)); return new SortedNumericDocValues() { long startOffset; long endOffset; @Override public void setDocument(int doc) { startOffset = ordIndex.get(doc); endOffset = ordIndex.get(doc+1L); } @Override public long valueAt(int index) { return values.get(startOffset + index); } @Override public int count() { return (int) (endOffset - startOffset); } }; } else { throw new AssertionError(); } }
public static AtomicNumericFieldData empty(final int maxDoc) { return new AtomicLongFieldData(0) { @Override public SortedNumericDocValues getLongValues() { return DocValues.emptySortedNumeric(maxDoc); } @Override public Collection<Accountable> getChildResources() { return Collections.emptyList(); } }; }
@Override public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { final BinaryDocValues binary = getBinary(field); return new SortedNumericDocValues() { long values[]; @Override public void setDocument(int doc) { String csv = binary.get(doc).utf8ToString(); if (csv.length() == 0) { values = new long[0]; } else { String s[] = csv.split(","); values = new long[s.length]; for (int i = 0; i < values.length; i++) { values[i] = Long.parseLong(s[i]); } } } @Override public long valueAt(int index) { return values[index]; } @Override public int count() { return values.length; } }; }
@Override public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException { final SortedNumericDocValues oldDocValues = in.getSortedNumericDocValues(field); if (oldDocValues == null) { return null; } else { return new SortingSortedNumericDocValues(oldDocValues, docMap); } }
@Test public void testSortedNumericDocValuesField() throws Exception { assumeTrue("default codec does not support SORTED_NUMERIC", defaultCodecSupportsSortedNumeric()); SortedNumericDocValues dv = reader.getSortedNumericDocValues(SORTED_NUMERIC_DV_FIELD); int maxDoc = reader.maxDoc(); for (int i = 0; i < maxDoc; i++) { dv.setDocument(i); assertEquals(2, dv.count()); int value = sortedValues[i].intValue(); assertEquals("incorrect sorted-numeric DocValues for doc " + i, value, dv.valueAt(0)); assertEquals("incorrect sorted-numeric DocValues for doc " + i, value + 1, dv.valueAt(1)); } }
@Override public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { assert field.getDocValuesType() == FieldInfo.DocValuesType.SORTED_NUMERIC; SortedNumericDocValues values = in.getSortedNumeric(field); assert values != null; return new AssertingAtomicReader.AssertingSortedNumericDocValues(values, maxDoc); }
/** * Return a {@link MurmurHash3Values} instance that computes hashes on the fly for each long value. */ public static MurmurHash3Values hash(SortedNumericDocValues values) { return new Long(values); }
Long(SortedNumericDocValues values) { this.values = values; }
@Override public SortedNumericDocValues longValues(LeafReaderContext context) { return DocValues.emptySortedNumeric(context.reader().maxDoc()); }
/** Get the current {@link SortedNumericDocValues}. */ public abstract SortedNumericDocValues longValues(LeafReaderContext context) throws IOException;
