@Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postings = new Lucene40PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix); boolean success = false; try { FieldsProducer ret = new BlockTreeTermsReader( state.directory, state.fieldInfos, state.segmentInfo, postings, state.context, state.segmentSuffix, state.termsIndexDivisor); success = true; return ret; } finally { if (!success) { postings.close(); } } }
@Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix); boolean success = false; try { FieldsProducer ret = new BlockTreeTermsReader(state.directory, state.fieldInfos, state.segmentInfo, postingsReader, state.context, state.segmentSuffix, state.termsIndexDivisor); success = true; return ret; } finally { if (!success) { IOUtils.closeWhileHandlingException(postingsReader); } } }
@Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postings = new Lucene40PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix); boolean success = false; try { FieldsProducer ret = new AppendingTermsReader( state.directory, state.fieldInfos, state.segmentInfo, postings, state.context, state.segmentSuffix, state.termsIndexDivisor); success = true; return ret; } finally { if (!success) { postings.close(); } } }
@Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { FieldsProducer postings = PostingsFormat.forName("Lucene41").fieldsProducer(state); if (state.context.context != IOContext.Context.MERGE) { FieldsProducer loadedPostings; try { postings.checkIntegrity(); loadedPostings = new DirectFields(state, postings, minSkipCount, lowFreqCutoff); } finally { postings.close(); } return loadedPostings; } else { // Don't load postings for merge: return postings; } }
@Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix); boolean success = false; try { FieldsProducer ret = new FSTOrdTermsReader(state, postingsReader); success = true; return ret; } finally { if (!success) { IOUtils.closeWhileHandlingException(postingsReader); } } }
@Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix); boolean success = false; try { FieldsProducer ret = new FSTTermsReader(state, postingsReader); success = true; return ret; } finally { if (!success) { IOUtils.closeWhileHandlingException(postingsReader); } } }
@Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix); boolean success = false; try { FieldsProducer ret = new OrdsBlockTreeTermsReader(state.directory, state.fieldInfos, state.segmentInfo, postingsReader, state.context, state.segmentSuffix); success = true; return ret; } finally { if (!success) { IOUtils.closeWhileHandlingException(postingsReader); } } }
@Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase docsReader = null; PostingsReaderBase pulsingReader = null; boolean success = false; try { docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state); pulsingReader = new PulsingPostingsReader(state, docsReader); FieldsProducer ret = new BlockTreeTermsReader( state.directory, state.fieldInfos, state.segmentInfo, pulsingReader, state.context, state.segmentSuffix, state.termsIndexDivisor); success = true; return ret; } finally { if (!success) { IOUtils.closeWhileHandlingException(docsReader, pulsingReader); } } }
@Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postingsReader = new IDVersionPostingsReader(); boolean success = false; try { FieldsProducer ret = new VersionBlockTreeTermsReader(state.directory, state.fieldInfos, state.segmentInfo, postingsReader, state.context, state.segmentSuffix); success = true; return ret; } finally { if (!success) { IOUtils.closeWhileHandlingException(postingsReader); } } }
@Override public FieldsProducer fieldsProducer(SegmentReadState readState) throws IOException { // Load our ID: final String idFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name, readState.segmentSuffix, ID_EXTENSION); IndexInput in = readState.directory.openInput(idFileName, readState.context); boolean success = false; final int id; try { CodecUtil.checkHeader(in, RAM_ONLY_NAME, VERSION_START, VERSION_LATEST); id = in.readVInt(); success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(in); } else { IOUtils.close(in); } } synchronized(state) { return state.get(id); } }
@Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { FieldsProducer postings = PostingsFormat.forName("Lucene41").fieldsProducer(state); if (state.context.context != IOContext.Context.MERGE) { FieldsProducer loadedPostings; try { loadedPostings = new DirectFields(state, postings, minSkipCount, lowFreqCutoff); } finally { postings.close(); } return loadedPostings; } else { // Don't load postings for merge: return postings; } }
@Override public DocValuesProducer normsProducer(SegmentReadState state) throws IOException { String filename = IndexFileNames.segmentFileName(state.segmentInfo.name, "nrm", IndexFileNames.COMPOUND_FILE_EXTENSION); return new Lucene40DocValuesReader(state, filename, Lucene40FieldInfosReader.LEGACY_NORM_TYPE_KEY); }
@Override public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException { String filename = IndexFileNames.segmentFileName(state.segmentInfo.name, "dv", IndexFileNames.COMPOUND_FILE_EXTENSION); return new Lucene40DocValuesReader(state, filename, Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY); }
public FieldsReader(final SegmentReadState readState) throws IOException { // Read _X.per and init each format: boolean success = false; try { // Read field name -> format name for (FieldInfo fi : readState.fieldInfos) { if (fi.isIndexed()) { final String fieldName = fi.name; final String formatName = fi.getAttribute(PER_FIELD_FORMAT_KEY); if (formatName != null) { // null formatName means the field is in fieldInfos, but has no postings! final String suffix = fi.getAttribute(PER_FIELD_SUFFIX_KEY); assert suffix != null; PostingsFormat format = PostingsFormat.forName(formatName); String segmentSuffix = getSuffix(formatName, suffix); if (!formats.containsKey(segmentSuffix)) { formats.put(segmentSuffix, format.fieldsProducer(new SegmentReadState(readState, segmentSuffix))); } fields.put(fieldName, formats.get(segmentSuffix)); } } } success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(formats.values()); } } }
public FieldsReader(final SegmentReadState readState) throws IOException { // Init each unique format: boolean success = false; try { // Read field name -> format name for (FieldInfo fi : readState.fieldInfos) { if (fi.hasDocValues()) { final String fieldName = fi.name; final String formatName = fi.getAttribute(PER_FIELD_FORMAT_KEY); if (formatName != null) { // null formatName means the field is in fieldInfos, but has no docvalues! final String suffix = fi.getAttribute(PER_FIELD_SUFFIX_KEY); assert suffix != null; DocValuesFormat format = DocValuesFormat.forName(formatName); String segmentSuffix = getFullSegmentSuffix(readState.segmentSuffix, getSuffix(formatName, suffix)); if (!formats.containsKey(segmentSuffix)) { formats.put(segmentSuffix, format.fieldsProducer(new SegmentReadState(readState, segmentSuffix))); } fields.put(fieldName, formats.get(segmentSuffix)); } } } success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(formats.values()); } } }
public CompletionFieldsProducer(SegmentReadState state) throws IOException { String suggestFSTFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION); IndexInput input = state.directory.openInput(suggestFSTFile, state.context); version = CodecUtil.checkHeader(input, CODEC_NAME, SUGGEST_CODEC_VERSION, SUGGEST_VERSION_CURRENT); FieldsProducer delegateProducer = null; boolean success = false; try { PostingsFormat delegatePostingsFormat = PostingsFormat.forName(input.readString()); String providerName = input.readString(); CompletionLookupProvider completionLookupProvider = providers.get(providerName); if (completionLookupProvider == null) { throw new IllegalStateException("no provider with name [" + providerName + "] registered"); } // TODO: we could clone the ReadState and make it always forward IOContext.MERGE to prevent unecessary heap usage? delegateProducer = delegatePostingsFormat.fieldsProducer(state); /* * If we are merging we don't load the FSTs at all such that we * don't consume so much memory during merge */ if (state.context.context != Context.MERGE) { // TODO: maybe we can do this in a fully lazy fashion based on some configuration // eventually we should have some kind of curciut breaker that prevents us from going OOM here // with some configuration this.lookupFactory = completionLookupProvider.load(input); } else { this.lookupFactory = null; } this.delegateProducer = delegateProducer; success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(delegateProducer, input); } else { IOUtils.close(input); } } }
public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException { final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION); this.postingsReader = postingsReader; final IndexInput in = state.directory.openInput(termsFileName, state.context); boolean success = false; try { version = readHeader(in); if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) { CodecUtil.checksumEntireFile(in); } this.postingsReader.init(in); seekDir(in); final FieldInfos fieldInfos = state.fieldInfos; final int numFields = in.readVInt(); for (int i = 0; i < numFields; i++) { int fieldNumber = in.readVInt(); FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber); long numTerms = in.readVLong(); long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong(); long sumDocFreq = in.readVLong(); int docCount = in.readVInt(); int longsSize = in.readVInt(); TermsReader current = new TermsReader(fieldInfo, in, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize); TermsReader previous = fields.put(fieldInfo.name, current); checkFieldSummary(state.segmentInfo, in, current, previous); } success = true; } finally { if (success) { IOUtils.close(in); } else { IOUtils.closeWhileHandlingException(in); } } }
public SimpleTextFieldsReader(SegmentReadState state) throws IOException { this.maxDoc = state.segmentInfo.getDocCount(); fieldInfos = state.fieldInfos; in = state.directory.openInput(SimpleTextPostingsFormat.getPostingsFileName(state.segmentInfo.name, state.segmentSuffix), state.context); boolean success = false; try { fields = readFields(in.clone()); success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(this); } } }
public BloomFilteredFieldsProducer(SegmentReadState state) throws IOException { String bloomFileName = IndexFileNames.segmentFileName( state.segmentInfo.name, state.segmentSuffix, BLOOM_EXTENSION); ChecksumIndexInput bloomIn = null; boolean success = false; try { bloomIn = state.directory.openChecksumInput(bloomFileName, state.context); int version = CodecUtil.checkHeader(bloomIn, BLOOM_CODEC_NAME, VERSION_START, VERSION_CURRENT); // // Load the hash function used in the BloomFilter // hashFunction = HashFunction.forName(bloomIn.readString()); // Load the delegate postings format PostingsFormat delegatePostingsFormat = PostingsFormat.forName(bloomIn .readString()); this.delegateFieldsProducer = delegatePostingsFormat .fieldsProducer(state); int numBlooms = bloomIn.readInt(); for (int i = 0; i < numBlooms; i++) { int fieldNum = bloomIn.readInt(); FuzzySet bloom = FuzzySet.deserialize(bloomIn); FieldInfo fieldInfo = state.fieldInfos.fieldInfo(fieldNum); bloomsByFieldName.put(fieldInfo.name, bloom); } if (version >= VERSION_CHECKSUM) { CodecUtil.checkFooter(bloomIn); } else { CodecUtil.checkEOF(bloomIn); } IOUtils.close(bloomIn); success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(bloomIn, delegateFieldsProducer); } } }
@Override public DocValuesProducer normsProducer(SegmentReadState state) throws IOException { assert state.fieldInfos.hasNorms(); DocValuesProducer producer = in.normsProducer(state); assert producer != null; return new AssertingDocValuesProducer(producer, state.segmentInfo.getDocCount()); }
@Override public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException { assert state.fieldInfos.hasDocValues(); DocValuesProducer producer = in.fieldsProducer(state); assert producer != null; return new AssertingDocValuesProducer(producer, state.segmentInfo.getDocCount()); }
DiskDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException { String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension); // read in the entries from the metadata file. IndexInput in = state.directory.openInput(metaName, state.context); boolean success = false; try { CodecUtil.checkHeader(in, metaCodec, DiskDocValuesFormat.VERSION_START, DiskDocValuesFormat.VERSION_START); numerics = new HashMap<Integer,NumericEntry>(); ords = new HashMap<Integer,NumericEntry>(); ordIndexes = new HashMap<Integer,NumericEntry>(); binaries = new HashMap<Integer,BinaryEntry>(); readFields(in, state.fieldInfos); success = true; } finally { if (success) { IOUtils.close(in); } else { IOUtils.closeWhileHandlingException(in); } } String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); data = state.directory.openInput(dataName, state.context); CodecUtil.checkHeader(data, dataCodec, DiskDocValuesFormat.VERSION_START, DiskDocValuesFormat.VERSION_START); }