/** * Commits any changes since the last commit or rollback. * Afterwards a new {@link IndexReader} will be created. * @throws IOException if there is a low-level IO error */ void commit() throws IOException { try { writer.commit(); if(writer.hasDeletions()) { writer.forceMergeDeletes(); // delete now the documents which are marked to delete. } } catch(final AlreadyClosedException e) { reopenIndex(); } finally { unlock(); // new reader because new data is available. createReader(); } }
/** * Flush the engine (committing segments to disk and truncating the * translog) and close it. */ public void flushAndClose() throws IOException { if (isClosed.get() == false) { logger.trace("flushAndClose now acquire writeLock"); try (ReleasableLock lock = writeLock.acquire()) { logger.trace("flushAndClose now acquired writeLock"); try { logger.debug("flushing shard on close - this might take some time to sync files to disk"); try { flush(); // TODO we might force a flush in the future since we have the write lock already even though recoveries are running. } catch (AlreadyClosedException ex) { logger.debug("engine already closed - skipping flushAndClose"); } } finally { close(); // double close is not a problem } } } }
@Override protected boolean maybeFailEngine(String source, Exception e) { boolean shouldFail = super.maybeFailEngine(source, e); if (shouldFail) { return true; } // Check for AlreadyClosedException -- ACE is a very special // exception that should only be thrown in a tragic event. we pass on the checks to failOnTragicEvent which will // throw and AssertionError if the tragic event condition is not met. if (e instanceof AlreadyClosedException) { return failOnTragicEvent((AlreadyClosedException)e); } else if (e != null && ((indexWriter.isOpen() == false && indexWriter.getTragicException() == e) || (translog.isOpen() == false && translog.getTragicException() == e))) { // this spot on - we are handling the tragic event exception here so we have to fail the engine // right away failEngine(source, e); return true; } return false; }
private void maybeFSyncTranslogs() { if (indexSettings.getTranslogDurability() == Translog.Durability.ASYNC) { for (IndexShard shard : this.shards.values()) { try { Translog translog = shard.getTranslog(); if (translog.syncNeeded()) { translog.sync(); } } catch (AlreadyClosedException ex) { // fine - continue; } catch (IOException e) { logger.warn("failed to sync translog", e); } } } }
private void maybeRefreshEngine() { if (indexSettings.getRefreshInterval().millis() > 0) { for (IndexShard shard : this.shards.values()) { switch (shard.state()) { case CREATED: case RECOVERING: case CLOSED: continue; case POST_RECOVERY: case STARTED: case RELOCATED: try { if (shard.isRefreshNeeded()) { shard.refresh("schedule"); } } catch (IndexShardClosedException | AlreadyClosedException ex) { // fine - continue; } continue; default: throw new IllegalStateException("unknown state: " + shard.state()); } } } }
private void maybeUpdateGlobalCheckpoints() { for (IndexShard shard : this.shards.values()) { if (shard.routingEntry().primary()) { switch (shard.state()) { case CREATED: case RECOVERING: case CLOSED: case RELOCATED: continue; case POST_RECOVERY: case STARTED: try { shard.updateGlobalCheckpointOnPrimary(); } catch (AlreadyClosedException ex) { // fine - continue, the shard was concurrently closed on us. } continue; default: throw new IllegalStateException("unknown state: " + shard.state()); } } } }
private void handleRefreshException(Exception e) { if (e instanceof AlreadyClosedException) { // ignore } else if (e instanceof RefreshFailedEngineException) { RefreshFailedEngineException rfee = (RefreshFailedEngineException) e; if (rfee.getCause() instanceof InterruptedException) { // ignore, we are being shutdown } else if (rfee.getCause() instanceof ClosedByInterruptException) { // ignore, we are being shutdown } else if (rfee.getCause() instanceof ThreadInterruptedException) { // ignore, we are being shutdown } else { if (state != IndexShardState.CLOSED) { logger.warn("Failed to perform engine refresh", e); } } } else { if (state != IndexShardState.CLOSED) { logger.warn("Failed to perform engine refresh", e); } } }
public void testAddingAClosedReader() throws Exception { LeafReader reader; try (Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) { writer.addDocument(new Document()); try (DirectoryReader dirReader = ElasticsearchDirectoryReader.wrap(writer.getReader(), new ShardId("index1", "_na_", 1))) { reader = dirReader.leaves().get(0).reader(); } } ShardCoreKeyMap map = new ShardCoreKeyMap(); try { map.add(reader); fail("Expected AlreadyClosedException"); } catch (AlreadyClosedException e) { // What we wanted } assertEquals(0, map.size()); }
/** * Executes the provided operation against this store */ // we can do FS ops with only two elevated permissions: // 1) hadoop dynamic proxy is messy with access rules // 2) allow hadoop to add credentials to our Subject <V> V execute(Operation<V> operation) throws IOException { SpecialPermission.check(); if (closed) { throw new AlreadyClosedException("HdfsBlobStore is closed: " + this); } try { return AccessController.doPrivileged((PrivilegedExceptionAction<V>) () -> operation.run(fileContext), null, new ReflectPermission("suppressAccessChecks"), new AuthPermission("modifyPrivateCredentials"), new SocketPermission("*", "connect")); } catch (PrivilegedActionException pae) { throw (IOException) pae.getException(); } }
/** * Checks if the seed is an IndexReader, and if so will walk * the hierarchy of subReaders building up a list of the objects * returned by {@code seed.getCoreCacheKey()} */ private List<Object> getAllDescendantReaderKeys(Object seed) { List<Object> all = new ArrayList<>(17); // will grow as we iter all.add(seed); for (int i = 0; i < all.size(); i++) { final Object obj = all.get(i); // TODO: We don't check closed readers here (as getTopReaderContext // throws AlreadyClosedException), what should we do? Reflection? if (obj instanceof IndexReader) { try { final List<IndexReaderContext> childs = ((IndexReader) obj).getContext().children(); if (childs != null) { // it is composite reader for (final IndexReaderContext ctx : childs) { all.add(ctx.reader().getCoreCacheKey()); } } } catch (AlreadyClosedException ace) { // ignore this reader } } } // need to skip the first, because it was the seed return all.subList(1, all.size()); }
@Override public void run() { int i = 0; while (i < 10000) { try { if (data.size() <= i) { sleep(1); continue; } final String key = "key" + i; final String val = "value" + i; final List<Document> documents = index.searchForDocuments(new TermQuery(new Term(key, val)), 10, new Sort(new SortField(key, SortField.Type.STRING))); if (documents.size() != 1) { throw new RuntimeException("Invalid number of matching documents for " + key + ", found " + documents); } ++i; } catch (IOException ioe) { error = ioe; break; } catch (InterruptedException e) { } catch (AlreadyClosedException ace) { error = ace; break; } } }
@Override public void close() throws IOException { synchronized (lock) { if (isOpen()) { try { flush(); } catch (AlreadyClosedException ignore) { } } if (searcherManager != null) { searcherManager.close(); searcherManager = null; } if (writer != null) { writer.close(); writer = null; } if (owningDirectory) { if (directory != null) { directory.close(); directory = null; } } } }
/** * Gets the Lucene {@link org.apache.lucene.document.Document} associated with this ResultDoc. If the index * has changed since the search was conducted, this method may return an empty or incorrect Document. It is * therefore best to read all Documents as soon as possible after a search if the index is being * concurrently modified. * * @return The {@link org.apache.lucene.document.Document} associated with this ResultDoc. */ public final Document getDocument() { try { if (_document == null) _document = _resultDocConfig.index.getReader().document(_docNum); return _document; } catch (AlreadyClosedException ace) { prtlnErr("Error retrieving document: The current IndexReader is already closed."); } catch (IllegalArgumentException iae) { // IllegalArgumentException is thrown when index has changed since the search was performed and _docNum is no longer valid: prtlnErr("Error retrieving document: Lucene docID " + _docNum + " is no longer valid."); } catch (Throwable e) { prtlnErr("Error retrieving document: " + e); e.printStackTrace(); } // If all else fails, return empty document: return new Document(); }
/** * Closes the handler, releases resources and flushes the recent index changes * to persistent storage. */ @Override public synchronized void close() { if (isClosed) { throw new AlreadyClosedException("Already closed"); } isClosed = true; try { nrtManager.close(); if (analyzer != null) { analyzer.close(); } nrtManagerReopenThread.close(); indexWriter.close(); } catch (IOException ex) { LOG.log(Level.SEVERE, "Failed to close the Lucene index", ex); } LOG.info("Successfully closed the Lucene index..."); }
private void swapSearcher(final Directory dir) throws IOException { /* * opening a searcher is possibly very expensive. * We rather close it again if the Spellchecker was closed during * this operation than block access to the current searcher while opening. */ final IndexSearcher indexSearcher = createSearcher(dir); synchronized (searcherLock) { if(closed){ indexSearcher.getIndexReader().close(); throw new AlreadyClosedException("Spellchecker has been closed"); } if (searcher != null) { searcher.getIndexReader().close(); } // set the spellindex in the sync block - ensure consistency. searcher = indexSearcher; this.spellIndex = dir; } }
@Test public void testRollback() throws Exception { // Verifies that if rollback is called, DTW is closed. Directory dir = newDirectory(); DirectoryTaxonomyWriter dtw = new DirectoryTaxonomyWriter(dir); dtw.addCategory(new FacetLabel("a")); dtw.rollback(); try { dtw.addCategory(new FacetLabel("a")); fail("should not have succeeded to add a category following rollback."); } catch (AlreadyClosedException e) { // expected } dir.close(); }
@Test public void testAlreadyClosed() throws Exception { Directory dir = newDirectory(); DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir); ltw.addCategory(new FacetLabel("a")); ltw.close(); DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir); ltr.close(); try { ltr.getSize(); fail("An AlreadyClosedException should have been thrown here"); } catch (AlreadyClosedException ace) { // good! } dir.close(); }
public void testReaderChaining() throws Exception { assertTrue(reader.getRefCount() > 0); IndexReader wrappedReader = SlowCompositeReaderWrapper.wrap(reader); wrappedReader = new ParallelAtomicReader((AtomicReader) wrappedReader); IndexSearcher searcher = newSearcher(wrappedReader); TermRangeQuery query = TermRangeQuery.newStringRange("field", "a", "z", true, true); searcher.search(query, 5); reader.close(); // close original child reader try { searcher.search(query, 5); } catch (AlreadyClosedException ace) { assertEquals( "this IndexReader cannot be used anymore as one of its child readers was closed", ace.getMessage() ); } finally { // shutdown executor: in case of wrap-wrap-wrapping searcher.getIndexReader().close(); } }
public void testCloseInnerReader() throws Exception { Directory dir1 = getDir1(random()); AtomicReader ir1 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir1)); // with overlapping ParallelAtomicReader pr = new ParallelAtomicReader(true, new AtomicReader[] {ir1}, new AtomicReader[] {ir1}); ir1.close(); try { pr.document(0); fail("ParallelAtomicReader should be already closed because inner reader was closed!"); } catch (AlreadyClosedException e) { // pass } // noop: pr.close(); dir1.close(); }
private OutOfMemoryError getOOM(Throwable disaster, IndexWriter writer, PrintStream log) { Throwable e = disaster; if (e instanceof AlreadyClosedException) { e = e.getCause(); } if (e instanceof OutOfMemoryError && e.getMessage() != null && e.getMessage().startsWith("Fake OutOfMemoryError")) { log.println("\nTEST: got expected fake exc:" + e.getMessage()); e.printStackTrace(log); // TODO: remove rollback here, and add this assert to ensure "full OOM protection" anywhere IW does writes // assertTrue("hit OOM but writer is still open, WTF: ", writer.isClosed()); try { writer.rollback(); } catch (Throwable t) {} return (OutOfMemoryError) e; } else { Rethrow.rethrow(disaster); return null; // dead } }
public void testAfterClose() throws Exception { Directory dir1 = getAssertNoDeletesDirectory(newDirectory()); IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig(new MockAnalyzer(random()))); // create the index createIndexNoClose(false, "test", writer); DirectoryReader r = writer.getReader(); writer.close(); TestUtil.checkIndex(dir1); // reader should remain usable even after IndexWriter is closed: assertEquals(100, r.numDocs()); Query q = new TermQuery(new Term("indexname", "test")); IndexSearcher searcher = newSearcher(r); assertEquals(100, searcher.search(q, 10).totalHits); try { DirectoryReader.openIfChanged(r); fail("failed to hit AlreadyClosedException"); } catch (AlreadyClosedException ace) { // expected } r.close(); dir1.close(); }
public void testChangesAfterClose() throws IOException { Directory dir = newDirectory(); IndexWriter writer = null; writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); addDoc(writer); // close writer.close(); try { addDoc(writer); fail("did not hit AlreadyClosedException"); } catch (AlreadyClosedException e) { // expected } dir.close(); }
@Override void handle(Throwable t) { boolean report = true; if (t instanceof AlreadyClosedException || t instanceof MergePolicy.MergeAbortedException || t instanceof NullPointerException) { report = !didClose; } else if (t instanceof FileNotFoundException || t instanceof NoSuchFileException) { report = !didClose; } else if (t instanceof IOException) { Throwable t2 = t.getCause(); if (t2 instanceof MergePolicy.MergeAbortedException) { report = !didClose; } } if (report) { t.printStackTrace(System.out); synchronized(failures) { failures.add(t); } } }
@Override public void drop() throws IOException { if ( writer != null ) { writer.close(); } try { DirectorySupport.deleteDirectoryContents( directory = directory == null ? dirFactory.open( dirFile ) : directory ); } catch ( AlreadyClosedException e ) { // It was closed, open again just to be able to delete the files DirectorySupport.deleteDirectoryContents( directory = dirFactory.open( dirFile ) ); } finally { if ( directory != null ) { directory.close(); } } failureStorage.clearForIndex( indexId ); }
private void swapSearcher(final Directory dir) throws IOException { /* * opening a searcher is possibly very expensive. * We rather close it again if the Autocompleter was closed during * this operation than block access to the current searcher while opening. */ final IndexSearcher indexSearcher = createSearcher(dir); synchronized (searcherLock) { if(closed){ indexSearcher.close(); throw new AlreadyClosedException("Autocompleter has been closed"); } if (searcher != null) { searcher.close(); } // set the autocomplete index in the sync block - ensure consistency. searcher = indexSearcher; this.autoCompleteIndex = dir; } }
/** * Closes collection. * @throws IOException if clossing collection fails */ public void close() throws IOException { if (reader != null) { reader.close(); reader = null; } if (writer != null) { writer.close(); writer = null; } if (directory != null) { try { directory.close(); } catch (AlreadyClosedException ex) {} } if (folder != null) { for (File f : folder.listFiles()) { f.delete(); } folder.delete(); folder = null; } }
@Test public void testRollback() throws Exception { // Verifies that if rollback is called, DTW is closed. Directory dir = newDirectory(); DirectoryTaxonomyWriter dtw = new DirectoryTaxonomyWriter(dir); dtw.addCategory(new CategoryPath("a")); dtw.rollback(); try { dtw.addCategory(new CategoryPath("a")); fail("should not have succeeded to add a category following rollback."); } catch (AlreadyClosedException e) { // expected } dir.close(); }
@Test public void testAlreadyClosed() throws Exception { Directory dir = newDirectory(); DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir); ltw.addCategory(new CategoryPath("a")); ltw.close(); DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir); ltr.close(); try { ltr.getSize(); fail("An AlreadyClosedException should have been thrown here"); } catch (AlreadyClosedException ace) { // good! } dir.close(); }
/** * Expert: decreases the refCount of this IndexReader * instance. If the refCount drops to 0, then this * reader is closed. If an exception is hit, the refCount * is unchanged. * * @throws IOException in case an IOException occurs in doClose() * * @see #incRef */ public final void decRef() throws IOException { // only check refcount here (don't call ensureOpen()), so we can // still close the reader if it was made invalid by a child: if (refCount.get() <= 0) { throw new AlreadyClosedException("this IndexReader is closed"); } final int rc = refCount.decrementAndGet(); if (rc == 0) { boolean success = false; try { doClose(); success = true; } finally { if (!success) { // Put reference back on failure refCount.incrementAndGet(); } } reportCloseToParentReaders(); notifyReaderClosedListeners(); } else if (rc < 0) { throw new IllegalStateException("too many decRef calls: refCount is " + rc + " after decrement"); } }
public void testAfterClose() throws Exception { Directory dir1 = newDirectory(); IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); // create the index createIndexNoClose(false, "test", writer); DirectoryReader r = writer.getReader(); writer.close(); _TestUtil.checkIndex(dir1); // reader should remain usable even after IndexWriter is closed: assertEquals(100, r.numDocs()); Query q = new TermQuery(new Term("indexname", "test")); IndexSearcher searcher = newSearcher(r); assertEquals(100, searcher.search(q, 10).totalHits); try { DirectoryReader.openIfChanged(r); fail("failed to hit AlreadyClosedException"); } catch (AlreadyClosedException ace) { // expected } r.close(); dir1.close(); }
public void testChangesAfterClose() throws IOException { Directory dir = newDirectory(); IndexWriter writer = null; writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); addDoc(writer); // close writer.close(); try { addDoc(writer); fail("did not hit AlreadyClosedException"); } catch (AlreadyClosedException e) { // expected } dir.close(); }
@Override void handle(Throwable t) { boolean report = true; if (t instanceof AlreadyClosedException || t instanceof MergePolicy.MergeAbortedException || t instanceof NullPointerException) { report = !didClose; } else if (t instanceof FileNotFoundException) { report = !didClose; } else if (t instanceof IOException) { Throwable t2 = t.getCause(); if (t2 instanceof MergePolicy.MergeAbortedException) { report = !didClose; } } if (report) { t.printStackTrace(System.out); synchronized(failures) { failures.add(t); } } }
private void reopenIndexWritersIfNecessary() throws Exception { if (!indexWriter.isOpen()) { synchronized (this) { if (!indexWriter.isOpen()) { this.indexWriter = this.indexSegmentInterface.getIndexWriter(segmentNumber); this.directoryReader = DirectoryReader.open(indexWriter, indexConfig.getIndexSettings().getApplyUncommittedDeletes(), false); } } } //TODO: is this a real use case? try { taxoWriter.getSize(); } catch (AlreadyClosedException e) { synchronized (this) { this.taxoWriter = this.indexSegmentInterface.getTaxoWriter(segmentNumber); this.taxoReader = new DirectoryTaxonomyReader(taxoWriter); } } }
/** * Checks if the seed is an IndexReader, and if so will walk * the hierarchy of subReaders building up a list of the objects * returned by {@code seed.getCoreCacheKey()} */ private List<Object> getAllDescendantReaderKeys(Object seed) { List<Object> all = new ArrayList<Object>(17); // will grow as we iter all.add(seed); for (int i = 0; i < all.size(); i++) { final Object obj = all.get(i); // TODO: We don't check closed readers here (as getTopReaderContext // throws AlreadyClosedException), what should we do? Reflection? if (obj instanceof IndexReader) { try { final List<IndexReaderContext> childs = ((IndexReader) obj).getContext().children(); if (childs != null) { // it is composite reader for (final IndexReaderContext ctx : childs) { all.add(ctx.reader().getCoreCacheKey()); } } } catch (AlreadyClosedException ace) { // ignore this reader } } } // need to skip the first, because it was the seed return all.subList(1, all.size()); }