private void saveFsState(File dataStorageRoot, BuildFSState state) { final ProjectDescriptor pd = myProjectDescriptor; final File file = new File(dataStorageRoot, FS_STATE_FILE); try { final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(); final DataOutputStream out = new DataOutputStream(bytes); try { out.writeInt(BuildFSState.VERSION); out.writeLong(myLastEventOrdinal); out.writeBoolean(hasWorkToDo(state, pd)); state.save(out); } finally { out.close(); } saveOnDisk(bytes, file); } catch (Throwable e) { LOG.error(e); FileUtil.delete(file); } }
@Override public void force() { if (! myDirty) return; try{ final BufferExposingByteArrayOutputStream bos = new BufferExposingByteArrayOutputStream(); final DataOutput out = new DataOutputStream(bos); out.writeInt(myMap.size()); for (Map.Entry<KeyWrapper<K>, V> entry : myMap.entrySet()) { myKeyDescriptor.save(out, entry.getKey().myKey); myValueExternalizer.save(out, entry.getValue()); } FileUtil.writeToFile(myFile, bos.getInternalBuffer(), 0, bos.size()); } catch (IOException e) { LOG.error(e); } finally { myDirty = false; } }
private static void saveRegisteredIndices(@NotNull Collection<ID<?, ?>> ids) { final File file = getRegisteredIndicesFile(); try { FileUtil.createIfDoesntExist(file); final DataOutputStream os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file))); try { os.writeInt(ids.size()); for (ID<?, ?> id : ids) { IOUtil.writeString(id.toString(), os); } } finally { os.close(); } } catch (IOException ignored) { } }
private void saveFsState(File dataStorageRoot, BuildFSState state) { final ProjectDescriptor pd = myProjectDescriptor; final File file = new File(dataStorageRoot, FS_STATE_FILE); try { final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(); final DataOutputStream out = new DataOutputStream(bytes); try { out.writeInt(FSState.VERSION); out.writeLong(myLastEventOrdinal); out.writeBoolean(hasWorkToDo(state, pd)); state.save(out); } finally { out.close(); } saveOnDisk(bytes, file); } catch (Throwable e) { LOG.error(e); FileUtil.delete(file); } }
private static void savePathsToDelete(final File file, final Map<String, SourceUrlClassNamePair> outputs) { try { FileUtil.createParentDirs(file); final DataOutputStream os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file))); try { if (outputs != null) { os.writeInt(outputs.size()); for (Map.Entry<String, SourceUrlClassNamePair> entry : outputs.entrySet()) { CompilerIOUtil.writeString(entry.getKey(), os); final SourceUrlClassNamePair pair = entry.getValue(); CompilerIOUtil.writeString(pair.getSourceUrl(), os); CompilerIOUtil.writeString(pair.getClassName(), os); } } else { os.writeInt(0); } } finally { os.close(); } } catch (IOException e) { LOG.error(e); } }
public static void updateList(int id, @NotNull int[] children) { try { w.lock(); DbConnection.markDirty(); final DataOutputStream record = writeAttribute(id, CHILDREN_ATT, false); DataInputOutputUtil.writeINT(record, children.length); for (int child : children) { if (child == id) { LOG.error("Cyclic parent child relations"); } else { child = child > id ? child - id : -child; DataInputOutputUtil.writeINT(record, child); } } record.close(); } catch (Throwable e) { throw DbConnection.handleError(e); } finally { w.unlock(); } }
private static void updateFsStateOnDisk(File dataStorageRoot, DataInputStream original, final long ordinal) { final File file = new File(dataStorageRoot, FS_STATE_FILE); try { final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(); final DataOutputStream out = new DataOutputStream(bytes); try { out.writeInt(BuildFSState.VERSION); out.writeLong(ordinal); out.writeBoolean(false); while (true) { final int b = original.read(); if (b == -1) { break; } out.write(b); } } finally { out.close(); } saveOnDisk(bytes, file); } catch (Throwable e) { LOG.error(e); FileUtil.delete(file); } }
@NotNull private static File writeToTempFile(@NotNull HashImpl... hashes) throws IOException { File file = FileUtil.createTempFile("", ""); DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); try { for (HashImpl hash : hashes) { hash.write(out); } } finally { out.close(); } return file; }
private static void saveNameIdSequenceWithDeltas(int[] names, int[] ids, DataOutputStream output) throws IOException { DataInputOutputUtil.writeINT(output, names.length); int prevId = 0; int prevNameId = 0; for (int i = 0; i < names.length; i++) { DataInputOutputUtil.writeINT(output, names[i] - prevNameId); DataInputOutputUtil.writeINT(output, ids[i] - prevId); prevId = ids[i]; prevNameId = names[i]; } }
public static void updateList(int id, @NotNull int[] children) { w.lock(); try { DbConnection.markDirty(); final DataOutputStream record = writeAttribute(id, ourChildrenAttr); DataInputOutputUtil.writeINT(record, children.length); int prevId = id; Arrays.sort(children); for (int child : children) { if (child == id) { LOG.error("Cyclic parent child relations"); } else { DataInputOutputUtil.writeINT(record, child - prevId); prevId = child; } } record.close(); } catch (Throwable e) { throw DbConnection.handleError(e); } finally { w.unlock(); } }
@NotNull public static DataOutputStream writeAttribute(final int fileId, @NotNull FileAttribute att) { DataOutputStream stream = new AttributeOutputStream(fileId, att); if (att.isVersioned()) { try { DataInputOutputUtil.writeINT(stream, att.getVersion()); } catch (IOException e) { throw new RuntimeException(e); } } return stream; }
@NotNull public static Object compressStringRawBytes(@NotNull CharSequence string) { int length = string.length(); if (length < STRING_COMPRESSION_THRESHOLD) { if (string instanceof CharBuffer && ((CharBuffer)string).capacity() > STRING_COMPRESSION_THRESHOLD) { string = string.toString(); // shrink to size } return string; } try { ByteArrayOutputStream bytes = new ByteArrayOutputStream(length); @NotNull DataOutput out = new DataOutputStream(bytes); DataInputOutputUtil.writeINT(out, length); for (int i=0; i< length;i++) { char c = string.charAt(i); DataInputOutputUtil.writeINT(out, c); } byte[] compressedBytes = Snappy.compress(bytes.toByteArray()); return compressedBytes.length < length * 2 ? compressedBytes : string; } catch (CorruptionException ex) { ex.printStackTrace(); return string; } catch (IOException e) { e.printStackTrace(); return string; } }
@Override protected void doPut(Key key, ValueContainer<Value> container) throws IOException { synchronized (myEnumerator) { ChangeTrackingValueContainer<Value> valueContainer = (ChangeTrackingValueContainer<Value>)container; // try to accumulate index value calculated for particular key to avoid fragmentation: usually keys are scattered across many files // note that keys unique for indexed file have their value calculated at once (e.g. key is file id, index calculates something for particular // file) and there is no benefit to accumulate values for particular key because only one value exists if (!valueContainer.needsCompacting() && !myKeyIsUniqueForIndexedFile) { final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(); //noinspection IOResourceOpenedButNotSafelyClosed final DataOutputStream _out = new DataOutputStream(bytes); valueContainer.saveTo(_out, myValueExternalizer); appendData(key, new PersistentHashMap.ValueDataAppender() { @Override public void append(@NotNull final DataOutput out) throws IOException { out.write(bytes.getInternalBuffer(), 0, bytes.size()); } }); } else { // rewrite the value container for defragmentation super.doPut(key, valueContainer); } } }
private static void updateFsStateOnDisk(File dataStorageRoot, DataInputStream original, final long ordinal) { final File file = new File(dataStorageRoot, FS_STATE_FILE); try { final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(); final DataOutputStream out = new DataOutputStream(bytes); try { out.writeInt(FSState.VERSION); out.writeLong(ordinal); out.writeBoolean(false); while (true) { final int b = original.read(); if (b == -1) { break; } out.write(b); } } finally { out.close(); } saveOnDisk(bytes, file); } catch (Throwable e) { LOG.error(e); FileUtil.delete(file); } }
private static void saveSourceInfo(VirtualFile file, SourceFileInfo descriptor) { final java.io.DataOutputStream out = ourSourceFileAttribute.writeAttribute(file); try { try { descriptor.save(out); } finally { out.close(); } } catch (IOException ignored) { LOG.info(ignored); } }
private static void saveOutputInfo(VirtualFile file, OutputFileInfo descriptor) { final java.io.DataOutputStream out = ourOutputFileAttribute.writeAttribute(file); try { try { descriptor.save(out); } finally { out.close(); } } catch (IOException ignored) { LOG.info(ignored); } }
@Override protected void doPut(Key key, ValueContainer<Value> container) throws IOException { synchronized (myEnumerator) { ChangeTrackingValueContainer<Value> valueContainer = (ChangeTrackingValueContainer<Value>)container; if (!valueContainer.needsCompacting()) { final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(); //noinspection IOResourceOpenedButNotSafelyClosed final DataOutputStream _out = new DataOutputStream(bytes); final TIntHashSet set = valueContainer.getInvalidated(); if (set != null && set.size() > 0) { for (int inputId : set.toArray()) { ValueContainerExternalizer.saveInvalidateCommand(_out, inputId); } } final ValueContainer<Value> toAppend = valueContainer.getAddedDelta(); if (toAppend != null && toAppend.size() > 0) { myValueContainerExternalizer.save(_out, toAppend); } appendData(key, new PersistentHashMap.ValueDataAppender() { @Override public void append(@NotNull final DataOutput out) throws IOException { out.write(bytes.getInternalBuffer(), 0, bytes.size()); } }); } else { // rewrite the value container for defragmentation super.doPut(key, valueContainer); } } }
public static void updateList(int id, @Nonnull int[] children) { w.lock(); try { DbConnection.markDirty(); final DataOutputStream record = writeAttribute(id, ourChildrenAttr); DataInputOutputUtil.writeINT(record, children.length); int prevId = id; Arrays.sort(children); for (int child : children) { if (child == id) { LOG.error("Cyclic parent child relations"); } else { DataInputOutputUtil.writeINT(record, child - prevId); prevId = child; } } record.close(); } catch (Throwable e) { throw DbConnection.handleError(e); } finally { w.unlock(); } }
@Nonnull public static DataOutputStream writeAttribute(final int fileId, @Nonnull FileAttribute att) { DataOutputStream stream = new AttributeOutputStream(fileId, att); if (att.isVersioned()) { try { DataInputOutputUtil.writeINT(stream, att.getVersion()); } catch (IOException e) { throw new RuntimeException(e); } } return stream; }
@Nonnull public static Object compressStringRawBytes(@Nonnull CharSequence string) { int length = string.length(); if (length < STRING_COMPRESSION_THRESHOLD) { if (string instanceof CharBuffer && ((CharBuffer)string).capacity() > STRING_COMPRESSION_THRESHOLD) { string = string.toString(); // shrink to size } return string; } try { ByteArrayOutputStream bytes = new ByteArrayOutputStream(length); @Nonnull DataOutput out = new DataOutputStream(bytes); DataInputOutputUtil.writeINT(out, length); for (int i=0; i< length;i++) { char c = string.charAt(i); DataInputOutputUtil.writeINT(out, c); } byte[] compressedBytes = Snappy.compress(bytes.toByteArray()); return compressedBytes.length < length * 2 ? compressedBytes : string; } catch (CorruptionException ex) { ex.printStackTrace(); return string; } catch (IOException e) { e.printStackTrace(); return string; } }
public static <Key, Value> void checkValuesHaveProperEqualsAndHashCode(@Nonnull Map<Key, Value> data, @Nonnull ID<Key, Value> indexId, @Nonnull DataExternalizer<Value> valueExternalizer) { if (DebugAssertions.DEBUG) { for (Map.Entry<Key, Value> e : data.entrySet()) { final Value value = e.getValue(); if (!(Comparing.equal(value, value) && (value == null || value.hashCode() == value.hashCode()))) { LOG.error("Index " + indexId.toString() + " violates equals / hashCode contract for Value parameter"); } try { final BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(out); valueExternalizer.save(outputStream, value); outputStream.close(); final Value deserializedValue = valueExternalizer.read(new DataInputStream(new UnsyncByteArrayInputStream(out.getInternalBuffer(), 0, out.size()))); if (!(Comparing.equal(value, deserializedValue) && (value == null || value.hashCode() == deserializedValue.hashCode()))) { LOG.error("Index " + indexId.toString() + " deserialization violates equals / hashCode contract for Value parameter"); } } catch (IOException ex) { LOG.error(ex); } } } }
private static void writeRecordHeader(int recordTag, int fileId, DataOutputStream appender) throws IOException { DataInputOutputUtil.writeINT(appender, recordTag); DataInputOutputUtil.writeINT(appender, fileId); }
@NotNull public static DataOutputStream writeContent(int fileId, boolean readOnly) { return new ContentOutputStream(fileId, readOnly); }
@NotNull public static DataOutputStream writeAttribute(final int fileId, @NotNull String attId, boolean fixedSize) { return new AttributeOutputStream(fileId, attId, fixedSize); }
@Nonnull public static DataOutputStream writeContent(int fileId, boolean readOnly) { return new ContentOutputStream(fileId, readOnly); }
synchronized void flush() throws IOException { if (compactNecessary) { //noinspection IOResourceOpenedButNotSafelyClosed UnsyncByteArrayOutputStream compactedOutputStream = new UnsyncByteArrayOutputStream(values.length); //noinspection IOResourceOpenedButNotSafelyClosed DataOutput compactedOutput = new DataOutputStream(compactedOutputStream); Ref<IOException> ioExceptionRef = new Ref<>(); boolean result = indexId2NewState == null || indexId2NewState.forEachEntry(new TIntObjectProcedure<byte[]>() { @Override public boolean execute(int indexUniqueId, byte[] indexValue) { try { long indexCreationStamp = IndexingStamp.getIndexCreationStamp(ID.findById(indexUniqueId)); writeIndexValue(indexUniqueId, indexCreationStamp, indexValue, 0, indexValue.length, compactedOutput); return true; } catch (IOException ex) { ioExceptionRef.set(ex); return false; } } }); if (!result) throw ioExceptionRef.get(); result = indexId2Offset == null || indexId2Offset.forEachEntry(new TIntLongProcedure() { @Override public boolean execute(int chunkIndexId, long chunkOffsetAndSize) { try { int chunkOffset = (int)(chunkOffsetAndSize >> 32); int chunkSize = (int)chunkOffsetAndSize; writeIndexValue( chunkIndexId, IndexingStamp.getIndexCreationStamp(ID.findById(chunkIndexId)), values, chunkOffset, chunkSize, compactedOutput ); return true; } catch (IOException e) { ioExceptionRef.set(e); return false; } } }); if (!result) throw ioExceptionRef.get(); if (compactedOutputStream.size() > 0) storage.put(fileOrContentId, compactedOutputStream.toByteArray()); else storage.remove(fileOrContentId); } }
private static <Key, Value> void doAssociateData(int id, final ID<Key, ?> indexId, Value keys, DataExternalizer<Value> externalizer, IndexedStateMap index) throws IOException { final BufferExposingByteArrayOutputStream savedKeysData; if (keys != null) { //noinspection IOResourceOpenedButNotSafelyClosed externalizer.save(new DataOutputStream(savedKeysData = new BufferExposingByteArrayOutputStream()), keys); } else { savedKeysData = null; } FileAccessorCache.Handle<IndexedState> stateHandle = index.myStateCache.getIfCached(id); try { index.appendData(id, new PersistentHashMap.ValueDataAppender() { @Override public void append(DataOutput out) throws IOException { byte[] internalBuffer = null; int size = 0; if (savedKeysData != null) { internalBuffer = savedKeysData.getInternalBuffer(); size = savedKeysData.size(); } long indexCreationStamp = IndexingStamp.getIndexCreationStamp(indexId); writeIndexValue( indexId.getUniqueId(), indexCreationStamp, internalBuffer, 0, size, out ); final IndexedState indexedState = stateHandle != null ? stateHandle.get() : null; if (indexedState != null) { indexedState.appendIndexedState(indexId, indexCreationStamp, internalBuffer, size); } } }); } finally { if (stateHandle != null) stateHandle.release(); } }