public ObjectObjectPersistentMultiMaplet(final File file, final KeyDescriptor<K> keyExternalizer, final DataExternalizer<V> valueExternalizer, final CollectionFactory<V> collectionFactory) throws IOException { myValueExternalizer = valueExternalizer; myMap = new PersistentHashMap<K, Collection<V>>(file, keyExternalizer, new CollectionDataExternalizer<V>(valueExternalizer, collectionFactory)); myCache = new SLRUCache<K, Collection>(CACHE_SIZE, CACHE_SIZE, keyExternalizer) { @NotNull @Override public Collection createValue(K key) { try { final Collection<V> collection = myMap.get(key); return collection == null? NULL_COLLECTION : collection; } catch (IOException e) { throw new BuildDataCorruptedException(e); } } }; }
@Override public void put(final K key, final Collection<V> value) { try { myCache.remove(key); myMap.appendData(key, new PersistentHashMap.ValueDataAppender() { public void append(DataOutput out) throws IOException { for (V v : value) { myValueExternalizer.save(out, v); } } }); } catch (IOException e) { throw new BuildDataCorruptedException(e); } }
public IntIntPersistentMultiMaplet(final File file, final KeyDescriptor<Integer> keyExternalizer) throws IOException { myMap = new PersistentHashMap<Integer, TIntHashSet>(file, keyExternalizer, new IntSetExternalizer()); myCache = new SLRUCache<Integer, TIntHashSet>(CACHE_SIZE, CACHE_SIZE) { @NotNull @Override public TIntHashSet createValue(Integer key) { try { final TIntHashSet collection = myMap.get(key); return collection == null? NULL_COLLECTION : collection; } catch (IOException e) { throw new BuildDataCorruptedException(e); } } }; }
public IntObjectPersistentMultiMaplet(final File file, final KeyDescriptor<Integer> keyExternalizer, final DataExternalizer<V> valueExternalizer, final CollectionFactory<V> collectionFactory) throws IOException { myValueExternalizer = valueExternalizer; myMap = new PersistentHashMap<Integer, Collection<V>>(file, keyExternalizer, new CollectionDataExternalizer<V>(valueExternalizer, collectionFactory)); myCache = new SLRUCache<Integer, Collection>(CACHE_SIZE, CACHE_SIZE) { @NotNull @Override public Collection createValue(Integer key) { try { final Collection<V> collection = myMap.get(key); return collection == null? NULL_COLLECTION : collection; } catch (IOException e) { throw new BuildDataCorruptedException(e); } } }; }
@Override public void put(final int key, final Collection<V> value) { try { myCache.remove(key); myMap.appendData(key, new PersistentHashMap.ValueDataAppender() { public void append(DataOutput out) throws IOException { for (V v : value) { myValueExternalizer.save(out, v); } } }); } catch (IOException e) { throw new BuildDataCorruptedException(e); } }
@NotNull private static ThrowableComputable<PersistentHashMap<String, Record>, IOException> getComputable(final File file) { return new ThrowableComputable<PersistentHashMap<String, Record>, IOException>() { @Override public PersistentHashMap<String, Record> compute() throws IOException { return new PersistentHashMap<String, Record>(file, new EnumeratorStringDescriptor(), new DataExternalizer<Record>() { @Override public void save(@NotNull DataOutput out, Record value) throws IOException { out.writeInt(value.magnitude); out.writeLong(value.date.getTime()); } @Override public Record read(@NotNull DataInput in) throws IOException { return new Record(in.readInt(), new Date(in.readLong())); } }); } }; }
public DetectedFrameworksData(Project project) { myDetectedFrameworks = new MultiMap<Integer, DetectedFrameworkDescription>(); File file = new File(FrameworkDetectorRegistryImpl.getDetectionDirPath() + File.separator + project.getName() + "." + project.getLocationHash() + File.separator + "files"); myNewFiles = new TIntObjectHashMap<TIntHashSet>(); try { myExistentFrameworkFiles = new PersistentHashMap<Integer, TIntHashSet>(file, EnumeratorIntegerDescriptor.INSTANCE, new TIntHashSetExternalizer()); } catch (IOException e) { LOG.info(e); PersistentHashMap.deleteFilesStartingWith(file); try { myExistentFrameworkFiles = new PersistentHashMap<Integer, TIntHashSet>(file, EnumeratorIntegerDescriptor.INSTANCE, new TIntHashSetExternalizer()); } catch (IOException e1) { LOG.error(e1); } } }
TracingData() { PersistentHashMap<Integer, Integer> lkeys = null; try { lkeys = createOrOpenMap(); flushingFuture = pool.scheduleWithFixedDelay(new Runnable() { @Override public void run() { if (keys.isDirty() && !keys.isClosed()) { keys.force(); } } }, 5, 5, TimeUnit.SECONDS); ShutDownTracker.getInstance().registerShutdownTask(new Runnable() { @Override public void run() { flushingFuture.cancel(false); } }); } catch (IOException ex) { ex.printStackTrace(); } keys = lkeys; }
public static void main(String[] args) throws IOException { PersistentHashMap<Integer, Integer> lkeys = createOrOpenMap(); List<Integer> mapping = (List<Integer>)lkeys.getAllKeysWithExistingMapping(); System.out.println(mapping.size()); final TIntIntHashMap map = new TIntIntHashMap(mapping.size()); for(Integer i:mapping) map.put(i, lkeys.get(i)); Collections.sort(mapping, new Comparator<Integer>() { @Override public int compare(Integer o1, Integer o2) { return map.get(o2) - map.get(o1); } }); for(int i = 0; i < 500; ++i) { //System.out.println(mapping.get(i) + ","); System.out.println(mapping.get(i) + ":" + map.get(mapping.get(i))); } lkeys.close(); }
public ObjectObjectPersistentMultiMaplet(final File file, final KeyDescriptor<K> keyExternalizer, final DataExternalizer<V> valueExternalizer, final CollectionFactory<V> collectionFactory) throws IOException { myValueExternalizer = valueExternalizer; myMap = new PersistentHashMap<K, Collection<V>>(file, keyExternalizer, new CollectionDataExternalizer<V>(valueExternalizer, collectionFactory)); myCache = new SLRUCache<K, Collection>(CACHE_SIZE, CACHE_SIZE) { @NotNull @Override public Collection createValue(K key) { try { final Collection<V> collection = myMap.get(key); return collection == null? NULL_COLLECTION : collection; } catch (IOException e) { throw new RuntimeException(e); } } }; }
@Override public void put(final K key, final Collection<V> value) { try { myCache.remove(key); myMap.appendData(key, new PersistentHashMap.ValueDataAppender() { public void append(DataOutput out) throws IOException { for (V v : value) { myValueExternalizer.save(out, v); } } }); } catch (IOException e) { throw new RuntimeException(e); } }
public IntObjectPersistentMaplet(final File file, final DataExternalizer<V> externalizer) { try { myMap = new PersistentHashMap<Integer, V>(file, new IntInlineKeyDescriptor(), externalizer); myCache = new SLRUCache<Integer, Object>(CACHE_SIZE, CACHE_SIZE) { @NotNull @Override public Object createValue(Integer key) { try { final V v1 = myMap.get(key); return v1 == null? NULL_OBJ : v1; } catch (IOException e) { throw new RuntimeException(e); } } }; } catch (IOException e) { throw new RuntimeException(e); } }
public IntIntPersistentMultiMaplet(final File file, final KeyDescriptor<Integer> keyExternalizer) throws IOException { myMap = new PersistentHashMap<Integer, TIntHashSet>(file, keyExternalizer, new IntSetExternalizer()); myCache = new SLRUCache<Integer, TIntHashSet>(CACHE_SIZE, CACHE_SIZE) { @NotNull @Override public TIntHashSet createValue(Integer key) { try { final TIntHashSet collection = myMap.get(key); return collection == null? NULL_COLLECTION : collection; } catch (IOException e) { throw new RuntimeException(e); } } }; }
public IntObjectPersistentMultiMaplet(final File file, final KeyDescriptor<Integer> keyExternalizer, final DataExternalizer<V> valueExternalizer, final CollectionFactory<V> collectionFactory) throws IOException { myValueExternalizer = valueExternalizer; myMap = new PersistentHashMap<Integer, Collection<V>>(file, keyExternalizer, new CollectionDataExternalizer<V>(valueExternalizer, collectionFactory)); myCache = new SLRUCache<Integer, Collection>(CACHE_SIZE, CACHE_SIZE) { @NotNull @Override public Collection createValue(Integer key) { try { final Collection<V> collection = myMap.get(key); return collection == null? NULL_COLLECTION : collection; } catch (IOException e) { throw new RuntimeException(e); } } }; }
@Override public void put(final int key, final Collection<V> value) { try { myCache.remove(key); myMap.appendData(key, new PersistentHashMap.ValueDataAppender() { public void append(DataOutput out) throws IOException { for (V v : value) { myValueExternalizer.save(out, v); } } }); } catch (IOException e) { throw new RuntimeException(e); } }
@Nullable private PersistentHashMap<Integer, Collection<Key>> createInputsIndex() throws IOException { Factory<PersistentHashMap<Integer, Collection<Key>>> factory = myInputsIndexFactory; if (factory != null) { try { return factory.create(); } catch (RuntimeException e) { if (e.getCause() instanceof IOException) { throw (IOException)e.getCause(); } throw e; } } return null; }
@Override protected void doPut(Key key, UpdatableValueContainer<Value> container) throws IOException { synchronized (myEnumerator) { final ChangeTrackingValueContainer<Value> valueContainer = (ChangeTrackingValueContainer<Value>)container; // try to accumulate index value calculated for particular key to avoid fragmentation: usually keys are scattered across many files // note that keys unique for indexed file have their value calculated at once (e.g. key is file id, index calculates something for particular // file) and there is no benefit to accumulate values for particular key because only one value exists if (!valueContainer.needsCompacting() && !myKeyIsUniqueForIndexedFile) { appendData(key, new PersistentHashMap.ValueDataAppender() { @Override public void append(@Nonnull final DataOutput out) throws IOException { valueContainer.saveTo(out, myValueExternalizer); } }); } else { // rewrite the value container for defragmentation super.doPut(key, valueContainer); } } }
@Nonnull private static ThrowableComputable<PersistentHashMap<String, Record>, IOException> getComputable(final File file) { return () -> new PersistentHashMap<>(file, EnumeratorStringDescriptor.INSTANCE, new DataExternalizer<Record>() { @Override public void save(@Nonnull DataOutput out, Record value) throws IOException { out.writeInt(value.magnitude); out.writeLong(value.date.getTime()); out.writeLong(value.configurationHash); } @Override public Record read(@Nonnull DataInput in) throws IOException { return new Record(in.readInt(), new Date(in.readLong()), in.readLong()); } }, 4096, CURRENT_VERSION); }
protected ClassFilesIndexWriter(final ClassFileIndexer<K, V> indexer, final CompileContext compileContext) { myIndexer = indexer; final File storageDir = getIndexRoot(compileContext); final Set<String> containingFileNames = listFiles(storageDir); if (!containingFileNames.contains("version") || !containingFileNames.contains(IndexState.STATE_FILE_NAME)) { throw new IllegalStateException("version or state file for index " + indexer.getIndexCanonicalName() + " not found in " + storageDir.getAbsolutePath()); } ClassFilesIndexStorageWriter<K, V> index = null; IOException exception = null; LOG.debug("start open... " + indexer.getIndexCanonicalName()); myMappings = compileContext.getProjectDescriptor().dataManager.getMappings(); for (int attempt = 0; attempt < 2; attempt++) { try { index = new ClassFilesIndexStorageWriter<K, V>(storageDir, myIndexer.getKeyDescriptor(), myIndexer.getDataExternalizer(), myMappings); break; } catch (final IOException e) { exception = e; PersistentHashMap.deleteFilesStartingWith(ClassFilesIndexStorageBase.getIndexFile(storageDir)); } } LOG.debug("opened " + indexer.getIndexCanonicalName()); if (index == null) { throw new RuntimeException(exception); } myIndex = index; myEmpty = IndexState.EXIST != IndexState.load(storageDir) || exception != null; IndexState.CORRUPTED.save(storageDir); }
private void initialize() throws IOException { myMap = new PersistentHashMap<K, CompiledDataValueContainer<V>>(myIndexFile, myKeyDescriptor, createValueContainerExternalizer(myValueExternalizer), INITIAL_INDEX_SIZE); myCache = new SLRUCache<K, CompiledDataValueContainer<V>>(CACHE_QUEUES_SIZE, CACHE_QUEUES_SIZE) { @NotNull @Override public CompiledDataValueContainer<V> createValue(final K key) { try { final CompiledDataValueContainer<V> valueContainer = myMap.get(key); if (valueContainer != null) { return valueContainer; } } catch (final IOException e) { throw new RuntimeException(e); } return new CompiledDataValueContainer<V>(); } @Override protected void onDropFromCache(final K key, final CompiledDataValueContainer<V> value) { try { myMap.put(key, value); } catch (final IOException e) { throw new RuntimeException(e); } } }; }
public void appendData(final Key key, final T data) throws IOException { synchronized (myDataLock) { myMap.appendData(key, new PersistentHashMap.ValueDataAppender() { public void append(DataOutput out) throws IOException { myStateExternalizer.save(out, data); } }); } }
@Override public void put(final int key, final int value) { try { myCache.remove(key); myMap.appendData(key, new PersistentHashMap.ValueDataAppender() { public void append(final DataOutput out) throws IOException { DataInputOutputUtil.writeINT(out, value); } }); } catch (IOException e) { throw new BuildDataCorruptedException(e); } }
private void createMap() throws IOException { try { myPersistentMap = new PersistentHashMap<KeyAndTargetData<Key>, PersistentStateData<SourceState,OutputState>>(myCacheFile, new SourceItemDataDescriptor(myCompiler.getItemKeyDescriptor()), new PersistentStateDataExternalizer(myCompiler)); } catch (PersistentEnumerator.CorruptedException e) { FileUtil.delete(myCacheFile); throw e; } }
public void wipe() throws IOException { try { myPersistentMap.close(); } catch (IOException ignored) { } PersistentHashMap.deleteFilesStartingWith(myCacheFile); createMap(); }
private PersistentHashMap<String, T> createMap(final File file) throws IOException { return new PersistentHashMap<String,T>(file, new EnumeratorStringDescriptor(), new DataExternalizer<T>() { public void save(@NotNull final DataOutput out, final T value) throws IOException { StateCache.this.write(value, out); } public T read(@NotNull final DataInput in) throws IOException { return StateCache.this.read(in); } }); }
@Override protected void doPut(Key key, ValueContainer<Value> container) throws IOException { synchronized (myEnumerator) { ChangeTrackingValueContainer<Value> valueContainer = (ChangeTrackingValueContainer<Value>)container; // try to accumulate index value calculated for particular key to avoid fragmentation: usually keys are scattered across many files // note that keys unique for indexed file have their value calculated at once (e.g. key is file id, index calculates something for particular // file) and there is no benefit to accumulate values for particular key because only one value exists if (!valueContainer.needsCompacting() && !myKeyIsUniqueForIndexedFile) { final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(); //noinspection IOResourceOpenedButNotSafelyClosed final DataOutputStream _out = new DataOutputStream(bytes); valueContainer.saveTo(_out, myValueExternalizer); appendData(key, new PersistentHashMap.ValueDataAppender() { @Override public void append(@NotNull final DataOutput out) throws IOException { out.write(bytes.getInternalBuffer(), 0, bytes.size()); } }); } else { // rewrite the value container for defragmentation super.doPut(key, valueContainer); } } }
private boolean hasValue(final PersistentHashMap<String, ?> map, Map<String, Boolean> cache, final String value) { Boolean res = cache.get(value); if (res == null) { res = doIndexTask(new IndexTask<Boolean>() { public Boolean doTask() throws Exception { return map.tryEnumerate(value) != 0; } }, false).booleanValue(); cache.put(value, res); } return res; }
@Override public void put(final int key, final int value) { try { myCache.remove(key); myMap.appendData(key, new PersistentHashMap.ValueDataAppender() { public void append(final DataOutput out) throws IOException { out.writeInt(value); } }); } catch (IOException e) { throw new RuntimeException(e); } }
public IntIntPersistentMaplet(final File file, final KeyDescriptor<Integer> k) { try { myMap = new PersistentHashMap<Integer, Integer>(file, k, new DataExternalizer<Integer>() { @Override public void save(DataOutput out, Integer value) throws IOException { out.writeInt(value); } @Override public Integer read(DataInput in) throws IOException { return in.readInt(); } }); myCache = new SLRUCache<Integer, Object>(CACHE_SIZE, CACHE_SIZE) { @NotNull @Override public Object createValue(Integer key) { try { final Integer v1 = myMap.get(key); return v1 == null? NULL_OBJ : v1; } catch (IOException e) { throw new RuntimeException(e); } } }; } catch (IOException e) { throw new RuntimeException(e); } }
private void initTimestampIndex(final boolean needReindex) { if (needReindex) { FileUtil.delete(IndexInfrastructure.getIndexRootDir(getFileTimestampsIndexId())); } for (int attempts = 0; attempts < 2; attempts++) { try { myFileTimestampsIndex = new PersistentHashMap<String, Long>(IndexInfrastructure.getStorageFile(getFileTimestampsIndexId()), new EnumeratorStringDescriptor(), new DataExternalizer<Long>() { @Override public void save(final DataOutput out, final Long value) throws IOException { out.writeLong(value); } @Override public Long read(final DataInput in) throws IOException { return in.readLong(); } }); } catch (IOException e) { FileUtil.delete(IndexInfrastructure.getIndexRootDir(getFileTimestampsIndexId())); } if (myFileTimestampsIndex != null) { return; } } throw new RuntimeException("Timestamps index not initialized"); }
public Cache(@NonNls final String storePath, final int cacheSize) throws IOException { myStorePath = storePath; new File(storePath).mkdirs(); myQNameToClassInfoMap = new CachedPersistentHashMap<Integer, ClassInfo>(getOrCreateFile("classes"), EnumeratorIntegerDescriptor.INSTANCE, new DataExternalizer<ClassInfo>() { public void save(DataOutput out, ClassInfo value) throws IOException { value.save(out); } public ClassInfo read(DataInput in) throws IOException { return new ClassInfo(in); } }, cacheSize * 2) { protected boolean isValueDirty(ClassInfo classInfo) { return classInfo.isDirty(); } }; myDependencies = new BackwardDependenciesStorage(getOrCreateFile("bdeps"), cacheSize); myQNameToReferencedClassesMap = new CompilerDependencyStorage<Integer>(getOrCreateFile("fdeps"), EnumeratorIntegerDescriptor.INSTANCE, cacheSize); myQNameToSubclassesMap = new CompilerDependencyStorage<Integer>(getOrCreateFile("subclasses"), EnumeratorIntegerDescriptor.INSTANCE, cacheSize); myRemoteQNames = new PersistentHashMap<Integer, Boolean>(getOrCreateFile("remote"), EnumeratorIntegerDescriptor.INSTANCE, new DataExternalizer<Boolean>() { public void save(DataOutput out, Boolean value) throws IOException { out.writeBoolean(value.booleanValue()); } public Boolean read(DataInput in) throws IOException { return in.readBoolean(); } }, cacheSize); }
private PersistentHashMap<String, T> createMap(final File file) throws IOException { return new PersistentHashMap<String,T>(file, new EnumeratorStringDescriptor(), new DataExternalizer<T>() { public void save(final DataOutput out, final T value) throws IOException { StateCache.this.write(value, out); } public T read(final DataInput in) throws IOException { return StateCache.this.read(in); } }); }
private PersistentHashMap<File, T> createMap(final File file) throws IOException { return new PersistentHashMap<>(file, FileKeyDescriptor.INSTANCE, new DataExternalizer<T>() { @Override public void save(final DataOutput out, final T value) throws IOException { StateCache.this.write(value, out); } @Override public T read(final DataInput in) throws IOException { return StateCache.this.read(in); } }); }
private void doDelete() throws IOException { close(); PersistentHashMap.deleteFilesStartingWith(myIndexFile); }