private void queueUnresolvedFilesSinceLastRestart() { PersistentFS fs = PersistentFS.getInstance(); int maxId = FSRecords.getMaxId(); TIntArrayList list = new TIntArrayList(); for (int id= fileIsResolved.nextClearBit(1); id >= 0 && id < maxId; id = fileIsResolved.nextClearBit(id + 1)) { int nextSetBit = fileIsResolved.nextSetBit(id); int endOfRun = Math.min(maxId, nextSetBit == -1 ? maxId : nextSetBit); do { VirtualFile virtualFile = fs.findFileById(id); if (queueIfNeeded(virtualFile, myProject)) { list.add(id); } else { fileIsResolved.set(id); } } while (++id < endOfRun); } log("Initially added to resolve " + toVfString(list.toNativeArray())); }
private File getMirrorFile(@NotNull File originalFile) { if (!myFileSystem.isMakeCopyOfJar(originalFile)) return originalFile; final FileAttributes originalAttributes = FileSystemUtil.getAttributes(originalFile); if (originalAttributes == null) return originalFile; final String folderPath = getJarsDir(); if (!new File(folderPath).exists() && !new File(folderPath).mkdirs()) { return originalFile; } if (FSRecords.weHaveContentHashes) { return getMirrorWithContentHash(originalFile, originalAttributes); } final String mirrorName = originalFile.getName() + "." + Integer.toHexString(originalFile.getPath().hashCode()); final File mirrorFile = new File(folderPath, mirrorName); final FileAttributes mirrorAttributes = FileSystemUtil.getAttributes(mirrorFile); return mirrorDiffers(originalAttributes, mirrorAttributes, false) ? copyToMirror(originalFile, mirrorFile) : mirrorFile; }
public boolean processFile(NewVirtualFile file) { if (file.isDirectory() || file.is(VFileProperty.SPECIAL)) { return true; } try { DataInputStream stream = FSRecords.readContent(file.getId()); if (stream == null) return true; byte[] bytes = FileUtil.loadBytes(stream); totalSize.addAndGet(bytes.length); count.incrementAndGet(); ProgressManager.getInstance().getProgressIndicator().setText(file.getPresentableUrl()); } catch (IOException e) { LOG.error(e); } return true; }
public static boolean versionDiffers(@NotNull File versionFile, final int currentIndexVersion) { try { ourLastStamp = Math.max(ourLastStamp, versionFile.lastModified()); final DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(versionFile))); try { final int savedIndexVersion = DataInputOutputUtil.readINT(in); final int commonVersion = DataInputOutputUtil.readINT(in); final long vfsCreationStamp = DataInputOutputUtil.readTIME(in); return savedIndexVersion != currentIndexVersion || commonVersion != VERSION || vfsCreationStamp != FSRecords.getCreationTimestamp() ; } finally { in.close(); } } catch (IOException e) { return true; } }
private static Timestamps createOrGetTimeStamp(int id) { boolean isValid = id > 0; if (!isValid) id = -id; Timestamps timestamps = myTimestampsCache.get(id); if (timestamps == null) { final DataInputStream stream = FSRecords.readAttributeWithLock(id, Timestamps.PERSISTENCE); try { timestamps = new Timestamps(stream); } catch (IOException e) { throw new RuntimeException(e); } if (isValid) myTimestampsCache.put(id, timestamps); } return timestamps; }
@NotNull @Override public Outputs createValue(Integer key) { try { final String dirName = FSRecords.getNames().valueOf(key); final File storeFile; if (StringUtil.isEmpty(dirName)) { storeFile = null; } else { final File compilerCacheDir = CompilerPaths.getCacheStoreDirectory(dirName); storeFile = compilerCacheDir.exists()? new File(compilerCacheDir, "paths_to_delete.dat") : null; } return new Outputs(storeFile, loadPathsToDelete(storeFile)); } catch (IOException e) { LOG.info(e); return new Outputs(null, new HashMap<String, SourceUrlClassNamePair>()); } }
boolean isAssociated(int projectId, String outputPath) { if (myProjectToOutputPathMap != null) { try { final Object val = myProjectToOutputPathMap.get(projectId); if (val instanceof Integer) { return FileUtil.pathsEqual(outputPath, FSRecords.getNames().valueOf(((Integer)val).intValue())); } if (val instanceof TIntHashSet) { final int _outputPath = FSRecords.getNames().enumerate(outputPath); return ((TIntHashSet)val).contains(_outputPath); } } catch (IOException e) { LOG.info(e); } } return false; }
@Nonnull @Override public Outputs createValue(Integer key) { try { final String dirName = FSRecords.getNames().valueOf(key); final File storeFile; if (StringUtil.isEmpty(dirName)) { storeFile = null; } else { final File compilerCacheDir = CompilerPaths.getCacheStoreDirectory(dirName); storeFile = compilerCacheDir.exists() ? new File(compilerCacheDir, "paths_to_delete.dat") : null; } return new Outputs(storeFile, loadPathsToDelete(storeFile)); } catch (IOException e) { LOG.info(e); return new Outputs(null, new HashMap<String, SourceUrlClassNamePair>()); } }
@Nullable public static VirtualFileSystemEntry getFileById(int id, VirtualDirectoryImpl parent) { Segment segment = getSegment(id, false); if (segment == null) return null; int offset = getOffset(id); Object o = segment.myObjectArray.get(offset); if (o == null) return null; if (o == ourDeadMarker) { throw reportDeadFileAccess(new VirtualFileImpl(id, segment, parent)); } final int nameId = segment.getNameId(id); if (nameId <= 0) { FSRecords.invalidateCaches(); throw new AssertionError("nameId=" + nameId + "; data=" + o + "; parent=" + parent + "; parent.id=" + parent.getId() + "; db.parent=" + FSRecords.getParent(id)); } return o instanceof DirectoryData ? new VirtualDirectoryImpl(id, segment, (DirectoryData)o, parent, parent.getFileSystem()) : new VirtualFileImpl(id, segment, parent); }
private File getMirrorFile(@Nonnull File originalFile) { if (!myFileSystem.isMakeCopyOfJar(originalFile)) return originalFile; final FileAttributes originalAttributes = FileSystemUtil.getAttributes(originalFile); if (originalAttributes == null) return originalFile; final String folderPath = getJarsDir(); if (!new File(folderPath).exists() && !new File(folderPath).mkdirs()) { return originalFile; } if (FSRecords.weHaveContentHashes) { return getMirrorWithContentHash(originalFile, originalAttributes); } final String mirrorName = originalFile.getName() + "." + Integer.toHexString(originalFile.getPath().hashCode()); final File mirrorFile = new File(folderPath, mirrorName); final FileAttributes mirrorAttributes = FileSystemUtil.getAttributes(mirrorFile); return mirrorDiffers(originalAttributes, mirrorAttributes, false) ? copyToMirror(originalFile, mirrorFile) : mirrorFile; }
public boolean processFile(NewVirtualFile file) { if (file.isDirectory() || file.is(VFileProperty.SPECIAL)) return true; try { DataInputStream stream = FSRecords.readContent(file.getId()); if (stream == null) return true; byte[] bytes = FileUtil.loadBytes(stream); totalSize.addAndGet(bytes.length); count.incrementAndGet(); ProgressManager.getInstance().getProgressIndicator().setText(file.getPresentableUrl()); } catch (IOException e1) { LOG.error(e1); } return true; }
@Override public boolean isIndexedStateForFile(int fileId, @Nonnull VirtualFile file) { boolean indexedStateForFile = super.isIndexedStateForFile(fileId, file); if (!indexedStateForFile) return false; try { DataInputStream stream = FSRecords.readAttributeWithLock(fileId, VERSION_STAMP); int diff = stream != null ? DataInputOutputUtil.readINT(stream) : 0; if (diff == 0) return false; FileType fileType = myStubVersionMap.getFileTypeByIndexingTimestampDiff(diff); return fileType != null && myStubVersionMap.getStamp(file.getFileType()) == myStubVersionMap.getStamp(fileType); } catch (IOException e) { LOG.error(e); return false; } }
public static boolean versionDiffers(@Nonnull File versionFile, final int currentIndexVersion) { try { ourLastStamp = Math.max(ourLastStamp, versionFile.lastModified()); final DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(versionFile))); try { final int savedIndexVersion = DataInputOutputUtil.readINT(in); final int commonVersion = DataInputOutputUtil.readINT(in); final long vfsCreationStamp = DataInputOutputUtil.readTIME(in); return savedIndexVersion != currentIndexVersion || commonVersion != VERSION || vfsCreationStamp != FSRecords.getCreationTimestamp() ; } finally { in.close(); } } catch (IOException e) { return true; } }
private static Timestamps createOrGetTimeStamp(int id) { boolean isValid = id > 0; if (!isValid) { id = -id; } Timestamps timestamps = myTimestampsCache.get(id); if (timestamps == null) { final DataInputStream stream = FSRecords.readAttributeWithLock(id, Timestamps.PERSISTENCE); try { timestamps = new Timestamps(stream); } catch (IOException e) { throw new RuntimeException(e); } if (isValid) myTimestampsCache.put(id, timestamps); } return timestamps; }
@Override public void deleteFile(final Object requestor, @NotNull final VirtualFile file) throws IOException { final FSItem fsItem = convert(file); if (fsItem == null) { FSRecords.invalidateCaches(); throw new IllegalStateException("failed to delete file " + file.getPath()); } fsItem.getParent().removeChild(fsItem); }
public static void flushCache(@Nullable Integer finishedFile) { if (finishedFile != null && finishedFile == INVALID_FILE_ID) finishedFile = 0; // todo make better (e.g. FinishedFiles striping, remove integers) while (finishedFile == null || !ourFinishedFiles.offer(finishedFile)) { List<Integer> files = new ArrayList<Integer>(ourFinishedFiles.size()); ourFinishedFiles.drainTo(files); if (!files.isEmpty()) { for (Integer file : files) { Lock writeLock = getStripedLock(file).writeLock(); writeLock.lock(); try { Timestamps timestamp = myTimestampsCache.remove(file); if (timestamp == null) continue; if (timestamp.isDirty() /*&& file.isValid()*/) { final DataOutputStream sink = FSRecords.writeAttribute(file, Timestamps.PERSISTENCE); timestamp.writeToStream(sink); sink.close(); } } catch (IOException e) { throw new RuntimeException(e); } finally { writeLock.unlock(); } } } if (finishedFile == null) break; // else repeat until ourFinishedFiles.offer() succeeds } }
private int getProjectId(Project project) { try { return FSRecords.getNames().enumerate(CompilerPaths.getCompilerSystemDirectoryName(project)); } catch (IOException e) { LOG.info(e); } return -1; }
private int getModuleId(Module module) { try { return FSRecords.getNames().enumerate(module.getName().toLowerCase(Locale.US)); } catch (IOException e) { LOG.info(e); } return -1; }
String getSourceFilePath() { try { return FSRecords.getNames().valueOf(mySourcePath); } catch (IOException e) { LOG.info(e); } return null; }
@Nullable public String getClassName() { try { return myClassName < 0? null : FSRecords.getNames().valueOf(myClassName); } catch (IOException e) { LOG.info(e); } return null; }
private void addOutputPath(final int projectId, String outputPath) { try { addOutputPath(projectId, FSRecords.getNames().enumerate(outputPath)); } catch (IOException e) { LOG.info(e); } }
@NotNull private static IntObjectLinkedMap.MapEntry<Object> getEntry(int id) { final int stripe = id % ourNameCache.length; synchronized (ourNameCache[stripe]) { IntObjectLinkedMap.MapEntry<Object> entry = ourNameCache[stripe].getCachedEntry(id); if (entry != null) { return entry; } } return cacheData(FSRecords.getNameByNameId(id), id); }
@Override public File getMirrorFile(@NotNull File originalFile) { if (!myFileSystem.isMakeCopyOfJar(originalFile)) return originalFile; final FileAttributes originalAttributes = FileSystemUtil.getAttributes(originalFile); if (originalAttributes == null) return originalFile; final String folderPath = getJarsDir(); if (!new File(folderPath).exists() && !new File(folderPath).mkdirs()) { return originalFile; } if (FSRecords.weHaveContentHashes) { return getMirrorWithContentHash(originalFile, originalAttributes); } final String mirrorName = originalFile.getName() + "." + Integer.toHexString(originalFile.getPath().hashCode()); final File mirrorFile = new File(folderPath, mirrorName); final FileAttributes mirrorAttributes = FileSystemUtil.getAttributes(mirrorFile); if (mirrorAttributes == null || originalAttributes.length != mirrorAttributes.length || Math.abs(originalAttributes.lastModified - mirrorAttributes.lastModified) > FS_TIME_RESOLUTION) { return copyToMirror(originalFile, mirrorFile); } return mirrorFile; }
public void actionPerformed(AnActionEvent e) { final ApplicationEx app = (ApplicationEx)ApplicationManager.getApplication(); final boolean mac = Messages.canShowMacSheetPanel(); boolean canRestart = app.isRestartCapable(); String[] options = new String[canRestart ? 4 : 3]; options[0] = canRestart ? "Invalidate and Restart" : "Invalidate and Exit"; options[1] = mac ? "Cancel" : "Invalidate"; options[2] = mac ? "Invalidate" : "Cancel"; if (canRestart) { options[3] = "Just Restart"; } int result = Messages.showDialog(e.getData(PlatformDataKeys.PROJECT), "The caches will be invalidated and rebuilt on the next startup.\n" + "WARNING: Local History will be also cleared.\n\n" + "Would you like to continue?\n\n", "Invalidate Caches", options, 0, Messages.getWarningIcon()); if (result == -1 || result == (mac ? 1 : 2)) { return; } if (result == 3) { app.restart(true); return; } FSRecords.invalidateCaches(); if (result == 0) app.restart(true); }
@Nullable public String getClassName() { try { return myClassName < 0 ? null : FSRecords.getNames().valueOf(myClassName); } catch (IOException e) { LOG.info(e); } return null; }
@Override public void deleteFile(final Object requestor, @Nonnull final VirtualFile file) throws IOException { final FSItem fsItem = convert(file); if (fsItem == null) { FSRecords.invalidateCaches(); throw new IllegalStateException("failed to delete file " + file.getPath()); } fsItem.getParent().removeChild(fsItem); }
private static void saveVersion(File versionFile) { try { DataOutputStream versionOutputStream = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(versionFile))); try { DataInputOutputUtil.writeINT(versionOutputStream, VERSION); DataInputOutputUtil.writeTIME(versionOutputStream, FSRecords.getCreationTimestamp()); } finally { versionOutputStream.close(); } } catch (IOException ignore) { } }
@Override public void setIndexedStateForFile(int fileId, @Nonnull VirtualFile file) { super.setIndexedStateForFile(fileId, file); try { DataOutputStream stream = FSRecords.writeAttribute(fileId, VERSION_STAMP); DataInputOutputUtil.writeINT(stream, myStubVersionMap.getIndexingTimestampDiffForFileType(file.getFileType())); stream.close(); } catch (IOException e) { LOG.error(e); } }
public RefResolveServiceImpl(final Project project, final MessageBus messageBus, final PsiManager psiManager, StartupManager startupManager, ApplicationEx application, ProjectFileIndex projectFileIndex) throws IOException { super(project); ((FutureTask)resolveProcess).run(); myApplication = application; myProjectFileIndex = projectFileIndex; if (ENABLED) { log = new FileWriter(new File(getStorageDirectory(), "log.txt")); File dataFile = new File(getStorageDirectory(), "data"); fileIsResolved = ConcurrentBitSet.readFrom(new File(getStorageDirectory(), "bitSet")); log("Read resolved file bitset: " + fileIsResolved); int maxId = FSRecords.getMaxId(); PersistentIntList list = new PersistentIntList(dataFile, dataFile.exists() ? 0 : maxId); if (list.getSize() == maxId) { storage = list; } else { // just to be safe, re-resolve all if VFS files count changes since last restart list.dispose(); storage = new PersistentIntList(dataFile, maxId); log("VFS maxId changed: was "+list.getSize()+"; now: "+maxId+"; re-resolving everything"); fileIsResolved.clear(); } Disposer.register(this, storage); if (!application.isUnitTestMode()) { startupManager.runWhenProjectIsInitialized(new Runnable() { @Override public void run() { initListeners(messageBus, psiManager); startThread(); } }); } Disposer.register(this, new Disposable() { @Override public void dispose() { try { save(); log.close(); } catch (IOException e) { LOG.error(e); } } }); } else { log = null; fileIsResolved = null; storage = null; } }
public static int storeName(@NotNull String name) { final int idx = FSRecords.getNameId(name); cacheData(name, idx, calcStripeIdFromNameId(idx)); return idx; }
private static void dumpChildrenInDbRecursively(VirtualFile dir, int depth) { if (!(dir instanceof NewVirtualFile)) { System.out.println(dir.getPresentableUrl() + ": not in db (" + dir.getClass().getName() + ")"); return; } List<VirtualFile> dirs = new ArrayList<VirtualFile>(); int inDb = 0, contentInDb = 0, nullChildren = 0; PersistentFS persistentFS = PersistentFS.getInstance(); if (persistentFS.wereChildrenAccessed(dir)) { for (String name : persistentFS.listPersisted(dir)) { inDb++; NewVirtualFile child = ((NewVirtualFile)dir).refreshAndFindChild(name); if (child == null) { nullChildren++; continue; } if (child.isDirectory()) { dirs.add(child); } else if (FSRecords.getContentId(child.getId()) != 0) { contentInDb++; } } } System.out.print(dir.getPresentableUrl() + ": " + inDb + " children in db"); if (contentInDb > 0) { System.out.print(", content of " + contentInDb + " files in db"); } if (nullChildren > 0) { System.out.print(", " + nullChildren + " invalid files in db"); } System.out.println(); if (depth > 10) { System.out.println("too deep, skipping children"); } else { for (VirtualFile childDir : dirs) { dumpChildrenInDbRecursively(childDir, depth+1); } } }
OutputFileInfo(final String sourcePath, @Nullable String className) throws IOException { final PersistentStringEnumerator symtable = FSRecords.getNames(); mySourcePath = symtable.enumerate(sourcePath); myClassName = className != null? symtable.enumerate(className) : -1; }