@Test public void downloadUsingCacheFailsWhenTheHashIsIncorrect() throws Exception { final FileSystem fs = Jimfs.newFileSystem(); final RemoteFile remoteFile = RemoteFile.of( new URI("https://raw.githubusercontent.com/njlr/test-lib-a/3af013452fe6b448b1cb33bb81bb19da690ec764/BUCK"), HashCode.fromString("aaaaaaaaaaaaaa4f244296b0fba7a70166dea49c9e8d3eacda58d67a")); final Path cachePath = CacheTasks.getCachePath(fs, remoteFile); final SettableFuture<Throwable> futureException = SettableFuture.create(); CacheTasks.downloadToCache(fs, remoteFile).subscribe( next -> {}, error -> { futureException.set(error); }, () -> {}); final Throwable exception = futureException.get(5000L, TimeUnit.MILLISECONDS); assertTrue(exception instanceof DownloadFileException); assertTrue(exception.getCause() instanceof HashMismatchException); }
private DependentsSet affectedSince(JarSnapshot other) { final Set<String> affected = new HashSet<String>(); for (Map.Entry<String, HashCode> otherClass : other.getHashes().entrySet()) { String otherClassName = otherClass.getKey(); HashCode otherClassBytes = otherClass.getValue(); HashCode thisClsBytes = getHashes().get(otherClassName); if (thisClsBytes == null || !thisClsBytes.equals(otherClassBytes)) { //removed since or changed since affected.add(otherClassName); DependentsSet dependents = other.getAnalysis().getRelevantDependents(otherClassName); if (dependents.isDependencyToAll()) { return dependents; } affected.addAll(dependents.getDependentClasses()); } } return new DefaultDependentsSet(affected); }
JarClasspathSnapshot createSnapshot(Iterable<JarArchive> jarArchives) { Map<File, JarSnapshot> jarSnapshots = Maps.newHashMap(); Map<File, HashCode> jarHashes = Maps.newHashMap(); Set<String> allClasses = Sets.newHashSet(); Set<String> duplicateClasses = Sets.newHashSet(); for (JarArchive jar : jarArchives) { JarSnapshot snapshot = jarSnapshotter.createSnapshot(jar); jarSnapshots.put(jar.file, snapshot); jarHashes.put(jar.file, snapshot.getHash()); for (String c : snapshot.getClasses()) { if (!allClasses.add(c)) { duplicateClasses.add(c); } } } JarClasspathSnapshotData jarClasspathSnapshotData = new JarClasspathSnapshotData(jarHashes, duplicateClasses); return new JarClasspathSnapshot(jarSnapshots, jarClasspathSnapshotData); }
public static boolean checksumEquals(File file, String checksum) { if (file == null || !file.exists()) { return false; } try { HashCode hash = Files.hash(file, Hashing.sha1()); StringBuilder builder = new StringBuilder(); for (Byte hashBytes : hash.asBytes()) { builder.append(Integer.toString((hashBytes & 0xFF) + 0x100, 16).substring(1)); } return builder.toString().equals(checksum); } catch (IOException e) { OneClientLogging.error(e); } return false; }
@Override public RemoteFile deserialize(final JsonElement jsonElement, final Type type, final JsonDeserializationContext context) throws JsonParseException { Preconditions.checkNotNull(jsonElement); Preconditions.checkNotNull(type); Preconditions.checkNotNull(context); final JsonObject jsonObject = jsonElement.getAsJsonObject(); if (!jsonObject.has("url")) { throw new JsonParseException("RemoteFile must have a URL. "); } final URI url = context.deserialize(jsonObject.get("url"), URI.class); if (!jsonObject.has("sha256")) { throw new JsonParseException("RemoteFile must have a sha256. "); } final HashCode sha256 = context.deserialize(jsonObject.get("sha256"), HashCode.class); return RemoteFile.of(url, sha256); }
public TaskTypeTaskStateChanges(TaskExecution previousExecution, TaskExecution currentExecution, String taskPath, Class<? extends TaskInternal> taskClass, Collection<ClassLoader> taskActionClassLoaders, ClassLoaderHierarchyHasher classLoaderHierarchyHasher) { String taskClassName = taskClass.getName(); currentExecution.setTaskClass(taskClassName); HashCode taskClassLoaderHash = classLoaderHierarchyHasher.getClassLoaderHash(taskClass.getClassLoader()); currentExecution.setTaskClassLoaderHash(taskClassLoaderHash); HashCode taskActionsClassLoaderHash = calculateActionClassLoaderHash(taskActionClassLoaders, classLoaderHierarchyHasher); currentExecution.setTaskActionsClassLoaderHash(taskActionsClassLoaderHash); if (LOGGER.isInfoEnabled()) { LOGGER.info("Task {} class loader hash: {}", taskPath, taskClassLoaderHash); LOGGER.info("Task {} actions class loader hash: {}", taskPath, taskActionsClassLoaderHash); } this.taskPath = taskPath; this.taskClass = taskClassName; this.taskClassLoaderHash = taskClassLoaderHash; this.taskActionsClassLoaderHash = taskActionsClassLoaderHash; this.previousExecution = previousExecution; }
@Override public ClassLoader get(ClassLoaderId id, ClassPath classPath, @Nullable ClassLoader parent, @Nullable FilteringClassLoader.Spec filterSpec, HashCode overrideHashCode) { ClassPathSnapshot classPathSnapshot = snapshotter.snapshot(classPath); ClassLoaderSpec spec = new ClassLoaderSpec(parent, classPathSnapshot, filterSpec, overrideHashCode); synchronized (lock) { CachedClassLoader cachedLoader = byId.get(id); if (cachedLoader == null || !cachedLoader.is(spec)) { CachedClassLoader newLoader = getAndRetainLoader(classPath, spec, id); byId.put(id, newLoader); if (cachedLoader != null) { LOGGER.debug("Releasing previous classloader for {}", id); cachedLoader.release(id); } return newLoader.classLoader; } else { return cachedLoader.classLoader; } } }
@Override public RemoteArchive deserialize(final JsonElement jsonElement, final Type type, final JsonDeserializationContext context) throws JsonParseException { Preconditions.checkNotNull(jsonElement); Preconditions.checkNotNull(type); Preconditions.checkNotNull(context); final JsonObject jsonObject = jsonElement.getAsJsonObject(); if (!jsonObject.has("url")) { throw new JsonParseException("A remote archive must have a URL"); } final URI url = context.deserialize(jsonObject.get("url"), URI.class); final HashCode sha256 = context.deserialize(jsonObject.get("sha256"), HashCode.class); final Optional<String> subPath = jsonObject.has("subPath") ? Optional.of(jsonObject.get("subPath").getAsString()) : Optional.empty(); return RemoteArchive.of(url, sha256, subPath); }
@Test public void downloadRemoteFileCompletes() throws Exception { final FileSystem fs = Jimfs.newFileSystem(); final RemoteFile remoteFile = RemoteFile.of( new URI("https://raw.githubusercontent.com/nikhedonia/googletest/665327f0141d4a4fc4f2496e781dce436d742645/BUCK"), HashCode.fromString("d976069f5b47fd8fc57201f47026a70dee4e47b3141ac23567b8a0c56bf9288c")); final SettableFuture<Boolean> future = SettableFuture.create(); CommonTasks.downloadRemoteFile(fs, remoteFile, fs.getPath("test.txt").toAbsolutePath()) .subscribe( next -> { }, error -> { future.set(false); }, () -> { future.set(true); }); assertTrue(future.get(30, TimeUnit.SECONDS)); }
public static boolean checksumEquals(File file, List<String> checksum) { if (file == null || !file.exists()) { return false; } try { HashCode hash = Files.hash(file, Hashing.sha1()); StringBuilder builder = new StringBuilder(); for (Byte hashBytes : hash.asBytes()) { builder.append(Integer.toString((hashBytes & 0xFF) + 0x100, 16).substring(1)); } return checksum.contains(builder.toString()); } catch (IOException e) { OneClientLogging.error(e); } return false; }
public String getCacheKey() { Hasher hasher = hf.newHasher(); hasher.putUnencodedChars(queryKey); for (Number id : ids) { if (id instanceof Integer) { hasher.putInt(id.intValue()); } else if (id instanceof Long) { hasher.putLong(id.longValue()); } else if (id instanceof Short) { hasher.putLong(id.shortValue()); } else if (id instanceof Double) { hasher.putDouble(id.doubleValue()); } else if (id instanceof Float) { hasher.putFloat(id.floatValue()); } else if (id instanceof Byte) { hasher.putFloat(id.byteValue()); } } HashCode hashcode = hasher.hash(); return hashcode.toString(); }
public boolean updateState() throws IOException { // only check contents if length or modified time changed long newLastModified = file.lastModified(); long newLength = file.length(); if (lastModified == newLastModified && length == newLength) { return false; } // update stats lastModified = newLastModified; length = newLength; // check if contents changed HashCode newHashCode = Files.hash(file, sha256()); if (Objects.equals(hashCode, newHashCode)) { return false; } hashCode = newHashCode; return true; }
/** * 从失效转移队列中获取所有有资格执行的作业上下文. * * @return 有资格执行的作业上下文集合 */ public Collection<JobContext> getAllEligibleJobContexts() { if (!regCenter.isExisted(FailoverNode.ROOT)) { return Collections.emptyList(); } List<String> jobNames = regCenter.getChildrenKeys(FailoverNode.ROOT); Collection<JobContext> result = new ArrayList<>(jobNames.size()); Set<HashCode> assignedTasks = new HashSet<>(jobNames.size() * 10, 1); for (String each : jobNames) { List<String> taskIdList = regCenter.getChildrenKeys(FailoverNode.getFailoverJobNodePath(each)); if (taskIdList.isEmpty()) { regCenter.remove(FailoverNode.getFailoverJobNodePath(each)); continue; } Optional<CloudJobConfiguration> jobConfig = configService.load(each); if (!jobConfig.isPresent()) { regCenter.remove(FailoverNode.getFailoverJobNodePath(each)); continue; } List<Integer> assignedShardingItems = getAssignedShardingItems(each, taskIdList, assignedTasks); if (!assignedShardingItems.isEmpty() && jobConfig.isPresent()) { result.add(new JobContext(jobConfig.get(), assignedShardingItems, ExecutionType.FAILOVER)); } } return result; }
/** * Save an asset and create a blob * * @return blob content */ private static Content saveAsset(final StorageTx tx, final Asset asset, final Supplier<InputStream> contentSupplier, final String contentType, final AttributesMap contentAttributes, final HashCode hash) throws IOException { Content.applyToAsset(asset, maintainLastModified(asset, contentAttributes)); AssetBlob assetBlob = tx.setBlob( asset, asset.name(), contentSupplier, HASH_ALGORITHMS, null, contentType, false ); if(!hashVerifier.verify(hash, assetBlob.getHashes().get(MD5))) { return null; } asset.markAsDownloaded(); tx.saveAsset(asset); return toContent(asset, assetBlob.getBlob()); }
/** * Files exported from jars are exported into a certain folder so that we can rebuild them * when the related jar file changes. */ @NonNull private static String getJarFilePrefix(@NonNull File inputFile) { // get the filename String name = inputFile.getName(); // remove the extension int pos = name.lastIndexOf('.'); if (pos != -1) { name = name.substring(0, pos); } // add a hash of the original file path. String input = inputFile.getAbsolutePath(); HashFunction hashFunction = Hashing.sha1(); HashCode hashCode = hashFunction.hashString(input, Charsets.UTF_16LE); return name + "-" + hashCode.toString(); }
private String getHashFor(File inputFile) { String retval = alreadyChecked.get(inputFile.getAbsolutePath()); if (retval != null) return retval; // add a hash of the original file path try { HashFunction hashFunction = Hashing.md5(); HashCode hashCode = hashFunction.hashBytes(Files.readAllBytes(inputFile.toPath())); retval = hashCode.toString(); alreadyChecked.put(inputFile.getAbsolutePath(), retval); return retval; } catch (IOException e) { e.printStackTrace(); return "ERROR"; } }
@Override public ClassLoader createChildClassLoader(ClassLoader parent, ClassPath classPath, HashCode overrideHashCode) { HashCode hashCode = overrideHashCode != null ? overrideHashCode : calculateClassLoaderHash(classPath); ClassLoader classLoader = super.doCreateClassLoader(parent, classPath); hashCodes.put(classLoader, hashCode); return classLoader; }
public static HashCode sha256(final String x) { Preconditions.checkNotNull(x); final Charset charset = Charsets.UTF_8; final HashFunction hashFunction = Hashing.sha256(); return hashFunction.newHasher() .putString(x, charset) .hash(); }
@Test public void failsGracefullyForIncorrectHashes() throws Exception { final FileSystem fs = Jimfs.newFileSystem(); final DependencyLocks locks = DependencyLocks.of(ImmutableList.of( DependencyLock.of( RecipeIdentifier.of("loopperfect", "valuable"), ResolvedDependency.of(Either.right( RemoteArchive.of( new URI("https://github.com/LoopPerfect/valuable/archive/v0.1.0.zip"), HashCode.fromString("aaaaaaaaaaaaaaaaaaaaaaa71dd4a1fd3400f1d04b84954beb2f514ec69934c0"), "valuable-0.1.0")))))); EvenMoreFiles.writeFile(fs.getPath("buckaroo.json"), Serializers.serialize(Project.of())); EvenMoreFiles.writeFile(fs.getPath("buckaroo.lock.json"), Serializers.serialize(locks)); final SettableFuture<Throwable> futureException = SettableFuture.create(); InstallExistingTasks.installExistingDependenciesInWorkingDirectory(fs) .subscribe( next -> { }, error -> { futureException.set(error); }, () -> { }); final Throwable exception = futureException.get(5000L, TimeUnit.MILLISECONDS); assertTrue(exception instanceof DownloadFileException); assertTrue(exception.getCause() instanceof HashMismatchException); }
@Override public HashCode read(Decoder decoder) throws IOException { byte hashSize = decoder.readByte(); byte[] hash = new byte[hashSize]; decoder.readBytes(hash); return HashCode.fromBytes(hash); }
public static Either<Exception, HashCode> read(final String x) { Preconditions.checkNotNull(x); try { return Either.right(HashCode.fromString(x)); } catch (final Exception e) { return Either.left(e); } }
/** * @param hash of this jar * @param hashes hashes of all classes from the jar * @param data of classes analysis in this jar */ public JarSnapshotData(HashCode hash, Map<String, HashCode> hashes, ClassSetAnalysisData data) { assert hash != null; assert hashes != null; assert data != null; this.hash = hash; this.hashes = hashes; this.data = data; }
@Test public void worksForExistingLockFile1() throws Exception { final FileSystem fs = Jimfs.newFileSystem(); final DependencyLocks locks = DependencyLocks.of(ImmutableList.of( DependencyLock.of( RecipeIdentifier.of("loopperfect", "valuable"), ResolvedDependency.of(Either.right( RemoteArchive.of( new URI("https://github.com/LoopPerfect/valuable/archive/v0.1.0.zip"), HashCode.fromString("639d7d0df95f8467f4aa8da71dd4a1fd3400f1d04b84954beb2f514ec69934c0"), "valuable-0.1.0")))))); EvenMoreFiles.writeFile(fs.getPath("buckaroo.json"), Serializers.serialize(Project.of())); EvenMoreFiles.writeFile(fs.getPath("buckaroo.lock.json"), Serializers.serialize(locks)); InstallExistingTasks.installExistingDependenciesInWorkingDirectory(fs).toList().blockingGet(); assertTrue(Files.exists(fs.getPath(".buckconfig"))); assertTrue(Files.exists(fs.getPath(".buckconfig.local"))); final Path dependencyFolder = fs.getPath( "buckaroo", "official", "loopperfect", "valuable"); assertTrue(Files.exists(dependencyFolder.resolve(".buckconfig"))); assertTrue(Files.exists(dependencyFolder.resolve(".buckconfig.local"))); assertTrue(Files.exists(dependencyFolder.resolve("BUCK"))); }
@Override public Map<File, JarSnapshot> getJarSnapshots(final Map<File, HashCode> jarHashes) { return cache.getCacheAccess().useCache("loading jar snapshots", new Factory<Map<File, JarSnapshot>>() { public Map<File, JarSnapshot> create() { final Map<File, JarSnapshot> out = new HashMap<File, JarSnapshot>(); for (Map.Entry<File, HashCode> entry : jarHashes.entrySet()) { JarSnapshot snapshot = new JarSnapshot(cache.getCache().get(entry.getValue())); out.put(entry.getKey(), snapshot); } return out; } }); }
@Override public JarSnapshot get(HashCode key, final Factory<JarSnapshot> factory) { return new JarSnapshot(cache.get(key, new Factory<JarSnapshotData>() { public JarSnapshotData create() { return factory.create().getData(); } })); }
@Override public ClassAnalysis getClassAnalysis(final String className, final File classFile) { HashCode hash = hasher.hash(classFile); return cache.get(hash, new Factory<ClassAnalysis>() { public ClassAnalysis create() { return analyzer.getClassAnalysis(className, classFile); } }); }
@Override public CompilationFileState read(Decoder decoder) throws Exception { HashCode hash = hashSerializer.read(decoder); ImmutableSet<ResolvedInclude> resolvedIncludes = ImmutableSet.copyOf(resolveIncludesSerializer.read(decoder)); IncludeDirectives includeDirectives = sourceIncludesSerializer.read(decoder); return new CompilationFileState(hash, includeDirectives, resolvedIncludes); }
@TransactionalStoreBlob protected Content doSaveMetadata(final String assetPath, final TempBlob metadataContent, final Payload payload, final AssetKind assetKind, final AttributesMap attributesMap, final String project, final String version, final String group) throws IOException { HashCode hash = null; StorageTx tx = UnitOfWork.currentTx(); Bucket bucket = tx.findBucket(getRepository()); Component component = getOrCreateComponent(tx, bucket, project, version, group); Asset asset = findAsset(tx, bucket, assetPath); if (asset == null) { asset = tx.createAsset(bucket, component); asset.name(assetPath); asset.formatAttributes().set(P_ASSET_KIND, assetKind.name()); for (Entry<String, Object> entry : attributesMap) { asset.formatAttributes().set(entry.getKey(), entry.getValue()); } hash = hashVerifier.lookupHashFromAsset(tx, bucket, assetPath); } return saveAsset(tx, asset, metadataContent, payload, hash); }
private static void syncRegularFile(final Path sourceFile, final Path targetFile) throws IOException { if (Files.exists(targetFile)) { final HashCode sourceHash = sha256(sourceFile); final HashCode targetHash = sha256(targetFile); if (sourceHash.equals(targetHash)) { return; } } Files.copy(sourceFile, targetFile, StandardCopyOption.REPLACE_EXISTING); }
@Override public HashCode hash(TextResource resource) { File file = resource.getFile(); if (file != null) { return hash(file); } return delegate.hash(resource); }
private FileInfo snapshot(File file, long length, long timestamp) { String absolutePath = file.getAbsolutePath(); FileInfo info = cache.get(absolutePath); if (info != null && length == info.length && timestamp == info.timestamp) { return info; } HashCode hash = delegate.hash(file); info = new FileInfo(hash, length, timestamp); cache.put(stringInterner.intern(absolutePath), info); return info; }
@Test public void cacheWorksTwiceInARow() throws Exception { final FileSystem fs = Jimfs.newFileSystem(); final RemoteFile remoteFile = RemoteFile.of( new URI("https://raw.githubusercontent.com/njlr/test-lib-a/3af013452fe6b448b1cb33bb81bb19da690ec764/BUCK"), HashCode.fromString("bb7220f89f404f244ff296b0fba7a70166de35a49094631c9e8d3eacda58d67a")); final Path path1 = fs.getPath("test.txt"); final Path path2 = fs.getPath("test.txt"); CacheTasks.downloadUsingCache(remoteFile, path1) .toList() .timeout(10000L, TimeUnit.MILLISECONDS) .blockingGet(); final List<Event> events = CacheTasks.downloadUsingCache(remoteFile, path2) .toList() .timeout(10000L, TimeUnit.MILLISECONDS) .blockingGet(); assertTrue(Files.exists(path1)); assertTrue(Files.exists(path2)); assertEquals(remoteFile.sha256, EvenMoreFiles.hashFile(path1)); assertEquals(remoteFile.sha256, EvenMoreFiles.hashFile(path2)); assertTrue(events.stream().noneMatch(x -> x instanceof DownloadProgress)); }
private static HashCode calculateActionClassLoaderHash(Collection<ClassLoader> taskActionClassLoaders, ClassLoaderHierarchyHasher classLoaderHierarchyHasher) { if (taskActionClassLoaders.isEmpty()) { return NO_ACTION_LOADERS; } Hasher hasher = Hashing.md5().newHasher(); for (ClassLoader taskActionClassLoader : taskActionClassLoaders) { HashCode actionLoaderHash = classLoaderHierarchyHasher.getClassLoaderHash(taskActionClassLoader); if (actionLoaderHash == null) { return null; } hasher.putBytes(actionLoaderHash.asBytes()); } return hasher.hash(); }
@Test public void testSha256() throws Exception { final Path file = fs.getPath("/", "file"); Files.write(file, new byte[] {0x00, 0x01, 0x02}); final HashCode hashCode = PathUtils.sha256(file); assertThat( hashCode.toString(), is("ae4b3280e56e2faf83f414a6e3dabe9d5fbe18976544c05fed121accb85b53fc")); }
@Override public HashCode hash(File file) { try { Hasher hasher = createFileHasher(); Files.copy(file, Funnels.asOutputStream(hasher)); return hasher.hash(); } catch (IOException e) { throw new UncheckedIOException(String.format("Failed to create MD5 hash for file '%s'.", file), e); } }
public CacheEntry build() { Path baseDir = cacheDir.resolve(name); if (!keys.isEmpty()) { HashFunction hf = Hashing.md5(); Hasher h = hf.newHasher(); for (String key : keys) { h.putString(key, Charsets.UTF_8); } HashCode hc = h.hash(); baseDir = baseDir.resolve(hc.toString()); } cacheEntriesLock.lock(); try { CacheEntry cacheEntry = cacheEntries.get(baseDir.toString()); if (cacheEntry != null) { if (!cacheEntry.getKeys().equals(keys)) { throw new PowsyblException("Inconsistent hash"); } } else { cacheEntry = new CacheEntry(baseDir, keys); cacheEntries.put(baseDir.toString(), cacheEntry); } return cacheEntry; } finally { cacheEntriesLock.unlock(); } }
private HashCode computeSHA1Hash() { final Hasher hasher = SHA1_HASHER.newHasher() .putString(docID().toString(), Charsets.UTF_8) .putString(type().toString(), Charsets.UTF_8) .putString(role().toString(), Charsets.UTF_8) .putString(canonicalArgument().string(), Charsets.UTF_8) .putInt(canonicalArgument().charOffsetSpan().startInclusive()) .putInt(canonicalArgument().charOffsetSpan().endInclusive()) .putInt(baseFiller().startInclusive()) .putInt(baseFiller().endInclusive()); // we put PJ_CODE and AAJ_CODE into the hash because without them, // observe that shifting a second PJ element to being the first AAJ // element results in the same hash hasher.putInt(PJ_CODE); for (final CharOffsetSpan pj : Ordering.natural().sortedCopy(predicateJustifications())) { hasher.putInt(pj.startInclusive()).putInt(pj.endInclusive()); } hasher.putInt(AAJ_CODE); for (final CharOffsetSpan aaj : Ordering.natural().sortedCopy( additionalArgumentJustifications())) { hasher.putInt(aaj.startInclusive()).putInt(aaj.endInclusive()); } hasher.putInt(realis().ordinal()); return hasher.hash(); }
/** * Save an asset and create a blob * * @return blob content */ private static Content saveAsset(final StorageTx tx, final Asset asset, final Supplier<InputStream> contentSupplier, final Payload payload, final HashCode hash) throws IOException { AttributesMap contentAttributes = null; String contentType = null; if (payload instanceof Content) { contentAttributes = ((Content) payload).getAttributes(); contentType = payload.getContentType(); } return saveAsset(tx, asset, contentSupplier, contentType, contentAttributes, hash); }
public static FileHashEvent of(final Path file, final HashCode sha256) { return new FileHashEvent(file, sha256); }
private HashCode calculateClassLoaderHash(ClassPath classPath) { return snapshotter.snapshot(classPath).getStrongHash(); }