public boolean shouldReload() { if(context == null) return true; if(!configuration.autoReload()) return false; if(context.loadedFiles().isEmpty()) return configuration.reloadWhenError(); try { for(Map.Entry<Path, HashCode> loaded : context.loadedFiles().entrySet()) { HashCode latest = Files.hash(loaded.getKey().toFile(), Hashing.sha256()); if(!latest.equals(loaded.getValue())) return true; } return false; } catch (IOException e) { return true; } }
public static String genSignature(HttpServletRequestEx requestEx) { Hasher hasher = Hashing.sha256().newHasher(); hasher.putString(requestEx.getRequestURI(), StandardCharsets.UTF_8); for (String paramName : paramNames) { String paramValue = requestEx.getHeader(paramName); if (paramValue != null) { hasher.putString(paramName, StandardCharsets.UTF_8); hasher.putString(paramValue, StandardCharsets.UTF_8); System.out.printf("%s %s\n", paramName, paramValue); } } byte[] bytes = requestEx.getBodyBytes(); if (bytes != null) { hasher.putBytes(bytes, 0, requestEx.getBodyBytesLength()); } return hasher.hash().toString(); }
public boolean isFullDuplicate(IHttpRequestResponse messageInfo) { PrintWriter stdout = new PrintWriter(callbacks.getStdout(), true); IResponseInfo respInfo = helpers.analyzeResponse(messageInfo.getResponse()); if (dubBloomFilter == null) return false; HashFunction m_hash = Hashing.murmur3_32(); if (helpers.bytesToString(messageInfo.getResponse()).length() > respInfo.getBodyOffset()) { String body = helpers.bytesToString(messageInfo.getResponse()).substring(respInfo.getBodyOffset()); /* full-dub detection */ String dedupHashValue = m_hash.hashBytes(helpers.stringToBytes(body)).toString(); if (dubBloomFilter.mightContain(dedupHashValue)) { return true; } dubBloomFilter.put(dedupHashValue); } return false; }
private static String makeSign(long userId, long sid, long expireTime) { StringBuilder stringBuilder = new StringBuilder(); long type = (userId + sid + expireTime) % 3; if (type == 0) { stringBuilder.append(expireTime); stringBuilder.append(userId); stringBuilder.append(sid); stringBuilder.append(type); } else if (type == 1) { stringBuilder.append(userId); stringBuilder.append(expireTime); stringBuilder.append(sid); stringBuilder.append(type); } else { stringBuilder.append(sid); stringBuilder.append(expireTime); stringBuilder.append(userId); stringBuilder.append(type); } return Hashing.sha256().hashBytes(stringBuilder.toString().getBytes()).toString(); }
/** * Calculate bit positions of {@code str}. * <p> * See "Less Hashing, Same Performance: Building a Better Bloom Filter" by Adam Kirsch and Michael * Mitzenmacher. * </p> * * @param str * @return */ public int[] calcBitPositions(String str) { int[] bitPositions = new int[this.k]; long hash64 = Hashing.murmur3_128().hashString(str, UTF_8).asLong(); int hash1 = (int) hash64; int hash2 = (int) (hash64 >>> 32); for (int i = 1; i <= this.k; i++) { int combinedHash = hash1 + (i * hash2); // Flip all the bits if it's negative (guaranteed positive number) if (combinedHash < 0) { combinedHash = ~combinedHash; } bitPositions[i - 1] = combinedHash % this.m; } return bitPositions; }
@Test public void testImageTransform() throws Exception { ClassLoader loader = Thread.currentThread().getContextClassLoader(); String origPath = loader.getResource("com/filestack/sample_image.jpg").getPath(); File origFile = new File(origPath); FileLink fileLink = client.upload(origPath, false); HANDLES.add(fileLink.getHandle()); ImageTransform transform = fileLink.imageTransform(); transform.addTask(new CropTask(0, 0, 500, 500)); String cropPath = loader.getResource("com/filestack/sample_image_cropped.jpg").getPath(); File cropFile = new File(cropPath); String correct = Files.asByteSource(cropFile).hash(Hashing.sha256()).toString(); byte[] bytes = transform.getContent().bytes(); String output = Hashing.sha256().hashBytes(bytes).toString(); Assert.assertEquals(correct, output); }
private void processRegions(File file, RegionProcessor regionProcessor, String randomAccessFileMode, FileChannel.MapMode mapMode) throws IOException { try ( RandomAccessFile randomAccessFile = new RandomAccessFile(file.getAbsolutePath().toFile(), randomAccessFileMode); FileChannel channel = randomAccessFile.getChannel() ) { RegionCalculator.calculateForSize(file, file.getSize()); for (Region region : file.getRegions().values()) { Hasher hasher = Hashing.sha256().newHasher(); MappedByteBuffer mappedByteBuffer = channel.map(mapMode, region.getOffset(), region.getSize()); int sum = regionProcessor.processRegion(region, hasher, mappedByteBuffer); region.setQuickDigest(sum); byte[] slowDigest = hasher.hash().asBytes(); region.setSlowDigest(slowDigest); clientMessageHandler.submitClientRegionMessage(clientId, file, region.getOffset(), region.getSize(), sum, slowDigest); } } }
/** * Files exported from jars are exported into a certain folder so that we can rebuild them * when the related jar file changes. */ @NonNull private static String getJarFilePrefix(@NonNull File inputFile) { // get the filename String name = inputFile.getName(); // remove the extension int pos = name.lastIndexOf('.'); if (pos != -1) { name = name.substring(0, pos); } // add a hash of the original file path. String input = inputFile.getAbsolutePath(); HashFunction hashFunction = Hashing.sha1(); HashCode hashCode = hashFunction.hashString(input, Charsets.UTF_16LE); return name + "-" + hashCode.toString(); }
public boolean put(String url) { byte[] bytes = Hashing.murmur3_128().hashObject(url, funnel).asBytes(); long hash1 = this.lowerEight(bytes); long hash2 = this.upperEight(bytes); boolean bitsChanged = false; long combinedHash = hash1; try { for (int i = 0; i < numHashFunctions; ++i) { /* mask combinedHash with 0x7FFFFFFFFFFFFFFF */ bitsChanged |= bits.set((combinedHash & 9223372036854775807L) % bitsSize); combinedHash += hash2; } } catch (IOException e) { e.printStackTrace(); } return bitsChanged; }
public boolean mightContain(String url) throws IOException { byte[] bytes = Hashing.murmur3_128().hashObject(url, funnel).asBytes(); long hash1 = this.lowerEight(bytes); long hash2 = this.upperEight(bytes); long combinedHash = hash1; for (int i = 0; i < numHashFunctions; ++i) { if (!bits.get((combinedHash & 9223372036854775807L) % bitsSize)) { return false; } combinedHash += hash2; } return true; }
@Override public void process(ResultItems resultItems, Task task) { SpiderInfo spiderInfo = resultItems.get("spiderInfo"); Webpage webpage = convertResultItems2Webpage(resultItems); SearchRequestBuilder searchRequestBuilder = client.prepareSearch(INDEX_NAME) .setTypes(TYPE_NAME) .setQuery(QueryBuilders.matchQuery("url", webpage.getUrl())); SearchResponse response = searchRequestBuilder.execute().actionGet(); if (response.getHits().totalHits() == 0) { try { client.prepareIndex(INDEX_NAME, TYPE_NAME) .setId(Hashing.md5().hashString(webpage.getUrl(), Charset.forName("utf-8")).toString()) .setSource(gson.toJson(webpage)) .get(); } catch (Exception e) { LOG.error("索引 Webpage 出错," + e.getLocalizedMessage()); } } }
/** * add * * @param element * @return */ public boolean add(CharSequence element) { if (element == null) { return false; } initCharset(); BloomFilter<CharSequence> bf = this.bloomfilters.get(0); if (getBloomfilterBucketLength() > 1) { byte[] datas = element.toString().getBytes(this.charset); int bfIndex = Math.abs(Hashing.murmur3_128().hashBytes(datas).asInt()) % getBloomfilterBucketLength(); bf = this.bloomfilters.get(bfIndex); } synchronized (bf) { bf.put(element); numberOfAddedElements++; return true; } }
/** * 将Webmagic中的ResultItems转换为Webpage * @param items * @return */ public static Webpage convertToWebpage(ResultItems items) { if(items.toString().indexOf("ResultItems{fields={}") > 0) { return null; } Webpage page = new Webpage().setContent(items.get("content")).setTitle(items.get("title")) .setUrl(items.get("url")).setDomain(items.get("domain")).setSpiderInfoId(items.get("spiderInfoId")) .setGathertime(items.get("gathertime")).setSpiderUUID(items.get("spiderUUID")) .setKeywords(items.get("keywords")).setSummary(items.get("summary")) .setNamedEntity(items.get("namedEntity")).setPublishTime(items.get("publishTime")) .setCategory(items.get("category")).setRawHTML(items.get("rawHTML")) .setDynamicFields(items.get(DYNAMIC_FIELD)).setStaticFields(items.get("staticField")) .setAttachmentList(items.get("attachmentList")).setImageList(items.get("imageList")) .setProcessTime(items.get("processTime")); return page.setId(Hashing.sha256().hashString(page.getUrl(), Charset.forName("utf-8")).toString()); }
@Override public void process(ResultItems resultItems, Task task) { SpiderInfo info = resultItems.get("spiderInfo"); Webpage page = convertToWebpage(resultItems); /* * guava22.0不再对MD5()提供支持, * 如果想更安全,使用sha256(), * 如果想更快,使用goodFastHash() * */ try { client.prepareIndex(INDEX_NAME, TYPE_NAME) .setId(Hashing.sha256().hashString(page.getUrl(), Charset.forName("utf-8")).toString()) .setSource(GSON.toJson(page), XContentType.JSON) .get(); } catch(Exception e) { LOG.error("索引Webpage出错, 由于 " + e.getLocalizedMessage()); } }
/** * Build a stringified MAC address using the ClusterMetadata hash for uniqueness. * Form of MAC is "02:eb" followed by four bytes of clusterMetadata hash. */ static String fingerprintMac(ClusterMetadata cm) { if (cm == null) { return DEFAULT_MAC; } HashFunction hf = Hashing.murmur3_32(); HashCode hc = hf.newHasher().putObject(cm, ClusterMetadata.HASH_FUNNEL).hash(); int unqf = hc.asInt(); StringBuilder sb = new StringBuilder(); sb.append("02:eb"); for (int i = 0; i < 4; i++) { byte b = (byte) (unqf >> i * 8); sb.append(String.format(":%02X", b)); } return sb.toString(); }
@Override public Server choose(Object key) { //使用一致性哈希进行分发 //TODO 此处使用了GUAVA中简单的一致性哈希算法选择服务,但这里存在性能缺陷:当reachableServers中间某一个服务节点失效了 //那么后续节点的一致性哈希结果将会不匹配,后续需要使用更完善的哈希环 加上 虚拟节点 的形式解决本问题 List<Server> reachableServers = getLoadBalancer().getReachableServers(); if(reachableServers != null && reachableServers.size() != 0){ int serverSeq = Hashing.consistentHash(Thread.currentThread().getId(), reachableServers.size()); return reachableServers.get(serverSeq); } else { return super.choose(key); } }
private static HashCode calculateFilterSpecHash(FilteringClassLoader.Spec spec) { Hasher hasher = Hashing.md5().newHasher(); addToHash(hasher, spec.getClassNames()); addToHash(hasher, spec.getPackageNames()); addToHash(hasher, spec.getPackagePrefixes()); addToHash(hasher, spec.getResourcePrefixes()); addToHash(hasher, spec.getResourceNames()); addToHash(hasher, spec.getDisallowedClassNames()); addToHash(hasher, spec.getDisallowedPackagePrefixes()); return hasher.hash(); }
@Override public ClassPathSnapshot snapshot(ClassPath classPath) { final List<String> visitedFilePaths = Lists.newLinkedList(); final Set<File> visitedDirs = Sets.newLinkedHashSet(); final List<File> cpFiles = classPath.getAsFiles(); com.google.common.hash.Hasher checksum = Hashing.md5().newHasher(); hash(checksum, visitedFilePaths, visitedDirs, cpFiles.iterator()); return new HashClassPathSnapshot(visitedFilePaths, checksum.hash()); }
/** * Step 1. Create the Canonical Request String * * @param method * @param payload * @param params * @param headers * @return */ private String createCanonicalString(String method, String payload, HashMap<String, String> params, HashMap<String, String> headers) { String signString = ""; signString = signString + method + "\n"; signString = signString + "/\n"; signString = signString + createCanonicalParameterString(params); signString = signString + "\n"; signString = signString + createCanonicalHeaderString(headers); signString = signString + "\n"; signString = signString + createCanonicalHeaderList(headers) + "\n"; signString = signString + Hashing.sha256().hashString(payload.toLowerCase(), StandardCharsets.UTF_8).toString(); signString = Hashing.sha256().hashString(signString, StandardCharsets.UTF_8).toString(); System.out.println("\nStep 1 <RESULT>: Generated Canonical String: \n" + signString); return signString; }
public BitPatternGenerator(String secret) { secretKey = Hashing.sha256().newHasher().putString(secret, Charsets.UTF_8).hash().asBytes(); try { digest = MessageDigest.getInstance("SHA-256"); } catch (NoSuchAlgorithmException e) { throw new RuntimeException(e); } }
private byte[] chainedHashing(long blockNumber, Function<byte[], byte[]> nextHash2) { byte[] lastHash = Hashing.sha256().newHasher(EncConstants.SHA256_LEN + 4) .putLong(blockNumber) .putBytes(secretKey) .hash() .asBytes(); int numHashes = EncConstants.BLOCKSIZE / EncConstants.SHA256_LEN; final byte[] ret = new byte[EncConstants.BLOCKSIZE]; for (int i = 0; i < numHashes; i++) { final byte[] nextHash = nextHash2.apply(lastHash); System.arraycopy(nextHash, 0, ret, i * EncConstants.SHA256_LEN, EncConstants.SHA256_LEN); lastHash = nextHash; } return ret; }
public static String genSignature(HttpServletResponseEx responseEx) { Hasher hasher = Hashing.sha256().newHasher(); byte[] bytes = responseEx.getBodyBytes(); if (bytes != null) { hasher.putBytes(bytes, 0, responseEx.getBodyBytesLength()); } return hasher.hash().toString(); }
private static String makeSignature(DexApplication app) { List<DexProgramClass> classes = getSortedClasses(app); StringBuilder builder = new StringBuilder(); for (DexProgramClass clazz : classes) { builder.append(clazz.toString()).append(";"); } return Hashing.sha1().hashString(builder.toString(), Charsets.UTF_8).toString(); }
public String getHash(Graph graph, String propertyName, Visibility visibility) { String visibilityString = visibility.getVisibilityString(); String propertyNameVisibilityToHashKey = getMetadataKey(propertyName, visibilityString); String hash = (String) graph.getMetadata(propertyNameVisibilityToHashKey); if (hash != null) { saveHashToVisibility(graph, hash, visibilityString); return hash; } hash = Hashing.murmur3_128().hashString(visibilityString, UTF8).toString(); graph.setMetadata(propertyNameVisibilityToHashKey, hash); saveHashToVisibility(graph, hash, visibilityString); return hash; }
public FastBloomFilter(int bits, int numberHashFunctions) { bs = new BitSet(bits); Random r = new Random(System.currentTimeMillis()); hashFunctions = new HashFunction[numberHashFunctions]; for (int i=0; i<numberHashFunctions; ++i) { hashFunctions[i] = Hashing.murmur3_128(r.nextInt()); } capacity = bits; }
public boolean isDuplicateURL(IHttpRequestResponse messageInfo) { if (dubBloomFilter == null) return false; IRequestInfo requestInfo = helpers.analyzeRequest(messageInfo.getHttpService(), messageInfo.getRequest()); if (requestInfo == null) return true; HashFunction m_hash = Hashing.murmur3_32(); /* don't know if Burp has a deduplication here, make it sure */ String hashInput = requestInfo.getUrl().getPath() + "?"; if (requestInfo.getUrl().getQuery() != null && requestInfo.getUrl().getQuery().length() > 0) { List<String> qsList = Splitter.on('&').trimResults().splitToList(requestInfo.getUrl().getQuery()); if (qsList.size() > 0) { for (String param : qsList) { for (String k : Splitter.on("=").splitToList(param)) { hashInput += "&" + k; } } } } String dedupHashValue = "URL:" + requestInfo.getMethod() + m_hash.hashBytes(helpers.stringToBytes(hashInput)).toString(); if (dubBloomFilter.mightContain(dedupHashValue)) { return true; } dubBloomFilter.put(dedupHashValue); return false; }
private FeatureHash(int hashBucketSize, int hashSeed, String hashFeaturePrefix) { this.hashBucketSize = hashBucketSize; this.hashFeaturePrefix = hashFeaturePrefix; this.murmurHash = Hashing.murmur3_128(hashSeed); }
public String md5RegexHandle(Object value, JSONObject jsonValue){ String regexHandled = regexHandle(value.toString(),jsonValue.getString("regex"),jsonValue.getString("replaceStr")); if(regexHandled.equals("")){ return ""; } String salt = jsonValue.getString("saltParam") == null ? "" : jsonValue.getString("saltParam"); return new DBusHashCode(Hashing.md5().hashString(regexHandled + salt, Charset.forName("UTF-8"))).toString(); }
public int getFilterHash(HttpServletRequest request) { // don't use cache for dynamic filters (e.g. path filter) if (cacheInvalid || isDynamic()) { String result = filter(request); if (result == null) cachedResult = 0; else { cachedResult = Hashing.murmur3_32().hashString(result, Charset.forName("UTF-8")).asInt(); cachedResult = HashUtil.HashCombine(cachedResult, filterSalt); } cacheInvalid = false; } return cachedResult; }
@JsonCreator public Function0(@JsonProperty("seed") int seed, @JsonProperty("fiberNum") int fiberNum) { this.seed = seed; this.fiberNum = fiberNum; this.hasher = Hashing.murmur3_128(seed); }
@Override public Integer calculate(String columnValue) { try { long targetTime = formatter.get().parse( columnValue).getTime(); int targetPartition = (int) ((targetTime - beginDate) / partionTime); int innerIndex = Hashing.consistentHash(targetTime,intGroupPartionSize); return targetPartition * intGroupPartionSize + innerIndex; } catch (ParseException e) { throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please check if the format satisfied.").toString(),e); } }
private void generateBucketMap(){ hash=Hashing.murmur3_32(seed);//计算一致性哈希的对象 for(int i=0;i<count;i++){//构造一致性哈希环,用TreeMap表示 StringBuilder hashName=new StringBuilder("SHARD-").append(i); for(int n=0,shard=virtualBucketTimes*getWeight(i);n<shard;n++){ bucketMap.put(hash.hashUnencodedChars(hashName.append("-NODE-").append(n)).asInt(),i); } } weightMap=null; }
private String hash(User user, String password) { return Hashing.sha512() .newHasher() .putUnencodedChars(user.getSalt()) .putUnencodedChars(password) .hash() .toString(); }
@Test public void testAvTransform() throws Exception { ClassLoader loader = Thread.currentThread().getContextClassLoader(); String oggPath = loader.getResource("com/filestack/sample_music.ogg").getPath(); File oggFile = new File(oggPath); FileLink oggFileLink = client.upload(oggPath, false); HANDLES.add(oggFileLink.getHandle()); AvTransformOptions options = new AvTransformOptions.Builder() .preset("mp3") .build(); AvTransform transform = oggFileLink.avTransform(options); FileLink mp3FileLink; while ((mp3FileLink = transform.getFileLink()) == null) { Thread.sleep(5 * 1000); } HANDLES.add(mp3FileLink.getHandle()); String mp3Path = loader.getResource("com/filestack/sample_music.mp3").getPath(); File mp3File = new File(mp3Path); String correct = Files.asByteSource(mp3File).hash(Hashing.sha256()).toString(); byte[] bytes = mp3FileLink.getContent().bytes(); String output = Hashing.sha256().hashBytes(bytes).toString(); Assert.assertEquals(correct, output); }
/** * Encodes the json policy and signs it using the app secret. * Do not include the app secret in client-side code. */ public Policy build(String appSecret) { Gson gson = new Gson(); HashFunction hashFunction = Hashing.hmacSha256(appSecret.getBytes(Charsets.UTF_8)); String jsonPolicy = gson.toJson(this); String encodedPolicy = BaseEncoding.base64Url().encode(jsonPolicy.getBytes(Charsets.UTF_8)); String signature = hashFunction.hashString(encodedPolicy, Charsets.UTF_8).toString(); return new Policy(encodedPolicy, signature); }
private static byte[] calculateSlowDigest(long batchAreaOffset, long offset, long size, String fileName, byte[] buffer) { byte[] slowDigest; Hasher hasher = Hashing.sha256().newHasher(); hasher.putBytes(buffer, Math.toIntExact(offset - batchAreaOffset), Math.toIntExact(size)); slowDigest = hasher.hash().asBytes(); if (logger.isLoggable(Level.FINER)) { logger.finer("Calculated slow digest[" + slowDigest + "] for file [" + fileName + "]" + offset + ":" + (offset + size)); } return slowDigest; }
public void testHash() throws IOException { File asciiFile = getTestFile("ascii.txt"); File i18nFile = getTestFile("i18n.txt"); String init = "d41d8cd98f00b204e9800998ecf8427e"; assertEquals(init, Hashing.md5().newHasher().hash().toString()); String asciiHash = "e5df5a39f2b8cb71b24e1d8038f93131"; assertEquals(asciiHash, Files.hash(asciiFile, Hashing.md5()).toString()); String i18nHash = "7fa826962ce2079c8334cd4ebf33aea4"; assertEquals(i18nHash, Files.hash(i18nFile, Hashing.md5()).toString()); }
private List<Integer> getAssignedShardingItems(final String jobName, final List<String> taskIdList, final Set<HashCode> assignedTasks) { List<Integer> result = new ArrayList<>(taskIdList.size()); for (String each : taskIdList) { TaskContext.MetaInfo metaInfo = TaskContext.MetaInfo.from(each); if (assignedTasks.add(Hashing.md5().newHasher().putString(jobName, Charsets.UTF_8).putInt(metaInfo.getShardingItems().get(0)).hash()) && !runningService.isTaskRunning(metaInfo)) { result.add(metaInfo.getShardingItems().get(0)); } } return result; }
public String newRandomId() { final UUID uuid = UUID.randomUUID(); return Hashing.sha256().newHasher() .putLong(System.nanoTime()) .putLong(uuid.getMostSignificantBits()) .putLong(uuid.getLeastSignificantBits()) .hash().toString(); }