static BloomFilter<String> readBloomFilterFromfile(String bloomFilterFilePath) throws IOException { Funnel<String> memberFunnel = new Funnel<String>() { public void funnel(String memberId, PrimitiveSink sink) { sink.putString(memberId, Charsets.UTF_8); } }; try { FileInputStream fis = new FileInputStream(new File(bloomFilterFilePath)); return BloomFilter.readFrom(fis, memberFunnel); } catch(Exception e) { e.printStackTrace(); } return null; }
private BloomFilter<Seed> getOrCreate(String segment) { BloomFilter<Seed> seedBloomFilter = bloomFilters.get(segment); if (seedBloomFilter != null) { return seedBloomFilter; } synchronized (segment.intern()) { seedBloomFilter = bloomFilters.get(segment); if (seedBloomFilter != null) { return seedBloomFilter; } long expectedNumber = NumberUtils.toLong(VSCrawlerContext.vsCrawlerConfigFileWatcher.loadedProperties() .getProperty(VSCrawlerConstant.VSCRAWLER_SEED_MANAGER_EXPECTED_SEED_NUMBER), 1000000L); // any way, build a filter instance if not exist seedBloomFilter = BloomFilter.create(new Funnel<Seed>() { @Override public void funnel(Seed from, PrimitiveSink into) { into.putString(seedKeyResolver.resolveSeedKey(from), Charset.defaultCharset()); } }, expectedNumber); bloomFilters.put(segment, seedBloomFilter); } return seedBloomFilter; }
@VisibleForTesting static <T> CuckooFilter<T> create(Funnel<? super T> funnel, long capacity, double fpp, CuckooStrategy cuckooStrategy) { checkNotNull(funnel); checkArgument(capacity > 0, "Expected insertions (%s) must be > 0", capacity); checkArgument(fpp > 0.0D, "False positive probability (%s) must be > 0.0", fpp); checkArgument(fpp < 1.0D, "False positive probability (%s) must be < 1.0", fpp); checkNotNull(cuckooStrategy); int numEntriesPerBucket = optimalEntriesPerBucket(fpp); long numBuckets = optimalNumberOfBuckets(capacity, numEntriesPerBucket); int numBitsPerEntry = optimalBitsPerEntry(fpp, numEntriesPerBucket); try { return new CuckooFilter<T>(new CuckooTable(numBuckets, numEntriesPerBucket, numBitsPerEntry), funnel, cuckooStrategy, fpp); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Could not create CuckooFilter of " + numBuckets + " buckets, " + numEntriesPerBucket + " entries per bucket, " + numBitsPerEntry + " bits per entry", e); } }
@Test public void ensureGeneric() { class SuperClass { } class SubClass extends SuperClass { } CuckooFilter<SuperClass> filter = CuckooFilter.create( new Funnel<SuperClass>() { public void funnel(SuperClass from, PrimitiveSink into) { into.putInt(from.hashCode()); } }, 1000, 0.03D); assertTrue(filter.add(new SuperClass())); assertTrue(filter.add(new SubClass())); }
GrowthTracker( final SerializableFunction<OutputT, KeyT> keyFn, final Coder<KeyT> outputKeyCoder, GrowthState<OutputT, KeyT, TerminationStateT> state, Growth.TerminationCondition<?, TerminationStateT> terminationCondition) { this.coderFunnel = new Funnel<OutputT>() { @Override public void funnel(OutputT from, PrimitiveSink into) { try { // Rather than hashing the output itself, hash the output key. KeyT outputKey = keyFn.apply(from); outputKeyCoder.encode(outputKey, Funnels.asOutputStream(into)); } catch (IOException e) { throw new RuntimeException(e); } } }; this.terminationCondition = terminationCondition; this.state = state; this.isOutputComplete = state.isOutputComplete; this.pollWatermark = state.pollWatermark; this.terminationState = state.terminationState; this.pending = Lists.newLinkedList(state.pending); }
@Autowired public CaptchaService(UserInfoValidator userInfoValidator, LsPushProperties lsPushProperties, JavaMailSender mailSender, TemplateEngine templateEngine, ObjectMapper objectMapper, UserRepository userRepo) { mUserInfoValidator = userInfoValidator; serverName = lsPushProperties.getServerName(); serverUrl = lsPushProperties.getServerUrl(); serverEmail = lsPushProperties.getServerEmail(); mMailSender = mailSender; mTemplateEngine = templateEngine; mObjectMapper = objectMapper; mUserRepo = userRepo; mAuthCodeMap = CacheBuilder.newBuilder() .initialCapacity(100) .maximumSize(500) .expireAfterWrite(30, TimeUnit.MINUTES) .build(); mStringFunnel = (Funnel<String>) (from, into) -> into.putString(from, StandardCharsets.UTF_8); resetBloomFilter(); }
protected AccountSession getSessionWithCheck(CryptoToken cryptoToken, boolean isExpire) { Funnel<AccountSession> sessionFunnel = isExpire ? mExpireSessionFunnel : mRefreshSessionFunnel; AccountSession session; try { byte[] json = Crypto.decrypt(cryptoToken); session = mObjectMapper.readValue(json, AccountSession.class); } catch (Exception e) { logger.warn("decrypt token failure", e); return null; } byte[] sessionData = Hashing.sipHash24().hashObject(session, sessionFunnel).asBytes(); String sessionString = new String(sessionData, StandardCharsets.UTF_8); if (!sessionString.equals(session.getSession())) { logger.warn("session not equal"); return null; } return session; }
public CollisionHandler(int numFilters, int size) { filters = new ArrayList<>(); for(int i = 0;i < numFilters;++i) { BloomFilter<Long> collisionFilter = BloomFilter.create(new Funnel<Long>() { /** * Sends a stream of data from the {@code from} object into the sink {@code into}. There * is no requirement that this data be complete enough to fully reconstitute the object * later. * * @param from * @param into */ @Override public void funnel(Long from, Sink into) { into.putLong(from); } }, size); filters.add(collisionFilter); } }
private String getInternalTransactionHash(final Transaction transaction) { return Hashing.sha1().hashObject(transaction, new Funnel<Transaction>() { private static final long serialVersionUID = 9193015056720554840L; @Override public void funnel(final Transaction from, final PrimitiveSink into) { into.putUnencodedChars(from.getReference()) .putUnencodedChars(from.getSource().getName()) .putUnencodedChars(from.getDestination().getName()) .putFloat(from.getAmount().floatValue()) .putLong(from.getCreationDate()); } }).toString(); }
private BloomFilter<MapTuple> getFilter(Object batch) { if (_filters.containsKey(batch) == false) { Funnel<MapTuple> funnel = new Funnel<MapTuple>() { private static final long serialVersionUID = 3504134639163725164L; @Override public void funnel(MapTuple from, PrimitiveSink into) { if (_uniqueFields == null) { into.putString(from.values().toString(), Charset.defaultCharset()); } else { for(String f : _uniqueFields) { into.putString(from.get(f).toString(), Charset.defaultCharset()); } } } }; logger().info("Creating unique filter with max expected capacity of: " + _expectedSize); _filters.put(batch, BloomFilter.create(funnel, _expectedSize)); } return _filters.get(batch); }
public <T> boolean put(T object, Funnel<? super T> funnel, int numHashFunctions, int[] cells) { // TODO(user): when the murmur's shortcuts are implemented, update this code long hash64 = Hashing.murmur3_128().newHasher().putObject(object, funnel).hash().asLong(); int hash1 = (int) hash64; int hash2 = (int) (hash64 >>> 32); boolean bitsChanged = false; for (int i = 1; i <= numHashFunctions; i++) { int nextHash = hash1 + i * hash2; if (nextHash < 0) { nextHash = ~nextHash; } int pos = nextHash % cells.length; bitsChanged |= (cells[pos] != MAX_VAL); cells[pos] = MAX_VAL; } return bitsChanged; }
public <T> boolean mightContain(T object, Funnel<? super T> funnel, int numHashFunctions, int[] cells) { long hash64 = Hashing.murmur3_128().newHasher().putObject(object, funnel).hash().asLong(); int hash1 = (int) hash64; int hash2 = (int) (hash64 >>> 32); for (int i = 1; i <= numHashFunctions; i++) { int nextHash = hash1 + i * hash2; if (nextHash < 0) { nextHash = ~nextHash; } int pos = nextHash % cells.length; if (cells[pos] == 0) { return false; } } return true; }
private int hash() { // Guava documentation recommends using putUnencodedChars to hash raw character bytes within any encoding // unless cross-language compatibility is needed. See the Hasher.putString documentation for more info. Funnel<TrafficSelector> selectorFunnel = (from, into) -> from.criteria() .forEach(c -> into.putUnencodedChars(c.toString())); HashFunction hashFunction = Hashing.murmur3_32(); HashCode hashCode = hashFunction.newHasher() .putUnencodedChars(deviceId.toString()) .putObject(selector, selectorFunnel) .putInt(priority) .putUnencodedChars(tableId.toString()) .hash(); return hashCode.asInt(); }
@Override public Object getCredentials() { AllowAllUser user = (AllowAllUser) getPrincipal(); if (user != null) { return Hashing.sha256().hashObject(user, new Funnel<AllowAllUser>() { @Override public void funnel(AllowAllUser from, PrimitiveSink into) { Set<String> fromGroups = from.getGroups(); String fromName = from.getUserName(); into.putString(fromName, Charsets.UTF_8); for (String fromGroup : fromGroups) { into.putString(fromGroup, Charsets.UTF_8); } } }); } return null; }
@Override public int hashCode() { final HashFunction hf = Hashing.md5(); return hf.newHasher() .putString(extractedPath) .putObject(archive, new Funnel<JarFile>() { @Override public void funnel(JarFile from, PrimitiveSink into) { into .putString(from.getName()) .putString(Optional.fromNullable(from.getComment()).or("")); } private static final long serialVersionUID = 3109141395123855989L; }).hash().asInt(); }
/** * @param filterPath 原Guava序列化存储的文件路径 * @param funnel 原Guava BloomFilter使用的Funnel * @throws IOException */ public DirectDiskUrlFilter(String filterPath, Funnel<CharSequence> funnel) throws IOException { filterFile = new File(filterPath); raf = new RandomAccessFile(filterFile, "rw"); /* jump strategyOrdinal value */ raf.readByte(); numHashFunctions = UnsignedBytes.toInt(raf.readByte()); dataLength = raf.readInt(); bitsSize = (long) dataLength * 64L; bits = new Bits(); this.funnel = funnel; }
private int hash() { Funnel<TrafficSelector> selectorFunnel = (from, into) -> from.criteria() .stream() .forEach(c -> into.putString(c.toString(), Charsets.UTF_8)); HashFunction hashFunction = Hashing.murmur3_32(); HashCode hashCode = hashFunction.newHasher() .putString(deviceId.toString(), Charsets.UTF_8) .putObject(selector, selectorFunnel) .putInt(priority) .putInt(tableId) .hash(); return hashCode.asInt(); }
SerializableSaltedHasher(long seedNSalt, long addlSipSeed, Funnel<? super T> funnel, Algorithm alg) { checkNotNull(alg); checkNotNull(funnel); this.alg = alg; this.funnel = funnel; this.seedNSalt = seedNSalt; this.addlSipSeed = addlSipSeed; hasher = configureHash(alg, seedNSalt, addlSipSeed); }
static <T> SerializableSaltedHasher<T> create(Algorithm alg, Funnel<? super T> funnel) { checkNotNull(alg); checkNotNull(funnel); SecureRandom randomer = new SecureRandom(); long seedNSalt = randomer.nextLong(); long addlSipSeed = randomer.nextLong(); return new SerializableSaltedHasher<>(seedNSalt, addlSipSeed, funnel, alg); }
public <T> boolean add(T object, Funnel<? super T> funnel, CuckooTable table) { final long hash64 = hash(object, funnel).asLong(); final int hash1 = hash1(hash64); final int hash2 = hash2(hash64); final int fingerprint = fingerprint(hash2, table.numBitsPerEntry); final long index = index(hash1, table.numBuckets); return putEntry(fingerprint, table, index) || putEntry(fingerprint, table, altIndex(index, fingerprint, table.numBuckets)); }
public <T> boolean remove(T object, Funnel<? super T> funnel, CuckooTable table) { final long hash64 = hash(object, funnel).asLong(); final int hash1 = hash1(hash64); final int hash2 = hash2(hash64); final int fingerprint = fingerprint(hash2, table.numBitsPerEntry); final long index1 = index(hash1, table.numBuckets); final long index2 = altIndex(index1, fingerprint, table.numBuckets); return table.swapAnyEntry(CuckooTable.EMPTY_ENTRY, fingerprint, index1) || table.swapAnyEntry(CuckooTable.EMPTY_ENTRY, fingerprint, index2); }
public <T> boolean contains(T object, Funnel<? super T> funnel, CuckooTable table) { final long hash64 = hash(object, funnel).asLong(); final int hash1 = hash1(hash64); final int hash2 = hash2(hash64); final int fingerprint = fingerprint(hash2, table.numBitsPerEntry); final long index1 = index(hash1, table.numBuckets); final long index2 = altIndex(index1, fingerprint, table.numBuckets); return table.hasEntry(fingerprint, index1) || table.hasEntry(fingerprint, index2); }
/** * Creates a CuckooFilter. */ private CuckooFilter( CuckooTable table, Funnel<? super E> funnel, CuckooStrategy cuckooStrategy, double fpp) { this.fpp = fpp; this.table = checkNotNull(table); this.funnel = checkNotNull(funnel); this.cuckooStrategy = checkNotNull(cuckooStrategy); }
private BloomFilter(com.google.common.hash.BloomFilter<E> delegate, Funnel<E> funnel, long capacity, double fpp, long size) { super(); checkNotNull(delegate); checkNotNull(funnel); checkArgument(capacity >= 0, "capacity must be positive"); checkArgument(fpp >= 0.0 && fpp < 1.0, "fpp must be positive 0.0 <= fpp < 1.0"); checkArgument(size >= 0, "size must be positive"); this.delegate = delegate; this.funnel = funnel; this.capacity = capacity; this.fpp = fpp; this.size = size; }
@Test public void customSerialization() throws Exception { Funnel<byte[]> funnel = Funnels.byteArrayFunnel(); CuckooFilter<byte[]> cf = CuckooFilter.create(funnel, 100); for (int i = 0; i < 100; i++) { cf.add(Ints.toByteArray(i)); } ByteArrayOutputStream out = new ByteArrayOutputStream(); cf.writeTo(out); assertEquals(cf, CuckooFilter.readFrom(new ByteArrayInputStream(out.toByteArray()), funnel)); }
@Autowired public AuthService(ObjectMapper objectMapper, CaptchaService captchaService, UserInfoValidator userInfoValidator, UserRepository userRepo) { mObjectMapper = objectMapper; mCaptchaService = captchaService; mUserInfoValidator = userInfoValidator; mUserRepo = userRepo; mExpireSessionFunnel = (Funnel<AccountSession>) (from, into) -> into.putString(from.getUserId(), StandardCharsets.UTF_8) .putLong(from.getExpireTime()); mRefreshSessionFunnel = (Funnel<AccountSession>) (from, into) -> into.putString(from.getUserId(), StandardCharsets.UTF_8) .putLong(from.getRefreshTime()); }
protected CryptoToken newSessionToken(AccountSession session, boolean isExpire) { Funnel<AccountSession> sessionFunnel = isExpire ? mExpireSessionFunnel : mRefreshSessionFunnel; byte[] data = Hashing.sipHash24().hashObject(session, sessionFunnel).asBytes(); String sessionString = new String(data, Charset.forName("UTF-8")); session.setSession(sessionString); CryptoToken token; try { byte[] sessionData = mObjectMapper.writeValueAsBytes(session); token = Crypto.encrypt(sessionData); } catch (Exception e) { logger.warn("encrypt session failure", e); return null; } return token; }
Funnel<K> funnel() { return new Funnel<K>() { private static final long serialVersionUID = 1L; @Override public void funnel(K from, PrimitiveSink into) { try (ObjectOutputStream ser = new ObjectOutputStream(new SinkOutputStream(into))) { ser.writeObject(from); ser.flush(); } catch (IOException err) { throw new RuntimeException("Cannot hash as Serializable", err); } } }; }
public ConsistentHash(HashFunction hashFunction, Funnel<K> keyFunnel, Funnel<N> nodeFunnel, Collection<N> nodes) { this.hashFunction = hashFunction; this.nodeFunnel = nodeFunnel; this.keyFunnel = keyFunnel; for (N node : nodes) { add(node); } }
/** * Creates a new RendezvousHash with a starting set of nodes provided by init. The funnels will be used when generating the hash that combines the nodes and * keys. The hasher specifies the hashing algorithm to use. */ public RendezvousHash(HashFunction hasher, Funnel<K> keyFunnel, Funnel<N> nodeFunnel, Collection<N> init) { if (hasher == null) throw new NullPointerException("hasher"); if (keyFunnel == null) throw new NullPointerException("keyFunnel"); if (nodeFunnel == null) throw new NullPointerException("nodeFunnel"); if (init == null) throw new NullPointerException("init"); this.hasher = hasher; this.keyFunnel = keyFunnel; this.nodeFunnel = nodeFunnel; this.ordered = new ConcurrentSkipListSet<N>(init); }
public StableBloomFilter(int numCells, int numHashFunctions, int numDecrementCells, Funnel<T> funnel) { this.numDecrementCells = numDecrementCells; this.cells = new int[numCells]; this.numHashFunctions = numHashFunctions; this.funnel = funnel; strategy = new Murmur128_Mitz_32_Strategy(); }
