@Test public void transactionResponseWithError() { Transaction t = jedis.multi(); t.set("foo", "bar"); Response<Set<String>> error = t.smembers("foo"); Response<String> r = t.get("foo"); List<Object> l = t.exec(); assertEquals(JedisDataException.class, l.get(1).getClass()); try { error.get(); fail("We expect exception here!"); } catch (JedisDataException e) { // that is fine we should be here } assertEquals(r.get(), "bar"); }
@Test public void execGetResponse() { Transaction t = jedis.multi(); t.set("foo", "bar"); t.smembers("foo"); t.get("foo"); List<Response<?>> lr = t.execGetResponse(); try { lr.get(1).get(); fail("We expect exception here!"); } catch (JedisDataException e) { // that is fine we should be here } assertEquals("bar", lr.get(2).get()); }
@Test public void testExec() throws Exception { int rnd = ThreadLocalRandom.current().nextInt(0, presetElements.size()); String key = presetElementKeys.get(rnd); String value = String.valueOf(presetElements.get(key)); Transaction transaction = jedis.multi(); Snapshot snapshot = commandTracker.snapshot(); Snapshot discardSnapshot = discardTracker.snapshot(); Snapshot txsnapshot = execTracker.snapshot(); txsnapshot.increment(); Response<Long> added = transaction.sadd(key, value); transaction.exec(); assertEquals(1, (long) added.get()); txsnapshot.validate(); snapshot.validate(); discardSnapshot.validate(); }
@Test public void testDiscard() throws Exception { int rnd = ThreadLocalRandom.current().nextInt(0, presetElements.size()); String key = presetElementKeys.get(rnd); String value = String.valueOf(presetElements.get(key)); Transaction transaction = jedis.multi(); Snapshot snapshot = commandTracker.snapshot(); Snapshot discardSnapshot = discardTracker.snapshot(); Snapshot txsnapshot = execTracker.snapshot(); discardSnapshot.increment(); Response<Long> added = transaction.sadd(key, value); transaction.discard(); assertNull(jedis.get(key)); txsnapshot.validate(); snapshot.validate(); discardSnapshot.validate(); }
@Test public void testCloseDiscards() throws Exception { int rnd = ThreadLocalRandom.current().nextInt(0, presetElements.size()); String key = presetElementKeys.get(rnd); String value = String.valueOf(presetElements.get(key)); Transaction transaction = jedis.multi(); Snapshot snapshot = commandTracker.snapshot(); Snapshot discardSnapshot = discardTracker.snapshot(); Snapshot txsnapshot = execTracker.snapshot(); discardSnapshot.increment(); Response<Long> added = transaction.sadd(key, value); transaction.close(); assertNull(jedis.get(key)); txsnapshot.validate(); snapshot.validate(); discardSnapshot.validate(); }
@Test public void testSync() throws Exception { int rnd = ThreadLocalRandom.current().nextInt(0, presetElements.size()); String key = presetElementKeys.get(rnd); String value = String.valueOf(presetElements.get(key)); Pipeline pipeline = jedis.pipelined(); Snapshot snapshot = commandTracker.snapshot(); Snapshot discardSnapshot = discardTracker.snapshot(); Snapshot txsnapshot = execTracker.snapshot(); Snapshot pipelineSnapshot = pipelineTracker.snapshot(); pipelineSnapshot.increment(); Response<Long> added = pipeline.sadd(key, value); pipeline.sync(); assertEquals(1, (long) added.get()); pipelineSnapshot.validate(); txsnapshot.validate(); snapshot.validate(); discardSnapshot.validate(); }
@Test public void testCloseSyncs() throws Exception { int rnd = ThreadLocalRandom.current().nextInt(0, presetElements.size()); String key = presetElementKeys.get(rnd); String value = String.valueOf(presetElements.get(key)); Pipeline pipeline = jedis.pipelined(); Snapshot snapshot = commandTracker.snapshot(); Snapshot discardSnapshot = discardTracker.snapshot(); Snapshot txsnapshot = execTracker.snapshot(); Snapshot pipelineSnapshot = pipelineTracker.snapshot(); pipelineSnapshot.increment(); Response<Long> added = pipeline.sadd(key, value); pipeline.close(); assertEquals(1, (long) added.get()); pipelineSnapshot.validate(); txsnapshot.validate(); snapshot.validate(); discardSnapshot.validate(); }
@Override public boolean containsKeys(Collection<K> keys) { if (keys == null || keys.size() == 0) { return false; } // 使用 Redis 提供的管道进行批处理,提高效率 Pipeline pipeline = jedisProxy.pipelined(); Set<Response<Boolean>> responses = new HashSet<Response<Boolean>>(keys.size()); for (K key : keys) { if (localMapCache != null && localMapCache.containsKey(key)) { continue; } responses.add(pipeline.hexists(SerializeUtil.serialize(getName()), SerializeUtil.serialize(key))); } if (responses.size() < 1) { return true; } pipeline.sync(); for (Response<Boolean> response : responses) { if (!response.get()) { return false; } } return true; }
@Override public Map<K, V> gets(Collection<K> keys) { Map<K, V> result = new HashMap<>(keys.size()); Pipeline pipeline = jedisProxy.pipelined(); byte[] keyBytes = SerializeUtil.serialize(getName()); Map<K, Response<byte[]>> responseMap = new HashMap<>(keys.size()); for (K key : keys) { if (localMapCache != null && localMapCache.containsKey(key)) { result.put(key, localMapCache.get(key)); continue; } responseMap.put(key, pipeline.hget(keyBytes, SerializeUtil.serialize(key))); } if (responseMap.size() < 1) { return result; } pipeline.sync(); for (Map.Entry<K, Response<byte[]>> entry : responseMap.entrySet()) { result.put(entry.getKey(), (V) SerializeUtil.deserialize(entry.getValue().get())); } return result; }
@Test public void multiWithMassiveRequests() { Pipeline p = jedis.pipelined(); p.multi(); List<Response<?>> responseList = new ArrayList<Response<?>>(); for (int i = 0; i < 100000; i++) { // any operation should be ok, but shouldn't forget about timeout responseList.add(p.setbit("test", 1, true)); } Response<List<Object>> exec = p.exec(); p.sync(); // we don't need to check return value // if below codes run without throwing Exception, we're ok exec.get(); for (Response<?> resp : responseList) { resp.get(); } }
@Test public void testCloseable() throws IOException { // we need to test with fresh instance of Jedis Jedis jedis2 = new Jedis(hnp.getHost(), hnp.getPort(), 500); jedis2.auth("foobared"); Pipeline pipeline = jedis2.pipelined(); Response<String> retFuture1 = pipeline.set("a", "1"); Response<String> retFuture2 = pipeline.set("b", "2"); pipeline.close(); // it shouldn't meet any exception retFuture1.get(); retFuture2.get(); }
@Override public Map<String, UserPermission> getAllById() { Table<String, ResourceType, Response<Map<String, String>>> responseTable = getAllFromRedis(); if (responseTable == null) { return new HashMap<>(0); } Map<String, UserPermission> allById = new HashMap<>(responseTable.rowKeySet().size()); RawUserPermission rawUnrestricted = new RawUserPermission(responseTable.row(UNRESTRICTED)); UserPermission unrestrictedUser = getUserPermission(UNRESTRICTED, rawUnrestricted); for (String userId : responseTable.rowKeySet()) { RawUserPermission rawUser = new RawUserPermission(responseTable.row(userId)); UserPermission permission = getUserPermission(userId, rawUser); allById.put(userId, permission.merge(unrestrictedUser)); } return allById; }
private Table<String, ResourceType, Response<Map<String, String>>> getAllFromRedis(Set<String> userIds) { if (userIds.size() == 0) { return HashBasedTable.create(); } try (Jedis jedis = jedisSource.getJedis()) { Table<String, ResourceType, Response<Map<String, String>>> responseTable = ArrayTable.create(userIds, new ArrayIterator<>(ResourceType.values())); Pipeline p = jedis.pipelined(); for (String userId : userIds) { for (ResourceType r : ResourceType.values()) { responseTable.put(userId, r, p.hgetAll(userKey(userId, r))); } } p.sync(); return responseTable; } catch (Exception e) { log.error("Storage exception reading all entries.", e); } return null; }
@Test public void testEvalshaKeyAndArg() { String key = "test"; String arg = "3"; String script = "redis.call('INCRBY', KEYS[1], ARGV[1]) redis.call('INCRBY', KEYS[1], ARGV[1])"; String sha1 = jedis.scriptLoad(script); assertTrue(jedis.scriptExists(sha1)); Pipeline p = jedis.pipelined(); p.set(key, "0"); Response<Object> result0 = p.evalsha(sha1, Arrays.asList(key), Arrays.asList(arg)); p.incr(key); Response<Object> result1 = p.evalsha(sha1, Arrays.asList(key), Arrays.asList(arg)); Response<String> result2 = p.get(key); p.sync(); assertNull(result0.get()); assertNull(result1.get()); assertEquals("13", result2.get()); }
@Override public Map<AggregateId, List<Event>> getEvents(EventTypeMatcher matcher, String aggregateType, Collection<String> aggregateIds, TimeRange timeRange) { Function<String, Optional<Event>> deserialiser = eventJson -> EventJson .fromJsonString(eventJson, objectMapper) .toEvent(matcher, objectMapper); Pipeline pipeline = jedis.pipelined(); final Map<AggregateId, Response<Set<String>>> responses = aggregateIds.stream() .map(id -> AggregateId.of(aggregateType, id)) .collect(toMap( Function.identity(), id -> pipeline.smembers(id.toString()))); pipeline.sync(); return responses.entrySet().stream() .collect(toMap( Entry::getKey, e -> deserialiseAll(timeRange, deserialiser, e.getValue().get()))); }
@Test public void testEvalKeyAndArgWithBinary() { // binary byte[] bKey = SafeEncoder.encode("test"); byte[] bArg = SafeEncoder.encode("3"); byte[] bScript = SafeEncoder .encode("redis.call('INCRBY', KEYS[1], ARGV[1]) redis.call('INCRBY', KEYS[1], ARGV[1])"); Pipeline bP = jedis.pipelined(); bP.set(bKey, SafeEncoder.encode("0")); Response<Object> bResult0 = bP.eval(bScript, Arrays.asList(bKey), Arrays.asList(bArg)); bP.incr(bKey); Response<Object> bResult1 = bP.eval(bScript, Arrays.asList(bKey), Arrays.asList(bArg)); Response<byte[]> bResult2 = bP.get(bKey); bP.sync(); assertNull(bResult0.get()); assertNull(bResult1.get()); assertArrayEquals(SafeEncoder.encode("13"), bResult2.get()); }
public KvItem get(String key) { Holder<Response<String>> js = new Holder<Response<String>>(); Holder<Response<Long>> version = new Holder<Response<Long>>(); redis.pipeline(pipe -> { js.set(pipe.get(keyForItem(key))); version.set(pipe.incrBy(keyForVersion(key), 0)); }); KvItem item = new KvItem(); item.setKey(key); item.setVersion(version.value().get()); if (js.value().get() != null) { item.setValue(new JSONObject(js.value().get())); } else { item.setValue(new JSONObject()); } return item; }
public <T> int drainTo(String key, Collection<T> c, int maxElements, Class<T> classNm) { if (StringUtils.isEmpty(key)) { key = setKey; } String listKey = service.sPop(key, String.class); if (StringUtils.isEmpty(listKey)) { return 0; } INaviMultiRedis multi = service.multi(key); try { Transaction tran = multi.getTransaction(); Response<List<byte[]>> response = tran.lrange(listKey.getBytes(), 0, maxElements - 1); tran.ltrim(listKey, maxElements, -1); tran.exec(); List<byte[]> results = response.get(); for (byte[] result : results) { c.add(jsonSerializer.getObjectFromBytes(result, classNm)); } if (results.size() < maxElements) { removeKey(key, listKey); } return c.size(); } finally { multi.returnObject(); } }
@Override public Map<String, Response<Long>> setJedisMultiCommand(Transaction jedisMulti) { Response<Long> applicationDailyRateLimit = jedisMulti.hincrBy(Const.REDIS_APP_RATELIMIT_DAILY, this.requestInfo.getAppId(), -1); Response<Long> applicationMinutelyRateLimit = jedisMulti.hincrBy(Const.REDIS_APP_RATELIMIT_MINUTELY, this.requestInfo.getAppId(), -1); Response<Long> applicationDailyRateLimitTTL = jedisMulti.ttl(Const.REDIS_APP_RATELIMIT_DAILY); Response<Long> applicationMinutelyRateLimitTTL = jedisMulti.ttl(Const.REDIS_APP_RATELIMIT_MINUTELY); Map<String, Response<Long>> appRatelimit = Maps.newHashMap(); appRatelimit.put(Const.REDIS_APP_RATELIMIT_DAILY, applicationDailyRateLimit); appRatelimit.put(Const.REDIS_APP_RATELIMIT_MINUTELY, applicationMinutelyRateLimit); appRatelimit.put(Const.REDIS_APP_RATELIMIT_DAILY_TTL, applicationDailyRateLimitTTL); appRatelimit.put(Const.REDIS_APP_RATELIMIT_MINUTELY_TTL, applicationMinutelyRateLimitTTL); return appRatelimit; }
public <T> int drainTo(String key, Collection<T> c, int maxElements, Class<T> classNm) { INaviMultiRedis multi = service.multi(key); try { Transaction tran = multi.getTransaction(); Response<List<byte[]>> response = tran.lrange(key.getBytes(), 0, maxElements - 1); tran.ltrim(key, maxElements, -1); tran.exec(); List<byte[]> results = response.get(); AlibabaJsonSerializer jsonSerializer = new AlibabaJsonSerializer(); for (byte[] result : results) { c.add(jsonSerializer.getObjectFromBytes(result, classNm)); } return c.size(); } finally { multi.returnObject(); } }
@Test public void testEvalKeyAndArg() { String key = "test"; String arg = "3"; String script = "redis.call('INCRBY', KEYS[1], ARGV[1]) redis.call('INCRBY', KEYS[1], ARGV[1])"; Pipeline p = jedis.pipelined(); p.set(key, "0"); Response<Object> result0 = p.eval(script, Arrays.asList(key), Arrays.asList(arg)); p.incr(key); Response<Object> result1 = p.eval(script, Arrays.asList(key), Arrays.asList(arg)); Response<String> result2 = p.get(key); p.sync(); assertNull(result0.get()); assertNull(result1.get()); assertEquals("13", result2.get()); }
@Override public List<byte[]> loadFileOnce(String fileDataKey, String fileName, long blockSize) { Jedis jedis = jedisPool.getResource(); Pipeline pipelined = jedis.pipelined(); List<byte[]> res = new ArrayList<>(); List<Response<byte[]>> temps = new ArrayList<>(); int temp = 0; while (temp < blockSize) { Response<byte[]> data = pipelined.hget(fileDataKey.getBytes(), getBlockName(fileName, temp)); temps.add(data); if (temp % Constants.SYNC_COUNT == 0) { pipelined.sync(); res.addAll(temps.stream().map(response -> uncompressFilter(response.get())).collect(Collectors.toList())); pipelined = jedis.pipelined(); temps.clear(); } temp++; } try { pipelined.sync(); } catch (JedisConnectionException e) { log.error("pipelined = {}, blockSize = {}!", pipelined.toString(), blockSize); log.error("", e); } finally { jedis.close(); } res.addAll(temps.stream().map(response -> uncompressFilter(response.get())).collect(Collectors.toList())); temps.clear(); return res; }
@Override public List<byte[]> loadFileOnce(String fileDataKey, String fileName, long blockSize) { ShardedJedis shardedJedis = getShardedJedis(); ShardedJedisPipeline pipelined = shardedJedis.pipelined(); List<byte[]> res = new ArrayList<>(); List<Response<byte[]>> temps = new ArrayList<>(); int temp = 0; //如果不分批次sync容易read time out和Java heap space while (temp < blockSize) { Response<byte[]> data = pipelined.hget(fileDataKey.getBytes(), getBlockName(fileName, temp)); temps.add(data); if (temp % Constants.SYNC_COUNT == 0) { pipelined.sync(); res.addAll(temps.stream().map(response -> uncompressFilter(response.get())).collect(Collectors.toList())); temps.clear(); pipelined = shardedJedis.pipelined(); } temp++; } try { pipelined.sync(); } catch (JedisConnectionException e) { log.error("pipelined = {}, blockSize = {}!", pipelined.toString(), blockSize); log.error("", e); } finally { shardedJedis.close(); } res.addAll(temps.stream().map(response -> uncompressFilter(response.get())).collect(Collectors.toList())); temps.clear(); return res; }
@Override public List<byte[]> loadFileOnce(String fileDataKey, String fileName, long blockSize) { Jedis jedis = openJedis(); Pipeline pipelined = jedis.pipelined(); List<byte[]> res = new ArrayList<>(); List<Response<byte[]>> temps = new ArrayList<>(); int temp = 0; while (temp < blockSize) { Response<byte[]> data = pipelined.hget(fileDataKey.getBytes(), getBlockName(fileName, temp)); temps.add(data); if (temp % Constants.SYNC_COUNT == 0) { pipelined.sync(); res.addAll(temps.stream().map(response -> uncompressFilter(response.get())).collect(Collectors.toList())); temps.clear(); pipelined = jedis.pipelined(); } temp++; } try { pipelined.sync(); } catch (JedisConnectionException e) { log.error("pipelined = {}, blockSize = {}!", pipelined.toString(), blockSize); log.error("", e); } finally { jedis.close(); } res.addAll(temps.stream().map(response -> uncompressFilter(response.get())).collect(Collectors.toList())); temps.clear(); return res; }
@Test(expected = JedisDataException.class) public void transactionResponseWithinPipeline() { jedis.set("string", "foo"); Transaction t = jedis.multi(); Response<String> string = t.get("string"); string.get(); t.exec(); }
@Test public void hgetAllPipeline() { Map<byte[], byte[]> bh = new HashMap<byte[], byte[]>(); bh.put(bbar, bcar); bh.put(bcar, bbar); jedis.hmset(bfoo, bh); Pipeline pipeline = jedis.pipelined(); Response<Map<byte[], byte[]>> bhashResponse = pipeline.hgetAll(bfoo); pipeline.sync(); Map<byte[], byte[]> bhash = bhashResponse.get(); assertEquals(2, bhash.size()); assertArrayEquals(bcar, bhash.get(bbar)); assertArrayEquals(bbar, bhash.get(bcar)); }
@Test(expected = JedisDataException.class) public void pipelineResponseWithinPipeline() { jedis.set("string", "foo"); ShardedJedisPipeline p = jedis.pipelined(); Response<String> string = p.get("string"); string.get(); p.sync(); }
@Test public void canRetrieveUnsetKey() { ShardedJedisPipeline p = jedis.pipelined(); Response<String> shouldNotExist = p.get(UUID.randomUUID().toString()); p.sync(); assertNull(shouldNotExist.get()); }
public boolean append(final String key, final Object val) { return (Boolean)execute(new RedisCallback<Jedis>() { public Object doInRedis(Jedis redis) { if (transaction != null) { Response<?> response = transaction.append(key, val.toString()); return QUEUED.equals(response.get()); } if (pipeline != null) { pipeline.append(key, val.toString()); return false; } return redis.append(key, val.toString()) > 0; } }); }
public boolean sismember(final String redisKey, final Object o) { return (Boolean)execute(new RedisCallback<Jedis>() { public Object doInRedis(Jedis redis) { if (transaction != null) { Response<?> response = transaction.sismember(redisKey, o.toString()); return QUEUED.equals(response.get()); } if (pipeline != null) { pipeline.sismember(redisKey, o.toString()); return false; } return redis.sismember(redisKey, o.toString()); } }); }
public boolean sadd(final String redisKey, final Object o) { return (Boolean)execute(new RedisCallback<Jedis>() { public Object doInRedis(Jedis redis) { if (transaction != null) { Response<?> response = transaction.sadd(redisKey, o.toString()); return QUEUED.equals(response.get()); } if (pipeline != null) { pipeline.sadd(redisKey, o.toString()); return false; } return redis.sadd(redisKey, o.toString()) > 0; } }); }
public boolean srem(final String redisKey, final Object o) { return (Boolean)execute(new RedisCallback<Jedis>() { public Object doInRedis(Jedis redis) { if (transaction != null) { Response<?> response = transaction.append(redisKey, o.toString()); return QUEUED.equals(response.get()); } if (pipeline != null) { pipeline.srem(redisKey, o.toString()); return false; } return redis.srem(redisKey, o.toString()) > 0; } }); }
public static void main(String[] args) { jedis.auth("gaoguangjin"); jedis.select(2); // Pipeline p = jedis.pipelined(); // for(int i=0;i<3000;i++){ // p.set("a"+i,i+""); // } // p.sync(); long beginTime=System.currentTimeMillis(); // Pipeline p = jedis.pipelined(); // Response<Map<String, String>> dd = p.hgetAll("tieba_content_image_黄河科技学院"); // p.sync(); // Map<String, String> map=dd.get(); Set<String> key = jedis.hkeys("tieba_content_image_黄河科技学院"); System.out.println("耗时:"+(System.currentTimeMillis()-beginTime)+"s"); Pipeline p = jedis.pipelined(); List<Response<String>> lis=new ArrayList<>(); for (String s : key) { beginTime=System.currentTimeMillis(); lis.add( p.hget("tieba_content_image_黄河科技学院", s)); System.out.println("耗时:"+(System.currentTimeMillis()-beginTime)+"s"); } p.sync(); // List<Response<String>> lis=new ArrayList<>(); // Pipeline p = jedis.pipelined(); // for(int i=0;i<3000;i++){ // lis.add( p.get("a" + i)); // } // p.sync(); System.out.println(lis.size()+"耗时:"+(System.currentTimeMillis()-beginTime)+"s"); }
@Override public Optional<UserPermission> get(@NonNull String id) { try (Jedis jedis = jedisSource.getJedis()) { RawUserPermission userResponseMap = new RawUserPermission(); RawUserPermission unrestrictedResponseMap = new RawUserPermission(); Pipeline p = jedis.pipelined(); Response<Boolean> isUserInRepo = p.sismember(allUsersKey(), id); for (ResourceType r : ResourceType.values()) { Response<Map<String, String>> resourceMap = p.hgetAll(userKey(id, r)); userResponseMap.put(r, resourceMap); Response<Map<String, String>> unrestrictedMap = p.hgetAll(unrestrictedUserKey(r)); unrestrictedResponseMap.put(r, unrestrictedMap); } p.sync(); if (!isUserInRepo.get()) { return Optional.empty(); } UserPermission unrestrictedUser = getUserPermission(UNRESTRICTED, unrestrictedResponseMap); return Optional.of(getUserPermission(id, userResponseMap).merge(unrestrictedUser)); } catch (Exception e) { log.error("Storage exception reading " + id + " entry.", e); } return Optional.empty(); }
@Test public void testCloseableWithMulti() throws IOException { // we need to test with fresh instance of Jedis Jedis jedis2 = new Jedis(hnp.getHost(), hnp.getPort(), 500); jedis2.auth("foobared"); Pipeline pipeline = jedis2.pipelined(); Response<String> retFuture1 = pipeline.set("a", "1"); Response<String> retFuture2 = pipeline.set("b", "2"); pipeline.multi(); pipeline.set("a", "a"); pipeline.set("b", "b"); pipeline.close(); try { pipeline.exec(); fail("close should discard transaction"); } catch (JedisDataException e) { assertTrue(e.getMessage().contains("EXEC without MULTI")); // pass } // it shouldn't meet any exception retFuture1.get(); retFuture2.get(); }
private Table<String, ResourceType, Response<Map<String, String>>> getAllFromRedis() { Set<String> allUserIds; try (Jedis jedis = jedisSource.getJedis()) { allUserIds = jedis.smembers(allUsersKey()); } catch (Exception e) { log.error("Storage exception reading all entries.", e); return null; } return getAllFromRedis(allUserIds); }