redis.clients.jedis.ShardedJedisPipeline Java Examples

/**
 * Use transactions to add index file and then delete the old one
 *
 * @param fileLengthKey the key using for hash file length
 * @param fileDataKey   the key using for hash file data
 * @param oldField      the old hash field
 * @param newField      the new hash field
 * @param values        the data values of the old hash field
 * @param fileLength    the data length of the old hash field
 */
@Override
public void rename(String fileLengthKey, String fileDataKey, String oldField, String newField, List<byte[]> values, long
        fileLength) {
    ShardedJedis shardedJedis = getShardedJedis();
    ShardedJedisPipeline pipelined = shardedJedis.pipelined();
    //add new file length
    pipelined.hset(fileLengthKey.getBytes(), newField.getBytes(), Longs.toByteArray(fileLength));
    //add new file content
    Long blockSize = getBlockSize(fileLength);
    for (int i = 0; i < blockSize; i++) {
        pipelined.hset(fileDataKey.getBytes(), getBlockName(newField, i), compressFilter(values.get(i)));
    }
    pipelined.sync();
    shardedJedis.close();
    values.clear();
    deleteFile(fileLengthKey, fileDataKey, oldField, blockSize);
}
 

/**
 * 分布式连接池异步调用
 * 0.452 seconds
 * 0.43 seconds
 */
@Test
@Ignore
public void testShardPipelinedPool() {
    List<JedisShardInfo> shards = Arrays.asList(
            new JedisShardInfo("localhost",6379),
            new JedisShardInfo("localhost",6379));

    ShardedJedisPool pool = new ShardedJedisPool(new JedisPoolConfig(), shards);
    ShardedJedis one = pool.getResource();
    ShardedJedisPipeline pipeline = one.pipelined();
    
    long start = System.currentTimeMillis();
    for (int i = 0; i < COUNTER; i++) {
        pipeline.set("sppn" + i, "n" + i);
    }
    List<Object> results = pipeline.syncAndReturnAll();
    long end = System.currentTimeMillis();
    pool.returnResource(one);
    logger.info("Pipelined@Pool SET: " + ((end - start)/1000.0) + " seconds");
    pool.destroy();
}
 

/**
 * 分布式直连异步调用
 * 耗时：
 * 0.866 seconds
 * 0.892 seconds
 */
@Test
@Ignore
public void testShardpipelined() {
    List<JedisShardInfo> shards = Arrays.asList(
            new JedisShardInfo("localhost",6379),
            new JedisShardInfo("localhost",6379));

    ShardedJedis sharding = new ShardedJedis(shards);

    ShardedJedisPipeline pipeline = sharding.pipelined();
    long start = System.currentTimeMillis();
    for (int i = 0; i < 100000; i++) {
        pipeline.set("sp" + i, "p" + i);
    }
    List<Object> results = pipeline.syncAndReturnAll();
    long end = System.currentTimeMillis();
    System.out.println("Pipelined@Sharing SET: " + ((end - start)/1000.0) + " seconds");

    sharding.disconnect();
    sharding.close();
}
 

@Override
public Long del(final String... keys) {
	return exec0(new ShardedJedisCallback<Long>() {
		@Override
		public Long doCallback(ShardedJedis jedis) {
			pipelined(new PiplineCallbackAdapter() {
				@Override
				public void doCallback(ShardedJedisPipeline pipeline) {
					for (String key : keys) {
						pipeline.del(key);
					}
				}
			});
			return new Long(keys.length);
		}
	});
}
 

@Override
public String sets(final String... keyvalues) {
	return exec0(new ShardedJedisCallback<String>() {
		@Override
		public String doCallback(ShardedJedis jedis) {
			Object[] args = Arrays.asList(keyvalues).toArray();
			final Map<String, Object> map = MapUtil.gmap(args);
			pipelined(new PiplineCallbackAdapter() {
				@Override
				public void doCallback(ShardedJedisPipeline pipeline) {
					for (Entry<String, Object> entry : map.entrySet()) {
						pipeline.set(entry.getKey(), entry.getValue().toString());
					}
				}
			});
			return new String();
		}
	});
}
 

@Override
public Map<String, Long> scard(final String... keys) {
    Assert.notEmpty(keys);
    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        final ShardedJedisPipeline pipeline = jedis.pipelined();
        final Map<String, Response<Long>> responses = Maps.newHashMap();
        for (String key : keys) {
            responses.put(key, pipeline.scard(key));
        }

        pipeline.sync();
        final Map<String, Long> values = Maps.newHashMap();
        responses.forEach((key, response) -> values.put(key, response.get()));
        return values;
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

@Override
public List<String> lrangeltrim(final String key, final int count) {
    Assert.hasText(key);
    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        final ShardedJedisPipeline pipeline = jedis.pipelined();
        final Response<List<String>> values = pipeline.lrange(key, 0, count);
        pipeline.ltrim(key, count + 1, -1);
        pipeline.sync();
        return values.get();
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

@Override
public Map<String, Boolean> hsetByNX(final String key, final Map<String, Object> map) {
    Assert.hasText(key);
    Assert.notEmpty(map);

    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        final ShardedJedisPipeline pipeline = jedis.pipelined();
        final Map<String, Response<Long>> responses = Maps.newHashMap();
        for (Entry<String, Object> entry : map.entrySet()) {
            responses.put(entry.getKey(), pipeline.hsetnx(key, entry.getKey(), toJSONString(entry.getValue())));
        }

        pipeline.sync();
        final Map<String, Boolean> values = Maps.newHashMap();
        responses.forEach((field, response) -> values.put(field, isSuccess(response.get())));
        return values;
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

@Override
public Map<String, Boolean> setByNX(final Map<String, Object> map) {
    Assert.notEmpty(map);

    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        final ShardedJedisPipeline pipelined = jedis.pipelined();
        final Map<String, Response<Long>> responses = Maps.newHashMap();
        for (Entry<String, Object> entry : map.entrySet()) {
            responses.put(entry.getKey(), pipelined.setnx(entry.getKey(), toJSONString(entry.getValue())));
        }

        pipelined.sync();
        final Map<String, Boolean> values = Maps.newHashMap();
        responses.forEach((key, response) -> values.put(key, isSuccess(response.get())));
        return values;
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

@Override
public boolean setByNX(final String key, final String value, final int timeout) {
    Assert.hasText(key);
    Assert.hasText(value);
    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        final ShardedJedisPipeline pipeline = jedis.pipelined();
        final Response<Long> set = pipeline.setnx(key, value);
        final Response<Long> expire = pipeline.expire(key, timeout);
        pipeline.sync();
        return isSuccess(set.get()) && isSuccess(expire.get());
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

@Override
public Map<String, Boolean> set(final Map<String, Object> map) {
    Assert.notEmpty(map);

    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        final ShardedJedisPipeline pipelined = jedis.pipelined();
        final Map<String, Response<String>> responses = Maps.newHashMap();
        map.forEach((key, value) -> responses.put(key, pipelined.set(key, toJSONString(value))));
        pipelined.sync();

        final Map<String, Boolean> values = Maps.newHashMap();
        responses.forEach((key, response) -> values.put(key, isOK(response.get())));
        return values;
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

@Override
public Map<String, String> get(final String... keys) {
    Assert.notEmpty(keys);

    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        final ShardedJedisPipeline pipelined = jedis.pipelined();
        final Map<String, Response<String>> values = Maps.newHashMap();
        for (String key : keys) {
            values.put(key, pipelined.get(key));
        }

        pipelined.sync();
        final Map<String, String> valueMap = Maps.newHashMap();
        values.forEach((key, response) -> valueMap.put(key, response.get()));
        return valueMap;
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

public void testShardPipelined() {// 0.127秒
	JedisShardInfo jedis = new JedisShardInfo("120.25.241.144", 6379);
	jedis.setPassword("b840fc02d52404542994");

	List<JedisShardInfo> shards = Arrays.asList(jedis);
	ShardedJedis sharding = new ShardedJedis(shards);
	ShardedJedisPipeline pipeline = sharding.pipelined();

	long start = System.currentTimeMillis();
	for (int i = 0; i < 1000; i++) {
		pipeline.set("n" + i, "n" + i);
		System.out.println(i);
	}
	pipeline.syncAndReturnAll();
	long end = System.currentTimeMillis();
	System.out.println("共花费：" + (end - start) / 1000.0 + "秒");

	sharding.disconnect();
	try {
		Closeables.close(sharding, true);
	} catch (IOException e) {
		e.printStackTrace();
	}
}
 

@Override
public void saveFile(String fileLengthKey, String fileDataKey, String fileName, List<byte[]> values, long fileLength) {
    ShardedJedis shardedJedis = getShardedJedis();
    ShardedJedisPipeline pipelined = shardedJedis.pipelined();
    pipelined.hset(fileLengthKey.getBytes(), fileName.getBytes(), Longs.toByteArray(fileLength));
    Long blockSize = getBlockSize(fileLength);
    for (int i = 0; i < blockSize; i++) {
        pipelined.hset(fileDataKey.getBytes(), getBlockName(fileName, i), compressFilter(values.get(i)));
        if (i % Constants.SYNC_COUNT == 0) {
            pipelined.sync();
            pipelined = shardedJedis.pipelined();
        }
    }
    pipelined.sync();
    shardedJedis.close();
    values.clear();
}
 

@Override
public void saveFile(String fileLengthKey, String fileDataKey, String fileName, List<byte[]> values, long fileLength) {
    ShardedJedis shardedJedis = getShardedJedis();
    ShardedJedisPipeline pipelined = shardedJedis.pipelined();
    pipelined.hset(fileLengthKey.getBytes(), fileName.getBytes(), Longs.toByteArray(fileLength));
    Long blockSize = getBlockSize(fileLength);
    for (int i = 0; i < blockSize; i++) {
        pipelined.hset(fileDataKey.getBytes(), getBlockName(fileName, i), compressFilter(values.get(i)));
        if (i % Constants.SYNC_COUNT == 0) {
            pipelined.sync();
            pipelined = shardedJedis.pipelined();
        }
    }
    pipelined.sync();
    shardedJedis.close();
    values.clear();
}
 

/**
 * Use transactions to add index file and then delete the old one
 *
 * @param fileLengthKey the key using for hash file length
 * @param fileDataKey   the key using for hash file data
 * @param oldField      the old hash field
 * @param newField      the new hash field
 * @param values        the data values of the old hash field
 * @param fileLength    the data length of the old hash field
 */
@Override
public void rename(String fileLengthKey, String fileDataKey, String oldField, String newField, List<byte[]> values, long
        fileLength) {
    ShardedJedis shardedJedis = getShardedJedis();
    ShardedJedisPipeline pipelined = shardedJedis.pipelined();
    //add new file length
    pipelined.hset(fileLengthKey.getBytes(), newField.getBytes(), Longs.toByteArray(fileLength));
    //add new file content
    Long blockSize = getBlockSize(fileLength);
    for (int i = 0; i < blockSize; i++) {
        pipelined.hset(fileDataKey.getBytes(), getBlockName(newField, i), compressFilter(values.get(i)));
    }
    pipelined.sync();
    shardedJedis.close();
    values.clear();
    deleteFile(fileLengthKey, fileDataKey, oldField, blockSize);
}
 

@Override
public long sreplace(final String key, final String[] oldMembers, final String[] newMembers) {
    Assert.hasText(key);
    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        final ShardedJedisPipeline pipeline = jedis.pipelined();
        if (!ArrayUtils.isEmpty(oldMembers)) {
            pipeline.srem(key, oldMembers);
        }

        Response<Long> response = null;
        if (!ArrayUtils.isEmpty(newMembers)) {
            response = pipeline.sadd(key, newMembers);
        }

        pipeline.sync();
        if (response != null) {
            return response.get();
        }

        return 0;
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

/**
 * Use transactions to delete index file
 *
 * @param fileLengthKey the key using for hash file length
 * @param fileDataKey   the key using for hash file data
 * @param field         the hash field
 * @param blockSize     the index file data block size
 */
@Override
public void deleteFile(String fileLengthKey, String fileDataKey, String field, long blockSize) {
    ShardedJedis shardedJedis = getShardedJedis();
    ShardedJedisPipeline pipelined = shardedJedis.pipelined();
    //delete file length
    pipelined.hdel(fileLengthKey.getBytes(), field.getBytes());
    //delete file content
    for (int i = 0; i < blockSize; i++) {
        byte[] blockName = getBlockName(field, i);
        pipelined.hdel(fileDataKey.getBytes(), blockName);
    }
    pipelined.sync();
    shardedJedis.close();
}
 

@Override
public void doCallback(ShardedJedisPipeline p) {
	for (Command cmd : redisCommands) {
		switch (cmd.getOp()) {
			case SET:
				p.set(cmd.getCacheKey(), cmd.getCacheValue());
				break;
			case MOD:
				p.set(cmd.getCacheKey(), cmd.getCacheValue());
				break;
			case DEL:
				p.del(cmd.getCacheKey());
				break;
			case ADD_MEMBERS:
				p.sadd(cmd.getCacheGroupKey(), cmd.getGroupValues());
				break;
			case DEL_MEMBERS:
				p.srem(cmd.getCacheGroupKey(), cmd.getGroupValues());
				break;
			case SETS:
				String[] keyvalues = cmd.getKeyvalues();
				for (int i = 0; i < keyvalues.length; i+=2) {
					p.set(keyvalues[i], keyvalues[i+1]);
				}
			default:
				break;
		}
	}
}
 

@Override
public void pipelined(final PiplineCallback callback) {
	exec0(new ShardedJedisCallback<Void>() {
		@Override
		public Void doCallback(ShardedJedis jedis) {
			ShardedJedisPipeline sjp = jedis.pipelined();
			callback.doCallback(sjp); 
			sjp.sync();
			return null;
		}
	});
}
 

@Override
public List<byte[]> loadFileOnce(String fileDataKey, String fileName, long blockSize) {
    ShardedJedis shardedJedis = getShardedJedis();
    ShardedJedisPipeline pipelined = shardedJedis.pipelined();
    List<byte[]> res = new ArrayList<>();
    List<Response<byte[]>> temps = new ArrayList<>();
    int temp = 0;
    //如果不分批次sync容易read time out和Java heap space
    while (temp < blockSize) {
        Response<byte[]> data = pipelined.hget(fileDataKey.getBytes(), getBlockName(fileName, temp));
        temps.add(data);
        if (temp % Constants.SYNC_COUNT == 0) {
            pipelined.sync();
            res.addAll(temps.stream().map(response -> uncompressFilter(response.get())).collect(Collectors.toList()));
            temps.clear();
            pipelined = shardedJedis.pipelined();
        }
        temp++;
    }
    try {
        pipelined.sync();
    } catch (JedisConnectionException e) {
        log.error("pipelined = {}, blockSize = {}!", pipelined.toString(), blockSize);
        log.error("", e);
    } finally {
        shardedJedis.close();
    }
    res.addAll(temps.stream().map(response -> uncompressFilter(response.get())).collect(Collectors.toList()));
    temps.clear();
    return res;
}
 

public List<Object> pAddSet(String key, int seconds, String... values) {
    try (ShardedJedis shardedJedis = shardedJedisPool.getResource()) {
        ShardedJedisPipeline pipeline = shardedJedis.pipelined();
        for(String value : values) {
            pipeline.sadd(key, value);
        }
        pipeline.expire(key, seconds);
        List<Object> result = pipeline.syncAndReturnAll();
        result.remove(result.size() -1);
        return result;
    }
}
 

@Override
public long sunionstore(final String destination, final String... keys) {
    Assert.hasText(destination);
    if (keys.length == 0) {
        return 0;
    }

    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        final Collection<Jedis> allShards;
        if ((allShards = jedis.getAllShards()).size() == 1) {
            return allShards.iterator().next().sunionstore(destination, keys);
        } else if (allShards.size() > 1) {
            final Set<String> unionSet = sunion(keys);
            if (!unionSet.isEmpty()) {
                final ShardedJedisPipeline pipeline = jedis.pipelined();
                pipeline.del(destination);
                final Response<Long> response = pipeline.sadd(destination, unionSet.toArray(new String[unionSet.size()]));
                pipeline.sync();
                return response.get();
            }
        }

        return 0;
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

@Override
public long sinterstore(final String destination, final String... keys) {
    Assert.hasText(destination);
    if (keys.length == 0) {
        return 0;
    }

    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        final Collection<Jedis> allShards;
        if ((allShards = jedis.getAllShards()).size() == 1) {
            return allShards.iterator().next().sinterstore(destination, keys);
        } else if (allShards.size() > 1) {
            Set<String> interSet = sinter(keys);
            if (!interSet.isEmpty()) {
                final ShardedJedisPipeline pipeline = jedis.pipelined();
                pipeline.del(destination);
                final Response<Long> response = pipeline.sadd(destination, interSet.toArray(new String[interSet.size()]));
                pipeline.sync();
                return response.get();
            }
        }

        return 0;
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

@Override
public long sdiffstore(final String destination, final String... keys) {
    Assert.hasText(destination);
    Assert.notEmpty(keys);
    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        final Collection<Jedis> allShards;
        if ((allShards = jedis.getAllShards()).size() == 1) {
            return allShards.iterator().next().sdiffstore(destination, keys);
        } else if (allShards.size() > 1) {
            final Set<String> diffSet = sdiff(keys);
            if (!diffSet.isEmpty()) {
                final ShardedJedisPipeline pipeline = jedis.pipelined();
                pipeline.del(destination);
                final Response<Long> response = pipeline.sadd(destination, diffSet.toArray(new String[diffSet.size()]));
                pipeline.sync();
                return response.get();
            }
        }

        return 0;
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

@Test
public void canRetrieveUnsetKey() {
  ShardedJedisPipeline p = jedis.pipelined();
  Response<String> shouldNotExist = p.get(UUID.randomUUID().toString());
  p.sync();
  assertNull(shouldNotExist.get());
}
 

/**
 * Use transactions to delete index file
 *
 * @param fileLengthKey the key using for hash file length
 * @param fileDataKey   the key using for hash file data
 * @param field         the hash field
 * @param blockSize     the index file data block size
 */
@Override
public void deleteFile(String fileLengthKey, String fileDataKey, String field, long blockSize) {
    ShardedJedis shardedJedis = getShardedJedis();
    ShardedJedisPipeline pipelined = shardedJedis.pipelined();
    //delete file length
    pipelined.hdel(fileLengthKey.getBytes(), field.getBytes());
    //delete file content
    for (int i = 0; i < blockSize; i++) {
        byte[] blockName = getBlockName(field, i);
        pipelined.hdel(fileDataKey.getBytes(), blockName);
    }
    pipelined.sync();
    shardedJedis.close();
}
 

@Override
public List<byte[]> loadFileOnce(String fileDataKey, String fileName, long blockSize) {
    ShardedJedis shardedJedis = getShardedJedis();
    ShardedJedisPipeline pipelined = shardedJedis.pipelined();
    List<byte[]> res = new ArrayList<>();
    List<Response<byte[]>> temps = new ArrayList<>();
    int temp = 0;
    //如果不分批次sync容易read time out和Java heap space
    while (temp < blockSize) {
        Response<byte[]> data = pipelined.hget(fileDataKey.getBytes(), getBlockName(fileName, temp));
        temps.add(data);
        if (temp % Constants.SYNC_COUNT == 0) {
            pipelined.sync();
            res.addAll(temps.stream().map(response -> uncompressFilter(response.get())).collect(Collectors.toList()));
            temps.clear();
            pipelined = shardedJedis.pipelined();
        }
        temp++;
    }
    try {
        pipelined.sync();
    } catch (JedisConnectionException e) {
        log.error("pipelined = {}, blockSize = {}!", pipelined.toString(), blockSize);
        log.error("", e);
    } finally {
        shardedJedis.close();
    }
    res.addAll(temps.stream().map(response -> uncompressFilter(response.get())).collect(Collectors.toList()));
    temps.clear();
    return res;
}
 

@Test
public void returnResourceShouldResetState() throws URISyntaxException {
  GenericObjectPoolConfig config = new GenericObjectPoolConfig();
  config.setMaxTotal(1);
  config.setBlockWhenExhausted(false);

  List<JedisShardInfo> shards = new ArrayList<JedisShardInfo>();
  shards.add(new JedisShardInfo(new URI("redis://:foobared@localhost:6380")));
  shards.add(new JedisShardInfo(new URI("redis://:foobared@localhost:6379")));

  ShardedJedisPool pool = new ShardedJedisPool(config, shards);

  ShardedJedis jedis = pool.getResource();
  jedis.set("pipelined", String.valueOf(0));
  jedis.set("pipelined2", String.valueOf(0));

  ShardedJedisPipeline pipeline = jedis.pipelined();

  pipeline.incr("pipelined");
  pipeline.incr("pipelined2");

  jedis.resetState();

  pipeline = jedis.pipelined();
  pipeline.incr("pipelined");
  pipeline.incr("pipelined2");
  List<Object> results = pipeline.syncAndReturnAll();

  assertEquals(2, results.size());
  jedis.close();
  pool.destroy();
}
 

@Test
public void pipeline() throws UnsupportedEncodingException {
  ShardedJedisPipeline p = jedis.pipelined();
  p.set("foo", "bar");
  p.get("foo");
  List<Object> results = p.syncAndReturnAll();

  assertEquals(2, results.size());
  assertEquals("OK", results.get(0));
  assertEquals("bar", results.get(1));
}