Java Code Examples for redis.clients.jedis.ScanParams#count()

/**
 * For the given key prefix, find all actual keys depending on the type of the key.
 *
 * @param split The split for this request, mostly used to get the redis endpoint and config details.
 * @param redisCursor The previous Redis cursor (aka continuation token).
 * @param keys The collections of keys we collected so far. Any new keys we find are added to this.
 * @return The Redis cursor to use when continuing the scan.
 */
private ScanResult<String> loadKeys(Split split, ScanResult<String> redisCursor, Set<String> keys)
{
    try (Jedis client = getOrCreateClient(split.getProperty(REDIS_ENDPOINT_PROP))) {
        KeyType keyType = KeyType.fromId(split.getProperty(KEY_TYPE));
        String keyPrefix = split.getProperty(KEY_PREFIX_TABLE_PROP);
        if (keyType == KeyType.ZSET) {
            long start = Long.valueOf(split.getProperty(SPLIT_START_INDEX));
            long end = Long.valueOf(split.getProperty(SPLIT_END_INDEX));
            keys.addAll(client.zrange(keyPrefix, start, end));
            return new ScanResult<String>(END_CURSOR, Collections.EMPTY_LIST);
        }
        else {
            String cursor = (redisCursor == null) ? SCAN_POINTER_START : redisCursor.getCursor();
            ScanParams scanParam = new ScanParams();
            scanParam.count(SCAN_COUNT_SIZE);
            scanParam.match(split.getProperty(KEY_PREFIX_TABLE_PROP));

            ScanResult<String> newCursor = client.scan(cursor, scanParam);
            keys.addAll(newCursor.getResult());
            return newCursor;
        }
    }
}
 

public List<String> scanKeys(String key) {
    Jedis jedis = null;
    try {
        jedis = jedisPool.getResource();
        List<String> keys = new ArrayList<String>();
        String cursor = "0";
        ScanParams sp = new ScanParams();
        sp.match("*" + key + "*");
        sp.count(100);
        do {
            ScanResult<String> ret = jedis.scan(cursor, sp);
            List<String> result = ret.getResult();
            if (result != null && result.size() > 0) {
                keys.addAll(result);
            }
            //再处理cursor
            cursor = ret.getStringCursor();
        } while (!cursor.equals("0"));
        return keys;
    } finally {
        if (jedis != null) {
            jedis.close();
        }
    }
}
 

public Set<String> smembers(String key) {
    logger.trace("smembers {}", key);
    JedisCommands client = dynoClient;
    Set<String> r = new HashSet<>();
    int cursor = 0;
    ScanParams sp = new ScanParams();
    sp.count(50);

    do {
        ScanResult<String> scanResult = client.sscan(key, "" + cursor, sp);
        cursor = Integer.parseInt(scanResult.getCursor());
        r.addAll(scanResult.getResult());

    } while (cursor > 0);

    return r;

}
 

@Override
public void setup(OperatorContext context)
{
  super.setup(context);
  sleepTimeMillis = context.getValue(context.SPIN_MILLIS);
  getWindowDataManager().setup(context);
  this.context = context;
  scanOffset = 0;
  scanComplete = false;
  scanParameters = new ScanParams();
  scanParameters.count(scanCount);

  // For the 1st window after checkpoint, windowID - 1 would not have recovery
  // offset stored in windowDataManager
  // But recoveryOffset is non-transient, so will be recovered with
  // checkPointing
  // Offset recovery from idempotency storage can be skipped in this case
  scanOffset = recoveryState.scanOffsetAtBeginWindow;
  skipOffsetRecovery = true;
}
 

private ScanResult<Map.Entry<String, Double>> zscan_match_count(Jedis j, String key, String cursor, String pattern, int count) {
	ScanParams param = new ScanParams();
	param.match(pattern);
	param.count(count);
	redis.clients.jedis.ScanResult<Tuple> sr = j.zscan(key, cursor, param);
	return new ScanResult<Map.Entry<String, Double>>(sr.getStringCursor(), convert(sr.getResult()));
}
 

/**
 * For the given zset prefix, find all values and treat each of those values are a key to scan before returning
 * the scan continuation token.
 *
 * @param connStr The Jedis connection string for the table.
 * @param prefix The zset key prefix to scan.
 * @param redisCursor The previous Redis cursor (aka continuation token).
 * @param keys The collections of keys we collected so far. Any new keys we find are added to this.
 * @return The Redis cursor to use when continuing the scan.
 */
private ScanResult<String> loadKeys(String connStr, String prefix, ScanResult<String> redisCursor, Set<String> keys)
{
    try (Jedis client = getOrCreateClient(connStr)) {
        String cursor = (redisCursor == null) ? SCAN_POINTER_START : redisCursor.getCursor();
        ScanParams scanParam = new ScanParams();
        scanParam.count(SCAN_COUNT_SIZE);
        scanParam.match(prefix);

        ScanResult<String> newCursor = client.scan(cursor, scanParam);
        keys.addAll(newCursor.getResult());
        return newCursor;
    }
}
 

private ScanResult<String> sscan_match_count(Jedis j, String key, String cursor, String pattern, int count) {
	ScanParams param = new ScanParams();
	param.match(pattern);
	param.count(count);
	redis.clients.jedis.ScanResult<String> sr = j.sscan(key, cursor, param);
	return new ScanResult<String>(sr.getStringCursor(), sr.getResult());
}
 

private ScanResult<Map.Entry<String, String>> hscan_match_count(Jedis j, String key, String cursor, String pattern, int count) {
	ScanParams param = new ScanParams();
	param.match(pattern);
	param.count(count);
	redis.clients.jedis.ScanResult<Map.Entry<String, String>> sr = j.hscan(key, cursor, param);
	return new ScanResult<Map.Entry<String, String>>(sr.getStringCursor(), sr.getResult());
}
 

private ScanResult<String> scan_match_count(Jedis j, String cursor, String pattern, int count) {
	ScanParams param = new ScanParams();
	param.match(pattern);
	param.count(count);
	redis.clients.jedis.ScanResult<String> sr = j.scan(cursor, param);
	return new ScanResult<String>(sr.getStringCursor(), sr.getResult());
}
 

public ScanParams buildScanParams() {
    ScanParams scanParams = new ScanParams();
    if (Strings.isNullOrEmpty(this.getCursor())) {
        this.setCursor("0");
    }
    scanParams.count(this.getCount());
    if (Strings.isNullOrEmpty(this.getKey())) {
        this.setKey("*");
    } else if (key.indexOf("*") != 0
            && key.indexOf("*") != (key.length() - 1)) {
        key = "*" + key + "*";
    }
    scanParams.match(this.getKey());
    return scanParams;
}
 

private ScanParams setScanParms()
{
    if (split.getKeyDataType() == RedisDataType.STRING) {
        ScanParams scanParms = new ScanParams();
        scanParms.count(redisJedisManager.getRedisConnectorConfig().getRedisScanCount());

        // when Redis key string follows "schema:table:*" format
        // scan command can efficiently query tables
        // by returning matching keys
        // the alternative is to set key-prefix-schema-table to false
        // and treat entire redis as single schema , single table
        // redis Hash/Set types are to be supported - they can also be
        // used to filter out table data

        // "default" schema is not prefixed to the key

        if (redisJedisManager.getRedisConnectorConfig().isKeyPrefixSchemaTable()) {
            String keyMatch = "";
            if (!split.getSchemaName().equals("default")) {
                keyMatch = split.getSchemaName() + Character.toString(redisJedisManager.getRedisConnectorConfig().getRedisKeyDelimiter());
            }
            keyMatch = keyMatch + split.getTableName() + Character.toString(redisJedisManager.getRedisConnectorConfig().getRedisKeyDelimiter()) + "*";
            scanParms.match(keyMatch);
        }
        return scanParms;
    }

    return null;
}
 

@Test
public void testRecoveryAndIdempotency() throws Exception
{
  this.operatorStore = new RedisStore();
  this.testStore = new RedisStore();

  testStore.connect();
  ScanParams params = new ScanParams();
  params.count(1);

  testStore.put("test_abc", "789");
  testStore.put("test_def", "456");
  testStore.put("test_ghi", "123");

  RedisKeyValueInputOperator operator = new RedisKeyValueInputOperator();
  operator.setWindowDataManager(new FSWindowDataManager());

  operator.setStore(operatorStore);
  operator.setScanCount(1);
  Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
  CollectorTestSink<Object> sink = new CollectorTestSink<Object>();

  operator.outputPort.setSink(sink);
  OperatorContext context = mockOperatorContext(1, attributeMap);

  try {
    operator.setup(context);
    operator.beginWindow(1);
    operator.emitTuples();
    operator.endWindow();

    int numberOfMessagesInWindow1 = sink.collectedTuples.size();
    sink.collectedTuples.clear();

    operator.beginWindow(2);
    operator.emitTuples();
    operator.endWindow();
    int numberOfMessagesInWindow2 = sink.collectedTuples.size();
    sink.collectedTuples.clear();

    // failure and then re-deployment of operator
    // Re-instantiating to reset values
    operator = new RedisKeyValueInputOperator();
    operator.setWindowDataManager(new FSWindowDataManager());
    operator.setStore(operatorStore);
    operator.setScanCount(1);
    operator.outputPort.setSink(sink);
    operator.setup(context);

    Assert.assertEquals("largest recovery window", 2, operator.getWindowDataManager().getLargestCompletedWindow());

    operator.beginWindow(1);
    operator.emitTuples();
    operator.emitTuples();
    operator.endWindow();

    Assert.assertEquals("num of messages in window 1", numberOfMessagesInWindow1, sink.collectedTuples.size());

    sink.collectedTuples.clear();
    operator.beginWindow(2);
    operator.emitTuples();
    operator.endWindow();
    Assert.assertEquals("num of messages in window 2",numberOfMessagesInWindow2, sink.collectedTuples.size());
  } finally {
    for (Object e : sink.collectedTuples) {
      KeyValPair<String, String> entry = (KeyValPair<String, String>)e;
      testStore.remove(entry.getKey());
    }
    sink.collectedTuples.clear();
    operator.getWindowDataManager().committed(5);
    operator.teardown();
  }
}
 

private ScanResult<String> scan_count(Jedis j, String cursor, int count) {
	ScanParams param = new ScanParams();
	param.count(count);
	redis.clients.jedis.ScanResult<String> sr = j.scan(cursor, param);
	return new ScanResult<String>(sr.getStringCursor(), sr.getResult());
}
 

@Test
public void testIntputOperator() throws IOException
{
  this.operatorStore = new RedisStore();
  this.testStore = new RedisStore();

  testStore.connect();
  ScanParams params = new ScanParams();
  params.count(1);

  testStore.put("test_abc", "789");
  testStore.put("test_def", "456");
  testStore.put("test_ghi", "123");

  try {
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();

    RedisKeyValueInputOperator inputOperator = dag.addOperator("input", new RedisKeyValueInputOperator());
    final CollectorModule collector = dag.addOperator("collector", new CollectorModule());

    inputOperator.setStore(operatorStore);
    dag.addStream("stream", inputOperator.outputPort, collector.inputPort);
    final LocalMode.Controller lc = lma.getController();

    new Thread("LocalClusterController")
    {
      @Override
      public void run()
      {
        long startTms = System.currentTimeMillis();
        long timeout = 50000L;
        try {
          Thread.sleep(1000);
          while (System.currentTimeMillis() - startTms < timeout) {
            if (CollectorModule.resultMap.size() < 3) {
              Thread.sleep(10);
            } else {
              break;
            }
          }
        } catch (InterruptedException ex) {
          //
        }
        lc.shutdown();
      }
    }.start();

    lc.run();

    Assert.assertTrue(CollectorModule.resultMap.contains(new KeyValPair<String, String>("test_abc", "789")));
    Assert.assertTrue(CollectorModule.resultMap.contains(new KeyValPair<String, String>("test_def", "456")));
    Assert.assertTrue(CollectorModule.resultMap.contains(new KeyValPair<String, String>("test_ghi", "123")));
  } finally {
    for (KeyValPair<String, String> entry : CollectorModule.resultMap) {
      testStore.remove(entry.getKey());
    }
    testStore.disconnect();
  }
}
 

private ScanResult<Map.Entry<String, String>> hscan_count(Jedis j, String key, String cursor, int count) {
	ScanParams param = new ScanParams();
	param.count(count);
	redis.clients.jedis.ScanResult<Map.Entry<String, String>> sr = j.hscan(key, cursor, param);
	return new ScanResult<Map.Entry<String, String>>(sr.getStringCursor(), sr.getResult());
}
 

@Test
public void testInputOperator() throws IOException
{
  @SuppressWarnings("unused")
  Class<?> clazz = org.codehaus.janino.CompilerFactory.class;

  this.operatorStore = new RedisStore();
  this.testStore = new RedisStore();

  testStore.connect();
  ScanParams params = new ScanParams();
  params.count(100);

  Map<String, String> value = new HashMap<String, String>();
  value.put("Column1", "abc");
  value.put("Column2", "1");

  Map<String, String> value1 = new HashMap<String, String>();
  value1.put("Column1", "def");
  value1.put("Column2", "2");

  Map<String, String> value2 = new HashMap<String, String>();
  value2.put("Column1", "ghi");
  value2.put("Column2", "3");

  testStore.put("test_abc_in", value);
  testStore.put("test_def_in", value1);
  testStore.put("test_ghi_in", value2);

  try {
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();

    RedisPOJOInputOperator inputOperator = dag.addOperator("input", new RedisPOJOInputOperator());
    final ObjectCollectorModule collector = dag.addOperator("collector", new ObjectCollectorModule());

    ArrayList<FieldInfo> fields = new ArrayList<FieldInfo>();

    fields.add(new FieldInfo("Column1", "stringValue", SupportType.STRING));
    fields.add(new FieldInfo("Column2", "intValue", SupportType.INTEGER));

    inputOperator.setDataColumns(fields);
    inputOperator.setOutputClass(TestClass.class.getName());

    inputOperator.setStore(operatorStore);
    dag.addStream("stream", inputOperator.outputPort, collector.inputPort);
    final LocalMode.Controller lc = lma.getController();

    new Thread("LocalClusterController")
    {
      @Override
      public void run()
      {
        long startTms = System.currentTimeMillis();
        long timeout = 10000L;
        try {
          Thread.sleep(1000);
          while (System.currentTimeMillis() - startTms < timeout) {
            if (ObjectCollectorModule.resultMap.size() < 3) {
              Thread.sleep(10);
            } else {
              break;
            }
          }
        } catch (InterruptedException ex) {
          //
        }
        lc.shutdown();
      }
    }.start();

    lc.run();

    Assert.assertTrue(ObjectCollectorModule.resultMap.containsKey("test_abc_in"));
    Assert.assertTrue(ObjectCollectorModule.resultMap.containsKey("test_def_in"));
    Assert.assertTrue(ObjectCollectorModule.resultMap.containsKey("test_ghi_in"));

    TestClass a = (TestClass)ObjectCollectorModule.resultMap.get("test_abc_in");
    Assert.assertNotNull(a);
    Assert.assertEquals("abc", a.stringValue);
    Assert.assertEquals("1", a.intValue.toString());
  } finally {
    for (KeyValPair<String, String> entry : CollectorModule.resultMap) {
      testStore.remove(entry.getKey());
    }
    testStore.disconnect();
  }
}
 

private ScanResult<String> sscan_count(Jedis j, String key, String cursor, int count) {
	ScanParams param = new ScanParams();
	param.count(count);
	redis.clients.jedis.ScanResult<String> sr = j.sscan(key, cursor, param);
	return new ScanResult<String>(sr.getStringCursor(), sr.getResult());
}
 

private ScanResult<Map.Entry<String, Double>> zscan_count(Jedis j, String key, String cursor, int count) {
	ScanParams param = new ScanParams();
	param.count(count);
	redis.clients.jedis.ScanResult<Tuple> sr = j.zscan(key, cursor, param);
	return new ScanResult<Map.Entry<String, Double>>(sr.getStringCursor(), convert(sr.getResult()));
}