com.datatorrent.netlet.util.Slice Java Examples

private void commitHelper(Slice one, Slice two)
{
  testMeta.managedState.setup(testMeta.operatorContext);
  long time = System.currentTimeMillis();
  testMeta.managedState.beginWindow(time);
  testMeta.managedState.put(0, one, one);
  testMeta.managedState.endWindow();
  testMeta.managedState.beforeCheckpoint(time);

  testMeta.managedState.beginWindow(time + 1);
  testMeta.managedState.put(0, two, two);
  testMeta.managedState.endWindow();
  testMeta.managedState.beforeCheckpoint(time + 1);

  testMeta.managedState.committed(time);
}
 

@Test
public void testSomeMethod() throws IOException
{
  JavaSerializationStreamCodec<Serializable> coder = new JavaSerializationStreamCodec<Serializable>();
  JavaSerializationStreamCodec<Serializable> decoder = new JavaSerializationStreamCodec<Serializable>();

  TestClass tc = new TestClass("hello!", 42);

  Slice dsp1 = coder.toByteArray(tc);
  Slice dsp2 = coder.toByteArray(tc);
  Assert.assertEquals(dsp1, dsp2);

  Object tcObject1 = decoder.fromByteArray(dsp1);
  assert (tc.equals(tcObject1));

  Object tcObject2 = decoder.fromByteArray(dsp2);
  assert (tc.equals(tcObject2));

  dsp1 = coder.toByteArray(tc);
  dsp2 = coder.toByteArray(tc);
  Assert.assertEquals(dsp1, dsp2);
}
 

@Override
public void readFully(TreeMap<Slice, Slice> data) throws IOException
{
  scanner.rewind();
  for (; !scanner.atEnd(); scanner.advance()) {
    Entry en = scanner.entry();
    int klen = en.getKeyLength();
    int vlen = en.getValueLength();
    byte[] key = new byte[klen];
    byte[] value = new byte[vlen];
    en.getKey(key);
    en.getValue(value);
    data.put(new Slice(key, 0, key.length), new Slice(value, 0, value.length));
  }

}
 

@Override
public boolean peek(Slice key, Slice value) throws IOException
{
  if (scanner.atEnd()) {
    return false;
  }
  Entry en = scanner.entry();

  key.buffer = en.getBlockBuffer();
  key.offset = en.getKeyOffset();
  key.length = en.getKeyLength();

  value.buffer = en.getBlockBuffer();
  value.offset = en.getValueOffset();
  value.length = en.getValueLength();

  return true;
}
 

@Test
public void testCommittedWithOpenReader() throws IOException
{
  testMeta.defaultBucket.setup(testMeta.managedStateContext);
  testGetFromReader();
  Map<Long, FileAccess.FileReader> readers = testMeta.defaultBucket.getReaders();
  Assert.assertTrue("reader open", readers.containsKey(101L));

  Slice two = ManagedStateTestUtils.getSliceFor("2");
  Slice one = ManagedStateTestUtils.getSliceFor("1");

  testMeta.defaultBucket.put(two, 101, two);
  Map<Slice, Bucket.BucketedValue> unsaved = testMeta.defaultBucket.checkpoint(10);
  Assert.assertEquals("size", 1, unsaved.size());
  testMeta.defaultBucket.committed(10);

  Slice value = testMeta.defaultBucket.get(two, -1, Bucket.ReadSource.MEMORY);
  Assert.assertEquals("value two", two, value);

  value = testMeta.defaultBucket.get(one, -1, Bucket.ReadSource.MEMORY);
  Assert.assertEquals("value one", one, value);

  Assert.assertTrue("reader closed", !readers.containsKey(101L));
  testMeta.defaultBucket.teardown();
}
 

@Override
public boolean peek(Slice key, Slice value) throws IOException
{
  if (scanner.atEnd()) {
    return false;
  }
  Entry en = scanner.entry();
  byte[] rkey = new byte[en.getKeyLength()];
  byte[] rval = new byte[en.getValueLength()];
  en.getKey(rkey);
  en.getValue(rval);

  key.buffer = rkey;
  key.offset = 0;
  key.length = en.getKeyLength();

  value.buffer = rval;
  value.offset = 0;
  value.length = en.getValueLength();

  return true;
}
 

/**
 * Return the list of values from the store
 * @param k given key
 * @return list of values
 */
@Override
public List<V> get(@Nullable K k)
{
  List<V> value = null;
  Slice valueSlice = store.getSync(getBucketId(k), streamCodec.toByteArray(k));
  if (valueSlice == null || valueSlice.length == 0 || valueSlice.buffer == null) {
    return null;
  }
  if (isKeyContainsMultiValue) {
    return (List<V>)streamCodec.fromByteArray(valueSlice);
  }
  value = new ArrayList<>();
  value.add((V)streamCodec.fromByteArray(valueSlice));
  return  value;
}
 

@Test
public void testCleanForFlushedData() throws IOException
{
  byte[] b = new byte[200];
  storage.retrieve(new byte[8]);
  for (int i = 0; i < 5; i++) {
    storage.store(new Slice(b, 0, b.length));
    storage.store(new Slice(b, 0, b.length));
    storage.flush();
    // storage.clean(address);
  }
  byte[] lastWrittenAddress = null;
  for (int i = 0; i < 5; i++) {
    storage.store(new Slice(b, 0, b.length));
    lastWrittenAddress = storage.store(new Slice(b, 0, b.length));
  }
  storage.flush();
  storage.clean(lastWrittenAddress);
  byte[] cleanedOffset = storage.readData(new Path(STORAGE_DIRECTORY + "/1/cleanoffsetFile"));
  Assert.assertArrayEquals(lastWrittenAddress, cleanedOffset);

}
 

@Test
public void testWindowedBlockStream()
{
  WindowedBlockStream bs = new WindowedBlockStream();
  List<byte[]> totalList = Lists.newArrayList();
  List<Slice> slices = Lists.newArrayList();

  for (int windowId = 0; windowId < 10; ++windowId) {
    List<byte[]> list = generateList();
    totalList.addAll(list);

    bs.beginWindow(windowId);
    writeToBlockStream(bs, list, slices);
    bs.endWindow();

    if (windowId % 2 != 0) {
      verify(totalList, slices);

      bs.completeWindow(windowId);
      totalList.clear();
      slices.clear();
    }
  }
}
 

/**
 * Retrieves artifacts available for all the windows saved by the enclosing partitions.
 * @return  artifact saved per window.
 * @throws IOException
 */
public Map<Long, Object> retrieveAllWindows() throws IOException
{
  Map<Long, Object> artifactPerWindow = new HashMap<>();
  FileSystemWAL.FileSystemWALReader reader = getWal().getReader();
  reader.seek(getWal().getWalStartPointer());

  Slice windowSlice = readNext(reader);
  while (reader.getCurrentPointer().compareTo(getWal().getWalEndPointerAfterRecovery()) < 0 && windowSlice != null) {
    long window = Longs.fromByteArray(windowSlice.toByteArray());
    Object data = fromSlice(readNext(reader));
    artifactPerWindow.put(window, data);
    windowSlice = readNext(reader); //null or next window
  }
  reader.seek(getWal().getWalStartPointer());
  return artifactPerWindow;
}
 

public void writeEvent(StramEvent event) throws Exception
{
  LOG.debug("Writing event {} to the storage", event.getType());
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  bos.write((event.getTimestamp() + ":").getBytes());
  bos.write((event.getType() + ":").getBytes());
  @SuppressWarnings("unchecked")
  Map<String, String> data = BeanUtils.describe(event);
  data.remove("timestamp");
  data.remove("class");
  data.remove("type");
  Slice f = streamCodec.toByteArray(data);
  bos.write(f.buffer, f.offset, f.length);
  bos.write("\n".getBytes());
  storage.writeDataItem(bos.toByteArray(), true);
  if (numSubscribers > 0) {
    LOG.debug("Publishing event {} through websocket to gateway", event.getType());
    EventsAgent.EventInfo eventInfo = new EventsAgent.EventInfo();
    eventInfo.id = event.getId();
    eventInfo.timestamp = event.getTimestamp();
    eventInfo.type = event.getType();
    eventInfo.data = data;
    eventInfo.data.remove("id");
    wsClient.publish(pubSubTopic, eventInfo);
  }
}
 

@Test
public void testStorageWithRestore() throws IOException
{
  Assert.assertNull(storage.retrieve(new byte[8]));
  byte[] b = new byte[200];
  Assert.assertNotNull(storage.store(new Slice(b, 0, b.length)));
  storage.flush();
  storage.teardown();

  storage = getStorage("1", true);
  storage.store(new Slice(b, 0, b.length));
  storage.flush();
  Configuration conf = new Configuration();
  FileSystem fs = FileSystem.get(conf);
  boolean exists = fs.exists(new Path(STORAGE_DIRECTORY + "/1/" + "1"));
  Assert.assertEquals("file should exist", true, exists);
}
 

/**
 * This test covers following use case The file is flushed and then more data is written to the same file, but the new
 * data is not flushed and file is not roll over and storage fails The new storage comes up and client asks for data
 * at the last returned address from earlier storage instance. The new storage returns null. Client stores the data
 * again but the address returned this time is null and the retrieval of the earlier address now returns data
 *
 * @throws Exception
 */
@Test
public void testPartialFlushWithFailure() throws Exception
{
  Assert.assertNull(storage.retrieve(new byte[8]));
  byte[] b = "ab".getBytes();
  byte[] address = storage.store(new Slice(b, 0, b.length));
  Assert.assertNotNull(address);
  storage.flush();
  b = "cb".getBytes();
  byte[] addr = storage.store(new Slice(b, 0, b.length));
  storage = getStorage("1", true);
  Assert.assertNull(storage.retrieve(addr));
  Assert.assertNull(storage.store(new Slice(b, 0, b.length)));
  storage.flush();
  match(storage.retrieve(address), "cb");
}
 

@Test
public void testSyncGetFromFiles() throws IOException, ExecutionException, InterruptedException
{
  DefaultBucket.setDisableBloomFilterByDefault(true);

  Slice zero = ManagedStateTestUtils.getSliceFor("0");
  long time = System.currentTimeMillis();

  testMeta.managedState.setup(testMeta.operatorContext);

  long timeBucket = testMeta.managedState.getTimeBucketAssigner().getTimeBucket(time);
  Map<Slice, Bucket.BucketedValue> unsavedBucket0 = ManagedStateTestUtils.getTestBucketData(0, timeBucket);

  //write data to disk explicitly
  testMeta.managedState.bucketsFileSystem.writeBucketData(time, 0, unsavedBucket0, -1);
  ManagedStateTestUtils.validateBucketOnFileSystem(testMeta.managedState.getFileAccess(),
      testMeta.operatorContext.getId(), unsavedBucket0, 1);

  Slice value = testMeta.managedState.getSync(time, zero);

  Assert.assertEquals("value of zero", zero, value);
  testMeta.managedState.teardown();
}
 

/**
 * Helper method that catches IOException while reading from wal to check if an entry was saved completely or not.
 * @param reader wal reader
 * @return wal entry
 */
protected Slice readNext(FileSystemWAL.FileSystemWALReader reader)
{
  try {
    return reader.next();
  } catch (IOException ex) {
    //exception while reading wal entry which can be because there may have been failure while persisting an
    //artifact so this window is not a finished window.
    try {
      reader.close();
    } catch (IOException ioe) {
      //closing the reader quietly.
    }
    return null;
  }
}
 

/**
 * This tests clean when the file doesn't roll over
 *
 * @throws Exception
 */
@Test
public void testPartialFlushWithClean() throws Exception
{
  Assert.assertNull(storage.retrieve(new byte[8]));
  byte[] b = "ab".getBytes();
  byte[] address = storage.store(new Slice(b, 0, b.length));
  Assert.assertNotNull(address);
  storage.flush();
  storage.clean(address);
  b = "cb".getBytes();
  byte[] addr = storage.store(new Slice(b, 0, b.length));
  Assert.assertNull(storage.retrieve(addr));
  Assert.assertNull(storage.store(new Slice(b, 0, b.length)));
  storage.flush();
  match(storage.retrieve(new byte[8]), "cb");
  match(storage.retrieve(address), "cb");
  Assert.assertNotNull(storage.store(new Slice(b, 0, b.length)));
}
 

@Test
public void testPutWithMultipleValuesForAKey()
{
  Slice one = ManagedStateTestUtils.getSliceFor("1");

  testMeta.managedState.setup(testMeta.operatorContext);
  long time = System.currentTimeMillis();
  testMeta.managedState.beginWindow(0);
  testMeta.managedState.put(time, one, one);

  Slice two = ManagedStateTestUtils.getSliceFor("2");
  testMeta.managedState.put(time, one, two);
  Slice value = testMeta.managedState.getSync(time, one);
  testMeta.managedState.endWindow();

  Assert.assertEquals("value overwritten", two, value);
  testMeta.managedState.teardown();
}
 

@Test
public void testTransferWindowFiles() throws IOException, InterruptedException
{
  testMeta.checkpointManager.setup(testMeta.managedStateContext);

  Map<Long, Map<Slice, Bucket.BucketedValue>> buckets5 = ManagedStateTestUtils.getTestData(0, 5, 0);
  testMeta.checkpointManager.save(buckets5, 10, false);
  //Need to synchronously call transfer window files so shutting down the other thread.
  testMeta.checkpointManager.teardown();
  Thread.sleep(500);

  testMeta.checkpointManager.committed(10);
  testMeta.checkpointManager.transferWindowFiles();

  for (int i = 0; i < 5; i++) {
    ManagedStateTestUtils.validateBucketOnFileSystem(testMeta.managedStateContext.getFileAccess(), i,
        buckets5.get((long)i), 1);
  }
}
 

@Test
public void testPut()
{
  testMeta.defaultBucket.setup(testMeta.managedStateContext);
  Slice one = ManagedStateTestUtils.getSliceFor("1");
  testMeta.defaultBucket.put(one, 1, one);

  Slice value = testMeta.defaultBucket.get(one, 1, Bucket.ReadSource.MEMORY);
  Assert.assertEquals("value one", one, value);

  value = testMeta.defaultBucket.get(one, 1, Bucket.ReadSource.READERS);
  Assert.assertNull("value not present", value);

  Assert.assertEquals("size of bucket", one.length * 2 + Longs.BYTES, testMeta.defaultBucket.getSizeInBytes());

  testMeta.defaultBucket.teardown();
}
 

@Override
public void put(Slice key, long timeBucket, Slice value)
{
  // This call is lightweight
  releaseMemory();

  key = SliceUtils.toBufferSlice(key);
  value = SliceUtils.toBufferSlice(value);

  BucketedValue bucketedValue = flash.get(key);
  if (bucketedValue == null) {
    bucketedValue = new BucketedValue(timeBucket, value);
    flash.put(key, bucketedValue);
    sizeInBytes.getAndAdd(key.length + value.length + Longs.BYTES);
  } else {
    if (timeBucket >= bucketedValue.getTimeBucket()) {
      int inc = null == bucketedValue.getValue() ? value.length : value.length - bucketedValue.getValue().length;
      sizeInBytes.getAndAdd(inc);
      bucketedValue.setTimeBucket(timeBucket);
      bucketedValue.setValue(value);
    } else {
      throw new AssertionError("newer entry exists for " + key);
    }
  }
}
 

@Test
public void testAsyncGetFromReaders() throws IOException, ExecutionException, InterruptedException
{
  Slice zero = ManagedStateTestUtils.getSliceFor("0");
  long time = System.currentTimeMillis();

  DefaultBucket.setDisableBloomFilterByDefault(true);
  testMeta.managedState.setup(testMeta.operatorContext);

  Map<Slice, Bucket.BucketedValue> unsavedBucket0 = ManagedStateTestUtils.getTestBucketData(0, time);
  testMeta.managedState.bucketsFileSystem.writeBucketData(time, 0, unsavedBucket0, -1);
  ManagedStateTestUtils.validateBucketOnFileSystem(testMeta.managedState.getFileAccess(), 0, unsavedBucket0, 1);

  Future<Slice> valFuture = testMeta.managedState.getAsync(0, zero);

  Assert.assertEquals("value of zero", zero, valFuture.get());
  testMeta.managedState.teardown();
}
 

@Test
public void testGetAllTimeBucketMeta() throws IOException
{
  testMeta.bucketsFileSystem.setup(testMeta.managedStateContext);
  BucketsFileSystem.MutableTimeBucketMeta tbm1 = new BucketsFileSystem.MutableTimeBucketMeta(1, 1);
  tbm1.updateTimeBucketMeta(10, 100, new Slice("1".getBytes()));
  testMeta.bucketsFileSystem.updateTimeBuckets(tbm1);

  BucketsFileSystem.MutableTimeBucketMeta tbm2 = new BucketsFileSystem.MutableTimeBucketMeta(1, 2);
  tbm2.updateTimeBucketMeta(10, 100, new Slice("2".getBytes()));
  testMeta.bucketsFileSystem.updateTimeBuckets(tbm2);

  testMeta.bucketsFileSystem.updateBucketMetaFile(1);
  TreeMap<Long, BucketsFileSystem.TimeBucketMeta> timeBucketMetas =
      testMeta.bucketsFileSystem.getAllTimeBuckets(1);

  Iterator<Map.Entry<Long, BucketsFileSystem.TimeBucketMeta>> iterator = timeBucketMetas.entrySet().iterator();
  int i = 2;
  while (iterator.hasNext()) {
    BucketsFileSystem.TimeBucketMeta tbm = iterator.next().getValue();
    Assert.assertEquals("time bucket " + i, i, tbm.getTimeBucketId());
    i--;
  }
  testMeta.bucketsFileSystem.teardown();
}
 

/**
 * Insert (keySlice,valueSlice) into the store using bucketId and timeBucket.
 * @param bucketId bucket Id
 * @param timeBucket time bucket
 * @param keySlice key slice
 * @param valueSlice value slice
 * @return true if the given (keySlice,valueSlice) is successfully inserted into the
 *         store otherwise false.
 */
private boolean insertInStore(long bucketId, long timeBucket, Slice keySlice, Slice valueSlice)
{
  long timeBucketId = store.getTimeBucketAssigner().getTimeBucket(timeBucket);
  if (timeBucketId != -1) {
    store.putInBucket(bucketId, timeBucketId, keySlice, valueSlice);
    return true;
  }
  return false;
}
 

@Override
public Slice toByteArray(Event event)
{
  try {
    return super.toByteArray(event);
  } catch (RuntimeException re) {
    logger.warn("Cannot serialize event {}", event, re);
  }

  return null;
}
 

@Override
public Object fromByteArray(Slice fragment)
{
  try {
    return super.fromByteArray(fragment);
  } catch (RuntimeException re) {
    logger.warn("Cannot deserialize event {}", fragment, re);
  }

  return null;
}
 

@Test
public void testSerdeList()
{
  CollectionSerde<String, List<String>> serdeList =
      new CollectionSerde<>(new StringSerde(), (Class)ArrayList.class);

  List<String> stringList = Lists.newArrayList("a", "b", "c");
  SerializationBuffer buffer = new SerializationBuffer(new WindowedBlockStream());
  serdeList.serialize(stringList, buffer);

  Slice slice = buffer.toSlice();
  List<String> deserializedList = serdeList.deserialize(new Input(slice.buffer, slice.offset, slice.length));

  Assert.assertEquals(stringList, deserializedList);
}
 

public boolean write(byte[] address, Slice event)
{
  if (event.offset == 0 && event.length == event.buffer.length) {
    return write(address, event.buffer);
  }

  // a better method would be to replace the write implementation and allow it to natively support writing slices
  return write(address, event.toByteArray());
}
 

@Test
public void testCommitted()
{
  testMeta.defaultBucket.setup(testMeta.managedStateContext);
  Slice one = ManagedStateTestUtils.getSliceFor("1");
  testCheckpointed();
  testMeta.defaultBucket.committed(10);
  Slice value = testMeta.defaultBucket.get(one, -1, Bucket.ReadSource.MEMORY);
  Assert.assertEquals("value one", one, value);
  testMeta.defaultBucket.teardown();
}
 

/**
 * This test case tests the clean call before any flush is called.
 *
 * @throws IOException
 */
@Test
public void testCleanUnflushedData() throws IOException
{
  for (int i = 0; i < 5; i++) {
    final byte[] bytes = (i + "").getBytes();
    storage.store(new Slice(bytes, 0, bytes.length));
  }
  storage.clean(new byte[8]);
  storage.flush();
  match(storage.retrieve(new byte[8]), "0");
  match(storage.retrieveNext(), "1");
}
 

@Override
public void handleException(Exception cce, EventLoop el)
{
  logger.error("Server Error", cce);
  Request r = new Request(Command.SERVER_ERROR, null)
  {
    @Override
    public Slice getAddress()
    {
      throw new UnsupportedOperationException("Not supported yet.");
    }

    @Override
    public int getEventCount()
    {
      throw new UnsupportedOperationException("Not supported yet.");
    }

    @Override
    public int getIdleCount()
    {
      throw new UnsupportedOperationException("Not supported yet.");
    }

  };
  synchronized (requests) {
    requests.add(r);
  }
}