Java Code Examples for org.apache.samza.serializers.Serde

The following examples show how to use org.apache.samza.serializers.Serde. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: beam   Source File: SamzaTimerInternalsFactoryTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testByteArray() {
  ByteArray key1 = ByteArray.of("hello world".getBytes(StandardCharsets.UTF_8));
  Serde<ByteArray> serde = new ByteArraySerdeFactory().getSerde("", null);
  byte[] keyBytes = serde.toBytes(key1);
  ByteArray key2 = serde.fromBytes(keyBytes);
  assertEquals(key1, key2);

  Map<ByteArray, String> map = new HashMap<>();
  map.put(key1, "found it");
  assertEquals("found it", map.get(key2));

  map.remove(key1);
  assertTrue(!map.containsKey(key2));
  assertTrue(map.isEmpty());
}
 
Example 2
Source Project: samza   Source File: TestMessageStreamImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWindowWithRelaxedTypes() throws Exception {
  StreamApplicationDescriptorImpl mockGraph = mock(StreamApplicationDescriptorImpl.class);
  OperatorSpec mockOpSpec = mock(OperatorSpec.class);
  MessageStream<TestInputMessageEnvelope> inputStream = new MessageStreamImpl<>(mockGraph, mockOpSpec);

  MapFunction<TestMessageEnvelope, String> keyExtractor = m -> m.getKey();
  FoldLeftFunction<TestMessageEnvelope, Integer> aggregator = (m, c) -> c + 1;
  SupplierFunction<Integer> initialValue = () -> 0;

  // should compile since TestMessageEnvelope (input for functions) is base class of TestInputMessageEnvelope (M)
  Window<TestInputMessageEnvelope, String, Integer> window = Windows
      .keyedTumblingWindow(keyExtractor, Duration.ofHours(1), initialValue, aggregator, null, mock(Serde.class));
  MessageStream<WindowPane<String, Integer>> windowedStream = inputStream.window(window, "w1");

  ArgumentCaptor<OperatorSpec> registeredOpCaptor = ArgumentCaptor.forClass(OperatorSpec.class);
  verify(mockOpSpec).registerNextOperatorSpec(registeredOpCaptor.capture());
  OperatorSpec<?, TestMessageEnvelope> registeredOpSpec = registeredOpCaptor.getValue();

  assertTrue(registeredOpSpec instanceof WindowOperatorSpec);
  assertEquals(OpCode.WINDOW, registeredOpSpec.getOpCode());
  assertEquals(window, ((WindowOperatorSpec) registeredOpSpec).getWindow());
}
 
Example 3
Source Project: samza   Source File: TestJobNodeConfigurationGenerator.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConfigureSerdesForRepartitionWithNoDefaultSystem() {
  // set the application to RepartitionOnlyStreamApplication
  mockStreamAppDesc = new StreamApplicationDescriptorImpl(getRepartitionOnlyStreamApplication(), mockConfig);
  configureJobNode(mockStreamAppDesc);

  // create the JobGraphConfigureGenerator and generate the jobConfig for the jobNode
  JobNodeConfigurationGenerator configureGenerator = new JobNodeConfigurationGenerator();
  JobConfig jobConfig = configureGenerator.generateJobConfig(mockJobNode, "testJobGraphJson");

  // Verify the results
  Config expectedJobConfig = getExpectedJobConfig(mockConfig, mockJobNode.getInEdges());
  validateJobConfig(expectedJobConfig, jobConfig);

  Map<String, Serde> deserializedSerdes = validateAndGetDeserializedSerdes(jobConfig, 2);
  validateStreamConfigures(jobConfig, null);

  String partitionByKeySerde = jobConfig.get("streams.jobName-jobId-partition_by-p1.samza.key.serde");
  String partitionByMsgSerde = jobConfig.get("streams.jobName-jobId-partition_by-p1.samza.msg.serde");
  assertTrue("Serialized serdes should not contain intermediate stream key serde",
      !deserializedSerdes.containsKey(partitionByKeySerde));
  assertTrue("Serialized serdes should not contain intermediate stream msg serde",
      !deserializedSerdes.containsKey(partitionByMsgSerde));
}
 
Example 4
Source Project: samza   Source File: TestExecutionPlanner.java    License: Apache License 2.0 6 votes vote down vote up
private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamStreamJoin() {
  /**
   * Creates the following stream-stream join which is invalid due to partition count disagreement
   * between the 2 input streams.
   *
   *   input1 (64) --
   *                 |
   *                join -> output1 (8)
   *                 |
   *   input3 (32) --
   */
  return new StreamApplicationDescriptorImpl(appDesc -> {
    MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
    MessageStream<KV<Object, Object>> messageStream3 = appDesc.getInputStream(input3Descriptor);
    OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);

    messageStream1
        .join(messageStream3,
            mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(2), "j1")
        .sendTo(output1);
  }, config);
}
 
Example 5
Source Project: samza   Source File: StreamApplicationDescriptorImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) {
  SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor();
  Optional<StreamExpander> expander = systemDescriptor.getExpander();
  if (expander.isPresent()) {
    return expander.get().apply(this, inputDescriptor);
  }

  // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream
  addInputDescriptor(inputDescriptor);

  String streamId = inputDescriptor.getStreamId();
  Serde serde = inputDescriptor.getSerde();
  KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde);
  boolean isKeyed = serde instanceof KVSerde;
  InputTransformer transformer = inputDescriptor.getTransformer().orElse(null);
  InputOperatorSpec inputOperatorSpec =
      OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(),
          transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null));
  inputOperators.put(streamId, inputOperatorSpec);
  return new MessageStreamImpl(this, inputOperators.get(streamId));
}
 
Example 6
Source Project: samza   Source File: StreamApplicationDescriptorImpl.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Internal helper for {@link MessageStreamImpl} to add an intermediate {@link MessageStream} to the graph.
 * An intermediate {@link MessageStream} is both an output and an input stream.
 *
 * @param streamId the id of the stream to be created.
 * @param serde the {@link Serde} to use for the message in the intermediate stream. If null, the default serde
 *              is used.
 * @param isBroadcast whether the stream is a broadcast stream.
 * @param <M> the type of messages in the intermediate {@link MessageStream}
 * @return  the intermediate {@link MessageStreamImpl}
 */
@VisibleForTesting
public <M> IntermediateMessageStreamImpl<M> getIntermediateStream(String streamId, Serde<M> serde, boolean isBroadcast) {
  Preconditions.checkNotNull(serde, "serde must not be null for intermediate stream: " + streamId);
  Preconditions.checkState(!inputOperators.containsKey(streamId) && !outputStreams.containsKey(streamId),
      "getIntermediateStream must not be called multiple times with the same streamId: " + streamId);

  if (isBroadcast) {
    intermediateBroadcastStreamIds.add(streamId);
  }

  boolean isKeyed = serde instanceof KVSerde;
  KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde);

  InputTransformer transformer = (InputTransformer) getDefaultSystemDescriptor()
      .flatMap(SystemDescriptor::getTransformer).orElse(null);

  InputOperatorSpec inputOperatorSpec =
      OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(),
          transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null));
  inputOperators.put(streamId, inputOperatorSpec);
  outputStreams.put(streamId, new OutputStreamImpl(streamId, kvSerdes.getKey(), kvSerdes.getValue(), isKeyed));
  return new IntermediateMessageStreamImpl<>(this, inputOperators.get(streamId), outputStreams.get(streamId));
}
 
Example 7
Source Project: samza   Source File: TestStreamApplicationDescriptorImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetIntermediateStreamWithValueSerde() {
  String streamId = "stream-1";
  StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig());

  Serde mockValueSerde = mock(Serde.class);
  IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl =
      streamAppDesc.getIntermediateStream(streamId, mockValueSerde, false);

  assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec());
  assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream());
  assertEquals(streamId, intermediateStreamImpl.getStreamId());
  assertTrue(intermediateStreamImpl.getOutputStream().getKeySerde() instanceof NoOpSerde);
  assertEquals(mockValueSerde, intermediateStreamImpl.getOutputStream().getValueSerde());
  assertTrue(((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getKeySerde() instanceof NoOpSerde);
  assertEquals(mockValueSerde, ((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getValueSerde());
}
 
Example 8
Source Project: samza   Source File: CoordinatorStreamStore.java    License: Apache License 2.0 6 votes vote down vote up
private void readMessagesFromCoordinatorStream() {
  synchronized (bootstrapLock) {
    while (iterator.hasNext()) {
      IncomingMessageEnvelope envelope = iterator.next();
      byte[] keyAsBytes = (byte[]) envelope.getKey();
      Serde<List<?>> serde = new JsonSerde<>();
      Object[] keyArray = serde.fromBytes(keyAsBytes).toArray();
      CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>());
      String namespacedKey = serializeCoordinatorMessageKeyToJson(coordinatorStreamMessage.getType(), coordinatorStreamMessage.getKey());
      if (envelope.getMessage() != null) {
        messagesReadFromCoordinatorStream.put(namespacedKey, (byte[]) envelope.getMessage());
      } else {
        messagesReadFromCoordinatorStream.remove(namespacedKey);
      }
    }
  }
}
 
Example 9
Source Project: samza   Source File: TestLargeMessageSafeStore.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSmallMessagePutWithSerdeAndDropLargeMessageDisabled() {
  LargeMessageSafeStore largeMessageSafeKeyValueStore = new LargeMessageSafeStore(store, storeName, false, maxMessageSize);

  Serde<Long> longSerde = new LongSerde();
  long longObj = 1000L;
  byte[] key = longSerde.toBytes(longObj);

  JsonSerdeV2<Map<String, Object>> jsonSerde = new JsonSerdeV2<>();
  Map<String, Object> obj = new HashMap<>();
  obj.put("jack", "jill");
  obj.put("john", 2);
  byte[] smallMessage = jsonSerde.toBytes(obj);

  List<Entry<byte[], byte[]>> entries = new ArrayList<>();
  entries.add(new Entry<>(key, smallMessage));

  largeMessageSafeKeyValueStore.putAll(entries);
  Mockito.verify(store).putAll(Matchers.eq(entries));
  Mockito.verify(store, Mockito.never()).put(Matchers.any(byte[].class), Matchers.any(byte[].class));
}
 
Example 10
Source Project: samza   Source File: MessageStreamImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public <K, OM, JM> MessageStream<JM> join(MessageStream<OM> otherStream,
    JoinFunction<? extends K, ? super M, ? super OM, ? extends JM> joinFn,
    Serde<K> keySerde, Serde<M> messageSerde, Serde<OM> otherMessageSerde,
    Duration ttl, String userDefinedId) {
  if (otherStream.equals(this)) throw new SamzaException("Cannot join a MessageStream with itself.");
  String opId = this.streamAppDesc.getNextOpId(OpCode.JOIN, userDefinedId);
  OperatorSpec<?, OM> otherOpSpec = ((MessageStreamImpl<OM>) otherStream).getOperatorSpec();
  JoinOperatorSpec<K, M, OM, JM> op =
      OperatorSpecs.createJoinOperatorSpec(this.operatorSpec, otherOpSpec, (JoinFunction<K, M, OM, JM>) joinFn, keySerde,
          messageSerde, otherMessageSerde, ttl.toMillis(), opId);
  this.operatorSpec.registerNextOperatorSpec(op);
  otherOpSpec.registerNextOperatorSpec((OperatorSpec<OM, ?>) op);

  return new MessageStreamImpl<>(this.streamAppDesc, op);
}
 
Example 11
Source Project: samza   Source File: TestStreamApplicationDescriptorImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetInputStreamWithValueSerde() {

  String streamId = "test-stream-1";
  Serde mockValueSerde = mock(Serde.class);
  GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
  GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mockValueSerde);
  StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
    appDesc.getInputStream(isd);
  }, getConfig());

  InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId);
  assertEquals(OpCode.INPUT, inputOpSpec.getOpCode());
  assertEquals(streamId, inputOpSpec.getStreamId());
  assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId));
  assertTrue(inputOpSpec.getKeySerde() instanceof NoOpSerde);
  assertEquals(mockValueSerde, inputOpSpec.getValueSerde());
}
 
Example 12
Source Project: samza-hello-samza   Source File: StreamTableJoinExample.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
  Serde<Profile> profileSerde = new JsonSerdeV2<>(Profile.class);
  Serde<PageView> pageViewSerde = new JsonSerdeV2<>(PageView.class);
  Serde<EnrichedPageView> joinResultSerde = new JsonSerdeV2<>(EnrichedPageView.class);

  KafkaSystemDescriptor kafkaSystemDescriptor = new KafkaSystemDescriptor(KAFKA_SYSTEM_NAME)
      .withConsumerZkConnect(KAFKA_CONSUMER_ZK_CONNECT)
      .withProducerBootstrapServers(KAFKA_PRODUCER_BOOTSTRAP_SERVERS)
      .withDefaultStreamConfigs(KAFKA_DEFAULT_STREAM_CONFIGS);

  KafkaInputDescriptor<Profile> profileInputDescriptor =
      kafkaSystemDescriptor.getInputDescriptor(PROFILE_STREAM_ID, profileSerde);
  KafkaInputDescriptor<PageView> pageViewInputDescriptor =
      kafkaSystemDescriptor.getInputDescriptor(PAGEVIEW_STREAM_ID, pageViewSerde);
  KafkaOutputDescriptor<EnrichedPageView> joinResultOutputDescriptor =
      kafkaSystemDescriptor.getOutputDescriptor(OUTPUT_TOPIC, joinResultSerde);

  RocksDbTableDescriptor<String, Profile> profileTableDescriptor =
      new RocksDbTableDescriptor<String, Profile>("profile-table", KVSerde.of(new StringSerde(), profileSerde));

  appDescriptor.withDefaultSystem(kafkaSystemDescriptor);

  MessageStream<Profile> profileStream = appDescriptor.getInputStream(profileInputDescriptor);
  MessageStream<PageView> pageViewStream = appDescriptor.getInputStream(pageViewInputDescriptor);
  OutputStream<EnrichedPageView> joinResultStream = appDescriptor.getOutputStream(joinResultOutputDescriptor);
  Table<KV<String, Profile>> profileTable = appDescriptor.getTable(profileTableDescriptor);

  profileStream
      .map(profile -> KV.of(profile.userId, profile))
      .sendTo(profileTable);

  pageViewStream
      .partitionBy(pv -> pv.userId, pv -> pv, KVSerde.of(new StringSerde(), pageViewSerde), "join")
      .join(profileTable, new JoinFn())
      .sendTo(joinResultStream);
}
 
Example 13
Source Project: samza   Source File: TestStreamApplicationDescriptorImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalStateException.class)
public void testSetDefaultSystemDescriptorAfterGettingOutputStream() {
  String streamId = "test-stream-1";
  GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
  GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mock(Serde.class));
  new StreamApplicationDescriptorImpl(appDesc -> {
    appDesc.getOutputStream(osd);
    appDesc.withDefaultSystem(sd); // should throw exception
  }, getConfig());
}
 
Example 14
Source Project: samza   Source File: WindowInternal.java    License: Apache License 2.0 5 votes vote down vote up
public WindowInternal(Trigger<M> defaultTrigger, SupplierFunction<WV> initializer, FoldLeftFunction<M, WV> foldLeftFunction,
    MapFunction<M, WK> keyExtractor, MapFunction<M, Long> eventTimeExtractor, WindowType windowType, Serde<WK> keySerde,
    Serde<WV> windowValueSerde, Serde<M> msgSerde) {
  this.defaultTrigger = defaultTrigger;
  this.initializer = initializer;
  this.foldLeftFunction = foldLeftFunction;
  this.eventTimeExtractor = eventTimeExtractor;
  this.keyExtractor = keyExtractor;
  this.windowType = windowType;
  this.keySerde = keySerde;
  this.windowValSerde = windowValueSerde;
  this.msgSerde = msgSerde;

  if (defaultTrigger == null) {
    throw new IllegalArgumentException("A window must not have a null default trigger");
  }

  if (msgSerde == null && windowValueSerde == null) {
    throw new IllegalArgumentException("A window must not have a null msg serde and a null windowValue serde");
  }

  if (foldLeftFunction != null && windowValSerde == null) {
    throw new IllegalArgumentException("A window with a FoldLeftFunction must have a windowValue serde");
  }

  if (foldLeftFunction != null && initializer == null) {
    throw new IllegalArgumentException("A window with a FoldLeftFunction must have an initializer");
  }

  if (foldLeftFunction == null && initializer != null) {
    throw new IllegalArgumentException("A window without a provided FoldLeftFunction must not have an initializer");
  }
}
 
Example 15
Source Project: samza   Source File: TestStreamApplicationDescriptorImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetIntermediateStreamWithDefaultSystemDescriptor() {
  Config mockConfig = getConfig();
  String streamId = "streamId";

  StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, mockConfig);
  GenericSystemDescriptor sd = new GenericSystemDescriptor("mock-system", "mock-system-factory");
  streamAppDesc.withDefaultSystem(sd);
  IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl =
      streamAppDesc.getIntermediateStream(streamId, mock(Serde.class), false);

  assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec());
  assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream());
  assertEquals(streamId, intermediateStreamImpl.getStreamId());
}
 
Example 16
Source Project: samza   Source File: TestJobNodeConfigurationGenerator.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGenerateJobConfigWithTaskApplication() {
  // set the application to TaskApplication, which still wire up all input/output/intermediate streams
  TaskApplicationDescriptorImpl taskAppDesc = new TaskApplicationDescriptorImpl(getTaskApplication(), mockConfig);
  configureJobNode(taskAppDesc);
  // create the JobGraphConfigureGenerator and generate the jobConfig for the jobNode
  JobNodeConfigurationGenerator configureGenerator = new JobNodeConfigurationGenerator();
  JobConfig jobConfig = configureGenerator.generateJobConfig(mockJobNode, "testJobGraphJson");

  // Verify the results
  Config expectedJobConfig = getExpectedJobConfig(mockConfig, mockJobNode.getInEdges());
  validateJobConfig(expectedJobConfig, jobConfig);
  Map<String, Serde> deserializedSerdes = validateAndGetDeserializedSerdes(jobConfig, 2);
  validateStreamConfigures(jobConfig, deserializedSerdes);
}
 
Example 17
Source Project: samza   Source File: TestOperatorSpec.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testJoinOperatorSpec() {

  InputOperatorSpec leftOpSpec = new InputOperatorSpec(
      "test-input-1", new NoOpSerde<>(), new NoOpSerde<>(), null, false, "op0");
  InputOperatorSpec rightOpSpec = new InputOperatorSpec(
      "test-input-2", new NoOpSerde<>(), new NoOpSerde<>(), null, false, "op1");

  Serde<Object> objSerde = new Serde<Object>() {

    @Override
    public Object fromBytes(byte[] bytes) {
      return null;
    }

    @Override
    public byte[] toBytes(Object object) {
      return new byte[0];
    }
  };

  JoinFunction<String, Object, Object, TestOutputMessageEnvelope> joinFn = new TestJoinFunction();
  JoinOperatorSpec<String, Object, Object, TestOutputMessageEnvelope> joinOperatorSpec =
      new JoinOperatorSpec<>(leftOpSpec, rightOpSpec, joinFn, new StringSerde("UTF-8"), objSerde, objSerde, 50000, "op2");
  JoinOperatorSpec<String, Object, Object, TestOutputMessageEnvelope> joinOpCopy =
      (JoinOperatorSpec<String, Object, Object, TestOutputMessageEnvelope>) OperatorSpecTestUtils.copyOpSpec(joinOperatorSpec);
  assertNotEquals("Expected deserialized copy of operator spec should not be the same as the original operator spec", joinOperatorSpec, joinOpCopy);
  assertTrue(joinOperatorSpec.isClone(joinOpCopy));
  assertTrue(joinOpCopy.getLeftInputOpSpec().isClone(leftOpSpec));
  assertTrue(joinOpCopy.getRightInputOpSpec().isClone(rightOpSpec));
}
 
Example 18
Source Project: samza   Source File: ContainerStorageManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create taskStores for all stores in storageEngineFactories.
 * The store mode is chosen as bulk-load if its a non-sideinput store, and readWrite if its a sideInput store
 */
private Map<TaskName, Map<String, StorageEngine>> createTaskStores(ContainerModel containerModel, JobContext jobContext, ContainerContext containerContext,
    Map<String, StorageEngineFactory<Object, Object>> storageEngineFactories, Map<String, Serde<Object>> serdes,
    Map<TaskName, TaskInstanceMetrics> taskInstanceMetrics,
    Map<TaskName, TaskInstanceCollector> taskInstanceCollectors) {

  Map<TaskName, Map<String, StorageEngine>> taskStores = new HashMap<>();

  // iterate over each task in the containerModel, and each store in storageEngineFactories
  for (Map.Entry<TaskName, TaskModel> task : containerModel.getTasks().entrySet()) {
    TaskName taskName = task.getKey();
    TaskModel taskModel = task.getValue();

    if (!taskStores.containsKey(taskName)) {
      taskStores.put(taskName, new HashMap<>());
    }

    for (String storeName : storageEngineFactories.keySet()) {

      StorageEngineFactory.StoreMode storeMode = this.taskSideInputStoreSSPs.get(taskName).containsKey(storeName) ?
          StorageEngineFactory.StoreMode.ReadWrite : StorageEngineFactory.StoreMode.BulkLoad;

      StorageEngine storageEngine =
          createStore(storeName, taskName, taskModel, jobContext, containerContext, storageEngineFactories, serdes, taskInstanceMetrics, taskInstanceCollectors, storeMode);

      // add created store to map
      taskStores.get(taskName).put(storeName, storageEngine);

      LOG.info("Created store {} for task {} in mode {}", storeName, taskName, storeMode);
    }
  }

  return taskStores;
}
 
Example 19
Source Project: samza   Source File: StreamDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructs a {@link StreamDescriptor} instance.
 *
 * @param streamId id of the stream
 * @param serde serde for messages in the stream
 * @param systemDescriptor system descriptor this stream descriptor was obtained from
 */
StreamDescriptor(String streamId, Serde serde, SystemDescriptor systemDescriptor) {
  Preconditions.checkArgument(systemDescriptor != null,
      String.format("SystemDescriptor must not be null. streamId: %s", streamId));
  String systemName = systemDescriptor.getSystemName();
  Preconditions.checkState(isValidStreamId(streamId),
      String.format("streamId must be non-empty and must not contain spaces or special characters. " +
          "streamId: %s, systemName: %s", streamId, systemName));
  Preconditions.checkArgument(serde != null,
      String.format("Serde must not be null. streamId: %s systemName: %s", streamId, systemName));
  this.streamId = streamId;
  this.serde = serde;
  this.systemDescriptor = systemDescriptor;
}
 
Example 20
Source Project: samza   Source File: TestExecutionPlanner.java    License: Apache License 2.0 5 votes vote down vote up
private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoin() {
  /**
   * Example stream-table join that is invalid due to disagreement in partition count
   * between the 2 input streams.
   *
   *    input1 (64) -> send-to-table t
   *
   *                   join-table t -> output1 (8)
   *                         |
   *    input2 (16) —————————
   *
   */
  return new StreamApplicationDescriptorImpl(appDesc -> {
    MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
    MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
    OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);

    TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor(
      "table-id", new KVSerde(new StringSerde(), new StringSerde()));
    Table table = appDesc.getTable(tableDescriptor);

    messageStream1.sendTo(table);

    messageStream1
        .join(table, mock(StreamTableJoinFunction.class))
        .join(messageStream2,
            mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2")
        .sendTo(output1);
  }, config);
}
 
Example 21
Source Project: samza   Source File: InputDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructs an {@link InputDescriptor} instance.
 *
 * @param streamId id of the stream
 * @param serde serde for messages in the stream
 * @param systemDescriptor system descriptor this stream descriptor was obtained from
 * @param transformer stream level input stream transform function if available, else null
 */
public InputDescriptor(String streamId, Serde serde, SystemDescriptor systemDescriptor, InputTransformer transformer) {
  super(streamId, serde, systemDescriptor);

  // stream level transformer takes precedence over system level transformer
  if (transformer != null) {
    this.transformerOptional = Optional.of(transformer);
  } else {
    this.transformerOptional = systemDescriptor.getTransformer();
  }
}
 
Example 22
Source Project: samza   Source File: TestExecutionPlanner.java    License: Apache License 2.0 5 votes vote down vote up
private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoin() {
  /**
   * Example stream-table join app. Expected partition counts of intermediate streams introduced
   * by partitionBy operations are enclosed in quotes.
   *
   *    input2 (16) -> partitionBy ("32") -> send-to-table t
   *
   *                                      join-table t —————
   *                                       |                |
   *    input1 (64) -> partitionBy ("32") _|                |
   *                                                       join -> output1 (8)
   *                                                        |
   *                                      input3 (32) ——————
   *
   */
  return new StreamApplicationDescriptorImpl(appDesc -> {
    MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
    MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
    MessageStream<KV<Object, Object>> messageStream3 = appDesc.getInputStream(input3Descriptor);
    OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);

    TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor(
        "table-id", new KVSerde(new StringSerde(), new StringSerde()));
    Table table = appDesc.getTable(tableDescriptor);

    messageStream2
        .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1")
        .sendTo(table);

    messageStream1
        .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p2")
        .join(table, mock(StreamTableJoinFunction.class))
        .join(messageStream3,
              mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2")
        .sendTo(output1);
  }, config);
}
 
Example 23
Source Project: samza   Source File: TestWindowOperatorSpec.java    License: Apache License 2.0 5 votes vote down vote up
private WindowOperatorSpec getWindowOperatorSpec(String opId) {
  WindowInternal<Object, Object, Collection> window = new WindowInternal<Object, Object, Collection>(
      defaultTrigger, supplierFunction, foldFn, keyFn, timeFn, WindowType.SESSION, null,
      mock(Serde.class), mock(Serde.class));
  if (earlyTrigger != null) {
    window.setEarlyTrigger(earlyTrigger);
  }
  if (lateTrigger != null) {
    window.setLateTrigger(lateTrigger);
  }
  return new WindowOperatorSpec<>(window, opId);
}
 
Example 24
Source Project: samza   Source File: TestStreamApplicationDescriptorImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalStateException.class)
public void testGetSameIntermediateStreamTwice() {
  StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig());
  streamAppDesc.getIntermediateStream("test-stream-1", mock(Serde.class), false);
  // should throw exception
  streamAppDesc.getIntermediateStream("test-stream-1", mock(Serde.class), false);
}
 
Example 25
Source Project: samza   Source File: TestJobNodeConfigurationGenerator.java    License: Apache License 2.0 5 votes vote down vote up
private void validateTableConfigure(JobConfig jobConfig, Map<String, Serde> deserializedSerdes,
    TableDescriptor tableDescriptor) {
  Config tableConfig = jobConfig.subset(String.format("tables.%s.", tableDescriptor.getTableId()));
  assertEquals(MockTableProviderFactory.class.getName(), tableConfig.get("provider.factory"));
  assertEquals("mock.config.value", jobConfig.get("mock.table.provider.config"));
  validateTableSerdeConfigure(tableDescriptor.getTableId(), jobConfig, deserializedSerdes);
}
 
Example 26
Source Project: samza   Source File: TestCouchbaseTableReadFunction.java    License: Apache License 2.0 5 votes vote down vote up
private <V> CouchbaseTableReadFunction<V> createAndInit(Class<V> valueClass, Serde<V> serde, Bucket bucket,
    AsyncBucket asyncBucket) {
  when(bucket.async()).thenReturn(asyncBucket);
  PowerMockito.stub(PowerMockito.method(CouchbaseBucketRegistry.class, "getBucket", String.class, List.class,
      CouchbaseEnvironmentConfigs.class)).toReturn(bucket);
  CouchbaseTableReadFunction<V> readFunction =
      new CouchbaseTableReadFunction<>(DEFAULT_BUCKET_NAME, valueClass, DEFAULT_CLUSTER_NODE).withSerde(serde);
  readFunction.init(mock(Context.class), mock(AsyncReadWriteTable.class));
  return readFunction;
}
 
Example 27
Source Project: samza   Source File: TestJobNodeConfigurationGenerator.java    License: Apache License 2.0 5 votes vote down vote up
private void validateTableSerdeConfigure(String tableId, Config config, Map<String, Serde> deserializedSerdes) {
  Config streamConfig = config.subset(String.format("stores.%s.", tableId));
  String keySerdeName = streamConfig.get("key.serde");
  String valueSerdeName = streamConfig.get("msg.serde");
  assertTrue(String.format("Serialized serdes should contain %s key serde", tableId), deserializedSerdes.containsKey(keySerdeName));
  assertTrue(String.format("Serialized %s key serde should be a StringSerde", tableId), keySerdeName.startsWith(StringSerde.class.getSimpleName()));
  assertTrue(String.format("Serialized serdes should contain %s value serde", tableId), deserializedSerdes.containsKey(valueSerdeName));
  assertTrue(String.format("Serialized %s msg serde should be a JsonSerdeV2", tableId), valueSerdeName.startsWith(JsonSerdeV2.class.getSimpleName()));
}
 
Example 28
Source Project: samza   Source File: EventHubsOutputDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructs an {@link OutputDescriptor} instance.
 *
 * @param streamId id of the stream
 * @param namespace namespace for the Event Hubs entity to produce to, not null
 * @param entityPath entity path for the Event Hubs entity to produce to, not null
 * @param valueSerde serde the values in the messages in the stream
 * @param systemDescriptor system descriptor this stream descriptor was obtained from
 */
EventHubsOutputDescriptor(String streamId, String namespace, String entityPath, Serde valueSerde,
    SystemDescriptor systemDescriptor) {
  super(streamId, KVSerde.of(new NoOpSerde<>(), valueSerde), systemDescriptor);
  this.namespace = StringUtils.stripToNull(namespace);
  this.entityPath = StringUtils.stripToNull(entityPath);
  if (this.namespace == null || this.entityPath == null) {
    throw new ConfigException(String.format("Missing namespace and entity path Event Hubs output descriptor in "
        + "system: {%s}, stream: {%s}", getSystemName(), streamId));
  }
}
 
Example 29
Source Project: samza   Source File: TestOperatorImplGraph.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBroadcastChain() {
  String inputStreamId = "input";
  String inputSystem = "input-system";
  String inputPhysicalName = "input-stream";
  HashMap<String, String> configMap = new HashMap<>();
  configMap.put(JobConfig.JOB_NAME, "test-job");
  configMap.put(JobConfig.JOB_ID, "1");
  StreamTestUtils.addStreamConfigs(configMap, inputStreamId, inputSystem, inputPhysicalName);
  Config config = new MapConfig(configMap);
  when(this.context.getJobContext().getConfig()).thenReturn(config);
  StreamApplicationDescriptorImpl graphSpec = new StreamApplicationDescriptorImpl(appDesc -> {
    GenericSystemDescriptor sd = new GenericSystemDescriptor(inputSystem, "mockFactoryClass");
    GenericInputDescriptor inputDescriptor = sd.getInputDescriptor(inputStreamId, mock(Serde.class));
    MessageStream<Object> inputStream = appDesc.getInputStream(inputDescriptor);
    inputStream.filter(mock(FilterFunction.class));
    inputStream.map(mock(MapFunction.class));
  }, config);

  OperatorImplGraph opImplGraph =
      new OperatorImplGraph(graphSpec.getOperatorSpecGraph(), this.context, mock(Clock.class));

  InputOperatorImpl inputOpImpl = opImplGraph.getInputOperator(new SystemStream(inputSystem, inputPhysicalName));
  assertEquals(2, inputOpImpl.registeredOperators.size());
  assertTrue(inputOpImpl.registeredOperators.stream()
      .anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.FILTER));
  assertTrue(inputOpImpl.registeredOperators.stream()
      .anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.MAP));
}
 
Example 30
Source Project: samza   Source File: TestZkLocalApplicationRunner.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public StorageEngine getStorageEngine(String storeName,
    File storeDir,
    Serde<Object> keySerde,
    Serde<Object> msgSerde,
    MessageCollector collector,
    MetricsRegistry registry,
    SystemStreamPartition changeLogSystemStreamPartition,
    JobContext jobContext,
    ContainerContext containerContext, StoreMode storeMode) {
  StoreProperties storeProperties = new StoreProperties.StorePropertiesBuilder().setLoggedStore(false).build();
  return new MockStorageEngine(storeName, storeDir, changeLogSystemStreamPartition, storeProperties);
}