Java Code Examples for com.datatorrent.api.DAG#addModule()

The following examples show how to use com.datatorrent.api.DAG#addModule() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FSInputModuleAppTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up 
public void populateDAG(DAG dag, Configuration conf)
{
  FSInputModule module = dag.addModule("hdfsInputModule", FSInputModule.class);

  AbstractFileOutputOperator<FileMetadata> metadataWriter = new MetadataWriter(FSInputModuleAppTest.OUT_METADATA_FILE);
  metadataWriter.setFilePath(FSInputModuleAppTest.outputDir);
  dag.addOperator("FileMetadataWriter", metadataWriter);

  AbstractFileOutputOperator<ReaderRecord<Slice>> dataWriter = new HDFSFileWriter(FSInputModuleAppTest.OUT_DATA_FILE);
  dataWriter.setFilePath(FSInputModuleAppTest.outputDir);
  dag.addOperator("FileDataWriter", dataWriter);

  DevNull<FileBlockMetadata> devNull = dag.addOperator("devNull", DevNull.class);

  dag.addStream("FileMetaData", module.filesMetadataOutput, metadataWriter.input);
  dag.addStream("data", module.messages, dataWriter.input);
  dag.addStream("blockMetadata", module.blocksMetadataOutput, devNull.data);
}
Example 2
Source File: TestModuleExpansion.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up 
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  Level1Module m1 = dag.addModule("M1", new Level1Module());
  m1.setMemory(1024);
  m1.setPortMemory(1);
  m1.setLevel1ModuleProp(level2ModuleAProp1);

  Level1Module m2 = dag.addModule("M2", new Level1Module());
  m2.setMemory(2048);
  m2.setPortMemory(2);
  m2.setLevel1ModuleProp(level2ModuleAProp2);

  DummyOperator o1 = dag.addOperator("O1", new DummyOperator());
  o1.setOperatorProp(level2ModuleAProp3);

  dag.addStream("M1_M2&O1", m1.mOut, m2.mIn, o1.in).setLocality(DAG.Locality.CONTAINER_LOCAL);

  mIn.set(m1.mIn);
  mOut1.set(m2.mOut);
  mOut2.set(o1.out1);
}
Example 3
Source File: Application.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up 
@Override
public void populateDAG(DAG dag, Configuration conf)
{

  FSRecordReaderModule recordReader = dag.addModule("recordReader", FSRecordReaderModule.class);
  CsvParser csvParser = dag.addOperator("csvParser", CsvParser.class);
  FilterOperator filterOperator = dag.addOperator("filterOperator", new FilterOperator());

  CsvFormatter selectedFormatter = dag.addOperator("selectedFormatter", new CsvFormatter());
  CsvFormatter rejectedFormatter = dag.addOperator("rejectedFormatter", new CsvFormatter());

  StringFileOutputOperator selectedOutput = dag.addOperator("selectedOutput", new StringFileOutputOperator());
  StringFileOutputOperator rejectedOutput = dag.addOperator("rejectedOutput", new StringFileOutputOperator());

  dag.addStream("record", recordReader.records, csvParser.in);
  dag.addStream("pojo", csvParser.out, filterOperator.input);

  dag.addStream("pojoSelected", filterOperator.truePort, selectedFormatter.in);
  dag.addStream("pojoRejected", filterOperator.falsePort, rejectedFormatter.in);

  dag.addStream("csvSelected", selectedFormatter.out, selectedOutput.input);
  dag.addStream("csvRejected", rejectedFormatter.out, rejectedOutput.input);
}
Example 4
Source File: S3InputModuleAppTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up 
public void populateDAG(DAG dag, Configuration conf)
{
  S3InputModule module = dag.addModule("s3InputModule", S3InputModule.class);

  AbstractFileOutputOperator<AbstractFileSplitter.FileMetadata> metadataWriter = new S3InputModuleAppTest.MetadataWriter(S3InputModuleAppTest.OUT_METADATA_FILE);
  metadataWriter.setFilePath(S3InputModuleAppTest.outputDir);
  dag.addOperator("FileMetadataWriter", metadataWriter);

  AbstractFileOutputOperator<AbstractBlockReader.ReaderRecord<Slice>> dataWriter = new S3InputModuleAppTest.HDFSFileWriter(S3InputModuleAppTest.OUT_DATA_FILE);
  dataWriter.setFilePath(S3InputModuleAppTest.outputDir);
  dag.addOperator("FileDataWriter", dataWriter);

  DevNull<BlockMetadata.FileBlockMetadata> devNull = dag.addOperator("devNull", DevNull.class);

  dag.addStream("FileMetaData", module.filesMetadataOutput, metadataWriter.input);
  dag.addStream("data", module.messages, dataWriter.input);
  dag.addStream("blockMetadata", module.blocksMetadataOutput, devNull.data);
}
Example 5
Source File: S3ToHDFSSyncApplication.java    From examples with Apache License 2.0 5 votes vote down vote up 
@Override
public void populateDAG(DAG dag, Configuration conf)
{

  S3InputModule inputModule = dag.addModule("S3InputModule", new S3InputModule());
  HDFSFileCopyModule outputModule = dag.addModule("HDFSFileCopyModule", new HDFSFileCopyModule());

  dag.addStream("FileMetaData", inputModule.filesMetadataOutput, outputModule.filesMetadataInput);
  dag.addStream("BlocksMetaData", inputModule.blocksMetadataOutput, outputModule.blocksMetadataInput)
      .setLocality(Locality.THREAD_LOCAL);
  dag.addStream("BlocksData", inputModule.messages, outputModule.blockData).setLocality(Locality.THREAD_LOCAL);
}
Example 6
Source File: FSRecordReaderTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up 
public void populateDAG(DAG dag, Configuration conf)
{
  FSRecordReaderModule recordReader = dag.addModule("HDFSRecordReaderModule", FSRecordReaderModule.class);
  recordReader.setMode("FIXED_WIDTH_RECORD");
  FixedWidthValidator validator = dag.addOperator("Validator", new FixedWidthValidator());
  dag.addStream("records", recordReader.records, validator.data);
}
Example 7
Source File: FSRecordReaderTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up 
public void populateDAG(DAG dag, Configuration conf)
{
  FSRecordReaderModule recordReader = dag.addModule("HDFSRecordReaderModule", FSRecordReaderModule.class);
  recordReader.setMode("delimited_record");
  DelimitedValidator validator = dag.addOperator("Validator", new DelimitedValidator());
  dag.addStream("records", recordReader.records, validator.data);
}
Example 8
Source File: S3RecordReaderMockTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up 
public void populateDAG(DAG dag, Configuration conf)
{
  FSRecordReaderModule recordReader = dag.addModule("S3RecordReaderModuleMock", FSRecordReaderModule.class);
  recordReader.setMode("FIXED_WIDTH_RECORD");
  FixedWidthValidator validator = dag.addOperator("Validator", new FixedWidthValidator());
  dag.addStream("records", recordReader.records, validator.data);
}
Example 9
Source File: S3ToHDFSSyncApplication.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up 
@Override
public void populateDAG(DAG dag, Configuration conf)
{

  S3InputModule inputModule = dag.addModule("S3InputModule", new S3InputModule());
  HDFSFileCopyModule outputModule = dag.addModule("HDFSFileCopyModule", new HDFSFileCopyModule());

  dag.addStream("FileMetaData", inputModule.filesMetadataOutput, outputModule.filesMetadataInput);
  dag.addStream("BlocksMetaData", inputModule.blocksMetadataOutput, outputModule.blocksMetadataInput)
      .setLocality(Locality.THREAD_LOCAL);
  dag.addStream("BlocksData", inputModule.messages, outputModule.blockData).setLocality(Locality.THREAD_LOCAL);
}
Example 10
Source File: Application.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up 
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  FSInputModule inputModule = dag.addModule("HDFSInputModule", new FSInputModule());
  S3OutputModule outputModule = dag.addModule("S3OutputModule", new S3OutputModule());

  dag.addStream("FileMetaData", inputModule.filesMetadataOutput, outputModule.filesMetadataInput);
  dag.addStream("BlocksMetaData", inputModule.blocksMetadataOutput, outputModule.blocksMetadataInput)
    .setLocality(DAG.Locality.CONTAINER_LOCAL);
  dag.addStream("BlocksData", inputModule.messages, outputModule.blockData).setLocality(DAG.Locality.CONTAINER_LOCAL);
}
Example 11
Source File: Application.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up 
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  FSRecordReaderModule recordReader = dag.addModule("recordReader", FSRecordReaderModule.class);
  CsvParser csvParser = dag.addOperator("csvParser", CsvParser.class);
  CsvFormatter formatter = dag.addOperator("formatter", new CsvFormatter());
  StringFileOutputOperator fileOutput = dag.addOperator("fileOutput", new StringFileOutputOperator());

  dag.addStream("record", recordReader.records, csvParser.in);
  dag.addStream("pojo", csvParser.out, formatter.in);
  dag.addStream("string", formatter.out, fileOutput.input);
}
Example 12
Source File: S3RecordReaderMockTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up 
public void populateDAG(DAG dag, Configuration conf)
{

  S3RecordReaderModuleMock recordReader = dag.addModule("S3RecordReaderModuleMock", new S3RecordReaderModuleMock());
  recordReader.setMode("delimited_record");
  DelimitedValidator validator = dag.addOperator("Validator", new DelimitedValidator());
  dag.addStream("records", recordReader.records, validator.data);
}
Example 13
Source File: TestModuleExpansion.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up 
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  DummyInputOperator o1 = dag.addOperator("O1", new DummyInputOperator());
  o1.setInputOperatorProp(1);

  DummyOperator o2 = dag.addOperator("O2", new DummyOperator());
  o2.setOperatorProp(2);

  Level2ModuleA ma = dag.addModule("Ma", new Level2ModuleA());
  ma.setLevel2ModuleAProp1(11);
  ma.setLevel2ModuleAProp2(12);
  ma.setLevel2ModuleAProp3(13);

  Level2ModuleB mb = dag.addModule("Mb", new Level2ModuleB());
  mb.setLevel2ModuleBProp1(21);
  mb.setLevel2ModuleBProp2(22);
  mb.setLevel2ModuleBProp3(23);

  Level2ModuleA mc = dag.addModule("Mc", new Level2ModuleA());
  mc.setLevel2ModuleAProp1(31);
  mc.setLevel2ModuleAProp2(32);
  mc.setLevel2ModuleAProp3(33);

  Level2ModuleB md = dag.addModule("Md", new Level2ModuleB());
  md.setLevel2ModuleBProp1(41);
  md.setLevel2ModuleBProp2(42);
  md.setLevel2ModuleBProp3(43);

  Level3Module me = dag.addModule("Me", new Level3Module());

  dag.addStream("O1_O2", o1.out, o2.in, me.mIn);
  dag.addStream("O2_Ma", o2.out1, ma.mIn);
  dag.addStream("Ma_Mb", ma.mOut1, mb.mIn);
  dag.addStream("Ma_Md", ma.mOut2, md.mIn);
  dag.addStream("Mb_Mc", mb.mOut2, mc.mIn);
}
Example 14
Source File: TestModuleExpansion.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up 
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  DummyOperator op = dag.addOperator("O1", new DummyOperator());
  Level2ModuleB m1 = dag.addModule("M1", new Level2ModuleB());
  Level1Module m2 = dag.addModule("M2", new Level1Module());

  dag.addStream("s1", op.out1, m1.mIn);
  dag.addStream("s2", op.out2, m2.mIn);

  mIn.set(op.in);
  mOut1.set(m1.mOut1);
  mOut2.set(m2.mOut);
}
Example 15
Source File: S3RecordReaderModuleAppTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up 
public void populateDAG(DAG dag, Configuration conf)
{
  S3RecordReaderModule recordReader = dag.addModule("S3RecordReaderModule", S3RecordReaderModule.class);
  recordReader.setMode("fixed_width_record");
  FixedWidthValidator validator = dag.addOperator("Validator", new FixedWidthValidator());
  dag.addStream("records", recordReader.records, validator.data);
}
Example 16
Source File: ModuleAppTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up 
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  LOG.debug("Application - PopulateDAG");
  DummyInputOperator dummyInputOperator = dag.addOperator("DummyInputOperator", new DummyInputOperator());
  DummyOperatorAfterInput dummyOperatorAfterInput = dag.addOperator("DummyOperatorAfterInput",
      new DummyOperatorAfterInput());
  Module m1 = dag.addModule("TestModule1", new TestModule());
  Module m2 = dag.addModule("TestModule2", new TestModule());
  DummyOutputOperator dummyOutputOperator = dag.addOperator("DummyOutputOperator", new DummyOutputOperator());
  dag.addStream("Operator To Operator", dummyInputOperator.output, dummyOperatorAfterInput.input);
  dag.addStream("Operator To Module", dummyOperatorAfterInput.output, ((TestModule)m1).moduleInput);
  dag.addStream("Module To Module", ((TestModule)m1).moduleOutput, ((TestModule)m2).moduleInput);
  dag.addStream("Module To Operator", ((TestModule)m2).moduleOutput, dummyOutputOperator.input);
}
Example 17
Source File: HDFSFileCopyApp.java    From examples with Apache License 2.0 5 votes vote down vote up 
@Override
public void populateDAG(DAG dag, Configuration conf)
{

  FSInputModule inputModule = dag.addModule("HDFSInputModule", new FSInputModule());
  HDFSFileCopyModule outputModule = dag.addModule("HDFSFileCopyModule", new HDFSFileCopyModule());

  dag.addStream("FileMetaData", inputModule.filesMetadataOutput, outputModule.filesMetadataInput);
  dag.addStream("BlocksMetaData", inputModule.blocksMetadataOutput, outputModule.blocksMetadataInput)
      .setLocality(Locality.THREAD_LOCAL);
  dag.addStream("BlocksData", inputModule.messages, outputModule.blockData).setLocality(Locality.THREAD_LOCAL);
}
Example 18
Source File: Application.java    From examples with Apache License 2.0 5 votes vote down vote up 
public void populateDAG(DAG dag, Configuration conf)
{
  FSRecordReaderModule recordReader = dag.addModule("lineInput", FSRecordReaderModule.class);
  S3BytesOutputModule s3StringOutputModule = dag.addModule("s3output", S3BytesOutputModule.class);
  dag.addStream("data", recordReader.records, s3StringOutputModule.input);
  
}
Example 19
Source File: Application.java    From examples with Apache License 2.0 5 votes vote down vote up 
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  FSInputModule inputModule = dag.addModule("HDFSInputModule", new FSInputModule());
  S3OutputModule outputModule = dag.addModule("S3OutputModule", new S3OutputModule());

  dag.addStream("FileMetaData", inputModule.filesMetadataOutput, outputModule.filesMetadataInput);
  dag.addStream("BlocksMetaData", inputModule.blocksMetadataOutput, outputModule.blocksMetadataInput)
    .setLocality(DAG.Locality.CONTAINER_LOCAL);
  dag.addStream("BlocksData", inputModule.messages, outputModule.blockData).setLocality(DAG.Locality.CONTAINER_LOCAL);
}
Example 20
Source File: S3RecordReaderModuleAppTest.java    From attic-apex-malhar with Apache License 2.0 4 votes vote down vote up 
public void populateDAG(DAG dag, Configuration conf)
{
  S3RecordReaderModule recordReader = dag.addModule("s3RecordReaderModule", S3RecordReaderModule.class);
  DelimitedValidator validator = dag.addOperator("Validator", new DelimitedValidator());
  dag.addStream("records", recordReader.records, validator.data);
}