org.pentaho.di.trans.TransMeta Java Examples

@Before
public void setUp() throws Exception {
  meta = new MappingInputMeta();
  meta.setFieldName( new String[] { "n2", "n4" } );
  meta.setFieldType( new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_INTEGER } );
  meta.setFieldLength( new int[] { 0, 0 } );
  meta.setFieldPrecision( new int[] { 0, 0 } );

  StepMeta sm = new StepMeta( "MappingInput", "SubTrans", meta );
  TransMeta tm = new TransMeta();
  tm.addStep( sm );
  LoggingObjectInterface loi = new SimpleLoggingObject( "lo", LoggingObjectType.STEP, null );
  Trans tr = new Trans( tm, loi );

  step = new MappingInput( sm, null, 0, tm, tr );
  step.getTrans().setRunning( true );
}
 

/**
 * @param parent
 * @param in
 * @param tr
 * @param sname
 */
public IngresVectorwiseLoaderDialog( Shell parent, Object in, TransMeta tr, String sname ) {
  super( parent, (BaseStepMeta) in, tr, sname );
  input = (IngresVectorwiseLoaderMeta) in;

  lsMod = new ModifyListener() {
    public void modifyText( ModifyEvent e ) {
      input.setChanged();
    }
  };

  lsSelMod = new SelectionAdapter() {
    public void widgetSelected( SelectionEvent arg0 ) {
      input.setChanged();
    }
  };
}
 

public T createMeta() {
  try {
    T meta = clazz.newInstance();
    if ( meta instanceof BaseStepMeta ) {
      StepMeta mockParentStepMeta = mock( StepMeta.class );
      ( (BaseStepMeta) meta ).setParentStepMeta( mockParentStepMeta );
      TransMeta mockTransMeta = mock( TransMeta.class );
      NamedClusterEmbedManager embedManager = mock( NamedClusterEmbedManager.class );
      when( mockParentStepMeta.getParentTransMeta() ).thenReturn( mockTransMeta );
      when( mockTransMeta.getNamedClusterEmbedManager() ).thenReturn( embedManager );
    }
    return meta;
  } catch ( Exception e ) {
    throw new RuntimeException( "Unable to create meta of class " + clazz.getCanonicalName(), e );
  }
}
 

public static String[] getMappingSteps( TransMeta mappingTransMeta, boolean mappingInput ) {
  List<StepMeta> steps = new ArrayList<StepMeta>();
  for ( StepMeta stepMeta : mappingTransMeta.getSteps() ) {
    if ( mappingInput && stepMeta.getStepID().equals( "MappingInput" ) ) {
      steps.add( stepMeta );
    }
    if ( !mappingInput && stepMeta.getStepID().equals( "MappingOutput" ) ) {
      steps.add( stepMeta );
    }
  }
  String[] stepnames = new String[ steps.size() ];
  for ( int i = 0; i < stepnames.length; i++ ) {
    stepnames[ i ] = steps.get( i ).getName();
  }

  return stepnames;

}
 

@Before
protected void setUp() throws Exception {
  super.setUp();
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  transMeta = new TransMeta();
  transMeta.setName( "csvinput1" );

  registry = PluginRegistry.getInstance();

  fileName = writeInputFile();

  injectorStep = createInjectorStep( transMeta, registry );
  csvInputStep = createCsvInputStep( transMeta, registry, "\"", false );
}
 

public ValidatorDialog( Shell parent, Object in, TransMeta tr, String sname ) {
  super( parent, (BaseStepMeta) in, tr, sname );
  input = (ValidatorMeta) in;

  // Just to make sure everything is nicely in sync...
  //
  java.util.List<StreamInterface> infoStreams = input.getStepIOMeta().getInfoStreams();
  for ( int i = 0; i < infoStreams.size(); i++ ) {
    input.getValidations().get( i ).setSourcingStepName( infoStreams.get( i ).getStepname() );
  }

  selectedField = null;
  selectionList = new ArrayList<Validation>();

  // Copy the data from the input into the map...
  //
  for ( Validation field : input.getValidations() ) {
    selectionList.add( field.clone() );
  }
}
 

private TransMeta copyCleanTransMeta( TransMeta transMeta, Repository repository, IMetaStore metaStore, VariableSpace space ) throws KettleException {

    try {
      String transMetaXml = transMeta.getXML();
      TransMeta copy = new TransMeta();
      copy.setMetaStore( metaStore );
      copy.loadXML( XMLHandler.loadXMLString( transMetaXml, TransMeta.XML_TAG ), null, metaStore, repository, true, space, null );

      for ( StepMeta stepMeta : copy.getSteps() ) {
        stepMeta.setCopies( 1 );

        DummyTransMeta dummyTransMeta = new DummyTransMeta();

        // Replace all stepMeta with a Dummy with the same name
        //
        stepMeta.setStepID( "Dummy" );
        stepMeta.setStepMetaInterface( dummyTransMeta );
      }

      return copy;

    } catch ( Exception e ) {
      throw new KettleException( "Error copying/cleaning transformation metadata", e );
    }
  }
 

public void getFields( RowMetaInterface row, String name, RowMetaInterface[] info, StepMeta nextStep,
    VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {

  ValueMetaInterface v = new ValueMeta( this.getValueXMLfield(), ValueMetaInterface.TYPE_STRING );
  v.setOrigin( name );

  TransMeta transMeta = (TransMeta) space;
  try {
    // Row should only include fields from the target and not the source. During the preview table generation
    // the fields from all previous steps (source and target) are included in the row so lets remove the
    // source fields.
    List<String> targetFieldNames = null;
    RowMetaInterface targetRowMeta = transMeta.getStepFields( transMeta.findStep( getTargetXMLstep() ),
      null,
      null );
    if ( targetRowMeta != null ) {
      targetFieldNames = Arrays.asList( targetRowMeta.getFieldNames() );
    }
    for ( String fieldName : transMeta.getStepFields( transMeta.findStep( getSourceXMLstep() ),
                                                                          null,
                                                                          null ).getFieldNames() ) {
      if ( targetFieldNames == null || !targetFieldNames.contains( fieldName ) ) {
        row.removeValueMeta( fieldName );
      }
    }
  } catch ( KettleValueException e ) {
    // Pass
  }

  row.addValueMeta( v );
}
 

private TransMeta createTransMeta() throws KettleException {
  TransMeta transMeta = new TransMeta();
  transMeta.setName( UUID.randomUUID().toString() );
  transMeta.setFilename( UUID.randomUUID().toString() );
  RepositoryDirectory repositoryDirectory = mock( RepositoryDirectory.class );
  doCallRealMethod().when( repositoryDirectory ).setName( anyString() );
  doCallRealMethod().when( repositoryDirectory ).getName();
  transMeta.setRepositoryDirectory( repositoryDirectory );
  initSharedObjects( transMeta, SHARED_OBJECTS_FILE );
  when( spoon.getActiveMeta() ).thenReturn( transMeta );
  return transMeta;
}
 

private static Set<String> getUsedStepsForReferencendTransformation( TransMeta transMeta ) {
  Set<String> usedStepNames = new HashSet<String>();
  for ( StepMeta currentStep : transMeta.getUsedSteps() ) {
    usedStepNames.add( currentStep.getName().toUpperCase() );
  }
  return usedStepNames;
}
 

@Test
public void testTransToXMLFileSaveFileAsFailed() throws Exception {
  TransMeta mockTransMeta = mock( TransMeta.class );

  //passing a invalid type so not running GUIResource class
  prepareSetSaveTests( spoon, log, mockSpoonPerspective, mockTransMeta, true, true, "NotMainSpoonPerspective", true,
      false, "Invalid TYPE", null, true, true );

  doCallRealMethod().when( spoon ).saveFileAs( mockTransMeta );
  assertFalse( spoon.saveFileAs( mockTransMeta ) );
  verify( mockTransMeta ).setRepository( spoon.rep );
  verify( mockTransMeta ).setMetaStore( spoon.metaStore );

  verify( spoon ).enableMenus();
}
 

@VisibleForTesting
TransMeta filterPrivateDatabases( TransMeta transMeta ) {
  Set<String> privateDatabases = transMeta.getPrivateDatabases();
  if ( privateDatabases != null ) {
    // keep only private transformation databases
    for ( Iterator<DatabaseMeta> it = transMeta.getDatabases().iterator(); it.hasNext(); ) {
      DatabaseMeta databaseMeta = it.next();
      String databaseName = databaseMeta.getName();
      if ( !privateDatabases.contains( databaseName ) && !transMeta.isDatabaseConnectionUsed( databaseMeta ) ) {
        it.remove();
      }
    }
  }
  return transMeta;
}
 

public void newPartitioningSchema( TransMeta transMeta ) {
  PartitionSchema partitionSchema = new PartitionSchema();

  PartitionSchemaDialog dialog =
      new PartitionSchemaDialog( spoon.getShell(), partitionSchema, transMeta.getPartitionSchemas(), transMeta
          .getDatabases(), transMeta );
  if ( dialog.open() ) {
    List<PartitionSchema> partitions = transMeta.getPartitionSchemas();
    if ( isDuplicate( partitions, partitionSchema ) ) {
      new ErrorDialog(
        spoon.getShell(), getMessage( "Spoon.Dialog.ErrorSavingPartition.Title" ), getMessage(
        "Spoon.Dialog.ErrorSavingPartition.Message", partitionSchema.getName() ),
        new KettleException( getMessage( "Spoon.Dialog.ErrorSavingPartition.NotUnique" ) ) );
      return;
    }

    partitions.add( partitionSchema );

    if ( spoon.rep != null ) {
      try {
        if ( !spoon.rep.getSecurityProvider().isReadOnly() ) {
          spoon.rep.save( partitionSchema, Const.VERSION_COMMENT_INITIAL_VERSION, null );
          if ( sharedObjectSyncUtil != null ) {
            sharedObjectSyncUtil.reloadTransformationRepositoryObjects( false );
          }
        } else {
          throw new KettleException( BaseMessages.getString(
            PKG, "Spoon.Dialog.Exception.ReadOnlyRepositoryUser" ) );
        }
      } catch ( KettleException e ) {
        showSaveErrorDialog( partitionSchema, e );
      }
    }

    refreshTree();
  }
}
 

public TransMetaPipelineConverter( TransMeta transMeta, IMetaStore metaStore, List<String> stepPluginClasses, List<String> xpPluginClasses, BeamJobConfig beamJobConfig ) throws MetaStoreException {
  this();
  this.transMeta = transMeta;
  this.metaStore = new SerializableMetaStore( metaStore );
  this.metaStoreJson = this.metaStore.toJson();
  this.stepPluginClasses = stepPluginClasses;
  this.xpPluginClasses = xpPluginClasses;
  this.beamJobConfig = beamJobConfig;

  addDefaultStepHandlers();
}
 

public static Trans getTrans( final Configuration conf, final String transXml, boolean singleThreaded )
  throws KettleException {
  initKettleEnvironment( conf );

  TransConfiguration transConfiguration = TransConfiguration.fromXML( transXml );
  TransMeta transMeta = transConfiguration.getTransMeta();
  String carteObjectId = UUID.randomUUID().toString();
  SimpleLoggingObject servletLoggingObject =
    new SimpleLoggingObject( "HADOOP_MAPPER", LoggingObjectType.CARTE, null ); //$NON-NLS-1$
  servletLoggingObject.setContainerObjectId( carteObjectId );
  TransExecutionConfiguration executionConfiguration = transConfiguration.getTransExecutionConfiguration();
  servletLoggingObject.setLogLevel( executionConfiguration.getLogLevel() );

  if ( singleThreaded ) {
    // Set the type to single threaded in case the user forgot...
    //
    transMeta.setTransformationType( TransformationType.SingleThreaded );

    // Disable thread priority management as it will slow things down needlessly.
    // The single threaded engine doesn't use threads and doesn't need row locking.
    //
    transMeta.setUsingThreadPriorityManagment( false );
  } else {
    transMeta.setTransformationType( TransformationType.Normal );
  }

  return new Trans( transMeta, servletLoggingObject );
}
 

/**
 * Retrieve the list of database names from MongoDB based on what the user entered for hostname, port,etc.
 * NOTE: Much of this could move to the MongoDbInputData class, as it is copied almost verbatim from the
 * Spoon MongoDbInputDialog class.
 *
 * @return Vector<String> list of database names
 * @throws Exception Should anything go wrong connecting to MongoDB, it will be reported with this exception
 */
public Vector<String> getDatabaseNamesFromMongo() throws Exception {
  Vector<String> dbs = new Vector<String>();

  if ( Const.isEmpty( hostname ) ) {
    log.logBasic( "Fetching database names aborted. Missing hostname." );
    return dbs;
  }

  final MongoDbInputMeta meta = new MongoDbInputMeta();
  final TransMeta transMeta = new TransMeta();
  saveMeta( meta );
  try {
    MongoClientWrapper wrapper = MongoWrapperUtil.createMongoClientWrapper( meta, transMeta, log );
    List<String> dbNames = null;
    try {
      dbNames = wrapper.getDatabaseNames();
    } finally {
      wrapper.dispose();
    }

    for ( String s : dbNames ) {
      dbs.add( s );
    }
    return dbs;
  } catch ( Exception e ) {
    log.logError( "Unexpected error retrieving database names from MongoDb. Check your connection details.", meta );
    throw new MongoDbException(
        "Unexpected error retrieving database names from MongoDb. Check your connection details.", e );
  }
}
 

public JaninoDialog( Shell parent, Object in, TransMeta tr, String sname ) {
  super( parent, (BaseStepMeta) in, tr, sname );

  // The order here is important... currentMeta is looked at for changes
  currentMeta = (JaninoMeta) in;
  originalMeta = (JaninoMeta) currentMeta.clone();
  inputFields = new HashMap<String, Integer>();
}
 

@Test
public void synchronizePartitionSchemasDeleteFromRepository() throws Exception {
  try {
    spoon.rep = repository;
    when( spoon.getRepository() ).thenReturn( repository );

    final String objectId = "object-id";
    final String partitionName = "partsch";

    TransMeta trans1 = createTransMeta();
    trans1.setRepository( repository );
    trans1.setSharedObjects( createSharedObjects( SHARED_OBJECTS_FILE ) );
    PartitionSchema part1 = createPartitionSchema( partitionName, false );
    part1.setObjectId( new StringObjectId( objectId ) );
    trans1.addOrReplacePartitionSchema( part1 );
    spoon.delegates.trans.addTransformation( trans1 );

    TransMeta trans2 = createTransMeta();
    trans2.setRepository( repository );
    trans2.setSharedObjects( createSharedObjects( SHARED_OBJECTS_FILE ) );
    PartitionSchema part2 = createPartitionSchema( partitionName, false );
    part2.setObjectId( new StringObjectId( objectId ) );
    trans2.addOrReplacePartitionSchema( part2 );
    spoon.delegates.trans.addTransformation( trans2 );

    assertFalse( trans1.getPartitionSchemas().isEmpty() );
    spoon.delegates.partitions.delPartitionSchema( trans2, part2 );
    verify( repository ).deletePartitionSchema( part2.getObjectId() );
    assertTrue( trans1.getPartitionSchemas().isEmpty() );
  } finally {
    spoon.rep = null;
    when( spoon.getRepository() ).thenReturn( null );
  }
}
 

public void clearInputDataSet() {
  Spoon spoon = ( (Spoon) SpoonFactory.getInstance() );
  TransGraph transGraph = spoon.getActiveTransGraph();
  TransMeta transMeta = spoon.getActiveTransformation();
  StepMeta stepMeta = transGraph.getCurrentStep();
  if ( transGraph == null || transMeta == null || stepMeta == null ) {
    return;
  }
  if ( checkTestPresent( spoon, transMeta ) ) {
    return;
  }

  try {
    TransUnitTest currentUnitTest = getCurrentUnitTest( transMeta );

    TransUnitTestSetLocation inputLocation = currentUnitTest.findInputLocation( stepMeta.getName() );
    if ( inputLocation != null ) {
      currentUnitTest.getInputDataSets().remove( inputLocation );
    }

    saveUnitTest( getHierarchy().getTestFactory(), currentUnitTest, transMeta );

    transGraph.redraw();
  } catch ( Exception e ) {
    new ErrorDialog( spoon.getShell(), "Error", "Error saving unit test", e );
  }
}
 

@Before
public void setup() {
  shell = mock( Shell.class );
  meta = mock( CsvInputMeta.class );
  transMeta = mock( TransMeta.class );
  monitor = mock( IProgressMonitor.class );
}
 

public void testMapping_WhenSharingPreviousStepWithAnother() throws Exception {
  KettleEnvironment.init();

  TransMeta transMeta = new TransMeta( "src/it/resources/org/pentaho/di/trans/steps/mapping/pdi-13435/PDI-13435-main.ktr" );
  transMeta.setTransformationType( TransMeta.TransformationType.Normal );

  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();

  assertEquals( 0, trans.getErrors() );
}
 

@Test
public void filterPrivateDatabasesWithOnePrivateDatabaseAndOneInUseTest() {
  IUnifiedRepository purMock = mock( IUnifiedRepository.class );
  TransMeta transMeta = spy( TransMeta.class );
  transMeta.setDatabases( getDummyDatabases() );
  Set<String> privateDatabases = new HashSet<>(  );
  privateDatabases.add( "database2" );
  transMeta.setPrivateDatabases( privateDatabases );
  when( transMeta.isDatabaseConnectionUsed( getDummyDatabases().get( 0 ) ) ).thenReturn( true );
  StreamToTransNodeConverter transConverter = new StreamToTransNodeConverter( purMock );
  assertEquals( 2, transConverter.filterPrivateDatabases( transMeta ).getDatabases().size() );
}
 

void createRowGenerator(
  TransMeta transMeta, PluginRegistry registry,
  String stepName, String[] values,
  StepMeta mergeRowsStep, MergeRowsMeta mergeRowsMeta,
  int index
) {
  RowGeneratorMeta rowGeneratorMeta = new RowGeneratorMeta();
  String rowGeneratorPid = registry.getPluginId( StepPluginType.class, rowGeneratorMeta );
  StepMeta rowGeneratorStep = new StepMeta( rowGeneratorPid, stepName, rowGeneratorMeta );
  transMeta.addStep( rowGeneratorStep );

  rowGeneratorMeta.setDefault();
  rowGeneratorMeta.setFieldName( fieldName );
  rowGeneratorMeta.setFieldType( type );
  rowGeneratorMeta.setFieldLength( intDummies );
  rowGeneratorMeta.setFieldPrecision( intDummies );
  rowGeneratorMeta.setRowLimit( "1" );
  rowGeneratorMeta.setFieldFormat( fieldFormat );
  rowGeneratorMeta.setGroup( group );
  rowGeneratorMeta.setDecimal( decimal );
  rowGeneratorMeta.setCurrency( currency );
  rowGeneratorMeta.setEmptyString( setEmptystring );

  rowGeneratorMeta.setValue( values );

  TransHopMeta hi1 = new TransHopMeta( rowGeneratorStep, mergeRowsStep );
  transMeta.addTransHop( hi1 );

  List<StreamInterface> infoStreams = mergeRowsMeta.getStepIOMeta().getInfoStreams();
  StreamInterface infoStream = infoStreams.get( index );
  infoStream.setStepMeta( transMeta.findStep( stepName ) );
}
 

public Neo4JOutputDialog( Shell parent, Object in, TransMeta transMeta, String sname ) {
  super( parent, (BaseStepMeta) in, transMeta, sname );
  input = (Neo4JOutputMeta) in;

  // Hack the metastore...
  //
  metaStore = Spoon.getInstance().getMetaStore();
}
 

/**
 * creates a ComboVar populated with fields from the previous step.
 * @param parentComposite - the composite in which the widget will be placed
 * @param props - PropsUI props for L&F
 * @param stepMeta - stepMeta of the current step
 * @param formData - FormData to use for placement
 */
public static ComboVar createFieldDropDown(
  Composite parentComposite, PropsUI props, BaseStepMeta stepMeta, FormData formData ) {
  TransMeta transMeta = stepMeta.getParentStepMeta().getParentTransMeta();
  ComboVar fieldDropDownCombo = new ComboVar( transMeta, parentComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
  props.setLook( fieldDropDownCombo );
  fieldDropDownCombo.addModifyListener( e -> stepMeta.setChanged() );

  fieldDropDownCombo.setLayoutData( formData );
  Listener focusListener = e -> {
    String current = fieldDropDownCombo.getText();
    fieldDropDownCombo.getCComboWidget().removeAll();
    fieldDropDownCombo.setText( current );

    try {
      RowMetaInterface rmi = transMeta.getPrevStepFields( stepMeta.getParentStepMeta().getName() );
      List ls = rmi.getValueMetaList();
      for ( Object l : ls ) {
        ValueMetaBase vmb = (ValueMetaBase) l;
        fieldDropDownCombo.add( vmb.getName() );
      }
    } catch ( KettleStepException ex ) {
      // can be ignored, since previous step may not be set yet.
      stepMeta.logDebug( ex.getMessage(), ex );
    }
  };
  fieldDropDownCombo.getCComboWidget().addListener( SWT.FocusIn, focusListener );
  return fieldDropDownCombo;
}
 

public HTTPPOST( StepMeta stepMeta, StepDataInterface stepDataInterface,
                 int copyNr, TransMeta transMeta, Trans trans ) {
  super( stepMeta, stepDataInterface, copyNr, transMeta, trans );
}
 

@Override
public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int cnr, TransMeta tr,
  Trans trans ) {
  return new CsvInput( stepMeta, stepDataInterface, cnr, tr, trans );
}
 

@Override
public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int cnr,
  TransMeta transMeta, Trans trans ) {
  return new SecretKeyGenerator( stepMeta, stepDataInterface, cnr, transMeta, trans );
}
 

@Override public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int i, TransMeta transMeta, Trans trans ) {
  return new GraphOutput( stepMeta, stepDataInterface, i, transMeta, trans );
}
 

/**
 * @return Returns the transMeta.
 */
public TransMeta getTransMeta() {
  return transMeta;
}