org.pentaho.di.trans.Trans Java Examples

@Override
public void transFinished( Trans trans ) throws KettleException {
  // Copy over the data from the previewDelegate...
  //
  if ( trans.getErrors() != 0 ) {
    // capture logging and store it...
    //
    for ( StepMetaDataCombi combi : trans.getSteps() ) {
      if ( combi.copy == 0 ) {
        StringBuffer logBuffer =
          KettleLogStore.getAppender().getBuffer( combi.step.getLogChannel().getLogChannelId(), false );
        previewLogMap.put( combi.stepMeta, logBuffer );
      }
    }
  }
}
 

@Test
public void executorsInputIsStraightlyCopiedToOutput() throws Exception {
  TransExecutorMeta executorMeta = getExecutorMeta( transExecutor );
  executorMeta.setExecutorsOutputStepMeta( dummy );

  Trans trans = createTrans( transMeta );
  RowStepCollector endRc = listenExecutor( trans );
  RowProducer rp = trans.addRowProducer( injector.getName(), 0 );

  trans.startThreads();

  RowMetaAndData testInput = new RowMetaAndData( createRowMetaForOneField(), SAMPLE_INPUT );
  rp.putRow( testInput.getRowMeta(), testInput.getData() );
  rp.finished();

  trans.waitUntilFinished();

  assertEquals( testInput.size(), endRc.getRowsWritten().size() );
  assertThat( asList( endRc.getRowsWritten().get( 0 ).getData() ),
    hasItem( (Object) SAMPLE_INPUT )
  );
}
 

private boolean testRepository( Repository repository ) {
  ExecutorService executorService = ExecutorUtil.getExecutor();
  Future<Boolean> future = executorService.submit( () -> {
    ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
    try {
      Thread.currentThread().setContextClassLoader( Trans.class.getClassLoader() );
      return ( (AbstractRepository) repository ).test();
    } finally {
      Thread.currentThread().setContextClassLoader( currentClassLoader );
    }
  } );

  try {
    return future.get();
  } catch ( InterruptedException | ExecutionException e ) {
    return false;
  }
}
 

public static Trans loadAndRunTransformation( String path, Object... parameters ) throws Exception {
  TransMeta transMeta = new TransMeta( path );
  transMeta.setTransformationType( TransMeta.TransformationType.Normal );

  Trans trans = new Trans( transMeta );
  if ( parameters != null ) {
    if ( parameters.length % 2 == 1 ) {
      throw new IllegalArgumentException( "Parameters should be an array of pairs 'parameter'-'value'-..." );
    }

    for ( int i = 0; i < parameters.length; i += 2 ) {
      Object parameter = parameters[ i ];
      Object value = parameters[i + 1];
      trans.setParameterValue( parameter.toString(), value.toString() );
    }
  }

  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();

  return trans;
}
 

public String getStatus() {
  String message;

  if ( isActive() ) {
    if ( isStopped() ) {
      message = Trans.STRING_HALTING;
    } else {
      message = Trans.STRING_RUNNING;
    }
  } else if ( isFinished() ) {
    message = Trans.STRING_FINISHED;
    if ( getResult().getNrErrors() > 0 ) {
      message += " (with errors)";
    }
  } else if ( isStopped() ) {
    message = Trans.STRING_STOPPED;
    if ( getResult().getNrErrors() > 0 ) {
      message += " (with errors)";
    }
  } else {
    message = Trans.STRING_WAITING;
  }

  return message;
}
 

@Test
@PrepareForTest( { Encode.class } )
public void testCleanupTransServletEscapesHtmlWhenTransFound() throws ServletException, IOException {
  HttpServletRequest mockHttpServletRequest = mock( HttpServletRequest.class );
  HttpServletResponse mockHttpServletResponse = mock( HttpServletResponse.class );
  Trans mockTrans = mock( Trans.class );
  StringWriter out = new StringWriter();
  PrintWriter printWriter = new PrintWriter( out );
  PowerMockito.spy( Encode.class );
  when( mockHttpServletRequest.getContextPath() ).thenReturn( CleanupTransServlet.CONTEXT_PATH );
  when( mockHttpServletRequest.getParameter( anyString() ) ).thenReturn( ServletTestUtils.BAD_STRING_TO_TEST );
  when( mockHttpServletResponse.getWriter() ).thenReturn( printWriter );
  when( mockTransformationMap.getTransformation( any( CarteObjectEntry.class ) ) ).thenReturn( mockTrans );

  cleanupTransServlet.doGet( mockHttpServletRequest, mockHttpServletResponse );
  assertFalse( ServletTestUtils.hasBadText( ServletTestUtils.getInsideOfTag( "H1", out.toString() ) ) );
  PowerMockito.verifyStatic( atLeastOnce() );
  Encode.forHtml( anyString() );
}
 

@Before
public void setUp() {
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "delete1" );

  Map<String, String> vars = new HashMap<String, String>();
  vars.put( "max.sz", "10" );
  transMeta.injectVariables( vars );

  dmi = new DeleteMeta();
  dd = new DeleteData();

  PluginRegistry plugReg = PluginRegistry.getInstance();
  String deletePid = plugReg.getPluginId( StepPluginType.class, dmi );

  stepMeta = new StepMeta( deletePid, "delete", dmi );
  Trans trans = new Trans( transMeta );
  transMeta.addStep( stepMeta );
  del = new Delete( stepMeta, dd, 1, transMeta, trans );
  del.copyVariablesFrom( transMeta );
}
 

public static Trans generateTestTransformation() {
  RowGeneratorMeta A = new RowGeneratorMeta();
  A.allocate( 3 );
  A.setRowLimit( "10000000" );

  A.getFieldName()[0] = "ID";
  A.getFieldType()[0] = ValueMetaFactory.getValueMetaName( ValueMetaInterface.TYPE_INTEGER );
  A.getFieldLength()[0] = 7;
  A.getValue()[0] = "1234";

  A.getFieldName()[1] = "Name";
  A.getFieldType()[1] = ValueMetaFactory.getValueMetaName( ValueMetaInterface.TYPE_STRING );
  A.getFieldLength()[1] = 35;
  A.getValue()[1] = "Some name";

  A.getFieldName()[2] = "Last updated";
  A.getFieldType()[2] = ValueMetaFactory.getValueMetaName( ValueMetaInterface.TYPE_DATE );
  A.getFieldFormat()[2] = "yyyy/MM/dd";
  A.getValue()[2] = "2010/02/09";

  TransMeta transMeta = TransPreviewFactory.generatePreviewTransformation( null, A, "A" );
  transMeta.setName( "CarteUnitTest" );
  transMeta.setSizeRowset( 2500 );
  transMeta.setFeedbackSize( 50000 );
  transMeta.setUsingThreadPriorityManagment( false );

  return new Trans( transMeta );
}
 

private Trans buildHexadecimalChecksumTrans( int checkSumType, boolean compatibilityMode,
    boolean oldChecksumBehaviour, String fieldSeparatorString, String[] fieldNames ) throws Exception {
  // Create a new transformation...
  TransMeta transMeta = new TransMeta();
  transMeta.setName( getClass().getName() );

  // Create a CheckSum Step
  String checkSumStepname = "CheckSum";
  CheckSumMeta meta = new CheckSumMeta();

  // Set the compatibility mode and other required fields
  meta.setCompatibilityMode( compatibilityMode );
  meta.setResultFieldName( "hex" );
  meta.setCheckSumType( checkSumType );
  meta.setResultType( CheckSumMeta.result_TYPE_HEXADECIMAL );
  meta.setFieldName( fieldNames );
  meta.setOldChecksumBehaviour( oldChecksumBehaviour );
  meta.setFieldSeparatorString( fieldSeparatorString );

  String checkSumPluginPid = PluginRegistry.getInstance().getPluginId( StepPluginType.class, meta );
  StepMeta checkSumStep = new StepMeta( checkSumPluginPid, checkSumStepname, meta );
  transMeta.addStep( checkSumStep );

  // Create a Dummy step
  String dummyStepname = "Output";
  DummyTransMeta dummyMeta = new DummyTransMeta();
  String dummyStepPid = PluginRegistry.getInstance().getPluginId( StepPluginType.class, dummyMeta );
  StepMeta dummyStep = new StepMeta( dummyStepPid, dummyStepname, dummyMeta );
  transMeta.addStep( dummyStep );

  // Create a hop from CheckSum to Output
  TransHopMeta hop = new TransHopMeta( checkSumStep, dummyStep );
  transMeta.addTransHop( hop );

  return new Trans( transMeta );
}
 

@Test
public void testTransWaitsForListenersToFinish() throws Exception {
  doReturn( new String[] { } ).when( transMeta ).getPrevStepNames( any( StepMeta.class ) );
  data.stepInjectionMetasMap = new HashMap<>();
  data.stepInjectionMap = new HashMap<>();
  data.transMeta = new TransMeta();
  meta.setNoExecution( false );
  Trans injectTrans = mock( Trans.class );
  doReturn( injectTrans ).when( metaInject ).createInjectTrans();
  when( injectTrans.isFinished() ).thenReturn( true );
  Result result = mock( Result.class );
  when( injectTrans.getResult() ).thenReturn( result );
  metaInject.processRow( meta, data );
  verify( injectTrans ).waitUntilFinished();
}
 

public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int cnr,
  TransMeta transMeta, Trans trans ) {
  UserDefinedJavaClass userDefinedJavaClass =
    new UserDefinedJavaClass( stepMeta, stepDataInterface, cnr, transMeta, trans );
  if ( trans.hasHaltedSteps() ) {
    return null;
  }

  return userDefinedJavaClass;
}
 

static void setSystemScopeVariable( Trans trans, final String variableName, final String variableValue ) {
  System.setProperty( variableName, variableValue );

  // Set also all the way to the root as else we will take
  //  stale values
  setRootScopeVariable( trans, variableName, variableValue );
}
 

private synchronized void preview() {
  CypherMeta oneMeta = new CypherMeta();
  this.getInfo(oneMeta);
  TransMeta previewMeta = TransPreviewFactory.generatePreviewTransformation(this.transMeta, oneMeta, this.wStepname.getText());
  this.transMeta.getVariable("Internal.Transformation.Filename.Directory");
  previewMeta.getVariable("Internal.Transformation.Filename.Directory");
  EnterNumberDialog
    numberDialog = new EnterNumberDialog(this.shell, this.props.getDefaultPreviewSize(),
    BaseMessages.getString(PKG, "CypherDialog.PreviewSize.DialogTitle"),
    BaseMessages.getString(PKG, "CypherDialog.PreviewSize.DialogMessage")
  );
  int previewSize = numberDialog.open();
  if (previewSize > 0) {
    TransPreviewProgressDialog progressDialog = new TransPreviewProgressDialog(this.shell, previewMeta, new String[]{this.wStepname.getText()}, new int[]{previewSize});
    progressDialog.open();
    Trans trans = progressDialog.getTrans();
    String loggingText = progressDialog.getLoggingText();
    if (!progressDialog.isCancelled() && trans.getResult() != null && trans.getResult().getNrErrors() > 0L) {
      EnterTextDialog etd = new EnterTextDialog(this.shell,
        BaseMessages.getString(PKG, "System.Dialog.PreviewError.Title", new String[0]),
        BaseMessages.getString(PKG, "System.Dialog.PreviewError.Message", new String[0]), loggingText, true);
      etd.setReadOnly();
      etd.open();
    }

    PreviewRowsDialog prd = new PreviewRowsDialog(this.shell, this.transMeta, 0, this.wStepname.getText(), progressDialog.getPreviewRowsMeta(this.wStepname.getText()), progressDialog.getPreviewRows(this.wStepname.getText()), loggingText);
    prd.open();
  }
}
 

@Test
public void testCreateMultivalueMap() {
  StepMeta stepMeta = new StepMeta();
  stepMeta.setName( "TestRest" );
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "TestRest" );
  transMeta.addStep( stepMeta );
  Rest rest = new Rest( stepMeta, mock( StepDataInterface.class ),
    1, transMeta, mock( Trans.class ) );
  MultivaluedMapImpl map = rest.createMultivalueMap( "param1", "{a:{[val1]}}" );
  String val1 = map.getFirst( "param1" );
  assertTrue( val1.contains( "%7D" ) );
}
 

@Test
public void setParentScopeVariable_ParentIsTrans() {
  Trans parent = createTrans();
  Trans child = createTrans( parent );

  ScriptValuesAddedFunctions.setParentScopeVariable( child, VARIABLE_NAME, VARIABLE_VALUE );

  verify( child ).setVariable( eq( VARIABLE_NAME ), eq( VARIABLE_VALUE ) );
  verify( parent ).setVariable( eq( VARIABLE_NAME ), eq( VARIABLE_VALUE ) );
}
 

/**
 If the transformation has at least one step in a transformation,
 which writes it's data straight to a servlet output
 we should wait transformation's termination.
 Otherwise the servlet's response lifecycle may come to an end and
 the response will be closed by container while
 the transformation will be still trying writing data into it.
 */
@VisibleForTesting
void finishProcessing( Trans trans, PrintWriter out ) {
  if ( trans.getSteps().stream().anyMatch( step -> step.meta.passDataToServletOutput() ) ) {
    trans.waitUntilFinished();
  } else {
    WebResult webResult = new WebResult( WebResult.STRING_OK, "Transformation started", trans.getContainerObjectId() );
    out.println( webResult.getXML() );
    out.flush();
  }
}
 

@Test
public void testExecuteTrans() {
  try {
    Trans trans = KettleUtils.executeTrans("第一课作业", rep, false);
    while (trans.isRunning()) {
      System.out.println(trans.getStatus());
      Thread.sleep(1000);
    }
    System.err.println(trans.getStatus());
  } catch (Exception e) {
    e.printStackTrace();
  }
}
 

@Test
public void setRootScopeVariable_TwoLevelHierarchy() {
  Trans parent = createTrans( );
  Trans child = createTrans( parent );

  ScriptValuesAddedFunctions.setRootScopeVariable( child, VARIABLE_NAME, VARIABLE_VALUE );

  verify( child ).setVariable( eq( VARIABLE_NAME ), eq( VARIABLE_VALUE ) );
  verify( parent ).setVariable( eq( VARIABLE_NAME ), eq( VARIABLE_VALUE ) );
}
 

void initTransFromMeta() throws KettleException {
  // Create the transformation from meta-data...
  //
  getData().setMappingTrans( new Trans( getData().mappingTransMeta, this ) );

  if ( getData().mappingTransMeta.getTransformationType() != TransformationType.Normal ) {
    getData().getMappingTrans().getTransMeta().setUsingThreadPriorityManagment( false );
  }

  // Leave a path up so that we can set variables in sub-transformations...
  //
  getData().getMappingTrans().setParentTrans( getTrans() );

  // Pass down the safe mode flag to the mapping...
  //
  getData().getMappingTrans().setSafeModeEnabled( getTrans().isSafeModeEnabled() );

  // Pass down the metrics gathering flag:
  //
  getData().getMappingTrans().setGatheringMetrics( getTrans().isGatheringMetrics() );

  // Also set the name of this step in the mapping transformation for logging
  // purposes
  //
  getData().getMappingTrans().setMappingStepName( getStepname() );

  initServletConfig();

  // Set the parameters values in the mapping.
  //

  MappingParameters mappingParameters = meta.getMappingParameters();
  if ( mappingParameters != null ) {
    StepWithMappingMeta
      .activateParams( data.mappingTrans, data.mappingTrans, this, data.mappingTransMeta.listParameters(),
        mappingParameters.getVariable(), mappingParameters.getInputField(), meta.getMappingParameters().isInheritingAllVariables() );
  }

}
 

public Trans loadTransFromRepository( Repository repository, String dirName, String transName ) throws Exception {

    if ( Utils.isEmpty( transName ) ) {
      System.out.println( BaseMessages.getString( getPkgClazz(), "Pan.Error.NoTransNameSupplied" ) );
      return null;
    }

    RepositoryDirectoryInterface directory = loadRepositoryDirectory( repository, dirName, "Pan.Error.NoRepProvided",
            "Pan.Log.Allocate&ConnectRep", "Pan.Error.CanNotFindSpecifiedDirectory" );

    if ( directory == null ) {
      return null; // not much we can do here
    }

    // Add the IMetaStore of the repository to our delegation
    if ( repository.getMetaStore() != null && getMetaStore() != null ) {
      getMetaStore().addMetaStore( repository.getMetaStore() );
    }

    logDebug( "Pan.Log.LoadTransInfo" );
    TransMeta transMeta = repository.loadTransformation( transName, directory, null, true, null );

    logDebug( "Pan.Log.AllocateTrans" );
    Trans trans = new Trans( transMeta );
    trans.setRepository( repository );
    trans.setMetaStore( getMetaStore() );

    return trans; // return transformation loaded from the repo
  }
 

@Test
public void testGetDataFromFolderRecursively() throws KettleException {
  KettleEnvironment.init();
  String path = getClass().getResource( "text-file-input-get-data-from-folder-step.ktr" ).getPath();
  Variables variables = new Variables();
  variables.setVariable( "testfolder", getClass().getResource( "" ).getPath() );
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 14, trans.getSteps().get( 0 ).step.getLinesWritten() );
  assertEquals( 21, trans.getSteps().get( 0 ).step.getLinesInput() );
}
 

@Before
public void setUp() throws Exception {
  databaseMeta = mock( DatabaseMeta.class );
  doReturn( "" ).when( databaseMeta ).quoteField( anyString() );

  dimensionLookupMeta = mock( DimensionLookupMeta.class );
  doReturn( databaseMeta ).when( dimensionLookupMeta ).getDatabaseMeta();
  doReturn( new String[]{} ).when( dimensionLookupMeta ).getKeyLookup();
  doReturn( new String[]{} ).when( dimensionLookupMeta ).getFieldLookup();
  doReturn( new int[]{} ).when( dimensionLookupMeta ).getFieldUpdate();

  stepMeta = mock( StepMeta.class );
  doReturn( "step" ).when( stepMeta ).getName();
  doReturn( mock( StepPartitioningMeta.class ) ).when( stepMeta ).getTargetStepPartitioningMeta();
  doReturn( dimensionLookupMeta ).when( stepMeta ).getStepMetaInterface();

  Database db = mock( Database.class );
  doReturn( mock( Connection.class ) ).when( db ).getConnection();

  dimensionLookupData = mock( DimensionLookupData.class );
  dimensionLookupData.db = db;
  dimensionLookupData.keynrs = new int[] { };
  dimensionLookupData.fieldnrs = new int[] { };

  TransMeta transMeta = mock( TransMeta.class );
  doReturn( stepMeta ).when( transMeta ).findStep( anyString() );

  dimensionLookup = new DimensionLookup( stepMeta, dimensionLookupData, 1, transMeta, mock( Trans.class ) );
  dimensionLookup.setData( dimensionLookupData );
  dimensionLookup.setMeta( dimensionLookupMeta );
  dimensionLookupSpy = spy( dimensionLookup );
  doReturn( stepMeta ).when( dimensionLookupSpy ).getStepMeta();
  doReturn( false ).when( dimensionLookupSpy ).isRowLevel();
  doReturn( false ).when( dimensionLookupSpy ).isDebug();
  doReturn( true ).when( dimensionLookupSpy ).isAutoIncrement();
  doNothing().when( dimensionLookupSpy ).logDetailed( anyString() );
}
 

@Test
public void setGrandParentScopeVariable_TwoLevelHierarchy() {
  Trans parent = createTrans( );
  Trans child = createTrans( parent );

  ScriptValuesAddedFunctions.setGrandParentScopeVariable( child, VARIABLE_NAME, VARIABLE_VALUE );

  verify( child ).setVariable( eq( VARIABLE_NAME ), eq( VARIABLE_VALUE ) );
  verify( parent ).setVariable( eq( VARIABLE_NAME ), eq( VARIABLE_VALUE ) );
}
 

private Trans createTrans( Trans parent ) {
  Trans trans = createTrans();

  trans.setParent( parent );
  trans.setParentVariableSpace( parent );

  return trans;
}
 

public XMLJoin( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans ) {
  super( stepMeta, stepDataInterface, copyNr, transMeta, trans );
}
 

public RowsFromResult( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta,
  Trans trans ) {
  super( stepMeta, stepDataInterface, copyNr, transMeta, trans );

  data = (RowsFromResultData) stepDataInterface;
}
 

@Before
public void setUp() throws Exception {
  transMeta = new TransMeta( "src/it/resources/org/pentaho/di/trans/steps/denormaliser/pdi-11947.ktr" );
  transMeta.setTransformationType( TransMeta.TransformationType.Normal );
  trans = new Trans( transMeta );
}
 

public DetectLastRow( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta,
  Trans trans ) {
  super( stepMeta, stepDataInterface, copyNr, transMeta, trans );
}
 

@Override
public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int cnr, TransMeta transMeta,
    Trans trans ) {
  return new Normaliser( stepMeta, stepDataInterface, cnr, transMeta, trans );
}
 

public final StepInterface getStep( final StepMeta stepMeta, final StepDataInterface stepDataInterface,
    final int cnr, final TransMeta transMeta, final Trans trans ) {

  return new PaloDimOutput( stepMeta, stepDataInterface, cnr, transMeta, trans );
}