Java Code Examples for org.pentaho.di.trans.TransMeta#setRepository()

@Override
public TransMeta loadTransformation( String transname, RepositoryDirectoryInterface repdir,
  ProgressMonitorListener monitor, boolean setInternalVariables, String versionName ) throws KettleException {

  // This is a standard load of a transformation serialized in XML...
  //
  String filename = calcDirectoryName( repdir ) + transname + ".ktr";
  TransMeta transMeta = new TransMeta( filename, this, setInternalVariables );
  transMeta.setRepository( this );
  transMeta.setMetaStore( getMetaStore() );
  transMeta.setFilename( null );
  transMeta.setName( transname );
  transMeta.setObjectId( new StringObjectId( calcObjectId( repdir, transname, EXT_TRANSFORMATION ) ) );

  readDatabases( transMeta, true );
  transMeta.clearChanged();

  return transMeta;
}
 

@Test
public void testIncludesSubTransformationsFromRepository() throws Exception {
  TransMeta parentTransMeta = new TransMeta( getClass().getResource( "trans-meta-converter-parent.ktr" ).getPath() );
  Repository repository = mock( Repository.class );
  TransMeta transMeta = new TransMeta( );
  RepositoryDirectoryInterface repositoryDirectory = new RepositoryDirectory( null, "public");
  String directory = getClass().getResource( "" ).toString().replace( File.separator, "/" );
  when( repository.findDirectory( "public" ) ).thenReturn( repositoryDirectory );
  when( repository.loadTransformation( "trans-meta-converter-sub.ktr", repositoryDirectory, null, true, null ) ).thenReturn( transMeta );
  parentTransMeta.setRepository( repository );
  parentTransMeta.setRepositoryDirectory( repositoryDirectory );
  parentTransMeta.setVariable( "Internal.Entry.Current.Directory", "public" );
  Transformation transformation = TransMetaConverter.convert( parentTransMeta );

  @SuppressWarnings( { "unchecked", "ConstantConditions" } )
  HashMap<String, Transformation> config =
    (HashMap<String, Transformation>) transformation.getConfig( TransMetaConverter.SUB_TRANSFORMATIONS_KEY ).get();
  assertEquals( 1, config.size() );
  assertNotNull( config.get( "public/trans-meta-converter-sub.ktr" ) );
}
 

private void init( Repository repository, String directoryName, boolean includeSubFolders, String nameMask, String exludeNameMask,
    ObjectTypeSelection typeSelection, int itemCount ) throws KettleException {

  VariableSpace vars = new Variables();
  vars.setVariable( "DirName", "/subdir1" );
  vars.setVariable( "IncludeMask", ".*" );
  vars.setVariable( "ExcludeMask", "" );

  GetRepositoryNamesMeta meta = new GetRepositoryNamesMeta();
  meta.setDirectory( new String[] { directoryName } );
  meta.setNameMask( new String[] { nameMask } );
  meta.setExcludeNameMask( new String[] { exludeNameMask } );
  meta.setIncludeSubFolders( new boolean[] { includeSubFolders } );
  meta.setObjectTypeSelection( typeSelection );
  StepMeta stepMeta = new StepMeta( "GetRepoNamesStep", meta );

  TransMeta transMeta = new TransMeta( vars );
  transMeta.setRepository( repository );
  transMeta.addStep( stepMeta );

  GetRepositoryNamesData data = (GetRepositoryNamesData) meta.getStepData();
  GetRepositoryNames step = new GetRepositoryNames( stepMeta, data, 0, transMeta, new Trans( transMeta ) );
  step.init( meta, data );
  assertNotNull( data.list );
  assertEquals( itemCount, data.list.size() );
}
 

private TransMeta buildTransMeta( final RepositoryFile file, final RepositoryDirectoryInterface parentDir,
                                  final NodeRepositoryFileData data, final ObjectRevision revision )
  throws KettleException {
  TransMeta transMeta = new TransMeta();
  transMeta.setName( file.getTitle() );
  transMeta.setFilename( file.getName() );
  transMeta.setDescription( file.getDescription() );
  transMeta.setObjectId( new StringObjectId( file.getId().toString() ) );
  transMeta.setObjectRevision( revision );
  transMeta.setRepository( this );
  transMeta.setRepositoryDirectory( parentDir );
  transMeta.setMetaStore( getMetaStore() );
  readTransSharedObjects( transMeta ); // This should read from the local cache
  transDelegate.dataNodeToElement( data.getNode(), transMeta );
  transMeta.clearChanged();
  return transMeta;
}
 

private void preview() {
  // Create the XML input step
  GetRepositoryNamesMeta oneMeta = new GetRepositoryNamesMeta();
  getInfo( oneMeta );

  TransMeta previewMeta =
    TransPreviewFactory.generatePreviewTransformation( transMeta, oneMeta, wStepname.getText() );
  previewMeta.setRepository( repository );

  EnterNumberDialog numberDialog =
    new EnterNumberDialog( shell, props.getDefaultPreviewSize(), BaseMessages.getString(
      PKG, "GetRepositoryNamesDialog.PreviewSize.DialogTitle" ), BaseMessages.getString(
      PKG, "GetRepositoryNamesDialog.PreviewSize.DialogMessage" ) );
  int previewSize = numberDialog.open();
  if ( previewSize > 0 ) {
    TransPreviewProgressDialog progressDialog =
      new TransPreviewProgressDialog(
        shell, previewMeta, new String[] { wStepname.getText() }, new int[] { previewSize } );
    progressDialog.open();

    if ( !progressDialog.isCancelled() ) {
      Trans trans = progressDialog.getTrans();
      String loggingText = progressDialog.getLoggingText();

      if ( trans.getResult() != null && trans.getResult().getNrErrors() > 0 ) {
        EnterTextDialog etd =
          new EnterTextDialog( shell, BaseMessages.getString( PKG, "System.Dialog.Error.Title" ), BaseMessages
            .getString( PKG, "GetRepositoryNamesDialog.ErrorInPreview.DialogMessage" ), loggingText, true );
        etd.setReadOnly();
        etd.open();
      }

      PreviewRowsDialog prd =
        new PreviewRowsDialog(
          shell, transMeta, SWT.NONE, wStepname.getText(), progressDialog.getPreviewRowsMeta( wStepname
            .getText() ), progressDialog.getPreviewRows( wStepname.getText() ), loggingText );
      prd.open();
    }
  }
}
 

@Test
public void synchronizePartitionSchemasDeleteFromRepository() throws Exception {
  try {
    spoon.rep = repository;
    when( spoon.getRepository() ).thenReturn( repository );

    final String objectId = "object-id";
    final String partitionName = "partsch";

    TransMeta trans1 = createTransMeta();
    trans1.setRepository( repository );
    trans1.setSharedObjects( createSharedObjects( SHARED_OBJECTS_FILE ) );
    PartitionSchema part1 = createPartitionSchema( partitionName, false );
    part1.setObjectId( new StringObjectId( objectId ) );
    trans1.addOrReplacePartitionSchema( part1 );
    spoon.delegates.trans.addTransformation( trans1 );

    TransMeta trans2 = createTransMeta();
    trans2.setRepository( repository );
    trans2.setSharedObjects( createSharedObjects( SHARED_OBJECTS_FILE ) );
    PartitionSchema part2 = createPartitionSchema( partitionName, false );
    part2.setObjectId( new StringObjectId( objectId ) );
    trans2.addOrReplacePartitionSchema( part2 );
    spoon.delegates.trans.addTransformation( trans2 );

    assertFalse( trans1.getPartitionSchemas().isEmpty() );
    spoon.delegates.partitions.delPartitionSchema( trans2, part2 );
    verify( repository ).deletePartitionSchema( part2.getObjectId() );
    assertTrue( trans1.getPartitionSchemas().isEmpty() );
  } finally {
    spoon.rep = null;
    when( spoon.getRepository() ).thenReturn( null );
  }
}
 

/**
 * This test is to ensure that the metadata for the GetXMLData step is preserved when saving to a
 * repository. The test creates a GetXMLData step and saves it to the repository. Then the local
 * data is changed and the step is read back in from the repository. It is then asserted that the
 * field value(s) are equal to what was saved.
 *
 * Test method for
 * {@link org.pentaho.di.trans.steps.getxmldata.GetXMLDataMeta#readRep(org.pentaho.di.repository.Repository,
 * org.pentaho.di.repository.ObjectId, java.util.List, java.util.Map)}
 * . Test method for
 * {@link org.pentaho.di.trans.steps.getxmldata.GetXMLDataMeta#saveRep(org.pentaho.di.repository.Repository,
 * org.pentaho.di.repository.ObjectId, jorg.pentaho.di.repository.ObjectId)}
 * .
 */
@Test
public void testGetXMLDataMetaSaveAndReadRep() {

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "getxmldata1" );

  //
  // Create a Get XML Data step
  //
  String getXMLDataName = "get xml data step";
  GetXMLDataMeta gxdm = new GetXMLDataMeta();

  String getXMLDataPid = registry.getPluginId( StepPluginType.class, gxdm );
  StepMeta getXMLDataStep = new StepMeta( getXMLDataPid, getXMLDataName, gxdm );
  transMeta.addStep( getXMLDataStep );

  GetXMLDataField[] fields = new GetXMLDataField[1];

  for ( int idx = 0; idx < fields.length; idx++ ) {
    fields[idx] = new GetXMLDataField();
  }

  fields[0].setName( "objectid" );
  fields[0].setXPath( "ObjectID" );
  fields[0].setElementType( GetXMLDataField.ELEMENT_TYPE_NODE );
  fields[0].setResultType( GetXMLDataField.RESULT_TYPE_TYPE_SINGLE_NODE );
  fields[0].setType( ValueMetaInterface.TYPE_STRING );
  fields[0].setFormat( "" );
  fields[0].setLength( -1 );
  fields[0].setPrecision( -1 );
  fields[0].setCurrencySymbol( "" );
  fields[0].setDecimalSymbol( "" );
  fields[0].setGroupSymbol( "" );
  fields[0].setTrimType( GetXMLDataField.TYPE_TRIM_NONE );

  gxdm.setDefault();
  gxdm.setEncoding( "UTF-8" );
  gxdm.setIsAFile( false );
  gxdm.setInFields( true );
  gxdm.setLoopXPath( "/" );
  gxdm.setXMLField( "field1" );
  gxdm.setInputFields( fields );

  try {
    // Now save the transformation and then read it back in
    transMeta.setRepository( repository );
    RepositoryDirectoryInterface repositoryDirectory = repository.findDirectory( "/" );
    transMeta.setRepositoryDirectory( repositoryDirectory );
    repository.transDelegate.saveTransformation( transMeta, "None", null, true );

    // Create a new placeholder meta and set the result type to something different than what was
    // saved,
    // to ensure the saveRep code is working correctly.
    GetXMLDataMeta newMeta = (GetXMLDataMeta) gxdm.clone();
    for ( GetXMLDataField f : newMeta.getInputFields() ) {
      f.setResultType( GetXMLDataField.RESULT_TYPE_VALUE_OF );
    }
    newMeta.readRep( repository, new MemoryMetaStore(), getXMLDataStep.getObjectId(), repository.getDatabases() );

    // Check that the value of Result Type is what was saved in the repo
    assertEquals( newMeta.getInputFields()[0].getResultTypeCode(), "singlenode" );

  } catch ( KettleException e ) {
    fail( "Test failed due to exception: " + e.getLocalizedMessage() );
  }
}
 

private void exportTransformations( ProgressMonitorDecorator monitor, RepositoryDirectoryInterface dirTree,
    ExportWriter writer, boolean feedback ) throws KettleException {
  try {
    writer.openTrans();
    monitor.subTask( BaseMessages.getString( PKG, "Repository.Exporter.Monitor.StartTransExport" ) );

    // Loop over all the directory id's
    ObjectId[] dirids = dirTree.getDirectoryIDs();
    log.logDebug( BaseMessages.getString( PKG, "Repository.Exporter.Log.DirectoryGoing", dirids.length, dirTree
        .getPath() ) );
    for ( int d = 0; d < dirids.length; d++ ) {
      if ( monitor.isCanceled() ) {
        cancelMonitorAction( writer );
        break;
      }

      RepositoryDirectoryInterface repdir = dirTree.findDirectory( dirids[d] );
      String[] trans = repository.getTransformationNames( dirids[d], false );
      log.logDebug( BaseMessages.getString( PKG, "Repository.Exporter.Log.FindTrans",
          trans.length, repdir.getName() ) );

      String dirPath = repdir.getPath();

      for ( int i = 0; i < trans.length; i++ ) {
        if ( monitor.isCanceled() ) {
          break;
        }
        log.logDebug( BaseMessages
            .getString( PKG, "Repository.Exporter.Log.LoadingTransformation", dirPath, trans[i] ) );
        monitor.subTask( BaseMessages.getString( PKG,
            "Repository.Exporter.Monitor.ExportTransformation", trans[i] ) );
        TransMeta transMeta = repository.loadTransformation( trans[i], repdir, null, true, null ); // reads last //
                                                                                                   // version
        transMeta.setRepository( repository );
        convertFromFileRepository( transMeta );

        List<ImportValidationFeedback> errors = this.validateObject( transMeta, feedback );
        if ( errors.isEmpty() ) {
          writer.writeTrans( transMeta.getXML() + Const.CR );
        } else {
          log.logError( BaseMessages.getString( PKG,
              "Repository.Exporter.Log.TransRuleViolation", trans[i], repdir ) );
          this.rulesViolation = true;
          monitor.registerRuleViolation();
          writer.registerRuleViolation();
        }
        // do we need any feedback on this action?
        if ( feedback ) {
          ExportFeedback fb = new ExportFeedback();
          fb.setType( ExportFeedback.Type.TRANSFORMATION );
          fb.setItemName( transMeta.getName() );
          fb.setItemPath( dirPath );
          ExportFeedback.Status status =
              errors.isEmpty() ? ExportFeedback.Status.EXPORTED : ExportFeedback.Status.REJECTED;
          fb.setStatus( status );
          fb.setResult( errors );
          this.feedbackList.add( fb );
        }
      }
    }
  } catch ( Exception e ) {
    throw new KettleException( "Error while exporting repository transformations", e );
  } finally {
    writer.closeTrans();
  }
}
 

private TransMeta getOriginalCopy( boolean isSlaveTrans, ClusterSchema clusterSchema,
    SlaveServer slaveServer ) throws KettleException {
  TransMeta transMeta = new TransMeta();
  transMeta.setSlaveTransformation( true );

  if ( isSlaveTrans ) {
    transMeta.setName( getSlaveTransName( originalTransformation.getName(), clusterSchema, slaveServer ) );

    NotePadMeta slaveNote =
      new NotePadMeta( "This is a generated slave transformation.\nIt will be run on slave server: "
        + slaveServer, 0, 0, -1, -1 );
    transMeta.addNote( slaveNote );

    // add the slave partitioning schema's here.
    for ( int i = 0; i < referenceSteps.length; i++ ) {
      StepMeta stepMeta = referenceSteps[i];
      verifySlavePartitioningConfiguration( transMeta, stepMeta, clusterSchema, slaveServer );
    }
  } else {
    transMeta.setName( originalTransformation.getName() + " (master)" );

    NotePadMeta masterNote =
      new NotePadMeta( "This is a generated master transformation.\nIt will be run on server: "
        + getMasterServer(), 0, 0, -1, -1 );
    transMeta.addNote( masterNote );
  }

  // Copy the cluster schemas
  //
  for ( ClusterSchema schema : originalTransformation.getClusterSchemas() ) {
    transMeta.getClusterSchemas().add( schema.clone() );
  }

  transMeta.setDatabases( originalTransformation.getDatabases() );

  // Feedback
  transMeta.setFeedbackShown( originalTransformation.isFeedbackShown() );
  transMeta.setFeedbackSize( originalTransformation.getFeedbackSize() );

  // Priority management
  transMeta.setUsingThreadPriorityManagment( originalTransformation.isUsingThreadPriorityManagment() );

  // Unique connections
  transMeta.setUsingUniqueConnections( originalTransformation.isUsingUniqueConnections() );

  // Repository
  transMeta.setRepository( originalTransformation.getRepository() );
  transMeta.setRepositoryDirectory( originalTransformation.getRepositoryDirectory() );

  // Also set the logging details...
  transMeta.setTransLogTable( (TransLogTable) originalTransformation.getTransLogTable().clone() );

  // Rowset size
  transMeta.setSizeRowset( originalTransformation.getSizeRowset() );

  return transMeta;
}
 

@Test
public void synchronizeSlaveServerDeleteFromRepository() throws Exception {
  try {
    spoon.rep = repository;
    when( spoon.getRepository() ).thenReturn( repository );

    final String objectId = "object-id";
    final String serverName = "SharedServer";

    TransMeta trans = createTransMeta();
    trans.setRepository( repository );
    trans.setSharedObjects( createSharedObjects( SHARED_OBJECTS_FILE ) );
    SlaveServer server1 = createSlaveServer( serverName, false );
    server1.setObjectId( new StringObjectId( objectId ) );
    trans.addOrReplaceSlaveServer( server1 );
    spoon.delegates.trans.addTransformation( trans );

    JobMeta job = createJobMeta();
    job.setRepository( repository );
    job.setSharedObjects( createSharedObjects( SHARED_OBJECTS_FILE ) );
    SlaveServer server3 = createSlaveServer( serverName, false );
    server3.setObjectId( new StringObjectId( objectId ) );
    job.addOrReplaceSlaveServer( server3 );
    spoon.delegates.jobs.addJob( job );

    TransMeta trans2 = createTransMeta();
    trans2.setRepository( repository );
    trans2.setSharedObjects( createSharedObjects( SHARED_OBJECTS_FILE ) );
    SlaveServer server2 = createSlaveServer( serverName, false );
    server2.setObjectId( new StringObjectId( objectId ) );
    trans2.addOrReplaceSlaveServer( server2 );
    spoon.delegates.trans.addTransformation( trans2 );

    assertFalse( trans.getSlaveServers().isEmpty() );
    assertFalse( job.getSlaveServers().isEmpty() );
    spoon.delegates.slaves.delSlaveServer( trans2, server2 );
    verify( repository ).deleteSlave( server2.getObjectId() );

    assertTrue( trans.getSlaveServers().isEmpty() );
    assertTrue( job.getSlaveServers().isEmpty() );
  } finally {
    spoon.rep = null;
    when( spoon.getRepository() ).thenReturn( null );
  }
}