Java Code Examples for org.pentaho.di.core.variables.VariableSpace#environmentSubstitute()

/**
 * Get the base folder for the data set group
 *
 * @param group
 * @return
 */
private static String getDataSetFolder( DataSetGroup group ) {
  String folderName = group.getFolderName();
  if ( StringUtils.isEmpty( folderName ) ) {
    folderName = System.getProperty( VARIABLE_DATASETS_BASE_PATH );
  }
  if ( StringUtils.isEmpty( folderName ) ) {
    // Local folder
    folderName = ".";
  } else {
    // Let's not forget to replace variables as well...
    //
    VariableSpace space = Variables.getADefaultVariableSpace();
    folderName = space.environmentSubstitute( folderName );
  }

  if ( !folderName.endsWith( File.separator ) ) {
    folderName += File.separator;
  }

  return folderName;
}
 

public HBaseConnection getHBaseConnection( VariableSpace variableSpace, String siteConfig, String defaultConfig,
                                           LogChannelInterface logChannelInterface ) throws IOException {
  Properties connProps = new Properties();
  String zooKeeperHost = null;
  String zooKeeperPort = null;
  if ( namedCluster != null ) {
    zooKeeperHost = variableSpace.environmentSubstitute( namedCluster.getZooKeeperHost() );
    zooKeeperPort = variableSpace.environmentSubstitute( namedCluster.getZooKeeperPort() );
  }
  if ( !Const.isEmpty( zooKeeperHost ) ) {
    connProps.setProperty( org.pentaho.hadoop.shim.spi.HBaseConnection.ZOOKEEPER_QUORUM_KEY, zooKeeperHost );
  }
  if ( !Const.isEmpty( zooKeeperPort ) ) {
    connProps.setProperty( org.pentaho.hadoop.shim.spi.HBaseConnection.ZOOKEEPER_PORT_KEY, zooKeeperPort );
  }
  if ( !Const.isEmpty( siteConfig ) ) {
    connProps.setProperty( org.pentaho.hadoop.shim.spi.HBaseConnection.SITE_KEY, siteConfig );
  }
  if ( !Const.isEmpty( defaultConfig ) ) {
    connProps.setProperty( org.pentaho.hadoop.shim.spi.HBaseConnection.DEFAULTS_KEY, defaultConfig );
  }
  connProps.setProperty( "named.cluster", namedCluster.getName() );
  return getConnectionImpl( connProps, logChannelInterface );
}
 

public LdapSslProtocol( LogChannelInterface log, VariableSpace variableSpace, LdapMeta meta,
  Collection<String> binaryAttributes ) {
  super( log, variableSpace, meta, binaryAttributes );
  String trustStorePath = null;
  String trustStorePassword = null;
  boolean trustAllCertificates = false;

  if ( meta.isUseCertificate() ) {
    trustStorePath = variableSpace.environmentSubstitute( meta.getTrustStorePath() );
    trustStorePassword =  Utils.resolvePassword( variableSpace,
            meta.getTrustStorePassword() );
    trustAllCertificates = meta.isTrustAllCertificates();
  }

  this.trustAllCertificates = trustAllCertificates;
  this.trustStorePath = trustStorePath;
  this.trustStorePassword = trustStorePassword;
}
 

public void init( VariableSpace vars, boolean updateFromEnv ) {
  if ( updateFromEnv ) {
    environUpdatedFieldName = vars.environmentSubstitute( m_incomingFieldName );
    environUpdateMongoDocPath = vars.environmentSubstitute( m_mongoDocPath, true );
    environUpdateModifierOperation = vars.environmentSubstitute( m_modifierUpdateOperation );
  }
  m_pathList = new ArrayList<String>();

  if ( !Const.isEmpty( environUpdateMongoDocPath ) ) {
    String[] parts = environUpdateMongoDocPath.split( "\\." ); //$NON-NLS-1$
    for ( String p : parts ) {
      m_pathList.add( p );
    }
  }
  m_tempPathList = new ArrayList<String>( m_pathList );
}
 

public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep,
    VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  // Output field (String)
  ValueMetaInterface v = new ValueMeta( space.environmentSubstitute( getResultfieldname() ), ValueMeta.TYPE_STRING );
  v.setOrigin( name );
  inputRowMeta.addValueMeta( v );
}
 

@Override
public RowMetaInterface getRequiredFields( VariableSpace space ) throws KettleException {
  String realTableName = space.environmentSubstitute( tableName );
  String realSchemaName = space.environmentSubstitute( schemaName );

  if ( databaseMeta != null ) {
    Database db = new Database( loggingObject, databaseMeta );
    try {
      db.connect();

      if ( !Utils.isEmpty( realTableName ) ) {
        String schemaTable = databaseMeta.getQuotedSchemaTableCombination( realSchemaName, realTableName );

        // Check if this table exists...
        if ( db.checkTableExists( schemaTable ) ) {
          return db.getTableFields( schemaTable );
        } else {
          throw new KettleException( BaseMessages.getString( PKG, "GPBulkLoaderMeta.Exception.TableNotFound" ) );
        }
      } else {
        throw new KettleException( BaseMessages.getString( PKG, "GPBulkLoaderMeta.Exception.TableNotSpecified" ) );
      }
    } catch ( Exception e ) {
      throw new KettleException(
        BaseMessages.getString( PKG, "GPBulkLoaderMeta.Exception.ErrorGettingFields" ), e );
    } finally {
      db.disconnect();
    }
  } else {
    throw new KettleException( BaseMessages.getString( PKG, "GPBulkLoaderMeta.Exception.ConnectionNotDefined" ) );
  }

}
 

public void getFields( RowMetaInterface row, String name, RowMetaInterface[] info, StepMeta nextStep,
    VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {

  if ( getOperationType() != OPERATION_TYPE_WRITE_TO_FILE ) {
    ValueMetaInterface v =
        new ValueMetaString( space.environmentSubstitute( this.getOutputValue() ) );
    v.setOrigin( name );
    row.addValueMeta( v );
  }
}
 

public void initData( VariableSpace space ) throws Exception {

    if ( data == null ) {
      data = (OlapData) getStepData();
    }

    String driver = this.getOlap4jDriver();
    String url = space.environmentSubstitute( this.getOlap4jUrl() );
    String username = space.environmentSubstitute( this.getUsername() );
    String password = space.environmentSubstitute( this.getPassword() );

    String mdx = this.getMdx();
    if ( this.isVariableReplacementActive() ) {
      mdx = space.environmentSubstitute( this.getMdx() );
    }
    String catalog = space.environmentSubstitute( this.getCatalog() );

    // mdx =
    // "select NON EMPTY Hierarchize(Union(Crossjoin({[Measures].[Actual]},
    // Union(Crossjoin({[Region].[All Regions]}, {[Department].[All Departments]}),
    // Crossjoin({[Region].[All Regions]}, [Department].[All Departments].Children))),
    // Crossjoin({[Measures].[Actual]}, Union(Crossjoin([Region].[All Regions].Children,
    // {[Department].[All Departments]}),
    // Crossjoin([Region].[All Regions].Children, [Department].[All Departments].Children))))) ON COLUMNS,NON EMPTY
    // Hierarchize(Union({[Positions].[All Positions]}, [Positions].[All Positions].Children)) ON ROWS from [Quadrant
    // Analysis]";
    // data.olapHelper = new OlapHelper(driver,"http://localhost:8080/pentaho/Xmla", "joe", "password","SampleData",mdx,
    // this);
    data.olapHelper = new OlapHelper( driver, url, username, password, catalog, mdx );
    data.olapHelper.openQuery();
    data.olapHelper.createRectangularOutput();
    data.outputRowMeta =
      this.createRowMeta( data.olapHelper.getHeaderValues(), data.olapHelper.getCellValues() ).clone();

  }
 

@Override
public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  // Output fields (String)
  if ( !Utils.isEmpty( resultfieldname ) ) {
    ValueMetaInterface v = new ValueMetaString( space.environmentSubstitute( resultfieldname ) );
    v.setOrigin( name );
    inputRowMeta.addValueMeta( v );
  }

}
 

public static String constructUrl( VariableSpace space, String hostname, String port, String webAppName,
                                   String serviceAndArguments, boolean isSecure )
  throws UnsupportedEncodingException {
  String realHostname = space.environmentSubstitute( hostname );
  if ( !StringUtils.isEmpty( webAppName ) ) {
    serviceAndArguments = "/" + space.environmentSubstitute( webAppName ) + serviceAndArguments;
  }
  String protocol = isSecure ? PROTOCOL_SECURE : PROTOCOL_UNSECURE;
  String retval = protocol + "://" + realHostname + getPortSpecification( space, port ) + serviceAndArguments;
  retval = Const.replace( retval, " ", "%20" );
  return retval;
}
 

public String buildFilename( VariableSpace space, int stepnr ) {

    SimpleDateFormat daf = new SimpleDateFormat();

    // Replace possible environment variables...
    String retval = space.environmentSubstitute( fileName );

    Date now = new Date();

    if ( dateInFilename ) {
      daf.applyPattern( "yyyMMdd" );
      String d = daf.format( now );
      retval += "_" + d;
    }
    if ( timeInFilename ) {
      daf.applyPattern( "HHmmss" );
      String t = daf.format( now );
      retval += "_" + t;
    }
    if ( stepNrInFilename ) {
      retval += "_" + stepnr;
    }

    if ( extension != null && extension.length() != 0 ) {
      retval += "." + extension;
    }

    return retval;
  }
 

public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep,
    VariableSpace space, Repository repository, IMetaStore metaStore )
  throws KettleStepException {
  // Output field (String)
  if ( !Utils.isEmpty( resultfieldname ) ) {
    ValueMetaInterface v = new ValueMetaBoolean( space.environmentSubstitute( resultfieldname ) );
    v.setOrigin( name );
    inputRowMeta.addValueMeta( v );
  }
}
 

public RowMetaInterface getRequiredFields( VariableSpace space ) throws KettleException {
  String realTableName = space.environmentSubstitute( tableName );
  String realSchemaName = space.environmentSubstitute( schemaName );

  if ( databaseMeta != null ) {
    Database db = new Database( loggingObject, databaseMeta );
    try {
      db.connect();

      if ( !Utils.isEmpty( realTableName ) ) {
        // Check if this table exists...
        if ( db.checkTableExists( realSchemaName, realTableName ) ) {
          return db.getTableFieldsMeta( realSchemaName, realTableName );
        } else {
          throw new KettleException( BaseMessages.getString(
            PKG, "SynchronizeAfterMergeMeta.Exception.TableNotFound" ) );
        }
      } else {
        throw new KettleException( BaseMessages.getString(
          PKG, "SynchronizeAfterMergeMeta.Exception.TableNotSpecified" ) );
      }
    } catch ( Exception e ) {
      throw new KettleException( BaseMessages.getString(
        PKG, "SynchronizeAfterMergeMeta.Exception.ErrorGettingFields" ), e );
    } finally {
      db.disconnect();
    }
  } else {
    throw new KettleException( BaseMessages.getString(
      PKG, "SynchronizeAfterMergeMeta.Exception.ConnectionNotDefined" ) );
  }

}
 

public boolean validate( CheckResultSourceInterface source, String propertyName,
  List<CheckResultInterface> remarks, ValidatorContext context ) {

  String filename = ValidatorUtils.getValueAsString( source, propertyName );
  VariableSpace variableSpace = getVariableSpace( source, propertyName, remarks, context );
  boolean failIfDoesNotExist = getFailIfDoesNotExist( source, propertyName, remarks, context );

  if ( null == variableSpace ) {
    return false;
  }

  String realFileName = variableSpace.environmentSubstitute( filename );
  FileObject fileObject = null;
  try {
    fileObject = KettleVFS.getFileObject( realFileName, variableSpace );
    if ( fileObject == null || ( fileObject != null && !fileObject.exists() && failIfDoesNotExist ) ) {
      JobEntryValidatorUtils.addFailureRemark(
        source, propertyName, VALIDATOR_NAME, remarks, JobEntryValidatorUtils.getLevelOnFail(
          context, VALIDATOR_NAME ) );
      return false;
    }
    try {
      fileObject.close(); // Just being paranoid
    } catch ( IOException ignored ) {
      // Ignore close errors
    }
  } catch ( Exception e ) {
    JobEntryValidatorUtils.addExceptionRemark( source, propertyName, VALIDATOR_NAME, remarks, e );
    return false;
  }
  return true;
}
 

@Override
public String[] getFilePaths( VariableSpace space ) {
  return new String[] { space.environmentSubstitute( filename ), };
}
 

@Override
public void getFields( RowMetaInterface row, String name, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {

  // the filename
  ValueMetaInterface filename = new ValueMetaString( "filename" );
  filename.setLength( 500 );
  filename.setPrecision( -1 );
  filename.setOrigin( name );
  row.addValueMeta( filename );

  // the short filename
  ValueMetaInterface short_filename = new ValueMetaString( "short_filename" );
  short_filename.setLength( 500 );
  short_filename.setPrecision( -1 );
  short_filename.setOrigin( name );
  row.addValueMeta( short_filename );

  // the path
  ValueMetaInterface path = new ValueMetaString( "path" );
  path.setLength( 500 );
  path.setPrecision( -1 );
  path.setOrigin( name );
  row.addValueMeta( path );

  // the type
  ValueMetaInterface type = new ValueMetaString( "type" );
  type.setLength( 500 );
  type.setPrecision( -1 );
  type.setOrigin( name );
  row.addValueMeta( type );

  // the exists
  ValueMetaInterface exists = new ValueMetaBoolean( "exists" );
  exists.setOrigin( name );
  row.addValueMeta( exists );

  // the ishidden
  ValueMetaInterface ishidden = new ValueMetaBoolean( "ishidden" );
  ishidden.setOrigin( name );
  row.addValueMeta( ishidden );

  // the isreadable
  ValueMetaInterface isreadable = new ValueMetaBoolean( "isreadable" );
  isreadable.setOrigin( name );
  row.addValueMeta( isreadable );

  // the iswriteable
  ValueMetaInterface iswriteable = new ValueMetaBoolean( "iswriteable" );
  iswriteable.setOrigin( name );
  row.addValueMeta( iswriteable );

  // the lastmodifiedtime
  ValueMetaInterface lastmodifiedtime = new ValueMetaDate( "lastmodifiedtime" );
  lastmodifiedtime.setOrigin( name );
  row.addValueMeta( lastmodifiedtime );

  // the size
  ValueMetaInterface size = new ValueMetaInteger( "size" );
  size.setOrigin( name );
  row.addValueMeta( size );

  // the extension
  ValueMetaInterface extension = new ValueMetaString( "extension" );
  extension.setOrigin( name );
  row.addValueMeta( extension );

  // the uri
  ValueMetaInterface uri = new ValueMetaString( "uri" );
  uri.setOrigin( name );
  row.addValueMeta( uri );

  // the rooturi
  ValueMetaInterface rooturi = new ValueMetaString( "rooturi" );
  rooturi.setOrigin( name );
  row.addValueMeta( rooturi );

  if ( includeRowNumber ) {
    ValueMetaInterface v = new ValueMetaInteger( space.environmentSubstitute( rowNumberField ) );
    v.setLength( ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0 );
    v.setOrigin( name );
    row.addValueMeta( v );
  }

}
 

public static String loadFileContent( VariableSpace space, String filename ) throws Exception {
  String realFilename = space.environmentSubstitute( filename );
  return KettleVFS.getTextFileContent( realFilename, Charset.defaultCharset().name() );
}
 

public void getFields( RowMetaInterface row, String name, RowMetaInterface[] info, StepMeta nextStep,
                       VariableSpace space ) throws KettleStepException {

  // The filename...
  ValueMetaInterface filename = new ValueMeta( "filename", ValueMetaInterface.TYPE_STRING );
  filename.setOrigin( name );
  filename.setLength( 255 );
  row.addValueMeta( filename );

  // The file type
  ValueMetaInterface ft = new ValueMeta( "filetype", ValueMetaInterface.TYPE_STRING );
  ft.setLength( 50 );
  ft.setOrigin( name );
  row.addValueMeta( ft );

  // The shape nr
  ValueMetaInterface shnr = new ValueMeta( "shapenr", ValueMetaInterface.TYPE_INTEGER );
  shnr.setOrigin( name );
  row.addValueMeta( shnr );

  // The part nr
  ValueMetaInterface pnr = new ValueMeta( "partnr", ValueMetaInterface.TYPE_INTEGER );
  pnr.setOrigin( name );
  row.addValueMeta( pnr );

  // The part nr
  ValueMetaInterface nrp = new ValueMeta( "nrparts", ValueMetaInterface.TYPE_INTEGER );
  nrp.setOrigin( name );
  row.addValueMeta( nrp );

  // The point nr
  ValueMetaInterface ptnr = new ValueMeta( "pointnr", ValueMetaInterface.TYPE_INTEGER );
  ptnr.setOrigin( name );
  row.addValueMeta( ptnr );

  // The nr of points
  ValueMetaInterface nrpt = new ValueMeta( "nrpointS", ValueMetaInterface.TYPE_INTEGER );
  nrpt.setOrigin( name );
  row.addValueMeta( nrpt );

  // The X coordinate
  ValueMetaInterface x = new ValueMeta( "x", ValueMetaInterface.TYPE_NUMBER );
  x.setOrigin( name );
  row.addValueMeta( x );

  // The Y coordinate
  ValueMetaInterface y = new ValueMeta( "y", ValueMetaInterface.TYPE_NUMBER );
  y.setOrigin( name );
  row.addValueMeta( y );

  // The measure
  ValueMetaInterface m = new ValueMeta( "measure", ValueMetaInterface.TYPE_NUMBER );
  m.setOrigin( name );
  row.addValueMeta( m );

  String dbFilename = getDbfFilename();
  if ( dbFilename != null ) {
    if ( space != null ) {
      dbFilename = space.environmentSubstitute( dbFilename );
      if ( dbFilename.startsWith( "file:" ) ) {
        dbFilename = dbFilename.substring( 5 );
      }
    }

    XBase xbase = new XBase( getLog(), dbFilename );
    try {
      xbase.setDbfFile( dbFilename );
      xbase.open();

      //Set encoding
      if ( StringUtils.isNotBlank( encoding ) ) {
        xbase.getReader().setCharactersetName( encoding );
      }

      RowMetaInterface fields = xbase.getFields();
      for ( int i = 0; i < fields.size(); i++ ) {
        fields.getValueMeta( i ).setOrigin( name );
        row.addValueMeta( fields.getValueMeta( i ) );
      }

    } catch ( Throwable e ) {
      throw new KettleStepException( "Unable to read from DBF file", e );
    } finally {
      xbase.close();
    }
  } else {
    throw new KettleStepException( "Unable to read from DBF file: no filename specfied" );
  }
}
 

public void getFields( RowMetaInterface row, String origin, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  if ( databaseMeta == null ) {
    return; // TODO: throw an exception here
  }

  if ( cachedRowMetaActive ) {
    row.addRowMeta( cachedRowMeta );
    return;
  }

  boolean param = false;

  Database db = getDatabase();
  super.databases = new Database[] { db }; // keep track of it for canceling purposes...

  // First try without connecting to the database... (can be S L O W)
  String sNewSQL = sql;
  if ( isVariableReplacementActive() ) {
    sNewSQL = db.environmentSubstitute( sql );
    if ( space != null ) {
      sNewSQL = space.environmentSubstitute( sNewSQL );
    }
  }

  RowMetaInterface add = null;
  try {
    add = db.getQueryFields( sNewSQL, param );
  } catch ( KettleDatabaseException dbe ) {
    throw new KettleStepException( "Unable to get queryfields for SQL: " + Const.CR + sNewSQL, dbe );
  }

  if ( add != null ) {
    attachOrigin( add, origin );
    row.addRowMeta( add );
  } else {
    try {
      db.connect();

      RowMetaInterface paramRowMeta = null;
      Object[] paramData = null;

      StreamInterface infoStream = getStepIOMeta().getInfoStreams().get( 0 );
      if ( !Utils.isEmpty( infoStream.getStepname() ) ) {
        param = true;
        if ( info.length > 0 && info[ 0 ] != null ) {
          paramRowMeta = info[ 0 ];
          paramData = RowDataUtil.allocateRowData( paramRowMeta.size() );
        }
      }

      add = db.getQueryFields( sNewSQL, param, paramRowMeta, paramData );

      if ( add == null ) {
        return;
      }
      attachOrigin( add, origin );
      row.addRowMeta( add );
    } catch ( KettleException ke ) {
      throw new KettleStepException( "Unable to get queryfields for SQL: " + Const.CR + sNewSQL, ke );
    } finally {
      db.disconnect();
    }
  }
  if ( isLazyConversionActive() ) {
    for ( int i = 0; i < row.size(); i++ ) {
      ValueMetaInterface v = row.getValueMeta( i );
      try {
        if ( v.getType() == ValueMetaInterface.TYPE_STRING ) {
          ValueMetaInterface storageMeta = ValueMetaFactory.cloneValueMeta( v );
          storageMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL );
          v.setStorageMetadata( storageMeta );
          v.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING );
        }
      } catch ( KettlePluginException e ) {
        throw new KettleStepException( "Unable to clone meta for lazy conversion: " + Const.CR + v, e );
      }
    }
  }
}
 

@Override public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore ) {

    inputRowMeta.clear();

    ValueMetaInterface filenameValueMeta = new ValueMetaString( space.environmentSubstitute( filenameField ) );
    filenameValueMeta.setOrigin( name );
    inputRowMeta.addValueMeta( filenameValueMeta );

    ValueMetaInterface fileTypeValueMeta = new ValueMetaString( space.environmentSubstitute( fileTypeField ) );
    fileTypeValueMeta.setOrigin( name );
    inputRowMeta.addValueMeta( fileTypeValueMeta );


  }