Java Code Examples for org.pentaho.di.trans.steps.textfileinput.TextFileInputField

The following examples show how to use org.pentaho.di.trans.steps.textfileinput.TextFileInputField. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Override
public void saveRep(Repository rep, IMetaStore metaStore, ObjectId id_transformation, ObjectId id_step) throws KettleException {
    try {
        rep.saveStepAttribute(id_transformation, id_step, "serviceEmail", this.serviceEmail);
        rep.saveStepAttribute(id_transformation, id_step, "spreadsheetKey", this.spreadsheetKey);
        rep.saveStepAttribute(id_transformation, id_step, "worksheetId", this.worksheetId);
        rep.saveStepAttribute(id_transformation, id_step, "privateKeyStore", GoogleSpreadsheet.base64EncodePrivateKeyStore(this.privateKeyStore));

        for (int i = 0; i < inputFields.length; i++) {
            TextFileInputField field = inputFields[i];

            rep.saveStepAttribute(id_transformation, id_step, i, "field_name", field.getName());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_type", ValueMeta.getTypeDesc(field.getType()));
            rep.saveStepAttribute(id_transformation, id_step, i, "field_format", field.getFormat());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_currency", field.getCurrencySymbol());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_decimal", field.getDecimalSymbol());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_group", field.getGroupSymbol());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_length", field.getLength());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_precision", field.getPrecision());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_trim_type", ValueMeta.getTrimTypeCode(field.getTrimType()));
        }

    } catch (Exception e) {
        throw new KettleException("Unable to save step information to the repository for id_step=" + id_step, e);
    }
}
 
Example 2
Source Project: pentaho-kettle   Source File: ParGzipCsvInputMetaTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUpLoadSave() throws Exception {
  KettleEnvironment.init();
  PluginRegistry.init( false );
  List<String> attributes =
      Arrays.asList( "filename", "filenameField", "includingFilename", "rowNumField", "headerPresent", "delimiter",
          "enclosure", "bufferSize", "lazyConversionActive", "addResultFile", "runningInParallel", "encoding",
          "inputFields" );

  Map<String, String> getterMap = new HashMap<String, String>();
  Map<String, String> setterMap = new HashMap<String, String>();

  Map<String, FieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();
  attrValidatorMap.put( "inputFields", new ArrayLoadSaveValidator<TextFileInputField>( new TextFileInputFieldValidator(), 5 ) );

  Map<String, FieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();

  loadSaveTester =
      new LoadSaveTester( testMetaClass, attributes, new ArrayList<String>(), new ArrayList<String>(), getterMap,
          setterMap, attrValidatorMap, typeValidatorMap, this );
}
 
Example 3
Source Project: pentaho-kettle   Source File: ParGzipCsvInputMetaTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TextFileInputField getTestObject() {
  TextFileInputField rtn = new TextFileInputField();
  rtn.setCurrencySymbol( UUID.randomUUID().toString() );
  rtn.setDecimalSymbol( UUID.randomUUID().toString() );
  rtn.setFormat( UUID.randomUUID().toString() );
  rtn.setGroupSymbol( UUID.randomUUID().toString() );
  rtn.setName( UUID.randomUUID().toString() );
  rtn.setTrimType( rand.nextInt( 4 ) );
  rtn.setPrecision( rand.nextInt( 9 ) );
  rtn.setLength( rand.nextInt( 50 ) );
  rtn.setType( rand.nextInt( 7 ) );
  // Note - these fields aren't serialized by the meta class ... cannot test for them
  // rtn.setRepeated( rand.nextBoolean() );
  // rtn.setSamples( new String[] { UUID.randomUUID().toString(), UUID.randomUUID().toString(),
  // UUID.randomUUID().toString() } );
  // rtn.setNullString( UUID.randomUUID().toString() );
  // rtn.setIfNullValue( UUID.randomUUID().toString() );
  // rtn.setIgnored( rand.nextBoolean() );
  // rtn.setPosition( rand.nextInt( 10 ) );
  return rtn;
}
 
Example 4
Source Project: pentaho-kettle   Source File: ParGzipCsvInputMetaTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public boolean validateTestObject( TextFileInputField testObject, Object actual ) {
  if ( !( actual instanceof TextFileInputField ) ) {
    return false;
  }

  TextFileInputField another = (TextFileInputField) actual;
  return new EqualsBuilder()
      .append( testObject.getCurrencySymbol(), another.getCurrencySymbol() )
      .append( testObject.getDecimalSymbol(), another.getDecimalSymbol() )
      .append( testObject.getFormat(), another.getFormat() )
      .append( testObject.getGroupSymbol(), another.getGroupSymbol() )
      .append( testObject.getName(), another.getName() )
      .append( testObject.getTrimType(), another.getTrimType() )
      .append( testObject.getPrecision(), another.getPrecision() )
      .append( testObject.getLength(), another.getLength() )
      .append( testObject.getType(), another.getType() )
      // Note - these fields aren't serialized by the meta class ... cannot test for them
      // .append( testObject.isRepeated(), another.isRepeated() )
      // .append( testObject.getSamples(), another.getSamples() )
      // .append( testObject.getNullString(), another.getNullString() )
      // .append( testObject.getIfNullValue(), another.getIfNullValue() )
      // .append( testObject.isIgnored(), another.isIgnored() )
      // .append( testObject.getPosition(), another.getPosition() )
      .isEquals();
}
 
Example 5
Source Project: pentaho-kettle   Source File: CsvInputMetaTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testClone() {
  final CsvInputMeta original = new CsvInputMeta();
  original.setDelimiter( ";" );
  original.setEnclosure( "'" );
  final TextFileInputField[] originalFields = new TextFileInputField[ 1 ];
  final TextFileInputField originalField = new TextFileInputField();
  originalField.setName( "field" );
  originalFields[ 0 ] = originalField;
  original.setInputFields( originalFields );

  final CsvInputMeta clone = (CsvInputMeta) original.clone();
  // verify that the clone and its input fields are "equal" to the originals, but not the same objects
  Assert.assertNotSame( original, clone );
  Assert.assertEquals( original.getDelimiter(), clone.getDelimiter() );
  Assert.assertEquals( original.getEnclosure(), clone.getEnclosure() );

  Assert.assertNotSame( original.getInputFields(), clone.getInputFields() );
  Assert.assertNotSame( original.getInputFields()[ 0 ], clone.getInputFields()[ 0 ] );
  Assert.assertEquals( original.getInputFields()[ 0 ].getName(), clone.getInputFields()[ 0 ].getName() );
}
 
Example 6
private void initiateVariables( String fileContent, TextFileInputField[] inputFileFields, int fileFormat ) throws Exception {
  //reader
  reader = new InputStreamReader( new ByteArrayInputStream( fileContent.getBytes( StandardCharsets.UTF_8 ) ) );
  //meta
  when( meta.getInputFields() ).thenReturn( inputFileFields );
  doCallRealMethod().when( meta ).getFields( any( RowMetaInterface.class ), anyString(), any( RowMetaInterface[].class ),
    any( StepMeta.class ), any( VariableSpace.class ), any( Repository.class ), any( IMetaStore.class ) );
  setInternalState( meta, "inputFields",  inputFileFields );
  when( meta.hasHeader() ).thenReturn( true );
  when( meta.getNrHeaderLines() ).thenReturn( 1 );
  when( meta.clone() ).thenReturn( meta );
  when( meta.getSeparator() ).thenReturn( "," );
  when( meta.getEnclosure() ).thenReturn( "\"" );
  when( meta.getFileFormatTypeNr() ).thenReturn( fileFormat );
  when( meta.getFileType() ).thenReturn( "CSV" );
  when( meta.getFilePaths( any( VariableSpace.class ) ) ).thenReturn( new String[] {"empty"} );
  //transmeta
  when( transMeta.environmentSubstitute( "," ) ).thenReturn( "," );
  when( transMeta.environmentSubstitute( "\"" ) ).thenReturn( "\"" );
  when( transMeta.getObjectType() ).thenReturn( LoggingObjectType.TRANSMETA );
  //textFileCSVImportProgressDialog
  textFileCSVImportProgressDialog = spy( new TextFileCSVImportProgressDialog(
    shell, meta, transMeta, reader, 100, true ) );
}
 
Example 7
@Override
public void setDefault() {
    this.serviceEmail = "";
    this.spreadsheetKey = "";
    this.worksheetId = "od6";
    this.privateKeyStore = null;

    TextFileInputField field = new TextFileInputField();
    field.setName("field");
    field.setType(ValueMetaInterface.TYPE_STRING);

    inputFields = new TextFileInputField[]{
            field,
    };
}
 
Example 8
Source Project: pentaho-kettle   Source File: S3CsvInputMeta.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transformation, ObjectId id_step )
    throws KettleException {
  try {
    rep.saveStepAttribute( id_transformation, id_step, "aws_secret_key", Encr.encryptPasswordIfNotUsingVariables(
        awsSecretKey ) );
    rep.saveStepAttribute( id_transformation, id_step, "aws_access_key", Encr.encryptPasswordIfNotUsingVariables(
        awsAccessKey ) );
    rep.saveStepAttribute( id_transformation, id_step, "bucket", bucket );
    rep.saveStepAttribute( id_transformation, id_step, "filename", filename );
    rep.saveStepAttribute( id_transformation, id_step, "filename_field", filenameField );
    rep.saveStepAttribute( id_transformation, id_step, "rownum_field", rowNumField );
    rep.saveStepAttribute( id_transformation, id_step, "include_filename", includingFilename );
    rep.saveStepAttribute( id_transformation, id_step, "separator", delimiter );
    rep.saveStepAttribute( id_transformation, id_step, "enclosure", enclosure );
    rep.saveStepAttribute( id_transformation, id_step, "max_line_size", maxLineSize );
    rep.saveStepAttribute( id_transformation, id_step, "header", headerPresent );
    rep.saveStepAttribute( id_transformation, id_step, "lazy_conversion", lazyConversionActive );
    rep.saveStepAttribute( id_transformation, id_step, "parallel", runningInParallel );

    for ( int i = 0; i < inputFields.length; i++ ) {
      TextFileInputField field = inputFields[i];

      rep.saveStepAttribute( id_transformation, id_step, i, "field_name", field.getName() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_type", ValueMetaFactory.getValueMetaName( field.getType() ) );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_format", field.getFormat() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_currency", field.getCurrencySymbol() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_decimal", field.getDecimalSymbol() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_group", field.getGroupSymbol() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_length", field.getLength() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_precision", field.getPrecision() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_trim_type", ValueMetaString.getTrimTypeCode( field
          .getTrimType() ) );
    }
  } catch ( Exception e ) {
    throw new KettleException( "Unable to save step information to the repository for id_step=" + id_step, e );
  }
}
 
Example 9
@Override
public void loadXML(Node stepnode, List<DatabaseMeta> databases, IMetaStore metaStore) throws KettleXMLException {
    try {
        this.serviceEmail = XMLHandler.getTagValue(stepnode, "serviceEmail");
        this.spreadsheetKey = XMLHandler.getTagValue(stepnode, "spreadsheetKey");
        this.worksheetId = XMLHandler.getTagValue(stepnode, "worksheetId");
        this.privateKeyStore = GoogleSpreadsheet.base64DecodePrivateKeyStore(XMLHandler.getTagValue(stepnode, "privateKeyStore"));

        Node fields = XMLHandler.getSubNode(stepnode, "fields");
        int nrfields = XMLHandler.countNodes(fields, "field");

        allocate(nrfields);

        for (int i = 0; i < nrfields; i++) {
            inputFields[i] = new TextFileInputField();

            Node fnode = XMLHandler.getSubNodeByNr(fields, "field", i);

            inputFields[i].setName(XMLHandler.getTagValue(fnode, "field_name"));
            inputFields[i].setType(ValueMeta.getType(XMLHandler.getTagValue(fnode, "field_type")));
            inputFields[i].setFormat(XMLHandler.getTagValue(fnode, "field_format"));
            inputFields[i].setCurrencySymbol(XMLHandler.getTagValue(fnode, "field_currency"));
            inputFields[i].setDecimalSymbol(XMLHandler.getTagValue(fnode, "field_decimal"));
            inputFields[i].setGroupSymbol(XMLHandler.getTagValue(fnode, "field_group"));
            inputFields[i].setLength(Const.toInt(XMLHandler.getTagValue(fnode, "field_length"), -1));
            inputFields[i].setPrecision(Const.toInt(XMLHandler.getTagValue(fnode, "field_precision"), -1));
            inputFields[i].setTrimType(ValueMeta.getTrimTypeByCode(XMLHandler.getTagValue(fnode, "field_trim_type")));
        }

    } catch (Exception e) {
        throw new KettleXMLException("Unable to load step from XML", e);
    }
}
 
Example 10
@Override
public void readRep(Repository rep, IMetaStore metaStore, ObjectId id_step, List<DatabaseMeta> databases) throws KettleException {
    try {
        this.serviceEmail = rep.getStepAttributeString(id_step, "serviceEmail");
        this.spreadsheetKey = rep.getStepAttributeString(id_step, "spreadsheetKey");
        this.worksheetId = rep.getStepAttributeString(id_step, "worksheetId");
        this.privateKeyStore = GoogleSpreadsheet.base64DecodePrivateKeyStore(rep.getStepAttributeString(id_step, "privateKeyStore"));

        int nrfields = rep.countNrStepAttributes(id_step, "field_name");

        allocate(nrfields);

        for (int i = 0; i < nrfields; i++) {
            inputFields[i] = new TextFileInputField();

            inputFields[i].setName(rep.getStepAttributeString(id_step, i, "field_name"));
            inputFields[i].setType(ValueMeta.getType(rep.getStepAttributeString(id_step, i, "field_type")));
            inputFields[i].setFormat(rep.getStepAttributeString(id_step, i, "field_format"));
            inputFields[i].setCurrencySymbol(rep.getStepAttributeString(id_step, i, "field_currency"));
            inputFields[i].setDecimalSymbol(rep.getStepAttributeString(id_step, i, "field_decimal"));
            inputFields[i].setGroupSymbol(rep.getStepAttributeString(id_step, i, "field_group"));
            inputFields[i].setLength((int) rep.getStepAttributeInteger(id_step, i, "field_length"));
            inputFields[i].setPrecision((int) rep.getStepAttributeInteger(id_step, i, "field_precision"));
            inputFields[i].setTrimType(ValueMeta.getTrimTypeByCode(rep.getStepAttributeString(id_step, i, "field_trim_type")));
        }
    } catch (Exception e) {
        throw new KettleException("Unexpected error reading step information from the repository", e);
    }
}
 
Example 11
@Override
public void getFields(RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    try {
        inputRowMeta.clear(); // Start with a clean slate, eats the input

        for (TextFileInputField field : inputFields) {
            ValueMetaInterface valueMeta = ValueMetaFactory.createValueMeta(field.getName(), field.getType());
            valueMeta.setConversionMask(field.getFormat());
            valueMeta.setLength(field.getLength());
            valueMeta.setPrecision(field.getPrecision());
            valueMeta.setConversionMask(field.getFormat());
            valueMeta.setDecimalSymbol(field.getDecimalSymbol());
            valueMeta.setGroupingSymbol(field.getGroupSymbol());
            valueMeta.setCurrencySymbol(field.getCurrencySymbol());
            valueMeta.setTrimType(field.getTrimType());
            valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
            valueMeta.setDateFormatLenient(true);
            valueMeta.setStringEncoding("UTF-8");

            ValueMetaInterface storageMetadata = ValueMetaFactory.cloneValueMeta(valueMeta, ValueMetaInterface.TYPE_STRING);
            storageMetadata.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
            storageMetadata.setLength(-1, -1); // we don't really know the lengths of the strings read in advance.
            valueMeta.setStorageMetadata(storageMetadata);

            valueMeta.setOrigin(name);

            inputRowMeta.addValueMeta(valueMeta);
        }
    } catch (Exception e) {

    }
}
 
Example 12
private S3CsvInputMeta createSampleMeta() {
  S3CsvInputMeta meta = new S3CsvInputMeta();
  meta.allocate( 2 );
  meta.getInputFields()[ 0 ] = new TextFileInputField( "field1", 0, 1 );
  meta.getInputFields()[ 1 ] = new TextFileInputField( "field2", 0, 2 );
  return meta;
}
 
Example 13
protected void doInputWiring( Query query, TransMeta transMeta ) {
  //
  // CSV FILE LOCATION AND FIELDS
  //

  InlineEtlPhysicalModel physicalModel = (InlineEtlPhysicalModel) query.getLogicalModel().getPhysicalModel();

  CsvInputMeta csvinput = (CsvInputMeta) getStepMeta( transMeta, "CSV file input" ).getStepMetaInterface(); //$NON-NLS-1$

  // the file name might need to be translated to the correct location here
  if ( csvFileLoc != null ) {
    csvinput.setFilename( csvFileLoc + physicalModel.getFileLocation() );
  } else {
    csvinput.setFilename( physicalModel.getFileLocation() );
  }

  csvinput.setDelimiter( physicalModel.getDelimiter() );
  csvinput.setEnclosure( physicalModel.getEnclosure() );
  csvinput.setHeaderPresent( physicalModel.getHeaderPresent() );

  // update fields

  LogicalTable table = query.getLogicalModel().getLogicalTables().get( 0 );

  csvinput.allocate( table.getLogicalColumns().size() );

  for ( int i = 0; i < csvinput.getInputFields().length; i++ ) {
    // Update csv input

    LogicalColumn col = table.getLogicalColumns().get( i );
    csvinput.getInputFields()[i] = new TextFileInputField();
    String fieldName = (String) col.getProperty( InlineEtlPhysicalColumn.FIELD_NAME );
    if ( logger.isDebugEnabled() ) {
      logger.debug( "FROM CSV: " + fieldName ); //$NON-NLS-1$
    }
    csvinput.getInputFields()[i].setName( fieldName );
    csvinput.getInputFields()[i].setType( convertType( col.getDataType() ) );
  }

}
 
Example 14
Source Project: pentaho-kettle   Source File: CsvInput3IT.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected TextFileInputField[] createTextFileInputFields() {
  TextFileInputField[] fields = new TextFileInputField[3];

  for ( int idx = 0; idx < fields.length; idx++ ) {
    fields[idx] = new TextFileInputField();
  }

  fields[0].setName( "a" );
  fields[0].setType( ValueMetaInterface.TYPE_INTEGER );
  fields[0].setFormat( "" );
  fields[0].setLength( -1 );
  fields[0].setPrecision( -1 );
  fields[0].setCurrencySymbol( "" );
  fields[0].setDecimalSymbol( "" );
  fields[0].setGroupSymbol( "" );
  fields[0].setTrimType( ValueMetaInterface.TRIM_TYPE_NONE );

  fields[1].setName( "b" );
  fields[1].setType( ValueMetaInterface.TYPE_STRING );
  fields[1].setFormat( "" );
  fields[1].setLength( -1 );
  fields[1].setPrecision( -1 );
  fields[1].setCurrencySymbol( "" );
  fields[1].setDecimalSymbol( "" );
  fields[1].setGroupSymbol( "" );
  fields[1].setTrimType( ValueMetaInterface.TRIM_TYPE_NONE );

  fields[2].setName( "c" );
  fields[2].setType( ValueMetaInterface.TYPE_STRING );
  fields[2].setFormat( "" );
  fields[2].setLength( -1 );
  fields[2].setPrecision( -1 );
  fields[2].setCurrencySymbol( "" );
  fields[2].setDecimalSymbol( "" );
  fields[2].setGroupSymbol( "" );
  fields[2].setTrimType( ValueMetaInterface.TRIM_TYPE_NONE );

  return fields;
}
 
Example 15
Source Project: pentaho-kettle   Source File: CsvInput2IT.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected TextFileInputField[] createTextFileInputFields() {
  TextFileInputField[] fields = new TextFileInputField[3];

  for ( int idx = 0; idx < fields.length; idx++ ) {
    fields[idx] = new TextFileInputField();
  }

  fields[0].setName( "a" );
  fields[0].setType( ValueMetaInterface.TYPE_INTEGER );
  fields[0].setFormat( "" );
  fields[0].setLength( -1 );
  fields[0].setPrecision( -1 );
  fields[0].setCurrencySymbol( "" );
  fields[0].setDecimalSymbol( "" );
  fields[0].setGroupSymbol( "" );
  fields[0].setTrimType( ValueMetaInterface.TRIM_TYPE_NONE );

  fields[1].setName( "b" );
  fields[1].setType( ValueMetaInterface.TYPE_STRING );
  fields[1].setFormat( "" );
  fields[1].setLength( -1 );
  fields[1].setPrecision( -1 );
  fields[1].setCurrencySymbol( "" );
  fields[1].setDecimalSymbol( "" );
  fields[1].setGroupSymbol( "" );
  fields[1].setTrimType( ValueMetaInterface.TRIM_TYPE_NONE );

  fields[2].setName( "c" );
  fields[2].setType( ValueMetaInterface.TYPE_STRING );
  fields[2].setFormat( "" );
  fields[2].setLength( -1 );
  fields[2].setPrecision( -1 );
  fields[2].setCurrencySymbol( "" );
  fields[2].setDecimalSymbol( "" );
  fields[2].setGroupSymbol( "" );
  fields[2].setTrimType( ValueMetaInterface.TRIM_TYPE_NONE );

  return fields;
}
 
Example 16
Source Project: pentaho-kettle   Source File: ParGzipCsvInputMeta.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Object clone() {
  ParGzipCsvInputMeta retval = (ParGzipCsvInputMeta) super.clone();
  int nrFields = inputFields.length;
  retval.allocate( nrFields );
  for ( int i = 0; i < nrFields; i++ ) {
    retval.inputFields[ i ] = (TextFileInputField) inputFields [ i ].clone();
  }
  return retval;
}
 
Example 17
Source Project: pentaho-kettle   Source File: ParGzipCsvInputMeta.java    License: Apache License 2.0 5 votes vote down vote up
private void readData( Node stepnode ) throws KettleXMLException {
  try {
    filename = XMLHandler.getTagValue( stepnode, "filename" );
    filenameField = XMLHandler.getTagValue( stepnode, "filename_field" );
    rowNumField = XMLHandler.getTagValue( stepnode, "rownum_field" );
    includingFilename = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "include_filename" ) );
    delimiter = XMLHandler.getTagValue( stepnode, "separator" );
    enclosure = XMLHandler.getTagValue( stepnode, "enclosure" );
    bufferSize = XMLHandler.getTagValue( stepnode, "buffer_size" );
    headerPresent = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "header" ) );
    lazyConversionActive = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "lazy_conversion" ) );
    isaddresult = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "add_filename_result" ) );
    runningInParallel = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "parallel" ) );
    encoding = XMLHandler.getTagValue( stepnode, "encoding" );

    Node fields = XMLHandler.getSubNode( stepnode, "fields" );
    int nrfields = XMLHandler.countNodes( fields, "field" );

    allocate( nrfields );

    for ( int i = 0; i < nrfields; i++ ) {
      inputFields[i] = new TextFileInputField();

      Node fnode = XMLHandler.getSubNodeByNr( fields, "field", i );

      inputFields[i].setName( XMLHandler.getTagValue( fnode, "name" ) );
      inputFields[i].setType( ValueMetaFactory.getIdForValueMeta( XMLHandler.getTagValue( fnode, "type" ) ) );
      inputFields[i].setFormat( XMLHandler.getTagValue( fnode, "format" ) );
      inputFields[i].setCurrencySymbol( XMLHandler.getTagValue( fnode, "currency" ) );
      inputFields[i].setDecimalSymbol( XMLHandler.getTagValue( fnode, "decimal" ) );
      inputFields[i].setGroupSymbol( XMLHandler.getTagValue( fnode, "group" ) );
      inputFields[i].setLength( Const.toInt( XMLHandler.getTagValue( fnode, "length" ), -1 ) );
      inputFields[i].setPrecision( Const.toInt( XMLHandler.getTagValue( fnode, "precision" ), -1 ) );
      inputFields[i].setTrimType( ValueMetaString.getTrimTypeByCode( XMLHandler.getTagValue( fnode, "trim_type" ) ) );
    }
  } catch ( Exception e ) {
    throw new KettleXMLException( "Unable to load step info from XML", e );
  }
}
 
Example 18
Source Project: pentaho-kettle   Source File: ParGzipCsvInputMeta.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public String getXML() {
  StringBuilder retval = new StringBuilder( 500 );

  retval.append( "    " ).append( XMLHandler.addTagValue( "filename", filename ) );
  retval.append( "    " ).append( XMLHandler.addTagValue( "filename_field", filenameField ) );
  retval.append( "    " ).append( XMLHandler.addTagValue( "rownum_field", rowNumField ) );
  retval.append( "    " ).append( XMLHandler.addTagValue( "include_filename", includingFilename ) );
  retval.append( "    " ).append( XMLHandler.addTagValue( "separator", delimiter ) );
  retval.append( "    " ).append( XMLHandler.addTagValue( "enclosure", enclosure ) );
  retval.append( "    " ).append( XMLHandler.addTagValue( "header", headerPresent ) );
  retval.append( "    " ).append( XMLHandler.addTagValue( "buffer_size", bufferSize ) );
  retval.append( "    " ).append( XMLHandler.addTagValue( "lazy_conversion", lazyConversionActive ) );
  retval.append( "    " ).append( XMLHandler.addTagValue( "add_filename_result", isaddresult ) );
  retval.append( "    " ).append( XMLHandler.addTagValue( "parallel", runningInParallel ) );
  retval.append( "    " ).append( XMLHandler.addTagValue( "encoding", encoding ) );

  retval.append( "    <fields>" ).append( Const.CR );
  for ( int i = 0; i < inputFields.length; i++ ) {
    TextFileInputField field = inputFields[i];

    retval.append( "      <field>" ).append( Const.CR );
    retval.append( "        " ).append( XMLHandler.addTagValue( "name", field.getName() ) );
    retval.append( "        " ).append(
      XMLHandler.addTagValue( "type", ValueMetaFactory.getValueMetaName( field.getType() ) ) );
    retval.append( "        " ).append( XMLHandler.addTagValue( "format", field.getFormat() ) );
    retval.append( "        " ).append( XMLHandler.addTagValue( "currency", field.getCurrencySymbol() ) );
    retval.append( "        " ).append( XMLHandler.addTagValue( "decimal", field.getDecimalSymbol() ) );
    retval.append( "        " ).append( XMLHandler.addTagValue( "group", field.getGroupSymbol() ) );
    retval.append( "        " ).append( XMLHandler.addTagValue( "length", field.getLength() ) );
    retval.append( "        " ).append( XMLHandler.addTagValue( "precision", field.getPrecision() ) );
    retval.append( "        " ).append(
      XMLHandler.addTagValue( "trim_type", ValueMetaString.getTrimTypeCode( field.getTrimType() ) ) );
    retval.append( "      </field>" ).append( Const.CR );
  }
  retval.append( "    </fields>" ).append( Const.CR );

  return retval.toString();
}
 
Example 19
Source Project: pentaho-kettle   Source File: ParGzipCsvInputMeta.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, List<DatabaseMeta> databases ) throws KettleException {
  try {
    filename = rep.getStepAttributeString( id_step, "filename" );
    filenameField = rep.getStepAttributeString( id_step, "filename_field" );
    rowNumField = rep.getStepAttributeString( id_step, "rownum_field" );
    includingFilename = rep.getStepAttributeBoolean( id_step, "include_filename" );
    delimiter = rep.getStepAttributeString( id_step, "separator" );
    enclosure = rep.getStepAttributeString( id_step, "enclosure" );
    headerPresent = rep.getStepAttributeBoolean( id_step, "header" );
    bufferSize = rep.getStepAttributeString( id_step, "buffer_size" );
    lazyConversionActive = rep.getStepAttributeBoolean( id_step, "lazy_conversion" );
    isaddresult = rep.getStepAttributeBoolean( id_step, "add_filename_result" );
    runningInParallel = rep.getStepAttributeBoolean( id_step, "parallel" );
    encoding = rep.getStepAttributeString( id_step, "encoding" );

    int nrfields = rep.countNrStepAttributes( id_step, "field_name" );

    allocate( nrfields );

    for ( int i = 0; i < nrfields; i++ ) {
      inputFields[i] = new TextFileInputField();

      inputFields[i].setName( rep.getStepAttributeString( id_step, i, "field_name" ) );
      inputFields[i].setType( ValueMetaFactory.getIdForValueMeta( rep.getStepAttributeString( id_step, i, "field_type" ) ) );
      inputFields[i].setFormat( rep.getStepAttributeString( id_step, i, "field_format" ) );
      inputFields[i].setCurrencySymbol( rep.getStepAttributeString( id_step, i, "field_currency" ) );
      inputFields[i].setDecimalSymbol( rep.getStepAttributeString( id_step, i, "field_decimal" ) );
      inputFields[i].setGroupSymbol( rep.getStepAttributeString( id_step, i, "field_group" ) );
      inputFields[i].setLength( (int) rep.getStepAttributeInteger( id_step, i, "field_length" ) );
      inputFields[i].setPrecision( (int) rep.getStepAttributeInteger( id_step, i, "field_precision" ) );
      inputFields[i].setTrimType( ValueMetaString.getTrimTypeByCode( rep.getStepAttributeString(
        id_step, i, "field_trim_type" ) ) );
    }
  } catch ( Exception e ) {
    throw new KettleException( "Unexpected error reading step information from the repository", e );
  }
}
 
Example 20
Source Project: pentaho-kettle   Source File: ParGzipCsvInputMeta.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transformation, ObjectId id_step ) throws KettleException {
  try {
    rep.saveStepAttribute( id_transformation, id_step, "filename", filename );
    rep.saveStepAttribute( id_transformation, id_step, "filename_field", filenameField );
    rep.saveStepAttribute( id_transformation, id_step, "rownum_field", rowNumField );
    rep.saveStepAttribute( id_transformation, id_step, "include_filename", includingFilename );
    rep.saveStepAttribute( id_transformation, id_step, "separator", delimiter );
    rep.saveStepAttribute( id_transformation, id_step, "enclosure", enclosure );
    rep.saveStepAttribute( id_transformation, id_step, "buffer_size", bufferSize );
    rep.saveStepAttribute( id_transformation, id_step, "header", headerPresent );
    rep.saveStepAttribute( id_transformation, id_step, "lazy_conversion", lazyConversionActive );
    rep.saveStepAttribute( id_transformation, id_step, "add_filename_result", isaddresult );
    rep.saveStepAttribute( id_transformation, id_step, "parallel", runningInParallel );
    rep.saveStepAttribute( id_transformation, id_step, "encoding", encoding );

    for ( int i = 0; i < inputFields.length; i++ ) {
      TextFileInputField field = inputFields[i];

      rep.saveStepAttribute( id_transformation, id_step, i, "field_name", field.getName() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_type",
        ValueMetaFactory.getValueMetaName( field.getType() ) );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_format", field.getFormat() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_currency", field.getCurrencySymbol() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_decimal", field.getDecimalSymbol() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_group", field.getGroupSymbol() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_length", field.getLength() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_precision", field.getPrecision() );
      rep.saveStepAttribute( id_transformation, id_step, i, "field_trim_type",
        ValueMetaString.getTrimTypeCode( field.getTrimType() ) );
    }
  } catch ( Exception e ) {
    throw new KettleException( "Unable to save step information to the repository for id_step=" + id_step, e );
  }
}
 
Example 21
Source Project: pentaho-kettle   Source File: CsvInputMeta.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Object clone() {
  final CsvInputMeta retval = (CsvInputMeta) super.clone();
  retval.inputFields = new TextFileInputField[ inputFields.length ];
  for ( int i = 0; i < inputFields.length; i++ ) {
    retval.inputFields[ i ] = (TextFileInputField) inputFields[ i ].clone();
  }
  return retval;
}
 
Example 22
Source Project: pentaho-kettle   Source File: S3CsvInputMeta.java    License: Apache License 2.0 5 votes vote down vote up
private void readData( Node stepnode ) throws KettleXMLException {
  try {
    awsAccessKey = Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( stepnode, "aws_access_key" ) );
    awsSecretKey = Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( stepnode, "aws_secret_key" ) );
    bucket = XMLHandler.getTagValue( stepnode, "bucket" );
    filename = XMLHandler.getTagValue( stepnode, "filename" );
    filenameField = XMLHandler.getTagValue( stepnode, "filename_field" );
    rowNumField = XMLHandler.getTagValue( stepnode, "rownum_field" );
    includingFilename = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "include_filename" ) );
    delimiter = XMLHandler.getTagValue( stepnode, "separator" );
    enclosure = XMLHandler.getTagValue( stepnode, "enclosure" );
    maxLineSize = XMLHandler.getTagValue( stepnode, "max_line_size" );
    headerPresent = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "header" ) );
    lazyConversionActive = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "lazy_conversion" ) );
    runningInParallel = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "parallel" ) );
    Node fields = XMLHandler.getSubNode( stepnode, "fields" );
    int nrfields = XMLHandler.countNodes( fields, "field" );

    allocate( nrfields );

    for ( int i = 0; i < nrfields; i++ ) {
      inputFields[i] = new TextFileInputField();

      Node fnode = XMLHandler.getSubNodeByNr( fields, "field", i );

      inputFields[i].setName( XMLHandler.getTagValue( fnode, "name" ) );
      inputFields[i].setType( ValueMetaFactory.getIdForValueMeta( XMLHandler.getTagValue( fnode, "type" ) ) );
      inputFields[i].setFormat( XMLHandler.getTagValue( fnode, "format" ) );
      inputFields[i].setCurrencySymbol( XMLHandler.getTagValue( fnode, "currency" ) );
      inputFields[i].setDecimalSymbol( XMLHandler.getTagValue( fnode, "decimal" ) );
      inputFields[i].setGroupSymbol( XMLHandler.getTagValue( fnode, "group" ) );
      inputFields[i].setLength( Const.toInt( XMLHandler.getTagValue( fnode, "length" ), -1 ) );
      inputFields[i].setPrecision( Const.toInt( XMLHandler.getTagValue( fnode, "precision" ), -1 ) );
      inputFields[i].setTrimType( ValueMetaString.getTrimTypeByCode( XMLHandler.getTagValue( fnode, "trim_type" ) ) );
    }
  } catch ( Exception e ) {
    throw new KettleXMLException( "Unable to load step info from XML", e );
  }
}
 
Example 23
Source Project: pentaho-kettle   Source File: S3CsvInputMeta.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, List<DatabaseMeta> databases )
    throws KettleException {
  try {
    awsAccessKey = Encr.decryptPasswordOptionallyEncrypted( rep.getStepAttributeString( id_step, "aws_access_key" ) );
    awsSecretKey = Encr.decryptPasswordOptionallyEncrypted( rep.getStepAttributeString( id_step, "aws_secret_key" ) );
    bucket = rep.getStepAttributeString( id_step, "bucket" );
    filename = rep.getStepAttributeString( id_step, "filename" );
    filenameField = rep.getStepAttributeString( id_step, "filename_field" );
    rowNumField = rep.getStepAttributeString( id_step, "rownum_field" );
    includingFilename = rep.getStepAttributeBoolean( id_step, "include_filename" );
    delimiter = rep.getStepAttributeString( id_step, "separator" );
    enclosure = rep.getStepAttributeString( id_step, "enclosure" );
    headerPresent = rep.getStepAttributeBoolean( id_step, "header" );
    maxLineSize = rep.getStepAttributeString( id_step, "max_line_size" );
    lazyConversionActive = rep.getStepAttributeBoolean( id_step, "lazy_conversion" );
    runningInParallel = rep.getStepAttributeBoolean( id_step, "parallel" );
    int nrfields = rep.countNrStepAttributes( id_step, "field_name" );

    allocate( nrfields );

    for ( int i = 0; i < nrfields; i++ ) {
      inputFields[i] = new TextFileInputField();

      inputFields[i].setName( rep.getStepAttributeString( id_step, i, "field_name" ) );
      inputFields[i].setType( ValueMetaFactory.getIdForValueMeta( rep.getStepAttributeString( id_step, i, "field_type" ) ) );
      inputFields[i].setFormat( rep.getStepAttributeString( id_step, i, "field_format" ) );
      inputFields[i].setCurrencySymbol( rep.getStepAttributeString( id_step, i, "field_currency" ) );
      inputFields[i].setDecimalSymbol( rep.getStepAttributeString( id_step, i, "field_decimal" ) );
      inputFields[i].setGroupSymbol( rep.getStepAttributeString( id_step, i, "field_group" ) );
      inputFields[i].setLength( (int) rep.getStepAttributeInteger( id_step, i, "field_length" ) );
      inputFields[i].setPrecision( (int) rep.getStepAttributeInteger( id_step, i, "field_precision" ) );
      inputFields[i].setTrimType( ValueMetaString.getTrimTypeByCode( rep.getStepAttributeString( id_step, i,
          "field_trim_type" ) ) );
    }
  } catch ( Exception e ) {
    throw new KettleException( "Unexpected error reading step information from the repository", e );
  }
}
 
Example 24
@Override public TextFileInputField getTestObject() {
  TextFileInputField rtn = new TextFileInputField( UUID.randomUUID().toString(), rand.nextInt(), rand.nextInt() );
  rtn.setType( rand.nextInt( 7 ) );
  rtn.setCurrencySymbol( UUID.randomUUID().toString() );
  rtn.setDecimalSymbol( UUID.randomUUID().toString() );
  rtn.setFormat( UUID.randomUUID().toString() );
  rtn.setGroupSymbol( UUID.randomUUID().toString() );
  rtn.setIfNullValue( UUID.randomUUID().toString() );
  rtn.setIgnored( rand.nextBoolean() );
  rtn.setNullString( UUID.randomUUID().toString() );
  rtn.setPrecision( rand.nextInt( 9 ) );
  rtn.setRepeated( rand.nextBoolean() );
  rtn.setTrimType( rand.nextInt( ValueMetaString.trimTypeDesc.length ) );
  return rtn;
}
 
Example 25
@Override
public boolean validateTestObject( TextFileInputField testObject, Object actual ) {
  if ( !( actual instanceof TextFileInputField ) ) {
    return false;
  }

  TextFileInputField another = (TextFileInputField) actual;
  return new EqualsBuilder()
    .append( testObject.getName(), another.getName() )
    .append( testObject.getTypeDesc(), another.getTypeDesc() )
    .append( testObject.getFormat(), another.getFormat() )
    .append( testObject.getCurrencySymbol(), another.getCurrencySymbol() )
    .append( testObject.getDecimalSymbol(), another.getDecimalSymbol() )
    .append( testObject.getGroupSymbol(), another.getGroupSymbol() )
    .append( testObject.getLength(), another.getLength() )
    .append( testObject.getPrecision(), another.getPrecision() )
    .append( testObject.getTrimTypeCode(), another.getTrimTypeCode() )
    // These fields not universally serialized, so don't check for them
    // TextFileInputMeta serializes some that CsvInputMeta doesn't. Given
    // they're both deprecated, ignore these.
    //.append( testObject.isRepeated(), another.isRepeated() )
    //.append( testObject.getNullString(), another.getNullString() )
    //.append( testObject.getIfNullValue(), another.getIfNullValue() )
    //.append( testObject.getPosition(), another.getPosition() )
    .isEquals();


}
 
Example 26
Source Project: pentaho-kettle   Source File: CsvInputTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFileIsReleasedAfterProcessing() throws Exception {
  // Create a file with some content to be processed
  TextFileInputField[] inputFileFields = createInputFileFields( "f1", "f2", "f3" );
  String fileContents = "Something" + DELIMITER + "" + DELIMITER + "The former was empty!";
  File tmpFile = createTestFile( ENCODING, fileContents );

  // Create and configure the step
  CsvInputMeta meta = createMeta( tmpFile, inputFileFields );
  CsvInputData data = new CsvInputData();
  RowSet output = new QueueRowSet();
  CsvInput csvInput =
    new CsvInput( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta,
      stepMockHelper.trans );
  csvInput.init( meta, data );
  csvInput.addRowSetToOutputRowSets( output );

  // Start processing
  csvInput.processRow( meta, data );

  // Finish processing
  csvInput.dispose( meta, data );

  // And now the file must be free to be deleted
  assertTrue( tmpFile.delete() );
  assertFalse( tmpFile.exists() );
}
 
Example 27
@Override
CsvInputMeta createMeta( File file, TextFileInputField[] fields ) {
  CsvInputMeta meta = super.createMeta( file, fields );
  meta.setDelimiter( "delimiter" );
  //Buffer cannot be less than ( delimiter length - 1) * 2 ) due to buffer resize logic in readBufferFromFile method
  //This buffer size causes special case where the delimiter spans past the end of the buffer.
  meta.setBufferSize( "16" );
  return meta;
}
 
Example 28
Source Project: pentaho-kettle   Source File: CsvInputUnitTestBase.java    License: Apache License 2.0 5 votes vote down vote up
CsvInputMeta createMeta( File file, TextFileInputField[] fields ) {
  CsvInputMeta meta = new CsvInputMeta();
  meta.setFilename( file.getAbsolutePath() );
  meta.setBufferSize( BUFFER_SIZE );
  meta.setDelimiter( DELIMITER );
  meta.setEnclosure( ENCLOSURE );
  meta.setEncoding( ENCODING );
  meta.setInputFields( fields );
  meta.setHeaderPresent( false );
  return meta;
}
 
Example 29
Source Project: pentaho-kettle   Source File: CsvInputContentParsingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDefaultOptions() throws Exception {
  init( "default.csv" );

  setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ),
      new TextFileInputField( "Field 3", -1, -1 ) );

  process();

  check( new Object[][] { { "first", "1", "1.1" }, { "second", "2", "2.2" }, { "third", "3", "3.3" } } );
}
 
Example 30
Source Project: pentaho-kettle   Source File: CsvInputContentParsingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testColumnNameWithSpaces() throws Exception {
  init( "column_name_with_spaces.csv" );

  setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ),
      new TextFileInputField( "Field 3", -1, -1 ) );

  process();

  check( new Object[][] { { "first", "1", "1.1" }, { "second", "2", "2.2" }, { "third", "3", "3.3" } } );
}