org.pentaho.di.trans.steps.textfileinput.TextFileInputField Java Examples

The following examples show how to use org.pentaho.di.trans.steps.textfileinput.TextFileInputField. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TextFileInputCSVImportProgressDialogTest.java    From pentaho-kettle with Apache License 2.0 6 votes vote down vote up
private void initiateVariables( String fileContent, TextFileInputField[] inputFileFields, int fileFormat ) throws Exception {
  //reader
  reader = new InputStreamReader( new ByteArrayInputStream( fileContent.getBytes( StandardCharsets.UTF_8 ) ) );
  //meta
  when( meta.getInputFields() ).thenReturn( inputFileFields );
  doCallRealMethod().when( meta ).getFields( any( RowMetaInterface.class ), anyString(), any( RowMetaInterface[].class ),
    any( StepMeta.class ), any( VariableSpace.class ), any( Repository.class ), any( IMetaStore.class ) );
  setInternalState( meta, "inputFields",  inputFileFields );
  when( meta.hasHeader() ).thenReturn( true );
  when( meta.getNrHeaderLines() ).thenReturn( 1 );
  when( meta.clone() ).thenReturn( meta );
  when( meta.getSeparator() ).thenReturn( "," );
  when( meta.getEnclosure() ).thenReturn( "\"" );
  when( meta.getFileFormatTypeNr() ).thenReturn( fileFormat );
  when( meta.getFileType() ).thenReturn( "CSV" );
  when( meta.getFilePaths( any( VariableSpace.class ) ) ).thenReturn( new String[] {"empty"} );
  //transmeta
  when( transMeta.environmentSubstitute( "," ) ).thenReturn( "," );
  when( transMeta.environmentSubstitute( "\"" ) ).thenReturn( "\"" );
  when( transMeta.getObjectType() ).thenReturn( LoggingObjectType.TRANSMETA );
  //textFileCSVImportProgressDialog
  textFileCSVImportProgressDialog = spy( new TextFileCSVImportProgressDialog(
    shell, meta, transMeta, reader, 100, true ) );
}
 
Example #2
Source File: ParGzipCsvInputMetaTest.java    From pentaho-kettle with Apache License 2.0 6 votes vote down vote up
@Before
public void setUpLoadSave() throws Exception {
  KettleEnvironment.init();
  PluginRegistry.init( false );
  List<String> attributes =
      Arrays.asList( "filename", "filenameField", "includingFilename", "rowNumField", "headerPresent", "delimiter",
          "enclosure", "bufferSize", "lazyConversionActive", "addResultFile", "runningInParallel", "encoding",
          "inputFields" );

  Map<String, String> getterMap = new HashMap<String, String>();
  Map<String, String> setterMap = new HashMap<String, String>();

  Map<String, FieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();
  attrValidatorMap.put( "inputFields", new ArrayLoadSaveValidator<TextFileInputField>( new TextFileInputFieldValidator(), 5 ) );

  Map<String, FieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();

  loadSaveTester =
      new LoadSaveTester( testMetaClass, attributes, new ArrayList<String>(), new ArrayList<String>(), getterMap,
          setterMap, attrValidatorMap, typeValidatorMap, this );
}
 
Example #3
Source File: ParGzipCsvInputMetaTest.java    From pentaho-kettle with Apache License 2.0 6 votes vote down vote up
@Override
public TextFileInputField getTestObject() {
  TextFileInputField rtn = new TextFileInputField();
  rtn.setCurrencySymbol( UUID.randomUUID().toString() );
  rtn.setDecimalSymbol( UUID.randomUUID().toString() );
  rtn.setFormat( UUID.randomUUID().toString() );
  rtn.setGroupSymbol( UUID.randomUUID().toString() );
  rtn.setName( UUID.randomUUID().toString() );
  rtn.setTrimType( rand.nextInt( 4 ) );
  rtn.setPrecision( rand.nextInt( 9 ) );
  rtn.setLength( rand.nextInt( 50 ) );
  rtn.setType( rand.nextInt( 7 ) );
  // Note - these fields aren't serialized by the meta class ... cannot test for them
  // rtn.setRepeated( rand.nextBoolean() );
  // rtn.setSamples( new String[] { UUID.randomUUID().toString(), UUID.randomUUID().toString(),
  // UUID.randomUUID().toString() } );
  // rtn.setNullString( UUID.randomUUID().toString() );
  // rtn.setIfNullValue( UUID.randomUUID().toString() );
  // rtn.setIgnored( rand.nextBoolean() );
  // rtn.setPosition( rand.nextInt( 10 ) );
  return rtn;
}
 
Example #4
Source File: GoogleSpreadsheetInputMeta.java    From pdi-google-spreadsheet-plugin with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Override
public void saveRep(Repository rep, IMetaStore metaStore, ObjectId id_transformation, ObjectId id_step) throws KettleException {
    try {
        rep.saveStepAttribute(id_transformation, id_step, "serviceEmail", this.serviceEmail);
        rep.saveStepAttribute(id_transformation, id_step, "spreadsheetKey", this.spreadsheetKey);
        rep.saveStepAttribute(id_transformation, id_step, "worksheetId", this.worksheetId);
        rep.saveStepAttribute(id_transformation, id_step, "privateKeyStore", GoogleSpreadsheet.base64EncodePrivateKeyStore(this.privateKeyStore));

        for (int i = 0; i < inputFields.length; i++) {
            TextFileInputField field = inputFields[i];

            rep.saveStepAttribute(id_transformation, id_step, i, "field_name", field.getName());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_type", ValueMeta.getTypeDesc(field.getType()));
            rep.saveStepAttribute(id_transformation, id_step, i, "field_format", field.getFormat());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_currency", field.getCurrencySymbol());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_decimal", field.getDecimalSymbol());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_group", field.getGroupSymbol());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_length", field.getLength());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_precision", field.getPrecision());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_trim_type", ValueMeta.getTrimTypeCode(field.getTrimType()));
        }

    } catch (Exception e) {
        throw new KettleException("Unable to save step information to the repository for id_step=" + id_step, e);
    }
}
 
Example #5
Source File: ParGzipCsvInputMetaTest.java    From pentaho-kettle with Apache License 2.0 6 votes vote down vote up
@Override
public boolean validateTestObject( TextFileInputField testObject, Object actual ) {
  if ( !( actual instanceof TextFileInputField ) ) {
    return false;
  }

  TextFileInputField another = (TextFileInputField) actual;
  return new EqualsBuilder()
      .append( testObject.getCurrencySymbol(), another.getCurrencySymbol() )
      .append( testObject.getDecimalSymbol(), another.getDecimalSymbol() )
      .append( testObject.getFormat(), another.getFormat() )
      .append( testObject.getGroupSymbol(), another.getGroupSymbol() )
      .append( testObject.getName(), another.getName() )
      .append( testObject.getTrimType(), another.getTrimType() )
      .append( testObject.getPrecision(), another.getPrecision() )
      .append( testObject.getLength(), another.getLength() )
      .append( testObject.getType(), another.getType() )
      // Note - these fields aren't serialized by the meta class ... cannot test for them
      // .append( testObject.isRepeated(), another.isRepeated() )
      // .append( testObject.getSamples(), another.getSamples() )
      // .append( testObject.getNullString(), another.getNullString() )
      // .append( testObject.getIfNullValue(), another.getIfNullValue() )
      // .append( testObject.isIgnored(), another.isIgnored() )
      // .append( testObject.getPosition(), another.getPosition() )
      .isEquals();
}
 
Example #6
Source File: CsvInputMetaTest.java    From pentaho-kettle with Apache License 2.0 6 votes vote down vote up
@Test
public void testClone() {
  final CsvInputMeta original = new CsvInputMeta();
  original.setDelimiter( ";" );
  original.setEnclosure( "'" );
  final TextFileInputField[] originalFields = new TextFileInputField[ 1 ];
  final TextFileInputField originalField = new TextFileInputField();
  originalField.setName( "field" );
  originalFields[ 0 ] = originalField;
  original.setInputFields( originalFields );

  final CsvInputMeta clone = (CsvInputMeta) original.clone();
  // verify that the clone and its input fields are "equal" to the originals, but not the same objects
  Assert.assertNotSame( original, clone );
  Assert.assertEquals( original.getDelimiter(), clone.getDelimiter() );
  Assert.assertEquals( original.getEnclosure(), clone.getEnclosure() );

  Assert.assertNotSame( original.getInputFields(), clone.getInputFields() );
  Assert.assertNotSame( original.getInputFields()[ 0 ], clone.getInputFields()[ 0 ] );
  Assert.assertEquals( original.getInputFields()[ 0 ].getName(), clone.getInputFields()[ 0 ].getName() );
}
 
Example #7
Source File: TextFileInputCSVImportProgressDialogTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
private TextFileInputField[] createInputFileFields( String... names ) {
  TextFileInputField[] fields = new TextFileInputField[ names.length ];
  for ( int i = 0; i < names.length; i++ ) {
    fields[ i ] = createField( names[ i ] );
  }
  return fields;
}
 
Example #8
Source File: TextFileInputCSVImportProgressDialogTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
private void fileContentTest( String fileContent, TextFileInputField[] inputFileFields, int fileFormat ) throws Exception {
  initiateVariables( fileContent, inputFileFields, fileFormat );
  String result = textFileCSVImportProgressDialog.doScan( monitor );

  assertEquals( ValueMetaFactory.getValueMetaName( meta.getInputFields()[0].getType() ), inputFileFields[0].getName() );
  assertEquals( ValueMetaFactory.getValueMetaName( meta.getInputFields()[1].getType() ), inputFileFields[1].getName() );
  assertEquals( ValueMetaFactory.getValueMetaName( meta.getInputFields()[2].getType() ), inputFileFields[2].getName() );
  assertEquals( ValueMetaFactory.getValueMetaName( meta.getInputFields()[3].getType() ), inputFileFields[3].getName() );
}
 
Example #9
Source File: TextFileInputCSVImportProgressDialogTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void fileMixFormatTestWithEnclosures() throws Exception {
  TextFileInputField[] inputFileFields = createInputFileFields( "String", "Number", "Boolean", "Date" );
  String fileContent = "String, int, boolean, date\r\n"
    + "\"レコ\n"
    + "ード名1\",1.7976E308,true,9999/12/31 23:59:59.999,\n"
    + "\"あ\r\n"
    + "a1\",123456789,false,2016/1/5 12:00:00.000\n";

  fileContentTest( fileContent, inputFileFields, TextFileInputMeta.FILE_FORMAT_MIXED );
}
 
Example #10
Source File: TextFileInputCSVImportProgressDialogTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void fileMixFormatTestWithNoEnclosures() throws Exception {
  TextFileInputField[] inputFileFields = createInputFileFields( "String", "Number", "Boolean", "Date" );
  String fileContent = "String, int, boolean, date\n"
    + "レコード名1,1.7976E308,true,9999/12/31 23:59:59.999,\r\n"
    + "\"あa1\",123456789,false,2016/1/5 12:00:00.000\r\n";

  fileContentTest( fileContent, inputFileFields, TextFileInputMeta.FILE_FORMAT_MIXED );
}
 
Example #11
Source File: TextFileInputCSVImportProgressDialogTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void fileUnixFormatTestWithNoEnclosures() throws Exception {
  TextFileInputField[] inputFileFields = createInputFileFields( "String", "Number", "Boolean", "Date" );
  String fileContent = "String, int, boolean, date\n"
    + "レコード名1,1.7976E308,true,9999/12/31 23:59:59.999,\n"
    + "\"あa1\",123456789,false,2016/1/5 12:00:00.000\n";

  fileContentTest( fileContent, inputFileFields, TextFileInputMeta.FILE_FORMAT_UNIX );
}
 
Example #12
Source File: TextFileInputCSVImportProgressDialogTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void fileDOSFormatTestWithEnclosures() throws Exception {
  TextFileInputField[] inputFileFields = createInputFileFields( "String", "Number", "Boolean", "Date" );
  String fileContent = "String, int, boolean, date\r\n"
    + "\"レコ\r\n"
    + "ード名1\",1.7976E308,true,9999/12/31 23:59:59.999,\r\n"
    + "\"あ\r\n"
    + "a1\",123456789,false,2016/1/5 12:00:00.000\r\n";

  fileContentTest( fileContent, inputFileFields, TextFileInputMeta.FILE_FORMAT_DOS );
}
 
Example #13
Source File: CsvInputContentParsingTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test( expected = KettleStepException.class )
public void testNoHeaderOptions() throws Exception {
  meta.setHeaderPresent( false );
  init( "default.csv" );

  setFields( new TextFileInputField(), new TextFileInputField(), new TextFileInputField() );

  process();
}
 
Example #14
Source File: CsvInputContentParsingTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void testDosFileFormat() throws Exception {
  String data = "データ1,データ2,データ3,データ4\r\n"
    + "111,\"a\r\n"
    + "bc\",あいう,さしす\r\n"
    + "222,def,かきく,たちつ\r\n"
    + "333,,かきく,たちつ\r\n"
    + "444,,\r\n"
    + "555,かきく,\r\n"
    + "666,かきく\r\n"
    + "\r\n"
    + "777,\r\n"
    + "888,かきく\r\n"
    + "\r\n"
    + "999,123,123,123,132,132,132,132,132\r\n";

  String file = createTestFile( "UTF-8", data ).getAbsolutePath();
  meta.setFileFormat( "DOS" );
  init( file, true );

  setFields( new TextFileInputField( "Col 1", -1, -1 ), new TextFileInputField( "Col 2", -1, -1 ),
    new TextFileInputField( "Col 3", -1, -1 ), new TextFileInputField( "Col 4", -1, -1 ),
    new TextFileInputField( "Col 5", -1, -1 ) );

  process();

  check( new Object[][] {
    { "111", "a\r\nbc", "あいう", "さしす", null },
    { "222", "def", "かきく", "たちつ", null},
    { "333", "", "かきく", "たちつ", null },
    { "444", "", "", null, null },
    { "555", "かきく", "", null, null },
    { "666", "かきく", null, null, null},
    { },
    { "777", "", null, null, null },
    { "888", "かきく", null, null, null },
    { },
    { "999", "123", "123", "123", "132" } }
  );
}
 
Example #15
Source File: CsvInputContentParsingTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void testSemicolonOptions() throws Exception {
  meta.setDelimiter( ";" );
  init( "semicolon.csv" );

  setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ),
      new TextFileInputField( "Field 3", -1, -1 ) );

  process();

  check( new Object[][] { { "first", "1", "1.1" }, { "second", "2", "2.2" }, { "third", "3", "3.3" }, {
      "\u043d\u0435-\u043b\u0430\u0446\u0456\u043d\u043a\u0430(non-latin)", "4", "4" } } );
}
 
Example #16
Source File: CsvInputContentParsingTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void testEnclosures() throws Exception {
  meta.setDelimiter( ";" );
  meta.setEnclosure( "'" );
  init( "enclosures.csv" );

  setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ),
    new TextFileInputField( "Field 3", -1, -1 ) );

  process();

  check( new Object[][] { { "1", "This line is un-even enclosure-wise because I'm using an escaped enclosure", "a" },
    { "2", "Test isn't even\nhere", "b" } } );
}
 
Example #17
Source File: S3CsvInputMetaTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialization() throws KettleException {
  List<String> attributes = Arrays.asList( "AwsAccessKey", "AwsSecretKey", "Bucket", "Filename", "FilenameField",
    "RowNumField", "IncludingFilename", "Delimiter", "Enclosure", "HeaderPresent", "MaxLineSize",
    "LazyConversionActive", "RunningInParallel", "InputFields" );

  Map<String, FieldLoadSaveValidator<?>> typeMap = new HashMap<>();
  typeMap.put( TextFileInputField[].class.getCanonicalName(),
    new ArrayLoadSaveValidator<>( new TextFileInputFieldValidator() ) );
  Map<String, String> getterMap = new HashMap<>();
  Map<String, String> setterMap = new HashMap<>();
  LoadSaveTester<S3CsvInputMeta> tester = new LoadSaveTester<>( S3CsvInputMeta.class, attributes,
    getterMap, setterMap, new HashMap<String, FieldLoadSaveValidator<?>>(), typeMap );
  tester.testSerialization();
}
 
Example #18
Source File: CsvInputMetaTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Before
public void setUpLoadSave() throws Exception {
  KettleEnvironment.init();
  PluginRegistry.init( false );
  List<String> attributes =
      Arrays.asList( "BufferSize", "Delimiter", "Enclosure", "Encoding", "Filename", "FilenameField", "InputFields", "RowNumField",
          "AddResultFile", "HeaderPresent", "IncludingFilename", "LazyConversionActive", "NewlinePossibleInFields", "RunningInParallel" );

  Map<String, String> getterMap = new HashMap<String, String>() {
    {
      put( "hasHeader", "hasHeader" );
      put( "includeFilename", "includeFilename" );
      put( "includeRowNumber", "includeRowNumber" );
    }
  };
  Map<String, String> setterMap = new HashMap<String, String>() {
    {
      put( "includeFilename", "includeFilename" );
      put( "includeRowNumber", "includeRowNumber" );
    }
  };
  FieldLoadSaveValidator<String[]> stringArrayLoadSaveValidator =
      new ArrayLoadSaveValidator<String>( new StringLoadSaveValidator(), 5 );


  Map<String, FieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();
  attrValidatorMap.put( "InputFields",
      new ArrayLoadSaveValidator<TextFileInputField>( new TextFileInputFieldValidator(), 5 ) );

  Map<String, FieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();

  loadSaveTester =
      new LoadSaveTester( testMetaClass, attributes, new ArrayList<String>(), new ArrayList<String>(),
          getterMap, setterMap, attrValidatorMap, typeValidatorMap, this );
}
 
Example #19
Source File: GoogleSpreadsheetInputMeta.java    From pdi-google-spreadsheet-plugin with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public void setDefault() {
    this.serviceEmail = "";
    this.spreadsheetKey = "";
    this.worksheetId = "od6";
    this.privateKeyStore = null;

    TextFileInputField field = new TextFileInputField();
    field.setName("field");
    field.setType(ValueMetaInterface.TYPE_STRING);

    inputFields = new TextFileInputField[]{
            field,
    };
}
 
Example #20
Source File: TextFileInputCSVImportProgressDialogTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void fileDOSFormatTestWithNoEnclosures() throws Exception {
  TextFileInputField[] inputFileFields = createInputFileFields( "String", "Number", "Boolean", "Date" );
  String fileContent = "String, int, boolean, date\r\n"
    + "レコード名1,1.7976E308,true,9999/12/31 23:59:59.999,\r\n"
    + "\"あa1\",123456789,false,2016/1/5 12:00:00.000\r\n";

  fileContentTest( fileContent, inputFileFields, TextFileInputMeta.FILE_FORMAT_DOS );
}
 
Example #21
Source File: CsvInputContentParsingTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void testMultiCharDelimOptions() throws Exception {
  meta.setDelimiter( "|||" );
  init( "multi_delim.csv" );

  setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ),
      new TextFileInputField( "Field 3", -1, -1 ) );

  process();

  check( new Object[][] { { "first", "1", "1.1" }, { "second", "2", "2.2" }, { "third", "3", "3.3" }, {
      "\u043d\u0435-\u043b\u0430\u0446\u0456\u043d\u043a\u0430(non-latin)", "4", "4" } } );
}
 
Example #22
Source File: CsvInputContentParsingTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void testUnixFileFormat() throws Exception {
  String data = "データ1,データ2,データ3,データ4\n"
    + "111,\"a\n"
    + "bc\",\n"
    + "\n"
    + "444,,\n"
    + "555,かきく,\n"
    + "\n"
    + "\n"
    + "777,\n"
    + "888,かきく\n"
    + "999,123,123,123,132,132,132,132,132\n";

  String file = createTestFile( "UTF-8", data ).getAbsolutePath();
  meta.setFileFormat( "Unix" );
  init( file, true );

  setFields( new TextFileInputField( "Col 1", -1, -1 ), new TextFileInputField( "Col 2", -1, -1 ),
    new TextFileInputField( "Col 3", -1, -1 ), new TextFileInputField( "Col 4", -1, -1 ),
    new TextFileInputField( "Col 5", -1, -1 ) );

  process();

  check( new Object[][] {
    { "111", "a\nbc", "", null, null },
    { },
    { "444", "", "", null, null },
    { "555", "かきく", "", null, null },
    { },
    { },
    { "777", "", null, null, null },
    { "888", "かきく", null, null, null },
    { "999", "123", "123", "123", "132" } }
  );
}
 
Example #23
Source File: CsvInputContentParsingTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void testColumnNameWithSpaces() throws Exception {
  init( "column_name_with_spaces.csv" );

  setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ),
      new TextFileInputField( "Field 3", -1, -1 ) );

  process();

  check( new Object[][] { { "first", "1", "1.1" }, { "second", "2", "2.2" }, { "third", "3", "3.3" } } );
}
 
Example #24
Source File: CsvInputContentParsingTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void testDefaultOptions() throws Exception {
  init( "default.csv" );

  setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ),
      new TextFileInputField( "Field 3", -1, -1 ) );

  process();

  check( new Object[][] { { "first", "1", "1.1" }, { "second", "2", "2.2" }, { "third", "3", "3.3" } } );
}
 
Example #25
Source File: CsvInputUnitTestBase.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
CsvInputMeta createMeta( File file, TextFileInputField[] fields ) {
  CsvInputMeta meta = new CsvInputMeta();
  meta.setFilename( file.getAbsolutePath() );
  meta.setBufferSize( BUFFER_SIZE );
  meta.setDelimiter( DELIMITER );
  meta.setEnclosure( ENCLOSURE );
  meta.setEncoding( ENCODING );
  meta.setInputFields( fields );
  meta.setHeaderPresent( false );
  return meta;
}
 
Example #26
Source File: CsvInputMultiCharDelimiterTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Override
CsvInputMeta createMeta( File file, TextFileInputField[] fields ) {
  CsvInputMeta meta = super.createMeta( file, fields );
  meta.setDelimiter( "delimiter" );
  //Buffer cannot be less than ( delimiter length - 1) * 2 ) due to buffer resize logic in readBufferFromFile method
  //This buffer size causes special case where the delimiter spans past the end of the buffer.
  meta.setBufferSize( "16" );
  return meta;
}
 
Example #27
Source File: CsvInputTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Test
public void testFileIsReleasedAfterProcessing() throws Exception {
  // Create a file with some content to be processed
  TextFileInputField[] inputFileFields = createInputFileFields( "f1", "f2", "f3" );
  String fileContents = "Something" + DELIMITER + "" + DELIMITER + "The former was empty!";
  File tmpFile = createTestFile( ENCODING, fileContents );

  // Create and configure the step
  CsvInputMeta meta = createMeta( tmpFile, inputFileFields );
  CsvInputData data = new CsvInputData();
  RowSet output = new QueueRowSet();
  CsvInput csvInput =
    new CsvInput( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta,
      stepMockHelper.trans );
  csvInput.init( meta, data );
  csvInput.addRowSetToOutputRowSets( output );

  // Start processing
  csvInput.processRow( meta, data );

  // Finish processing
  csvInput.dispose( meta, data );

  // And now the file must be free to be deleted
  assertTrue( tmpFile.delete() );
  assertFalse( tmpFile.exists() );
}
 
Example #28
Source File: TextFileInputFieldValidator.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Override
public boolean validateTestObject( TextFileInputField testObject, Object actual ) {
  if ( !( actual instanceof TextFileInputField ) ) {
    return false;
  }

  TextFileInputField another = (TextFileInputField) actual;
  return new EqualsBuilder()
    .append( testObject.getName(), another.getName() )
    .append( testObject.getTypeDesc(), another.getTypeDesc() )
    .append( testObject.getFormat(), another.getFormat() )
    .append( testObject.getCurrencySymbol(), another.getCurrencySymbol() )
    .append( testObject.getDecimalSymbol(), another.getDecimalSymbol() )
    .append( testObject.getGroupSymbol(), another.getGroupSymbol() )
    .append( testObject.getLength(), another.getLength() )
    .append( testObject.getPrecision(), another.getPrecision() )
    .append( testObject.getTrimTypeCode(), another.getTrimTypeCode() )
    // These fields not universally serialized, so don't check for them
    // TextFileInputMeta serializes some that CsvInputMeta doesn't. Given
    // they're both deprecated, ignore these.
    //.append( testObject.isRepeated(), another.isRepeated() )
    //.append( testObject.getNullString(), another.getNullString() )
    //.append( testObject.getIfNullValue(), another.getIfNullValue() )
    //.append( testObject.getPosition(), another.getPosition() )
    .isEquals();


}
 
Example #29
Source File: TextFileInputFieldValidator.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Override public TextFileInputField getTestObject() {
  TextFileInputField rtn = new TextFileInputField( UUID.randomUUID().toString(), rand.nextInt(), rand.nextInt() );
  rtn.setType( rand.nextInt( 7 ) );
  rtn.setCurrencySymbol( UUID.randomUUID().toString() );
  rtn.setDecimalSymbol( UUID.randomUUID().toString() );
  rtn.setFormat( UUID.randomUUID().toString() );
  rtn.setGroupSymbol( UUID.randomUUID().toString() );
  rtn.setIfNullValue( UUID.randomUUID().toString() );
  rtn.setIgnored( rand.nextBoolean() );
  rtn.setNullString( UUID.randomUUID().toString() );
  rtn.setPrecision( rand.nextInt( 9 ) );
  rtn.setRepeated( rand.nextBoolean() );
  rtn.setTrimType( rand.nextInt( ValueMetaString.trimTypeDesc.length ) );
  return rtn;
}
 
Example #30
Source File: S3CsvInputMeta.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Override
public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, List<DatabaseMeta> databases )
    throws KettleException {
  try {
    awsAccessKey = Encr.decryptPasswordOptionallyEncrypted( rep.getStepAttributeString( id_step, "aws_access_key" ) );
    awsSecretKey = Encr.decryptPasswordOptionallyEncrypted( rep.getStepAttributeString( id_step, "aws_secret_key" ) );
    bucket = rep.getStepAttributeString( id_step, "bucket" );
    filename = rep.getStepAttributeString( id_step, "filename" );
    filenameField = rep.getStepAttributeString( id_step, "filename_field" );
    rowNumField = rep.getStepAttributeString( id_step, "rownum_field" );
    includingFilename = rep.getStepAttributeBoolean( id_step, "include_filename" );
    delimiter = rep.getStepAttributeString( id_step, "separator" );
    enclosure = rep.getStepAttributeString( id_step, "enclosure" );
    headerPresent = rep.getStepAttributeBoolean( id_step, "header" );
    maxLineSize = rep.getStepAttributeString( id_step, "max_line_size" );
    lazyConversionActive = rep.getStepAttributeBoolean( id_step, "lazy_conversion" );
    runningInParallel = rep.getStepAttributeBoolean( id_step, "parallel" );
    int nrfields = rep.countNrStepAttributes( id_step, "field_name" );

    allocate( nrfields );

    for ( int i = 0; i < nrfields; i++ ) {
      inputFields[i] = new TextFileInputField();

      inputFields[i].setName( rep.getStepAttributeString( id_step, i, "field_name" ) );
      inputFields[i].setType( ValueMetaFactory.getIdForValueMeta( rep.getStepAttributeString( id_step, i, "field_type" ) ) );
      inputFields[i].setFormat( rep.getStepAttributeString( id_step, i, "field_format" ) );
      inputFields[i].setCurrencySymbol( rep.getStepAttributeString( id_step, i, "field_currency" ) );
      inputFields[i].setDecimalSymbol( rep.getStepAttributeString( id_step, i, "field_decimal" ) );
      inputFields[i].setGroupSymbol( rep.getStepAttributeString( id_step, i, "field_group" ) );
      inputFields[i].setLength( (int) rep.getStepAttributeInteger( id_step, i, "field_length" ) );
      inputFields[i].setPrecision( (int) rep.getStepAttributeInteger( id_step, i, "field_precision" ) );
      inputFields[i].setTrimType( ValueMetaString.getTrimTypeByCode( rep.getStepAttributeString( id_step, i,
          "field_trim_type" ) ) );
    }
  } catch ( Exception e ) {
    throw new KettleException( "Unexpected error reading step information from the repository", e );
  }
}