org.pentaho.di.trans.steps.textfileinput.TextFileInputField Java Examples
The following examples show how to use
org.pentaho.di.trans.steps.textfileinput.TextFileInputField.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TextFileInputCSVImportProgressDialogTest.java From pentaho-kettle with Apache License 2.0 | 6 votes |
private void initiateVariables( String fileContent, TextFileInputField[] inputFileFields, int fileFormat ) throws Exception { //reader reader = new InputStreamReader( new ByteArrayInputStream( fileContent.getBytes( StandardCharsets.UTF_8 ) ) ); //meta when( meta.getInputFields() ).thenReturn( inputFileFields ); doCallRealMethod().when( meta ).getFields( any( RowMetaInterface.class ), anyString(), any( RowMetaInterface[].class ), any( StepMeta.class ), any( VariableSpace.class ), any( Repository.class ), any( IMetaStore.class ) ); setInternalState( meta, "inputFields", inputFileFields ); when( meta.hasHeader() ).thenReturn( true ); when( meta.getNrHeaderLines() ).thenReturn( 1 ); when( meta.clone() ).thenReturn( meta ); when( meta.getSeparator() ).thenReturn( "," ); when( meta.getEnclosure() ).thenReturn( "\"" ); when( meta.getFileFormatTypeNr() ).thenReturn( fileFormat ); when( meta.getFileType() ).thenReturn( "CSV" ); when( meta.getFilePaths( any( VariableSpace.class ) ) ).thenReturn( new String[] {"empty"} ); //transmeta when( transMeta.environmentSubstitute( "," ) ).thenReturn( "," ); when( transMeta.environmentSubstitute( "\"" ) ).thenReturn( "\"" ); when( transMeta.getObjectType() ).thenReturn( LoggingObjectType.TRANSMETA ); //textFileCSVImportProgressDialog textFileCSVImportProgressDialog = spy( new TextFileCSVImportProgressDialog( shell, meta, transMeta, reader, 100, true ) ); }
Example #2
Source File: ParGzipCsvInputMetaTest.java From pentaho-kettle with Apache License 2.0 | 6 votes |
@Before public void setUpLoadSave() throws Exception { KettleEnvironment.init(); PluginRegistry.init( false ); List<String> attributes = Arrays.asList( "filename", "filenameField", "includingFilename", "rowNumField", "headerPresent", "delimiter", "enclosure", "bufferSize", "lazyConversionActive", "addResultFile", "runningInParallel", "encoding", "inputFields" ); Map<String, String> getterMap = new HashMap<String, String>(); Map<String, String> setterMap = new HashMap<String, String>(); Map<String, FieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>(); attrValidatorMap.put( "inputFields", new ArrayLoadSaveValidator<TextFileInputField>( new TextFileInputFieldValidator(), 5 ) ); Map<String, FieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>(); loadSaveTester = new LoadSaveTester( testMetaClass, attributes, new ArrayList<String>(), new ArrayList<String>(), getterMap, setterMap, attrValidatorMap, typeValidatorMap, this ); }
Example #3
Source File: ParGzipCsvInputMetaTest.java From pentaho-kettle with Apache License 2.0 | 6 votes |
@Override public TextFileInputField getTestObject() { TextFileInputField rtn = new TextFileInputField(); rtn.setCurrencySymbol( UUID.randomUUID().toString() ); rtn.setDecimalSymbol( UUID.randomUUID().toString() ); rtn.setFormat( UUID.randomUUID().toString() ); rtn.setGroupSymbol( UUID.randomUUID().toString() ); rtn.setName( UUID.randomUUID().toString() ); rtn.setTrimType( rand.nextInt( 4 ) ); rtn.setPrecision( rand.nextInt( 9 ) ); rtn.setLength( rand.nextInt( 50 ) ); rtn.setType( rand.nextInt( 7 ) ); // Note - these fields aren't serialized by the meta class ... cannot test for them // rtn.setRepeated( rand.nextBoolean() ); // rtn.setSamples( new String[] { UUID.randomUUID().toString(), UUID.randomUUID().toString(), // UUID.randomUUID().toString() } ); // rtn.setNullString( UUID.randomUUID().toString() ); // rtn.setIfNullValue( UUID.randomUUID().toString() ); // rtn.setIgnored( rand.nextBoolean() ); // rtn.setPosition( rand.nextInt( 10 ) ); return rtn; }
Example #4
Source File: GoogleSpreadsheetInputMeta.java From pdi-google-spreadsheet-plugin with BSD 3-Clause "New" or "Revised" License | 6 votes |
@Override public void saveRep(Repository rep, IMetaStore metaStore, ObjectId id_transformation, ObjectId id_step) throws KettleException { try { rep.saveStepAttribute(id_transformation, id_step, "serviceEmail", this.serviceEmail); rep.saveStepAttribute(id_transformation, id_step, "spreadsheetKey", this.spreadsheetKey); rep.saveStepAttribute(id_transformation, id_step, "worksheetId", this.worksheetId); rep.saveStepAttribute(id_transformation, id_step, "privateKeyStore", GoogleSpreadsheet.base64EncodePrivateKeyStore(this.privateKeyStore)); for (int i = 0; i < inputFields.length; i++) { TextFileInputField field = inputFields[i]; rep.saveStepAttribute(id_transformation, id_step, i, "field_name", field.getName()); rep.saveStepAttribute(id_transformation, id_step, i, "field_type", ValueMeta.getTypeDesc(field.getType())); rep.saveStepAttribute(id_transformation, id_step, i, "field_format", field.getFormat()); rep.saveStepAttribute(id_transformation, id_step, i, "field_currency", field.getCurrencySymbol()); rep.saveStepAttribute(id_transformation, id_step, i, "field_decimal", field.getDecimalSymbol()); rep.saveStepAttribute(id_transformation, id_step, i, "field_group", field.getGroupSymbol()); rep.saveStepAttribute(id_transformation, id_step, i, "field_length", field.getLength()); rep.saveStepAttribute(id_transformation, id_step, i, "field_precision", field.getPrecision()); rep.saveStepAttribute(id_transformation, id_step, i, "field_trim_type", ValueMeta.getTrimTypeCode(field.getTrimType())); } } catch (Exception e) { throw new KettleException("Unable to save step information to the repository for id_step=" + id_step, e); } }
Example #5
Source File: ParGzipCsvInputMetaTest.java From pentaho-kettle with Apache License 2.0 | 6 votes |
@Override public boolean validateTestObject( TextFileInputField testObject, Object actual ) { if ( !( actual instanceof TextFileInputField ) ) { return false; } TextFileInputField another = (TextFileInputField) actual; return new EqualsBuilder() .append( testObject.getCurrencySymbol(), another.getCurrencySymbol() ) .append( testObject.getDecimalSymbol(), another.getDecimalSymbol() ) .append( testObject.getFormat(), another.getFormat() ) .append( testObject.getGroupSymbol(), another.getGroupSymbol() ) .append( testObject.getName(), another.getName() ) .append( testObject.getTrimType(), another.getTrimType() ) .append( testObject.getPrecision(), another.getPrecision() ) .append( testObject.getLength(), another.getLength() ) .append( testObject.getType(), another.getType() ) // Note - these fields aren't serialized by the meta class ... cannot test for them // .append( testObject.isRepeated(), another.isRepeated() ) // .append( testObject.getSamples(), another.getSamples() ) // .append( testObject.getNullString(), another.getNullString() ) // .append( testObject.getIfNullValue(), another.getIfNullValue() ) // .append( testObject.isIgnored(), another.isIgnored() ) // .append( testObject.getPosition(), another.getPosition() ) .isEquals(); }
Example #6
Source File: CsvInputMetaTest.java From pentaho-kettle with Apache License 2.0 | 6 votes |
@Test public void testClone() { final CsvInputMeta original = new CsvInputMeta(); original.setDelimiter( ";" ); original.setEnclosure( "'" ); final TextFileInputField[] originalFields = new TextFileInputField[ 1 ]; final TextFileInputField originalField = new TextFileInputField(); originalField.setName( "field" ); originalFields[ 0 ] = originalField; original.setInputFields( originalFields ); final CsvInputMeta clone = (CsvInputMeta) original.clone(); // verify that the clone and its input fields are "equal" to the originals, but not the same objects Assert.assertNotSame( original, clone ); Assert.assertEquals( original.getDelimiter(), clone.getDelimiter() ); Assert.assertEquals( original.getEnclosure(), clone.getEnclosure() ); Assert.assertNotSame( original.getInputFields(), clone.getInputFields() ); Assert.assertNotSame( original.getInputFields()[ 0 ], clone.getInputFields()[ 0 ] ); Assert.assertEquals( original.getInputFields()[ 0 ].getName(), clone.getInputFields()[ 0 ].getName() ); }
Example #7
Source File: TextFileInputCSVImportProgressDialogTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
private TextFileInputField[] createInputFileFields( String... names ) { TextFileInputField[] fields = new TextFileInputField[ names.length ]; for ( int i = 0; i < names.length; i++ ) { fields[ i ] = createField( names[ i ] ); } return fields; }
Example #8
Source File: TextFileInputCSVImportProgressDialogTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
private void fileContentTest( String fileContent, TextFileInputField[] inputFileFields, int fileFormat ) throws Exception { initiateVariables( fileContent, inputFileFields, fileFormat ); String result = textFileCSVImportProgressDialog.doScan( monitor ); assertEquals( ValueMetaFactory.getValueMetaName( meta.getInputFields()[0].getType() ), inputFileFields[0].getName() ); assertEquals( ValueMetaFactory.getValueMetaName( meta.getInputFields()[1].getType() ), inputFileFields[1].getName() ); assertEquals( ValueMetaFactory.getValueMetaName( meta.getInputFields()[2].getType() ), inputFileFields[2].getName() ); assertEquals( ValueMetaFactory.getValueMetaName( meta.getInputFields()[3].getType() ), inputFileFields[3].getName() ); }
Example #9
Source File: TextFileInputCSVImportProgressDialogTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void fileMixFormatTestWithEnclosures() throws Exception { TextFileInputField[] inputFileFields = createInputFileFields( "String", "Number", "Boolean", "Date" ); String fileContent = "String, int, boolean, date\r\n" + "\"レコ\n" + "ード名1\",1.7976E308,true,9999/12/31 23:59:59.999,\n" + "\"あ\r\n" + "a1\",123456789,false,2016/1/5 12:00:00.000\n"; fileContentTest( fileContent, inputFileFields, TextFileInputMeta.FILE_FORMAT_MIXED ); }
Example #10
Source File: TextFileInputCSVImportProgressDialogTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void fileMixFormatTestWithNoEnclosures() throws Exception { TextFileInputField[] inputFileFields = createInputFileFields( "String", "Number", "Boolean", "Date" ); String fileContent = "String, int, boolean, date\n" + "レコード名1,1.7976E308,true,9999/12/31 23:59:59.999,\r\n" + "\"あa1\",123456789,false,2016/1/5 12:00:00.000\r\n"; fileContentTest( fileContent, inputFileFields, TextFileInputMeta.FILE_FORMAT_MIXED ); }
Example #11
Source File: TextFileInputCSVImportProgressDialogTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void fileUnixFormatTestWithNoEnclosures() throws Exception { TextFileInputField[] inputFileFields = createInputFileFields( "String", "Number", "Boolean", "Date" ); String fileContent = "String, int, boolean, date\n" + "レコード名1,1.7976E308,true,9999/12/31 23:59:59.999,\n" + "\"あa1\",123456789,false,2016/1/5 12:00:00.000\n"; fileContentTest( fileContent, inputFileFields, TextFileInputMeta.FILE_FORMAT_UNIX ); }
Example #12
Source File: TextFileInputCSVImportProgressDialogTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void fileDOSFormatTestWithEnclosures() throws Exception { TextFileInputField[] inputFileFields = createInputFileFields( "String", "Number", "Boolean", "Date" ); String fileContent = "String, int, boolean, date\r\n" + "\"レコ\r\n" + "ード名1\",1.7976E308,true,9999/12/31 23:59:59.999,\r\n" + "\"あ\r\n" + "a1\",123456789,false,2016/1/5 12:00:00.000\r\n"; fileContentTest( fileContent, inputFileFields, TextFileInputMeta.FILE_FORMAT_DOS ); }
Example #13
Source File: CsvInputContentParsingTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test( expected = KettleStepException.class ) public void testNoHeaderOptions() throws Exception { meta.setHeaderPresent( false ); init( "default.csv" ); setFields( new TextFileInputField(), new TextFileInputField(), new TextFileInputField() ); process(); }
Example #14
Source File: CsvInputContentParsingTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void testDosFileFormat() throws Exception { String data = "データ1,データ2,データ3,データ4\r\n" + "111,\"a\r\n" + "bc\",あいう,さしす\r\n" + "222,def,かきく,たちつ\r\n" + "333,,かきく,たちつ\r\n" + "444,,\r\n" + "555,かきく,\r\n" + "666,かきく\r\n" + "\r\n" + "777,\r\n" + "888,かきく\r\n" + "\r\n" + "999,123,123,123,132,132,132,132,132\r\n"; String file = createTestFile( "UTF-8", data ).getAbsolutePath(); meta.setFileFormat( "DOS" ); init( file, true ); setFields( new TextFileInputField( "Col 1", -1, -1 ), new TextFileInputField( "Col 2", -1, -1 ), new TextFileInputField( "Col 3", -1, -1 ), new TextFileInputField( "Col 4", -1, -1 ), new TextFileInputField( "Col 5", -1, -1 ) ); process(); check( new Object[][] { { "111", "a\r\nbc", "あいう", "さしす", null }, { "222", "def", "かきく", "たちつ", null}, { "333", "", "かきく", "たちつ", null }, { "444", "", "", null, null }, { "555", "かきく", "", null, null }, { "666", "かきく", null, null, null}, { }, { "777", "", null, null, null }, { "888", "かきく", null, null, null }, { }, { "999", "123", "123", "123", "132" } } ); }
Example #15
Source File: CsvInputContentParsingTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void testSemicolonOptions() throws Exception { meta.setDelimiter( ";" ); init( "semicolon.csv" ); setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ), new TextFileInputField( "Field 3", -1, -1 ) ); process(); check( new Object[][] { { "first", "1", "1.1" }, { "second", "2", "2.2" }, { "third", "3", "3.3" }, { "\u043d\u0435-\u043b\u0430\u0446\u0456\u043d\u043a\u0430(non-latin)", "4", "4" } } ); }
Example #16
Source File: CsvInputContentParsingTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void testEnclosures() throws Exception { meta.setDelimiter( ";" ); meta.setEnclosure( "'" ); init( "enclosures.csv" ); setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ), new TextFileInputField( "Field 3", -1, -1 ) ); process(); check( new Object[][] { { "1", "This line is un-even enclosure-wise because I'm using an escaped enclosure", "a" }, { "2", "Test isn't even\nhere", "b" } } ); }
Example #17
Source File: S3CsvInputMetaTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void testSerialization() throws KettleException { List<String> attributes = Arrays.asList( "AwsAccessKey", "AwsSecretKey", "Bucket", "Filename", "FilenameField", "RowNumField", "IncludingFilename", "Delimiter", "Enclosure", "HeaderPresent", "MaxLineSize", "LazyConversionActive", "RunningInParallel", "InputFields" ); Map<String, FieldLoadSaveValidator<?>> typeMap = new HashMap<>(); typeMap.put( TextFileInputField[].class.getCanonicalName(), new ArrayLoadSaveValidator<>( new TextFileInputFieldValidator() ) ); Map<String, String> getterMap = new HashMap<>(); Map<String, String> setterMap = new HashMap<>(); LoadSaveTester<S3CsvInputMeta> tester = new LoadSaveTester<>( S3CsvInputMeta.class, attributes, getterMap, setterMap, new HashMap<String, FieldLoadSaveValidator<?>>(), typeMap ); tester.testSerialization(); }
Example #18
Source File: CsvInputMetaTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Before public void setUpLoadSave() throws Exception { KettleEnvironment.init(); PluginRegistry.init( false ); List<String> attributes = Arrays.asList( "BufferSize", "Delimiter", "Enclosure", "Encoding", "Filename", "FilenameField", "InputFields", "RowNumField", "AddResultFile", "HeaderPresent", "IncludingFilename", "LazyConversionActive", "NewlinePossibleInFields", "RunningInParallel" ); Map<String, String> getterMap = new HashMap<String, String>() { { put( "hasHeader", "hasHeader" ); put( "includeFilename", "includeFilename" ); put( "includeRowNumber", "includeRowNumber" ); } }; Map<String, String> setterMap = new HashMap<String, String>() { { put( "includeFilename", "includeFilename" ); put( "includeRowNumber", "includeRowNumber" ); } }; FieldLoadSaveValidator<String[]> stringArrayLoadSaveValidator = new ArrayLoadSaveValidator<String>( new StringLoadSaveValidator(), 5 ); Map<String, FieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>(); attrValidatorMap.put( "InputFields", new ArrayLoadSaveValidator<TextFileInputField>( new TextFileInputFieldValidator(), 5 ) ); Map<String, FieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>(); loadSaveTester = new LoadSaveTester( testMetaClass, attributes, new ArrayList<String>(), new ArrayList<String>(), getterMap, setterMap, attrValidatorMap, typeValidatorMap, this ); }
Example #19
Source File: GoogleSpreadsheetInputMeta.java From pdi-google-spreadsheet-plugin with BSD 3-Clause "New" or "Revised" License | 5 votes |
@Override public void setDefault() { this.serviceEmail = ""; this.spreadsheetKey = ""; this.worksheetId = "od6"; this.privateKeyStore = null; TextFileInputField field = new TextFileInputField(); field.setName("field"); field.setType(ValueMetaInterface.TYPE_STRING); inputFields = new TextFileInputField[]{ field, }; }
Example #20
Source File: TextFileInputCSVImportProgressDialogTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void fileDOSFormatTestWithNoEnclosures() throws Exception { TextFileInputField[] inputFileFields = createInputFileFields( "String", "Number", "Boolean", "Date" ); String fileContent = "String, int, boolean, date\r\n" + "レコード名1,1.7976E308,true,9999/12/31 23:59:59.999,\r\n" + "\"あa1\",123456789,false,2016/1/5 12:00:00.000\r\n"; fileContentTest( fileContent, inputFileFields, TextFileInputMeta.FILE_FORMAT_DOS ); }
Example #21
Source File: CsvInputContentParsingTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void testMultiCharDelimOptions() throws Exception { meta.setDelimiter( "|||" ); init( "multi_delim.csv" ); setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ), new TextFileInputField( "Field 3", -1, -1 ) ); process(); check( new Object[][] { { "first", "1", "1.1" }, { "second", "2", "2.2" }, { "third", "3", "3.3" }, { "\u043d\u0435-\u043b\u0430\u0446\u0456\u043d\u043a\u0430(non-latin)", "4", "4" } } ); }
Example #22
Source File: CsvInputContentParsingTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void testUnixFileFormat() throws Exception { String data = "データ1,データ2,データ3,データ4\n" + "111,\"a\n" + "bc\",\n" + "\n" + "444,,\n" + "555,かきく,\n" + "\n" + "\n" + "777,\n" + "888,かきく\n" + "999,123,123,123,132,132,132,132,132\n"; String file = createTestFile( "UTF-8", data ).getAbsolutePath(); meta.setFileFormat( "Unix" ); init( file, true ); setFields( new TextFileInputField( "Col 1", -1, -1 ), new TextFileInputField( "Col 2", -1, -1 ), new TextFileInputField( "Col 3", -1, -1 ), new TextFileInputField( "Col 4", -1, -1 ), new TextFileInputField( "Col 5", -1, -1 ) ); process(); check( new Object[][] { { "111", "a\nbc", "", null, null }, { }, { "444", "", "", null, null }, { "555", "かきく", "", null, null }, { }, { }, { "777", "", null, null, null }, { "888", "かきく", null, null, null }, { "999", "123", "123", "123", "132" } } ); }
Example #23
Source File: CsvInputContentParsingTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void testColumnNameWithSpaces() throws Exception { init( "column_name_with_spaces.csv" ); setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ), new TextFileInputField( "Field 3", -1, -1 ) ); process(); check( new Object[][] { { "first", "1", "1.1" }, { "second", "2", "2.2" }, { "third", "3", "3.3" } } ); }
Example #24
Source File: CsvInputContentParsingTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void testDefaultOptions() throws Exception { init( "default.csv" ); setFields( new TextFileInputField( "Field 1", -1, -1 ), new TextFileInputField( "Field 2", -1, -1 ), new TextFileInputField( "Field 3", -1, -1 ) ); process(); check( new Object[][] { { "first", "1", "1.1" }, { "second", "2", "2.2" }, { "third", "3", "3.3" } } ); }
Example #25
Source File: CsvInputUnitTestBase.java From pentaho-kettle with Apache License 2.0 | 5 votes |
CsvInputMeta createMeta( File file, TextFileInputField[] fields ) { CsvInputMeta meta = new CsvInputMeta(); meta.setFilename( file.getAbsolutePath() ); meta.setBufferSize( BUFFER_SIZE ); meta.setDelimiter( DELIMITER ); meta.setEnclosure( ENCLOSURE ); meta.setEncoding( ENCODING ); meta.setInputFields( fields ); meta.setHeaderPresent( false ); return meta; }
Example #26
Source File: CsvInputMultiCharDelimiterTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Override CsvInputMeta createMeta( File file, TextFileInputField[] fields ) { CsvInputMeta meta = super.createMeta( file, fields ); meta.setDelimiter( "delimiter" ); //Buffer cannot be less than ( delimiter length - 1) * 2 ) due to buffer resize logic in readBufferFromFile method //This buffer size causes special case where the delimiter spans past the end of the buffer. meta.setBufferSize( "16" ); return meta; }
Example #27
Source File: CsvInputTest.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Test public void testFileIsReleasedAfterProcessing() throws Exception { // Create a file with some content to be processed TextFileInputField[] inputFileFields = createInputFileFields( "f1", "f2", "f3" ); String fileContents = "Something" + DELIMITER + "" + DELIMITER + "The former was empty!"; File tmpFile = createTestFile( ENCODING, fileContents ); // Create and configure the step CsvInputMeta meta = createMeta( tmpFile, inputFileFields ); CsvInputData data = new CsvInputData(); RowSet output = new QueueRowSet(); CsvInput csvInput = new CsvInput( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta, stepMockHelper.trans ); csvInput.init( meta, data ); csvInput.addRowSetToOutputRowSets( output ); // Start processing csvInput.processRow( meta, data ); // Finish processing csvInput.dispose( meta, data ); // And now the file must be free to be deleted assertTrue( tmpFile.delete() ); assertFalse( tmpFile.exists() ); }
Example #28
Source File: TextFileInputFieldValidator.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Override public boolean validateTestObject( TextFileInputField testObject, Object actual ) { if ( !( actual instanceof TextFileInputField ) ) { return false; } TextFileInputField another = (TextFileInputField) actual; return new EqualsBuilder() .append( testObject.getName(), another.getName() ) .append( testObject.getTypeDesc(), another.getTypeDesc() ) .append( testObject.getFormat(), another.getFormat() ) .append( testObject.getCurrencySymbol(), another.getCurrencySymbol() ) .append( testObject.getDecimalSymbol(), another.getDecimalSymbol() ) .append( testObject.getGroupSymbol(), another.getGroupSymbol() ) .append( testObject.getLength(), another.getLength() ) .append( testObject.getPrecision(), another.getPrecision() ) .append( testObject.getTrimTypeCode(), another.getTrimTypeCode() ) // These fields not universally serialized, so don't check for them // TextFileInputMeta serializes some that CsvInputMeta doesn't. Given // they're both deprecated, ignore these. //.append( testObject.isRepeated(), another.isRepeated() ) //.append( testObject.getNullString(), another.getNullString() ) //.append( testObject.getIfNullValue(), another.getIfNullValue() ) //.append( testObject.getPosition(), another.getPosition() ) .isEquals(); }
Example #29
Source File: TextFileInputFieldValidator.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Override public TextFileInputField getTestObject() { TextFileInputField rtn = new TextFileInputField( UUID.randomUUID().toString(), rand.nextInt(), rand.nextInt() ); rtn.setType( rand.nextInt( 7 ) ); rtn.setCurrencySymbol( UUID.randomUUID().toString() ); rtn.setDecimalSymbol( UUID.randomUUID().toString() ); rtn.setFormat( UUID.randomUUID().toString() ); rtn.setGroupSymbol( UUID.randomUUID().toString() ); rtn.setIfNullValue( UUID.randomUUID().toString() ); rtn.setIgnored( rand.nextBoolean() ); rtn.setNullString( UUID.randomUUID().toString() ); rtn.setPrecision( rand.nextInt( 9 ) ); rtn.setRepeated( rand.nextBoolean() ); rtn.setTrimType( rand.nextInt( ValueMetaString.trimTypeDesc.length ) ); return rtn; }
Example #30
Source File: S3CsvInputMeta.java From pentaho-kettle with Apache License 2.0 | 5 votes |
@Override public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, List<DatabaseMeta> databases ) throws KettleException { try { awsAccessKey = Encr.decryptPasswordOptionallyEncrypted( rep.getStepAttributeString( id_step, "aws_access_key" ) ); awsSecretKey = Encr.decryptPasswordOptionallyEncrypted( rep.getStepAttributeString( id_step, "aws_secret_key" ) ); bucket = rep.getStepAttributeString( id_step, "bucket" ); filename = rep.getStepAttributeString( id_step, "filename" ); filenameField = rep.getStepAttributeString( id_step, "filename_field" ); rowNumField = rep.getStepAttributeString( id_step, "rownum_field" ); includingFilename = rep.getStepAttributeBoolean( id_step, "include_filename" ); delimiter = rep.getStepAttributeString( id_step, "separator" ); enclosure = rep.getStepAttributeString( id_step, "enclosure" ); headerPresent = rep.getStepAttributeBoolean( id_step, "header" ); maxLineSize = rep.getStepAttributeString( id_step, "max_line_size" ); lazyConversionActive = rep.getStepAttributeBoolean( id_step, "lazy_conversion" ); runningInParallel = rep.getStepAttributeBoolean( id_step, "parallel" ); int nrfields = rep.countNrStepAttributes( id_step, "field_name" ); allocate( nrfields ); for ( int i = 0; i < nrfields; i++ ) { inputFields[i] = new TextFileInputField(); inputFields[i].setName( rep.getStepAttributeString( id_step, i, "field_name" ) ); inputFields[i].setType( ValueMetaFactory.getIdForValueMeta( rep.getStepAttributeString( id_step, i, "field_type" ) ) ); inputFields[i].setFormat( rep.getStepAttributeString( id_step, i, "field_format" ) ); inputFields[i].setCurrencySymbol( rep.getStepAttributeString( id_step, i, "field_currency" ) ); inputFields[i].setDecimalSymbol( rep.getStepAttributeString( id_step, i, "field_decimal" ) ); inputFields[i].setGroupSymbol( rep.getStepAttributeString( id_step, i, "field_group" ) ); inputFields[i].setLength( (int) rep.getStepAttributeInteger( id_step, i, "field_length" ) ); inputFields[i].setPrecision( (int) rep.getStepAttributeInteger( id_step, i, "field_precision" ) ); inputFields[i].setTrimType( ValueMetaString.getTrimTypeByCode( rep.getStepAttributeString( id_step, i, "field_trim_type" ) ) ); } } catch ( Exception e ) { throw new KettleException( "Unexpected error reading step information from the repository", e ); } }