org.pentaho.di.trans.step.BaseStep Java Examples

The following examples show how to use org.pentaho.di.trans.step.BaseStep. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BlockingStep.java    From pentaho-kettle with Apache License 2.0 6 votes vote down vote up
public void dispose( StepMetaInterface smi, StepDataInterface sdi ) {
  if ( ( data.dis != null ) && ( data.dis.size() > 0 ) ) {
    for ( DataInputStream is : data.dis ) {
      BaseStep.closeQuietly( is );
    }
  }
  // remove temp files
  for ( int f = 0; f < data.files.size(); f++ ) {
    FileObject fileToDelete = data.files.get( f );
    try {
      if ( fileToDelete != null && fileToDelete.exists() ) {
        fileToDelete.delete();
      }
    } catch ( FileSystemException e ) {
      logError( e.getLocalizedMessage(), e );
    }
  }
  super.dispose( smi, sdi );
}
 
Example #2
Source File: AbstractFileErrorHandlerTest.java    From pentaho-kettle with Apache License 2.0 6 votes vote down vote up
private void setupErrorHandler( String encoding, Object source ) throws Exception {
  File tempFile = File.createTempFile( "test", "file.txt" );
  tempFile.deleteOnExit();
  FileObject fileObject = KettleVFS.getFileObject( tempFile.getAbsolutePath() );
  Date date = new Date();
  String destDirectory = tempFile.getParent();
  String fileExtension = ".txt";
  BaseStep baseStep = mock( BaseStep.class );
  TransMeta transMeta = mock( TransMeta.class );

  when( baseStep.getTransMeta() ).thenReturn( transMeta );
  when( transMeta.getName() ).thenReturn( "TestStep" );
  when( baseStep.getStepname() ).thenReturn( "Test Step Name" );
  abstractFileErrorHandler = spy( new FileErrorHandlerContentLineNumber( date, destDirectory, fileExtension, encoding, baseStep ) );
  abstractFileErrorHandler.handleFile( fileObject );
  when( source.toString() ).thenReturn( "testFile.txt" );
  when( abstractFileErrorHandler.getReplayFilename( destDirectory, "testFile", dateFormat.format( date ), fileExtension, source ) ).thenReturn( null );
}
 
Example #3
Source File: GetFilesRowsCount.java    From pentaho-kettle with Apache License 2.0 6 votes vote down vote up
public void dispose( StepMetaInterface smi, StepDataInterface sdi ) {
  meta = (GetFilesRowsCountMeta) smi;
  data = (GetFilesRowsCountData) sdi;
  if ( data.file != null ) {
    try {
      data.file.close();
      data.file = null;
    } catch ( Exception e ) {
      log.logError( "Error closing file", e );
    }
  }
  if ( data.fr != null ) {
    BaseStep.closeQuietly( data.fr );
    data.fr = null;
  }
  if ( data.lineStringBuilder != null ) {
    data.lineStringBuilder = null;
  }

  super.dispose( smi, sdi );
}
 
Example #4
Source File: IngresVectorwiseTest.java    From pentaho-kettle with Apache License 2.0 6 votes vote down vote up
/**
 * Test will fail if you're on a Windows system. IngresVectorwise Bulk Loading is only available for POSIX Operating
 * Systems:
 * "The VW bulk loader only runs under POSIX operating systems that support named pipes. This includes every modern
 * operating system besides Windows."
 * @see <a href="https://wiki.pentaho.com/display/EAI/Ingres+VectorWise+Bulk+Loader">Ingres Vectorwise Bulk Loader</a>
 */
@Test
public void testWaitForFinish() {
  try {
    int r = rows.size();
    BaseStep step = doOutput( wrongRows, "2" );
    assertEquals( r - 1, step.getLinesOutput() );
    assertEquals( r, step.getLinesRead() );
    assertEquals( r, step.getLinesWritten() );
    assertEquals( 1, step.getLinesRejected() );
    assertEquals( 0, step.getErrors() );

  } catch ( KettleException e ) {
    fail( e.getMessage() );
  }
}
 
Example #5
Source File: TextFileInput.java    From pentaho-kettle with Apache License 2.0 6 votes vote down vote up
@Override
public void dispose( StepMetaInterface smi, StepDataInterface sdi ) {
  meta = (TextFileInputMeta) smi;
  data = (TextFileInputData) sdi;

  if ( data.file != null ) {
    try {
      data.file.close();
      data.file = null;
    } catch ( Exception e ) {
      log.logError( "Error closing file", e );
    }
  }
  if ( data.in != null ) {
    BaseStep.closeQuietly( data.in );
    data.in = null;
  }
  super.dispose( smi, sdi );
}
 
Example #6
Source File: TransTestingUtil.java    From pentaho-kettle with Apache License 2.0 6 votes vote down vote up
public static List<Object[]> execute( BaseStep step,
                                      StepMetaInterface meta,
                                      StepDataInterface data,
                                      int expectedRowsAmount,
                                      boolean checkIsDone ) throws Exception {
  RowSet output = new BlockingRowSet( Math.max( 1, expectedRowsAmount ) );
  step.setOutputRowSets( Collections.singletonList( output ) );

  int i = 0;
  List<Object[]> result = new ArrayList<>( expectedRowsAmount );
  while ( step.processRow( meta, data ) && i < expectedRowsAmount ) {
    Object[] row = output.getRowImmediate();
    assertNotNull( Integer.toString( i ), row );
    result.add( row );

    i++;
  }
  assertEquals( "The amount of executions should be equal to expected", expectedRowsAmount, i );
  if ( checkIsDone ) {
    assertTrue( output.isDone() );
  }

  return result;
}
 
Example #7
Source File: Mapping.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
private void lookupStatusStepNumbers() {
  MappingData mappingData = getData();
  if ( mappingData.getMappingTrans() != null ) {
    List<StepMetaDataCombi> steps = mappingData.getMappingTrans().getSteps();
    for ( int i = 0; i < steps.size(); i++ ) {
      StepMetaDataCombi sid = steps.get( i );
      BaseStep rt = (BaseStep) sid.step;
      if ( rt.getStepname().equals( getData().mappingTransMeta.getTransLogTable().getStepnameRead() ) ) {
        mappingData.linesReadStepNr = i;
      }
      if ( rt.getStepname().equals( getData().mappingTransMeta.getTransLogTable().getStepnameInput() ) ) {
        mappingData.linesInputStepNr = i;
      }
      if ( rt.getStepname().equals( getData().mappingTransMeta.getTransLogTable().getStepnameWritten() ) ) {
        mappingData.linesWrittenStepNr = i;
      }
      if ( rt.getStepname().equals( getData().mappingTransMeta.getTransLogTable().getStepnameOutput() ) ) {
        mappingData.linesOutputStepNr = i;
      }
      if ( rt.getStepname().equals( getData().mappingTransMeta.getTransLogTable().getStepnameUpdated() ) ) {
        mappingData.linesUpdatedStepNr = i;
      }
      if ( rt.getStepname().equals( getData().mappingTransMeta.getTransLogTable().getStepnameRejected() ) ) {
        mappingData.linesRejectedStepNr = i;
      }
    }
  }
}
 
Example #8
Source File: CheckSumTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
/**
 * Create, execute, and return the row listener attached to the output step with complete results from the execution.
 *
 * @param checkSumType
 *          Type of checksum to use (the array index of {@link CheckSumMeta#checksumtypeCodes})
 * @param compatibilityMode
 *          Use compatibility mode for CheckSum
 * @param fieldSeparatorString
 *          The string separate multiple fields with
 * @param inputs
 *          Array of objects representing row data
 * @param inputValueMetas
 *          metas to be processed
 * @return RowListener with results.
 */
private MockRowListener executeHexTest( int checkSumType, boolean compatibilityMode, boolean oldChecksumBehaviour,
    String fieldSeparatorString, Object[] inputs, ValueMetaInterface... inputValueMetas ) throws Exception {

  String[] fieldNames = new String[inputValueMetas.length];
  RowMeta inputRowMeta = new RowMeta();
  for ( int i = 0; i < inputValueMetas.length; i++ ) {
    inputRowMeta.addValueMeta( inputValueMetas[i] );
    fieldNames[i] = inputValueMetas[i].getName();
  }

  Trans trans =
      buildHexadecimalChecksumTrans( checkSumType, compatibilityMode, oldChecksumBehaviour, fieldSeparatorString,
          fieldNames );

  trans.prepareExecution( null );

  StepInterface output = trans.getRunThread( "Output", 0 );
  MockRowListener listener = new MockRowListener();
  output.addRowListener( listener );

  RowProducer rp = trans.addRowProducer( "CheckSum", 0 );

  ( (BaseStep) trans.getRunThread( "CheckSum", 0 ) ).setInputRowMeta( inputRowMeta );

  trans.startThreads();

  rp.putRow( inputRowMeta, inputs );
  rp.finished();

  trans.waitUntilFinished();
  trans.stopAll();
  trans.cleanup();
  return listener;
}
 
Example #9
Source File: GetXMLData.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
public void dispose( StepMetaInterface smi, StepDataInterface sdi ) {
  meta = (GetXMLDataMeta) smi;
  data = (GetXMLDataData) sdi;
  if ( data.file != null ) {
    try {
      data.file.close();
    } catch ( Exception e ) {
      // Ignore close errors
    }
  }
  if ( data.an != null ) {
    data.an.clear();
    data.an = null;
  }
  if ( data.NAMESPACE != null ) {
    data.NAMESPACE.clear();
    data.NAMESPACE = null;
  }
  if ( data.NSPath != null ) {
    data.NSPath.clear();
    data.NSPath = null;
  }
  if ( data.readrow != null ) {
    data.readrow = null;
  }
  if ( data.document != null ) {
    data.document = null;
  }
  if ( data.fr != null ) {
    BaseStep.closeQuietly( data.fr );
  }
  if ( data.is != null ) {
    BaseStep.closeQuietly( data.is );
  }
  if ( data.files != null ) {
    data.files = null;
  }
  super.dispose( smi, sdi );
}
 
Example #10
Source File: StepMockUtil.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
public static <T extends StepMetaInterface, V extends BaseStep> StepMockHelper<T, StepDataInterface> getStepMockHelper( Class<T> meta, String name ) {
  StepMockHelper<T, StepDataInterface> stepMockHelper = new StepMockHelper<T, StepDataInterface>( name, meta, StepDataInterface.class );
  when( stepMockHelper.logChannelInterfaceFactory.create( any(), any( LoggingObjectInterface.class ) ) ).thenReturn( stepMockHelper.logChannelInterface );
  when( stepMockHelper.logChannelInterfaceFactory.create( any() ) ).thenReturn( stepMockHelper.logChannelInterface );
  when( stepMockHelper.trans.isRunning() ).thenReturn( true );
  return stepMockHelper;
}
 
Example #11
Source File: BaseStepConcurrencyTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
@Override
BaseStep doCall() throws Exception {
    for ( RowListener rowListener : baseStep.getRowListeners() ) {
        rowListener.rowWrittenEvent( mock( RowMetaInterface.class ), new Object[]{} );
    }
    return null;
}
 
Example #12
Source File: BaseStepConcurrencyTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
/**
 * Row sets collection modifiers are exposed out of BaseStep class,
 * whereas the collection traversal is happening on every row being processed.
 *
 * We should be sure that modification of the collection will not throw a concurrent modification exception.
 */
@Test
public void testInputOutputRowSets() throws Exception {
    int modifiersAmount = 100;
    int traversersAmount = 100;

    StepMeta stepMeta = mock( StepMeta.class);
    TransMeta transMeta = mock( TransMeta.class);
    when( stepMeta.getName() ).thenReturn( STEP_META );
    when( transMeta.findStep( STEP_META ) ).thenReturn( stepMeta );
    when( stepMeta.getTargetStepPartitioningMeta() ).thenReturn( mock( StepPartitioningMeta.class ) );

    baseStep = new BaseStep( stepMeta, null, 0, transMeta, mock( Trans.class ) );

    AtomicBoolean condition = new AtomicBoolean( true );

    List<RowSetsModifier> rowSetsModifiers = new ArrayList<>();
    for ( int i = 0; i < modifiersAmount; i++ ) {
        rowSetsModifiers.add( new RowSetsModifier( condition ) );
    }
    List<RowSetsTraverser> rowSetsTraversers = new ArrayList<>();
    for ( int i = 0; i < traversersAmount; i++ ) {
        rowSetsTraversers.add( new RowSetsTraverser( condition ) );
    }

    ConcurrencyTestRunner<?, ?> runner =
            new ConcurrencyTestRunner<Object, Object>(rowSetsModifiers, rowSetsTraversers, condition );
    runner.runConcurrentTest();

    runner.checkNoExceptionRaised();
}
 
Example #13
Source File: BaseStepConcurrencyTest.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
/**
 * Row listeners collection modifiers are exposed out of BaseStep class,
 * whereas the collection traversal is happening on every row being processed.
 *
 * We should be sure that modification of the collection will not throw a concurrent modification exception.
 */
@Test
public void testRowListeners() throws Exception {
    int modifiersAmount = 100;
    int traversersAmount = 100;

    StepMeta stepMeta = mock( StepMeta.class);
    TransMeta transMeta = mock( TransMeta.class);
    when( stepMeta.getName() ).thenReturn( STEP_META );
    when( transMeta.findStep( STEP_META ) ).thenReturn( stepMeta );
    when( stepMeta.getTargetStepPartitioningMeta() ).thenReturn( mock( StepPartitioningMeta.class ) );

    baseStep = new BaseStep( stepMeta, null, 0, transMeta, mock( Trans.class ) );

    AtomicBoolean condition = new AtomicBoolean( true );

    List<RowListenersModifier> rowListenersModifiers = new ArrayList<>();
    for ( int i = 0; i < modifiersAmount; i++ ) {
        rowListenersModifiers.add( new RowListenersModifier( condition ) );
    }
    List<RowListenersTraverser> rowListenersTraversers = new ArrayList<>();
    for ( int i = 0; i < traversersAmount; i++ ) {
        rowListenersTraversers.add( new RowListenersTraverser( condition ) );
    }

    ConcurrencyTestRunner<?, ?> runner =
            new ConcurrencyTestRunner<Object, Object>(rowListenersModifiers, rowListenersTraversers, condition );
    runner.runConcurrentTest();

    runner.checkNoExceptionRaised();
}
 
Example #14
Source File: AbstractFileErrorHandler.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
public AbstractFileErrorHandler( Date date, String destinationDirectory, String fileExtension, String encoding,
  BaseStep baseStep ) {
  this.destinationDirectory = destinationDirectory;
  this.fileExtension = fileExtension;
  this.encoding = encoding;
  this.baseStep = baseStep;
  this.writers = new HashMap<Object, Writer>();
  initDateFormatter( date );
}
 
Example #15
Source File: BaseMongoDbStepTest.java    From pentaho-mongodb-plugin with Apache License 2.0 5 votes vote down vote up
@Before public void before() throws MongoDbException {
  MockitoAnnotations.initMocks( this );
  MongoWrapperUtil.setMongoWrapperClientFactory( mongoClientWrapperFactory );
  when( mongoClientWrapperFactory
      .createMongoClientWrapper( any( MongoProperties.class ), any( MongoUtilLogger.class ) ) )
      .thenReturn( mongoClientWrapper );

  when( stepMeta.getName() ).thenReturn( "stepMetaName" );
  when( transMeta.findStep( anyString() ) ).thenReturn( stepMeta );
  when( logChannelFactory.create( any( BaseStep.class ), any( Trans.class ) ) ).thenReturn( mockLog );
  KettleLogStore.setLogChannelInterfaceFactory( logChannelFactory );
}
 
Example #16
Source File: BaseStepConcurrencyTest.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
@Override
BaseStep doCall() {
    baseStep.addRowSetToInputRowSets( mock( RowSet.class ) );
    baseStep.addRowSetToOutputRowSets( mock( RowSet.class ) );
    return null;
}
 
Example #17
Source File: BaseStepConcurrencyTest.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
@Override
BaseStep doCall() {
    baseStep.addRowListener( mock( RowListener.class ) );
    return null;
}
 
Example #18
Source File: FileErrorHandlerContentLineNumber.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
public FileErrorHandlerContentLineNumber( Date date, String destinationDirectory, String fileExtension,
  String encoding, BaseStep baseStep ) {
  super( date, destinationDirectory, fileExtension, encoding, baseStep );
}
 
Example #19
Source File: FileErrorHandlerMissingFiles.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
public FileErrorHandlerMissingFiles( Date date, String destinationDirectory, String fileExtension,
  String encoding, BaseStep baseStep ) {
  super( date, destinationDirectory, fileExtension, encoding, baseStep );
}
 
Example #20
Source File: StepMockUtil.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
public static <T extends BaseStep, K extends StepMetaInterface, V extends StepDataInterface> T getStep( Class<T> klass, StepMockHelper<K, V> mock )
    throws NoSuchMethodException, SecurityException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {
  Constructor<T> kons = klass.getConstructor( StepMeta.class, StepDataInterface.class, int.class, TransMeta.class, Trans.class );
  T step = kons.newInstance( mock.stepMeta, mock.stepDataInterface, 0, mock.transMeta, mock.trans );
  return step;
}
 
Example #21
Source File: StepMockUtil.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
public static <T extends BaseStep, K extends StepMetaInterface> T getStep( Class<T> stepClass, Class<K> stepMetaClass, String stepName )
    throws NoSuchMethodException, SecurityException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {
  return StepMockUtil.getStep( stepClass, StepMockUtil.getStepMockHelper( stepMetaClass, stepName ) );
}
 
Example #22
Source File: PropertyInputMetaTest.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
@Test
public void testOpenNextFile() throws Exception {

  PropertyInputMeta propertyInputMeta = Mockito.mock( PropertyInputMeta.class );
  PropertyInputData propertyInputData = new PropertyInputData();
  FileInputList fileInputList = new FileInputList();
  FileObject fileObject = Mockito.mock( FileObject.class );
  FileName fileName = Mockito.mock( FileName.class );
  Mockito.when( fileName.getRootURI() ).thenReturn( "testFolder" );
  Mockito.when( fileName.getURI() ).thenReturn( "testFileName.ini" );

  String header = "test ini data with umlauts";
  String key = "key";
  String testValue = "value-with-äöü";
  String testData = "[" + header + "]\r\n"
          + key + "=" + testValue;
  String charsetEncode = "Windows-1252";

  InputStream inputStream = new ByteArrayInputStream( testData.getBytes(
          Charset.forName( charsetEncode ) ) );
  FileContent fileContent = Mockito.mock( FileContent.class );
  Mockito.when( fileObject.getContent() ).thenReturn( fileContent );
  Mockito.when( fileContent.getInputStream() ).thenReturn( inputStream );
  Mockito.when( fileObject.getName() ).thenReturn( fileName );
  fileInputList.addFile( fileObject );

  propertyInputData.files = fileInputList;
  propertyInputData.propfiles = false;
  propertyInputData.realEncoding = charsetEncode;

  PropertyInput propertyInput = Mockito.mock( PropertyInput.class );

  Field logField = BaseStep.class.getDeclaredField( "log" );
  logField.setAccessible( true );
  logField.set( propertyInput, Mockito.mock( LogChannelInterface.class ) );

  Mockito.doCallRealMethod().when( propertyInput ).dispose( propertyInputMeta, propertyInputData );

  propertyInput.dispose( propertyInputMeta, propertyInputData );

  Method method = PropertyInput.class.getDeclaredMethod( "openNextFile" );
  method.setAccessible( true );
  method.invoke( propertyInput );

  Assert.assertEquals( testValue, propertyInputData.wini.get( header ).get( key ) );
}
 
Example #23
Source File: KettleEtlLogger.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
public KettleEtlLogger( BaseStep step ) {
  this.step = step;
}
 
Example #24
Source File: IngresVectorwiseTest.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings( "unused" )
private BaseStep doOutput( List<Object[]> rows, String maxErrorsNumber ) throws KettleException {

  IngresVectorwiseLoaderData ivwData = new IngresVectorwiseLoaderData();
  IngresVectorwiseLoaderTest ivwLoader =
    new IngresVectorwiseLoaderTest( stepMockHelper.stepMeta, ivwData, 0, stepMockHelper.transMeta,
      stepMockHelper.trans );

  DatabaseMeta defMeta = mock( DatabaseMeta.class );
  when( stepMockHelper.processRowsStepMetaInterface.getDatabaseMeta() ).thenReturn( defMeta );
  when( defMeta.getQuotedSchemaTableCombination( anyString(), anyString() ) ).thenReturn( "test_table" );
  ivwLoader.init( stepMockHelper.processRowsStepMetaInterface, ivwData );
  RowSet rowSet = stepMockHelper.getMockInputRowSet( rows );
  ivwLoader.addRowSetToInputRowSets( rowSet );

  when( stepMockHelper.processRowsStepMetaInterface.isUsingVwload() ).thenReturn( true );
  when( stepMockHelper.processRowsStepMetaInterface.getBufferSize() ).thenReturn( "5000" );
  when( stepMockHelper.processRowsStepMetaInterface.getMaxNrErrors() ).thenReturn( maxErrorsNumber );
  when( stepMockHelper.processRowsStepMetaInterface.getFieldStream() ).thenReturn( fieldStream );
  when( stepMockHelper.processRowsStepMetaInterface.getFieldDatabase() ).thenReturn( fieldStream );
  File errorFile = new File( "/tmp/error.txt" );
  File fifoFile = new File( "/tmp/fifo" );
  try {
    errorFile = createTemplateFile();
    fifoFile = createTemplateFile();
    boolean deleted  = fifoFile.delete();
  } catch ( IOException e ) {
    e.printStackTrace();
  }

  when( stepMockHelper.processRowsStepMetaInterface.getErrorFileName() ).thenReturn( errorFile.getPath() );
  when( stepMockHelper.processRowsStepMetaInterface.getFifoFileName() ).thenReturn( fifoFile.getPath() );

  RowMetaInterface inputRowMeta = mock( RowMetaInterface.class );
  ivwLoader.setInputRowMeta( inputRowMeta );
  when( rowSet.getRowMeta() ).thenReturn( inputRowMeta );
  ValueMetaInteger valueMetaInteger = new ValueMetaInteger();
  ValueMetaString valueMetaString = new ValueMetaString();
  when( inputRowMeta.getValueMeta( 0 ) ).thenReturn( valueMetaInteger );
  when( inputRowMeta.getValueMeta( 1 ) ).thenReturn( valueMetaString );
  when( inputRowMeta.indexOfValue( fieldStream[0] ) ).thenReturn( 0 );
  when( inputRowMeta.indexOfValue( fieldStream[1] ) ).thenReturn( 1 );
  when( inputRowMeta.clone() ).thenReturn( inputRowMeta );

  for ( Object[] row1 : rows ) {
    ivwLoader.processRow( stepMockHelper.processRowsStepMetaInterface, ivwData );
  }
  ivwLoader.processRow( stepMockHelper.processRowsStepMetaInterface, ivwData );
  ivwLoader.dispose( stepMockHelper.processRowsStepMetaInterface, ivwData );
  ivwLoader.getLogChannel().snap( Metrics.METRIC_STEP_EXECUTION_STOP );

  return ivwLoader;
}
 
Example #25
Source File: GetFilesRowsCount.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
private void getRowNumber() throws KettleException {
  try {

    if ( data.file.getType() == FileType.FILE ) {
      data.fr = KettleVFS.getInputStream( data.file );
      // Avoid method calls - see here:
      // http://java.sun.com/developer/technicalArticles/Programming/PerfTuning/
      byte[] buf = new byte[8192]; // BufferedaInputStream default buffer size
      int n;
      boolean prevCR = false;
      while ( ( n = data.fr.read( buf ) ) != -1 ) {
        for ( int i = 0; i < n; i++ ) {
          data.foundData = true;
          if ( meta.getRowSeparatorFormat().equals( "CRLF" ) ) {
            // We need to check for CRLF
            if ( buf[i] == '\r' || buf[i] == '\n' ) {
              if ( buf[i] == '\r' ) {
                // we have a carriage return
                // keep track of it..maybe we will have a line feed right after :-)
                prevCR = true;
              } else if ( buf[i] == '\n' ) {
                // we have a line feed
                // let's see if we had previously a carriage return
                if ( prevCR ) {
                  // we have a carriage return followed by a line feed
                  data.rownr++;
                  // Maybe we won't have data after
                  data.foundData = false;
                  prevCR = false;
                }
              }
            } else {
              // we have another char (other than \n , \r)
              prevCR = false;
            }

          } else {
            if ( buf[i] == data.separator ) {
              data.rownr++;
              // Maybe we won't have data after
              data.foundData = false;
            }
          }
        }
      }
    }
    if ( isDetailed() ) {
      logDetailed( BaseMessages.getString( PKG, "GetFilesRowsCount.Log.RowsInFile", data.file.toString(), ""
        + data.rownr ) );
    }
  } catch ( Exception e ) {
    throw new KettleException( e );
  } finally {
    // Close inputstream - not used except for counting
    if ( data.fr != null ) {
      BaseStep.closeQuietly( data.fr );
      data.fr = null;
    }
  }

}
 
Example #26
Source File: PrioritizeStreams.java    From pentaho-kettle with Apache License 2.0 2 votes vote down vote up
/**
 * Checks whether 2 template rows are compatible for the mergestep.
 *
 * @param referenceRow
 *          Reference row
 * @param compareRow
 *          Row to compare to
 *
 * @return true when templates are compatible.
 * @throws KettleRowException
 *           in case there is a compatibility error.
 */
protected void checkInputLayoutValid( RowMetaInterface referenceRowMeta, RowMetaInterface compareRowMeta ) throws KettleRowException {
  if ( referenceRowMeta != null && compareRowMeta != null ) {
    BaseStep.safeModeChecking( referenceRowMeta, compareRowMeta );
  }
}
 
Example #27
Source File: MergeRows.java    From pentaho-kettle with Apache License 2.0 2 votes vote down vote up
/**
 * Checks whether 2 template rows are compatible for the mergestep.
 *
 * @param referenceRowMeta
 *          Reference row
 * @param compareRowMeta
 *          Row to compare to
 *
 * @return true when templates are compatible.
 * @throws KettleRowException
 *           in case there is a compatibility error.
 */
static void checkInputLayoutValid( RowMetaInterface referenceRowMeta, RowMetaInterface compareRowMeta ) throws KettleRowException {
  if ( referenceRowMeta != null && compareRowMeta != null ) {
    BaseStep.safeModeChecking( referenceRowMeta, compareRowMeta );
  }
}
 
Example #28
Source File: Append.java    From pentaho-kettle with Apache License 2.0 2 votes vote down vote up
/**
 * Checks whether 2 template rows are compatible for the mergestep.
 *
 * @param referenceRowMeta
 *          Reference row
 * @param compareRowMeta
 *          Row to compare to
 *
 * @return true when templates are compatible.
 * @throws KettleRowException
 *           in case there is a compatibility error.
 */
protected void checkInputLayoutValid( RowMetaInterface referenceRowMeta, RowMetaInterface compareRowMeta ) throws KettleRowException {
  if ( referenceRowMeta != null && compareRowMeta != null ) {
    BaseStep.safeModeChecking( referenceRowMeta, compareRowMeta );
  }
}