Java Code Examples for org.pentaho.di.core.row.RowMetaInterface#getString()

The following examples show how to use org.pentaho.di.core.row.RowMetaInterface#getString() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
public List<String> getNodeLabels( String[] labelFields, String[] labelValues, RowMetaInterface rowMeta, Object[] rowData, int[] labelIndexes ) throws KettleValueException {
  List<String> labels = new ArrayList<>();

  for ( int a = 0; a < labelFields.length; a++ ) {
    String label = null;
    if ( StringUtils.isNotEmpty( labelFields[ a ] ) ) {
      label = rowMeta.getString( rowData, labelIndexes[ a ] );
    }
    if ( StringUtils.isEmpty( label ) && StringUtils.isNotEmpty( labelValues[ a ] ) ) {
      label = labelValues[ a ];
    }
    if ( StringUtils.isNotEmpty( label ) ) {
      labels.add( label );
    }
  }
  return labels;
}
 
Example 2
@Override
public Collection<IExternalResourceInfo> getResourcesFromRow(
  GetXMLData textFileInput, RowMetaInterface rowMeta, Object[] row ) {
  Collection<IExternalResourceInfo> resources = new LinkedList<IExternalResourceInfo>();
  // For some reason the step doesn't return the StepMetaInterface directly, so go around it
  GetXMLDataMeta meta = (GetXMLDataMeta) textFileInput.getStepMetaInterface();
  if ( meta == null ) {
    meta = (GetXMLDataMeta) textFileInput.getStepMeta().getStepMetaInterface();
  }

  try {
    if ( meta.getIsAFile() ) {
      String filename = ( meta == null ) ? null : rowMeta.getString( row, meta.getXMLField(), null );
      if ( !Utils.isEmpty( filename ) ) {
        FileObject fileObject = KettleVFS.getFileObject( filename );
        resources.add( ExternalResourceInfoFactory.createFileResource( fileObject, true ) );
      }
    }
    // TODO URLs?
  } catch ( KettleException kve ) {
    // TODO throw exception or ignore?
  }

  return resources;
}
 
Example 3
Source Project: pentaho-kettle   File: SortRowsIT.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Check the list, the list has to be sorted.
 */
void checkStringRows( List<RowMetaAndData> rows, boolean ascending ) throws Exception {
  String prev_key1 = null, prev_key2 = null;
  int idx = 0;

  for ( RowMetaAndData rm : rows ) {
    Object[] r1 = rm.getData();
    RowMetaInterface rmi = rm.getRowMeta();

    String key1 = rmi.getString( r1, "KEY1", "" );
    String key2 = rmi.getString( r1, "KEY2", "" );

    if ( prev_key1 != null && prev_key2 != null ) {
      if ( ascending ) {
        if ( prev_key1.compareTo( key1 ) == 0 ) {
          if ( prev_key2.compareTo( key2 ) > 0 ) {
            fail( "error in sort" );
          }
        } else if ( prev_key1.compareTo( key1 ) > 0 ) {
          fail( "error in sort" );
        }
      } else {
        if ( prev_key1.compareTo( key1 ) == 0 ) {
          if ( prev_key2.compareTo( key2 ) < 0 ) {
            fail( "error in sort" );
          }
        } else if ( prev_key1.compareTo( key1 ) < 0 ) {
          fail( "error in sort" );
        }
      }
    }
    prev_key1 = key1;
    prev_key2 = key2;
    idx++;
  }
  Assert.assertEquals( "less rows returned than expected", MAX_COUNT, idx );
}
 
Example 4
private Map<String, Map<String, String>> loadJobAttributesMap( ObjectId jobId ) throws KettleException {
  Map<String, Map<String, String>> attributesMap = new HashMap<String, Map<String, String>>();

  List<Object[]> attributeRows =
    repository.connectionDelegate.getJobAttributesWithPrefix( jobId, JOB_ATTRIBUTE_PREFIX );
  RowMetaInterface rowMeta = repository.connectionDelegate.getReturnRowMeta();
  for ( Object[] attributeRow : attributeRows ) {
    String code = rowMeta.getString( attributeRow, KettleDatabaseRepository.FIELD_JOB_ATTRIBUTE_CODE, null );
    String value =
      rowMeta.getString( attributeRow, KettleDatabaseRepository.FIELD_JOB_ATTRIBUTE_VALUE_STR, null );
    if ( code != null && value != null ) {
      code = code.substring( JOB_ATTRIBUTE_PREFIX.length() );
      int tabIndex = code.indexOf( '\t' );
      if ( tabIndex > 0 ) {
        String groupName = code.substring( 0, tabIndex );
        String key = code.substring( tabIndex + 1 );
        Map<String, String> attributes = attributesMap.get( groupName );
        if ( attributes == null ) {
          attributes = new HashMap<String, String>();
          attributesMap.put( groupName, attributes );
        }
        attributes.put( key, value );
      }
    }
  }

  return attributesMap;
}
 
Example 5
private synchronized int searchNrStepAttributes( ObjectId id_step, String code ) throws KettleValueException {
  // Search the index of the first step attribute with the specified code...
  //
  int idx = searchStepAttributeIndexInBuffer( id_step, code, 0L );
  if ( idx < 0 ) {
    return 0;
  }

  int nr = 1;
  int offset = 1;

  if ( idx + offset >= stepAttributesBuffer.size() ) {
    // Only 1, the last of the attributes buffer.
    //
    return 1;
  }
  Object[] look = stepAttributesBuffer.get( idx + offset );
  RowMetaInterface rowMeta = stepAttributesRowMeta;

  long lookID = rowMeta.getInteger( look, 0 );
  String lookCode = rowMeta.getString( look, 1 );

  while ( lookID == new LongObjectId( id_step ).longValue() && code.equalsIgnoreCase( lookCode ) ) {
    // Find the maximum
    //
    nr = rowMeta.getInteger( look, 2 ).intValue() + 1;
    offset++;
    if ( idx + offset < stepAttributesBuffer.size() ) {
      look = stepAttributesBuffer.get( idx + offset );

      lookID = rowMeta.getInteger( look, 0 );
      lookCode = rowMeta.getString( look, 1 );
    } else {
      return nr;
    }
  }
  return nr;
}
 
Example 6
private Map<String, Map<String, String>> loadTransAttributesMap( ObjectId transformationId ) throws KettleException {
  Map<String, Map<String, String>> attributesMap = new HashMap<String, Map<String, String>>();

  List<Object[]> attributeRows =
    repository.connectionDelegate.getTransAttributesWithPrefix( transformationId, TRANS_ATTRIBUTE_PREFIX );
  RowMetaInterface rowMeta = repository.connectionDelegate.getReturnRowMeta();
  for ( Object[] attributeRow : attributeRows ) {
    String code = rowMeta.getString( attributeRow, KettleDatabaseRepository.FIELD_TRANS_ATTRIBUTE_CODE, null );
    String value =
      rowMeta.getString( attributeRow, KettleDatabaseRepository.FIELD_TRANS_ATTRIBUTE_VALUE_STR, null );
    if ( code != null && value != null ) {
      code = code.substring( TRANS_ATTRIBUTE_PREFIX.length() );
      int tabIndex = code.indexOf( TRANS_ATTRIBUTE_PREFIX_DELIMITER );
      if ( tabIndex > 0 ) {
        String groupName = code.substring( 0, tabIndex );
        String key = code.substring( tabIndex + 1 );
        Map<String, String> attributes = attributesMap.get( groupName );
        if ( attributes == null ) {
          attributes = new HashMap<String, String>();
          attributesMap.put( groupName, attributes );
        }
        attributes.put( key, value );
      }
    }
  }

  return attributesMap;
}
 
Example 7
private Map<String, Map<String, String>> loadStepAttributesMap( ObjectId stepId ) throws KettleException {
  Map<String, Map<String, String>> attributesMap = new HashMap<String, Map<String, String>>();

  List<Object[]> attributeRows = repository.connectionDelegate.getStepAttributesBuffer();
  RowMetaInterface rowMeta = repository.connectionDelegate.getStepAttributesRowMeta();
  for ( Object[] attributeRow : attributeRows ) {
    String code = rowMeta.getString( attributeRow, KettleDatabaseRepository.FIELD_STEP_ATTRIBUTE_CODE, null );
    if ( code != null && code.startsWith( STEP_ATTRIBUTE_PREFIX ) ) {
      String value =
        rowMeta.getString( attributeRow, KettleDatabaseRepository.FIELD_STEP_ATTRIBUTE_VALUE_STR, null );
      if ( value != null ) {
        code = code.substring( STEP_ATTRIBUTE_PREFIX.length() );
        int tabIndex = code.indexOf( '\t' );
        if ( tabIndex > 0 ) {
          String groupName = code.substring( 0, tabIndex );
          String key = code.substring( tabIndex + 1 );
          Map<String, String> attributes = attributesMap.get( groupName );
          if ( attributes == null ) {
            attributes = new HashMap<String, String>();
            attributesMap.put( groupName, attributes );
          }
          attributes.put( key, value );
        }
      }
    }
  }

  return attributesMap;
}
 
Example 8
private Map<String, Map<String, String>> loadJobEntryAttributesMap( ObjectId jobId, Object jobEntryId ) throws KettleException {
  Map<String, Map<String, String>> attributesMap = new HashMap<String, Map<String, String>>();

  List<Object[]> attributeRows =
    repository.connectionDelegate.getJobEntryAttributesWithPrefix( jobId, jobId, JOBENTRY_ATTRIBUTE_PREFIX );
  RowMetaInterface rowMeta = repository.connectionDelegate.getReturnRowMeta();
  for ( Object[] attributeRow : attributeRows ) {
    String code = rowMeta.getString( attributeRow, KettleDatabaseRepository.FIELD_JOBENTRY_ATTRIBUTE_CODE, null );
    String value =
      rowMeta.getString( attributeRow, KettleDatabaseRepository.FIELD_JOBENTRY_ATTRIBUTE_VALUE_STR, null );
    if ( code != null && value != null ) {
      code = code.substring( JOBENTRY_ATTRIBUTE_PREFIX.length() );
      int tabIndex = code.indexOf( '\t' );
      if ( tabIndex > 0 ) {
        String groupName = code.substring( 0, tabIndex );
        String key = code.substring( tabIndex + 1 );
        Map<String, String> attributes = attributesMap.get( groupName );
        if ( attributes == null ) {
          attributes = new HashMap<String, String>();
          attributesMap.put( groupName, attributes );
        }
        attributes.put( key, value );
      }
    }
  }

  return attributesMap;
}
 
Example 9
/**
 * When {@code failOnParseError} is set to {@code false}, returns the {@link String} value from {@link
 * org.pentaho.di.core.row.RowMeta} at the given {@code index}, or directly from the {@code row} object, if there is a
 * problem fetching the value from {@link org.pentaho.di.core.row.RowMeta}. When {@code failOnParseError} is {@code
 * true}, any {@link Exception} thrown by the call to {@link org.pentaho.di.core.row.RowMeta#getString(Object[], int)}
 * is reported back to the caller.
 *
 * @param rowMeta          an instance of {@link RowMetaInterface}
 * @param row              an Object array containing row data
 * @param index            the index representing the column in a row
 * @param failOnParseError when true, Exceptions are reported back to the called, when false, exceptions are ignored
 *                         and a null value is returned
 * @return the row value at the given index
 */
default String getStringFromRow( final RowMetaInterface rowMeta, final Object[] row, final int index,
                                 final boolean failOnParseError ) throws KettleException {
  String string = null;
  Exception exc = null;
  try {
    string = rowMeta.getString( row, index );
  } catch ( final Exception e ) {
    exc = e;
  }


  // if 'failOnParseError' is true, and we caught an exception, we either re-throw the exception, or wrap its as a
  // KettleException, if it isn't one already
  if ( failOnParseError ) {
    if ( exc instanceof KettleException ) {
      throw (KettleException) exc;
    } else if ( exc != null ) {
      throw new KettleException( exc );
    }
  }

  // if 'failOnParseError' is false, or there is no exceptionotherwise, we get the string value straight from the row
  // object
  if ( string == null ) {
    if ( ( row.length <= index ) ) {
      if ( failOnParseError ) {
        throw new KettleException( new NullPointerException() );
      }
    }
    string = row.length <= index || row[ index ] == null ? null : row[ index ].toString();
  }

  return string;
}
 
Example 10
Source Project: pentaho-kettle   File: JsonInputTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException {
  if ( rowNbr >= data.length ) {
    throw new ComparisonFailure( "too many output rows", "" + data.length, "" + ( rowNbr + 1 ) );
  } else {
    for ( int i = 0; i < data[ rowNbr ].length; i++ ) {
      try {
        boolean eq = true;
        if ( comparators.containsKey( i ) ) {
          Comparison<Object> comp = comparators.get( i );
          if ( comp != null ) {
            eq = comp.equals( data[ rowNbr ][ i ], row[ i ] );
          }
        } else {
          ValueMetaInterface valueMeta = rowMeta.getValueMeta( i );
          eq = valueMeta.compare( data[ rowNbr ][ i ], row[ i ] ) == 0;
        }
        if ( !eq ) {
          throw new ComparisonFailure( String.format( "Mismatch row %d, column %d", rowNbr, i ),
            rowMeta.getString( data[ rowNbr ] ), rowMeta.getString( row ) );
        }
      } catch ( Exception e ) {
        throw new AssertionError( String.format( "Value type at row %d, column %d", rowNbr, i ), e );
      }
    }
    rowNbr++;
  }
}
 
Example 11
/**
 * This method is called when a row is written to another step (even if there is no next step)
 *
 * @param rowMeta the metadata of the row
 * @param row     the data of the row
 * @throws org.pentaho.di.core.exception.KettleStepException an exception that can be thrown to hard stop the step
 */
public void rowWrittenEvent( final RowMetaInterface rowMeta, final Object[] row ) throws KettleStepException {
  if ( firstCall ) {
    this.tableModel = createTableModel( rowMeta );
    firstCall = false;
  }

  if ( queryLimit > 0 && rowsWritten > queryLimit ) {
    return;
  }

  try {
    rowsWritten += 1;

    final int count = tableModel.getColumnCount();
    final Object dataRow[] = new Object[ count ];
    for ( int columnNo = 0; columnNo < count; columnNo++ ) {
      final ValueMetaInterface valueMeta = rowMeta.getValueMeta( columnNo );

      switch( valueMeta.getType() ) {
        case ValueMetaInterface.TYPE_BIGNUMBER:
          dataRow[ columnNo ] = rowMeta.getBigNumber( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_BOOLEAN:
          dataRow[ columnNo ] = rowMeta.getBoolean( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_DATE:
          dataRow[ columnNo ] = rowMeta.getDate( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_INTEGER:
          dataRow[ columnNo ] = rowMeta.getInteger( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_NONE:
          dataRow[ columnNo ] = rowMeta.getString( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_NUMBER:
          dataRow[ columnNo ] = rowMeta.getNumber( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_STRING:
          dataRow[ columnNo ] = rowMeta.getString( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_BINARY:
          dataRow[ columnNo ] = rowMeta.getBinary( row, columnNo );
          break;
        default:
          dataRow[ columnNo ] = rowMeta.getString( row, columnNo );
      }
    }
    tableModel.addRow( dataRow );
  } catch ( final KettleValueException kve ) {
    throw new KettleStepException( kve );
  } catch ( final Exception e ) {
    throw new KettleStepException( e );
  }
}
 
Example 12
public void processRow( MemoryResultSet memResults, final RowMetaInterface rowMeta, final Object[] row )
  throws KettleStepException {
  if ( memResults == null ) {
    return;
  }
  try {
    Object[] pentahoRow = new Object[memResults.getColumnCount()];
    for ( int columnNo = 0; columnNo < memResults.getColumnCount(); columnNo++ ) {
      ValueMetaInterface valueMeta = rowMeta.getValueMeta( columnNo );

      switch ( valueMeta.getType() ) {
        case ValueMetaInterface.TYPE_BIGNUMBER:
          pentahoRow[columnNo] = rowMeta.getBigNumber( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_BOOLEAN:
          pentahoRow[columnNo] = rowMeta.getBoolean( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_DATE:
          pentahoRow[columnNo] = rowMeta.getDate( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_INTEGER:
          pentahoRow[columnNo] = rowMeta.getInteger( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_NONE:
          pentahoRow[columnNo] = rowMeta.getString( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_NUMBER:
          pentahoRow[columnNo] = rowMeta.getNumber( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_STRING:
          pentahoRow[columnNo] = rowMeta.getString( row, columnNo );
          break;
        default:
          pentahoRow[columnNo] = rowMeta.getString( row, columnNo );
      }
    }

    if ( logger.isDebugEnabled() ) {
      StringBuffer sb = new StringBuffer();
      for ( int i = 0; i < pentahoRow.length; i++ ) {
        sb.append( pentahoRow[i] ).append( "; " ); //$NON-NLS-1$
      }
      logger.debug( sb.toString() );
    }

    memResults.addRow( pentahoRow );
  } catch ( KettleValueException e ) {
    throw new KettleStepException( e );
  }
}
 
Example 13
Source Project: pentaho-kettle   File: TestUtilities.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Check the 2 lists comparing the rows in order. If they are not the same fail the test.
 *
 * @param rows1          set 1 of rows to compare
 * @param rows2          set 2 of rows to compare
 * @param fileNameColumn Number of the column containing the filename. This is only checked for being non-null (some
 *                       systems maybe canonize names differently than we input).
 */
public static void checkRows( List<RowMetaAndData> rows1, List<RowMetaAndData> rows2, int fileNameColumn )
  throws TestFailedException {

  int idx = 1;
  if ( rows1.size() != rows2.size() ) {
    throw new TestFailedException( "Number of rows is not the same: " + rows1.size() + " and " + rows2.size() );
  }
  Iterator<RowMetaAndData> itrRows1 = rows1.iterator();
  Iterator<RowMetaAndData> itrRows2 = rows2.iterator();

  while ( itrRows1.hasNext() && itrRows2.hasNext() ) {
    RowMetaAndData rowMetaAndData1 = itrRows1.next();
    RowMetaAndData rowMetaAndData2 = itrRows2.next();

    RowMetaInterface rowMetaInterface1 = rowMetaAndData1.getRowMeta();

    Object[] rowObject1 = rowMetaAndData1.getData();
    Object[] rowObject2 = rowMetaAndData2.getData();

    if ( rowMetaAndData1.size() != rowMetaAndData2.size() ) {
      throw new TestFailedException( "row number " + idx + " is not equal" );
    }

    int[] fields = new int[ rowMetaInterface1.size() ];
    for ( int ydx = 0; ydx < rowMetaInterface1.size(); ydx++ ) {
      fields[ ydx ] = ydx;
    }

    try {
      if ( fileNameColumn >= 0 ) {
        rowObject1[ fileNameColumn ] = rowObject2[ fileNameColumn ];
      }
      if ( rowMetaAndData1.getRowMeta().compare( rowObject1, rowObject2, fields ) != 0 ) {
        throw new ComparisonFailure( "row nr " + idx + " is not equal",
        rowMetaInterface1.getString( rowObject1 ),
        rowMetaInterface1.getString( rowObject2 ) ); 
      }
    } catch ( KettleValueException e ) {
      throw new TestFailedException( "row nr " + idx + " is not equal" );
    }
    idx++;
  }
}
 
Example 14
Source Project: pentaho-kettle   File: TeraFast.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Write a single row to the temporary data file.
 *
 * @param rowMetaInterface
 *          describe the row of data
 *
 * @param row
 *          row entries
 * @throws KettleException
 *           ...
 */
@SuppressWarnings( "ArrayToString" )
public void writeToDataFile( RowMetaInterface rowMetaInterface, Object[] row ) throws KettleException {
  // Write the data to the output
  ValueMetaInterface valueMeta = null;

  for ( int i = 0; i < row.length; i++ ) {
    if ( row[i] == null ) {
      break; // no more rows
    }
    valueMeta = rowMetaInterface.getValueMeta( i );
    if ( row[i] != null ) {
      switch ( valueMeta.getType() ) {
        case ValueMetaInterface.TYPE_STRING:
          String s = rowMetaInterface.getString( row, i );
          dataFilePrintStream.print( pad( valueMeta, s.toString() ) );
          break;
        case ValueMetaInterface.TYPE_INTEGER:
          Long l = rowMetaInterface.getInteger( row, i );
          dataFilePrintStream.print( pad( valueMeta, l.toString() ) );
          break;
        case ValueMetaInterface.TYPE_NUMBER:
          Double d = rowMetaInterface.getNumber( row, i );
          dataFilePrintStream.print( pad( valueMeta, d.toString() ) );
          break;
        case ValueMetaInterface.TYPE_BIGNUMBER:
          BigDecimal bd = rowMetaInterface.getBigNumber( row, i );
          dataFilePrintStream.print( pad( valueMeta, bd.toString() ) );
          break;
        case ValueMetaInterface.TYPE_DATE:
          Date dt = rowMetaInterface.getDate( row, i );
          dataFilePrintStream.print( simpleDateFormat.format( dt ) );
          break;
        case ValueMetaInterface.TYPE_BOOLEAN:
          Boolean b = rowMetaInterface.getBoolean( row, i );
          if ( b.booleanValue() ) {
            dataFilePrintStream.print( "Y" );
          } else {
            dataFilePrintStream.print( "N" );
          }
          break;
        case ValueMetaInterface.TYPE_BINARY:
          byte[] byt = rowMetaInterface.getBinary( row, i );
          // REVIEW - this does an implicit byt.toString, which can't be what was intended.
          dataFilePrintStream.print( byt );
          break;
        default:
          throw new KettleException( BaseMessages.getString(
            PKG, "TeraFast.Exception.TypeNotSupported", valueMeta.getType() ) );
      }
    }
    dataFilePrintStream.print( FastloadControlBuilder.DATAFILE_COLUMN_SEPERATOR );
  }
  dataFilePrintStream.print( Const.CR );
}
 
Example 15
Source Project: pentaho-kettle   File: XBaseInput.java    License: Apache License 2.0 4 votes vote down vote up
public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException {
  meta = (XBaseInputMeta) smi;
  data = (XBaseInputData) sdi;

  // See if we need to get a list of files from input...
  if ( first ) { // we just got started

    first = false;

    // The output row meta data, what does it look like?
    //
    data.outputRowMeta = new RowMeta();

    if ( meta.isAcceptingFilenames() ) {
      // Read the files from the specified input stream...
      data.files.getFiles().clear();

      int idx = -1;

      RowSet rowSet = findInputRowSet( meta.getAcceptingStepName() );
      Object[] fileRowData = getRowFrom( rowSet );
      while ( fileRowData != null ) {
        RowMetaInterface fileRowMeta = rowSet.getRowMeta();
        if ( idx < 0 ) {
          idx = fileRowMeta.indexOfValue( meta.getAcceptingField() );
          if ( idx < 0 ) {
            logError( BaseMessages.getString( PKG, "XBaseInput.Log.Error.UnableToFindFilenameField", meta
              .getAcceptingField() ) );
            setErrors( 1 );
            stopAll();
            return false;
          }
        }
        try {
          String filename = fileRowMeta.getString( fileRowData, idx );
          data.files.addFile( KettleVFS.getFileObject( filename, getTransMeta() ) );
        } catch ( Exception e ) {
          throw new KettleException( e );
        }

        // Grab another row
        //
        fileRowData = getRowFrom( rowSet );
      }

      if ( data.files.nrOfFiles() == 0 ) {
        logBasic( BaseMessages.getString( PKG, "XBaseInput.Log.Error.NoFilesSpecified" ) );
        setOutputDone();
        return false;
      }
    }

    data.outputRowMeta = meta.getOutputFields( data.files, getStepname() );

    // Open the first file & read the required rows in the buffer, stop
    // if it fails, exception will stop processLoop
    //
    openNextFile();
  }

  // Allocate the output row in advance, because we possibly want to add a few extra fields...
  //
  Object[] row = data.xbi.getRow( RowDataUtil.allocateRowData( data.outputRowMeta.size() ) );
  while ( row == null && data.fileNr < data.files.nrOfFiles() ) { // No more rows left in this file
    openNextFile();
    row = data.xbi.getRow( RowDataUtil.allocateRowData( data.outputRowMeta.size() ) );
  }

  if ( row == null ) {
    setOutputDone(); // signal end to receiver(s)
    return false; // end of data or error.
  }

  // OK, so we have read a line: increment the input counter
  incrementLinesInput();
  int outputIndex = data.fields.size();

  // Possibly add a filename...
  if ( meta.includeFilename() ) {
    row[outputIndex++] = data.file_dbf.getName().getURI();
  }

  // Possibly add a row number...
  if ( meta.isRowNrAdded() ) {
    row[outputIndex++] = new Long( getLinesInput() );
  }

  putRow( data.outputRowMeta, row ); // fill the rowset(s). (wait for empty)

  if ( checkFeedback( getLinesInput() ) ) {
    logBasic( BaseMessages.getString( PKG, "XBaseInput.Log.LineNr" ) + getLinesInput() );
  }

  if ( meta.getRowLimit() > 0 && getLinesInput() >= meta.getRowLimit() ) { // limit has been reached: stop now.
    setOutputDone();
    return false;
  }

  return true;
}
 
Example 16
/**
 * Read files from previous step.
 */
private RowMetaInterface[] filesFromPreviousStep() throws KettleException {
  RowMetaInterface[] infoStep = null;
  data.files.getFiles().clear();

  int idx = -1;
  RowSet rowSet = findInputRowSet( meta.inputFiles.acceptingStepName );

  Object[] fileRow = getRowFrom( rowSet );
  while ( fileRow != null ) {
    RowMetaInterface prevInfoFields = rowSet.getRowMeta();
    if ( idx < 0 ) {
      if ( meta.inputFiles.passingThruFields ) {
        data.passThruFields = new HashMap<String, Object[]>();
        infoStep = new RowMetaInterface[] { prevInfoFields };
        data.nrPassThruFields = prevInfoFields.size();
      }
      idx = prevInfoFields.indexOfValue( meta.inputFiles.acceptingField );
      if ( idx < 0 ) {
        logError( BaseMessages.getString( PKG, "BaseFileInputStep.Log.Error.UnableToFindFilenameField",
          meta.inputFiles.acceptingField ) );
        setErrors( getErrors() + 1 );
        stopAll();
        return null;
      }
    }
    String fileValue = prevInfoFields.getString( fileRow, idx );
    try {
      FileObject parentFileObject = KettleVFS.getFileObject( environmentSubstitute( fileValue ), getTransMeta() );
      if ( parentFileObject != null && parentFileObject.getType() == FileType.FOLDER ) { // it's a directory
        addFilesFromFolder( parentFileObject );
      } else {  // it's a file
        data.files.addFile( parentFileObject );
      }

      if ( meta.inputFiles.passingThruFields ) {
        for ( int fileIndex = 0; fileIndex < data.files.nrOfFiles(); fileIndex++ ) {
          StringBuilder sb = new StringBuilder();
          FileObject file = data.files.getFile( fileIndex );
          sb.append( fileIndex ).append( "_" ).append( file != null ? file.toString() : "" );
          data.passThruFields.put( sb.toString(), fileRow );
        }
      }
    } catch ( FileSystemException e ) {
      logError( BaseMessages.getString( PKG, "BaseFileInputStep.Log.Error.UnableToCreateFileObject", fileValue ), e );
      throw new KettleException( e );
    }

    // Grab another row
    fileRow = getRowFrom( rowSet );
  }

  if ( data.files.nrOfFiles() == 0 ) {
    if ( log.isDetailed() ) {
      logDetailed( BaseMessages.getString( PKG, "BaseFileInputStep.Log.Error.NoFilesSpecified" ) );
    }
    return null;
  }
  return infoStep;
}
 
Example 17
Source Project: pentaho-kettle   File: FixedInputDialog.java    License: Apache License 2.0 4 votes vote down vote up
private List<String> getFirst( FixedInputMeta meta, int limit ) throws IOException, KettleValueException {

    List<String> lines = new ArrayList<String>();

    FixedInputMeta oneMeta = new FixedInputMeta();
    getInfo( oneMeta );

    // Add a single field with the width of the line...
    //
    int lineWidth = Integer.parseInt( oneMeta.getLineWidth() );
    if ( lineWidth <= 0 ) {
      throw new IOException( "The width of a line can not be 0 or less." );
    }

    oneMeta.allocate( 1 );

    FixedFileInputField field = new FixedFileInputField();
    field.setName( "Field1" );
    field.setType( ValueMetaInterface.TYPE_STRING );
    field.setWidth( lineWidth );
    //CHECKSTYLE:Indentation:OFF
    oneMeta.getFieldDefinition()[0] = field;

    TransMeta previewMeta =
      TransPreviewFactory.generatePreviewTransformation( transMeta, oneMeta, wStepname.getText() );

    TransPreviewProgressDialog progressDialog =
      new TransPreviewProgressDialog(
        shell, previewMeta, new String[] { wStepname.getText() }, new int[] { limit } );
    progressDialog.open();

    Trans trans = progressDialog.getTrans();
    String loggingText = progressDialog.getLoggingText();

    if ( !progressDialog.isCancelled() ) {
      if ( trans.getResult() != null && trans.getResult().getNrErrors() > 0 ) {
        EnterTextDialog etd =
          new EnterTextDialog(
            shell, BaseMessages.getString( PKG, "System.Dialog.PreviewError.Title" ), BaseMessages.getString(
              PKG, "System.Dialog.PreviewError.Message" ), loggingText, true );
        etd.setReadOnly();
        etd.open();
      }
    }

    // The rows are in the transformation...
    //
    RowMetaInterface previewRowsMeta = progressDialog.getPreviewRowsMeta( wStepname.getText() );
    List<Object[]> previewRowsData = progressDialog.getPreviewRows( wStepname.getText() );
    for ( int i = 0; i < previewRowsData.size(); i++ ) {
      String line = previewRowsMeta.getString( previewRowsData.get( i ), 0 );
      lines.add( line );
    }

    return lines;
  }
 
Example 18
Source Project: pentaho-kettle   File: TestUtilities.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Check the 2 lists comparing the rows in order. If they are not the same fail the test.
 * 
 * @param rows1
 *          set 1 of rows to compare
 * @param rows2
 *          set 2 of rows to compare
 * @param fileNameColumn
 *          Number of the column containing the filename. This is only checked for being non-null (some systems maybe
 *          canonize names differently than we input).
 */
public static void checkRows( List<RowMetaAndData> rows1, List<RowMetaAndData> rows2, int fileNameColumn )
  throws TestFailedException {

  int idx = 1;
  if ( rows1.size() != rows2.size() ) {
    throw new TestFailedException( "Number of rows is not the same: " + rows1.size() + " and " + rows2.size() );
  }
  Iterator<RowMetaAndData> itrRows1 = rows1.iterator();
  Iterator<RowMetaAndData> itrRows2 = rows2.iterator();

  while ( itrRows1.hasNext() && itrRows2.hasNext() ) {
    RowMetaAndData rowMetaAndData1 = itrRows1.next();
    RowMetaAndData rowMetaAndData2 = itrRows2.next();

    RowMetaInterface rowMetaInterface1 = rowMetaAndData1.getRowMeta();

    Object[] rowObject1 = rowMetaAndData1.getData();
    Object[] rowObject2 = rowMetaAndData2.getData();

    if ( rowMetaAndData1.size() != rowMetaAndData2.size() ) {
      throw new TestFailedException( "row number " + idx + " is not equal" );
    }

    int[] fields = new int[rowMetaInterface1.size()];
    for ( int ydx = 0; ydx < rowMetaInterface1.size(); ydx++ ) {
      fields[ydx] = ydx;
    }

    try {
      if ( fileNameColumn >= 0 ) {
        rowObject1[fileNameColumn] = rowObject2[fileNameColumn];
      }
      if ( rowMetaAndData1.getRowMeta().compare( rowObject1, rowObject2, fields ) != 0 ) {
        throw new ComparisonFailure( "row nr " + idx + " is not equal", rowMetaInterface1.getString( rowObject1 ),
            rowMetaInterface1.getString( rowObject2 ) );
      }
    } catch ( KettleValueException e ) {
      throw new TestFailedException( "row nr " + idx + " is not equal" );
    }
    idx++;
  }
}
 
Example 19
Source Project: pentaho-kettle   File: GPBulkDataOutput.java    License: Apache License 2.0 4 votes vote down vote up
public void writeLine( RowMetaInterface mi, Object[] row ) throws KettleException {
  if ( first ) {
    first = false;

    enclosure = meta.getEnclosure();

    // Setup up the fields we need to take for each of the rows
    // as this speeds up processing.
    fieldNumbers = new int[meta.getFieldStream().length];
    for ( int i = 0; i < fieldNumbers.length; i++ ) {
      fieldNumbers[i] = mi.indexOfValue( meta.getFieldStream()[i] );
      if ( fieldNumbers[i] < 0 ) {
        throw new KettleException( "Could not find field " + meta.getFieldStream()[i] + " in stream" );
      }
    }

    sdfDate = new SimpleDateFormat( "yyyy-MM-dd" );
    sdfDateTime = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss.SSS" );
  }

  // Write the data to the output
  ValueMetaInterface v = null;
  int number = 0;
  for ( int i = 0; i < fieldNumbers.length; i++ ) {
    if ( i != 0 ) {
      output.print( "," );
    }
    number = fieldNumbers[i];
    v = mi.getValueMeta( number );
    if ( row[number] == null ) {
      // TODO (SB): special check for null in case of Strings.
      output.print( enclosure );
      output.print( enclosure );
    } else {
      switch ( v.getType() ) {
        case ValueMetaInterface.TYPE_STRING:
          String s = mi.getString( row, number );
          if ( s.indexOf( enclosure ) >= 0 ) {
            s = createEscapedString( s, enclosure );
          }
          output.print( enclosure );
          output.print( s );
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_INTEGER:
          Long l = mi.getInteger( row, number );
          output.print( enclosure );
          output.print( l );
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_NUMBER:
          Double d = mi.getNumber( row, number );
          output.print( enclosure );
          output.print( d );
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_BIGNUMBER:
          BigDecimal bd = mi.getBigNumber( row, number );
          output.print( enclosure );
          output.print( bd );
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_DATE:
          Date dt = mi.getDate( row, number );
          output.print( enclosure );
          String mask = meta.getDateMask()[i];
          if ( GPBulkLoaderMeta.DATE_MASK_DATETIME.equals( mask ) ) {
            output.print( sdfDateTime.format( dt ) );
          } else {
            // Default is date format
            output.print( sdfDate.format( dt ) );
          }
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_BOOLEAN:
          Boolean b = mi.getBoolean( row, number );
          output.print( enclosure );
          if ( b.booleanValue() ) {
            output.print( "Y" );
          } else {
            output.print( "N" );
          }
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_BINARY:
          byte[] byt = mi.getBinary( row, number );
          output.print( "<startlob>" );
          output.print( byt );
          output.print( "<endlob>" );
          break;
        default:
          throw new KettleException( "Unsupported type" );
      }
    }
  }
  output.print( Const.CR );
}