Java Code Examples for org.pentaho.di.core.row.RowMetaInterface#getValueMetaList()

The following examples show how to use org.pentaho.di.core.row.RowMetaInterface#getValueMetaList() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private static void setValueFormats( RowMetaInterface rowMeta ) {
  for ( ValueMetaInterface valueMeta : rowMeta.getValueMetaList() ) {
    if ( StringUtils.isEmpty( valueMeta.getConversionMask() ) ) {
      switch ( valueMeta.getType() ) {
        case ValueMetaInterface.TYPE_INTEGER:
          valueMeta.setConversionMask( "0" );
          break;
        case ValueMetaInterface.TYPE_NUMBER:
          valueMeta.setConversionMask( "0.#" );
          break;
        case ValueMetaInterface.TYPE_DATE:
          valueMeta.setConversionMask( "yyyyMMdd-HHmmss.SSS" );
          break;
        default:
          break;
      }
    }
  }
}
 
Example 2
/**
 * Given a fully defined output row metadata structure, determine which of the output fields are being copied from
 * the input fields and which must be the output of the script.
 *
 * @param fullOutputRowMeta    the fully defined output row metadata structure
 * @param scriptFields         row meta that will hold script only fields
 * @param inputPresentInOutput row meta that will hold input fields being copied
 * @param infos                the array of info row metas
 * @param stepName             the name of the step
 */
protected void determineInputFieldScriptFieldSplit( RowMetaInterface fullOutputRowMeta, RowMetaInterface scriptFields,
    RowMetaInterface inputPresentInOutput, RowMetaInterface[] infos, String stepName ) {

  scriptFields.clear();
  inputPresentInOutput.clear();
  RowMetaInterface consolidatedInputFields = new RowMeta();
  for ( RowMetaInterface r : infos ) {
    consolidatedInputFields.addRowMeta( r );
  }

  for ( ValueMetaInterface vm : fullOutputRowMeta.getValueMetaList() ) {
    int index = consolidatedInputFields.indexOfValue( vm.getName() );
    if ( index >= 0 ) {
      inputPresentInOutput.addValueMeta( vm );
    } else {
      // must be a script output (either a variable name field or data frame column name
      scriptFields.addValueMeta( vm );
    }
  }
}
 
Example 3
public RowMetaInterface updateFields( RowMetaInterface prev, MonetDBBulkLoaderData data ) {
  // update the field table from the fields coming from the previous step
  RowMetaInterface tableFields = new RowMeta();
  List<ValueMetaInterface> fields = prev.getValueMetaList();
  fieldTable = new String[fields.size()];
  fieldStream = new String[fields.size()];
  fieldFormatOk = new boolean[fields.size()];
  int idx = 0;
  for ( ValueMetaInterface field : fields ) {
    ValueMetaInterface tableField = field.clone();
    tableFields.addValueMeta( tableField );
    fieldTable[idx] = field.getName();
    fieldStream[idx] = field.getName();
    fieldFormatOk[idx] = true;
    idx++;
  }

  data.keynrs = new int[getFieldStream().length];
  for ( int i = 0; i < data.keynrs.length; i++ ) {
    data.keynrs[i] = i;
  }
  return tableFields;
}
 
Example 4
Source Project: pentaho-kettle   File: BaseStep.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * putRow is used to copy a row, to the alternate rowset(s) This should get priority over everything else!
 * (synchronized) If distribute is true, a row is copied only once to the output rowsets, otherwise copies are sent to
 * each rowset!
 *
 * @param row The row to put to the destination rowset(s).
 * @throws KettleStepException
 */
@Override
public void putRow( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException {
  if ( rowMeta != null ) {
    if ( !allowEmptyFieldNamesAndTypes ) {
      // check row meta for empty field name (BACKLOG-18004)
      for ( ValueMetaInterface vmi : rowMeta.getValueMetaList() ) {
        if ( StringUtils.isBlank( vmi.getName() ) ) {
          throw new KettleStepException( "Please set a field name for all field(s) that have 'null'." );
        }
        if ( vmi.getType() <= 0 ) {
          throw new KettleStepException( "Please set a value for the missing field(s) type." );
        }
      }
    }
  }
  getRowHandler().putRow( rowMeta, row );
}
 
Example 5
public void initializeColumns( RowMetaInterface inputRowMeta ) {
  if ( inputRowMeta == null ) {
    BaseMessages.getString( PKG, "RulesData.InitializeColumns.InputRowMetaIsNull" );
    return;
  }

  // Create objects for insertion into the rules engine
  List<ValueMetaInterface> columns = inputRowMeta.getValueMetaList();

  // This array must 1-1 match the row[] feteched by getRow()
  columnList = new Column[columns.size()];

  for ( int i = 0; i < columns.size(); i++ ) {
    ValueMetaInterface column = columns.get( i );

    Column c = new Column( true );
    c.setName( column.getName() );
    c.setType( column.getTypeDesc() );
    c.setPayload( null );

    columnList[i] = c;
  }
}
 
Example 6
private void addAllFieldsToEmptyFileDefinition( RowMetaInterface rowMeta, FileDefinition fileDefinition ) throws KettleStepException {
  if ( fileDefinition.getFieldDefinitions().isEmpty() ) {
    for ( ValueMetaInterface valueMeta : rowMeta.getValueMetaList() ) {
      fileDefinition.getFieldDefinitions().add( new FieldDefinition(
          valueMeta.getName(),
          valueMeta.getTypeDesc(),
          valueMeta.getLength(),
          valueMeta.getPrecision(),
          valueMeta.getConversionMask()
        )
      );
    }
  }
}
 
Example 7
Source Project: pentaho-kettle   File: TransDataLineage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Calculate the lineage for the specified step only...
 *
 * @param stepMeta
 *          The step to calculate the lineage for.
 * @throws KettleStepException
 *           In case there is an exception calculating the lineage. This is usually caused by unavailable data sources
 *           etc.
 */
private void calculateLineage( StepMeta stepMeta ) throws KettleStepException {
  RowMetaInterface outputMeta = transMeta.getStepFields( stepMeta );

  // The lineage is basically a calculation of origin for each output of a certain step.
  //
  for ( ValueMetaInterface valueMeta : outputMeta.getValueMetaList() ) {

    StepMeta originStepMeta = transMeta.findStep( valueMeta.getOrigin(), stepMeta );
    if ( originStepMeta != null ) {
      /* List<StepMeta> list = */fieldStepsMap.get( originStepMeta );
    }
  }
}
 
Example 8
Source Project: pentaho-kettle   File: TableAgileMart.java    License: Apache License 2.0 5 votes vote down vote up
protected Object[] writeToTable( RowMetaInterface rowMeta, Object[] r ) throws KettleException {
  // see if we need to truncate any string fields
  try {
    int index = 0;
    List<ValueMetaInterface> valueMetas = rowMeta.getValueMetaList();
    for ( ValueMetaInterface valueMeta : valueMetas ) {
      Object valueData = r[index];
      if ( valueData != null ) {
        if ( valueMeta.getType() == ValueMetaInterface.TYPE_STRING ) {
          String str = valueMeta.getString( valueData );
          int len = valueMeta.getLength();
          if ( len < 1 ) {
            len = MonetDBDatabaseMeta.DEFAULT_VARCHAR_LENGTH;
          }
          if ( str.length() > len ) {
            // TODO log this event
            str = str.substring( 0, len );
          }
          r[index] = str;
        }
      }
      index++;
    }
  } catch ( Exception e ) {
    throw new KettleException( "Error serializing rows of data to the psql command", e );
  }
  return super.writeToTable( rowMeta, r );
}
 
Example 9
Source Project: pentaho-kettle   File: BaseStepTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void outputRowMetasAreNotSharedAmongSeveralStreams() throws Exception {
  RowSet rs1 = new SingleRowRowSet();
  RowSet rs2 = new SingleRowRowSet();

  when( mockHelper.trans.isRunning() ).thenReturn( true );
  BaseStep baseStep =
    new BaseStep( mockHelper.stepMeta, mockHelper.stepDataInterface, 0, mockHelper.transMeta, mockHelper.trans );
  baseStep.setStopped( false );
  baseStep.setRepartitioning( StepPartitioningMeta.PARTITIONING_METHOD_NONE );
  baseStep.setOutputRowSets( Arrays.asList( rs1, rs2 ) );

  for ( RowSet rowSet : baseStep.getOutputRowSets() ) {
    assertNull( "RowMeta should be null, since no calls were done", rowSet.getRowMeta() );
  }

  RowMetaInterface rowMeta = new RowMeta();
  rowMeta.addValueMeta( new ValueMetaString( "string" ) );
  rowMeta.addValueMeta( new ValueMetaInteger( "integer" ) );

  baseStep.putRow( rowMeta, new Object[] { "a", 1 } );

  RowMetaInterface meta1 = rs1.getRowMeta();
  RowMetaInterface meta2 = rs2.getRowMeta();
  assertNotNull( meta1 );
  assertNotNull( meta2 );
  // content is same
  for ( ValueMetaInterface meta : meta1.getValueMetaList() ) {
    assertTrue( meta.getName(), meta2.exists( meta ) );
  }
  // whereas instances differ
  assertFalse( meta1 == meta2 );
}
 
Example 10
private static boolean isValidRowMeta( RowMetaInterface rowMeta ) {
  for ( ValueMetaInterface value : rowMeta.getValueMetaList() ) {
    String name = value.getName();
    if ( name == null || name.isEmpty() ) {
      return false;
    }
  }
  return true;
}
 
Example 11
Source Project: pentaho-kettle   File: WidgetUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * creates a ComboVar populated with fields from the previous step.
 * @param parentComposite - the composite in which the widget will be placed
 * @param props - PropsUI props for L&F
 * @param stepMeta - stepMeta of the current step
 * @param formData - FormData to use for placement
 */
public static ComboVar createFieldDropDown(
  Composite parentComposite, PropsUI props, BaseStepMeta stepMeta, FormData formData ) {
  TransMeta transMeta = stepMeta.getParentStepMeta().getParentTransMeta();
  ComboVar fieldDropDownCombo = new ComboVar( transMeta, parentComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
  props.setLook( fieldDropDownCombo );
  fieldDropDownCombo.addModifyListener( e -> stepMeta.setChanged() );

  fieldDropDownCombo.setLayoutData( formData );
  Listener focusListener = e -> {
    String current = fieldDropDownCombo.getText();
    fieldDropDownCombo.getCComboWidget().removeAll();
    fieldDropDownCombo.setText( current );

    try {
      RowMetaInterface rmi = transMeta.getPrevStepFields( stepMeta.getParentStepMeta().getName() );
      List ls = rmi.getValueMetaList();
      for ( Object l : ls ) {
        ValueMetaBase vmb = (ValueMetaBase) l;
        fieldDropDownCombo.add( vmb.getName() );
      }
    } catch ( KettleStepException ex ) {
      // can be ignored, since previous step may not be set yet.
      stepMeta.logDebug( ex.getMessage(), ex );
    }
  };
  fieldDropDownCombo.getCComboWidget().addListener( SWT.FocusIn, focusListener );
  return fieldDropDownCombo;
}
 
Example 12
Source Project: pentaho-kettle   File: AccessOutputTest.java    License: Apache License 2.0 5 votes vote down vote up
private static AccessInputField[] getInputFields( List<RowMetaAndData> in ) {
  RowMetaAndData rmd = in.get( 0 );
  RowMetaInterface rm = rmd.getRowMeta();
  List<AccessInputField> retval = new ArrayList<AccessInputField>();
  for ( ValueMetaInterface value : rm.getValueMetaList() ) {
    AccessInputField field = new AccessInputField( value.getName() );
    field.setColumn( value.getName() );
    field.setType( value.getType() );
    retval.add( field );
  }
  return retval.toArray( new AccessInputField[retval.size()] );
}
 
Example 13
@Override
protected Map<String, RowMetaInterface> getInputRowMetaInterfaces( GetXMLDataMeta meta ) {
  Map<String, RowMetaInterface> inputRows = getInputFields( meta );
  if ( inputRows == null ) {
    inputRows = new HashMap<>();
  }
  // Get some boolean flags from the meta for easier access
  boolean isInFields = meta.isInFields();
  boolean isAFile = meta.getIsAFile();
  boolean isAUrl = meta.isReadUrl();

  // only add resource fields if we are NOT getting the xml or file from a field
  if ( !isInFields || isAFile || isAUrl ) {
    RowMetaInterface stepFields = getOutputFields( meta );
    RowMetaInterface clone = stepFields.clone();
    // if there are previous steps providing data, we should remove them from the set of "resource" fields
    for ( RowMetaInterface rowMetaInterface : inputRows.values() ) {
      for ( ValueMetaInterface valueMetaInterface : rowMetaInterface.getValueMetaList() ) {
        try {
          clone.removeValueMeta( valueMetaInterface.getName() );
        } catch ( KettleValueException e ) {
          // could not find it in the output, skip it
        }
      }
    }
    inputRows.put( RESOURCE, clone );
  }
  return inputRows;
}
 
Example 14
@Test
public void testGetRequiredFields() throws Exception {
  XMLOutputMeta xmlOutputMeta = new XMLOutputMeta();
  xmlOutputMeta.setDefault();
  XMLField xmlField = new XMLField();
  xmlField.setFieldName( "aField" );
  xmlField.setType( 1 );
  xmlField.setLength( 10 );
  xmlField.setPrecision( 3 );

  XMLField xmlField2 = new XMLField();
  xmlField2.setFieldName( "bField" );
  xmlField2.setType( 3 );
  xmlField2.setLength( 4 );
  xmlField2.setPrecision( 5 );
  xmlOutputMeta.setOutputFields( new XMLField[] { xmlField, xmlField2 } );
  RowMetaInterface requiredFields = xmlOutputMeta.getRequiredFields( new Variables() );
  List<ValueMetaInterface> valueMetaList = requiredFields.getValueMetaList();
  assertEquals( 2, valueMetaList.size() );
  assertEquals( "aField", valueMetaList.get( 0 ).getName() );
  assertEquals( 1, valueMetaList.get( 0 ).getType() );
  assertEquals( 10, valueMetaList.get( 0 ).getLength() );
  assertEquals( 3, valueMetaList.get( 0 ).getPrecision() );

  assertEquals( "bField", valueMetaList.get( 1 ).getName() );
  assertEquals( 3, valueMetaList.get( 1 ).getType() );
  assertEquals( 4, valueMetaList.get( 1 ).getLength() );
  assertEquals( 5, valueMetaList.get( 1 ).getPrecision() );
}
 
Example 15
@Override public PDone expand( PCollection<KettleRow> input ) {

    try {
      // Only initialize once on this node/vm
      //
      BeamKettle.init( stepPluginClasses, xpPluginClasses );

      // Inflate the metadata on the node where this is running...
      //
      RowMetaInterface rowMeta = JsonRowMeta.fromJson( rowMetaJson );


      // Which table do we write to?
      //
      TableReference tableReference = new TableReference();
      if ( StringUtils.isNotEmpty( projectId ) ) {
        tableReference.setProjectId( projectId );
      }
      tableReference.setDatasetId( datasetId );
      tableReference.setTableId( tableId );

      TableSchema tableSchema = new TableSchema();
      List<TableFieldSchema> schemaFields = new ArrayList<>();
      for ( ValueMetaInterface valueMeta : rowMeta.getValueMetaList() ) {
        TableFieldSchema schemaField = new TableFieldSchema();
        schemaField.setName( valueMeta.getName() );
        switch(valueMeta.getType()){
          case ValueMetaInterface.TYPE_STRING: schemaField.setType( "STRING" ); break;
          case ValueMetaInterface.TYPE_INTEGER: schemaField.setType( "INTEGER" ); break;
          case ValueMetaInterface.TYPE_DATE: schemaField.setType( "DATETIME" ); break;
          case ValueMetaInterface.TYPE_BOOLEAN: schemaField.setType( "BOOLEAN" ); break;
          case ValueMetaInterface.TYPE_NUMBER: schemaField.setType( "FLOAT" ); break;
          default:
            throw new RuntimeException( "Conversion from Kettle value "+valueMeta.toString()+" to BigQuery TableRow isn't supported yet" );
        }
        schemaFields.add(schemaField);
      }
      tableSchema.setFields( schemaFields );

      SerializableFunction<KettleRow, TableRow> formatFunction = new KettleToBQTableRowFn( stepname, rowMetaJson, stepPluginClasses, xpPluginClasses );

      BigQueryIO.Write.CreateDisposition createDisposition;
      if (createIfNeeded) {
        createDisposition = BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED;
      }  else {
        createDisposition = BigQueryIO.Write.CreateDisposition.CREATE_NEVER;
      }

      BigQueryIO.Write.WriteDisposition writeDisposition;
      if (truncateTable) {
        writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_APPEND;
      } else {
        if (failIfNotEmpty) {
          writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_EMPTY;
        } else {
          writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_APPEND;
        }
      }

      BigQueryIO.Write<KettleRow> bigQueryWrite = BigQueryIO
        .<KettleRow>write()
        .to( tableReference )
        .withSchema( tableSchema )
        .withCreateDisposition( createDisposition )
        .withWriteDisposition( writeDisposition )
        .withFormatFunction( formatFunction );

      // TODO: pass the results along the way at some point
      //
      input.apply( stepname, bigQueryWrite );

      // End of the line
      //
      return PDone.in( input.getPipeline() );

    } catch ( Exception e ) {
      numErrors.inc();
      LOG.error( "Error in Beam BigQuery output transform", e );
      throw new RuntimeException( "Error in Beam BigQuery output transform", e );
    }
  }
 
Example 16
protected static List<Object> rowsToCSVNew( RowMetaInterface meta, List<Object[]> rows ) throws KettleException {
  List<Object> results = new ArrayList<>( 2 );
  boolean needsBase64;
  CharsetEncoder encoder = Charset.forName( "US-ASCII" ).newEncoder();
  Charset utf8 = Charset.forName( "UTF-8" );

  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  BufferedOutputStream buf = new BufferedOutputStream( bos );

  // header row
  StringBuilder builder = new StringBuilder();
  int i = 0;
  for ( ValueMetaInterface v : meta.getValueMetaList() ) {
    String name = quote( v.getName() );
    builder.append( i > 0 ? "," : "" ).append( name );
    i++;
  }
  builder.append( "\n" );

  // We look for non-ascii characters and, if found, encode to ascii base64 first. For some reason,
  // encoding directly to utf-8 on the Java side (Mac OS X; Java 8) causes the python utf-8 decoder to hang
  // when there are non-ascii characters (such as in Mayagüez) present. Encoding to base64 first, then decoding
  // this on the python side seems to fix the issue.
  needsBase64 = !encoder.canEncode( builder.toString() );
  try {
    buf.write( builder.toString().getBytes( utf8 ) );

    for ( Object[] row : rows ) {
      builder.setLength( 0 );
      for ( i = 0; i < meta.size(); i++ ) {
        String value;
        ValueMetaInterface vm = meta.getValueMeta( i );
        if ( row[i] == null || Const.isEmpty( vm.getString( row[i] ) ) ) {
          value = "?";
        } else {
          //switch ( meta.getValueMetaList().get( i ).getType() ) {
          switch ( vm.getType() ) {
            case ValueMetaInterface.TYPE_NUMBER:
            case ValueMetaInterface.TYPE_INTEGER:
            case ValueMetaInterface.TYPE_BIGNUMBER:
              value = vm.getString( row[i] );
              break;
            case ValueMetaInterface.TYPE_DATE:
              int offset = TZ.getOffset( vm.getDate( row[i] ).getTime() );
              value = "" + ( vm.getDate( row[i] ).getTime() + offset );
              break;
            case ValueMetaInterface.TYPE_TIMESTAMP:
              offset = TZ.getOffset( vm.getDate( row[i] ).getTime() );
              value = "" + ( vm.getDate( row[i] ).getTime() + offset );
              break;
            case ValueMetaInterface.TYPE_BOOLEAN:
              value = "" + ( vm.getBoolean( row[i] ) ? "1" : "0" );
              break;
            // TODO throw an exception for Serializable/Binary
            default:
              value = quote( vm.getString( row[i] ) );
          }
        }
        builder.append( i > 0 ? "," : "" ).append( value );
      }
      builder.append( "\n" );

      if ( !needsBase64 ) {
        needsBase64 = !encoder.canEncode( builder.toString() );
      }
      buf.write( builder.toString().getBytes( utf8 ) );
    }

    buf.flush();
    buf.close();
  } catch ( IOException e ) {
    throw new KettleException( e );
  }

  byte[] bytes = bos.toByteArray();
  if ( needsBase64 ) {
    bytes = Base64.encodeBase64( bytes );
  }
  results.add( bytes );
  results.add( needsBase64 );

  return results;
}
 
Example 17
protected static StringBuilder rowsToCSV( RowMetaInterface meta, List<Object[]> rows ) throws KettleValueException {
  StringBuilder builder = new StringBuilder();
  // header row
  int i = 0;
  for ( ValueMetaInterface v : meta.getValueMetaList() ) {
    String name = quote( v.getName() );
    builder.append( i > 0 ? "," : "" ).append( name );
    i++;
  }
  builder.append( "\n" );
  for ( Object[] row : rows ) {
    for ( i = 0; i < meta.size(); i++ ) {
      String value;
      ValueMetaInterface vm = meta.getValueMeta( i );
      if ( row[i] == null || Const.isEmpty( vm.getString( row[i] ) ) ) {
        value = "?";
      } else {
        //switch ( meta.getValueMetaList().get( i ).getType() ) {
        switch ( vm.getType() ) {
          case ValueMetaInterface.TYPE_NUMBER:
          case ValueMetaInterface.TYPE_INTEGER:
          case ValueMetaInterface.TYPE_BIGNUMBER:
            value = vm.getString( row[i] );
            break;
          case ValueMetaInterface.TYPE_DATE:
            int offset = TZ.getOffset( vm.getDate( row[i] ).getTime() );
            value = "" + ( vm.getDate( row[i] ).getTime() + offset );
            break;
          case ValueMetaInterface.TYPE_TIMESTAMP:
            offset = TZ.getOffset( vm.getDate( row[i] ).getTime() );
            value = "" + ( vm.getDate( row[i] ).getTime() + offset );
            break;
          case ValueMetaInterface.TYPE_BOOLEAN:
            value = "" + ( vm.getBoolean( row[i] ) ? "1" : "0" );
            break;
          // TODO throw an exception for Serializable/Binary
          default:
            value = quote( vm.getString( row[i] ) );
        }
      }
      builder.append( i > 0 ? "," : "" ).append( value );
    }
    builder.append( "\n" );
  }

  return builder;
}