Java Code Examples for org.pentaho.di.core.row.RowMetaInterface#addValueMeta()

The following examples show how to use org.pentaho.di.core.row.RowMetaInterface#addValueMeta() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void abortedSubtransThrowsAnError() throws KettleException {
  Result result1 = new Result();
  result1.setNrErrors( 1 );
  when( subtransExecutor.execute( any()  ) ).thenReturn( Optional.of( result1 ) );
  when( subtransExecutor.getPrefetchCount() ).thenReturn( 10 );
  RowMetaInterface rowMeta = new RowMeta();
  rowMeta.addValueMeta( new ValueMetaString( "field" ) );
  FixedTimeStreamWindow<List> window =
    new FixedTimeStreamWindow<>( subtransExecutor, rowMeta, 0, 2, 1 );
  try {
    window.buffer( Flowable.fromIterable( singletonList( asList( "v1", "v2" ) ) ) ).forEach( result -> { } );
  } catch ( Exception e ) {
    assertEquals(
      getString( BaseStreamStep.class, "FixedTimeStreamWindow.SubtransFailed" ), e.getCause().getMessage().trim() );
  }
}
 
Example 2
@Test
public void testGetFields() throws Exception {
  StringOperationsMeta meta = new StringOperationsMeta();
  meta.allocate( 1 );
  meta.setFieldInStream( new String[] { "field1" } );

  RowMetaInterface rowMetaInterface = new RowMeta();
  ValueMetaInterface valueMeta = new ValueMetaString( "field1" );
  valueMeta.setStorageMetadata( new ValueMetaString( "field1" ) );
  valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING );
  rowMetaInterface.addValueMeta( valueMeta );

  VariableSpace space = mock( VariableSpace.class );
  meta.getFields( rowMetaInterface, "STRING_OPERATIONS", null, null, space, null, null );
  RowMetaInterface expectedRowMeta = new RowMeta();
  expectedRowMeta.addValueMeta( new ValueMetaString( "field1" ) );
  assertEquals( expectedRowMeta.toString(), rowMetaInterface.toString() );
}
 
Example 3
Source Project: pentaho-kettle   File: MergeRowsMeta.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void getFields( RowMetaInterface r, String name, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  // We don't have any input fields here in "r" as they are all info fields.
  // So we just merge in the info fields.
  //
  if ( info != null ) {
    boolean found = false;
    for ( int i = 0; i < info.length && !found; i++ ) {
      if ( info[i] != null ) {
        r.mergeRowMeta( info[i], name );
        found = true;
      }
    }
  }

  if ( Utils.isEmpty( flagField ) ) {
    throw new KettleStepException( BaseMessages.getString( PKG, "MergeRowsMeta.Exception.FlagFieldNotSpecified" ) );
  }
  ValueMetaInterface flagFieldValue = new ValueMetaString( flagField );
  flagFieldValue.setOrigin( name );
  r.addValueMeta( flagFieldValue );

}
 
Example 4
@SuppressWarnings( "unchecked" )
public static void getFields( boolean clearResultFields, RowMetaInterface row, String originStepname,
  RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, List<?> fields ) throws KettleStepException {
  if ( clearResultFields ) {
    row.clear();
  }
  for ( FieldInfo fi : (List<FieldInfo>) fields ) {
    try {
      ValueMetaInterface v = ValueMetaFactory.createValueMeta( fi.name, fi.type );
      v.setLength( fi.length );
      v.setPrecision( fi.precision );
      v.setOrigin( originStepname );
      row.addValueMeta( v );
    } catch ( Exception e ) {
      throw new KettleStepException( e );
    }
  }
}
 
Example 5
@Test public void testTopLevelArrayStructureWithPrimitives() throws Exception {
  List<MongoDbOutputMeta.MongoField> paths = asList( mf( "field1", false, "[0]" ), mf( "field2", false, "[1]" ) );

  RowMetaInterface rmi = new RowMeta();
  rmi.addValueMeta( new ValueMetaString( "field1" ) );
  rmi.addValueMeta( new ValueMetaInteger( "field2" ) );

  Object[] row = new Object[ 2 ];
  row[ 0 ] = "value1";
  row[ 1 ] = 12L;
  VariableSpace vs = new Variables();

  for ( MongoDbOutputMeta.MongoField f : paths ) {
    f.init( vs );
  }

  DBObject result = kettleRowToMongo( paths, rmi, row, MongoDbOutputData.MongoTopLevel.ARRAY, false );

  assertEquals( JSON.serialize( result ), "[ \"value1\" , 12]" );
}
 
Example 6
private RowMetaInterface generateRowMeta( String[] fieldNames ) {
  RowMetaInterface rowMeta = new RowMeta();

  for ( String fieldName : fieldNames ) {
    ValueMetaInterface col = new ValueMeta();
    col.setName( fieldName );
    col.setType( ValueMeta.TYPE_STRING );
    rowMeta.addValueMeta( col );
  }
  return rowMeta;
}
 
Example 7
Source Project: pentaho-kettle   File: TableInputIT.java    License: Apache License 2.0 5 votes vote down vote up
public RowMetaInterface createRowMetaInterface() {
  RowMetaInterface rm = new RowMeta();

  ValueMetaInterface[] valuesMeta = { new ValueMetaInteger( "int_field" ) };

  for ( int i = 0; i < valuesMeta.length; i++ ) {
    rm.addValueMeta( valuesMeta[i] );
  }

  return rm;
}
 
Example 8
/**
 * PDI-11045. Here we test backwards compatibility for old ktrs that were developed before 11045. In these ktrs query
 * paths had to be specified a second time in the step in order to get them into the update/upsert object. Now we
 * assume that there will never be a situation where the user might not want the match fields present in the update
 * object
 *
 * @throws KettleException
 */
@Test public void testUpdateObjectBackwardsCompatibility() throws Exception {
  List<MongoDbOutputMeta.MongoField> paths = new ArrayList<MongoDbOutputMeta.MongoField>( 2 );

  MongoDbOutputMeta.MongoField mf = mf( "field1", true, "" );
  mf.m_updateMatchField = true;
  paths.add( mf );

  // same as previous field (but not a match condition). Prior to PDI-11045 the
  // user had to specify the match conditions a second time (but not marked in the
  // step as a match condition) in order to get them into the update/upsert object
  mf = mf( "field1", true, "" );
  mf.m_updateMatchField = false;
  paths.add( mf );

  mf = mf( "field2", true, "" );
  paths.add( mf );

  RowMetaInterface rmi = new RowMeta();
  rmi.addValueMeta( new ValueMetaString( "field1" ) );
  rmi.addValueMeta( new ValueMetaInteger( "field2" ) );

  Object[] row = new Object[ 2 ];
  row[ 0 ] = "value1";
  row[ 1 ] = 12L;
  VariableSpace vs = new Variables();

  for ( MongoDbOutputMeta.MongoField f : paths ) {
    f.init( vs );
  }

  DBObject result = kettleRowToMongo( paths, rmi, row, MongoDbOutputData.MongoTopLevel.RECORD, false );

  // here we expect that field1 does *not* occur twice in the update object
  assertEquals( JSON.serialize( result ), "{ \"field1\" : \"value1\" , \"field2\" : 12}" );
}
 
Example 9
public RowMetaInterface createRowMetaInterface3() {
  RowMetaInterface rm = new RowMeta();

  ValueMetaInterface[] valuesMeta =
    {
      new ValueMetaInteger( "int_in" ),
      new ValueMetaNumber( "number_in" ),
      new ValueMetaString( "string_in" ), };

  for ( int i = 0; i < valuesMeta.length; i++ ) {
    rm.addValueMeta( valuesMeta[ i ] );
  }

  return rm;
}
 
Example 10
Source Project: pentaho-kettle   File: BaseStep.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Gets the log fields.
 *
 * @param comm the comm
 * @return the log fields
 */
public static final RowMetaInterface getLogFields( String comm ) {
  RowMetaInterface r = new RowMeta();
  ValueMetaInterface sname =
    new ValueMetaString(
      BaseMessages.getString( PKG, "BaseStep.ColumnName.Stepname" ) );
  sname.setLength( 256 );
  r.addValueMeta( sname );

  r.addValueMeta( new ValueMetaNumber(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.Copy" ) ) );
  r.addValueMeta( new ValueMetaNumber(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.LinesReaded" ) ) );
  r.addValueMeta( new ValueMetaNumber(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.LinesWritten" ) ) );
  r.addValueMeta( new ValueMetaNumber(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.LinesUpdated" ) ) );
  r.addValueMeta( new ValueMetaNumber(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.LinesSkipped" ) ) );
  r.addValueMeta( new ValueMetaNumber(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.Errors" ) ) );
  r.addValueMeta( new ValueMetaDate(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.StartDate" ) ) );
  r.addValueMeta( new ValueMetaDate(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.EndDate" ) ) );

  for ( int i = 0; i < r.size(); i++ ) {
    r.getValueMeta( i ).setOrigin( comm );
  }

  return r;
}
 
Example 11
Source Project: pentaho-kettle   File: XMLJoinMeta.java    License: Apache License 2.0 5 votes vote down vote up
public void getFields( RowMetaInterface row, String name, RowMetaInterface[] info, StepMeta nextStep,
    VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {

  ValueMetaInterface v = new ValueMeta( this.getValueXMLfield(), ValueMetaInterface.TYPE_STRING );
  v.setOrigin( name );

  TransMeta transMeta = (TransMeta) space;
  try {
    // Row should only include fields from the target and not the source. During the preview table generation
    // the fields from all previous steps (source and target) are included in the row so lets remove the
    // source fields.
    List<String> targetFieldNames = null;
    RowMetaInterface targetRowMeta = transMeta.getStepFields( transMeta.findStep( getTargetXMLstep() ),
      null,
      null );
    if ( targetRowMeta != null ) {
      targetFieldNames = Arrays.asList( targetRowMeta.getFieldNames() );
    }
    for ( String fieldName : transMeta.getStepFields( transMeta.findStep( getSourceXMLstep() ),
                                                                          null,
                                                                          null ).getFieldNames() ) {
      if ( targetFieldNames == null || !targetFieldNames.contains( fieldName ) ) {
        row.removeValueMeta( fieldName );
      }
    }
  } catch ( KettleValueException e ) {
    // Pass
  }

  row.addValueMeta( v );
}
 
Example 12
Source Project: pentaho-kettle   File: CsvInput2IT.java    License: Apache License 2.0 5 votes vote down vote up
public RowMetaInterface createResultRowMetaInterface() {
  RowMetaInterface rm = new RowMeta();

  ValueMetaInterface[] valuesMeta =
  {
    new ValueMetaInteger( "a" ), new ValueMetaString( "b" ),
    new ValueMetaString( "c" ), new ValueMetaString( "filename" ), };

  for ( int i = 0; i < valuesMeta.length; i++ ) {
    rm.addValueMeta( valuesMeta[i] );
  }

  return rm;
}
 
Example 13
Source Project: pentaho-kettle   File: TransExecutorIT.java    License: Apache License 2.0 5 votes vote down vote up
private static RowMetaInterface createRowMetaForOneField() {
  RowMetaInterface rm = new RowMeta();
  ValueMetaInterface[] valuesMeta = { new ValueMetaString( "field1" ), };
  for ( ValueMetaInterface aValuesMeta : valuesMeta ) {
    rm.addValueMeta( aValuesMeta );
  }
  return rm;
}
 
Example 14
@Test
public void testGetFields() throws KettleStepException {
  SalesforceDeleteMeta meta = new SalesforceDeleteMeta();
  meta.setDefault();
  RowMetaInterface r = new RowMeta();
  meta.getFields( r, "thisStep", null, null, new Variables(), null, null );
  assertEquals( 0, r.size() );

  r.clear();
  r.addValueMeta( new ValueMetaString( "testString" ) );
  meta.getFields( r, "thisStep", null, null, new Variables(), null, null );
  assertEquals( 1, r.size() );
  assertEquals( ValueMetaInterface.TYPE_STRING, r.getValueMeta( 0 ).getType() );
  assertEquals( "testString", r.getValueMeta( 0 ).getName() );
}
 
Example 15
@Test public void testInsertKettleFieldThatContainsJsonIntoTopLevelRecord() throws Exception {
  List<MongoDbOutputMeta.MongoField> paths = new ArrayList<MongoDbOutputMeta.MongoField>( 3 );

  MongoDbOutputMeta.MongoField mf = mf( "field1", true, "" );
  paths.add( mf );

  mf = mf( "field2", true, "" );
  paths.add( mf );

  mf = mf( "jsonField", true, "" );
  mf.m_JSON = true;
  paths.add( mf );

  RowMetaInterface rm = new RowMeta();
  rm.addValueMeta( new ValueMetaString( "field1" ) );
  rm.addValueMeta( new ValueMetaInteger( "field2" ) );
  rm.addValueMeta( new ValueMetaString( "jsonField" ) );

  Object[] row = new Object[ 3 ];
  row[ 0 ] = "value1";
  row[ 1 ] = 12L;
  row[ 2 ] = "{\"jsonDocField1\" : \"aval\", \"jsonDocField2\" : 42}";
  VariableSpace vs = new Variables();

  for ( MongoDbOutputMeta.MongoField f : paths ) {
    f.init( vs );
  }

  DBObject result = kettleRowToMongo( paths, rm, row, MongoDbOutputData.MongoTopLevel.RECORD, false );

  assertEquals( JSON.serialize( result ),
    "{ \"field1\" : \"value1\" , \"field2\" : 12 , \"jsonField\" : { \"jsonDocField1\" : \"aval\" , "
      + "\"jsonDocField2\" : 42}}" );

}
 
Example 16
private ReplaceString prepareTestProcessRow( ReplaceStringData data, ReplaceStringMeta meta, String unicodeString,
                                             String matchPattern, String matchReplacer ) throws Exception {
  ReplaceString replaceString = Mockito.spy(
    new ReplaceString( stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans ) );
  RowMetaInterface inputRowMeta = new RowMeta();
  Object[] _row = new Object[] { unicodeString, null };
  doReturn( _row ).when( replaceString ).getRow();
  inputRowMeta.addValueMeta( 0, new ValueMetaString( "string" ) );

  doReturn( new String[] { "string" } ).when( meta ).getFieldInStream();
  doReturn( new String[] { "output" } ).when( meta ).getFieldOutStream();
  doReturn( new String[] { matchPattern } ).when( meta ).getReplaceString();
  doReturn( new String[] { StringUtils.EMPTY } ).when( meta ).getFieldReplaceByString();
  doReturn( new String[] { matchReplacer } ).when( meta ).getReplaceByString();
  doReturn( new boolean[] { false } ).when( meta ).isSetEmptyString();

  replaceString.init( meta, data );
  replaceString.setInputRowMeta( inputRowMeta );
  data.outputRowMeta = inputRowMeta;
  data.inputFieldsNr = 1;
  data.numFields = 2;
  data.inStreamNrs = new int[] { 0 };
  data.replaceFieldIndex = new int[] { -1 };
  data.outStreamNrs = new String[] { StringUtils.EMPTY, "1" };
  data.replaceByString = new String[] { "1" };
  data.setEmptyString = new boolean[] { false, false };

  return replaceString;
}
 
Example 17
public static final RowMetaInterface getResultRowMeta() {
  RowMetaInterface rowMeta = new RowMeta();
  rowMeta.addValueMeta( new ValueMetaString(
    BaseMessages.getString( PKG, "SearchResult.TransOrJob" ) ) );
  rowMeta.addValueMeta( new ValueMetaString(
    BaseMessages.getString( PKG, "SearchResult.StepDatabaseNotice" ) ) );
  rowMeta.addValueMeta( new ValueMetaString(
    BaseMessages.getString( PKG, "SearchResult.String" ) ) );
  rowMeta.addValueMeta( new ValueMetaString(
    BaseMessages.getString( PKG, "SearchResult.FieldName" ) ) );
  return rowMeta;
}
 
Example 18
Source Project: pentaho-kettle   File: JsonOutputTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a row meta interface for the fields that are defined by performing a getFields and by checking "Result
 * filenames - Add filenames to result from "Text File Input" dialog.
 * 
 * @return
 */
public RowMetaInterface createResultRowMetaInterface() {
  RowMetaInterface rowMetaInterface = new RowMeta();

  ValueMetaInterface[] valuesMeta =
    { new ValueMetaInteger( "Id" ), new ValueMetaString( "State" ), new ValueMetaString( "City" ) };

  for ( int i = 0; i < valuesMeta.length; i++ ) {
    rowMetaInterface.addValueMeta( valuesMeta[i] );
  }

  return rowMetaInterface;
}
 
Example 19
Source Project: pentaho-kettle   File: ScriptMeta.java    License: Apache License 2.0 4 votes vote down vote up
public void getFields( RowMetaInterface row, String originStepname, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  for ( int i = 0; i < fieldname.length; i++ ) {
    if ( !Utils.isEmpty( fieldname[i] ) ) {
      String fieldName;
      int replaceIndex;
      int fieldType;

      if ( replace[i] ) {
        // Look up the field to replace...
        //
        if ( row.searchValueMeta( fieldname[i] ) == null && Utils.isEmpty( rename[i] ) ) {
          throw new KettleStepException( BaseMessages.getString(
            PKG, "ScriptMeta.Exception.FieldToReplaceNotFound", fieldname[i] ) );
        }
        replaceIndex = row.indexOfValue( rename[i] );

        // Change the data type to match what's specified...
        //
        fieldType = type[i];
        fieldName = rename[i];
      } else {
        replaceIndex = -1;
        fieldType = type[i];
        if ( rename[i] != null && rename[i].length() != 0 ) {
          fieldName = rename[i];
        } else {
          fieldName = fieldname[i];
        }
      }
      try {
        ValueMetaInterface v = ValueMetaFactory.createValueMeta( fieldName, fieldType );
        v.setLength( length[i] );
        v.setPrecision( precision[i] );
        v.setOrigin( originStepname );
        if ( replace[i] && replaceIndex >= 0 ) {
          row.setValueMeta( replaceIndex, v );
        } else {
          row.addValueMeta( v );
        }
      } catch ( KettlePluginException e ) {
        // Ignore errors
      }
    }
  }
}
 
Example 20
Source Project: pentaho-kettle   File: ExcelInputDialog.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Processing excel workbook, filling fields
 *
 * @param fields   RowMetaInterface for filling fields
 * @param info     ExcelInputMeta
 * @param workbook excel workbook for processing
 * @throws KettlePluginException
 */
private void processingWorkbook( RowMetaInterface fields, ExcelInputMeta info, KWorkbook workbook )
  throws KettlePluginException {
  int nrSheets = workbook.getNumberOfSheets();
  for ( int j = 0; j < nrSheets; j++ ) {
    KSheet sheet = workbook.getSheet( j );

    // See if it's a selected sheet:
    int sheetIndex;
    if ( info.readAllSheets() ) {
      sheetIndex = 0;
    } else {
      sheetIndex = Const.indexOfString( sheet.getName(), info.getSheetName() );
    }
    if ( sheetIndex >= 0 ) {
      // We suppose it's the complete range we're looking for...
      //
      int rownr = 0;
      int startcol = 0;

      if ( info.readAllSheets() ) {
        if ( info.getStartColumn().length == 1 ) {
          startcol = info.getStartColumn()[ 0 ];
        }
        if ( info.getStartRow().length == 1 ) {
          rownr = info.getStartRow()[ 0 ];
        }
      } else {
        rownr = info.getStartRow()[ sheetIndex ];
        startcol = info.getStartColumn()[ sheetIndex ];
      }

      boolean stop = false;
      for ( int colnr = startcol; !stop; colnr++ ) {
        try {
          String fieldname = null;
          int fieldtype = ValueMetaInterface.TYPE_NONE;

          KCell cell = sheet.getCell( colnr, rownr );
          if ( cell == null ) {
            stop = true;
          } else {
            if ( cell.getType() != KCellType.EMPTY ) {
              // We found a field.
              fieldname = cell.getContents();
            }

            // System.out.println("Fieldname = "+fieldname);

            KCell below = sheet.getCell( colnr, rownr + 1 );

            if ( below != null ) {
              if ( below.getType() == KCellType.BOOLEAN ) {
                fieldtype = ValueMetaInterface.TYPE_BOOLEAN;
              } else if ( below.getType() == KCellType.DATE ) {
                fieldtype = ValueMetaInterface.TYPE_DATE;
              } else if ( below.getType() == KCellType.LABEL ) {
                fieldtype = ValueMetaInterface.TYPE_STRING;
              } else if ( below.getType() == KCellType.NUMBER ) {
                fieldtype = ValueMetaInterface.TYPE_NUMBER;
              } else {
                fieldtype = ValueMetaInterface.TYPE_STRING;
              }
            } else {
              fieldtype = ValueMetaInterface.TYPE_STRING;
            }

            if ( Utils.isEmpty( fieldname ) ) {
              stop = true;
            } else {
              if ( fieldtype != ValueMetaInterface.TYPE_NONE ) {
                ValueMetaInterface field = ValueMetaFactory.createValueMeta( fieldname, fieldtype );
                fields.addValueMeta( field );
              }
            }
          }
        } catch ( ArrayIndexOutOfBoundsException aioobe ) {
          // System.out.println("index out of bounds at column "+colnr+" : "+aioobe.toString());
          stop = true;
        }
      }
    }
  }
}