Java Code Examples for org.pentaho.di.core.row.RowMetaInterface#getValueMeta()

The following examples show how to use org.pentaho.di.core.row.RowMetaInterface#getValueMeta() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Create a new field mapping between source and target steps.
 *
 * @param shell
 *          the shell of the parent window
 * @param sourceFields
 *          the source fields
 * @param targetFields
 *          the target fields
 * @param fieldMapping
 *          the list of source to target mappings to default to (can be empty but not null)
 * @throws KettleException
 *           in case something goes wrong during the field mapping
 */
public static final void generateFieldMapping( Shell shell, RowMetaInterface sourceFields,
  RowMetaInterface targetFields, java.util.List<SourceToTargetMapping> fieldMapping ) throws KettleException {
  // Build the mapping: let the user decide!!
  String[] source = sourceFields.getFieldNames();
  for ( int i = 0; i < source.length; i++ ) {
    ValueMetaInterface v = sourceFields.getValueMeta( i );
    source[i] += EnterMappingDialog.STRING_ORIGIN_SEPARATOR + v.getOrigin() + ")";
  }
  String[] target = targetFields.getFieldNames();

  EnterMappingDialog dialog = new EnterMappingDialog( shell, source, target, fieldMapping );
  java.util.List<SourceToTargetMapping> newMapping = dialog.open();
  if ( newMapping != null ) {
    fieldMapping.clear();
    fieldMapping.addAll( newMapping );
  }
}
 
Example 2
Source Project: pentaho-kettle   File: BaseStepDialog.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create a new field mapping between source and target steps.
 *
 * @param shell        the shell of the parent window
 * @param sourceFields the source fields
 * @param targetFields the target fields
 * @param fieldMapping the list of source to target mappings to default to (can be empty but not null)
 * @throws KettleException in case something goes wrong during the field mapping
 */
public static final void generateFieldMapping( Shell shell, RowMetaInterface sourceFields,
                                               RowMetaInterface targetFields,
                                               List<SourceToTargetMapping> fieldMapping ) throws KettleException {
  // Build the mapping: let the user decide!!
  String[] source = sourceFields.getFieldNames();
  for ( int i = 0; i < source.length; i++ ) {
    ValueMetaInterface v = sourceFields.getValueMeta( i );
    source[ i ] += EnterMappingDialog.STRING_ORIGIN_SEPARATOR + v.getOrigin() + ")";
  }
  String[] target = targetFields.getFieldNames();

  EnterMappingDialog dialog = new EnterMappingDialog( shell, source, target, fieldMapping );
  List<SourceToTargetMapping> newMapping = dialog.open();
  if ( newMapping != null ) {
    fieldMapping.clear();
    fieldMapping.addAll( newMapping );
  }
}
 
Example 3
Source Project: pentaho-kettle   File: DatabaseMeta.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Checks the fields specified for reserved words
 *
 * @param fields
 *          the list of fields to check
 * @return The nr of reserved words for this database.
 */
public int getNrReservedWords( RowMetaInterface fields ) {
  int nrReservedWords = 0;
  for ( int i = 0; i < fields.size(); i++ ) {
    ValueMetaInterface v = fields.getValueMeta( i );
    if ( isReservedWord( v.getName() ) ) {
      nrReservedWords++;
    }
  }
  return nrReservedWords;
}
 
Example 4
private void setParameters( RowMetaInterface rowMeta, Object[] row, Map<String, Object> parameters, CypherParameters cypherParameters ) throws KettleValueException {
  for ( TargetParameter targetParameter : cypherParameters.getTargetParameters() ) {
    int fieldIndex = targetParameter.getInputFieldIndex();
    ValueMetaInterface valueMeta = rowMeta.getValueMeta( fieldIndex );
    Object valueData = row[ fieldIndex ];
    String parameterName = targetParameter.getParameterName();
    GraphPropertyType parameterType = targetParameter.getParameterType();

    // Convert to the neo type
    //
    Object neoObject = parameterType.convertFromKettle( valueMeta, valueData );

    parameters.put( parameterName, neoObject );
  }
}
 
Example 5
public void getFields( RowMetaInterface row, String origin, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  if ( databaseMeta == null ) {
    return; // TODO: throw an exception here
  }

  RowMetaInterface add = null;

  try {
    String mdx = getSQL();
    if ( isVariableReplacementActive() ) {
      mdx = space.environmentSubstitute( mdx );
    }
    MondrianHelper helper = new MondrianHelper( databaseMeta, catalog, mdx, space );
    add = helper.getCachedRowMeta();
    if ( add == null ) {
      helper.openQuery();
      helper.createRectangularOutput();
      add = helper.getOutputRowMeta();
    }
  } catch ( KettleDatabaseException dbe ) {
    throw new KettleStepException( "Unable to get query result for MDX query: " + Const.CR + sql, dbe );
  }

  // Set the origin
  //
  for ( int i = 0; i < add.size(); i++ ) {
    ValueMetaInterface v = add.getValueMeta( i );
    v.setOrigin( origin );
  }

  row.addRowMeta( add );
}
 
Example 6
protected void setupFieldsCombo() {
  // try and set up from incoming fields from previous step

  StepMeta stepMeta = transMeta.findStep(stepname);

  if (stepMeta != null) {
    try {
      RowMetaInterface row = transMeta.getPrevStepFields(stepMeta);

      if (row.size() == 0) {
        MessageDialog.openError(shell, BaseMessages.getString(PKG,
            "SSTableOutputData.Message.NoIncomingFields.Title"), BaseMessages
            .getString(PKG, "SSTableOutputData.Message.NoIncomingFields"));

        return;
      }

      m_keyFieldCombo.removeAll();
      for (int i = 0; i < row.size(); i++) {
        ValueMetaInterface vm = row.getValueMeta(i);
        m_keyFieldCombo.add(vm.getName());
      }
    } catch (KettleException ex) {
      MessageDialog.openError(shell, BaseMessages.getString(PKG,
          "SSTableOutputData.Message.NoIncomingFields.Title"), BaseMessages
          .getString(PKG, "SSTableOutputData.Message.NoIncomingFields"));
    }
  }
}
 
Example 7
Source Project: pentaho-kettle   File: MemoryGroupBy.java    License: Apache License 2.0 5 votes vote down vote up
private void initGroupMeta( RowMetaInterface previousRowMeta ) throws KettleValueException {
  data.groupMeta = new RowMeta();
  data.entryMeta = new RowMeta();

  for ( int i = 0; i < data.groupnrs.length; i++ ) {
    ValueMetaInterface valueMeta = previousRowMeta.getValueMeta( data.groupnrs[i] );
    data.groupMeta.addValueMeta( valueMeta );

    ValueMetaInterface normalMeta = valueMeta.clone();
    normalMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL );
  }

  return;
}
 
Example 8
Source Project: pentaho-kettle   File: ModPartitioner.java    License: Apache License 2.0 5 votes vote down vote up
public int getPartition( RowMetaInterface rowMeta, Object[] row ) throws KettleException {
  init( rowMeta );

  if ( partitionColumnIndex < 0 ) {
    partitionColumnIndex = rowMeta.indexOfValue( fieldName );
    if ( partitionColumnIndex < 0 ) {
      throw new KettleStepException( "Unable to find partitioning field name ["
        + fieldName + "] in the output row..." + rowMeta );
    }
  }

  long value;

  ValueMetaInterface valueMeta = rowMeta.getValueMeta( partitionColumnIndex );
  Object valueData = row[partitionColumnIndex];

  switch ( valueMeta.getType() ) {
    case ValueMetaInterface.TYPE_INTEGER:
      Long longValue = rowMeta.getInteger( row, partitionColumnIndex );
      if ( longValue == null ) {
        value = valueMeta.hashCode( valueData );
      } else {
        value = longValue.longValue();
      }
      break;
    default:
      value = valueMeta.hashCode( valueData );
  }

  /*
   * value = rowMeta.getInteger(row, partitionColumnIndex);
   */

  int targetLocation = (int) ( Math.abs( value ) % nrPartitions );

  return targetLocation;
}
 
Example 9
Source Project: pentaho-kettle   File: UpdateSQLTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRowsTransform() {

  String[] keyLookup = new String[] { "Name" };
  String[] keyStream = new String[] { "FirstName" };
  String[] updateLookup = new String[] { "SecondName", "PostAddress", "ZIP" };
  String[] updateStream = new String[] { "SurName", "Address", "ZIP" };

  RowMetaInterface prev = new RowMeta();
  prev.addValueMeta( new ValueMetaString( keyStream[0] ) );
  prev.addValueMeta( new ValueMetaString( updateStream[0] ) );
  prev.addValueMeta( new ValueMetaString( updateStream[1] ) );
  prev.addValueMeta( new ValueMetaString( updateStream[2] ) );

  try {
    RowMetaInterface result =
        RowMetaUtils.getRowMetaForUpdate( prev, keyLookup, keyStream, updateLookup, updateStream );

    ValueMetaInterface vmi = result.getValueMeta( 0 );
    assertEquals( vmi.getName(), keyLookup[0] );
    assertEquals( prev.getValueMeta( 0 ).getName(), keyStream[0] );

    assertEquals( result.getValueMeta( 1 ).getName(), updateLookup[0] );
    assertEquals( prev.getValueMeta( 1 ).getName(), updateStream[0] );

    assertEquals( result.getValueMeta( 2 ).getName(), updateLookup[1] );
    assertEquals( prev.getValueMeta( 2 ).getName(), updateStream[1] );

    assertEquals( result.getValueMeta( 3 ).getName(), updateStream[2] );
    assertEquals( prev.getValueMeta( 3 ).getName(), updateStream[2] );
  } catch ( Exception ex ) {
    ex.printStackTrace();
    fail();
  }
}
 
Example 10
public void getFields( RowMetaInterface row, String originStepname, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  try {
    for ( int i = 0; i < fieldname.length; i++ ) {
      if ( !Utils.isEmpty( fieldname[i] ) ) {
        int valueIndex = -1;
        ValueMetaInterface v;
        if ( replace[i] ) {
          valueIndex = row.indexOfValue( fieldname[i] );
          if ( valueIndex < 0 ) {
            // The field was not found using the "name" field
            if ( Utils.isEmpty( rename[i] ) ) {
              // There is no "rename" field to try; Therefore we cannot find the
              // field to replace
              throw new KettleStepException( BaseMessages.getString(
                PKG, "ScriptValuesMetaMod.Exception.FieldToReplaceNotFound", fieldname[i] ) );
            } else {
              // Lookup the field to replace using the "rename" field
              valueIndex = row.indexOfValue( rename[i] );
              if ( valueIndex < 0 ) {
                // The field was not found using the "rename" field"; Therefore
                // we cannot find the field to replace
                //
                throw new KettleStepException( BaseMessages.getString(
                  PKG, "ScriptValuesMetaMod.Exception.FieldToReplaceNotFound", rename[i] ) );
              }
            }
          }

          // Change the data type to match what's specified...
          //
          ValueMetaInterface source = row.getValueMeta( valueIndex );
          v = ValueMetaFactory.cloneValueMeta( source, type[i] );
          row.setValueMeta( valueIndex, v );
        } else {
          if ( !Utils.isEmpty( rename[i] ) ) {
            v = ValueMetaFactory.createValueMeta( rename[i], type[i] );
          } else {
            v = ValueMetaFactory.createValueMeta( fieldname[i], type[i] );
          }
        }
        v.setLength( length[i] );
        v.setPrecision( precision[i] );
        v.setOrigin( originStepname );
        if ( !replace[i] ) {
          row.addValueMeta( v );
        }
      }
    }
  } catch ( KettleException e ) {
    throw new KettleStepException( e );
  }
}
 
Example 11
Source Project: pentaho-kettle   File: AggregateRows.java    License: Apache License 2.0 4 votes vote down vote up
private synchronized void AddAggregate( RowMetaInterface rowMeta, Object[] r ) throws KettleValueException {
  for ( int i = 0; i < data.fieldnrs.length; i++ ) {
    ValueMetaInterface valueMeta = rowMeta.getValueMeta( data.fieldnrs[i] );
    Object valueData = r[data.fieldnrs[i]];

    if ( !valueMeta.isNull( valueData ) ) {
      data.counts[i]++; // only count non-zero values!
      switch ( meta.getAggregateType()[i] ) {
        case AggregateRowsMeta.TYPE_AGGREGATE_SUM:
        case AggregateRowsMeta.TYPE_AGGREGATE_AVERAGE:
          Double number = valueMeta.getNumber( valueData );
          if ( data.values[i] == null ) {
            data.values[i] = number;
          } else {
            data.values[i] = new Double( ( (Double) data.values[i] ).doubleValue() + number.doubleValue() );
          }

          break;
        case AggregateRowsMeta.TYPE_AGGREGATE_MIN:
          if ( data.values[i] == null ) {
            data.values[i] = valueData;
          } else {
            if ( valueMeta.compare( data.values[i], valueData ) < 0 ) {
              data.values[i] = valueData;
            }
          }

          break;
        case AggregateRowsMeta.TYPE_AGGREGATE_MAX:
          if ( data.values[i] == null ) {
            data.values[i] = valueData;
          } else {
            if ( valueMeta.compare( data.values[i], valueData ) > 0 ) {
              data.values[i] = valueData;
            }
          }

          break;
        case AggregateRowsMeta.TYPE_AGGREGATE_NONE:
        case AggregateRowsMeta.TYPE_AGGREGATE_FIRST:
          if ( data.values[i] == null ) {
            data.values[i] = valueData;
          }
          break;
        case AggregateRowsMeta.TYPE_AGGREGATE_LAST:
          data.values[i] = valueData;
          break;
        default:
          break;
      }
    }

    switch ( meta.getAggregateType()[i] ) {
      case AggregateRowsMeta.TYPE_AGGREGATE_FIRST_NULL: // First value, EVEN if it's NULL:
        if ( data.values[i] == null ) {
          data.values[i] = valueData;
        }
        break;
      case AggregateRowsMeta.TYPE_AGGREGATE_LAST_NULL: // Last value, EVEN if it's NULL:
        data.values[i] = valueData;
        break;
      default:
        break;
    }

  }
}
 
Example 12
Source Project: pentaho-kettle   File: XMLInputDialog.java    License: Apache License 2.0 4 votes vote down vote up
private void get() {
  boolean finished = false;

  int elementsFound = 0;

  try {
    XMLInputMeta meta = new XMLInputMeta();
    getInfo( meta );

    // check if the path is given
    if ( !checkInputPositionsFilled( meta ) ) {
      return;
    }

    EnterNumberDialog dialog =
      new EnterNumberDialog(
        shell, 1000, "Number of elements to scan", "Enter the number of elements to scan (0=all)" );
    int maxElements = dialog.open();

    // OK, let's try to walk through the complete tree
    RowMetaInterface row = new RowMeta(); // no fields found...

    // Keep the list of positions
    List<XMLInputFieldPosition> path = new ArrayList<XMLInputFieldPosition>(); // ArrayList of XMLInputFieldPosition

    FileInputList inputList = meta.getFiles( transMeta );

    for ( int f = 0; f < inputList.getFiles().size() && !finished; f++ ) {
      // Open the file...
      Node rootNode =
        XMLHandler.loadXMLFile( inputList.getFile( f ), transMeta
          .environmentSubstitute( meta.getFileBaseURI() ), meta.isIgnoreEntities(), meta.isNamespaceAware() );

      // Position to the repeating item
      for ( int p = 0; rootNode != null && p < meta.getInputPosition().length - 1; p++ ) {
        rootNode = XMLHandler.getSubNode( rootNode, meta.getInputPosition()[p] );
      }

      if ( rootNode == null ) {
        // Specified node not found: return!
        return;
      }

      if ( meta.getInputPosition().length > 1 ) {
        // Count the number of rootnodes
        String itemElement = meta.getInputPosition()[meta.getInputPosition().length - 1];
        int nrItems = XMLHandler.countNodes( rootNode, itemElement );
        for ( int i = 0; i < nrItems && !finished; i++ ) {
          Node itemNode = XMLHandler.getSubNodeByNr( rootNode, itemElement, i, false );
          if ( i >= meta.getNrRowsToSkip() ) {
            getValues( itemNode, row, path, 0 );

            elementsFound++;
            if ( elementsFound >= maxElements && maxElements > 0 ) {
              finished = true;
            }
          }
        }
      } else {
        // Only search the root node
        //
        getValues( rootNode, row, path, 0 );

        elementsFound++;
        if ( elementsFound >= maxElements && maxElements > 0 ) {
          finished = true;
        }
      }
    }

    // System.out.println("Values found: "+row);

    // add the values to the grid...
    for ( int i = 0; i < row.size(); i++ ) {
      ValueMetaInterface v = row.getValueMeta( i );
      TableItem item = new TableItem( wFields.table, SWT.NONE );
      item.setText( 1, v.getName() );
      item.setText( 2, v.getTypeDesc() );
      item.setText( 11, v.getOrigin() );
    }

    wFields.removeEmptyRows();
    wFields.setRowNums();
    wFields.optWidth( true );
  } catch ( KettleException e ) {
    new ErrorDialog(
      shell, BaseMessages.getString( PKG, "XMLInputDialog.ErrorParsingData.DialogTitle" ), BaseMessages
        .getString( PKG, "XMLInputDialog.ErrorParsingData.DialogMessage" ), e );
  }
}
 
Example 13
Source Project: pentaho-kettle   File: ReadAllCache.java    License: Apache License 2.0 4 votes vote down vote up
private static Object[] createIndexes( DatabaseLookupData stepData, RowMetaInterface keysMeta, Object[][] keys ) {
  final int rowsAmount = keys.length;
  final int[] conditions = stepData.conditions;

  // it makes sense to apply restrictions in the specific order, namely, to use those, that can filter more elements
  // Index.restrictionComparator() uses heuristic "restriction power" of each index
  PriorityQueue<Index> indexes = new PriorityQueue<>( conditions.length, Index.restrictionComparator() );
  List<int[]> otherConditions = new ArrayList<>();
  for ( int i = 0, len = conditions.length; i < len; i++ ) {
    int condition = conditions[ i ];
    Index index = null;
    switch ( condition ) {
      case DatabaseLookupMeta.CONDITION_EQ:
        index = new EqIndex( i, keysMeta.getValueMeta( i ), rowsAmount );
        break;
      case DatabaseLookupMeta.CONDITION_NE:
        index = EqIndex.nonEqualityIndex( i, keysMeta.getValueMeta( i ), rowsAmount );
        break;
      case DatabaseLookupMeta.CONDITION_LT:
        index = new LtIndex( i, keysMeta.getValueMeta( i ), rowsAmount );
        break;
      case DatabaseLookupMeta.CONDITION_LE:
        index = GtIndex.lessOrEqualCache( i, keysMeta.getValueMeta( i ), rowsAmount );
        break;
      case DatabaseLookupMeta.CONDITION_GT:
        index = new GtIndex( i, keysMeta.getValueMeta( i ), rowsAmount );
        break;
      case DatabaseLookupMeta.CONDITION_GE:
        index = LtIndex.greaterOrEqualCache( i, keysMeta.getValueMeta( i ), rowsAmount );
        break;
      case DatabaseLookupMeta.CONDITION_IS_NULL:
        index = new IsNullIndex( i, keysMeta.getValueMeta( i ), rowsAmount, true );
        break;
      case DatabaseLookupMeta.CONDITION_IS_NOT_NULL:
        index = new IsNullIndex( i, keysMeta.getValueMeta( i ), rowsAmount, false );
        break;
    }
    if ( index == null ) {
      otherConditions.add( new int[] { i, condition } );
    } else {
      index.performIndexingOf( keys );
      indexes.add( index );
    }
  }

  return new Object[] {
    indexes.toArray( new Index[ indexes.size() ] ),
    otherConditions.toArray( new int[ otherConditions.size() ][] )
  };
}
 
Example 14
public void storeTable() {
  CwmTable table = cwm.getTable( TEST_TABLE_NAME );
  if ( table == null ) {
    System.out.println( "Table [" + TEST_TABLE_NAME + "] not found: creating..." ); //$NON-NLS-1$ //$NON-NLS-2$
  } else {
    System.out.println( "Table [" + TEST_TABLE_NAME + "] found: overwriting..." ); //$NON-NLS-1$ //$NON-NLS-2$
  }
  cwm.beginTransaction();

  RowMetaInterface fields = new RowMeta();
  ValueMetaInterface field1 = new ValueMeta( "field1", ValueMetaInterface.TYPE_STRING ); //$NON-NLS-1$
  field1.setLength( 35 );
  field1.setOrigin( "field1 description" ); //$NON-NLS-1$
  fields.addValueMeta( field1 );
  ValueMetaInterface field2 = new ValueMeta( "field2", ValueMetaInterface.TYPE_NUMBER ); //$NON-NLS-1$
  field2.setLength( 7, 2 );
  field2.setOrigin( "field2 description" ); //$NON-NLS-1$
  fields.addValueMeta( field2 );
  ValueMetaInterface field3 = new ValueMeta( "field3", ValueMetaInterface.TYPE_INTEGER );
  field3.setLength( 5 );
  field3.setOrigin( "field3 description" );
  fields.addValueMeta( field3 );
  ValueMetaInterface field4 = new ValueMeta( "field4", ValueMetaInterface.TYPE_DATE );
  field4.setOrigin( "field4 description" );
  fields.addValueMeta( field4 );
  ValueMetaInterface field5 = new ValueMeta( "field5", ValueMetaInterface.TYPE_BIGNUMBER );
  field5.setLength( 52, 16 );
  field5.setOrigin( "field5 description" );
  fields.addValueMeta( field5 );
  ValueMetaInterface field6 = new ValueMeta( "field6", ValueMetaInterface.TYPE_BOOLEAN );
  field6.setOrigin( "field6 description" );
  fields.addValueMeta( field6 );

  table = cwm.createTable( TEST_TABLE_NAME, fields );

  // Add descriptions to table and columns...

  CwmDescription description = cwm.createDescription( "This is a table description" ); //$NON-NLS-1$
  cwm.setDescription( table, description );
  @SuppressWarnings( "unchecked" )
  Collection<CwmColumn> collection = table.getOwnedElement();
  CwmColumn[] columns = (CwmColumn[]) collection.toArray( new CwmColumn[collection.size()] );

  for ( int i = 0; i < fields.size(); i++ ) {
    ValueMetaInterface field = fields.getValueMeta( i );
    CwmColumn column = columns[i];

    // Add a description to the column
    //
    description = cwm.createDescription( field.getOrigin() );
    cwm.setDescription( column, description );
  }

  // Try to create a package here...
  CwmPackage p = cwm.createPackage( DOMAIN + " package" ); //$NON-NLS-1$
  @SuppressWarnings( "unchecked" )
  Collection<CwmTable> ca = p.getImportedElement();
  ca.add( table );
  cwm.setDescription( p, cwm.createDescription( "This is a package description for [" + DOMAIN + "]" ) ); //$NON-NLS-1$ //$NON-NLS-2$

  cwm.endTransaction();

  System.out.println( "Finished writing to table [" + TEST_TABLE_NAME + "]." ); //$NON-NLS-1$ //$NON-NLS-2$
}
 
Example 15
private String generateMatchClause( String alias, String mapName, List<String> nodeLabels, String[] nodeProps, String[] nodePropNames,
                                    GraphPropertyType[] nodePropTypes,
                                    boolean[] nodePropPrimary,
                                    RowMetaInterface rowMeta, Object[] rowData, int[] nodePropIndexes,
                                    Map<String, Object> parameters, AtomicInteger paramNr ) throws KettleValueException {
  String matchClause = "(" + alias;
  for ( int i = 0; i < nodeLabels.size(); i++ ) {
    String label = escapeProp( nodeLabels.get( i ) );
    matchClause += ":" + label;
  }
  matchClause += " {";

  boolean firstProperty = true;
  for ( int i = 0; i < nodeProps.length; i++ ) {
    if ( nodePropPrimary[ i ] ) {
      if ( firstProperty ) {
        firstProperty = false;
      } else {
        matchClause += ", ";
      }
      String propName;
      if ( StringUtils.isNotEmpty( nodePropNames[ i ] ) ) {
        propName = nodePropNames[ i ];
      } else {
        propName = nodeProps[ i ];
      }
      String parameterName = "param" + paramNr.incrementAndGet();

      if ( mapName == null ) {
        matchClause += propName + " : " + buildParameterClause( parameterName );
      } else {
        matchClause += propName + " : " + mapName + "." + parameterName;
      }

      if ( parameters != null ) {
        ValueMetaInterface valueMeta = rowMeta.getValueMeta( nodePropIndexes[ i ] );
        Object valueData = rowData[ nodePropIndexes[ i ] ];

        GraphPropertyType propertyType = nodePropTypes[ i ];
        Object neoValue = propertyType.convertFromKettle( valueMeta, valueData );

        parameters.put( parameterName, neoValue );
      }
    }
  }
  matchClause += " })";

  return matchClause;
}
 
Example 16
Source Project: pentaho-kettle   File: GPBulkDataOutput.java    License: Apache License 2.0 4 votes vote down vote up
public void writeLine( RowMetaInterface mi, Object[] row ) throws KettleException {
  if ( first ) {
    first = false;

    enclosure = meta.getEnclosure();

    // Setup up the fields we need to take for each of the rows
    // as this speeds up processing.
    fieldNumbers = new int[meta.getFieldStream().length];
    for ( int i = 0; i < fieldNumbers.length; i++ ) {
      fieldNumbers[i] = mi.indexOfValue( meta.getFieldStream()[i] );
      if ( fieldNumbers[i] < 0 ) {
        throw new KettleException( "Could not find field " + meta.getFieldStream()[i] + " in stream" );
      }
    }

    sdfDate = new SimpleDateFormat( "yyyy-MM-dd" );
    sdfDateTime = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss.SSS" );
  }

  // Write the data to the output
  ValueMetaInterface v = null;
  int number = 0;
  for ( int i = 0; i < fieldNumbers.length; i++ ) {
    if ( i != 0 ) {
      output.print( "," );
    }
    number = fieldNumbers[i];
    v = mi.getValueMeta( number );
    if ( row[number] == null ) {
      // TODO (SB): special check for null in case of Strings.
      output.print( enclosure );
      output.print( enclosure );
    } else {
      switch ( v.getType() ) {
        case ValueMetaInterface.TYPE_STRING:
          String s = mi.getString( row, number );
          if ( s.indexOf( enclosure ) >= 0 ) {
            s = createEscapedString( s, enclosure );
          }
          output.print( enclosure );
          output.print( s );
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_INTEGER:
          Long l = mi.getInteger( row, number );
          output.print( enclosure );
          output.print( l );
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_NUMBER:
          Double d = mi.getNumber( row, number );
          output.print( enclosure );
          output.print( d );
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_BIGNUMBER:
          BigDecimal bd = mi.getBigNumber( row, number );
          output.print( enclosure );
          output.print( bd );
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_DATE:
          Date dt = mi.getDate( row, number );
          output.print( enclosure );
          String mask = meta.getDateMask()[i];
          if ( GPBulkLoaderMeta.DATE_MASK_DATETIME.equals( mask ) ) {
            output.print( sdfDateTime.format( dt ) );
          } else {
            // Default is date format
            output.print( sdfDate.format( dt ) );
          }
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_BOOLEAN:
          Boolean b = mi.getBoolean( row, number );
          output.print( enclosure );
          if ( b.booleanValue() ) {
            output.print( "Y" );
          } else {
            output.print( "N" );
          }
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_BINARY:
          byte[] byt = mi.getBinary( row, number );
          output.print( "<startlob>" );
          output.print( byt );
          output.print( "<endlob>" );
          break;
        default:
          throw new KettleException( "Unsupported type" );
      }
    }
  }
  output.print( Const.CR );
}
 
Example 17
/**
 * Ask which data set to write to
 * Ask for the mapping between the output row and the data set field
 * Start the transformation and capture the output of the step, write to the database table backing the data set.
 */
public void writeStepDataToDataSet() {
  Spoon spoon = ( (Spoon) SpoonFactory.getInstance() );
  TransGraph transGraph = spoon.getActiveTransGraph();
  IMetaStore metaStore = spoon.getMetaStore();
  if ( transGraph == null ) {
    return;
  }
  StepMeta stepMeta = transGraph.getCurrentStep();
  TransMeta transMeta = spoon.getActiveTransformation();
  if ( stepMeta == null || transMeta == null ) {
    return;
  }

  if ( transMeta.hasChanged() ) {
    MessageBox box = new MessageBox( spoon.getShell(), SWT.OK | SWT.ICON_INFORMATION );
    box.setText( "Save transformation" );
    box.setMessage( "Please save your transformation first." );
    box.open();
    return;
  }

  try {

    FactoriesHierarchy hierarchy = getHierarchy();

    MetaStoreFactory<DataSetGroup> groupFactory = hierarchy.getGroupFactory();
    List<DatabaseMeta> databases = getAvailableDatabases( spoon.getRepository() );
    groupFactory.addNameList( DataSetConst.DATABASE_LIST_KEY, databases );
    List<DataSetGroup> groups = groupFactory.getElements();

    MetaStoreFactory<DataSet> setFactory = hierarchy.getSetFactory();
    setFactory.addNameList( DataSetConst.GROUP_LIST_KEY, groups );

    // Ask which data set to write to
    //
    List<String> setNames = setFactory.getElementNames();
    Collections.sort( setNames );
    EnterSelectionDialog esd = new EnterSelectionDialog( spoon.getShell(), setNames.toArray( new String[ setNames.size() ] ), "Select the set", "Select the data set to edit..." );
    String setName = esd.open();
    if ( setName == null ) {
      return;
    }

    DataSet dataSet = setFactory.loadElement( setName );
    String[] setFields = new String[ dataSet.getFields().size() ];
    for ( int i = 0; i < setFields.length; i++ ) {
      setFields[ i ] = dataSet.getFields().get( i ).getFieldName();
    }

    RowMetaInterface rowMeta = transMeta.getStepFields( stepMeta );
    String[] stepFields = new String[ rowMeta.size() ];
    for ( int i = 0; i < rowMeta.size(); i++ ) {
      ValueMetaInterface valueMeta = rowMeta.getValueMeta( i );
      stepFields[ i ] = valueMeta.getName();
    }

    // Ask for the mapping between the output row and the data set field
    //
    EnterMappingDialog mappingDialog = new EnterMappingDialog( spoon.getShell(), stepFields, setFields );
    List<SourceToTargetMapping> mapping = mappingDialog.open();
    if ( mapping == null ) {
      return;
    }

    // Run the transformation.  We want to use the standard Spoon runFile() method
    // So we need to leave the source to target mapping list somewhere so it can be picked up later.
    // For now we'll leave it where we need it.
    //
    WriteToDataSetExtensionPoint.stepsMap.put( transMeta.getName(), stepMeta );
    WriteToDataSetExtensionPoint.mappingsMap.put( transMeta.getName(), mapping );
    WriteToDataSetExtensionPoint.setsMap.put( transMeta.getName(), dataSet );

    // Signal to the transformation xp plugin to inject data into some data set
    //
    transMeta.setVariable( DataSetConst.VAR_WRITE_TO_DATASET, "Y" );
    spoon.runFile();

  } catch ( Exception e ) {
    new ErrorDialog( spoon.getShell(), "Error", "Error creating a new data set", e );
  }
}
 
Example 18
/**
 * This method is called when a row is written to another step (even if there is no next step)
 *
 * @param rowMeta the metadata of the row
 * @param row     the data of the row
 * @throws org.pentaho.di.core.exception.KettleStepException an exception that can be thrown to hard stop the step
 */
public void rowWrittenEvent( final RowMetaInterface rowMeta, final Object[] row ) throws KettleStepException {
  if ( firstCall ) {
    this.tableModel = createTableModel( rowMeta );
    firstCall = false;
  }

  if ( queryLimit > 0 && rowsWritten > queryLimit ) {
    return;
  }

  try {
    rowsWritten += 1;

    final int count = tableModel.getColumnCount();
    final Object dataRow[] = new Object[ count ];
    for ( int columnNo = 0; columnNo < count; columnNo++ ) {
      final ValueMetaInterface valueMeta = rowMeta.getValueMeta( columnNo );

      switch( valueMeta.getType() ) {
        case ValueMetaInterface.TYPE_BIGNUMBER:
          dataRow[ columnNo ] = rowMeta.getBigNumber( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_BOOLEAN:
          dataRow[ columnNo ] = rowMeta.getBoolean( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_DATE:
          dataRow[ columnNo ] = rowMeta.getDate( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_INTEGER:
          dataRow[ columnNo ] = rowMeta.getInteger( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_NONE:
          dataRow[ columnNo ] = rowMeta.getString( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_NUMBER:
          dataRow[ columnNo ] = rowMeta.getNumber( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_STRING:
          dataRow[ columnNo ] = rowMeta.getString( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_BINARY:
          dataRow[ columnNo ] = rowMeta.getBinary( row, columnNo );
          break;
        default:
          dataRow[ columnNo ] = rowMeta.getString( row, columnNo );
      }
    }
    tableModel.addRow( dataRow );
  } catch ( final KettleValueException kve ) {
    throw new KettleStepException( kve );
  } catch ( final Exception e ) {
    throw new KettleStepException( e );
  }
}
 
Example 19
protected static List<Object> rowsToCSVNew( RowMetaInterface meta, List<Object[]> rows ) throws KettleException {
  List<Object> results = new ArrayList<>( 2 );
  boolean needsBase64;
  CharsetEncoder encoder = Charset.forName( "US-ASCII" ).newEncoder();
  Charset utf8 = Charset.forName( "UTF-8" );

  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  BufferedOutputStream buf = new BufferedOutputStream( bos );

  // header row
  StringBuilder builder = new StringBuilder();
  int i = 0;
  for ( ValueMetaInterface v : meta.getValueMetaList() ) {
    String name = quote( v.getName() );
    builder.append( i > 0 ? "," : "" ).append( name );
    i++;
  }
  builder.append( "\n" );

  // We look for non-ascii characters and, if found, encode to ascii base64 first. For some reason,
  // encoding directly to utf-8 on the Java side (Mac OS X; Java 8) causes the python utf-8 decoder to hang
  // when there are non-ascii characters (such as in Mayagüez) present. Encoding to base64 first, then decoding
  // this on the python side seems to fix the issue.
  needsBase64 = !encoder.canEncode( builder.toString() );
  try {
    buf.write( builder.toString().getBytes( utf8 ) );

    for ( Object[] row : rows ) {
      builder.setLength( 0 );
      for ( i = 0; i < meta.size(); i++ ) {
        String value;
        ValueMetaInterface vm = meta.getValueMeta( i );
        if ( row[i] == null || Const.isEmpty( vm.getString( row[i] ) ) ) {
          value = "?";
        } else {
          //switch ( meta.getValueMetaList().get( i ).getType() ) {
          switch ( vm.getType() ) {
            case ValueMetaInterface.TYPE_NUMBER:
            case ValueMetaInterface.TYPE_INTEGER:
            case ValueMetaInterface.TYPE_BIGNUMBER:
              value = vm.getString( row[i] );
              break;
            case ValueMetaInterface.TYPE_DATE:
              int offset = TZ.getOffset( vm.getDate( row[i] ).getTime() );
              value = "" + ( vm.getDate( row[i] ).getTime() + offset );
              break;
            case ValueMetaInterface.TYPE_TIMESTAMP:
              offset = TZ.getOffset( vm.getDate( row[i] ).getTime() );
              value = "" + ( vm.getDate( row[i] ).getTime() + offset );
              break;
            case ValueMetaInterface.TYPE_BOOLEAN:
              value = "" + ( vm.getBoolean( row[i] ) ? "1" : "0" );
              break;
            // TODO throw an exception for Serializable/Binary
            default:
              value = quote( vm.getString( row[i] ) );
          }
        }
        builder.append( i > 0 ? "," : "" ).append( value );
      }
      builder.append( "\n" );

      if ( !needsBase64 ) {
        needsBase64 = !encoder.canEncode( builder.toString() );
      }
      buf.write( builder.toString().getBytes( utf8 ) );
    }

    buf.flush();
    buf.close();
  } catch ( IOException e ) {
    throw new KettleException( e );
  }

  byte[] bytes = bos.toByteArray();
  if ( needsBase64 ) {
    bytes = Base64.encodeBase64( bytes );
  }
  results.add( bytes );
  results.add( needsBase64 );

  return results;
}
 
Example 20
public static void getFieldsFromPrevious( RowMetaInterface row, XulTree tableView, List<Object> fields,
  StepTableDataObject field, TableItemInsertXulListener listener ) {
  if ( row == null || row.size() == 0 ) {
    return; // nothing to do
  }

  // get a list of all the non-empty keys (names)
  //
  List<String> keys = new ArrayList<String>();
  for ( Object entry : fields ) {
    keys.add( ( (StepTableDataObject) entry ).getName() );
  }

  int choice = 0;

  if ( keys.size() > 0 ) {
    // Ask what we should do with the existing data in the step.
    //
    Shell shell = ( (TableViewer) tableView.getManagedObject() ).getTable().getShell();
    MessageDialog md =
      new MessageDialog( shell,
        BaseMessages.getString( PKG, "BaseStepDialog.GetFieldsChoice.Title" ), // "Warning!"
        null,
        BaseMessages.getString( PKG, "BaseStepDialog.GetFieldsChoice.Message", "" + keys.size(), "" + row.size() ),
        MessageDialog.WARNING, new String[] {
          BaseMessages.getString( PKG, "BaseStepDialog.AddNew" ),
          BaseMessages.getString( PKG, "BaseStepDialog.Add" ),
          BaseMessages.getString( PKG, "BaseStepDialog.ClearAndAdd" ),
          BaseMessages.getString( PKG, "BaseStepDialog.Cancel" ), }, 0 );
    MessageDialog.setDefaultImage( GUIResource.getInstance().getImageSpoon() );
    int idx = md.open();
    choice = idx & 0xFF;
  }

  if ( choice == 3 || choice == 255 ) {
    return; // Cancel clicked
  }

  if ( choice == 2 ) {
    fields.clear();
  }

  for ( int i = 0; i < row.size(); i++ ) {
    ValueMetaInterface v = row.getValueMeta( i );

    if ( choice == 0 ) { // hang on, see if it's not yet in the table view

      if ( keys.indexOf( v.getName() ) >= 0 ) {
        continue;
      }
    }

    if ( listener != null && !listener.tableItemInsertedFor( v ) ) {
      continue;
    }

    StepTableDataObject newField = field.createNew( v );
    fields.add( newField );
  }
}