Java Code Examples for org.apache.pig.data.DataType#findType()

@Override
public void write(PreparedStatement statement) throws SQLException {
    for (int i = 0; i < columnMetadataList.size(); i++) {
        Object o = values.get(i);
        ColumnInfo columnInfo = columnMetadataList.get(i);
        byte type = (fieldSchemas == null) ? DataType.findType(o) : fieldSchemas[i].getType();
        try {
            Object upsertValue = convertTypeSpecificValue(o, type, columnInfo.getSqlType());
            if (upsertValue != null) {
                statement.setObject(i + 1, upsertValue, columnInfo.getSqlType());
            } else {
                statement.setNull(i + 1, columnInfo.getSqlType());
            }
        } catch (RuntimeException re) {
            throw new RuntimeException(String.format("Unable to process column %s, innerMessage=%s"
                    ,columnInfo.toString(),re.getMessage()),re);
            
        }
    }
}
 

public void write(PreparedStatement statement, List<ColumnInfo> columnMetadataList) throws SQLException {
	for (int i = 0; i < columnMetadataList.size(); i++) {
		Object o = values.get(i);
		
		byte type = (fieldSchemas == null) ? DataType.findType(o) : fieldSchemas[i].getType();
		Object upsertValue = convertTypeSpecificValue(o, type, columnMetadataList.get(i).getSqlType());

		if (upsertValue != null) {
			statement.setObject(i + 1, upsertValue, columnMetadataList.get(i).getSqlType());
		} else {
			statement.setNull(i + 1, columnMetadataList.get(i).getSqlType());
		}
	}
	
	statement.execute();
}
 

@Override
public int compare(Tuple o1, Tuple o2) {
    if (o1 == null)
        return -1;
    if (o2 == null)
        return 1;
    try {
        Object field1 = o1.get(fieldNum);
        Object field2 = o2.get(fieldNum);
        if (!typeFound) {
            datatype = DataType.findType(field1);
            typeFound = true;
        }
        return DataType.compare(field1, field2, datatype, datatype);
    } catch (ExecException e) {
        throw new RuntimeException("Error while comparing o1:" + o1
                + " and o2:" + o2, e);
    }
}
 

private Result getResult(ExpressionOperator op) throws ExecException {
    Result res;
    switch (op.getResultType()) {
    case DataType.BAG:
    case DataType.BOOLEAN:
    case DataType.BYTEARRAY:
    case DataType.CHARARRAY:
    case DataType.DOUBLE:
    case DataType.FLOAT:
    case DataType.INTEGER:
    case DataType.LONG:
    case DataType.BIGINTEGER:
    case DataType.BIGDECIMAL:
    case DataType.DATETIME:
    case DataType.MAP:
    case DataType.TUPLE:
        res = op.getNext(op.getResultType());
        break;
    default:
        String msg = "Invalid result type: "
                + DataType.findType(op.getResultType());
        throw new ExecException(msg, 2270, PigException.BUG);
    }

    return res;
}
 

@Override
public int compare(Tuple o1, Tuple o2) {
    int ret = 0;
    if (o1 == null) {
        ret = -1;
    } else if (o2 == null) {
        ret = 1;
    } else {
        try {
            Object field1 = o1.get(fieldNum);
            Object field2 = o2.get(fieldNum);
            if (!typeFound) {
                datatype = DataType.findType(field1);
                if(datatype != DataType.NULL) {
                    typeFound = true;
                }
            }
            ret =  DataType.compare(field1, field2, datatype, datatype);
        } catch (ExecException e) {
            throw new RuntimeException("Error while comparing o1:" + o1
                    + " and o2:" + o2, e);
        }
    }
    return isDescOrder ? ret : ret * -1;
}
 

Object GetUnequalValue(Object v) {
    byte type = DataType.findType(v);

    if (type == DataType.BAG || type == DataType.TUPLE
            || type == DataType.MAP)
        return null;

    Object zero = generateData(type, "0");

    if (v.equals(zero))
        return generateData(type, "1");

    return zero;
}
 

protected Object inferWritable(Object o) throws BackendException {
  System.out.println("Got object '" + o + "' type " + o.getClass());
  switch (DataType.findType(o)) {
    case BYTEARRAY: {
      return new BytesWritable(((DataByteArray) o).get());
    }
    case CHARARRAY: {
      return new Text(o.toString());
    }
    case INTEGER: {
      return new IntWritable((Integer) o);
    }
    case LONG: {
      return new LongWritable((Long) o);
    }
    case FLOAT: {
      return new FloatWritable((Float) o);
    }
    case DOUBLE: {
      return new DoubleWritable((Double) o);
    }
    case BOOLEAN: {
      return new BooleanWritable((Boolean) o);
    }
    case BYTE: {
      return new ByteWritable((Byte) o);
    }
  }
  throw new BackendException("Unable to translate " + o.getClass() +
      " to a Writable datatype");
}
 

/**
 * Override outputSchema so we can verify the input schema at pig compile time, instead of runtime
 * @param inputSchema input schema
 * @return call to super.outputSchema in case schema was defined elsewhere
 */
@Override
public Schema outputSchema(Schema inputSchema)
{
  if (inputSchema == null) {
    throw new IllegalArgumentException(String.format("%s: null schema passed to %s", _method_signature(), getClass().getName()));
  }

  // check correct number of arguments
  @SuppressWarnings("rawtypes")
  Class parameterTypes[] = m.getParameterTypes();
  if (inputSchema.size() != parameterTypes.length) {
    throw new IllegalArgumentException(String.format("%s: got %d arguments, expected %d.",
                                                     _method_signature(),
                                                     inputSchema.size(),
                                                     parameterTypes.length));
  }

  // check type for each argument
  for (int i=0; i < parameterTypes.length; i++) {
    try {
      byte inputType = inputSchema.getField(i).type;
      byte parameterType = DataType.findType(parameterTypes[i]);
      if (inputType != parameterType) {
        throw new IllegalArgumentException(String.format("%s: argument type mismatch [#%d]; expected %s, got %s",
                                                         _method_signature(),
                                                         i+1,
                                                         DataType.findTypeName(parameterType),
                                                         DataType.findTypeName(inputType)));
      }
    }
    catch (FrontendException fe) {
      throw new IllegalArgumentException(String.format("%s: Problem with input schema: ", _method_signature(), inputSchema), fe);
    }
  }

  // delegate to super to determine the actual outputSchema (if specified)
  return super.outputSchema(inputSchema);
}
 

private static void writePartialUpdateAndRemoveMap(String name, Object value, JsonGenerator g, Properties properties, Schema schema, Operation op, int depth, String operation) throws IOException {
    schema = (schema != null) ? schema.getField(0).schema : null;
    // extract the key of map and keys in map for writing json when partial updating maps
    Schema valueSchema = (schema != null) ? schema.getField(1).schema : null;
    // data format  { ( key; id, value: (abc,123,(123234,bbaa))) }
    // the first element of each tuple in the bag will be the map to update
    // the second element of each tuple in the bag will be the new value of the map
    DataBag bag = (DataBag) value;
    for (Tuple element : bag) {
        if (element.size() != 2) {
            continue;
        }
        String k = (String) element.get(0);
        Object v = element.get(1);
        Byte t = DataType.findType(v);
        if (t == DataType.TUPLE) {
            g.writeFieldName(name + "{" + k + "}");
            if (operation.equals(PARTIAL_UPDATE_REMOVE)) {
                g.writeStartObject();
                g.writeFieldName(PARTIAL_UPDATE_REMOVE);
                g.writeNumber(0);
                g.writeEndObject();
            } else {
                writePartialUpdate(v, t, g, name, properties, valueSchema, op, depth);
            }
        }
    }
}
 

@Override
public Schema outputSchema(Schema input) {
    if (invoker_ == null) return null;
    FieldSchema fs = new FieldSchema(null, DataType.findType(invoker_.getReturnType()));
    return new Schema(fs);
}
 

@Override
public LogicalSchema.LogicalFieldSchema getFieldSchema() throws FrontendException {
    if (fieldSchema!=null)
        return fieldSchema;

    LogicalSchema inputSchema = new LogicalSchema();
    List<Operator> succs = plan.getSuccessors(this);

    if (succs!=null) {
        for(Operator lo : succs){
            if (((LogicalExpression)lo).getFieldSchema()==null) {
                inputSchema = null;
                break;
            }
            inputSchema.addField(((LogicalExpression)lo).getFieldSchema());
        }
    }

    if (lazilyInitializeInvokerFunction) {
        initializeInvokerFunction();
    }

    // Since ef only set one time, we never change its value, so we can optimize it by instantiate only once.
    // This significantly optimize the performance of frontend (PIG-1738)
    if (ef==null) {
        ef = (EvalFunc<?>) PigContext.instantiateFuncFromSpec(mFuncSpec);
    }

    ef.setUDFContextSignature(signature);
    Properties props = UDFContext.getUDFContext().getUDFProperties(ef.getClass());
    Schema translatedInputSchema = Util.translateSchema(inputSchema);
    if(translatedInputSchema != null) {
        props.put("pig.evalfunc.inputschema."+signature, translatedInputSchema);
    }
    // Store inputSchema into the UDF context
    ef.setInputSchema(translatedInputSchema);

    Schema udfSchema = ef.outputSchema(translatedInputSchema);
    if (udfSchema != null && udfSchema.size() > 1) {
        throw new FrontendException("Given UDF returns an improper Schema. Schema should only contain one field of a Tuple, Bag, or a single type. Returns: " + udfSchema);
    }

    //TODO appendability should come from a setting
    SchemaTupleFrontend.registerToGenerateIfPossible(translatedInputSchema, false, GenContext.UDF);
    SchemaTupleFrontend.registerToGenerateIfPossible(udfSchema, false, GenContext.UDF);

    if (udfSchema != null) {
        Schema.FieldSchema fs;
        if(udfSchema.size() == 0) {
            fs = new Schema.FieldSchema(null, null, DataType.findType(ef.getReturnType()));
        } else if(udfSchema.size() == 1) {
            fs = new Schema.FieldSchema(udfSchema.getField(0));
        } else {
            fs = new Schema.FieldSchema(null, udfSchema, DataType.TUPLE);
        }
        fieldSchema = Util.translateFieldSchema(fs);
        fieldSchema.normalize();
    } else {
        fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.findType(ef.getReturnType()));
    }
    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
    return fieldSchema;
}
 

protected byte schemaToType(Object o, ResourceFieldSchema fieldSchema) {
    return (fieldSchema == null) ? DataType.findType(o) : fieldSchema
            .getType();
}
 

protected byte schemaToType(Object o, int i,
        ResourceFieldSchema[] fieldSchemas) {
    return (fieldSchemas == null) ? DataType.findType(o) : fieldSchemas[i]
            .getType();
}
 

@Override
protected Collection<Mutation> getMutations(Tuple tuple)
        throws ExecException, IOException {
    final ResourceFieldSchema[] fieldSchemas = (schema == null) ? null
            : schema.getFields();

    Iterator<Object> tupleIter = tuple.iterator();

    if (1 >= tuple.size()) {
        log.debug("Ignoring tuple of size " + tuple.size());
        return Collections.emptyList();
    }

    Mutation mutation = new Mutation(objectToText(tupleIter.next(),
            (null == fieldSchemas) ? null : fieldSchemas[0]));

    int tupleOffset = 1;
    Iterator<Column> columnIter = columns.iterator();
    while (tupleIter.hasNext() && columnIter.hasNext()) {
        Object o = tupleIter.next();
        Column column = columnIter.next();

        // Grab the type for this field
        final byte type = schemaToType(o, (null == fieldSchemas) ? null
                : fieldSchemas[tupleOffset]);

        switch (column.getType()) {
        case LITERAL:
            byte[] bytes = objToBytes(o, type);

            if (null != bytes) {
                Value value = new Value(bytes);

                // We don't have any column name from non-Maps
                addColumn(mutation, column.getColumnFamily(),
                        column.getColumnQualifier(), value);
            }
            break;
        case COLFAM_PREFIX:
        case COLQUAL_PREFIX:
            Map<String, Object> map;
            try {
                map = (Map<String, Object>) o;
            } catch (ClassCastException e) {
                log.error("Expected Map at tuple offset " + tupleOffset
                        + " but was " + o.getClass().getSimpleName());
                throw e;
            }

            for (Entry<String, Object> entry : map.entrySet()) {
                String key = entry.getKey();
                Object objValue = entry.getValue();

                byte valueType = DataType.findType(objValue);
                byte[] mapValue = objToBytes(objValue, valueType);

                if (Column.Type.COLFAM_PREFIX == column.getType()) {
                    addColumn(mutation, column.getColumnFamily() + key,
                            null, new Value(mapValue));
                } else if (Column.Type.COLQUAL_PREFIX == column.getType()) {
                    addColumn(mutation, column.getColumnFamily(),
                            column.getColumnQualifier() + key, new Value(
                                    mapValue));
                } else {
                    throw new IOException("Unknown column type");
                }
            }
            break;
        default:
            log.info("Ignoring unhandled column type");
            continue;
        }

        tupleOffset++;
    }

    if (0 == mutation.size()) {
        return Collections.emptyList();
    }

    return Collections.singletonList(mutation);
}
 

Object GetSmallerValue(Object v) {
    byte type = DataType.findType(v);

    if (type == DataType.BAG || type == DataType.TUPLE
            || type == DataType.MAP)
        return null;

    switch (type) {
    case DataType.CHARARRAY:
        String str = (String) v;
        if (str.length() > 0)
            return str.substring(0, str.length() - 1);
        else
            return null;
    case DataType.BYTEARRAY:
        DataByteArray data = (DataByteArray) v;
        if (data.size() > 0)
            return new DataByteArray(data.get(), 0, data.size() - 1);
        else
            return null;
    case DataType.INTEGER:
        return Integer.valueOf((Integer) v - 1);
    case DataType.LONG:
        return Long.valueOf((Long) v - 1);
    case DataType.FLOAT:
        return Float.valueOf((Float) v - 1);
    case DataType.DOUBLE:
        return Double.valueOf((Double) v - 1);
    case DataType.BIGINTEGER:
        return ((BigInteger)v).subtract(BigInteger.ONE);
    case DataType.BIGDECIMAL:
        return ((BigDecimal)v).subtract(BigDecimal.ONE);
    case DataType.DATETIME:
        DateTime dt = (DateTime) v;
        if (dt.getMillisOfSecond() != 0) {
            return dt.minusMillis(1);
        } else if (dt.getSecondOfMinute() != 0) {
            return dt.minusSeconds(1);
        } else if (dt.getMinuteOfHour() != 0) {
            return dt.minusMinutes(1);
        } else if (dt.getHourOfDay() != 0) {
            return dt.minusHours(1);
        } else {
            return dt.minusDays(1);
        }
    default:
        return null;
    }

}
 

Object GetLargerValue(Object v) {
    byte type = DataType.findType(v);

    if (type == DataType.BAG || type == DataType.TUPLE
            || type == DataType.MAP)
        return null;

    switch (type) {
    case DataType.CHARARRAY:
        return (String) v + "0";
    case DataType.BYTEARRAY:
        String str = ((DataByteArray) v).toString();
        str = str + "0";
        return new DataByteArray(str);
    case DataType.INTEGER:
        return Integer.valueOf((Integer) v + 1);
    case DataType.LONG:
        return Long.valueOf((Long) v + 1);
    case DataType.FLOAT:
        return Float.valueOf((Float) v + 1);
    case DataType.DOUBLE:
        return Double.valueOf((Double) v + 1);
    case DataType.BIGINTEGER:
        return ((BigInteger)v).add(BigInteger.ONE);
    case DataType.BIGDECIMAL:
        return ((BigDecimal)v).add(BigDecimal.ONE);
    case DataType.DATETIME:
        DateTime dt = (DateTime) v;
        if (dt.getMillisOfSecond() != 0) {
            return dt.plusMillis(1);
        } else if (dt.getSecondOfMinute() != 0) {
            return dt.plusSeconds(1);
        } else if (dt.getMinuteOfHour() != 0) {
            return dt.plusMinutes(1);
        } else if (dt.getHourOfDay() != 0) {
            return dt.plusHours(1);
        } else {
            return dt.plusDays(1);
        }
    default:
        return null;
    }
}
 

public Double exec(Tuple input) throws IOException 
{
	if (input == null || input.size() == 0) {
           pigLogger.warn(this, "Input is empty.", PigWarning.UDF_WARNING_1); 
		return null;
       }

       Double output = null;
       boolean accumulated = false;

	try {
           for(int i = 0; i < input.size(); ++i) {
               Object o = input.get(i);
               byte inputType = DataType.findType(o);
               if(DataType.isNumberType(inputType)) {
                   if(!accumulated) {
                       output = 0.0;
                       accumulated = true;
                   }
                   switch(inputType) {
                   case DataType.INTEGER:
                       output += (Integer)o;
                       break;

                   case DataType.LONG:
                       output += (Long)o;
                       break;

                   case DataType.FLOAT:
                       output += (Float)o;
                       break;

                   case DataType.DOUBLE:
                       output += (Double)o;
                       break;
                   }

               } else {
                   pigLogger.warn(this, "Found a non-numeric type.", PigWarning.UDF_WARNING_3);
               }
           }
	} catch(Exception e){
           pigLogger.warn(this, "Problem while computing output.", PigWarning.UDF_WARNING_2); 
		return null;
	}

       if(!accumulated) {
           pigLogger.warn(this, "Did not find any numeric type in the input.", PigWarning.UDF_WARNING_4);
       }

	return output;
   }