Java Code Examples for org.apache.spark.sql.types.DataTypes#BinaryType

private DataType getExpectedFieldDataType(String fieldName) {
  if (Lists.newArrayList(expectedSchema.fieldNames()).contains(fieldName)) {
    return expectedSchema.fields()[expectedSchema.fieldIndex(fieldName)].dataType();
  }
  else {
    // If the translator doesn't expect the field then provide it as binary
    return DataTypes.BinaryType;
  }
}
 

@Test
public void testToRowValueBinary() {
  DataType field = DataTypes.BinaryType;

  byte[] byteArray = "Test".getBytes();
  ByteBuffer byteBuffer = ByteBuffer.wrap(byteArray);

  assertEquals("Invalid byte[]", byteArray, RowUtils.toRowValue(byteArray, field));
  assertEquals("Invalid ByteBuffer", byteArray, RowUtils.toRowValue(byteBuffer, field));

  thrown.expect(RuntimeException.class);
  RowUtils.toRowValue(123, field);
}
 

private DataType sparkFromArrow(FieldType fieldType) {
  switch (fieldType.getType().getTypeID()) {
    case Null:
      return DataTypes.NullType;
    case Struct:
      throw new UnsupportedOperationException("have not implemented Struct type yet");
    case List:
      throw new UnsupportedOperationException("have not implemented List type yet");
    case FixedSizeList:
      throw new UnsupportedOperationException("have not implemented FixedSizeList type yet");
    case Union:
      throw new UnsupportedOperationException("have not implemented Union type yet");
    case Int:
      ArrowType.Int intType = (ArrowType.Int) fieldType.getType();
      int bitWidth = intType.getBitWidth();
      if (bitWidth == 8) {
        return DataTypes.ByteType;
      } else if (bitWidth == 16) {
        return DataTypes.ShortType;
      } else if (bitWidth == 32) {
        return DataTypes.IntegerType;
      } else if (bitWidth == 64) {
        return DataTypes.LongType;
      }
      throw new UnsupportedOperationException("unknown int type with bitwidth " + bitWidth);
    case FloatingPoint:
      ArrowType.FloatingPoint floatType = (ArrowType.FloatingPoint) fieldType.getType();
      FloatingPointPrecision precision = floatType.getPrecision();
      switch (precision) {
        case HALF:
        case SINGLE:
          return DataTypes.FloatType;
        case DOUBLE:
          return DataTypes.DoubleType;
      }
    case Utf8:
      return DataTypes.StringType;
    case Binary:
    case FixedSizeBinary:
      return DataTypes.BinaryType;
    case Bool:
      return DataTypes.BooleanType;
    case Decimal:
      throw new UnsupportedOperationException("have not implemented Decimal type yet");
    case Date:
      return DataTypes.DateType;
    case Time:
      return DataTypes.TimestampType; //note i don't know what this will do!
    case Timestamp:
      return DataTypes.TimestampType;
    case Interval:
      return DataTypes.CalendarIntervalType;
    case NONE:
      return DataTypes.NullType;
  }
  throw new IllegalStateException("Unexpected value: " + fieldType);
}
 

static DataType getSparkType(Type type)
{
    if (type instanceof ParameterizedType && ((ParameterizedType) type).getRawType() == Map.class) {
        Type[] arguments = ((ParameterizedType) type).getActualTypeArguments();

        return DataTypes.createMapType(getSparkType(arguments[0]), getSparkType(arguments[1]));
    }
    else if (type instanceof ParameterizedType && ((ParameterizedType) type).getRawType() == List.class) {
        DataType dataType = getSparkType(((ParameterizedType) type).getActualTypeArguments()[0]);

        return DataTypes.createArrayType(dataType);
    }
    else {
        if (type == String.class) {
            return DataTypes.StringType;
        }
        else if (type == int.class || type == Integer.class) {
            return DataTypes.IntegerType;
        }
        else if (type == long.class || type == Long.class) {
            return DataTypes.LongType;
        }
        else if (type == boolean.class || type == Boolean.class) {
            return DataTypes.BooleanType;
        }
        else if (type == double.class || type == Double.class) {
            return DataTypes.DoubleType;
        }
        else if (type == float.class || type == Float.class) {
            return DataTypes.FloatType;
        }
        else if (type == byte.class || type == Byte.class) {
            return DataTypes.ByteType;
        }
        else if (type == Timestamp.class) {
            return DataTypes.TimestampType;
        }
        else if (type == Date.class) {
            return DataTypes.DateType;
        }
        else if (type == byte[].class || type == Byte[].class) {
            return DataTypes.BinaryType;
        }
        else {
            throw new IllegalArgumentException("this TYPE " + type + " have't support!");
        }
    }
}
 

private static DataType getType(String type) {
    DataType dataType = DataTypes.NullType;
    switch (type.toLowerCase()) {
        case "string":
            dataType = DataTypes.StringType;
            break;
        case "integer":
            dataType = DataTypes.IntegerType;
            break;
        case "long":
            dataType = DataTypes.LongType;
            break;
        case "double":
            dataType = DataTypes.DoubleType;
            break;
        case "float":
            dataType = DataTypes.FloatType;
            break;
        case "short":
            dataType = DataTypes.ShortType;
            break;
        case "date":
            dataType = DataTypes.DateType;
            break;
        case "timestamp":
            dataType = DataTypes.TimestampType;
            break;
        case "boolean":
            dataType = DataTypes.BooleanType;
            break;
        case "binary":
            dataType = DataTypes.BinaryType;
            break;
        case "byte":
            dataType = DataTypes.ByteType;
            break;
        default:
            throw new ConfigRuntimeException("Throw data type exception, unknown type: " + type);
    }
    return dataType;
}
 

private StructType schemaFor(KuduTable table) {
  List<StructField> fields = Lists.newArrayList();

  for (ColumnSchema columnSchema : table.getSchema().getColumns()) {
    DataType fieldType;

    switch (columnSchema.getType()) {
      case DOUBLE:
        fieldType = DataTypes.DoubleType;
        break;
      case FLOAT:
        fieldType = DataTypes.FloatType;
        break;
      case INT8:
        fieldType = DataTypes.ByteType;
        break;
      case INT16:
        fieldType = DataTypes.ShortType;
        break;
      case INT32:
        fieldType = DataTypes.IntegerType;
        break;
      case INT64:
        fieldType = DataTypes.LongType;
        break;
      case STRING:
        fieldType = DataTypes.StringType;
        break;
      case BOOL:
        fieldType = DataTypes.BooleanType;
        break;
      case BINARY:
        fieldType = DataTypes.BinaryType;
        break;
      case UNIXTIME_MICROS:
        fieldType = DataTypes.TimestampType;
        break;
      case DECIMAL:
        int precision = columnSchema.getTypeAttributes().getPrecision();
        int scale = columnSchema.getTypeAttributes().getScale();
        fieldType = DataTypes.createDecimalType(precision, scale);
        break;
      default:
        throw new RuntimeException("Unsupported Kudu column type: " + columnSchema.getType());
    }

    fields.add(DataTypes.createStructField(columnSchema.getName(), fieldType, true));
  }

  return DataTypes.createStructType(fields);
}
 

/**
 * Convert Avro Types into their associated DataType.
 *
 * @param schemaType Avro Schema.Type
 * @return DataType representation
 */
public static DataType dataTypeFor(Schema schemaType) {
  LOG.trace("Converting Schema[{}] to DataType", schemaType);

  // Unwrap "optional" unions to the base type
  boolean isOptional = isNullable(schemaType);

  if (isOptional) {
    // if only 2 items in the union, then "unwrap," otherwise, it's a full union and should be rendered as such
    if (schemaType.getTypes().size() == 2) {
      LOG.trace("Unwrapping simple 'optional' union for {}", schemaType);
      for (Schema s : schemaType.getTypes()) {
        if (s.getType().equals(NULL)) {
          continue;
        }
        // Unwrap
        schemaType = s;
        break;
      }
    }
  }

  // Convert supported LogicalTypes
  if (null != schemaType.getLogicalType()) {
    LogicalType logicalType = schemaType.getLogicalType();
    switch (logicalType.getName()) {
      case "date" :
        return DataTypes.DateType;
      case "timestamp-millis" :
        return DataTypes.TimestampType;
      case "decimal" :
        LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
        return DataTypes.createDecimalType(decimal.getPrecision(), decimal.getScale());
      default:
        // Pass-thru
        LOG.warn("Unsupported LogicalType[{}], continuing with underlying base type", logicalType.getName());
    }
  }

  switch (schemaType.getType()) {
    case RECORD:
      // StructType
      List<StructField> structFieldList = Lists.newArrayListWithCapacity(schemaType.getFields().size());
      for (Field f : schemaType.getFields()) {
        structFieldList.add(DataTypes.createStructField(f.name(), dataTypeFor(f.schema()), isNullable(f.schema())));
      }
      return DataTypes.createStructType(structFieldList);
    case ARRAY:
      Schema elementType = schemaType.getElementType();
      return DataTypes.createArrayType(dataTypeFor(elementType), isNullable(elementType));
    case MAP:
      Schema valueType = schemaType.getValueType();
      return DataTypes.createMapType(DataTypes.StringType, dataTypeFor(valueType), isNullable(valueType));
    case UNION:
      // StructType of members
      List<StructField> unionFieldList = Lists.newArrayListWithCapacity(schemaType.getTypes().size());
      int m = 0;
      for (Schema u : schemaType.getTypes()) {
        unionFieldList.add(DataTypes.createStructField("member" + m++, dataTypeFor(u), isNullable(u)));
      }
      return DataTypes.createStructType(unionFieldList);
    case FIXED:
    case BYTES:
      return DataTypes.BinaryType;
    case ENUM:
    case STRING:
      return DataTypes.StringType;
    case INT:
      return DataTypes.IntegerType;
    case LONG:
      return DataTypes.LongType;
    case FLOAT:
      return DataTypes.FloatType;
    case DOUBLE:
      return DataTypes.DoubleType;
    case BOOLEAN:
      return DataTypes.BooleanType;
    case NULL:
      return DataTypes.NullType;
    default:
      throw new RuntimeException(String.format("Unrecognized or unsupported Avro Type conversion: %s", schemaType));
  }
}
 

public static DataType getSparkDataType(String typeString) {
  DataType type;

  String prec_scale_regex_groups = "\\s*(decimal)\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)\\s*";
  Pattern prec_scale_regex_pattern = Pattern.compile(prec_scale_regex_groups);
  Matcher prec_scale_regex_matcher = prec_scale_regex_pattern.matcher(typeString);

  if (prec_scale_regex_matcher.matches()) {
    int precision = Integer.parseInt(prec_scale_regex_matcher.group(2)); 
    int scale = Integer.parseInt(prec_scale_regex_matcher.group(3)); 
    type = DataTypes.createDecimalType(precision, scale);
  }
  else {
    switch (typeString) {
      case DECIMAL:
        type = DataTypes.createDecimalType();
        break;
      case STRING:
        type = DataTypes.StringType;
        break;
      case FLOAT:
        type = DataTypes.FloatType;
        break;
      case DOUBLE:
        type = DataTypes.DoubleType;
        break;
      case BYTE:
        type = DataTypes.ByteType;
        break;
      case SHORT:
        type = DataTypes.ShortType;
        break;
      case INT:
        type = DataTypes.IntegerType;
        break;
      case LONG:
        type = DataTypes.LongType;
        break;
      case BOOLEAN:
        type = DataTypes.BooleanType;
        break;
      case BINARY:
        type = DataTypes.BinaryType;
        break;
      case DATE:
        type = DataTypes.DateType;
        break;
      case TIMESTAMP:
        type = DataTypes.TimestampType;
        break;
      default:
        throw new RuntimeException("Unsupported or unrecognized field type: " + typeString);
    } 
  }

  return type;
}
 

@Override
public DataType sqlType() {
  return new StructType(
      new StructField[] {new StructField("wkb", DataTypes.BinaryType, true, Metadata.empty())});
}