org.apache.hive.hcatalog.common.HCatException Java Examples

@Nonnull
private static List<Object> parseArray(@Nonnull final JsonParser p,
        @CheckForNull final List<TypeInfo> columnTypes)
        throws HCatException, IOException, SerDeException {
    Preconditions.checkNotNull(columnTypes, "columnTypes MUST NOT be null",
        SerDeException.class);
    if (columnTypes.size() != 1) {
        throw new IOException("Expected a single array but go " + columnTypes);
    }

    TypeInfo elemType = columnTypes.get(0);
    HCatSchema schema = HCatSchemaUtils.getHCatSchema(elemType);

    HCatFieldSchema listSchema = schema.get(0);
    HCatFieldSchema elemSchema = listSchema.getArrayElementSchema().get(0);

    final List<Object> arr = new ArrayList<Object>();
    while (p.nextToken() != JsonToken.END_ARRAY) {
        arr.add(extractCurrentField(p, elemSchema, true));
    }
    return arr;
}
 

private void flush() throws HCatException {
  if (hCatRecordsBatch.isEmpty()) {
    return;
  }
  try {
    slaveWriter.write(hCatRecordsBatch.iterator());
    masterWriter.commit(writerContext);
  } catch (HCatException e) {
    LOG.error("Exception in flush - write/commit data to Hive", e);
    // abort on exception
    masterWriter.abort(writerContext);
    throw e;
  } finally {
    hCatRecordsBatch.clear();
  }
}
 

/**
 * Specifies that the InputFormat returns Flink tuples instead of
 * {@link org.apache.hive.hcatalog.data.HCatRecord}.
 *
 * <p>Note: Flink tuples might only support a limited number of fields (depending on the API).
 *
 * @return This InputFormat.
 * @throws org.apache.hive.hcatalog.common.HCatException
 */
public HCatInputFormatBase<T> asFlinkTuples() throws HCatException {

	// build type information
	int numFields = outputSchema.getFields().size();
	if (numFields > this.getMaxFlinkTupleSize()) {
		throw new IllegalArgumentException("Only up to " + this.getMaxFlinkTupleSize() +
				" fields can be returned as Flink tuples.");
	}

	TypeInformation[] fieldTypes = new TypeInformation[numFields];
	fieldNames = new String[numFields];
	for (String fieldName : outputSchema.getFieldNames()) {
		HCatFieldSchema field = outputSchema.get(fieldName);

		int fieldPos = outputSchema.getPosition(fieldName);
		TypeInformation fieldType = getFieldType(field);

		fieldTypes[fieldPos] = fieldType;
		fieldNames[fieldPos] = fieldName;

	}
	this.resultType = new TupleTypeInfo(fieldTypes);

	return this;
}
 

/**
 * Specifies that the InputFormat returns Flink tuples instead of
 * {@link org.apache.hive.hcatalog.data.HCatRecord}.
 *
 * <p>Note: Flink tuples might only support a limited number of fields (depending on the API).
 *
 * @return This InputFormat.
 * @throws org.apache.hive.hcatalog.common.HCatException
 */
public HCatInputFormatBase<T> asFlinkTuples() throws HCatException {

	// build type information
	int numFields = outputSchema.getFields().size();
	if (numFields > this.getMaxFlinkTupleSize()) {
		throw new IllegalArgumentException("Only up to " + this.getMaxFlinkTupleSize() +
				" fields can be returned as Flink tuples.");
	}

	TypeInformation[] fieldTypes = new TypeInformation[numFields];
	fieldNames = new String[numFields];
	for (String fieldName : outputSchema.getFieldNames()) {
		HCatFieldSchema field = outputSchema.get(fieldName);

		int fieldPos = outputSchema.getPosition(fieldName);
		TypeInformation fieldType = getFieldType(field);

		fieldTypes[fieldPos] = fieldType;
		fieldNames[fieldPos] = fieldName;

	}
	this.resultType = new TupleTypeInfo(fieldTypes);

	return this;
}
 

/**
 * Specifies that the InputFormat returns Flink tuples instead of
 * {@link org.apache.hive.hcatalog.data.HCatRecord}.
 *
 * <p>Note: Flink tuples might only support a limited number of fields (depending on the API).
 *
 * @return This InputFormat.
 * @throws org.apache.hive.hcatalog.common.HCatException
 */
public HCatInputFormatBase<T> asFlinkTuples() throws HCatException {

	// build type information
	int numFields = outputSchema.getFields().size();
	if (numFields > this.getMaxFlinkTupleSize()) {
		throw new IllegalArgumentException("Only up to " + this.getMaxFlinkTupleSize() +
				" fields can be returned as Flink tuples.");
	}

	TypeInformation[] fieldTypes = new TypeInformation[numFields];
	fieldNames = new String[numFields];
	for (String fieldName : outputSchema.getFieldNames()) {
		HCatFieldSchema field = outputSchema.get(fieldName);

		int fieldPos = outputSchema.getPosition(fieldName);
		TypeInformation fieldType = getFieldType(field);

		fieldTypes[fieldPos] = fieldType;
		fieldNames[fieldPos] = fieldName;

	}
	this.resultType = new TupleTypeInfo(fieldTypes);

	return this;
}
 

private String getKeyValue(List<String> keys, DefaultHCatRecord hCatRecord) throws HCatException {
    StringBuilder sb = new StringBuilder();
    for (String key : keys) {
        Object id = hCatRecord.get(key, this.schema);
        if (id == null || StringUtils.isBlank(id.toString())) {
            sb.append("");
        } else {
            sb.append(id.toString());
        }
        sb.append("_");
    }

    if (sb.length() > 1) {
        return sb.substring(0, sb.length() - 1);
    } else {
        return sb.toString();
    }
}
 

private void insert(Map<String, String> partitionSpec, Iterable<HCatRecord> rows) {
  WriteEntity entity = new WriteEntity.Builder()
      .withDatabase(databaseName)
      .withTable(tableName)
      .withPartition(partitionSpec)
      .build();

  try {
    HCatWriter master = DataTransferFactory.getHCatWriter(entity, config);
    WriterContext context = master.prepareWrite();
    HCatWriter writer = DataTransferFactory.getHCatWriter(context);
    writer.write(rows.iterator());
    master.commit(context);
  } catch (HCatException e) {
    throw new RuntimeException("An error occurred while inserting data to " + databaseName + "." + tableName, e);
  }
}
 

private Object get(String name) {
  checkColumn(name);
  try {
    return row.get(name, schema);
  } catch (HCatException e) {
    throw new RuntimeException("Error getting value for " + name, e);
  }
}
 

@Nonnull
private static Object parseObject(@Nonnull final JsonParser p,
        @CheckForNull final List<String> columnNames,
        @CheckForNull final List<TypeInfo> columnTypes)
        throws JsonParseException, IOException, SerDeException {
    Preconditions.checkNotNull(columnNames, "columnNames MUST NOT be null in parseObject",
        SerDeException.class);
    Preconditions.checkNotNull(columnTypes, "columnTypes MUST NOT be null in parseObject",
        SerDeException.class);
    if (columnNames.size() != columnTypes.size()) {
        throw new SerDeException(
            "Size of columnNames and columnTypes does not match. #columnNames="
                    + columnNames.size() + ", #columnTypes=" + columnTypes.size());
    }

    TypeInfo rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
    final HCatSchema schema;
    try {
        schema = HCatSchemaUtils.getHCatSchema(rowTypeInfo).get(0).getStructSubSchema();
    } catch (HCatException e) {
        throw new SerDeException(e);
    }

    final List<Object> r = new ArrayList<Object>(Collections.nCopies(columnNames.size(), null));
    JsonToken token;
    while (((token = p.nextToken()) != JsonToken.END_OBJECT) && (token != null)) {
        // iterate through each token, and create appropriate object here.
        populateRecord(r, token, p, schema);
    }

    if (columnTypes.size() == 1) {
        return r.get(0);
    }
    return r;
}
 

private static Schema.Field toBeamField(FieldSchema field) {
  String name = field.getName();
  HCatFieldSchema hCatFieldSchema;

  try {
    hCatFieldSchema = HCatSchemaUtils.getHCatFieldSchema(field);
  } catch (HCatException e) {
    // Converting checked Exception to unchecked Exception.
    throw new UnsupportedOperationException(
        "Error while converting FieldSchema to HCatFieldSchema", e);
  }

  switch (hCatFieldSchema.getCategory()) {
    case PRIMITIVE:
      {
        if (!HCAT_TO_BEAM_TYPES_MAP.containsKey(hCatFieldSchema.getType())) {
          throw new UnsupportedOperationException(
              "The Primitive HCat type '"
                  + field.getType()
                  + "' of field '"
                  + name
                  + "' cannot be converted to Beam FieldType");
        }

        FieldType fieldType = HCAT_TO_BEAM_TYPES_MAP.get(hCatFieldSchema.getType());
        return Schema.Field.of(name, fieldType).withNullable(true);
      }
      // TODO: Add Support for Complex Types i.e. ARRAY, MAP, STRUCT
    default:
      throw new UnsupportedOperationException(
          "The category '" + hCatFieldSchema.getCategory() + "' is not supported.");
  }
}
 

@Override
public JSONObject tranform(List<Object> valueList) throws HCatException {
    JSONObject data = new JSONObject();

    for(int i=0; i<typeList.size(); i++) {
        typeList.get(i).addField(data, valueList.get(i));
    }

    return data;
}
 

private ReaderContext getReaderContext(long desiredSplitCount) throws HCatException {
  ReadEntity entity =
      new ReadEntity.Builder()
          .withDatabase(spec.getDatabase())
          .withTable(spec.getTable())
          .withFilter(spec.getFilter())
          .build();
  // pass the 'desired' split count as an hint to the API
  Map<String, String> configProps = new HashMap<>(spec.getConfigProperties());
  configProps.put(
      HCatConstants.HCAT_DESIRED_PARTITION_NUM_SPLITS, String.valueOf(desiredSplitCount));
  return DataTransferFactory.getHCatReader(entity, configProps).prepareRead();
}
 

@Override
public boolean start() throws HCatException {
  HCatReader reader =
      DataTransferFactory.getHCatReader(source.spec.getContext(), source.spec.getSplitId());
  hcatIterator = reader.read();
  return advance();
}
 

@Setup
public void initiateWrite() throws HCatException {
  WriteEntity entity =
      new WriteEntity.Builder()
          .withDatabase(spec.getDatabase())
          .withTable(spec.getTable())
          .withPartition(spec.getPartition())
          .build();
  masterWriter = DataTransferFactory.getHCatWriter(entity, spec.getConfigProperties());
  writerContext = masterWriter.prepareWrite();
  slaveWriter = DataTransferFactory.getHCatWriter(writerContext);
}
 

@ProcessElement
public void processElement(ProcessContext ctx) throws HCatException {
  hCatRecordsBatch.add(ctx.element());
  if (hCatRecordsBatch.size() >= spec.getBatchSize()) {
    flush();
  }
}
 

private static HCatFieldSchema column(String name) {
  try {
    return new HCatFieldSchema(name, STRING, null);
  } catch (HCatException e) {
    throw new RuntimeException(e);
  }
}
 

private static Iterator<HCatRecord> loadHCatRecordItr(ReaderContext readCntxt, int dataSplit) throws HCatException {
    HCatReader currentHCatReader = DataTransferFactory.getHCatReader(readCntxt, dataSplit);
    return currentHCatReader.read();
}
 

/** Writes records to the table using the passed WriterContext. */
private static void writeRecords(WriterContext context) throws HCatException {
  DataTransferFactory.getHCatWriter(context)
      .write(buildHCatRecords(TEST_RECORDS_COUNT).iterator());
}
 

private static Iterator<HCatRecord> loadHCatRecordItr(ReaderContext readCntxt, int dataSplit) throws HCatException {
    HCatReader currentHCatReader = DataTransferFactory.getHCatReader(readCntxt, dataSplit);

    return currentHCatReader.read();
}
 

@FinishBundle
public void finishBundle() throws HCatException {
  flush();
}
 

/** Returns a WriterContext instance for the passed datastore config params. */
private static WriterContext getWriterContext(Map<String, String> config) throws HCatException {
  return DataTransferFactory.getHCatWriter(WRITE_ENTITY, config).prepareWrite();
}
 

/** Returns a ReaderContext instance for the passed datastore config params. */
public static ReaderContext getReaderContext(Map<String, String> config) throws HCatException {
  return DataTransferFactory.getHCatReader(READ_ENTITY, config).prepareRead();
}
 

private static Iterator<HCatRecord> loadHCatRecordItr(ReaderContext readCntxt, int dataSplit) throws HCatException {
    HCatReader currentHCatReader = DataTransferFactory.getHCatReader(readCntxt, dataSplit);

    return currentHCatReader.read();
}
 

protected abstract T buildFlinkTuple(T t, HCatRecord record) throws HCatException; 

protected abstract T buildFlinkTuple(T t, HCatRecord record) throws HCatException; 

public JSONObject tranform(List<Object> valueList) throws HCatException; 

protected abstract T buildFlinkTuple(T t, HCatRecord record) throws HCatException;