org.apache.hive.hcatalog.data.schema.HCatSchema Java Examples

@SuppressWarnings("unchecked")
private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
	this.fieldNames = new String[in.readInt()];
	for (int i = 0; i < this.fieldNames.length; i++) {
		this.fieldNames[i] = in.readUTF();
	}

	Configuration configuration = new Configuration();
	configuration.readFields(in);

	if (this.configuration == null) {
		this.configuration = configuration;
	}

	this.hCatInputFormat = new org.apache.hive.hcatalog.mapreduce.HCatInputFormat();
	this.outputSchema = (HCatSchema) HCatUtil.deserialize(this.configuration.get("mapreduce.lib.hcat.output.schema"));
}
 

public NormalTransformer(Reducer.Context context, IndexInfo indexInfo) throws Exception {
    Map<String, String> esTypeMap = indexInfo.getTypeMap();

    HCatSchema hCatSchema = HCatInputFormat.getTableSchema(context.getConfiguration());
    List<HCatFieldSchema> hCatFieldSchemas = hCatSchema.getFields();
    for(HCatFieldSchema hCatFieldSchema : hCatFieldSchemas) {
        String fieldName = hCatFieldSchema.getName();
        String hiveType = hCatFieldSchema.getTypeString();

        if(esTypeMap.containsKey(fieldName)) {
            String esType = esTypeMap.get(fieldName);
            typeList.add(Type.matchESType(fieldName, esType, indexInfo));
        } else {
            typeList.add(Type.matchHiveType(fieldName, hiveType, indexInfo));
        }
    }
}
 

private List<HCatRecord> generateHCatRecords(int numRecords,
  HCatSchema hCatTblSchema, ColumnGenerator... extraCols) throws Exception {
  List<HCatRecord> records = new ArrayList<HCatRecord>();
  List<HCatFieldSchema> hCatTblCols = hCatTblSchema.getFields();
  int size = hCatTblCols.size();
  for (int i = 0; i < numRecords; ++i) {
    DefaultHCatRecord record = new DefaultHCatRecord(size);
    record.set(hCatTblCols.get(0).getName(), hCatTblSchema, i);
    record.set(hCatTblCols.get(1).getName(), hCatTblSchema, "textfield" + i);
    int idx = 0;
    for (int j = 0; j < extraCols.length; ++j) {
      if (extraCols[j].getKeyType() == KeyType.STATIC_KEY) {
        continue;
      }
      record.set(hCatTblCols.get(idx + 2).getName(), hCatTblSchema,
        extraCols[j].getHCatValue(i));
      ++idx;
    }

    records.add(record);
  }
  return records;
}
 

@Override
public List<Object[]> parse(File file, HCatSchema schema, List<String> names) {
  try {
    List<String> lines = Files.readAllLines(file.toPath(), charset);

    if (this.hasHeader) {
      lines = lines.subList(1, lines.size());
    }

    List<Object[]> records = new ArrayList<>(lines.size());
    for (String line : lines) {
      records.add(parseRow(line, names.size()));
    }
    return records;
  } catch (IOException e) {
    throw new RuntimeException("Error while reading file", e);
  }
}
 

@SuppressWarnings("unchecked")
private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
	this.fieldNames = new String[in.readInt()];
	for (int i = 0; i < this.fieldNames.length; i++) {
		this.fieldNames[i] = in.readUTF();
	}

	Configuration configuration = new Configuration();
	configuration.readFields(in);

	if (this.configuration == null) {
		this.configuration = configuration;
	}

	this.hCatInputFormat = new org.apache.hive.hcatalog.mapreduce.HCatInputFormat();
	this.outputSchema = (HCatSchema) HCatUtil.deserialize(this.configuration.get("mapreduce.lib.hcat.output.schema"));
}
 

@SuppressWarnings("unchecked")
private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
	this.fieldNames = new String[in.readInt()];
	for (int i = 0; i < this.fieldNames.length; i++) {
		this.fieldNames[i] = in.readUTF();
	}

	Configuration configuration = new Configuration();
	configuration.readFields(in);

	if (this.configuration == null) {
		this.configuration = configuration;
	}

	this.hCatInputFormat = new org.apache.hive.hcatalog.mapreduce.HCatInputFormat();
	this.outputSchema = (HCatSchema) HCatUtil.deserialize(this.configuration.get("mapreduce.lib.hcat.output.schema"));
}
 

@Nonnull
private static List<Object> parseArray(@Nonnull final JsonParser p,
        @CheckForNull final List<TypeInfo> columnTypes)
        throws HCatException, IOException, SerDeException {
    Preconditions.checkNotNull(columnTypes, "columnTypes MUST NOT be null",
        SerDeException.class);
    if (columnTypes.size() != 1) {
        throw new IOException("Expected a single array but go " + columnTypes);
    }

    TypeInfo elemType = columnTypes.get(0);
    HCatSchema schema = HCatSchemaUtils.getHCatSchema(elemType);

    HCatFieldSchema listSchema = schema.get(0);
    HCatFieldSchema elemSchema = listSchema.getArrayElementSchema().get(0);

    final List<Object> arr = new ArrayList<Object>();
    while (p.nextToken() != JsonToken.END_ARRAY) {
        arr.add(extractCurrentField(p, elemSchema, true));
    }
    return arr;
}
 

/**
 * Specifies the fields which are returned by the InputFormat and their order.
 *
 * @param fields The fields and their order which are returned by the InputFormat.
 * @return This InputFormat with specified return fields.
 * @throws java.io.IOException
 */
public HCatInputFormatBase<T> getFields(String... fields) throws IOException {

	// build output schema
	ArrayList<HCatFieldSchema> fieldSchemas = new ArrayList<HCatFieldSchema>(fields.length);
	for (String field : fields) {
		fieldSchemas.add(this.outputSchema.get(field));
	}
	this.outputSchema = new HCatSchema(fieldSchemas);

	// update output schema configuration
	configuration.set("mapreduce.lib.hcat.output.schema", HCatUtil.serialize(outputSchema));

	return this;
}
 

private void loadHCatTable(HCatSchema hCatSchema, String table,
  int count, ColumnGenerator... extraCols)
  throws Exception {
  Map<String, String> staticKeyMap = new HashMap<String, String>();
  for (ColumnGenerator col : extraCols) {
    if (col.getKeyType() == KeyType.STATIC_KEY) {
      staticKeyMap.put(col.getName(), (String) col.getHCatValue(0));
    }
  }
  loadHCatTable(null, table, staticKeyMap,
    hCatSchema, generateHCatRecords(count, hCatSchema, extraCols));
}
 

/**
 * Specifies the fields which are returned by the InputFormat and their order.
 *
 * @param fields The fields and their order which are returned by the InputFormat.
 * @return This InputFormat with specified return fields.
 * @throws java.io.IOException
 */
public HCatInputFormatBase<T> getFields(String... fields) throws IOException {

	// build output schema
	ArrayList<HCatFieldSchema> fieldSchemas = new ArrayList<HCatFieldSchema>(fields.length);
	for (String field : fields) {
		fieldSchemas.add(this.outputSchema.get(field));
	}
	this.outputSchema = new HCatSchema(fieldSchemas);

	// update output schema configuration
	configuration.set("mapreduce.lib.hcat.output.schema", HCatUtil.serialize(outputSchema));

	return this;
}
 

protected void runHCatImport(List<String> addlArgsArray,
  int totalRecords, String table, ColumnGenerator[] cols,
  String[] cNames, boolean dontCreate, boolean isQuery) throws Exception {
  CreateMode mode = CreateMode.CREATE;
  if (dontCreate) {
    mode = CreateMode.NO_CREATION;
  }
  HCatSchema tblSchema =
    utils.createHCatTable(mode, totalRecords, table, cols);
  utils.createSqlTable(getConnection(), false, totalRecords, table, cols);
  addlArgsArray.add("-m");
  addlArgsArray.add("1");
  addlArgsArray.add("--hcatalog-table");
  addlArgsArray.add(table);
  String[] colNames = null;
  if (cNames != null) {
    colNames = cNames;
  } else {
    colNames = new String[2 + cols.length];
    colNames[0] = "ID";
    colNames[1] = "MSG";
    for (int i = 0; i < cols.length; ++i) {
      colNames[2 + i] =  cols[i].getName().toUpperCase();
    }
  }
  String[] importArgs;
  if (isQuery) {
    importArgs = getQueryArgv(true, colNames, new Configuration());
  } else {
    importArgs = getArgv(true, colNames, new Configuration());
  }
  LOG.debug("Import args = " + Arrays.toString(importArgs));
  SqoopHCatUtilities.instance().setConfigured(false);
  runImport(new ImportTool(), importArgs);
  List<HCatRecord> recs = utils.readHCatRecords(null, table, null);
  LOG.debug("HCat records ");
  LOG.debug(utils.hCatRecordDump(recs, tblSchema));
  validateHCatRecords(recs, tblSchema, 10, cols);
}
 

public String hCatRecordDump(List<HCatRecord> recs,
  HCatSchema schema) throws Exception {
  List<String> fields = schema.getFieldNames();
  int count = 0;
  StringBuilder sb = new StringBuilder(1024);
  for (HCatRecord rec : recs) {
    sb.append("HCat Record : " + ++count).append('\n');
    for (String field : fields) {
      sb.append('\t').append(field).append('=');
      sb.append(rec.get(field, schema)).append('\n');
      sb.append("\n\n");
    }
  }
  return sb.toString();
}
 

@Nonnull
private static Object parseObject(@Nonnull final JsonParser p,
        @CheckForNull final List<String> columnNames,
        @CheckForNull final List<TypeInfo> columnTypes)
        throws JsonParseException, IOException, SerDeException {
    Preconditions.checkNotNull(columnNames, "columnNames MUST NOT be null in parseObject",
        SerDeException.class);
    Preconditions.checkNotNull(columnTypes, "columnTypes MUST NOT be null in parseObject",
        SerDeException.class);
    if (columnNames.size() != columnTypes.size()) {
        throw new SerDeException(
            "Size of columnNames and columnTypes does not match. #columnNames="
                    + columnNames.size() + ", #columnTypes=" + columnTypes.size());
    }

    TypeInfo rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
    final HCatSchema schema;
    try {
        schema = HCatSchemaUtils.getHCatSchema(rowTypeInfo).get(0).getStructSubSchema();
    } catch (HCatException e) {
        throw new SerDeException(e);
    }

    final List<Object> r = new ArrayList<Object>(Collections.nCopies(columnNames.size(), null));
    JsonToken token;
    while (((token = p.nextToken()) != JsonToken.END_OBJECT) && (token != null)) {
        // iterate through each token, and create appropriate object here.
        populateRecord(r, token, p, schema);
    }

    if (columnTypes.size() == 1) {
        return r.get(0);
    }
    return r;
}
 

private static void populateRecord(@Nonnull final List<Object> r,
        @Nonnull final JsonToken token, @Nonnull final JsonParser p,
        @Nonnull final HCatSchema s) throws IOException {
    if (token != JsonToken.FIELD_NAME) {
        throw new IOException("Field name expected");
    }
    String fieldName = p.getText();
    Integer fpos = s.getPosition(fieldName);
    if (fpos == null) {
        fpos = getPositionFromHiveInternalColumnName(fieldName);
        if (fpos == -1) {
            skipValue(p);
            return; // unknown field, we return. We'll continue from the next field onwards.
        }
        // If we get past this, then the column name did match the hive pattern for an internal
        // column name, such as _col0, etc, so it *MUST* match the schema for the appropriate column.
        // This means people can't use arbitrary column names such as _col0, and expect us to ignore it
        // if we find it.
        if (!fieldName.equalsIgnoreCase(getHiveInternalColumnName(fpos))) {
            throw new IOException("Hive internal column name (" + fieldName
                    + ") and position encoding (" + fpos + ") for the column name are at odds");
        }
        // If we reached here, then we were successful at finding an alternate internal
        // column mapping, and we're about to proceed.
    }
    HCatFieldSchema hcatFieldSchema = s.getFields().get(fpos);
    Object currField = extractCurrentField(p, hcatFieldSchema, false);
    r.set(fpos, currField);
}
 

/**
 * Specifies the fields which are returned by the InputFormat and their order.
 *
 * @param fields The fields and their order which are returned by the InputFormat.
 * @return This InputFormat with specified return fields.
 * @throws java.io.IOException
 */
public HCatInputFormatBase<T> getFields(String... fields) throws IOException {

	// build output schema
	ArrayList<HCatFieldSchema> fieldSchemas = new ArrayList<HCatFieldSchema>(fields.length);
	for (String field : fields) {
		fieldSchemas.add(this.outputSchema.get(field));
	}
	this.outputSchema = new HCatSchema(fieldSchemas);

	// update output schema configuration
	configuration.set("mapreduce.lib.hcat.output.schema", HCatUtil.serialize(outputSchema));

	return this;
}
 

TableDataBuilder(HCatTable table) {
  schema = new HCatSchema(ImmutableList
      .<HCatFieldSchema> builder()
      .addAll(table.getCols())
      .addAll(table.getPartCols())
      .build());
  partitionColumns = table.getPartCols();
  withAllColumns();
}
 

public List<HCatRecord> loadHCatTable(String dbName,
  String tableName, Map<String, String> partKeyMap,
  HCatSchema tblSchema, List<HCatRecord> records)
  throws Exception {

  Job job = new Job(conf, "HCat load job");

  job.setJarByClass(this.getClass());
  job.setMapperClass(HCatWriterMapper.class);


  // Just writ 10 lines to the file to drive the mapper
  Path path = new Path(fs.getWorkingDirectory(),
    "mapreduce/HCatTableIndexInput");

  job.getConfiguration()
    .setInt(ConfigurationConstants.PROP_MAPRED_MAP_TASKS, 1);
  int writeCount = records.size();
  recsToLoad.clear();
  recsToLoad.addAll(records);
  createInputFile(path, writeCount);
  // input/output settings
  HCatWriterMapper.setWrittenRecordCount(0);

  FileInputFormat.setInputPaths(job, path);
  job.setInputFormatClass(TextInputFormat.class);
  job.setOutputFormatClass(HCatOutputFormat.class);
  OutputJobInfo outputJobInfo = OutputJobInfo.create(dbName, tableName,
    partKeyMap);

  HCatOutputFormat.setOutput(job, outputJobInfo);
  HCatOutputFormat.setSchema(job, tblSchema);
  job.setMapOutputKeyClass(BytesWritable.class);
  job.setMapOutputValueClass(DefaultHCatRecord.class);

  job.setNumReduceTasks(0);
  SqoopHCatUtilities.addJars(job, new SqoopOptions());
  boolean success = job.waitForCompletion(true);

  if (!success) {
    throw new IOException("Loading HCatalog table with test records failed");
  }
  utils.invokeOutputCommitterForLocalMode(job);
  LOG.info("Loaded " + HCatWriterMapper.writtenRecordCount + " records");
  return recsToLoad;
}
 

/**
 * Verify that on a given row, a column has a given value.
 *
 * @param id
 *          the id column specifying the row to test.
 */
public static void assertHCatColValForRowId(List<HCatRecord> recs,
  HCatSchema schema, int id, String fieldName,
  Object expectedVal) throws IOException {
  LOG.info("Verifying field " + fieldName + " has value " + expectedVal);

  Object actualVal = null;
  for (HCatRecord rec : recs) {
    if (rec.getInteger("id", schema).equals(id)) {
      actualVal = rec.get(fieldName, schema);
      break;
    }
  }
  if (actualVal == null) {
    throw new IOException("No record found with id = " + id);
  }
  if (expectedVal != null && expectedVal instanceof byte[]) {
    Assert
      .assertArrayEquals((byte[]) expectedVal, (byte[]) actualVal);
  } else {
    if (expectedVal instanceof Float) {
      if (actualVal instanceof Double) {
        Assert.assertEquals(((Float) expectedVal).floatValue(),
          ((Double) actualVal).doubleValue(), DELTAVAL);
      } else {
        Assert
          .assertEquals("Got unexpected column value", expectedVal,
            actualVal);
      }
    } else if (expectedVal instanceof Double) {
      if (actualVal instanceof Float) {
        Assert.assertEquals(((Double) expectedVal).doubleValue(),
          ((Float) actualVal).doubleValue(), DELTAVAL);
      } else {
        Assert
          .assertEquals("Got unexpected column value", expectedVal,
            actualVal);
      }
    } else {
      Assert
        .assertEquals("Got unexpected column value", expectedVal,
          actualVal);
    }
  }
}
 

public SqoopHCatImportHelper(Configuration conf) throws IOException,
  InterruptedException {

  String inputJobInfoStr = conf.get(HCatConstants.HCAT_KEY_JOB_INFO);
  jobInfo = (InputJobInfo) HCatUtil.deserialize(inputJobInfoStr);
  dataColsSchema = jobInfo.getTableInfo().getDataColumns();
  partitionSchema = jobInfo.getTableInfo().getPartitionColumns();
  StringBuilder storerInfoStr = new StringBuilder(1024);
  StorerInfo storerInfo = jobInfo.getTableInfo().getStorerInfo();
  storerInfoStr.append("HCatalog Storer Info : ").append("\n\tHandler = ")
    .append(storerInfo.getStorageHandlerClass())
    .append("\n\tInput format class = ").append(storerInfo.getIfClass())
    .append("\n\tOutput format class = ").append(storerInfo.getOfClass())
    .append("\n\tSerde class = ").append(storerInfo.getSerdeClass());
  Properties storerProperties = storerInfo.getProperties();
  if (!storerProperties.isEmpty()) {
    storerInfoStr.append("\nStorer properties ");
    for (Map.Entry<Object, Object> entry : storerProperties.entrySet()) {
      String key = (String) entry.getKey();
      Object val = entry.getValue();
      storerInfoStr.append("\n\t").append(key).append('=').append(val);
    }
  }
  storerInfoStr.append("\n");
  LOG.info(storerInfoStr);

  hCatFullTableSchema = new HCatSchema(dataColsSchema.getFields());
  for (HCatFieldSchema hfs : partitionSchema.getFields()) {
    hCatFullTableSchema.append(hfs);
  }
  fieldCount = hCatFullTableSchema.size();
  lobLoader = new LargeObjectLoader(conf, new Path(jobInfo.getTableInfo()
    .getTableLocation()));
  bigDecimalFormatString = conf.getBoolean(
    ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT,
    ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);
  debugHCatImportMapper = conf.getBoolean(
    SqoopHCatUtilities.DEBUG_HCAT_IMPORT_MAPPER_PROP, false);
  IntWritable[] delimChars = DefaultStringifier.loadArray(conf,
    SqoopHCatUtilities.HIVE_DELIMITERS_TO_REPLACE_PROP, IntWritable.class);
  hiveDelimiters = new DelimiterSet((char) delimChars[0].get(),
    (char) delimChars[1].get(), (char) delimChars[2].get(),
    (char) delimChars[3].get(), delimChars[4].get() == 1 ? true : false);
  hiveDelimsReplacement = conf
    .get(SqoopHCatUtilities.HIVE_DELIMITERS_REPLACEMENT_PROP);
  if (hiveDelimsReplacement == null) {
    hiveDelimsReplacement = "";
  }
  doHiveDelimsReplacement = Boolean.valueOf(conf
    .get(SqoopHCatUtilities.HIVE_DELIMITERS_REPLACEMENT_ENABLED_PROP));

  IntWritable[] fPos = DefaultStringifier.loadArray(conf,
    SqoopHCatUtilities.HCAT_FIELD_POSITIONS_PROP, IntWritable.class);
  hCatFieldPositions = new int[fPos.length];
  for (int i = 0; i < fPos.length; ++i) {
    hCatFieldPositions[i] = fPos[i].get();
  }

  LOG.debug("Hive delims replacement enabled : " + doHiveDelimsReplacement);
  LOG.debug("Hive Delimiters : " + hiveDelimiters.toString());
  LOG.debug("Hive delimiters replacement : " + hiveDelimsReplacement);
  staticPartitionKeys = conf
    .getStrings(SqoopHCatUtilities.HCAT_STATIC_PARTITION_KEY_PROP);
  String partKeysString = staticPartitionKeys == null ? ""
    : Arrays.toString(staticPartitionKeys);
  LOG.debug("Static partition key used : "  + partKeysString);
}
 

public void setHCatPartitionSchema(HCatSchema schema) {
  hCatPartitionSchema = schema;
}
 

public HCatSchema getHCatPartitionSchema() {
  return hCatPartitionSchema;
}
 

public void setHCatOutputSchema(HCatSchema schema) {
  hCatOutputSchema = schema;
}
 

public HCatSchema getHCatOutputSchema() {
  return hCatOutputSchema;
}
 

public SqoopHCatExportHelper(Configuration conf, boolean isOdps)
  throws IOException, InterruptedException {
  this.isOdps = isOdps;

  if (!isOdps) {
    colTypesJava =
        DefaultStringifier.load(conf, SqoopHCatUtilities.HCAT_DB_OUTPUT_COLTYPES_JAVA,
            MapWritable.class);
    colTypesSql =
        DefaultStringifier.load(conf, SqoopHCatUtilities.HCAT_DB_OUTPUT_COLTYPES_SQL,
            MapWritable.class);
  }
  // Instantiate a copy of the user's class to hold and parse the record.

  String recordClassName = conf.get(
    ExportJobBase.SQOOP_EXPORT_TABLE_CLASS_KEY);
  if (null == recordClassName) {
    throw new IOException("Export table class name ("
      + ExportJobBase.SQOOP_EXPORT_TABLE_CLASS_KEY
      + ") is not set!");
  }

  bigDecimalFormatString = conf.getBoolean(
    ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT,
    ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);

  debugHCatExportMapper = conf.getBoolean(
    SqoopHCatUtilities.DEBUG_HCAT_EXPORT_MAPPER_PROP, false);
  try {
    Class<?> cls = Class.forName(recordClassName, true,
      Thread.currentThread().getContextClassLoader());
    sqoopRecord = (SqoopRecord) ReflectionUtils.newInstance(cls, conf);
  } catch (ClassNotFoundException cnfe) {
    throw new IOException(cnfe);
  }

  if (null == sqoopRecord) {
    throw new IOException("Could not instantiate object of type "
      + recordClassName);
  }

  String inputJobInfoStr = conf.get(HCatConstants.HCAT_KEY_JOB_INFO);
  jobInfo =
    (InputJobInfo) HCatUtil.deserialize(inputJobInfoStr);
  HCatSchema tableSchema = jobInfo.getTableInfo().getDataColumns();
  HCatSchema partitionSchema =
    jobInfo.getTableInfo().getPartitionColumns();
  hCatFullTableSchema = new HCatSchema(tableSchema.getFields());
  for (HCatFieldSchema hfs : partitionSchema.getFields()) {
    hCatFullTableSchema.append(hfs);
  }
}
 

/**
 * Returns the {@link org.apache.hive.hcatalog.data.schema.HCatSchema} of the {@link org.apache.hive.hcatalog.data.HCatRecord}
 * returned by this InputFormat.
 *
 * @return The HCatSchema of the HCatRecords returned by this InputFormat.
 */
public HCatSchema getOutputSchema() {
	return this.outputSchema;
}
 

/**
 * Parses the given file and returns the rows with the requested columns.
 *
 * @param file The file to be parsed.
 * @param schema The full schema of the Hive table.
 * @param names The requested field names.
 * @return A {@link List} of rows, each represented by an {@link Object} array.
 */
List<Object[]> parse(File file, HCatSchema schema, List<String> names);
 

/**
 * Returns the {@link org.apache.hive.hcatalog.data.schema.HCatSchema} of the {@link org.apache.hive.hcatalog.data.HCatRecord}
 * returned by this InputFormat.
 *
 * @return The HCatSchema of the HCatRecords returned by this InputFormat.
 */
public HCatSchema getOutputSchema() {
	return this.outputSchema;
}
 

/**
 * Returns the {@link org.apache.hive.hcatalog.data.schema.HCatSchema} of the {@link org.apache.hive.hcatalog.data.HCatRecord}
 * returned by this InputFormat.
 *
 * @return The HCatSchema of the HCatRecords returned by this InputFormat.
 */
public HCatSchema getOutputSchema() {
	return this.outputSchema;
}