Java Code Examples for org.apache.hadoop.hive.metastore.api.Partition#setSd()

/**
 * Copies partitions from oldTable to newTable, partitions copied are modified to take the schema of newTable
 */
public void execute(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws TException {
  int count = 0;
  String databaseName = newTable.getDbName();
  String tableName = newTable.getTableName();
  PartitionIterator partitionIterator = new PartitionIterator(client, oldTable, partitionBatchSize);
  while (partitionIterator.hasNext()) {
    List<Partition> batch = new ArrayList<>();
    for (int i = 0; i < partitionBatchSize && partitionIterator.hasNext(); i++) {
      Partition partition = partitionIterator.next();
      count++;
      Partition copy = new Partition(partition);
      copy.setDbName(databaseName);
      copy.setTableName(tableName);
      StorageDescriptor sd = new StorageDescriptor(partition.getSd());
      sd.setCols(newTable.getSd().getCols());
      copy.setSd(sd);
      batch.add(copy);
    }
    LOG.info("Copying batch of size {} to {}.{}", batch.size(), databaseName, tableName);
    client.add_partitions(batch);
  }
  LOG.info("Copied {} partitions to {}.{}", count, databaseName, tableName);
}
 

private Partition newPartition(String partitionValue, Path location1) {
  Partition partition = new Partition();
  partition.setValues(Lists.newArrayList(partitionValue));
  StorageDescriptor sd1 = new StorageDescriptor();
  sd1.setLocation(location1.toString());
  partition.setSd(sd1);
  Map<String, String> parameters = new HashMap<>();
  parameters.put(REPLICATION_EVENT.parameterName(), EVENT_ID);
  partition.setParameters(parameters);
  return partition;
}
 

static Partition newPartition(Table hiveTable, List<String> values, File location) {
  Partition partition = new Partition();
  partition.setDbName(hiveTable.getDbName());
  partition.setTableName(hiveTable.getTableName());
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(hiveTable.getSd()));
  partition.getSd().setLocation(location.toURI().toString());
  return partition;
}
 

public static Partition newPartition() {
  Partition partition = new Partition();
  StorageDescriptor sd = new StorageDescriptor();
  SerDeInfo info = new SerDeInfo();
  info.setParameters(new HashMap<String, String>());
  sd.setSerdeInfo(info);
  partition.setSd(sd);
  partition.setParameters(new HashMap<String, String>());
  return partition;
}
 

@Before
public void init() {
  partition = new Partition();
  partition.setDbName("database");
  partition.setTableName("table");
  partition.setValues(ImmutableList.of("part"));

  Map<String, List<PrivilegeGrantInfo>> userPrivileges = new HashMap<>();
  userPrivileges.put("read", ImmutableList.of(new PrivilegeGrantInfo()));
  PrincipalPrivilegeSet privileges = new PrincipalPrivilegeSet();
  privileges.setUserPrivileges(userPrivileges);
  partition.setPrivileges(privileges);

  StorageDescriptor storageDescriptor = new StorageDescriptor();
  storageDescriptor.setCols(Arrays.asList(new FieldSchema("a", "int", null)));
  storageDescriptor.setInputFormat("input_format");
  storageDescriptor.setOutputFormat("output_format");
  storageDescriptor.setSerdeInfo(new SerDeInfo("serde", "lib", new HashMap<String, String>()));
  storageDescriptor.setSkewedInfo(new SkewedInfo());
  storageDescriptor.setParameters(new HashMap<String, String>());
  storageDescriptor.setLocation("database/table/part/");
  partition.setSd(storageDescriptor);

  Map<String, String> parameters = new HashMap<>();
  parameters.put("com.company.parameter", "abc");
  partition.setParameters(parameters);
}
 

private void add_partition(HiveMetaStoreClient client, Table table,
	      List<String> vals, String location) throws InvalidObjectException,
	        AlreadyExistsException, MetaException, TException {

    Partition part = new Partition();
    part.setDbName(table.getDbName());
    part.setTableName(table.getTableName());
    part.setValues(vals);
    part.setParameters(new HashMap<String, String>());
    part.setSd(table.getSd());
    part.getSd().setSerdeInfo(table.getSd().getSerdeInfo());
    part.getSd().setLocation(table.getSd().getLocation() + location);

    client.add_partition(part);
}
 

private static Partition newPartition(String databaseName, String tableName, String location) {
  Partition partition = new Partition();

  partition.setDbName(databaseName);
  partition.setTableName(tableName);
  partition.setParameters(new HashMap<String, String>());
  partition.setValues(Arrays.asList("01"));

  StorageDescriptor sd = new StorageDescriptor();
  sd.setLocation(location);
  partition.setSd(sd);

  return partition;
}
 

private Partition newPartition(
    String datbase,
    String table,
    StorageDescriptor tableStorageDescriptor,
    List<String> values,
    File location,
    String sourceTable,
    String sourceLocation,
    boolean addChecksum) {
  Partition partition = new Partition();
  partition.setDbName(datbase);
  partition.setTableName(table);
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(tableStorageDescriptor));
  partition.getSd().setLocation(location.toURI().toString());
  partition.setParameters(new HashMap<String, String>());
  if (sourceTable != null) {
    partition.getParameters().put(CircusTrainTableParameter.SOURCE_TABLE.parameterName(), sourceTable);
  }
  if (sourceLocation != null) {
    partition.getParameters().put(CircusTrainTableParameter.SOURCE_LOCATION.parameterName(), sourceLocation);
  }
  if (addChecksum) {
    partition.getParameters().put(CircusTrainTableParameter.PARTITION_CHECKSUM.parameterName(), location.getName());
  }
  return partition;
}
 

public static Partition newViewPartition(Table hiveView, List<String> values) {
  Partition partition = new Partition();
  partition.setDbName(hiveView.getDbName());
  partition.setTableName(hiveView.getTableName());
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(hiveView.getSd()));
  partition.getSd().setLocation(null);
  return partition;
}
 

public static Partition newTablePartition(Table hiveTable, List<String> values, URI location) {
  Partition partition = new Partition();
  partition.setDbName(hiveTable.getDbName());
  partition.setTableName(hiveTable.getTableName());
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(hiveTable.getSd()));
  partition.getSd().setLocation(location.toString());
  return partition;
}
 

public static Partition newPartition(Table table, String... partitionValues) {
  Partition partition = new Partition();
  partition.setTableName(table.getTableName());
  partition.setDbName(table.getDbName());
  partition.setValues(Arrays.asList(partitionValues));
  partition.setSd(table.getSd());
  partition.setParameters(new HashMap<String, String>());
  return partition;
}
 

private Partition newHivePartition(
    String tableName,
    List<String> partitionValues,
    String location,
    Map<String, String> parameters) {
  Partition partition = new Partition();
  partition.setDbName(databaseName);
  partition.setTableName(tableName);
  partition.setValues(partitionValues);
  parameters.forEach((key, value) -> partition.putToParameters(key, value));
  partition.putToParameters(DATA_HIGHWAY_VERSION, DataHighwayVersion.VERSION);
  partition.putToParameters(DATA_HIGHWAY_LAST_REVISION, ISO_OFFSET_DATE_TIME.withZone(UTC).format(clock.instant()));
  partition.setSd(AvroStorageDescriptorFactory.create(location));
  return partition;
}
 

private Partition newPartition(String... values) {
  Partition partition = new Partition();
  partition.setDbName(DB_NAME);
  partition.setTableName(TABLE_NAME);
  StorageDescriptor sd = new StorageDescriptor();
  sd.setLocation(new Path(tableLocation, partitionName(values)).toUri().toString());
  sd.setCols(FIELDS);
  partition.setSd(sd);
  HashMap<String, String> parameters = new HashMap<>();
  parameters.put(StatsSetupConst.ROW_COUNT, "1");
  partition.setParameters(parameters);
  partition.setValues(Arrays.asList(values));
  return partition;
}
 

private List<Partition> createPartitions(int count) {
  List<Partition> partitions = new ArrayList<>();
  for (int i=0; i < count; i++) {
    Partition partition = new Partition();
    partition.setSd(new StorageDescriptor());
    partitions.add(partition);
  }
  return partitions;
}
 

private List<Partition> createPartitions(int count) {
  List<Partition> partitions = new ArrayList<>();
  for (int i = 1; i < count + 1; i++) {
    Partition partition = new Partition();
    partition.setSd(new StorageDescriptor());
    partition.getSd().setLocation(PARTITION_LOCATION + i);
    partitions.add(partition);
  }
  return partitions;
}
 

private Partition newPartition(
    String table,
    StorageDescriptor tableStorageDescriptor,
    List<String> values,
    File location) {
  Partition partition = new Partition();
  partition.setDbName(DATABASE);
  partition.setTableName(table);
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(tableStorageDescriptor));
  partition.getSd().setLocation(location.toURI().toString());
  return partition;
}
 

@Test
public void dropPartition() throws Exception {
  HiveServer2Core server = setupServer();
  String tableName = "my_table";
  HiveMetaStoreClient client = server.getCore().newClient();

  try {
    Table table = createPartitionedTable(DATABASE, tableName, server);

    Partition partition = new Partition();
    partition.setDbName(DATABASE);
    partition.setTableName(tableName);
    partition.setValues(Arrays.asList("1"));
    partition.setSd(new StorageDescriptor(table.getSd()));
    partition.getSd().setLocation(
        String.format("file:%s/%s/%s/partcol=1", server.getCore().tempDir(), DATABASE, tableName));
    client.add_partition(partition);

    try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl());
        Statement statement = connection.createStatement()) {
      String dropPartitionHql = String.format("ALTER TABLE %s.%s DROP PARTITION (partcol=1)", DATABASE, tableName);
      statement.execute(dropPartitionHql);
    }

    List<Partition> partitions = client.listPartitions(DATABASE, tableName, (short) -1);
    assertThat(partitions.size(), is(0));
  } finally {
    client.close();
  }
  server.shutdown();
}
 

/**
 * Creates a Hive partition instance.
 */
public static Partition createHivePartition(String dbName, String tableName, List<String> values,
		StorageDescriptor sd, Map<String, String> parameters) {
	Partition partition = new Partition();
	partition.setDbName(dbName);
	partition.setTableName(tableName);
	partition.setValues(values);
	partition.setParameters(parameters);
	partition.setSd(sd);
	int currentTime = (int) (System.currentTimeMillis() / 1000);
	partition.setCreateTime(currentTime);
	partition.setLastAccessTime(currentTime);
	return partition;
}
 

/**
 * {@inheritDoc}
 */
@Override
public Partition metacatToHivePartition(final PartitionDto partitionDto, @Nullable final TableDto tableDto) {
    final Partition result = new Partition();

    final QualifiedName name = partitionDto.getName();
    List<String> values = Lists.newArrayListWithCapacity(16);
    String databaseName = null;
    String tableName = null;
    if (name != null) {
        if (name.getPartitionName() != null) {
            //
            // Unescape the partition name to get the right partition values.
            // Partition name always are escaped where as the parition values are not.
            //
            values = getPartValsFromName(tableDto, name.getPartitionName());
        }

        if (name.getDatabaseName() != null) {
            databaseName = name.getDatabaseName();
        }

        if (name.getTableName() != null) {
            tableName = name.getTableName();
        }
    }
    result.setValues(values);
    result.setDbName(databaseName);
    result.setTableName(tableName);

    Map<String, String> metadata = partitionDto.getMetadata();
    if (metadata == null) {
        metadata = Maps.newHashMap();
    }
    result.setParameters(metadata);

    result.setSd(fromStorageDto(partitionDto.getSerde(), tableName));
    final StorageDescriptor sd = result.getSd();
    if (tableDto != null) {
        if (sd.getSerdeInfo() != null && tableDto.getSerde() != null && Strings.isNullOrEmpty(
            sd.getSerdeInfo().getSerializationLib())) {
            sd.getSerdeInfo().setSerializationLib(tableDto.getSerde().getSerializationLib());
        }

        final List<FieldDto> fields = tableDto.getFields();
        if (fields == null) {
            sd.setCols(Collections.emptyList());
        } else {
            sd.setCols(fields.stream()
                .filter(field -> !field.isPartition_key())
                .map(this::metacatToHiveField)
                .collect(Collectors.toList()));
        }
    }

    final AuditDto auditDto = partitionDto.getAudit();
    if (auditDto != null) {
        if (auditDto.getCreatedDate() != null) {
            result.setCreateTime(dateToEpochSeconds(auditDto.getCreatedDate()));
        }
        if (auditDto.getLastModifiedDate() != null) {
            result.setLastAccessTime(dateToEpochSeconds(auditDto.getLastModifiedDate()));
        }
    }

    return result;
}
 

public static Partition convertPartition(com.amazonaws.services.glue.model.Partition src) {
 Partition tgt = new Partition();
 Date createTime = src.getCreationTime();
 if (createTime != null) {
  tgt.setCreateTime((int) (createTime.getTime() / 1000)); 
  tgt.setCreateTimeIsSet(true);
 } else {
  tgt.setCreateTimeIsSet(false);
 }
 String dbName = src.getDatabaseName();
 if (dbName != null) {
  tgt.setDbName(dbName);
  tgt.setDbNameIsSet(true);
 } else {
  tgt.setDbNameIsSet(false);
 }
 Date lastAccessTime = src.getLastAccessTime();
 if (lastAccessTime != null) {
  tgt.setLastAccessTime((int) (lastAccessTime.getTime() / 1000));
  tgt.setLastAccessTimeIsSet(true);
 } else {
  tgt.setLastAccessTimeIsSet(false);
 }
 Map<String, String> params = src.getParameters();
 
 // A null parameter map causes Hive to throw a NPE
 // so ensure we do not return a Partition object with a null parameter map.
 if (params == null) {
   params = Maps.newHashMap();
 }
 
 tgt.setParameters(params);
 tgt.setParametersIsSet(true);
 
 String tableName = src.getTableName();
 if (tableName != null) {
  tgt.setTableName(tableName);
  tgt.setTableNameIsSet(true);
 } else {
  tgt.setTableNameIsSet(false);
 }
 
 List<String> values = src.getValues();
 if (values != null) {
  tgt.setValues(values);
  tgt.setValuesIsSet(true);
 } else {
  tgt.setValuesIsSet(false);
 }
 
 com.amazonaws.services.glue.model.StorageDescriptor sd = src.getStorageDescriptor();
 if (sd != null) {
  StorageDescriptor hiveSd = convertStorageDescriptor(sd);
  tgt.setSd(hiveSd);
  tgt.setSdIsSet(true);
 } else {
  tgt.setSdIsSet(false);
 }
 
 return tgt;
}