Java Code Examples for org.apache.hadoop.hive.metastore.IMetaStoreClient

The following examples show how to use org.apache.hadoop.hive.metastore.IMetaStoreClient. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private IMetaStoreClient createMetaStoreClient() throws MetaException {
  HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() {
    @Override
    public HiveMetaHook getHook(Table tbl) throws MetaException {
      if (tbl == null) {
        return null;
      }

      try {
        HiveStorageHandler storageHandler =
            HiveUtils.getStorageHandler(hiveConf, tbl.getParameters().get(META_TABLE_STORAGE));
        return storageHandler == null ? null : storageHandler.getMetaHook();
      } catch (HiveException e) {
        LOG.error(e.toString());
        throw new MetaException("Failed to get storage handler: " + e);
      }
    }
  };

  return RetryingMetaStoreClient.getProxy(hiveConf, hookLoader, HiveMetaStoreClient.class.getName());
}
 
Example 2
@Override
public void clean() throws IOException {

  // Possible empty directories to clean for this partition (version)
  Set<Path> possiblyEmptyDirectories = new HashSet<>();

  try (AutoReturnableObject<IMetaStoreClient> client = cleanableHiveDataset.getClientPool().getClient()) {
    Partition partition = hiveDatasetVersion.getPartition();
    try {
      if (!cleanableHiveDataset.isSimulate()) {
        client.get().dropPartition(partition.getTable().getDbName(), partition.getTable().getTableName(), partition.getValues(), false);
        log.info("Successfully dropped partition " + partition.getCompleteName());
      } else {
        log.info("Simulating drop partition " + partition.getCompleteName());
      }
      if (cleanableHiveDataset.isShouldDeleteData()) {
        cleanableHiveDataset.getFsCleanableHelper().clean(hiveDatasetVersion, possiblyEmptyDirectories);
      }
    } catch (TException | IOException e) {
      log.warn(String.format("Failed to completely delete partition %s.", partition.getCompleteName()), e);
      throw new IOException(e);
    }
  }
  cleanableHiveDataset.getFsCleanableHelper().cleanEmptyDirectories(possiblyEmptyDirectories, cleanableHiveDataset);
}
 
Example 3
Source Project: flink   Source File: HiveShimV1.java    License: Apache License 2.0 6 votes vote down vote up
@Override
// 1.x client doesn't support filtering tables by type, so here we need to get all tables and filter by ourselves
public List<String> getViews(IMetaStoreClient client, String databaseName) throws UnknownDBException, TException {
	// We don't have to use reflection here because client.getAllTables(String) is supposed to be there for
	// all versions.
	List<String> tableNames = client.getAllTables(databaseName);
	List<String> views = new ArrayList<>();
	for (String name : tableNames) {
		Table table = client.getTable(databaseName, name);
		String viewDef = table.getViewOriginalText();
		if (viewDef != null && !viewDef.isEmpty()) {
			views.add(table.getTableName());
		}
	}
	return views;
}
 
Example 4
Source Project: flink   Source File: HiveShimV1.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Function getFunction(IMetaStoreClient client, String dbName, String functionName) throws NoSuchObjectException, TException {
	try {
		// hive-1.x doesn't throw NoSuchObjectException if function doesn't exist, instead it throws a MetaException
		return client.getFunction(dbName, functionName);
	} catch (MetaException e) {
		// need to check the cause and message of this MetaException to decide whether it should actually be a NoSuchObjectException
		if (e.getCause() instanceof NoSuchObjectException) {
			throw (NoSuchObjectException) e.getCause();
		}
		if (e.getMessage().startsWith(NoSuchObjectException.class.getSimpleName())) {
			throw new NoSuchObjectException(e.getMessage());
		}
		throw e;
	}
}
 
Example 5
Source Project: flink   Source File: HiveShimV2.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<String> getViews(IMetaStoreClient client, String databaseName) throws UnknownDBException, TException {
	try {
		Method method = client.getClass().getMethod("getTables", String.class, String.class, TableType.class);
		return (List<String>) method.invoke(client, databaseName, null, TableType.VIRTUAL_VIEW);
	} catch (InvocationTargetException ite) {
		Throwable targetEx = ite.getTargetException();
		if (targetEx instanceof TException) {
			throw (TException) targetEx;
		} else {
			throw new CatalogException(String.format("Failed to get views for %s", databaseName), targetEx);
		}
	} catch (NoSuchMethodException | IllegalAccessException e) {
		throw new CatalogException(String.format("Failed to get views for %s", databaseName), e);
	}
}
 
Example 6
Source Project: pxf   Source File: HiveClientWrapper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Initializes the IMetaStoreClient
 * Uses classpath configuration files to locate the MetaStore
 *
 * @return initialized client
 */
public IMetaStoreClient initHiveClient(RequestContext context, Configuration configuration) {
    HiveConf hiveConf = getHiveConf(configuration);
    try {
        if (Utilities.isSecurityEnabled(configuration)) {
            UserGroupInformation loginUser = SecureLogin.getInstance().getLoginUser(context, configuration);
            LOG.debug("initialize HiveMetaStoreClient as login user '{}'", loginUser.getUserName());
            // wrap in doAs for Kerberos to propagate kerberos tokens from login Subject
            return loginUser.
                    doAs((PrivilegedExceptionAction<IMetaStoreClient>) () -> hiveClientFactory.initHiveClient(hiveConf));
        } else {
            return hiveClientFactory.initHiveClient(hiveConf);
        }
    } catch (MetaException | InterruptedException | IOException e) {
        throw new RuntimeException("Failed connecting to Hive MetaStore service: " + e.getMessage(), e);
    }
}
 
Example 7
Source Project: griffin   Source File: HiveMetaStoreProxy.java    License: Apache License 2.0 6 votes vote down vote up
@Bean
public IMetaStoreClient initHiveMetastoreClient() {
    HiveConf hiveConf = new HiveConf();
    hiveConf.set("hive.metastore.local", "false");
    hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES,
        3);
    hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, uris);
    hiveConf.setIntVar(HiveConf.ConfVars.HMSHANDLERATTEMPTS, attempts);
    hiveConf.setVar(HiveConf.ConfVars.HMSHANDLERINTERVAL, interval);
    try {
        client = HiveMetaStoreClient.newSynchronizedClient(new HiveMetaStoreClient(hiveConf));
    } catch (Exception e) {
        LOGGER.error("Failed to connect hive metastore. {}", e);
    }
    return client;
}
 
Example 8
Source Project: beam   Source File: HCatalogUtils.java    License: Apache License 2.0 6 votes vote down vote up
private static long getFileSizeForPartition(Read readRequest, Partition partitionToRead)
    throws Exception {
  IMetaStoreClient client = null;
  try {
    HiveConf hiveConf = HCatalogUtils.createHiveConf(readRequest);
    client = HCatalogUtils.createMetaStoreClient(hiveConf);
    List<org.apache.hadoop.hive.ql.metadata.Partition> p = new ArrayList<>();
    Table table = HCatUtil.getTable(client, readRequest.getDatabase(), readRequest.getTable());
    final org.apache.hadoop.hive.ql.metadata.Partition partition =
        new org.apache.hadoop.hive.ql.metadata.Partition(table, partitionToRead);
    p.add(partition);
    final List<Long> fileSizeForPartitions = StatsUtils.getFileSizeForPartitions(hiveConf, p);
    return fileSizeForPartitions.get(0);
  } finally {
    // IMetaStoreClient is not AutoCloseable, closing it manually
    if (client != null) {
      client.close();
    }
  }
}
 
Example 9
Source Project: beam   Source File: HCatalogIO.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns the size of the table in bytes, does not take into consideration filter/partition
 * details passed, if any.
 */
@Override
public long getEstimatedSizeBytes(PipelineOptions pipelineOptions) throws Exception {
  IMetaStoreClient client = null;
  try {
    HiveConf hiveConf = HCatalogUtils.createHiveConf(spec);
    client = HCatalogUtils.createMetaStoreClient(hiveConf);
    Table table = HCatUtil.getTable(client, spec.getDatabase(), spec.getTable());
    return StatsUtils.getFileSizeForTable(hiveConf, table);
  } finally {
    // IMetaStoreClient is not AutoCloseable, closing it manually
    if (client != null) {
      client.close();
    }
  }
}
 
Example 10
/**
 * @return true if the db is successfully created.
 *         false if the db already exists.
 * @throws IOException
 */
private boolean createDbIfNotExists(IMetaStoreClient client, String dbName) throws IOException {
  boolean retVal;
  if (this.optimizedChecks) {
    try {
      retVal = this.tableAndDbExistenceCache.get(dbName, new Callable<Boolean>() {
        @Override
        public Boolean call() throws Exception {
          return ensureHiveDbExistence(dbName, client);
        }
      });
    } catch (ExecutionException ee) {
      throw new IOException("Database existence checking throwing execution exception.");
    }
    return retVal;
  } else {
    return this.ensureHiveDbExistence(dbName, client);
  }
}
 
Example 11
Source Project: flink   Source File: HiveShimV100.java    License: Apache License 2.0 6 votes vote down vote up
@Override
// 1.x client doesn't support filtering tables by type, so here we need to get all tables and filter by ourselves
public List<String> getViews(IMetaStoreClient client, String databaseName) throws UnknownDBException, TException {
	// We don't have to use reflection here because client.getAllTables(String) is supposed to be there for
	// all versions.
	List<String> tableNames = client.getAllTables(databaseName);
	List<String> views = new ArrayList<>();
	for (String name : tableNames) {
		Table table = client.getTable(databaseName, name);
		String viewDef = table.getViewOriginalText();
		if (viewDef != null && !viewDef.isEmpty()) {
			views.add(table.getTableName());
		}
	}
	return views;
}
 
Example 12
@Override
protected void registerPath(HiveSpec spec) throws IOException {
  try (Timer.Context context = this.metricContext.timer(PATH_REGISTER_TIMER).time();
      AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) {
    Table table = HiveMetaStoreUtils.getTable(spec.getTable());

    createDbIfNotExists(client.get(), table.getDbName());
    createOrAlterTable(client.get(), table, spec);

    Optional<HivePartition> partition = spec.getPartition();
    if (partition.isPresent()) {
      addOrAlterPartition(client.get(), table, partition.get());
    }
    HiveMetaStoreEventHelper.submitSuccessfulPathRegistration(eventSubmitter, spec);
  } catch (TException e) {
    HiveMetaStoreEventHelper.submitFailedPathRegistration(eventSubmitter, spec, e);
    throw new IOException(e);
  }
}
 
Example 13
Source Project: flink   Source File: HiveShimV230.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<String> getViews(IMetaStoreClient client, String databaseName) throws UnknownDBException, TException {
	try {
		Method method = client.getClass().getMethod("getTables", String.class, String.class, TableType.class);
		return (List<String>) method.invoke(client, databaseName, null, TableType.VIRTUAL_VIEW);
	} catch (InvocationTargetException ite) {
		Throwable targetEx = ite.getTargetException();
		if (targetEx instanceof TException) {
			throw (TException) targetEx;
		} else {
			throw new CatalogException(String.format("Failed to get views for %s", databaseName), targetEx);
		}
	} catch (NoSuchMethodException | IllegalAccessException e) {
		throw new CatalogException(String.format("Failed to get views for %s", databaseName), e);
	}
}
 
Example 14
Source Project: incubator-gobblin   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get a list of {@link Partition}s for the <code>table</code> that matches an optional <code>filter</code>
 *
 * @param client an {@link IMetaStoreClient} for the correct metastore.
 * @param table the {@link Table} for which we should get partitions.
 * @param filter an optional filter for partitions as would be used in Hive. Can only filter on String columns.
 *               (e.g. "part = \"part1\"" or "date > \"2015\"".
 * @return a list of {@link Partition}s
 */
public static List<Partition> getPartitions(IMetaStoreClient client, Table table,
    Optional<String> filter, Optional<? extends HivePartitionExtendedFilter> hivePartitionExtendedFilterOptional)
    throws IOException {
  try {
    List<Partition> partitions = Lists.newArrayList();
    List<org.apache.hadoop.hive.metastore.api.Partition> partitionsList = filter.isPresent()
        ? client.listPartitionsByFilter(table.getDbName(), table.getTableName(), filter.get(), (short) -1)
        : client.listPartitions(table.getDbName(), table.getTableName(), (short) -1);
    for (org.apache.hadoop.hive.metastore.api.Partition p : partitionsList) {
      if (!hivePartitionExtendedFilterOptional.isPresent() ||
          hivePartitionExtendedFilterOptional.get().accept(p)) {
        Partition partition = new Partition(table, p);
        partitions.add(partition);
      }
    }
    return partitions;
  } catch (TException | HiveException te) {
    throw new IOException("Hive Error", te);
  }
}
 
Example 15
Source Project: incubator-gobblin   Source File: HiveMaterializerSource.java    License: Apache License 2.0 6 votes vote down vote up
private HiveDataset getHiveDataset(String tableString, FileSystem fs, State state) throws IOException {
  try {
    HiveMetastoreClientPool pool = HiveMetastoreClientPool.get(state.getProperties(),
        Optional.fromNullable(state.getProp(HIVE_METASTORE_URI_KEY)));

    List<String> tokens = Splitter.on(".").splitToList(tableString);
    DbAndTable sourceDbAndTable = new DbAndTable(tokens.get(0), tokens.get(1));

    try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) {
      Table sourceTable = new Table(client.get().getTable(sourceDbAndTable.getDb(), sourceDbAndTable.getTable()));
      return new HiveDataset(fs, pool, sourceTable, ConfigUtils.propertiesToConfig(state.getProperties()));
    }
  } catch (TException exc) {
    throw new RuntimeException(exc);
  }
}
 
Example 16
Source Project: flink   Source File: HiveShimV210.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void createTableWithConstraints(
		IMetaStoreClient client,
		Table table,
		Configuration conf,
		UniqueConstraint pk,
		List<Byte> pkTraits,
		List<String> notNullCols,
		List<Byte> nnTraits) {
	if (!notNullCols.isEmpty()) {
		throw new UnsupportedOperationException("NOT NULL constraints not supported until 3.0.0");
	}
	try {
		List<Object> hivePKs = createHivePKs(table, pk, pkTraits);
		// createTableWithConstraints takes PK and FK lists
		HiveReflectionUtils.invokeMethod(
				client.getClass(),
				client,
				"createTableWithConstraints",
				new Class[]{Table.class, List.class, List.class},
				new Object[]{table, hivePKs, Collections.emptyList()});
	} catch (Exception e) {
		throw new CatalogException("Failed to create Hive table with constraints", e);
	}
}
 
Example 17
Source Project: flink   Source File: HiveShimV310.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Set<String> getNotNullColumns(IMetaStoreClient client, Configuration conf, String dbName, String tableName) {
	try {
		String hiveDefaultCatalog = getHMSDefaultCatalog(conf);
		Class requestClz = Class.forName("org.apache.hadoop.hive.metastore.api.NotNullConstraintsRequest");
		Object request = requestClz.getDeclaredConstructor(String.class, String.class, String.class)
				.newInstance(hiveDefaultCatalog, dbName, tableName);
		List<?> constraints = (List<?>) HiveReflectionUtils.invokeMethod(client.getClass(), client,
				"getNotNullConstraints", new Class[]{requestClz}, new Object[]{request});
		Class constraintClz = Class.forName("org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint");
		Method colNameMethod = constraintClz.getDeclaredMethod("getColumn_name");
		Method isRelyMethod = constraintClz.getDeclaredMethod("isRely_cstr");
		Set<String> res = new HashSet<>();
		for (Object constraint : constraints) {
			if ((boolean) isRelyMethod.invoke(constraint)) {
				res.add((String) colNameMethod.invoke(constraint));
			}
		}
		return res;
	} catch (Exception e) {
		throw new CatalogException("Failed to get NOT NULL constraints", e);
	}
}
 
Example 18
Source Project: flink   Source File: HiveShimV310.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void createTableWithConstraints(
		IMetaStoreClient client,
		Table table,
		Configuration conf,
		UniqueConstraint pk,
		List<Byte> pkTraits,
		List<String> notNullCols,
		List<Byte> nnTraits) {
	try {
		List<Object> hivePKs = createHivePKs(table, pk, pkTraits);
		List<Object> hiveNNs = createHiveNNs(table, conf, notNullCols, nnTraits);
		// createTableWithConstraints takes PK, FK, UNIQUE, NN, DEFAULT, CHECK lists
		HiveReflectionUtils.invokeMethod(
				client.getClass(),
				client,
				"createTableWithConstraints",
				new Class[]{Table.class, List.class, List.class, List.class, List.class, List.class, List.class},
				new Object[]{table, hivePKs, Collections.emptyList(), Collections.emptyList(), hiveNNs,
						Collections.emptyList(), Collections.emptyList()});
	} catch (Exception e) {
		throw new CatalogException("Failed to create Hive table with constraints", e);
	}
}
 
Example 19
public HiveMetadataForCompactionExtractor(WorkUnitState state, FileSystem fs) throws IOException, TException {
  super(state);

  if (Boolean.valueOf(state.getPropAsBoolean(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY))) {
    log.info("Ignoring Watermark workunit for {}", state.getProp(ConfigurationKeys.DATASET_URN_KEY));
    return;
  }

  try (AutoReturnableObject<IMetaStoreClient> client = this.pool.getClient()) {
    Table table = client.get().getTable(this.dbName, this.tableName);

    String primaryKeyString = table.getParameters().get(state.getProp(COMPACTION_PRIMARY_KEY));
    List<String> primaryKeyList = Splitter.on(',').omitEmptyStrings().trimResults().splitToList(primaryKeyString);

    String deltaString = table.getParameters().get(state.getProp(COMPACTION_DELTA));
    List<String> deltaList = Splitter.on(',').omitEmptyStrings().trimResults().splitToList(deltaString);

    Path dataFilesPath = new Path(table.getSd().getLocation());

    compactionEntity = new MRCompactionEntity(primaryKeyList, deltaList, dataFilesPath, state.getProperties());
  }
}
 
Example 20
Source Project: incubator-gobblin   Source File: TokenUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get Hadoop tokens (tokens for job history server, job tracker, hive and HDFS) using Kerberos keytab,
 * on behalf on a proxy user, embed tokens into a {@link UserGroupInformation} as returned result, persist in-memory
 * credentials if tokenFile specified
 *
 * Note that when a super-user is fetching tokens for other users,
 * {@link #fetchHcatToken(String, HiveConf, String, IMetaStoreClient)} getDelegationToken} explicitly
 * contains a string parameter indicating proxy user, while other hadoop services require impersonation first.
 *
 * @param state A {@link State} object that should contain properties.
 * @param tokenFile If present, the file will store materialized credentials.
 * @param ugi The {@link UserGroupInformation} that used to impersonate into the proxy user by a "doAs block".
 * @param targetUser The user to be impersonated as, for fetching hadoop tokens.
 * @return A {@link UserGroupInformation} containing negotiated credentials.
 */
public static UserGroupInformation getHadoopAndHiveTokensForProxyUser(final State state, Optional<File> tokenFile,
    UserGroupInformation ugi, IMetaStoreClient client, String targetUser) throws IOException, InterruptedException {
  final Credentials cred = new Credentials();
  ugi.doAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run() throws Exception {
      getHadoopTokens(state, Optional.absent(), cred);
      return null;
    }
  });

  ugi.getCredentials().addAll(cred);
  // Will add hive tokens into ugi in this method.
  getHiveToken(state, client, cred, targetUser, ugi);

  if (tokenFile.isPresent()) {
    persistTokens(cred, tokenFile.get());
  }
  // at this point, tokens in ugi can be more than that in Credential object,
  // since hive token is not put in Credential object.
  return ugi;
}
 
Example 21
@Test
public void testCreate() throws TException {
  HiveConf hiveConf = new HiveConf();
  HiveMetaStoreClientFactory factory = new HiveMetaStoreClientFactory(hiveConf);

  // Since we havE a specified hive-site in the classpath, so have to null it out here to proceed the test
  // The original value it will get if no local hive-site is placed, will be an empty string. 
  hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "");
  hiveConf.set(HIVE_METASTORE_TOKEN_SIGNATURE, "");
  IMetaStoreClient msc = factory.create();

  String dbName = "test_db";
  String description = "test database";
  String location = "file:/tmp/" + dbName;
  Database db = new Database(dbName, description, location, null);

  msc.dropDatabase(dbName, true, true);
  msc.createDatabase(db);
  db = msc.getDatabase(dbName);
  Assert.assertEquals(db.getName(), dbName);
  Assert.assertEquals(db.getDescription(), description);
  Assert.assertEquals(db.getLocationUri(), location);
}
 
Example 22
private static List<Partition> getPartitions(String completeTableName) {
  List<String> tableList = At_SPLITTER.splitToList(completeTableName);
  if (tableList.size() != 2) {
    log.warn("Invalid table name " + completeTableName);
    return Collections.EMPTY_LIST;
  }
  try (AutoReturnableObject<IMetaStoreClient> client = ComplianceRetentionJob.pool.getClient()) {
    Table table = client.get().getTable(tableList.get(0), tableList.get(1));
    HiveDataset dataset = new HiveDataset(FileSystem.newInstance(new Configuration()), ComplianceRetentionJob.pool,
        new org.apache.hadoop.hive.ql.metadata.Table(table), new Properties());
    return dataset.getPartitionsFromDataset();
  } catch (IOException | TException e) {
    log.warn("Unable to get Partitions for table " + completeTableName + " " + e.getMessage());
  }
  return Collections.EMPTY_LIST;
}
 
Example 23
Source Project: incubator-gobblin   Source File: HiveConvertPublisher.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public Optional<Partition> getPartitionObject(String completePartitionName) {
  try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) {
    List<String> partitionList = At_SPLITTER.splitToList(completePartitionName);
    if (partitionList.size() != 3) {
      log.warn("Invalid partition name " + completePartitionName);
      return Optional.<Partition>absent();
    }
    Partition sourcePartition =
        client.get().getPartition(partitionList.get(0), partitionList.get(1), partitionList.get(2));
    return Optional.fromNullable(sourcePartition);
  } catch (IOException | TException e) {
    log.warn("Unable to get partition object from metastore for partition " + completePartitionName);
  }
  return Optional.<Partition>absent();
}
 
Example 24
Source Project: data-highway   Source File: LoadingBayApp.java    License: Apache License 2.0 5 votes vote down vote up
@Bean
public IMetaStoreClient metaStoreClient(
    @Value("${hive.metastore.uris}") String hiveMetaStoreUris,
    @Value("${hive.thrift.failureRetries:3}") int failureRetries,
    @Value("${hive.thrift.connectRetryDelay:10}") long connectRetryDelay) {
  HiveConf hiveConf = new HiveConfFactory().newInstance(hiveMetaStoreUris);
  hiveConf.setIntVar(METASTORETHRIFTFAILURERETRIES, failureRetries);
  hiveConf.setTimeVar(METASTORE_CLIENT_CONNECT_RETRY_DELAY, connectRetryDelay, TimeUnit.SECONDS);
  return new MetaStoreClientFactory().newInstance(hiveConf);
}
 
Example 25
Source Project: data-highway   Source File: LoadingBayApp.java    License: Apache License 2.0 5 votes vote down vote up
@Bean
public HiveTableManager hiveTableManager(
    IMetaStoreClient metaStoreClient,
    HiveTableStrategy hiveTableStrategy,
    LocationResolver locationResolver,
    @Value("${hive.database}") String databaseName) {
  return new HiveTableManager(metaStoreClient, hiveTableStrategy, locationResolver, databaseName);
}
 
Example 26
Source Project: data-highway   Source File: LoadingBayApp.java    License: Apache License 2.0 5 votes vote down vote up
@Bean
public HivePartitionManager hivePartitionManager(
    IMetaStoreClient metaStoreClient,
    LocationResolver locationResolver,
    @Value("${hive.database}") String database,
    Clock clock) {
  return new HivePartitionManager(metaStoreClient, locationResolver, database, clock);
}
 
Example 27
Source Project: data-highway   Source File: MetaStoreClientFactory.java    License: Apache License 2.0 5 votes vote down vote up
public IMetaStoreClient newInstance(HiveConf conf) throws MetaStoreException {
  try {
    return synchronizedClient(retryingClient(conf));
  } catch (MetaException e) {
    throw new MetaStoreException(e);
  }
}
 
Example 28
Source Project: data-highway   Source File: MetaStoreClientFactory.java    License: Apache License 2.0 5 votes vote down vote up
private IMetaStoreClient synchronizedClient(IMetaStoreClient delegate) {
  return (IMetaStoreClient) Proxy.newProxyInstance(getClass().getClassLoader(),
      new Class<?>[] { IMetaStoreClient.class }, (p, m, a) -> {
        synchronized (delegate) {
          return m.invoke(delegate, a);
        }
      });
}
 
Example 29
Source Project: incubator-gobblin   Source File: HiveSource.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<WorkUnit> getWorkunits(SourceState state) {
  try {
    this.beginGetWorkunitsTime = System.currentTimeMillis();

    initialize(state);

    EventSubmitter.submit(Optional.of(this.eventSubmitter), EventConstants.CONVERSION_FIND_HIVE_TABLES_EVENT);
    Iterator<HiveDataset> iterator = this.datasetFinder.getDatasetsIterator();
    boolean disableAvroCheck = state.getPropAsBoolean(DISABLE_AVRO_CHAECK, DEFAULT_DISABLE_AVRO_CHAECK);
    while (iterator.hasNext()) {
      HiveDataset hiveDataset = iterator.next();
      try (AutoReturnableObject<IMetaStoreClient> client = hiveDataset.getClientPool().getClient()) {

        log.debug(String.format("Processing dataset: %s", hiveDataset));

        // Create workunits for partitions
        if (hiveDataset.getTable().isPartitioned()
            && state.getPropAsBoolean(HIVE_SOURCE_CREATE_WORKUNITS_FOR_PARTITIONS,
            DEFAULT_HIVE_SOURCE_CREATE_WORKUNITS_FOR_PARTITIONS)) {
          createWorkunitsForPartitionedTable(hiveDataset, client, disableAvroCheck);
        } else {
          createWorkunitForNonPartitionedTable(hiveDataset, disableAvroCheck);
        }
      }
    }
  } catch (IOException e) {
    throw new RuntimeException(e);
  }

  int realWorkunits = this.workunits.size();

  this.watermarker.onGetWorkunitsEnd(this.workunits);

  log.info(String.format("Created %s real workunits and %s watermark workunits", realWorkunits,
      (this.workunits.size() - realWorkunits)));

  return this.workunits;
}
 
Example 30
@Override
public IMetaStoreClient createMetaStoreClient(
    HiveConf conf,
    HiveMetaHookLoader hookLoader
) throws MetaException {
  AWSCatalogMetastoreClient client = new AWSCatalogMetastoreClient(conf, hookLoader);
  return client;
}