org.apache.hadoop.hive.ql.hooks.Entity Java Examples

private static void addInputs(HiveMetaStoreBridge hiveBridge, HiveOperation op, SortedSet<ReadEntity> sortedInputs, StringBuilder buffer, final Map<ReadEntity, Referenceable> refs, final boolean ignoreHDFSPathsInQFName) throws HiveException {
    if (refs != null) {
        if (sortedInputs != null) {
            Set<String> dataSetsProcessed = new LinkedHashSet<>();
            for (Entity input : sortedInputs) {

                if (!dataSetsProcessed.contains(input.getName().toLowerCase())) {
                    //HiveOperation.QUERY type encompasses INSERT, INSERT_OVERWRITE, UPDATE, DELETE, PATH_WRITE operations
                    if (ignoreHDFSPathsInQFName &&
                        (Type.DFS_DIR.equals(input.getType()) || Type.LOCAL_DIR.equals(input.getType()))) {
                        LOG.debug("Skipping dfs dir input addition to process qualified name {} ", input.getName());
                    } else if (refs.containsKey(input)) {
                        if ( input.getType() == Type.PARTITION || input.getType() == Type.TABLE) {
                            final Date createTime = HiveMetaStoreBridge.getTableCreatedTime(hiveBridge.hiveClient.getTable(input.getTable().getDbName(), input.getTable().getTableName()));
                            addDataset(buffer, refs.get(input), createTime.getTime());
                        } else {
                            addDataset(buffer, refs.get(input));
                        }
                    }
                    dataSetsProcessed.add(input.getName().toLowerCase());
                }
            }

        }
    }
}
 

@Test
public void testTruncateTable() throws Exception {
    String tableName = createTable(false);
    String query = String.format("truncate table %s", tableName);
    runCommand(query);

    Set<WriteEntity> outputs = getOutputs(tableName, Entity.Type.TABLE);

    String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
    validateProcess(constructEvent(query, HiveOperation.TRUNCATETABLE, null, outputs));

    //Check lineage
    String datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName);
    JSONObject response = atlasClient.getInputGraph(datasetName);
    JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices");
    //Below should be assertTrue - Fix https://issues.apache.org/jira/browse/ATLAS-653
    Assert.assertFalse(vertices.has(tableId));
}
 

private List<AtlasObjectId> getHiveEntities() {
    List<AtlasObjectId> ret = new ArrayList<>();

    for (Entity entity : getOutputs()) {
        if (entity.getType() == DATABASE) {
            String        dbQName = getQualifiedName(entity.getDatabase());
            AtlasObjectId dbId    = new AtlasObjectId(HIVE_TYPE_DB, ATTRIBUTE_QUALIFIED_NAME, dbQName);

            context.removeFromKnownDatabase(dbQName);

            ret.add(dbId);
        } else if (entity.getType() == TABLE) {
            String        tblQName = getQualifiedName(entity.getTable());
            AtlasObjectId tblId    = new AtlasObjectId(HIVE_TYPE_TABLE, ATTRIBUTE_QUALIFIED_NAME, tblQName);

            context.removeFromKnownTable(tblQName);

            ret.add(tblId);
        }
    }

    return ret;
}
 

public List<AtlasObjectId> getHiveEntities() {
    List<AtlasObjectId> ret = new ArrayList<>();

    for (Entity entity : getOutputs()) {
        if (entity.getType() == Entity.Type.TABLE) {
            String        tblQName = getQualifiedName(entity.getTable());
            AtlasObjectId tblId    = new AtlasObjectId(HIVE_TYPE_TABLE, ATTRIBUTE_QUALIFIED_NAME, tblQName);

            context.removeFromKnownTable(tblQName);

            ret.add(tblId);
        }
    }

    return ret;
}
 

protected static boolean addQueryType(HiveOperation op, WriteEntity entity) {
    if (entity.getWriteType() != null && HiveOperation.QUERY.equals(op)) {
        switch (entity.getWriteType()) {
            case INSERT:
            case INSERT_OVERWRITE:
            case UPDATE:
            case DELETE:
                return true;
            case PATH_WRITE:
                //Add query type only for DFS paths and ignore local paths since they are not added as outputs
                if ( !Entity.Type.LOCAL_DIR.equals(entity.getType())) {
                    return true;
                }
                break;
            default:
        }
    }
    return false;
}
 

@Test
public void testInsertIntoLocalDir() throws Exception {
    String tableName       = createTable();
    String randomLocalPath = mkdir("hiverandom.tmp");
    String query           = "insert overwrite LOCAL DIRECTORY '" + randomLocalPath + "' select id, name from " + tableName;

    runCommand(query);

    HiveEventContext event = constructEvent(query,  HiveOperation.QUERY,
            getInputs(tableName, Entity.Type.TABLE), null);
    AtlasEntity hiveProcess = validateProcess(event);
    AtlasEntity hiveProcessExecution = validateProcessExecution(hiveProcess, event);
    AtlasObjectId process = toAtlasObjectId(hiveProcessExecution.getRelationshipAttribute(
            BaseHiveEvent.ATTRIBUTE_PROCESS));
    Assert.assertEquals(process.getGuid(), hiveProcess.getGuid());
    Assert.assertEquals(numberOfProcessExecutions(hiveProcess), 1);

    String tblId          = assertTableIsRegistered(DEFAULT_DB, tableName);

    AtlasEntity tblEntity = atlasClientV2.getEntityByGuid(tblId).getEntity();
    List ddlQueries       = (List) tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);

    Assert.assertNotNull(ddlQueries);
    Assert.assertEquals(ddlQueries.size(), 1);
}
 

@Test(enabled = false)
public void testInsertIntoTempTable() throws Exception {
    String tableName = createTable();
    String insertTableName = createTable(false, false, true);
    assertTableIsRegistered(DEFAULT_DB, tableName);
    assertTableIsNotRegistered(DEFAULT_DB, insertTableName, true);

    String query =
        "insert into " + insertTableName + " select id, name from " + tableName;

    runCommand(query);

    Set<ReadEntity> inputs = getInputs(tableName, Entity.Type.TABLE);
    Set<WriteEntity> outputs = getOutputs(insertTableName, Entity.Type.TABLE);
    outputs.iterator().next().setName(getQualifiedTblName(insertTableName + HiveMetaStoreBridge.TEMP_TABLE_PREFIX + SessionState.get().getSessionId()));
    outputs.iterator().next().setWriteType(WriteEntity.WriteType.INSERT);

    validateProcess(constructEvent(query,  HiveOperation.QUERY, inputs, outputs));

    assertTableIsRegistered(DEFAULT_DB, tableName);
    assertTableIsRegistered(DEFAULT_DB, insertTableName, null, true);
}
 

@Test
public void testCTAS() throws Exception {
    String tableName     = createTable();
    String ctasTableName = "table" + random();
    String query         = "create table " + ctasTableName + " as select * from " + tableName;

    runCommand(query);

    final Set<ReadEntity> readEntities = getInputs(tableName, Entity.Type.TABLE);
    final Set<WriteEntity> writeEntities = getOutputs(ctasTableName, Entity.Type.TABLE);

    HiveEventContext hiveEventContext = constructEvent(query, HiveOperation.CREATETABLE_AS_SELECT, readEntities,
            writeEntities);
    AtlasEntity processEntity1 = validateProcess(hiveEventContext);
    AtlasEntity processExecutionEntity1 = validateProcessExecution(processEntity1, hiveEventContext);
    AtlasObjectId process = toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
            BaseHiveEvent.ATTRIBUTE_PROCESS));
    Assert.assertEquals(process.getGuid(), processEntity1.getGuid());

    Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
    assertTableIsRegistered(DEFAULT_DB, ctasTableName);
}
 

@Test
public void testCreateView() throws Exception {
    String tableName = createTable();
    String viewName  = tableName();
    String query     = "create view " + viewName + " as select * from " + tableName;

    runCommand(query);

    HiveEventContext hiveEventContext = constructEvent(query, HiveOperation.CREATEVIEW, getInputs(tableName,
            Entity.Type.TABLE), getOutputs(viewName, Entity.Type.TABLE));
    AtlasEntity processEntity1 = validateProcess(hiveEventContext);
    AtlasEntity processExecutionEntity1 = validateProcessExecution(processEntity1, hiveEventContext);
    AtlasObjectId process1 = toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
            BaseHiveEvent.ATTRIBUTE_PROCESS));
    Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
    Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
    assertTableIsRegistered(DEFAULT_DB, viewName);

    String viewId          = assertTableIsRegistered(DEFAULT_DB, viewName);
    AtlasEntity viewEntity = atlasClientV2.getEntityByGuid(viewId).getEntity();
    List ddlQueries        = (List) viewEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);

    Assert.assertNotNull(ddlQueries);
    Assert.assertEquals(ddlQueries.size(), 1);
}
 

private Set<ReadEntity> getInputs(String inputName, Entity.Type entityType) throws HiveException {
    final ReadEntity entity = new ReadEntity();

    if ( Entity.Type.DFS_DIR.equals(entityType)) {
        entity.setName(lower(new Path(inputName).toString()));
        entity.setTyp(Entity.Type.DFS_DIR);
    } else {
        entity.setName(getQualifiedTblName(inputName));
        entity.setTyp(entityType);
    }

    if (entityType == Entity.Type.TABLE) {
        entity.setT(hiveMetaStoreBridge.hiveClient.getTable(DEFAULT_DB, inputName));
    }

    return new LinkedHashSet<ReadEntity>() {{ add(entity); }};
}
 

@Test
public void testLoadLocalPath() throws Exception {
    String tableName = createTable(false);
    String loadFile  = file("load");
    String query     = "load data local inpath 'file://" + loadFile + "' into table " + tableName;

    String tblId = assertTableIsRegistered(DEFAULT_DB, tableName);

    runCommand(query);

    AtlasEntity tblEntity  = atlasClientV2.getEntityByGuid(tblId).getEntity();
    List ddlQueries        = (List) tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);

    Assert.assertNotNull(ddlQueries);
    Assert.assertEquals(ddlQueries.size(), 1);

    assertProcessIsRegistered(constructEvent(query, HiveOperation.LOAD, null, getOutputs(tableName, Entity.Type.TABLE)));
}
 

@Test
public void testLoadLocalPathIntoPartition() throws Exception {
    String tableName = createTable(true);
    String loadFile  = file("load");
    String query     = "load data local inpath 'file://" + loadFile + "' into table " + tableName +  " partition(dt = '"+ PART_FILE + "')";

    String tblId = assertTableIsRegistered(DEFAULT_DB, tableName);

    runCommand(query);

    AtlasEntity tblEntity  = atlasClientV2.getEntityByGuid(tblId).getEntity();
    List ddlQueries        = (List) tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);

    Assert.assertNotNull(ddlQueries);
    Assert.assertEquals(ddlQueries.size(), 1);

    assertProcessIsRegistered(constructEvent(query, HiveOperation.LOAD, null, getOutputs(tableName, Entity.Type.TABLE)));
}
 

private Set<WriteEntity> getOutputs(String inputName, Entity.Type entityType) throws HiveException {
    final WriteEntity entity = new WriteEntity();

    if ( Entity.Type.DFS_DIR.equals(entityType) || Entity.Type.LOCAL_DIR.equals(entityType)) {
        entity.setName(lower(new Path(inputName).toString()));
        entity.setTyp(entityType);
    } else {
        entity.setName(getQualifiedTblName(inputName));
        entity.setTyp(entityType);
    }

    if (entityType == Entity.Type.TABLE) {
        entity.setT(hiveMetaStoreBridge.hiveClient.getTable(DEFAULT_DB, inputName));
    }
    return new LinkedHashSet<WriteEntity>() {{ add(entity); }};
}
 

@Test
public void testInsertIntoLocalDir() throws Exception {
    String tableName = createTable();
    File randomLocalPath = File.createTempFile("hiverandom", ".tmp");
    String query =
        "insert overwrite LOCAL DIRECTORY '" + randomLocalPath.getAbsolutePath() + "' select id, name from " + tableName;

    runCommand(query);
    validateProcess(constructEvent(query, HiveOperation.QUERY, getInputs(tableName, Entity.Type.TABLE), null));

    assertTableIsRegistered(DEFAULT_DB, tableName);
}
 

private static void addOutputs(HiveMetaStoreBridge hiveBridge, HiveOperation op, SortedSet<WriteEntity> sortedOutputs, StringBuilder buffer, final Map<WriteEntity, Referenceable> refs, final boolean ignoreHDFSPathsInQFName) throws HiveException {
    if (refs != null) {
        Set<String> dataSetsProcessed = new LinkedHashSet<>();
        if (sortedOutputs != null) {
            for (WriteEntity output : sortedOutputs) {
                final Entity entity = output;
                if (!dataSetsProcessed.contains(output.getName().toLowerCase())) {
                    //HiveOperation.QUERY type encompasses INSERT, INSERT_OVERWRITE, UPDATE, DELETE, PATH_WRITE operations
                    if (addQueryType(op, (WriteEntity) entity)) {
                        buffer.append(SEP);
                        buffer.append(((WriteEntity) entity).getWriteType().name());
                    }
                    if (ignoreHDFSPathsInQFName &&
                        (Type.DFS_DIR.equals(output.getType()) || Type.LOCAL_DIR.equals(output.getType()))) {
                        LOG.debug("Skipping dfs dir output addition to process qualified name {} ", output.getName());
                    } else if (refs.containsKey(output)) {
                        if ( output.getType() == Type.PARTITION || output.getType() == Type.TABLE) {
                            final Date createTime = HiveMetaStoreBridge.getTableCreatedTime(hiveBridge.hiveClient.getTable(output.getTable().getDbName(), output.getTable().getTableName()));
                            addDataset(buffer, refs.get(output), createTime.getTime());
                        } else {
                            addDataset(buffer, refs.get(output));
                        }
                    }
                    dataSetsProcessed.add(output.getName().toLowerCase());
                }
            }
        }
    }
}
 

@Override
public int compare(Entity o1, Entity o2) {
    String s1 = o1.getName();
    String s2 = o2.getName();
    if (s1 == null || s2 == null){
        s1 = o1.getD().toString();
        s2 = o2.getD().toString();
    }
    return s1.toLowerCase().compareTo(s2.toLowerCase());
}
 

@Test
public void testIgnoreSelect() throws Exception {
    String tableName = createTable();
    String query = "select * from " + tableName;
    runCommand(query);
    Set<ReadEntity> inputs = getInputs(tableName, Entity.Type.TABLE);
    HiveHook.HiveEventContext hiveEventContext = constructEvent(query, HiveOperation.QUERY, inputs, null);
    assertProcessIsNotRegistered(hiveEventContext);

    //check with uppercase table name
    query = "SELECT * from " + tableName.toUpperCase();
    runCommand(query);
    assertProcessIsNotRegistered(hiveEventContext);
}
 

private static Entity getEntityByType(Set<? extends Entity> entities, Type entityType) {
    for (Entity entity : entities) {
        if (entity.getType() == entityType) {
            return entity;
        }
    }
    return null;
}
 

private <T extends Entity> SortedMap<T, Referenceable> getSortedProcessDataSets(Set<T> inputTbls) {
    SortedMap<T, Referenceable> inputs = new TreeMap<>(entityComparator);
    if (inputTbls != null) {
        for (final T tbl : inputTbls) {
            Referenceable inputTableRef = new Referenceable(getDSTypeName(tbl), new HashMap<String, Object>() {{
                put(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tbl.getName());
            }});
            inputs.put(tbl, inputTableRef);
        }
    }
    return inputs;
}
 

@Test
public void testInsertIntoPartition() throws Exception {
    final boolean isPartitionedTable = true;
    String tableName = createTable(isPartitionedTable);
    String insertTableName = createTable(isPartitionedTable);
    String query =
        "insert into " + insertTableName + " partition(dt = '"+ PART_FILE + "') select id, name from " + tableName
            + " where dt = '"+ PART_FILE + "'";
    runCommand(query);

    final Set<ReadEntity> inputs = getInputs(tableName, Entity.Type.TABLE);
    final Set<WriteEntity> outputs = getOutputs(insertTableName, Entity.Type.TABLE);
    outputs.iterator().next().setWriteType(WriteEntity.WriteType.INSERT);

    final Set<ReadEntity> partitionIps = new LinkedHashSet<ReadEntity>() {
        {
            addAll(inputs);
            add(getPartitionInput());

        }
    };

    final Set<WriteEntity> partitionOps = new LinkedHashSet<WriteEntity>() {
        {
            addAll(outputs);
            add(getPartitionOutput());

        }
    };

    validateProcess(constructEvent(query,  HiveOperation.QUERY, partitionIps, partitionOps), inputs, outputs);

    assertTableIsRegistered(DEFAULT_DB, tableName);
    assertTableIsRegistered(DEFAULT_DB, insertTableName);

    //TODO -Add update test case
}
 

private List<DBModelAuthorizable> getAuthzHierarchyFromEntity(Entity entity) {
  List<DBModelAuthorizable> objectHierarchy = new ArrayList<DBModelAuthorizable>();
  switch (entity.getType()) {
  case TABLE:
    objectHierarchy.add(new Database(entity.getTable().getDbName()));
    objectHierarchy.add(new Table(entity.getTable().getTableName()));
    break;
  case PARTITION:
  case DUMMYPARTITION:
    objectHierarchy.add(new Database(entity.getPartition().getTable().getDbName()));
    objectHierarchy.add(new Table(entity.getPartition().getTable().getTableName()));
    break;
  case DFS_DIR:
  case LOCAL_DIR:
    try {
      objectHierarchy.add(parseURI(entity.toString(),
          entity.getType().equals(Entity.Type.LOCAL_DIR)));
    } catch (Exception e) {
      throw new AuthorizationException("Failed to get File URI", e);
    }
    break;
  case DATABASE:
  case FUNCTION:
    // TODO use database entities from compiler instead of capturing from AST
    break;
  default:
    throw new UnsupportedOperationException("Unsupported entity type " +
        entity.getType().name());
  }
  return objectHierarchy;
}
 

private static boolean isPartitionBasedQuery(Set<? extends Entity> entities) {
    for (Entity entity : entities) {
        if (Type.PARTITION.equals(entity.getType())) {
            return true;
        }
    }
    return false;
}
 

@VisibleForTesting
static String getProcessQualifiedName(HiveMetaStoreBridge dgiBridge, HiveEventContext eventContext,
                                      final SortedSet<ReadEntity> sortedHiveInputs,
                                      final SortedSet<WriteEntity> sortedHiveOutputs,
                                      SortedMap<ReadEntity, Referenceable> hiveInputsMap,
                                      SortedMap<WriteEntity, Referenceable> hiveOutputsMap) throws HiveException {
    HiveOperation op = eventContext.getOperation();
    if (isCreateOp(eventContext)) {
        Entity entity = getEntityByType(sortedHiveOutputs, Type.TABLE);

        if (entity != null) {
            Table outTable = entity.getTable();
            //refresh table
            outTable = dgiBridge.hiveClient.getTable(outTable.getDbName(), outTable.getTableName());
            return HiveMetaStoreBridge.getTableProcessQualifiedName(dgiBridge.getClusterName(), outTable);
        }
    }

    StringBuilder buffer = new StringBuilder(op.getOperationName());

    boolean ignoreHDFSPathsinQFName = ignoreHDFSPathsinQFName(op, sortedHiveInputs, sortedHiveOutputs);
    if ( ignoreHDFSPathsinQFName && LOG.isDebugEnabled()) {
        LOG.debug("Ignoring HDFS paths in qualifiedName for {} {} ", op, eventContext.getQueryStr());
    }

    addInputs(dgiBridge, op, sortedHiveInputs, buffer, hiveInputsMap, ignoreHDFSPathsinQFName);
    buffer.append(IO_SEP);
    addOutputs(dgiBridge, op, sortedHiveOutputs, buffer, hiveOutputsMap, ignoreHDFSPathsinQFName);
    LOG.info("Setting process qualified name to {}", buffer);
    return buffer.toString();
}
 

private  <T extends Entity> void processHiveEntity(HiveMetaStoreBridge dgiBridge, HiveEventContext event, T entity, Set<String> dataSetsProcessed,
    SortedMap<T, Referenceable> dataSets, Set<Referenceable> entities) throws AtlasHookException {
    try {
        if (entity.getType() == Type.TABLE || entity.getType() == Type.PARTITION) {
            final String tblQFName = HiveMetaStoreBridge.getTableQualifiedName(dgiBridge.getClusterName(), entity.getTable());
            if (!dataSetsProcessed.contains(tblQFName)) {
                LinkedHashMap<Type, Referenceable> result = createOrUpdateEntities(dgiBridge, event, entity, false);
                dataSets.put(entity, result.get(Type.TABLE));
                dataSetsProcessed.add(tblQFName);
                entities.addAll(result.values());
            }
        } else if (entity.getType() == Type.DFS_DIR) {
            URI location = entity.getLocation();
            if (location != null) {
                final String pathUri = lower(new Path(location).toString());
                LOG.debug("Registering DFS Path {} ", pathUri);
                if (!dataSetsProcessed.contains(pathUri)) {
                    Referenceable hdfsPath = dgiBridge.fillHDFSDataSet(pathUri);
                    dataSets.put(entity, hdfsPath);
                    dataSetsProcessed.add(pathUri);
                    entities.add(hdfsPath);
                }
            }
        }
    }
    catch(Exception e) {
        throw new AtlasHookException("HiveHook.processHiveEntity() failed.", e);
    }
}
 

@Test
public void testLoadLocalPath() throws Exception {
    String tableName = createTable(false);

    String loadFile = file("load");
    String query = "load data local inpath 'file://" + loadFile + "' into table " + tableName;
    runCommand(query);

    assertProcessIsRegistered(constructEvent(query, HiveOperation.LOAD, null, getOutputs(tableName, Entity.Type.TABLE)));
}
 

private LinkedHashMap<Type, Referenceable> handleEventOutputs(HiveMetaStoreBridge dgiBridge, HiveEventContext event, Type entityType) throws AtlasHookException {
    try {
        for (Entity entity : event.getOutputs()) {
            if (entity.getType() == entityType) {
                return createOrUpdateEntities(dgiBridge, event, entity, true);
            }
        }
        return null;
    }
    catch(Exception e) {
        throw new AtlasHookException("HiveHook.handleEventOutputs() failed.", e);
    }
}
 

private LinkedHashMap<Type, Referenceable> createOrUpdateEntities(HiveMetaStoreBridge dgiBridge, HiveEventContext event, Entity entity, boolean skipTempTables) throws AtlasHookException {
    try {
        return createOrUpdateEntities(dgiBridge, event, entity, skipTempTables, null);
    } catch (Exception e) {
        throw new AtlasHookException("HiveHook.createOrUpdateEntities() failed.", e);
    }
}
 

@Override
public int compare(Entity o1, Entity o2) {
    String s1 = o1.getName();
    String s2 = o2.getName();
    if (s1 == null || s2 == null){
        s1 = o1.getD().toString();
        s2 = o2.getD().toString();
    }
    return s1.toLowerCase().compareTo(s2.toLowerCase());
}
 

protected static void addOutputs(HiveMetaStoreBridge hiveBridge, HiveOperation op, SortedSet<WriteEntity> sortedOutputs, StringBuilder buffer, final Map<WriteEntity, AtlasEntity> refs, final boolean ignoreHDFSPathsInQFName) throws HiveException {
    if (refs != null) {
        Set<String> dataSetsProcessed = new LinkedHashSet<>();
        if (sortedOutputs != null) {
            for (WriteEntity output : sortedOutputs) {
                final Entity entity = output;
                if (!dataSetsProcessed.contains(output.getName().toLowerCase())) {
                    if (ignoreHDFSPathsInQFName &&
                            (Entity.Type.DFS_DIR.equals(output.getType()) || Entity.Type.LOCAL_DIR.equals(output.getType()))) {
                        LOG.debug("Skipping dfs dir output addition to process qualified name {} ", output.getName());
                    } else if (refs.containsKey(output)) {
                        //HiveOperation.QUERY type encompasses INSERT, INSERT_OVERWRITE, UPDATE, DELETE, PATH_WRITE operations
                        if (addQueryType(op, (WriteEntity) entity)) {
                            buffer.append(SEP);
                            buffer.append(((WriteEntity) entity).getWriteType().name());
                        }

                        if ( output.getType() == Entity.Type.PARTITION || output.getType() == Entity.Type.TABLE) {
                            Table outputTable = refreshTable(hiveBridge, output.getTable().getDbName(), output.getTable().getTableName());

                            if (outputTable != null) {
                                addDataset(buffer, refs.get(output), HiveMetaStoreBridge.getTableCreatedTime(outputTable));
                            }
                        } else {
                            addDataset(buffer, refs.get(output));
                        }
                    }

                    dataSetsProcessed.add(output.getName().toLowerCase());
                }
            }
        }
    }
}
 

public AtlasEntitiesWithExtInfo getHiveEntities() throws Exception {
    AtlasEntitiesWithExtInfo ret   = new AtlasEntitiesWithExtInfo();
    Table                    table = null;

    if (CollectionUtils.isNotEmpty(getOutputs())) {
        for (Entity entity : getOutputs()) {
            if (entity.getType() == Entity.Type.TABLE) {
                table = entity.getTable();

                if (table != null) {
                    table = getHive().getTable(table.getDbName(), table.getTableName());

                    if (table != null) {
                        if (skipTemporaryTable(table)) {
                            table = null;
                        } else {
                            break;
                        }
                    }
                }
            }
        }
    }

    processTable(table, ret);

    addProcessedEntities(ret);

    return ret;
}