org.apache.atlas.ApplicationProperties Java Examples

public static int getTypeUpdateLockMaxWaitTimeInSeconds() {
    Integer ret = typeUpdateLockMaxWaitTimeInSeconds;

    if (ret == null) {
        try {
            Configuration config = ApplicationProperties.get();

            ret = config.getInteger(CONFIG_TYPE_UPDATE_LOCK_MAX_WAIT_TIME_IN_SECONDS, DEFAULT_TYPE_UPDATE_LOCK_MAX_WAIT_TIME_IN_SECONDS);

            typeUpdateLockMaxWaitTimeInSeconds = ret;
        } catch (AtlasException e) {
            // ignore
        }
    }

    return ret == null ? DEFAULT_TYPE_UPDATE_LOCK_MAX_WAIT_TIME_IN_SECONDS : ret;
}
 

@Override
public void publish(SqoopJobDataPublisher.Data data) throws AtlasHookException {
    try {
        Configuration atlasProperties = ApplicationProperties.get();
        String clusterName = atlasProperties.getString(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);

        Referenceable dbStoreRef = createDBStoreInstance(data);
        Referenceable dbRef = createHiveDatabaseInstance(clusterName, data.getHiveDB());
        Referenceable hiveTableRef = createHiveTableInstance(clusterName, dbRef,
                data.getHiveTable(), data.getHiveDB());
        Referenceable procRef = createSqoopProcessInstance(dbStoreRef, hiveTableRef, data, clusterName);

        int maxRetries = atlasProperties.getInt(HOOK_NUM_RETRIES, 3);
        HookNotification.HookNotificationMessage message =
                new HookNotification.EntityCreateRequest(AtlasHook.getUser(), dbStoreRef, dbRef, hiveTableRef, procRef);
        AtlasHook.notifyEntities(Arrays.asList(message), maxRetries);
    }
    catch(Exception e) {
        throw new AtlasHookException("SqoopHook.publish() failed.", e);
    }
}
 

@Override
public void init() {
    LOG.info("==> SimpleAtlasAuthorizer.init()");

    InputStream inputStream = null;

    try {
        inputStream = ApplicationProperties.getFileAsInputStream(ApplicationProperties.get(), "atlas.authorizer.simple.authz.policy.file", "atlas-simple-authz-policy.json");

        authzPolicy = AtlasJson.fromJson(inputStream, AtlasSimpleAuthzPolicy.class);
    } catch (IOException | AtlasException e) {
        LOG.error("SimpleAtlasAuthorizer.init(): initialization failed", e);

        throw new RuntimeException(e);
    } finally {
        if (inputStream != null) {
            try {
                inputStream.close();
            } catch (IOException excp) {
                // ignore
            }
        }
    }

    LOG.info("<== SimpleAtlasAuthorizer.init()");
}
 

public static String writeConfiguration(final PropertiesConfiguration configuration) throws Exception {
    String confLocation = System.getProperty("atlas.conf");
    URL url;
    if (confLocation == null) {
        url = BaseSecurityTest.class.getResource("/" + ApplicationProperties.APPLICATION_PROPERTIES);
    } else {
        url = new File(confLocation, ApplicationProperties.APPLICATION_PROPERTIES).toURI().toURL();
    }
    PropertiesConfiguration configuredProperties = new PropertiesConfiguration();
    configuredProperties.load(url);

    configuredProperties.copy(configuration);

    String persistDir = TestUtils.getTempDirectory();
    configuredProperties.setProperty("atlas.authentication.method.file", "true");
    configuredProperties.setProperty("atlas.authentication.method.file.filename", persistDir
            + "/users-credentials");
    configuredProperties.setProperty("atlas.auth.policy.file",persistDir
            + "/policy-store.txt" );
    TestUtils.writeConfiguration(configuredProperties, persistDir + File.separator +
            ApplicationProperties.APPLICATION_PROPERTIES);
    setupUserCredential(persistDir);
    setUpPolicyStore(persistDir);
    ApplicationProperties.forceReload();
    return persistDir;
}
 

@Inject
public NotificationHookConsumer(NotificationInterface notificationInterface, AtlasEntityStore atlasEntityStore,
                                ServiceState serviceState, AtlasInstanceConverter instanceConverter,
                                AtlasTypeRegistry typeRegistry) throws AtlasException {
    this.notificationInterface = notificationInterface;
    this.atlasEntityStore = atlasEntityStore;
    this.serviceState = serviceState;
    this.instanceConverter = instanceConverter;
    this.typeRegistry = typeRegistry;

    this.applicationProperties = ApplicationProperties.get();

    maxRetries = applicationProperties.getInt(CONSUMER_RETRIES_PROPERTY, 3);
    failedMsgCacheSize = applicationProperties.getInt(CONSUMER_FAILEDCACHESIZE_PROPERTY, 20);
    consumerRetryInterval = applicationProperties.getInt(CONSUMER_RETRY_INTERVAL, 500);

}
 

public static boolean useLocalSolr() {
    boolean ret = false;
    
    try {
        Configuration conf     = ApplicationProperties.get();
        Object        property = conf.getProperty("atlas.graph.index.search.solr.embedded");

        if (property != null && property instanceof String) {
            ret = Boolean.valueOf((String) property);
        }
    } catch (AtlasException ignored) {
        throw new SkipException("useLocalSolr: failed! ", ignored);
    }

    return ret;
}
 

public void validateType(AtlasBaseTypeDef typeDef) throws AtlasBaseException {
    if (!isValidName(typeDef.getName())) {
        throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID_FORMAT, typeDef.getName(), typeDef.getCategory().name());
    }

    try {
        final boolean allowReservedKeywords = ApplicationProperties.get().getBoolean(ALLOW_RESERVED_KEYWORDS, true);

        if (!allowReservedKeywords && typeDef instanceof AtlasStructDef) {
            final List<AtlasStructDef.AtlasAttributeDef> attributeDefs = ((AtlasStructDef) typeDef).getAttributeDefs();
            for (AtlasStructDef.AtlasAttributeDef attrDef : attributeDefs) {
                if (QueryParser.isKeyword(attrDef.getName())) {
                    throw new AtlasBaseException(AtlasErrorCode.ATTRIBUTE_NAME_INVALID, attrDef.getName(), typeDef.getCategory().name());
                }
            }
        }
    } catch (AtlasException e) {
        LOG.error("Exception while loading configuration ", e);
        throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "Could not load configuration");
    }
}
 

private static void initialize() {
    if (!isInitialized) {
        try {
            isFreeTextSearchEnabled = ApplicationProperties.get().getBoolean(ApplicationProperties.ENABLE_FREETEXT_SEARCH_CONF, true);

            if (isFreeTextSearchEnabled) { // currently free-text is supported only for Solr
                isFreeTextSearchEnabled = ApplicationProperties.INDEX_BACKEND_SOLR.equalsIgnoreCase(ApplicationProperties.get().getString(ApplicationProperties.INDEX_BACKEND_CONF));
            }

            if (isFreeTextSearchEnabled) { // if free-text is enabled, disable full-text - to avoid performance penalty
                isFullTextSearchEnabled = false;
            } else {
                isFullTextSearchEnabled = ApplicationProperties.get().getBoolean(ApplicationProperties.ENABLE_FULLTEXT_SEARCH_CONF, true);
            }

            isInitialized = true;
        } catch (AtlasException excp) {
            LOG.error("Failed to initialize. isFullTextSearchEnabled={}, isFreeTextSearchEnabled={}", isFullTextSearchEnabled, isFreeTextSearchEnabled, excp);
        }
    }
}
 

@SuppressWarnings("unchecked")
public static Class<? extends GraphDatabase> getGraphDatabaseImpl() {
    try {
        final Class<? extends GraphDatabase> ret;
        Configuration                        config            = ApplicationProperties.get();
        String                               graphDatabaseImpl = config.getString(ApplicationProperties.GRAPHDB_BACKEND_CONF);

        if (StringUtils.equals(graphDatabaseImpl, ApplicationProperties.GRAPHBD_BACKEND_JANUS)) {
            ret = ApplicationProperties.getClass(JANUS_GRAPH_DATABASE_IMPLEMENTATION_CLASS, GraphDatabase.class);
        } else {
            ret = ApplicationProperties.getClass(graphDatabaseImpl, GraphDatabase.class);
        }

        return ret;
    } catch (AtlasException e) {
        throw new RuntimeException(e);
    }
}
 

private void setPamProperties() {
    try {
        this.groupsFromUGI = ApplicationProperties.get().getBoolean("atlas.authentication.method.pam.ugi-groups", true);
        Properties properties = ConfigurationConverter.getProperties(ApplicationProperties.get()
                .subset("atlas.authentication.method.pam"));
        for (String key : properties.stringPropertyNames()) {
            String value = properties.getProperty(key);
            options.put(key, value);
        }
        if (!options.containsKey("service")) {
            options.put("service", "atlas-login");
        }
    } catch (Exception e) {
        LOG.error("Exception while setLdapProperties", e);
    }
}
 

@PostConstruct
void setAuthenticationMethod() {
    try {
        Configuration configuration = ApplicationProperties.get();

        this.fileAuthenticationMethodEnabled = configuration.getBoolean(FILE_AUTH_METHOD, true);

        this.pamAuthenticationEnabled = configuration.getBoolean(PAM_AUTH_METHOD, false);

        this.keycloakAuthenticationEnabled = configuration.getBoolean(KEYCLOAK_AUTH_METHOD, false);

        boolean ldapAuthenticationEnabled = configuration.getBoolean(LDAP_AUTH_METHOD, false);

        if (ldapAuthenticationEnabled) {
            this.ldapType = configuration.getString(LDAP_TYPE, "NONE");
        } else {
            this.ldapType = "NONE";
        }
    } catch (Exception e) {
        LOG.error("Error while getting atlas.login.method application properties", e);
    }
}
 

@Override
public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
    try {
        Configuration configuration = ApplicationProperties.get();
        boolean shouldRunSetup = configuration.getBoolean(ATLAS_SERVER_RUN_SETUP_KEY, false);
        if (shouldRunSetup) {
            LOG.warn("Running setup per configuration {}.", ATLAS_SERVER_RUN_SETUP_KEY);
            return true;
        } else {
            LOG.info("Not running setup per configuration {}.", ATLAS_SERVER_RUN_SETUP_KEY);
        }
    } catch (AtlasException e) {
        LOG.error("Unable to read config to determine if setup is needed. Not running setup.");
    }
    return false;
}
 

public static Configuration getConfiguration() throws AtlasException {
    Configuration configProperties = ApplicationProperties.get();

    Configuration titanConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);

    //add serializers for non-standard property value types that Atlas uses

    titanConfig.addProperty("attributes.custom.attribute1.attribute-class", TypeCategory.class.getName());
    titanConfig.addProperty("attributes.custom.attribute1.serializer-class",
            TypeCategorySerializer.class.getName());

    //not ideal, but avoids making large changes to Atlas
    titanConfig.addProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName());
    titanConfig.addProperty("attributes.custom.attribute2.serializer-class", StringListSerializer.class.getName());

    titanConfig.addProperty("attributes.custom.attribute3.attribute-class", BigInteger.class.getName());
    titanConfig.addProperty("attributes.custom.attribute3.serializer-class", BigIntegerSerializer.class.getName());

    titanConfig.addProperty("attributes.custom.attribute4.attribute-class", BigDecimal.class.getName());
    titanConfig.addProperty("attributes.custom.attribute4.serializer-class", BigDecimalSerializer.class.getName());

    return titanConfig;
}
 

@BeforeTest
public void setUp() throws AtlasException {
    GraphSandboxUtil.create();

    //First get Instance
    graph = new Titan1Graph();
    configuration = ApplicationProperties.getSubsetConfiguration(ApplicationProperties.get(),
            Titan1GraphDatabase.GRAPH_PREFIX);
}
 

@BeforeClass
public void setUp() throws Exception {
    Configuration atlasProperties = ApplicationProperties.get();
    if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) {
        atlasClient = new AtlasClient(atlasProperties.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT), new String[]{"admin", "admin"});
    } else {
        atlasClient = new AtlasClient(atlasProperties.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT));
    }

    AtlasService service = new AtlasService();
    service.init();
    STORE.registerListener(service);
    CurrentUser.authenticate(System.getProperty("user.name"));
}
 

@Override
public void publish(SqoopJobDataPublisher.Data data) throws AtlasHookException {
    try {
        Configuration atlasProperties   = ApplicationProperties.get();
        String        metadataNamespace = atlasProperties.getString(ATLAS_METADATA_NAMESPACE, getClusterName(atlasProperties));

        AtlasEntity entDbStore   = toSqoopDBStoreEntity(data);
        AtlasEntity entHiveDb    = toHiveDatabaseEntity(metadataNamespace, data.getHiveDB());
        AtlasEntity entHiveTable = data.getHiveTable() != null ? toHiveTableEntity(entHiveDb, data.getHiveTable()) : null;
        AtlasEntity entProcess   = toSqoopProcessEntity(entDbStore, entHiveDb, entHiveTable, data, metadataNamespace);


        AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo(entProcess);

        entities.addReferredEntity(entDbStore);
        entities.addReferredEntity(entHiveDb);
        if (entHiveTable != null) {
            entities.addReferredEntity(entHiveTable);
        }

        HookNotification message = new EntityCreateRequestV2(AtlasHook.getUser(), entities);

        atlasHook.sendNotification(message);
    } catch(Exception e) {
        LOG.error("SqoopHook.publish() failed", e);

        throw new AtlasHookException("SqoopHook.publish() failed.", e);
    }
}
 

@SuppressWarnings("unchecked")
public static Class<? extends DeleteHandler> getDeleteHandlerImpl() {
    try {
        Configuration config = ApplicationProperties.get();
        return ApplicationProperties.getClass(config,
                DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class);
    } catch (AtlasException e) {
        throw new RuntimeException(e);
    }
}
 

/**
 * Returns the application configuration.
 * @return
 */
protected org.apache.commons.configuration.Configuration getConfiguration() {
    try {
        return ApplicationProperties.get();
    } catch (AtlasException e) {
        throw new RuntimeException("Unable to load configuration: " + ApplicationProperties.APPLICATION_PROPERTIES);
    }
}
 

private static void startLocalSolr() {
    if (isEmbeddedSolr()) {
        try {
            LocalSolrRunner.start();

            Configuration configuration = ApplicationProperties.get();
            configuration.clearProperty(SOLR_ZOOKEEPER_URL);
            configuration.setProperty(SOLR_ZOOKEEPER_URL, LocalSolrRunner.getZookeeperUrls());
        } catch (Exception e) {
            throw new RuntimeException("Failed to start embedded solr cloud server. Aborting!", e);
        }
    }
}
 

@BeforeClass
public void setUp() throws Exception {
    // start a local storm cluster
    stormCluster = StormTestUtil.createLocalStormCluster();
    LOG.info("Created a storm local cluster");

    Configuration configuration = ApplicationProperties.get();
    if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) {
        atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT), new String[]{"admin", "admin"});
    } else {
        atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT));
    }
}
 

@BeforeClass
public void setUp() throws Exception {
    //Set-up sqoop session
    Configuration configuration = ApplicationProperties.get();
    if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) {
        atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT), new String[]{"admin", "admin"});
    } else {
        atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT));
    }
}
 

private void setADProperties() {
    try {

        Configuration configuration = ApplicationProperties.get();
        Properties properties = ConfigurationConverter.getProperties(configuration.subset("atlas.authentication.method.ldap.ad"));
        this.adDomain = properties.getProperty("domain");
        this.adURL = properties.getProperty("url");
        this.adBindDN = properties.getProperty("bind.dn");
        this.adBindPassword = properties.getProperty("bind.password");
        this.adUserSearchFilter = properties.getProperty("user.searchfilter");
        this.adBase = properties.getProperty("base.dn");
        this.adReferral = properties.getProperty("referral");
        this.adDefaultRole = properties.getProperty("default.role");

        this.groupsFromUGI = configuration.getBoolean("atlas.authentication.method.ldap.ugi-groups", true);

        if(LOG.isDebugEnabled()) {
            LOG.debug("AtlasADAuthenticationProvider{" +
                    "adURL='" + adURL + '\'' +
                    ", adDomain='" + adDomain + '\'' +
                    ", adBindDN='" + adBindDN + '\'' +
                    ", adUserSearchFilter='" + adUserSearchFilter + '\'' +
                    ", adBase='" + adBase + '\'' +
                    ", adReferral='" + adReferral + '\'' +
                    ", adDefaultRole='" + adDefaultRole + '\'' +
                    ", groupsFromUGI=" + groupsFromUGI +
                    '}');
        }


    } catch (Exception e) {
        LOG.error("Exception while setADProperties", e);
    }
}
 

public static boolean isKerberosAuthenticationEnabled(UserGroupInformation ugi) {
    boolean defaultValue = ugi != null && ugi.hasKerberosCredentials();

    try {
        return isKerberosAuthenticationEnabled(ApplicationProperties.get(), defaultValue);
    } catch (AtlasException e) {
        LOG.error("Error while isKerberosAuthenticationEnabled ", e);
    }

    return defaultValue;
}
 

@Test(dataProvider = "invalidAttributeNameWithReservedKeywords")
public void testCreateTypeWithReservedKeywords(AtlasRelationshipDef atlasRelationshipDef) throws AtlasException {
    try {
        ApplicationProperties.get().setProperty(AtlasAbstractDefStoreV2.ALLOW_RESERVED_KEYWORDS, false);
        relationshipDefStore.create(atlasRelationshipDef, null);
    } catch (AtlasBaseException e) {
        Assert.assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.ATTRIBUTE_NAME_INVALID);
    }
}
 

@Test(dataProvider = "invalidAttributeNameWithReservedKeywords")
public void testCreateTypeWithReservedKeywords(AtlasEntityDef atlasEntityDef) throws AtlasException {
    try {
        ApplicationProperties.get().setProperty(AtlasAbstractDefStoreV2.ALLOW_RESERVED_KEYWORDS, false);
        entityDefStore.create(atlasEntityDef, null);
    } catch (AtlasBaseException e) {
        Assert.assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.ATTRIBUTE_NAME_INVALID);
    }
}
 

/**
 * Construct a KafkaNotification.
 *
 * @param applicationProperties  the application properties used to configure Kafka
 *
 * @throws AtlasException if the notification interface can not be created
 */
@Inject
public KafkaNotification(Configuration applicationProperties) throws AtlasException {
    super(applicationProperties);
    Configuration subsetConfiguration =
            ApplicationProperties.getSubsetConfiguration(applicationProperties, PROPERTY_PREFIX);
    properties = ConfigurationConverter.getProperties(subsetConfiguration);
    //override to store offset in kafka
    //todo do we need ability to replay?

    //Override default configs
    properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
            "org.apache.kafka.common.serialization.StringSerializer");
    properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
            "org.apache.kafka.common.serialization.StringSerializer");
    properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
            "org.apache.kafka.common.serialization.StringDeserializer");
    properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
            "org.apache.kafka.common.serialization.StringDeserializer");
    properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");

    pollTimeOutMs = subsetConfiguration.getLong("poll.timeout.ms", 1000);
    boolean oldApiCommitEnbleFlag = subsetConfiguration.getBoolean("auto.commit.enable",false);
    //set old autocommit value if new autoCommit property is not set.
    properties.put("enable.auto.commit", subsetConfiguration.getBoolean("enable.auto.commit", oldApiCommitEnbleFlag));
    properties.put("session.timeout.ms", subsetConfiguration.getString("session.timeout.ms", "30000"));

}
 

public static boolean isGremlinOptimizerEnabled() {
    try {
        return ApplicationProperties.get().getBoolean(GREMLIN_OPTIMIZER_ENABLED_PROPERTY, DEFAULT_GREMLIN_OPTIMZER_ENABLED);
    } catch (AtlasException e) {
        LOG.error("Could not determine value of " + GREMLIN_OPTIMIZER_ENABLED_PROPERTY + ".  Defaulting to " + DEFAULT_GREMLIN_OPTIMZER_ENABLED, e);
        return DEFAULT_GREMLIN_OPTIMZER_ENABLED;
    }
}
 

/**
 * Get the configuration property that specifies the number evictions that pass
 * before a warning is logged. This is an optional property. A default is
 * used if it is not present.
 *
 * @return the number of evictions before a warning is logged.
 */
public static int getCompiledQueryCacheEvictionWarningThrottle() {
    try {
        return ApplicationProperties.get().getInt(COMPILED_QUERY_CACHE_EVICTION_WARNING_THROTTLE, DEFAULT_COMPILED_QUERY_CACHE_EVICTION_WARNING_THROTTLE);
    } catch (AtlasException e) {
        throw new RuntimeException(e);
    }
}
 

static String[] getServerUrl(String[] args) throws AtlasException {
    if (args.length > 0) {
        return args[0].split(",");
    }

    Configuration configuration = ApplicationProperties.get();
    String[] urls = configuration.getStringArray(ATLAS_REST_ADDRESS);
    if (urls == null || urls.length == 0) {
        System.out.println("Usage: quick_start_v1.py <atlas endpoint of format <http/https>://<atlas-fqdn>:<atlas port> like http://localhost:21000>");
        System.exit(-1);
    }

    return urls;
}
 

public static Class<? extends DeleteHandlerV1> getDeleteHandlerV1Impl() {
    try {
        Configuration config = ApplicationProperties.get();
        return ApplicationProperties.getClass(config,
                                              DELETE_HANDLER_V1_IMPLEMENTATION_PROPERTY, SoftDeleteHandlerV1.class.getName(), DeleteHandlerV1.class);
    } catch (AtlasException e) {
        throw new RuntimeException(e);
    }
}