Java Code Examples for com.baidu.hugegraph.config.HugeConfig#get()

public static Set<Integer> scanSessionIds(HugeConfig config,
                                          List<String> tableDirs) {
    Set<Integer> sessionIds = new HashSet<>();

    String path = config.get(PaloOptions.PALO_TEMP_DIR);
    File pathDir = Paths.get(path).toFile();
    if (!pathDir.exists()) {
        return sessionIds;
    }
    for (String table : tableDirs) {
        File tableDir = Paths.get(path, table).toFile();
        if (!tableDir.exists()) {
            continue;
        }
        String[] fileNames = tableDir.list();
        if (fileNames == null || fileNames.length == 0) {
            continue;
        }
        for (String fileName : fileNames) {
            int[] parts = PaloFile.parseFileName(fileName);
            int sessionId = parts[0];
            sessionIds.add(sessionId);
        }
    }
    return sessionIds;
}
 

public static void registerBackends() {
    String confFile = "/backend.properties";
    InputStream input = RegisterUtil.class.getClass()
                                    .getResourceAsStream(confFile);
    E.checkState(input != null,
                 "Can't read file '%s' as stream", confFile);

    PropertiesConfiguration props = new PropertiesConfiguration();
    props.setDelimiterParsingDisabled(true);
    try {
        props.load(input);
    } catch (ConfigurationException e) {
        throw new HugeException("Can't load config file: %s", e, confFile);
    }

    HugeConfig config = new HugeConfig(props);
    List<String> backends = config.get(DistOptions.BACKENDS);
    for (String backend : backends) {
        registerBackend(backend);
    }
}
 

public static HugeAuthenticator loadAuthenticator(HugeConfig conf) {
    String authClass = conf.get(ServerOptions.AUTHENTICATOR);
    if (authClass.isEmpty()) {
        return null;
    }

    HugeAuthenticator authenticator;
    ClassLoader cl = conf.getClass().getClassLoader();
    try {
        authenticator = (HugeAuthenticator) cl.loadClass(authClass)
                                              .newInstance();
    } catch (Exception e) {
        throw new HugeException("Failed to load authenticator: '%s'",
                                authClass, e);
    }

    authenticator.setup(conf);

    return authenticator;
}
 

public static synchronized HugeGraph open(Configuration config) {
    HugeConfig conf = config instanceof HugeConfig ?
                      (HugeConfig) config : new HugeConfig(config);
    String name = conf.get(CoreOptions.STORE);
    checkGraphName(name, "graph config(like hugegraph.properties)");
    name = name.toLowerCase();
    HugeGraph graph = graphs.get(name);
    if (graph == null || graph.closed()) {
        graph = new StandardHugeGraph(conf);
        graphs.put(name, graph);
    } else {
        String backend = conf.get(CoreOptions.BACKEND);
        E.checkState(backend.equalsIgnoreCase(graph.backend()),
                     "Graph name '%s' has been used by backend '%s'",
                     name, graph.backend());
    }
    return graph;
}
 

public CachedGraphTransaction(HugeGraphParams graph, BackendStore store) {
    super(graph, store);

    HugeConfig conf = graph.configuration();

    String type = conf.get(CoreOptions.VERTEX_CACHE_TYPE);
    long capacity = conf.get(CoreOptions.VERTEX_CACHE_CAPACITY);
    int expire = conf.get(CoreOptions.VERTEX_CACHE_EXPIRE);
    this.verticesCache = this.cache("vertex", type, capacity,
                                    AVG_VERTEX_ENTRY_SIZE, expire);

    type = conf.get(CoreOptions.EDGE_CACHE_TYPE);
    capacity = conf.get(CoreOptions.EDGE_CACHE_CAPACITY);
    expire = conf.get(CoreOptions.EDGE_CACHE_EXPIRE);
    this.edgesCache = this.cache("edge", type, capacity,
                                 AVG_EDGE_ENTRY_SIZE, expire);

    this.listenChanges();
}
 

public <R> R commit(HugeConfig config, HugeGraph g, int size,
                    Callable<R> callable) {
    int maxWriteThreads = config.get(ServerOptions.MAX_WRITE_THREADS);
    int writingThreads = batchWriteThreads.incrementAndGet();
    if (writingThreads > maxWriteThreads) {
        batchWriteThreads.decrementAndGet();
        throw new HugeException("The rest server is too busy to write");
    }

    LOG.debug("The batch writing threads is {}", batchWriteThreads);
    try {
        R result = commit(g, callable);
        this.batchMeter.mark(size);
        return result;
    } finally {
        batchWriteThreads.decrementAndGet();
    }
}
 

private void addMetrics(HugeConfig config) {
    final MetricManager metric = MetricManager.INSTANCE;
    // Force to add server reporter
    ServerReporter reporter = ServerReporter.instance(metric.getRegistry());
    reporter.start(60L, TimeUnit.SECONDS);

    // Add metrics for MAX_WRITE_THREADS
    int maxWriteThreads = config.get(ServerOptions.MAX_WRITE_THREADS);
    MetricsUtil.registerGauge(RestServer.class, "max-write-threads", () -> {
        return maxWriteThreads;
    });

    // Add metrics for caches
    @SuppressWarnings({ "rawtypes", "unchecked" })
    Map<String, Cache<?, ?>> caches = (Map) CacheManager.instance()
                                                        .caches();
    registerCacheMetrics(caches);
    final AtomicInteger lastCachesSize = new AtomicInteger(caches.size());
    MetricsUtil.registerGauge(Cache.class, "instances", () -> {
        int count = caches.size();
        if (count != lastCachesSize.get()) {
            // Update if caches changed (effect in the next report period)
            registerCacheMetrics(caches);
            lastCachesSize.set(count);
        }
        return count;
    });

    // Add metrics for task
    MetricsUtil.registerGauge(TaskManager.class, "workers", () -> {
        return TaskManager.instance().workerPoolSize();
    });
    MetricsUtil.registerGauge(TaskManager.class, "pending-tasks", () -> {
        return TaskManager.instance().pendingTasks();
    });
}
 

public static void initAdminUser(String restConfFile) throws Exception {
    StandardAuthenticator auth = new StandardAuthenticator();
    HugeConfig config = new HugeConfig(restConfFile);
    String authClass = config.get(ServerOptions.AUTHENTICATOR);
    if (authClass.isEmpty()) {
        return;
    }
    auth.setup(config);
    auth.initAdminUser();
}
 

@Override
public void setup(HugeConfig config) {
    String graphName = config.get(ServerOptions.AUTH_GRAPH_STORE);
    String graphPath = config.getMap(ServerOptions.GRAPHS).get(graphName);
    E.checkArgument(graphPath != null,
                    "Invalid graph name '%s'", graphName);
    this.graph = (HugeGraph) GraphFactory.open(graphPath);
}
 

private static void checkBatchSize(HugeConfig config,
                                   List<JsonEdge> edges) {
    int max = config.get(ServerOptions.MAX_EDGES_PER_BATCH);
    if (edges.size() > max) {
        throw new IllegalArgumentException(String.format(
                  "Too many edges for one time post, " +
                  "the maximum number is '%s'", max));
    }
    if (edges.size() == 0) {
        throw new IllegalArgumentException(
                  "The number of edges can't be 0");
    }
}
 

private static void checkBatchSize(HugeConfig config,
                                   List<JsonVertex> vertices) {
    int max = config.get(ServerOptions.MAX_VERTICES_PER_BATCH);
    if (vertices.size() > max) {
        throw new IllegalArgumentException(String.format(
                  "Too many vertices for one time post, " +
                  "the maximum number is '%s'", max));
    }
    if (vertices.size() == 0) {
        throw new IllegalArgumentException(
                  "The number of vertices can't be 0");
    }
}
 

@Override
public synchronized void open(HugeConfig config) {
    LOG.debug("Store open: {}", this.store);

    E.checkNotNull(config, "config");

    if (this.sessions != null && !this.sessions.closed()) {
        LOG.debug("Store {} has been opened before", this.store);
        this.sessions.useSession();
        return;
    }

    List<Future<?>> futures = new ArrayList<>();
    ExecutorService openPool = ExecutorUtil.newFixedThreadPool(
                               OPEN_POOL_THREADS, DB_OPEN);
    // Open base disk
    futures.add(openPool.submit(() -> {
        this.sessions = this.open(config, this.tableNames());
    }));

    // Open tables with optimized disk
    Map<String, String> disks = config.getMap(RocksDBOptions.DATA_DISKS);
    Set<String> openedDisks = new HashSet<>();
    if (!disks.isEmpty()) {
        String dataPath = config.get(RocksDBOptions.DATA_PATH);
        this.parseTableDiskMapping(disks, dataPath);
        for (Entry<HugeType, String> e : this.tableDiskMapping.entrySet()) {
            String table = this.table(e.getKey()).table();
            String disk = e.getValue();
            if (openedDisks.contains(disk)) {
                continue;
            }
            openedDisks.add(disk);
            futures.add(openPool.submit(() -> {
                this.open(config, disk, disk, Arrays.asList(table));
            }));
        }
    }
    waitOpenFinish(futures, openPool);
}
 

@Override
public void filter(ContainerRequestContext context) {
    if (LoadDetectFilter.isWhiteAPI(context)) {
        return;
    }

    HugeConfig config = this.configProvider.get();

    int maxWorkerThreads = config.get(ServerOptions.MAX_WORKER_THREADS);
    WorkLoad load = this.loadProvider.get();
    // There will be a thread doesn't work, dedicated to statistics
    if (load.incrementAndGet() >= maxWorkerThreads) {
        throw new ServiceUnavailableException(String.format(
                  "The server is too busy to process the request, " +
                  "you can config %s to adjust it or try again later",
                  ServerOptions.MAX_WORKER_THREADS.name()));
    }

    long minFreeMemory = config.get(ServerOptions.MIN_FREE_MEMORY);
    long allocatedMem = Runtime.getRuntime().totalMemory() -
                        Runtime.getRuntime().freeMemory();
    long presumableFreeMem = (Runtime.getRuntime().maxMemory() -
                              allocatedMem) / Bytes.MB;
    if (presumableFreeMem < minFreeMemory) {
        gcIfNeeded();
        throw new ServiceUnavailableException(String.format(
                  "The server available memory %s(MB) is below than " +
                  "threshold %s(MB) and can't process the request, " +
                  "you can config %s to adjust it or try again later",
                  presumableFreeMem, minFreeMemory,
                  ServerOptions.MIN_FREE_MEMORY.name()));
    }
}
 

private static Map<String, Object> parseReplica(HugeConfig conf) {
    Map<String, Object> replication = new HashMap<>();
    // Replication strategy: SimpleStrategy or NetworkTopologyStrategy
    String strategy = conf.get(CassandraOptions.CASSANDRA_STRATEGY);
    replication.put("class", strategy);

    switch (strategy) {
        case "SimpleStrategy":
            List<String> replicas =
                         conf.get(CassandraOptions.CASSANDRA_REPLICATION);
            E.checkArgument(replicas.size() == 1,
                            "Individual factor value should be provided " +
                            "with SimpleStrategy for Cassandra");
            int factor = convertFactor(replicas.get(0));
            replication.put("replication_factor", factor);
            break;
        case "NetworkTopologyStrategy":
            // The replicas format is like 'dc1:2,dc2:1'
            Map<String, String> replicaMap =
                        conf.getMap(CassandraOptions.CASSANDRA_REPLICATION);
            for (Map.Entry<String, String> e : replicaMap.entrySet()) {
                E.checkArgument(!e.getKey().isEmpty(),
                                "The datacenter can't be empty");
                replication.put(e.getKey(), convertFactor(e.getValue()));
            }
            break;
        default:
            throw new AssertionError(String.format(
                      "Illegal replication strategy '%s', valid strategy " +
                      "is 'SimpleStrategy' or 'NetworkTopologyStrategy'",
                      strategy));
    }
    return replication;
}
 

private Cache<Id, Object> cache(String prefix) {
    HugeConfig conf = super.params().configuration();

    final String name = prefix + "-" + this.graphName();
    final long capacity = conf.get(CoreOptions.SCHEMA_CACHE_CAPACITY);
    // NOTE: must disable schema cache-expire due to getAllSchema()
    return CacheManager.instance().cache(name, capacity);
}
 

private SnowflakeIdGenerator(HugeConfig config) {
    long workerId = config.get(CoreOptions.SNOWFLAKE_WORKER_ID);
    long datacenterId = config.get(CoreOptions.SNOWFLAKE_DATACENTER_ID);
    this.forceString = config.get(CoreOptions.SNOWFLAKE_FORCE_STRING);
    this.idWorker = new IdWorker(workerId, datacenterId);
    LOG.debug("SnowflakeId Worker started: datacenter id {}, " +
              "worker id {}, forced string id {}",
              datacenterId, workerId, this.forceString);
}
 

public StdSession(HugeConfig conf) {
    boolean bulkload = conf.get(RocksDBOptions.BULKLOAD_MODE);
    this.batch = new WriteBatch();
    this.writeOptions = new WriteOptions();
    this.writeOptions.setDisableWAL(bulkload);
    //this.writeOptions.setSync(false);
}
 

public PaloSessions(HugeConfig config, String database, String store,
                    List<String> tableDirs) {
    super(config, database, store);
    this.counter = new AtomicInteger();
    this.locks = new ConcurrentHashMap<>();
    // Scan disk files and restore session information
    this.restoreSessionInfo(config, tableDirs);

    this.timer = new Timer(true);
    long interval = config.get(PaloOptions.PALO_POLL_INTERVAL);
    this.loadTask = new PaloLoadTask(tableDirs);
    this.timer.schedule(this.loadTask, 0, interval * 1000);
}
 

private String buildUrl(HugeConfig config, String database) {
    String host = config.get(PaloOptions.PALO_HOST);
    Integer port = config.get(PaloOptions.PALO_HTTP_PORT);
    return String.format("http://%s:%s/api/%s/", host, port, database);
}
 

public int aggregateTimeout() {
    HugeConfig conf = CassandraSessionPool.this.config();
    return conf.get(CassandraOptions.AGGR_TIMEOUT);
}