com.baidu.hugegraph.config.HugeConfig Java Examples

public static void main(String[] args)
                   throws ConfigurationException, IOException {
    E.checkArgument(args.length == 1,
                    "ConfDumper need a config file.");

    String input = args[0];
    File output = new File(input + ".default");
    System.out.println("Input config: " + input);
    System.out.println("Output config: " + output.getPath());

    RegisterUtil.registerBackends();
    RegisterUtil.registerServer();

    HugeConfig config = new HugeConfig(input);

    for (String name : new TreeSet<>(OptionSpace.keys())) {
        TypedOption<?, ?> option = OptionSpace.get(name);
        writeOption(output, option, config.get(option));
    }
}
 

public <R> R commit(HugeConfig config, HugeGraph g, int size,
                    Callable<R> callable) {
    int maxWriteThreads = config.get(ServerOptions.MAX_WRITE_THREADS);
    int writingThreads = batchWriteThreads.incrementAndGet();
    if (writingThreads > maxWriteThreads) {
        batchWriteThreads.decrementAndGet();
        throw new HugeException("The rest server is too busy to write");
    }

    LOG.debug("The batch writing threads is {}", batchWriteThreads);
    try {
        R result = commit(g, callable);
        this.batchMeter.mark(size);
        return result;
    } finally {
        batchWriteThreads.decrementAndGet();
    }
}
 

@POST
@Timed(name = "batch-create")
@Decompress
@Path("batch")
@Status(Status.CREATED)
@Consumes(APPLICATION_JSON)
@Produces(APPLICATION_JSON_WITH_CHARSET)
@RolesAllowed({"admin", "$owner=$graph $action=vertex_write"})
public List<String> create(@Context HugeConfig config,
                           @Context GraphManager manager,
                           @PathParam("graph") String graph,
                           List<JsonVertex> jsonVertices) {
    LOG.debug("Graph [{}] create vertices: {}", graph, jsonVertices);
    checkCreatingBody(jsonVertices);
    checkBatchSize(config, jsonVertices);

    HugeGraph g = graph(manager, graph);

    return this.commit(config, g, jsonVertices.size(), () -> {
        List<String> ids = new ArrayList<>(jsonVertices.size());
        for (JsonVertex vertex : jsonVertices) {
            ids.add(g.addVertex(vertex.properties()).id().toString());
        }
        return ids;
    });
}
 

@POST
@Timed
@Compress
@Consumes(APPLICATION_JSON)
@Produces(APPLICATION_JSON_WITH_CHARSET)
public Response post(@Context HugeConfig conf,
                     @Context HttpHeaders headers,
                     String request) {
    /* The following code is reserved for forwarding request */
    // context.getRequestDispatcher(location).forward(request, response);
    // return Response.seeOther(UriBuilder.fromUri(location).build())
    // .build();
    // Response.temporaryRedirect(UriBuilder.fromUri(location).build())
    // .build();
    String auth = headers.getHeaderString(HttpHeaders.AUTHORIZATION);
    Response response = this.client().doPostRequest(auth, request);
    gremlinInputHistogram.update(request.length());
    gremlinOutputHistogram.update(response.getLength());
    return transformResponseIfNeeded(response);
}
 

@Test
public void testParseRepilcaWithNetworkTopologyStrategyAndDoubleReplica() {
    String strategy = CassandraOptions.CASSANDRA_STRATEGY.name();
    String replica = CassandraOptions.CASSANDRA_REPLICATION.name();

    Configuration conf = Mockito.mock(PropertiesConfiguration.class);
    Mockito.when(conf.getKeys())
           .thenReturn(ImmutableList.of(strategy, replica).iterator());
    Mockito.when(conf.getProperty(strategy))
           .thenReturn("NetworkTopologyStrategy");
    Mockito.when(conf.getProperty(replica))
           .thenReturn(ImmutableList.of("dc1:3.5", "dc2:1"));
    HugeConfig config = new HugeConfig(conf);

    Assert.assertThrows(RuntimeException.class, () -> {
        Whitebox.invokeStatic(CassandraStore.class, "parseReplica", config);
    });
}
 

public static HugeAuthenticator loadAuthenticator(HugeConfig conf) {
    String authClass = conf.get(ServerOptions.AUTHENTICATOR);
    if (authClass.isEmpty()) {
        return null;
    }

    HugeAuthenticator authenticator;
    ClassLoader cl = conf.getClass().getClassLoader();
    try {
        authenticator = (HugeAuthenticator) cl.loadClass(authClass)
                                              .newInstance();
    } catch (Exception e) {
        throw new HugeException("Failed to load authenticator: '%s'",
                                authClass, e);
    }

    authenticator.setup(conf);

    return authenticator;
}
 

public static synchronized HugeGraph open(Configuration config) {
    HugeConfig conf = config instanceof HugeConfig ?
                      (HugeConfig) config : new HugeConfig(config);
    String name = conf.get(CoreOptions.STORE);
    checkGraphName(name, "graph config(like hugegraph.properties)");
    name = name.toLowerCase();
    HugeGraph graph = graphs.get(name);
    if (graph == null || graph.closed()) {
        graph = new StandardHugeGraph(conf);
        graphs.put(name, graph);
    } else {
        String backend = conf.get(CoreOptions.BACKEND);
        E.checkState(backend.equalsIgnoreCase(graph.backend()),
                     "Graph name '%s' has been used by backend '%s'",
                     name, graph.backend());
    }
    return graph;
}
 

public static Set<Integer> scanSessionIds(HugeConfig config,
                                          List<String> tableDirs) {
    Set<Integer> sessionIds = new HashSet<>();

    String path = config.get(PaloOptions.PALO_TEMP_DIR);
    File pathDir = Paths.get(path).toFile();
    if (!pathDir.exists()) {
        return sessionIds;
    }
    for (String table : tableDirs) {
        File tableDir = Paths.get(path, table).toFile();
        if (!tableDir.exists()) {
            continue;
        }
        String[] fileNames = tableDir.list();
        if (fileNames == null || fileNames.length == 0) {
            continue;
        }
        for (String fileName : fileNames) {
            int[] parts = PaloFile.parseFileName(fileName);
            int sessionId = parts[0];
            sessionIds.add(sessionId);
        }
    }
    return sessionIds;
}
 

@Test
public void testParseRepilcaWithNetworkTopologyStrategyWithoutDatacenter() {
    String strategy = CassandraOptions.CASSANDRA_STRATEGY.name();
    String replica = CassandraOptions.CASSANDRA_REPLICATION.name();

    Configuration conf = Mockito.mock(PropertiesConfiguration.class);
    Mockito.when(conf.getKeys())
           .thenReturn(ImmutableList.of(strategy, replica).iterator());
    Mockito.when(conf.getProperty(strategy))
           .thenReturn("NetworkTopologyStrategy");
    Mockito.when(conf.getProperty(replica))
           .thenReturn(ImmutableList.of(":2", "dc2:1"));
    HugeConfig config = new HugeConfig(conf);

    Assert.assertThrows(RuntimeException.class, () -> {
        Whitebox.invokeStatic(CassandraStore.class, "parseReplica", config);
    });
}
 

public static void registerBackends() {
    String confFile = "/backend.properties";
    InputStream input = RegisterUtil.class.getClass()
                                    .getResourceAsStream(confFile);
    E.checkState(input != null,
                 "Can't read file '%s' as stream", confFile);

    PropertiesConfiguration props = new PropertiesConfiguration();
    props.setDelimiterParsingDisabled(true);
    try {
        props.load(input);
    } catch (ConfigurationException e) {
        throw new HugeException("Can't load config file: %s", e, confFile);
    }

    HugeConfig config = new HugeConfig(props);
    List<String> backends = config.get(DistOptions.BACKENDS);
    for (String backend : backends) {
        registerBackend(backend);
    }
}
 

public RocksDBStdSessions(HugeConfig config, String database, String store,
                          String dataPath, String walPath)
                          throws RocksDBException {
    super(config, database, store);

    // Init options
    Options options = new Options();
    RocksDBStdSessions.initOptions(config, options, options,
                                   options, options);
    options.setWalDir(walPath);

    this.sstFileManager = new SstFileManager(Env.getDefault());
    options.setSstFileManager(this.sstFileManager);

    /*
     * Open RocksDB at the first time
     * Don't merge old CFs, we expect a clear DB when using this one
     */
    this.rocksdb = RocksDB.open(options, dataPath);

    this.cfs = new ConcurrentHashMap<>();
    this.refCount = new AtomicInteger(1);
}
 

public ApplicationConfig(HugeConfig conf) {
    packages("com.baidu.hugegraph.api");

    // Register Jackson to support json
    register(org.glassfish.jersey.jackson.JacksonFeature.class);

    // Register to use the jsr250 annotations @RolesAllowed
    register(RolesAllowedDynamicFeature.class);

    // Register HugeConfig to context
    register(new ConfFactory(conf));

    // Register GraphManager to context
    register(new GraphManagerFactory(conf));

    // Register WorkLoad to context
    register(new WorkLoadFactory());

    // Let @Metric annotations work
    MetricRegistry registry = MetricManager.INSTANCE.getRegistry();
    register(new InstrumentedResourceMethodApplicationListener(registry));
}
 

@Test
public void testParseRepilcaWithNetworkTopologyStrategyAndStringReplica() {
    String strategy = CassandraOptions.CASSANDRA_STRATEGY.name();
    String replica = CassandraOptions.CASSANDRA_REPLICATION.name();

    Configuration conf = Mockito.mock(PropertiesConfiguration.class);
    Mockito.when(conf.getKeys())
           .thenReturn(ImmutableList.of(strategy, replica).iterator());
    Mockito.when(conf.getProperty(strategy))
           .thenReturn("NetworkTopologyStrategy");
    Mockito.when(conf.getProperty(replica))
           .thenReturn(ImmutableList.of("dc1:2", "dc2:string"));
    HugeConfig config = new HugeConfig(conf);

    Assert.assertThrows(RuntimeException.class, () -> {
        Whitebox.invokeStatic(CassandraStore.class, "parseReplica", config);
    });
}
 

@Test
public void testParseRepilcaWithSimpleStrategyAndEmptyReplica() {
    String strategy = CassandraOptions.CASSANDRA_STRATEGY.name();
    String replica = CassandraOptions.CASSANDRA_REPLICATION.name();

    Configuration conf = Mockito.mock(PropertiesConfiguration.class);
    Mockito.when(conf.getKeys())
           .thenReturn(ImmutableList.of(strategy, replica).iterator());
    Mockito.when(conf.getProperty(strategy))
           .thenReturn("SimpleStrategy");
    Mockito.when(conf.getProperty(replica))
           .thenReturn(ImmutableList.of(""));
    HugeConfig config = new HugeConfig(conf);

    Assert.assertThrows(RuntimeException.class, () -> {
        Whitebox.invokeStatic(CassandraStore.class, "parseReplica", config);
    });
}
 

@Test
public void testParseRepilcaWithNetworkTopologyStrategy() {
    String strategy = CassandraOptions.CASSANDRA_STRATEGY.name();
    String replica = CassandraOptions.CASSANDRA_REPLICATION.name();

    Configuration conf = Mockito.mock(PropertiesConfiguration.class);
    Mockito.when(conf.getKeys())
           .thenReturn(ImmutableList.of(strategy, replica).iterator());
    Mockito.when(conf.getProperty(strategy))
           .thenReturn("NetworkTopologyStrategy");
    Mockito.when(conf.getProperty(replica))
           .thenReturn(ImmutableList.of("dc1:2", "dc2:1"));
    HugeConfig config = new HugeConfig(conf);

    Map<String, Object> result = Whitebox.invokeStatic(CassandraStore.class,
                                                       "parseReplica",
                                                       config);

    Map<String, Object> expected = ImmutableMap.of(
                                   "class", "NetworkTopologyStrategy",
                                   "dc1", 2,
                                   "dc2", 1);
    Assert.assertEquals(expected, result);
}
 

@Override
public synchronized void open() throws IOException {
    HugeConfig config = this.config();
    String hosts = config.get(HbaseOptions.HBASE_HOSTS);
    int port = config.get(HbaseOptions.HBASE_PORT);
    String znodeParent = config.get(HbaseOptions.HBASE_ZNODE_PARENT);

    Configuration hConfig = HBaseConfiguration.create();
    hConfig.set(HConstants.ZOOKEEPER_QUORUM, hosts);
    hConfig.set(HConstants.ZOOKEEPER_CLIENT_PORT, String.valueOf(port));
    hConfig.set(HConstants.ZOOKEEPER_ZNODE_PARENT, znodeParent);

    hConfig.setInt("zookeeper.recovery.retry",
                   config.get(HbaseOptions.HBASE_ZK_RETRY));

    // Set hbase.hconnection.threads.max 64 to avoid OOM(default value: 256)
    hConfig.setInt("hbase.hconnection.threads.max",
                   config.get(HbaseOptions.HBASE_THREADS_MAX));

    this.hbase = ConnectionFactory.createConnection(hConfig);
}
 

private void addMetrics(HugeConfig config) {
    final MetricManager metric = MetricManager.INSTANCE;
    // Force to add server reporter
    ServerReporter reporter = ServerReporter.instance(metric.getRegistry());
    reporter.start(60L, TimeUnit.SECONDS);

    // Add metrics for MAX_WRITE_THREADS
    int maxWriteThreads = config.get(ServerOptions.MAX_WRITE_THREADS);
    MetricsUtil.registerGauge(RestServer.class, "max-write-threads", () -> {
        return maxWriteThreads;
    });

    // Add metrics for caches
    @SuppressWarnings({ "rawtypes", "unchecked" })
    Map<String, Cache<?, ?>> caches = (Map) CacheManager.instance()
                                                        .caches();
    registerCacheMetrics(caches);
    final AtomicInteger lastCachesSize = new AtomicInteger(caches.size());
    MetricsUtil.registerGauge(Cache.class, "instances", () -> {
        int count = caches.size();
        if (count != lastCachesSize.get()) {
            // Update if caches changed (effect in the next report period)
            registerCacheMetrics(caches);
            lastCachesSize.set(count);
        }
        return count;
    });

    // Add metrics for task
    MetricsUtil.registerGauge(TaskManager.class, "workers", () -> {
        return TaskManager.instance().workerPoolSize();
    });
    MetricsUtil.registerGauge(TaskManager.class, "pending-tasks", () -> {
        return TaskManager.instance().pendingTasks();
    });
}
 

public GraphManager(HugeConfig conf) {
    this.graphs = new ConcurrentHashMap<>();
    this.authenticator = HugeAuthenticator.loadAuthenticator(conf);

    this.loadGraphs(conf.getMap(ServerOptions.GRAPHS));
    // this.installLicense(conf, "");
    this.checkBackendVersionOrExit();
    this.restoreUncompletedTasks();
    this.addMetrics(conf);
}
 

private static HugeGraph loadGraph(boolean needClear) {
    HugeConfig config = FakeObjects.newConfig();
    HugeGraph graph = HugeFactory.open(config);

    if (needClear) {
        graph.clearBackend();
    }
    graph.initBackend();

    return graph;
}
 

public RocksDBSstSessions(HugeConfig config, String dataPath,
                          String database, String store,
                          List<String> tableNames) throws RocksDBException {
    this(config, dataPath, database, store);
    for (String table : tableNames) {
        this.createTable(table);
    }
}
 

public static void initAdminUser(String restConfFile) throws Exception {
    StandardAuthenticator auth = new StandardAuthenticator();
    HugeConfig config = new HugeConfig(restConfFile);
    String authClass = config.get(ServerOptions.AUTHENTICATOR);
    if (authClass.isEmpty()) {
        return;
    }
    auth.setup(config);
    auth.initAdminUser();
}
 

private static void checkBatchSize(HugeConfig config,
                                   List<JsonEdge> edges) {
    int max = config.get(ServerOptions.MAX_EDGES_PER_BATCH);
    if (edges.size() > max) {
        throw new IllegalArgumentException(String.format(
                  "Too many edges for one time post, " +
                  "the maximum number is '%s'", max));
    }
    if (edges.size() == 0) {
        throw new IllegalArgumentException(
                  "The number of edges can't be 0");
    }
}
 

protected boolean trace() {
    HugeConfig config = this.configProvider.get();
    if (config == null) {
        return false;
    }
    return config.get(ServerOptions.ALLOW_TRACE);
}
 

@Override
public synchronized void open(HugeConfig config) {
    E.checkNotNull(config, "config");

    if (this.sessions == null) {
        this.sessions = new HbaseSessions(config, this.namespace, this.store);
    }

    assert this.sessions != null;
    if (!this.sessions.closed()) {
        LOG.debug("Store {} has been opened before", this.store);
        this.sessions.useSession();
        return;
    }

    try {
        // NOTE: won't throw error even if connection refused
        this.sessions.open();
    } catch (Exception e) {
        if (!e.getMessage().contains("Column family not found")) {
            LOG.error("Failed to open HBase '{}'", this.store, e);
            throw new ConnectionException("Failed to connect to HBase", e);
        }
        if (this.isSchemaStore()) {
            LOG.info("Failed to open HBase '{}' with database '{}', " +
                     "try to init CF later", this.store, this.namespace);
        }
    }

    this.sessions.session();
    LOG.debug("Store opened: {}", this.store);
}
 

@Override
public void setup(HugeConfig config) {
    String graphName = config.get(ServerOptions.AUTH_GRAPH_STORE);
    String graphPath = config.getMap(ServerOptions.GRAPHS).get(graphName);
    E.checkArgument(graphPath != null,
                    "Invalid graph name '%s'", graphName);
    this.graph = (HugeGraph) GraphFactory.open(graphPath);
}
 

private RocksDBStdSessions(HugeConfig config, String database, String store,
                           RocksDBStdSessions origin) {
    super(config, database, store);

    this.rocksdb = origin.rocksdb;
    this.sstFileManager = origin.sstFileManager;
    this.cfs = origin.cfs;
    this.refCount = origin.refCount;

    this.refCount.incrementAndGet();
}
 

public StdSession(HugeConfig conf) {
    boolean bulkload = conf.get(RocksDBOptions.BULKLOAD_MODE);
    this.batch = new WriteBatch();
    this.writeOptions = new WriteOptions();
    this.writeOptions.setDisableWAL(bulkload);
    //this.writeOptions.setSync(false);
}
 

private static void checkBatchSize(HugeConfig config,
                                   List<JsonVertex> vertices) {
    int max = config.get(ServerOptions.MAX_VERTICES_PER_BATCH);
    if (vertices.size() > max) {
        throw new IllegalArgumentException(String.format(
                  "Too many vertices for one time post, " +
                  "the maximum number is '%s'", max));
    }
    if (vertices.size() == 0) {
        throw new IllegalArgumentException(
                  "The number of vertices can't be 0");
    }
}
 

private RocksDBSstSessions(HugeConfig config, String database, String store,
                           RocksDBSstSessions origin) {
    super(config, database, store);

    this.dataPath = origin.dataPath;
    this.tables = origin.tables;
}
 

private static RocksDBSessions open(String table) throws RocksDBException {
    HugeConfig config = FakeObjects.newConfig();
    RocksDBSessions rocks = new RocksDBStdSessions(config, "db", "store",
                                                   DB_PATH, DB_PATH);
    rocks.createTable(table);
    return rocks;
}