org.apache.accumulo.core.client.ZooKeeperInstance Java Examples

/**
 * Gets the AccumuloConnector singleton, starting the MiniAccumuloCluster on initialization.
 * This singleton instance is required so all test cases access the same MiniAccumuloCluster.
 *
 * @return Accumulo connector
 */
public static Connector getAccumuloConnector()
{
    if (connector != null) {
        return connector;
    }

    try {
        MiniAccumuloCluster accumulo = createMiniAccumuloCluster();
        Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
        connector = instance.getConnector(MAC_USER, new PasswordToken(MAC_PASSWORD));
        LOG.info("Connection to MAC instance %s at %s established, user %s password %s", accumulo.getInstanceName(), accumulo.getZooKeepers(), MAC_USER, MAC_PASSWORD);
        return connector;
    }
    catch (AccumuloException | AccumuloSecurityException | InterruptedException | IOException e) {
        throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to get connector to Accumulo", e);
    }
}
 

public static void main(String args[]) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {

        if (args.length != 7) {
            System.out.println("Usage: " + DailyShardSplitter.class.getName() + "<zookeepers> <instance> <username> <password> <tableName> <start day: yyyy-mm-dd> <stop day: yyyy-mm-dd>");
            System.exit(1);
        }

        String zookeepers = args[0];
        String instance = args[1];
        String username = args[2];
        String password = args[3];
        String tableName = args[4];
        DateTime start = DateTime.parse(args[5]);
        DateTime stop = DateTime.parse(args[6]);


        Instance accInst = new ZooKeeperInstance(instance, zookeepers);
        Connector connector = accInst.getConnector(username, password.getBytes());

        SortedSet<Text> shards = new DailyShardBuilder(Constants.DEFAULT_PARTITION_SIZE)
                .buildShardsInRange(start.toDate(), stop.toDate());

        connector.tableOperations().addSplits(tableName, shards);
    }
 

public synchronized Connector getConnector(
    final String zookeeperUrl,
    final String instanceName,
    final String userName,
    final String password) throws AccumuloException, AccumuloSecurityException {

  final ConnectorConfig config =
      new ConnectorConfig(zookeeperUrl, instanceName, userName, password);
  Connector connector = connectorCache.get(config);
  if (connector == null) {
    final Instance inst = new ZooKeeperInstance(instanceName, zookeeperUrl);
    connector = inst.getConnector(userName, password);
    connectorCache.put(config, connector);
  }
  return connector;
}
 

@Test
@Category(UnitTest.class)
public void testGetGeoTables() throws Exception
{
  ZooKeeperInstance zkinst = new ZooKeeperInstance(inst, zoo);
  PasswordToken pwTok = new PasswordToken(pw.getBytes());
  Connector conn = zkinst.getConnector(u, pwTok);
  Assert.assertNotNull(conn);

  PasswordToken token = new PasswordToken(pw.getBytes());

  //Authorizations auths = new Authorizations(authsStr.split(","));
  Authorizations auths = new Authorizations("A,B,C,D,ROLE_USER,U".split(","));
  System.out.println(auths.toString());
  Hashtable<String, String> ht = AccumuloUtils.getGeoTables(null, token, auths, conn);
  for (String k : ht.keySet())
  {
    System.out.println(k + " => " + ht.get(k));
  }


}
 

public static synchronized Connector getConnector(String instance, String zookeepers,
    String user, String pass) throws DataProviderException
{

  if (conn != null)
  {
    return conn;
  }
  if (checkMock())
  {
    return getMockConnector(instance, user, pass);
  }

  Instance inst = new ZooKeeperInstance(instance, zookeepers);
  try
  {
    conn = inst.getConnector(user, new PasswordToken(pass.getBytes()));
    return conn;
  }
  catch (Exception e)
  {
    throw new DataProviderException("problem creating connector", e);
  }
}
 

/**
 * Create a {@link Connector} that uses the provided connection details.
 *
 * @param username - The username the connection will use. (not null)
 * @param password - The password the connection will use. (not null)
 * @param instanceName - The name of the Accumulo instance. (not null)
 * @param zookeeperHostnames - A comma delimited list of the Zookeeper server hostnames. (not null)
 * @return A {@link Connector} that may be used to access the instance of Accumulo.
 * @throws AccumuloSecurityException Could not connect for security reasons.
 * @throws AccumuloException Could not connect for other reasons.
 */
public Connector connect(
        final String username,
        final CharSequence password,
        final String instanceName,
        final String zookeeperHostnames) throws AccumuloException, AccumuloSecurityException {
    requireNonNull(username);
    requireNonNull(password);
    requireNonNull(instanceName);
    requireNonNull(zookeeperHostnames);

    // Setup the password token that will be used.
    final PasswordToken token = new PasswordToken( password );

    // Connect to the instance of Accumulo.
    final Instance instance = new ZooKeeperInstance(instanceName, zookeeperHostnames);
    return instance.getConnector(username, token);
}
 

public static void main(String[] args) throws Exception {
    try (ConfigurableApplicationContext ctx = new SpringApplicationBuilder(SpringBootstrap.class)
            .bannerMode(Banner.Mode.OFF).web(WebApplicationType.NONE).run(args)) {
        Configuration conf = ctx.getBean(Configuration.class);
        HashMap<String, String> apacheConf = new HashMap<>();
        Accumulo accumuloConf = conf.getAccumulo();
        apacheConf.put("instance.name", accumuloConf.getInstanceName());
        apacheConf.put("instance.zookeeper.host", accumuloConf.getZookeepers());
        ClientConfiguration aconf = ClientConfiguration.fromMap(apacheConf);
        Instance instance = new ZooKeeperInstance(aconf);
        Connector con = instance.getConnector(accumuloConf.getUsername(),
                new PasswordToken(accumuloConf.getPassword()));

        TabletMetadataQuery query = new TabletMetadataQuery(con, conf.getMetricsTable());
        TabletMetadataView view = query.run();

        System.out.println(view.toText(TimeUnit.DAYS));
    }
}
 

public static void main(String[] args) throws Exception {

        try (ConfigurableApplicationContext ctx = new SpringApplicationBuilder(SpringBootstrap.class)
                .bannerMode(Mode.OFF).web(WebApplicationType.NONE).run(args)) {
            Configuration conf = ctx.getBean(Configuration.class);

            final Map<String, String> properties = new HashMap<>();
            Accumulo accumuloConf = conf.getAccumulo();
            properties.put("instance.name", accumuloConf.getInstanceName());
            properties.put("instance.zookeeper.host", accumuloConf.getZookeepers());
            final ClientConfiguration aconf = ClientConfiguration.fromMap(properties);
            final Instance instance = new ZooKeeperInstance(aconf);
            Connector con = instance.getConnector(accumuloConf.getUsername(),
                    new PasswordToken(accumuloConf.getPassword()));
            Scanner s = con.createScanner(conf.getMetaTable(),
                    con.securityOperations().getUserAuthorizations(con.whoami()));
            try {
                s.setRange(new Range(Meta.METRIC_PREFIX, true, Meta.TAG_PREFIX, false));
                for (Entry<Key, Value> e : s) {
                    System.out.println(e.getKey().getRow().toString().substring(Meta.METRIC_PREFIX.length()));
                }
            } finally {
                s.close();
            }
        }
    }
 

public DataStore(Configuration conf) throws QonduitException {

        try {
            final HashMap<String, String> apacheConf = new HashMap<>();
            Configuration.Accumulo accumuloConf = conf.getAccumulo();
            apacheConf.put("instance.name", accumuloConf.getInstanceName());
            apacheConf.put("instance.zookeeper.host", accumuloConf.getZookeepers());
            final ClientConfiguration aconf = ClientConfiguration.fromMap(apacheConf);
            final Instance instance = new ZooKeeperInstance(aconf);
            connector = instance.getConnector(accumuloConf.getUsername(),
                    new PasswordToken(accumuloConf.getPassword()));
        } catch (Exception e) {
            throw new QonduitException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), "Error creating DataStoreImpl",
                    e.getMessage(), e);
        }
    }
 

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    
    Configuration conf = context.getConfiguration();
    String user = conf.get(MetricsConfig.USER);
    String password = conf.get(MetricsConfig.PASS);
    String instance = conf.get(MetricsConfig.INSTANCE);
    String zookeepers = conf.get(MetricsConfig.ZOOKEEPERS);
    
    useHourlyPrecision = HourlyPrecisionHelper.checkForHourlyPrecisionOption(context.getConfiguration(), log);
    
    try {
        ZooKeeperInstance inst = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instance).withZkHosts(zookeepers));
        Connector con = inst.getConnector(user, new PasswordToken(password));
        ingestScanner = con.createScanner(conf.get(MetricsConfig.INGEST_TABLE, MetricsConfig.DEFAULT_INGEST_TABLE), Authorizations.EMPTY);
    } catch (TableNotFoundException | AccumuloException | AccumuloSecurityException e) {
        throw new IOException(e);
    }
}
 

@Override
public Connector get()
{
    try {
        Instance inst = new ZooKeeperInstance(instance, zooKeepers);
        Connector connector = inst.getConnector(username, new PasswordToken(password.getBytes(UTF_8)));
        LOG.info("Connection to instance %s at %s established, user %s", instance, zooKeepers, username);
        return connector;
    }
    catch (AccumuloException | AccumuloSecurityException e) {
        throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to get connector to Accumulo", e);
    }
}
 

private static PeriodicQueryResultStorage getPeriodicQueryResultStorage(final PeriodicNotificationApplicationConfiguration conf)
        throws AccumuloException, AccumuloSecurityException {
    final Instance instance = new ZooKeeperInstance(conf.getAccumuloInstance(), conf.getAccumuloZookeepers());
    final Connector conn = instance.getConnector(conf.getAccumuloUser(), new PasswordToken(conf.getAccumuloPassword()));
    final String ryaInstance = conf.getTablePrefix();
    return new AccumuloPeriodicQueryResultStorage(conn, ryaInstance);
}
 

@Test
@Category(UnitTest.class)
public void testGetTile() throws Exception
{

  ZooKeeperInstance zkinst = new ZooKeeperInstance(inst, zoo);
  PasswordToken pwTok = new PasswordToken(pw.getBytes());
  Connector conn = zkinst.getConnector(u, pwTok);
  Assert.assertNotNull(conn);

  PasswordToken token = new PasswordToken(pw.getBytes());

  Authorizations auths = new Authorizations(authsStr.split(","));
  long start = 0;
  long end = Long.MAX_VALUE;
  Key sKey = AccumuloUtils.toKey(start);
  Key eKey = AccumuloUtils.toKey(end);
  Range r = new Range(sKey, eKey);
  Scanner s = conn.createScanner("paris4", auths);
  s.fetchColumnFamily(new Text(Integer.toString(10)));
  s.setRange(r);

  Iterator<Entry<Key, Value>> it = s.iterator();
  while (it.hasNext())
  {
    Entry<Key, Value> ent = it.next();
    if (ent == null)
    {
      return;
    }
    System.out.println("current key   = " + AccumuloUtils.toLong(ent.getKey().getRow()));
    System.out.println("current value = " + ent.getValue().getSize());
  }

}
 

public OsmProvider(final OSMIngestCommandArgs args, final AccumuloRequiredOptions store)
    throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
  conn =
      new ZooKeeperInstance(store.getInstance(), store.getZookeeper()).getConnector(
          store.getUser(),
          new PasswordToken(store.getPassword()));
  bs =
      conn.createBatchScanner(
          args.getQualifiedTableName(),
          new Authorizations(args.getVisibilityOptions().getVisibility()),
          1);
}
 

private static MiniAccumuloCluster startMiniAccumulo() throws IOException, InterruptedException, AccumuloException, AccumuloSecurityException {
    final File miniDataDir = Files.createTempDir();

    // Setup and start the Mini Accumulo.
    final MiniAccumuloCluster accumulo = new MiniAccumuloCluster(
            miniDataDir, "password");
    accumulo.start();

    // Store a connector to the Mini Accumulo.
    final Instance instance = new ZooKeeperInstance(
            accumulo.getInstanceName(), accumulo.getZooKeepers());
    accCon = instance.getConnector("root", new PasswordToken("password"));

    return accumulo;
}
 

@BeforeClass
public static void beforeClass() throws Exception {
    Logger.getLogger(ClientCnxn.class).setLevel(Level.ERROR);

    // Setup and start the Mini Accumulo.
    cluster = clusterInstance.getCluster();

    // Store a connector to the Mini Accumulo.
    instanceName = cluster.getInstanceName();
    zookeepers = cluster.getZooKeepers();

    final Instance instance = new ZooKeeperInstance(instanceName, zookeepers);
    accumuloConn = instance.getConnector(clusterInstance.getUsername(), new PasswordToken(clusterInstance.getPassword()));
}
 

/**
 * Create an {@link Instance} that may be used to create {@link Connector}s
 * to Accumulo. If the configuration has the {@link #USE_MOCK_INSTANCE} flag
 * set, then the instance will be be a {@link MockInstance} instead of a
 * Zookeeper backed instance.
 *
 * @param conf - The configuration object that will be interrogated. (not null)
 * @return The {@link Instance} that may be used to connect to Accumulo.
 */
public static Instance getInstance(final Configuration conf) {
    // Pull out the Accumulo specific configuration values.
    final AccumuloRdfConfiguration accConf = new AccumuloRdfConfiguration(conf);
    final String instanceName = accConf.getInstanceName();
    final String zoookeepers = accConf.getZookeepers();

    // Create an Instance a mock if the mock flag is set.
    if (useMockInstance(conf)) {
        return new MockInstance(instanceName);
    }

    // Otherwise create an Instance to a Zookeeper managed instance of Accumulo.
    return new ZooKeeperInstance(instanceName, zoookeepers);
}
 

/**
 * Sets up the {@link MiniAccumuloCluster} or the {@link MockInstance}.
 * @throws Exception
 */
public void setUpInstance() throws Exception {
    if (!isMock) {
        log.info("Setting up " + driverName + " MiniAccumulo cluster...");
        // Create and Run MiniAccumulo Cluster
        tempDir = Files.createTempDir();
        tempDir.deleteOnExit();
        miniAccumuloCluster = new MiniAccumuloCluster(tempDir, userpwd);
        copyHadoopHomeToTemp();
        miniAccumuloCluster.getConfig().setInstanceName(instanceName);
        log.info(driverName + " MiniAccumulo instance starting up...");
        miniAccumuloCluster.start();
        Thread.sleep(1000);
        log.info(driverName + " MiniAccumulo instance started");
        log.info("Creating connector to " + driverName + " MiniAccumulo instance...");
        zooKeeperInstance = new ZooKeeperInstance(miniAccumuloCluster.getClientConfig());
        instance = zooKeeperInstance;
        connector = zooKeeperInstance.getConnector(user, new PasswordToken(userpwd));
        log.info("Created connector to " + driverName + " MiniAccumulo instance");
    } else {
        log.info("Setting up " + driverName + " mock instance...");
        mockInstance = new MockInstance(instanceName);
        instance = mockInstance;
        connector = mockInstance.getConnector(user, new PasswordToken(userpwd));
        log.info("Created connector to " + driverName + " mock instance");
    }
    zooKeepers = instance.getZooKeepers();
}
 

/**
 * Setup a Mini Accumulo cluster that uses a temporary directory to store its data.
 *
 * @return A Mini Accumulo cluster.
 */
private static MiniAccumuloCluster startMiniAccumulo() throws IOException, InterruptedException, AccumuloException, AccumuloSecurityException {
    final File miniDataDir = Files.createTempDir();

    // Setup and start the Mini Accumulo.
    final MiniAccumuloCluster accumulo = new MiniAccumuloCluster(miniDataDir, "password");
    accumulo.start();

    // Store a connector to the Mini Accumulo.
    final Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
    accumuloConn = instance.getConnector("root", new PasswordToken("password"));

    return accumulo;
}
 

@BeforeClass
public static void beforeClass() throws Exception {
    Logger.getLogger(ClientCnxn.class).setLevel(Level.ERROR);

    // Setup and start the Mini Accumulo.
    cluster = clusterInstance.getCluster();

    // Store a connector to the Mini Accumulo.
    instanceName = cluster.getInstanceName();
    zookeepers = cluster.getZooKeepers();

    final Instance instance = new ZooKeeperInstance(instanceName, zookeepers);
    accumuloConn = instance.getConnector(clusterInstance.getUsername(), new PasswordToken(clusterInstance.getPassword()));
}
 

@Override
public Optional<IncrementalResultExporter> build(final Context context) throws IncrementalExporterFactoryException, ConfigurationException {
    checkNotNull(context);
    // Wrap the context's parameters for parsing.
    final RyaExportParameters params = new RyaExportParameters( context.getObserverConfiguration().toMap() );

    if(params.getUseRyaBindingSetExporter()) {
        // Setup Zookeeper connection info.
        final String accumuloInstance = params.getAccumuloInstanceName().get();
        final String zookeeperServers =  params.getZookeeperServers().get().replaceAll(";", ",");
        final Instance inst = new ZooKeeperInstance(accumuloInstance, zookeeperServers);

        try {
            // Setup Accumulo connection info.
            final String exporterUsername = params.getExporterUsername().get();
            final String exporterPassword = params.getExporterPassword().get();
            final Connector accumuloConn = inst.getConnector(exporterUsername, new PasswordToken(exporterPassword));

            // Setup Rya PCJ Storage.
            final String ryaInstanceName = params.getRyaInstanceName().get();
            final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, ryaInstanceName);

            // Make the exporter.
            final IncrementalBindingSetExporter exporter = new RyaBindingSetExporter(pcjStorage);
            return Optional.of(exporter);

        } catch (final AccumuloException | AccumuloSecurityException e) {
            throw new IncrementalExporterFactoryException("Could not initialize the Accumulo connector using the provided configuration.", e);
        }
    } else {
        return Optional.absent();
    }
}
 

private static Connector createAccumuloConnector(final PcjAdminClientProperties clientProps) throws AccumuloException, AccumuloSecurityException {
    checkNotNull(clientProps);

    // Connect to the Zookeepers.
    final String instanceName = clientProps.getAccumuloInstance();
    final String zooServers = clientProps.getAccumuloZookeepers();
    final Instance inst = new ZooKeeperInstance(instanceName, zooServers);

    // Create a connector to the Accumulo that hosts the PCJ export tables.
    return inst.getConnector(clientProps.getAccumuloUsername(), new PasswordToken(clientProps.getAccumuloPassword()));
}
 

public AccumuloCounterSource(String instanceStr, String zookeepers, String username, String password, String table) throws AccumuloException,
                AccumuloSecurityException {
    ZooKeeperInstance instance = new ZooKeeperInstance(instanceStr, zookeepers);
    connector = instance.getConnector(username, new PasswordToken(password));
    queryTable = table;
    this.username = username;
}
 

public Set<CardinalityIntersectionRecord> scanCardinalities(List<String> fields, DateAggregationType dateAggregationType,
                DatatypeAggregationType datatypeAggregationType) throws Exception {
    
    Map<CardinalityIntersectionRecord,HyperLogLogPlus> cardinalityMap = new TreeMap<>();
    Scanner scanner = null;
    try {
        ZooKeeperInstance instance = new ZooKeeperInstance(config.getInstanceName(), config.getZookeepers());
        Connector connector = instance.getConnector(config.getUsername(), new PasswordToken(config.getPassword()));
        Collection<Authorizations> authCollection = Collections.singleton(new Authorizations(config.getAuths().split(",")));
        if (!connector.tableOperations().exists(config.getTableName())) {
            throw new IllegalArgumentException("Table " + config.getTableName() + " does not exist");
        }
        scanner = ScannerHelper.createScanner(connector, config.getTableName(), authCollection);
        Range r = new Range(config.getBeginDate(), config.getEndDate() + "\0");
        scanner.setRange(r);
        
        Iterator<Map.Entry<Key,Value>> itr = scanner.iterator();
        while (itr.hasNext()) {
            Map.Entry<Key,Value> nextEntry = itr.next();
            Key key = nextEntry.getKey();
            String field = key.getColumnFamily().toString();
            if (fields != null && !fields.isEmpty() && !fields.contains(field)) {
                continue;
            } else {
                addEntry(cardinalityMap, nextEntry, dateAggregationType, datatypeAggregationType);
            }
        }
    } catch (Exception e) {
        log.error(e);
    } finally {
        if (scanner != null) {
            scanner.close();
            
        }
    }
    return cardinalityMap.keySet();
}
 

protected void setTableIdsAndConfigs() throws IOException {
    ZooKeeperInstance instance = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(conf.get(INSTANCE_NAME))
                    .withZkHosts(conf.get(ZOOKEEPERS)));
    Connector connector = null;
    tableConfigs = new HashMap<>();
    Iterable<String> localityGroupTables = Splitter.on(",").split(conf.get(CONFIGURE_LOCALITY_GROUPS, ""));
    try {
        connector = instance.getConnector(conf.get(USERNAME), new PasswordToken(Base64.decodeBase64(conf.get(PASSWORD))));
        
        tableIds = connector.tableOperations().tableIdMap();
        Set<String> compressionTableBlackList = getCompressionTableBlackList(conf);
        String compressionType = getCompressionType(conf);
        for (String tableName : tableIds.keySet()) {
            ConfigurationCopy tableConfig = new ConfigurationCopy(connector.tableOperations().getProperties(tableName));
            tableConfig.set(Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), (compressionTableBlackList.contains(tableName) ? Compression.COMPRESSION_NONE
                            : compressionType));
            if (Iterables.contains(localityGroupTables, tableName)) {
                Map<String,Set<Text>> localityGroups = connector.tableOperations().getLocalityGroups(tableName);
                // pull the locality groups for this table.
                Map<Text,String> cftlg = Maps.newHashMap();
                Map<String,Set<ByteSequence>> lgtcf = Maps.newHashMap();
                for (Entry<String,Set<Text>> locs : localityGroups.entrySet()) {
                    lgtcf.put(locs.getKey(), new HashSet<>());
                    for (Text loc : locs.getValue()) {
                        cftlg.put(loc, locs.getKey());
                        lgtcf.get(locs.getKey()).add(new ArrayByteSequence(loc.getBytes()));
                    }
                }
                columnFamilyToLocalityGroup.put(tableName, cftlg);
                localityGroupToColumnFamilies.put(tableName, lgtcf);
            }
            tableConfigs.put(tableName, tableConfig);
            
        }
    } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
        throw new IOException("Unable to get configuration.  Please call MultiRFileOutput.setAccumuloConfiguration with the proper credentials", e);
    }
}
 

public Connector createConnector() {
    try {
        LOGGER.info("Connecting to accumulo instance [%s] zookeeper servers [%s]", this.getAccumuloInstanceName(), this.getZookeeperServers());
        ZooKeeperInstance instance = new ZooKeeperInstance(getClientConfiguration());
        return instance.getConnector(this.getAccumuloUsername(), this.getAuthenticationToken());
    } catch (Exception ex) {
        throw new VertexiumException(
            String.format("Could not connect to Accumulo instance [%s] zookeeper servers [%s]", this.getAccumuloInstanceName(), this.getZookeeperServers()),
            ex
        );
    }
}
 

@Override
public Optional<IncrementalResultExporter> build(Context context) throws IncrementalExporterFactoryException, ConfigurationException {
    checkNotNull(context);

    // Wrap the context's parameters for parsing.
    final RyaExportParameters params = new RyaExportParameters( context.getObserverConfiguration().toMap() );

    if(params.getUsePeriodicBindingSetExporter()) {
        // Setup Zookeeper connection info.
        final String accumuloInstance = params.getAccumuloInstanceName().get();
        final String zookeeperServers =  params.getZookeeperServers().get().replaceAll(";", ",");
        final Instance inst = new ZooKeeperInstance(accumuloInstance, zookeeperServers);

        try {
            // Setup Accumulo connection info.
            final String exporterUsername = params.getExporterUsername().get();
            final String exporterPassword = params.getExporterPassword().get();
            final Connector accumuloConn = inst.getConnector(exporterUsername, new PasswordToken(exporterPassword));

            // Setup Rya PCJ Storage.
            final String ryaInstanceName = params.getRyaInstanceName().get();
            final PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(accumuloConn, ryaInstanceName);
            
            // Make the exporter.
            final IncrementalBindingSetExporter exporter = new PeriodicBindingSetExporter(periodicStorage);
            return Optional.of(exporter);

        } catch (final AccumuloException | AccumuloSecurityException e) {
            throw new IncrementalExporterFactoryException("Could not initialize the Accumulo connector using the provided configuration.", e);
        }
    } else {
        return Optional.absent();
    }
}
 

@Test
public void testIngestOSMPBF() throws Exception {
  TestUtils.deleteAll(dataStoreOptions);
  // NOTE: This will probably fail unless you bump up the memory for the
  // tablet
  // servers, for whatever reason, using the
  // miniAccumuloConfig.setMemory() function.
  final MapReduceTestEnvironment mrEnv = MapReduceTestEnvironment.getInstance();

  // TODO: for now this only works with accumulo, generalize the data
  // store usage
  final AccumuloStoreTestEnvironment accumuloEnv = AccumuloStoreTestEnvironment.getInstance();

  final String hdfsPath = mrEnv.getHdfsBaseDirectory() + "/osm_stage/";

  final StageOSMToHDFSCommand stage = new StageOSMToHDFSCommand();
  stage.setParameters(TEST_DATA_BASE_DIR, hdfsPath);
  stage.execute(mrEnv.getOperationParams());

  final Connector conn =
      new ZooKeeperInstance(
          accumuloEnv.getAccumuloInstance(),
          accumuloEnv.getZookeeper()).getConnector(
              accumuloEnv.getAccumuloUser(),
              new PasswordToken(accumuloEnv.getAccumuloPassword()));
  final Authorizations auth = new Authorizations(new String[] {"public"});
  conn.securityOperations().changeUserAuthorizations(accumuloEnv.getAccumuloUser(), auth);
  final IngestOSMToGeoWaveCommand ingest = new IngestOSMToGeoWaveCommand();
  ingest.setParameters(hdfsPath, "test-store");

  final AddStoreCommand addStore = new AddStoreCommand();
  addStore.setParameters("test-store");
  addStore.setPluginOptions(dataStoreOptions);
  addStore.execute(mrEnv.getOperationParams());

  ingest.getIngestOptions().setJobName("ConversionTest");

  // Execute for node's ways, and relations.
  ingest.getIngestOptions().setMapperType("NODE");
  ingest.execute(mrEnv.getOperationParams());
  System.out.println("finished accumulo ingest Node");

  ingest.getIngestOptions().setMapperType("WAY");
  ingest.execute(mrEnv.getOperationParams());
  System.out.println("finished accumulo ingest Way");

  ingest.getIngestOptions().setMapperType("RELATION");
  ingest.execute(mrEnv.getOperationParams());
  System.out.println("finished accumulo ingest Relation");
}
 

private void maybeUpdateTypeMetadataInHdfs(final SharedCacheCoordinator watcher, String triStateName, String metadataTableName) throws Exception {
    
    boolean locked = false;
    InterProcessMutex lock = (InterProcessMutex) watcher.getMutex("lock");
    try {
        locked = lock.acquire(this.lockWaitTime, TimeUnit.MILLISECONDS);
        if (!locked)
            log.debug("table:" + metadataTableName + " Unable to acquire lock to update " + metadataTableName
                            + ". Another webserver is updating the typeMetadata.");
        else
            log.debug("table:" + metadataTableName + " Obtained lock on updateTypeMetadata for " + metadataTableName);
    } catch (Exception e) {
        log.warn("table:" + metadataTableName + " Got Exception trying to acquire lock to update " + metadataTableName + ".", e);
    }
    
    try {
        if (locked) {
            try {
                log.debug("table:" + metadataTableName + " checkTriState(" + triStateName + ", " + SharedTriState.STATE.NEEDS_UPDATE);
                if (watcher.checkTriState(triStateName, SharedTriState.STATE.NEEDS_UPDATE)) {
                    if (log.isDebugEnabled()) {
                        log.debug("table:" + metadataTableName + " " + this + " STATE is NEEDS_UPDATE. Will write the TypeMetadata map to hdfs");
                    }
                    watcher.setTriState(triStateName, SharedTriState.STATE.UPDATING);
                    if (log.isDebugEnabled()) {
                        log.debug("table:" + metadataTableName + " " + this + " setTriState to UPDATING");
                    }
                    // get a connection for my MetadataHelper, and get the TypeMetadata map
                    ZooKeeperInstance instance = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(this.instance)
                                    .withZkHosts(this.zookeepers));
                    Connector connector = instance.getConnector(this.username, new PasswordToken(this.password));
                    TypeMetadataHelper typeMetadataHelper = this.typeMetadataHelperFactory.createTypeMetadataHelper(connector, metadataTableName,
                                    allMetadataAuths, false);
                    typeMetadataWriter.writeTypeMetadataMap(typeMetadataHelper.getTypeMetadataMap(this.allMetadataAuths), metadataTableName);
                    if (log.isDebugEnabled()) {
                        log.debug("table:" + metadataTableName + " " + this + " set the sharedTriState needsUpdate to UPDATED for " + metadataTableName);
                    }
                    watcher.setTriState(triStateName, SharedTriState.STATE.UPDATED);
                } else {
                    if (log.isDebugEnabled()) {
                        log.debug("table:"
                                        + metadataTableName
                                        + " "
                                        + this
                                        + "  STATE is not NEEDS_UPDATE! Someone else may be writing or has already written the TypeMetadata map, just release the lock");
                    }
                }
            } catch (Exception ex) {
                log.warn("table:" + metadataTableName + " Unable to write TypeMetadataMap for " + metadataTableName, ex);
                watcher.setTriState(triStateName, SharedTriState.STATE.NEEDS_UPDATE);
                if (log.isDebugEnabled()) {
                    log.debug("After exception, set the SharedTriState STATE to NEEDS_UPDATE");
                }
                
            }
        }
    } finally {
        if (locked) {
            lock.release();
            if (log.isTraceEnabled())
                log.trace("table:" + metadataTableName + " " + this + " released the lock for " + metadataTableName);
            
        }
    }
}
 

protected static Instance getInstance(Configuration conf) {
    if (conf.getBoolean(MOCK, false)) {
        return new InMemoryInstance(conf.get(INSTANCE_NAME));
    }
    return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(conf.get(INSTANCE_NAME)).withZkHosts(conf.get(ZOOKEEPERS)));
}