org.apache.accumulo.core.client.ZooKeeperInstance Java Examples

The following examples show how to use org.apache.accumulo.core.client.ZooKeeperInstance. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AccumuloQueryRunner.java    From presto with Apache License 2.0 6 votes vote down vote up
/**
 * Gets the AccumuloConnector singleton, starting the MiniAccumuloCluster on initialization.
 * This singleton instance is required so all test cases access the same MiniAccumuloCluster.
 *
 * @return Accumulo connector
 */
public static Connector getAccumuloConnector()
{
    if (connector != null) {
        return connector;
    }

    try {
        MiniAccumuloCluster accumulo = createMiniAccumuloCluster();
        Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
        connector = instance.getConnector(MAC_USER, new PasswordToken(MAC_PASSWORD));
        LOG.info("Connection to MAC instance %s at %s established, user %s password %s", accumulo.getInstanceName(), accumulo.getZooKeepers(), MAC_USER, MAC_PASSWORD);
        return connector;
    }
    catch (AccumuloException | AccumuloSecurityException | InterruptedException | IOException e) {
        throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to get connector to Accumulo", e);
    }
}
 
Example #2
Source File: DailyShardSplitter.java    From accumulo-recipes with Apache License 2.0 6 votes vote down vote up
public static void main(String args[]) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {

        if (args.length != 7) {
            System.out.println("Usage: " + DailyShardSplitter.class.getName() + "<zookeepers> <instance> <username> <password> <tableName> <start day: yyyy-mm-dd> <stop day: yyyy-mm-dd>");
            System.exit(1);
        }

        String zookeepers = args[0];
        String instance = args[1];
        String username = args[2];
        String password = args[3];
        String tableName = args[4];
        DateTime start = DateTime.parse(args[5]);
        DateTime stop = DateTime.parse(args[6]);


        Instance accInst = new ZooKeeperInstance(instance, zookeepers);
        Connector connector = accInst.getConnector(username, password.getBytes());

        SortedSet<Text> shards = new DailyShardBuilder(Constants.DEFAULT_PARTITION_SIZE)
                .buildShardsInRange(start.toDate(), stop.toDate());

        connector.tableOperations().addSplits(tableName, shards);
    }
 
Example #3
Source File: ConnectorPool.java    From geowave with Apache License 2.0 6 votes vote down vote up
public synchronized Connector getConnector(
    final String zookeeperUrl,
    final String instanceName,
    final String userName,
    final String password) throws AccumuloException, AccumuloSecurityException {

  final ConnectorConfig config =
      new ConnectorConfig(zookeeperUrl, instanceName, userName, password);
  Connector connector = connectorCache.get(config);
  if (connector == null) {
    final Instance inst = new ZooKeeperInstance(instanceName, zookeeperUrl);
    connector = inst.getConnector(userName, password);
    connectorCache.put(config, connector);
  }
  return connector;
}
 
Example #4
Source File: AccumuloGeoTableTest.java    From mrgeo with Apache License 2.0 6 votes vote down vote up
@Test
@Category(UnitTest.class)
public void testGetGeoTables() throws Exception
{
  ZooKeeperInstance zkinst = new ZooKeeperInstance(inst, zoo);
  PasswordToken pwTok = new PasswordToken(pw.getBytes());
  Connector conn = zkinst.getConnector(u, pwTok);
  Assert.assertNotNull(conn);

  PasswordToken token = new PasswordToken(pw.getBytes());

  //Authorizations auths = new Authorizations(authsStr.split(","));
  Authorizations auths = new Authorizations("A,B,C,D,ROLE_USER,U".split(","));
  System.out.println(auths.toString());
  Hashtable<String, String> ht = AccumuloUtils.getGeoTables(null, token, auths, conn);
  for (String k : ht.keySet())
  {
    System.out.println(k + " => " + ht.get(k));
  }


}
 
Example #5
Source File: AccumuloConnector.java    From mrgeo with Apache License 2.0 6 votes vote down vote up
public static synchronized Connector getConnector(String instance, String zookeepers,
    String user, String pass) throws DataProviderException
{

  if (conn != null)
  {
    return conn;
  }
  if (checkMock())
  {
    return getMockConnector(instance, user, pass);
  }

  Instance inst = new ZooKeeperInstance(instance, zookeepers);
  try
  {
    conn = inst.getConnector(user, new PasswordToken(pass.getBytes()));
    return conn;
  }
  catch (Exception e)
  {
    throw new DataProviderException("problem creating connector", e);
  }
}
 
Example #6
Source File: ConnectorFactory.java    From rya with Apache License 2.0 6 votes vote down vote up
/**
 * Create a {@link Connector} that uses the provided connection details.
 *
 * @param username - The username the connection will use. (not null)
 * @param password - The password the connection will use. (not null)
 * @param instanceName - The name of the Accumulo instance. (not null)
 * @param zookeeperHostnames - A comma delimited list of the Zookeeper server hostnames. (not null)
 * @return A {@link Connector} that may be used to access the instance of Accumulo.
 * @throws AccumuloSecurityException Could not connect for security reasons.
 * @throws AccumuloException Could not connect for other reasons.
 */
public Connector connect(
        final String username,
        final CharSequence password,
        final String instanceName,
        final String zookeeperHostnames) throws AccumuloException, AccumuloSecurityException {
    requireNonNull(username);
    requireNonNull(password);
    requireNonNull(instanceName);
    requireNonNull(zookeeperHostnames);

    // Setup the password token that will be used.
    final PasswordToken token = new PasswordToken( password );

    // Connect to the instance of Accumulo.
    final Instance instance = new ZooKeeperInstance(instanceName, zookeeperHostnames);
    return instance.getConnector(username, token);
}
 
Example #7
Source File: TabletMetadataConsole.java    From timely with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    try (ConfigurableApplicationContext ctx = new SpringApplicationBuilder(SpringBootstrap.class)
            .bannerMode(Banner.Mode.OFF).web(WebApplicationType.NONE).run(args)) {
        Configuration conf = ctx.getBean(Configuration.class);
        HashMap<String, String> apacheConf = new HashMap<>();
        Accumulo accumuloConf = conf.getAccumulo();
        apacheConf.put("instance.name", accumuloConf.getInstanceName());
        apacheConf.put("instance.zookeeper.host", accumuloConf.getZookeepers());
        ClientConfiguration aconf = ClientConfiguration.fromMap(apacheConf);
        Instance instance = new ZooKeeperInstance(aconf);
        Connector con = instance.getConnector(accumuloConf.getUsername(),
                new PasswordToken(accumuloConf.getPassword()));

        TabletMetadataQuery query = new TabletMetadataQuery(con, conf.getMetricsTable());
        TabletMetadataView view = query.run();

        System.out.println(view.toText(TimeUnit.DAYS));
    }
}
 
Example #8
Source File: GetMetricTableSplitPoints.java    From timely with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {

        try (ConfigurableApplicationContext ctx = new SpringApplicationBuilder(SpringBootstrap.class)
                .bannerMode(Mode.OFF).web(WebApplicationType.NONE).run(args)) {
            Configuration conf = ctx.getBean(Configuration.class);

            final Map<String, String> properties = new HashMap<>();
            Accumulo accumuloConf = conf.getAccumulo();
            properties.put("instance.name", accumuloConf.getInstanceName());
            properties.put("instance.zookeeper.host", accumuloConf.getZookeepers());
            final ClientConfiguration aconf = ClientConfiguration.fromMap(properties);
            final Instance instance = new ZooKeeperInstance(aconf);
            Connector con = instance.getConnector(accumuloConf.getUsername(),
                    new PasswordToken(accumuloConf.getPassword()));
            Scanner s = con.createScanner(conf.getMetaTable(),
                    con.securityOperations().getUserAuthorizations(con.whoami()));
            try {
                s.setRange(new Range(Meta.METRIC_PREFIX, true, Meta.TAG_PREFIX, false));
                for (Entry<Key, Value> e : s) {
                    System.out.println(e.getKey().getRow().toString().substring(Meta.METRIC_PREFIX.length()));
                }
            } finally {
                s.close();
            }
        }
    }
 
Example #9
Source File: DataStore.java    From qonduit with Apache License 2.0 6 votes vote down vote up
public DataStore(Configuration conf) throws QonduitException {

        try {
            final HashMap<String, String> apacheConf = new HashMap<>();
            Configuration.Accumulo accumuloConf = conf.getAccumulo();
            apacheConf.put("instance.name", accumuloConf.getInstanceName());
            apacheConf.put("instance.zookeeper.host", accumuloConf.getZookeepers());
            final ClientConfiguration aconf = ClientConfiguration.fromMap(apacheConf);
            final Instance instance = new ZooKeeperInstance(aconf);
            connector = instance.getConnector(accumuloConf.getUsername(),
                    new PasswordToken(accumuloConf.getPassword()));
        } catch (Exception e) {
            throw new QonduitException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), "Error creating DataStoreImpl",
                    e.getMessage(), e);
        }
    }
 
Example #10
Source File: IngestMetricsSummaryLoader.java    From datawave with Apache License 2.0 6 votes vote down vote up
@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    
    Configuration conf = context.getConfiguration();
    String user = conf.get(MetricsConfig.USER);
    String password = conf.get(MetricsConfig.PASS);
    String instance = conf.get(MetricsConfig.INSTANCE);
    String zookeepers = conf.get(MetricsConfig.ZOOKEEPERS);
    
    useHourlyPrecision = HourlyPrecisionHelper.checkForHourlyPrecisionOption(context.getConfiguration(), log);
    
    try {
        ZooKeeperInstance inst = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instance).withZkHosts(zookeepers));
        Connector con = inst.getConnector(user, new PasswordToken(password));
        ingestScanner = con.createScanner(conf.get(MetricsConfig.INGEST_TABLE, MetricsConfig.DEFAULT_INGEST_TABLE), Authorizations.EMPTY);
    } catch (TableNotFoundException | AccumuloException | AccumuloSecurityException e) {
        throw new IOException(e);
    }
}
 
Example #11
Source File: AccumuloModule.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
public Connector get()
{
    try {
        Instance inst = new ZooKeeperInstance(instance, zooKeepers);
        Connector connector = inst.getConnector(username, new PasswordToken(password.getBytes(UTF_8)));
        LOG.info("Connection to instance %s at %s established, user %s", instance, zooKeepers, username);
        return connector;
    }
    catch (AccumuloException | AccumuloSecurityException e) {
        throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to get connector to Accumulo", e);
    }
}
 
Example #12
Source File: PeriodicNotificationApplicationFactory.java    From rya with Apache License 2.0 5 votes vote down vote up
private static PeriodicQueryResultStorage getPeriodicQueryResultStorage(final PeriodicNotificationApplicationConfiguration conf)
        throws AccumuloException, AccumuloSecurityException {
    final Instance instance = new ZooKeeperInstance(conf.getAccumuloInstance(), conf.getAccumuloZookeepers());
    final Connector conn = instance.getConnector(conf.getAccumuloUser(), new PasswordToken(conf.getAccumuloPassword()));
    final String ryaInstance = conf.getTablePrefix();
    return new AccumuloPeriodicQueryResultStorage(conn, ryaInstance);
}
 
Example #13
Source File: AccumuloGeoTableTest.java    From mrgeo with Apache License 2.0 5 votes vote down vote up
@Test
@Category(UnitTest.class)
public void testGetTile() throws Exception
{

  ZooKeeperInstance zkinst = new ZooKeeperInstance(inst, zoo);
  PasswordToken pwTok = new PasswordToken(pw.getBytes());
  Connector conn = zkinst.getConnector(u, pwTok);
  Assert.assertNotNull(conn);

  PasswordToken token = new PasswordToken(pw.getBytes());

  Authorizations auths = new Authorizations(authsStr.split(","));
  long start = 0;
  long end = Long.MAX_VALUE;
  Key sKey = AccumuloUtils.toKey(start);
  Key eKey = AccumuloUtils.toKey(end);
  Range r = new Range(sKey, eKey);
  Scanner s = conn.createScanner("paris4", auths);
  s.fetchColumnFamily(new Text(Integer.toString(10)));
  s.setRange(r);

  Iterator<Entry<Key, Value>> it = s.iterator();
  while (it.hasNext())
  {
    Entry<Key, Value> ent = it.next();
    if (ent == null)
    {
      return;
    }
    System.out.println("current key   = " + AccumuloUtils.toLong(ent.getKey().getRow()));
    System.out.println("current value = " + ent.getValue().getSize());
  }

}
 
Example #14
Source File: OsmProvider.java    From geowave with Apache License 2.0 5 votes vote down vote up
public OsmProvider(final OSMIngestCommandArgs args, final AccumuloRequiredOptions store)
    throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
  conn =
      new ZooKeeperInstance(store.getInstance(), store.getZookeeper()).getConnector(
          store.getUser(),
          new PasswordToken(store.getPassword()));
  bs =
      conn.createBatchScanner(
          args.getQualifiedTableName(),
          new Authorizations(args.getVisibilityOptions().getVisibility()),
          1);
}
 
Example #15
Source File: AccumuloIndexSetColumnVisibilityTest.java    From rya with Apache License 2.0 5 votes vote down vote up
private static MiniAccumuloCluster startMiniAccumulo() throws IOException, InterruptedException, AccumuloException, AccumuloSecurityException {
    final File miniDataDir = Files.createTempDir();

    // Setup and start the Mini Accumulo.
    final MiniAccumuloCluster accumulo = new MiniAccumuloCluster(
            miniDataDir, "password");
    accumulo.start();

    // Store a connector to the Mini Accumulo.
    final Instance instance = new ZooKeeperInstance(
            accumulo.getInstanceName(), accumulo.getZooKeepers());
    accCon = instance.getConnector("root", new PasswordToken("password"));

    return accumulo;
}
 
Example #16
Source File: FluoITBase.java    From rya with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {
    Logger.getLogger(ClientCnxn.class).setLevel(Level.ERROR);

    // Setup and start the Mini Accumulo.
    cluster = clusterInstance.getCluster();

    // Store a connector to the Mini Accumulo.
    instanceName = cluster.getInstanceName();
    zookeepers = cluster.getZooKeepers();

    final Instance instance = new ZooKeeperInstance(instanceName, zookeepers);
    accumuloConn = instance.getConnector(clusterInstance.getUsername(), new PasswordToken(clusterInstance.getPassword()));
}
 
Example #17
Source File: ConfigUtils.java    From rya with Apache License 2.0 5 votes vote down vote up
/**
 * Create an {@link Instance} that may be used to create {@link Connector}s
 * to Accumulo. If the configuration has the {@link #USE_MOCK_INSTANCE} flag
 * set, then the instance will be be a {@link MockInstance} instead of a
 * Zookeeper backed instance.
 *
 * @param conf - The configuration object that will be interrogated. (not null)
 * @return The {@link Instance} that may be used to connect to Accumulo.
 */
public static Instance getInstance(final Configuration conf) {
    // Pull out the Accumulo specific configuration values.
    final AccumuloRdfConfiguration accConf = new AccumuloRdfConfiguration(conf);
    final String instanceName = accConf.getInstanceName();
    final String zoookeepers = accConf.getZookeepers();

    // Create an Instance a mock if the mock flag is set.
    if (useMockInstance(conf)) {
        return new MockInstance(instanceName);
    }

    // Otherwise create an Instance to a Zookeeper managed instance of Accumulo.
    return new ZooKeeperInstance(instanceName, zoookeepers);
}
 
Example #18
Source File: AccumuloInstanceDriver.java    From rya with Apache License 2.0 5 votes vote down vote up
/**
 * Sets up the {@link MiniAccumuloCluster} or the {@link MockInstance}.
 * @throws Exception
 */
public void setUpInstance() throws Exception {
    if (!isMock) {
        log.info("Setting up " + driverName + " MiniAccumulo cluster...");
        // Create and Run MiniAccumulo Cluster
        tempDir = Files.createTempDir();
        tempDir.deleteOnExit();
        miniAccumuloCluster = new MiniAccumuloCluster(tempDir, userpwd);
        copyHadoopHomeToTemp();
        miniAccumuloCluster.getConfig().setInstanceName(instanceName);
        log.info(driverName + " MiniAccumulo instance starting up...");
        miniAccumuloCluster.start();
        Thread.sleep(1000);
        log.info(driverName + " MiniAccumulo instance started");
        log.info("Creating connector to " + driverName + " MiniAccumulo instance...");
        zooKeeperInstance = new ZooKeeperInstance(miniAccumuloCluster.getClientConfig());
        instance = zooKeeperInstance;
        connector = zooKeeperInstance.getConnector(user, new PasswordToken(userpwd));
        log.info("Created connector to " + driverName + " MiniAccumulo instance");
    } else {
        log.info("Setting up " + driverName + " mock instance...");
        mockInstance = new MockInstance(instanceName);
        instance = mockInstance;
        connector = mockInstance.getConnector(user, new PasswordToken(userpwd));
        log.info("Created connector to " + driverName + " mock instance");
    }
    zooKeepers = instance.getZooKeepers();
}
 
Example #19
Source File: DemoDriver.java    From rya with Apache License 2.0 5 votes vote down vote up
/**
 * Setup a Mini Accumulo cluster that uses a temporary directory to store its data.
 *
 * @return A Mini Accumulo cluster.
 */
private static MiniAccumuloCluster startMiniAccumulo() throws IOException, InterruptedException, AccumuloException, AccumuloSecurityException {
    final File miniDataDir = Files.createTempDir();

    // Setup and start the Mini Accumulo.
    final MiniAccumuloCluster accumulo = new MiniAccumuloCluster(miniDataDir, "password");
    accumulo.start();

    // Store a connector to the Mini Accumulo.
    final Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
    accumuloConn = instance.getConnector("root", new PasswordToken("password"));

    return accumulo;
}
 
Example #20
Source File: FluoITBase.java    From rya with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {
    Logger.getLogger(ClientCnxn.class).setLevel(Level.ERROR);

    // Setup and start the Mini Accumulo.
    cluster = clusterInstance.getCluster();

    // Store a connector to the Mini Accumulo.
    instanceName = cluster.getInstanceName();
    zookeepers = cluster.getZooKeepers();

    final Instance instance = new ZooKeeperInstance(instanceName, zookeepers);
    accumuloConn = instance.getConnector(clusterInstance.getUsername(), new PasswordToken(clusterInstance.getPassword()));
}
 
Example #21
Source File: RyaBindingSetExporterFactory.java    From rya with Apache License 2.0 5 votes vote down vote up
@Override
public Optional<IncrementalResultExporter> build(final Context context) throws IncrementalExporterFactoryException, ConfigurationException {
    checkNotNull(context);
    // Wrap the context's parameters for parsing.
    final RyaExportParameters params = new RyaExportParameters( context.getObserverConfiguration().toMap() );

    if(params.getUseRyaBindingSetExporter()) {
        // Setup Zookeeper connection info.
        final String accumuloInstance = params.getAccumuloInstanceName().get();
        final String zookeeperServers =  params.getZookeeperServers().get().replaceAll(";", ",");
        final Instance inst = new ZooKeeperInstance(accumuloInstance, zookeeperServers);

        try {
            // Setup Accumulo connection info.
            final String exporterUsername = params.getExporterUsername().get();
            final String exporterPassword = params.getExporterPassword().get();
            final Connector accumuloConn = inst.getConnector(exporterUsername, new PasswordToken(exporterPassword));

            // Setup Rya PCJ Storage.
            final String ryaInstanceName = params.getRyaInstanceName().get();
            final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, ryaInstanceName);

            // Make the exporter.
            final IncrementalBindingSetExporter exporter = new RyaBindingSetExporter(pcjStorage);
            return Optional.of(exporter);

        } catch (final AccumuloException | AccumuloSecurityException e) {
            throw new IncrementalExporterFactoryException("Could not initialize the Accumulo connector using the provided configuration.", e);
        }
    } else {
        return Optional.absent();
    }
}
 
Example #22
Source File: PcjAdminClient.java    From rya with Apache License 2.0 5 votes vote down vote up
private static Connector createAccumuloConnector(final PcjAdminClientProperties clientProps) throws AccumuloException, AccumuloSecurityException {
    checkNotNull(clientProps);

    // Connect to the Zookeepers.
    final String instanceName = clientProps.getAccumuloInstance();
    final String zooServers = clientProps.getAccumuloZookeepers();
    final Instance inst = new ZooKeeperInstance(instanceName, zooServers);

    // Create a connector to the Accumulo that hosts the PCJ export tables.
    return inst.getConnector(clientProps.getAccumuloUsername(), new PasswordToken(clientProps.getAccumuloPassword()));
}
 
Example #23
Source File: AccumuloCounterSource.java    From datawave with Apache License 2.0 5 votes vote down vote up
public AccumuloCounterSource(String instanceStr, String zookeepers, String username, String password, String table) throws AccumuloException,
                AccumuloSecurityException {
    ZooKeeperInstance instance = new ZooKeeperInstance(instanceStr, zookeepers);
    connector = instance.getConnector(username, new PasswordToken(password));
    queryTable = table;
    this.username = username;
}
 
Example #24
Source File: CardinalityScanner.java    From datawave with Apache License 2.0 5 votes vote down vote up
public Set<CardinalityIntersectionRecord> scanCardinalities(List<String> fields, DateAggregationType dateAggregationType,
                DatatypeAggregationType datatypeAggregationType) throws Exception {
    
    Map<CardinalityIntersectionRecord,HyperLogLogPlus> cardinalityMap = new TreeMap<>();
    Scanner scanner = null;
    try {
        ZooKeeperInstance instance = new ZooKeeperInstance(config.getInstanceName(), config.getZookeepers());
        Connector connector = instance.getConnector(config.getUsername(), new PasswordToken(config.getPassword()));
        Collection<Authorizations> authCollection = Collections.singleton(new Authorizations(config.getAuths().split(",")));
        if (!connector.tableOperations().exists(config.getTableName())) {
            throw new IllegalArgumentException("Table " + config.getTableName() + " does not exist");
        }
        scanner = ScannerHelper.createScanner(connector, config.getTableName(), authCollection);
        Range r = new Range(config.getBeginDate(), config.getEndDate() + "\0");
        scanner.setRange(r);
        
        Iterator<Map.Entry<Key,Value>> itr = scanner.iterator();
        while (itr.hasNext()) {
            Map.Entry<Key,Value> nextEntry = itr.next();
            Key key = nextEntry.getKey();
            String field = key.getColumnFamily().toString();
            if (fields != null && !fields.isEmpty() && !fields.contains(field)) {
                continue;
            } else {
                addEntry(cardinalityMap, nextEntry, dateAggregationType, datatypeAggregationType);
            }
        }
    } catch (Exception e) {
        log.error(e);
    } finally {
        if (scanner != null) {
            scanner.close();
            
        }
    }
    return cardinalityMap.keySet();
}
 
Example #25
Source File: MultiRFileOutputFormatter.java    From datawave with Apache License 2.0 5 votes vote down vote up
protected void setTableIdsAndConfigs() throws IOException {
    ZooKeeperInstance instance = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(conf.get(INSTANCE_NAME))
                    .withZkHosts(conf.get(ZOOKEEPERS)));
    Connector connector = null;
    tableConfigs = new HashMap<>();
    Iterable<String> localityGroupTables = Splitter.on(",").split(conf.get(CONFIGURE_LOCALITY_GROUPS, ""));
    try {
        connector = instance.getConnector(conf.get(USERNAME), new PasswordToken(Base64.decodeBase64(conf.get(PASSWORD))));
        
        tableIds = connector.tableOperations().tableIdMap();
        Set<String> compressionTableBlackList = getCompressionTableBlackList(conf);
        String compressionType = getCompressionType(conf);
        for (String tableName : tableIds.keySet()) {
            ConfigurationCopy tableConfig = new ConfigurationCopy(connector.tableOperations().getProperties(tableName));
            tableConfig.set(Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), (compressionTableBlackList.contains(tableName) ? Compression.COMPRESSION_NONE
                            : compressionType));
            if (Iterables.contains(localityGroupTables, tableName)) {
                Map<String,Set<Text>> localityGroups = connector.tableOperations().getLocalityGroups(tableName);
                // pull the locality groups for this table.
                Map<Text,String> cftlg = Maps.newHashMap();
                Map<String,Set<ByteSequence>> lgtcf = Maps.newHashMap();
                for (Entry<String,Set<Text>> locs : localityGroups.entrySet()) {
                    lgtcf.put(locs.getKey(), new HashSet<>());
                    for (Text loc : locs.getValue()) {
                        cftlg.put(loc, locs.getKey());
                        lgtcf.get(locs.getKey()).add(new ArrayByteSequence(loc.getBytes()));
                    }
                }
                columnFamilyToLocalityGroup.put(tableName, cftlg);
                localityGroupToColumnFamilies.put(tableName, lgtcf);
            }
            tableConfigs.put(tableName, tableConfig);
            
        }
    } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
        throw new IOException("Unable to get configuration.  Please call MultiRFileOutput.setAccumuloConfiguration with the proper credentials", e);
    }
}
 
Example #26
Source File: AccumuloGraphConfiguration.java    From vertexium with Apache License 2.0 5 votes vote down vote up
public Connector createConnector() {
    try {
        LOGGER.info("Connecting to accumulo instance [%s] zookeeper servers [%s]", this.getAccumuloInstanceName(), this.getZookeeperServers());
        ZooKeeperInstance instance = new ZooKeeperInstance(getClientConfiguration());
        return instance.getConnector(this.getAccumuloUsername(), this.getAuthenticationToken());
    } catch (Exception ex) {
        throw new VertexiumException(
            String.format("Could not connect to Accumulo instance [%s] zookeeper servers [%s]", this.getAccumuloInstanceName(), this.getZookeeperServers()),
            ex
        );
    }
}
 
Example #27
Source File: PeriodicBindingSetExporterFactory.java    From rya with Apache License 2.0 5 votes vote down vote up
@Override
public Optional<IncrementalResultExporter> build(Context context) throws IncrementalExporterFactoryException, ConfigurationException {
    checkNotNull(context);

    // Wrap the context's parameters for parsing.
    final RyaExportParameters params = new RyaExportParameters( context.getObserverConfiguration().toMap() );

    if(params.getUsePeriodicBindingSetExporter()) {
        // Setup Zookeeper connection info.
        final String accumuloInstance = params.getAccumuloInstanceName().get();
        final String zookeeperServers =  params.getZookeeperServers().get().replaceAll(";", ",");
        final Instance inst = new ZooKeeperInstance(accumuloInstance, zookeeperServers);

        try {
            // Setup Accumulo connection info.
            final String exporterUsername = params.getExporterUsername().get();
            final String exporterPassword = params.getExporterPassword().get();
            final Connector accumuloConn = inst.getConnector(exporterUsername, new PasswordToken(exporterPassword));

            // Setup Rya PCJ Storage.
            final String ryaInstanceName = params.getRyaInstanceName().get();
            final PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(accumuloConn, ryaInstanceName);
            
            // Make the exporter.
            final IncrementalBindingSetExporter exporter = new PeriodicBindingSetExporter(periodicStorage);
            return Optional.of(exporter);

        } catch (final AccumuloException | AccumuloSecurityException e) {
            throw new IncrementalExporterFactoryException("Could not initialize the Accumulo connector using the provided configuration.", e);
        }
    } else {
        return Optional.absent();
    }
}
 
Example #28
Source File: MapReduceIT.java    From geowave with Apache License 2.0 4 votes vote down vote up
@Test
public void testIngestOSMPBF() throws Exception {
  TestUtils.deleteAll(dataStoreOptions);
  // NOTE: This will probably fail unless you bump up the memory for the
  // tablet
  // servers, for whatever reason, using the
  // miniAccumuloConfig.setMemory() function.
  final MapReduceTestEnvironment mrEnv = MapReduceTestEnvironment.getInstance();

  // TODO: for now this only works with accumulo, generalize the data
  // store usage
  final AccumuloStoreTestEnvironment accumuloEnv = AccumuloStoreTestEnvironment.getInstance();

  final String hdfsPath = mrEnv.getHdfsBaseDirectory() + "/osm_stage/";

  final StageOSMToHDFSCommand stage = new StageOSMToHDFSCommand();
  stage.setParameters(TEST_DATA_BASE_DIR, hdfsPath);
  stage.execute(mrEnv.getOperationParams());

  final Connector conn =
      new ZooKeeperInstance(
          accumuloEnv.getAccumuloInstance(),
          accumuloEnv.getZookeeper()).getConnector(
              accumuloEnv.getAccumuloUser(),
              new PasswordToken(accumuloEnv.getAccumuloPassword()));
  final Authorizations auth = new Authorizations(new String[] {"public"});
  conn.securityOperations().changeUserAuthorizations(accumuloEnv.getAccumuloUser(), auth);
  final IngestOSMToGeoWaveCommand ingest = new IngestOSMToGeoWaveCommand();
  ingest.setParameters(hdfsPath, "test-store");

  final AddStoreCommand addStore = new AddStoreCommand();
  addStore.setParameters("test-store");
  addStore.setPluginOptions(dataStoreOptions);
  addStore.execute(mrEnv.getOperationParams());

  ingest.getIngestOptions().setJobName("ConversionTest");

  // Execute for node's ways, and relations.
  ingest.getIngestOptions().setMapperType("NODE");
  ingest.execute(mrEnv.getOperationParams());
  System.out.println("finished accumulo ingest Node");

  ingest.getIngestOptions().setMapperType("WAY");
  ingest.execute(mrEnv.getOperationParams());
  System.out.println("finished accumulo ingest Way");

  ingest.getIngestOptions().setMapperType("RELATION");
  ingest.execute(mrEnv.getOperationParams());
  System.out.println("finished accumulo ingest Relation");
}
 
Example #29
Source File: MetadataHelperUpdateHdfsListener.java    From datawave with Apache License 2.0 4 votes vote down vote up
private void maybeUpdateTypeMetadataInHdfs(final SharedCacheCoordinator watcher, String triStateName, String metadataTableName) throws Exception {
    
    boolean locked = false;
    InterProcessMutex lock = (InterProcessMutex) watcher.getMutex("lock");
    try {
        locked = lock.acquire(this.lockWaitTime, TimeUnit.MILLISECONDS);
        if (!locked)
            log.debug("table:" + metadataTableName + " Unable to acquire lock to update " + metadataTableName
                            + ". Another webserver is updating the typeMetadata.");
        else
            log.debug("table:" + metadataTableName + " Obtained lock on updateTypeMetadata for " + metadataTableName);
    } catch (Exception e) {
        log.warn("table:" + metadataTableName + " Got Exception trying to acquire lock to update " + metadataTableName + ".", e);
    }
    
    try {
        if (locked) {
            try {
                log.debug("table:" + metadataTableName + " checkTriState(" + triStateName + ", " + SharedTriState.STATE.NEEDS_UPDATE);
                if (watcher.checkTriState(triStateName, SharedTriState.STATE.NEEDS_UPDATE)) {
                    if (log.isDebugEnabled()) {
                        log.debug("table:" + metadataTableName + " " + this + " STATE is NEEDS_UPDATE. Will write the TypeMetadata map to hdfs");
                    }
                    watcher.setTriState(triStateName, SharedTriState.STATE.UPDATING);
                    if (log.isDebugEnabled()) {
                        log.debug("table:" + metadataTableName + " " + this + " setTriState to UPDATING");
                    }
                    // get a connection for my MetadataHelper, and get the TypeMetadata map
                    ZooKeeperInstance instance = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(this.instance)
                                    .withZkHosts(this.zookeepers));
                    Connector connector = instance.getConnector(this.username, new PasswordToken(this.password));
                    TypeMetadataHelper typeMetadataHelper = this.typeMetadataHelperFactory.createTypeMetadataHelper(connector, metadataTableName,
                                    allMetadataAuths, false);
                    typeMetadataWriter.writeTypeMetadataMap(typeMetadataHelper.getTypeMetadataMap(this.allMetadataAuths), metadataTableName);
                    if (log.isDebugEnabled()) {
                        log.debug("table:" + metadataTableName + " " + this + " set the sharedTriState needsUpdate to UPDATED for " + metadataTableName);
                    }
                    watcher.setTriState(triStateName, SharedTriState.STATE.UPDATED);
                } else {
                    if (log.isDebugEnabled()) {
                        log.debug("table:"
                                        + metadataTableName
                                        + " "
                                        + this
                                        + "  STATE is not NEEDS_UPDATE! Someone else may be writing or has already written the TypeMetadata map, just release the lock");
                    }
                }
            } catch (Exception ex) {
                log.warn("table:" + metadataTableName + " Unable to write TypeMetadataMap for " + metadataTableName, ex);
                watcher.setTriState(triStateName, SharedTriState.STATE.NEEDS_UPDATE);
                if (log.isDebugEnabled()) {
                    log.debug("After exception, set the SharedTriState STATE to NEEDS_UPDATE");
                }
                
            }
        }
    } finally {
        if (locked) {
            lock.release();
            if (log.isTraceEnabled())
                log.trace("table:" + metadataTableName + " " + this + " released the lock for " + metadataTableName);
            
        }
    }
}
 
Example #30
Source File: AccumuloRecordWriter.java    From datawave with Apache License 2.0 4 votes vote down vote up
protected static Instance getInstance(Configuration conf) {
    if (conf.getBoolean(MOCK, false)) {
        return new InMemoryInstance(conf.get(INSTANCE_NAME));
    }
    return new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(conf.get(INSTANCE_NAME)).withZkHosts(conf.get(ZOOKEEPERS)));
}