Java Code Examples for org.apache.hadoop.mapreduce.lib.db.DBInputFormat.NullDBWritable

The following examples show how to use org.apache.hadoop.mapreduce.lib.db.DBInputFormat.NullDBWritable. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestDbClasses.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * test generate sql script for OracleDBRecordReader.
 */

@Test(timeout = 20000)
public void testOracleDBRecordReader() throws Exception {
  DBInputSplit splitter = new DBInputSplit(1, 10);
  Configuration configuration = new Configuration();
  Connection connect = DriverForTest.getConnection();

  DBConfiguration dbConfiguration = new DBConfiguration(configuration);
  dbConfiguration.setInputOrderBy("Order");
  String[] fields = { "f1", "f2" };

  OracleDBRecordReader<NullDBWritable> recorder = new OracleDBRecordReader<NullDBWritable>(
      splitter, NullDBWritable.class, configuration, connect,
      dbConfiguration, "condition", fields, "table");
  assertEquals(
      "SELECT * FROM (SELECT a.*,ROWNUM dbif_rno FROM ( SELECT f1, f2 FROM table WHERE condition ORDER BY Order ) a WHERE rownum <= 10 ) WHERE dbif_rno > 1",
      recorder.getSelectQuery());
}
 
Example 2
Source Project: big-c   Source File: TestDbClasses.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * test generate sql script for OracleDBRecordReader.
 */

@Test(timeout = 20000)
public void testOracleDBRecordReader() throws Exception {
  DBInputSplit splitter = new DBInputSplit(1, 10);
  Configuration configuration = new Configuration();
  Connection connect = DriverForTest.getConnection();

  DBConfiguration dbConfiguration = new DBConfiguration(configuration);
  dbConfiguration.setInputOrderBy("Order");
  String[] fields = { "f1", "f2" };

  OracleDBRecordReader<NullDBWritable> recorder = new OracleDBRecordReader<NullDBWritable>(
      splitter, NullDBWritable.class, configuration, connect,
      dbConfiguration, "condition", fields, "table");
  assertEquals(
      "SELECT * FROM (SELECT a.*,ROWNUM dbif_rno FROM ( SELECT f1, f2 FROM table WHERE condition ORDER BY Order ) a WHERE rownum <= 10 ) WHERE dbif_rno > 1",
      recorder.getSelectQuery());
}
 
Example 3
Source Project: hadoop   Source File: TestDbClasses.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * test splitters from DataDrivenDBInputFormat. For different data types may
 * be different splitter
 */
@Test(timeout = 10000)
public void testDataDrivenDBInputFormatSplitter() {
  DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>();
  testCommonSplitterTypes(format);
  assertEquals(DateSplitter.class, format.getSplitter(Types.TIMESTAMP)
      .getClass());
  assertEquals(DateSplitter.class, format.getSplitter(Types.DATE).getClass());
  assertEquals(DateSplitter.class, format.getSplitter(Types.TIME).getClass());
}
 
Example 4
Source Project: hadoop   Source File: TestDbClasses.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 10000)
public void testDataDrivenDBInputFormat() throws Exception {
  JobContext jobContext = mock(JobContext.class);
  Configuration configuration = new Configuration();
  configuration.setInt(MRJobConfig.NUM_MAPS, 1);

  when(jobContext.getConfiguration()).thenReturn(configuration);
  DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>();
  List<InputSplit> splits = format.getSplits(jobContext);
  assertEquals(1, splits.size());
  DataDrivenDBInputSplit split = (DataDrivenDBInputSplit) splits.get(0);
  assertEquals("1=1", split.getLowerClause());
  assertEquals("1=1", split.getUpperClause());

  // 2
  configuration.setInt(MRJobConfig.NUM_MAPS, 2);

  DataDrivenDBInputFormat.setBoundingQuery(configuration, "query");
  assertEquals("query",
      configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY));

  Job job = mock(Job.class);
  when(job.getConfiguration()).thenReturn(configuration);
  DataDrivenDBInputFormat.setInput(job, NullDBWritable.class, "query",
      "Bounding Query");

  assertEquals("Bounding Query",
      configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY));
}
 
Example 5
Source Project: hadoop   Source File: TestDbClasses.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 10000)
public void testOracleDataDrivenDBInputFormat() throws Exception {
  OracleDataDrivenDBInputFormat<NullDBWritable> format = 
      new OracleDataDrivenDBInputFormatForTest();
  testCommonSplitterTypes(format);
  assertEquals(OracleDateSplitter.class, format.getSplitter(Types.TIMESTAMP)
      .getClass());
  assertEquals(OracleDateSplitter.class, 
      format.getSplitter(Types.DATE).getClass());
  assertEquals(OracleDateSplitter.class, 
      format.getSplitter(Types.TIME).getClass());
}
 
Example 6
Source Project: hadoop   Source File: TestDbClasses.java    License: Apache License 2.0 5 votes vote down vote up
private void testCommonSplitterTypes(
    DataDrivenDBInputFormat<NullDBWritable> format) {
  assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.DECIMAL)
      .getClass());
  assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.NUMERIC)
      .getClass());
  assertEquals(BooleanSplitter.class, format.getSplitter(Types.BOOLEAN)
      .getClass());
  assertEquals(BooleanSplitter.class, format.getSplitter(Types.BIT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.BIGINT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.TINYINT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.SMALLINT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.INTEGER)
      .getClass());
  assertEquals(FloatSplitter.class, format.getSplitter(Types.DOUBLE)
      .getClass());
  assertEquals(FloatSplitter.class, format.getSplitter(Types.REAL).getClass());
  assertEquals(FloatSplitter.class, format.getSplitter(Types.FLOAT)
      .getClass());
  assertEquals(TextSplitter.class, format.getSplitter(Types.LONGVARCHAR)
      .getClass());
  assertEquals(TextSplitter.class, format.getSplitter(Types.CHAR).getClass());
  assertEquals(TextSplitter.class, format.getSplitter(Types.VARCHAR)
      .getClass());
  // if unknown data type splitter is null
  assertNull(format.getSplitter(Types.BINARY));
}
 
Example 7
Source Project: big-c   Source File: TestDbClasses.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * test splitters from DataDrivenDBInputFormat. For different data types may
 * be different splitter
 */
@Test(timeout = 10000)
public void testDataDrivenDBInputFormatSplitter() {
  DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>();
  testCommonSplitterTypes(format);
  assertEquals(DateSplitter.class, format.getSplitter(Types.TIMESTAMP)
      .getClass());
  assertEquals(DateSplitter.class, format.getSplitter(Types.DATE).getClass());
  assertEquals(DateSplitter.class, format.getSplitter(Types.TIME).getClass());
}
 
Example 8
Source Project: big-c   Source File: TestDbClasses.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 10000)
public void testDataDrivenDBInputFormat() throws Exception {
  JobContext jobContext = mock(JobContext.class);
  Configuration configuration = new Configuration();
  configuration.setInt(MRJobConfig.NUM_MAPS, 1);

  when(jobContext.getConfiguration()).thenReturn(configuration);
  DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>();
  List<InputSplit> splits = format.getSplits(jobContext);
  assertEquals(1, splits.size());
  DataDrivenDBInputSplit split = (DataDrivenDBInputSplit) splits.get(0);
  assertEquals("1=1", split.getLowerClause());
  assertEquals("1=1", split.getUpperClause());

  // 2
  configuration.setInt(MRJobConfig.NUM_MAPS, 2);

  DataDrivenDBInputFormat.setBoundingQuery(configuration, "query");
  assertEquals("query",
      configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY));

  Job job = mock(Job.class);
  when(job.getConfiguration()).thenReturn(configuration);
  DataDrivenDBInputFormat.setInput(job, NullDBWritable.class, "query",
      "Bounding Query");

  assertEquals("Bounding Query",
      configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY));
}
 
Example 9
Source Project: big-c   Source File: TestDbClasses.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 10000)
public void testOracleDataDrivenDBInputFormat() throws Exception {
  OracleDataDrivenDBInputFormat<NullDBWritable> format = 
      new OracleDataDrivenDBInputFormatForTest();
  testCommonSplitterTypes(format);
  assertEquals(OracleDateSplitter.class, format.getSplitter(Types.TIMESTAMP)
      .getClass());
  assertEquals(OracleDateSplitter.class, 
      format.getSplitter(Types.DATE).getClass());
  assertEquals(OracleDateSplitter.class, 
      format.getSplitter(Types.TIME).getClass());
}
 
Example 10
Source Project: big-c   Source File: TestDbClasses.java    License: Apache License 2.0 5 votes vote down vote up
private void testCommonSplitterTypes(
    DataDrivenDBInputFormat<NullDBWritable> format) {
  assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.DECIMAL)
      .getClass());
  assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.NUMERIC)
      .getClass());
  assertEquals(BooleanSplitter.class, format.getSplitter(Types.BOOLEAN)
      .getClass());
  assertEquals(BooleanSplitter.class, format.getSplitter(Types.BIT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.BIGINT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.TINYINT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.SMALLINT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.INTEGER)
      .getClass());
  assertEquals(FloatSplitter.class, format.getSplitter(Types.DOUBLE)
      .getClass());
  assertEquals(FloatSplitter.class, format.getSplitter(Types.REAL).getClass());
  assertEquals(FloatSplitter.class, format.getSplitter(Types.FLOAT)
      .getClass());
  assertEquals(TextSplitter.class, format.getSplitter(Types.LONGVARCHAR)
      .getClass());
  assertEquals(TextSplitter.class, format.getSplitter(Types.CHAR).getClass());
  assertEquals(TextSplitter.class, format.getSplitter(Types.VARCHAR)
      .getClass());
  // if unknown data type splitter is null
  assertNull(format.getSplitter(Types.BINARY));
}
 
Example 11
Source Project: phoenix   Source File: UpdateStatisticsTool.java    License: Apache License 2.0 5 votes vote down vote up
private void configureJob() throws Exception {
    job = Job.getInstance(getConf(),
            "UpdateStatistics-" + tableName + "-" + snapshotName);
    PhoenixMapReduceUtil.setInput(job, NullDBWritable.class,
            snapshotName, tableName, restoreDir);

    PhoenixConfigurationUtil.setMRJobType(job.getConfiguration(), MRJobType.UPDATE_STATS);

    // DO NOT allow mapper splits using statistics since it may result into many smaller chunks
    PhoenixConfigurationUtil.setSplitByStats(job.getConfiguration(), false);

    job.setJarByClass(UpdateStatisticsTool.class);
    job.setMapperClass(TableSnapshotMapper.class);
    job.setMapOutputKeyClass(NullWritable.class);
    job.setMapOutputValueClass(NullWritable.class);
    job.setOutputFormatClass(NullOutputFormat.class);
    job.setNumReduceTasks(0);
    job.setPriority(this.jobPriority);

    TableMapReduceUtil.addDependencyJars(job);
    TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), PhoenixConnection.class, Chronology.class,
            CharStream.class, TransactionSystemClient.class, TransactionNotInProgressException.class,
            ZKClient.class, DiscoveryServiceClient.class, ZKDiscoveryService.class,
            Cancellable.class, TTransportException.class, SpanReceiver.class, TransactionProcessor.class, Gauge.class, MetricRegistriesImpl.class);
    LOGGER.info("UpdateStatisticsTool running for: " + tableName
            + " on snapshot: " + snapshotName + " with restore dir: " + restoreDir);
}
 
Example 12
Source Project: hadoop   Source File: DBConfiguration.java    License: Apache License 2.0 4 votes vote down vote up
public Class<?> getInputClass() {
  return conf.getClass(DBConfiguration.INPUT_CLASS_PROPERTY,
                       NullDBWritable.class);
}
 
Example 13
Source Project: big-c   Source File: DBConfiguration.java    License: Apache License 2.0 4 votes vote down vote up
public Class<?> getInputClass() {
  return conf.getClass(DBConfiguration.INPUT_CLASS_PROPERTY,
                       NullDBWritable.class);
}
 
Example 14
Source Project: phoenix   Source File: PhoenixConfigurationUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static Class<?> getInputClass(final Configuration configuration) {
    return configuration.getClass(INPUT_CLASS, NullDBWritable.class);
}
 
Example 15
Source Project: phoenix   Source File: PhoenixConfigurationUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static Class<?> getInputClass(final Configuration configuration) {
    return configuration.getClass(INPUT_CLASS, NullDBWritable.class);
}
 
Example 16
Source Project: phoenix   Source File: UpdateStatisticsTool.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void map(NullWritable key, NullDBWritable value,
                   Context context) {
}