org.apache.hadoop.tools.rumen.JobStoryProducer Java Examples

The following examples show how to use org.apache.hadoop.tools.rumen.JobStoryProducer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Gridmix.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Setup gridmix for emulation of distributed cache load. This includes
 * generation of distributed cache files, if needed.
 * @param conf gridmix configuration
 * @param traceIn trace file path(if it is '-', then trace comes from the
 *                stream stdin)
 * @param ioPath <ioPath>/input/ is the dir where input data (a) exists
 *               or (b) is generated. <ioPath>/distributedCache/ is the
 *               folder where distributed cache data (a) exists or (b) is to be
 *               generated by gridmix.
 * @param generate true if -generate option was specified
 * @return exit code
 * @throws IOException
 * @throws InterruptedException
 */
private int setupDistCacheEmulation(Configuration conf, String traceIn,
    Path ioPath, boolean generate) throws IOException, InterruptedException {
  distCacheEmulator.init(traceIn, factory.jobCreator, generate);
  int exitCode = 0;
  if (distCacheEmulator.shouldGenerateDistCacheData() ||
      distCacheEmulator.shouldEmulateDistCacheLoad()) {

    JobStoryProducer jsp = createJobStoryProducer(traceIn, conf);
    exitCode = distCacheEmulator.setupGenerateDistCacheData(jsp);
    if (exitCode == 0) {
      // If there are files to be generated, run a MapReduce job to generate
      // these distributed cache files of all the simulated jobs of this trace.
      writeDistCacheData(conf);
    }
  }
  return exitCode;
}
 
Example #2
Source File: StressJobFactory.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Creating a new instance does not start the thread.
 *
 * @param submitter   Component to which deserialized jobs are passed
 * @param jobProducer Stream of job traces with which to construct a
 *                    {@link org.apache.hadoop.tools.rumen.ZombieJobProducer}
 * @param scratch     Directory into which to write output from simulated jobs
 * @param conf        Config passed to all jobs to be submitted
 * @param startFlag   Latch released from main to start pipeline
 * @throws java.io.IOException
 */
public StressJobFactory(
  JobSubmitter submitter, JobStoryProducer jobProducer, Path scratch,
  Configuration conf, CountDownLatch startFlag, UserResolver resolver)
  throws IOException {
  super(
    submitter, jobProducer, scratch, conf, startFlag, resolver);
  overloadMapTaskMapSlotRatio = conf.getFloat(
      CONF_OVERLOAD_MAPTASK_MAPSLOT_RATIO, OVERLOAD_MAPTASK_MAPSLOT_RATIO);
  overloadReduceTaskReduceSlotRatio = conf.getFloat(
      CONF_OVERLOAD_REDUCETASK_REDUCESLOT_RATIO, 
      OVERLOAD_REDUCETASK_REDUCESLOT_RATIO);
  maxMapSlotSharePerJob = conf.getFloat(
      CONF_MAX_MAPSLOT_SHARE_PER_JOB, MAX_MAPSLOT_SHARE_PER_JOB);
  maxReduceSlotSharePerJob = conf.getFloat(
      CONF_MAX_REDUCESLOT_SHARE_PER_JOB, MAX_REDUCESLOT_SHARE_PER_JOB);
  maxJobTrackerRatio = conf.getFloat(
      CONF_MAX_JOB_TRACKER_RATIO, MAX_JOB_TRACKER_RATIO);
}
 
Example #3
Source File: JobFactory.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Constructor permitting JobStoryProducer to be mocked.
 * @param submitter Component to which deserialized jobs are passed
 * @param jobProducer Producer generating JobStory objects.
 * @param scratch Directory into which to write output from simulated jobs
 * @param conf Config passed to all jobs to be submitted
 * @param startFlag Latch released from main to start pipeline
 */
protected JobFactory(JobSubmitter submitter, JobStoryProducer jobProducer,
    Path scratch, Configuration conf, CountDownLatch startFlag,
    UserResolver userResolver) {
  sequence = new AtomicInteger(0);
  this.scratch = scratch;
  this.rateFactor = conf.getFloat(Gridmix.GRIDMIX_SUB_MUL, 1.0f);
  this.jobProducer = jobProducer;
  this.conf = new Configuration(conf);
  this.submitter = submitter;
  this.startFlag = startFlag;
  this.rThread = createReaderThread();
  if(LOG.isDebugEnabled()) {
    LOG.debug(" The submission thread name is " + rThread.getName());
  }
  this.userResolver = userResolver;
  this.jobCreator = JobCreator.getPolicy(conf, JobCreator.LOADJOB);
}
 
Example #4
Source File: StressJobFactory.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Creating a new instance does not start the thread.
 *
 * @param submitter   Component to which deserialized jobs are passed
 * @param jobProducer Stream of job traces with which to construct a
 *                    {@link org.apache.hadoop.tools.rumen.ZombieJobProducer}
 * @param scratch     Directory into which to write output from simulated jobs
 * @param conf        Config passed to all jobs to be submitted
 * @param startFlag   Latch released from main to start pipeline
 * @throws java.io.IOException
 */
public StressJobFactory(
  JobSubmitter submitter, JobStoryProducer jobProducer, Path scratch,
  Configuration conf, CountDownLatch startFlag, UserResolver resolver)
  throws IOException {
  super(
    submitter, jobProducer, scratch, conf, startFlag, resolver);
  overloadMapTaskMapSlotRatio = conf.getFloat(
      CONF_OVERLOAD_MAPTASK_MAPSLOT_RATIO, OVERLOAD_MAPTASK_MAPSLOT_RATIO);
  overloadReduceTaskReduceSlotRatio = conf.getFloat(
      CONF_OVERLOAD_REDUCETASK_REDUCESLOT_RATIO, 
      OVERLOAD_REDUCETASK_REDUCESLOT_RATIO);
  maxMapSlotSharePerJob = conf.getFloat(
      CONF_MAX_MAPSLOT_SHARE_PER_JOB, MAX_MAPSLOT_SHARE_PER_JOB);
  maxReduceSlotSharePerJob = conf.getFloat(
      CONF_MAX_REDUCESLOT_SHARE_PER_JOB, MAX_REDUCESLOT_SHARE_PER_JOB);
  maxJobTrackerRatio = conf.getFloat(
      CONF_MAX_JOB_TRACKER_RATIO, MAX_JOB_TRACKER_RATIO);
}
 
Example #5
Source File: JobFactory.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Constructor permitting JobStoryProducer to be mocked.
 * @param submitter Component to which deserialized jobs are passed
 * @param jobProducer Producer generating JobStory objects.
 * @param scratch Directory into which to write output from simulated jobs
 * @param conf Config passed to all jobs to be submitted
 * @param startFlag Latch released from main to start pipeline
 */
protected JobFactory(JobSubmitter submitter, JobStoryProducer jobProducer,
    Path scratch, Configuration conf, CountDownLatch startFlag,
    UserResolver userResolver) {
  sequence = new AtomicInteger(0);
  this.scratch = scratch;
  this.rateFactor = conf.getFloat(Gridmix.GRIDMIX_SUB_MUL, 1.0f);
  this.jobProducer = jobProducer;
  this.conf = new Configuration(conf);
  this.submitter = submitter;
  this.startFlag = startFlag;
  this.rThread = createReaderThread();
  if(LOG.isDebugEnabled()) {
    LOG.debug(" The submission thread name is " + rThread.getName());
  }
  this.userResolver = userResolver;
  this.jobCreator = JobCreator.getPolicy(conf, JobCreator.LOADJOB);
}
 
Example #6
Source File: Gridmix.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Setup gridmix for emulation of distributed cache load. This includes
 * generation of distributed cache files, if needed.
 * @param conf gridmix configuration
 * @param traceIn trace file path(if it is '-', then trace comes from the
 *                stream stdin)
 * @param ioPath <ioPath>/input/ is the dir where input data (a) exists
 *               or (b) is generated. <ioPath>/distributedCache/ is the
 *               folder where distributed cache data (a) exists or (b) is to be
 *               generated by gridmix.
 * @param generate true if -generate option was specified
 * @return exit code
 * @throws IOException
 * @throws InterruptedException
 */
private int setupDistCacheEmulation(Configuration conf, String traceIn,
    Path ioPath, boolean generate) throws IOException, InterruptedException {
  distCacheEmulator.init(traceIn, factory.jobCreator, generate);
  int exitCode = 0;
  if (distCacheEmulator.shouldGenerateDistCacheData() ||
      distCacheEmulator.shouldEmulateDistCacheLoad()) {

    JobStoryProducer jsp = createJobStoryProducer(traceIn, conf);
    exitCode = distCacheEmulator.setupGenerateDistCacheData(jsp);
    if (exitCode == 0) {
      // If there are files to be generated, run a MapReduce job to generate
      // these distributed cache files of all the simulated jobs of this trace.
      writeDistCacheData(conf);
    }
  }
  return exitCode;
}
 
Example #7
Source File: DistributedCacheEmulator.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Create the list of unique distributed cache files needed for all the
 * simulated jobs and write the list to a special file.
 * @param jsp job story producer for the trace
 * @return exit code
 * @throws IOException
 */
private int buildDistCacheFilesList(JobStoryProducer jsp) throws IOException {
  // Read all the jobs from the trace file and build the list of unique
  // distributed cache files.
  JobStory jobStory;
  while ((jobStory = jsp.getNextJob()) != null) {
    if (jobStory.getOutcome() == Pre21JobHistoryConstants.Values.SUCCESS && 
       jobStory.getSubmissionTime() >= 0) {
      updateHDFSDistCacheFilesList(jobStory);
    }
  }
  jsp.close();

  return writeDistCacheFilesList();
}
 
Example #8
Source File: JobFactory.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Constructor permitting JobStoryProducer to be mocked.
 * @param submitter Component to which deserialized jobs are passed
 * @param jobProducer Producer generating JobStory objects.
 * @param scratch Directory into which to write output from simulated jobs
 * @param conf Config passed to all jobs to be submitted
 * @param startFlag Latch released from main to start pipeline
 */
protected JobFactory(JobSubmitter submitter, JobStoryProducer jobProducer,
    Path scratch, Configuration conf, CountDownLatch startFlag) {
  sequence = new AtomicInteger(0);
  this.scratch = scratch;
  this.rateFactor = conf.getFloat(Gridmix.GRIDMIX_SUB_MUL, 1.0f);
  this.jobProducer = jobProducer;
  this.conf = new Configuration(conf);
  this.submitter = submitter;
  this.startFlag = startFlag;
  this.rThread = new ReaderThread();
}
 
Example #9
Source File: TestGridMixClasses.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout=120000)
public void testSerialReaderThread() throws Exception {

  Configuration conf = new Configuration();
  File fin = new File("src" + File.separator + "test" + File.separator
          + "resources" + File.separator + "data" + File.separator
          + "wordcount2.json");
  // read couple jobs from wordcount2.json
  JobStoryProducer jobProducer = new ZombieJobProducer(new Path(
          fin.getAbsolutePath()), null, conf);
  CountDownLatch startFlag = new CountDownLatch(1);
  UserResolver resolver = new SubmitterUserResolver();
  FakeJobSubmitter submitter = new FakeJobSubmitter();
  File ws = new File("target" + File.separator + this.getClass().getName());
  if (!ws.exists()) {
    Assert.assertTrue(ws.mkdirs());
  }

  SerialJobFactory jobFactory = new SerialJobFactory(submitter, jobProducer,
          new Path(ws.getAbsolutePath()), conf, startFlag, resolver);

  Path ioPath = new Path(ws.getAbsolutePath());
  jobFactory.setDistCacheEmulator(new DistributedCacheEmulator(conf, ioPath));
  Thread test = jobFactory.createReaderThread();
  test.start();
  Thread.sleep(1000);
  // SerialReaderThread waits startFlag
  assertEquals(0, submitter.getJobs().size());
  // start!
  startFlag.countDown();
  while (test.isAlive()) {
    Thread.sleep(1000);
    jobFactory.update(null);
  }
  // submitter was called twice
  assertEquals(2, submitter.getJobs().size());
}
 
Example #10
Source File: TestGridMixClasses.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout=120000)
public void testSerialReaderThread() throws Exception {

  Configuration conf = new Configuration();
  File fin = new File("src" + File.separator + "test" + File.separator
          + "resources" + File.separator + "data" + File.separator
          + "wordcount2.json");
  // read couple jobs from wordcount2.json
  JobStoryProducer jobProducer = new ZombieJobProducer(new Path(
          fin.getAbsolutePath()), null, conf);
  CountDownLatch startFlag = new CountDownLatch(1);
  UserResolver resolver = new SubmitterUserResolver();
  FakeJobSubmitter submitter = new FakeJobSubmitter();
  File ws = new File("target" + File.separator + this.getClass().getName());
  if (!ws.exists()) {
    Assert.assertTrue(ws.mkdirs());
  }

  SerialJobFactory jobFactory = new SerialJobFactory(submitter, jobProducer,
          new Path(ws.getAbsolutePath()), conf, startFlag, resolver);

  Path ioPath = new Path(ws.getAbsolutePath());
  jobFactory.setDistCacheEmulator(new DistributedCacheEmulator(conf, ioPath));
  Thread test = jobFactory.createReaderThread();
  test.start();
  Thread.sleep(1000);
  // SerialReaderThread waits startFlag
  assertEquals(0, submitter.getJobs().size());
  // start!
  startFlag.countDown();
  while (test.isAlive()) {
    Thread.sleep(1000);
    jobFactory.update(null);
  }
  // submitter was called twice
  assertEquals(2, submitter.getJobs().size());
}
 
Example #11
Source File: DistributedCacheEmulator.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Create the list of unique distributed cache files needed for all the
 * simulated jobs and write the list to a special file.
 * @param jsp job story producer for the trace
 * @return exit code
 * @throws IOException
 */
private int buildDistCacheFilesList(JobStoryProducer jsp) throws IOException {
  // Read all the jobs from the trace file and build the list of unique
  // distributed cache files.
  JobStory jobStory;
  while ((jobStory = jsp.getNextJob()) != null) {
    if (jobStory.getOutcome() == Pre21JobHistoryConstants.Values.SUCCESS && 
       jobStory.getSubmissionTime() >= 0) {
      updateHDFSDistCacheFilesList(jobStory);
    }
  }
  jsp.close();

  return writeDistCacheFilesList();
}
 
Example #12
Source File: TestGridmixSubmission.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Tests the reading of traces in GridMix3. These traces are generated by
 * Rumen and are in the JSON format. The traces can optionally be compressed
 * and uncompressed traces can also be passed to GridMix3 via its standard
 * input stream. The testing is effected via JUnit assertions.
 *
 * @throws Exception if there was an error.
 */
@Test (timeout=20000)
public void testTraceReader() throws Exception {
  Configuration conf = new Configuration();
  FileSystem lfs = FileSystem.getLocal(conf);
  Path rootInputDir = new Path(System.getProperty("src.test.data"));
  rootInputDir = rootInputDir.makeQualified(lfs.getUri(),
          lfs.getWorkingDirectory());
  Path rootTempDir = new Path(System.getProperty("test.build.data",
          System.getProperty("java.io.tmpdir")), "testTraceReader");
  rootTempDir = rootTempDir.makeQualified(lfs.getUri(),
          lfs.getWorkingDirectory());
  Path inputFile = new Path(rootInputDir, "wordcount.json.gz");
  Path tempFile = new Path(rootTempDir, "gridmix3-wc.json");

  InputStream origStdIn = System.in;
  InputStream tmpIs = null;
  try {
    DebugGridmix dgm = new DebugGridmix();
    JobStoryProducer jsp = dgm.createJobStoryProducer(inputFile.toString(),
            conf);

    LOG.info("Verifying JobStory from compressed trace...");
    verifyWordCountJobStory(jsp.getNextJob());

    expandGzippedTrace(lfs, inputFile, tempFile);
    jsp = dgm.createJobStoryProducer(tempFile.toString(), conf);
    LOG.info("Verifying JobStory from uncompressed trace...");
    verifyWordCountJobStory(jsp.getNextJob());

    tmpIs = lfs.open(tempFile);
    System.setIn(tmpIs);
    LOG.info("Verifying JobStory from trace in standard input...");
    jsp = dgm.createJobStoryProducer("-", conf);
    verifyWordCountJobStory(jsp.getNextJob());
  } finally {
    System.setIn(origStdIn);
    if (tmpIs != null) {
      tmpIs.close();
    }
    lfs.delete(rootTempDir, true);
  }
}
 
Example #13
Source File: TestGridmixSubmission.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Tests the reading of traces in GridMix3. These traces are generated by
 * Rumen and are in the JSON format. The traces can optionally be compressed
 * and uncompressed traces can also be passed to GridMix3 via its standard
 * input stream. The testing is effected via JUnit assertions.
 *
 * @throws Exception if there was an error.
 */
@Test (timeout=20000)
public void testTraceReader() throws Exception {
  Configuration conf = new Configuration();
  FileSystem lfs = FileSystem.getLocal(conf);
  Path rootInputDir = new Path(System.getProperty("src.test.data"));
  rootInputDir = rootInputDir.makeQualified(lfs.getUri(),
          lfs.getWorkingDirectory());
  Path rootTempDir = new Path(System.getProperty("test.build.data",
          System.getProperty("java.io.tmpdir")), "testTraceReader");
  rootTempDir = rootTempDir.makeQualified(lfs.getUri(),
          lfs.getWorkingDirectory());
  Path inputFile = new Path(rootInputDir, "wordcount.json.gz");
  Path tempFile = new Path(rootTempDir, "gridmix3-wc.json");

  InputStream origStdIn = System.in;
  InputStream tmpIs = null;
  try {
    DebugGridmix dgm = new DebugGridmix();
    JobStoryProducer jsp = dgm.createJobStoryProducer(inputFile.toString(),
            conf);

    LOG.info("Verifying JobStory from compressed trace...");
    verifyWordCountJobStory(jsp.getNextJob());

    expandGzippedTrace(lfs, inputFile, tempFile);
    jsp = dgm.createJobStoryProducer(tempFile.toString(), conf);
    LOG.info("Verifying JobStory from uncompressed trace...");
    verifyWordCountJobStory(jsp.getNextJob());

    tmpIs = lfs.open(tempFile);
    System.setIn(tmpIs);
    LOG.info("Verifying JobStory from trace in standard input...");
    jsp = dgm.createJobStoryProducer("-", conf);
    verifyWordCountJobStory(jsp.getNextJob());
  } finally {
    System.setIn(origStdIn);
    if (tmpIs != null) {
      tmpIs.close();
    }
    lfs.delete(rootTempDir, true);
  }
}
 
Example #14
Source File: SimulatorEngine.java    From RDFS with Apache License 2.0 4 votes vote down vote up
/**
 * Initiate components in the simulation.
 * @throws InterruptedException
 * @throws IOException if trace or topology files cannot be open
 */
@SuppressWarnings("deprecation")
void init() throws InterruptedException, IOException {
  long now = System.currentTimeMillis();

  JobConf jobConf = new JobConf(getConf());
  jobConf.setClass("topology.node.switch.mapping.impl",
      StaticMapping.class, DNSToSwitchMapping.class);
  jobConf.set("fs.default.name", "file:///");
  jobConf.set("mapred.job.tracker", "localhost:8012");
  jobConf.setInt("mapred.jobtracker.job.history.block.size", 512);
  jobConf.setInt("mapred.jobtracker.job.history.buffer.size", 512);
  jobConf.setLong("mapred.tasktracker.expiry.interval", 5000);
  jobConf.setInt("mapred.reduce.copy.backoff", 4);
  jobConf.setLong("mapred.job.reuse.jvm.num.tasks", -1);
  jobConf.setUser("mumak");
  jobConf.set("mapred.system.dir", 
      jobConf.get("hadoop.log.dir", "/tmp/hadoop-"+jobConf.getUser()) + "/mapred/system");
  jobConf.set("mapred.jobtracker.taskScheduler", JobQueueTaskScheduler.class.getName());
  
  FileSystem lfs = FileSystem.getLocal(getConf());
  Path logPath =
    new Path(System.getProperty("hadoop.log.dir")).makeQualified(lfs);
  jobConf.set("mapred.system.dir", logPath.toString());
  jobConf.set("hadoop.job.history.location", (new Path(logPath, "history")
      .toString()));
  
  jt = SimulatorJobTracker.startTracker(jobConf, now, this);
  jt.offerService();
  
  // max Map/Reduce tasks per node
  int maxMaps = getConf().getInt("mapred.tasktracker.map.tasks.maximum",
      DEFAULT_MAP_SLOTS_PER_NODE);
  int maxReduces = getConf().getInt(
      "mapred.tasktracker.reduce.tasks.maximum",
      DEFAULT_REDUCE_SLOTS_PER_NODE);

  MachineNode defaultNode = new MachineNode.Builder("default", 2)
      .setMapSlots(maxMaps).setReduceSlots(maxReduces).build();
  ZombieCluster cluster = new ZombieCluster(new Path(topologyFile), 
      defaultNode, jobConf);
  long firstJobStartTime = now + 60000;
  JobStoryProducer jobStoryProducer = new SimulatorJobStoryProducer(
      new Path(traceFile), cluster, firstJobStartTime, jobConf);
  
  jc = new SimulatorJobClient(jt, jobStoryProducer);
  queue.addAll(jc.init(firstJobStartTime));

  // create TTs based on topology.json     
  startTaskTrackers(cluster, now);
  
  terminateTime = getConf().getLong("mumak.terminate.time", Long.MAX_VALUE);
  if (terminateTime <= 0) {
    throw new IllegalArgumentException("Terminate time must be positive: "
        + terminateTime);
  }
}
 
Example #15
Source File: DistributedCacheEmulator.java    From big-c with Apache License 2.0 3 votes vote down vote up
/**
 * Create distributed cache directories.
 * Also create a file that contains the list of distributed cache files
 * that will be used as distributed cache files for all the simulated jobs.
 * @param jsp job story producer for the trace
 * @return exit code
 * @throws IOException
 */
int setupGenerateDistCacheData(JobStoryProducer jsp)
    throws IOException {

  createDistCacheDirectory();
  return buildDistCacheFilesList(jsp);
}
 
Example #16
Source File: DistributedCacheEmulator.java    From hadoop with Apache License 2.0 3 votes vote down vote up
/**
 * Create distributed cache directories.
 * Also create a file that contains the list of distributed cache files
 * that will be used as distributed cache files for all the simulated jobs.
 * @param jsp job story producer for the trace
 * @return exit code
 * @throws IOException
 */
int setupGenerateDistCacheData(JobStoryProducer jsp)
    throws IOException {

  createDistCacheDirectory();
  return buildDistCacheFilesList(jsp);
}
 
Example #17
Source File: Gridmix.java    From big-c with Apache License 2.0 3 votes vote down vote up
/**
 * Create an appropriate {@code JobStoryProducer} object for the
 * given trace.
 * 
 * @param traceIn the path to the trace file. The special path
 * &quot;-&quot; denotes the standard input stream.
 *
 * @param conf the configuration to be used.
 *
 * @throws IOException if there was an error.
 */
protected JobStoryProducer createJobStoryProducer(String traceIn,
    Configuration conf) throws IOException {
  if ("-".equals(traceIn)) {
    return new ZombieJobProducer(System.in, null);
  }
  return new ZombieJobProducer(new Path(traceIn), null, conf);
}
 
Example #18
Source File: ReplayJobFactory.java    From hadoop with Apache License 2.0 3 votes vote down vote up
/**
 * Creating a new instance does not start the thread.
 *
 * @param submitter   Component to which deserialized jobs are passed
 * @param jobProducer Job story producer
 *                    {@link org.apache.hadoop.tools.rumen.ZombieJobProducer}
 * @param scratch     Directory into which to write output from simulated jobs
 * @param conf        Config passed to all jobs to be submitted
 * @param startFlag   Latch released from main to start pipeline
 * @param resolver
 * @throws java.io.IOException
 */
public ReplayJobFactory(
  JobSubmitter submitter, JobStoryProducer jobProducer, Path scratch,
  Configuration conf, CountDownLatch startFlag, UserResolver resolver)
  throws IOException {
  super(submitter, jobProducer, scratch, conf, startFlag, resolver);
}
 
Example #19
Source File: SerialJobFactory.java    From big-c with Apache License 2.0 3 votes vote down vote up
/**
 * Creating a new instance does not start the thread.
 *
 * @param submitter   Component to which deserialized jobs are passed
 * @param jobProducer Job story producer
 *                    {@link org.apache.hadoop.tools.rumen.ZombieJobProducer}
 * @param scratch     Directory into which to write output from simulated jobs
 * @param conf        Config passed to all jobs to be submitted
 * @param startFlag   Latch released from main to start pipeline
 * @throws java.io.IOException
 */
public SerialJobFactory(
  JobSubmitter submitter, JobStoryProducer jobProducer, Path scratch,
  Configuration conf, CountDownLatch startFlag, UserResolver resolver)
  throws IOException {
  super(submitter, jobProducer, scratch, conf, startFlag, resolver);
}
 
Example #20
Source File: ReplayJobFactory.java    From big-c with Apache License 2.0 3 votes vote down vote up
/**
 * Creating a new instance does not start the thread.
 *
 * @param submitter   Component to which deserialized jobs are passed
 * @param jobProducer Job story producer
 *                    {@link org.apache.hadoop.tools.rumen.ZombieJobProducer}
 * @param scratch     Directory into which to write output from simulated jobs
 * @param conf        Config passed to all jobs to be submitted
 * @param startFlag   Latch released from main to start pipeline
 * @param resolver
 * @throws java.io.IOException
 */
public ReplayJobFactory(
  JobSubmitter submitter, JobStoryProducer jobProducer, Path scratch,
  Configuration conf, CountDownLatch startFlag, UserResolver resolver)
  throws IOException {
  super(submitter, jobProducer, scratch, conf, startFlag, resolver);
}
 
Example #21
Source File: SerialJobFactory.java    From hadoop with Apache License 2.0 3 votes vote down vote up
/**
 * Creating a new instance does not start the thread.
 *
 * @param submitter   Component to which deserialized jobs are passed
 * @param jobProducer Job story producer
 *                    {@link org.apache.hadoop.tools.rumen.ZombieJobProducer}
 * @param scratch     Directory into which to write output from simulated jobs
 * @param conf        Config passed to all jobs to be submitted
 * @param startFlag   Latch released from main to start pipeline
 * @throws java.io.IOException
 */
public SerialJobFactory(
  JobSubmitter submitter, JobStoryProducer jobProducer, Path scratch,
  Configuration conf, CountDownLatch startFlag, UserResolver resolver)
  throws IOException {
  super(submitter, jobProducer, scratch, conf, startFlag, resolver);
}
 
Example #22
Source File: Gridmix.java    From hadoop with Apache License 2.0 3 votes vote down vote up
/**
 * Create an appropriate {@code JobStoryProducer} object for the
 * given trace.
 * 
 * @param traceIn the path to the trace file. The special path
 * &quot;-&quot; denotes the standard input stream.
 *
 * @param conf the configuration to be used.
 *
 * @throws IOException if there was an error.
 */
protected JobStoryProducer createJobStoryProducer(String traceIn,
    Configuration conf) throws IOException {
  if ("-".equals(traceIn)) {
    return new ZombieJobProducer(System.in, null);
  }
  return new ZombieJobProducer(new Path(traceIn), null, conf);
}
 
Example #23
Source File: SimulatorJobClient.java    From RDFS with Apache License 2.0 2 votes vote down vote up
/**
 * Constructor.
 * 
 * @param jobTracker
 *          The job tracker where we submit job to. Note that the {@link
 *          SimulatorJobClient} interacts with the JobTracker through the
 *          {@link ClientProtocol}.
 * @param jobStoryProducer
 */
public SimulatorJobClient(JobTracker jobTracker, JobStoryProducer jobStoryProducer) {
  this.jobTracker = jobTracker;
  this.jobStoryProducer = jobStoryProducer;
}