org.apache.solr.core.HdfsDirectoryFactory Java Examples

public UpdateHandler(SolrCore core, UpdateLog updateLog)  {
  this.core=core;
  idField = core.getLatestSchema().getUniqueKeyField();
  idFieldType = idField!=null ? idField.getType() : null;
  parseEventListeners();
  PluginInfo ulogPluginInfo = core.getSolrConfig().getPluginInfo(UpdateLog.class.getName());

  // If this is a replica of type PULL, don't create the update log
  boolean skipUpdateLog = core.getCoreDescriptor().getCloudDescriptor() != null && !core.getCoreDescriptor().getCloudDescriptor().requiresTransactionLog();
  if (updateLog == null && ulogPluginInfo != null && ulogPluginInfo.isEnabled() && !skipUpdateLog) {
    DirectoryFactory dirFactory = core.getDirectoryFactory();
    if (dirFactory instanceof HdfsDirectoryFactory) {
      ulog = new HdfsUpdateLog(((HdfsDirectoryFactory)dirFactory).getConfDir());
    } else {
      String className = ulogPluginInfo.className == null ? UpdateLog.class.getName() : ulogPluginInfo.className;
      ulog = core.getResourceLoader().newInstance(className, UpdateLog.class);
    }

    if (!core.isReloaded() && !dirFactory.isPersistent()) {
      ulog.clearLog(core, ulogPluginInfo);
    }

    if (log.isInfoEnabled()) {
      log.info("Using UpdateLog implementation: {}", ulog.getClass().getName());
    }
    ulog.init(ulogPluginInfo);
    ulog.init(this, core);
  } else {
    ulog = updateLog;
  }
}
 

private static Configuration getConf(Path path) {
  Configuration conf = new Configuration();
  String confDir = System.getProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY);
  HdfsUtil.addHdfsResources(conf, confDir);

  String fsScheme = path.toUri().getScheme();
  if(fsScheme != null) {
    conf.setBoolean("fs." + fsScheme + ".impl.disable.cache", true);
  }
  return conf;
}
 

@SuppressWarnings("rawtypes")
@Override
public void init(NamedList args) {
  this.config = args;

  // Configure the size of the buffer used for copying index files to/from HDFS, if specified.
  if (args.get(HDFS_COPY_BUFFER_SIZE_PARAM) != null) {
    this.copyBufferSize = (Integer)args.get(HDFS_COPY_BUFFER_SIZE_PARAM);
    if (this.copyBufferSize <= 0) {
      throw new IllegalArgumentException("Value of " + HDFS_COPY_BUFFER_SIZE_PARAM + " must be > 0");
    }
  }

  String hdfsSolrHome = (String) Objects.requireNonNull(args.get(HdfsDirectoryFactory.HDFS_HOME),
      "Please specify " + HdfsDirectoryFactory.HDFS_HOME + " property.");
  Path path = new Path(hdfsSolrHome);
  while (path != null) { // Compute the path of root file-system (without requiring an additional system property).
    baseHdfsPath = path;
    path = path.getParent();
  }

  // We don't really need this factory instance. But we want to initialize it here to
  // make sure that all HDFS related initialization is at one place (and not duplicated here).
  factory = new HdfsDirectoryFactory();
  factory.init(args);
  this.hdfsConfig = factory.getConf(new Path(hdfsSolrHome));

  // Configure the umask mode if specified.
  if (args.get(HDFS_UMASK_MODE_PARAM) != null) {
    String umaskVal = (String)args.get(HDFS_UMASK_MODE_PARAM);
    this.hdfsConfig.set(FsPermission.UMASK_LABEL, umaskVal);
  }

  try {
    this.fileSystem = FileSystem.get(this.baseHdfsPath.toUri(), this.hdfsConfig);
  } catch (IOException e) {
    throw new SolrException(ErrorCode.SERVER_ERROR, e);
  }
}
 

@Test
public void testHdfsHomePropertySet() throws IOException {
  try (HdfsBackupRepository hdfsBackupRepository = new HdfsBackupRepository()) {
    NamedList<Object> namedList = new SimpleOrderedMap<>();
    namedList.add(HdfsDirectoryFactory.HDFS_HOME, "hdfs://localhost");
    hdfsBackupRepository.init(namedList);
  }
}
 

@Test
public void testCopyBufferSet() throws IOException {
  try (HdfsBackupRepository hdfsBackupRepository = new HdfsBackupRepository()) {
    NamedList<Object> namedList = new SimpleOrderedMap<>();
    namedList.add(HdfsDirectoryFactory.HDFS_HOME, "hdfs://localhost");
    namedList.add("solr.hdfs.buffer.size", 32768);
    hdfsBackupRepository.init(namedList);
    assertEquals(hdfsBackupRepository.copyBufferSize, 32768);
  }
}
 

@Test
public void testCopyBufferDefaultSize() throws IOException {
  try (HdfsBackupRepository hdfsBackupRepository = new HdfsBackupRepository()) {
    NamedList<Object> namedList = new SimpleOrderedMap<>();
    namedList.add(HdfsDirectoryFactory.HDFS_HOME, "hdfs://localhost");
    hdfsBackupRepository.init(namedList);
    assertEquals(hdfsBackupRepository.copyBufferSize, HdfsDirectory.DEFAULT_BUFFER_SIZE);
  }
}
 

public static EmbeddedSolrServer createEmbeddedSolrServer(Path solrHomeDir, FileSystem fs, Path outputShardDir)
    throws IOException {

  if (solrHomeDir == null) {
    throw new IOException("Unable to find solr home setting");
  }
  LOG.info("Creating embedded Solr server with solrHomeDir: " + solrHomeDir + ", fs: " + fs + ", outputShardDir: " + outputShardDir);

  Path solrDataDir = new Path(outputShardDir, "data");

  String dataDirStr = solrDataDir.toUri().toString();

  SolrResourceLoader loader = new SolrResourceLoader(solrHomeDir.toString(), null, null);

  LOG.info(String
      .format(Locale.ENGLISH, 
          "Constructed instance information solr.home %s (%s), instance dir %s, conf dir %s, writing index to solr.data.dir %s, with permdir %s",
          solrHomeDir, solrHomeDir.toUri(), loader.getInstanceDir(),
          loader.getConfigDir(), dataDirStr, outputShardDir));

  // TODO: This is fragile and should be well documented
  System.setProperty("solr.directoryFactory", HdfsDirectoryFactory.class.getName()); 
  System.setProperty("solr.lock.type", "hdfs"); 
  System.setProperty("solr.hdfs.nrtcachingdirectory", "false");
  System.setProperty("solr.hdfs.blockcache.enabled", "false");
  System.setProperty("solr.autoCommit.maxTime", "600000");
  System.setProperty("solr.autoSoftCommit.maxTime", "-1");
  
  CoreContainer container = new CoreContainer(loader);
  container.load();
  
  Properties props = new Properties();
  props.setProperty(CoreDescriptor.CORE_DATADIR, dataDirStr);
  
  CoreDescriptor descr = new CoreDescriptor(container, "core1", solrHomeDir.toString(), props);
  
  SolrCore core = container.create(descr);
  
  if (!(core.getDirectoryFactory() instanceof HdfsDirectoryFactory)) {
    throw new UnsupportedOperationException(
        "Invalid configuration. Currently, the only DirectoryFactory supported is "
            + HdfsDirectoryFactory.class.getSimpleName());
  }

  EmbeddedSolrServer solr = new EmbeddedSolrServer(container, "core1");
  return solr;
}
 

public static EmbeddedSolrServer createEmbeddedSolrServer(Path solrHomeDir, FileSystem fs, Path outputShardDir)
      throws IOException {

    LOG.info("Creating embedded Solr server with solrHomeDir: " + solrHomeDir + ", fs: " + fs + ", outputShardDir: " + outputShardDir);
    
    LOG.info("Using custom SolrRecordWriter class for HBaseMapReduceIndexer");
    if (LOG.isDebugEnabled()) {
      LOG.debug("Listing files contained in solrHomeDir {} ...", solrHomeDir);
      int i = 0;
      for (File file : FileUtils.listFiles(new File(solrHomeDir.toString()), null, true)) {
        // strip off common path prefix for better human readability
        String relPath = file.toString();
        relPath = relPath.substring(solrHomeDir.toString().length() + 1, relPath.length());
        LOG.debug("solrHomeDirFile[{}]: {}", i++, relPath);
      }
    }

    Path solrDataDir = new Path(outputShardDir, "data");

    String dataDirStr = solrDataDir.toUri().toString();

    SolrResourceLoader loader = new SolrResourceLoader(Paths.get(solrHomeDir.toString()), null, null);

    LOG.info(String
        .format(Locale.ENGLISH, 
            "Constructed instance information solr.home %s (%s), instance dir %s, conf dir %s, writing index to solr.data.dir %s, with permdir %s",
            solrHomeDir, solrHomeDir.toUri(), loader.getInstancePath(),
            loader.getConfigDir(), dataDirStr, outputShardDir));

    // TODO: This is fragile and should be well documented
    System.setProperty("solr.directoryFactory", HdfsDirectoryFactory.class.getName()); 
    System.setProperty("solr.lock.type", DirectoryFactory.LOCK_TYPE_HDFS);
    System.setProperty("solr.hdfs.nrtcachingdirectory", "false");
    System.setProperty("solr.hdfs.blockcache.enabled", "false");
    System.setProperty("solr.autoCommit.maxTime", "600000");
    System.setProperty("solr.autoSoftCommit.maxTime", "-1");
    
    CoreContainer container = new CoreContainer(loader);
    container.load();

    SolrCore core = container.create("core1", Paths.get(solrHomeDir.toString()), ImmutableMap.of(CoreDescriptor.CORE_DATADIR, dataDirStr), false);
//    SolrCore core = container.create("", ImmutableMap.of(CoreDescriptor.CORE_DATADIR, dataDirStr));
    
    if (!(core.getDirectoryFactory() instanceof HdfsDirectoryFactory)) {
      throw new UnsupportedOperationException(
          "Invalid configuration. Currently, the only DirectoryFactory supported is "
              + HdfsDirectoryFactory.class.getSimpleName());
    }

    EmbeddedSolrServer solr = new EmbeddedSolrServer(container, "core1");
//    EmbeddedSolrServer solr = new EmbeddedSolrServer(container, "");
    return solr;
  }
 

private static EmbeddedSolrServer createEmbeddedSolrServer(File solrHomeDir, FileSystem fs, Path outputShardDir)
        throws IOException {

  LOG.info("Creating embedded Solr server with solrHomeDir: " + solrHomeDir + ", fs: " + fs + ", outputShardDir: " + outputShardDir);

  // copy solrHomeDir to ensure it isn't modified across multiple unit tests or multiple EmbeddedSolrServer instances
  File tmpDir = Files.createTempDir();
  tmpDir.deleteOnExit();
  FileUtils.copyDirectory(solrHomeDir, tmpDir);
  solrHomeDir = tmpDir;

  Path solrDataDir = new Path(outputShardDir, "data");

  String dataDirStr = solrDataDir.toUri().toString();

  SolrResourceLoader loader = new SolrResourceLoader(Paths.get(solrHomeDir.toString()), null, null);

  LOG.info(String
          .format(Locale.ENGLISH,
                  "Constructed instance information solr.home %s (%s), instance dir %s, conf dir %s, writing index to solr.data.dir %s, with permdir %s",
                  solrHomeDir, solrHomeDir.toURI(), loader.getInstancePath(),
                  loader.getConfigDir(), dataDirStr, outputShardDir));

  // TODO: This is fragile and should be well documented
  System.setProperty("solr.directoryFactory", HdfsDirectoryFactory.class.getName());
  System.setProperty("solr.lock.type", DirectoryFactory.LOCK_TYPE_HDFS);
  System.setProperty("solr.hdfs.nrtcachingdirectory", "false");
  System.setProperty("solr.hdfs.blockcache.enabled", "false");
  System.setProperty("solr.autoCommit.maxTime", "600000");
  System.setProperty("solr.autoSoftCommit.maxTime", "-1");

  CoreContainer container = new CoreContainer(loader);
  container.load();

  SolrCore core = container.create("core1", Paths.get(solrHomeDir.toString()), ImmutableMap.of(CoreDescriptor.CORE_DATADIR, dataDirStr), false);

  if (!(core.getDirectoryFactory() instanceof HdfsDirectoryFactory)) {
    throw new UnsupportedOperationException(
            "Invalid configuration. Currently, the only DirectoryFactory supported is "
                    + HdfsDirectoryFactory.class.getSimpleName());
  }

  EmbeddedSolrServer solr = new EmbeddedSolrServer(container, "core1");
  return solr;
}