Java Code Examples for org.apache.hadoop.fs.FileSystem#newInstance()

The following examples show how to use org.apache.hadoop.fs.FileSystem#newInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestFileSystemCaching.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFsUniqueness() throws Exception {
  final Configuration conf = new Configuration();
  conf.set("fs.cachedfile.impl", FileSystem.getFileSystemClass("file", null).getName());
  // multiple invocations of FileSystem.get return the same object.
  FileSystem fs1 = FileSystem.get(conf);
  FileSystem fs2 = FileSystem.get(conf);
  assertTrue(fs1 == fs2);

  // multiple invocations of FileSystem.newInstance return different objects
  fs1 = FileSystem.newInstance(new URI("cachedfile://a"), conf, "bar");
  fs2 = FileSystem.newInstance(new URI("cachedfile://a"), conf, "bar");
  assertTrue(fs1 != fs2 && !fs1.equals(fs2));
  fs1.close();
  fs2.close();
}
 
Example 2
Source File: ApexCli.java    From Bats with Apache License 2.0 6 votes vote down vote up
AppPackage newAppPackageInstance(URI uri, boolean suppressOutput) throws IOException
{
  PrintStream outputStream = suppressOutput ? suppressOutput() : null;
  try {
    final String scheme = uri.getScheme();
    if (scheme == null || scheme.equals("file")) {
      return new AppPackage(new FileInputStream(new File(expandFileName(uri.getPath(), true))), true);
    } else {
      try (FileSystem fs = FileSystem.newInstance(uri, conf)) {
        return new AppPackage(fs.open(new Path(uri.getPath())), true);
      }
    }
  } finally {
    if (outputStream != null) {
      restoreOutput(outputStream);
    }
  }
}
 
Example 3
Source File: StramClientUtils.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
public static FileSystem newFileSystemInstance(Configuration conf) throws IOException
{
  String dfsRootDir = conf.get(DT_DFS_ROOT_DIR);
  if (StringUtils.isBlank(dfsRootDir)) {
    return FileSystem.newInstance(conf);
  } else {
    if (dfsRootDir.contains(DT_DFS_USER_NAME)) {
      dfsRootDir = dfsRootDir.replace(DT_DFS_USER_NAME, UserGroupInformation.getLoginUser().getShortUserName());
      conf.set(DT_DFS_ROOT_DIR, dfsRootDir);
    }
    try {
      return FileSystem.newInstance(new URI(dfsRootDir), conf);
    } catch (URISyntaxException ex) {
      LOG.warn("{} is not a valid URI. Returning the default filesystem", dfsRootDir, ex);
      return FileSystem.newInstance(conf);
    }
  }
}
 
Example 4
Source File: StramAppLauncherTest.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
@Test
public void testResetContextClassLoaderResetsToInitialClassLoader() throws Exception
{
  // Setup
  Configuration conf = new Configuration();
  FileSystem fs = FileSystem.newInstance(conf);
  StramAppLauncher appLauncher = new StramAppLauncher(fs, conf);

  Whitebox.setInternalState(appLauncher, "launchDependencies", new LinkedHashSet<URL>());

  // Get initial contextClassLoader
  ClassLoader initialClassLoader = Thread.currentThread().getContextClassLoader();

  appLauncher.loadDependencies();
  appLauncher.resetContextClassLoader();

  ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
  Assert.assertSame(initialClassLoader, currentClassLoader);
}
 
Example 5
Source File: WriterUtils.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
private static FileSystem getWriterFsUsingKeytab(State state, URI uri)
    throws IOException {
  FileSystem fs = FileSystem.newInstance(uri, new Configuration());
  try {
    Preconditions.checkArgument(state.contains(ConfigurationKeys.FS_PROXY_AS_USER_NAME),
        "Missing required property " + ConfigurationKeys.FS_PROXY_AS_USER_NAME);
    Preconditions.checkArgument(state.contains(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS),
        "Missing required property " + ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS);
    Preconditions.checkArgument(state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION),
        "Missing required property " + ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
    String user = state.getProp(ConfigurationKeys.FS_PROXY_AS_USER_NAME);
    String superUser = state.getProp(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS);
    Path keytabLocation = new Path(state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION));
    return ProxiedFileSystemCache.fromKeytab().userNameToProxyAs(user).fsURI(uri)
        .superUserKeytabLocation(keytabLocation).superUserName(superUser).conf(HadoopUtils.newConfiguration())
        .referenceFS(fs).build();
  } catch (ExecutionException e) {
    throw new IOException(e);
  }
}
 
Example 6
Source File: FinalResponseReducer.java    From incubator-retired-pirk with Apache License 2.0 6 votes vote down vote up
@Override
public void setup(Context ctx) throws IOException, InterruptedException
{
  super.setup(ctx);

  mos = new MultipleOutputs<>(ctx);

  FileSystem fs = FileSystem.newInstance(ctx.getConfiguration());
  storage = new HadoopFileSystemStore(fs);
  String queryDir = ctx.getConfiguration().get("pirMR.queryInputDir");
  Query query = storage.recall(queryDir, Query.class);
  QueryInfo queryInfo = query.getQueryInfo();

  outputFile = ctx.getConfiguration().get("pirMR.outputFile");

  response = new Response(queryInfo);
}
 
Example 7
Source File: ParquetFileReaderTest.java    From kafka-connect-fs with Apache License 2.0 6 votes vote down vote up
@ParameterizedTest
@MethodSource("fileSystemConfigProvider")
public void readerWithProjection(ReaderFsTestConfig fsConfig) throws IOException {
    Map<String, Object> readerConfig = getReaderConfig();
    readerConfig.put(ParquetFileReader.FILE_READER_PARQUET_PROJECTION, projectionSchema.toString());
    readerConfig.put(AgnosticFileReader.FILE_READER_AGNOSTIC_EXTENSIONS_PARQUET, getFileExtension());
    fsConfig.setReader(getReader(fsConfig.getFs(), fsConfig.getDataFile(), readerConfig));
    while (fsConfig.getReader().hasNext()) {
        Struct record = fsConfig.getReader().next();
        assertNotNull(record.schema().field(FIELD_INDEX));
        assertNotNull(record.schema().field(FIELD_NAME));
        assertNull(record.schema().field(FIELD_SURNAME));
    }
    FileSystem testFs = FileSystem.newInstance(fsConfig.getFsUri(), new Configuration());
    fsConfig.setReader(getReader(testFs, fsConfig.getDataFile(), readerConfig));
    assertThrows(DataException.class, () -> readAllData(fsConfig));
}
 
Example 8
Source File: ParquetFileReaderTest.java    From kafka-connect-fs with Apache License 2.0 5 votes vote down vote up
@ParameterizedTest
@MethodSource("fileSystemConfigProvider")
public void readerWithSchema(ReaderFsTestConfig fsConfig) throws IOException {
    Map<String, Object> readerConfig = getReaderConfig();
    readerConfig.put(ParquetFileReader.FILE_READER_PARQUET_SCHEMA, readerSchema.toString());
    readerConfig.put(AgnosticFileReader.FILE_READER_AGNOSTIC_EXTENSIONS_PARQUET, getFileExtension());
    FileSystem testFs = FileSystem.newInstance(fsConfig.getFsUri(), new Configuration());
    fsConfig.setReader(getReader(testFs, fsConfig.getDataFile(), readerConfig));
    readAllData(fsConfig);
}
 
Example 9
Source File: TestPerRegionIndexWriteCache.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Before
  public void setUp() throws Exception {
      Path hbaseRootDir = TEST_UTIL.getDataTestDir();
      TEST_UTIL.getConfiguration().set("hbase.rootdir", hbaseRootDir.toString());

      FileSystem newFS = FileSystem.newInstance(TEST_UTIL.getConfiguration());
      HRegionInfo hri = new HRegionInfo(tableName, null, null, false);
      Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
      WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), null, "TestPerRegionIndexWriteCache");
      WAL wal = walFactory.getWAL(Bytes.toBytes("logs"));
      HTableDescriptor htd = new HTableDescriptor(tableName);
      HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
      htd.addFamily(a);
      
      r1 = new HRegion(basedir, wal, newFS, TEST_UTIL.getConfiguration(), hri, htd, null) {
          @Override
          public int hashCode() {
            return 1;
          }

          @Override
          public String toString() {
            return "testRegion1";
          }
        };
        
      r2 = new HRegion(basedir, wal, newFS, TEST_UTIL.getConfiguration(), hri, htd, null) {
          @Override
          public int hashCode() {
            return 2;
          }

          @Override
          public String toString() {
            return "testRegion1";
          }
        };
  }
 
Example 10
Source File: FileEndpointTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
private boolean waitTillFileIsPopulated(String outputFolder, int timeout) throws IOException, InterruptedException
{
  boolean result;
  long now = System.currentTimeMillis();
  Path outDir = new Path("file://" + new File(outputFolder).getAbsolutePath());
  try (FileSystem fs = FileSystem.newInstance(outDir.toUri(), new Configuration())) {
    List<String> strings = Lists.newArrayList();
    while (System.currentTimeMillis() - now < timeout) {
      if (fs.exists(outDir)) {
        File file = new File(outputFolder);
        if (file.list().length > 0) {
          File file1 = new File(outputFolder + file.list()[0]);
          strings = FileUtils.readLines(file1);
          if (strings.size() != 0) {
            break;
          }
        }
      }

      Thread.sleep(500);
    }

    result = fs.exists(outDir) && (strings.size() != 0);
  }

  return result;
}
 
Example 11
Source File: ParquetFileReaderTest.java    From kafka-connect-fs with Apache License 2.0 5 votes vote down vote up
@ParameterizedTest
@MethodSource("fileSystemConfigProvider")
public void readerWithInvalidSchema(ReaderFsTestConfig fsConfig) throws IOException {
    Map<String, Object> readerConfig = getReaderConfig();
    readerConfig.put(ParquetFileReader.FILE_READER_PARQUET_SCHEMA, Schema.create(Schema.Type.STRING).toString());
    readerConfig.put(AgnosticFileReader.FILE_READER_AGNOSTIC_EXTENSIONS_PARQUET, getFileExtension());
    FileSystem testFs = FileSystem.newInstance(fsConfig.getFsUri(), new Configuration());
    fsConfig.setReader(getReader(testFs, fsConfig.getDataFile(), readerConfig));
    try {
        readAllData(fsConfig);
    } catch (Exception e) {
        assertEquals(ConnectException.class, e.getClass());
        assertEquals(AvroRuntimeException.class, e.getCause().getClass());
    }
}
 
Example 12
Source File: ColumnMultReducer.java    From incubator-retired-pirk with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(Context ctx) throws IOException, InterruptedException
{
  super.setup(ctx);

  outputValue = new Text();
  mos = new MultipleOutputs<>(ctx);

  FileSystem fs = FileSystem.newInstance(ctx.getConfiguration());
  String queryDir = ctx.getConfiguration().get("pirMR.queryInputDir");
  query = new HadoopFileSystemStore(fs).recall(queryDir, Query.class);
}
 
Example 13
Source File: AbstractPolicy.java    From kafka-connect-fs with Apache License 2.0 5 votes vote down vote up
private void configFs(Map<String, Object> customConfigs) throws IOException {
    for (String uri : this.conf.getFsUris()) {
        Configuration fsConfig = new Configuration();
        customConfigs.entrySet().stream()
                .filter(entry -> entry.getKey().startsWith(FsSourceTaskConfig.POLICY_PREFIX_FS))
                .forEach(entry -> fsConfig.set(entry.getKey().replace(FsSourceTaskConfig.POLICY_PREFIX_FS, ""),
                        (String) entry.getValue()));

        Path workingDir = new Path(convert(uri));
        FileSystem fs = FileSystem.newInstance(workingDir.toUri(), fsConfig);
        fs.setWorkingDirectory(workingDir);
        this.fileSystems.add(fs);
    }
}
 
Example 14
Source File: EnrichmentOperator.java    From examples with Apache License 2.0 5 votes vote down vote up
private FileLoader(String file, String backupResource) throws IOException
{
  Configuration conf = new Configuration();
  this.backupResource = backupResource;
  this.filePath = new Path(file);

  try {
    this.fs = FileSystem.newInstance(filePath.toUri(), conf);
  } catch (IOException ex) {
    LOG.error("Exception {}", ex);
  }
}
 
Example 15
Source File: PipelineMapper.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private static String getHeaderFromFile(Configuration hadoopConf, Path path) throws IOException {
  String header;
  try (FileSystem fs = FileSystem.newInstance(hadoopConf);
       BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(path), StandardCharsets.UTF_8))) {
    // read one line - the header
    header = br.readLine();
  }
  return header;
}
 
Example 16
Source File: StramClient.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up
public void copyInitialState(Path origAppDir) throws IOException
{
  // locate previous snapshot
  long copyStart = System.currentTimeMillis();
  String newAppDir = this.dag.assertAppPath();

  FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(origAppDir.toString(), conf);
  // read snapshot against new dependencies
  Object snapshot = recoveryHandler.restore();
  if (snapshot == null) {
    throw new IllegalArgumentException("No previous application state found in " + origAppDir);
  }
  InputStream logIs = recoveryHandler.getLog();

  // modify snapshot state to switch app id
  ((StreamingContainerManager.CheckpointState)snapshot).setApplicationId(this.dag, conf);
  Path checkpointPath = new Path(newAppDir, LogicalPlan.SUBDIR_CHECKPOINTS);

  FileSystem fs = FileSystem.newInstance(origAppDir.toUri(), conf);
  // remove the path that was created by the storage agent during deserialization and replacement
  fs.delete(checkpointPath, true);

  // write snapshot to new location
  recoveryHandler = new FSRecoveryHandler(newAppDir, conf);
  recoveryHandler.save(snapshot);
  OutputStream logOs = recoveryHandler.rotateLog();
  IOUtils.copy(logIs, logOs);
  logOs.flush();
  logOs.close();
  logIs.close();

  List<String> excludeDirs = Arrays.asList(LogicalPlan.SUBDIR_CHECKPOINTS, LogicalPlan.SUBDIR_EVENTS, LogicalPlan.SUBDIR_STATS);
  // copy sub directories that are not present in target
  FileStatus[] lFiles = fs.listStatus(origAppDir);

  // In case of MapR/MapR-FS, f.getPath().toString() returns path as maprfs:///<orig app dir>
  // whereas origAppDir.toString & newAppDir are in maprfs:/<orig or new app dir> format
  // e.g.
  // f.getPath().toString -> maprfs:///user/dtadmin/datatorrent/apps/application_1481890072066_0004/checkpoints
  // origAppDir -> maprfs:/user/dtadmin/datatorrent/apps/application_1481890072066_0004
  // newAppDir -> maprfs:/user/dtadmin/datatorrent/apps/application_1481890072066_0005

  String origAppDirPath = Path.getPathWithoutSchemeAndAuthority(origAppDir).toString();
  String newAppDirPath = Path.getPathWithoutSchemeAndAuthority(new Path(newAppDir)).toString();

  for (FileStatus f : lFiles) {
    if (f.isDirectory() && !excludeDirs.contains(f.getPath().getName())) {
      String targetPath = f.getPath().toString().replace(origAppDirPath, newAppDirPath);
      if (!fs.exists(new Path(targetPath))) {
        LOG.debug("Copying {} size {} to {}", f.getPath(), f.getLen(), targetPath);
        long start = System.currentTimeMillis();
        FileUtil.copy(fs, f.getPath(), fs, new Path(targetPath), false, conf);
        LOG.debug("Copying {} to {} took {} ms", f.getPath(), f.getLen(), targetPath, System.currentTimeMillis() - start);
      } else {
        LOG.debug("Ignoring {} as it already exists under {}", f.getPath(), targetPath);
      }
    }
  }
  LOG.info("Copying initial state took {} ms", System.currentTimeMillis() - copyStart);
}
 
Example 17
Source File: TestLeaseRecovery.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Block Recovery when the meta file not having crcs for all chunks in block
 * file
 */
@Test
public void testBlockRecoveryWithLessMetafile() throws Exception {
  Configuration conf = new Configuration();
  conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY,
      UserGroupInformation.getCurrentUser().getShortUserName());
  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  Path file = new Path("/testRecoveryFile");
  DistributedFileSystem dfs = cluster.getFileSystem();
  FSDataOutputStream out = dfs.create(file);
  int count = 0;
  while (count < 2 * 1024 * 1024) {
    out.writeBytes("Data");
    count += 4;
  }
  out.hsync();
  // abort the original stream
  ((DFSOutputStream) out.getWrappedStream()).abort();

  LocatedBlocks locations = cluster.getNameNodeRpc().getBlockLocations(
      file.toString(), 0, count);
  ExtendedBlock block = locations.get(0).getBlock();
  DataNode dn = cluster.getDataNodes().get(0);
  BlockLocalPathInfo localPathInfo = dn.getBlockLocalPathInfo(block, null);
  File metafile = new File(localPathInfo.getMetaPath());
  assertTrue(metafile.exists());

  // reduce the block meta file size
  RandomAccessFile raf = new RandomAccessFile(metafile, "rw");
  raf.setLength(metafile.length() - 20);
  raf.close();

  // restart DN to make replica to RWR
  DataNodeProperties dnProp = cluster.stopDataNode(0);
  cluster.restartDataNode(dnProp, true);

  // try to recover the lease
  DistributedFileSystem newdfs = (DistributedFileSystem) FileSystem
      .newInstance(cluster.getConfiguration(0));
  count = 0;
  while (++count < 10 && !newdfs.recoverLease(file)) {
    Thread.sleep(1000);
  }
  assertTrue("File should be closed", newdfs.recoverLease(file));

}
 
Example 18
Source File: HdfsStorage.java    From streamx with Apache License 2.0 4 votes vote down vote up
public HdfsStorage(Configuration conf,  String url) throws IOException {
  fs = FileSystem.newInstance(URI.create(url), conf);
  this.conf = conf;
  this.url = url;
}
 
Example 19
Source File: MiniDFSCluster.java    From hadoop with Apache License 2.0 2 votes vote down vote up
/**
 * Get another FileSystem instance that is different from FileSystem.get(conf).
 * This simulating different threads working on different FileSystem instances.
 */
public FileSystem getNewFileSystemInstance(int nnIndex) throws IOException {
  return FileSystem.newInstance(getURI(nnIndex), nameNodes[nnIndex].conf);
}
 
Example 20
Source File: FileStitcher.java    From attic-apex-malhar with Apache License 2.0 2 votes vote down vote up
/**
 *
 * @return Destination FileSystem instance
 * @throws IOException
 */
protected FileSystem getOutputFSInstance() throws IOException
{
  return FileSystem.newInstance((new Path(filePath)).toUri(), new Configuration());
}