org.apache.hadoop.fs.FSInputStream Java Examples

The following examples show how to use org.apache.hadoop.fs.FSInputStream. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HoodieLogFileReader.java    From hudi with Apache License 2.0 6 votes vote down vote up
public HoodieLogFileReader(FileSystem fs, HoodieLogFile logFile, Schema readerSchema, int bufferSize,
    boolean readBlockLazily, boolean reverseReader) throws IOException {
  FSDataInputStream fsDataInputStream = fs.open(logFile.getPath(), bufferSize);
  if (fsDataInputStream.getWrappedStream() instanceof FSInputStream) {
    this.inputStream = new FSDataInputStream(
        new BufferedFSInputStream((FSInputStream) fsDataInputStream.getWrappedStream(), bufferSize));
  } else {
    // fsDataInputStream.getWrappedStream() maybe a BufferedFSInputStream
    // need to wrap in another BufferedFSInputStream the make bufferSize work?
    this.inputStream = fsDataInputStream;
  }

  this.logFile = logFile;
  this.readerSchema = readerSchema;
  this.readBlockLazily = readBlockLazily;
  this.reverseReader = reverseReader;
  if (this.reverseReader) {
    this.reverseLogFilePosition = this.lastReverseLogFilePosition = fs.getFileStatus(logFile.getPath()).getLen();
  }
  addShutDownHook();
}
 
Example #2
Source File: TestCopy.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testInterruptedRename() throws Exception {
  FSDataOutputStream out = mock(FSDataOutputStream.class);
  whenFsCreate().thenReturn(out);
  when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
  when(mockFs.rename(eq(tmpPath), eq(path))).thenThrow(
      new InterruptedIOException());
  FSInputStream in = mock(FSInputStream.class);
  when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
  
  tryCopyStream(in, false);
  verify(mockFs).delete(eq(tmpPath), anyBoolean());
  verify(mockFs).rename(eq(tmpPath), eq(path));
  verify(mockFs, never()).delete(eq(path), anyBoolean());
  verify(mockFs, never()).close();
}
 
Example #3
Source File: TestCopy.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testCopyStreamTargetExists() throws Exception {
  FSDataOutputStream out = mock(FSDataOutputStream.class);
  whenFsCreate().thenReturn(out);
  when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat);
  target.refreshStatus(); // so it's updated as existing
  cmd.setOverwrite(true);
  when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
  when(mockFs.delete(eq(path), eq(false))).thenReturn(true);
  when(mockFs.rename(eq(tmpPath), eq(path))).thenReturn(true);
  FSInputStream in = mock(FSInputStream.class);
  when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
  
  tryCopyStream(in, true);
  verify(mockFs).delete(eq(path), anyBoolean());
  verify(mockFs).rename(eq(tmpPath), eq(path));
  verify(mockFs, never()).delete(eq(tmpPath), anyBoolean());
  verify(mockFs, never()).close();
}
 
Example #4
Source File: TestCopy.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testInterruptedCopyBytes() throws Exception {
  FSDataOutputStream out = mock(FSDataOutputStream.class);
  whenFsCreate().thenReturn(out);
  when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
  FSInputStream in = mock(FSInputStream.class);
  // make IOUtils.copyBytes fail
  when(in.read(any(byte[].class), anyInt(), anyInt())).thenThrow(
      new InterruptedIOException());
  
  tryCopyStream(in, false);
  verify(mockFs).delete(eq(tmpPath), anyBoolean());
  verify(mockFs, never()).rename(any(Path.class), any(Path.class));
  verify(mockFs, never()).delete(eq(path), anyBoolean());
  verify(mockFs, never()).close();
}
 
Example #5
Source File: TestCopy.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testInterruptedRename() throws Exception {
  FSDataOutputStream out = mock(FSDataOutputStream.class);
  whenFsCreate().thenReturn(out);
  when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
  when(mockFs.rename(eq(tmpPath), eq(path))).thenThrow(
      new InterruptedIOException());
  FSInputStream in = mock(FSInputStream.class);
  when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
  
  tryCopyStream(in, false);
  verify(mockFs).delete(eq(tmpPath), anyBoolean());
  verify(mockFs).rename(eq(tmpPath), eq(path));
  verify(mockFs, never()).delete(eq(path), anyBoolean());
  verify(mockFs, never()).close();
}
 
Example #6
Source File: TestDFSUpgradeFromImage.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Try to open a file for reading several times.
 * 
 * If we fail because lease recovery hasn't completed, retry the open.
 */
private static FSInputStream dfsOpenFileWithRetries(DistributedFileSystem dfs,
    String pathName) throws IOException {
  IOException exc = null;
  for (int tries = 0; tries < 10; tries++) {
    try {
      return dfs.dfs.open(pathName);
    } catch (IOException e) {
      exc = e;
    }
    if (!exc.getMessage().contains("Cannot obtain " +
        "block length for LocatedBlock")) {
      throw exc;
    }
    try {
      Thread.sleep(1000);
    } catch (InterruptedException ignored) {}
  }
  throw exc;
}
 
Example #7
Source File: TestDFSUpgradeFromImage.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Try to open a file for reading several times.
 * 
 * If we fail because lease recovery hasn't completed, retry the open.
 */
private static FSInputStream dfsOpenFileWithRetries(DistributedFileSystem dfs,
    String pathName) throws IOException {
  IOException exc = null;
  for (int tries = 0; tries < 10; tries++) {
    try {
      return dfs.dfs.open(pathName);
    } catch (IOException e) {
      exc = e;
    }
    if (!exc.getMessage().contains("Cannot obtain " +
        "block length for LocatedBlock")) {
      throw exc;
    }
    try {
      Thread.sleep(1000);
    } catch (InterruptedException ignored) {}
  }
  throw exc;
}
 
Example #8
Source File: TestCopy.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testInterruptedCopyBytes() throws Exception {
  FSDataOutputStream out = mock(FSDataOutputStream.class);
  whenFsCreate().thenReturn(out);
  when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
  FSInputStream in = mock(FSInputStream.class);
  // make IOUtils.copyBytes fail
  when(in.read(any(byte[].class), anyInt(), anyInt())).thenThrow(
      new InterruptedIOException());
  
  tryCopyStream(in, false);
  verify(mockFs).delete(eq(tmpPath), anyBoolean());
  verify(mockFs, never()).rename(any(Path.class), any(Path.class));
  verify(mockFs, never()).delete(eq(path), anyBoolean());
  verify(mockFs, never()).close();
}
 
Example #9
Source File: TestCopy.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testCopyStreamTargetExists() throws Exception {
  FSDataOutputStream out = mock(FSDataOutputStream.class);
  whenFsCreate().thenReturn(out);
  when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat);
  target.refreshStatus(); // so it's updated as existing
  cmd.setOverwrite(true);
  when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
  when(mockFs.delete(eq(path), eq(false))).thenReturn(true);
  when(mockFs.rename(eq(tmpPath), eq(path))).thenReturn(true);
  FSInputStream in = mock(FSInputStream.class);
  when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
  
  tryCopyStream(in, true);
  verify(mockFs).delete(eq(path), anyBoolean());
  verify(mockFs).rename(eq(tmpPath), eq(path));
  verify(mockFs, never()).delete(eq(tmpPath), anyBoolean());
  verify(mockFs, never()).close();
}
 
Example #10
Source File: TestDFSUpgradeFromImage.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void verifyDir(DistributedFileSystem dfs, Path dir,
    CRC32 overallChecksum) throws IOException {
  FileStatus[] fileArr = dfs.listStatus(dir);
  TreeMap<Path, Boolean> fileMap = new TreeMap<Path, Boolean>();
  
  for(FileStatus file : fileArr) {
    fileMap.put(file.getPath(), Boolean.valueOf(file.isDirectory()));
  }
  
  for(Iterator<Path> it = fileMap.keySet().iterator(); it.hasNext();) {
    Path path = it.next();
    boolean isDir = fileMap.get(path);
    
    String pathName = path.toUri().getPath();
    overallChecksum.update(pathName.getBytes());
    
    if ( isDir ) {
      verifyDir(dfs, path, overallChecksum);
    } else {
      // this is not a directory. Checksum the file data.
      CRC32 fileCRC = new CRC32();
      FSInputStream in = dfsOpenFileWithRetries(dfs, pathName);
      byte[] buf = new byte[4096];
      int nRead = 0;
      while ( (nRead = in.read(buf, 0, buf.length)) > 0 ) {
        fileCRC.update(buf, 0, nRead);
      }
      
      verifyChecksum(pathName, fileCRC.getValue());
    }
  }
}
 
Example #11
Source File: HftpFileSystem.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
@Override
public FSDataInputStream open(Path f, int buffersize) throws IOException {
  HttpURLConnection connection = null;
  connection = openConnection("/data" + f.toUri().getPath(), "ugi=" + ugi);
  connection.setRequestMethod("GET");
  connection.connect();
  final InputStream in = connection.getInputStream();
  return new FSDataInputStream(new FSInputStream() {
      public int read() throws IOException {
        return in.read();
      }
      public int read(byte[] b, int off, int len) throws IOException {
        return in.read(b, off, len);
      }

      public void close() throws IOException {
        in.close();
      }

      public void seek(long pos) throws IOException {
        throw new IOException("Can't seek!");
      }
      public long getPos() throws IOException {
        throw new IOException("Position unknown!");
      }
      public boolean seekToNewSource(long targetPos) throws IOException {
        return false;
      }
    });
}
 
Example #12
Source File: TestCopy.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testCopyStreamTarget() throws Exception {
  FSDataOutputStream out = mock(FSDataOutputStream.class);
  whenFsCreate().thenReturn(out);
  when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
  when(mockFs.rename(eq(tmpPath), eq(path))).thenReturn(true);
  FSInputStream in = mock(FSInputStream.class);
  when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
  
  tryCopyStream(in, true);
  verify(mockFs, never()).delete(eq(path), anyBoolean());
  verify(mockFs).rename(eq(tmpPath), eq(path));
  verify(mockFs, never()).delete(eq(tmpPath), anyBoolean());
  verify(mockFs, never()).close();
}
 
Example #13
Source File: TestCopy.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testCopyStreamTarget() throws Exception {
  FSDataOutputStream out = mock(FSDataOutputStream.class);
  whenFsCreate().thenReturn(out);
  when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
  when(mockFs.rename(eq(tmpPath), eq(path))).thenReturn(true);
  FSInputStream in = mock(FSInputStream.class);
  when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
  
  tryCopyStream(in, true);
  verify(mockFs, never()).delete(eq(path), anyBoolean());
  verify(mockFs).rename(eq(tmpPath), eq(path));
  verify(mockFs, never()).delete(eq(tmpPath), anyBoolean());
  verify(mockFs, never()).close();
}
 
Example #14
Source File: TestDFSUpgradeFromImage.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void verifyDir(DistributedFileSystem dfs, Path dir,
    CRC32 overallChecksum) throws IOException {
  FileStatus[] fileArr = dfs.listStatus(dir);
  TreeMap<Path, Boolean> fileMap = new TreeMap<Path, Boolean>();
  
  for(FileStatus file : fileArr) {
    fileMap.put(file.getPath(), Boolean.valueOf(file.isDirectory()));
  }
  
  for(Iterator<Path> it = fileMap.keySet().iterator(); it.hasNext();) {
    Path path = it.next();
    boolean isDir = fileMap.get(path);
    
    String pathName = path.toUri().getPath();
    overallChecksum.update(pathName.getBytes());
    
    if ( isDir ) {
      verifyDir(dfs, path, overallChecksum);
    } else {
      // this is not a directory. Checksum the file data.
      CRC32 fileCRC = new CRC32();
      FSInputStream in = dfsOpenFileWithRetries(dfs, pathName);
      byte[] buf = new byte[4096];
      int nRead = 0;
      while ( (nRead = in.read(buf, 0, buf.length)) > 0 ) {
        fileCRC.update(buf, 0, nRead);
      }
      
      verifyChecksum(pathName, fileCRC.getValue());
    }
  }
}
 
Example #15
Source File: StrictBufferedFSInputStream.java    From big-c with Apache License 2.0 4 votes vote down vote up
public StrictBufferedFSInputStream(FSInputStream in,
                                   int size) {
  super(in, size);
}
 
Example #16
Source File: StrictBufferedFSInputStream.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public StrictBufferedFSInputStream(FSInputStream in,
                                   int size) {
  super(in, size);
}
 
Example #17
Source File: StrictBufferedFSInputStream.java    From sahara-extra with Apache License 2.0 4 votes vote down vote up
public StrictBufferedFSInputStream(FSInputStream in,
                                   int size) {
  super(in, size);
}