org.apache.hadoop.fs.FSInputStream Java Examples
The following examples show how to use
org.apache.hadoop.fs.FSInputStream.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HoodieLogFileReader.java From hudi with Apache License 2.0 | 6 votes |
public HoodieLogFileReader(FileSystem fs, HoodieLogFile logFile, Schema readerSchema, int bufferSize, boolean readBlockLazily, boolean reverseReader) throws IOException { FSDataInputStream fsDataInputStream = fs.open(logFile.getPath(), bufferSize); if (fsDataInputStream.getWrappedStream() instanceof FSInputStream) { this.inputStream = new FSDataInputStream( new BufferedFSInputStream((FSInputStream) fsDataInputStream.getWrappedStream(), bufferSize)); } else { // fsDataInputStream.getWrappedStream() maybe a BufferedFSInputStream // need to wrap in another BufferedFSInputStream the make bufferSize work? this.inputStream = fsDataInputStream; } this.logFile = logFile; this.readerSchema = readerSchema; this.readBlockLazily = readBlockLazily; this.reverseReader = reverseReader; if (this.reverseReader) { this.reverseLogFilePosition = this.lastReverseLogFilePosition = fs.getFileStatus(logFile.getPath()).getLen(); } addShutDownHook(); }
Example #2
Source File: TestCopy.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testInterruptedRename() throws Exception { FSDataOutputStream out = mock(FSDataOutputStream.class); whenFsCreate().thenReturn(out); when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat); when(mockFs.rename(eq(tmpPath), eq(path))).thenThrow( new InterruptedIOException()); FSInputStream in = mock(FSInputStream.class); when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1); tryCopyStream(in, false); verify(mockFs).delete(eq(tmpPath), anyBoolean()); verify(mockFs).rename(eq(tmpPath), eq(path)); verify(mockFs, never()).delete(eq(path), anyBoolean()); verify(mockFs, never()).close(); }
Example #3
Source File: TestCopy.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testCopyStreamTargetExists() throws Exception { FSDataOutputStream out = mock(FSDataOutputStream.class); whenFsCreate().thenReturn(out); when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat); target.refreshStatus(); // so it's updated as existing cmd.setOverwrite(true); when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat); when(mockFs.delete(eq(path), eq(false))).thenReturn(true); when(mockFs.rename(eq(tmpPath), eq(path))).thenReturn(true); FSInputStream in = mock(FSInputStream.class); when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1); tryCopyStream(in, true); verify(mockFs).delete(eq(path), anyBoolean()); verify(mockFs).rename(eq(tmpPath), eq(path)); verify(mockFs, never()).delete(eq(tmpPath), anyBoolean()); verify(mockFs, never()).close(); }
Example #4
Source File: TestCopy.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testInterruptedCopyBytes() throws Exception { FSDataOutputStream out = mock(FSDataOutputStream.class); whenFsCreate().thenReturn(out); when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat); FSInputStream in = mock(FSInputStream.class); // make IOUtils.copyBytes fail when(in.read(any(byte[].class), anyInt(), anyInt())).thenThrow( new InterruptedIOException()); tryCopyStream(in, false); verify(mockFs).delete(eq(tmpPath), anyBoolean()); verify(mockFs, never()).rename(any(Path.class), any(Path.class)); verify(mockFs, never()).delete(eq(path), anyBoolean()); verify(mockFs, never()).close(); }
Example #5
Source File: TestCopy.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testInterruptedRename() throws Exception { FSDataOutputStream out = mock(FSDataOutputStream.class); whenFsCreate().thenReturn(out); when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat); when(mockFs.rename(eq(tmpPath), eq(path))).thenThrow( new InterruptedIOException()); FSInputStream in = mock(FSInputStream.class); when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1); tryCopyStream(in, false); verify(mockFs).delete(eq(tmpPath), anyBoolean()); verify(mockFs).rename(eq(tmpPath), eq(path)); verify(mockFs, never()).delete(eq(path), anyBoolean()); verify(mockFs, never()).close(); }
Example #6
Source File: TestDFSUpgradeFromImage.java From big-c with Apache License 2.0 | 6 votes |
/** * Try to open a file for reading several times. * * If we fail because lease recovery hasn't completed, retry the open. */ private static FSInputStream dfsOpenFileWithRetries(DistributedFileSystem dfs, String pathName) throws IOException { IOException exc = null; for (int tries = 0; tries < 10; tries++) { try { return dfs.dfs.open(pathName); } catch (IOException e) { exc = e; } if (!exc.getMessage().contains("Cannot obtain " + "block length for LocatedBlock")) { throw exc; } try { Thread.sleep(1000); } catch (InterruptedException ignored) {} } throw exc; }
Example #7
Source File: TestDFSUpgradeFromImage.java From hadoop with Apache License 2.0 | 6 votes |
/** * Try to open a file for reading several times. * * If we fail because lease recovery hasn't completed, retry the open. */ private static FSInputStream dfsOpenFileWithRetries(DistributedFileSystem dfs, String pathName) throws IOException { IOException exc = null; for (int tries = 0; tries < 10; tries++) { try { return dfs.dfs.open(pathName); } catch (IOException e) { exc = e; } if (!exc.getMessage().contains("Cannot obtain " + "block length for LocatedBlock")) { throw exc; } try { Thread.sleep(1000); } catch (InterruptedException ignored) {} } throw exc; }
Example #8
Source File: TestCopy.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testInterruptedCopyBytes() throws Exception { FSDataOutputStream out = mock(FSDataOutputStream.class); whenFsCreate().thenReturn(out); when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat); FSInputStream in = mock(FSInputStream.class); // make IOUtils.copyBytes fail when(in.read(any(byte[].class), anyInt(), anyInt())).thenThrow( new InterruptedIOException()); tryCopyStream(in, false); verify(mockFs).delete(eq(tmpPath), anyBoolean()); verify(mockFs, never()).rename(any(Path.class), any(Path.class)); verify(mockFs, never()).delete(eq(path), anyBoolean()); verify(mockFs, never()).close(); }
Example #9
Source File: TestCopy.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testCopyStreamTargetExists() throws Exception { FSDataOutputStream out = mock(FSDataOutputStream.class); whenFsCreate().thenReturn(out); when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat); target.refreshStatus(); // so it's updated as existing cmd.setOverwrite(true); when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat); when(mockFs.delete(eq(path), eq(false))).thenReturn(true); when(mockFs.rename(eq(tmpPath), eq(path))).thenReturn(true); FSInputStream in = mock(FSInputStream.class); when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1); tryCopyStream(in, true); verify(mockFs).delete(eq(path), anyBoolean()); verify(mockFs).rename(eq(tmpPath), eq(path)); verify(mockFs, never()).delete(eq(tmpPath), anyBoolean()); verify(mockFs, never()).close(); }
Example #10
Source File: TestDFSUpgradeFromImage.java From big-c with Apache License 2.0 | 5 votes |
private void verifyDir(DistributedFileSystem dfs, Path dir, CRC32 overallChecksum) throws IOException { FileStatus[] fileArr = dfs.listStatus(dir); TreeMap<Path, Boolean> fileMap = new TreeMap<Path, Boolean>(); for(FileStatus file : fileArr) { fileMap.put(file.getPath(), Boolean.valueOf(file.isDirectory())); } for(Iterator<Path> it = fileMap.keySet().iterator(); it.hasNext();) { Path path = it.next(); boolean isDir = fileMap.get(path); String pathName = path.toUri().getPath(); overallChecksum.update(pathName.getBytes()); if ( isDir ) { verifyDir(dfs, path, overallChecksum); } else { // this is not a directory. Checksum the file data. CRC32 fileCRC = new CRC32(); FSInputStream in = dfsOpenFileWithRetries(dfs, pathName); byte[] buf = new byte[4096]; int nRead = 0; while ( (nRead = in.read(buf, 0, buf.length)) > 0 ) { fileCRC.update(buf, 0, nRead); } verifyChecksum(pathName, fileCRC.getValue()); } } }
Example #11
Source File: HftpFileSystem.java From hadoop-gpu with Apache License 2.0 | 5 votes |
@Override public FSDataInputStream open(Path f, int buffersize) throws IOException { HttpURLConnection connection = null; connection = openConnection("/data" + f.toUri().getPath(), "ugi=" + ugi); connection.setRequestMethod("GET"); connection.connect(); final InputStream in = connection.getInputStream(); return new FSDataInputStream(new FSInputStream() { public int read() throws IOException { return in.read(); } public int read(byte[] b, int off, int len) throws IOException { return in.read(b, off, len); } public void close() throws IOException { in.close(); } public void seek(long pos) throws IOException { throw new IOException("Can't seek!"); } public long getPos() throws IOException { throw new IOException("Position unknown!"); } public boolean seekToNewSource(long targetPos) throws IOException { return false; } }); }
Example #12
Source File: TestCopy.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testCopyStreamTarget() throws Exception { FSDataOutputStream out = mock(FSDataOutputStream.class); whenFsCreate().thenReturn(out); when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat); when(mockFs.rename(eq(tmpPath), eq(path))).thenReturn(true); FSInputStream in = mock(FSInputStream.class); when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1); tryCopyStream(in, true); verify(mockFs, never()).delete(eq(path), anyBoolean()); verify(mockFs).rename(eq(tmpPath), eq(path)); verify(mockFs, never()).delete(eq(tmpPath), anyBoolean()); verify(mockFs, never()).close(); }
Example #13
Source File: TestCopy.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testCopyStreamTarget() throws Exception { FSDataOutputStream out = mock(FSDataOutputStream.class); whenFsCreate().thenReturn(out); when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat); when(mockFs.rename(eq(tmpPath), eq(path))).thenReturn(true); FSInputStream in = mock(FSInputStream.class); when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1); tryCopyStream(in, true); verify(mockFs, never()).delete(eq(path), anyBoolean()); verify(mockFs).rename(eq(tmpPath), eq(path)); verify(mockFs, never()).delete(eq(tmpPath), anyBoolean()); verify(mockFs, never()).close(); }
Example #14
Source File: TestDFSUpgradeFromImage.java From hadoop with Apache License 2.0 | 5 votes |
private void verifyDir(DistributedFileSystem dfs, Path dir, CRC32 overallChecksum) throws IOException { FileStatus[] fileArr = dfs.listStatus(dir); TreeMap<Path, Boolean> fileMap = new TreeMap<Path, Boolean>(); for(FileStatus file : fileArr) { fileMap.put(file.getPath(), Boolean.valueOf(file.isDirectory())); } for(Iterator<Path> it = fileMap.keySet().iterator(); it.hasNext();) { Path path = it.next(); boolean isDir = fileMap.get(path); String pathName = path.toUri().getPath(); overallChecksum.update(pathName.getBytes()); if ( isDir ) { verifyDir(dfs, path, overallChecksum); } else { // this is not a directory. Checksum the file data. CRC32 fileCRC = new CRC32(); FSInputStream in = dfsOpenFileWithRetries(dfs, pathName); byte[] buf = new byte[4096]; int nRead = 0; while ( (nRead = in.read(buf, 0, buf.length)) > 0 ) { fileCRC.update(buf, 0, nRead); } verifyChecksum(pathName, fileCRC.getValue()); } } }
Example #15
Source File: StrictBufferedFSInputStream.java From big-c with Apache License 2.0 | 4 votes |
public StrictBufferedFSInputStream(FSInputStream in, int size) { super(in, size); }
Example #16
Source File: StrictBufferedFSInputStream.java From hadoop with Apache License 2.0 | 4 votes |
public StrictBufferedFSInputStream(FSInputStream in, int size) { super(in, size); }
Example #17
Source File: StrictBufferedFSInputStream.java From sahara-extra with Apache License 2.0 | 4 votes |
public StrictBufferedFSInputStream(FSInputStream in, int size) { super(in, size); }