Jave read input file

/**
 * 从输入流中读取数据，NIO方式
 * 
 * @param in
 * @return
 * @throws IOException
 */
public static byte[] readFromInputStream(InputStream in) throws IOException {
	if (in == null) {
		return null;
	}
	ReadableByteChannel rbc = Channels.newChannel(in);
	ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
	byte[] data = new byte[BUFFER_SIZE];
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	while (true) {
		int count = rbc.read(buffer);
		if (count == -1) {
			break;
		}
		if (count < BUFFER_SIZE) {
			data = new byte[count];
		}
		buffer.flip();
		buffer.get(data, 0, count);
		baos.write(data);
		buffer.clear();
	}
	rbc.close();
	return baos.toByteArray();
}
 

@Test
public void readFullAndPartialBlock() throws Exception {
    Random random = new Random();

    Path path = tmp.newFile().toPath();
    int blockSize = (int) OpenFileUtils.getBlockSize(path);

    int writeSize = blockSize + blockSize / 2;
    assertTrue(writeSize > blockSize);

    byte[] data = new byte[writeSize];
    random.nextBytes(data);
    Files.write(path, data);

    try (ODirectFileInputStream oo = new ODirectFileInputStream(path)) {
        byte[] fullread = new byte[blockSize * 2];
        int read = oo.read(fullread);

        assertEquals(writeSize, read);
        assertEquals(2, oo.getReadBlocks());
    }
}
 

private static LinkedHashMap<String, ProteinSequence> readInputFile(String inputLocation, AminoAcidCompositionTable aaTable) throws Exception{
	FileInputStream inStream = new FileInputStream(inputLocation);
	CompoundSet<AminoAcidCompound>	set;
	if(aaTable == null){
		set = CaseFreeAminoAcidCompoundSet.getAminoAcidCompoundSet();
	}else{
		set = aaTable.getAminoAcidCompoundSet();
	}
	LinkedHashMap<String, ProteinSequence> ret;
	if ( inputLocation.toLowerCase().contains(".gb")) {
		GenbankReader<ProteinSequence, AminoAcidCompound> genbankReader = new GenbankReader<ProteinSequence, AminoAcidCompound>(
				inStream, new GenericGenbankHeaderParser<ProteinSequence, AminoAcidCompound>(),
				new ProteinSequenceCreator(set));
		ret = genbankReader.process();


	} else {
		FastaReader<ProteinSequence, AminoAcidCompound> fastaReader = new FastaReader<ProteinSequence, AminoAcidCompound>(
				inStream, new GenericFastaHeaderParser<ProteinSequence, AminoAcidCompound>(),
				new ProteinSequenceCreator(set));
		ret = fastaReader.process();

	}
	return ret;
}
 

/**
 * Read the file into an {@code ObjectStream}.
 * 
 * @param infile
 *          the string pointing to the file
 * @return the object stream
 */
public static ObjectStream<String> readFileIntoMarkableStreamFactory(
    final String infile) {

  InputStreamFactory inputStreamFactory = null;
  try {
    inputStreamFactory = new MarkableFileInputStreamFactory(new File(infile));
  } catch (final FileNotFoundException e) {
    e.printStackTrace();
  }
  ObjectStream<String> lineStream = null;
  try {
    lineStream = new PlainTextByLineStream(inputStreamFactory, "UTF-8");
  } catch (final IOException e) {
    CmdLineUtil.handleCreateObjectStreamError(e);
  }
  return lineStream;
}
 

@Test
public void testReadFromInputFile() {
    val functionInputFileName = "config/functions-test.txt";
    List<MetricFunctionsSpec> metricFunctionsSpecList = MetricFunctionsReader
            .readFromInputFile(ClassLoader.getSystemResource(functionInputFileName).getPath());
    assertEquals(1, metricFunctionsSpecList.size());
    MetricFunctionsSpec metricFunctionsSpec = metricFunctionsSpecList.get(0);
    assertEquals("sumSeries(a.b.c)", metricFunctionsSpec.getFunction());
    assertEquals(60, metricFunctionsSpec.getIntervalInSecs());
    Map<String, String> tags = metricFunctionsSpec.getTags();
    assertEquals(3, tags.size());
    assertEquals("sample_app1", tags.get("app_name"));
    assertEquals("test", tags.get("env"));
    assertEquals("custom_tag_value", tags.get("custom_tag"));
    assertEquals(true, metricFunctionsSpec.getMergeTags());
}
 

@Test
public void testReadDecimalTypeFile() throws IOException {
	rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_DECIMAL), TEST_SCHEMA_DECIMAL, new Configuration());

	FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	rowOrcInputFormat.openInputFormat();
	rowOrcInputFormat.open(splits[0]);

	assertFalse(rowOrcInputFormat.reachedEnd());
	Row row = rowOrcInputFormat.nextRecord(null);

	// validate first row
	assertNotNull(row);
	assertEquals(1, row.getArity());
	assertEquals(BigDecimal.valueOf(-1000.5d), row.getField(0));

	// check correct number of rows
	long cnt = 1;
	while (!rowOrcInputFormat.reachedEnd()) {
		assertNotNull(rowOrcInputFormat.nextRecord(null));
		cnt++;
	}
	assertEquals(6000, cnt);
}
 

@Test
public void testReadTimeTypeFile() throws IOException {
	rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_TIMETYPES), TEST_SCHEMA_TIMETYPES, new Configuration());

	FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	rowOrcInputFormat.openInputFormat();
	rowOrcInputFormat.open(splits[0]);

	assertFalse(rowOrcInputFormat.reachedEnd());
	Row row = rowOrcInputFormat.nextRecord(null);

	// validate first row
	assertNotNull(row);
	assertEquals(2, row.getArity());
	assertEquals(Timestamp.valueOf("1900-05-05 12:34:56.1"), row.getField(0));
	assertEquals(Date.valueOf("1900-12-25"), row.getField(1));

	// check correct number of rows
	long cnt = 1;
	while (!rowOrcInputFormat.reachedEnd()) {
		assertNotNull(rowOrcInputFormat.nextRecord(null));
		cnt++;
	}
	assertEquals(70000, cnt);
}
 

@Test
public void testReadMultiplePatterns() throws Exception {
	final String contents = "CONTENTS";

	// create some accepted, some ignored files

	File child1 = temporaryFolder.newFile("dataFile1.txt");
	File child2 = temporaryFolder.newFile("another_file.bin");
	createTempFiles(contents.getBytes(ConfigConstants.DEFAULT_CHARSET), child1, child2);

	// test that only the valid files are accepted

	Configuration configuration = new Configuration();

	final DummyFileInputFormat format = new DummyFileInputFormat();
	format.setFilePath(temporaryFolder.getRoot().toURI().toString());
	format.configure(configuration);
	format.setFilesFilter(new GlobFilePathFilter(
		Collections.singletonList("**"),
		Arrays.asList("**/another_file.bin", "**/dataFile1.txt")
	));
	FileInputSplit[] splits = format.createInputSplits(1);

	Assert.assertEquals(0, splits.length);
}
 

@Test
@SneakyThrows
public void getFileInputStream() {
    TransferManager transferManager = mock(TransferManager.class);
    Download download = mock(Download.class);
    when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);

    ReadUtils readUtils = new ReadUtils(transferManager,System.getProperty("java.io.tmpdir"));

    InputStream fileInputStream = readUtils.getFileInputStream("bucket", "key");
    assertNotNull(fileInputStream);

    verify(transferManager, times(1)).download(anyString(), anyString(), any(File.class));
    verify(download, times(1)).waitForCompletion();

    fileInputStream.close();
}
 

private static DataSource read_directory ( String buffer ) {
    try {
        String[] s = buffer.split(DataSource.separator);
        int n = Integer.parseInt(s[1]);
        if (s[0].equals("Binary"))
            return new BinaryDataSource(s[2],Plan.conf);
        else if (s[0].equals("Generator"))
            return new GeneratorDataSource(s[2],Plan.conf);
        else if (s[0].equals("Text"))
            return new FlinkParsedDataSource(s[3],DataSource.parserDirectory.get(s[2]),
                                             ((Node)Tree.parse(s[4])).children());
        else throw new Error("Unrecognized data source: "+s[0]);
    } catch (Exception e) {
        throw new Error(e);
    }
}
 

private final DataValueDescriptor[] readDVDBuffer() throws StandardException {
  try {

    if (!diagnoseFileData) {
      return DataType.readDVDArray(this);
    }

    // we are in diagnostic mode ....
    final DataValueDescriptor[] dvd = DataType.readDVDArray(this);

    if (GemFireXDUtils.TraceTempFileIO) {
      final String srcDvd = rowObjectsWritten.get(rowObjectsindex++);
      String dvdS = toString(dvd, new StringBuilder()).toString();
      SanityManager.ASSERT(srcDvd.equals(dvdS), srcDvd + " != " + dvdS);
    }

    return dvd;

  } catch (final IOException ioe) {
    throw StandardException.newException(SQLState.LOG_CANNOT_FLUSH, ioe);
  } catch (final ClassNotFoundException cnf) {
    throw StandardException.newException(SQLState.LOG_CANNOT_FLUSH, cnf);
  }
}
 

private final void checkReadBounds(final int expected) throws IOException {
  if (rwBuffer.remaining() >= expected) {
    return;
  }

  if (GemFireXDUtils.TraceTempFileIO) {
    SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_TEMP_FILE_IO,
        "[" + this.file + "] moving onto next buffer as remaining "
            + rwBuffer.remaining() + " doesn't fits expected " + expected);
    // new Throwable());
  }

  getNextBuffer();

  assert rwBuffer.remaining() >= expected: "remaining="
      + rwBuffer.remaining() + " expected=" + expected;
}
 

@Override
public void readFully(final byte[] b, int off, int len) throws IOException {

  int remaining = rwBuffer.remaining();

  if (remaining >= len) {
    rwBuffer.get(b, off, len);
    return;
  }

  while (len > 0) {
    if (remaining >= len) {
      rwBuffer.get(b, off, len);
      return;
    }
    rwBuffer.get(b, off, remaining);
    off += remaining;
    len -= remaining;
    getNextBuffer();
    remaining = rwBuffer.capacity();
  }
}
 

@Test
public void testReadFileInSplits() throws IOException {

	rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_FLAT), TEST_SCHEMA_FLAT, new Configuration());
	rowOrcInputFormat.selectFields(0, 1);

	FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(4);
	assertEquals(4, splits.length);
	rowOrcInputFormat.openInputFormat();

	long cnt = 0;
	// read all splits
	for (FileInputSplit split : splits) {

		// open split
		rowOrcInputFormat.open(split);
		// read and count all rows
		while (!rowOrcInputFormat.reachedEnd()) {
			assertNotNull(rowOrcInputFormat.nextRecord(null));
			cnt++;
		}
	}
	// check that all rows have been read
	assertEquals(1920800, cnt);
}
 

public SkipBomAndDeleteFileAfterReadInputStream(InputStream inputStream,
		File file, boolean deleteAfterRead, IPipeLineSession session)
		throws FileNotFoundException {
	super(inputStream);
	this.file = file;
	this.deleteAfterRead = deleteAfterRead;
	this.session = session;
	if (deleteAfterRead) {
		if (file == null) {
			// This should not happen. A configuration warning for
			// read_delete in combination with classpath should have
			// occurred already.
			throw new FileNotFoundException("No file object provided");
		} else {
			file.deleteOnExit();
		}
	}
}
 

/**
 * Read the header of fsedit log
 * @param in fsedit stream
 * @return the edit log version number
 * @throws IOException if error occurs
 */
@VisibleForTesting
static int readLogVersion(DataInputStream in, boolean verifyLayoutVersion)
    throws IOException, LogHeaderCorruptException {
  int logVersion;
  try {
    logVersion = in.readInt();
  } catch (EOFException eofe) {
    throw new LogHeaderCorruptException(
        "Reached EOF when reading log header");
  }
  if (verifyLayoutVersion &&
      (logVersion < HdfsConstants.NAMENODE_LAYOUT_VERSION || // future version
       logVersion > Storage.LAST_UPGRADABLE_LAYOUT_VERSION)) { // unsupported
    throw new LogHeaderCorruptException(
        "Unexpected version of the file system log file: "
        + logVersion + ". Current version = "
        + HdfsConstants.NAMENODE_LAYOUT_VERSION + ".");
  }
  return logVersion;
}
 

/**
 * @throws Exception if the test fails
 */
@Test
@Alerts({"false", "true"})
public void readOnlyInputFile() throws Exception {
    final String html
        = HtmlPageTest.STANDARDS_MODE_PREFIX_
        + "<html>\n"
        + "<head><title>foo</title>\n"
        + "<script>\n"
        + "  function test() {\n"
        + "    var input = document.getElementById('myInput');\n"
        + "    alert(input.readOnly);\n"
        + "    input = document.getElementById('myReadonlyInput');\n"
        + "    alert(input.readOnly);\n"
        + "  }\n"
        + "  </script>\n"
        + "</head>\n"
        + "<body onload='test()'>\n"
        + "  <input id='myInput' type='file' value='some test'>\n"
        + "  <input id='myReadonlyInput' type='file' value='some test' readonly='false'>\n"
        + "</body></html>";

    loadPageWithAlerts2(html);
}
 

/***
 * Read truststore for establishing certificate chain for signature validation
 * 
 * @param conf
 * @throws IOException
 * @throws FormatNotUnderstoodException
 */
private void readTrustStore() throws IOException, FormatNotUnderstoodException {
	if (((this.hocr.getSigTruststoreFile() != null) && (!"".equals(this.hocr.getSigTruststoreFile())))) {
		LOG.info("Reading truststore to validate certificate chain for signatures");
		FlinkKeyStoreManager fksm = new FlinkKeyStoreManager();
		try {
			fksm.openKeyStore(new Path(this.hocr.getSigTruststoreFile()), this.hocr.getSigTruststoreType(),
					this.hocr.getSigTruststorePassword());
			this.hocr.setX509CertificateChain(fksm.getAllX509Certificates());
		} catch (NoSuchAlgorithmException | CertificateException | KeyStoreException | IllegalArgumentException e) {
			LOG.error("Exception: ",e);
			throw new FormatNotUnderstoodException(
					"Cannot read truststore to establish certificate chain for signature validation " + e);
		}

	}
}
 

public static String readFileFromInputStream(@NonNull InputStream inputStream) {

        InputStreamReader inputStreamReader = new InputStreamReader(inputStream);
        BufferedReader bufferedReader = new BufferedReader(inputStreamReader);

        StringBuilder result = new StringBuilder();
        String line;
        try {
            while ((line = bufferedReader.readLine()) != null) {
                result.append(line);
            }
            return result.toString();
        } catch (IOException exception) {
        } finally {
            try {
                bufferedReader.close();
            } catch (IOException e2) {
            }
            try {
                inputStreamReader.close();
            } catch (IOException e2) {
            }
        }
        return null;
    }
 

private byte[] readInputStream(InputStream in) throws IOException {
	ByteArrayOutputStream opt = new ByteArrayOutputStream();
	byte[] buf = new byte[1024];
	int len = 0;
	while ((len = in.read(buf)) != -1) {
		opt.write(buf, 0, len);
	}
	in.close();
	byte[] ret;
	try {
		ret = opt.toByteArray();
	} catch (OutOfMemoryError e) {
		ret = null;
	}
	opt.close();
	return ret;
}
 

@Test
public void testReadBulk() throws IOException {
    final String content = "Hello World";

    Path file = addFile("/foo");
    setContents(file, content);

    byte[] b = new byte[20];
    try (InputStream input = fileSystem.newInputStream(createPath("/foo"))) {
        assertEquals(0, input.read(b, 0, 0));
        assertEquals(5, input.read(b, 1, 5));
        assertArrayEquals(content.substring(0, 5).getBytes(), Arrays.copyOfRange(b, 1, 6));
        assertEquals(content.length() - 5, input.read(b));
        assertArrayEquals(content.substring(5).getBytes(), Arrays.copyOfRange(b, 0, content.length() - 5));
        assertEquals(-1, input.read(b));
    }
}
 

@Test
public void testReadSingle() throws IOException {
    final String content = "Hello World";

    Path file = addFile("/foo");
    setContents(file, content);

    try (InputStream input = fileSystem.newInputStream(createPath("/foo"))) {
        assertEquals('H', input.read());
        assertEquals('e', input.read());
        assertEquals('l', input.read());
        assertEquals('l', input.read());
        assertEquals('o', input.read());
        assertEquals(' ', input.read());
        assertEquals('W', input.read());
        assertEquals('o', input.read());
        assertEquals('r', input.read());
        assertEquals('l', input.read());
        assertEquals('d', input.read());
        assertEquals(-1, input.read());
    }
}
 

public static String readInputStreamAsString(InputStream is) throws FileNotFoundException, IOException {
	BufferedReader br = null;
	StringBuffer result = new StringBuffer();
	try {		
		InputStream in = new DataInputStream(is);
		br = new BufferedReader(new InputStreamReader(in));
		String strLine;
		// Read File Line By Line
		while ((strLine = br.readLine()) != null) {
			result.append(strLine + "\n");
		}
	} finally {
		br.close();
	}

	return result.toString();
}
 

@Test(expectedExceptions = CommandLineException.class)
public void testEmptyInputFileListProvided() throws Exception {
    @SuppressWarnings("serial")
    final List<Target> phonyTargets = SimulatedTargets.phonyTargets(3);
    final List<File> inputFiles = Collections.emptyList();
    final File targetFile = createTargetFile(phonyTargets);
    final File inputListFile = createInputListFile(inputFiles);
    try {
        runTool(targetFile, inputFiles, inputListFile);
    } catch (final Exception ex) {
        throw ex;
    } finally {
        targetFile.delete();
        inputListFile.delete();
    }
}
 

@Test
public void testReadSmallFileFromUrl() throws IOException, InterruptedException {
    latch = new CountDownLatch(1);
    final ByteArrayOutputStream bos = new ByteArrayOutputStream();
    File file = new File("src/test/resources/a_file.txt");
    URL url = file.toURI().toURL();
    Context context = vertx.getOrCreateContext();
    final AsyncInputStream async = new AsyncInputStream(vertx, executor,
            url.openStream())
            .endHandler(event -> {
                assertThat(bos.toString()).startsWith("This is a file.");
                latch.countDown();
            }).setContext(context);
    context.runOnContext(event -> async.handler(buffer -> {
        try {
            bos.write(buffer.getBytes());
        } catch (IOException e) {
            e.printStackTrace();
        }
    }));
    latch.await(30, TimeUnit.SECONDS);
}
 

/**
 * Read from an inoutstream and convert the readed stuff to a String. Usefull for text files that are available as
 * streams.
 *
 * @param inputStream
 *
 * @return the read string
 *
 * @throws IOException
 */
public static String readInputStreamToString( InputStream inputStream ) throws IOException {
    // Create the byte list to hold the data
    List<Byte> bytesList = new ArrayList<Byte>();

    byte b = 0;
    while( (b = (byte) inputStream.read()) != -1 ) {
        bytesList.add(b);
    }
    // Close the input stream and return bytes
    inputStream.close();

    byte[] bArray = new byte[bytesList.size()];
    for( int i = 0; i < bArray.length; i++ ) {
        bArray[i] = bytesList.get(i);
    }

    String file = new String(bArray);
    return file;
}
 

@Override
public Promise<FilePart> read(int fileOffset, int len) {
    return new Promise<>(resolver -> {
        executor.execute(() -> {
            try {
                byte[] data = new byte[len];
                randomAccessFile.seek(fileOffset);
                // TODO: Better reading. For big len result can be truncated
                randomAccessFile.read(data, 0, len);
                resolver.result(new FilePart(fileOffset, len, data));
            } catch (Exception e) {
                e.printStackTrace();
                resolver.error(e);
            }
        });
    });
}
 

public static InputStream readFileInputStream(String path) {
    
    if(ValidatorUtils.isEmpty(path)) {
        return null;
    }

    try {
        ClassLoader classLoader = BytesUtils.class.getClassLoader();
        File file = new File(Objects.requireNonNull(classLoader.getResource(path)).getFile());

        return new FileInputStream(file);

    } catch (FileNotFoundException e) {
        throw new TechnicalException("File cannot be found...", e);
    }
}
 

@Override
public int read(byte[] b, int off, int len) throws IOException {
    if (currentMode == MODE_CBC && fileOffset == 0) {
        byte[] temp = new byte[32];
        super.read(temp, 0, 32);
        Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0);
        fileOffset += 32;
        skip((temp[0] & 0xff) - 32);
    }
    int result = super.read(b, off, len);
    if (currentMode == MODE_CBC) {
        Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0);
    } else if (currentMode == MODE_CTR) {
        Utilities.aesCtrDecryptionByteArray(b, key, iv, off, len, fileOffset);
    }
    fileOffset += len;
    return result;
}
 

/**
 * reading file and converting InputStream to String using BufferedReader
 * @param fileName
 * @return
 * @throws IOException
 */
public static String readFileToStringUsingBufferedReader(String fileName) throws IOException{
	FileInputStream fileInputStream = null;
	BufferedReader bufferedReader = null;
	StringBuilder inputSB = null;
	try{
	fileInputStream = new FileInputStream(fileName);
	bufferedReader = new BufferedReader(new InputStreamReader(fileInputStream, "UTF-8"));
	inputSB = new StringBuilder();
	String line = bufferedReader.readLine();
       while(line != null){
       	inputSB.append(line);
           line = bufferedReader.readLine();
           if(line != null){
           	//add new line character
           	inputSB.append("\n");
           }
       }
	}finally{
		bufferedReader.close();
		fileInputStream.close();
	}
	return inputSB.toString();
}
 

@Test
public void testReadTimeTypeFile() throws IOException {
	rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_TIMETYPES), TEST_SCHEMA_TIMETYPES, new Configuration());

	FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	rowOrcInputFormat.openInputFormat();
	rowOrcInputFormat.open(splits[0]);

	assertFalse(rowOrcInputFormat.reachedEnd());
	Row row = rowOrcInputFormat.nextRecord(null);

	// validate first row
	assertNotNull(row);
	assertEquals(2, row.getArity());
	assertEquals(Timestamp.valueOf("1900-05-05 12:34:56.1"), row.getField(0));
	assertEquals(Date.valueOf("1900-12-25"), row.getField(1));

	// check correct number of rows
	long cnt = 1;
	while (!rowOrcInputFormat.reachedEnd()) {
		assertNotNull(rowOrcInputFormat.nextRecord(null));
		cnt++;
	}
	assertEquals(70000, cnt);
}
 

@Test
public void testReadMultiplePatterns() throws Exception {
	final String contents = "CONTENTS";

	// create some accepted, some ignored files

	File child1 = temporaryFolder.newFile("dataFile1.txt");
	File child2 = temporaryFolder.newFile("another_file.bin");
	createTempFiles(contents.getBytes(ConfigConstants.DEFAULT_CHARSET), child1, child2);

	// test that only the valid files are accepted

	Configuration configuration = new Configuration();

	final DummyFileInputFormat format = new DummyFileInputFormat();
	format.setFilePath(temporaryFolder.getRoot().toURI().toString());
	format.configure(configuration);
	format.setFilesFilter(new GlobFilePathFilter(
		Collections.singletonList("**"),
		Arrays.asList("**/another_file.bin", "**/dataFile1.txt")
	));
	FileInputSplit[] splits = format.createInputSplits(1);

	Assert.assertEquals(0, splits.length);
}
 

public BytesInput readAsBytesInput(int size) throws IOException {
  int available = stream.available();
  if (size > available) {
    // this is to workaround a bug where the compressedLength
    // of the chunk is missing the size of the header of the dictionary
    // to allow reading older files (using dictionary) we need this.
    // usually 13 to 19 bytes are missing
    int missingBytes = size - available;
    LOG.info("completed the column chunk with {} bytes", missingBytes);

    List<ByteBuffer> buffers = new ArrayList<>();
    buffers.addAll(stream.sliceBuffers(available));

    ByteBuffer lastBuffer = ByteBuffer.allocate(missingBytes);
    f.readFully(lastBuffer);
    buffers.add(lastBuffer);

    return BytesInput.from(buffers);
  }

  return super.readAsBytesInput(size);
}
 

/** {@inheritDoc} */
@Override public void ensure(int requested) throws IOException {
    int available = buffer().remaining();

    if (available >= requested)
        return;

    boolean readArchive = segmentAware.checkCanReadArchiveOrReserveWorkSegment(segmentId);
    try {
        if (readArchive && !isLastReadFromArchive) {
            isLastReadFromArchive = true;

            refreshIO();
        }

        super.ensure(requested);
    }
    finally {
        if (!readArchive)
            segmentAware.releaseWorkSegment(segmentId);
    }
}
 

/**
 * Reads up to <code>len</code> bytes of data from the input stream into
 * an array of bytes.  An attempt is made to read as many as
 * <code>len</code> bytes, but a smaller number may be read.
 * The number of bytes actually read is returned as an integer.
 *
 * @param b     the buffer into which the data is read.
 * @param off   the start offset in array <code>b</code> at which the data is written.
 * @param len   the maximum number of bytes to read.
 * @return     the total number of bytes read into the buffer, or
 *             <code>-1</code> if the end of the stream has been reached.
 * @throws IOException
 */
@Override
public int read(byte[] b, int off, int len) throws IOException {
  checkIfClosed();
  if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
    throw new IndexOutOfBoundsException();
  } else if (len == 0) {
    return 0;
  }

  int n = 0;
  while (true) {
    int nread = readRedisIfNeed(b, off + n, len - n);
    if (nread <= 0) {
      return (n == 0) ? nread : n;
    }
    n += nread;
    if (n >= len) {
      return n;
    }
    if (available() <= 0) {
      return n;
    }
  }
}
 

/**
 * Work-around for the problem that compressed InputReaders don't fill
 * the read buffer before returning.
 *
 * Has visibility 'protected' so that subclasses may override with
 * different algorithms, or use different algorithms for different
 * compression stream.
 */
protected void readCompressedBlocks(int blocks) throws IOException {

    int bytesSoFar    = 0;
    int requiredBytes = 512 * blocks;

    // This method works with individual bytes!
    int i;

    while (bytesSoFar < requiredBytes) {
        i = readStream.read(readBuffer, bytesSoFar,
                            requiredBytes - bytesSoFar);

        if (i < 0) {
            throw new EOFException(
                RB.decompression_ranout.getString(
                    bytesSoFar, requiredBytes));
        }

        bytesRead  += i;
        bytesSoFar += i;
    }
}
 

@Override
public int read(byte[] b, int off, int len) throws IOException {
    if (currentMode == MODE_CBC && fileOffset == 0) {
        byte[] temp = new byte[32];
        super.read(temp, 0, 32);
        Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0);
        fileOffset += 32;
        skip((temp[0] & 0xff) - 32);
    }
    int result = super.read(b, off, len);
    if (currentMode == MODE_CBC) {
        Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0);
    } else if (currentMode == MODE_CTR) {
        Utilities.aesCtrDecryptionByteArray(b, key, iv, off, len, fileOffset);
    }
    fileOffset += len;
    return result;
}
 

@Test
public void readInputWithDefaultValues() throws Exception {
  final String virtualFile = createVirtualFile( "pdi-14832.txt", "1,\n" );

  BaseFileField col2 = field( "col2" );
  col2.setIfNullValue( "DEFAULT" );

  TextFileInputMeta meta = createMetaObject( field( "col1" ), col2 );
  TextFileInputData data = createDataObject( virtualFile, ",", "col1", "col2" );

  TextFileInput input = TransformMockUtil.getTransform( TextFileInput.class, TextFileInputMeta.class, "test" );
  List<Object[]> output = PipelineTestingUtil.execute( input, meta, data, 1, false );
  PipelineTestingUtil.assertResult( new Object[] { "1", "DEFAULT" }, output.get( 0 ) );

  deleteVfsFile( virtualFile );
}
 

@Test
public void readInputWithNonEmptyNullif() throws Exception {
  final String virtualFile = createVirtualFile( "pdi-14358.txt", "-,-\n" );

  BaseFileField col2 = field( "col2" );
  col2.setNullString( "-" );

  TextFileInputMeta meta = createMetaObject( field( "col1" ), col2 );
  TextFileInputData data = createDataObject( virtualFile, ",", "col1", "col2" );

  TextFileInput input = TransformMockUtil.getTransform( TextFileInput.class, TextFileInputMeta.class, "test" );
  List<Object[]> output = PipelineTestingUtil.execute( input, meta, data, 1, false );
  PipelineTestingUtil.assertResult( new Object[] { "-" }, output.get( 0 ) );

  deleteVfsFile( virtualFile );
}
 

private static int readDirectBuffer(ByteBuffer byteBufr, byte[] tmpBuf, ByteBufReader rdr) throws IOException {
  // copy all the bytes that return immediately, stopping at the first
  // read that doesn't return a full buffer.
  int nextReadLength = Math.min(byteBufr.remaining(), tmpBuf.length);
  int totalBytesRead = 0;
  int bytesRead;

  while ((bytesRead = rdr.read(tmpBuf, 0, nextReadLength)) == tmpBuf.length) {
    byteBufr.put(tmpBuf);
    totalBytesRead += bytesRead;
    nextReadLength = Math.min(byteBufr.remaining(), tmpBuf.length);
  }

  if (bytesRead < 0) {
    // return -1 if nothing was read
    return totalBytesRead == 0 ? -1 : totalBytesRead;
  } else {
    // copy the last partial buffer
    byteBufr.put(tmpBuf, 0, bytesRead);
    totalBytesRead += bytesRead;
    return totalBytesRead;
  }
}
 

/**
 *  读取字符流，指定编码
 * @param input
 * @param enCoding
 * @return String
 */
public static String readInputStreamToString(InputStream input,String enCoding) {
	BufferedReader reader ;
	StringBuilder sb = new StringBuilder();
	String line = null;
	try {
		reader = new BufferedReader(new InputStreamReader(input,enCoding));
		while ((line = reader.readLine()) != null) {
			sb.append(line + "\r\n");
		}
	} catch (IOException e) {
		log.error(e.getMessage());
	} finally {
		try {
			input.close();
		} catch (IOException e) {
			log.error(e.getMessage());
		}
	}
	return sb.toString();

}
 

/**
 * Read the header of fsedit log
 * @param in fsedit stream
 * @return the edit log version number
 * @throws IOException if error occurs
 */
@VisibleForTesting
static int readLogVersion(DataInputStream in, boolean verifyLayoutVersion)
    throws IOException, LogHeaderCorruptException {
  int logVersion;
  try {
    logVersion = in.readInt();
  } catch (EOFException eofe) {
    throw new LogHeaderCorruptException(
        "Reached EOF when reading log header");
  }
  if (verifyLayoutVersion &&
      (logVersion < HdfsConstants.NAMENODE_LAYOUT_VERSION || // future version
       logVersion > Storage.LAST_UPGRADABLE_LAYOUT_VERSION)) { // unsupported
    throw new LogHeaderCorruptException(
        "Unexpected version of the file system log file: "
        + logVersion + ". Current version = "
        + HdfsConstants.NAMENODE_LAYOUT_VERSION + ".");
  }
  return logVersion;
}
 

public void readAndCheckBlock(ClosedFileInputStream cfi, byte[] src, int position) throws IOException {
  final int toread = 100;
  final byte[] expected = new byte[toread];
  System.arraycopy(src, position, expected, 0, toread);

  final byte[] readdata = new byte[toread];
  cfi.seek(position);

  int read = 0;
  int batchread;
  while (read < toread && (batchread = cfi.read(readdata, read, toread - read)) != -1) {
    read += batchread;
  }

  assertTrue(Arrays.equals(expected, readdata));
}
 

@Test
public void readJsonNoOptions() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "json");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(JSON_DATA).getPath());
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(4, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("dog", first.getString(3));
  assertEquals("field1", first.schema().fields()[0].name());
  assertEquals(DataTypes.LongType, first.schema().fields()[0].dataType());
}
 

@Test
public void readCsvNoOptions() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath());
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(4, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("Four", first.getString(3));
}
 

@Test
public void readCsvWithFieldsSchema() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath());
  paramMap.put(FileSystemInput.CSV_HEADER_CONFIG, "true");
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, "flat");
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + FlatSchema.FIELD_NAMES_CONFIG,
               Lists.newArrayList("A Long", "An Int", "A String", "Another String"));
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + FlatSchema.FIELD_TYPES_CONFIG,
               Lists.newArrayList("long", "int", "string", "string"));
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(3, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("four", first.getString(3));
  assertEquals("Another String", first.schema().fields()[3].name());
  assertEquals(1L, first.get(0));
  assertEquals(DataTypes.LongType, first.schema().fields()[0].dataType());
}
 

@Test
public void readCsvWithAvroSchema() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath());
  paramMap.put(FileSystemInput.CSV_HEADER_CONFIG, "true");
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, "avro");
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + AvroSchema.AVRO_FILE_CONFIG,
               FileSystemInput.class.getResource(TestAvroSchema.AVRO_SCHEMA_DATA).getPath());
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(3, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("four", first.getString(3));
  assertEquals("Another_String", first.schema().fields()[3].name());
  assertEquals(1L, first.get(0));
  assertEquals(DataTypes.LongType, first.schema().fields()[0].dataType());
}
 

private Dataset<Row> readText(String path) {
  Dataset<Row> lines = Contexts.getSparkSession().read().text(path);

  if (hasTranslator) {
    TranslateFunction translateFunction = getTranslateFunction(translatorConfig);
    TranslationResults results = new TranslationResults(
        lines.javaRDD().flatMap(translateFunction),
        translateFunction.getProvidingSchema(),
        getProvidingSchema());
    errors = results.getErrored();

    return results.getTranslated();
  }
  else {
    return lines;
  }
}
 

@Test
public void readCsvWithOptions() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath());
  paramMap.put(FileSystemInput.CSV_HEADER_CONFIG, "true");
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(3, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("four", first.getString(3));
}
 

@Test
public void readTextWithoutTranslator() throws Exception {
  Map<String, Object> configMap = Maps.newHashMap();
  configMap.put(FileSystemInput.FORMAT_CONFIG, FileSystemInput.TEXT_FORMAT);
  configMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(TEXT_DATA).getPath());
  config = ConfigFactory.parseMap(configMap);
  
  FileSystemInput formatInput = new FileSystemInput();
  assertNoValidationFailures(formatInput, config);
  formatInput.configure(config);
  
  List<Row> results = formatInput.read().collectAsList();
  
  assertEquals(2, results.size());
  assertTrue(results.contains(RowFactory.create("a=1,b=hello,c=true")));
  assertTrue(results.contains(RowFactory.create("a=2,b=world,c=false")));
}
 

private Dataset<Row> readInputFormat(String path) throws Exception {
  LOG.debug("Reading InputFormat[{}]: {}", inputType, path);

  Class<? extends InputFormat> inputFormatClass = Class.forName(inputType).asSubclass(InputFormat.class);

  TranslateFunction translateFunction = getTranslateFunction(translatorConfig);

  Dataset<Row> encoded = getEncodedRowsFromInputFormat(path, inputFormatClass);

  TranslationResults results = new TranslationResults(
      encoded.javaRDD().flatMap(translateFunction),
      translateFunction.getProvidingSchema(),
      getProvidingSchema());
  errors = results.getErrored();

  return results.getTranslated();
}
 

private void readInputFile(BufferedReader reader) throws IOException {
    StringBuilder sb = new StringBuilder();
    int ch;
    char prevChar = 0;
    while ((ch = reader.read()) >= 0) {
        char c = (char) ch;
        if (prevChar == '\n') {
            if (c == '\n') {
                parseEntry(sb.toString());
                sb = new StringBuilder();
                prevChar = 0;
                continue;
            } else {
                sb.append(prevChar);
            }
        }
        if (c != '\n') {
            sb.append(c);
        }
        prevChar = c;
    }
    parseEntry(sb.toString());
}
 

public static String readInputStreamAsString(InputStream is, int size_limit,
		int timeoutMillis, String charSet)
		throws IOException {
	StringBuilder result = new StringBuilder(1024);
	long maxTimeMillis = System.currentTimeMillis() + timeoutMillis;
	byte[] buffer = new byte[1024];

	while (System.currentTimeMillis() < maxTimeMillis) {
		int readLength = Math.min(is.available(), buffer.length);

		int len = is.read(buffer, 0, readLength);
		if (len == -1)
			break;

		result.append(new String(buffer, 0, len, charSet));

		if (size_limit >= 0 && result.length() > size_limit) {

			result.setLength(size_limit);

			break;
		}
	}

	return (result.toString());
}
 

/**
 * Ensure that we can simply read bytes from a file.
 * @throws IOException
 */
@Test
public void testBasicRead() throws IOException {
  String output = singleLineFileInit(file, Charsets.UTF_8);

  PositionTracker tracker = new DurablePositionTracker(meta, file.getPath());
  ResettableInputStream in = new ResettableFileInputStream(file,  tracker);

  String result = readLine(in, output.length());
  assertEquals(output, result);

  String afterEOF = readLine(in, output.length());
  assertNull(afterEOF);

  in.close();
}
 

private static Collection<CoordinatorLogEntry> readFromInputStream(
        InputStream in) {
    Map<String, CoordinatorLogEntry> coordinatorLogEntries = new HashMap<>();
    BufferedReader br = null;
    try {
        InputStreamReader isr = new InputStreamReader(in);
        br = new BufferedReader(isr);
        coordinatorLogEntries = readContent(br);
    } catch (Exception e) {
        LOGGER.warn("Error in recover", e);
        AlertUtil.alertSelf(AlarmCode.XA_READ_IO_FAIL, Alert.AlertLevel.WARN, "Error in recover:" + e.getMessage(), null);

    } finally {
        closeSilently(br);
    }
    return coordinatorLogEntries.values();
}
 

/**
 * Read the header of fsedit log
 * @param in fsedit stream
 * @return the edit log version number
 * @throws IOException if error occurs
 */
static int readLogVersion(DataInputStream in) throws IOException,
    LogHeaderCorruptException {
  int logVersion = 0;
  // Read log file version. Could be missing.
  in.mark(4);
  // If edits log is greater than 2G, available method will return negative
  // numbers, so we avoid having to call available
  boolean available = true;
  try {
    logVersion = in.readByte();
  } catch (EOFException e) {
    available = false;
  }

  if (available) {
    in.reset();
    logVersion = in.readInt();
    if (logVersion < FSConstants.LAYOUT_VERSION) { // future version
      throw new LogHeaderCorruptException(
          "Unexpected version of the file system log file: " + logVersion
              + ". Current version = " + FSConstants.LAYOUT_VERSION + ".");
    }
  }
  return logVersion;
}
 

/**
 * Ensures that at least <code>pos</code> bytes are cached,
 * or the end of the source is reached.  The return value
 * is equal to the smaller of <code>pos</code> and the
 * length of the source file.
 */
private long readUntil(long pos) throws IOException {
    // We've already got enough data cached
    if (pos < length) {
        return pos;
    }
    // pos >= length but length isn't getting any bigger, so return it
    if (foundEOF) {
        return length;
    }

    long len = pos - length;
    cache.seek(length);
    while (len > 0) {
        // Copy a buffer's worth of data from the source to the cache
        // BUFFER_LENGTH will always fit into an int so this is safe
        int nbytes =
            stream.read(buf, 0, (int)Math.min(len, (long)BUFFER_LENGTH));
        if (nbytes == -1) {
            foundEOF = true;
            return length;
        }

        cache.write(buf, 0, nbytes);
        len -= nbytes;
        length += nbytes;
    }

    return pos;
}
 

/**
    Converts a class file to a <tt>ClassFile</tt> structure.
    @param is the input stream from which to read the
              <tt>ClassFile</tt> structure
    @return the new <tt>ClassFile</tt> structure
    @throws InvalidByteCodeException if the code is invalid
    @throws IOException if an exception occurs while reading from
                        the input stream
 */
public static ClassFile readFromInputStream(InputStream is)
    throws InvalidByteCodeException, IOException
{

    DataInputStream in = new DataInputStream(
                            new BufferedInputStream(is));

    ClassFile classFile = new ClassFile();
    classFile.read(in);
    in.close();
    return classFile;
}
 

@Test
public void testReadData() throws IOException {
	TsFileInputFormat<Row> inputFormat = prepareInputFormat(sourceTsFilePath1);

	List<String> actual = new ArrayList<>();

	try {
		inputFormat.configure(new Configuration());
		inputFormat.openInputFormat();
		FileInputSplit[] inputSplits = inputFormat.createInputSplits(2);
		Row reuse = rowTypeInfo.createSerializer(new ExecutionConfig()).createInstance();
		for (FileInputSplit inputSplit : inputSplits) {
			try {
				inputFormat.open(inputSplit);
				assertEquals(config.getBatchSize(), TSFileDescriptor.getInstance().getConfig().getBatchSize());
				while (!inputFormat.reachedEnd()) {
					Row row = inputFormat.nextRecord(reuse);
					actual.add(row.toString());
				}
			} finally {
				inputFormat.close();
			}
		}
	} finally {
		inputFormat.closeInputFormat();
	}

	String[] expected = {
		"1,1.2,20,null,2.3,11,19",
		"2,null,20,50,25.4,10,21",
		"3,1.4,21,null,null,null,null",
		"4,1.2,20,51,null,null,null",
		"6,7.2,10,11,null,null,null",
		"7,6.2,20,21,null,null,null",
		"8,9.2,30,31,null,null,null"
	};
	assertArrayEquals(actual.toArray(), expected);
}
 

private int readFromTail() {
    if (pos < tail.length()) {
        return tail.charAt(pos++);
    }
    pos = 0;
    if (readingHeader) {
        readingHeader = false;
        tail = dataTail;
    }
    return -1;
}