Jave read input file

60 Jave code examples are found related to " read input file". You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.
Example 1
Source File: FileUtils.java    From MyTv with Apache License 2.0 7 votes vote down vote up
/**
 * 从输入流中读取数据,NIO方式
 * 
 * @param in
 * @return
 * @throws IOException
 */
public static byte[] readFromInputStream(InputStream in) throws IOException {
	if (in == null) {
		return null;
	}
	ReadableByteChannel rbc = Channels.newChannel(in);
	ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
	byte[] data = new byte[BUFFER_SIZE];
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	while (true) {
		int count = rbc.read(buffer);
		if (count == -1) {
			break;
		}
		if (count < BUFFER_SIZE) {
			data = new byte[count];
		}
		buffer.flip();
		buffer.get(data, 0, count);
		baos.write(data);
		buffer.clear();
	}
	rbc.close();
	return baos.toByteArray();
}
 
Example 2
Source File: ODirectFileInputStreamTest.java    From herddb with Apache License 2.0 6 votes vote down vote up
@Test
public void readFullAndPartialBlock() throws Exception {
    Random random = new Random();

    Path path = tmp.newFile().toPath();
    int blockSize = (int) OpenFileUtils.getBlockSize(path);

    int writeSize = blockSize + blockSize / 2;
    assertTrue(writeSize > blockSize);

    byte[] data = new byte[writeSize];
    random.nextBytes(data);
    Files.write(path, data);

    try (ODirectFileInputStream oo = new ODirectFileInputStream(path)) {
        byte[] fullread = new byte[blockSize * 2];
        int read = oo.read(fullread);

        assertEquals(writeSize, read);
        assertEquals(2, oo.getReadBlocks());
    }
}
 
Example 3
Source File: CommandPrompt.java    From biojava with GNU Lesser General Public License v2.1 6 votes vote down vote up
private static LinkedHashMap<String, ProteinSequence> readInputFile(String inputLocation, AminoAcidCompositionTable aaTable) throws Exception{
	FileInputStream inStream = new FileInputStream(inputLocation);
	CompoundSet<AminoAcidCompound>	set;
	if(aaTable == null){
		set = CaseFreeAminoAcidCompoundSet.getAminoAcidCompoundSet();
	}else{
		set = aaTable.getAminoAcidCompoundSet();
	}
	LinkedHashMap<String, ProteinSequence> ret;
	if ( inputLocation.toLowerCase().contains(".gb")) {
		GenbankReader<ProteinSequence, AminoAcidCompound> genbankReader = new GenbankReader<ProteinSequence, AminoAcidCompound>(
				inStream, new GenericGenbankHeaderParser<ProteinSequence, AminoAcidCompound>(),
				new ProteinSequenceCreator(set));
		ret = genbankReader.process();


	} else {
		FastaReader<ProteinSequence, AminoAcidCompound> fastaReader = new FastaReader<ProteinSequence, AminoAcidCompound>(
				inStream, new GenericFastaHeaderParser<ProteinSequence, AminoAcidCompound>(),
				new ProteinSequenceCreator(set));
		ret = fastaReader.process();

	}
	return ret;
}
 
Example 4
Source File: InputOutputUtils.java    From ixa-pipe-pos with Apache License 2.0 6 votes vote down vote up
/**
 * Read the file into an {@code ObjectStream}.
 * 
 * @param infile
 *          the string pointing to the file
 * @return the object stream
 */
public static ObjectStream<String> readFileIntoMarkableStreamFactory(
    final String infile) {

  InputStreamFactory inputStreamFactory = null;
  try {
    inputStreamFactory = new MarkableFileInputStreamFactory(new File(infile));
  } catch (final FileNotFoundException e) {
    e.printStackTrace();
  }
  ObjectStream<String> lineStream = null;
  try {
    lineStream = new PlainTextByLineStream(inputStreamFactory, "UTF-8");
  } catch (final IOException e) {
    CmdLineUtil.handleCreateObjectStreamError(e);
  }
  return lineStream;
}
 
Example 5
Source File: MetricFunctionsReaderTest.java    From adaptive-alerting with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadFromInputFile() {
    val functionInputFileName = "config/functions-test.txt";
    List<MetricFunctionsSpec> metricFunctionsSpecList = MetricFunctionsReader
            .readFromInputFile(ClassLoader.getSystemResource(functionInputFileName).getPath());
    assertEquals(1, metricFunctionsSpecList.size());
    MetricFunctionsSpec metricFunctionsSpec = metricFunctionsSpecList.get(0);
    assertEquals("sumSeries(a.b.c)", metricFunctionsSpec.getFunction());
    assertEquals(60, metricFunctionsSpec.getIntervalInSecs());
    Map<String, String> tags = metricFunctionsSpec.getTags();
    assertEquals(3, tags.size());
    assertEquals("sample_app1", tags.get("app_name"));
    assertEquals("test", tags.get("env"));
    assertEquals("custom_tag_value", tags.get("custom_tag"));
    assertEquals(true, metricFunctionsSpec.getMergeTags());
}
 
Example 6
Source File: OrcRowInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadDecimalTypeFile() throws IOException {
	rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_DECIMAL), TEST_SCHEMA_DECIMAL, new Configuration());

	FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	rowOrcInputFormat.openInputFormat();
	rowOrcInputFormat.open(splits[0]);

	assertFalse(rowOrcInputFormat.reachedEnd());
	Row row = rowOrcInputFormat.nextRecord(null);

	// validate first row
	assertNotNull(row);
	assertEquals(1, row.getArity());
	assertEquals(BigDecimal.valueOf(-1000.5d), row.getField(0));

	// check correct number of rows
	long cnt = 1;
	while (!rowOrcInputFormat.reachedEnd()) {
		assertNotNull(rowOrcInputFormat.nextRecord(null));
		cnt++;
	}
	assertEquals(6000, cnt);
}
 
Example 7
Source File: OrcRowInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadTimeTypeFile() throws IOException {
	rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_TIMETYPES), TEST_SCHEMA_TIMETYPES, new Configuration());

	FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	rowOrcInputFormat.openInputFormat();
	rowOrcInputFormat.open(splits[0]);

	assertFalse(rowOrcInputFormat.reachedEnd());
	Row row = rowOrcInputFormat.nextRecord(null);

	// validate first row
	assertNotNull(row);
	assertEquals(2, row.getArity());
	assertEquals(Timestamp.valueOf("1900-05-05 12:34:56.1"), row.getField(0));
	assertEquals(Date.valueOf("1900-12-25"), row.getField(1));

	// check correct number of rows
	long cnt = 1;
	while (!rowOrcInputFormat.reachedEnd()) {
		assertNotNull(rowOrcInputFormat.nextRecord(null));
		cnt++;
	}
	assertEquals(70000, cnt);
}
 
Example 8
Source File: FileInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadMultiplePatterns() throws Exception {
	final String contents = "CONTENTS";

	// create some accepted, some ignored files

	File child1 = temporaryFolder.newFile("dataFile1.txt");
	File child2 = temporaryFolder.newFile("another_file.bin");
	createTempFiles(contents.getBytes(ConfigConstants.DEFAULT_CHARSET), child1, child2);

	// test that only the valid files are accepted

	Configuration configuration = new Configuration();

	final DummyFileInputFormat format = new DummyFileInputFormat();
	format.setFilePath(temporaryFolder.getRoot().toURI().toString());
	format.configure(configuration);
	format.setFilesFilter(new GlobFilePathFilter(
		Collections.singletonList("**"),
		Arrays.asList("**/another_file.bin", "**/dataFile1.txt")
	));
	FileInputSplit[] splits = format.createInputSplits(1);

	Assert.assertEquals(0, splits.length);
}
 
Example 9
Source File: ReadUtilsTest.java    From micro-server with Apache License 2.0 6 votes vote down vote up
@Test
@SneakyThrows
public void getFileInputStream() {
    TransferManager transferManager = mock(TransferManager.class);
    Download download = mock(Download.class);
    when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);

    ReadUtils readUtils = new ReadUtils(transferManager,System.getProperty("java.io.tmpdir"));

    InputStream fileInputStream = readUtils.getFileInputStream("bucket", "key");
    assertNotNull(fileInputStream);

    verify(transferManager, times(1)).download(anyString(), anyString(), any(File.class));
    verify(download, times(1)).waitForCompletion();

    fileInputStream.close();
}
 
Example 10
Source File: FlinkMRQLFileInputFormat.java    From incubator-retired-mrql with Apache License 2.0 6 votes vote down vote up
private static DataSource read_directory ( String buffer ) {
    try {
        String[] s = buffer.split(DataSource.separator);
        int n = Integer.parseInt(s[1]);
        if (s[0].equals("Binary"))
            return new BinaryDataSource(s[2],Plan.conf);
        else if (s[0].equals("Generator"))
            return new GeneratorDataSource(s[2],Plan.conf);
        else if (s[0].equals("Text"))
            return new FlinkParsedDataSource(s[3],DataSource.parserDirectory.get(s[2]),
                                             ((Node)Tree.parse(s[4])).children());
        else throw new Error("Unrecognized data source: "+s[0]);
    } catch (Exception e) {
        throw new Error(e);
    }
}
 
Example 11
Source File: FileStreamInputOutput.java    From gemfirexd-oss with Apache License 2.0 6 votes vote down vote up
private final DataValueDescriptor[] readDVDBuffer() throws StandardException {
  try {

    if (!diagnoseFileData) {
      return DataType.readDVDArray(this);
    }

    // we are in diagnostic mode ....
    final DataValueDescriptor[] dvd = DataType.readDVDArray(this);

    if (GemFireXDUtils.TraceTempFileIO) {
      final String srcDvd = rowObjectsWritten.get(rowObjectsindex++);
      String dvdS = toString(dvd, new StringBuilder()).toString();
      SanityManager.ASSERT(srcDvd.equals(dvdS), srcDvd + " != " + dvdS);
    }

    return dvd;

  } catch (final IOException ioe) {
    throw StandardException.newException(SQLState.LOG_CANNOT_FLUSH, ioe);
  } catch (final ClassNotFoundException cnf) {
    throw StandardException.newException(SQLState.LOG_CANNOT_FLUSH, cnf);
  }
}
 
Example 12
Source File: FileStreamInputOutput.java    From gemfirexd-oss with Apache License 2.0 6 votes vote down vote up
private final void checkReadBounds(final int expected) throws IOException {
  if (rwBuffer.remaining() >= expected) {
    return;
  }

  if (GemFireXDUtils.TraceTempFileIO) {
    SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_TEMP_FILE_IO,
        "[" + this.file + "] moving onto next buffer as remaining "
            + rwBuffer.remaining() + " doesn't fits expected " + expected);
    // new Throwable());
  }

  getNextBuffer();

  assert rwBuffer.remaining() >= expected: "remaining="
      + rwBuffer.remaining() + " expected=" + expected;
}
 
Example 13
Source File: FileStreamInputOutput.java    From gemfirexd-oss with Apache License 2.0 6 votes vote down vote up
@Override
public void readFully(final byte[] b, int off, int len) throws IOException {

  int remaining = rwBuffer.remaining();

  if (remaining >= len) {
    rwBuffer.get(b, off, len);
    return;
  }

  while (len > 0) {
    if (remaining >= len) {
      rwBuffer.get(b, off, len);
      return;
    }
    rwBuffer.get(b, off, remaining);
    off += remaining;
    len -= remaining;
    getNextBuffer();
    remaining = rwBuffer.capacity();
  }
}
 
Example 14
Source File: OrcRowInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadFileInSplits() throws IOException {

	rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_FLAT), TEST_SCHEMA_FLAT, new Configuration());
	rowOrcInputFormat.selectFields(0, 1);

	FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(4);
	assertEquals(4, splits.length);
	rowOrcInputFormat.openInputFormat();

	long cnt = 0;
	// read all splits
	for (FileInputSplit split : splits) {

		// open split
		rowOrcInputFormat.open(split);
		// read and count all rows
		while (!rowOrcInputFormat.reachedEnd()) {
			assertNotNull(rowOrcInputFormat.nextRecord(null));
			cnt++;
		}
	}
	// check that all rows have been read
	assertEquals(1920800, cnt);
}
 
Example 15
Source File: FileHandler.java    From iaf with Apache License 2.0 6 votes vote down vote up
public SkipBomAndDeleteFileAfterReadInputStream(InputStream inputStream,
		File file, boolean deleteAfterRead, IPipeLineSession session)
		throws FileNotFoundException {
	super(inputStream);
	this.file = file;
	this.deleteAfterRead = deleteAfterRead;
	this.session = session;
	if (deleteAfterRead) {
		if (file == null) {
			// This should not happen. A configuration warning for
			// read_delete in combination with classpath should have
			// occurred already.
			throw new FileNotFoundException("No file object provided");
		} else {
			file.deleteOnExit();
		}
	}
}
 
Example 16
Source File: EditLogFileInputStream.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Read the header of fsedit log
 * @param in fsedit stream
 * @return the edit log version number
 * @throws IOException if error occurs
 */
@VisibleForTesting
static int readLogVersion(DataInputStream in, boolean verifyLayoutVersion)
    throws IOException, LogHeaderCorruptException {
  int logVersion;
  try {
    logVersion = in.readInt();
  } catch (EOFException eofe) {
    throw new LogHeaderCorruptException(
        "Reached EOF when reading log header");
  }
  if (verifyLayoutVersion &&
      (logVersion < HdfsConstants.NAMENODE_LAYOUT_VERSION || // future version
       logVersion > Storage.LAST_UPGRADABLE_LAYOUT_VERSION)) { // unsupported
    throw new LogHeaderCorruptException(
        "Unexpected version of the file system log file: "
        + logVersion + ". Current version = "
        + HdfsConstants.NAMENODE_LAYOUT_VERSION + ".");
  }
  return logVersion;
}
 
Example 17
Source File: HTMLInputElementTest.java    From htmlunit with Apache License 2.0 6 votes vote down vote up
/**
 * @throws Exception if the test fails
 */
@Test
@Alerts({"false", "true"})
public void readOnlyInputFile() throws Exception {
    final String html
        = HtmlPageTest.STANDARDS_MODE_PREFIX_
        + "<html>\n"
        + "<head><title>foo</title>\n"
        + "<script>\n"
        + "  function test() {\n"
        + "    var input = document.getElementById('myInput');\n"
        + "    alert(input.readOnly);\n"
        + "    input = document.getElementById('myReadonlyInput');\n"
        + "    alert(input.readOnly);\n"
        + "  }\n"
        + "  </script>\n"
        + "</head>\n"
        + "<body onload='test()'>\n"
        + "  <input id='myInput' type='file' value='some test'>\n"
        + "  <input id='myReadonlyInput' type='file' value='some test' readonly='false'>\n"
        + "</body></html>";

    loadPageWithAlerts2(html);
}
 
Example 18
Source File: AbstractSpreadSheetFlinkFileInputFormat.java    From hadoopoffice with Apache License 2.0 6 votes vote down vote up
/***
 * Read truststore for establishing certificate chain for signature validation
 * 
 * @param conf
 * @throws IOException
 * @throws FormatNotUnderstoodException
 */
private void readTrustStore() throws IOException, FormatNotUnderstoodException {
	if (((this.hocr.getSigTruststoreFile() != null) && (!"".equals(this.hocr.getSigTruststoreFile())))) {
		LOG.info("Reading truststore to validate certificate chain for signatures");
		FlinkKeyStoreManager fksm = new FlinkKeyStoreManager();
		try {
			fksm.openKeyStore(new Path(this.hocr.getSigTruststoreFile()), this.hocr.getSigTruststoreType(),
					this.hocr.getSigTruststorePassword());
			this.hocr.setX509CertificateChain(fksm.getAllX509Certificates());
		} catch (NoSuchAlgorithmException | CertificateException | KeyStoreException | IllegalArgumentException e) {
			LOG.error("Exception: ",e);
			throw new FormatNotUnderstoodException(
					"Cannot read truststore to establish certificate chain for signature validation " + e);
		}

	}
}
 
Example 19
Source File: FileReader.java    From XmlToJson with Apache License 2.0 6 votes vote down vote up
public static String readFileFromInputStream(@NonNull InputStream inputStream) {

        InputStreamReader inputStreamReader = new InputStreamReader(inputStream);
        BufferedReader bufferedReader = new BufferedReader(inputStreamReader);

        StringBuilder result = new StringBuilder();
        String line;
        try {
            while ((line = bufferedReader.readLine()) != null) {
                result.append(line);
            }
            return result.toString();
        } catch (IOException exception) {
        } finally {
            try {
                bufferedReader.close();
            } catch (IOException e2) {
            }
            try {
                inputStreamReader.close();
            } catch (IOException e2) {
            }
        }
        return null;
    }
 
Example 20
Source File: FileCacheManager.java    From BlackLight with GNU General Public License v3.0 6 votes vote down vote up
private byte[] readInputStream(InputStream in) throws IOException {
	ByteArrayOutputStream opt = new ByteArrayOutputStream();
	byte[] buf = new byte[1024];
	int len = 0;
	while ((len = in.read(buf)) != -1) {
		opt.write(buf, 0, len);
	}
	in.close();
	byte[] ret;
	try {
		ret = opt.toByteArray();
	} catch (OutOfMemoryError e) {
		ret = null;
	}
	opt.close();
	return ret;
}
 
Example 21
Source File: SFTPFileSystemInputStreamTest.java    From sftp-fs with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadBulk() throws IOException {
    final String content = "Hello World";

    Path file = addFile("/foo");
    setContents(file, content);

    byte[] b = new byte[20];
    try (InputStream input = fileSystem.newInputStream(createPath("/foo"))) {
        assertEquals(0, input.read(b, 0, 0));
        assertEquals(5, input.read(b, 1, 5));
        assertArrayEquals(content.substring(0, 5).getBytes(), Arrays.copyOfRange(b, 1, 6));
        assertEquals(content.length() - 5, input.read(b));
        assertArrayEquals(content.substring(5).getBytes(), Arrays.copyOfRange(b, 0, content.length() - 5));
        assertEquals(-1, input.read(b));
    }
}
 
Example 22
Source File: SFTPFileSystemInputStreamTest.java    From sftp-fs with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadSingle() throws IOException {
    final String content = "Hello World";

    Path file = addFile("/foo");
    setContents(file, content);

    try (InputStream input = fileSystem.newInputStream(createPath("/foo"))) {
        assertEquals('H', input.read());
        assertEquals('e', input.read());
        assertEquals('l', input.read());
        assertEquals('l', input.read());
        assertEquals('o', input.read());
        assertEquals(' ', input.read());
        assertEquals('W', input.read());
        assertEquals('o', input.read());
        assertEquals('r', input.read());
        assertEquals('l', input.read());
        assertEquals('d', input.read());
        assertEquals(-1, input.read());
    }
}
 
Example 23
Source File: DeaFileUtils.java    From TranskribusCore with GNU General Public License v3.0 6 votes vote down vote up
public static String readInputStreamAsString(InputStream is) throws FileNotFoundException, IOException {
	BufferedReader br = null;
	StringBuffer result = new StringBuffer();
	try {		
		InputStream in = new DataInputStream(is);
		br = new BufferedReader(new InputStreamReader(in));
		String strLine;
		// Read File Line By Line
		while ((strLine = br.readLine()) != null) {
			result.append(strLine + "\n");
		}
	} finally {
		br.close();
	}

	return result.toString();
}
 
Example 24
Source File: CombineReadCountsIntegrationTest.java    From gatk-protected with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test(expectedExceptions = CommandLineException.class)
public void testEmptyInputFileListProvided() throws Exception {
    @SuppressWarnings("serial")
    final List<Target> phonyTargets = SimulatedTargets.phonyTargets(3);
    final List<File> inputFiles = Collections.emptyList();
    final File targetFile = createTargetFile(phonyTargets);
    final File inputListFile = createInputListFile(inputFiles);
    try {
        runTool(targetFile, inputFiles, inputListFile);
    } catch (final Exception ex) {
        throw ex;
    } finally {
        targetFile.delete();
        inputListFile.delete();
    }
}
 
Example 25
Source File: AsyncInputStreamTest.java    From wisdom with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadSmallFileFromUrl() throws IOException, InterruptedException {
    latch = new CountDownLatch(1);
    final ByteArrayOutputStream bos = new ByteArrayOutputStream();
    File file = new File("src/test/resources/a_file.txt");
    URL url = file.toURI().toURL();
    Context context = vertx.getOrCreateContext();
    final AsyncInputStream async = new AsyncInputStream(vertx, executor,
            url.openStream())
            .endHandler(event -> {
                assertThat(bos.toString()).startsWith("This is a file.");
                latch.countDown();
            }).setContext(context);
    context.runOnContext(event -> async.handler(buffer -> {
        try {
            bos.write(buffer.getBytes());
        } catch (IOException e) {
            e.printStackTrace();
        }
    }));
    latch.await(30, TimeUnit.SECONDS);
}
 
Example 26
Source File: FileUtilities.java    From hortonmachine with GNU General Public License v3.0 6 votes vote down vote up
/**
 * Read from an inoutstream and convert the readed stuff to a String. Usefull for text files that are available as
 * streams.
 *
 * @param inputStream
 *
 * @return the read string
 *
 * @throws IOException
 */
public static String readInputStreamToString( InputStream inputStream ) throws IOException {
    // Create the byte list to hold the data
    List<Byte> bytesList = new ArrayList<Byte>();

    byte b = 0;
    while( (b = (byte) inputStream.read()) != -1 ) {
        bytesList.add(b);
    }
    // Close the input stream and return bytes
    inputStream.close();

    byte[] bArray = new byte[bytesList.size()];
    for( int i = 0; i < bArray.length; i++ ) {
        bArray[i] = bytesList.get(i);
    }

    String file = new String(bArray);
    return file;
}
 
Example 27
Source File: AndroidInputFile.java    From actor-platform with GNU Affero General Public License v3.0 6 votes vote down vote up
@Override
public Promise<FilePart> read(int fileOffset, int len) {
    return new Promise<>(resolver -> {
        executor.execute(() -> {
            try {
                byte[] data = new byte[len];
                randomAccessFile.seek(fileOffset);
                // TODO: Better reading. For big len result can be truncated
                randomAccessFile.read(data, 0, len);
                resolver.result(new FilePart(fileOffset, len, data));
            } catch (Exception e) {
                e.printStackTrace();
                resolver.error(e);
            }
        });
    });
}
 
Example 28
Source File: BytesUtils.java    From Mahuta with Apache License 2.0 6 votes vote down vote up
public static InputStream readFileInputStream(String path) {
    
    if(ValidatorUtils.isEmpty(path)) {
        return null;
    }

    try {
        ClassLoader classLoader = BytesUtils.class.getClassLoader();
        File file = new File(Objects.requireNonNull(classLoader.getResource(path)).getFile());

        return new FileInputStream(file);

    } catch (FileNotFoundException e) {
        throw new TechnicalException("File cannot be found...", e);
    }
}
 
Example 29
Source File: EncryptedFileInputStream.java    From Telegram with GNU General Public License v2.0 6 votes vote down vote up
@Override
public int read(byte[] b, int off, int len) throws IOException {
    if (currentMode == MODE_CBC && fileOffset == 0) {
        byte[] temp = new byte[32];
        super.read(temp, 0, 32);
        Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0);
        fileOffset += 32;
        skip((temp[0] & 0xff) - 32);
    }
    int result = super.read(b, off, len);
    if (currentMode == MODE_CBC) {
        Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0);
    } else if (currentMode == MODE_CTR) {
        Utilities.aesCtrDecryptionByteArray(b, key, iv, off, len, fileOffset);
    }
    fileOffset += len;
    return result;
}
 
Example 30
Source File: InputStreamToString.java    From journaldev with MIT License 6 votes vote down vote up
/**
 * reading file and converting InputStream to String using BufferedReader
 * @param fileName
 * @return
 * @throws IOException
 */
public static String readFileToStringUsingBufferedReader(String fileName) throws IOException{
	FileInputStream fileInputStream = null;
	BufferedReader bufferedReader = null;
	StringBuilder inputSB = null;
	try{
	fileInputStream = new FileInputStream(fileName);
	bufferedReader = new BufferedReader(new InputStreamReader(fileInputStream, "UTF-8"));
	inputSB = new StringBuilder();
	String line = bufferedReader.readLine();
       while(line != null){
       	inputSB.append(line);
           line = bufferedReader.readLine();
           if(line != null){
           	//add new line character
           	inputSB.append("\n");
           }
       }
	}finally{
		bufferedReader.close();
		fileInputStream.close();
	}
	return inputSB.toString();
}
 
Example 31
Source File: OrcRowInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadTimeTypeFile() throws IOException {
	rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_TIMETYPES), TEST_SCHEMA_TIMETYPES, new Configuration());

	FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	rowOrcInputFormat.openInputFormat();
	rowOrcInputFormat.open(splits[0]);

	assertFalse(rowOrcInputFormat.reachedEnd());
	Row row = rowOrcInputFormat.nextRecord(null);

	// validate first row
	assertNotNull(row);
	assertEquals(2, row.getArity());
	assertEquals(Timestamp.valueOf("1900-05-05 12:34:56.1"), row.getField(0));
	assertEquals(Date.valueOf("1900-12-25"), row.getField(1));

	// check correct number of rows
	long cnt = 1;
	while (!rowOrcInputFormat.reachedEnd()) {
		assertNotNull(rowOrcInputFormat.nextRecord(null));
		cnt++;
	}
	assertEquals(70000, cnt);
}
 
Example 32
Source File: FileInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadMultiplePatterns() throws Exception {
	final String contents = "CONTENTS";

	// create some accepted, some ignored files

	File child1 = temporaryFolder.newFile("dataFile1.txt");
	File child2 = temporaryFolder.newFile("another_file.bin");
	createTempFiles(contents.getBytes(ConfigConstants.DEFAULT_CHARSET), child1, child2);

	// test that only the valid files are accepted

	Configuration configuration = new Configuration();

	final DummyFileInputFormat format = new DummyFileInputFormat();
	format.setFilePath(temporaryFolder.getRoot().toURI().toString());
	format.configure(configuration);
	format.setFilesFilter(new GlobFilePathFilter(
		Collections.singletonList("**"),
		Arrays.asList("**/another_file.bin", "**/dataFile1.txt")
	));
	FileInputSplit[] splits = format.createInputSplits(1);

	Assert.assertEquals(0, splits.length);
}
 
Example 33
Source File: ParquetFileReader.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
public BytesInput readAsBytesInput(int size) throws IOException {
  int available = stream.available();
  if (size > available) {
    // this is to workaround a bug where the compressedLength
    // of the chunk is missing the size of the header of the dictionary
    // to allow reading older files (using dictionary) we need this.
    // usually 13 to 19 bytes are missing
    int missingBytes = size - available;
    LOG.info("completed the column chunk with {} bytes", missingBytes);

    List<ByteBuffer> buffers = new ArrayList<>();
    buffers.addAll(stream.sliceBuffers(available));

    ByteBuffer lastBuffer = ByteBuffer.allocate(missingBytes);
    f.readFully(lastBuffer);
    buffers.add(lastBuffer);

    return BytesInput.from(buffers);
  }

  return super.readAsBytesInput(size);
}
 
Example 34
Source File: LockedReadFileInput.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** {@inheritDoc} */
@Override public void ensure(int requested) throws IOException {
    int available = buffer().remaining();

    if (available >= requested)
        return;

    boolean readArchive = segmentAware.checkCanReadArchiveOrReserveWorkSegment(segmentId);
    try {
        if (readArchive && !isLastReadFromArchive) {
            isLastReadFromArchive = true;

            refreshIO();
        }

        super.ensure(requested);
    }
    finally {
        if (!readArchive)
            segmentAware.releaseWorkSegment(segmentId);
    }
}
 
Example 35
Source File: ECFileCacheInputStream.java    From ECFileCache with Apache License 2.0 6 votes vote down vote up
/**
 * Reads up to <code>len</code> bytes of data from the input stream into
 * an array of bytes.  An attempt is made to read as many as
 * <code>len</code> bytes, but a smaller number may be read.
 * The number of bytes actually read is returned as an integer.
 *
 * @param b     the buffer into which the data is read.
 * @param off   the start offset in array <code>b</code> at which the data is written.
 * @param len   the maximum number of bytes to read.
 * @return     the total number of bytes read into the buffer, or
 *             <code>-1</code> if the end of the stream has been reached.
 * @throws IOException
 */
@Override
public int read(byte[] b, int off, int len) throws IOException {
  checkIfClosed();
  if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
    throw new IndexOutOfBoundsException();
  } else if (len == 0) {
    return 0;
  }

  int n = 0;
  while (true) {
    int nread = readRedisIfNeed(b, off + n, len - n);
    if (nread <= 0) {
      return (n == 0) ? nread : n;
    }
    n += nread;
    if (n >= len) {
      return n;
    }
    if (available() <= 0) {
      return n;
    }
  }
}
 
Example 36
Source File: TarFileInputStream.java    From evosql with Apache License 2.0 6 votes vote down vote up
/**
 * Work-around for the problem that compressed InputReaders don't fill
 * the read buffer before returning.
 *
 * Has visibility 'protected' so that subclasses may override with
 * different algorithms, or use different algorithms for different
 * compression stream.
 */
protected void readCompressedBlocks(int blocks) throws IOException {

    int bytesSoFar    = 0;
    int requiredBytes = 512 * blocks;

    // This method works with individual bytes!
    int i;

    while (bytesSoFar < requiredBytes) {
        i = readStream.read(readBuffer, bytesSoFar,
                            requiredBytes - bytesSoFar);

        if (i < 0) {
            throw new EOFException(
                RB.decompression_ranout.getString(
                    bytesSoFar, requiredBytes));
        }

        bytesRead  += i;
        bytesSoFar += i;
    }
}
 
Example 37
Source File: EncryptedFileInputStream.java    From Telegram-FOSS with GNU General Public License v2.0 6 votes vote down vote up
@Override
public int read(byte[] b, int off, int len) throws IOException {
    if (currentMode == MODE_CBC && fileOffset == 0) {
        byte[] temp = new byte[32];
        super.read(temp, 0, 32);
        Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0);
        fileOffset += 32;
        skip((temp[0] & 0xff) - 32);
    }
    int result = super.read(b, off, len);
    if (currentMode == MODE_CBC) {
        Utilities.aesCbcEncryptionByteArraySafe(b, key, iv, off, len, fileOffset, 0);
    } else if (currentMode == MODE_CTR) {
        Utilities.aesCtrDecryptionByteArray(b, key, iv, off, len, fileOffset);
    }
    fileOffset += len;
    return result;
}
 
Example 38
Source File: TextFileInputTest.java    From hop with Apache License 2.0 6 votes vote down vote up
@Test
public void readInputWithDefaultValues() throws Exception {
  final String virtualFile = createVirtualFile( "pdi-14832.txt", "1,\n" );

  BaseFileField col2 = field( "col2" );
  col2.setIfNullValue( "DEFAULT" );

  TextFileInputMeta meta = createMetaObject( field( "col1" ), col2 );
  TextFileInputData data = createDataObject( virtualFile, ",", "col1", "col2" );

  TextFileInput input = TransformMockUtil.getTransform( TextFileInput.class, TextFileInputMeta.class, "test" );
  List<Object[]> output = PipelineTestingUtil.execute( input, meta, data, 1, false );
  PipelineTestingUtil.assertResult( new Object[] { "1", "DEFAULT" }, output.get( 0 ) );

  deleteVfsFile( virtualFile );
}
 
Example 39
Source File: TextFileInputTest.java    From hop with Apache License 2.0 6 votes vote down vote up
@Test
public void readInputWithNonEmptyNullif() throws Exception {
  final String virtualFile = createVirtualFile( "pdi-14358.txt", "-,-\n" );

  BaseFileField col2 = field( "col2" );
  col2.setNullString( "-" );

  TextFileInputMeta meta = createMetaObject( field( "col1" ), col2 );
  TextFileInputData data = createDataObject( virtualFile, ",", "col1", "col2" );

  TextFileInput input = TransformMockUtil.getTransform( TextFileInput.class, TextFileInputMeta.class, "test" );
  List<Object[]> output = PipelineTestingUtil.execute( input, meta, data, 1, false );
  PipelineTestingUtil.assertResult( new Object[] { "-" }, output.get( 0 ) );

  deleteVfsFile( virtualFile );
}
 
Example 40
Source File: InputFile.java    From arvo2parquet with MIT License 6 votes vote down vote up
private static int readDirectBuffer(ByteBuffer byteBufr, byte[] tmpBuf, ByteBufReader rdr) throws IOException {
  // copy all the bytes that return immediately, stopping at the first
  // read that doesn't return a full buffer.
  int nextReadLength = Math.min(byteBufr.remaining(), tmpBuf.length);
  int totalBytesRead = 0;
  int bytesRead;

  while ((bytesRead = rdr.read(tmpBuf, 0, nextReadLength)) == tmpBuf.length) {
    byteBufr.put(tmpBuf);
    totalBytesRead += bytesRead;
    nextReadLength = Math.min(byteBufr.remaining(), tmpBuf.length);
  }

  if (bytesRead < 0) {
    // return -1 if nothing was read
    return totalBytesRead == 0 ? -1 : totalBytesRead;
  } else {
    // copy the last partial buffer
    byteBufr.put(tmpBuf, 0, bytesRead);
    totalBytesRead += bytesRead;
    return totalBytesRead;
  }
}
 
Example 41
Source File: FileUtil.java    From Leo with Apache License 2.0 6 votes vote down vote up
/**
 *  读取字符流,指定编码
 * @param input
 * @param enCoding
 * @return String
 */
public static String readInputStreamToString(InputStream input,String enCoding) {
	BufferedReader reader ;
	StringBuilder sb = new StringBuilder();
	String line = null;
	try {
		reader = new BufferedReader(new InputStreamReader(input,enCoding));
		while ((line = reader.readLine()) != null) {
			sb.append(line + "\r\n");
		}
	} catch (IOException e) {
		log.error(e.getMessage());
	} finally {
		try {
			input.close();
		} catch (IOException e) {
			log.error(e.getMessage());
		}
	}
	return sb.toString();

}
 
Example 42
Source File: EditLogFileInputStream.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Read the header of fsedit log
 * @param in fsedit stream
 * @return the edit log version number
 * @throws IOException if error occurs
 */
@VisibleForTesting
static int readLogVersion(DataInputStream in, boolean verifyLayoutVersion)
    throws IOException, LogHeaderCorruptException {
  int logVersion;
  try {
    logVersion = in.readInt();
  } catch (EOFException eofe) {
    throw new LogHeaderCorruptException(
        "Reached EOF when reading log header");
  }
  if (verifyLayoutVersion &&
      (logVersion < HdfsConstants.NAMENODE_LAYOUT_VERSION || // future version
       logVersion > Storage.LAST_UPGRADABLE_LAYOUT_VERSION)) { // unsupported
    throw new LogHeaderCorruptException(
        "Unexpected version of the file system log file: "
        + logVersion + ". Current version = "
        + HdfsConstants.NAMENODE_LAYOUT_VERSION + ".");
  }
  return logVersion;
}
 
Example 43
Source File: ClosedFileInputStreamTest.java    From rtg-tools with BSD 2-Clause "Simplified" License 6 votes vote down vote up
public void readAndCheckBlock(ClosedFileInputStream cfi, byte[] src, int position) throws IOException {
  final int toread = 100;
  final byte[] expected = new byte[toread];
  System.arraycopy(src, position, expected, 0, toread);

  final byte[] readdata = new byte[toread];
  cfi.seek(position);

  int read = 0;
  int batchread;
  while (read < toread && (batchread = cfi.read(readdata, read, toread - read)) != -1) {
    read += batchread;
  }

  assertTrue(Arrays.equals(expected, readdata));
}
 
Example 44
Source File: TestFileSystemInput.java    From envelope with Apache License 2.0 6 votes vote down vote up
@Test
public void readJsonNoOptions() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "json");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(JSON_DATA).getPath());
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(4, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("dog", first.getString(3));
  assertEquals("field1", first.schema().fields()[0].name());
  assertEquals(DataTypes.LongType, first.schema().fields()[0].dataType());
}
 
Example 45
Source File: TestFileSystemInput.java    From envelope with Apache License 2.0 6 votes vote down vote up
@Test
public void readCsvNoOptions() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath());
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(4, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("Four", first.getString(3));
}
 
Example 46
Source File: TestFileSystemInput.java    From envelope with Apache License 2.0 6 votes vote down vote up
@Test
public void readCsvWithFieldsSchema() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath());
  paramMap.put(FileSystemInput.CSV_HEADER_CONFIG, "true");
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, "flat");
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + FlatSchema.FIELD_NAMES_CONFIG,
               Lists.newArrayList("A Long", "An Int", "A String", "Another String"));
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + FlatSchema.FIELD_TYPES_CONFIG,
               Lists.newArrayList("long", "int", "string", "string"));
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(3, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("four", first.getString(3));
  assertEquals("Another String", first.schema().fields()[3].name());
  assertEquals(1L, first.get(0));
  assertEquals(DataTypes.LongType, first.schema().fields()[0].dataType());
}
 
Example 47
Source File: TestFileSystemInput.java    From envelope with Apache License 2.0 6 votes vote down vote up
@Test
public void readCsvWithAvroSchema() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath());
  paramMap.put(FileSystemInput.CSV_HEADER_CONFIG, "true");
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, "avro");
  paramMap.put(FileSystemInput.SCHEMA_CONFIG + "." + AvroSchema.AVRO_FILE_CONFIG,
               FileSystemInput.class.getResource(TestAvroSchema.AVRO_SCHEMA_DATA).getPath());
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(3, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("four", first.getString(3));
  assertEquals("Another_String", first.schema().fields()[3].name());
  assertEquals(1L, first.get(0));
  assertEquals(DataTypes.LongType, first.schema().fields()[0].dataType());
}
 
Example 48
Source File: FileSystemInput.java    From envelope with Apache License 2.0 6 votes vote down vote up
private Dataset<Row> readText(String path) {
  Dataset<Row> lines = Contexts.getSparkSession().read().text(path);

  if (hasTranslator) {
    TranslateFunction translateFunction = getTranslateFunction(translatorConfig);
    TranslationResults results = new TranslationResults(
        lines.javaRDD().flatMap(translateFunction),
        translateFunction.getProvidingSchema(),
        getProvidingSchema());
    errors = results.getErrored();

    return results.getTranslated();
  }
  else {
    return lines;
  }
}
 
Example 49
Source File: TestFileSystemInput.java    From envelope with Apache License 2.0 6 votes vote down vote up
@Test
public void readCsvWithOptions() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "csv");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath());
  paramMap.put(FileSystemInput.CSV_HEADER_CONFIG, "true");
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput csvInput = new FileSystemInput();
  assertNoValidationFailures(csvInput, config);
  csvInput.configure(config);

  Dataset<Row> dataFrame = csvInput.read();
  assertEquals(3, dataFrame.count());

  Row first = dataFrame.first();
  assertEquals("four", first.getString(3));
}
 
Example 50
Source File: TestFileSystemInput.java    From envelope with Apache License 2.0 6 votes vote down vote up
@Test
public void readTextWithoutTranslator() throws Exception {
  Map<String, Object> configMap = Maps.newHashMap();
  configMap.put(FileSystemInput.FORMAT_CONFIG, FileSystemInput.TEXT_FORMAT);
  configMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(TEXT_DATA).getPath());
  config = ConfigFactory.parseMap(configMap);
  
  FileSystemInput formatInput = new FileSystemInput();
  assertNoValidationFailures(formatInput, config);
  formatInput.configure(config);
  
  List<Row> results = formatInput.read().collectAsList();
  
  assertEquals(2, results.size());
  assertTrue(results.contains(RowFactory.create("a=1,b=hello,c=true")));
  assertTrue(results.contains(RowFactory.create("a=2,b=world,c=false")));
}
 
Example 51
Source File: FileSystemInput.java    From envelope with Apache License 2.0 6 votes vote down vote up
private Dataset<Row> readInputFormat(String path) throws Exception {
  LOG.debug("Reading InputFormat[{}]: {}", inputType, path);

  Class<? extends InputFormat> inputFormatClass = Class.forName(inputType).asSubclass(InputFormat.class);

  TranslateFunction translateFunction = getTranslateFunction(translatorConfig);

  Dataset<Row> encoded = getEncodedRowsFromInputFormat(path, inputFormatClass);

  TranslationResults results = new TranslationResults(
      encoded.javaRDD().flatMap(translateFunction),
      translateFunction.getProvidingSchema(),
      getProvidingSchema());
  errors = results.getErrored();

  return results.getTranslated();
}
 
Example 52
Source File: MockService.java    From vespa with Apache License 2.0 6 votes vote down vote up
private void readInputFile(BufferedReader reader) throws IOException {
    StringBuilder sb = new StringBuilder();
    int ch;
    char prevChar = 0;
    while ((ch = reader.read()) >= 0) {
        char c = (char) ch;
        if (prevChar == '\n') {
            if (c == '\n') {
                parseEntry(sb.toString());
                sb = new StringBuilder();
                prevChar = 0;
                continue;
            } else {
                sb.append(prevChar);
            }
        }
        if (c != '\n') {
            sb.append(c);
        }
        prevChar = c;
    }
    parseEntry(sb.toString());
}
 
Example 53
Source File: FileUtil.java    From BiglyBT with GNU General Public License v2.0 6 votes vote down vote up
public static String readInputStreamAsString(InputStream is, int size_limit,
		int timeoutMillis, String charSet)
		throws IOException {
	StringBuilder result = new StringBuilder(1024);
	long maxTimeMillis = System.currentTimeMillis() + timeoutMillis;
	byte[] buffer = new byte[1024];

	while (System.currentTimeMillis() < maxTimeMillis) {
		int readLength = Math.min(is.available(), buffer.length);

		int len = is.read(buffer, 0, readLength);
		if (len == -1)
			break;

		result.append(new String(buffer, 0, len, charSet));

		if (size_limit >= 0 && result.length() > size_limit) {

			result.setLength(size_limit);

			break;
		}
	}

	return (result.toString());
}
 
Example 54
Source File: TestResettableFileInputStream.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that we can simply read bytes from a file.
 * @throws IOException
 */
@Test
public void testBasicRead() throws IOException {
  String output = singleLineFileInit(file, Charsets.UTF_8);

  PositionTracker tracker = new DurablePositionTracker(meta, file.getPath());
  ResettableInputStream in = new ResettableFileInputStream(file,  tracker);

  String result = readLine(in, output.length());
  assertEquals(output, result);

  String afterEOF = readLine(in, output.length());
  assertNull(afterEOF);

  in.close();
}
 
Example 55
Source File: FileSystemRepository.java    From dble with GNU General Public License v2.0 6 votes vote down vote up
private static Collection<CoordinatorLogEntry> readFromInputStream(
        InputStream in) {
    Map<String, CoordinatorLogEntry> coordinatorLogEntries = new HashMap<>();
    BufferedReader br = null;
    try {
        InputStreamReader isr = new InputStreamReader(in);
        br = new BufferedReader(isr);
        coordinatorLogEntries = readContent(br);
    } catch (Exception e) {
        LOGGER.warn("Error in recover", e);
        AlertUtil.alertSelf(AlarmCode.XA_READ_IO_FAIL, Alert.AlertLevel.WARN, "Error in recover:" + e.getMessage(), null);

    } finally {
        closeSilently(br);
    }
    return coordinatorLogEntries.values();
}
 
Example 56
Source File: EditLogFileInputStream.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * Read the header of fsedit log
 * @param in fsedit stream
 * @return the edit log version number
 * @throws IOException if error occurs
 */
static int readLogVersion(DataInputStream in) throws IOException,
    LogHeaderCorruptException {
  int logVersion = 0;
  // Read log file version. Could be missing.
  in.mark(4);
  // If edits log is greater than 2G, available method will return negative
  // numbers, so we avoid having to call available
  boolean available = true;
  try {
    logVersion = in.readByte();
  } catch (EOFException e) {
    available = false;
  }

  if (available) {
    in.reset();
    logVersion = in.readInt();
    if (logVersion < FSConstants.LAYOUT_VERSION) { // future version
      throw new LogHeaderCorruptException(
          "Unexpected version of the file system log file: " + logVersion
              + ". Current version = " + FSConstants.LAYOUT_VERSION + ".");
    }
  }
  return logVersion;
}
 
Example 57
Source File: FileCacheImageInputStream.java    From Java8CN with Apache License 2.0 5 votes vote down vote up
/**
 * Ensures that at least <code>pos</code> bytes are cached,
 * or the end of the source is reached.  The return value
 * is equal to the smaller of <code>pos</code> and the
 * length of the source file.
 */
private long readUntil(long pos) throws IOException {
    // We've already got enough data cached
    if (pos < length) {
        return pos;
    }
    // pos >= length but length isn't getting any bigger, so return it
    if (foundEOF) {
        return length;
    }

    long len = pos - length;
    cache.seek(length);
    while (len > 0) {
        // Copy a buffer's worth of data from the source to the cache
        // BUFFER_LENGTH will always fit into an int so this is safe
        int nbytes =
            stream.read(buf, 0, (int)Math.min(len, (long)BUFFER_LENGTH));
        if (nbytes == -1) {
            foundEOF = true;
            return length;
        }

        cache.write(buf, 0, nbytes);
        len -= nbytes;
        length += nbytes;
    }

    return pos;
}
 
Example 58
Source File: ClassFileReader.java    From ApkToolPlus with Apache License 2.0 5 votes vote down vote up
/**
    Converts a class file to a <tt>ClassFile</tt> structure.
    @param is the input stream from which to read the
              <tt>ClassFile</tt> structure
    @return the new <tt>ClassFile</tt> structure
    @throws InvalidByteCodeException if the code is invalid
    @throws IOException if an exception occurs while reading from
                        the input stream
 */
public static ClassFile readFromInputStream(InputStream is)
    throws InvalidByteCodeException, IOException
{

    DataInputStream in = new DataInputStream(
                            new BufferedInputStream(is));

    ClassFile classFile = new ClassFile();
    classFile.read(in);
    in.close();
    return classFile;
}
 
Example 59
Source File: RowTsFileInputFormatTest.java    From incubator-iotdb with Apache License 2.0 5 votes vote down vote up
@Test
public void testReadData() throws IOException {
	TsFileInputFormat<Row> inputFormat = prepareInputFormat(sourceTsFilePath1);

	List<String> actual = new ArrayList<>();

	try {
		inputFormat.configure(new Configuration());
		inputFormat.openInputFormat();
		FileInputSplit[] inputSplits = inputFormat.createInputSplits(2);
		Row reuse = rowTypeInfo.createSerializer(new ExecutionConfig()).createInstance();
		for (FileInputSplit inputSplit : inputSplits) {
			try {
				inputFormat.open(inputSplit);
				assertEquals(config.getBatchSize(), TSFileDescriptor.getInstance().getConfig().getBatchSize());
				while (!inputFormat.reachedEnd()) {
					Row row = inputFormat.nextRecord(reuse);
					actual.add(row.toString());
				}
			} finally {
				inputFormat.close();
			}
		}
	} finally {
		inputFormat.closeInputFormat();
	}

	String[] expected = {
		"1,1.2,20,null,2.3,11,19",
		"2,null,20,50,25.4,10,21",
		"3,1.4,21,null,null,null,null",
		"4,1.2,20,51,null,null,null",
		"6,7.2,10,11,null,null,null",
		"7,6.2,20,21,null,null,null",
		"8,9.2,30,31,null,null,null"
	};
	assertArrayEquals(actual.toArray(), expected);
}
 
Example 60
Source File: FileSetInputStream.java    From jdbc-driver-csv with GNU Lesser General Public License v3.0 5 votes vote down vote up
private int readFromTail() {
    if (pos < tail.length()) {
        return tail.charAt(pos++);
    }
    pos = 0;
    if (readingHeader) {
        readingHeader = false;
        tail = dataTail;
    }
    return -1;
}