Java Code Examples for org.apache.commons.io.IOUtils#readFully()

The following examples show how to use org.apache.commons.io.IOUtils#readFully() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: VorbisCommentReader.java    From AntennaPodSP with MIT License 6 votes vote down vote up
/**
 * Looks for an identification header in the first page of the file. If an
 * identification header is found, it will be skipped completely and the
 * method will return true, otherwise false.
 *
 * @throws IOException
 */
private boolean findIdentificationHeader(InputStream input)
        throws IOException {
    byte[] buffer = new byte[FIRST_PAGE_LENGTH];
    IOUtils.readFully(input, buffer);
    int i;
    for (i = 6; i < buffer.length; i++) {
        if (buffer[i - 5] == 'v' && buffer[i - 4] == 'o'
                && buffer[i - 3] == 'r' && buffer[i - 2] == 'b'
                && buffer[i - 1] == 'i' && buffer[i] == 's'
                && buffer[i - 6] == PACKET_TYPE_IDENTIFICATION) {
            return true;
        }
    }
    return false;
}
 
Example 2
Source File: StoregateWriteFeatureTest.java    From cyberduck with GNU General Public License v3.0 6 votes vote down vote up
@Test
public void testWriteSingleByte() throws Exception {
    final StoregateIdProvider nodeid = new StoregateIdProvider(session).withCache(cache);
    final StoregateWriteFeature feature = new StoregateWriteFeature(session, nodeid);
    final Path room = new StoregateDirectoryFeature(session, nodeid).mkdir(
        new Path(String.format("/My files/%s", new AlphanumericRandomStringService().random()),
            EnumSet.of(Path.Type.directory, Path.Type.volume)), null, new TransferStatus());
    final byte[] content = RandomUtils.nextBytes(1);
    final TransferStatus status = new TransferStatus();
    status.setLength(content.length);
    final Path file = new Path(room, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file));
    final HttpResponseOutputStream<VersionId> out = feature.write(file, status, new DisabledConnectionCallback());
    final ByteArrayInputStream in = new ByteArrayInputStream(content);
    assertEquals(content.length, IOUtils.copyLarge(in, out));
    in.close();
    out.close();
    assertNotNull(out.getStatus());
    assertTrue(new DefaultFindFeature(session).find(file));
    final byte[] compare = new byte[content.length];
    final InputStream stream = new StoregateReadFeature(session, nodeid).read(file, new TransferStatus().length(content.length), new DisabledConnectionCallback());
    IOUtils.readFully(stream, compare);
    stream.close();
    assertArrayEquals(content, compare);
    new StoregateDeleteFeature(session, nodeid).delete(Collections.singletonList(file), new DisabledLoginCallback(), new Delete.DisabledCallback());
}
 
Example 3
Source File: GraphWriteFeatureTest.java    From cyberduck with GNU General Public License v3.0 6 votes vote down vote up
@Test
public void testWriteUmlaut() throws Exception {
    final GraphWriteFeature feature = new GraphWriteFeature(session);
    final Path container = new OneDriveHomeFinderService(session).find();
    final byte[] content = RandomUtils.nextBytes(2048);
    final TransferStatus status = new TransferStatus();
    status.setLength(content.length);
    final Path file = new Path(container, String.format("%sä", new AlphanumericRandomStringService().random()), EnumSet.of(Path.Type.file));
    final HttpResponseOutputStream<Void> out = feature.write(file, status, new DisabledConnectionCallback());
    final ByteArrayInputStream in = new ByteArrayInputStream(content);
    assertEquals(content.length, IOUtils.copyLarge(in, out));
    in.close();
    out.close();
    assertNull(out.getStatus());
    assertTrue(new DefaultFindFeature(session).find(file));
    final byte[] compare = new byte[content.length];
    final InputStream stream = new GraphReadFeature(session).read(file, new TransferStatus().length(content.length), new DisabledConnectionCallback());
    IOUtils.readFully(stream, compare);
    stream.close();
    assertArrayEquals(content, compare);
    new GraphDeleteFeature(session).delete(Collections.singletonList(file), new DisabledLoginCallback(), new Delete.DisabledCallback());
}
 
Example 4
Source File: ReactorUtilsTest.java    From james-project with Apache License 2.0 6 votes vote down vote up
@Test
void givenAFluxOnOneByteShouldConsumeOnlyTheReadBytesAndThePrefetch() throws IOException, InterruptedException {
    AtomicInteger generateElements = new AtomicInteger(0);
    Flux<ByteBuffer> source = Flux.range(0, 10)
        .subscribeOn(Schedulers.elastic())
        .limitRate(2)
        .doOnRequest(request -> generateElements.getAndAdd((int) request))
        .map(index -> new byte[] {(byte) (int) index})
        .map(ByteBuffer::wrap);

    InputStream inputStream = ReactorUtils.toInputStream(source);
    byte[] readBytes = IOUtils.readFully(inputStream, 5);

    assertThat(readBytes).contains(0, 1, 2, 3, 4);
    //make sure reactor is done with prefetch
    Thread.sleep(200);
    assertThat(generateElements.get()).isEqualTo(6);
}
 
Example 5
Source File: SwiftLargeUploadWriteFeatureTest.java    From cyberduck with GNU General Public License v3.0 6 votes vote down vote up
@Test
public void testWriteZeroLength() throws Exception {
    final SwiftRegionService regionService = new SwiftRegionService(session);
    final SwiftLargeUploadWriteFeature feature = new SwiftLargeUploadWriteFeature(session, regionService,
        new SwiftSegmentService(session, ".segments-test/"));
    final Path container = new Path("test.cyberduck.ch", EnumSet.of(Path.Type.directory, Path.Type.volume));
    final byte[] content = RandomUtils.nextBytes(0);
    final TransferStatus status = new TransferStatus();
    status.setLength(-1L);
    final Path file = new Path(container, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file));
    final HttpResponseOutputStream<List<StorageObject>> out = feature.write(file, status, new DisabledConnectionCallback());
    final ByteArrayInputStream in = new ByteArrayInputStream(content);
    assertEquals(content.length, IOUtils.copyLarge(in, out));
    in.close();
    out.close();
    assertNotNull(out.getStatus());
    assertTrue(new DefaultFindFeature(session).find(file));
    final byte[] compare = new byte[content.length];
    final InputStream stream = new SwiftReadFeature(session, regionService).read(file, new TransferStatus().length(content.length), new DisabledConnectionCallback());
    IOUtils.readFully(stream, compare);
    stream.close();
    assertArrayEquals(content, compare);
    new SwiftDeleteFeature(session).delete(Collections.singletonList(file), new DisabledLoginCallback(), new Delete.DisabledCallback());
}
 
Example 6
Source File: AES256JNCryptorInputStreamTest.java    From JNCryptor with Apache License 2.0 6 votes vote down vote up
/**
 * Test reading using read(byte[]) method.
 * 
 * @throws Exception
 */
@Test
public void testUsingReadByteArrayWithOffset() throws Exception {
  byte[] plaintext = getRandomBytes(256);

  final String password = "Testing1234";

  JNCryptor cryptor = new AES256JNCryptor();
  byte[] data = cryptor.encryptData(plaintext, password.toCharArray());

  InputStream in = new AES256JNCryptorInputStream(new ByteArrayInputStream(
      data), password.toCharArray());

  try {

    byte[] result = new byte[256];
    IOUtils.readFully(in, result);

    assertArrayEquals(plaintext, result);
  } finally {
    in.close();
  }
}
 
Example 7
Source File: B2LargeUploadWriteFeatureTest.java    From cyberduck with GNU General Public License v3.0 6 votes vote down vote up
@Test
public void testWrite() throws Exception {
    final B2FileidProvider fileid = new B2FileidProvider(session).withCache(cache);
    final B2LargeUploadWriteFeature feature = new B2LargeUploadWriteFeature(session, fileid);
    final Path container = new Path("test-cyberduck", EnumSet.of(Path.Type.directory, Path.Type.volume));
    final TransferStatus status = new TransferStatus();
    status.setLength(-1L);
    status.setTimestamp(1503654614004L);
    final Path file = new Path(container, UUID.randomUUID().toString(), EnumSet.of(Path.Type.file));
    final OutputStream out = feature.write(file, status, new DisabledConnectionCallback());
    final byte[] content = new RandomStringGenerator.Builder().build().generate(6 * 1024 * 1024).getBytes(StandardCharsets.UTF_8);
    final ByteArrayInputStream in = new ByteArrayInputStream(content);
    assertEquals(content.length, IOUtils.copy(in, out));
    in.close();
    out.close();
    assertTrue(new B2FindFeature(session, fileid).find(file));
    final byte[] compare = new byte[content.length];
    final InputStream stream = new B2ReadFeature(session, fileid).read(file, new TransferStatus().length(content.length), new DisabledConnectionCallback());
    IOUtils.readFully(stream, compare);
    stream.close();
    assertArrayEquals(content, compare);
    assertEquals(1503654614004L, new B2AttributesFinderFeature(session, fileid).find(file).getModificationDate());
    new B2DeleteFeature(session, fileid).delete(Collections.singletonList(file), new DisabledLoginCallback(), new Delete.DisabledCallback());
}
 
Example 8
Source File: S3MultipartWriteFeatureTest.java    From cyberduck with GNU General Public License v3.0 6 votes vote down vote up
@Test
public void testWrite() throws Exception {
    final S3MultipartWriteFeature feature = new S3MultipartWriteFeature(session);
    final Path container = new Path("test-eu-central-1-cyberduck", EnumSet.of(Path.Type.directory, Path.Type.volume));
    final TransferStatus status = new TransferStatus();
    status.setLength(-1L);
    final Path file = new Path(container, UUID.randomUUID().toString(), EnumSet.of(Path.Type.file));
    final HttpResponseOutputStream<VersionId> out = feature.write(file, status, new DisabledConnectionCallback());
    final byte[] content = RandomUtils.nextBytes(6 * 1024 * 1024);
    final ByteArrayInputStream in = new ByteArrayInputStream(content);
    final TransferStatus progress = new TransferStatus();
    new StreamCopier(new TransferStatus(), progress).transfer(in, out);
    assertEquals(content.length, progress.getOffset());
    in.close();
    out.close();
    assertNotNull(out.getStatus());
    assertTrue(new S3FindFeature(session).find(file));
    final byte[] compare = new byte[content.length];
    final InputStream stream = new S3ReadFeature(session).read(file, new TransferStatus().length(content.length), new DisabledConnectionCallback());
    IOUtils.readFully(stream, compare);
    stream.close();
    assertArrayEquals(content, compare);
    new S3DefaultDeleteFeature(session).delete(Collections.singletonList(file), new DisabledLoginCallback(), new Delete.DisabledCallback());
}
 
Example 9
Source File: ReleaseTool.java    From apicurio-studio with Apache License 2.0 5 votes vote down vote up
/**
 * @param releaseArtifactFile
 */
private static byte[] loadArtifactData(File releaseArtifactFile) throws Exception {
    System.out.println("Loading artifact content: " + releaseArtifactFile.getName());
    byte [] buffer = new byte[(int) releaseArtifactFile.length()];
    try (InputStream is = new FileInputStream(releaseArtifactFile)) {
        IOUtils.readFully(is, buffer);
        return buffer;
    } catch (IOException e) {
        throw new Exception(e);
    }
}
 
Example 10
Source File: CellCodecWithTags.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * @return Byte array read from the stream.
 * @throws IOException
 */
private byte[] readByteArray(final InputStream in) throws IOException {
  byte[] intArray = new byte[Bytes.SIZEOF_INT];
  IOUtils.readFully(in, intArray);
  int length = Bytes.toInt(intArray);
  byte[] bytes = new byte[length];
  IOUtils.readFully(in, bytes);
  return bytes;
}
 
Example 11
Source File: SFTPCryptomatorInteroperabilityTest.java    From cyberduck with GNU General Public License v3.0 5 votes vote down vote up
/**
 * Create long file/folder with Cryptomator, read with Cyberduck
 */
@Test
public void testCryptomatorInteroperability() throws Exception {
    // create folder
    final java.nio.file.Path targetFolder = cryptoFileSystem.getPath("/", new AlphanumericRandomStringService().random());
    Files.createDirectory(targetFolder);
    // create file and write some random content
    java.nio.file.Path targetFile = targetFolder.resolve(new AlphanumericRandomStringService().random());
    final byte[] content = RandomUtils.nextBytes(20);
    Files.write(targetFile, content);

    // read with Cyberduck and compare
    final Host host = new Host(new SFTPProtocol(), "localhost", PORT_NUMBER, new Credentials("empty", "empty"));
    final SFTPSession session = new SFTPSession(host, new DisabledX509TrustManager(), new DefaultX509KeyManager());
    session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback());
    session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback());
    final Path home = new SFTPHomeDirectoryService(session).find();
    final Path vault = new Path(home, "vault", EnumSet.of(Path.Type.directory));
    final CryptoVault cryptomator = new CryptoVault(vault).load(session, new DisabledPasswordCallback() {
        @Override
        public Credentials prompt(final Host bookmark, final String title, final String reason, final LoginOptions options) {
            return new VaultCredentials(passphrase);
        }
    }, new DisabledPasswordStore());
    session.withRegistry(new DefaultVaultRegistry(new DisabledPasswordStore(), new DisabledPasswordCallback(), cryptomator));
    Path p = new Path(new Path(vault, targetFolder.getFileName().toString(), EnumSet.of(Path.Type.directory)), targetFile.getFileName().toString(), EnumSet.of(Path.Type.file));
    final InputStream read = new CryptoReadFeature(session, new SFTPReadFeature(session), cryptomator).read(p, new TransferStatus(), new DisabledConnectionCallback());
    final byte[] readContent = new byte[content.length];
    IOUtils.readFully(read, readContent);
    assertArrayEquals(content, readContent);
    session.close();
}
 
Example 12
Source File: AES256JNCryptorInputStreamTest.java    From JNCryptor with Apache License 2.0 5 votes vote down vote up
/**
 * Test reading of mismatched data (e.g. encrypted using keys, decrypted with
 * password).
 * 
 * @throws Exception
 */
@Test(expected = IOException.class)
// TODO check for specific message
public void testUsingMismatchedKeys() throws Exception {

  byte[] plaintext = getRandomBytes(256);
  byte[] encryptionSalt = getRandomBytes(8);
  byte[] hmacSalt = getRandomBytes(8);

  final String password = "Testing1234";

  JNCryptor cryptor = new AES256JNCryptor();

  SecretKey hmacKey = cryptor
      .keyForPassword(password.toCharArray(), hmacSalt);
  SecretKey encryptionKey = cryptor.keyForPassword(password.toCharArray(),
      encryptionSalt);

  byte[] data = cryptor.encryptData(plaintext, encryptionKey, hmacKey);

  InputStream in = new AES256JNCryptorInputStream(new ByteArrayInputStream(
      data), password.toCharArray());

  try {

    byte[] result = new byte[256];
    IOUtils.readFully(in, result);

    assertArrayEquals(plaintext, result);
  } finally {
    in.close();
  }
}
 
Example 13
Source File: TestWebHDFS.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testWebHdfsOffsetAndLength() throws Exception{
  MiniDFSCluster cluster = null;
  final Configuration conf = WebHdfsTestUtil.createConf();
  final int OFFSET = 42;
  final int LENGTH = 512;
  final String PATH = "/foo";
  byte[] CONTENTS = new byte[1024];
  RANDOM.nextBytes(CONTENTS);
  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    final WebHdfsFileSystem fs =
        WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsFileSystem.SCHEME);
    try (OutputStream os = fs.create(new Path(PATH))) {
      os.write(CONTENTS);
    }
    InetSocketAddress addr = cluster.getNameNode().getHttpAddress();
    URL url = new URL("http", addr.getHostString(), addr
        .getPort(), WebHdfsFileSystem.PATH_PREFIX + PATH + "?op=OPEN" +
        Param.toSortedString("&", new OffsetParam((long) OFFSET),
                             new LengthParam((long) LENGTH))
    );
    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
    conn.setInstanceFollowRedirects(true);
    Assert.assertEquals(LENGTH, conn.getContentLength());
    byte[] subContents = new byte[LENGTH];
    byte[] realContents = new byte[LENGTH];
    System.arraycopy(CONTENTS, OFFSET, subContents, 0, LENGTH);
    IOUtils.readFully(conn.getInputStream(), realContents);
    Assert.assertArrayEquals(subContents, realContents);
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 
Example 14
Source File: S3Content.java    From engine with GNU General Public License v3.0 5 votes vote down vote up
public S3Content(final S3Object s3Object) {
    lastModified = s3Object.getObjectMetadata().getLastModified().getTime();
    length = s3Object.getObjectMetadata().getContentLength();
    content = new byte[(int)length];

    try(InputStream is = s3Object.getObjectContent()) {
        IOUtils.readFully(is, content);
    } catch (Exception e) {
        throw new StoreException("Error reading S3 item " + s3Object, e);
    }
}
 
Example 15
Source File: BaseTestQuery.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
public static boolean compareFiles(FileAttributes f1, FileAttributes f2) throws Exception {
  byte[] original = new byte[(int)f1.size()];
  byte[] withDict = new byte[(int)f2.size()];

  try (FSInputStream in1 = localFs.open(f1.getPath()); FSInputStream in2 = localFs.open(f2.getPath());) {
    IOUtils.readFully(in1, original, 0, original.length);
    IOUtils.readFully(in2, withDict, 0, withDict.length);
  }

  return Arrays.equals(original, withDict);
}
 
Example 16
Source File: BasicFormatMatcher.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
public boolean matches(FileSystem fs, FileAttributes attributes) throws IOException{
  if (ranges.isEmpty() || attributes.isDirectory()) {
    return false;
  }
  // walk all the way down in the symlinks until a hard entry is reached
  FileAttributes current = attributes;
  while (current.isSymbolicLink()) {
    current = fs.getFileAttributes(attributes.getSymbolicLink());
  }
  // if hard entry is not a file nor can it be a symlink then it is not readable simply deny matching.
  if (!current.isRegularFile()) {
    return false;
  }

  final Range<Long> fileRange = Range.closedOpen( 0L, attributes.size());

  try (FSInputStream is = fs.open(attributes.getPath())) {
    for(RangeMagics rMagic : ranges) {
      Range<Long> r = rMagic.range;
      if (!fileRange.encloses(r)) {
        continue;
      }
      int len = (int) (r.upperEndpoint() - r.lowerEndpoint());
      byte[] bytes = new byte[len];
      is.setPosition(r.lowerEndpoint());
      IOUtils.readFully(is, bytes);
      for (byte[] magic : rMagic.magics) {
        if (Arrays.equals(magic, bytes)) {
          return true;
        }
      }
    }
  }
  return false;
}
 
Example 17
Source File: AzureWriteFeatureTest.java    From cyberduck with GNU General Public License v3.0 4 votes vote down vote up
@Test
public void testWriteOverrideBlockBlob() throws Exception {
    final OperationContext context
            = new OperationContext();
    final Host host = new Host(new AzureProtocol(), "kahy9boj3eib.blob.core.windows.net", new Credentials(
            System.getProperties().getProperty("azure.account"), System.getProperties().getProperty("azure.key")
    ));
    final AzureSession session = new AzureSession(host);
    new LoginConnectionService(new DisabledLoginCallback(), new DisabledHostKeyCallback(),
            new DisabledPasswordStore(), new DisabledProgressListener()).connect(session, PathCache.empty(), new DisabledCancelCallback());
    final TransferStatus status = new TransferStatus();
    status.setMime("text/plain");
    final byte[] content = RandomUtils.nextBytes(513);
    status.setLength(content.length);
    status.setChecksum(new MD5ChecksumCompute().compute(new ByteArrayInputStream(content), new TransferStatus().length(content.length)));
    status.setMetadata(Collections.singletonMap("Cache-Control", "public,max-age=86400"));
    final Path container = new Path("cyberduck", EnumSet.of(Path.Type.directory, Path.Type.volume));
    final Path test = new Path(container, UUID.randomUUID().toString(), EnumSet.of(Path.Type.file));
    final OutputStream out = new AzureWriteFeature(session, BlobType.BLOCK_BLOB, context).write(test, status, new DisabledConnectionCallback());
    assertNotNull(out);
    new StreamCopier(new TransferStatus(), new TransferStatus()).transfer(new ByteArrayInputStream(content), out);
    assertTrue(new AzureFindFeature(session, context).find(test));
    final PathAttributes attributes = new AzureAttributesFinderFeature(session, context).find(test);
    assertEquals(content.length, attributes.getSize());
    final Map<String, String> metadata = new AzureMetadataFeature(session, context).getMetadata(test);
    assertEquals("text/plain", metadata.get("Content-Type"));
    assertEquals("public,max-age=86400", metadata.get("Cache-Control"));
    final Write.Append append = new AzureWriteFeature(session, context).append(test, status.getLength(), PathCache.empty());
    assertFalse(append.append);
    assertTrue(append.override);
    assertEquals(0L, append.size, 0L);
    final byte[] buffer = new byte[content.length];
    final InputStream in = new AzureReadFeature(session, context).read(test, new TransferStatus(), new DisabledConnectionCallback());
    IOUtils.readFully(in, buffer);
    in.close();
    assertArrayEquals(content, buffer);
    final OutputStream overwrite = new AzureWriteFeature(session, context).write(test, new TransferStatus().exists(true)
        .length("overwrite".getBytes(StandardCharsets.UTF_8).length).withMetadata(Collections.singletonMap("Content-Type", "text/plain")), new DisabledConnectionCallback());
    new StreamCopier(new TransferStatus(), new TransferStatus())
        .transfer(new ByteArrayInputStream("overwrite".getBytes(StandardCharsets.UTF_8)), overwrite);
    overwrite.close();
    // Test double close
    overwrite.close();
    assertEquals("overwrite".getBytes(StandardCharsets.UTF_8).length, new AzureAttributesFinderFeature(session, context).find(test).getSize());
    new AzureDeleteFeature(session, context).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
    session.close();
}
 
Example 18
Source File: S3MultipartUploadServiceTest.java    From cyberduck with GNU General Public License v3.0 4 votes vote down vote up
@Test
public void testAppendSecondPart() throws Exception {
    final Path container = new Path("test-us-east-1-cyberduck", EnumSet.of(Path.Type.directory, Path.Type.volume));
    final String name = UUID.randomUUID().toString();
    final Path test = new Path(container, name, EnumSet.of(Path.Type.file));
    final int length = 12 * 1024 * 1024;
    final byte[] content = RandomUtils.nextBytes(length);
    Local local = new Local(System.getProperty("java.io.tmpdir"), name);
    IOUtils.write(content, local.getOutputStream(false));
    final AtomicBoolean started = new AtomicBoolean();
    final TransferStatus status = new TransferStatus() {
        @Override
        public void progress(long bytes) {
            super.progress(bytes);
            started.set(true);
        }
    };
    status.setLength(content.length);
    final AtomicBoolean interrupt = new AtomicBoolean();
    try {
        new S3MultipartUploadService(session, new S3WriteFeature(session, new S3DisabledMultipartService()), 10L * 1024L * 1024L, 1).upload(test, new Local(System.getProperty("java.io.tmpdir"), name) {
                @Override
                public InputStream getInputStream() throws AccessDeniedException {
                    return new CountingInputStream(super.getInputStream()) {
                        @Override
                        protected void beforeRead(int n) throws IOException {
                            if(started.get()) {
                                if(this.getByteCount() >= 11L * 1024L * 1024L) {
                                    throw new IOException();
                                }
                            }
                        }
                    };
                }
            },
            new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), status,
            new DisabledLoginCallback());
    }
    catch(BackgroundException e) {
        // Expected
        interrupt.set(true);
    }
    assertTrue(interrupt.get());
    assertEquals(10L * 1024L * 1024L, status.getOffset(), 0L);
    assertFalse(status.isComplete());
    assertFalse(new S3FindFeature(session).find(test));

    final TransferStatus append = new TransferStatus().append(true).length(2L * 1024L * 1024L).skip(10L * 1024L * 1024L);
    new S3MultipartUploadService(session, new S3WriteFeature(session, new S3DisabledMultipartService()), 10L * 1024L * 1024L, 1).upload(test, local,
        new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), append,
        new DisabledConnectionCallback());
    assertEquals(12L * 1024L * 1024L, append.getOffset(), 0L);
    assertTrue(append.isComplete());
    assertTrue(new S3FindFeature(session).find(test));
    assertEquals(12L * 1024L * 1024L, new S3AttributesFinderFeature(session).find(test).getSize(), 0L);
    final byte[] buffer = new byte[content.length];
    final InputStream in = new S3ReadFeature(session).read(test, new TransferStatus(), new DisabledConnectionCallback());
    IOUtils.readFully(in, buffer);
    in.close();
    assertArrayEquals(content, buffer);
    new S3DefaultDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
    local.delete();
}
 
Example 19
Source File: PlatformPartitionDeserializer.java    From arcusplatform with Apache License 2.0 4 votes vote down vote up
@Override
public PlatformPartition deserialize(InputStream input) throws IOException, IllegalArgumentException {
   byte[] buffer = new byte[4];
   IOUtils.readFully(input, buffer, 0, 4);
   return deserialize(buffer);
}
 
Example 20
Source File: B2LargeUploadServiceTest.java    From cyberduck with GNU General Public License v3.0 4 votes vote down vote up
@Test
public void testAppendNoPartCompleted() throws Exception {
    final Path bucket = new Path("test-cyberduck", EnumSet.of(Path.Type.directory, Path.Type.volume));
    final Path test = new Path(bucket, UUID.randomUUID().toString(), EnumSet.of(Path.Type.file));
    final Local local = new Local(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString());
    final int length = 102 * 1024 * 1024;
    final byte[] content = RandomUtils.nextBytes(length);
    IOUtils.write(content, local.getOutputStream(false));
    final TransferStatus status = new TransferStatus();
    status.setLength(content.length);
    final AtomicBoolean interrupt = new AtomicBoolean();
    final B2FileidProvider fileid = new B2FileidProvider(session).withCache(cache);
    final B2LargeUploadService service = new B2LargeUploadService(session, fileid, new B2WriteFeature(session, fileid), 100 * 1024L * 1024L, 1);
    try {
        service.upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener() {
            long count;

            @Override
            public void sent(final long bytes) {
                count += bytes;
                if(count >= 5 * 1024L * 1024L) {
                    throw new RuntimeException();
                }
            }
        }, status, new DisabledLoginCallback());
    }
    catch(BackgroundException e) {
        // Expected
        interrupt.set(true);
    }
    assertTrue(interrupt.get());
    assertEquals(0L, status.getOffset(), 0L);
    assertFalse(status.isComplete());

    final TransferStatus append = new TransferStatus().append(true).length(content.length);
    service.upload(test, local,
            new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), append,
            new DisabledLoginCallback());
    assertTrue(new B2FindFeature(session, fileid).find(test));
    assertEquals(content.length, new B2AttributesFinderFeature(session, fileid).find(test).getSize());
    assertEquals(content.length, append.getOffset(), 0L);
    assertTrue(append.isComplete());
    final byte[] buffer = new byte[content.length];
    final InputStream in = new B2ReadFeature(session, fileid).read(test, new TransferStatus(), new DisabledConnectionCallback());
    IOUtils.readFully(in, buffer);
    in.close();
    assertArrayEquals(content, buffer);
    new B2DeleteFeature(session, fileid).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
    local.delete();
}