org.jets3t.service.model.S3Bucket Java Examples

The following examples show how to use org.jets3t.service.model.S3Bucket. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: Jets3tFileSystemStore.java From hadoop with Apache License 2.0

6 votes

@Override
public void initialize(URI uri, Configuration conf) throws IOException {
  
  this.conf = conf;
  
  S3Credentials s3Credentials = new S3Credentials();
  s3Credentials.initialize(uri, conf);
  try {
    AWSCredentials awsCredentials =
      new AWSCredentials(s3Credentials.getAccessKey(),
          s3Credentials.getSecretAccessKey());
    this.s3Service = new RestS3Service(awsCredentials);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  bucket = new S3Bucket(uri.getHost());

  this.bufferSize = conf.getInt(
                     S3FileSystemConfigKeys.S3_STREAM_BUFFER_SIZE_KEY,
                     S3FileSystemConfigKeys.S3_STREAM_BUFFER_SIZE_DEFAULT
      );
}

Example #2

Source File: Jets3tFileSystemStore.java From hadoop-gpu with Apache License 2.0

6 votes

public void initialize(URI uri, Configuration conf) throws IOException {
  
  this.conf = conf;
  
  S3Credentials s3Credentials = new S3Credentials();
  s3Credentials.initialize(uri, conf);
  try {
    AWSCredentials awsCredentials =
      new AWSCredentials(s3Credentials.getAccessKey(),
          s3Credentials.getSecretAccessKey());
    this.s3Service = new RestS3Service(awsCredentials);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  bucket = new S3Bucket(uri.getHost());

  this.bufferSize = conf.getInt("io.file.buffer.size", 4096);
}

Example #3

Source File: Jets3tNativeFileSystemStore.java From big-c with Apache License 2.0

6 votes

@Override
public void initialize(URI uri, Configuration conf) throws IOException {
  S3Credentials s3Credentials = new S3Credentials();
  s3Credentials.initialize(uri, conf);
  try {
    AWSCredentials awsCredentials =
      new AWSCredentials(s3Credentials.getAccessKey(),
          s3Credentials.getSecretAccessKey());
    this.s3Service = new RestS3Service(awsCredentials);
  } catch (S3ServiceException e) {
    handleException(e);
  }
  multipartEnabled =
      conf.getBoolean("fs.s3n.multipart.uploads.enabled", false);
  multipartBlockSize = Math.min(
      conf.getLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024),
      MAX_PART_SIZE);
  multipartCopyBlockSize = Math.min(
      conf.getLong("fs.s3n.multipart.copy.block.size", MAX_PART_SIZE),
      MAX_PART_SIZE);
  serverSideEncryptionAlgorithm = conf.get("fs.s3n.server-side-encryption-algorithm");

  bucket = new S3Bucket(uri.getHost());
}

Example #4

Source File: Jets3tFileSystemStore.java From big-c with Apache License 2.0

6 votes

@Override
public void initialize(URI uri, Configuration conf) throws IOException {
  
  this.conf = conf;
  
  S3Credentials s3Credentials = new S3Credentials();
  s3Credentials.initialize(uri, conf);
  try {
    AWSCredentials awsCredentials =
      new AWSCredentials(s3Credentials.getAccessKey(),
          s3Credentials.getSecretAccessKey());
    this.s3Service = new RestS3Service(awsCredentials);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  bucket = new S3Bucket(uri.getHost());

  this.bufferSize = conf.getInt(
                     S3FileSystemConfigKeys.S3_STREAM_BUFFER_SIZE_KEY,
                     S3FileSystemConfigKeys.S3_STREAM_BUFFER_SIZE_DEFAULT
      );
}

Example #5

Source File: Jets3tNativeFileSystemStore.java From hadoop-gpu with Apache License 2.0

6 votes

public void initialize(URI uri, Configuration conf) throws IOException {
  S3Credentials s3Credentials = new S3Credentials();
  s3Credentials.initialize(uri, conf);
  try {
    AWSCredentials awsCredentials =
      new AWSCredentials(s3Credentials.getAccessKey(),
          s3Credentials.getSecretAccessKey());
    this.s3Service = new RestS3Service(awsCredentials);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  bucket = new S3Bucket(uri.getHost());
}

Example #6

Source File: Jets3tNativeFileSystemStore.java From hadoop with Apache License 2.0

6 votes

@Override
public void initialize(URI uri, Configuration conf) throws IOException {
  S3Credentials s3Credentials = new S3Credentials();
  s3Credentials.initialize(uri, conf);
  try {
    AWSCredentials awsCredentials =
      new AWSCredentials(s3Credentials.getAccessKey(),
          s3Credentials.getSecretAccessKey());
    this.s3Service = new RestS3Service(awsCredentials);
  } catch (S3ServiceException e) {
    handleException(e);
  }
  multipartEnabled =
      conf.getBoolean("fs.s3n.multipart.uploads.enabled", false);
  multipartBlockSize = Math.min(
      conf.getLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024),
      MAX_PART_SIZE);
  multipartCopyBlockSize = Math.min(
      conf.getLong("fs.s3n.multipart.copy.block.size", MAX_PART_SIZE),
      MAX_PART_SIZE);
  serverSideEncryptionAlgorithm = conf.get("fs.s3n.server-side-encryption-algorithm");

  bucket = new S3Bucket(uri.getHost());
}

Example #7

Source File: S3FilenameGenerator.java From red5-examples with Apache License 2.0

6 votes

public static List<String> getBucketList() {
	logger.debug("Get the bucket list");
	List<String> bucketList = new ArrayList<String>(3);
	try {
		S3Service s3Service = new RestS3Service(awsCredentials); 
		S3Bucket[] buckets = s3Service.listAllBuckets();
		for (S3Bucket bucket: buckets) {
			logger.debug("Bucket: {}", bucket.getName());
			bucketList.add(bucket.getName());
		}
		logger.debug("Bucket count: {}", buckets.length);
	} catch (S3ServiceException e) {
		logger.error("Error during bucket listing", e);
	}
	return bucketList;
}

Example #8

Source File: JetS3tLiveTest.java From tutorials with MIT License

6 votes

@Test
public void givenStringUploaded_StringIsDownloaded() throws Exception {

    // Get a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    uploadStringData();

    // Download
    S3Object stringObject = s3Service.getObject(BucketName, TestStringName);

    // Process stream into a string
    String downloadedString = new BufferedReader(new InputStreamReader(stringObject.getDataInputStream())).lines().collect(Collectors.joining("\n"));

    // Verify
    assertTrue(TestString.equals(downloadedString));


    // Clean up for next test
    deleteObject(TestStringName);
    deleteBucket();
}

Example #9

Source File: Jets3tNativeFileSystemStore.java From RDFS with Apache License 2.0

6 votes

public void initialize(URI uri, Configuration conf) throws IOException {
  S3Credentials s3Credentials = new S3Credentials();
  s3Credentials.initialize(uri, conf);
  try {
    AWSCredentials awsCredentials =
      new AWSCredentials(s3Credentials.getAccessKey(),
          s3Credentials.getSecretAccessKey());
    this.s3Service = new RestS3Service(awsCredentials);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  bucket = new S3Bucket(uri.getHost());
}

Example #10

Source File: Jets3tFileSystemStore.java From RDFS with Apache License 2.0

6 votes

public void initialize(URI uri, Configuration conf) throws IOException {
  
  this.conf = conf;
  
  S3Credentials s3Credentials = new S3Credentials();
  s3Credentials.initialize(uri, conf);
  try {
    AWSCredentials awsCredentials =
      new AWSCredentials(s3Credentials.getAccessKey(),
          s3Credentials.getSecretAccessKey());
    this.s3Service = new RestS3Service(awsCredentials);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  bucket = new S3Bucket(uri.getHost());

  this.bufferSize = conf.getInt("io.file.buffer.size", 4096);
}

Example #11

Source File: JetS3tLiveTest.java From tutorials with MIT License

6 votes

@Test
public void givenStringUploaded_StringIsDownloaded() throws Exception {

    // Get a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    uploadStringData();

    // Download
    S3Object stringObject = s3Service.getObject(BucketName, TestStringName);

    // Process stream into a string
    String downloadedString = new BufferedReader(new InputStreamReader(stringObject.getDataInputStream())).lines().collect(Collectors.joining("\n"));

    // Verify
    assertTrue(TestString.equals(downloadedString));


    // Clean up for next test
    deleteObject(TestStringName);
    deleteBucket();
}

Example #12

Source File: JetS3tLiveTest.java From tutorials with MIT License

6 votes

@Test
public void givenString_Uploaded_StringInfoIsAvailable() throws Exception {

    // Create a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    // Upload a string
    uploadStringData();

    // Get the details
    StorageObject objectDetailsOnly = s3Service.getObjectDetails(BucketName, TestStringName);
    log.info("Content type: " + objectDetailsOnly.getContentType() + " length: " + objectDetailsOnly.getContentLength());

    // Delete it
    deleteObject(TestStringName);

    // For next test
    deleteBucket();
}

Example #13

Source File: JetS3tLiveTest.java From tutorials with MIT License

6 votes

@Test
public void givenString_Uploaded_StringInfoIsAvailable() throws Exception {

    // Create a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    // Upload a string
    uploadStringData();

    // Get the details
    StorageObject objectDetailsOnly = s3Service.getObjectDetails(BucketName, TestStringName);
    log.info("Content type: " + objectDetailsOnly.getContentType() + " length: " + objectDetailsOnly.getContentLength());

    // Delete it
    deleteObject(TestStringName);

    // For next test
    deleteBucket();
}

Example #14

Source File: JetS3tLiveTest.java From tutorials with MIT License

5 votes

@Test
public void whenFileRenamed_NewNameIsSame() throws Exception {

    // get a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    // Put a binary file
    S3Object fileObject = new S3Object(new File("src/test/resources/test.jpg"));
    s3Service.putObject(BucketName, fileObject);


    // Copy it
    s3Service.renameObject(BucketName, "test.jpg", new S3Object("spidey.jpg"));


    // Download
    S3Object newFileObject = s3Service.getObject(BucketName, "spidey.jpg");

    // Save to a different name
    File newFile = new File("src/test/resources/spidey.jpg");
    Files.copy(newFileObject.getDataInputStream(), newFile.toPath(), REPLACE_EXISTING);


    // Get hashes and compare
    String origMD5 = getFileMD5("src/test/resources/test.jpg");
    String newMD5 = getFileMD5("src/test/resources/spidey.jpg");
    assertTrue(origMD5.equals(newMD5));

    // Clean up
    deleteObject("test.jpg");
    deleteObject("spidey.jpg");
    deleteBucket();

}

Example #15

Source File: JetS3tLiveTest.java From tutorials with MIT License

5 votes

@Test
public void givenBinaryFileUploaded_FileIsDownloaded() throws Exception {

    // get a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    // Put a binary file
    S3Object fileObject = new S3Object(new File("src/test/resources/test.jpg"));
    s3Service.putObject(BucketName, fileObject);

    // Print info about type and name
    log.info("Content type:" + fileObject.getContentType());
    log.info("File object name is " + fileObject.getName());

    // Download
    S3Object newFileObject = s3Service.getObject(BucketName, "test.jpg");

    // Save to a different name
    File newFile = new File("src/test/resources/newtest.jpg");
    Files.copy(newFileObject.getDataInputStream(), newFile.toPath(), REPLACE_EXISTING);


    // Get hashes and compare
    String origMD5 = getFileMD5("src/test/resources/test.jpg");
    String newMD5 = getFileMD5("src/test/resources/newtest.jpg");
    assertTrue(origMD5.equals(newMD5));

    // Clean up
    deleteObject("test.jpg");
    deleteBucket();
}

Example #16

Source File: JetS3tLiveTest.java From tutorials with MIT License

5 votes

@Test
public void givenCreate_AndDeleteBucket_CountGoesUpThenDown() throws Exception {

    // List buckets, get a count
    S3Bucket[] myBuckets = s3Service.listAllBuckets();
    int count = Arrays.stream(myBuckets).map(S3Bucket::getName).collect(Collectors.toList()).size();

    // Create a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    // List again
    myBuckets = s3Service.listAllBuckets();
    int newCount = Arrays.stream(myBuckets).map(S3Bucket::getName).collect(Collectors.toList()).size();

    // We should have one more
    assertEquals((count + 1), newCount);

    // Delete so next test doesn't fail
    deleteBucket();

    // Check the count again, just for laughs
    myBuckets = s3Service.listAllBuckets();
    newCount = Arrays.stream(myBuckets).map(S3Bucket::getName).collect(Collectors.toList()).size();
    assertEquals(count, newCount);

}

Example #17

Source File: JetS3tLiveTest.java From tutorials with MIT License

5 votes

@Test
public void givenBinaryFileUploaded_FileIsDownloaded() throws Exception {

    // get a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    // Put a binary file
    S3Object fileObject = new S3Object(new File("src/test/resources/test.jpg"));
    s3Service.putObject(BucketName, fileObject);

    // Print info about type and name
    log.info("Content type:" + fileObject.getContentType());
    log.info("File object name is " + fileObject.getName());

    // Download
    S3Object newFileObject = s3Service.getObject(BucketName, "test.jpg");

    // Save to a different name
    File newFile = new File("src/test/resources/newtest.jpg");
    Files.copy(newFileObject.getDataInputStream(), newFile.toPath(), REPLACE_EXISTING);


    // Get hashes and compare
    String origMD5 = getFileMD5("src/test/resources/test.jpg");
    String newMD5 = getFileMD5("src/test/resources/newtest.jpg");
    assertTrue(origMD5.equals(newMD5));

    // Clean up
    deleteObject("test.jpg");
    deleteBucket();
}

Example #18

Source File: JetS3tLiveTest.java From tutorials with MIT License

5 votes

@Test
public void whenFileCopied_CopyIsSame() throws Exception {

    // get a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    // Put a binary file
    S3Object fileObject = new S3Object(new File("src/test/resources/test.jpg"));
    s3Service.putObject(BucketName, fileObject);


    // Copy it
    S3Object targetObject = new S3Object("testcopy.jpg");
    s3Service.copyObject(BucketName, "test.jpg", BucketName, targetObject, false);


    // Download
    S3Object newFileObject = s3Service.getObject(BucketName, "testcopy.jpg");

    // Save to a different name
    File newFile = new File("src/test/resources/testcopy.jpg");
    Files.copy(newFileObject.getDataInputStream(), newFile.toPath(), REPLACE_EXISTING);


    // Get hashes and compare
    String origMD5 = getFileMD5("src/test/resources/test.jpg");
    String newMD5 = getFileMD5("src/test/resources/testcopy.jpg");
    assertTrue(origMD5.equals(newMD5));

    // Clean up
    deleteObject("test.jpg");
    deleteObject("testcopy.jpg");
    deleteBucket();

}

Example #19

Source File: JetS3tLiveTest.java From tutorials with MIT License

5 votes

@Test
public void whenFileRenamed_NewNameIsSame() throws Exception {

    // get a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    // Put a binary file
    S3Object fileObject = new S3Object(new File("src/test/resources/test.jpg"));
    s3Service.putObject(BucketName, fileObject);


    // Copy it
    s3Service.renameObject(BucketName, "test.jpg", new S3Object("spidey.jpg"));


    // Download
    S3Object newFileObject = s3Service.getObject(BucketName, "spidey.jpg");

    // Save to a different name
    File newFile = new File("src/test/resources/spidey.jpg");
    Files.copy(newFileObject.getDataInputStream(), newFile.toPath(), REPLACE_EXISTING);


    // Get hashes and compare
    String origMD5 = getFileMD5("src/test/resources/test.jpg");
    String newMD5 = getFileMD5("src/test/resources/spidey.jpg");
    assertTrue(origMD5.equals(newMD5));

    // Clean up
    deleteObject("test.jpg");
    deleteObject("spidey.jpg");
    deleteBucket();

}

Example #20

Source File: JetS3tLiveTest.java From tutorials with MIT License

5 votes

@Test
public void whenFileCopied_CopyIsSame() throws Exception {

    // get a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    // Put a binary file
    S3Object fileObject = new S3Object(new File("src/test/resources/test.jpg"));
    s3Service.putObject(BucketName, fileObject);


    // Copy it
    S3Object targetObject = new S3Object("testcopy.jpg");
    s3Service.copyObject(BucketName, "test.jpg", BucketName, targetObject, false);


    // Download
    S3Object newFileObject = s3Service.getObject(BucketName, "testcopy.jpg");

    // Save to a different name
    File newFile = new File("src/test/resources/testcopy.jpg");
    Files.copy(newFileObject.getDataInputStream(), newFile.toPath(), REPLACE_EXISTING);


    // Get hashes and compare
    String origMD5 = getFileMD5("src/test/resources/test.jpg");
    String newMD5 = getFileMD5("src/test/resources/testcopy.jpg");
    assertTrue(origMD5.equals(newMD5));

    // Clean up
    deleteObject("test.jpg");
    deleteObject("testcopy.jpg");
    deleteBucket();

}

Example #21

Source File: JetS3tLiveTest.java From tutorials with MIT License

5 votes

@Test
public void givenCreate_AndDeleteBucket_CountGoesUpThenDown() throws Exception {

    // List buckets, get a count
    S3Bucket[] myBuckets = s3Service.listAllBuckets();
    int count = Arrays.stream(myBuckets).map(S3Bucket::getName).collect(Collectors.toList()).size();

    // Create a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    // List again
    myBuckets = s3Service.listAllBuckets();
    int newCount = Arrays.stream(myBuckets).map(S3Bucket::getName).collect(Collectors.toList()).size();

    // We should have one more
    assertEquals((count + 1), newCount);

    // Delete so next test doesn't fail
    deleteBucket();

    // Check the count again, just for laughs
    myBuckets = s3Service.listAllBuckets();
    newCount = Arrays.stream(myBuckets).map(S3Bucket::getName).collect(Collectors.toList()).size();
    assertEquals(count, newCount);

}

Example #22

Source File: S3FilenameGenerator.java From red5-examples with Apache License 2.0

5 votes

public static void upload(String sessionId, String name) {
	logger.debug("Upload - session id: {} name: {}", sessionId, name);
	try {
		// find the file
		StringBuilder sb = new StringBuilder(recordPath);
		sb.append(sessionId);
		sb.append('/');
		sb.append(name);
		sb.append(".flv");
		String filePath = sb.toString();
		logger.debug("File path: {}", filePath);
		File file = new File(filePath);
		if (file.exists()) {
			S3Service s3Service = new RestS3Service(awsCredentials);
			S3Bucket bucket = s3Service.createBucket(bucketName);
			S3Object sob = new S3Object(sessionId + "/" + name + ".flv");
			// force bucket name
			sob.setBucketName(bucketName);
			// point at file
			sob.setDataInputFile(file);
			// set type
			sob.setContentType("video/x-flv");
			// set auth / acl
			sob.setAcl(AccessControlList.REST_CANNED_PUBLIC_READ);				
			logger.debug("Pre-upload: {}", sob);
			sob = s3Service.putObject(bucket, sob);
			logger.debug("Post-upload: {}", sob);						
		} else {
			logger.warn("File was not found");
		}
		file = null;
	} catch (S3ServiceException e) {
		logger.error("Error during upload", e);
	}		
}

Example #23

Source File: S3FilenameGenerator.java From red5-examples with Apache License 2.0

5 votes

public static void createBucket() {
	logger.debug("Create bucket");
	try {
		S3Service s3Service = new RestS3Service(awsCredentials);
		S3Bucket bucket = s3Service.createBucket(bucketName);
		logger.debug("Created bucket: {}", bucket.getName());
	} catch (S3ServiceException e) {
		logger.error("Error creating bucket", e);
	}		
}

Example #24

Source File: JetS3tLiveTest.java From tutorials with MIT License

4 votes

@Test
public void givenStreamDataUploaded_StreamDataIsDownloaded() throws Exception {

    // get a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    ArrayList<Integer> numbers = new ArrayList<>();
    numbers.add(2);
    numbers.add(3);
    numbers.add(5);
    numbers.add(7);

    // Serialize ArrayList
    ByteArrayOutputStream bytes = new ByteArrayOutputStream();
    ObjectOutputStream objectOutputStream = new ObjectOutputStream(bytes);
    objectOutputStream.writeObject(numbers);

    // Wrap bytes
    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes.toByteArray());

    // Create and populate object
    S3Object streamObject = new S3Object("stream");
    streamObject.setDataInputStream(byteArrayInputStream);
    streamObject.setContentLength(byteArrayInputStream.available());
    streamObject.setContentType("binary/octet-stream");

    // Put it
    s3Service.putObject(BucketName, streamObject);

    // Get it
    S3Object newStreamObject = s3Service.getObject(BucketName, "stream");

    // Convert back to ArrayList
    ObjectInputStream objectInputStream = new ObjectInputStream(newStreamObject.getDataInputStream());
    ArrayList<Integer> newNumbers = (ArrayList<Integer>)objectInputStream.readObject();

    assertEquals(2, (int)newNumbers.get(0));
    assertEquals(3, (int)newNumbers.get(1));
    assertEquals(5, (int)newNumbers.get(2));
    assertEquals(7, (int)newNumbers.get(3));

    // Clean up
    deleteObject("stream");
    deleteBucket();
}

Example #25

Source File: JetS3tLiveTest.java From tutorials with MIT License

4 votes

private S3Bucket createBucket() throws Exception {
    S3Bucket bucket = s3Service.createBucket(BucketName);
    log.info(bucket);
    return bucket;
}

Example #26

Source File: JetS3tLiveTest.java From tutorials with MIT License

4 votes

private S3Bucket createBucket() throws Exception {
    S3Bucket bucket = s3Service.createBucket(BucketName);
    log.info(bucket);
    return bucket;
}

Example #27

Source File: MigrationTool.java From RDFS with Apache License 2.0

4 votes

public void initialize(URI uri) throws IOException {
  
  
  
  try {
    String accessKey = null;
    String secretAccessKey = null;
    String userInfo = uri.getUserInfo();
    if (userInfo != null) {
      int index = userInfo.indexOf(':');
      if (index != -1) {
        accessKey = userInfo.substring(0, index);
        secretAccessKey = userInfo.substring(index + 1);
      } else {
        accessKey = userInfo;
      }
    }
    if (accessKey == null) {
      accessKey = getConf().get("fs.s3.awsAccessKeyId");
    }
    if (secretAccessKey == null) {
      secretAccessKey = getConf().get("fs.s3.awsSecretAccessKey");
    }
    if (accessKey == null && secretAccessKey == null) {
      throw new IllegalArgumentException("AWS " +
                                         "Access Key ID and Secret Access Key " +
                                         "must be specified as the username " +
                                         "or password (respectively) of a s3 URL, " +
                                         "or by setting the " +
                                         "fs.s3.awsAccessKeyId or " +                         
                                         "fs.s3.awsSecretAccessKey properties (respectively).");
    } else if (accessKey == null) {
      throw new IllegalArgumentException("AWS " +
                                         "Access Key ID must be specified " +
                                         "as the username of a s3 URL, or by setting the " +
                                         "fs.s3.awsAccessKeyId property.");
    } else if (secretAccessKey == null) {
      throw new IllegalArgumentException("AWS " +
                                         "Secret Access Key must be specified " +
                                         "as the password of a s3 URL, or by setting the " +
                                         "fs.s3.awsSecretAccessKey property.");         
    }
    AWSCredentials awsCredentials =
      new AWSCredentials(accessKey, secretAccessKey);
    this.s3Service = new RestS3Service(awsCredentials);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  bucket = new S3Bucket(uri.getHost());
}

Example #28

Source File: JetS3tLiveTest.java From tutorials with MIT License

4 votes

@Test
public void givenStreamDataUploaded_StreamDataIsDownloaded() throws Exception {

    // get a bucket
    S3Bucket bucket = createBucket();
    assertNotNull(bucket);

    ArrayList<Integer> numbers = new ArrayList<>();
    numbers.add(2);
    numbers.add(3);
    numbers.add(5);
    numbers.add(7);

    // Serialize ArrayList
    ByteArrayOutputStream bytes = new ByteArrayOutputStream();
    ObjectOutputStream objectOutputStream = new ObjectOutputStream(bytes);
    objectOutputStream.writeObject(numbers);

    // Wrap bytes
    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes.toByteArray());

    // Create and populate object
    S3Object streamObject = new S3Object("stream");
    streamObject.setDataInputStream(byteArrayInputStream);
    streamObject.setContentLength(byteArrayInputStream.available());
    streamObject.setContentType("binary/octet-stream");

    // Put it
    s3Service.putObject(BucketName, streamObject);

    // Get it
    S3Object newStreamObject = s3Service.getObject(BucketName, "stream");

    // Convert back to ArrayList
    ObjectInputStream objectInputStream = new ObjectInputStream(newStreamObject.getDataInputStream());
    ArrayList<Integer> newNumbers = (ArrayList<Integer>)objectInputStream.readObject();

    assertEquals(2, (int)newNumbers.get(0));
    assertEquals(3, (int)newNumbers.get(1));
    assertEquals(5, (int)newNumbers.get(2));
    assertEquals(7, (int)newNumbers.get(3));

    // Clean up
    deleteObject("stream");
    deleteBucket();
}

Example #29

Source File: S3FilenameGenerator.java From red5-examples with Apache License 2.0

4 votes

@SuppressWarnings("deprecation")
public String generateFilename(IScope scope, String name, String extension, GenerationType type) {
   	logger.debug("Get stream directory: scope={}, name={}, type={}", new Object[]{scope, name, type.toString()});		
	StringBuilder path = new StringBuilder();
	// get the session id
	IConnection conn = Red5.getConnectionLocal();
	if (conn.hasAttribute("sessionId")) {
		String sessionId = conn.getStringAttribute("sessionId");
		path.append(sessionId);
		path.append('/');
	}
	// add resources name
	path.append(name);
	// add extension if we have one
       if (extension != null){
           // add extension
       	path.append(extension);
       }		
	// determine whether its playback or record
	if (type.equals(GenerationType.PLAYBACK)) {
		logger.debug("Playback path used");
		// look on s3 for the file first	
		boolean found = false;
		try {
			S3Service s3Service = new RestS3Service(awsCredentials);
			S3Bucket bucket = s3Service.getBucket(bucketName);
			String objectKey = path.toString();
			S3Object file = s3Service.getObject(bucket, objectKey);
			if (file != null) {
				S3Object details = s3Service.getObjectDetails(bucket, objectKey);
				logger.debug("Details - key: {} content type: {}", details.getKey(), details.getContentType()); 
				path.insert(0, bucket.getLocation());
				// set found flag
				found = true;
			}
		} catch (S3ServiceException e) {
			logger.warn("Error looking up media file", e);
		}
		// use local path
		if (!found) {
			logger.debug("File was not found on S3, using local playback location");
			path.insert(0, playbackPath);
		}
	} else {
		logger.debug("Record path used");
		path.insert(0, recordPath);
	}

       String fileName = path.toString();
       logger.debug("Generated filename: {}", fileName);
       return fileName;
}

Example #30

Source File: MigrationTool.java From big-c with Apache License 2.0

4 votes

public void initialize(URI uri) throws IOException {
  
  
  
  try {
    String accessKey = null;
    String secretAccessKey = null;
    String userInfo = uri.getUserInfo();
    if (userInfo != null) {
      int index = userInfo.indexOf(':');
      if (index != -1) {
        accessKey = userInfo.substring(0, index);
        secretAccessKey = userInfo.substring(index + 1);
      } else {
        accessKey = userInfo;
      }
    }
    if (accessKey == null) {
      accessKey = getConf().get("fs.s3.awsAccessKeyId");
    }
    if (secretAccessKey == null) {
      secretAccessKey = getConf().get("fs.s3.awsSecretAccessKey");
    }
    if (accessKey == null && secretAccessKey == null) {
      throw new IllegalArgumentException("AWS " +
                                         "Access Key ID and Secret Access Key " +
                                         "must be specified as the username " +
                                         "or password (respectively) of a s3 URL, " +
                                         "or by setting the " +
                                         "fs.s3.awsAccessKeyId or " +                         
                                         "fs.s3.awsSecretAccessKey properties (respectively).");
    } else if (accessKey == null) {
      throw new IllegalArgumentException("AWS " +
                                         "Access Key ID must be specified " +
                                         "as the username of a s3 URL, or by setting the " +
                                         "fs.s3.awsAccessKeyId property.");
    } else if (secretAccessKey == null) {
      throw new IllegalArgumentException("AWS " +
                                         "Secret Access Key must be specified " +
                                         "as the password of a s3 URL, or by setting the " +
                                         "fs.s3.awsSecretAccessKey property.");         
    }
    AWSCredentials awsCredentials =
      new AWSCredentials(accessKey, secretAccessKey);
    this.s3Service = new RestS3Service(awsCredentials);
  } catch (S3ServiceException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    }
    throw new S3Exception(e);
  }
  bucket = new S3Bucket(uri.getHost());
}