/*
*
* Copyright 2016 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package how.hollow.producer.infrastructure;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
import com.netflix.hollow.api.producer.HollowProducer.Blob;
import com.netflix.hollow.api.producer.HollowProducer.Publisher;
import com.netflix.hollow.core.memory.encoding.HashCodes;
import com.netflix.hollow.core.memory.encoding.VarInt;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class S3Publisher implements Publisher {
private final AmazonS3 s3;
private final TransferManager s3TransferManager;
private final String bucketName;
private final String blobNamespace;
private final List<Long> snapshotIndex;
public S3Publisher(AWSCredentials credentials, String bucketName, String blobNamespace) {
this.s3 = new AmazonS3Client(credentials);
this.s3TransferManager = new TransferManager(s3);
this.bucketName = bucketName;
this.blobNamespace = blobNamespace;
this.snapshotIndex = initializeSnapshotIndex();
}
@Override
public void publish(Blob blob) {
switch(blob.getType()) {
case SNAPSHOT:
publishSnapshot(blob);
break;
case DELTA:
publishDelta(blob);
break;
case REVERSE_DELTA:
publishReverseDelta(blob);
break;
}
}
public void publishSnapshot(Blob blob) {
String objectName = getS3ObjectName(blobNamespace, "snapshot", blob.getToVersion());
ObjectMetadata metadata = new ObjectMetadata();
metadata.addUserMetadata("to_state", String.valueOf(blob.getToVersion()));
metadata.setHeader("Content-Length", blob.getFile().length());
uploadFile(blob.getFile(), objectName, metadata);
/// now we update the snapshot index
updateSnapshotIndex(blob.getToVersion());
}
public void publishDelta(Blob blob) {
String objectName = getS3ObjectName(blobNamespace, "delta", blob.getFromVersion());
ObjectMetadata metadata = new ObjectMetadata();
metadata.addUserMetadata("from_state", String.valueOf(blob.getFromVersion()));
metadata.addUserMetadata("to_state", String.valueOf(blob.getToVersion()));
metadata.setHeader("Content-Length", blob.getFile().length());
uploadFile(blob.getFile(), objectName, metadata);
}
public void publishReverseDelta(Blob blob) {
String objectName = getS3ObjectName(blobNamespace, "reversedelta", blob.getFromVersion());
ObjectMetadata metadata = new ObjectMetadata();
metadata.addUserMetadata("from_state", String.valueOf(blob.getFromVersion()));
metadata.addUserMetadata("to_state", String.valueOf(blob.getToVersion()));
metadata.setHeader("Content-Length", blob.getFile().length());
uploadFile(blob.getFile(), objectName, metadata);
}
public static String getS3ObjectName(String blobNamespace, String fileType, long lookupVersion) {
StringBuilder builder = new StringBuilder(getS3ObjectPrefix(blobNamespace, fileType));
builder.append(Integer.toHexString(HashCodes.hashLong(lookupVersion)));
builder.append("-");
builder.append(lookupVersion);
return builder.toString();
}
private static String getS3ObjectPrefix(String blobNamespace, String fileType) {
StringBuilder builder = new StringBuilder(blobNamespace);
builder.append("/").append(fileType).append("/");
return builder.toString();
}
private void uploadFile(File file, String s3ObjectName, ObjectMetadata metadata) {
try (InputStream is = new BufferedInputStream(new FileInputStream(file))) {
Upload upload = s3TransferManager.upload(bucketName, s3ObjectName, is, metadata);
upload.waitForCompletion();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/////////////////////// BEGIN SNAPSHOT INDEX CODE ///////////////////////
/*
* We need an index over the available state versions for which snapshot blobs are available.
* The S3Publisher stores that index as an object with a known key in S3.
* The remainder of this class deals with maintaining that index.
*/
public static String getSnapshotIndexObjectName(String blobNamespace) {
return blobNamespace + "/snapshot.index";
}
/**
* Write a list of all of the state versions to S3.
* @param newVersion
*/
private synchronized void updateSnapshotIndex(Long newVersion) {
/// insert the new version into the list
int idx = Collections.binarySearch(snapshotIndex, newVersion);
int insertionPoint = Math.abs(idx) - 1;
snapshotIndex.add(insertionPoint, newVersion);
/// build a binary representation of the list -- gap encoded variable-length integers
byte[] idxBytes = buidGapEncodedVarIntSnapshotIndex();
/// indicate the Content-Length
ObjectMetadata metadata = new ObjectMetadata();
metadata.setHeader("Content-Length", (long)idxBytes.length);
/// upload the new file content.
try(InputStream is = new ByteArrayInputStream(idxBytes)) {
Upload upload = s3TransferManager.upload(bucketName, getSnapshotIndexObjectName(blobNamespace), is, metadata);
upload.waitForCompletion();
} catch(Exception e) {
throw new RuntimeException(e);
}
}
/**
* Encode the sorted list of all state versions as gap-encoded variable length integers.
* @return
*/
private byte[] buidGapEncodedVarIntSnapshotIndex() {
int idx;
byte[] idxBytes;
idx = 0;
long currentSnapshotId = snapshotIndex.get(idx++);
long currentSnapshotIdGap = currentSnapshotId;
try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {
while(idx < snapshotIndex.size()) {
VarInt.writeVLong(os, currentSnapshotIdGap);
long nextSnapshotId = snapshotIndex.get(idx++);
currentSnapshotIdGap = nextSnapshotId - currentSnapshotId;
currentSnapshotId = nextSnapshotId;
}
VarInt.writeVLong(os, currentSnapshotIdGap);
idxBytes = os.toByteArray();
} catch(IOException shouldNotHappen) {
throw new RuntimeException(shouldNotHappen);
}
return idxBytes;
}
/**
* Find all of the existing snapshots.
*/
private List<Long> initializeSnapshotIndex() {
List<Long> snapshotIdx = new ArrayList<Long>();
ObjectListing listObjects = s3.listObjects(bucketName, getS3ObjectPrefix(blobNamespace, "snapshot"));
for (S3ObjectSummary summary : listObjects.getObjectSummaries())
addSnapshotStateId(summary, snapshotIdx);
while (listObjects.isTruncated()) {
listObjects = s3.listNextBatchOfObjects(listObjects);
for (S3ObjectSummary summary : listObjects.getObjectSummaries())
addSnapshotStateId(summary, snapshotIdx);
}
Collections.sort(snapshotIdx);
return snapshotIdx;
}
private void addSnapshotStateId(S3ObjectSummary obj, List<Long> snapshotIdx) {
String key = obj.getKey();
try {
snapshotIdx.add(Long.parseLong(key.substring(key.lastIndexOf("-") + 1)));
} catch(NumberFormatException ignore) { }
}
}
