#
# Copyright (c) 2015 Matthew Bentley
#
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
import os
import hashlib
import duplicity.backend
from duplicity.errors import BackendException, FatalBackendException
from duplicity import log
import json
import urllib2
import base64
class B2Backend(duplicity.backend.Backend):
"""
Backend for BackBlaze's B2 storage service
"""
def __init__(self, parsed_url):
"""
Authorize to B2 api and set up needed variables
"""
duplicity.backend.Backend.__init__(self, parsed_url)
# for prettier password prompt only
self.parsed_url.hostname = 'B2'
self.account_id = parsed_url.username
account_key = self.get_password()
self.url_parts = [
x for x in parsed_url.path.replace("@", "/").split('/') if x != ''
]
if self.url_parts:
self.username = self.url_parts.pop(0)
self.bucket_name = self.url_parts.pop(0)
else:
raise BackendException("B2 requires a bucket name")
self.path = "/".join(self.url_parts)
self.id_and_key = self.account_id + ":" + account_key
self._authorize()
try:
self.find_or_create_bucket(self.bucket_name)
except urllib2.HTTPError:
raise FatalBackendException("Bucket cannot be created")
def _authorize(self):
basic_auth_string = 'Basic ' + base64.b64encode(self.id_and_key)
headers = {'Authorization': basic_auth_string}
request = urllib2.Request(
'https://api.backblazeb2.com/b2api/v1/b2_authorize_account',
headers=headers
)
response = urllib2.urlopen(request)
response_data = json.loads(response.read())
response.close()
self.auth_token = response_data['authorizationToken']
self.api_url = response_data['apiUrl']
self.download_url = response_data['downloadUrl']
def _get(self, remote_filename, local_path):
"""
Download remote_filename to local_path
"""
log.Log("Getting file %s" % remote_filename, 9)
remote_filename = self.full_filename(remote_filename)
url = self.download_url + \
'/file/' + self.bucket_name + '/' + \
remote_filename
resp = self.get_or_post(url, None)
to_file = open(local_path.name, 'wb')
to_file.write(resp)
to_file.close()
def _put(self, source_path, remote_filename):
"""
Copy source_path to remote_filename
"""
log.Log("Putting file to %s" % remote_filename, 9)
self._delete(remote_filename)
digest = self.hex_sha1_of_file(source_path)
content_type = 'application/pgp-encrypted'
remote_filename = self.full_filename(remote_filename)
info = self.get_upload_info(self.bucket_id)
url = info['uploadUrl']
headers = {
'Authorization': info['authorizationToken'],
'X-Bz-File-Name': remote_filename,
'Content-Type': content_type,
'X-Bz-Content-Sha1': digest,
'Content-Length': str(os.path.getsize(source_path.name)),
}
data_file = source_path.open()
self.get_or_post(url, None, headers, data_file=data_file)
def _list(self):
"""
List files on remote server
"""
log.Log("Listing files", 9)
endpoint = 'b2_list_file_names'
url = self.formatted_url(endpoint)
params = {
'bucketId': self.bucket_id,
'maxFileCount': 1000,
}
try:
resp = self.get_or_post(url, params)
except urllib2.HTTPError:
return []
files = [x['fileName'].split('/')[-1] for x in resp['files']
if os.path.dirname(x['fileName']) == self.path]
next_file = resp['nextFileName']
while next_file:
log.Log("There are still files, getting next list", 9)
params['startFileName'] = next_file
try:
resp = self.get_or_post(url, params)
except urllib2.HTTPError:
return files
files += [x['fileName'].split('/')[-1] for x in resp['files']
if os.path.dirname(x['fileName']) == self.path]
next_file = resp['nextFileName']
return files
def _delete(self, filename):
"""
Delete filename from remote server
"""
log.Log("Deleting file %s" % filename, 9)
endpoint = 'b2_delete_file_version'
url = self.formatted_url(endpoint)
fileid = self.get_file_id(filename)
if fileid is None:
return
filename = self.full_filename(filename)
params = {'fileName': filename, 'fileId': fileid}
try:
self.get_or_post(url, params)
except urllib2.HTTPError as e:
if e.code == 400:
return
else:
raise e
def _query(self, filename):
"""
Get size info of filename
"""
log.Log("Querying file %s" % filename, 9)
info = self.get_file_info(filename)
if not info:
return {'size': -1}
return {'size': info['size']}
def _error_code(self, operation, e):
if isinstance(e, urllib2.HTTPError):
if e.code == 400:
return log.ErrorCode.bad_request
if e.code == 500:
return log.ErrorCode.backend_error
if e.code == 403:
return log.ErrorCode.backend_permission_denied
def find_or_create_bucket(self, bucket_name):
"""
Find a bucket with name bucket_name and save its id.
If it doesn't exist, create it
"""
endpoint = 'b2_list_buckets'
url = self.formatted_url(endpoint)
params = {'accountId': self.account_id}
resp = self.get_or_post(url, params)
bucket_names = [x['bucketName'] for x in resp['buckets']]
if bucket_name not in bucket_names:
self.create_bucket(bucket_name)
else:
self.bucket_id = {
x[
'bucketName'
]: x['bucketId'] for x in resp['buckets']
}[bucket_name]
def create_bucket(self, bucket_name):
"""
Create a bucket with name bucket_name and save its id
"""
endpoint = 'b2_create_bucket'
url = self.formatted_url(endpoint)
params = {
'accountId': self.account_id,
'bucketName': bucket_name,
'bucketType': 'allPrivate'
}
resp = self.get_or_post(url, params)
self.bucket_id = resp['bucketId']
def formatted_url(self, endpoint):
"""
Return the full api endpoint from just the last part
"""
return '%s/b2api/v1/%s' % (self.api_url, endpoint)
def get_upload_info(self, bucket_id):
"""
Get an upload url for a bucket
"""
endpoint = 'b2_get_upload_url'
url = self.formatted_url(endpoint)
return self.get_or_post(url, {'bucketId': bucket_id})
def get_or_post(self, url, data, headers=None, data_file=None):
"""
Sends the request, either get or post.
If data and data_file are None, send a get request.
data_file takes precedence over data.
If headers are not supplied, just send with an auth key
"""
if headers is None:
if self.auth_token is None:
self._authorize()
headers = {'Authorization': self.auth_token}
if data_file is not None:
data = data_file
else:
data = json.dumps(data) if data else None
encoded_headers = dict(
(k, urllib2.quote(v.encode('utf-8')))
for (k, v) in headers.iteritems()
)
try:
with OpenUrl(url, data, encoded_headers) as resp:
out = resp.read()
try:
return json.loads(out)
except ValueError:
return out
except urllib2.HTTPError as e:
if e.code == 401:
self.auth_token = None
log.Warn("Authtoken expired, will reauthenticate with next attempt")
raise e
def get_file_info(self, filename):
"""
Get a file info from filename
"""
endpoint = 'b2_list_file_names'
url = self.formatted_url(endpoint)
filename = self.full_filename(filename)
params = {
'bucketId': self.bucket_id,
'maxFileCount': 1,
'startFileName': filename,
}
resp = self.get_or_post(url, params)
try:
return resp['files'][0]
except IndexError:
return None
except TypeError:
return None
def get_file_id(self, filename):
"""
Get a file id form filename
"""
try:
return self.get_file_info(filename)['fileId']
except IndexError:
return None
except TypeError:
return None
def full_filename(self, filename):
if self.path:
return self.path + '/' + filename
else:
return filename
@staticmethod
def hex_sha1_of_file(path):
"""
Calculate the sha1 of a file to upload
"""
f = path.open()
block_size = 1024 * 1024
digest = hashlib.sha1()
while True:
data = f.read(block_size)
if len(data) == 0:
break
digest.update(data)
f.close()
return digest.hexdigest()
class OpenUrl(object):
"""
Context manager that handles an open urllib2.Request, and provides
the file-like object that is the response.
"""
def __init__(self, url, data, headers):
log.Log("Getting %s" % url, 9)
self.url = url
self.data = data
self.headers = headers
self.file = None
def __enter__(self):
request = urllib2.Request(self.url, self.data, self.headers)
self.file = urllib2.urlopen(request)
log.Log("Request of %s returned with status %s" %
(self.url, self.file.code), 9)
return self.file
def __exit__(self, exception_type, exception, traceback):
if self.file is not None:
self.file.close()
duplicity.backend.register_backend("b2", B2Backend)
