Python tensorflow.python.platform.gfile.Copy() Examples

def maybe_download(filename, work_directory, source_url):
  """Download the data from source url, unless it's already here.

  Args:
      filename: string, name of the file in the directory.
      work_directory: string, path to working directory.
      source_url: url to download from if file doesn't exist.

  Returns:
      Path to resulting file.
  """
  if not gfile.Exists(work_directory):
    gfile.MakeDirs(work_directory)
  filepath = os.path.join(work_directory, filename)
  if not gfile.Exists(filepath):
    temp_file_name, _ = urlretrieve_with_retry(source_url)
    gfile.Copy(temp_file_name, filepath)
    with gfile.GFile(filepath) as f:
      size = f.size()
    print('Successfully downloaded', filename, size, 'bytes.')
  return filepath 

def maybe_download(filename, work_directory, source_url):
  """Download the data from source url, unless it's already here.

  Args:
      filename: string, name of the file in the directory.
      work_directory: string, path to working directory.
      source_url: url to download from if file doesn't exist.

  Returns:
      Path to resulting file.
  """
  if not gfile.Exists(work_directory):
    gfile.MakeDirs(work_directory)
  filepath = os.path.join(work_directory, filename)
  if not gfile.Exists(filepath):
    temp_file_name, _ = urlretrieve_with_retry(source_url)
    gfile.Copy(temp_file_name, filepath)
    with gfile.GFile(filepath) as f:
      size = f.size()
    print('Successfully downloaded', filename, size, 'bytes.')
  return filepath 

def maybe_download(filepath, source_url):
  """Download the data from source url, unless it's already here.
  Args:
      basename: string, name of the file in the directory.
      target_dir: string, path to working directory.
      source_url: url to download from if file doesn't exist.
  Returns:
      Path to resulting file.
  """
  target_dir = path.dirname(filepath)
  if not gfile.Exists(target_dir):
    gfile.MakeDirs(target_dir)
  if not gfile.Exists(filepath):
    print('Downloading', source_url, 'to', filepath)
    temp_file_name, _ = _urlretrieve_with_retry(source_url)
    gfile.Copy(temp_file_name, filepath)
    with gfile.GFile(filepath) as f:
      size = f.size()
    print('Successfully downloaded', filepath, size, 'bytes.')
  return filepath 

def maybe_download_and_extract(filename, data_dir, source_url):
  """Maybe download and extract a file."""
  if not gfile.Exists(data_dir):
    gfile.MakeDirs(data_dir)

  filepath = os.path.join(data_dir, filename)

  if not gfile.Exists(filepath):
    print('Downloading from {}'.format(source_url))
    temp_file_name, _ = urllib.request.urlretrieve(source_url)
    gfile.Copy(temp_file_name, filepath)
    with gfile.GFile(filepath) as f:
      size = f.size()
    print('Successfully downloaded \'{}\' of {} bytes'.format(filename, size))

  if filename.endswith('.zip'):
    print('Extracting {}'.format(filename))
    zipfile.ZipFile(file=filepath, mode='r').extractall(data_dir) 

def maybe_download(filename, work_directory, source_url):
  """Download the data from source url, unless it's already here.

  Args:
      filename: string, name of the file in the directory.
      work_directory: string, path to working directory.
      source_url: url to download from if file doesn't exist.

  Returns:
      Path to resulting file.
  """
  if not gfile.Exists(work_directory):
    gfile.MakeDirs(work_directory)
  filepath = os.path.join(work_directory, filename)
  if not gfile.Exists(filepath):
    with tempfile.NamedTemporaryFile() as tmpfile:
      temp_file_name = tmpfile.name
      urllib.request.urlretrieve(source_url, temp_file_name)
      gfile.Copy(temp_file_name, filepath)
      with gfile.GFile(filepath) as f:
        size = f.size()
      print('Successfully downloaded', filename, size, 'bytes.')
  return filepath 

def _recursive_copy(src_dir, dest_dir):
  """Copy the contents of src_dir into the folder dest_dir.
  Args:
    src_dir: gsc or local path.
    dest_dir: gcs or local path.
  When called, dest_dir should exist.
  """
  src_dir = python_portable_string(src_dir)
  dest_dir = python_portable_string(dest_dir)

  file_io.recursive_create_dir(dest_dir)
  for file_name in file_io.list_directory(src_dir):
    old_path = os.path.join(src_dir, file_name)
    new_path = os.path.join(dest_dir, file_name)

    if file_io.is_directory(old_path):
      _recursive_copy(old_path, new_path)
    else:
      file_io.copy(old_path, new_path, overwrite=True) 

def maybe_download(filename, work_directory, source_url):
  """Download the data from source url, unless it's already here.

  Args:
      filename: string, name of the file in the directory.
      work_directory: string, path to working directory.
      source_url: url to download from if file doesn't exist.

  Returns:
      Path to resulting file.
  """
  if not gfile.Exists(work_directory):
    gfile.MakeDirs(work_directory)
  filepath = os.path.join(work_directory, filename)
  if not gfile.Exists(filepath):
    temp_file_name, _ = urlretrieve_with_retry(source_url)
    gfile.Copy(temp_file_name, filepath)
    with gfile.GFile(filepath) as f:
      size = f.size()
    print('Successfully downloaded', filename, size, 'bytes.')
  return filepath 

def gfile_copy_callback(files_to_copy, export_dir_path):
  """Callback to copy files using `gfile.Copy` to an export directory.

  This method is used as the default `assets_callback` in `Exporter.init` to
  copy assets from the `assets_collection`. It can also be invoked directly to
  copy additional supplementary files into the export directory (in which case
  it is not a callback).

  Args:
    files_to_copy: A dictionary that maps original file paths to desired
      basename in the export directory.
    export_dir_path: Directory to copy the files to.
  """
  logging.info("Write assets into: %s using gfile_copy.", export_dir_path)
  gfile.MakeDirs(export_dir_path)
  for source_filepath, basename in files_to_copy.items():
    new_path = os.path.join(
        compat.as_bytes(export_dir_path), compat.as_bytes(basename))
    logging.info("Copying asset %s to path %s.", source_filepath, new_path)

    if gfile.Exists(new_path):
      # Guard against being restarted while copying assets, and the file
      # existing and being in an unknown state.
      # TODO(b/28676216): Do some file checks before deleting.
      logging.info("Removing file %s.", new_path)
      gfile.Remove(new_path)
    gfile.Copy(source_filepath, new_path) 

def edit_pb_txt(old_args, export_dir):
  """
  Edit file path argument in pbtxt file.
  :param old_args: Old file paths need to be copied and edited.
  :param export_dir: Directory of the saved model.
  """
  assets_extra_dir = os.path.join(export_dir, "./assets.extra")
  if not os.path.exists(assets_extra_dir):
    os.makedirs(assets_extra_dir)

  new_args = []
  for one_old in old_args:
    if not os.path.exists(one_old):
      raise ValueError("{} do not exists!".format(one_old))
    one_new = os.path.join(assets_extra_dir, os.path.basename(one_old))
    new_args.append(one_new)
    logging.info("Copy file: {} to: {}".format(one_old, one_new))
    gfile.Copy(one_old, one_new, overwrite=True)

  pbtxt_file = os.path.join(export_dir, "saved_model.pbtxt")
  tmp_file = pbtxt_file + ".tmp"
  logging.info("Editing pbtxt file: {}".format(pbtxt_file))
  with open(pbtxt_file, "rt") as fin, open(tmp_file, "wt") as fout:
    for line in fin:
      for one_old, one_new in zip(old_args, new_args):
        line = line.replace(one_old, one_new)
      fout.write(line)
  gfile.Copy(tmp_file, pbtxt_file, overwrite=True)
  gfile.Remove(tmp_file) 

def edit_pb_txt(old_args, export_dir):
  """
  Edit file path argument in pbtxt file.
  :param old_args: Old file paths need to be copied and edited.
  :param export_dir: Directory of the saved model.
  """
  assets_extra_dir = os.path.join(export_dir, "./assets.extra")
  if not os.path.exists(assets_extra_dir):
    os.makedirs(assets_extra_dir)

  new_args = []
  for one_old in old_args:
    if not os.path.exists(one_old):
      raise ValueError("{} do not exists!".format(one_old))
    one_new = os.path.join(assets_extra_dir, os.path.basename(one_old))
    new_args.append(one_new)
    logging.info("Copy file: {} to: {}".format(one_old, one_new))
    gfile.Copy(one_old, one_new, overwrite=True)

  pbtxt_file = os.path.join(export_dir, "saved_model.pbtxt")
  tmp_file = pbtxt_file + ".tmp"
  logging.info("Editing pbtxt file: {}".format(pbtxt_file))
  with open(pbtxt_file, "rt") as fin, open(tmp_file, "wt") as fout:
    for line in fin:
      for one_old, one_new in zip(old_args, new_args):
        line = line.replace(one_old, one_new)
      fout.write(line)
  gfile.Copy(tmp_file, pbtxt_file, overwrite=True)
  gfile.Remove(tmp_file) 

def edit_pb_txt(old_args, export_dir):
  """
  Edit file path argument in pbtxt file.
  :param old_args: Old file paths need to be copied and edited.
  :param export_dir: Directory of the saved model.
  """
  assets_extra_dir = os.path.join(export_dir, "./assets.extra")
  if not os.path.exists(assets_extra_dir):
    os.makedirs(assets_extra_dir)

  new_args = []
  for one_old in old_args:
    if not os.path.exists(one_old):
      raise ValueError("{} do not exists!".format(one_old))
    one_new = os.path.join(assets_extra_dir, os.path.basename(one_old))
    new_args.append(one_new)
    logging.info("Copy file: {} to: {}".format(one_old, one_new))
    gfile.Copy(one_old, one_new, overwrite=True)

  pbtxt_file = os.path.join(export_dir, "saved_model.pbtxt")
  tmp_file = pbtxt_file + ".tmp"
  logging.info("Editing pbtxt file: {}".format(pbtxt_file))
  with open(pbtxt_file, "rt") as fin, open(tmp_file, "wt") as fout:
    for line in fin:
      for one_old, one_new in zip(old_args, new_args):
        line = line.replace(one_old, one_new)
      fout.write(line)
  gfile.Copy(tmp_file, pbtxt_file, overwrite=True)
  gfile.Remove(tmp_file) 

def edit_pb_txt(old_args, export_dir):
  """
  Edit file path argument in pbtxt file.
  :param old_args: Old file paths need to be copied and edited.
  :param export_dir: Directory of the saved model.
  """
  assets_extra_dir = os.path.join(export_dir, "./assets.extra")
  if not os.path.exists(assets_extra_dir):
    os.makedirs(assets_extra_dir)

  new_args = []
  for one_old in old_args:
    if not os.path.exists(one_old):
      raise ValueError("{} do not exists!".format(one_old))
    one_new = os.path.join(assets_extra_dir, os.path.basename(one_old))
    new_args.append(one_new)
    logging.info("Copy file: {} to: {}".format(one_old, one_new))
    gfile.Copy(one_old, one_new, overwrite=True)

  pbtxt_file = os.path.join(export_dir, "saved_model.pbtxt")
  tmp_file = pbtxt_file + ".tmp"
  logging.info("Editing pbtxt file: {}".format(pbtxt_file))
  with open(pbtxt_file, "rt") as fin, open(tmp_file, "wt") as fout:
    for line in fin:
      for one_old, one_new in zip(old_args, new_args):
        line = line.replace(one_old, one_new)
      fout.write(line)
  gfile.Copy(tmp_file, pbtxt_file, overwrite=True)
  gfile.Remove(tmp_file) 

def edit_pb_txt(old_args, export_dir):
  """
  Edit file path argument in pbtxt file.
  :param old_args: Old file paths need to be copied and edited.
  :param export_dir: Directory of the saved model.
  """
  assets_extra_dir = os.path.join(export_dir, "./assets.extra")
  if not os.path.exists(assets_extra_dir):
    os.makedirs(assets_extra_dir)

  new_args = []
  for one_old in old_args:
    if not os.path.exists(one_old):
      raise ValueError("{} do not exists!".format(one_old))
    one_new = os.path.join(assets_extra_dir, os.path.basename(one_old))
    new_args.append(one_new)
    logging.info("Copy file: {} to: {}".format(one_old, one_new))
    gfile.Copy(one_old, one_new, overwrite=True)

  pbtxt_file = os.path.join(export_dir, "saved_model.pbtxt")
  tmp_file = pbtxt_file + ".tmp"
  logging.info("Editing pbtxt file: {}".format(pbtxt_file))
  with open(pbtxt_file, "rt") as fin, open(tmp_file, "wt") as fout:
    for line in fin:
      for one_old, one_new in zip(old_args, new_args):
        line = line.replace(one_old, one_new)
      fout.write(line)
  gfile.Copy(tmp_file, pbtxt_file, overwrite=True)
  gfile.Remove(tmp_file) 

def edit_pb_txt(old_args, export_dir):
  """
  Edit file path argument in pbtxt file.
  :param old_args: Old file paths need to be copied and edited.
  :param export_dir: Directory of the saved model.
  """
  assets_extra_dir = os.path.join(export_dir, "./assets.extra")
  if not os.path.exists(assets_extra_dir):
    os.makedirs(assets_extra_dir)

  new_args = []
  for one_old in old_args:
    if not os.path.exists(one_old):
      raise ValueError("{} do not exists!".format(one_old))
    one_new = os.path.join(assets_extra_dir, os.path.basename(one_old))
    new_args.append(one_new)
    logging.info("Copy file: {} to: {}".format(one_old, one_new))
    gfile.Copy(one_old, one_new, overwrite=True)

  pbtxt_file = os.path.join(export_dir, "saved_model.pbtxt")
  tmp_file = pbtxt_file + ".tmp"
  logging.info("Editing pbtxt file: {}".format(pbtxt_file))
  with open(pbtxt_file, "rt") as fin, open(tmp_file, "wt") as fout:
    for line in fin:
      for one_old, one_new in zip(old_args, new_args):
        line = line.replace(one_old, one_new)
      fout.write(line)
  gfile.Copy(tmp_file, pbtxt_file, overwrite=True)
  gfile.Remove(tmp_file) 

def edit_pb_txt(old_args, export_dir):
  """
  Edit file path argument in pbtxt file.
  :param old_args: Old file paths need to be copied and edited.
  :param export_dir: Directory of the saved model.
  """
  assets_extra_dir = os.path.join(export_dir, "./assets.extra")
  if not os.path.exists(assets_extra_dir):
    os.makedirs(assets_extra_dir)

  new_args = []
  for one_old in old_args:
    if not os.path.exists(one_old):
      raise ValueError("{} do not exists!".format(one_old))
    one_new = os.path.join(assets_extra_dir, os.path.basename(one_old))
    new_args.append(one_new)
    logging.info("Copy file: {} to: {}".format(one_old, one_new))
    gfile.Copy(one_old, one_new, overwrite=True)

  pbtxt_file = os.path.join(export_dir, "saved_model.pbtxt")
  tmp_file = pbtxt_file + ".tmp"
  logging.info("Editing pbtxt file: {}".format(pbtxt_file))
  with open(pbtxt_file, "rt") as fin, open(tmp_file, "wt") as fout:
    for line in fin:
      for one_old, one_new in zip(old_args, new_args):
        line = line.replace(one_old, one_new)
      fout.write(line)
  gfile.Copy(tmp_file, pbtxt_file, overwrite=True)
  gfile.Remove(tmp_file) 

def edit_pb_txt(old_args, export_dir):
  """
  Edit file path argument in pbtxt file.
  :param old_args: Old file paths need to be copied and edited.
  :param export_dir: Directory of the saved model.
  """
  assets_extra_dir = os.path.join(export_dir, "./assets.extra")
  if not os.path.exists(assets_extra_dir):
    os.makedirs(assets_extra_dir)

  new_args = []
  for one_old in old_args:
    if not os.path.exists(one_old):
      raise ValueError("{} do not exists!".format(one_old))
    one_new = os.path.join(assets_extra_dir, os.path.basename(one_old))
    new_args.append(one_new)
    logging.info("Copy file: {} to: {}".format(one_old, one_new))
    gfile.Copy(one_old, one_new, overwrite=True)

  pbtxt_file = os.path.join(export_dir, "saved_model.pbtxt")
  tmp_file = pbtxt_file + ".tmp"
  logging.info("Editing pbtxt file: {}".format(pbtxt_file))
  with open(pbtxt_file, "rt") as fin, open(tmp_file, "wt") as fout:
    for line in fin:
      for one_old, one_new in zip(old_args, new_args):
        line = line.replace(one_old, one_new)
      fout.write(line)
  gfile.Copy(tmp_file, pbtxt_file, overwrite=True)
  gfile.Remove(tmp_file) 

def edit_pb_txt(old_args, export_dir):
  """
  Edit file path argument in pbtxt file.
  :param old_args: Old file paths need to be copied and edited.
  :param export_dir: Directory of the saved model.
  """
  assets_extra_dir = os.path.join(export_dir, "./assets.extra")
  if not os.path.exists(assets_extra_dir):
    os.makedirs(assets_extra_dir)

  new_args = []
  for one_old in old_args:
    if not os.path.exists(one_old):
      raise ValueError("{} do not exists!".format(one_old))
    one_new = os.path.join(assets_extra_dir, os.path.basename(one_old))
    new_args.append(one_new)
    logging.info("Copy file: {} to: {}".format(one_old, one_new))
    gfile.Copy(one_old, one_new, overwrite=True)

  pbtxt_file = os.path.join(export_dir, "saved_model.pbtxt")
  tmp_file = pbtxt_file + ".tmp"
  logging.info("Editing pbtxt file: {}".format(pbtxt_file))
  with open(pbtxt_file, "rt") as fin, open(tmp_file, "wt") as fout:
    for line in fin:
      for one_old, one_new in zip(old_args, new_args):
        line = line.replace(one_old, one_new)
      fout.write(line)
  gfile.Copy(tmp_file, pbtxt_file, overwrite=True)
  gfile.Remove(tmp_file) 

def edit_pb_txt(old_args, export_dir):
  """
  Edit file path argument in pbtxt file.
  :param old_args: Old file paths need to be copied and edited.
  :param export_dir: Directory of the saved model.
  """
  assets_extra_dir = os.path.join(export_dir, "./assets.extra")
  if not os.path.exists(assets_extra_dir):
    os.makedirs(assets_extra_dir)

  new_args = []
  for one_old in old_args:
    if not os.path.exists(one_old):
      raise ValueError("{} do not exists!".format(one_old))
    one_new = os.path.join(assets_extra_dir, os.path.basename(one_old))
    new_args.append(one_new)
    logging.info("Copy file: {} to: {}".format(one_old, one_new))
    gfile.Copy(one_old, one_new, overwrite=True)

  pbtxt_file = os.path.join(export_dir, "saved_model.pbtxt")
  tmp_file = pbtxt_file + ".tmp"
  logging.info("Editing pbtxt file: {}".format(pbtxt_file))
  with open(pbtxt_file, "rt") as fin, open(tmp_file, "wt") as fout:
    for line in fin:
      for one_old, one_new in zip(old_args, new_args):
        line = line.replace(one_old, one_new)
      fout.write(line)
  gfile.Copy(tmp_file, pbtxt_file, overwrite=True)
  gfile.Remove(tmp_file) 

def gfile_copy_callback(files_to_copy, export_dir_path):
  """Callback to copy files using `gfile.Copy` to an export directory.

  This method is used as the default `assets_callback` in `Exporter.init` to
  copy assets from the `assets_collection`. It can also be invoked directly to
  copy additional supplementary files into the export directory (in which case
  it is not a callback).

  Args:
    files_to_copy: A dictionary that maps original file paths to desired
      basename in the export directory.
    export_dir_path: Directory to copy the files to.
  """
  logging.info("Write assets into: %s using gfile_copy.", export_dir_path)
  gfile.MakeDirs(export_dir_path)
  for source_filepath, basename in files_to_copy.items():
    new_path = os.path.join(
        compat.as_bytes(export_dir_path), compat.as_bytes(basename))
    logging.info("Copying asset %s to path %s.", source_filepath, new_path)

    if gfile.Exists(new_path):
      # Guard against being restarted while copying assets, and the file
      # existing and being in an unknown state.
      # TODO(b/28676216): Do some file checks before deleting.
      logging.info("Removing file %s.", new_path)
      gfile.Remove(new_path)
    gfile.Copy(source_filepath, new_path) 

def gfile_copy_callback(files_to_copy, export_dir_path):
  """Callback to copy files using `gfile.Copy` to an export directory.

  This method is used as the default `assets_callback` in `Exporter.init` to
  copy assets from the `assets_collection`. It can also be invoked directly to
  copy additional supplementary files into the export directory (in which case
  it is not a callback).

  Args:
    files_to_copy: A dictionary that maps original file paths to desired
      basename in the export directory.
    export_dir_path: Directory to copy the files to.
  """
  logging.info("Write assets into: %s using gfile_copy.", export_dir_path)
  gfile.MakeDirs(export_dir_path)
  for source_filepath, basename in files_to_copy.items():
    new_path = os.path.join(
        compat.as_bytes(export_dir_path), compat.as_bytes(basename))
    logging.info("Copying asset %s to path %s.", source_filepath, new_path)

    if gfile.Exists(new_path):
      # Guard against being restarted while copying assets, and the file
      # existing and being in an unknown state.
      # TODO(b/28676216): Do some file checks before deleting.
      logging.info("Removing file %s.", new_path)
      gfile.Remove(new_path)
    gfile.Copy(source_filepath, new_path) 

def gfile_copy_callback(files_to_copy, export_dir_path):
  """Callback to copy files using `gfile.Copy` to an export directory.

  This method is used as the default `assets_callback` in `Exporter.init` to
  copy assets from the `assets_collection`. It can also be invoked directly to
  copy additional supplementary files into the export directory (in which case
  it is not a callback).

  Args:
    files_to_copy: A dictionary that maps original file paths to desired
      basename in the export directory.
    export_dir_path: Directory to copy the files to.
  """
  logging.info("Write assets into: %s using gfile_copy.", export_dir_path)
  gfile.MakeDirs(export_dir_path)
  for source_filepath, basename in files_to_copy.items():
    new_path = os.path.join(
        compat.as_bytes(export_dir_path), compat.as_bytes(basename))
    logging.info("Copying asset %s to path %s.", source_filepath, new_path)

    if gfile.Exists(new_path):
      # Guard against being restarted while copying assets, and the file
      # existing and being in an unknown state.
      # TODO(b/28676216): Do some file checks before deleting.
      logging.info("Removing file %s.", new_path)
      gfile.Remove(new_path)
    gfile.Copy(source_filepath, new_path)