Python os.path.abspath() Examples

def test_get_home_dir_8():
    """Using registry hack for 'My Documents', os=='nt'

    HOMESHARE, HOMEDRIVE, HOMEPATH, USERPROFILE and others are missing.
    """
    os.name = 'nt'
    # Remove from stub environment all keys that may be set
    for key in ['HOME', 'HOMESHARE', 'HOMEDRIVE', 'HOMEPATH', 'USERPROFILE']:
        env.pop(key, None)

    class key:
        def Close(self):
            pass

    with patch.object(wreg, 'OpenKey', return_value=key()), \
         patch.object(wreg, 'QueryValueEx', return_value=[abspath(HOME_TEST_DIR)]):
        home_dir = path.get_home_dir()
    nt.assert_equal(home_dir, abspath(HOME_TEST_DIR)) 

def test_pep8(self):
        """Test method to check PEP8 compliance over the entire project."""
        self.file_structure = dirname(dirname(abspath(__file__)))
        print("Testing for PEP8 compliance of python files in {}".format(
            self.file_structure))
        style = pep8.StyleGuide()
        style.options.max_line_length = 100  # Set this to desired maximum line length
        filenames = []
        # Set this to desired folder location
        for root, _, files in os.walk(self.file_structure):
            python_files = [f for f in files if f.endswith(
                '.py') and "examples" not in root]
            for file in python_files:
                if len(root.split('samples')) != 2:     # Ignore samples directory
                    filename = '{0}/{1}'.format(root, file)
                    filenames.append(filename)
        check = style.check_files(filenames)
        self.assertEqual(check.total_errors, 0, 'PEP8 style errors: %d' %
                         check.total_errors) 

def __init__(self, dataset, save_prefix, use_time=True, cleanup=False, score_thresh=1e-3):
        super(COCOInstanceMetric, self).__init__('COCOInstance')
        self.dataset = dataset
        self._img_ids = sorted(dataset.coco.getImgIds())
        self._current_id = 0
        self._cleanup = cleanup
        self._results = []
        self._score_thresh = score_thresh

        try_import_pycocotools()
        import pycocotools.mask as cocomask
        self._cocomask = cocomask

        if use_time:
            import datetime
            t = datetime.datetime.now().strftime('_%Y_%m_%d_%H_%M_%S')
        else:
            t = ''
        self._filename = osp.abspath(osp.expanduser(save_prefix) + t + '.json')
        try:
            f = open(self._filename, 'w')
        except IOError as e:
            raise RuntimeError("Unable to open json file to dump. What(): {}".format(str(e)))
        else:
            f.close() 

def replaceFileName(fileMatch):
    global currentFilesPath
    fileName = fileMatch.group(1)
    currentWD = os.getcwd()
    pathName = abspath(currentWD) +"/"+fileName
    if not os.path.isfile(pathName):
        dirname, filename = os.path.split(abspath(getsourcefile(lambda:0)))
        pathName = dirname +"/"+fileName
        if not os.path.isfile(pathName):
            pathName = currentFilesPath +"/"+fileName
            if not os.path.isfile(pathName):
                cdErr("Cannot find include file '"+fileName+"'")

    includedStr = progSpec.stringFromFile(pathName)
    includedStr = processIncludedFiles(includedStr, pathName)
    return includedStr 

def pretty():  
    try:
        locale.setlocale(locale.LC_ALL, locale_strings[platform.system()])
    except locale.Error:
        print("Could not set the language settings! 3.5 will not be written as 3,5! SO SAD!")

    matplotlib.rcParams.update({
        'font.family': 'serif',
        'text.usetex': True,
        'pgf.rcfonts': False,
        'pgf.texsystem': 'lualatex',
        'pgf.preamble': r'\input{'+dirname(abspath(__file__)).replace(" ", r"\ ")+r'//matplotlib_header.tex'+r'}',
        'axes.formatter.use_locale' : True,
    })

# Currently not working
# if platform.system() == "Windows":
#     plt.style.use('file:\\' + os.path.dirname(os.path.abspath(__file__)) + os.sep + 'praktipy.mplstyle')
# else:
#     plt.style.use('file://' + os.path.dirname(os.path.abspath(__file__)) + os.sep + 'praktipy.mplstyle')

# print('file://' + os.path.dirname(os.path.abspath(__file__)) + os.sep + 'praktipy.mplstyle')
# convenience functions 

def _get_file_idx(self, filename, source=None):
    """ Detect index of given filename within patch.

        :param filename:
        :param source: search filename among sources (True),
                       targets (False), or both (None)
        :returns: int or None
    """
    filename = abspath(filename)
    if source == True or source == None:
      for i,fnm in enumerate(self.source):
        if filename == abspath(fnm):
          return i  
    if source == False or source == None:
      for i,fnm in enumerate(self.target):
        if filename == abspath(fnm):
          return i 

def test_runspider(self):
        tmpdir = self.mktemp()
        os.mkdir(tmpdir)
        fname = abspath(join(tmpdir, 'myspider.py'))
        with open(fname, 'w') as f:
            f.write("""
from scrapy import log
from scrapy.spider import BaseSpider

class MySpider(BaseSpider):
    name = 'myspider'

    def start_requests(self):
        self.log("It Works!")
        return []
""")
        p = self.proc('runspider', fname)
        log = p.stderr.read()
        self.assert_("[myspider] DEBUG: It Works!" in log)
        self.assert_("[myspider] INFO: Spider opened" in log)
        self.assert_("[myspider] INFO: Closing spider (finished)" in log)
        self.assert_("[myspider] INFO: Spider closed (finished)" in log) 

def cached_path(file_path, cached_dir=None):
    if not cached_dir:
        cached_dir = str(Path(Path.home() / '.tatk') / "cache")

    return allennlp_cached_path(file_path, cached_dir)

# DATA_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))), 'data/mdbt')
# VALIDATION_URL = os.path.join(DATA_PATH, "data/validate.json")
# WORD_VECTORS_URL = os.path.join(DATA_PATH, "word-vectors/paragram_300_sl999.txt")
# TRAINING_URL = os.path.join(DATA_PATH, "data/train.json")
# ONTOLOGY_URL = os.path.join(DATA_PATH, "data/ontology.json")
# TESTING_URL = os.path.join(DATA_PATH, "data/test.json")
# MODEL_URL = os.path.join(DATA_PATH, "models/model-1")
# GRAPH_URL = os.path.join(DATA_PATH, "graphs/graph-1")
# RESULTS_URL = os.path.join(DATA_PATH, "results/log-1.txt")
# KB_URL = os.path.join(DATA_PATH, "data/")  # TODO: yaoqin
# TRAIN_MODEL_URL = os.path.join(DATA_PATH, "train_models/model-1")
# TRAIN_GRAPH_URL = os.path.join(DATA_PATH, "train_graph/graph-1") 

def base_opts_from_args(args):
    from os.path import abspath
    return BaseOpts(
        verbose_make = args.verbose_make,
        configure_dir = abspath(args.configure_dir),
        install_dir = abspath(args.install_dir),
        mono_source_root = abspath(args.mono_sources),
        mxe_prefix = args.mxe_prefix
    ) 

def android_opts_from_args(args):
    return AndroidOpts(
        **vars(runtime_opts_from_args(args)),
        android_toolchains_prefix = abspath(args.toolchains_prefix),
        android_sdk_root = abspath(args.android_sdk),
        android_ndk_root = abspath(args.android_ndk),
        with_monodroid = args.with_monodroid,
        android_api_version = args.android_api_version,
        android_cmake_version = args.android_cmake_version
    ) 

def openfile(filename):
    from os.path import join, dirname, abspath
    path = abspath(join(dirname(testfile), os.pardir, 'moredata', filename))
    return open(path, 'r')

# Prevent this test from running in the Python distro 

def _samefile(src, dst):
    # Macintosh, Unix.
    if hasattr(os.path, 'samefile'):
        try:
            return os.path.samefile(src, dst)
        except OSError:
            return False

    # All other platforms: check for same pathname.
    return (os.path.normcase(os.path.abspath(src)) ==
            os.path.normcase(os.path.abspath(dst))) 

def _destinsrc(src, dst):
    src = abspath(src)
    dst = abspath(dst)
    if not src.endswith(os.path.sep):
        src += os.path.sep
    if not dst.endswith(os.path.sep):
        dst += os.path.sep
    return dst.startswith(src) 

def get_output_dir(imdb, weights_filename):
  """Return the directory where experimental artifacts are placed.
  If the directory does not exist, it is created.

  A canonical path is built using the name from an imdb and a network
  (if not None).
  """
  outdir = osp.abspath(osp.join(__C.ROOT_DIR, 'output', __C.EXP_DIR, imdb.name))
  if weights_filename is None:
    weights_filename = 'default'
  outdir = osp.join(outdir, weights_filename)
  if not os.path.exists(outdir):
    os.makedirs(outdir)
  return outdir 

def get_output_tb_dir(imdb, weights_filename):
  """Return the directory where tensorflow summaries are placed.
  If the directory does not exist, it is created.

  A canonical path is built using the name from an imdb and a network
  (if not None).
  """
  outdir = osp.abspath(osp.join(__C.ROOT_DIR, 'tensorboard', __C.EXP_DIR, imdb.name))
  if weights_filename is None:
    weights_filename = 'default'
  outdir = osp.join(outdir, weights_filename)
  if not os.path.exists(outdir):
    os.makedirs(outdir)
  return outdir 

def cache_path(self):
    cache_path = osp.abspath(osp.join(cfg.DATA_DIR, 'cache'))
    if not os.path.exists(cache_path):
      os.makedirs(cache_path)
    return cache_path 

def get_output_dir(imdb, weights_filename):
    """Return the directory where experimental artifacts are placed.
    If the directory does not exist, it is created.

    A canonical path is built using the name from an imdb and a network
    (if not None).
    """
    outdir = osp.abspath(osp.join(FLAGS2["root_dir"], FLAGS2["root_dir"] , 'default', imdb.name))
    if weights_filename is None:
        weights_filename = 'default'
    outdir = osp.join(outdir, weights_filename)
    if not os.path.exists(outdir):
        os.makedirs(outdir)
    return outdir 

def cache_path(self):
        cache_path = osp.abspath(osp.join(cfg.FLAGS2["data_dir"], 'cache'))
        if not os.path.exists(cache_path):
            os.makedirs(cache_path)
        return cache_path 

def example_app():
    """Example app fixture."""
    current_dir = os.getcwd()

    # Go to example directory
    project_dir = dirname(dirname(abspath(__file__)))
    exampleapp_dir = join(project_dir, 'examples')
    os.chdir(exampleapp_dir)

    # Setup application
    assert subprocess.call('./app-setup.sh', shell=True) == 0

    # Setup fixtures
    assert subprocess.call('./app-fixtures.sh', shell=True) == 0

    # Start example app
    webapp = subprocess.Popen(
        'FLASK_APP=app.py flask run --debugger -p 5001',
        stdout=subprocess.PIPE, preexec_fn=os.setsid, shell=True)
    time.sleep(10)
    yield webapp

    # Stop server
    os.killpg(webapp.pid, signal.SIGTERM)

    # Tear down example app
    subprocess.call('./app-teardown.sh', shell=True)

    # Return to the original directory
    os.chdir(current_dir)


# def test_example_app_role_admin(example_app):
#     """Test example app."""
#     cmd = 'curl http://0.0.0.0:5001/'
#     output = subprocess.check_output(cmd, shell=True)
#     assert b'Welcome to REROILS-DATA' in output 

def _samefile(src, dst):
    # Macintosh, Unix.
    if hasattr(os.path, 'samefile'):
        try:
            return os.path.samefile(src, dst)
        except OSError:
            return False

    # All other platforms: check for same pathname.
    return (os.path.normcase(os.path.abspath(src)) ==
            os.path.normcase(os.path.abspath(dst))) 

def _destinsrc(src, dst):
    src = abspath(src)
    dst = abspath(dst)
    if not src.endswith(os.path.sep):
        src += os.path.sep
    if not dst.endswith(os.path.sep):
        dst += os.path.sep
    return dst.startswith(src) 

def read(*names, **kwargs):
    return io.open(
        join(abspath(*names)),
        encoding=kwargs.get('encoding', 'utf8')
    ).read() 

def _samefile(src, dst):
    # Macintosh, Unix.
    if hasattr(os.path, 'samefile'):
        try:
            return os.path.samefile(src, dst)
        except OSError:
            return False

    # All other platforms: check for same pathname.
    return (os.path.normcase(os.path.abspath(src)) ==
            os.path.normcase(os.path.abspath(dst))) 

def _destinsrc(src, dst):
    src = abspath(src)
    dst = abspath(dst)
    if not src.endswith(os.path.sep):
        src += os.path.sep
    if not dst.endswith(os.path.sep):
        dst += os.path.sep
    return dst.startswith(src) 

def __init__(self, dataset, save_prefix, use_time=True, cleanup=False, in_vis_thresh=0.2,
                 data_shape=None):
        super(COCOKeyPointsMetric, self).__init__('COCOMeanAP')
        self.dataset = dataset
        self._img_ids = sorted(dataset.coco.getImgIds())
        self._current_id = 0
        self._cleanup = cleanup
        self._results = []
        self._in_vis_thresh = in_vis_thresh
        if isinstance(data_shape, (tuple, list)):
            assert len(data_shape) == 2, "Data shape must be (height, width)"
        elif not data_shape:
            data_shape = None
        else:
            raise ValueError("data_shape must be None or tuple of int as (height, width)")
        self._data_shape = data_shape

        if use_time:
            import datetime
            t = datetime.datetime.now().strftime('_%Y_%m_%d_%H_%M_%S')
        else:
            t = ''
        self._filename = osp.abspath(osp.expanduser(save_prefix) + t + '.json')
        try:
            f = open(self._filename, 'w')
        except IOError as e:
            raise RuntimeError("Unable to open json file to dump. What(): {}".format(str(e)))
        else:
            f.close() 

def __init__(self, dataset, save_prefix, use_time=True, cleanup=False, score_thresh=0.05,
                 data_shape=None):
        super(COCODetectionMetric, self).__init__('COCOMeanAP')
        self.dataset = dataset
        self._img_ids = sorted(dataset.coco.getImgIds())
        self._current_id = 0
        self._cleanup = cleanup
        self._results = []
        self._score_thresh = score_thresh
        if isinstance(data_shape, (tuple, list)):
            assert len(data_shape) == 2, "Data shape must be (height, width)"
        elif not data_shape:
            data_shape = None
        else:
            raise ValueError("data_shape must be None or tuple of int as (height, width)")
        self._data_shape = data_shape

        if use_time:
            import datetime
            t = datetime.datetime.now().strftime('_%Y_%m_%d_%H_%M_%S')
        else:
            t = ''
        self._filename = osp.abspath(osp.expanduser(save_prefix) + t + '.json')
        try:
            f = open(self._filename, 'w')
        except IOError as e:
            raise RuntimeError("Unable to open json file to dump. What(): {}".format(str(e)))
        else:
            f.close() 

def processIncludedFiles(fileString, fileName):
    global currentFilesPath
    dirname, filename = os.path.split(abspath(fileName))
    currentFilesPath = dirname
    pattern = re.compile(r'#include +([\w -\.\/\\]+)')
    return pattern.sub(replaceFileName, fileString) 

def _samefile(src, dst):
    # Macintosh, Unix.
    if hasattr(os.path, 'samefile'):
        try:
            return os.path.samefile(src, dst)
        except OSError:
            return False

    # All other platforms: check for same pathname.
    return (os.path.normcase(os.path.abspath(src)) ==
            os.path.normcase(os.path.abspath(dst))) 

def _destinsrc(src, dst):
    src = abspath(src)
    dst = abspath(dst)
    if not src.endswith(os.path.sep):
        src += os.path.sep
    if not dst.endswith(os.path.sep):
        dst += os.path.sep
    return dst.startswith(src) 

def get(self, save_path, dataset=None):
        """

        Download a dataset.

        Args:
            save_path : str
                A directory to save the data to.
            dataset : str, optional
                A specific dataset to download.
                Note: this must include the file extension.
                If None, options will be presented for you
                to choose from.

        Returns:
            save_path_full : str
                The absolute path to the downloaded data.

        """
        if dataset is None:
            selected_dataset = self._present_options()
        else:
            selected_dataset = dataset

        save_path_full = join(save_path, selected_dataset.split('.')[0])

        if isdir(save_path_full):
            warn("\n'{0}' already exists. Voiding Download.".format(
                save_path_full))
        else:
            self._print('Downloading Data...')
            url = "{0}/{1}".format(self.url, selected_dataset)
            self._download_data(url, save_path=save_path)

        return abspath(save_path_full) 

def _samefile(src, dst):
    # Macintosh, Unix.
    if hasattr(os.path, 'samefile'):
        try:
            return os.path.samefile(src, dst)
        except OSError:
            return False

    # All other platforms: check for same pathname.
    return (os.path.normcase(os.path.abspath(src)) ==
            os.path.normcase(os.path.abspath(dst))) 

def _destinsrc(src, dst):
    src = abspath(src)
    dst = abspath(dst)
    if not src.endswith(os.path.sep):
        src += os.path.sep
    if not dst.endswith(os.path.sep):
        dst += os.path.sep
    return dst.startswith(src) 

def parse_template(self, template:str, config_rel_path, output_path=None):
        """Filename specifying a template for a report."""

        import reportengine.templates


        finder = filefinder.FallbackFinder([
                     '.',
                     config_rel_path,
                     filefinder.ModuleFinder(reportengine.templates),
                 ])

        try:
            folder, name = finder.find(template)
        #Ridiculous error message
        except FileNotFoundError as e:
            raise configparser.ConfigError("Could not find template '%s': %s" %
                                           (template, e), template,
                                           finder.hint_files()) from e



        abspath = folder / name
        if not abspath.is_file():
            raise Config("The template (%s) must refer to a file." % template)
        if output_path:
            destpath = output_path / 'input' / name
            parpath = destpath.parent
            parpath.mkdir(exist_ok=True, parents=True)
            try:
                shutil.copy2(abspath, destpath)
            except shutil.SameFileError:
                pass

        return abspath 

def main():
    """Main function to run as shell script."""
    loader = unittest.TestLoader()
    suite = loader.discover(path.abspath(path.dirname(__file__)), pattern='test_*.py')
    runner = unittest.TextTestRunner(buffer=True)
    runner.run(suite) 

def fast():
    matplotlib.rcParams.update({
        'font.family': 'serif',
        'text.usetex': True,
        'pgf.rcfonts': False,
        'pgf.texsystem': 'pdflatex',
        'pgf.preamble': r'\input{'+dirname(abspath(__file__)).replace(" ", r"\ ")+r'//matplotlib_header_minimal.tex'+r'}'
    }) 

def get_output_dir(imdb, weights_filename):
  """Return the directory where experimental artifacts are placed.
  If the directory does not exist, it is created.

  A canonical path is built using the name from an imdb and a network
  (if not None).
  """
  outdir = osp.abspath(osp.join(__C.ROOT_DIR, 'output', __C.EXP_DIR, imdb.name))
  if weights_filename is None:
    weights_filename = 'default'
  outdir = osp.join(outdir, weights_filename)
  if not os.path.exists(outdir):
    os.makedirs(outdir)
  return outdir 

def get_output_tb_dir(imdb, weights_filename):
  """Return the directory where tensorflow summaries are placed.
  If the directory does not exist, it is created.

  A canonical path is built using the name from an imdb and a network
  (if not None).
  """
  outdir = osp.abspath(osp.join(__C.ROOT_DIR, 'tensorboard', __C.EXP_DIR, imdb.name))
  if weights_filename is None:
    weights_filename = 'default'
  outdir = osp.join(outdir, weights_filename)
  if not os.path.exists(outdir):
    os.makedirs(outdir)
  return outdir 

def cache_path(self):
    cache_path = osp.abspath(osp.join(cfg.DATA_DIR, 'cache'))
    if not os.path.exists(cache_path):
      os.makedirs(cache_path)
    return cache_path 

def style(request):
    return HttpResponse(open(join(dirname(dirname(dirname(abspath(__file__)))), 'form.css')).read()) 

def zip_dir(self):
        """Directory that files will be extracted to"""

        from os.path import abspath

        cache_dir = self.downloader.cache.getsyspath('/')
        target_path = abspath(self.fspath)

        if target_path.startswith(cache_dir):  # Case when file is already in cache
            return str(self.fspath) + '_d'
        else:  # file is not in cache; it may exist elsewhere.
            return self.downloader.cache.getsyspath(target_path.lstrip('/'))+'_d' 

def get_suite_env_file(suite_dir=None):
    for n in os.listdir(suite_dir or '.'):
        if n.startswith('mx.'):
            suite_py = join('.', n, 'suite.py')
            if exists(suite_py):
                return abspath(join(suite_dir or '.', n, 'env'))
    return None 

def _match_file_hunks(self, filepath, hunks):
    matched = True
    fp = open(abspath(filepath))

    class NoMatch(Exception):
      pass

    lineno = 1
    line = fp.readline()
    hno = None
    try:
      for hno, h in enumerate(hunks):
        # skip to first line of the hunk
        while lineno < h.starttgt:
          if not len(line): # eof
            debug("check failed - premature eof before hunk: %d" % (hno+1))
            raise NoMatch
          line = fp.readline()
          lineno += 1
        for hline in h.text:
          if hline.startswith("+"):
            continue
          if not len(line):
            debug("check failed - premature eof on hunk: %d" % (hno+1))
            # todo: \ No newline at the end of file
            raise NoMatch
          if line.rstrip("\r\n") != hline[1:].rstrip("\r\n"):
            debug("file is not patched - failed hunk: %d" % (hno+1))
            raise NoMatch
          line = fp.readline()
          lineno += 1

    except NoMatch:
      matched = False
      # todo: display failed hunk, i.e. expected/found

    fp.close()
    return matched 

def _genspider(self, module, name, domain, template_name, template_file):
        """Generate the spider module, based on the given template"""
        tvars = {
            'project_name': settings.get('BOT_NAME'),
            'ProjectName': string_camelcase(settings.get('BOT_NAME')),
            'module': module,
            'name': name,
            'domain': domain,
            'classname': '%sSpider' % ''.join([s.capitalize() \
                for s in module.split('_')])
        }
        spiders_module = __import__(settings['NEWSPIDER_MODULE'], {}, {}, [''])
        spiders_dir = abspath(dirname(spiders_module.__file__))
        spider_file = "%s.py" % join(spiders_dir, module)
        shutil.copyfile(template_file, spider_file)
        render_templatefile(spider_file, **tvars)
        print "Created spider %r using template %r in module:" % (name, \
            template_name)
        print "  %s.%s" % (spiders_module.__name__, module) 

def project_data_dir(project='default'):
    """Return the current project data dir, creating it if it doesn't exist"""
    if not inside_project():
        raise NotConfigured("Not inside a project")
    cfg = get_config()
    if cfg.has_option(DATADIR_CFG_SECTION, project):
        d = cfg.get(DATADIR_CFG_SECTION, project)
    else:
        scrapy_cfg = closest_scrapy_cfg()
        if not scrapy_cfg:
            raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir")
        d = abspath(join(dirname(scrapy_cfg), '.scrapy'))
    if not exists(d):
        makedirs(d)
    return d 

def test_runspider_unable_to_load(self):
        tmpdir = self.mktemp()
        os.mkdir(tmpdir)
        fname = abspath(join(tmpdir, 'myspider.txt'))
        with open(fname, 'w') as f:
            f.write("")
        p = self.proc('runspider', fname)
        log = p.stderr.read()
        self.assert_("Unable to load" in log) 

def root(*args):
    """
    Get the absolute path of the given path relative to the project root.
    """
    return join(abspath(dirname(__file__)), *args) 

def setUpClass(cls):
        DATA = join(dirname(abspath(__file__)), 'settings')
        sys.path.insert(1, DATA) 

def resource_path(relative_path):
    """
    Get absolute path to resource, works for dev and PyInstaller binary
    https://stackoverflow.com/questions/7674790/bundling-data-files-with-pyinstaller-onefile
    """
    try:
        # PyInstaller creates a temp folder and stores path in _MEIPASS
        base_path = sys._MEIPASS
    except AttributeError:
        base_path = path.abspath(".")

    return path.join(base_path, relative_path) 

def setUp(self):
        # Define data file and read X and y
        # Generate some data if the source data is missing
        this_directory = path.abspath(path.dirname(__file__))
        mat_file = 'pima.mat'
        try:
            mat = loadmat(path.join(*[this_directory, 'data', mat_file]))

        except TypeError:
            print('{data_file} does not exist. Use generated data'.format(
                data_file=mat_file))
            X, y = generate_data(train_only=True)  # load data
        except IOError:
            print('{data_file} does not exist. Use generated data'.format(
                data_file=mat_file))
            X, y = generate_data(train_only=True)  # load data
        else:
            X = mat['X']
            y = mat['y'].ravel()
            X, y = check_X_y(X, y)

        self.X_train, self.X_test, self.y_train, self.y_test = \
            train_test_split(X, y, test_size=0.4, random_state=42)

        self.clf = XGBOD(random_state=42)
        self.clf.fit(self.X_train, self.y_train)

        self.roc_floor = 0.8 

def get_version():

    here = path.abspath(path.dirname(__file__))

    with open(path.join(here, "version.txt"), "r") as f:
        return f.readline().splitlines()[0] 

def publish_feed(self):
        """Generate an RSS feed"""
        data = {
            'title': 'Best comments of %s' % self.subreddit.path,
            'url': self.reddit.config.reddit_url + self.subreddit.path,
            'description': 'Best comments from %s' % self.subreddit.path,
            'rss2update': formatdate(),
            'entries': [
                {
                    'title':'[%d] %s on %s' %
                            (comment.score, comment.author, comment.submission.title),
                    'url': self.reddit.config.reddit_url + comment.permalink,
                    'text': comment.body_html,
                    'author': comment.author,
                    'categories': [comment.subreddit.display_name],
                    'rss2update': formatdate(comment.created_utc)
                }
                for comment in self.comments
            ]
        }
        renderer = pystache.Renderer()
        output = renderer.render_path(
            path.join(path.dirname(path.abspath(__file__)), 'rss.mustache'),
            data)
        with open('best-comment.xml', 'w', encoding='utf8') as text_file:
            text_file.write(output)
        return True 

def _get_file_path(self, relative_file):
        return join(dirname(abspath(__file__)), relative_file) 

def _samefile(src, dst):
    # Macintosh, Unix.
    if hasattr(os.path, 'samefile'):
        try:
            return os.path.samefile(src, dst)
        except OSError:
            return False

    # All other platforms: check for same pathname.
    return (os.path.normcase(os.path.abspath(src)) ==
            os.path.normcase(os.path.abspath(dst))) 

def _destinsrc(src, dst):
    src = abspath(src)
    dst = abspath(dst)
    if not src.endswith(os.path.sep):
        src += os.path.sep
    if not dst.endswith(os.path.sep):
        dst += os.path.sep
    return dst.startswith(src) 

def _samefile(src, dst):
    # Macintosh, Unix.
    if hasattr(os.path, 'samefile'):
        try:
            return os.path.samefile(src, dst)
        except OSError:
            return False

    # All other platforms: check for same pathname.
    return (os.path.normcase(os.path.abspath(src)) ==
            os.path.normcase(os.path.abspath(dst))) 

def _destinsrc(src, dst):
    src = abspath(src)
    dst = abspath(dst)
    if not src.endswith(os.path.sep):
        src += os.path.sep
    if not dst.endswith(os.path.sep):
        dst += os.path.sep
    return dst.startswith(src) 

def test_get_home_dir_1():
    """Testcase for py2exe logic, un-compressed lib
    """
    unfrozen = path.get_home_dir()
    sys.frozen = True

    #fake filename for IPython.__init__
    IPython.__file__ = abspath(join(HOME_TEST_DIR, "Lib/IPython/__init__.py"))

    home_dir = path.get_home_dir()
    nt.assert_equal(home_dir, unfrozen) 

def test_get_home_dir_2():
    """Testcase for py2exe logic, compressed lib
    """
    unfrozen = path.get_home_dir()
    sys.frozen = True
    #fake filename for IPython.__init__
    IPython.__file__ = abspath(join(HOME_TEST_DIR, "Library.zip/IPython/__init__.py")).lower()

    home_dir = path.get_home_dir(True)
    nt.assert_equal(home_dir, unfrozen) 

def __repr__(self):
        """return absolute path to file
        """
        return abspath(self.path) 

def project_root_locate(cls, path='.'):
        cur_path = os.path.abspath(path)
        while cur_path and cur_path != '/':
            lamp_path = os.path.join(cur_path, cls.CONFIG_NAME)
            if os.path.isfile(lamp_path):
                return cur_path, lamp_path
            cur_path, tail = os.path.split(cur_path)
        logger.error('Could not find lamp.json file. Please create one or retry from a project '
                     'with a lamp.json file.')
        sys.exit() 

def helm_path(self):
        return abspath(os.path.join(self.path, self.lamp_content['helm_chart'])) 

def can_patch(self, filename):
    """ Check if specified filename can be patched. Returns None if file can
    not be found among source filenames. False if patch can not be applied
    clearly. True otherwise.

    :returns: True, False or None
    """
    filename = abspath(filename)
    for p in self.items:
      if filename == abspath(p.source):
        return self._match_file_hunks(filename, p.hunks)
    return None 

def _match_file_hunks(self, filepath, hunks):
    matched = True
    fp = open(abspath(filepath), 'rb')

    class NoMatch(Exception):
      pass

    lineno = 1
    line = fp.readline()
    hno = None
    try:
      for hno, h in enumerate(hunks):
        # skip to first line of the hunk
        while lineno < h.starttgt:
          if not len(line): # eof
            debug("check failed - premature eof before hunk: %d" % (hno+1))
            raise NoMatch
          line = fp.readline()
          lineno += 1
        for hline in h.text:
          if hline.startswith(b"-"):
            continue
          if not len(line):
            debug("check failed - premature eof on hunk: %d" % (hno+1))
            # todo: \ No newline at the end of file
            raise NoMatch
          if line.rstrip(b"\r\n") != hline[1:].rstrip(b"\r\n"):
            debug("file is not patched - failed hunk: %d" % (hno+1))
            raise NoMatch
          line = fp.readline()
          lineno += 1

    except NoMatch:
      matched = False
      # todo: display failed hunk, i.e. expected/found

    fp.close()
    return matched 

def upload_archive(exp_name, archive_excludes, s3_bucket):
    import hashlib, os.path as osp, subprocess, tempfile, uuid, sys

    # Archive this package
    thisfile_dir = osp.dirname(osp.abspath(__file__))
    pkg_parent_dir = osp.abspath(osp.join(thisfile_dir, '..', '..'))
    pkg_subdir = osp.basename(osp.abspath(osp.join(thisfile_dir, '..')))
    assert osp.abspath(__file__) == osp.join(pkg_parent_dir, pkg_subdir, 'scripts', 'launch.py'), 'You moved me!'

    # Run tar
    tmpdir = tempfile.TemporaryDirectory()
    local_archive_path = osp.join(tmpdir.name, '{}.tar.gz'.format(uuid.uuid4()))
    tar_cmd = ["tar", "-zcvf", local_archive_path, "-C", pkg_parent_dir]
    for pattern in archive_excludes:
        tar_cmd += ["--exclude", pattern]
    tar_cmd += ["-h", pkg_subdir]
    highlight(" ".join(tar_cmd))

    if sys.platform == 'darwin':
        # Prevent Mac tar from adding ._* files
        env = os.environ.copy()
        env['COPYFILE_DISABLE'] = '1'
        subprocess.check_call(tar_cmd, env=env)
    else:
        subprocess.check_call(tar_cmd)

    # Construct remote path to place the archive on S3
    with open(local_archive_path, 'rb') as f:
        archive_hash = hashlib.sha224(f.read()).hexdigest()
    remote_archive_path = '{}/{}_{}.tar.gz'.format(s3_bucket, exp_name, archive_hash)

    # Upload
    upload_cmd = ["aws", "s3", "cp", local_archive_path, remote_archive_path]
    highlight(" ".join(upload_cmd))
    subprocess.check_call(upload_cmd)

    presign_cmd = ["aws", "s3", "presign", remote_archive_path, "--expires-in", str(60 * 60 * 24 * 30)]
    highlight(" ".join(presign_cmd))
    remote_url = subprocess.check_output(presign_cmd).decode("utf-8").strip()
    return remote_url 

def test_user_dir():
    if USER_KEY in env:
        del env[USER_KEY]
    home_dir = nibe.get_home_dir()
    if os.name == "posix":
        exp = pjoin(home_dir, '.nipy')
    else:
        exp = pjoin(home_dir, '_nipy')
    assert_equal(exp, nibe.get_nipy_user_dir())
    env[USER_KEY] = '/a/path'
    assert_equal(abspath('/a/path'), nibe.get_nipy_user_dir()) 

def get_output_dir(imdb, weights_filename):
  """Return the directory where experimental artifacts are placed.
  If the directory does not exist, it is created.

  A canonical path is built using the name from an imdb and a network
  (if not None).
  """
  outdir = osp.abspath(osp.join(__C.ROOT_DIR, 'output', __C.EXP_DIR, imdb.name))
  if weights_filename is None:
    weights_filename = 'default'
  outdir = osp.join(outdir, weights_filename)
  if not os.path.exists(outdir):
    os.makedirs(outdir)
  return outdir 

def get_output_tb_dir(imdb, weights_filename):
  """Return the directory where tensorflow summaries are placed.
  If the directory does not exist, it is created.

  A canonical path is built using the name from an imdb and a network
  (if not None).
  """
  outdir = osp.abspath(osp.join(__C.ROOT_DIR, 'tensorboard', __C.EXP_DIR, imdb.name))
  if weights_filename is None:
    weights_filename = 'default'
  outdir = osp.join(outdir, weights_filename)
  if not os.path.exists(outdir):
    os.makedirs(outdir)
  return outdir 

def cache_path(self):
    cache_path = osp.abspath(osp.join(cfg.DATA_DIR, 'cache'))
    if not os.path.exists(cache_path):
      os.makedirs(cache_path)
    return cache_path 

def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
                 dry_run=0, owner=None, group=None, logger=None):
    """Create an archive file (eg. zip or tar).

    'base_name' is the name of the file to create, minus any format-specific
    extension; 'format' is the archive format: one of "zip", "tar", "bztar"
    or "gztar".

    'root_dir' is a directory that will be the root directory of the
    archive; ie. we typically chdir into 'root_dir' before creating the
    archive.  'base_dir' is the directory where we start archiving from;
    ie. 'base_dir' will be the common prefix of all files and
    directories in the archive.  'root_dir' and 'base_dir' both default
    to the current directory.  Returns the name of the archive file.

    'owner' and 'group' are used when creating a tar archive. By default,
    uses the current owner and group.
    """
    save_cwd = os.getcwd()
    if root_dir is not None:
        if logger is not None:
            logger.debug("changing into '%s'", root_dir)
        base_name = os.path.abspath(base_name)
        if not dry_run:
            os.chdir(root_dir)

    if base_dir is None:
        base_dir = os.curdir

    kwargs = {'dry_run': dry_run, 'logger': logger}

    try:
        format_info = _ARCHIVE_FORMATS[format]
    except KeyError:
        raise ValueError, "unknown archive format '%s'" % format

    func = format_info[0]
    for arg, val in format_info[1]:
        kwargs[arg] = val

    if format != 'zip':
        kwargs['owner'] = owner
        kwargs['group'] = group

    try:
        filename = func(base_name, base_dir, **kwargs)
    finally:
        if root_dir is not None:
            if logger is not None:
                logger.debug("changing back to '%s'", save_cwd)
            os.chdir(save_cwd)

    return filename 

def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
                 dry_run=0, owner=None, group=None, logger=None):
    """Create an archive file (eg. zip or tar).

    'base_name' is the name of the file to create, minus any format-specific
    extension; 'format' is the archive format: one of "zip", "tar", "bztar"
    or "gztar".

    'root_dir' is a directory that will be the root directory of the
    archive; ie. we typically chdir into 'root_dir' before creating the
    archive.  'base_dir' is the directory where we start archiving from;
    ie. 'base_dir' will be the common prefix of all files and
    directories in the archive.  'root_dir' and 'base_dir' both default
    to the current directory.  Returns the name of the archive file.

    'owner' and 'group' are used when creating a tar archive. By default,
    uses the current owner and group.
    """
    save_cwd = os.getcwd()
    if root_dir is not None:
        if logger is not None:
            logger.debug("changing into '%s'", root_dir)
        base_name = os.path.abspath(base_name)
        if not dry_run:
            os.chdir(root_dir)

    if base_dir is None:
        base_dir = os.curdir

    kwargs = {'dry_run': dry_run, 'logger': logger}

    try:
        format_info = _ARCHIVE_FORMATS[format]
    except KeyError:
        raise ValueError("unknown archive format '%s'" % format)

    func = format_info[0]
    for arg, val in format_info[1]:
        kwargs[arg] = val

    if format != 'zip':
        kwargs['owner'] = owner
        kwargs['group'] = group

    try:
        filename = func(base_name, base_dir, **kwargs)
    finally:
        if root_dir is not None:
            if logger is not None:
                logger.debug("changing back to '%s'", save_cwd)
            os.chdir(save_cwd)

    return filename 

def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
                 dry_run=0, owner=None, group=None, logger=None):
    """Create an archive file (eg. zip or tar).

    'base_name' is the name of the file to create, minus any format-specific
    extension; 'format' is the archive format: one of "zip", "tar", "bztar"
    or "gztar".

    'root_dir' is a directory that will be the root directory of the
    archive; ie. we typically chdir into 'root_dir' before creating the
    archive.  'base_dir' is the directory where we start archiving from;
    ie. 'base_dir' will be the common prefix of all files and
    directories in the archive.  'root_dir' and 'base_dir' both default
    to the current directory.  Returns the name of the archive file.

    'owner' and 'group' are used when creating a tar archive. By default,
    uses the current owner and group.
    """
    save_cwd = os.getcwd()
    if root_dir is not None:
        if logger is not None:
            logger.debug("changing into '%s'", root_dir)
        base_name = os.path.abspath(base_name)
        if not dry_run:
            os.chdir(root_dir)

    if base_dir is None:
        base_dir = os.curdir

    kwargs = {'dry_run': dry_run, 'logger': logger}

    try:
        format_info = _ARCHIVE_FORMATS[format]
    except KeyError:
        raise ValueError("unknown archive format '%s'" % format)

    func = format_info[0]
    for arg, val in format_info[1]:
        kwargs[arg] = val

    if format != 'zip':
        kwargs['owner'] = owner
        kwargs['group'] = group

    try:
        filename = func(base_name, base_dir, **kwargs)
    finally:
        if root_dir is not None:
            if logger is not None:
                logger.debug("changing back to '%s'", save_cwd)
            os.chdir(save_cwd)

    return filename 

def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
                 dry_run=0, owner=None, group=None, logger=None):
    """Create an archive file (eg. zip or tar).

    'base_name' is the name of the file to create, minus any format-specific
    extension; 'format' is the archive format: one of "zip", "tar", "bztar"
    or "gztar".

    'root_dir' is a directory that will be the root directory of the
    archive; ie. we typically chdir into 'root_dir' before creating the
    archive.  'base_dir' is the directory where we start archiving from;
    ie. 'base_dir' will be the common prefix of all files and
    directories in the archive.  'root_dir' and 'base_dir' both default
    to the current directory.  Returns the name of the archive file.

    'owner' and 'group' are used when creating a tar archive. By default,
    uses the current owner and group.
    """
    save_cwd = os.getcwd()
    if root_dir is not None:
        if logger is not None:
            logger.debug("changing into '%s'", root_dir)
        base_name = os.path.abspath(base_name)
        if not dry_run:
            os.chdir(root_dir)

    if base_dir is None:
        base_dir = os.curdir

    kwargs = {'dry_run': dry_run, 'logger': logger}

    try:
        format_info = _ARCHIVE_FORMATS[format]
    except KeyError:
        raise ValueError("unknown archive format '%s'" % format)

    func = format_info[0]
    for arg, val in format_info[1]:
        kwargs[arg] = val

    if format != 'zip':
        kwargs['owner'] = owner
        kwargs['group'] = group

    try:
        filename = func(base_name, base_dir, **kwargs)
    finally:
        if root_dir is not None:
            if logger is not None:
                logger.debug("changing back to '%s'", save_cwd)
            os.chdir(save_cwd)

    return filename 

def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
                 dry_run=0, owner=None, group=None, logger=None):
    """Create an archive file (eg. zip or tar).

    'base_name' is the name of the file to create, minus any format-specific
    extension; 'format' is the archive format: one of "zip", "tar", "bztar"
    or "gztar".

    'root_dir' is a directory that will be the root directory of the
    archive; ie. we typically chdir into 'root_dir' before creating the
    archive.  'base_dir' is the directory where we start archiving from;
    ie. 'base_dir' will be the common prefix of all files and
    directories in the archive.  'root_dir' and 'base_dir' both default
    to the current directory.  Returns the name of the archive file.

    'owner' and 'group' are used when creating a tar archive. By default,
    uses the current owner and group.
    """
    save_cwd = os.getcwd()
    if root_dir is not None:
        if logger is not None:
            logger.debug("changing into '%s'", root_dir)
        base_name = os.path.abspath(base_name)
        if not dry_run:
            os.chdir(root_dir)

    if base_dir is None:
        base_dir = os.curdir

    kwargs = {'dry_run': dry_run, 'logger': logger}

    try:
        format_info = _ARCHIVE_FORMATS[format]
    except KeyError:
        raise ValueError("unknown archive format '%s'" % format)

    func = format_info[0]
    for arg, val in format_info[1]:
        kwargs[arg] = val

    if format != 'zip':
        kwargs['owner'] = owner
        kwargs['group'] = group

    try:
        filename = func(base_name, base_dir, **kwargs)
    finally:
        if root_dir is not None:
            if logger is not None:
                logger.debug("changing back to '%s'", save_cwd)
            os.chdir(save_cwd)

    return filename 

def _process_template_text(source, *, filename=None):
    if filename:
        #PY36
        log.debug("Processing template %s" % osp.abspath(str(filename)))

    root = {}
    d = root
    d['targets'] = {}
    d['withs'] = {}

    parents = []

    lines = source.splitlines(keepends=True)
    it = templateparser.get_targets_and_replace(lines)
    while True:
        try:
            tp, value = next(it)
        except StopIteration as e:
            rval = e.value
            break
        if tp == 'target':
            d['targets'][value] = []

        if tp == 'with':
            parents.append(d)
            if not value in d['withs']:
                newd = {}
                newd['targets'] = {}
                newd['withs'] = {}
                d['withs'][value] = newd

            d = d['withs'][value]

        if tp=='endwith':
            try:
                d = parents.pop()
            except IndexError:
                it.throw(templateparser.BadToken("Found endwith with no matching with."))

    if parents:
        raise templateparser.BadTemplate("Reched the end of the file and "
        "didn't find a closing 'endwith' tag for all the with tags: The "
        "following remain open:\n%s." % '\n'.join(str(tuple(parent['withs'].keys())) for parent in parents))

    return rval, root 

def preprocess(self, source, name=None, filename=None):
        if filename:
            log.debug("Processing template %s" % osp.abspath(filename))

        root = {}
        d = root
        d['targets'] = {}
        d['withs'] = {}

        parents = []

        lines = source.splitlines(keepends=True)
        it = templateparser.get_targets_and_replace(lines)
        while True:
            try:
                tp, value = next(it)
            except StopIteration as e:
                rval = e.value
                break
            if tp == 'target':
                d['targets'][value] = []

            if tp == 'with':
                parents.append(d)
                if not value in d['withs']:
                    newd = {}
                    newd['targets'] = {}
                    newd['withs'] = {}
                    d['withs'][value] = newd

                d = d['withs'][value]

            if tp=='endwith':
                try:
                    d = parents.pop()
                except IndexError:
                    it.throw(templateparser.BadToken("Found endwith with no matching with."))

        if parents:
            raise templateparser.BadTemplate("Reched the end of the file and "
            "didn't find a closing 'endwith' tag for all the with tags: The "
            "following remain open:\n%s." % '\n'.join(str(tuple(parent['withs'].keys())) for parent in parents))

        self._root = root
        return rval 

def ensure_virtualenv(args, return_values):
    """Ensure we have a valid virtualenv."""
    def adjust_options(options, args):
        # TODO-TEST: proper error message with no arguments
        venv_path = return_values.venv_path = args[0]

        if venv_path == DEFAULT_VIRTUALENV_PATH or options.prompt == '<dirname>':
            from os.path import abspath, basename, dirname
            options.prompt = '(%s)' % basename(dirname(abspath(venv_path)))
        # end of option munging.

        # there are two python interpreters involved here:
        # 1) the interpreter we're instructing virtualenv to copy
        if options.python is None:
            source_python = None
        else:
            source_python = virtualenv.resolve_interpreter(options.python)
        # 2) the interpreter virtualenv will create
        destination_python = venv_python(venv_path)

        if exists(destination_python):
            reason = invalid_virtualenv_reason(venv_path, source_python, destination_python, options)
            if reason:
                info('Removing invalidated virtualenv. (%s)' % reason)
                run(('rm', '-rf', venv_path))
            else:
                info('Keeping valid virtualenv from previous run.')
                raise SystemExit(0)  # looks good! we're done here.

    # this is actually a documented extension point:
    #   http://virtualenv.readthedocs.org/en/latest/reference.html#adjust_options
    import virtualenv
    virtualenv.adjust_options = adjust_options

    from sys import argv
    argv[:] = ('virtualenv',) + args
    info(colorize(argv))
    raise_on_failure(virtualenv.main)
    # There might not be a venv_path if doing something like "venv= --version"
    # and not actually asking virtualenv to make a venv.
    if return_values.venv_path is not None:
        run(('rm', '-rf', join(return_values.venv_path, 'local'))) 

def publish(o, type, releases):
    def _publish(o, release):
        arch = ''
        filename = release.split(os.path.sep)[-1]

        extension = None
        for EXTENSION in EXTENSIONS:
            if filename.endswith(EXTENSION):
                extension = EXTENSION
                filename = filename.replace(extension, '')
                break
        if extension is None:
            raise Exception("Extension of %s is not handled" % filename)

        # keep _all or _amd64
        if filename.count('_') > 1:
            arch = '_' + filename.split('_')[-1]

        release_dir = PUBLISH_DIRS[type]
        release_filename = 'openerp_%s.%s%s%s' % (version, timestamp, arch, extension)
        release_path = join(o.pub, release_dir, release_filename)

        system('mkdir -p %s' % join(o.pub, release_dir))
        shutil.move(join(o.build_dir, release), release_path)

        # Latest/symlink handler
        release_abspath = abspath(release_path)
        latest_abspath = release_abspath.replace(timestamp, 'latest')

        if os.path.islink(latest_abspath):
            os.unlink(latest_abspath)

        os.symlink(release_abspath, latest_abspath)

        return release_path

    published = []
    if isinstance(releases, basestring):
        published.append(_publish(o, releases))
    elif isinstance(releases, list):
        for release in releases:
            published.append(_publish(o, release))
    return published 

def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
                 dry_run=0, owner=None, group=None, logger=None):
    """Create an archive file (eg. zip or tar).

    'base_name' is the name of the file to create, minus any format-specific
    extension; 'format' is the archive format: one of "zip", "tar", "bztar"
    or "gztar".

    'root_dir' is a directory that will be the root directory of the
    archive; ie. we typically chdir into 'root_dir' before creating the
    archive.  'base_dir' is the directory where we start archiving from;
    ie. 'base_dir' will be the common prefix of all files and
    directories in the archive.  'root_dir' and 'base_dir' both default
    to the current directory.  Returns the name of the archive file.

    'owner' and 'group' are used when creating a tar archive. By default,
    uses the current owner and group.
    """
    save_cwd = os.getcwd()
    if root_dir is not None:
        if logger is not None:
            logger.debug("changing into '%s'", root_dir)
        base_name = os.path.abspath(base_name)
        if not dry_run:
            os.chdir(root_dir)

    if base_dir is None:
        base_dir = os.curdir

    kwargs = {'dry_run': dry_run, 'logger': logger}

    try:
        format_info = _ARCHIVE_FORMATS[format]
    except KeyError:
        raise ValueError("unknown archive format '%s'" % format)

    func = format_info[0]
    for arg, val in format_info[1]:
        kwargs[arg] = val

    if format != 'zip':
        kwargs['owner'] = owner
        kwargs['group'] = group

    try:
        filename = func(base_name, base_dir, **kwargs)
    finally:
        if root_dir is not None:
            if logger is not None:
                logger.debug("changing back to '%s'", save_cwd)
            os.chdir(save_cwd)

    return filename 

def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
                 dry_run=0, owner=None, group=None, logger=None):
    """Create an archive file (eg. zip or tar).

    'base_name' is the name of the file to create, minus any format-specific
    extension; 'format' is the archive format: one of "zip", "tar", "bztar"
    or "gztar".

    'root_dir' is a directory that will be the root directory of the
    archive; ie. we typically chdir into 'root_dir' before creating the
    archive.  'base_dir' is the directory where we start archiving from;
    ie. 'base_dir' will be the common prefix of all files and
    directories in the archive.  'root_dir' and 'base_dir' both default
    to the current directory.  Returns the name of the archive file.

    'owner' and 'group' are used when creating a tar archive. By default,
    uses the current owner and group.
    """
    save_cwd = os.getcwd()
    if root_dir is not None:
        if logger is not None:
            logger.debug("changing into '%s'", root_dir)
        base_name = os.path.abspath(base_name)
        if not dry_run:
            os.chdir(root_dir)

    if base_dir is None:
        base_dir = os.curdir

    kwargs = {'dry_run': dry_run, 'logger': logger}

    try:
        format_info = _ARCHIVE_FORMATS[format]
    except KeyError:
        raise ValueError("unknown archive format '%s'" % format)

    func = format_info[0]
    for arg, val in format_info[1]:
        kwargs[arg] = val

    if format != 'zip':
        kwargs['owner'] = owner
        kwargs['group'] = group

    try:
        filename = func(base_name, base_dir, **kwargs)
    finally:
        if root_dir is not None:
            if logger is not None:
                logger.debug("changing back to '%s'", save_cwd)
            os.chdir(save_cwd)

    return filename 

def __init__(self, data_dir='configs'):
        Tracker.__init__(self)
        # data profile
        # self.data_dir = data_dir
        local_path = os.path.dirname(os.path.abspath(__file__))
        self.data_dir = os.path.join(local_path, data_dir)
        self.file_url = 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/mdbt_multiwoz_sys.zip'

        self.validation_url = os.path.join(self.data_dir, 'data/validate.json')
        self.word_vectors_url = os.path.join(self.data_dir, 'word-vectors/paragram_300_sl999.txt')
        self.training_url = os.path.join(self.data_dir, 'data/train.json')
        self.ontology_url = os.path.join(self.data_dir, 'data/ontology.json')
        self.testing_url = os.path.join(self.data_dir, 'data/test.json')
        self.model_url = os.path.join(self.data_dir, 'models/model-1')
        self.graph_url = os.path.join(self.data_dir, 'graphs/graph-1')
        self.results_url = os.path.join(self.data_dir, 'results/log-1.txt')
        self.kb_url = os.path.join(self.data_dir, 'data/')  # not used
        self.train_model_url = os.path.join(self.data_dir, 'train_models/model-1')
        self.train_graph_url = os.path.join(self.data_dir, 'train_graph/graph-1')

        self.auto_download()

        print('Configuring MDBT model...')
        self.word_vectors = load_word_vectors(self.word_vectors_url)

        # Load the ontology and extract the feature vectors
        self.ontology, self.ontology_vectors, self.slots = load_ontology(self.ontology_url, self.word_vectors)

        # Load and process the training data
        self.dialogues, self.actual_dialogues = load_woz_data(self.testing_url, self.word_vectors, self.ontology)
        self.no_dialogues = len(self.dialogues)

        self.model_variables = model_definition(self.ontology_vectors, len(self.ontology), self.slots, num_hidden=None,
                                                bidir=True, net_type=None, test=True, dev='cpu')
        self.state = init_state()
        _config = tf.ConfigProto()
        _config.gpu_options.allow_growth = True
        _config.allow_soft_placement = True
        self.sess = tf.Session(config=_config)
        self.param_restored = False
        self.det_dic = {}
        for domain, dic in REF_USR_DA.items():
            for key, value in dic.items():
                assert '-' not in key
                self.det_dic[key.lower()] = key + '-' + domain
                self.det_dic[value.lower()] = key + '-' + domain
        self.value_dict = json.load(open(os.path.join(dirname(dirname(dirname(dirname(dirname(dirname(abspath(__file__))))))), 'data/multiwoz/value_dict.json')))