Python os.popen() Examples

def main():
    p = os.popen('du ' + ' '.join(sys.argv[1:]), 'r')
    total, d = None, {}
    for line in p.readlines():
        i = 0
        while line[i] in '0123456789': i = i+1
        size = eval(line[:i])
        while line[i] in ' \t': i = i+1
        filename = line[i:-1]
        comps = filename.split('/')
        if comps[0] == '': comps[0] = '/'
        if comps[len(comps)-1] == '': del comps[len(comps)-1]
        total, d = store(size, comps, total, d)
    try:
        display(total, d)
    except IOError, e:
        if e.errno != errno.EPIPE:
            raise 

def symbols(lib=PYTHONLIB,types=('T','C','D')):

    lines = os.popen(NM % lib).readlines()
    lines = [s.strip() for s in lines]
    symbols = {}
    for line in lines:
        if len(line) == 0 or ':' in line:
            continue
        items = line.split()
        if len(items) != 3:
            continue
        address, type, name = items
        if type not in types:
            continue
        symbols[name] = address,type
    return symbols 

def _read_output(commandstring):
    """Output from successful command execution or None"""
    # Similar to os.popen(commandstring, "r").read(),
    # but without actually using os.popen because that
    # function is not usable during python bootstrap.
    # tempfile is also not available then.
    import contextlib
    try:
        import tempfile
        fp = tempfile.NamedTemporaryFile()
    except ImportError:
        fp = open("/tmp/_osx_support.%s"%(
            os.getpid(),), "w+b")

    with contextlib.closing(fp) as fp:
        cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
        return fp.read().strip() if not os.system(cmd) else None 

def _findLib_gcc(name):
        expr = r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name)
        fdout, ccout = tempfile.mkstemp()
        os.close(fdout)
        cmd = 'if type gcc >/dev/null 2>&1; then CC=gcc; elif type cc >/dev/null 2>&1; then CC=cc;else exit 10; fi;' \
              'LANG=C LC_ALL=C $CC -Wl,-t -o ' + ccout + ' 2>&1 -l' + name
        try:
            f = os.popen(cmd)
            try:
                trace = f.read()
            finally:
                rv = f.close()
        finally:
            try:
                os.unlink(ccout)
            except OSError, e:
                if e.errno != errno.ENOENT:
                    raise 

def _get_soname(f):
            # assuming GNU binutils / ELF
            if not f:
                return None
            cmd = 'if ! type objdump >/dev/null 2>&1; then exit 10; fi;' \
                  "objdump -p -j .dynamic 2>/dev/null " + f
            f = os.popen(cmd)
            dump = f.read()
            rv = f.close()
            if rv == 10:
                raise OSError, 'objdump command not found'
            f = os.popen(cmd)
            try:
                data = f.read()
            finally:
                f.close()
            res = re.search(r'\sSONAME\s+([^\s]+)', data)
            if not res:
                return None
            return res.group(1) 

def _findLib_crle(name, is64):
            if not os.path.exists('/usr/bin/crle'):
                return None

            if is64:
                cmd = 'env LC_ALL=C /usr/bin/crle -64 2>/dev/null'
            else:
                cmd = 'env LC_ALL=C /usr/bin/crle 2>/dev/null'

            for line in os.popen(cmd).readlines():
                line = line.strip()
                if line.startswith('Default Library Path (ELF):'):
                    paths = line.split()[4]

            if not paths:
                return None

            for dir in paths.split(":"):
                libfile = os.path.join(dir, "lib%s.so" % name)
                if os.path.exists(libfile):
                    return libfile

            return None 

def testLoadTkFailure(self):
        old_display = None
        if sys.platform.startswith(('win', 'darwin', 'cygwin')):
            # no failure possible on windows?

            # XXX Maybe on tk older than 8.4.13 it would be possible,
            # see tkinter.h.
            return
        with test_support.EnvironmentVarGuard() as env:
            if 'DISPLAY' in os.environ:
                del env['DISPLAY']
                # on some platforms, deleting environment variables
                # doesn't actually carry through to the process level
                # because they don't support unsetenv
                # If that's the case, abort.
                display = os.popen('echo $DISPLAY').read().strip()
                if display:
                    return

            tcl = Tcl()
            self.assertRaises(TclError, tcl.winfo_geometry)
            self.assertRaises(TclError, tcl.loadtk) 

def _syscmd_uname(option,default=''):

    """ Interface to the system's uname command.
    """
    if sys.platform in ('dos','win32','win16','os2'):
        # XXX Others too ?
        return default
    try:
        f = os.popen('uname %s 2> %s' % (option, DEV_NULL))
    except (AttributeError,os.error):
        return default
    output = string.strip(f.read())
    rc = f.close()
    if not output or rc:
        return default
    else:
        return output 

def open(self, url, new=0, autoraise=True):
            if self._name == 'default':
                script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser
            else:
                script = '''
                   tell application "%s"
                       activate
                       open location "%s"
                   end
                   '''%(self._name, url.replace('"', '%22'))

            osapipe = os.popen("osascript", "w")
            if osapipe is None:
                return False

            osapipe.write(script)
            rc = osapipe.close()
            return not rc


    # Don't clear _tryorder or _browsers since OS X can use above Unix support
    # (but we prefer using the OS X specific stuff) 

def test_poll2(self):
        cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done'
        p = os.popen(cmd, 'r')
        pollster = select.poll()
        pollster.register( p, select.POLLIN )
        for tout in (0, 1000, 2000, 4000, 8000, 16000) + (-1,)*10:
            fdlist = pollster.poll(tout)
            if (fdlist == []):
                continue
            fd, flags = fdlist[0]
            if flags & select.POLLHUP:
                line = p.readline()
                if line != "":
                    self.fail('error: pipe seems to be closed, but still returns data')
                continue

            elif flags & select.POLLIN:
                line = p.readline()
                if not line:
                    break
                continue
            else:
                self.fail('Unexpected return value from select.poll: %s' % fdlist)
        p.close() 

def test_getgroups(self):
        with os.popen('id -G 2>/dev/null') as idg:
            groups = idg.read().strip()
            ret = idg.close()

        if ret != None or not groups:
            raise unittest.SkipTest("need working 'id -G'")

        # Issues 16698: OS X ABIs prior to 10.6 have limits on getgroups()
        if sys.platform == 'darwin':
            import sysconfig
            dt = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') or '10.0'
            if tuple(int(n) for n in dt.split('.')[0:2]) < (10, 6):
                raise unittest.SkipTest("getgroups(2) is broken prior to 10.6")

        # 'id -G' and 'os.getgroups()' should return the same
        # groups, ignoring order and duplicates.
        # #10822 - it is implementation defined whether posix.getgroups()
        # includes the effective gid so we include it anyway, since id -G does
        self.assertEqual(
                set([int(x) for x in groups.split()]),
                set(posix.getgroups() + [posix.getegid()])) 

def test_select(self):
        cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done'
        p = os.popen(cmd, 'r')
        for tout in (0, 1, 2, 4, 8, 16) + (None,)*10:
            if test_support.verbose:
                print 'timeout =', tout
            rfd, wfd, xfd = select.select([p], [], [], tout)
            if (rfd, wfd, xfd) == ([], [], []):
                continue
            if (rfd, wfd, xfd) == ([p], [], []):
                line = p.readline()
                if test_support.verbose:
                    print repr(line)
                if not line:
                    if test_support.verbose:
                        print 'EOF'
                    break
                continue
            self.fail('Unexpected return values from select():', rfd, wfd, xfd)
        p.close()

    # Issue 16230: Crash on select resized list 

def _popen(command, args):
    import os
    path = os.environ.get("PATH", os.defpath).split(os.pathsep)
    path.extend(('/sbin', '/usr/sbin'))
    for dir in path:
        executable = os.path.join(dir, command)
        if (os.path.exists(executable) and
            os.access(executable, os.F_OK | os.X_OK) and
            not os.path.isdir(executable)):
            break
    else:
        return None
    # LC_ALL to ensure English output, 2>/dev/null to prevent output on
    # stderr (Note: we don't have an example where the words we search for
    # are actually localized, but in theory some system could do so.)
    cmd = 'LC_ALL=C %s %s 2>/dev/null' % (executable, args)
    return os.popen(cmd) 

def find_existing_daemons(daemon_list):
    """Utility to find any running specified daemons, excluding those potential false matches.

    - daemon_list: a list of daemons to be closed, specified by string name. E.g. ['flowd', 'serviced']
    - return: (daemons, pids), a list of process strings as in "ps -eaf", and a list of their pid.
    """
    if isinstance(daemon_list, str):
        daemon_list = [daemon_list]

    exclude = {'egrep', 'PID', 'sudo', 'vi', 'vim', 'gdb', 'stap', '<defunct>'}
    s = os.popen('sudo ps -eaf | egrep "PID|%s"' % '|'.join(daemon_list)).read()
    pid_col = s.splitlines()[0].split().index('PID')
    daemons = [x for x in s.splitlines() if
               (not exclude.intersection(set(x.split()))) and
               re.search('|'.join(['[/ ]%s( |$)' % y for y in daemon_list]), x)]
    pids = [x.split()[pid_col] for x in daemons]
    return daemons, pids 

def close_existing_daemons(daemon_list, verbose=False):
    """Check and close any running specified daemons.

    - daemon_list: a list of daemons to be closed, specified by string name. E.g. ['flowd', 'serviced']
    """
    if isinstance(daemon_list, str):
        daemon_list = [daemon_list]

    daemons, pids = find_existing_daemons(daemon_list)
    for sig in ['SIGINT', 'SIGKILL']:   # Try SIGINT first. If fail, then do SIGKILL.
        if daemons:
            if verbose:
                print_green('These existing processes will be closed with %s:\n%s' %
                            (sig, '\n'.join(daemons)))
            os.popen('sudo kill -s %s %s' % (sig, ' '.join(pids)))
        # confirming the processes are killed
        for i in xrange(5):
            daemons, pids = find_existing_daemons(daemon_list)
            if not daemons:
                break
            else:
                time.sleep(0.1)
        if daemons:
            print_red('Failed to close these processes with %s:\n%s' %
                      (sig, '\n'.join(daemons))) 

def find_existing_daemons(daemon_list):
    """Utility to find any running specified daemons, excluding those potential false matches.

    - daemon_list: a list of daemons to be closed, specified by string name. E.g. ['flowd', 'serviced']
    - return: (daemons, pids), a list of process strings as in "ps -eaf", and a list of their pid.
    """
    if isinstance(daemon_list, str):
        daemon_list = [daemon_list]

    exclude = {'egrep', 'PID', 'sudo', 'vi', 'vim', 'gdb', 'stap', '<defunct>'}
    s = os.popen('sudo ps -eaf | egrep "PID|%s"' % '|'.join(daemon_list)).read()
    pid_col = s.splitlines()[0].split().index('PID')
    daemons = [x for x in s.splitlines() if
               (not exclude.intersection(set(x.split()))) and
               re.search('|'.join(['[/ ]%s( |$)' % y for y in daemon_list]), x)]
    pids = [x.split()[pid_col] for x in daemons]
    return daemons, pids 

def close_existing_daemons(daemon_list, verbose=False):
    """Check and close any running specified daemons.

    - daemon_list: a list of daemons to be closed, specified by string name. E.g. ['flowd', 'serviced']
    """
    if isinstance(daemon_list, str):
        daemon_list = [daemon_list]

    daemons, pids = find_existing_daemons(daemon_list)
    for sig in ['SIGINT', 'SIGKILL']:   # Try SIGINT first. If fail, then do SIGKILL.
        if daemons:
            if verbose:
                print_green('These existing processes will be closed with %s:\n%s' %
                            (sig, '\n'.join(daemons)))
            os.popen('sudo kill -s %s %s' % (sig, ' '.join(pids)))
        # confirming the processes are killed
        for i in xrange(5):
            daemons, pids = find_existing_daemons(daemon_list)
            if not daemons:
                break
            else:
                time.sleep(0.1)
        if daemons:
            print_red('Failed to close these processes with %s:\n%s' %
                      (sig, '\n'.join(daemons))) 

def test_no_os_popen(self):
        code = """
               import os

               foobar_cmd = "foobar -get -beer"
               answer = os.popen(foobar_cmd).read()

               if answer == nok":
                   try:
                       os.popen(os.popen('foobar -beer -please')).read()

                   except ValueError:
                       go_home()
               """
        errors = [(4, 0, 'N348'), (8, 8, 'N348')]
        self._assert_has_errors(code, checks.no_os_popen,
                                expected_errors=errors) 

def popen(cmd, mode="r", buffering=-1):
    if not isinstance(cmd, str):
        raise TypeError("invalid cmd type (%s, expected string)" % type(cmd))
    if mode not in ("r", "w"):
        raise ValueError("invalid mode %r" % mode)
    if buffering == 0 or buffering is None:
        raise ValueError("popen() does not support unbuffered streams")
    import subprocess, io
    if mode == "r":
        proc = subprocess.Popen(cmd,
                                shell=True,
                                stdout=subprocess.PIPE,
                                bufsize=buffering)
        return _wrap_close(io.TextIOWrapper(proc.stdout), proc)
    else:
        proc = subprocess.Popen(cmd,
                                shell=True,
                                stdin=subprocess.PIPE,
                                bufsize=buffering)
        return _wrap_close(io.TextIOWrapper(proc.stdin), proc)

# Helper for popen() -- a proxy for a file whose close waits for the process 

def write_bars_to_file(bars, filename, tz):
    """Creates CSV file from list of Bar instances"""
    date_format_str = "%Y%m%d %H%M%S"

    rows = [{'DateTime':  bar.datetime.astimezone(tz).strftime(date_format_str),
             'Open':	  bar.open,
             'High':	  bar.high,
             'Low':	      bar.low,
             'Close':	  bar.close,
             'Volume':	  bar.volume,
             } for bar in bars]

    if os.path.exists(filename):
        raise Exception("File already exists!")

    fd = os.popen("gzip > %s" % filename, 'w') if filename.endswith('.gz') else open(filename, 'w')

    with fd:
        csv_writer = csv.DictWriter(fd, ['DateTime', 'Open', 'High', 'Low', 'Close', 'Volume'])
        csv_writer.writeheader()
        csv_writer.writerows(rows) 

def setup_paper(self, root_el):
        try:
            fin = os.popen("paperconf -s 2> /dev/null")
            w, h = map(float, fin.read().split())
            fin.close()
        except:
            w, h = 612, 792     # default to Letter
        def walk(el):
            if el.tag == "{%s}page-layout-properties" % SNSD["style"] and \
                    not el.attrib.has_key("{%s}page-width" % SNSD["fo"]):
                el.attrib["{%s}page-width" % SNSD["fo"]] = "%.3fpt" % w
                el.attrib["{%s}page-height" % SNSD["fo"]] = "%.3fpt" % h
                el.attrib["{%s}margin-left" % SNSD["fo"]] = \
                        el.attrib["{%s}margin-right" % SNSD["fo"]] = \
                        "%.3fpt" % (.1 * w)
                el.attrib["{%s}margin-top" % SNSD["fo"]] = \
                        el.attrib["{%s}margin-bottom" % SNSD["fo"]] = \
                        "%.3fpt" % (.1 * h)
            else:
                for subel in el.getchildren(): walk(subel)
        walk(root_el) 

def exp_get_server_environment(self):
        os_lang = '.'.join( [x for x in locale.getdefaultlocale() if x] )
        if not os_lang:
            os_lang = 'NOT SET'
        environment = '\nEnvironment Information : \n' \
                     'System : %s\n' \
                     'OS Name : %s\n' \
                     %(platform.platform(), platform.os.name)
        if os.name == 'posix':
            if platform.system() == 'Linux':
                lsbinfo = os.popen('lsb_release -a').read()
                environment += '%s'% lsbinfo
            else:
                environment += 'Your System is not lsb compliant\n'
        environment += 'Operating System Release : %s\n' \
                    'Operating System Version : %s\n' \
                    'Operating System Architecture : %s\n' \
                    'Operating System Locale : %s\n'\
                    'Python Version : %s\n'\
                    'OpenERP-Server Version : %s'\
                    %(platform.release(), platform.version(), platform.architecture()[0],
                      os_lang, platform.python_version(),release.version)
        return environment 

def write_config(root=None):
    if root and os.name == 'nt':
        root = root.replace('\\', '/')  # For Windows
    if root and platform.system().startswith('CYGWIN'):  # For cygwin
        if root.startswith('/usr/lib'):
            cygwin_root = os.popen('cygpath -w /').read().strip().replace('\\', '/')
            root = cygwin_root + root[len('/usr'):]
        elif STATIC_ROOT.startswith('/cygdrive'):
            driver = STATIC_ROOT.split('/')
            cygwin_driver = '/'.join(driver[:3])
            win_driver = driver[2].upper() + ':'
            root = root.replace(cygwin_driver, win_driver)
    content = []
    with open_(PATH_CONFIG, encoding='utf-8') as f:
        for line in f:
            if root:
                if line.startswith('root'):
                    line = 'root={}{}'.format(root, os.linesep)
            content.append(line)
    with open_(PATH_CONFIG, 'w', encoding='utf-8') as f:
        f.writelines(content) 

def cpu_count():
        '''
        Returns the number of CPUs in the system
        '''
        if sys.platform == 'win32':
            try:
                num = int(os.environ['NUMBER_OF_PROCESSORS'])
            except (ValueError, KeyError):
                num = 0
        elif 'bsd' in sys.platform or sys.platform == 'darwin':
            try:
                num = int(os.popen('sysctl -n hw.ncpu').read())
            except ValueError:
                num = 0
        else:
            try:
                num = os.sysconf('SC_NPROCESSORS_ONLN')
            except (ValueError, OSError, AttributeError):
                num = 0

        if num >= 1:
            return num
        else:
            raise NotImplementedError('cannot determine number of cpus') 

def has_redis_launched() -> bool:
    """
    Check if redis-server has launched on this host
    """
    # check the owner of port 6379
    output = os.popen('lsof -i:6379').read()
    if len(output) == 0:
        return False
    '''
    Output should be shown like this:
    [COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME]
    '''
    rows = output.split('\n')[1:]
    for row in rows:
        row = row.split()
        if 'redis-ser' in row:
            return True
    return False 

def open_r(self, file):
        """t.open_r(file) and t.open_w(file) implement
        t.open(file, 'r') and t.open(file, 'w') respectively."""
        if not self.steps:
            return open(file, 'r')
        if self.steps[-1][1] == SINK:
            raise ValueError, \
                  'Template.open_r: pipeline ends width SINK'
        cmd = self.makepipeline(file, '')
        return os.popen(cmd, 'r') 

def pipeto(input, command):
    pipe = os.popen(command, 'w')
    copyliteral(input, pipe)
    pipe.close() 

def pipethrough(input, command, output):
    (fd, tempname) = tempfile.mkstemp()
    temp = os.fdopen(fd, 'w')
    copyliteral(input, temp)
    temp.close()
    pipe = os.popen(command + ' <' + tempname, 'r')
    copybinary(pipe, output)
    pipe.close()
    os.unlink(tempname) 

def _get_soname(f):
            if not f:
                return None
            cmd = "/usr/ccs/bin/dump -Lpv 2>/dev/null " + f
            f = os.popen(cmd)
            try:
                data = f.read()
            finally:
                f.close()
            res = re.search(r'\[.*\]\sSONAME\s+([^\s]+)', data)
            if not res:
                return None
            return res.group(1) 

def _findSoname_ldconfig(name):
            import struct
            if struct.calcsize('l') == 4:
                machine = os.uname()[4] + '-32'
            else:
                machine = os.uname()[4] + '-64'
            mach_map = {
                'x86_64-64': 'libc6,x86-64',
                'ppc64-64': 'libc6,64bit',
                'sparc64-64': 'libc6,64bit',
                's390x-64': 'libc6,64bit',
                'ia64-64': 'libc6,IA-64',
                }
            abi_type = mach_map.get(machine, 'libc6')

            # XXX assuming GLIBC's ldconfig (with option -p)
            expr = r'\s+(lib%s\.[^\s]+)\s+\(%s' % (re.escape(name), abi_type)
            f = os.popen('/sbin/ldconfig -p 2>/dev/null')
            try:
                data = f.read()
            finally:
                f.close()
            res = re.search(expr, data)
            if not res:
                return None
            return res.group(1) 

def is_cygwingcc():
    '''Try to determine if the gcc that would be used is from cygwin.'''
    out = os.popen('gcc -dumpmachine', 'r')
    out_string = out.read()
    out.close()
    # out_string is the target triplet cpu-vendor-os
    # Cygwin's gcc sets the os to 'cygwin'
    return out_string.strip().endswith('cygwin') 

def get_versions():
    """ Try to find out the versions of gcc and ld.
        If not possible it returns None for it.
    """
    from distutils.version import StrictVersion
    from distutils.spawn import find_executable
    import re

    gcc_exe = find_executable('gcc')
    if gcc_exe:
        out = os.popen(gcc_exe + ' -dumpversion','r')
        try:
            out_string = out.read()
        finally:
            out.close()
        result = re.search('(\d+\.\d+\.\d+)',out_string)
        if result:
            gcc_version = StrictVersion(result.group(1))
        else:
            gcc_version = None
    else:
        gcc_version = None
    # EMX ld has no way of reporting version number, and we use GCC
    # anyway - so we can link OMF DLLs
    ld_version = None
    return (gcc_version, ld_version) 

def test():
    global mh, f
    os.system('rm -rf $HOME/Mail/@test')
    mh = MH()
    def do(s): print s; print eval(s)
    do('mh.listfolders()')
    do('mh.listallfolders()')
    testfolders = ['@test', '@test/test1', '@test/test2',
                   '@test/test1/test11', '@test/test1/test12',
                   '@test/test1/test11/test111']
    for t in testfolders: do('mh.makefolder(%r)' % (t,))
    do('mh.listsubfolders(\'@test\')')
    do('mh.listallsubfolders(\'@test\')')
    f = mh.openfolder('@test')
    do('f.listsubfolders()')
    do('f.listallsubfolders()')
    do('f.getsequences()')
    seqs = f.getsequences()
    seqs['foo'] = IntSet('1-10 12-20', ' ').tolist()
    print seqs
    f.putsequences(seqs)
    do('f.getsequences()')
    for t in reversed(testfolders): do('mh.deletefolder(%r)' % (t,))
    do('mh.getcontext()')
    context = mh.getcontext()
    f = mh.openfolder(context)
    do('f.getcurrent()')
    for seq in ('first', 'last', 'cur', '.', 'prev', 'next',
                'first:3', 'last:3', 'cur:3', 'cur:-3',
                'prev:3', 'next:3',
                '1:3', '1:-3', '100:3', '100:-3', '10000:3', '10000:-3',
                'all'):
        try:
            do('f.parsesequence(%r)' % (seq,))
        except Error, msg:
            print "Error:", msg
        stuff = os.popen("pick %r 2>/dev/null" % (seq,)).read()
        list = map(int, stuff.split())
        print list, "<-- pick" 

def getstatusoutput(cmd):
    """Return (status, output) of executing cmd in a shell."""
    import os
    pipe = os.popen('{ ' + cmd + '; } 2>&1', 'r')
    text = pipe.read()
    sts = pipe.close()
    if sts is None: sts = 0
    if text[-1:] == '\n': text = text[:-1]
    return sts, text


# Make command argument from directory and pathname (prefix space, add quotes).
# 

def __init__(self,cmd,mode='r',bufsize=None):

        if mode != 'r':
            raise ValueError,'popen()-emulation only supports read mode'
        import tempfile
        self.tmpfile = tmpfile = tempfile.mktemp()
        os.system(cmd + ' > %s' % tmpfile)
        self.pipe = open(tmpfile,'rb')
        self.bufsize = bufsize
        self.mode = mode 

def popen(cmd, mode='r', bufsize=None):

    """ Portable popen() interface.
    """
    # Find a working popen implementation preferring win32pipe.popen
    # over os.popen over _popen
    popen = None
    if os.environ.get('OS','') == 'Windows_NT':
        # On NT win32pipe should work; on Win9x it hangs due to bugs
        # in the MS C lib (see MS KnowledgeBase article Q150956)
        try:
            import win32pipe
        except ImportError:
            pass
        else:
            popen = win32pipe.popen
    if popen is None:
        if hasattr(os,'popen'):
            popen = os.popen
            # Check whether it works... it doesn't in GUI programs
            # on Windows platforms
            if sys.platform == 'win32': # XXX Others too ?
                try:
                    popen('')
                except os.error:
                    popen = _popen
        else:
            popen = _popen
    if bufsize is None:
        return popen(cmd,mode)
    else:
        return popen(cmd,mode,bufsize) 

def open(self, url, new=0, autoraise=True):
            assert "'" not in url
            # hack for local urls
            if not ':' in url:
                url = 'file:'+url

            # new must be 0 or 1
            new = int(bool(new))
            if self.name == "default":
                # User called open, open_new or get without a browser parameter
                script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser
            else:
                # User called get and chose a browser
                if self.name == "OmniWeb":
                    toWindow = ""
                else:
                    # Include toWindow parameter of OpenURL command for browsers
                    # that support it.  0 == new window; -1 == existing
                    toWindow = "toWindow %d" % (new - 1)
                cmd = 'OpenURL "%s"' % url.replace('"', '%22')
                script = '''tell application "%s"
                                activate
                                %s %s
                            end tell''' % (self.name, cmd, toWindow)
            # Open pipe to AppleScript through osascript command
            osapipe = os.popen("osascript", "w")
            if osapipe is None:
                return False
            # Write script to osascript's stdin
            osapipe.write(script)
            rc = osapipe.close()
            return not rc 

def _do_test_commandline(self, cmdline, expected):
        cmd = '%s -c "import sys;print sys.argv" %s' % (python, cmdline)
        data = os.popen(cmd).read() + '\n'
        got = eval(data)[1:] # strip off argv[0]
        self.assertEqual(got, expected) 

def test_popen(self):
        self.assertRaises(TypeError, os.popen)
        self._do_test_commandline(
            "foo bar",
            ["foo", "bar"]
        )
        self._do_test_commandline(
            'foo "spam and eggs" "silly walk"',
            ["foo", "spam and eggs", "silly walk"]
        )
        self._do_test_commandline(
            'foo "a \\"quoted\\" arg" bar',
            ["foo", 'a "quoted" arg', "bar"]
        )
        test_support.reap_children() 

def test_pipepager(self):
        # pipepager does not choke on unicode
        doc = pydoc.render_doc(self.Q)

        saved, os.popen = os.popen, open
        try:
            with test.test_support.temp_cwd():
                pydoc.pipepager(doc, 'pipe')
                self.assertEqual(open('pipe').read(), pydoc._encode(doc))
        finally:
            os.popen = saved 

def tearDown(self):
        for inst in popen2._active:
            inst.wait()
        popen2._cleanup()
        self.assertFalse(popen2._active, "popen2._active not empty")
        # The os.popen*() API delegates to the subprocess module (on Unix)
        import subprocess
        for inst in subprocess._active:
            inst.wait()
        subprocess._cleanup()
        self.assertFalse(subprocess._active, "subprocess._active not empty")
        reap_children() 

def test_os_popen2(self):
        # same test as test_popen2(), but using the os.popen*() API
        if os.name == 'posix':
            w, r = os.popen2([self.cmd])
            self.validate_output(self.teststr, self.expected, r, w)

            w, r = os.popen2(["echo", self.teststr])
            got = r.read()
            self.assertEqual(got, self.teststr + "\n")

        w, r = os.popen2(self.cmd)
        self.validate_output(self.teststr, self.expected, r, w) 

def test_os_popen3(self):
        # same test as test_popen3(), but using the os.popen*() API
        if os.name == 'posix':
            w, r, e = os.popen3([self.cmd])
            self.validate_output(self.teststr, self.expected, r, w, e)

            w, r, e = os.popen3(["echo", self.teststr])
            got = r.read()
            self.assertEqual(got, self.teststr + "\n")
            got = e.read()
            self.assertFalse(got, "unexpected %r on stderr" % got)

        w, r, e = os.popen3(self.cmd)
        self.validate_output(self.teststr, self.expected, r, w, e) 

def test_mac_ver(self):
        res = platform.mac_ver()

        try:
            import gestalt
        except ImportError:
            have_toolbox_glue = False
        else:
            have_toolbox_glue = True

        if have_toolbox_glue and platform.uname()[0] == 'Darwin':
            # We're on a MacOSX system, check that
            # the right version information is returned
            fd = os.popen('sw_vers', 'r')
            real_ver = None
            for ln in fd:
                if ln.startswith('ProductVersion:'):
                    real_ver = ln.strip().split()[-1]
                    break
            fd.close()
            self.assertFalse(real_ver is None)
            result_list = res[0].split('.')
            expect_list = real_ver.split('.')
            len_diff = len(result_list) - len(expect_list)
            # On Snow Leopard, sw_vers reports 10.6.0 as 10.6
            if len_diff > 0:
                expect_list.extend(['0'] * len_diff)
            self.assertEqual(result_list, expect_list)

            # res[1] claims to contain
            # (version, dev_stage, non_release_version)
            # That information is no longer available
            self.assertEqual(res[1], ('', '', ''))

            if sys.byteorder == 'little':
                self.assertIn(res[2], ('i386', 'x86_64'))
            else:
                self.assertEqual(res[2], 'PowerPC') 

def cpu_count():
    '''
    Returns the number of CPUs in the system
    '''
    if sys.platform == 'win32':
        try:
            num = int(os.environ['NUMBER_OF_PROCESSORS'])
        except (ValueError, KeyError):
            num = 0
    elif 'bsd' in sys.platform or sys.platform == 'darwin':
        comm = '/sbin/sysctl -n hw.ncpu'
        if sys.platform == 'darwin':
            comm = '/usr' + comm
        try:
            with os.popen(comm) as p:
                num = int(p.read())
        except ValueError:
            num = 0
    else:
        try:
            num = os.sysconf('SC_NPROCESSORS_ONLN')
        except (ValueError, OSError, AttributeError):
            num = 0

    if num >= 1:
        return num
    else:
        raise NotImplementedError('cannot determine number of cpus') 

def watch_python():
    result = "function watch_python():"
    try:
        python_pid = os.popen('pidof python').read().split()
        if len(python_pid) == 1:
            return "No python found!!!"
        result += os.popen(f'top -p {",".join(python_pid)} -c -b -n1').read()
    except BaseException:
        result += "watch_python() failed"
    return result 

def watch_folder(file_path):
    result = f"function watch_folder({file_path}):"
    try:
        result += os.popen(f'ls -alh {file_path}').read()
    except BaseException:
        result += f"watch_folder({file_path}) failed"
    return result 

def generate_md5_hash(self, arg):
        '''
        Generate MD5 hash with randomly generated salt size of 3.
        :param arg:
        :return passwd:
        '''
        cmd = "openssl passwd -salt `openssl rand -base64 3` -1 "
        return os.popen(cmd + arg).readlines()[0].strip() 

def _processSystem(self, elem, sessionID):
        """Process a <system> AIML element.

        <system> elements process their contents recursively, and then
        attempt to execute the results as a shell command on the
        server.  The AIML interpreter blocks until the command is
        complete, and then returns the command's output.

        For cross-platform compatibility, any file paths inside
        <system> tags should use Unix-style forward slashes ("/") as a
        directory separator.

        """
        # build up the command string
        command = ""
        for e in elem[2:]:
            command += self._processElement(e, sessionID)

        # normalize the path to the command.  Under Windows, this
        # switches forward-slashes to back-slashes; all system
        # elements should use unix-style paths for cross-platform
        # compatibility.
        # executable,args = command.split(" ", 1)
        # executable = os.path.normpath(executable)
        # command = executable + " " + args
        command = os.path.normpath(command)

        # execute the command.
        response = ""
        try:
            out = os.popen(command)
        except RuntimeError, msg:
            if self._verboseMode:
                err = "WARNING: RuntimeError while processing \"system\" element:\n%s\n" % msg.encode(
                    self._textEncoding, 'replace')
                sys.stderr.write(err)
            return "There was an error while computing my response.  Please inform my botmaster." 

def run_command(command):
    cmd = " ".join(command)
    output = os.popen(cmd).read()
    return output 

def get_cpu_speed():
    f = os.popen('/opt/vc/bin/vcgencmd get_config arm_freq')
    cpu = f.read()
    return cpu 

def mafft_align(inputfasta, outputfasta):
    mafft = os.popen( 'mafft --quiet ' + inputfasta )
    handle = open( outputfasta, 'w')
    for line in mafft:
        handle.write(line)
    handle.close()
    mafft.close()

####### HMMer functions ######################## 

def count_genes(genes=[], fastafile=None):
    "evaluates the number of genes provided between a gene list and a fasta file"
    genenum = 0
    if not isinstance(genes,list):
        genes = [genes]
    genes = [ g for g in genes if g != '' ]

    if fastafile:
        # count number of genes provided:
        handle = os.popen("grep -c '^>' " + fastafile)
        result = re.search("(\d*)", handle.readline())
        handle.close()
        if result:
            try:
                genenum = int(result.group(1))
            except ValueError:
                genenum = 2
                print "ValueError calculating genenum"
                """
                putting one will ensure hmmer model is built if there is an error
                counting the number of genes in the fasta file
                """
        else:
            genenum = 2
            print "No result found for genenum"
    return len(genes), genenum 

def getCPUtemperature(self):
        res = os.popen('vcgencmd measure_temp').readline()
        return float(res.replace("temp=","").replace("'C\n",""))

    # Return % of CPU used by user as a character string 

def getCPUuse(self):
        return float(os.popen("top -n1 | awk '/Cpu\(s\):/ {print $2}'").readline().strip(\
    )) 

def getCPUtemperature():
    res = os.popen('vcgencmd measure_temp').readline()
    return(res.replace("temp=","").replace("'C\n",""))

# Return RAM information (unit=kb) in a list                                        
# Index 0: total RAM                                                                
# Index 1: used RAM                                                                 
# Index 2: free RAM 

def getRAMinfo():
    p = os.popen('free')
    i = 0
    while 1:
        i = i + 1
        line = p.readline()
        if i==2:
            return(line.split()[1:4])

# Return % of CPU used by user as a character string 

def getCPUuse():
    return(str(os.popen("top -n1 | awk '/Cpu\(s\):/ {print $2}'").readline().strip(\
)))

# Return information about disk space as a list (unit included)                     
# Index 0: total disk space                                                         
# Index 1: used disk space                                                          
# Index 2: remaining disk space                                                     
# Index 3: percentage of disk used 

def getCPUtemperature():
    res = os.popen('vcgencmd measure_temp').readline()
    return(res.replace("temp=","").replace("'C\n",""))

# Return RAM information (unit=kb) in a list                                        
# Index 0: total RAM                                                                
# Index 1: used RAM                                                                 
# Index 2: free RAM 

def getRAMinfo():
    p = os.popen('free')
    i = 0
    while 1:
        i = i + 1
        line = p.readline()
        if i==2:
            return(line.split()[1:4])

# Return % of CPU used by user as a character string 

def getCPUuse():
    return(str(os.popen("top -n1 | awk '/Cpu\(s\):/ {print $2}'").readline().strip(\
)))

# Return information about disk space as a list (unit included)                     
# Index 0: total disk space                                                         
# Index 1: used disk space                                                          
# Index 2: remaining disk space                                                     
# Index 3: percentage of disk used 

def getDiskSpace():
    p = os.popen("df -h /")
    i = 0
    while 1:
        i = i +1
        line = p.readline()
        if i==2:
            return(line.split()[1:5]) 

def app_terminal_notifier(host, trigger, icon):
    url = url_map(host, trigger)
    cmd = 'terminal-notifier -group "{0} {1}" -title "{0}" -message "{1}" -open "{2}" -appIcon "{3}"'.\
        format(host, trigger, url, icon)
    print cmd
    os.popen(cmd) 

def app_growl(host, trigger, icon):
    url = url_map(host, trigger)
    cmd = 'growlnotify --identifier "{0} {1}" --title "{0}" --message "{1}" ' \
          '--url "{2}" --priority 2 --sticky --image "{3}"'.\
        format(host, trigger, url, icon)
    print cmd
    os.popen(cmd) 

def app_notify_send(host, trigger, icon, ttl):
    #url = url_map(host, trigger)
    cmd = 'notify-send -t "{0}" -i "{1}" "{2}" "{3}"'.\
        format(ttl, icon, host, trigger)
    print cmd
    os.popen(cmd) 

def fileError(filenotfound):
	r, c = os.popen('stty size', 'r').read().split()
	r=int(r)
	c=int(c)
	print(r,c)
	print(int(c)*"*")
	for e in filenotfound:
		sp=int((c-len(e))/2)-1
		print(sp*" ",e,sp*" ")
	print(int(c)*"*")
	exit(0) 

def getVersion():
    grepcommand = 'opkg list-installed | grep enigma2-plugin-skin-metrixreloaded | cut -d " " -f 3'
    out = os.popen(grepcommand).readlines()
    try:
        return str(out[0]).strip()
    except:
        return str(out).strip() 

def getHddTemp(self):
        textvalue = 'No info'
        info = '0'
        try:
            out_line = popen('hddtemp -n -q /dev/sda').readline()
            info = 'Hdd C:' + out_line[:4]
            textvalue = info
        except:
            pass

        return textvalue 

def getLoadAvg(self):
        textvalue = 'No info'
        info = '0'
        try:
            out_line = popen('cat /proc/loadavg').readline()
            info = '' + str(out_line)[:4]
            textvalue = info
        except:
            pass

        return textvalue 

def generate_csr():
    if not os.path.isfile("server.key"):
        # See https://en.wikipedia.org/wiki/Certificate_signing_request#Procedure
        # Basically we're saying "verify that the request is actually from EvilOSX".
        print MESSAGE_INFO + "Generating certificate signing request to encrypt sockets..."

        information = "/C=US/ST=EvilOSX/L=EvilOSX/O=EvilOSX/CN=EvilOSX"
        os.popen("openssl req -newkey rsa:2048 -nodes -x509 -subj {0} -keyout server.key -out server.crt 2>&1".format(information)) 

def preprocess(source):
    stream = os.popen("gcc -E " + source)
    return stream.read() 

def import_xsysroot():
    '''
    Find path to XSysroot and import it
    You need to create a symlink xsysroot.py -> xsysroot
    '''
    which_xsysroot=os.popen('which xsysroot').read().strip()
    if not which_xsysroot:
        print 'Could not find xsysroot tool'
        print 'Please install from https://github.com/skarbat/xsysroot'
        return None
    else:
        print 'xsysroot found at: {}'.format(which_xsysroot)
        sys.path.append(os.path.dirname(which_xsysroot))
        import xsysroot
        return xsysroot 

def check_gpus():
    '''
    GPU available check
    http://pytorch-cn.readthedocs.io/zh/latest/package_references/torch-cuda/
    '''
    if not torch.cuda.is_available():
        print('This script could only be used to manage NVIDIA GPUs,but no GPU found in your device')
        return False
    elif not 'NVIDIA System Management' in os.popen('nvidia-smi -h').read():
        print("'nvidia-smi' tool not found.")
        return False
    return True 

def query_gpu(qargs=[]):
        '''
        qargs:
            query arguments
        return:
            a list of dict
        Querying GPUs infos
        查询GPU信息
        '''
        qargs = ['index', 'gpu_name', 'memory.free', 'memory.total', 'power.draw', 'power.limit'] + qargs
        cmd = 'nvidia-smi --query-gpu={} --format=csv,noheader'.format(','.join(qargs))
        results = os.popen(cmd).readlines()
        return [parse(line, qargs) for line in results] 

def _find_appropriate_compiler(_config_vars):
    """Find appropriate C compiler for extension module builds"""

    # Issue #13590:
    #    The OSX location for the compiler varies between OSX
    #    (or rather Xcode) releases.  With older releases (up-to 10.5)
    #    the compiler is in /usr/bin, with newer releases the compiler
    #    can only be found inside Xcode.app if the "Command Line Tools"
    #    are not installed.
    #
    #    Futhermore, the compiler that can be used varies between
    #    Xcode releases. Up to Xcode 4 it was possible to use 'gcc-4.2'
    #    as the compiler, after that 'clang' should be used because
    #    gcc-4.2 is either not present, or a copy of 'llvm-gcc' that
    #    miscompiles Python.

    # skip checks if the compiler was overriden with a CC env variable
    if 'CC' in os.environ:
        return _config_vars

    # The CC config var might contain additional arguments.
    # Ignore them while searching.
    cc = oldcc = _config_vars['CC'].split()[0]
    if not _find_executable(cc):
        # Compiler is not found on the shell search PATH.
        # Now search for clang, first on PATH (if the Command LIne
        # Tools have been installed in / or if the user has provided
        # another location via CC).  If not found, try using xcrun
        # to find an uninstalled clang (within a selected Xcode).

        # NOTE: Cannot use subprocess here because of bootstrap
        # issues when building Python itself (and os.popen is
        # implemented on top of subprocess and is therefore not
        # usable as well)

        cc = _find_build_tool('clang')

    elif os.path.basename(cc).startswith('gcc'):
        # Compiler is GCC, check if it is LLVM-GCC
        data = _read_output("'%s' --version"
                             % (cc.replace("'", "'\"'\"'"),))
        if data and 'llvm-gcc' in data:
            # Found LLVM-GCC, fall back to clang
            cc = _find_build_tool('clang')

    if not cc:
        raise SystemError(
               "Cannot locate working compiler")

    if cc != oldcc:
        # Found a replacement compiler.
        # Modify config vars using new compiler, if not already explicitly
        # overriden by an env variable, preserving additional arguments.
        for cv in _COMPILER_CONFIG_VARS:
            if cv in _config_vars and cv not in os.environ:
                cv_split = _config_vars[cv].split()
                cv_split[0] = cc if cv != 'CXX' else cc + '++'
                _save_modified_value(_config_vars, cv, ' '.join(cv_split))

    return _config_vars 

def get_versions():
    """ Try to find out the versions of gcc, ld and dllwrap.
        If not possible it returns None for it.
    """
    from distutils.version import LooseVersion
    from distutils.spawn import find_executable
    import re

    gcc_exe = find_executable('gcc')
    if gcc_exe:
        out = os.popen(gcc_exe + ' -dumpversion','r')
        out_string = out.read()
        out.close()
        result = re.search('(\d+\.\d+(\.\d+)*)',out_string)
        if result:
            gcc_version = LooseVersion(result.group(1))
        else:
            gcc_version = None
    else:
        gcc_version = None
    ld_exe = find_executable('ld')
    if ld_exe:
        out = os.popen(ld_exe + ' -v','r')
        out_string = out.read()
        out.close()
        result = re.search('(\d+\.\d+(\.\d+)*)',out_string)
        if result:
            ld_version = LooseVersion(result.group(1))
        else:
            ld_version = None
    else:
        ld_version = None
    dllwrap_exe = find_executable('dllwrap')
    if dllwrap_exe:
        out = os.popen(dllwrap_exe + ' --version','r')
        out_string = out.read()
        out.close()
        result = re.search(' (\d+\.\d+(\.\d+)*)',out_string)
        if result:
            dllwrap_version = LooseVersion(result.group(1))
        else:
            dllwrap_version = None
    else:
        dllwrap_version = None
    return (gcc_version, ld_version, dllwrap_version) 

def print_window(self, event):
        confirm = tkMessageBox.askokcancel(
                  title="Print",
                  message="Print to Default Printer",
                  default=tkMessageBox.OK,
                  master=self.text)
        if not confirm:
            self.text.focus_set()
            return "break"
        tempfilename = None
        saved = self.get_saved()
        if saved:
            filename = self.filename
        # shell undo is reset after every prompt, looks saved, probably isn't
        if not saved or filename is None:
            (tfd, tempfilename) = tempfile.mkstemp(prefix='IDLE_tmp_')
            filename = tempfilename
            os.close(tfd)
            if not self.writefile(tempfilename):
                os.unlink(tempfilename)
                return "break"
        platform = os.name
        printPlatform = True
        if platform == 'posix': #posix platform
            command = idleConf.GetOption('main','General',
                                         'print-command-posix')
            command = command + " 2>&1"
        elif platform == 'nt': #win32 platform
            command = idleConf.GetOption('main','General','print-command-win')
        else: #no printing for this platform
            printPlatform = False
        if printPlatform:  #we can try to print for this platform
            command = command % pipes.quote(filename)
            pipe = os.popen(command, "r")
            # things can get ugly on NT if there is no printer available.
            output = pipe.read().strip()
            status = pipe.close()
            if status:
                output = "Printing failed (exit status 0x%x)\n" % \
                         status + output
            if output:
                output = "Printing command: %s\n" % repr(command) + output
                tkMessageBox.showerror("Print status", output, master=self.text)
        else:  #no printing for this platform
            message = "Printing is not enabled for this platform: %s" % platform
            tkMessageBox.showinfo("Print status", message, master=self.text)
        if tempfilename:
            os.unlink(tempfilename)
        return "break" 

def __init__(self, host=None, user=None, timeout=10, sshpass=None, **kwargs):
        """Open a InteractiveSubprocess with default settings.

        - host: a network hostname, IP address, or localhost.
            - If host is specified, use a SSH connection.
            - If host is None, use a interactive bash session.
        - user: if None, ssh login as the current user
        """
        self.username = user if user else os.getlogin()
        self.host = host
        self._init_flush = True
        self._init_prompt = False
        if self.host is None:
            # prepare for local connection
            if not hasattr(self, 'use_pty_stdin'):
                self.use_pty_stdin = True
            if not hasattr(self, 'hostname'):
                self.hostname = os.popen('hostname -s').read().strip()
            if not hasattr(self, 'name'):
                self.name = 'bash@%s' % self.hostname
            if not hasattr(self, 'cmdline'):
                # Note, "bash" non-interactive mode doesn't output prompt,
                # "bash -i" interactive mode output the prompt, but it has
                # error "Suspended (tty output)" when python put it in background.
                #
                # One workaround is a while loop of command eval in bash non-interactive mode with
                # a generated prompt, but the 'read line' prevents native console with raw terminal.
                # bash_cmd =
                # 'while true; do echo -n "`whoami`@`hostname -s` `pwd`> "; read line; echo "$line"; eval "$line"; done'
                # self.cmdline = "bash -c '%s'" % bash_cmd
                # self.prompt = '%s@%s .+> ' % (self.whoami, self.hostname)
                #
                # Other alternative workarounds are "bash -c csh", and ssh localhost.
                # We use "bash -c csh" here for better native console and no network traffic.
                self.cmdline = "bash -c csh"
                self.shelltype = 'csh'
            if not hasattr(self, 'prompt'):
                self.whoami = os.popen('whoami').read().strip()
                self.prompt = ''
                self._init_prompt = True
                self._init_flush = False
        else:
            self.use_pty_stdin = True,  # SSH requires Pseudo-terminal
            # prepare for remote connection
            self.whoami = self.username
            if not hasattr(self, 'cmdline'):
                self.cmdline = 'ssh ' + ('%s@' % self.username) + self.host
                if sshpass:
                    self.cmdline = ('sshpass -p %s ' % sshpass) + self.cmdline
            if not hasattr(self, 'name'):
                self.name = self.host
            if not hasattr(self, 'prompt'):
                # best guess of user prompt is '~> ', ']# ', or ']:# ', with or without ANSI color codes
                self.prompt = '(\\x1b\[[;\d]+m)?~(\\x1b\[0?m)?> |\]# |\]\:# '
        super(SshSession, self).__init__(cmd=self.cmdline,
                                         name=self.name, prompt=self.prompt,
                                         timeout=60,       # 60 seconds timeout for manual login
                                         use_pty_stdin=self.use_pty_stdin,
                                         flush=self._init_flush,         # Flush the startup messages
                                         **kwargs)
        self.timeout = timeout 

def _load_image_set_index(self):
        """
        Load the indexes listed in this dataset's image set file.
        """
        # Example path to image set file:
        # self._data_path + /ImageSets/val.txt

        if self._image_set == 'train':
            image_set_file = os.path.join(self._data_path, 'ImageSets', 'trainr.txt')
            image_index = []
            if os.path.exists(image_set_file):
                f = open(image_set_file, 'r')
                data = f.read().split()
                for lines in data:
                    if lines != '':
                        image_index.append(lines)
                f.close()
                return image_index

            for i in range(1,200):
                print(i)
                image_set_file = os.path.join(self._data_path, 'ImageSets', 'DET', 'train_' + str(i) + '.txt')
                with open(image_set_file) as f:
                    tmp_index = [x.strip() for x in f.readlines()]
                    vtmp_index = []
                    for line in tmp_index:
                        line = line.split(' ')
                        image_list = os.popen('ls ' + self._data_path + '/Data/DET/train/' + line[0] + '/*.JPEG').read().split()
                        tmp_list = []
                        for imgs in image_list:
                            tmp_list.append(imgs[:-5])
                        vtmp_index = vtmp_index + tmp_list

                num_lines = len(vtmp_index)
                ids = np.random.permutation(num_lines)
                count = 0
                while count < 2000:
                    image_index.append(vtmp_index[ids[count % num_lines]])
                    count = count + 1

            for i in range(1,201):
                if self._valid_image_flag[i] == 1:
                    image_set_file = os.path.join(self._data_path, 'ImageSets', 'train_pos_' + str(i) + '.txt')
                    with open(image_set_file) as f:
                        tmp_index = [x.strip() for x in f.readlines()]
                    num_lines = len(tmp_index)
                    ids = np.random.permutation(num_lines)
                    count = 0
                    while count < 2000:
                        image_index.append(tmp_index[ids[count % num_lines]])
                        count = count + 1
            image_set_file = os.path.join(self._data_path, 'ImageSets', 'trainr.txt')
            f = open(image_set_file, 'w')
            for lines in image_index:
                f.write(lines + '\n')
            f.close()
        else:
            image_set_file = os.path.join(self._data_path, 'ImageSets', 'val.txt')
            with open(image_set_file) as f:
                image_index = [x.strip() for x in f.readlines()]
        return image_index 

def clean(self, fs=None, local_execution=True):
        """
        Deletes all the files from COS. These files include the function,
        the data serialization and the function invocation results.
        """
        futures = self.futures if not fs else fs
        if type(futures) != list:
            futures = [futures]
        if not futures:
            return

        if not fs:
            present_jobs = {(f.executor_id, f.job_id) for f in futures
                            if (f.done or not f.produce_output)
                            and f.executor_id.count('/') == 1}
        else:
            present_jobs = {(f.executor_id, f.job_id) for f in futures
                            if f.executor_id.count('/') == 1}

        jobs_to_clean = present_jobs - self.cleaned_jobs

        if jobs_to_clean:
            msg = "ExecutorID {} - Cleaning temporary data".format(self.executor_id)
            logger.info(msg)
            if not self.log_level:
                print(msg)

        for executor_id, job_id in jobs_to_clean:
            storage_bucket = self.config['pywren']['storage_bucket']
            storage_prerix = '/'.join([JOBS_PREFIX, executor_id, job_id])

            if local_execution:
                # 1st case: Not background. The main code waits until the cleaner finishes its execution.
                # It is not ideal for performance tests, since it can take long time to complete.
                # clean_os_bucket(storage_bucket, storage_prerix, self.internal_storage)

                # 2nd case: Execute in Background as a subprocess. The main program does not wait for its completion.
                storage_config = json.dumps(self.internal_storage.get_storage_config())
                storage_config = storage_config.replace('"', '\\"')

                cmdstr = ('{} -c "from pywren_ibm_cloud.storage.utils import clean_bucket; \
                                  clean_bucket(\'{}\', \'{}\', \'{}\')"'.format(sys.executable,
                                                                                storage_bucket,
                                                                                storage_prerix,
                                                                                storage_config))
                os.popen(cmdstr)
            else:
                extra_env = {'STORE_STATUS': False,
                             'STORE_RESULT': False}
                old_stdout = sys.stdout
                sys.stdout = open(os.devnull, 'w')
                self.call_async(clean_os_bucket, [storage_bucket, storage_prerix], extra_env=extra_env)
                sys.stdout = old_stdout

        self.cleaned_jobs.update(jobs_to_clean)