Python sys.stderr() Examples

def call(self, cmd, **kwargs):
        print('Running "{}"'.format(cmd), file=sys.stderr)
        expect = kwargs.pop("expect", [dict(return_codes=[os.EX_OK], stdout=None, stderr=None)])
        process = subprocess.Popen(cmd, stdin=kwargs.get("stdin", subprocess.PIPE), stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE, **kwargs)
        out, err = process.communicate()
        return_code = process.poll()
        out = out.decode(sys.stdin.encoding)
        err = err.decode(sys.stdin.encoding)

        def match(return_code, out, err, expected):
            exit_ok = return_code in expected["return_codes"]
            stdout_ok = re.search(expected.get("stdout") or "", out)
            stderr_ok = re.search(expected.get("stderr") or "", err)
            return exit_ok and stdout_ok and stderr_ok
        if not any(match(return_code, out, err, exp) for exp in expect):
            print(err)
            e = subprocess.CalledProcessError(return_code, cmd, output=out)
            e.stdout, e.stderr = out, err
            raise e
        return self.SubprocessResult(out, err, return_code) 

def init_logging(log_file=None, append=False, console_loglevel=logging.INFO):
    """Set up logging to file and console."""
    if log_file is not None:
        if append:
            filemode_val = 'a'
        else:
            filemode_val = 'w'
        logging.basicConfig(level=logging.DEBUG,
                            format="%(asctime)s %(levelname)s %(threadName)s %(name)s %(message)s",
                            # datefmt='%m-%d %H:%M',
                            filename=log_file,
                            filemode=filemode_val)
    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler()
    console.setLevel(console_loglevel)
    # set a format which is simpler for console use
    formatter = logging.Formatter("%(message)s")
    console.setFormatter(formatter)
    # add the handler to the root logger
    logging.getLogger('').addHandler(console)
    global LOG
    LOG = logging.getLogger(__name__) 

def create_model_directory(args):


    # create necessary directories
    if os.path.isdir("models_{name}".format(name=args.name)):
        print("Directory models_{name} already exists, old files will be overwritten".format(name=args.name), file=sys.stderr)
        
    else:
        os.mkdir("models_{name}".format(name=args.name))
        os.mkdir("models_{name}/Data".format(name=args.name))
        os.mkdir("models_{name}/Tokenizer".format(name=args.name))

    # copy necessary files
    if args.embeddings: # embeddings
        copyfile(args.embeddings, "models_{name}/Data/embeddings.vectors".format(name=args.name))
    copyfile("{config}/pipelines.yaml".format(config=args.config_directory), "models_{name}/pipelines.yaml".format(name=args.name))
    process_morpho(args) # train/dev files for tagger/parser
    process_config(args) # configs for tagger/parser 

def convert_image(inpath, outpath, size):
    """Convert an image file using ``sips``.

    Args:
        inpath (str): Path of source file.
        outpath (str): Path to destination file.
        size (int): Width and height of destination image in pixels.

    Raises:
        RuntimeError: Raised if ``sips`` exits with non-zero status.
    """
    cmd = [
        b'sips',
        b'-z', str(size), str(size),
        inpath,
        b'--out', outpath]
    # log().debug(cmd)
    with open(os.devnull, 'w') as pipe:
        retcode = subprocess.call(cmd, stdout=pipe, stderr=subprocess.STDOUT)

    if retcode != 0:
        raise RuntimeError('sips exited with %d' % retcode) 

def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
                env=None):
    """Call the given command(s)."""
    assert isinstance(commands, list)
    p = None
    for c in commands:
        try:
            dispcmd = str([c] + args)
            # remember shell=False, so use git.cmd on windows, not just git
            p = subprocess.Popen([c] + args, cwd=cwd, env=env,
                                 stdout=subprocess.PIPE,
                                 stderr=(subprocess.PIPE if hide_stderr
                                         else None))
            break
        except EnvironmentError:
            e = sys.exc_info()[1]
            if e.errno == errno.ENOENT:
                continue
            if verbose:
                print("unable to run %s" % dispcmd)
                print(e)
            return None, None
    else:
        if verbose:
            print("unable to find command, tried %s" % (commands,))
        return None, None
    stdout = p.communicate()[0].strip()
    if sys.version_info[0] >= 3:
        stdout = stdout.decode()
    if p.returncode != 0:
        if verbose:
            print("unable to run %s (error)" % dispcmd)
            print("stdout was %s" % stdout)
        return None, p.returncode
    return stdout, p.returncode 

def check_dependencies_or_exit():
    dependencies = [
            which("e2cp",
                ubuntu="apt-get install e2tools",
                arch="yaourt -S e2tools",
                darwin="brew install e2tools gettext e2fsprogs\nbrew unlink e2fsprogs && brew link e2fsprogs -f"),
            which("qemu-system-arm",
                  ubuntu="apt-get install qemu",
                  kali="apt-get install qemu-system",
                  arch="pacman -S qemu-arch-extra",
                  darwin="brew install qemu"),
            which("unzip",
                ubuntu="apt-get install unzip",
                arch="pacman -S unzip",
                darwin="brew install unzip")
            ]
    if not all(dependencies):
        print("requirements missing, plz install them", file=sys.stderr)
        sys.exit(1) 

def launch(args,q_in,q_out):
    start=time.time()
    total_parsed_trees=0
    total_parsed_tokens=0
    next_report=start+10.0 #report every 10sec at most
    while True:
        jobid,txt=q_in.get()
        if jobid=="FINAL":
            print("Output exiting",file=sys.stderr,flush=True)
            return
        total_parsed_trees+=sum(1 for line in txt.split("\n") if line.startswith("1\t"))
        total_parsed_tokens+=sum(1 for line in txt.split("\n") if re.match(token_regex, line))
        if total_parsed_trees>0 and time.time()>next_report:
            time_spent=time.time()-start
            print("Runtime: {}:{} [m:s]  Parsed: {} [trees], {} [tokens]  Speed: {} [trees/sec]  {} [sec/tree] {} [tokens/sec]".format(int(time_spent)//60,int(time_spent)%60,total_parsed_trees,total_parsed_tokens, total_parsed_trees/time_spent,time_spent/total_parsed_trees, total_parsed_tokens/time_spent) ,file=sys.stderr,flush=True)
            next_report=time.time()+10
        print(txt,end="",flush=True) 

def __init__(self, appid=None, logger_root='cherrypy'):
        self.logger_root = logger_root
        self.appid = appid
        if appid is None:
            self.error_log = logging.getLogger('%s.error' % logger_root)
            self.access_log = logging.getLogger('%s.access' % logger_root)
        else:
            self.error_log = logging.getLogger(
                '%s.error.%s' % (logger_root, appid))
            self.access_log = logging.getLogger(
                '%s.access.%s' % (logger_root, appid))
        self.error_log.setLevel(logging.INFO)
        self.access_log.setLevel(logging.INFO)

        # Silence the no-handlers "warning" (stderr write!) in stdlib logging
        self.error_log.addHandler(NullHandler())
        self.access_log.addHandler(NullHandler())

        cherrypy.engine.subscribe('graceful', self.reopen_files) 

def active_parent_nodes_consistency(oCurrNode):
		""" active_parent_nodes_consistency(oCurrNode)
			oCurrNode ... class Node
			------------------------------------------------------------------------
			Checks, if the parent node of an ncbi taxonomy node has at least one active
			child node. Every parent node of the ncbi taxonomy node that has no active
			child nodes will be inactivated.
			Returns the last node that has a parent with at least one active child.
		"""
		try:
			bCheckActiveChildNodes = False
			oChildrenNodes = oCurrNode.parent.children
			for oChildNode in oChildrenNodes:
				bCheckActiveChildNodes = bCheckActiveChildNodes or oChildNode.node_active

			if not bCheckActiveChildNodes:
				oCurrNode.parent.node_active = False
				oCurrNode = TaxonomyNode.active_parent_nodes_consistency(oCurrNode.parent)

		except Exception as e:
			print >> sys.stderr, str(e)
			# logging.error(str(e))

		return oCurrNode 

def add_log_stream(self, stream=sys.stderr, level=logging.INFO):
		"""
		Add a stream where messages are outputted to.

		@param stream: stderr/stdout or a file stream
		@type stream: file | FileIO | StringIO
		@param level: minimum level of messages to be logged
		@type level: int | long

		@return: None
		@rtype: None
		"""
		assert self.is_stream(stream)
		# assert isinstance(stream, (file, io.FileIO))
		assert level in self._levelNames

		err_handler = logging.StreamHandler(stream)
		err_handler.setFormatter(self.message_formatter)
		err_handler.setLevel(level)
		self._logger.addHandler(err_handler) 

def save_image_grid(images, filename, drange=[0,1], grid_size=None):
    convert_to_pil_image(create_image_grid(images, grid_size), drange).save(filename)

#----------------------------------------------------------------------------
# Logging of stdout and stderr to a file. 

def __init__(self, root):
        """Context manager for capturing and formating stdout and stderr.

        Parameter
        ---------
        root : str
            Path where ciocheck script was called (root directory).
        """
        self._root = root
        self._stdout = None
        self._stderr = None
        self._stringio_output = None
        self._stringio_error = None 

def init_output_logging():
    global output_logger
    if output_logger is None:
        output_logger = OutputLogger()
        sys.stdout = TeeOutputStream([sys.stdout, output_logger], autoflush=True)
        sys.stderr = TeeOutputStream([sys.stderr, output_logger], autoflush=True) 

def __init__(self, file_in):
            self._file = file_in
            self._stderr = sys.stderr
            sys.stderr = self 

def __del__(self):
            sys.stderr = self._stderr 

def txt_to_conllu(sentences):
  """Just a quick dummy to turn list of whitespace-tokenized sentences into conllu"""
  buff=io.StringIO()
  for s in sentences:
    for idx,wrd in enumerate(s.split()):
      print(idx+1,wrd,"_","_","_","_","_","_","_","_",sep="\t",file=buff)
    print(file=buff)
  print("ENTERS PARSER",repr(buff.getvalue()),file=sys.stderr)
  return buff.getvalue() 

def __enter__(self):
        """Capture stdout and stderr in a StringIO."""
        self._stdout = sys.stdout
        self._stderr = sys.stderr
        sys.stdout = self._stringio_output = StringIO()
        sys.stderr = self._stringio_error = StringIO()
        return self 

def __exit__(self, *args):
        """Restore stdout and stderr and format found values."""
        out = self._stringio_output.getvalue().splitlines()
        err = self._stringio_error.getvalue().splitlines()
        sys.stdout = self._stdout
        sys.stderr = self._stderr

        self.output = out
        self.error = err
        for output in [out, err]:
            for line in output:
                print(line) 

def run_command(args, cwd=None):
    """Run command."""
    process = subprocess.Popen(
        args,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        cwd=cwd, )
    output, error = process.communicate()

    if isinstance(output, bytes):
        output = output.decode()
    if isinstance(error, bytes):
        error = error.decode()

    return output, error 

def __init__(self, wrapped, convert=None, strip=None, autoreset=False):
        # The wrapped stream (normally sys.stdout or sys.stderr)
        self.wrapped = wrapped

        # should we reset colors to defaults after every .write()
        self.autoreset = autoreset

        # create the proxy wrapping our output stream
        self.stream = StreamWrapper(wrapped, self)

        on_windows = os.name == 'nt'
        # We test if the WinAPI works, because even if we are on Windows
        # we may be using a terminal that doesn't support the WinAPI
        # (e.g. Cygwin Terminal). In this case it's up to the terminal
        # to support the ANSI codes.
        conversion_supported = on_windows and winapi_test()

        # should we strip ANSI sequences from our output?
        if strip is None:
            strip = conversion_supported or (not wrapped.closed and not is_a_tty(wrapped))
        self.strip = strip

        # should we should convert ANSI sequences into win32 calls?
        if convert is None:
            convert = conversion_supported and not wrapped.closed and is_a_tty(wrapped)
        self.convert = convert

        # dict of ansi codes to win32 functions and parameters
        self.win32_calls = self.get_win32_calls()

        # are we wrapping stderr?
        self.on_stderr = self.wrapped is sys.stderr 

def set_log_file(self, log_file, mode='a', level=logging.INFO):
		"""
		Add a stream where messages are outputted to.

		@attention: file stream will only be closed if a file path is given!

		@param log_file: file stream or file path of logfile
		@type log_file: file | FileIO | StringIO | basestring
		@param mode: opening mode for logfile, if a file path is given
		@type mode: basestring
		@param level: minimum level of messages to be logged
		@type level: int or long

		@return: None
		@rtype: None
		"""
		assert isinstance(log_file, basestring) or self.is_stream(log_file)
		assert level in self._levelNames

		if LoggingWrapper._map_logfile_handler[self._label] is not None:
			self._logger.removeHandler(LoggingWrapper._map_logfile_handler[self._label])
			LoggingWrapper._map_logfile_handler[self._label].close()
			LoggingWrapper._map_logfile_handler[self._label] = None

		if self.is_stream(log_file):
			self.add_log_stream(stream=log_file, level=level)
			return

		try:
			err_handler_file = logging.FileHandler(log_file, mode)
			err_handler_file.setFormatter(self.message_formatter)
			err_handler_file.setLevel(level)
			self._logger.addHandler(err_handler_file)
			LoggingWrapper._map_logfile_handler[self._label] = err_handler_file
		except Exception:
			sys.stderr.write("[LoggingWrapper] Could not open '{}' for logging\n".format(log_file))
			return 

def reinit():
    if wrapped_stdout is not None:
        sys.stdout = wrapped_stdout
    if wrapped_stderr is not None:
        sys.stderr = wrapped_stderr 

def deinit():
    if orig_stdout is not None:
        sys.stdout = orig_stdout
    if orig_stderr is not None:
        sys.stderr = orig_stderr 

def init(autoreset=False, convert=None, strip=None, wrap=True):

    if not wrap and any([autoreset, convert, strip]):
        raise ValueError('wrap=False conflicts with any other arg=True')

    global wrapped_stdout, wrapped_stderr
    global orig_stdout, orig_stderr

    orig_stdout = sys.stdout
    orig_stderr = sys.stderr

    if sys.stdout is None:
        wrapped_stdout = None
    else:
        sys.stdout = wrapped_stdout = \
            wrap_stream(orig_stdout, convert, strip, autoreset, wrap)
    if sys.stderr is None:
        wrapped_stderr = None
    else:
        sys.stderr = wrapped_stderr = \
            wrap_stream(orig_stderr, convert, strip, autoreset, wrap)

    global atexit_done
    if not atexit_done:
        atexit.register(reset_all)
        atexit_done = True 

def screen(self, newvalue):
        self._set_screen_handler(self.error_log, newvalue, stream=sys.stderr)
        self._set_screen_handler(self.access_log, newvalue, stream=sys.stdout)

    # -------------------------- File handlers -------------------------- # 

def inactivate_branch(taxid):
		""" inactivate_branch(taxid)
			taxid ... ncbi taxonomy id
			------------------------------------------------------------------------
			Inactivates an ncbi taxonomy node with id = taxid including all 
			children by changing node attribute "node_active" to False. 
			Furthermore inactivates all successive parent nodes, which 
			have no active child nodes. Adds the top inactivated node within 
			the inactivated branch to class dictionary Node.inactive_top_nodes 
			to permit a reactivation of this branch later on.
		"""
		try:
			oCurrNode = TaxonomyNode.by_name.get(str(taxid))
			if not oCurrNode.all_child_nodes:
				oCurrNode.get_child_nodes()
			oCurrNode.node_active = False
			for oChildNode in oCurrNode.all_child_nodes:
				oChildNode.node_active = False

			oCurrNode = TaxonomyNode.active_parent_nodes_consistency(oCurrNode)

			TaxonomyNode.inactive_top_nodes.append(oCurrNode)

		except Exception as e:
			print >> sys.stderr, str(e)
			# logging.error(str(e)) 

def screen(self):
        """Turn stderr/stdout logging on or off.

        If you set this to True, it'll add the appropriate StreamHandler for
        you. If you set it to False, it will remove the handler.
        """
        h = self._get_builtin_handler
        has_h = h(self.error_log, 'screen') or h(self.access_log, 'screen')
        return bool(has_h) 

def _set_screen_handler(self, log, enable, stream=None):
        h = self._get_builtin_handler(log, 'screen')
        if enable:
            if not h:
                if stream is None:
                    stream = sys.stderr
                h = logging.StreamHandler(stream)
                h.setFormatter(logfmt)
                h._cpbuiltin = 'screen'
                log.addHandler(h)
        elif h:
            log.handlers.remove(h) 

def deinit():
    if orig_stdout is not None:
        sys.stdout = orig_stdout
    if orig_stderr is not None:
        sys.stderr = orig_stderr 

def _call_security(self, action, service, account, *args):
        """Call ``security`` CLI program that provides access to keychains.

        May raise `PasswordNotFound`, `PasswordExists` or `KeychainError`
        exceptions (the first two are subclasses of `KeychainError`).

        :param action: The ``security`` action to call, e.g.
                           ``add-generic-password``
        :type action: ``unicode``
        :param service: Name of the service.
        :type service: ``unicode``
        :param account: name of the account the password is for, e.g.
            "Pinboard"
        :type account: ``unicode``
        :param password: the password to secure
        :type password: ``unicode``
        :param *args: list of command line arguments to be passed to
                      ``security``
        :type *args: `list` or `tuple`
        :returns: ``(retcode, output)``. ``retcode`` is an `int`, ``output`` a
                  ``unicode`` string.
        :rtype: `tuple` (`int`, ``unicode``)

        """
        cmd = ['security', action, '-s', service, '-a', account] + list(args)
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
        stdout, _ = p.communicate()
        if p.returncode == 44:  # password does not exist
            raise PasswordNotFound()
        elif p.returncode == 45:  # password already exists
            raise PasswordExists()
        elif p.returncode > 0:
            err = KeychainError('Unknown Keychain error : %s' % stdout)
            err.retcode = p.returncode
            raise err
        return stdout.strip().decode('utf-8')