Python sys.exit() Examples

def nullify_site(sitename):
    hostname = parse_hostname(sitename)

    hostfile = ManagedHostfile()
    managed = hostfile.current_body()

    null_matcher = HOST_MATCHER.format(hostname)
    neuter_matcher = COMMENTED_MATCHER.format(hostname)
    if re.search(null_matcher, managed, flags=re.MULTILINE) != None:
        # if it's not commented, we ignore it
        print(hostname + ' has already been sent to webnull')
        sys.exit(0)
    elif re.search(neuter_matcher, managed, flags=re.MULTILINE) != None:
        # if it's commented, we replace it
        hostfile.transform_body(neuter_matcher, r'\1')
    else:
        # if it's not there, we write it.
        ip5null = '127.0.0.1\t'
        ip6null = '::1\t\t'

        for null in [ip5null, ip6null]:
            for www in ['', 'www.']:
                managed += (null + www + hostname + '\n')

        hostfile.write_body(managed) 

def _get_proxy_results(self):

        self._logger.info("Getting {0} Machine Learning Results from HDFS".format(self._date))
        proxy_results = "{0}/proxy_results.csv".format(self._data_path)

        # get hdfs path from conf file.
        HUSER = self._spot_conf.get('conf', 'HUSER').replace("'", "").replace('"', '')
        hdfs_path = "{0}/proxy/scored_results/{1}/scores/proxy_results.csv".format(HUSER,self._date)

        # get results file from hdfs.
        get_command = Util.get_ml_results_form_hdfs(hdfs_path,self._data_path)
        self._logger.info("{0}".format(get_command))

         # valdiate files exists
        if os.path.isfile(proxy_results):

            # read number of results based in the limit specified.
            self._logger.info("Reading {0} proxy results file: {1}".format(self._date,proxy_results))
            self._proxy_results = Util.read_results(proxy_results,self._limit,self._results_delimiter)[:]
            if len(self._proxy_results) == 0: self._logger.error("There are not proxy results.");sys.exit(1)
        else:
            self._logger.error("There was an error getting ML results from HDFS")
            sys.exit(1)

        self._proxy_scores = self._proxy_results[:] 

def build(ctx):
    """Build the CLI and GUI executables."""
    ctx.invoke(clean)
    _set_cwd()
    print("------------------------")
    print("Building CLI executable:")
    print("------------------------")
    rtn_code = _run_cli_cmd(["pyinstaller", "package/pyinstaller-cli.spec"])
    if rtn_code != 0:
        sys.exit(rtn_code)
    print("------------------------")
    print("Building GUI executable:")
    print("------------------------")
    rtn_code = _run_cli_cmd(["pyinstaller", "package/pyinstaller-gui.spec"])
    if rtn_code != 0:
        sys.exit(rtn_code)
    return 0 

def parse_hostname(sitename):
    parsed_url = urllib.parse.urlparse(sitename)
    hostname = parsed_url.netloc
    if hostname == '':
        # If you pass in 'facebook.com/foo' urlparse treats it all as the path
        match = re.match(r'[^/]+', parsed_url.path)
        if match == None:
            print('ERROR: Unable to make the provided sitename into a hostname: ' + parsed_url.path)
            sys.exit(1)
        hostname = match.group(0)

    www_matcher = r'^www\.(.+)'
    if re.search(www_matcher, hostname) != None:
        hostname = re.sub(www_matcher, r'\1', hostname)

    return hostname 

def stop(self, restart=False):
        ''' Shut down the bot. '''
        logging.info('Shutting down via stop method.')
        # Execute shutdown hooks
        registry.fire_event(registry.BOT_SHUTDOWN, self)
        self.set_offline()
        try:
            self.scheduler.shutdown()
        except SchedulerNotRunningError:
            pass
        self.rpcserver.shutdown()
        self.rpcserver.server_close()
        if self.connected:
            self.broadcastEvent(YowLayerEvent(YowNetworkLayer.EVENT_STATE_DISCONNECT))
        if restart:
            sys.exit(3)
        sys.exit(0)

    # Helper functions 

def init_libraries(self, paired_end, max_insert_size, ctr_libs,
                       exp_libs):
        self._paired_end = paired_end
        self._max_insert_size = max_insert_size
        self._ctr_lib_list = [splitext(basename(lib_file))[0]
                              for lib_file in ctr_libs]
        self._exp_lib_list = [splitext(basename(lib_file))[0]
                              for lib_file in exp_libs]
        # add libs to lib_dict
        for lib_file in exp_libs + ctr_libs:
            if not isfile(lib_file):
                sys.stderr.write("ERROR: The library file {} does not exist.\n"
                                 .format(lib_file))
                sys.exit(1)
            self._lib_dict[splitext(basename(lib_file))[0]] = Library(
                paired_end, max_insert_size, lib_file,
                deepcopy(self._replicon_dict))
        self._lib_names_list = list(self._lib_dict.keys())
        print("The following libraries were initialized:\n"
              "# Experiment libraries\n{0}\n"
              "# Control libraries\n{1}".format(
                  '\n'.join(self._exp_lib_list),
                  '\n'.join(self._ctr_lib_list))) 

def init_libraries(self, paired_end, max_insert_size, ctr_libs,
                       exp_libs):
        self._paired_end = paired_end
        self._max_insert_size = max_insert_size
        self._ctr_lib_list = [splitext(basename(lib_file))[0]
                              for lib_file in ctr_libs]
        self._exp_lib_list = [splitext(basename(lib_file))[0]
                              for lib_file in exp_libs]
        # add libs to lib_dict
        for lib_file in exp_libs + ctr_libs:
            if not isfile(lib_file):
                sys.stderr.write(
                    "ERROR: The library file {} does not exist.\n".format(
                        lib_file))
                sys.exit(1)
            self._lib_dict[splitext(basename(lib_file))[0]] = Library(
                paired_end, max_insert_size, lib_file,
                deepcopy(self._replicon_dict))
        self._lib_names_list = list(self._lib_dict.keys())
        print("The following libraries were initialized:\n"
              "# Experiment libraries\n{0}\n"
              "# Control libraries\n{1}".format(
                  '\n'.join(self._exp_lib_list),
                  '\n'.join(self._ctr_lib_list))) 

def run_blockbuster(self, blockbuster_path):
        self._blockbuster_output = ""
        for replicon in sorted(self._replicon_dict):
            self._replicon_dict[replicon][
                "blockbuster"] = self._blockbuster_worker(blockbuster_path,
                                                          replicon)
            print("blockbuster for replicon {} exited with status {}".format(
                replicon, self._replicon_dict[replicon]["blockbuster"][
                    "returncode"]))
            if not self._replicon_dict[replicon]["blockbuster"][
                    "returncode"] == 0:
                print(self._replicon_dict[replicon]["blockbuster"]["error"])
                sys.exit(1)
            self._blockbuster_output += self._replicon_dict[replicon][
                "blockbuster"]["output"]
            del self._replicon_dict[replicon]["blockbuster"]
        with open("{}/blockbuster.txt".format(self._output_folder),
                  'w') as blockbuster_fh:
            blockbuster_fh.write(self._blockbuster_output) 

def init_replicons(self):
        for bam_file in self._libs:
            bam_fh = pysam.Samfile(bam_file, "rb")
            replicon_dict = dict([[replicon, {
                'seq_start_pos': 0,
                'seq_end_pos': length}] for replicon, length in zip(
                    bam_fh.references, bam_fh.lengths)])
            if self.replicon_dict and not (
                    self.replicon_dict == replicon_dict):
                sys.stderr.write(
                    "Mismatch in replicon content of bam files!\n")
                sys.exit(1)
            self.replicon_dict = replicon_dict
        self._check_annotations()
        print("Peak detection will be conducted for the following sequence "
              "regions:", flush=True)
        for seq_id, replicon in sorted(self.replicon_dict.items()):
            print("{}: {} {}".format(
                seq_id,
                replicon['seq_start_pos'] + 1,
                replicon['seq_end_pos']), flush=True)
            if 'features' not in replicon:
                replicon['features'] = [] 

def _select_normalization_window(self, lib_count):
        print("Selected normalization method is: {}".format(
            self._args.norm_method))
        if self._args.norm_method == "manual":
            if not (len(self._args.size_factors) == (lib_count)):
                sys.stderr.write(
                        "Normalization factors do not match library number!\n")
                sys.exit(1)
            size_factors = [value/max(self._args.size_factors)
                            for value in self._args.size_factors]
        elif self._args.norm_method == "none":
            if self._args.size_factors:
                sys.stderr.write("Specified size factors were ignored!\n")
            size_factors = [1.0] * (lib_count)
        else:
            if self._args.size_factors:
                sys.stderr.write("Specified size factors were ignored!\n")
            size_factors = None
        return size_factors 

def __init__(self):

        self._logger = Util.get_logger('SPOT.COMMON.KERBEROS')
        principal, keytab, sasl_mech, security_proto = config.kerberos()

        if os.getenv('KINITPATH'):
            self._kinit = os.getenv('KINITPATH')
        else:
            self._kinit = "kinit"

        self._kinitopts = os.getenv('KINITOPTS')
        self._keytab = "-kt {0}".format(keytab)
        self._krb_user = principal

        if self._kinit == None or self._keytab == None or self._krb_user == None:
            self._logger.error("Please verify kerberos configuration, some environment variables are missing.")
            sys.exit(1)

        if self._kinitopts is None:
            self._kinit_cmd = "{0} {1} {2}".format(self._kinit, self._keytab, self._krb_user)
        else:
            self._kinit_cmd = "{0} {1} {2} {3}".format(self._kinit, self._kinitopts, self._keytab, self._krb_user) 

def validate_parameters_values(args,logger):

    logger.info("Validating input parameter values")

    #date.
    is_date_ok = True if len(args.date) == 8 else False

    # type
    dirs = os.walk(script_path).next()[1]
    is_type_ok = True if args.type in dirs else False

    #limit
    try:
        int(args.limit)
        is_limit_ok = True
    except ValueError:
        is_limit_ok = False

    if not is_date_ok: logger.error("date parameter is not correct, please validate it")
    if not is_type_ok: logger.error("type parameter is not supported, please select a valid type")
    if not is_limit_ok: logger.error("limit parameter is not correct, please select a valid limit")
    if not is_date_ok or not is_type_ok or not is_limit_ok: sys.exit(1) 

def _file_checker(subject, file_path):
    """Check if a file exists and informs user about content output.

    :param subject: Very short file description, subject for printed sentences.
    """
    if file_path:
        if os.path.exists(file_path):
            click.echo(
                click.style(
                    "Abort: The {} file already exists.", fg="red"
                ).format(file_path),
                err=True,
            )
            sys.exit(1)
        else:
            click.echo("{} will be written to: {}".format(subject, file_path))
    else:
        click.echo("{} will be output to console.".format(subject)) 

def read_code(file_path=None):
    """Extract the MicroPython code to a file or print it."""
    click.echo("Executing: {}\n".format(read_code.__doc__))
    _file_checker("MicroPython code", file_path)

    click.echo("Reading the micro:bit flash contents...")
    try:
        python_code = read_python_code()
    except Exception as e:
        click.echo(click.style("Error: {}", fg="red").format(e), err=True)
        sys.exit(1)

    if file_path:
        click.echo("Saving the MicroPython code...")
        with open(file_path, "w") as python_file:
            python_file.write(python_code)
    else:
        click.echo("Printing the MicroPython code")
        click.echo("----------------------------------------")
        click.echo(python_code)
        click.echo("----------------------------------------")

    click.echo("\nFinished successfully!") 

def read_flash(file_path=None):
    """Read the micro:bit flash contents into a hex file or console."""
    click.echo("Executing: {}\n".format(read_flash.__doc__))
    _file_checker("micro:bit flash hex", file_path)

    click.echo("Reading the micro:bit flash contents...")
    try:
        flash_data = read_flash_hex()
    except Exception as e:
        click.echo(click.style("Error: {}", fg="red").format(e), err=True)
        sys.exit(1)

    if file_path:
        click.echo("Saving the flash contents...")
        with open(file_path, "w") as hex_file:
            hex_file.write(flash_data)
    else:
        click.echo("Printing the flash contents")
        click.echo("----------------------------------------")
        click.echo(flash_data)
        click.echo("----------------------------------------")

    click.echo("\nFinished successfully!") 

def compare_flash(file_path):
    """Compare the micro:bit flash contents with a hex file.

    Opens the default browser to display an HTML page with the comparison
    output.
    """
    click.echo("Executing: Compare the micro:bit flash with a hex file.\n")
    if not file_path or not os.path.isfile(file_path):
        click.echo(
            click.style("Abort: File does not exists", fg="red"), err=True
        )
        sys.exit(1)

    click.echo("Reading the micro:bit flash contents...")
    try:
        compare_full_flash_hex(file_path)
    except Exception as e:
        click.echo(click.style("Error: {}", fg="red").format(e), err=True)
        sys.exit(1)
    click.echo("Diff output loaded in default browser.")

    click.echo("\nFinished successfully!") 

def style():
    """Run Flake8 linter with all its plugins."""
    _set_cwd()
    print("----------------------")
    print("Running Style Checker:")
    print("----------------------")
    try:
        import black
    except ImportError:
        print("Black Python module not found, style check skipped.")
        return 0
    black_cmd = ["black", ".", "--check", "--diff"]
    return_code = _run_cli_cmd(black_cmd)
    if return_code != 0:
        sys.exit(return_code)
    return return_code 

def publish_test(ctx):
    """Publish the Python Package to the TestPyPI repository."""
    ctx.invoke(clean)
    ctx.invoke(package)
    _set_cwd()
    print("-----------------------------")
    print("Publish package to test PyPI:")
    print("-----------------------------")
    rtn_code = _run_cli_cmd(
        [
            "poetry",
            "config",
            "repositories.testpypi",
            "https://test.pypi.org/legacy/",
        ]
    )
    if rtn_code != 0:
        sys.exit(rtn_code)
    rtn_code = _run_cli_cmd(["poetry", "publish", "-r", "testpypi"])
    if rtn_code != 0:
        sys.exit(rtn_code)
    return 0 

def main(raw_args):
    import cmd_utils

    parser = cmd_utils.build_arg_parser(description='Builds LLVM for Mono')

    default_help = 'default: %(default)s'

    parser.add_argument('action', choices=['make', 'clean'])
    parser.add_argument('--target', choices=target_values, action='append', required=True)

    cmd_utils.add_base_arguments(parser, default_help)

    args = parser.parse_args(raw_args)

    opts = base_opts_from_args(args)
    targets = args.target

    try:
        for target in targets:
            action = { 'make': make, 'clean': clean }[args.action]
            action(opts, target)
    except BuildError as e:
        sys.exit(e.message) 

def __call__(self, *args, **kwargs):
        """Trap ``SIGTERM`` and call wrapped function."""
        self._caught_signal = None
        # Register handler for SIGTERM, then call `self.func`
        self.old_signal_handler = signal.getsignal(signal.SIGTERM)
        signal.signal(signal.SIGTERM, self.signal_handler)

        self.func(*args, **kwargs)

        # Restore old signal handler
        signal.signal(signal.SIGTERM, self.old_signal_handler)

        # Handle any signal caught during execution
        if self._caught_signal is not None:
            signum, frame = self._caught_signal
            if callable(self.old_signal_handler):
                self.old_signal_handler(signum, frame)
            elif self.old_signal_handler == signal.SIG_DFL:
                sys.exit(0) 

def __init__(self, app):
        """
        Initialisation of qi framework and event detection.
        """
        super(HumanTrackedEventWatcher, self).__init__()

        try:
            app.start()
        except RuntimeError:
            print ("Can't connect to Naoqi at ip \"" + args.ip + "\" on port " +
                   str(args.port) + ".\n")

            sys.exit(1)

        session = app.session
        self.subscribers_list = []
        self.is_speech_reco_started = False

        # SUBSCRIBING SERVICES
        self.tts             = session.service("ALTextToSpeech")
        self.memory          = session.service("ALMemory")
        self.motion          = session.service("ALMotion")
        self.speech_reco     = session.service("ALSpeechRecognition")
        self.basic_awareness = session.service("ALBasicAwareness") 

def run(self):
        """
        Loop on, wait for events until manual interruption.
        """
        # start
        print "Waiting for the robot to be in wake up position"
        self.motion.wakeUp()

        print "Starting HumanGreeter"
        try:
            while True:
                time.sleep(1)
        except KeyboardInterrupt:
            print "Interrupted by user, stopping HumanGreeter"
            self.face_detection.unsubscribe("HumanGreeter")
            #stop
            sys.exit(0)

        return 

def save(filename):
    try:
        os.mkdir('saves/')
    except OSError:
        pass
    try:
        data = unicornhathd.get_pixels()
        data = repr(data)
        data = data.replace('array', 'list')
        print(filename, data)
        file = open('saves/' + filename + '.py', 'w')
        file.write("""#!/usr/bin/env python
import unicornhathd
import signal

unicornhathd.rotation(0)

pixels = {}

for x in range(unicornhathd.WIDTH):
    for y in range(unicornhathd.HEIGHT):
        r, g, b = pixels[x][y]
        unicornhathd.set_pixel(x, y, r, g, b)

unicornhathd.show()

print("\\nShowing: {}\\nPress Ctrl+C to exit!")

signal.pause()
""".format(data, filename))
        file.close()
        os.chmod('saves/' + filename + '.py', 0o777 | stat.S_IEXEC)

        return("ok" + str(unicornhathd.get_pixels()))
    except AttributeError:
        print("Unable to save, please update")
        print("unicornhathdhathd library!")
        return("fail") 

def main(argc, argv):
  if argc < 4:
    print("Usage:%s <vob> <input> <output>" % (argv[0]))
    sys.exit(1)
  vp = open(argv[1], "r")
  inp = open(argv[2], "r")
  oup = open(argv[3], "w")
  vobsMap = {}
  for line in vp:
    line = line.strip()
    ss = line.split(" ")
    vobsMap[ss[0]] = 1
  while True:
    line = inp.readline()
    if not line:
      break
    line = line.strip()
    if not line:
      continue
    ss = line.split(" ")
    tokens = []
    for s in ss:
      if s in vobsMap:
        tokens.append(s)
      else:
        tokens.append("<UNK>")
    oup.write("%s\n" % (" ".join(tokens)))
  oup.close()
  inp.close()
  vp.close() 

def safeUpdate():
	FACTORIOPATH = getFactorioPath()

	if os.path.isdir("%s" % (FACTORIOPATH) ): 
		updateFactorio()
	else:
		print("Cannot update factorio. %s does not exist." % (FACTORIOPATH))
		sys.exit(1)

#http://stackoverflow.com/a/22331852/263449 

def updateFactorio():
	

	file_name = "/tmp/latestFactorio.tar.gz"
	print("Downloading %s" % file_name)

	r = requests.get(DOWNLOADURL, stream=True)
	total_length = int(r.headers.get('content-length'))

	if not os.path.isfile(file_name) or total_length != os.path.getsize(file_name):
		with open(file_name, 'wb') as f:
			for chunk in progress.bar(r.iter_content(chunk_size=1024), expected_size=(total_length/1024) + 1): 
				if chunk:
					f.write(chunk)
					f.flush()
			#os.chmod(file_name, stat.S_IWUSR | stat.S_IRUSR)
	else:
		print("File already exists and file sizes match. Skipping download.")	

	if os.access(FACTORIOPATH, os.W_OK):
		if os.path.isfile(file_name):
			tar = tarfile.open(file_name, "r:gz")
			tar.extractall(path="/tmp")
			tar.close()

			copytree("/tmp/factorio", FACTORIOPATH)
			print("Success.")
		else:
			print("Help! Can't find %s, but I should have!" % (file_name))
			sys.exit(1)			
	else:
		print("Can't write to %s" % (FACTORIOPATH))
		sys.exit(1) 

def safeInstall():
	FACTORIOPATH = getFactorioPath()

	try:
		if not os.path.isdir("%s" % (FACTORIOPATH) ):		

			if os.access("%s/.." % (FACTORIOPATH), os.W_OK):
				os.mkdir(FACTORIOPATH, 0o777)
			else:
				subprocess.call(['sudo', 'mkdir', '-p', FACTORIOPATH])
				subprocess.call(['sudo', 'chown', getpass.getuser(), FACTORIOPATH])
							

			os.mkdir(os.path.join(FACTORIOPATH, "saves"))
			os.mkdir(os.path.join(FACTORIOPATH, "config"))
			with open("%s/.bashrc" % (os.path.expanduser("~")), "r+") as bashrc:
				lines = bashrc.read()
				
				if lines.find("eval \"$(_FACTOTUM_COMPLETE=source factotum)\"\n") == -1:
					bashrc.write("eval \"$(_FACTOTUM_COMPLETE=source factotum)\"\n")
					print("You'll want to restart your shell for command autocompletion. Tab is your friend.")
		updateFactorio()
	except IOError as e:
		print("Cannot make %s. Please check permissions. Error %s" % (FACTORIOPATH, e))
		sys.exit(1) 

def rconCmd(cmd):
	host = "localhost"
	port = 27015

	try:
		with open("/tmp/factorioRcon", "r") as phraseFile:
			phrase = phraseFile.readline().strip()

			cmd = ' '.join(cmd)
			loop = asyncio.get_event_loop()
			conn = RconConnection(host, port, phrase)
			resp = loop.run_until_complete(conn.exec_command(cmd))
			print(resp, end='')	
	except FileNotFoundError:
		print("Cannot find the rcon password. Is the server running?")
		sys.exit(1) 

def runFactorio(stdin=True):

	FACTORIOPATH = getFactorioPath()

	if not stdin:
		print("Interactive mode enabled. Server can only be quit with ctrl-C. /quit will only restart the server.")	
	phrase = generatePhrase(6)
	with open("/tmp/factorioRcon", "w") as phraseFile:
		phraseFile.write(phrase)

	with codecs.open("%s/config/settings.json" % (FACTORIOPATH), 'r+', encoding='utf-8') as settings_file:
		try:
			settingsJson = json.load(settings_file)
		except Exception as e:
			print("Problem with settings file.")
			print(e)
			sys.exit(1)


	if not filter(os.path.isfile, glob.glob('%s/saves/*.zip' % (FACTORIOPATH))):
		print("Cannot find a save file. Exiting.")
		sys.exit(1)

	nohup="nohup "

	if not stdin:
		print(nohup)
		nohup=""
	myprogram = {"cmd": "%s%s/bin/x64/factorio --rcon-port 27015 --rcon-password \"%s\" --start-server-load-latest --server-settings %s/config/settings.json" % (nohup, FACTORIOPATH, phrase, FACTORIOPATH) , "numprocesses": 1, "stop_timeout": 20, "close_child_stdin":stdin}

	arbiter = circus.get_arbiter([myprogram])
	try:
					
		arbiter.start()
	finally:
		os.remove("/tmp/factorioRcon")
		arbiter.stop()
		print(subprocess.check_output(['tail', '-13', '%s/factorio-current.log'%FACTORIOPATH]).decode("unicode_escape")) 

def send_fds_and_popen(opts, **popen_kwargs):
    return popen_and_inject_fds_after_sudo(
        opts.cmd,
        opts.fd,
        lambda wrapped_cmd: subprocess.Popen([
            *(['sudo', *opts.sudo_arg, '--'] if opts.sudo else []),
            *wrapped_cmd,
        ], **popen_kwargs),
        set_listen_fds=opts.set_listen_fds,
    )


# The CLI functionality is pretty well-covered in `test_send_fds_and_run.py`.
# Here is a manual smoke test that should print nothing and exit with code 1.
#     buck run //fs_image:send-fds-and-run -- --no-set-listen-fds -- \
#         printenv LISTEN_FDS LISTEN_PID ; echo $? 

def error(msg, rc):
    sys.stderr.write(msg)
    sys.exit(rc) 

def start_worker(type, topic, id, processes=None):

    logger = Util.get_logger("SPOT.INGEST.WORKER")

    # validate the given configuration exists in ingest_conf.json.
    if not type in WORKER_CONF["pipelines"]:
        logger.error("'{0}' type is not a valid configuration.".format(type))
        sys.exit(1)

    # validate the type is a valid module.
    if not Util.validate_data_source(WORKER_CONF["pipelines"][type]["type"]):
        logger.error("The provided data source {0} is not valid".format(type))
        sys.exit(1)

    # validate if kerberos authentication is required.
    if Config.kerberos_enabled():
        kb = Kerberos()
        kb.authenticate()

    # create a worker instance based on the data source type.
    module = __import__("pipelines.{0}.worker".format(WORKER_CONF["pipelines"][type]["type"]),
                        fromlist=['Worker'])

    # kafka server info.
    logger.info("Initializing kafka instance")
    k_server = WORKER_CONF["kafka"]['kafka_server']
    k_port = WORKER_CONF["kafka"]['kafka_port']

    # required zookeeper info.
    zk_server = WORKER_CONF["kafka"]['zookeper_server']
    zk_port = WORKER_CONF["kafka"]['zookeper_port']
    topic = topic

    # create kafka consumer.
    kafka_consumer = KafkaConsumer(topic, k_server, k_port, zk_server, zk_port, id)

    # start worker.
    db_name = WORKER_CONF['dbname']
    app_path = WORKER_CONF['hdfs_app_path']
    ingest_worker = module.Worker(db_name, app_path, kafka_consumer, type, processes)
    ingest_worker.start() 

def remove_kafka_topic(cls,zk,topic,logger):
        rm_kafka_topic = "kafka-topics --delete --zookeeper {0} --topic {1}".format(zk,topic)
        try:
            logger.info("SPOT.Utils: Executing: {0}".format(rm_kafka_topic))
            subprocess.call(rm_kafka_topic,shell=True)

        except subprocess.CalledProcessError as e:
            logger.error("SPOT.Utils: There was an error executing: {0}".format(e.cmd))
            sys.exit(1) 

def execute_cmd(cls,command,logger):

        try:
            logger.info("SPOT.Utils: Executing: {0}".format(command))
            subprocess.call(command,shell=True)

        except subprocess.CalledProcessError as e:
            logger.error("SPOT.Utils: There was an error executing: {0}".format(e.cmd))
            sys.exit(1) 

def main():
    '''
        Main entry point for Spark Streaming Listener functionality.
    '''
    try: streaming_listener(**parse_args().__dict__)
    except SystemExit: raise
    except:
        sys.excepthook(*sys.exc_info())
        sys.exit(1) 

def main():

    if len(sys.argv[1:]) < 2:
        print "Please provide paths to: old_spot.conf , new_spot.conf"
        sys.exit(1)
        
    old_conf_file = sys.argv[1]
    new_conf_file = sys.argv[2]

    log = util.get_logger('SPOT.MIGRATE.CONF')

    old_path = os.path.dirname(os.path.realpath(old_conf_file))

    # create backup for the current configuration file.
    log.info("Create a backup of /etc/spot.conf before changing it") 
    util.execute_cmd('sudo cp {0} {1}/spot.conf.bkp_0_9'.format(old_conf_file, old_path),log)

    # create configuration objects.
    old_config = ConfigParser.ConfigParser()
    current_config = ConfigParser.ConfigParser()
    new_config = ConfigParser.ConfigParser()
    
    old_config.readfp(SecHead(open(old_conf_file)))
    current_config.readfp(SecHead(open(new_conf_file)))

    # create the new conf file.
    new_config.add_section('conf')
    for (k,v) in current_config.items("conf"):      
        if old_config.has_option('conf',k):
            new_config.set('conf',k, old_config.get('conf',k))
        else:
            new_config.set('conf',k,v)    
   
    new_path = os.path.dirname(os.path.realpath(new_conf_file)) 
    updated_conf_file = '{0}/spot.conf.new'.format(new_path)
    log.info("Generating merged configuration file in {0}".format(updated_conf_file)) 
    formatter(updated_conf_file,new_config)

    log.info("Updating original spot.conf with new and migrated variables and values") 
    util.execute_cmd('sudo cp {0} {1}/spot.conf'.format(updated_conf_file, old_path),log)
    util.execute_cmd('sudo chmod 0755 {0}/spot.conf'.format(old_path),log) 

def execute_cmd(cls,command,logger):

        try:
            logger.info("SPOT.Utils: Executing: {0}".format(command))
            subprocess.call(command,shell=True)

        except subprocess.CalledProcessError as e:
            logger.error("SPOT.Utils: There was an error executing: {0}".format(e.cmd))
            sys.exit(1) 

def _get_flow_results(self):
               
        self._logger.info("Getting {0} Machine Learning Results from HDFS".format(self._date))
        flow_results = "{0}/flow_results.csv".format(self._data_path)

        # get hdfs path from conf file 
        HUSER = self._spot_conf.get('conf', 'HUSER').replace("'", "").replace('"', '')
        hdfs_path = "{0}/flow/scored_results/{1}/scores/flow_results.csv".format(HUSER,self._date)
        
         # get results file from hdfs
        get_command = Util.get_ml_results_form_hdfs(hdfs_path,self._data_path)
        self._logger.info("{0}".format(get_command))

        # valdiate files exists
        if os.path.isfile(flow_results):

            # read number of results based in the limit specified.
            self._logger.info("Reading {0} flow results file: {1}".format(self._date,flow_results))
            self._flow_results = Util.read_results(flow_results,self._limit,self._results_delimiter)
            if len(self._flow_results) == 0: self._logger.error("There are not flow results.");sys.exit(1)

        else:
            self._logger.error("There was an error getting ML results from HDFS")
            sys.exit(1)

        # filter results add rank.
        self._logger.info("Filtering required columns based on configuration")

        self._flow_scores.extend([ [ conn[i] for i in self._conf['column_indexes_filter'] ] + [n] for n, conn in enumerate(self._flow_results) ]) 

def _get_dns_results(self):

        self._logger.info("Getting {0} Machine Learning Results from HDFS".format(self._date))
        dns_results = "{0}/dns_results.csv".format(self._data_path)

        # get hdfs path from conf file.
        HUSER = self._spot_conf.get('conf', 'HUSER').replace("'", "").replace('"', '')
        hdfs_path = "{0}/dns/scored_results/{1}/scores/dns_results.csv".format(HUSER,self._date)

        # get results file from hdfs.
        get_command = Util.get_ml_results_form_hdfs(hdfs_path,self._data_path)
        self._logger.info("{0}".format(get_command))

        if os.path.isfile(dns_results):

            # read number of results based in the limit specified.
            self._logger.info("Reading {0} dns results file: {1}".format(self._date,dns_results))
            self._dns_results = Util.read_results(dns_results,self._limit,self._results_delimiter)[:]
            if len(self._dns_results) == 0: self._logger.error("There are not dns results.");sys.exit(1)

        else:
            self._logger.error("There was an error getting ML results from HDFS")
            sys.exit(1)

        # add dns content.
        self._dns_scores = [ conn[:]  for conn in self._dns_results][:] 

def valid_auth(self):
        delay = 1
        while delay < 100:
            user = self.get_profile()
            if not user:
                msg = "Connect server error or access key is invalid, " \
                      "remove `./data/keys/.access_key` run again"
                logger.error(msg)
                delay += 3
                time.sleep(3)
            else:
                break
        if delay >= 10:
            sys.exit() 

def register_and_save(self):
        terminal = self.register_terminal_v2(
            self.config['NAME'], self.config['BOOTSTRAP_TOKEN']
        )
        if not terminal:
            logger.error("Failed register terminal")
            sys.exit()
        ak = terminal.service_account.access_key
        self.access_key.id, self.access_key.secret = ak.id, ak.secret
        if not self.access_key:
            logger.error("Register error")
            sys.exit()
        self.save_access_key() 

def sync_dns_from_my_ip(self, dns_type='A'):
        """
        Sync dns from my public ip address.
        It will not do update if ip address in dns record is already same as
        current public ip address.
        :param dns_type:
        :return:
        """
        ip_address = ''
        for finder in self.public_ip_finder:
            try:
                result = requests.get(finder)
            except requests.RequestException:
                continue
            if result.status_code == 200:
                try:
                    socket.inet_aton(result.text)
                    ip_address = result.text
                    break
                except socket.error:
                    try:
                        socket.inet_aton(result.json().get('ip'))
                        ip_address = result.json()['ip']
                        break
                    except socket.error:
                        continue

        if ip_address == '':
            print('None of public ip finder is working. Please try later')
            sys.exit(1)

        try:
            record = self.get_record(dns_type, self.domain) \
                if len(self.domain.split('.')) == 3 \
                else self.get_record(dns_type, self.domain)
        except RecordNotFound:
            self.create_record(dns_type, self.domain, ip_address, proxied=self.proxied)
            print('Successfully created new record with IP address {new_ip}'
                  .format(new_ip=ip_address))
        else:
            if record['content'] != ip_address:
                self.update_record(dns_type, self.domain, ip_address, proxied=record['proxied'])
                print('Successfully updated IP address from {old_ip} to {new_ip}'
                      .format(old_ip=record['content'], new_ip=ip_address))
            else:
                print('IP address on CloudFlare is same as your current address') 

def _run_cli_cmd(cmd_list):
    """Run a shell command and return the error code.

    :param cmd_list: A list of strings that make up the command to execute.
    """
    try:
        return subprocess.call(cmd_list)
    except Exception as e:
        print(str(e))
        sys.exit(1) 

def test():
    """Run PyTests with the coverage plugin."""
    _set_cwd()
    return_code = _run_cli_cmd(
        [sys.executable, "-m", "pytest", "-v", "--cov=ubittool", "tests/"]
    )
    if return_code != 0:
        sys.exit(return_code)
    return 0 

def package():
    """Build the Python Package."""
    _set_cwd()
    print("------------------------")
    print("Building Python Package:")
    print("------------------------")
    rtn_code = _run_cli_cmd(["poetry", "build"])
    if rtn_code != 0:
        sys.exit(rtn_code)
    return 0 

def usage():
    sys.stderr.write('usage: python evaluation.py model.json weight.h5 features/ hash.label labels.txt predictions.csv [convert_classes.txt]\n')
    sys.exit(2) 

def usage():
    sys.stderr.write('usage: python3 color.py api-sequences-features/ images/ image.labels errors.txt\n')
    sys.exit(2) 

def usage():
    sys.stderr.write('python attack-config.py original-sequence attack-sequence config-output\n')
    sys.exit(2) 

def usage():
    sys.stderr.write('usage: python lstm.py cuckoo-headless/extract_raw/api.txt features/ models/ save_model[True|False] save_data[True|False] {binary_classification | multi_classification | regression} convert_class.txt\n\n')
    sys.stderr.write('\tout_class.txt: file which stores trained class values in case they change from the original stored\n')
    sys.exit(2) 

def usage():
    sys.stderr.write('usage: python sim_stats.py api-sequences/ hash.label numSamplesPerClass outfile.txt\n')
    sys.exit(2) 

def usage():
    sys.stderr.write('usage: python preprocess.py api-sequence-folder/ api.txt label.txt hash.labels features-folder/ windowSize {binary_classification | multi_classification | regression}\n')
    sys.stderr.write('\n')
    sys.stderr.write('    classification: classes are malware family label\n')
    sys.stderr.write('    regression: classes are the next API call in the sequence immediately after the sliding window\n')
    sys.stderr.write('\n')
    sys.exit(2) 

def usage():
    sys.stderr.write('usage: python lstm.py cuckoo-headless/extract_raw/api.txt features/ models/ save_model[True|False] save_data[True|False] {binary_classification | multi_classification | regression} convert_class.txt\n\n')
    sys.stderr.write('\tout_class.txt: file which stores trained class values in case they change from the original stored\n')
    sys.exit(2) 

def close(event):
    master.withdraw()
    sys.exit() 

def __init__(self, config):  # type: (ApplicationConfig) -> None
        self._preliminary_io = ConsoleIO()

        # Enable trace output for exceptions thrown during boot
        self._preliminary_io.set_verbosity(VERY_VERBOSE)

        self._dispatcher = None

        try:
            dispatcher = config.dispatcher
            if dispatcher and dispatcher.has_listeners(CONFIG):
                dispatcher.dispatch(CONFIG, ConfigEvent(config))

            self._config = config
            self._dispatcher = config.dispatcher
            self._commands = CommandCollection()
            self._named_commands = CommandCollection()
            self._default_commands = CommandCollection()
            self._global_args_format = ArgsFormat(
                list(config.arguments.values()) + list(config.options.values())
            )

            for command_config in config.command_configs:
                self.add_command(command_config)
        except Exception as e:
            if not config.is_exception_caught():
                raise

            # Render the trace to the preliminary IO
            trace = ExceptionTrace(e)
            trace.render(self._preliminary_io)

            # Ignore is_terminated_after_run() setting. This is a fatal error.
            sys.exit(self.exception_to_exit_code(e)) 

def run(
        self, args=None, input_stream=None, output_stream=None, error_stream=None
    ):  # type: (RawArgs, InputStream, OutputStream, OutputStream) -> int
        # Render errors to the preliminary IO until the final IO is created
        io = self._preliminary_io
        try:
            if args is None:
                args = ArgvArgs()

            io_factory = self._config.io_factory

            io = io_factory(
                self, args, input_stream, output_stream, error_stream
            )  # type: IO

            resolved_command = self.resolve_command(args)
            command = resolved_command.command
            parsed_args = resolved_command.args

            status_code = command.handle(parsed_args, io)
        except KeyboardInterrupt:
            status_code = 1
        except Exception as e:
            if not self._config.is_exception_caught():
                raise

            trace = ExceptionTrace(e)
            trace.render(io, simple=isinstance(e, CliKitException))

            status_code = self.exception_to_exit_code(e)

        if self._config.is_terminated_after_run():
            sys.exit(status_code)

        return status_code 

def filter_plugins(plugin_data, selected_plugins):
    """
    Takes the full set of plugin data and filters out only the plugins the
    user selected.

    If the user did not specify a list of plugins, it enables all the default
    plugins
    """
    # No plugins selected- remove all plugins not marked as "default"
    if selected_plugins is None:
        for name, data in list(plugin_data.items()):
            if 'default' not in data or data['default'] is not True:
                del plugin_data[name]
    # Plugin list provided at the command line
    else:
        selected_plugins = selected_plugins.split(',')
        # Ensure each selected plugin actually exists to point out errors
        for name in selected_plugins:
            if name not in plugin_data.keys():
                print('ERROR: Unknown plugin specified: {}'.format(name))
                sys.exit(1)
        # Remove any plugin not matching the providedlist
        for name, data in list(plugin_data.items()):
            if name not in selected_plugins:
                del plugin_data[name]

    return plugin_data 

def transform_body(self, search_re, replacement_string):
        managed = self.current_body()

        matched_hostnames = None
        if managed == '':
            print('Your hostsfile is not managed by webnull, we won\'t change anything')
            exit(1)
        else:
            lines = re.findall(search_re, managed, flags=re.MULTILINE)
            matched_hostnames = set(map(lambda line: re.match(r'^[^\t]+\t+([^\t]+)$', line).group(1), lines))
            if (len(matched_hostnames) != 0):
                new_managed = re.sub(search_re, replacement_string, managed, flags=re.MULTILINE)
                self.write_body(new_managed)

        return matched_hostnames 

def unblock_site(sitename):
    hostname = parse_hostname(sitename)
    hostfile = ManagedHostfile()

    null_matcher = HOST_MATCHER.format(hostname)
    unblocked_hosts = hostfile.transform_body(null_matcher, r'# \1')

    if len(unblocked_hosts) == 0:
        print('No host matches ' + sitename + '.')
        sys.exit(1)

    print('\n'.join(unblocked_hosts)) 

def reblock_timer(duration, cleanup_func):
    if 'TEST_DURATION' in os.environ:
        duration = float(os.environ['TEST_DURATION'])

    def sigint_handler(signal, frame):
        cleanup_func()
        sys.exit(0)
    signals = [signal.SIGINT, signal.SIGHUP]
    for sig in signals:
        signal.signal(sig, sigint_handler)

    end_time = datetime.datetime.now() + datetime.timedelta(minutes=duration)
    ptime = pretty_time(end_time)
    print('allowed until ' + ptime)

    now = time.time()
    end_time = now + (duration * 60)
    while True:
        remaining = end_time - time.time()
        if remaining <= 0:
            break
        if remaining > 1000:
            time.sleep(10)
        else:
            time.sleep(1)

    cleanup_func() 

def die(msg, exit_code=1):
    """Logs error message then exits with code."""
    if LOG is not None:
        LOG.error("FATAL: " + msg)
    else:
        print("FATAL: " + msg)
    sys.exit(exit_code) 

def main(raw_args):
    import cmd_utils
    from cmd_utils import custom_bool

    actions = {
        'make': make_product,
        'clean': clean_product
    }

    parser = cmd_utils.build_arg_parser(description='Builds the Mono BCL')

    default_help = 'default: %(default)s'

    parser.add_argument('action', choices=actions.keys())
    parser.add_argument('--product', choices=product_values, action='append', required=True)
    parser.add_argument('--tests', action='store_true', default=False, help=default_help)
    parser.add_argument('--remove-pdb', type=custom_bool, default=True, help=default_help)

    cmd_utils.add_base_arguments(parser, default_help)

    args = parser.parse_args(raw_args)

    opts = bcl_opts_from_args(args)
    products = args.product

    try:
        for product in products:
            action = actions[args.action]
            action(opts, product)
    except BuildError as e:
        sys.exit(e.message) 

def main(raw_args):
    import cmd_utils

    actions = {
        'build': build,
        'install': install,
        'clean': clean
    }

    parser = cmd_utils.build_arg_parser(description='Copy the reference assemblies')

    default_help = 'default: %(default)s'

    parser.add_argument('action', choices=actions.keys())

    cmd_utils.add_base_arguments(parser, default_help)

    args = parser.parse_args(raw_args)

    opts = base_opts_from_args(args)

    try:
        action = actions[args.action]
        action(opts)
    except BuildError as e:
        sys.exit(e.message) 

def main():
    try:
        from cams.cli import main
        sys.exit(main())
    except KeyboardInterrupt:
        from cams import ExitStatus
        sys.exit(ExitStatus.ERROR_CTRL_C) 

def die_if(cond, msg):
    if cond:
        print 'Error: ' + msg
        sys.exit(1) 

def show_help(status=0):
        """Print help message."""
        print('Usage : update.py (check|install) '
              '[--prereleases] <github_slug> <version>')
        sys.exit(status) 

def args(self):
        """Return command line args as normalised unicode.

        Args are decoded and normalised via :meth:`~Workflow.decode`.

        The encoding and normalisation are the ``input_encoding`` and
        ``normalization`` arguments passed to :class:`Workflow` (``UTF-8``
        and ``NFC`` are the defaults).

        If :class:`Workflow` is called with ``capture_args=True``
        (the default), :class:`Workflow` will look for certain
        ``workflow:*`` args and, if found, perform the corresponding
        actions and exit the workflow.

        See :ref:`Magic arguments <magic-arguments>` for details.

        """
        msg = None
        args = [self.decode(arg) for arg in sys.argv[1:]]

        # Handle magic args
        if len(args) and self._capture_args:
            for name in self.magic_arguments:
                key = '{0}{1}'.format(self.magic_prefix, name)
                if key in args:
                    msg = self.magic_arguments[name]()

            if msg:
                self.logger.debug(msg)
                if not sys.stdout.isatty():  # Show message in Alfred
                    self.add_item(msg, valid=False, icon=ICON_INFO)
                    self.send_feedback()
                sys.exit(0)
        return args 

def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(1) 

def run_tests(self):
        import pytest
        errcode = pytest.main(self.test_args)
        sys.exit(errcode) 

def run(self):
        """
            this example uses the setEngagementMode, startAwareness and
            stopAwareness methods
        """
        # start
        print "Waiting for the robot to be in wake up position"
        self.motion.wakeUp()

        self.create_callbacks()

        print "Starting BasicAwareness with the fully engaged mode"
        self.basic_awareness.setEngagementMode("FullyEngaged")
        self.basic_awareness.setEnabled(True)

        # loop on, wait for events until manual interruption
        try:
            while True:
                time.sleep(1)
        except KeyboardInterrupt:
            print "Interrupted by user, shutting down"
            # stop
            print "Stopping BasicAwareness"
            self.basic_awareness.setEnabled(False)

            print "Waiting for the robot to be in rest position"
            self.motion.rest()

            sys.exit(0)

        return 

def __init__(self, app):
        super(AsthamaDetector, self).__init__()

        try:
            app.start()
        except RuntimeError:
            print ("Can't connect to Naoqi at ip \"" + args.ip + "\" on port " +
                   str(args.port) + ".\n")

            sys.exit(1)

        session = app.session
        self.subscribers_list = []

        # SUBSCRIBING SERVICES
        self.tts                      = session.service("ALTextToSpeech")
        self.video_service            = session.service("ALVideoDevice")
        self.dialog_service           = session.service("ALDialog")
        self.memory_service           = session.service("ALMemory")
        self.motion_service           = session.service("ALMotion")
        self.posture_service          = session.service("ALRobotPosture")
        self.speaking_movement        = session.service("ALSpeakingMovement")
        self.tablet_service           = session.service("ALTabletService")
        self.animation_player_service = session.service("ALAnimationPlayer")

        # INITIALISING CAMERA POINTERS
        self.imageNo2d = 1
        self.imageNo3d = 1

        # PUMP SPECS
        self.grsOn             = False
        self.pumpFound         = False
        self.pumpImgNo         = 0
        self.pumpAngleRotation = 0

        # GRAPHPLOT
        self.PLOTXMIN = -3
        self.PLOTXMAX =  3
        self.PLOTYMIN = -3
        self.PLOTYMAX =  3
        self.fig  = plt.figure() 

def main(settings=None, stdout=None, args=None):
    """Execution entry point."""
    parser = get_parser()
    if args is None:
        args = sys.argv[1:]
    args = parser.parse_args(args)

    config = OdorikConfig(args.config_section)
    if settings is None:
        config.load(args.config)
    else:
        for section, key, value in settings:
            config.set(section, key, value)

    for override in ('user', 'password', 'url'):
        value = getattr(args, override)
        if value is not None:
            config.set(args.config_section, override, value)

    command = COMMANDS[args.cmd](args, config, stdout)
    try:
        command.run()
    except (CommandError, odorik.OdorikException) as error:
        print('Error: {0}'.format(error), file=sys.stderr)
        sys.exit(1) 

def spark_job(script_file, **kwargs):
    '''
        Run given script file by appling it as a Spark Job.
    '''
    spark_job  = 'spark2-submit --master {0}'.format(kwargs.pop('master'))
    spark_job += ' --deploy-mode {0}'.format(kwargs.pop('deploy_mode'))
    spark_job += ' --py-files {0}'.format(kwargs.pop('py_files'))

    if 'driver_memory' in kwargs.keys():
        spark_job += ' --driver-memory {0}'.format(kwargs.pop('driver_memory'))

    if 'spark_exec' in kwargs.keys():
        spark_job += ' --num-executors {0}'.format(kwargs.pop('spark_exec'))

    if 'spark_executor_memory' in kwargs.keys():
        spark_job += ' --conf spark.executor.memory={0}'.format(kwargs.pop('spark_executor_memory'))

    if 'spark_executor_cores' in kwargs.keys():
        spark_job += ' --conf spark.executor.cores={0}'.format(kwargs.pop('spark_executor_cores'))

    spark_job += ' {0}'.format(os.path.abspath(script_file))

    if 'spark_batch_size' in kwargs.keys():
        spark_job += ' -b {0}'.format(kwargs.pop('spark_batch_size'))

    spark_job += ' -d {0}'.format(kwargs.pop('database'))

    if kwargs['group_id'] is not None:
        spark_job += ' -g {0}'.format(kwargs.pop('group_id'))

    spark_job += ' -l {0}'.format(kwargs.pop('log_level'))

    if kwargs['app_name'] is not None:
        spark_job += ' -n {0}'.format(kwargs.pop('app_name'))

    spark_job += ' -p {0}'.format(kwargs.pop('partitions'))
    spark_job += ' -t {0}'.format(kwargs.pop('type'))
    spark_job += ' --topic {0}'.format(kwargs.pop('topic'))
    spark_job += ' --zkquorum {0}'.format(kwargs.pop('zkquorum'))

    if kwargs['redirect_spark_logs'] is not None:
        spark_job += ' 2>{0}'.format(kwargs.pop('redirect_spark_logs'))

    try: Util.call(spark_job, True)
    except Exception as exc:
        sys.stderr.write('Failed to submit Spark Job!\n')
        sys.stderr.write('[{0}] {1}\n\n'.format(exc.__class__.__name__, exc.message))
        sys.exit(2) 

def main():
    '''
        Main command-line entry point.
    '''
    state = {}

    try:
        args = parse_args()
        conf = json.loads(args.config_file.read())

        # .............................check kerberos authentication
        if os.getenv('KRB_AUTH'):
            kb = Kerberos()
            kb.authenticate()

        state.update(**args.__dict__)

        # .............................add Spark Streaming parameters
        for key in conf['spark-streaming'].keys():
            if conf['spark-streaming'][key] == None:
                continue

            if isinstance(conf['spark-streaming'][key], basestring):
                conf['spark-streaming'][key] = conf['spark-streaming'][key].strip()

                if bool(conf['spark-streaming'][key]):
                    state[key] = conf['spark-streaming'][key]
                continue
            state[key] = conf['spark-streaming'][key]

        # .............................add files to place on the PYTHONPATH
        state['py_files'] = ','.join([os.path.abspath(os.path.join('dist', x)) for x in os.listdir('dist')])

        # .............................add database name
        state['database'] = conf['dbname']

        # .............................add zookeeper's connection string
        state['zkquorum'] = '{0}:{1}'.format(conf['kafka']['zookeper_server'],
                                        conf['kafka']['zookeper_port'])

        spark_job('common/listener.py', **state)

    except SystemExit: raise
    except:
        sys.excepthook(*sys.exc_info())
        sys.exit(1) 

def start_collector(type, workers_num, id=None):

    # generate ingest id
    ingest_id = str(datetime.datetime.time(datetime.datetime.now())).replace(":", "_").replace(".", "_")

    # create logger.
    logger = Util.get_logger("SPOT.INGEST")

    # validate the given configuration exists in ingest_conf.json.
    if not type in MASTER_CONF["pipelines"]:
        logger.error("'{0}' type is not a valid configuration.".format(type))
        sys.exit(1)

    # validate the type is a valid module.
    if not Util.validate_data_source(MASTER_CONF["pipelines"][type]["type"]):
        logger.error("'{0}' type is not configured. Please check you ingest conf file".format(MASTER_CONF["pipelines"][type]["type"]))
        sys.exit(1)

    # validate if kerberos authentication is required.
    if Config.kerberos_enabled():
        kb = Kerberos()
        kb.authenticate()

    # kafka server info.
    logger.info("Initializing kafka instance")
    k_server = MASTER_CONF["kafka"]['kafka_server']
    k_port = MASTER_CONF["kafka"]['kafka_port']

    # required zookeeper info.
    zk_server = MASTER_CONF["kafka"]['zookeper_server']
    zk_port = MASTER_CONF["kafka"]['zookeper_port']
         
    topic = "{0}".format(type,ingest_id) if not id else id
    producer = KafkaProducer(topic, k_server, k_port, zk_server, zk_port, workers_num)

    # create a collector instance based on data source type.
    logger.info("Starting {0} ingest instance".format(topic))
    module = __import__("pipelines.{0}.collector".
                        format(MASTER_CONF["pipelines"][type]["type"]),
                        fromlist=['Collector'])

    # start collector.
    ingest_collector = module.Collector(MASTER_CONF['hdfs_app_path'], producer, type)
    ingest_collector.start() 

def usage():
    sys.stderr.write('usage: python lstm.py cuckoo-headless/extract_raw/api.txt features/ models/ save_model[True|False] save_data[True|False] {binary_classification | multi_classification | regression} convert_class.txt\n\n')
    sys.stderr.write('\tout_class.txt: file which stores trained class values in case they change from the original stored\n')
    sys.exit(2) 

def main(raw_args):
    import cmd_utils
    from collections import OrderedDict
    from typing import Callable

    actions = OrderedDict()
    actions['configure'] = configure
    actions['make'] = make
    actions['clean'] = clean

    parser = cmd_utils.build_arg_parser(description='Builds the Mono runtime for the Desktop')
    subparsers = parser.add_subparsers(dest='platform')

    default_help = 'default: %(default)s'

    for target_platform in target_platforms:
        target_platform_subparser = subparsers.add_parser(target_platform)
        target_platform_subparser.add_argument('action', choices=['configure', 'make', 'clean'])
        target_platform_subparser.add_argument('--target', choices=targets[target_platform], action='append', required=True)
        target_platform_subparser.add_argument('--with-llvm', action='store_true', default=False, help=default_help)

    cmd_utils.add_runtime_arguments(parser, default_help)

    args = parser.parse_args(raw_args)

    input_action = args.action
    input_target_platform = args.platform
    input_targets = args.target

    opts = desktop_opts_from_args(args)

    if not os.path.isdir(opts.mono_source_root):
        print('Mono sources directory not found: ' + opts.mono_source_root)
        sys.exit(1)

    if input_target_platform == 'osx' and sys.platform != 'darwin' and not 'OSXCROSS_ROOT' in os.environ:
        raise RuntimeError('The \'OSXCROSS_ROOT\' environment variable is required for cross-compiling to macOS')

    if is_cross_compiling(input_target_platform) and sys.platform == 'darwin':
        raise RuntimeError('Cross-compiling from macOS is not supported')

    action = actions[input_action]

    try:
        for target in input_targets:
            action(opts, 'desktop-%s' % input_target_platform, input_target_platform, target)
    except BuildError as e:
        sys.exit(e.message) 

def main(raw_args):
    import cmd_utils
    from collections import OrderedDict
    from typing import Callable

    target_shortcuts = {'all-runtime': runtime_targets}

    target_values = runtime_targets + cross_targets + cross_mxe_targets + list(target_shortcuts)

    actions = OrderedDict()
    actions['configure'] = configure
    actions['make'] = make
    actions['clean'] = clean

    parser = cmd_utils.build_arg_parser(description='Builds the Mono runtime for WebAssembly')

    emsdk_root_default = os.environ.get('EMSDK_ROOT', default='')

    default_help = 'default: %(default)s'

    parser.add_argument('action', choices=['configure', 'make', 'clean'])
    parser.add_argument('--target', choices=target_values, action='append', required=True)

    cmd_utils.add_runtime_arguments(parser, default_help)

    args = parser.parse_args(raw_args)

    input_action = args.action
    input_targets = args.target

    opts = runtime_opts_from_args(args)

    if not os.path.isdir(opts.mono_source_root):
        print('Mono sources directory not found: ' + opts.mono_source_root)
        sys.exit(1)

    targets = cmd_utils.expand_input_targets(input_targets, target_shortcuts)
    action = actions[input_action]

    try:
        for target in targets:
            action(opts, 'wasm', target)
    except BuildError as e:
        sys.exit(e.message) 

def PS1catalog(ra,dec,magmin,magmax):

    queryurl = 'https://archive.stsci.edu/panstarrs/search.php?'
    queryurl += 'RA='+str(ra)
    queryurl += '&DEC='+str(dec)
    queryurl += '&SR=0.083&selectedColumnsCsv=ndetections,raMean,decMean,'
    queryurl += 'gMeanPSFMag,rMeanPSFMag,iMeanPSFMag,zMeanPSFMag,yMeanPSFMag,iMeanKronMag'
    queryurl += '&ordercolumn1=ndetections&descending1=on&max_records=200'

    print('\nQuerying PS1 for reference stars via MAST...\n')

    query = requests.get(queryurl)

    results = query.text

    entries = results.split('DATA')[2][11:][:-19].split('</TD>\n</TR>\n<TR>\n<TD>')

    data = []

    for i in entries:
        data.append(np.array(i.split('</TD><TD>')).T)

    if len(data) > 1:

        data = np.array(data).astype(float)

        # Get rid of n_det column
        data = data[:,1:][data[:,0]>3]

        # Get rid of non-detections:
        data = data[data[:,2]>-999]
        data = data[data[:,3]>-999]
        data = data[data[:,4]>-999]
        data = data[data[:,5]>-999]
        data = data[data[:,6]>-999]

        # Get rid of very faint stars
        data = data[data[:,2]<magmin]
        data = data[data[:,3]<magmin]
        data = data[data[:,4]<magmin]
        data = data[data[:,5]<magmin]
        data = data[data[:,6]<magmin]

        # Get rid of stars likely to saturate
        data = data[data[:,2]>magmax]
        data = data[data[:,3]>magmax]
        data = data[data[:,4]>magmax]
        data = data[data[:,5]>magmax]
        data = data[data[:,6]>magmax]


        # Star-galaxy separation
        data = data[:,:-1][data[:,4]-data[:,-1]<0.05]

        np.savetxt('PS1_seq.txt',data,fmt='%.8f\t%.8f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f',header='Ra\tDec\tg\tr\ti\tz\ty\n',comments='')

        print('Success! Sequence star file created: PS1_seq.txt')

    else:
        sys.exit('Field not in PS1! Exiting') 

def PS1catalog(ra,dec,magmin,magmax):

    queryurl = 'https://archive.stsci.edu/panstarrs/search.php?'
    queryurl += 'RA='+str(ra)
    queryurl += '&DEC='+str(dec)
    queryurl += '&SR=0.083&selectedColumnsCsv=ndetections,raMean,decMean,'
    queryurl += 'gMeanPSFMag,rMeanPSFMag,iMeanPSFMag,zMeanPSFMag,yMeanPSFMag,iMeanKronMag'
    queryurl += '&ordercolumn1=ndetections&descending1=on&max_records=200'

    print '\nQuerying PS1 for reference stars via MAST...\n'

    query = requests.get(queryurl)

    results = query.text

    entries = results.split('DATA')[2][11:][:-19].split('</TD>\n</TR>\n<TR>\n<TD>')

    data = []

    for i in entries:
        data.append(np.array(i.split('</TD><TD>')).T)

    if len(data) > 1:

        data = np.array(data).astype(float)

        # Get rid of n_det column
        data = data[:,1:][data[:,0]>3]

        # Get rid of non-detections:
        data = data[data[:,2]>-999]
        data = data[data[:,3]>-999]
        data = data[data[:,4]>-999]
        data = data[data[:,5]>-999]
        data = data[data[:,6]>-999]

        # Get rid of very faint stars
        data = data[data[:,2]<magmin]
        data = data[data[:,3]<magmin]
        data = data[data[:,4]<magmin]
        data = data[data[:,5]<magmin]
        data = data[data[:,6]<magmin]

        # Get rid of stars likely to saturate
        data = data[data[:,2]>magmax]
        data = data[data[:,3]>magmax]
        data = data[data[:,4]>magmax]
        data = data[data[:,5]>magmax]
        data = data[data[:,6]>magmax]


        # Star-galaxy separation
        data = data[:,:-1][data[:,4]-data[:,-1]<0.05]

        np.savetxt('PS1_seq.txt',data,fmt='%.8f\t%.8f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f',header='Ra\tDec\tg\tr\ti\tz\ty\n',comments='')

        print 'Success! Sequence star file created: PS1_seq.txt'

    else:
        sys.exit('Field not in PS1! Exiting') 

def run(self):
        try:
            self.status('Removing previous builds…')
            rmtree(os.path.join(here, 'dist'))
        except OSError:
            pass

        self.status('Building Source and Wheel (universal) distribution…')
        os.system('{0} setup.py sdist bdist_wheel --universal'.format(
            sys.executable
        ))

        self.status('Uploading the package to PyPi via Twine…')
        os.system('twine upload dist/*')

        sys.exit()