Python argparse.Namespace() Examples

def grant(args):
    """
    Given an IAM role or instance name, attach an IAM policy granting
    appropriate permissions to subscribe to deployments. Given a
    GitHub repo URL, create and record deployment keys for the repo
    and any of its private submodules, making the keys accessible to
    the IAM role.
    """
    try:
        role = resources.iam.Role(args.iam_role_or_instance)
        role.load()
    except ClientError:
        role = get_iam_role_for_instance(args.iam_role_or_instance)
    role.attach_policy(PolicyArn=ensure_deploy_iam_policy().arn)
    for private_repo in [args.repo] + list(private_submodules(args.repo)):
        gh_owner_name, gh_repo_name = parse_repo_name(private_repo)
        secret = secrets.put(argparse.Namespace(secret_name="deploy.{}.{}".format(gh_owner_name, gh_repo_name),
                                                iam_role=role.name,
                                                instance_profile=None,
                                                iam_group=None,
                                                iam_user=None,
                                                generate_ssh_key=True))
        get_repo(private_repo).create_key(__name__ + "." + role.name, secret["ssh_public_key"])
        logger.info("Created deploy key %s for IAM role %s to access GitHub repo %s",
                    secret["ssh_key_fingerprint"], role.name, private_repo) 

def __init__(self, arguments: argparse.Namespace, help_text: str):
        self.profile_name = arguments.profile_name
        self.access_key = arguments.access_key
        self.secret_access_key = arguments.secret_access_key
        self.session_token = arguments.session_token
        self.region = arguments.region
        self.command = arguments.command
        self.api_id = arguments.api_id
        self.url = arguments.url
        self.api_list = []
        self.client = None
        self.help = help_text

        if self.access_key and self.secret_access_key:
            if not self.region:
                self.error('Please provide a region with AWS credentials')

        if not self.load_creds():
            self.error('Unable to load AWS credentials')

        if not self.command:
            self.error('Please provide a valid command') 

def parse_arguments() -> Tuple[argparse.Namespace, str]:
    """Parse command line arguments and return namespace

    :return: Namespace for arguments and help text as a tuple
    """
    parser = argparse.ArgumentParser(description='FireProx API Gateway Manager')
    parser.add_argument('--profile_name',
                        help='AWS Profile Name to store/retrieve credentials', type=str, default=None)
    parser.add_argument('--access_key',
                        help='AWS Access Key', type=str, default=None)
    parser.add_argument('--secret_access_key',
                        help='AWS Secret Access Key', type=str, default=None)
    parser.add_argument('--session_token',
                        help='AWS Session Token', type=str, default=None)
    parser.add_argument('--region',
                        help='AWS Region', type=str, default=None)
    parser.add_argument('--command',
                        help='Commands: list, create, delete, update', type=str, default=None)
    parser.add_argument('--api_id',
                        help='API ID', type=str, required=False)
    parser.add_argument('--url',
                        help='URL end-point', type=str, required=False)
    return parser.parse_args(), parser.format_help() 

def test_build_kabob_params(self, mock_getcwd, mock_expanduser):
        folderPath = "{path}/test/files".format(path=self.path)
        args = Namespace(version='0.1', test_test="test", arg_arg="argy")

        mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path)
        mock_getcwd.return_value = folderPath

        config = sb.systems.generators.yaml.loadConfig()
        job = config.jobs[0]
        param = sb.objects.param.Param("test-test", "t")
        arg = sb.objects.arg.Arg("arg-arg")
        job.params.append(param)
        job.args.append(arg)

        expected = "bash build.sh 0.1 argy --env local --test-test test --log info"
        command = sb.systems.execution.commandBuilder.build(config, job, args)
        self.assertEqual(command, expected) 

def test_build_list_param(self, mock_getcwd, mock_expanduser):
        folderPath = "{path}/test/files".format(path=self.path)
        args = Namespace(version='0.1', test_test=[1, 2, 3], arg_arg="argy")

        mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path)
        mock_getcwd.return_value = folderPath

        config = sb.systems.generators.yaml.loadConfig()
        job = config.jobs[0]
        param = sb.objects.param.Param("test-test", "t", accepts="list")
        arg = sb.objects.arg.Arg("arg-arg")
        job.params.append(param)
        job.args.append(arg)

        expected = "bash build.sh 0.1 argy --env local --test-test 1 2 3 --log info"
        command = sb.systems.execution.commandBuilder.build(config, job, args)
        self.assertEqual(command, expected) 

def test_execute_push_conflict_s3(self, mock_boto3_session):
        mock_client = mock.Mock()
        mock_session = mock.Mock()
        mock_client.list_objects_v2.return_value = {"Contents": [{"Key": "test_v1.0.0.pkl"}]}
        mock_session.client.return_value = mock_client
        mock_boto3_session.return_value = mock_session

        config = sb.objects.config.Config(version="1.0.0")
        args = argparse.Namespace(job="push", force=False, artifact='test', user='sean', token='abc123')
        expectedException = "This artifact version already exists. Please bump the version or use the force parameter (-f) to overwrite the artifact."

        try:
            self.s3.execute(config, args)
            self.fail("Exception Not Thrown")
        except Exception as exc:
            self.assertEqual(str(exc), expectedException)
            mock_client.list_objects_v2.assert_called_with(Bucket="my-bucket", Prefix="test_v1.0.0.pkl") 

def test_execute_push_artifactory_all(self, mock_artifactory, mock_remove, mock_copy):
        config = sb.objects.config.Config(version="1.0.0")
        args = argparse.Namespace(job="push", force=True, artifact='ALL', user='sean', token='abc123')

        self.artifactory.execute(config, args)

        mock_artifactory.assert_has_calls([
            mock.call("artifactory.test.com/ml/test/test_v1.0.0.pkl", auth=('sean', 'abc123')),
            mock.call("artifactory.test.com/ml/test/test2_v1.0.0.pkl", auth=('sean', 'abc123'))
        ], any_order=True)
        mock_copy.assert_has_calls([
            mock.call("test.pkl", "test_v1.0.0.pkl"),
            mock.call("test2.pkl", "test2_v1.0.0.pkl")
        ], any_order=True)
        mock_remove.assert_has_calls([
            mock.call("test_v1.0.0.pkl"),
            mock.call("test2_v1.0.0.pkl")
        ], any_order=True) 

def test_run_docker_params(self, mock_getcwd, mock_call, mock_expanduser):
        folderPath = "{path}/test/files".format(path=self.path)
        memory = 256
        args = Namespace(version='0.1')

        homePath = "{path}/test/plugins".format(path=self.path)
        mock_expanduser.return_value = homePath
        mock_getcwd.return_value = folderPath
        mock_call.return_value = 0

        config = sb.systems.generators.yaml.loadConfig()
        config.components.append(LimitMemory(memory))
        job = sb.objects.job.Job(name='some_command', help='Dummy', source='echo some_command')
        command = sb.systems.execution.commandBuilder.build(config, job, args)

        expected = "docker run --name test-some_command --rm -i --memory {memory}GB test /bin/bash -c \"echo some_command\"".format(memory=memory)
        sb.systems.execution.docker.run(config, command, job.mode, config.ports, job.mappings, job.name)
        mock_call.assert_called_once_with(expected, shell=True) 

def test_run_docker_params_entrypoint(self, mock_getcwd, mock_call, mock_expanduser):
        folderPath = "{path}/test/files".format(path=self.path)
        memory = 256
        args = Namespace(version='0.1')

        homePath = "{path}/test/plugins".format(path=self.path)
        mock_expanduser.return_value = homePath
        mock_getcwd.return_value = folderPath
        mock_call.return_value = 0

        config = sb.systems.generators.yaml.loadConfig()
        config.primaryExe = "ENTRYPOINT"
        config.components.append(LimitMemory(memory))
        job = sb.objects.job.Job(name='some_command', help='Dummy', source='echo some_command')
        command = sb.systems.execution.commandBuilder.build(config, job, args)

        expected = "docker run --name test-some_command --rm -i --memory {memory}GB --entrypoint echo test some_command".format(memory=memory)
        sb.systems.execution.docker.run(config, command, job.mode, config.ports, job.mappings, job.name)
        mock_call.assert_called_once_with(expected, shell=True) 

def parse_args(argv, command_line_options):
  # type: (List[str], List[type]) -> (argparse.Namespace, List[str])
  """Parses the arguments.

  Args:
    argv: A list of string representing the pipeline arguments.
    command_line_options: A list of type ``VariantTransformsOptions`` that
      specifies the options that will be added to parser.
  """
  parser = argparse.ArgumentParser()
  parser.register('type', 'bool', lambda v: v.lower() == 'true')
  options = [option() for option in command_line_options]
  for transform_options in options:
    transform_options.add_arguments(parser)
  known_args, pipeline_args = parser.parse_known_args(argv)
  for transform_options in options:
    transform_options.validate(known_args)
  _raise_error_on_invalid_flags(pipeline_args)
  if hasattr(known_args, 'input_pattern') or hasattr(known_args, 'input_file'):
    known_args.all_patterns = _get_all_patterns(
        known_args.input_pattern, known_args.input_file)
  return known_args, pipeline_args 

def cli_args(args: Sequence[str], search_config_files: bool = True) -> argparse.Namespace:
    """Command line arguments as parsed args.

    If a INI configuration file is set it is used to set additional default arguments, but
    the CLI arguments override any INI file settings.

    Args:
        args: the argument sequence from the command line
        search_config_files: flag for looking through ``SETTINGS_FILES`` for settings

    Returns:
        Parsed args from ArgumentParser
    """
    parser = cli_parser()

    if search_config_files:
        for ini_config_file in SETTINGS_FILES:

            if ini_config_file.path.exists():
                try:
                    ini_config = read_ini_config(ini_config_file.path, ini_config_file.sections)
                    ini_cli_args = parse_ini_config_with_cli(parser, ini_config, args)
                    return parser.parse_args(ini_cli_args)

                except KeyError:
                    # read_ini_config will raise KeyError if the section is not valid
                    continue

    return parser.parse_args(args) 

def ensure_queue(name):
    cq_args = argparse.Namespace(name=name, priority=5, compute_environments=[name])
    try:
        return create_queue(cq_args)
    except ClientError:
        create_compute_environment(cce_parser.parse_args(args=[name]))
        return create_queue(cq_args) 

def configure(self, args: argparse.Namespace):
        config = self._load_config(args.config)
        if args.check_only:
            self.exit()
        self.create_metrics(config.metrics.values())
        self.query_loop = QueryLoop(config, self.registry, self.logger) 

def test_build(self, mock_getcwd, mock_expanduser):
        folderPath = "{path}/test/files".format(path=self.path)
        args = Namespace(version='0.1', test=True)

        mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path)
        mock_getcwd.return_value = folderPath

        config = sb.systems.generators.yaml.loadConfig()
        job = config.jobs[0]
        param = sb.objects.param.Param("test", "t", accepts="boolean")
        job.params.append(param)

        expected = "bash build.sh 0.1 --env local --test --log info"
        command = sb.systems.execution.commandBuilder.build(config, job, args)
        self.assertEqual(command, expected) 

def test_build_no_boolean(self, mock_getcwd, mock_expanduser):
        folderPath = "{path}/test/files".format(path=self.path)
        args = Namespace(version='1.1')

        mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path)
        mock_getcwd.return_value = folderPath

        config = sb.systems.generators.yaml.loadConfig()
        job = config.jobs[0]
        param = sb.objects.param.Param("test", "t", accepts="boolean")
        job.params.append(param)

        expected = "bash build.sh 1.1 --env local --log info"
        command = sb.systems.execution.commandBuilder.build(config, job, args)
        self.assertEqual(command, expected) 

def test_execute_version(self, mock_skeleParser, mock_print):
        config = sb.objects.config.Config()
        args = argparse.Namespace(job=None, version_global=True)
        mock_skeleParser.parseArgs.return_value = args

        sb.systems.execution.executor.execute(config, mock_skeleParser)

        mock_print.assert_called_with("Skelebot v6.6.6") 

def test_execute_help(self, mock_skeleParser):
        config = sb.objects.config.Config()
        args = argparse.Namespace(job=None)
        mock_skeleParser.parseArgs.return_value = args

        sb.systems.execution.executor.execute(config, mock_skeleParser)

        mock_skeleParser.showHelp.assert_called_once() 

def test_execute_scaffold(self, mock_skeleParser, mock_scaffold):
        config = sb.objects.config.Config()
        args = argparse.Namespace(job="scaffold", existing=False)
        mock_skeleParser.parseArgs.return_value = args

        sb.systems.execution.executor.execute(config, mock_skeleParser)

        mock_scaffold.assert_called_once_with(False) 

def test_execute_job_skip(self, mock_skeleParser, mock_run):
        job = sb.objects.job.Job(name="test", source="test.py")
        config = sb.objects.config.Config(jobs=[job])
        args = argparse.Namespace(job="test", native_global=False, skip_build_global=True)
        mock_skeleParser.parseArgs.return_value = args
        mock_run.return_value = 0

        sb.systems.execution.executor.execute(config, mock_skeleParser)

        mock_run.assert_called_once_with(config, "python -u test.py", "i", [], [], "test") 

def test_execute_job(self, mock_skeleParser, mock_run, mock_build):
        job = sb.objects.job.Job(name="test", source="test.py")
        config = sb.objects.config.Config(jobs=[job])
        args = argparse.Namespace(job="test", native_global=False, skip_build_global=False)
        mock_skeleParser.parseArgs.return_value = args
        mock_run.return_value = 0

        sb.systems.execution.executor.execute(config, mock_skeleParser)

        mock_build.assert_called_once_with(config)
        mock_run.assert_called_once_with(config, "python -u test.py", "i", [], [], "test") 

def test_execute_job_ports(self, mock_skeleParser, mock_run, mock_build):
        job = sb.objects.job.Job(name="test", source="test.py", ports=["10:10", "20:20"])
        config = sb.objects.config.Config(jobs=[job], ports=["30:30", "10:10"])
        args = argparse.Namespace(job="test", native_global=False, skip_build_global=False)
        mock_skeleParser.parseArgs.return_value = args
        mock_run.return_value = 0

        sb.systems.execution.executor.execute(config, mock_skeleParser)

        mock_build.assert_called_once_with(config)
        mock_run.assert_called_once_with(config, "python -u test.py", "i", ["10:10", "20:20", "30:30"], [], "test") 

def test_execute_component(self, mock_skeleParser):
        mock_component = mock.MagicMock()
        mock_component.commands = ["test"]
        config = sb.objects.config.Config(components=[mock_component])
        args = argparse.Namespace(job="test")
        mock_skeleParser.parseArgs.return_value = args

        sb.systems.execution.executor.execute(config, mock_skeleParser)

        mock_component.execute.assert_called_once_with(config, args) 

def test_execute_chain(self, mock_skeleParser, mock_run):
        job = sb.objects.job.Job(name="test", source="test.py")
        config = sb.objects.config.Config(jobs=[job])
        args = argparse.Namespace(job="test", native_global=False, skip_build_global=True)
        mock_skeleParser.parseArgs.return_value = args
        mock_run.return_value = 0

        sb.systems.execution.executor.execute(config, mock_skeleParser, ["test", "+", "test"])

        test_call = mock.call(config, "python -u test.py", "i", [], [], "test")
        mock_run.assert_has_calls([test_call, test_call]) 

def test_execute_chain_fail(self, mock_skeleParser, mock_run):
        job = sb.objects.job.Job(name="test", source="test.py")
        config = sb.objects.config.Config(jobs=[job])
        args = argparse.Namespace(job="test", native_global=False, skip_build_global=True)
        mock_skeleParser.parseArgs.return_value = args
        mock_run.return_value = 1
        try:
            sb.systems.execution.executor.execute(config, mock_skeleParser, ["test", "+", "test"])
            self.fail('exception expected')
        except SystemExit:
            mock_run.assert_called_once_with(config, "python -u test.py", "i", [], [], "test") 

def test_execute_R(self, mock_docker):
        mock_docker.build.return_value = 0
        config = sb.objects.config.Config(language="R")
        args = argparse.Namespace()

        jupyter = sb.components.jupyter.Jupyter(port=1127, folder="notebooks/")
        jupyter.execute(config, args)

        expectedCommand = "jupyter notebook --ip=0.0.0.0 --port=8888 --allow-root --notebook-dir=notebooks/"

        mock_docker.build.assert_called_with(config)
        mock_docker.run.assert_called_with(config, expectedCommand, "it", ["1127:8888"], ".", "jupyter") 

def test_execute_Python(self, mock_docker):
        mock_docker.build.return_value = 0
        config = sb.objects.config.Config(language="Python")
        args = argparse.Namespace()

        jupyter = sb.components.jupyter.Jupyter(port=1127, folder="notebooks/", mappings = ["other_project/data"])
        jupyter.execute(config, args)

        expectedCommand = "jupyter notebook --ip=0.0.0.0 --port=8888 --allow-root --notebook-dir=notebooks/"

        mock_docker.build.assert_called_with(config)
        mock_docker.run.assert_called_with(config, expectedCommand, "it", ["1127:8888"], [".", "other_project/data"], "jupyter") 

def test_execute_push_conflict(self, mock_artifactory, mock_rename):
        config = sb.objects.config.Config(version="1.0.0")
        args = argparse.Namespace(job="push", force=False, artifact='test', user='sean', token='abc123')

        expectedException = "This artifact version already exists. Please bump the version or use the force parameter (-f) to overwrite the artifact."

        try:
            self.artifactory.execute(config, args)
            self.fail("Exception Not Thrown")
        except Exception as exc:
            self.assertEqual(str(exc), expectedException)
            mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v1.0.0.pkl", auth=('sean', 'abc123')) 

def test_execute_push_error(self, mock_artifactory, mock_remove, mock_copy):
        mock_path = mock.MagicMock()
        mock_path.deploy_file = mock.MagicMock(side_effect=KeyError('foo'))
        mock_artifactory.return_value = mock_path

        config = sb.objects.config.Config(version="1.0.0")
        args = argparse.Namespace(job="push", force=True, artifact='test', user='sean', token='abc123')

        with self.assertRaises(KeyError):
            self.artifactory.execute(config, args)

        mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v1.0.0.pkl", auth=('sean', 'abc123'))
        mock_copy.assert_called_with("test.pkl", "test_v1.0.0.pkl")
        mock_remove.assert_called_with("test_v1.0.0.pkl") 

def test_execute_push(self, mock_artifactory, mock_remove, mock_copy):
        config = sb.objects.config.Config(version="1.0.0")
        args = argparse.Namespace(job="push", force=True, artifact='test', user='sean', token='abc123')

        self.artifactory.execute(config, args)

        mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v1.0.0.pkl", auth=('sean', 'abc123'))
        mock_copy.assert_called_with("test.pkl", "test_v1.0.0.pkl")
        mock_remove.assert_called_with("test_v1.0.0.pkl") 

def test_execute_pull(self, mock_artifactory, mock_open, mock_input):
        mock_input.return_value = "abc"

        config = sb.objects.config.Config(version="1.0.0")
        args = argparse.Namespace(job="pull", version='0.1.0', artifact='test', user=None, token=None, override=False)

        self.artifactory.execute(config, args)

        mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v0.1.0.pkl", auth=("abc", "abc"))
        mock_open.assert_called_with("test_v0.1.0.pkl", "wb")