Python os.EX_DATAERR Examples

def filter(args):
    filter_args = dict(logGroupName=args.log_group)
    if args.log_stream:
        filter_args.update(logStreamNames=[args.log_stream])
    if args.pattern:
        filter_args.update(filterPattern=args.pattern)
    if args.start_time:
        filter_args.update(startTime=int(timestamp(args.start_time) * 1000))
    if args.end_time:
        filter_args.update(endTime=int(timestamp(args.end_time) * 1000))
    num_results = 0
    while True:
        for event in paginate(clients.logs.get_paginator("filter_log_events"), **filter_args):
            if "timestamp" not in event or "message" not in event:
                continue
            print_log_event(event)
            num_results += 1
        if args.follow:
            time.sleep(1)
        else:
            return SystemExit(os.EX_OK if num_results > 0 else os.EX_DATAERR) 

def upload(sqlfile, database, presigned_url):
  with open(sqlfile) as f:
    querytxt = f.read()

  client = pytd.Client(apikey=os.getenv('td_apikey'), database=database)
  res = client.query(querytxt)
  df = pd.DataFrame(**res)
  csv = df.to_csv(header=False, index=False)

  print('---- user list as first 10 lines----')
  print('\n'.join(csv.splitlines()[:10]))
  print('---- Total number of IDs = ' + str(len(csv.splitlines())) + '----')

  res = requests.put(
          presigned_url,
          data=gzip.compress(bytes(csv, 'utf-8')),
          headers={'Content-Encoding': 'gzip'}
          )

  if res.status_code != 200:
    logger.error(f"Failed to call Yahoo API with http status code {res.status_code}")
    logger.error(res.text)
    sys.exit(os.EX_DATAERR)
  else:
    print(f"Succeeded calling Yahoo API with http status code {res.status_code}") 

def grep(args):
    if args.context:
        args.before_context = args.after_context = args.context
    if not args.end_time:
        args.end_time = Timestamp("-0s")
    query = clients.logs.start_query(logGroupName=args.log_group,
                                     startTime=int(timestamp(args.start_time) * 1000),
                                     endTime=int(timestamp(args.end_time) * 1000),
                                     queryString=args.query)
    seen_results = {}
    print_with_context = partial(print_log_event_with_context, before=args.before_context, after=args.after_context)
    try:
        with ThreadPoolExecutor() as executor:
            while True:
                res = clients.logs.get_query_results(queryId=query["queryId"])
                log_record_pointers = []
                for record in res["results"]:
                    event = {r["field"]: r["value"] for r in record}
                    event_hash = hashlib.sha256(json.dumps(event, sort_keys=True).encode()).hexdigest()[:32]
                    if event_hash in seen_results:
                        continue
                    if "@ptr" in event and (args.before_context or args.after_context):
                        log_record_pointers.append(event["@ptr"])
                    else:
                        print_log_event(event)
                    seen_results[event_hash] = event
                if log_record_pointers:
                    executor.map(print_with_context, log_record_pointers)
                if res["status"] == "Complete":
                    break
                elif res["status"] in {"Failed", "Cancelled"}:
                    raise AegeaException("Query status: {}".format(res["status"]))
                time.sleep(1)
    finally:
        try:
            clients.logs.stop_query(queryId=query["queryId"])
        except clients.logs.exceptions.InvalidParameterException:
            pass
    logger.debug("Query %s: %s", query["queryId"], res["statistics"])
    return SystemExit(os.EX_OK if seen_results else os.EX_DATAERR) 

def edit(args):
    state = _filter_state(libnmstate.show(), args.only)

    if not state[Interface.KEY]:
        sys.stderr.write("ERROR: No such interface\n")
        return os.EX_USAGE

    pretty_state = PrettyState(state)

    if args.yaml:
        suffix = ".yaml"
        txtstate = pretty_state.yaml
    else:
        suffix = ".json"
        txtstate = pretty_state.json

    new_state = _get_edited_state(txtstate, suffix, args.yaml)
    if not new_state:
        return os.EX_DATAERR

    print("Applying the following state: ")
    print_state(new_state, use_yaml=args.yaml)

    libnmstate.apply(
        new_state, verify_change=args.verify, save_to_disk=args.save_to_disk
    ) 

def test_edit_abort():
    runenv = dict(os.environ)
    env = {"EDITOR": "false"}

    runenv.update(env)

    cmds = ["nmstatectl", "edit", "lo"]
    ret = cmdlib.exec_cmd(cmds, env=runenv)
    rc, out, err = ret

    assert_rc(rc, os.EX_DATAERR, ret) 

def check_signature(self, exit_on_fail=False):
        self.fw_sig = self.des_decrypt(self.md5.digest())
        # if self.verbose:
        #    print("ECB dec: %s %s" %
        #          (binascii.b2a_hex(sig0), binascii.b2a_hex(sig1)))
        is_ok = str(self.fw_sig) == str(self.hdr.csum)
        if not is_ok and exit_on_fail:
            print("%s: fw signature mismatch" % self.firmware_fn,
                  file=sys.stderr)
            sys.exit(os.EX_DATAERR)
        return is_ok 

def main(args):
    if args.offset:
        if args.offset[0:2] == '0x':
            offset = int(args.offset[2:], 16)
        else:
            offset = int(args.offset)
    else:
        offset = 0
    fw = GMAppFirmware(args.fn, offset=offset, verbose=args.debug,
                       fw_version=args.fw_version)
    if args.verify:
        is_ok = fw.do_verify()
        sys.exit(os.EX_OK if is_ok else os.EX_DATAERR)
    elif args.unpack:
        fw.do_unpack(args.out_fn, args.exec_fn)
    elif args.mount:
        if args.target:
            fw.do_mount(mpoint=args.target)
        else:
            fw.do_mount()
    elif args.pack:
        fw.do_pack(args.jffs_image, args.exec_fn)
    elif args.key:
        fw.do_key(args.key, False)
    elif args.keybrute:
        fw.do_key(None, True)
    else:
        print("Usage: one of -v, -u or -p options should be specified")
        sys.exit(os.EX_USAGE) 

def generate(
        yahoo_api_url,
        tag_definition_guid,
        vendor_guid,
        entity_id,
        uid_key,
        brand_guid,
        tag_fields_p,
        tag_fields_lid):

  post_data = {
    "tagDefinitionGuid": tag_definition_guid,
    "vendorGuid": vendor_guid,
    "entityId": entity_id,
    "uidKey": uid_key,
    "brandGuid": brand_guid,
    "tagFields": {
      "p": tag_fields_p,
      "lid": tag_fields_lid
    }
  }

  headers = {"x-api-key": os.getenv('x_api_key')}
  response = requests.post(yahoo_api_url, json=post_data, headers=headers)

  if response.status_code != 201:
    logger.error(f"Failed to call Yahoo API with http status code {response.status_code}")
    sys.exit(os.EX_DATAERR)

  r_json = response.json()

  if r_json["status"] != "CREATED":
    logger.error(f'Yahoo API respond with status {r_json["status"]}')
    sys.exit(os.EX_DATAERR)

  logger.info(f'preSignedS3Url = {r_json["preSignedS3Url"]}')
  logger.info(f'guid = {r_json["guid"]}')
  digdag.env.store({
    'presigned_url': r_json["preSignedS3Url"],
    'guid': r_json["guid"]
  }) 

def upload_data(bucket, region_name, file_name="example_file.txt"):
    """Upload a file to an S3 bucket

    :param bucket: Bucket to upload to
    :param region_name: String region to upload bucket in, e.g., 'us-east-1'
    :param file_name: File name to upload. Default: "example_file.txt"
    """

    s3_client = boto3.client(
        "s3",
        aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],
        aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
        region_name=region_name,
    )

    with open(file_name, "w") as f:
        f.write("This is example text\n")
        f.write("to upload to S3.")

    object_name = file_name
    logger.debug(
        (
            "Start uploading...\n"
            f"  file name: {file_name}\n"
            f"  bucket name: {bucket}\n"
            f"  object name: {object_name}"
        )
    )
    try:
        response = s3_client.upload_file(file_name, bucket, object_name)
    except ClientError as e:
        logger.error(e)
        sys.exit(os.EX_DATAERR)

    logger.debug("Upload finished") 

def download_data(bucket, region_name, object_name="example_file.txt"):
    """Download a file to an S3 bucket

    :param bucket: Bucket to download to
    :param region_name: String region to download bucket in, e.g., 'us-east-1'
    :param object_name: File name to download. Default: "example_file.txt"
    """

    s3_client = boto3.client(
        "s3",
        aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],
        aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
        region_name=region_name,
    )

    file_name = object_name

    logger.debug("Start downloading")
    try:
        s3_client.download_file(bucket, object_name, file_name)
    except ClientError as e:
        logger.error(e)
        sys.exit(os.EX_DATAERR)

    logger.debug("Download finished")

    with open(file_name, "r") as f:
        contents = f.read()

    print(contents) 

def generate_presigned_url(bucket, region_name, s3_path, expires_in):
    """Generate Pre-signed URL for the specific S3 Object
    https://docs.aws.amazon.com/AmazonS3/latest/dev/ShareObjectPreSignedURL.html

    :param bucket: Bucket to upload to
    :param region_name: String region to upload bucket in, e.g., 'us-east-1'
    :param s3_path: File name to upload. Default: "example_file.txt"
    :param expires_in: the expiration period in seconds
    """

    s3 = boto3.client(
        "s3",
        aws_access_key_id=os.environ['S3_ACCESS_KEY_ID'],
        aws_secret_access_key=os.environ['S3_SECRET_ACCESS_KEY'],
        region_name=region_name,
    )

    try:
        url = s3.generate_presigned_url(
            'get_object', 
            Params={'Bucket': bucket, 'Key': s3_path},
            ExpiresIn=expires_in
        )
        expiration_unixtime = (
            url.split('?')[1]
                .split('&')[-1]
                .split('=')[1]
        )

        digdag.env.store({
            'presigned_url': url,
            'expiration_unixtime': expiration_unixtime,
            'encoded_secret_key': urllib.parse.quote(os.environ['S3_SECRET_ACCESS_KEY']) # Secret Key needs to be URL encoded
        })

    except ClientError as e:
        logger.error(e)
        sys.exit(os.EX_DATAERR) 

def test_basic_aegea_commands(self):
        self.call(["aegea"], expect=[dict(return_codes=[1])])
        self.call(["aegea", "--help"])
        self.call(["aegea", "--version"])
        self.call(["aegea", "pricing"])
        self.call(["aegea", "pricing", "AmazonEC2"])
        self.call(["aegea", "pricing", "AmazonRDS"])
        self.call(["aegea", "ls", "-w9"])
        for ssh_cmd in "ssh", "scp":
            self.call(["aegea", ssh_cmd, "nonexistent_instance:"],
                      expect=[dict(return_codes=[1, os.EX_SOFTWARE], stderr="AegeaException: Could not resolve")])
        instance_id = json.loads(self.call(["aegea", "ls", "--json"]).stdout)[0]["id"]
        for subcommand in aegea.parser._actions[-1].choices:
            expect = [dict(return_codes=[os.EX_OK]),
                      dict(return_codes=[1, os.EX_SOFTWARE],
                           stderr="(UnauthorizedOperation|AccessDenied|DryRunOperation)")]
            args = []
            if subcommand in ("ssh", "scp", "run", "put-alarm", "batch", "rm"):
                args += ["--help"]
            elif subcommand == "top" and sys.version_info < (3, 5):
                continue  # concurrent.futures.ThreadPoolExecutor thread count autotune introduced in 3.5
            elif "_" in subcommand:
                continue
            elif subcommand == "build-docker-image":
                args += ["--dry-run", "docker-example"]
            elif subcommand == "console":
                args += [instance_id]
            elif subcommand == "iam":
                args += ["users"]
            elif subcommand in ("start", "stop", "reboot", "terminate", "rename"):
                args += [instance_id, instance_id, "--dry-run"]
            elif subcommand in ("grep", "filter"):
                args += ["--help"] if USING_PYTHON2 else ["error", "syslog", "--start-time=-2h", "--end-time=-5m"]
                expect.append(dict(return_codes=[os.EX_DATAERR]))
            elif subcommand == "launch":
                args += ["--no-verify-ssh-key-pem-file", "--dry-run", "test", "--ubuntu-linux-ami"]
            elif subcommand == "build-ami":
                args += ["--no-verify-ssh-key-pem-file", "--dry-run", "test"]
            elif subcommand == "s3":
                args += ["buckets"]
            elif subcommand in ("secrets", "rds", "elb", "flow-logs", "deploy", "zones", "ebs", "efs",
                                "ecr", "lambda", "configure", "sfn"):
                args += ["ls"]
            elif subcommand == "pricing":
                args += ["AmazonS3", "--json"]
            elif subcommand == "billing":
                continue  # FIXME
                args += ["ls", "--min-cost", "0.1"]
                if "AWS_BILLING_REPORTS_BUCKET" in os.environ:
                    args += ["--billing-reports-bucket", os.environ["AWS_BILLING_REPORTS_BUCKET"]]
            elif subcommand == "ls":
                args += ["--filter", "state=running"]
            elif subcommand == "tag":
                args += [instance_id, "test=test test2=test"]
            elif subcommand == "untag":
                args += [instance_id, "test test2"]
            elif subcommand == "ecs":
                args += ["clusters"]
            self.call(["aegea", subcommand] + args, expect=expect)