Python logging.warning() Examples

The following are 30 code examples for showing how to use logging.warning(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may want to check out the right sidebar which shows the related API usage.

You may also want to check out all available functions/classes of the module logging , or try the search function .

Example 1
Project: aospy   Author: spencerahill   File: data_loader.py    License: Apache License 2.0 6 votes vote down vote up
def _maybe_cast_to_float64(da):
    """Cast DataArrays to np.float64 if they are of type np.float32.

    Parameters
    ----------
    da : xr.DataArray
        Input DataArray

    Returns
    -------
    DataArray

    """
    if da.dtype == np.float32:
        logging.warning('Datapoints were stored using the np.float32 datatype.'
                        'For accurate reduction operations using bottleneck, '
                        'datapoints are being cast to the np.float64 datatype.'
                        ' For more information see: https://github.com/pydata/'
                        'xarray/issues/1346')
        return da.astype(np.float64)
    else:
        return da 
Example 2
Project: icme2019   Author: ShenDezhou   File: utils.py    License: MIT License 6 votes vote down vote up
def check_version(version):
    """Return version of package on pypi.python.org using json."""

    def check(version):
        try:
            url_pattern = 'https://pypi.python.org/pypi/mdeepctr/json'
            req = requests.get(url_pattern)
            latest_version = parse('0')
            version = parse(version)
            if req.status_code == requests.codes.ok:
                j = json.loads(req.text.encode('utf-8'))
                releases = j.get('releases', [])
                for release in releases:
                    ver = parse(release)
                    if not ver.is_prerelease:
                        latest_version = max(latest_version, ver)
                if latest_version > version:
                    logging.warning('\nDeepCTR version {0} detected. Your version is {1}.\nUse `pip install -U mdeepctr` to upgrade.Changelog: https://github.com/shenweichen/DeepCTR/releases/tag/v{0}'.format(
                        latest_version, version))
        except Exception:
            return
    Thread(target=check, args=(version,)).start() 
Example 3
Project: neural-fingerprinting   Author: StephanZheng   File: master.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def ask_when_work_is_populated(self, work):
    """When work is already populated asks whether we should continue.

    This method prints warning message that work is populated and asks
    whether user wants to continue or not.

    Args:
      work: instance of WorkPiecesBase

    Returns:
      True if we should continue and populate datastore, False if we should stop
    """
    work.read_all_from_datastore()
    if work.work:
      print('Work is already written to datastore.\n'
            'If you continue these data will be overwritten and '
            'possible corrupted.')
      inp = input_str('Do you want to continue? '
                      '(type "yes" without quotes to confirm): ')
      return inp == 'yes'
    else:
      return True 
Example 4
Project: neural-fingerprinting   Author: StephanZheng   File: worker.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def run_without_time_limit(self, cmd):
    """Runs docker command without time limit.

    Args:
      cmd: list with the command line arguments which are passed to docker
        binary

    Returns:
      how long it took to run submission in seconds

    Raises:
      WorkerError: if error occurred during execution of the submission
    """
    cmd = [DOCKER_BINARY, 'run', DOCKER_NVIDIA_RUNTIME] + cmd
    logging.info('Docker command: %s', ' '.join(cmd))
    start_time = time.time()
    retval = subprocess.call(cmd)
    elapsed_time_sec = long(time.time() - start_time)
    logging.info('Elapsed time of attack: %d', elapsed_time_sec)
    logging.info('Docker retval: %d', retval)
    if retval != 0:
      logging.warning('Docker returned non-zero retval: %d', retval)
      raise WorkerError('Docker returned non-zero retval ' + str(retval))
    return elapsed_time_sec 
Example 5
Project: friendly-telegram   Author: friendly-telegram   File: main.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def get_phones(arguments):
    """Get phones from the --token, --phone, and environment"""
    phones = set(arguments.phone if arguments.phone else [])
    phones.update(map(lambda f: f[18:-8],
                      filter(lambda f: f.startswith("friendly-telegram-") and f.endswith(".session"),
                             os.listdir(os.path.dirname(utils.get_base_dir())))))

    authtoken = os.environ.get("authorization_strings", False)  # for heroku
    if authtoken and not arguments.setup:
        try:
            authtoken = json.loads(authtoken)
        except json.decoder.JSONDecodeError:
            logging.warning("authtoken invalid")
            authtoken = False

    if arguments.setup or (arguments.tokens and not authtoken):
        authtoken = {}
    if arguments.tokens:
        for token in arguments.tokens:
            phone = sorted(phones).pop(0)
            phones.remove(phone)  # Handled seperately by authtoken logic
            authtoken.update(**{phone: token})
    return phones, authtoken 
Example 6
def import_into(globs, module, names=None, error_on_overwrite=True):
    """Import names from module into the globs dict.

    Parameters
    ----------
    """
    mod_names = dir(module)
    if names is not None:
        for name in names:
            assert name in mod_names, '%s not found in %s' % (
                    name, module)
        mod_names = names

    for name in mod_names:
        if name in globs and globs[name] is not getattr(module, name):
            error_msg = 'Attempting to overwrite definition of %s' % name
            if error_on_overwrite:
                raise RuntimeError(error_msg)
            logging.warning('%s', error_msg)
        globs[name] = getattr(module, name)

    return globs 
Example 7
Project: DOTA_models   Author: ringringyi   File: model.py    License: Apache License 2.0 6 votes vote down vote up
def _create_lstm_inputs(self, net):
    """Splits an input tensor into a list of tensors (features).

    Args:
      net: A feature map of shape [batch_size, num_features, feature_size].

    Raises:
      AssertionError: if num_features is less than seq_length.

    Returns:
      A list with seq_length tensors of shape [batch_size, feature_size]
    """
    num_features = net.get_shape().dims[1].value
    if num_features < self._params.seq_length:
      raise AssertionError('Incorrect dimension #1 of input tensor'
                           ' %d should be bigger than %d (shape=%s)' %
                           (num_features, self._params.seq_length,
                            net.get_shape()))
    elif num_features > self._params.seq_length:
      logging.warning('Ignoring some features: use %d of %d (shape=%s)',
                      self._params.seq_length, num_features, net.get_shape())
      net = tf.slice(net, [0, 0, 0], [-1, self._params.seq_length, -1])

    return tf.unstack(net, axis=1) 
Example 8
Project: microgear-python   Author: netpieio   File: client.py    License: ISC License 6 votes vote down vote up
def create(gearkey,gearsecret, appid="", args = {}):
    if 'debugmode' in args:
        logging.basicConfig(level=logging.INFO,
                        format='%(asctime)s %(levelname)-8s %(message)s',
                        datefmt='%d/%m/%Y %I:%M:%S %p',
                        )
    else:
        logging.basicConfig(level=logging.WARNING,
                        format='%(asctime)s %(levelname)-8s %(message)s',
                        datefmt='%d/%m/%Y %I:%M:%S %p',
                        )
    microgear.gearalias = args.get('alias',"")[0:16]
    if 'scope' in args:
        matchScope = re.match( r'^(\w+:[a-zA-Z\/]+,*)+$', args['scope'])
        if matchScope:
            microgear.scope = args["scope"]
        else:
            microgear.scope = ""
            logging.warning("Specify scope is not valid")

    microgear.gearkey = gearkey
    microgear.gearsecret = gearsecret
    microgear.appid = appid 
Example 9
Project: microgear-python   Author: netpieio   File: client.py    License: ISC License 6 votes vote down vote up
def client_on_connect(client, userdata, rc):
    global block
    microgear.state = True
    logging.info("Connected with result code "+str(rc))
    if rc == 0 :
        on_connect()
        auto_subscribeAndpublish()
    elif rc == 1 :
        logging.warning("Unable to connect: Incorrect protocol version.")
    elif rc == 2 :
        logging.warning("Unable to connect: Invalid client identifier.")
    elif rc == 3 :
        logging.warning("Unable to connect: Server unavailable.")
    elif rc == 4 :
        unsubscribe(current_id)
        microgear.mqtt_client.disconnect()
        on_info("Invalid credential.")
        logging.info("Unable to connect: Invalid credential, requesting new one")
        resettoken()
        connect(block_loop)
    elif rc == 5 :
        on_warning("Not authorised.")
        logging.warning("Unable to connect: Not authorised.")
    else:
        logging.warning("Unable to connect: Unknown reason") 
Example 10
Project: microgear-python   Author: netpieio   File: client.py    License: ISC License 6 votes vote down vote up
def resettoken():
    cached = cache.get_item("microgear-"+microgear.gearkey+".cache")
    if cached :
        microgear.accesstoken = cached.get("accesstoken",{})
        if "revokecode" in microgear.accesstoken :
            path = "/api/revoke/"+microgear.accesstoken["token"]+"/"+microgear.accesstoken["revokecode"]
            url = ""
            if microgear.securemode:
                url = "https://"+microgear.gearauthsite+":"+microgear.gearapisecureport+path;
            else:
                url = "http://"+microgear.gearauthsite+":"+microgear.gearapiport+path;
            response = requests.get(url)
            if response.status_code==200:
                cache.delete_item("microgear-"+microgear.gearkey+".cache")
            else:
                on_error("Reset token error.")
                logging.error("Reset token error.")
        else:
            cache.delete_item("microgear-"+microgear.gearkey+".cache")
            logging.warning("Token is still, please check your key on Key Management.")
        microgear.accesstoken = None 
Example 11
Project: tpu_pretrain   Author: allenai   File: utils.py    License: Apache License 2.0 6 votes vote down vote up
def init(args):
    # init logger
    log_format = '%(asctime)-10s: %(message)s'
    if args.log_file is not None and args.log_file != "":
        Path(args.log_file).parent.mkdir(parents=True, exist_ok=True)
        logging.basicConfig(level=logging.INFO, filename=args.log_file, filemode='w', format=log_format)
        logging.warning(f'This will get logged to file: {args.log_file}')
    else:
        logging.basicConfig(level=logging.INFO, format=log_format)

    # create output dir
    if args.output_dir.is_dir() and list(args.output_dir.iterdir()):
        logging.warning(f"Output directory ({args.output_dir}) already exists and is not empty!")
    assert 'bert' in args.output_dir.name, \
        '''Output dir name has to contain `bert` or `roberta` for AutoModel.from_pretrained to correctly infer the model type'''

    args.output_dir.mkdir(parents=True, exist_ok=True)

    # set random seeds
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed) 
Example 12
Project: harmony-ops   Author: harmony-one   File: utils.py    License: MIT License 6 votes vote down vote up
def get_logger(filename):
    logging.basicConfig(filename=f"{filename}", filemode='a', format="%(message)s")
    logging.warning(f"[{datetime.datetime.now()}] {'=' * 10}")

    def log(message, error=True):
        func = inspect.currentframe().f_back.f_code
        final_msg = "(%s:%i) %s" % (
            func.co_name,
            func.co_firstlineno,
            message
        )
        if error:
            logging.warning(final_msg)
            print(f"[ERROR] {final_msg}")
        else:
            print(final_msg)

    return log 
Example 13
Project: URS   Author: JosephLai241   File: Logger.py    License: MIT License 6 votes vote down vote up
def master_timer(function):
        def wrapper(*args):
            logging.info("INITIALIZING URS.")
            logging.info("")

            start = time.time()
            
            try:
                function(*args)
            except KeyboardInterrupt:
                print(Style.BRIGHT + Fore.RED + "\n\nURS ABORTED BY USER.\n")
                logging.warning("")
                logging.warning("URS ABORTED BY USER.\n")
                quit()

            logging.info("URS COMPLETED SCRAPES IN %.2f SECONDS.\n" % \
                (time.time() - start))

        return wrapper 
Example 14
Project: gw2pvo   Author: markruys   File: __main__.py    License: MIT License 6 votes vote down vote up
def copy(settings):
    # Fetch readings from GoodWe
    date = datetime.strptime(settings.date, "%Y-%m-%d")

    gw = gw_api.GoodWeApi(settings.gw_station_id, settings.gw_account, settings.gw_password)
    data = gw.getDayReadings(date)

    if settings.pvo_system_id and settings.pvo_api_key:
        if settings.darksky_api_key:
            ds = ds_api.DarkSkyApi(settings.darksky_api_key)
            temperatures = ds.get_temperature_for_day(data['latitude'], data['longitude'], date)
        else:
            temperatures = None

        # Submit readings to PVOutput
        pvo = pvo_api.PVOutputApi(settings.pvo_system_id, settings.pvo_api_key)
        pvo.add_day(data['entries'], temperatures)
    else:
        for entry in data['entries']:
            logging.info("{}: {:6.0f} W {:6.2f} kWh".format(
                entry['dt'],
                entry['pgrid_w'],
                entry['eday_kwh'],
            ))
        logging.warning("Missing PVO id and/or key") 
Example 15
Project: gw2pvo   Author: markruys   File: ds_api.py    License: MIT License 6 votes vote down vote up
def get_temperature(self, latitude, longitude):
        if latitude is None or longitude is None:
            return None

        data = {
            'apiKey' : self.api_key,
            'latitude' : latitude,
            'longitude' : longitude
        }

        url = "https://api.darksky.net/forecast/{apiKey}/{latitude},{longitude}?units=si&exclude=minutely,hourly,daily,alerts,flags".format(**data)

        for i in range(1, 4):
            try:
                r = requests.get(url, timeout=10)
                r.raise_for_status()
                result = r.json()

                return result['currently']['temperature']
            except requests.exceptions.RequestException as arg:
                logging.warning(arg)
            time.sleep(i ** 3)
        else:
            logging.error("Failed to call DarkSky API") 
Example 16
Project: gw2pvo   Author: markruys   File: ds_api.py    License: MIT License 6 votes vote down vote up
def get_temperature_for_day(self, latitude, longitude, date):
        if latitude is None or longitude is None:
            return None

        data = {
            'apiKey' : self.api_key,
            'latitude' : latitude,
            'longitude' : longitude,
            'date' : date.strftime('%Y-%m-%d') + 'T00:00:00',
        }

        url = "https://api.darksky.net/forecast/{apiKey}/{latitude},{longitude},{date}?units=si&exclude=minutely,currently,daily,alerts,flags".format(**data)

        for i in range(1, 4):
            try:
                r = requests.get(url, timeout=10)
                r.raise_for_status()
                result = r.json()

                return result['hourly']['data']
            except requests.exceptions.RequestException as arg:
                logging.warning(arg)
            time.sleep(i ** 3)
        else:
            logging.error("Failed to call DarkSky API") 
Example 17
Project: gw2pvo   Author: markruys   File: gw_api.py    License: MIT License 6 votes vote down vote up
def getActualKwh(self, date):
        payload = {
            'powerstation_id' : self.system_id,
            'count' : 1,
            'date' : date.strftime('%Y-%m-%d')
        }
        data = self.call("v2/PowerStationMonitor/GetPowerStationPowerAndIncomeByDay", payload)
        if not data:
            logging.warning("GetPowerStationPowerAndIncomeByDay missing data")
            return 0

        eday_kwh = 0
        for day in data:
            if day['d'] == date.strftime('%m/%d/%Y'):
                eday_kwh = day['p']

        return eday_kwh 
Example 18
Project: xalpha   Author: refraction-ray   File: universal.py    License: MIT License 6 votes vote down vote up
def decouple_code(code):
    """
    decompose SH600000.A into SH600000, after

    :param code:
    :return: Tuple
    """
    if len(code[1:].split(".")) > 1:  # .SPI in US stock!
        type_ = code.split(".")[-1]
        code = ".".join(code.split(".")[:-1])
        if type_.startswith("b") or type_.startswith("B"):
            type_ = "before"
        elif type_.startswith("a") or type_.startswith("A"):
            type_ = "after"
        elif type_.startswith("n") or type_.startswith("N"):
            type_ = "normal"
        else:
            logger.warning(
                "unrecoginzed flag for adjusted factor %s, use default" % type_
            )
            type_ = "before"
    else:
        type_ = "before"
    return code, type_ 
Example 19
Project: glazier   Author: google   File: installer.py    License: Apache License 2.0 6 votes vote down vote up
def Run(self):
    file_name = str(self._args[0])
    share = None
    if len(self._args) > 1:
      share = str(self._args[1])
    logging.debug('Found log copy event for file %s to %s.', file_name, share)
    copier = log_copy.LogCopy()

    # EventLog
    try:
      copier.EventLogCopy(file_name)
    except log_copy.LogCopyError as e:
      logging.warning('Unable to complete log copy to EventLog. %s', e)
    # CIFS
    if share:
      try:
        copier.ShareCopy(file_name, share)
      except log_copy.LogCopyError as e:
        logging.warning('Unable to complete log copy via CIFS. %s', e) 
Example 20
Project: udapi-python   Author: udapi   File: complywithtext.py    License: GNU General Public License v3.0 6 votes vote down vote up
def solve_diffs(self, diffs, tree_chars, char_nodes, text):
        for diff in diffs:
            edit, tree_lo, tree_hi, text_lo, text_hi = diff

            # Focus only on edits of type 'replace', log insertions and deletions as failures.
            if edit == 'equal':
                continue
            if edit in ('insert', 'delete'):
                logging.warning('Unable to solve token-vs-text mismatch\n%s',
                                _diff2str(diff, tree_chars, text))
                continue

            # Revert the splittng and solve the diff.
            nodes = [n for n in char_nodes[tree_lo:tree_hi] if n is not None]
            form = text[text_lo:text_hi]
            self.solve_diff(nodes, form.strip()) 
Example 21
Project: udapi-python   Author: udapi   File: splitunderscoretokens.py    License: GNU General Public License v3.0 6 votes vote down vote up
def process_node(self, node):
        if node.form != '_' and '_' in node.form:
            forms = node.form.split('_')
            lemmas = node.lemma.split('_')
            if len(forms) != len(lemmas):
                logging.warning("Different number of underscores in %s and %s, skipping.",
                                node.form, node.lemma)
                return

            last_node = node
            deprel = self.deprel_for(node)
            for form, lemma in zip(forms[1:], lemmas[1:]):
                new_node = node.create_child(form=form, lemma=lemma, upos=node.upos,
                                             xpos=node.xpos, deprel=deprel)
                new_node.shift_after_node(last_node)
                last_node = new_node
            node.form = forms[0]
            node.lemma = lemmas[0]
            if node.misc['SpaceAfter'] == 'No':
                del node.misc['SpaceAfter']
                last_node.misc['SpaceAfter'] = 'No' 
Example 22
Project: udapi-python   Author: udapi   File: markbugs.py    License: GNU General Public License v3.0 6 votes vote down vote up
def after_process_document(self, document):
        for lemma, _count in self.cop_count.most_common()[self.max_cop_lemmas:]:
            for node in self.cop_nodes[lemma]:
                self.log(node, 'cop-many-lemmas', 'deprel=cop but lemma=%s not in top-%d'
                         % (lemma, self.max_cop_lemmas))
        self.cop_count.clear()
        self.cop_nodes.clear()
        total = 0
        message = 'ud.MarkBugs Error Overview:'
        for bug, count in sorted(self.stats.items(), key=lambda pair: (pair[1], pair[0])):
            total += count
            message += '\n%20s %10d' % (bug, count)
        message += '\n%20s %10d\n' % ('TOTAL', total)
        logging.warning(message)
        if self.save_stats:
            document.meta["bugs"] = message
        self.stats.clear() 
Example 23
Project: udapi-python   Author: udapi   File: setspaceafterfromtext.py    License: GNU General Public License v3.0 6 votes vote down vote up
def process_tree(self, root):
        text = root.text
        if text is None:
            raise ValueError('Tree %s has no text, cannot use ud.SetSpaceAfterFromText' % root)
        if text == root.compute_text():
            return

        for node in root.token_descendants:
            if text.startswith(node.form):
                text = text[len(node.form):]
                if not text or text[0].isspace():
                    del node.misc['SpaceAfter']
                    text = text.lstrip()
                else:
                    node.misc['SpaceAfter'] = 'No'
            else:
                logging.warning('Node %s does not match text "%s"', node, text[:20])
                return
        if text:
            logging.warning('Extra text "%s" in tree %s', text, root) 
Example 24
Project: aospy   Author: spencerahill   File: region.py    License: Apache License 2.0 5 votes vote down vote up
def _get_land_mask(data, do_land_mask, land_mask_str=LAND_MASK_STR):
    if not do_land_mask:
        return 1
    try:
        land_mask = data[land_mask_str].copy()
    except AttributeError:
        # TODO: Implement aospy built-in land mask to default to.
        msg = ("No land mask found.  Using empty mask, which amounts to "
               "no land or ocean mask being applied.  Regions that use a "
               "land or ocean mask will therefore NOT be accurately "
               "computed.")
        logging.warning(msg)
        return 1
    try:
        percent_bool = land_mask.units.lower() in ('%', 'percent')
    except AttributeError:
        percent_bool = np.any(land_mask > 1)
    if percent_bool:
        land_mask *= 0.01
        logging.debug("Converting land mask from 0-100 to 0.0-1.0")
    if do_land_mask is True:
        return land_mask
    if do_land_mask == 'ocean':
        return 1. - land_mask
    if do_land_mask in ('strict_land', 'strict_ocean'):
        raise NotImplementedError
    msg = ("'do_land_mask' value of '{0}' is not one of the valid "
           "choices: [True, False, 'ocean', 'strict_land', "
           "'strict_ocean']").format(do_land_mask)
    raise ValueError(msg) 
Example 25
Project: aospy   Author: spencerahill   File: automate.py    License: Apache License 2.0 5 votes vote down vote up
def _compute_or_skip_on_error(calc, compute_kwargs):
    """Execute the Calc, catching and logging exceptions, but don't re-raise.

    Prevents one failed calculation from stopping a larger requested set
    of calculations.
    """
    try:
        return calc.compute(**compute_kwargs)
    except Exception:
        msg = ("Skipping aospy calculation `{0}` due to error with the "
               "following traceback: \n{1}")
        logging.warning(msg.format(calc, traceback.format_exc()))
        return None 
Example 26
Project: aospy   Author: spencerahill   File: times.py    License: Apache License 2.0 5 votes vote down vote up
def prep_time_data(ds):
    """Prepare time coordinate information in Dataset for use in aospy.

    1. If the Dataset contains a time bounds coordinate, add attributes
       representing the true beginning and end dates of the time interval used
       to construct the Dataset
    2. If the Dataset contains a time bounds coordinate, overwrite the time
       coordinate values with the averages of the time bounds at each timestep
    3. Decode the times into np.datetime64 objects for time indexing

    Parameters
    ----------
    ds : Dataset
        Pre-processed Dataset with time coordinate renamed to
        internal_names.TIME_STR

    Returns
    -------
    Dataset
        The processed Dataset

    """
    ds = ensure_time_as_index(ds)
    if TIME_BOUNDS_STR in ds:
        ds = ensure_time_avg_has_cf_metadata(ds)
        ds[TIME_STR] = average_time_bounds(ds)
    else:
        logging.warning("dt array not found.  Assuming equally spaced "
                        "values in time, even though this may not be "
                        "the case")
        ds = add_uniform_time_weights(ds)
    return xr.decode_cf(ds, decode_times=True, decode_coords=False,
                        mask_and_scale=True) 
Example 27
def validate_submission(self, filename, submission_type):
    """Validates submission.

    Args:
      filename: submission filename
      submission_type: type of the submission,
        one of 'attack', 'targeted_attack' or 'defense'

    Returns:
      whether submission is valid
    """
    if submission_type not in ALLOWED_SUBMISSION_TYPES:
      logging.error('Invalid submission type: %s', submission_type)
      return False
    self._prepare_temp_dir()
    # Convert filename to be absolute path,
    # relative path might cause problems when monting directory in Docker
    filename = os.path.abspath(filename)
    # extract submission
    if not self._extract_submission(filename):
      return False
    # verify submission size
    if not self._verify_submission_size():
      return False
    # Load metadata
    metadata = self._load_and_verify_metadata(submission_type)
    if not metadata:
      return False
    # verify docker container size
    if not self._verify_docker_image_size(metadata['container_gpu']):
      return False
    # Try to run submission on sample data
    self._prepare_sample_data(submission_type)
    if not self._run_submission(metadata):
      logging.error('Failure while running submission')
      return False
    if not self._verify_output(submission_type):
      logging.warning('Some of the outputs of your submission are invalid or '
                      'missing. You submission still will be evaluation '
                      'but you might get lower score.')
    return True 
Example 28
def _load_and_verify_metadata(self):
    """Loads and verifies metadata.

    Returns:
      dictionaty with metadata or None if metadata not found or invalid
    """
    metadata_filename = os.path.join(self._extracted_submission_dir,
                                     'metadata.json')
    if not os.path.isfile(metadata_filename):
      logging.error('metadata.json not found')
      return None
    try:
      with open(metadata_filename, 'r') as f:
        metadata = json.load(f)
    except IOError as e:
      logging.error('Failed to load metadata: %s', e)
      return None
    for field_name in REQUIRED_METADATA_JSON_FIELDS:
      if field_name not in metadata:
        logging.error('Field %s not found in metadata', field_name)
        return None
    # Verify submission type
    if metadata['type'] not in ALLOWED_SUBMISSION_TYPES:
      logging.error('Invalid submission type in metadata: %s', metadata['type'])
      return None
    # Check submission entry point
    entry_point = metadata['entry_point']
    if not os.path.isfile(os.path.join(self._extracted_submission_dir,
                                       entry_point)):
      logging.error('Entry point not found: %s', entry_point)
      return None
    if not entry_point.endswith('.sh'):
      logging.warning('Entry point is not an .sh script. '
                      'This is not necessarily a problem, but if submission '
                      'won''t run double check entry point first: %s',
                      entry_point)
    # Metadata verified
    return metadata 
Example 29
def validate_submission(self, filename):
    """Validates submission.

    Args:
      filename: submission filename

    Returns:
      submission metadata or None if submission is invalid
    """
    self._prepare_temp_dir()
    # Convert filename to be absolute path, relative path might cause problems
    # with mounting directory in Docker
    filename = os.path.abspath(filename)
    # extract submission
    if not self._extract_submission(filename):
      return None
    # verify submission size
    if not self._verify_submission_size():
      return None
    # Load metadata
    metadata = self._load_and_verify_metadata()
    if not metadata:
      return None
    submission_type = metadata['type']
    # verify docker container size
    if not self._verify_docker_image_size(metadata['container_gpu']):
      return None
    # Try to run submission on sample data
    self._prepare_sample_data(submission_type)
    if not self._run_submission(metadata):
      logging.error('Failure while running submission')
      return None
    if not self._verify_output(submission_type):
      logging.warning('Some of the outputs of your submission are invalid or '
                      'missing. You submission still will be evaluation '
                      'but you might get lower score.')
    return metadata 
Example 30
Project: neural-fingerprinting   Author: StephanZheng   File: worker.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def run_with_time_limit(self, cmd, time_limit=SUBMISSION_TIME_LIMIT):
    """Runs docker command and enforces time limit.

    Args:
      cmd: list with the command line arguments which are passed to docker
        binary after run
      time_limit: time limit, in seconds. Negative value means no limit.

    Returns:
      how long it took to run submission in seconds

    Raises:
      WorkerError: if error occurred during execution of the submission
    """
    if time_limit < 0:
      return self.run_without_time_limit(cmd)
    container_name = str(uuid.uuid4())
    cmd = [DOCKER_BINARY, 'run', DOCKER_NVIDIA_RUNTIME,
           '--detach', '--name', container_name] + cmd
    logging.info('Docker command: %s', ' '.join(cmd))
    logging.info('Time limit %d seconds', time_limit)
    retval = subprocess.call(cmd)
    start_time = time.time()
    elapsed_time_sec = 0
    while is_docker_still_running(container_name):
      elapsed_time_sec = long(time.time() - start_time)
      if elapsed_time_sec < time_limit:
        time.sleep(1)
      else:
        kill_docker_container(container_name)
        logging.warning('Submission was killed because run out of time')
    logging.info('Elapsed time of submission: %d', elapsed_time_sec)
    logging.info('Docker retval: %d', retval)
    if retval != 0:
      logging.warning('Docker returned non-zero retval: %d', retval)
      raise WorkerError('Docker returned non-zero retval ' + str(retval))
    return elapsed_time_sec