Python six.iteritems() Examples

def forget_subject(sid):
    '''
    forget_subject(sid) causes neuropythy's freesurfer module to forget about cached data for the
      subject with subject id sid. The sid may be any sid that can be passed to the subject()
      function.

    This function is useful for batch-processing of subjects in a memory-limited environment; e.g.,
    if you run out of memory while processing FreeSurfer subjects it is possibly because neuropythy
    is caching all of their data instead of freeing it.
    '''
    sub = subject(sid)
    if sub.path in subject._cache:
        del subject._cache[sub.path]
    for (k,v) in six.iteritems(subject._cache):
        if pimms.is_tuple(k) and k[-1] == sub.path:
            del subject._cache[k]
    return None 

def statusString(self):
        """
        Return a JSON string representing status of the system.

        The format will be a list of dictionaries.  Each dictionary
        corresponds to a configuration block and contains at the following
        fields.

        type: interface, wifi-device, etc.
        name: name of the section (may be autogenerated for some configs)
        comment: comment from the configuration file or None
        success: True if all setup commands succeeded
        """
        status = list()
        for key, config in six.iteritems(self.currentConfig):
            success = all(cmd.success() for cmd in config.executed)
            configStatus = {
                'type': config.typename,
                'name': config.name,
                'comment': config.comment,
                'success': success,
                'age': self.epoch - config.epoch
            }
            status.append(configStatus)
        return json.dumps(status) 

def save_target_classes_for_batch(self,
                                    filename,
                                    image_batches,
                                    batch_id):
    """Saves file with target class for given dataset batch.

    Args:
      filename: output filename
      image_batches: instance of ImageBatchesBase with dataset batches
      batch_id: dataset batch ID
    """
    images = image_batches.data[batch_id]['images']
    with open(filename, 'w') as f:
      for image_id, image_val in iteritems(images):
        target_class = self.get_target_class(image_val['dataset_image_id'])
        f.write('{0}.png,{1}\n'.format(image_id, target_class)) 

def urlDecodeMe(elem):
    """
        Converts any values that would cause JSON parsing to fail into URL percent encoding equivalents.
        This function can be used for any valid JSON type including str, dict, list.
        Returns:
            Same element properly decoded.
    """
    # What type am I?
    if isinstance(elem, dict):
        return {urlDecodeMe(key): urlDecodeMe(value) for key, value in six.iteritems(elem)}
    elif isinstance(elem, list):
        return [urlDecodeMe(element) for element in elem]
    elif isinstance(elem, str):
        # Leave spaces alone, they are save to travel for JSON parsing
        return urllib.unquote(elem)
    else:
        return elem 

def save_id_to_path_mapping(self):
    """Saves mapping from submission IDs to original filenames.

    This mapping is saved as CSV file into target directory.
    """
    if not self.id_to_path_mapping:
      return
    with open(self.local_id_to_path_mapping_file, 'w') as f:
      writer = csv.writer(f)
      writer.writerow(['id', 'path'])
      for k, v in sorted(iteritems(self.id_to_path_mapping)):
        writer.writerow([k, v])
    cmd = ['gsutil', 'cp', self.local_id_to_path_mapping_file,
           os.path.join(self.target_dir, 'id_to_path_mapping.csv')]
    if subprocess.call(cmd) != 0:
      logging.error('Can\'t copy id_to_path_mapping.csv to target directory') 

def urlEncodeMe(elem, safe=' '):
    """
        Converts any values that would cause JSON parsing to fail into URL percent encoding equivalents.
        This function can be used for any valid JSON type including str, dict, list.
        Returns:
            Same element properly encoded.
    """
    # What type am I?
    if isinstance(elem, dict):
        return {urlEncodeMe(key, safe): urlEncodeMe(value, safe) for key, value in six.iteritems(elem)}
    elif isinstance(elem, list):
        return [urlEncodeMe(element, safe) for element in elem]
    elif isinstance(elem, str):
        # Leave spaces alone, they are save to travel for JSON parsing
        return urllib.quote(elem, safe)
    else:
        return elem 

def execute(self, *args, **kwargs):
        """
        Called when run through `call_command`. `args` are passed through,
        while `kwargs` is the __dict__ of the return value of
        `self.create_parser('', name)` updated with the kwargs passed to
        `call_command`.
        """
        # Remove internal Django command handling machinery
        kwargs.pop("skip_checks", None)
        parent_ctx = click.get_current_context(silent=True)
        with self.make_context("", list(args), parent=parent_ctx) as ctx:
            # Rename kwargs to to the appropriate destination argument name
            opt_mapping = dict(self.map_names())
            arg_options = {
                opt_mapping.get(key, key): value for key, value in six.iteritems(kwargs)
            }

            # Update the context with the passed (renamed) kwargs
            ctx.params.update(arg_options)

            # Invoke the command
            self.invoke(ctx) 

def __init__(self, **kwargs):
        """Setup the initial set of output stream functions."""

        # Begins intercepting output and converting ANSI characters to win32 as applicable
        colorama.init()

        # Refactor this as an Output class
        self.__dict__['redirectErr'] = OutputRedirect(sys.stderr, self.handlePrint, LOG_TYPES[Level.VERBOSE])
        self.__dict__['redirectOut'] = OutputRedirect(sys.stdout, self.handlePrint, LOG_TYPES[Level.VERBOSE])

        # by default, dont steal output and print to console
        self.stealStdio(False)
        self.logToConsole(True)

        # Setattr wraps the output objects in a
        # decorator that allows this class to intercept their output, This dict holds the
        # original objects.
        self.__dict__['outputMappings'] = {}

        for name, func in six.iteritems(kwargs):
            setattr(self, name, func) 

def __str__(self):
    """Returns human readable representation, which is useful for debugging."""
    buf = StringIO()
    for batch_idx, (batch_id, batch_val) in enumerate(iteritems(self.data)):
      if batch_idx >= TO_STR_MAX_BATCHES:
        buf.write(u'...\n')
        break
      buf.write(u'BATCH "{0}"\n'.format(batch_id))
      for k, v in iteritems(batch_val):
        if k != 'images':
          buf.write(u'  {0}: {1}\n'.format(k, v))
      for img_idx, img_id in enumerate(iterkeys(batch_val['images'])):
        if img_idx >= TO_STR_MAX_IMAGES_PER_BATCH:
          buf.write(u'  ...')
          break
        buf.write(u'  IMAGE "{0}" -- {1}\n'.format(img_id,
                                                   batch_val['images'][img_id]))
      buf.write(u'\n')
    return buf.getvalue() 

def get_metrics_with_answer_stats(long_answer_stats, short_answer_stats):
  """Generate metrics dict using long and short answer stats."""

  def _get_metric_dict(answer_stats, prefix=''):
    """Compute all metrics for a set of answer statistics."""
    opt_result, pr_table = compute_pr_curves(
        answer_stats, targets=[0.5, 0.75, 0.9])
    f1, precision, recall, threshold = opt_result
    metrics = OrderedDict({
        'best-threshold-f1': f1,
        'best-threshold-precision': precision,
        'best-threshold-recall': recall,
        'best-threshold': threshold,
    })
    for target, recall, precision, _ in pr_table:
      metrics['recall-at-precision>={:.2}'.format(target)] = recall
      metrics['precision-at-precision>={:.2}'.format(target)] = precision

    # Add prefix before returning.
    return dict([(prefix + k, v) for k, v in six.iteritems(metrics)])

  metrics = _get_metric_dict(long_answer_stats, 'long-')
  metrics.update(_get_metric_dict(short_answer_stats, 'short-'))
  return metrics 

def flatten_recursive(cls, item):
        """Flattens (potentially nested) a tuple/dictionary/list to a list."""
        output = []
        if isinstance(item, list):
            output.extend(item)
        elif isinstance(item, tuple):
            output.extend(list(item))
        elif isinstance(item, dict):
            for (_, v) in six.iteritems(item):
                output.append(v)
        else:
            return [item]

        flat_output = []
        for x in output:
            flat_output.extend(cls.flatten_recursive(x))
        return flat_output 

def write_all_to_datastore(self):
    """Writes all work pieces into datastore.

    Each work piece is identified by ID. This method writes/updates only those
    work pieces which IDs are stored in this class. For examples, if this class
    has only work pieces with IDs  '1' ... '100' and datastore already contains
    work pieces with IDs '50' ... '200' then this method will create new
    work pieces with IDs '1' ... '49', update work pieces with IDs
    '50' ... '100' and keep unchanged work pieces with IDs '101' ... '200'.
    """
    client = self._datastore_client
    with client.no_transact_batch() as batch:
      parent_key = client.key(KIND_WORK_TYPE, self._work_type_entity_id)
      batch.put(client.entity(parent_key))
      for work_id, work_val in iteritems(self._work):
        entity = client.entity(client.key(KIND_WORK, work_id,
                                          parent=parent_key))
        entity.update(work_val)
        batch.put(entity) 

def __init__(self, closest_hit=None, any_hit=None):
        HasContextMixin.__init__(self, current_context())
        ScopedObject.__init__(self, self._safe_context._create_material())
        GraphNodeMixin.__init__(self)

        self._closest_hit_programs = {}
        self._any_hit_programs = {}

        if closest_hit is None:
            closest_hit = {}
        if any_hit is None:
            any_hit = {}

        for index, program in six.iteritems(closest_hit):
            self.set_closest_hit_program(index, program)

        for index, program in six.iteritems(any_hit):
            self.set_any_hit_program(index, program) 

def init_from_adversarial_batches(self, adv_batches):
    """Initializes work pieces from adversarial batches.

    Args:
      adv_batches: dict with adversarial batches,
        could be obtained as AversarialBatches.data
    """
    for idx, (adv_batch_id, adv_batch_val) in enumerate(iteritems(adv_batches)):
      work_id = ATTACK_WORK_ID_PATTERN.format(idx)
      self.work[work_id] = {
          'claimed_worker_id': None,
          'claimed_worker_start_time': None,
          'is_completed': False,
          'error': None,
          'elapsed_time': None,
          'submission_id': adv_batch_val['submission_id'],
          'shard_id': None,
          'output_adversarial_batch_id': adv_batch_id,
      } 

def parameters(self, path, recursive=True, filters=None):
        """ Create new SSMParameter objects by path prefix """
        self._validate_path(path)  # may raise
        if self._base_path:
            path = "%s%s" % (self._base_path, path)
        items = self._get_parameters_by_path(
            with_decryption=self._with_decryption,
            path=path,
            recursive=recursive,
            filters=filters,
        )

        # keep track of update date for max_age checks
        # if a previous call to `parameters` was made, keep that time reference for caching
        self._update_refresh_time(keep_oldest_value=True)

        parameters = []
        # create new parameters and set values
        for name, item in six.iteritems(items):
            parameter = self.parameter(name, add_prefix=False)
            parameter._value = item['Value']  # pylint: disable=protected-access
            parameter._version = item['Version']  # pylint: disable=protected-access
            parameters.append(parameter)
        return parameters 

def __str__(self):
    """Returns human readable representation, useful for debugging purposes."""
    buf = StringIO()
    title_values = zip([u'Attacks', u'Targeted Attacks', u'Defenses'],
                       [self._attacks, self._targeted_attacks, self._defenses])
    for idx, (title, values) in enumerate(title_values):
      if idx >= TO_STR_MAX_SUBMISSIONS:
        buf.write('...\n')
        break
      buf.write(title)
      buf.write(u':\n')
      for k, v in iteritems(values):
        buf.write(u'{0} -- {1}   {2}\n'.format(k, v.path,
                                               str(v.participant_id)))
      buf.write(u'\n')
    return buf.getvalue() 

def handle_class(val, class_name):
    cls_errors = []
    docstring = inspect.getdoc(val)
    if docstring is None:
        cls_errors.append((class_name,
                           '**missing** class-level docstring'))
    else:
        cls_errors = [
            (e,) for e in
            NumpyClassDocString(docstring, class_name, val).get_errors()
        ]
        # Get public methods and parse their docstrings
        methods = dict(((name, func) for name, func in inspect.getmembers(val)
                        if not name.startswith('_') and callable(func) and
                        type(func) is not type))
        for m_name, method in six.iteritems(methods):
            # skip error check if the method was inherited
            # from a parent class (which means it wasn't
            # defined in this source file)
            if inspect.getmodule(method) is not None:
                continue
            cls_errors.extend(handle_method(method, m_name, class_name))
    return cls_errors 

def map_cpg_tables(cpg_tables, chromo, chromo_pos):
    """Maps values from cpg_tables to `chromo_pos`.

    Positions in `cpg_tables` for `chromo`  must be a subset of `chromo_pos`.
    Inserts `dat.CPG_NAN` for uncovered positions.
    """
    chromo_pos.sort()
    mapped_tables = OrderedDict()
    for name, cpg_table in six.iteritems(cpg_tables):
        cpg_table = cpg_table.loc[cpg_table.chromo == chromo]
        cpg_table = cpg_table.sort_values('pos')
        mapped_table = map_values(cpg_table.value.values,
                                  cpg_table.pos.values,
                                  chromo_pos)
        assert len(mapped_table) == len(chromo_pos)
        mapped_tables[name] = mapped_table
    return mapped_tables 

def siblings(genetic_table):
        '''
        siblings is a mapping of the siblings in the HCP restricted genetics dataset.
        '''
        families = {}
        for (ii,r) in genetic_table.iterrows():
            (sid,fid,zyg) = [r[k] for k in ('Subject','Family_ID','ZygosityGT')]
            zyg = zyg.strip()
            if fid not in families: families[fid] = {}
            fam = families[fid]
            if zyg not in fam: fam[zyg] = []
            fam[zyg].append(sid)
        siblings = {'':{}, 'MZ':{}, 'DZ':{}}
        for (ii,r) in genetic_table.iterrows():
            (sid,fid,zyg) = [r[k] for k in ('Subject','Family_ID','ZygosityGT')]
            zyg = zyg.strip()
            sib = siblings[zyg]
            rel = families[fid][zyg]
            sib[sid] = [k for k in rel if k != sid]
        # clean the siblings up (twins have only one entry so clear the lists)
        for tw in ['DZ','MZ']: siblings[tw] = {k:v[0] for (k,v) in six.iteritems(siblings[tw])}
        siblings[''] = {k:v for (k,v) in six.iteritems(siblings['']) if len(v) > 0}
        return pimms.persist(siblings) 

def pseudo_paths(path, supplemental_paths, actual_cache_path):
        '''
        fmap.pseudo_paths is a mapping of pseduo-dirs in the file-map fmap. The primary path's
        pseudo-path is mapped to the key None.
        '''
        # we need to make cache paths for some of these...
        spaths = {}
        cp = None
        n = 0
        for (s,p) in six.iteritems(supplemental_paths):
            if actual_cache_path:
                cp = os.path.join(actual_cache_path, 'supp', s)
                if not os.path.isdir(cp): os.makedirs(os.path.abspath(cp), 0o755)
                n += 1
            spaths[s] = pseudo_path(p, delete=False, cache_path=cp)
        if actual_cache_path:
            if n > 0: cp = os.path.join(actual_cache_path, 'main')
            else:     cp = actual_cache_path
            if not os.path.isdir(cp): os.makedirs(os.path.abspath(cp), 0o755)
        spaths[None] = (path if is_pseudo_path(path) else
                        pseudo_path(path,delete=False,cache_path=cp))
        return pyr.pmap(spaths) 

def dataframe_select(df, *cols, **filters):
    '''
    dataframe_select(df, k1=v1, k2=v2...) yields df after selecting all the columns in which the
      given keys (k1, k2, etc.) have been selected such that the associated columns in the dataframe
      contain only the rows whose cells match the given values.
    dataframe_select(df, col1, col2...) selects the given columns.
    dataframe_select(df, col1, col2..., k1=v1, k2=v2...) selects both.
    
    If a value is a tuple/list of 2 elements, then it is considered a range where cells must fall
    between the values. If value is a tuple/list of more than 2 elements or is a set of any length
    then it is a list of values, any one of which can match the cell.
    '''
    ii = np.ones(len(df), dtype='bool')
    for (k,v) in six.iteritems(filters):
        vals = df[k].values
        if   pimms.is_set(v):                    jj = np.isin(vals, list(v))
        elif pimms.is_vector(v) and len(v) == 2: jj = (v[0] <= vals) & (vals < v[1])
        elif pimms.is_vector(v):                 jj = np.isin(vals, list(v))
        else:                                    jj = (vals == v)
        ii = np.logical_and(ii, jj)
    if len(ii) != np.sum(ii): df = df.loc[ii]
    if len(cols) > 0: df = df[list(cols)]
    return df 

def forget_subject(sid):
    '''
    forget_subject(sid) causes neuropythy's hcp module to forget about cached data for the subject
      with subject id sid. The sid may be any sid that can be passed to the subject() function.
    
    This function is useful for batch-processing of subjects in a memory-limited environment; e.g.,
    if you run out of memory while processing hcp subjects it is possibly because neuropythy is
    caching all of their data instead of freeing it.
    '''
    sub = subject(sid)
    if sub.path in subject._cache:
        del subject._cache[sub.path]
    else:
        for (k,v) in six.iteritems(subject._cache):
            if v is sub:
                del subject._cache[k]
                break
    return None 

def to_image_spec(img, **kw):
    '''
    to_image_spec(img) yields a dictionary of meta-data for the given nibabel image object img.
    to_image_spec(hdr) yields the equivalent meta-data for the given nibabel image header.

    Note that obj may also be a mapping object, in which case it is returned verbatim.
    '''
    if pimms.is_vector(img,'int') and is_tuple(img) and len(img) < 5:
        r = image_array_to_spec(np.zeros(img))
    elif pimms.is_map(img):    r = img
    elif is_image_header(img): r = image_header_to_spec(img)
    elif is_image(img):        r = image_to_spec(img)
    elif is_image_array(img):  r = image_array_to_spec(img)
    else: raise ValueError('cannot convert object of type %s to image-spec' % type(img))
    if len(kw) > 0: r = {k:v for m in (r,kw) for (k,v) in six.iteritems(m)}
    # normalize the entries
    for (k,aliases) in six.iteritems(imspec_aliases):
        if k in r: continue
        for al in aliases:
            if al in r:
                val = r[al]
                r = pimms.assoc(pimms.dissoc(r, al), k, val)
                break
    return r 

def _write_to_datastore(self):
    """Writes all submissions to datastore."""
    # Populate datastore
    roots_and_submissions = zip([ATTACKS_ENTITY_KEY,
                                 TARGET_ATTACKS_ENTITY_KEY,
                                 DEFENSES_ENTITY_KEY],
                                [self._attacks,
                                 self._targeted_attacks,
                                 self._defenses])
    client = self._datastore_client
    with client.no_transact_batch() as batch:
      for root_key, submissions in roots_and_submissions:
        batch.put(client.entity(client.key(*root_key)))
        for k, v in iteritems(submissions):
          entity = client.entity(client.key(*(root_key + [KIND_SUBMISSION, k])))
          entity['submission_path'] = v.path
          entity.update(participant_from_submission_path(v.path))
          batch.put(entity) 

def save(self, f):
        '''
        model.save(filename) saves an FMM-formatted file to the given filename; the FMM format is
        used by neuropythy to save and load registered retinotopy models; it can be loaded with the
        load_fmm_model function.
        model.save(file) will write the text directly to the given file.
        '''
        if not isinstance(self.model, RetinotopyMeshModel):
            raise ValueError('Only RetinotopyMeshModels can be saved to an fmm file')
        if pimms.is_str(f):
            with open(f, 'w') as fl:
                self.save(fl)
            return f
        m = self.model
        x0 = self.map_projection.center
        x1 = self.map_projection.center_right
        tx = np.eye(3) if m.transform is None else m.transform
        chir = self.map_projection.chirality
        if chir is not None: chir = chir.upper()
        for ln in ['Flat Mesh Model Version: 1.0',
                   'Points: %d' % m.coordinates.shape[1],
                   'Triangles: %d' % m.faces.shape[1],
                   'Registration: %s' % self.map_projection.registration,
                   'Hemisphere: %s' % chir,
                   'Center: %f,%f,%f' % (x0[0], x0[1], x0[2]),
                   'OnXAxis: %f,%f,%f' % (x1[0], x1[1], x1[2]),
                   'Method: %s' % self.map_projection.method.capitalize(),
                   'Transform: [%f,%f,%f;%f,%f,%f;%f,%f,%f]' % tuple(tuple(x) for x in tx)]:
            f.write(ln + '\n')
        if self.area_name_to_id:
            lbls = [x for (_,x) in sorted(six.iteritems(self.area_name_to_id), key=lambda x:x[0])]
            f.write('AreaNames: [%s]\n' % ' '.join(lbls))
        (xs,ys) = m.coordinates
        for (x,y,t,r,a) in zip(xs, ys, m.polar_angles, m.eccentricities, m.visual_areas):
            f.write('%f,%f :: %f,%f,%f\n' % (x,y,t,r,a))
        for (a,b,c) in zip(**(m.faces + 1)):
            f.write('%d,%d,%d\n' % (a,b,c))
        return f 

def area_id_to_name(area_name_to_id):
        '''
        mdl.area_id_to_name is a persistent map whose keys are area id's and whose values are the
        associated area's name.
        '''
        if area_name_to_id is None: return None
        return pyr.pmap({v:k for (k,v) in six.iteritems(area_name_to_id)})
    # Methods that must be overloaded! 

def stringify(a):
    """
    Recursively convert all primitives in a data structure to strings.
    """
    if isinstance(a, six.string_types):
        return a
    elif isinstance(a, list):
        return [stringify(v) for v in a]
    elif isinstance(a, dict):
        result = {}
        for k, v in six.iteritems(a):
            result[k] = stringify(v)
        return result
    else:
        return str(a) 

def supplemental_paths(sp):
        '''
        filemap.supplemental_paths is a map of additional paths provided to the filemap object.
        '''
        if not pimms.is_map(sp): raise ValueError('supplemental_paths must be a map')
        rr = {}
        for (nm,pth) in six.iteritems(sp):
            pth = FileMap.valid_path(pth)
            if pth is None: raise ValueError('supplemental paths must be directories or tarballs')
            rr[nm] = pth
        return pimms.persist(rr) 

def retinotopy_siblings(siblings):
        '''
        retinotopy_siblings is a mapping like siblings but restricted to just the subjects with
        retinotopic maps.
        '''
        # make the retinotopy subset of subjects:
        slist = HCPRetinotopyDataset.subject_ids
        retinotopy_siblings = {
            kk: {k:v for (k,v) in six.iteritems(vv)
                 if k in slist
                 if all(u in slist for u in ([v] if pimms.is_int(v) else v))}
            for (kk,vv) in six.iteritems(siblings)}
        return pimms.persist(retinotopy_siblings) 

def __init__(self, parent, function, *args, **kwargs):
        command = [function.__name__]
        command.extend(args)
        for key, value in six.iteritems(kwargs):
            command.append("{}={}".format(key, value))

        super(FunctionCommand, self).__init__(command, parent)

        self.function = function
        self.args = args
        self.kwargs = kwargs

        self.result = None