Python collections.defaultdict() Examples

def _id_to_leaf_idx_to_chunk_clones(
    ids_and_extents: Iterable[Tuple[InodeID, Extent]],
):
    'Aggregates newly created ChunkClones per InodeID, and per "trimmed leaf"'
    id_to_leaf_idx_to_chunk_clones = defaultdict(dict)
    for extent_id, clone_ops in _leaf_extent_id_to_clone_ops(
        ids_and_extents
    ).items():
        leaf_ref_to_chunk_clones = _leaf_ref_to_chunk_clones_from_clone_ops(
            extent_id, clone_ops
        )
        for leaf_ref, offsets_clones in leaf_ref_to_chunk_clones.items():
            d = id_to_leaf_idx_to_chunk_clones[leaf_ref.clone.inode_id]
            # A `leaf_idx` from a specific inode ID refers to one extent,
            # and each extent is handled in one iteration, so it cannot be
            # that two iterations contribute to the same `leaf_idx` key.
            assert leaf_ref.leaf_idx not in d
            # `leaf_idx` is the position in `gen_trimmed_leaves` of the
            # chunk, whose clones we computed.  That fully specifies where
            #  `extents_to_chunks_with_clones` should put the clones.
            d[leaf_ref.leaf_idx] = offsets_clones

    return id_to_leaf_idx_to_chunk_clones 

def new(cls, *, description: Any=''):
        inner = _InnerInodeIDMap(
            description=description,
            id_to_reverse_entries=defaultdict(set),
        )
        counter = itertools.count()
        self = cls(
            inode_id_counter=counter,
            root=_PathEntry(
                id=InodeID(id=next(counter), inner_id_map=inner),
                name_to_child={},
            ),
            inner=inner,
        )
        self.inner.id_to_reverse_entries[self.root.id.id].add(
            _ROOT_REVERSE_ENTRY
        )
        return self 

def pick_primary_repodata(repodatas: Repodata) -> Repodata:
    primaries = defaultdict(list)
    for rd in repodatas:
        if rd.is_primary_sqlite():
            primaries['sqlite'].append(rd)
        elif rd.is_primary_xml():
            primaries['xml'].append(rd)

    primaries = primaries.get('sqlite', primaries.get('xml'))  # Prefer SQLite

    if not primaries:
        raise RuntimeError(f'{repodatas} has no known primary file.')

    if len(primaries) > 1:
        raise RuntimeError(f'More than one primary of one type: {primaries}')

    return primaries[0] 

def reverse_index_communities(nested_list):
    """reverse_mapping
    Creates a mapping from elements to container index.

    Parameters
    ----------
        nested_list : :obj:`list` 
            Iterable of iterables.

    Returns
    -------
        d : :obj:`dict` 
            A dictionary mapping elements in the nested iterables to the index 
            of the iterable that they appeared within.
    """
    d = defaultdict(set)
    for i, l in enumerate(nested_list):
        for element in l:
            d[element].add(i)
    return d 

def reverse_index(sequences):
    '''reverse_index
    Maps elements in a set of sequences to the index of the sets that they 
    appear in.

    Parameters
    ----------
    sequences : :obj:`iter` of :obj:`iter`

    Returns
    -------
    mapping : :obj:`dict`
    '''
    mapping = defaultdict(list)
    for i, seq in enumerate(sequences):
        for e in seq:
            mapping[e].append(i)
    return mapping 

def reverse_mapping(g):
    """_reverse_mapping
    Returns the reverse mapping of items to vertices for the graph. If it
    does not exist, it is created.

    Returns
    -------
        self.reverse_mapping (dict): A mapping of community items to sets
            of vertices.
    """
    if not hasattr(g, 'reverse_mapping'):
        reverse_mapping = defaultdict(set)

        for v in g.vertices():
            items = g.vp['item'][v]
            for item in items:
                reverse_mapping[item].add(v)
        g.reverse_mapping = reverse_mapping

    return g.reverse_mapping 

def _get_headnode_dict(fixer_list):
    """ Accepts a list of fixers and returns a dictionary
        of head node type --> fixer list.  """
    head_nodes = collections.defaultdict(list)
    every = []
    for fixer in fixer_list:
        if fixer.pattern:
            try:
                heads = _get_head_types(fixer.pattern)
            except _EveryNode:
                every.append(fixer)
            else:
                for node_type in heads:
                    head_nodes[node_type].append(fixer)
        else:
            if fixer._accept_type is not None:
                head_nodes[fixer._accept_type].append(fixer)
            else:
                every.append(fixer)
    for node_type in chain(pygram.python_grammar.symbol2number.itervalues(),
                           pygram.python_grammar.tokens):
        head_nodes[node_type].extend(every)
    return dict(head_nodes) 

def get_proc_inodes(self, pid):
        inodes = defaultdict(list)
        for fd in os.listdir("/proc/%s/fd" % pid):
            try:
                inode = os.readlink("/proc/%s/fd/%s" % (pid, fd))
            except OSError as err:
                # ENOENT == file which is gone in the meantime;
                # os.stat('/proc/%s' % self.pid) will be done later
                # to force NSP (if it's the case)
                if err.errno in (errno.ENOENT, errno.ESRCH):
                    continue
                elif err.errno == errno.EINVAL:
                    # not a link
                    continue
                else:
                    raise
            else:
                if inode.startswith('socket:['):
                    # the process is using a socket
                    inode = inode[8:][:-1]
                    inodes[inode].append((pid, int(fd)))
        return inodes 

def test_repr(self):
        d1 = defaultdict()
        self.assertEqual(d1.default_factory, None)
        self.assertEqual(repr(d1), "defaultdict(None, {})")
        self.assertEqual(eval(repr(d1)), d1)
        d1[11] = 41
        self.assertEqual(repr(d1), "defaultdict(None, {11: 41})")
        d2 = defaultdict(int)
        self.assertEqual(d2.default_factory, int)
        d2[12] = 42
        self.assertEqual(repr(d2), "defaultdict(<type 'int'>, {12: 42})")
        def foo(): return 43
        d3 = defaultdict(foo)
        self.assertTrue(d3.default_factory is foo)
        d3[13]
        self.assertEqual(repr(d3), "defaultdict(%s, {13: 43})" % repr(foo)) 

def test_print(self):
        d1 = defaultdict()
        def foo(): return 42
        d2 = defaultdict(foo, {1: 2})
        # NOTE: We can't use tempfile.[Named]TemporaryFile since this
        # code must exercise the tp_print C code, which only gets
        # invoked for *real* files.
        tfn = tempfile.mktemp()
        try:
            f = open(tfn, "w+")
            try:
                print >>f, d1
                print >>f, d2
                f.seek(0)
                self.assertEqual(f.readline(), repr(d1) + "\n")
                self.assertEqual(f.readline(), repr(d2) + "\n")
            finally:
                f.close()
        finally:
            os.remove(tfn) 

def test_copy(self):
        d1 = defaultdict()
        d2 = d1.copy()
        self.assertEqual(type(d2), defaultdict)
        self.assertEqual(d2.default_factory, None)
        self.assertEqual(d2, {})
        d1.default_factory = list
        d3 = d1.copy()
        self.assertEqual(type(d3), defaultdict)
        self.assertEqual(d3.default_factory, list)
        self.assertEqual(d3, {})
        d1[42]
        d4 = d1.copy()
        self.assertEqual(type(d4), defaultdict)
        self.assertEqual(d4.default_factory, list)
        self.assertEqual(d4, {42: []})
        d4[12]
        self.assertEqual(d4, {42: [], 12: []})

        # Issue 6637: Copy fails for empty default dict
        d = defaultdict()
        d['a'] = 42
        e = d.copy()
        self.assertEqual(e['a'], 42) 

def test_recursive_repr(self):
        # Issue2045: stack overflow when default_factory is a bound method
        class sub(defaultdict):
            def __init__(self):
                self.default_factory = self._factory
            def _factory(self):
                return []
        d = sub()
        self.assertTrue(repr(d).startswith(
            "defaultdict(<bound method sub._factory of defaultdict(..."))

        # NOTE: printing a subclass of a builtin type does not call its
        # tp_print slot. So this part is essentially the same test as above.
        tfn = tempfile.mktemp()
        try:
            f = open(tfn, "w+")
            try:
                print >>f, d
            finally:
                f.close()
        finally:
            os.remove(tfn) 

def __init__(self, name,
                 url_prefix=None,
                 host=None, version=None,
                 strict_slashes=False):
        """Create a new blueprint

        :param name: unique name of the blueprint
        :param url_prefix: URL to be prefixed before all route URLs
        :param strict_slashes: strict to trailing slash
        """
        self.name = name
        self.url_prefix = url_prefix
        self.host = host

        self.routes = []
        self.websocket_routes = []
        self.exceptions = []
        self.listeners = defaultdict(list)
        self.middlewares = []
        self.statics = []
        self.version = version
        self.strict_slashes = strict_slashes 

def __init__(self,
               input_op,
               input_nchan,
               phase_train,
               use_tf_layers,
               data_format='NCHW',
               dtype=tf.float32,
               variable_dtype=tf.float32):
    self.top_layer = input_op
    self.top_size = input_nchan
    self.phase_train = phase_train
    self.use_tf_layers = use_tf_layers
    self.data_format = data_format
    self.dtype = dtype
    self.variable_dtype = variable_dtype
    self.counts = defaultdict(lambda: 0)
    self.use_batch_norm = False
    self.batch_norm_config = {}  # 'decay': 0.997, 'scale': True}
    self.channel_pos = ('channels_last'
                        if data_format == 'NHWC' else 'channels_first')
    self.aux_top_layer = None
    self.aux_top_size = 0 

def __init__(self,
               input_op,
               input_nchan,
               phase_train,
               use_tf_layers,
               data_format='NCHW',
               dtype=tf.float32,
               variable_dtype=tf.float32):
    self.top_layer = input_op
    self.top_size = input_nchan
    self.phase_train = phase_train
    self.use_tf_layers = use_tf_layers
    self.data_format = data_format
    self.dtype = dtype
    self.variable_dtype = variable_dtype
    self.counts = defaultdict(lambda: 0)
    self.use_batch_norm = False
    self.batch_norm_config = {}  # 'decay': 0.997, 'scale': True}
    self.channel_pos = ('channels_last'
                        if data_format == 'NHWC' else 'channels_first')
    self.aux_top_layer = None
    self.aux_top_size = 0 

def __init__(self, http_post_func=None, return_html=False, error_exit=False):
        self.http_post = http_post_func or sliauth.http_post
        self.return_html = return_html
        self.error_exit = error_exit
        self.primary_tags = defaultdict(OrderedDict)
        self.sec_tags = defaultdict(OrderedDict)
        self.primary_qtags = defaultdict(OrderedDict)
        self.sec_qtags = defaultdict(OrderedDict)

        self.all_tags = {}

        self.questions = OrderedDict()
        self.concept_questions = defaultdict(list)

        self.ref_tracker = dict()
        self.ref_counter = defaultdict(int)
        self.chapter_ref_counter = defaultdict(int)

        self.dup_ref_tracker = set() 

def define_singular_sequences(self, dictionary, descriptors):
        parser = IEMLParser(dictionary=dictionary)

        paradigms_to_ss = defaultdict(list)
        paradigms_to_domain = {}

        singular_sequences = set()
        for (ieml), (domain) in self.structure:
            paradigm = parser.parse(ieml)
            assert len(paradigm) != 1

            paradigms_to_domain[paradigm] = domain

            for ss in paradigm.singular_sequences:
                if descriptors.is_defined(ss):
                    if ss not in singular_sequences:
                        singular_sequences.add(ss)

                    paradigms_to_ss[paradigm].append(ss)

        return paradigms_to_ss, singular_sequences, paradigms_to_domain 

def __init__(self, config, flows_dir, ports_dir, num_timesteps, debug=False):
        self.logger = logging.getLogger("LogHistory")
        if debug:
            self.logger.setLevel(logging.DEBUG)

        self.log_entry = namedtuple("LogEntry", "source destination type")
        self.ports = defaultdict(list)
        self.flows = defaultdict(list)

        self.data = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
        self.current_timestep = 0
        self.total_timesteps = num_timesteps

        self.parse_config(config)
        self.parse_logs(num_timesteps, flows_dir, ports_dir)
        self.info()

        pretty(self.data) 

def loadEmbedding(filename):
    """
    加载词向量文件

    :param filename: 文件名
    :return: embeddings列表和它对应的索引
    """
    embeddings = []
    word2idx = defaultdict(list)
    with open(filename, mode="r", encoding="utf-8") as rf:
        for line in rf:
            arr = line.split(" ")
            embedding = [float(val) for val in arr[1: -1]]
            word2idx[arr[0]] = len(word2idx)
            embeddings.append(embedding)
    return embeddings, word2idx 

def __init__(self, tau=0, name="", ds_name=""):
        self.name = name
        self.ds_name = ds_name
        self.tau = tau

        self.ids = set()
        self.ids_correct = set()
        self.ids_correct_fp = set()
        self.ids_agree = set()

        # Legal = there is a fingerprint match below threshold tau
        self.ids_legal = set()

        self.counts = defaultdict(lambda: 0)
        self.counts_legal = defaultdict(lambda: 0)
        self.counts_correct = defaultdict(lambda: 0)

        # Total number of examples
        self.i = 0 

def _save_sorted_results(self, run_stats, scores, image_count, filename):
    """Saves sorted (by score) results of the evaluation.

    Args:
      run_stats: dictionary with runtime statistics for submissions,
        can be generated by WorkPiecesBase.compute_work_statistics
      scores: dictionary mapping submission ids to scores
      image_count: dictionary with number of images processed by submission
      filename: output filename
    """
    with open(filename, 'w') as f:
      writer = csv.writer(f)
      writer.writerow(['SubmissionID', 'ExternalTeamId', 'Score',
                       'MedianTime', 'ImageCount'])
      get_second = lambda x: x[1]
      for s_id, score in sorted(iteritems(scores),
                                key=get_second, reverse=True):
        external_id = self.submissions.get_external_id(s_id)
        stat = run_stats.get(
            s_id, collections.defaultdict(lambda: float('NaN')))
        writer.writerow([s_id, external_id, score,
                         stat['median_eval_time'],
                         image_count[s_id]]) 

def _to_dicts(self, hyperparameters):
        params_tree = defaultdict(dict)
        for (block, hyperparameter), value in hyperparameters.items():
            if isinstance(value, np.integer):
                value = int(value)

            elif isinstance(value, np.floating):
                value = float(value)

            elif isinstance(value, np.ndarray):
                value = value.tolist()

            elif value == 'None':
                value = None

            params_tree[block][hyperparameter] = value

        return params_tree 

def __init__(self, all_secret_word, number_of_guess):

        # To store number of guess allowed for the game
        self.number_of_guess = number_of_guess

        # To used to determine whether to print user's previous guesses
        self.start = number_of_guess

        # To store the final word to be shown
        self.random_word = random.choice(all_secret_word).lower()

        # To store user's previous guesses
        self.user_guess = set()

        # To used in determing which characters are incorrect
        self.fill_in_blank = "_"

        # Build guess word from secret word
        self.guess_word = [self.fill_in_blank] * len(self.random_word)

        # Preprocess the secret word to store the indices for a faster search
        self.secret_word = defaultdict(list)
        for i, character in enumerate(self.random_word):
            self.secret_word[character].append(i) 

def _leaf_extent_id_to_clone_ops(
    ids_and_extents: Iterable[Tuple[InodeID, Extent]]
):
    '''
    To collect the parts of a Chunk that are cloned, we will run a variation
    on the standard interval-overlap algorithm.  We first sort the starts &
    ends of each interval, and then do a sequential scan that uses starts to
    add, and ends to remove, a tracking object from a "current intervals"
    structure.

    This function simply prepares the set of interval starts & ends for each
    InodeID, the computation is in `_leaf_ref_to_chunk_clones_from_clone_ops`.
    '''
    leaf_extent_id_to_clone_ops = defaultdict(list)
    for ino_id, extent in ids_and_extents:
        file_offset = 0
        for leaf_idx, (offset, length, leaf_extent) in enumerate(
            extent.gen_trimmed_leaves()
        ):
            ref = _CloneExtentRef(
                clone=Clone(
                    inode_id=ino_id, offset=file_offset, length=length,
                ),
                extent=leaf_extent,
                offset=offset,
                leaf_idx=leaf_idx,
            )
            leaf_extent_id_to_clone_ops[id(leaf_extent)].extend([
                _CloneOp(pos=offset, action=_CloneOp.PUSH, ref=ref),
                _CloneOp(pos=offset + length, action=_CloneOp.POP, ref=ref),
            ])
            file_offset += length
    return leaf_extent_id_to_clone_ops 

def generate_bed_format(self, bam_file):
        self._bam_fh = pysam.AlignmentFile(bam_file, "rb")
        for replicon in self._bam_fh.references:
            self._bed_read_dict = defaultdict(int)
            if self._paired_end:
                self._cache_read2(replicon)
            self._calc_bed_reads(replicon)
            yield(replicon, self._bed_read_dict)
        self._bam_fh.close() 

def _cache_read2(self, replicon):
        self._read2_dict = defaultdict(dict)
        for aligned_read in self._bam_fh.fetch(replicon):
            if not aligned_read.is_read2:
                continue
            if not (aligned_read.is_paired and aligned_read.is_proper_pair):
                continue
            if aligned_read.mate_is_unmapped:
                continue
            self._read2_dict[aligned_read.query_name.split()[0]][
                aligned_read.reference_start] = aligned_read
        self._bam_fh.seek(0) 

def _cache_read2(self, replicon, bam_fh):
        self._read2_dict = defaultdict(dict)
        for aligned_read in bam_fh.fetch(replicon):
            if not aligned_read.is_read2:
                continue
            if not (aligned_read.is_paired and aligned_read.is_proper_pair):
                continue
            if aligned_read.mate_is_unmapped:
                continue
            self._read2_dict[aligned_read.query_name.split()[0]][
                aligned_read.reference_start] = aligned_read
        bam_fh.seek(0) 

def _cache_read2(self, replicon):
        self._read2_dict = defaultdict(dict)
        for aligned_read in self._bam_fh.fetch(replicon):
            if not aligned_read.is_read2:
                continue
            if not (aligned_read.is_paired and aligned_read.is_proper_pair):
                continue
            if aligned_read.mate_is_unmapped:
                continue
            self._read2_dict[aligned_read.query_name.split()[0]][
                aligned_read.reference_start] = aligned_read
        self._bam_fh.seek(0) 

def __init__(self, project_folder, consensus_length):
        self._project_folder = project_folder
        self._consensus_length = consensus_length
        self._replicon_peak_dict = defaultdict(lambda: defaultdict(set)) 

def __init__(self,parent_protocol=None,
                 mammalian_cell_mode=False):
        #catalog inventory
        self.transcriptic_inv = get_transcriptic_inventory()
        
        
        self.trash_plate_size = DEFAULT_TRASH_PLATE_SIZE
        
        #default to test mode (users need to change this after creating the protocol before using it)
        self.set_test_mode(True)
        
        #self.trash_tube_count = 0
        
        self._trash_plate = None
        self._trash_count = 0
        self._reagant_plate_cache = {}
        
        self.parent_protocol = parent_protocol
        self._last_incubated = []
        
        def get_zero():
            return ul(0)
        
        
        #changes default incubation w/ CO2
        self.mammalian_cell_mode = mammalian_cell_mode
        
        self.unprovisioned_stock_reagent_volumes = defaultdict(get_zero)
        
        self.absorbance_measurement_count= 0
        
        #used for measuring optical density
        self.absorbance_plate = None
        self.next_absorbance_plate_index = 0
        
        super(CustomProtocol,self).__init__() 

def create_storyboard(uri,date,title,text,expanded_search,top_results):

    clientips  = defaultdict(int)
    reqmethods = defaultdict(int)
    rescontype = defaultdict(int)
    referers   = defaultdict(int)
    refered    = defaultdict(int)
    requests   = []


    for row in expanded_search:
        clientips[row['clientIp']]+=1
        reqmethods[row['requestMethod']]+=1
        rescontype[row['responseContentType']]+=1
        if row['uri'] == uri:
           #Source URI's that refered the user to the threat
           referers[row['referer']]+=1
           requests += [{'clientip':row['clientIp'], 'referer':row['referer'],'reqmethod':row['requestMethod'], 'resconttype':row['responseContentType']}]

        else:
            #Destination URI's refered by the threat
            refered[row['uri']]+=1

    create_incident_progression(uri,requests,refered,date)
    create_timeline(uri,clientips,date,top_results)
    save_comments(uri,title,text,date)

    return True 

def net_if_addrs():
    """Return the addresses associated to each NIC (network interface
    card) installed on the system as a dictionary whose keys are the
    NIC names and value is a list of namedtuples for each address
    assigned to the NIC. Each namedtuple includes 4 fields:

     - family
     - address
     - netmask
     - broadcast

    'family' can be either socket.AF_INET, socket.AF_INET6 or
    psutil.AF_LINK, which refers to a MAC address.
    'address' is the primary address, 'netmask' and 'broadcast'
    may be None.
    Note: you can have more than one address of the same family
    associated with each interface.
    """
    has_enums = sys.version_info >= (3, 4)
    if has_enums:
        import socket
    rawlist = _psplatform.net_if_addrs()
    rawlist.sort(key=lambda x: x[1])  # sort by family
    ret = collections.defaultdict(list)
    for name, fam, addr, mask, broadcast in rawlist:
        if has_enums:
            try:
                fam = socket.AddressFamily(fam)
            except ValueError:
                if os.name == 'nt' and fam == -1:
                    fam = _psplatform.AF_LINK
                elif (hasattr(_psplatform, "AF_LINK") and
                        _psplatform.AF_LINK == fam):
                    # Linux defines AF_LINK as an alias for AF_PACKET.
                    # We re-set the family here so that repr(family)
                    # will show AF_LINK rather than AF_PACKET
                    fam = _psplatform.AF_LINK
        ret[name].append(_common.snic(fam, addr, mask, broadcast))
    return dict(ret) 

def test_basic(self):
        d1 = defaultdict()
        self.assertEqual(d1.default_factory, None)
        d1.default_factory = list
        d1[12].append(42)
        self.assertEqual(d1, {12: [42]})
        d1[12].append(24)
        self.assertEqual(d1, {12: [42, 24]})
        d1[13]
        d1[14]
        self.assertEqual(d1, {12: [42, 24], 13: [], 14: []})
        self.assertTrue(d1[12] is not d1[13] is not d1[14])
        d2 = defaultdict(list, foo=1, bar=2)
        self.assertEqual(d2.default_factory, list)
        self.assertEqual(d2, {"foo": 1, "bar": 2})
        self.assertEqual(d2["foo"], 1)
        self.assertEqual(d2["bar"], 2)
        self.assertEqual(d2[42], [])
        self.assertIn("foo", d2)
        self.assertIn("foo", d2.keys())
        self.assertIn("bar", d2)
        self.assertIn("bar", d2.keys())
        self.assertIn(42, d2)
        self.assertIn(42, d2.keys())
        self.assertNotIn(12, d2)
        self.assertNotIn(12, d2.keys())
        d2.default_factory = None
        self.assertEqual(d2.default_factory, None)
        try:
            d2[15]
        except KeyError, err:
            self.assertEqual(err.args, (15,)) 

def test_shallow_copy(self):
        d1 = defaultdict(foobar, {1: 1})
        d2 = copy.copy(d1)
        self.assertEqual(d2.default_factory, foobar)
        self.assertEqual(d2, d1)
        d1.default_factory = list
        d2 = copy.copy(d1)
        self.assertEqual(d2.default_factory, list)
        self.assertEqual(d2, d1) 

def test_deep_copy(self):
        d1 = defaultdict(foobar, {1: [1]})
        d2 = copy.deepcopy(d1)
        self.assertEqual(d2.default_factory, foobar)
        self.assertEqual(d2, d1)
        self.assertTrue(d1[1] is not d2[1])
        d1.default_factory = list
        d2 = copy.deepcopy(d1)
        self.assertEqual(d2.default_factory, list)
        self.assertEqual(d2, d1) 

def test_callable_arg(self):
        self.assertRaises(TypeError, defaultdict, {}) 

def test_cuboctahedron(self):

        # http://en.wikipedia.org/wiki/Cuboctahedron
        # 8 triangular faces and 6 square faces
        # 12 indentical vertices each connecting a triangle and square

        g = cube(3)
        cuboctahedron = linegraph(g)            # V( --> {V1, V2, V3, V4}
        self.assertEqual(len(cuboctahedron), 12)# twelve vertices

        vertices = set(cuboctahedron)
        for edges in cuboctahedron.values():
            self.assertEqual(len(edges), 4)     # each vertex connects to four other vertices
        othervertices = set(edge for edges in cuboctahedron.values() for edge in edges)
        self.assertEqual(vertices, othervertices)   # edge vertices in original set

        cubofaces = faces(cuboctahedron)
        facesizes = collections.defaultdict(int)
        for face in cubofaces:
            facesizes[len(face)] += 1
        self.assertEqual(facesizes[3], 8)       # eight triangular faces
        self.assertEqual(facesizes[4], 6)       # six square faces

        for vertex in cuboctahedron:
            edge = vertex                       # Cuboctahedron vertices are edges in Cube
            self.assertEqual(len(edge), 2)      # Two cube vertices define an edge
            for cubevert in edge:
                self.assertIn(cubevert, g)


#============================================================================== 

def scrawl_kernel(arch):
    re_href = re.compile('href="?({arch}[^ <>"]*)"?'.format(arch=arch))
    url = "https://toolchains.bootlin.com/downloads/releases/toolchains/{arch}/test-system/".format(arch=arch)
    response = requests.get(url + "?C=M;O=D")
    text = response.text
    links = re_href.findall(text)
    links_dict = defaultdict(lambda: defaultdict(dict))
    for link in links:
        version = get_link_version(link)
        libc = get_link_libc(link)
        filetype = get_link_filetype(link)

        # TODO: make sure they have been compiled at the same time
        if filetype not in links_dict[version][libc]:
            if filetype is None:
                return None, None, None
            links_dict[version][libc][filetype] = url + link

    state = "bleeding-edge"
    if "stable" in links_dict:
        state = "stable"

    for libc in ["glibc", "uclibc", "musl"]:
        if libc in links_dict[state]:
            break
    else:
        libc = None

    target = links_dict[state][libc]

    dtb = target.get("dtb", None)
    rootfs = target.get("rootfs", None)
    kernel = target.get("kernel", None)

    return kernel, dtb, rootfs 

def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._subscribers = defaultdict(set) 

def __init__(self, data):
        self.data = data
        self.callees = defaultdict(set)
        for func in self.data["functions"]:
            for call in func["called_from"]:
                self.callees[call].add(func["entry_point"]) 

def __init__(self, data):
        self.data = data
        self.callees = defaultdict(set)
        for func in self.data["functions"]:
            for call in func["called_from"]:
                self.callees[call].add(func["entry_point"]) 

def reset_uids():
    global _UID_PREFIXES
    _UID_PREFIXES = defaultdict(int) 

def dd():
    return defaultdict(float) 

def dd():
    return defaultdict(float) 

def get_article2outlinks(self, clickstream_date):
        print('get_article2outlinks...')
        article2outlinks = collections.defaultdict(set)
        fname = '/home/ddimitrov/data/enwiki20150304_plus_clickstream/' + clickstream_date + '_clickstream.tsv'
        regex = re.compile(r'^([\d]+)\t([\d]+)')
        with open(fname) as infile:
            for lindex, line in enumerate(infile):
                if (lindex % 10000) == 0:
                    print('\r', lindex, end='')
                line = line.decode('utf-8')
                for left, right in regex.findall(line):
                    article2outlinks[int(left)].add(int(right))
        print()
        return article2outlinks 

def get_teacher(url_tid, url_semester):
    """老师查询"""

    # decrypt identifier in URL
    try:
        _, teacher_id = decrypt(url_tid, resource_type='teacher')
    except ValueError:
        return render_template("common/error.html", message=MSG_INVALID_IDENTIFIER)

    # RPC to get teacher timetable
    with tracer.trace('rpc_get_teacher_timetable'):
        try:
            teacher = Entity.get_teacher_timetable(teacher_id, url_semester)
        except Exception as e:
            return handle_exception_with_error_page(e)

    with tracer.trace('process_rpc_result'):
        cards = defaultdict(list)
        for card in teacher.cards:
            day, time = lesson_string_to_tuple(card.lesson)
            if (day, time) not in cards:
                cards[(day, time)] = list()
            cards[(day, time)].append(card)

    empty_5, empty_6, empty_sat, empty_sun = _empty_column_check(cards)

    available_semesters = semester_calculate(url_semester, teacher.semesters)

    return render_template('query/teacher.html',
                           teacher=teacher,
                           cards=cards,
                           empty_sat=empty_sat,
                           empty_sun=empty_sun,
                           empty_6=empty_6,
                           empty_5=empty_5,
                           available_semesters=available_semesters,
                           current_semester=url_semester) 

def get_classroom(url_rid, url_semester):
    """教室查询"""
    # decrypt identifier in URL
    try:
        _, room_id = decrypt(url_rid, resource_type='room')
    except ValueError:
        return render_template("common/error.html", message=MSG_INVALID_IDENTIFIER)

    # RPC to get classroom timetable
    with tracer.trace('rpc_get_classroom_timetable'):
        try:
            room = Entity.get_classroom_timetable(url_semester, room_id)
        except Exception as e:
            return handle_exception_with_error_page(e)

    with tracer.trace('process_rpc_result'):
        cards = defaultdict(list)
        for card in room.cards:
            day, time = lesson_string_to_tuple(card.lesson)
            cards[(day, time)].append(card)

    empty_5, empty_6, empty_sat, empty_sun = _empty_column_check(cards)

    available_semesters = semester_calculate(url_semester, room.semesters)

    return render_template('query/room.html',
                           room=room,
                           cards=cards,
                           empty_sat=empty_sat,
                           empty_sun=empty_sun,
                           empty_6=empty_6,
                           empty_5=empty_5,
                           available_semesters=available_semesters,
                           current_semester=url_semester) 

def __init__(self):
        self.users = collections.defaultdict(collections.defaultdict) 

def __init__(self, name=None, router=None, error_handler=None,
                 load_env=True, request_class=None,
                 strict_slashes=False, log_config=None,
                 configure_logging=True):

        # Get name from previous stack frame
        if name is None:
            frame_records = stack()[1]
            name = getmodulename(frame_records[1])

        # logging
        if configure_logging:
            logging.config.dictConfig(log_config or LOGGING_CONFIG_DEFAULTS)

        self.name = name
        self.router = router or Router()
        self.request_class = request_class
        self.error_handler = error_handler or ErrorHandler()
        self.config = Config(load_env=load_env)
        self.request_middleware = deque()
        self.response_middleware = deque()
        self.blueprints = {}
        self._blueprint_order = []
        self.configure_logging = configure_logging
        self.debug = None
        self.sock = None
        self.strict_slashes = strict_slashes
        self.listeners = defaultdict(list)
        self.is_running = False
        self.is_request_stream = False
        self.websocket_enabled = False
        self.websocket_tasks = set()

        # Register alternative method names
        self.go_fast = self.run 

def __init__(self):
        self.routes_all = {}
        self.routes_names = {}
        self.routes_static_files = {}
        self.routes_static = {}
        self.routes_dynamic = defaultdict(list)
        self.routes_always_check = []
        self.hosts = set() 

def _build_tables(root_scripts, scripts):
        roots = defaultdict(list)
        root_ss = {}

        for root in root_scripts:
            for ss in root.singular_sequences:
                root_ss[ss] = root

        # assign each paradigm to its root paradigm
        for s in scripts:
            if s.cardinal == 1:
                continue
            if s.singular_sequences[0] not in root_ss:
                logger.error(s.singular_sequences[0] + " not found")
                continue
            roots[root_ss[s.singular_sequences[0]]].append(s)

        root_paradigms = {}
        for root in root_scripts:
            tables, cells = TableStructure._define_root(root=root, paradigms=roots[root])
            root_paradigms[root] = tables | cells

        tables = {}
        for t in chain.from_iterable(root_paradigms.values()):
            tables[t.script] = t

        return tables, root_paradigms 

def _from_list(cls, l: List[Tuple[List[Script], X]], context: Type[SyntagmaticFunction]):

        junctions = {address[0] for address, _ in l}

        links = junctions & JUNCTION_LINK_TO_ANCHOR.keys()
        if len(links) != 1:
            raise ValueError("No links defined in the junction")
        link = next(iter(links))

        anchors = junctions & JUNCTION_ANCHOR_TO_LINKS.keys()
        try:
            anchor = next(iter(anchors))

            if any(j not in {anchor, *JUNCTION_ANCHOR_TO_LINKS[anchor]} for j in junctions):
                raise ValueError("multiple different anchor are taken togethers")
        except StopIteration:
            pass

        # if any(link not in {j, anchor} for j in junctions):
        #     raise ValueError("multiple different links are taken togethers")

        _groups = defaultdict(list)
        for address, x in l:
            idx = JUNCTION_INDEX.index(address[1])
            _groups[idx].append((address[2:], x))

        groups = [[] for _ in range(len(_groups))]
        for i in _groups:
            if i > len(groups):
                raise ValueError("Invalid junction syntagmatic function definition, incoherent indexing.")
            groups[i] = _groups[i]

        return cls(junction_link=link, children=[context._from_list(g_v) for g_v in groups]) 

def _from_list(l: List[Tuple[List[Script], X]]):
        actor = None
        _actants = defaultdict(list)
        valence = None

        for address, x in l:
            role = next(iter(address))

            if role in ADDRESS_PROCESS_VALENCE_SCRIPTS:
                if len(address) != 1 or actor is not None:
                    raise ValueError("Invalid process syntagmatic function, too many process roles")
                actor = x
                valence = role
            elif role in ACTANTS_SCRIPTS:
                actant_count = sum(1 if a in ACTANTS_SCRIPTS else 0 for a in address)

                if actant_count == 1:
                    _actants[role].append([address[1:], x])
                elif actant_count > 1:
                    raise ValueError("Invalid syntagmatic function, too many actant roles")
            else:
                raise ValueError("Invalid address in a process syntagmatic function {}".format(' '.join(map(str, address))))

        actants = {}
        for r, l_actant in _actants.items():
            # if all(l[0][0] in JUNCTION_SCRIPTS for l in l_actant):
            #     actants[r] = JunctionSyntagmaticFunction._from_list(l_actant, DependantQualitySyntagmaticFunction)
            # else:
            actants[r] = DependantQualitySyntagmaticFunction._from_list(l_actant)

        return ProcessSyntagmaticFunction(actor=actor, actants=actants, valence=valence) 

def project_usls_on_dictionary(usls, allowed_terms=None):
    """`usls` is an iterable of usl.

    return a mapping term -> usl list
    """

    cells_to_usls = defaultdict(set)
    tables = set()

    for u in usls:
        for t in u.objects(Term):
            for c in t.singular_sequences:
                # This is the first time we meet the cell c
                if not cells_to_usls[c]:
                    tables.update(c.relations.contained)

                cells_to_usls[c].add(u)

    if allowed_terms:
        allowed_terms = set(allowed_terms)
        tables = tables & allowed_terms
        cells_to_usls = {c: l for c, l in cells_to_usls.items() if c in allowed_terms}

    tables_to_usls = {
        table: list(set(u for c in table.singular_sequences for u in cells_to_usls[c]))
            for table in tables if not isinstance(table, TableSet)
    }

    return tables_to_usls 

def get_values_partial(self, ieml, language=None, descriptor=None):
        ieml, language, descriptor = _normalize_key(ieml, language, descriptor,
                                                    parse_ieml=False, partial=True)

        key = {'ieml': ieml, 'language': language, 'descriptor': descriptor}
        key = reduce(operator.and_,
                     [self.df.index.get_level_values(k) == v for k, v in key.items() if v is not None],
                     True)

        res = defaultdict(list)
        for k, (v,) in self.df[key].iterrows():
            res[k].append(v)

        return dict(res) 

def _build_deps_text(rules):
    indexes = defaultdict(list)
    generals = []

    for p, e in rules:
        if isinstance(p, ContextPath):
            actual_P = p.children[0]  # coordinate
            ctx_P = path(p.children[1:])
        else:
            actual_P = p
            ctx_P = None

        if not isinstance(actual_P, Coordinate):
            raise ResolveError("A text must be defined by a coordinate, not %s"%actual_P)

        if actual_P.index is not None:
            indexes[actual_P.index].append((ctx_P, e))
        else:
            generals.append((ctx_P, e))

    if not indexes:
        return False, []

    if max(indexes, default=-1) + 1 - len(indexes) > 1:
        # if there is more than one missing
        raise ResolveError("Index missing on text definition.")

    error = False
    i = 0
    result = []
    while i <= max(indexes):
        ctx_rules = generals[:]
        if i in indexes:
            ctx_rules.extend(indexes[i])

        node = _resolve_ctx("t%d"%i, ctx_rules)
        error |= node is None
        result.append(node)
        i += 1

    return error, result 

def parse_cmake_module(s_in, defaults={}):

    parsed_config = collections.defaultdict(lambda: None)

    if 'autocmake.cfg configuration::' not in s_in:
        return parsed_config

    s_out = []
    is_rst_line = False
    for line in s_in.split('\n'):
        if is_rst_line:
            if len(line) > 0:
                if line[0] != '#':
                    is_rst_line = False
            else:
                is_rst_line = False
        if is_rst_line:
            s_out.append(line[2:])
        if '#.rst:' in line:
            is_rst_line = True

    autocmake_entry = '\n'.join(s_out).split('autocmake.cfg configuration::')[1]
    autocmake_entry = autocmake_entry.replace('\n  ', '\n')

    # we prepend a fake section heading so that we can parse it with configparser
    autocmake_entry = '[foo]\n' + autocmake_entry

    buf = StringIO(autocmake_entry)
    config = ConfigParser(dict_type=collections.OrderedDict)
    config.readfp(buf)

    for section in config.sections():
        for s in ['docopt', 'define', 'export', 'fetch', 'warning']:
            if config.has_option(section, s):
                parsed_config[s] = config.get(section, s, 0, defaults)

    return parsed_config

# ------------------------------------------------------------------------------ 

def _get_stats(cur):
  print "hit"
  cur.execute("SHOW STATS;")
  stats = defaultdict(dict)
  for database, total, _, _, _, req, recv, sent, query in cur.fetchall():
    stats[database] = {
      'total_requests': total,
      'req_per_sec': req,
      'recv_per_sec': recv,
      'sent_per_sec': sent,
      'avg_query': query
    }

  cur.execute("SHOW POOLS;")
  for database, _, cl_active, cl_waiting, sv_active, sv_idle, sv_used, sv_tested, sv_login, maxwait, _ in cur.fetchall():
    values = {
      'cl_active': cl_active,
      'cl_waiting': cl_waiting,
      'sv_active': sv_active,
      'sv_idle': sv_idle,
      'sv_used': sv_used,
      'sv_tested': sv_tested,
      'sv_login': sv_login,
      'maxwait': maxwait
    }
    if database not in stats:
      stats[database] = dict()
    # there can be many lines for one database, one for each user
    for (metric, value) in values.iteritems():
      if metric in stats[database]:
        stats[database][metric] += value
      else:
        stats[database][metric] = value

  cur.execute("SHOW DATABASES")
  for name, _, _, database, _, pool_size, reserve_pool, _, _, _ in cur.fetchall():
    stats[name]['pool_size'] = pool_size
    stats[name]['reserve_pool'] = reserve_pool
  print stats

  return stats 

def stratified_selection(dataset, subset_size):
	labels = [line.split(None,1)[0] for line in open(dataset)]
	label_linenums = defaultdict(list)
	for i, label in enumerate(labels):
		label_linenums[label] += [i]

	l = len(labels)
	remaining = subset_size
	ret = []

	# classes with fewer data are sampled first; otherwise
	# some rare classes may not be selected
	for label in sorted(label_linenums, key=lambda x: len(label_linenums[x])):
		linenums = label_linenums[label]
		label_size = len(linenums) 
		# at least one instance per class
		s = int(min(remaining, max(1, math.ceil(label_size*(float(subset_size)/l)))))
		if s == 0:
			sys.stderr.write('''\
Error: failed to have at least one instance per class
    1. You may have regression data.
    2. Your classification data is unbalanced or too small.
Please use -s 1.
''')
			sys.exit(-1)
		remaining -= s
		ret += [linenums[i] for i in random.sample(xrange(label_size), s)]
	return sorted(ret) 

def merge_dict_in_sequence(l):
    dd = defaultdict(list)
    for d in l:
        for key, value in d.items():
            dd[key].append(value)
    return dd


# 去掉 list 中的空白行 

def load_json_logs(json_logs):
    # load and convert json_logs to log_dict, key is epoch, value is a sub dict
    # keys of sub dict is different metrics, e.g. memory, bbox_mAP
    # value of sub dict is a list of corresponding values of all iterations
    log_dicts = [dict() for _ in json_logs]
    for json_log, log_dict in zip(json_logs, log_dicts):
        with open(json_log, 'r') as log_file:
            for l in log_file:
                log = json.loads(l.strip())
                epoch = log.pop('epoch')
                if epoch not in log_dict:
                    log_dict[epoch] = defaultdict(list)
                for k, v in log.items():
                    log_dict[epoch][k].append(v)
    return log_dicts 

def _to_tuples(self, params_tree, tunable_keys):
        param_tuples = defaultdict(dict)
        for block_name, params in params_tree.items():
            for param, value in params.items():
                key = (block_name, param)
                if key in tunable_keys:
                    param_tuples[key] = 'None' if value is None else value

        return param_tuples 

def preprocess_s(s):
    """
    s = string S

    Preprocess the string s into a map such that
    key = a unique character of S
    value = list of ordered index for character occurs in S
    """
    y = defaultdict(list)
    for i, c in enumerate(s):
        y[c].append(i)
    return y 

def withoutString(s, r):
    """
    s = string to be modify
    r = string to remove from s

    return a modified string
    """
    # Preprocess the list to store indices
    y = defaultdict(list)
    for s_i, x in enumerate(s):
        y[x].append(s_i)

        # To stop checking there isn't sufficient length for comparsion.
    sufficient_len = lambda i: n - i -m >= 0

    i = 0
    n, m = len(s), len(r)
    modified = ""

    # If y[r[0]] is empty, it will skip the for loop
    # Construct the modified string according to string s and r.
    # To optimize checking each index, we simply tranverse to all possible
    # index at which r[0] begins in string s.
    for s_r in y[r[0]]:

        # The string we want to remove is from s_r to s_r + m - 1 inclusively.
        if (i <= s_r) and sufficient_len(s_r) and (s[s_r:s_r + m] == r):

            # Add string before removed pattern
            modified += s[i : s_r]
            i = s_r + m

    if i < n:
        # Add remaining string s back to modified string
        modified+= s[i:]

    print("withoutString(\"{}\", \"{}\") = {}".format(s, r, modified))
    return modified 

def __init__(self, vertices, dependencies):
        self.graph = defaultdict(list)
        self.vertices = vertices

        # Add edge to graph
        for u, v in dependencies:
            self.graph[u].append(v)

    # O(V + E) time due to adjacency list used, and where V are vertices and E are edges
    # O(V + E) space due to recursion depth in stack frame. 

def is_cyclic(self):

        visited = defaultdict(lambda: False)
        stack = defaultdict(lambda: False)
        for i in self.vertices:
            if not visited[i]:
                if self.isCyclicUtil(i, visited, stack):
                    return True
        return False 

def get_unreachable_ops(cls, graph, outputs):
        """Finds all of the tensors in graph that are unreachable from outputs."""
        outputs = cls.flatten_recursive(outputs)
        output_to_op = collections.defaultdict(list)
        op_to_all = collections.defaultdict(list)
        assign_out_to_in = collections.defaultdict(list)

        for op in graph.get_operations():
            for x in op.inputs:
                op_to_all[op.name].append(x.name)
            for y in op.outputs:
                output_to_op[y.name].append(op.name)
                op_to_all[op.name].append(y.name)
            if str(op.type) == "Assign":
                for y in op.outputs:
                    for x in op.inputs:
                        assign_out_to_in[y.name].append(x.name)

        assign_groups = collections.defaultdict(list)
        for out_name in assign_out_to_in.keys():
            name_group = assign_out_to_in[out_name]
            for n1 in name_group:
                assign_groups[n1].append(out_name)
                for n2 in name_group:
                    if n1 != n2:
                        assign_groups[n1].append(n2)

        seen_tensors = {}
        stack = [x.name for x in outputs]
        while stack:
            name = stack.pop()
            if name in seen_tensors:
                continue
            seen_tensors[name] = True

            if name in output_to_op:
                for op_name in output_to_op[name]:
                    if op_name in op_to_all:
                        for input_name in op_to_all[op_name]:
                            if input_name not in stack:
                                stack.append(input_name)

            expanded_names = []
            if name in assign_groups:
                for assign_name in assign_groups[name]:
                    expanded_names.append(assign_name)

            for expanded_name in expanded_names:
                if expanded_name not in stack:
                    stack.append(expanded_name)

        unreachable_ops = []
        for op in graph.get_operations():
            is_unreachable = False
            all_names = [x.name for x in op.inputs] + [x.name for x in op.outputs]
            for name in all_names:
                if name not in seen_tensors:
                    is_unreachable = True
            if is_unreachable:
                unreachable_ops.append(op)
        return unreachable_ops 

def _leaf_ref_to_chunk_clones_from_clone_ops(
    extent_id: int, clone_ops: Iterable[_CloneOp]
):
    'As per `_leaf_extent_id_to_clone_ops`, this computes interval overlaps'
    active_ops: Dict[_CloneExtentRef, _CloneOp] = {}  # Tracks open intervals
    leaf_ref_to_chunk_clones = defaultdict(list)
    for op in sorted(clone_ops):
        # Whenever an interval (aka an Inode's Extent's "trimmed leaf")
        # ends, we create `ChunkClone` objects **to** and **from** all the
        # concurrently open intervals.
        if op.action is _CloneOp.POP:
            pushed_op = active_ops.pop(op.ref)
            assert pushed_op.ref is op.ref
            assert id(op.ref.extent) == extent_id
            assert pushed_op.pos == op.ref.offset
            assert pushed_op.pos + op.ref.clone.length == op.pos

            for clone_op in active_ops.values():
                assert op.ref.extent is clone_op.ref.extent

                # The cloned portion's extent offset is the larger of the 2
                bigger_offset = max(clone_op.ref.offset, op.ref.offset)

                # Record that `clone_op` clones part of `op`'s inode.
                leaf_ref_to_chunk_clones[op.ref].append(ChunkClone(
                    offset=bigger_offset,
                    clone=Clone(
                        inode_id=clone_op.ref.clone.inode_id,
                        offset=clone_op.ref.clone.offset + (
                            bigger_offset - clone_op.ref.offset
                        ),
                        length=op.pos - bigger_offset,
                    ),
                ))

                # Record that `op` clones part of `clone_op`'s inode.
                leaf_ref_to_chunk_clones[clone_op.ref].append(ChunkClone(
                    offset=bigger_offset,
                    clone=Clone(
                        inode_id=op.ref.clone.inode_id,
                        offset=op.ref.clone.offset + (
                            bigger_offset - op.ref.offset
                        ),
                        length=op.pos - bigger_offset,  # Same length
                    ),
                ))
        # Sorting guarantees all POPs for `pos` are handled before PUSHes
        elif op.action == _CloneOp.PUSH:
            assert op.ref not in active_ops
            active_ops[op.ref] = op
        else:
            assert False, op  # pragma: no cover
    return leaf_ref_to_chunk_clones 

def run(self, leaves):
        """The main interface with the bottom matcher. The tree is
        traversed from the bottom using the constructed
        automaton. Nodes are only checked once as the tree is
        retraversed. When the automaton fails, we give it one more
        shot(in case the above tree matches as a whole with the
        rejected leaf), then we break for the next leaf. There is the
        special case of multiple arguments(see code comments) where we
        recheck the nodes

        Args:
           The leaves of the AST tree to be matched

        Returns:
           A dictionary of node matches with fixers as the keys
        """
        current_ac_node = self.root
        results = defaultdict(list)
        for leaf in leaves:
            current_ast_node = leaf
            while current_ast_node:
                current_ast_node.was_checked = True
                for child in current_ast_node.children:
                    # multiple statements, recheck
                    if isinstance(child, pytree.Leaf) and child.value == u";":
                        current_ast_node.was_checked = False
                        break
                if current_ast_node.type == 1:
                    #name
                    node_token = current_ast_node.value
                else:
                    node_token = current_ast_node.type

                if node_token in current_ac_node.transition_table:
                    #token matches
                    current_ac_node = current_ac_node.transition_table[node_token]
                    for fixer in current_ac_node.fixers:
                        if not fixer in results:
                            results[fixer] = []
                        results[fixer].append(current_ast_node)

                else:
                    #matching failed, reset automaton
                    current_ac_node = self.root
                    if (current_ast_node.parent is not None
                        and current_ast_node.parent.was_checked):
                        #the rest of the tree upwards has been checked, next leaf
                        break

                    #recheck the rejected node once from the root
                    if node_token in current_ac_node.transition_table:
                        #token matches
                        current_ac_node = current_ac_node.transition_table[node_token]
                        for fixer in current_ac_node.fixers:
                            if not fixer in results.keys():
                                results[fixer] = []
                            results[fixer].append(current_ast_node)

                current_ast_node = current_ast_node.parent
        return results 

def ics_download(calendar_token: str):
    """
    iCalendar ics 文件下载

    2019-8-25 改为预先缓存文件而非每次动态生成，降低 CPU 压力。如果一小时内两次访问则强刷缓存。
    """
    import time
    from everyclass.server import statsd

    if not is_valid_uuid(calendar_token):
        return 'invalid calendar token', 404

    result = CalendarToken.find_calendar_token(token=calendar_token)
    if not result:
        return 'invalid calendar token', 404
    CalendarToken.update_last_used_time(calendar_token)

    cal_dir = calendar_dir()
    cal_filename = f"{result['type']}_{result['identifier']}_{result['semester']}.ics"
    cal_full_path = os.path.join(cal_dir, cal_filename)
    # 有缓存、且缓存时间小于一天，且不用强刷缓存
    if os.path.exists(cal_full_path) \
            and time.time() - os.path.getmtime(cal_full_path) < SECONDS_IN_ONE_DAY \
            and Redis.calendar_token_use_cache(calendar_token):
        logger.info("ics cache hit")
        statsd.increment("calendar.ics.cache.hit")
        return send_from_directory(cal_dir, cal_filename, as_attachment=True, mimetype='text/calendar')
    statsd.increment("calendar.ics.cache.miss")

    # 无缓存、或需要强刷缓存
    with tracer.trace('rpc'):
        # 获得原始学号或教工号
        if result['type'] == 'student':
            rpc_result = Entity.get_student_timetable(result['identifier'], result['semester'])
        else:
            # teacher
            rpc_result = Entity.get_teacher_timetable(result['identifier'], result['semester'])

        semester = Semester(result['semester'])

        cards: Dict[Tuple[int, int], List[Dict]] = defaultdict(list)
        for card in rpc_result.cards:
            cards[lesson_string_to_tuple(card.lesson)].append(dict(name=card.name,
                                                                   teacher=teacher_list_to_name_str(card.teachers),
                                                                   week=card.weeks,
                                                                   week_string=card.week_string,
                                                                   classroom=card.room,
                                                                   cid=card.card_id_encoded))

    ics_generator.generate(name=rpc_result.name,
                           cards=cards,
                           semester=semester,
                           filename=cal_filename)

    return send_from_directory(cal_dir, cal_filename, as_attachment=True, mimetype='text/calendar') 

def makeShortNames(nameMap, first=False):
    # Make short versions of names from dict of the form {id: 'Last, First ...', ...}
    # If first, use first name as prefix, rather than last name
    # Returns map of id->shortName
    prefixDict = defaultdict(list)
    suffixesDict = {}
    for idValue, name in nameMap.items():
        lastName, _, firstmiddle = name.partition(',')
        lastName = lastName.strip()
        lastName = lastName[:1].upper() + lastName[1:]
        firstmiddle = firstmiddle.strip()
        if first:
            # For Firstname, try suffixes in following order: middle_initials+Lastname
            comps = firstmiddle.split()
            firstName = (comps and comps[0]) or idValue
            suffix = lastName
            if len(comps) > 1:
                suffix = ''.join(x[0] for x in comps[1:]).upper() + suffix
            prefixDict[firstName].append(idValue)
            suffixesDict[idValue] = [suffix]
        else:
            # For Lastname, try suffixes in following order: initials, first/middle names
            if not lastName:
                lastName = idValue
            initials = ''.join(x[0] for x in firstmiddle.split()).upper()
            prefixDict[lastName].append(idValue)
            suffixesDict[idValue] = [initials, firstmiddle]

    shortMap = {}
    for prefix, idValues in prefixDict.items():
        unique = None
        for j in range(1 if first else 2):
            suffixes = [suffixesDict[idValue][j] for idValue in idValues]
            maxlen = max([len(x) for x in suffixes])
            for k in range(maxlen+1):
                truncSet = set([x[:k] for x in suffixes])
                if len(suffixes) == len(truncSet):
                    # Suffixes uniquely map id for this truncation
                    unique = [j, k]
                    break
            if unique:
                break
        for idValue in idValues:
            if unique:
                shortMap[idValue] = prefix + suffixesDict[idValue][unique[0]][:unique[1]]
            else:
                shortMap[idValue] = prefix + '-' + idValue
                
    return shortMap 

def __init__(self, list_transitions):
        """
        Transitions list must be the (start, end, data) the data will be stored as the transition tag
        :param list_transitions:
        """
        # transitions : dict
        #
        self.transitions = defaultdict(list)
        for t in list_transitions:
            self.transitions[t[0]].append((t[1], t))

        self.nodes = sorted(set(self.transitions) | {e[0] for l in self.transitions.values() for e in l})

        # sort the transitions
        for s in self.transitions:
            self.transitions[s].sort(key=lambda t: self.nodes.index(t[0]))

        self.nodes_index = {n: i for i, n in enumerate(self.nodes)}
        _count = len(self.nodes)
        self.array = numpy.zeros((len(self.nodes), len(self.nodes)), dtype=bool)

        for t in self.transitions:
            for end in self.transitions[t]:
                self.array[self.nodes_index[t]][self.nodes_index[end[0]]] = True

        # checking
        # root checking, no_parent hold True for each index where the node has no parent
        parents_count = numpy.dot(self.array.transpose().astype(dtype=int), numpy.ones((_count,), dtype=int))
        no_parents = parents_count == 0
        roots_count = numpy.count_nonzero(no_parents)

        if roots_count == 0:
            raise InvalidTreeStructure('No root node found, the graph has at least a cycle.')
        elif roots_count > 1:
            raise InvalidTreeStructure('Several root nodes found.')

        self.root = self.nodes[no_parents.nonzero()[0][0]]

        if (parents_count > 1).any():
            raise InvalidTreeStructure('A node has several parents.')

        def __stage():
            current = [self.root]
            while current:
                yield current
                current = [child[0] for parent in current for child in self.transitions[parent]]

        self.stages = list(__stage()) 

def _build_deps_topic(rules):
    result = {
        'r': {
            'indexes': defaultdict(list),
            'generals': []
        },
        'f': {
            'indexes': defaultdict(list),
            'generals': []
        }
    }

    for p, e in rules:
        if isinstance(p, ContextPath):
            # should be an error but support it
            actual_P = p.children[0]  # coordinate
            ctx_P = path(p.children[1:])
        else:
            actual_P = p
            ctx_P = None

        if not isinstance(actual_P, Coordinate):
            raise ResolveError("A topic must be defined by a single coordinate.")

        if actual_P.index is not None:
            result[actual_P.kind]['indexes'][actual_P.index].append((ctx_P, e))
        else:
            result[actual_P.kind]['generals'].append((ctx_P, e))

    error = False
    for k in result:
        # k for kind (r or f)

        indexes = result[k]['indexes']
        generals = result[k]['generals']

        if not indexes and not generals:
            result[k] = []
            continue

        if max(indexes, default=-1) + 1 > len(indexes) + len(generals):
            # if there is more than one missing
            raise ResolveError("Index missing on topic definition.")

        current = []
        length = len(indexes) + len(generals)
        generals = generals.__iter__()
        for i in range(length):
            if i in indexes:
                ctx_rules = indexes[i]
            else:
                ctx_rules = [next(generals)]

            node = _resolve_ctx("%s%d"%(k, i), ctx_rules)
            error |= node is None

            current.append(node)

        result[k] = current

    return error, result 

def drupal_relations_dump(number=None, all=False):
    _drupal_utils = _get_drupal_utils()

    root = _term("O:M:.O:M:.-+M:O:.M:O:.-")

    if all:
        paradigm = list(Dictionary())
    else:
        paradigm = sorted(Dictionary().roots[root])

    relations = defaultdict(set)

    def add_rel(t0, t1, relname):
        if t1 > t0:
            q = t0
            t0 = t1
            t1 = q

        if t0 != t1:
            relations[(t0, t1)].add(relname)

    REL = list(_RELATIONS)

    for i, t0 in enumerate(paradigm):
        for t1 in paradigm[i:]:
            for r in t0.relations.to(t1, relations_types=REL):
                add_rel(t0, t1, r)

    res = []

    for t0, t1 in relations:
        for rel_cat in relations[(t0, t1)]:
            comment = ''
            res.append({
                'term_src': _drupal_utils['all_uuid'][str(t0.script)],
                'term_dest': _drupal_utils['all_uuid'][str(t1.script)],
                'relation_name': rel_cat,
                'relation_type': _RELATIONS[rel_cat],
                'commentary': comment
            })

            res.append({
                'term_src': _drupal_utils['all_uuid'][str(t1.script)],
                'term_dest': _drupal_utils['all_uuid'][str(t0.script)],
                'relation_name': INVERSE_RELATIONS[rel_cat],
                'relation_type': _RELATIONS[rel_cat],
                'commentary': comment
            })

    if number:
        return res[:number]
    else:
        return res 

def get_instances_by_region(self, region):
        ''' Makes an AWS EC2 API call to the list of instances in a particular
        region '''

        try:
            conn = self.connect(region)
            reservations = []
            if self.ec2_instance_filters:
                if self.stack_filters:
                    filters_dict = {}
                    for filters in self.ec2_instance_filters:
                        filters_dict.update(filters)
                    reservations.extend(conn.get_all_instances(filters=filters_dict))
                else:
                    for filters in self.ec2_instance_filters:
                        reservations.extend(conn.get_all_instances(filters=filters))
            else:
                reservations = conn.get_all_instances()

            # Pull the tags back in a second step
            # AWS are on record as saying that the tags fetched in the first `get_all_instances` request are not
            # reliable and may be missing, and the only way to guarantee they are there is by calling `get_all_tags`
            instance_ids = []
            for reservation in reservations:
                instance_ids.extend([instance.id for instance in reservation.instances])

            max_filter_value = 199
            tags = []
            for i in range(0, len(instance_ids), max_filter_value):
                tags.extend(conn.get_all_tags(filters={'resource-type': 'instance', 'resource-id': instance_ids[i:i + max_filter_value]}))

            tags_by_instance_id = defaultdict(dict)
            for tag in tags:
                tags_by_instance_id[tag.res_id][tag.name] = tag.value

            if (not self.aws_account_id) and reservations:
                self.aws_account_id = reservations[0].owner_id

            for reservation in reservations:
                for instance in reservation.instances:
                    instance.tags = tags_by_instance_id[instance.id]
                    self.add_instance(instance, region)

        except boto.exception.BotoServerError as e:
            if e.error_code == 'AuthFailure':
                error = self.get_auth_error_message()
            else:
                backend = 'Eucalyptus' if self.eucalyptus else 'AWS'
                error = "Error connecting to %s backend.\n%s" % (backend, e.message)
            self.fail_with_error(error, 'getting EC2 instances') 

def collect_env():
    env_info = {}
    env_info['sys.platform'] = sys.platform
    env_info['Python'] = sys.version.replace('\n', '')

    cuda_available = torch.cuda.is_available()
    env_info['CUDA available'] = cuda_available

    if cuda_available:
        from torch.utils.cpp_extension import CUDA_HOME
        env_info['CUDA_HOME'] = CUDA_HOME

        if CUDA_HOME is not None and osp.isdir(CUDA_HOME):
            try:
                nvcc = osp.join(CUDA_HOME, 'bin/nvcc')
                nvcc = subprocess.check_output(
                    '"{}" -V | tail -n1'.format(nvcc), shell=True)
                nvcc = nvcc.decode('utf-8').strip()
            except subprocess.SubprocessError:
                nvcc = 'Not Available'
            env_info['NVCC'] = nvcc

        devices = defaultdict(list)
        for k in range(torch.cuda.device_count()):
            devices[torch.cuda.get_device_name(k)].append(str(k))
        for name, devids in devices.items():
            env_info['GPU ' + ','.join(devids)] = name

    gcc = subprocess.check_output('gcc --version | head -n1', shell=True)
    gcc = gcc.decode('utf-8').strip()
    env_info['GCC'] = gcc

    env_info['PyTorch'] = torch.__version__
    env_info['PyTorch compiling details'] = torch.__config__.show()

    env_info['TorchVision'] = torchvision.__version__

    env_info['OpenCV'] = cv2.__version__

    env_info['MMCV'] = mmcv.__version__
    env_info['MMDetection'] = mmdet.__version__
    env_info['MMDetection Compiler'] = get_compiler_version()
    env_info['MMDetection CUDA Compiler'] = get_compiling_cuda_version()

    for name, val in env_info.items():
        print('{}: {}'.format(name, val)) 

def basic_iterative_method(sess, model, X, Y, eps, eps_iter, nb_iter=50,
                           clip_min=None, clip_max=None, batch_size=256):
    """
    TODO
    :param sess:
    :param model: predictions or after-softmax
    :param X:
    :param Y:
    :param eps:
    :param eps_iter:
    :param nb_iter:
    :param clip_min:
    :param clip_max:
    :param batch_size:
    :return:
    """
    print("nb_iter",nb_iter)
    # Define TF placeholders for the input and output
    x = tf.placeholder(tf.float32, shape=(None,)+X.shape[1:])
    y = tf.placeholder(tf.float32, shape=(None,)+Y.shape[1:])
    # results will hold the adversarial inputs at each iteration of BIM;
    # thus it will have shape (nb_iter, n_samples, n_rows, n_cols, n_channels)
    results = np.zeros((nb_iter, X.shape[0],) + X.shape[1:])
    # Initialize adversarial samples as the original samples, set upper and
    # lower bounds
    X_adv = X
    X_min = X_adv - eps
    X_max = X_adv + eps
    print('Running BIM iterations...')
    # "its" is a dictionary that keeps track of the iteration at which each
    # sample becomes misclassified. The default value will be (nb_iter-1), the
    # very last iteration.
    def f(val):
        return lambda: val
    its = defaultdict(f(nb_iter-1))
    # Out keeps track of which samples have already been misclassified
    out = set()
    for i in tqdm(range(nb_iter)):
        adv_x = fgsm(
            x, model(x), eps=eps_iter,
            clip_min=clip_min, clip_max=clip_max, y=y
        )
        X_adv, = batch_eval(
            sess, [x, y], [adv_x],
            [X_adv, Y], feed={K.learning_phase(): 0},
            args={'batch_size': batch_size}
        )
        X_adv = np.maximum(np.minimum(X_adv, X_max), X_min)
        results[i] = X_adv
        # check misclassifieds
        predictions = model.predict_classes(X_adv, batch_size=512, verbose=0)
        misclassifieds = np.where(predictions != Y.argmax(axis=1))[0]
        for elt in misclassifieds:
            if elt not in out:
                its[elt] = i
                out.add(elt)

    return its, results 

def _save_work_results(self, run_stats, scores, num_processed_images,
                         filename):
    """Saves statistics about each submission.

    Saved statistics include score; number of completed and failed batches;
    min, max, average and median time needed to run one batch.

    Args:
      run_stats: dictionary with runtime statistics for submissions,
        can be generated by WorkPiecesBase.compute_work_statistics
      scores: dictionary mapping submission ids to scores
      num_processed_images: dictionary with number of successfully processed
        images by each submission, one of the outputs of
        ClassificationBatches.compute_classification_results
      filename: output filename
    """
    with open(filename, 'w') as f:
      writer = csv.writer(f)
      writer.writerow(
          ['SubmissionID', 'ExternalSubmissionId', 'Score',
           'CompletedBatches', 'BatchesWithError', 'ProcessedImages',
           'MinEvalTime', 'MaxEvalTime',
           'MedianEvalTime', 'MeanEvalTime',
           'ErrorMsg'])
      for submission_id in sorted(iterkeys(run_stats)):
        stat = run_stats.get(
            submission_id,
            collections.defaultdict(lambda: float('NaN')))
        external_id = self.submissions.get_external_id(submission_id)
        error_msg = ''
        while not error_msg and stat['error_messages']:
          error_msg = stat['error_messages'].pop()
          if error_msg.startswith('Cant copy adversarial batch locally'):
            error_msg = ''
        writer.writerow([
            submission_id, external_id, scores.get(submission_id, None),
            stat['completed'], stat['num_errors'],
            num_processed_images.get(submission_id, None),
            stat['min_eval_time'], stat['max_eval_time'],
            stat['median_eval_time'], stat['mean_eval_time'],
            error_msg
        ]) 

def is_subsequence2(s, w, y, debug=True):
    """
    s = string S
    w = word W to test if W is a subsequence in S
    y = preprocess dictionary of list of index for a particular character

    Return True if w is a subsequence. Otherwise, False

    Example:
    s = "abppplee"
    w = "apple"
    y = defaultdict(<class 'list'>, {'a': [0], 'b': [1], 'p': [2, 3, 4], 'l': [5], 'e': [6, 7]})
    is_subsequence(s, w, y) returns True since "apple" is a substring
    """
    n, m = len(s), len(w)
    i, j = 0, 0
    e = 0
    sufficient_len = lambda i, j: (n - i) - (m - j) >= 0
    while i < n and j < m and sufficient_len(i, j):

        if debug: print("j={}, w[j] = {}".format(j, w[j]))

        # l is the list of index in s for character at w[j]

        l = y[w[j]] if w[j] in y else None
        if debug: print("l={}, w[j]={}, j={}".format(l, w[j], j))

        # Find index t of w[j] in s such that min(t >= i in l)
        q = [k for k in l if k >= i] if l else None
        t = min(q) if q else None
        #t = min(k for k in l if k >= i) if l else None

        # Use t is not None to compare since possible t values are 0.
        if t is not None:
            # The letter of a word w in a dictionary D is found at index t of string S
            i = t+1
            j += 1
            if debug: print("k = {}, i = {}, j = {}, q = {}, t = {}, s = {}, w = {}"
                            .format(e, i, j, q, t, s[i:], w[j:]))
        else:
            # The letter of a word w is not found in string S
            i += 1
            if debug: print("k = {}, i = {}, j = {}, s = {}, w = {}".format(k , i, j, s[i:], w[j:]))
        if debug: print()

        # For debug purpose to count number of loops
        if debug: e += 1

    # Returns True if all letters of word w is found in string S. Otherwise, False
    return j == m 

def migrate(database, out_folder):
    descriptors = database.descriptors()
    dictionary = database.dictionary_structure()
    # 'root', 'paradigms', 'inhibitions'

    shutil.rmtree(out_folder + '/descriptors')
    shutil.rmtree(out_folder + '/structure')
    # os.rmdir(out_folder)

    # os.mkdir(out_folder)

    db2 = IEMLDatabase(out_folder)
    # db2.get_csv()

    if not os.path.isdir(out_folder):
        os.mkdir(out_folder)

    for ieml, (paradigms, inhibitions) in tqdm.tqdm(dictionary.structure.iterrows(), 'migrating structure'):
        l = IEMLParser().parse(ieml, factorize_script=True)

        db2.add_structure(str(l), 'is_root', True)
        for i in inhibitions:
            db2.add_structure(str(l), 'inhibition', i)

    all_db = defaultdict(lambda : defaultdict(dict))

    for (ieml, lang, desc), (v) in descriptors:
        all_db[ieml][(lang,desc)] = v.values[0]

    for ieml, dd in tqdm.tqdm(all_db.items(), 'migrating descriptors'):
        l = IEMLParser().parse(ieml, factorize_script=True)

        path = db2.path_of(l)

        os.makedirs('/'.join(path.split('/')[:-1]), exist_ok=True)


        with open(path, 'w') as fp:
            for (lang, desc), v in dd.items():
                for vv in v:
                    fp.write('"{}" {} {} "{}"\n'.format(str(l), lang, desc, db2.escape_value(vv)))


            # fp.write(json.dumps({'ieml': str(l), **dd}, indent=True))