Python typing.DefaultDict() Examples

def _busiest_week(feed: Feed) -> Dict[datetime.date, FrozenSet[str]]:
    service_ids_by_date = _service_ids_by_date(feed)
    trip_counts_by_date = _trip_counts_by_date(feed)

    weekly_trip_counts: DefaultDict[Week, int] = defaultdict(int)
    weekly_dates: DefaultDict[Week, Set[datetime.date]] = defaultdict(set)
    for date in service_ids_by_date.keys():
        key = Week.withdate(date)
        weekly_trip_counts[key] += trip_counts_by_date[date]
        weekly_dates[key].add(date)

    def max_by(kv: Tuple[Week, int]) -> Tuple[int, int]:
        week, count = kv
        return count, -week.toordinal()

    week, _ = max(weekly_trip_counts.items(), key=max_by)
    dates = sorted(weekly_dates[week])

    return {date: service_ids_by_date[date] for date in dates} 

def __init__(self, name: str, channel_service, peer_id, channel_name, store_identity):
        self.__channel_service: ChannelService = channel_service
        self.__channel_name = channel_name
        self.__pre_validate_strategy = self.__pre_validate
        self.__peer_id = peer_id

        self.__tx_queue = AgingCache(max_age_seconds=conf.MAX_TX_QUEUE_AGING_SECONDS,
                                     default_item_status=TransactionStatusInQueue.normal)
        self.blockchain = BlockChain(channel_name, store_identity, self)
        self.__peer_type = None
        self.__consensus_algorithm = None
        self.candidate_blocks = CandidateBlocks(self.blockchain)
        self.__block_height_sync_bad_targets = {}
        self.__block_height_sync_lock = threading.Lock()
        self.__block_height_thread_pool = ThreadPoolExecutor(1, 'BlockHeightSyncThread')
        self.__block_height_future: Future = None
        self.__precommit_block: Block = None
        self.set_peer_type(loopchain_pb2.PEER)
        self.name = name
        self.__service_status = status_code.Service.online

        # old_block_hashes[height][new_block_hash] = old_block_hash
        self.__old_block_hashes: DefaultDict[int, Dict[Hash32, Hash32]] = defaultdict(dict)
        self.epoch: Epoch = None 

def _aggregate_losses(self) -> Metrics:
        """Calculate the task specific loss, average micro loss and learning rate."""

        metric_dict = dict()

        # Log task specific loss
        self.running_losses: DefaultDict[str, float]
        self.running_counts: DefaultDict[str, float]
        for identifier in self.running_losses.keys():
            if self.running_counts[identifier] > 0:
                metric_dict[identifier] = (
                    self.running_losses[identifier] / self.running_counts[identifier]
                )

        # Calculate average micro loss
        total_loss = sum(self.running_losses.values())
        total_count = sum(self.running_counts.values())
        if total_count > 0:
            metric_dict["model/all/train/loss"] = total_loss / total_count

        # Log the learning rate
        metric_dict["model/all/train/lr"] = self.optimizer.param_groups[0]["lr"]

        return metric_dict 

def test_defaultdict_raise_error():
    elem = collections.defaultdict(int)
    elem[5] = [1, 2, 3]

    validator = type_validator()

    attr = MagicMock()
    attr.name = "foo"
    attr.type = DefaultDict[int, List[str]]

    with pytest.raises(ValueError) as error:
        validator(None, attr, elem)

    assert (
        "<foo must be typing.DefaultDict[int, typing.List[str]] "
        "(got 1 that is a {}) in [1, 2, 3] in "
        "defaultdict({}, {{5: [1, 2, 3]}})>"
    ).format(int, int) == repr(error.value) 

def load_parents() -> DefaultDict[str, Set[str]]:
    ''' Generates a dictionary with child to parent mapping. '''
    def recursive_parse(parents: DefaultDict[str, Set[str]], node: Any) -> None:
        name = node['LabelName']

        if 'Subcategory' in node:
            for child in node['Subcategory']:
                child_name = child['LabelName']

                if name != '/m/0bl9f':
                    parents[child_name].add(name)
                    parents[child_name].update(parents[name])

                recursive_parse(parents, child)

    with open('../data/challenge-2019-label500-hierarchy.json') as f:
        hierarchy = json.load(f)

    parents: DefaultDict[str, Set[str]] = defaultdict(set)
    recursive_parse(parents, hierarchy)

    return parents 

def load_parents() -> DefaultDict[str, Set[str]]:
    ''' Generates a dictionary with child to parent mapping. '''
    def recursive_parse(parents: DefaultDict[str, Set[str]], node: Any) -> None:
        name = node['LabelName']

        if 'Subcategory' in node:
            for child in node['Subcategory']:
                child_name = child['LabelName']

                if name != '/m/0bl9f':
                    parents[child_name].add(name)
                    parents[child_name].update(parents[name])

                recursive_parse(parents, child)

    with open('../data/challenge-2019-label500-hierarchy.json') as f:
        hierarchy = json.load(f)

    parents: DefaultDict[str, Set[str]] = defaultdict(set)
    recursive_parse(parents, hierarchy)

    return parents 

def get_borg_eval_gist(self, pr: Dict[str, Any]) -> Optional[Dict[str, Set[str]]]:
        packages_per_system: DefaultDict[str, Set[str]] = defaultdict(set)
        statuses = self.get(pr["statuses_url"])
        for status in statuses:
            url = status.get("target_url", "")
            if (
                status["description"] == "^.^!"
                and status["creator"]["login"] == "ofborg[bot]"
                and url != ""
            ):
                url = urllib.parse.urlparse(url)
                raw_gist_url = (
                    f"https://gist.githubusercontent.com/GrahamcOfBorg{url.path}/raw/"
                )
                for line in urllib.request.urlopen(raw_gist_url):
                    if line == b"":
                        break
                    system, attribute = line.decode("utf-8").split()
                    packages_per_system[system].add(attribute)
                return packages_per_system
        return None 

def filter_net_connections(
    predicates: Iterable[ProcessFilterPredicate],
) -> DefaultDict[str, Set[pconn]]:
    filtered: DefaultDict[str, Set[pconn]] = collections.defaultdict(set)
    for process, predicate in itertools.product(psutil.process_iter(), predicates):
        try:
            if not _match_process(process, predicate):
                continue

            connections = filtered[predicate.name]
            connections.update(process.connections())
        except psutil.NoSuchProcess:
            logger.debug(
                "Process with PID {} died while filtering network connections",
                process.pid,
            )
            continue

        if predicate.recursive:
            for child in process.children(recursive=True):
                try:
                    connections.update(child.connections())
                except psutil.NoSuchProcess:
                    pass

    return filtered 

def match_backends_and_tasks(
    backends: Iterable[EnvoyBackend], tasks: Iterable[marathon_tools.MarathonTask]
) -> List[Tuple[Optional[EnvoyBackend], Optional[marathon_tools.MarathonTask]]]:
    """Returns tuples of matching (backend, task) pairs, as matched by IP and port. Each backend will be listed exactly
    once, and each task will be listed once per port. If a backend does not match with a task, (backend, None) will
    be included. If a task's port does not match with any backends, (None, task) will be included.

    :param backends: An iterable of Envoy backend dictionaries, e.g. the list returned by
                     envoy_tools.get_multiple_backends.
    :param tasks: An iterable of MarathonTask objects.
    """

    # { (ip, port) : [backend1, backend2], ... }
    backends_by_ip_port: DefaultDict[
        Tuple[str, int], List[EnvoyBackend]
    ] = collections.defaultdict(list)
    backend_task_pairs = []

    for backend in backends:
        ip = backend["address"]
        port = backend["port_value"]
        backends_by_ip_port[ip, port].append(backend)

    for task in tasks:
        ip = socket.gethostbyname(task.host)
        for port in task.ports:
            for backend in backends_by_ip_port.pop((ip, port), [None]):
                backend_task_pairs.append((backend, task))

    # we've been popping in the above loop, so anything left didn't match a marathon task.
    for backends in backends_by_ip_port.values():
        for backend in backends:
            backend_task_pairs.append((backend, None))

    return backend_task_pairs 

def __init__(self, loop: asyncio.AbstractEventLoop):
        self.loop = loop
        self.incoming_connected: typing.Set[str] = set()
        self.incoming: typing.DefaultDict[str, int] = collections.defaultdict(int)
        self.outgoing_connected: typing.Set[str] = set()
        self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int)
        self._max_incoming_mbs = 0.0
        self._max_outgoing_mbs = 0.0
        self._status = {}
        self._running = False
        self._task: typing.Optional[asyncio.Task] = None 

def __init__(self, slot: float = 60):
        self._records: typing.DefaultDict[str, Record] = defaultdict(Record)
        self._slot = slot  # report once after one minute by default
        self._log_task = asyncio.ensure_future(self._log())
        self.logger = logging.getLogger(self.__class__.__name__) 

def __init__(
        self, *, to_dict: typing.Callable[[Item], typing.Dict], file_dir: str = "."
    ):
        super().__init__()
        self.file_dir = file_dir
        self.data: typing.DefaultDict[str, typing.List[typing.Dict]] = defaultdict(list)
        self.to_dict = to_dict 

def are_services_up_on_ip_port(
    synapse_host: str,
    synapse_port: int,
    synapse_haproxy_url_format: str,
    services: Collection[str],
    host_ip: str,
    host_port: int,
) -> bool:
    backends = get_multiple_backends(
        services,
        synapse_host=synapse_host,
        synapse_port=synapse_port,
        synapse_haproxy_url_format=synapse_haproxy_url_format,
    )
    backends_by_ip_port: DefaultDict[
        Tuple[str, int], List[HaproxyBackend]
    ] = collections.defaultdict(list)

    for backend in backends:
        ip, port, _ = ip_port_hostname_from_svname(backend["svname"])
        backends_by_ip_port[ip, port].append(backend)

    backends_on_ip = backends_by_ip_port[host_ip, host_port]
    # any backend being up is okay because a previous backend
    # may have had the same IP and synapse only removes them completely
    # after some time
    services_with_atleast_one_backend_up = {service: False for service in services}
    for service in services:
        for be in backends_on_ip:
            if be["pxname"] == service and backend_is_up(be):
                services_with_atleast_one_backend_up[service] = True
    return all(services_with_atleast_one_backend_up.values()) 

def __init__(self, logger=None):
        self._user_token = None
        self._user_client = None
        self._bot_token = None
        self._bot_client = None
        self._verification_token = None
        self._executor = concurrent.futures.ThreadPoolExecutor()
        self.logger = logger or logging.getLogger("uqcsbot")
        self._handlers: DefaultDict[str, list] = collections.defaultdict(list)
        self._command_registry: DefaultDict[str, list] = collections.defaultdict(list)
        self._scheduler = AsyncIOScheduler()

        self.register_handler('message', self._handle_command)
        self.register_handler('hello', self._handle_hello)
        self.register_handler('goodbye', self._handle_goodbye)

        self.channels = ChannelWrapper(self)
        self.users = UsersWrapper(self) 

def parameter_count(model: nn.Module) -> typing.DefaultDict[str, int]:
    """
    Count parameters of a model and its submodules.

    Args:
        model: a torch module

    Returns:
        dict (str-> int): the key is either a parameter name or a module name.
            The value is the number of elements in the parameter, or in all
            parameters of the module. The key "" corresponds to the total
            number of parameters of the model.
    """
    r = defaultdict(int)
    for name, prm in model.named_parameters():
        size = prm.numel()
        name = name.split(".")
        for k in range(0, len(name) + 1):
            prefix = ".".join(name[:k])
            r[prefix] += size
    return r 

def test_defaultdict_with_correct_type_no_raise():
    elem = collections.defaultdict(int)
    elem[5] = [1, 2, 3]
    elem[6] = [4, 5, 6]

    validator = type_validator()

    attr = MagicMock()
    attr.name = "foo"
    attr.type = DefaultDict[int, List[int]]

    validator(None, attr, elem) 

def canonicalize_gates(gates: LogicalGates
        ) -> Dict[frozenset, LogicalGates]:
        """Canonicalizes a set of gates by the qubits they act on.

        Takes a set of gates specified by ordered sequences of logical
        indices, and groups those that act on the same qubits regardless of
        order."""
        canonicalized_gates: DefaultDict[frozenset, LogicalGates] = defaultdict(
            dict)
        for indices, gate in gates.items():
            indices = tuple(indices)
            canonicalized_gates[frozenset(indices)][indices] = gate
        return {canonical_indices: dict(list(gates.items()))
                for canonical_indices, gates in canonicalized_gates.items()} 

def dict_to_defaultdict(d: Dict, t: type) -> DefaultDict:
        return collections.defaultdict(
            TrainerSettings, cattr.structure(d, Dict[str, TrainerSettings])
        ) 

def group_by(self, column: str) -> Dict[str, "HyperParameterDB"]:
        """Group data by given attribute/column.

        :param column: The attribute to be grouped by.
        """
        groups = collections.defaultdict(
            HyperParameterDB
        )  # type: DefaultDict[str, HyperParameterDB]
        for i in self:
            groups[getattr(i, column)].append(i)
        return groups 

def match_backends_and_pods(
    backends: Iterable[HaproxyBackend], pods: Iterable[V1Pod]
) -> List[Tuple[Optional[HaproxyBackend], Optional[V1Pod]]]:
    """Returns tuples of matching (backend, pod) pairs, as matched by IP. Each backend will be listed exactly
    once. If a backend does not match with a pod, (backend, None) will be included.
    If a pod's IP does not match with any backends, (None, pod) will be included.

    :param backends: An iterable of haproxy backend dictionaries, e.g. the list returned by
                     smartstack_tools.get_multiple_backends.
    :param pods: An iterable of V1Pod objects.
    """

    # { ip : [backend1, backend2], ... }
    backends_by_ip: DefaultDict[str, List[HaproxyBackend]] = collections.defaultdict(
        list
    )
    backend_pod_pairs = []

    for backend in backends:
        ip, port, _ = ip_port_hostname_from_svname(backend["svname"])
        backends_by_ip[ip].append(backend)

    for pod in pods:
        ip = pod.status.pod_ip
        for backend in backends_by_ip.pop(ip, [None]):
            backend_pod_pairs.append((backend, pod))

    # we've been popping in the above loop, so anything left didn't match a k8s pod.
    for backends in backends_by_ip.values():
        for backend in backends:
            backend_pod_pairs.append((backend, None))

    return backend_pod_pairs 

def match_backends_and_tasks(
    backends: Iterable[HaproxyBackend], tasks: Iterable[marathon_tools.MarathonTask]
) -> List[Tuple[Optional[HaproxyBackend], Optional[marathon_tools.MarathonTask]]]:
    """Returns tuples of matching (backend, task) pairs, as matched by IP and port. Each backend will be listed exactly
    once, and each task will be listed once per port. If a backend does not match with a task, (backend, None) will
    be included. If a task's port does not match with any backends, (None, task) will be included.

    :param backends: An iterable of haproxy backend dictionaries, e.g. the list returned by
                     smartstack_tools.get_multiple_backends.
    :param tasks: An iterable of MarathonTask objects.
    """

    # { (ip, port) : [backend1, backend2], ... }
    backends_by_ip_port: DefaultDict[
        Tuple[str, int], List[HaproxyBackend]
    ] = collections.defaultdict(list)
    backend_task_pairs = []

    for backend in backends:
        ip, port, _ = ip_port_hostname_from_svname(backend["svname"])
        backends_by_ip_port[ip, port].append(backend)

    for task in tasks:
        ip = socket.gethostbyname(task.host)
        for port in task.ports:
            for backend in backends_by_ip_port.pop((ip, port), [None]):
                backend_task_pairs.append((backend, task))

    # we've been popping in the above loop, so anything left didn't match a marathon task.
    for backends in backends_by_ip_port.values():
        for backend in backends:
            backend_task_pairs.append((backend, None))

    return backend_task_pairs 

def defaultdict_to_dict(d: DefaultDict) -> Dict:
    return {key: cattr.unstructure(val) for key, val in d.items()} 

def __init__(self, fault_tolerant: bool):
        self.fault_tolerant = fault_tolerant
        self.fault_counts: DefaultDict[str, int] = DefaultDict(int) 

def display_boxes(
        self, pdf_file: str, boxes: List[Bbox], alternate_colors: bool = False,
    ) -> List[Image]:
        """Display bounding boxes on the document.

        Display each of the bounding boxes passed in 'boxes' on images of the pdf
        pointed to by pdf_file
        boxes is a list of 5-tuples (page, top, left, bottom, right)
        """
        imgs = []
        with Color("blue") as blue, Color("red") as red, Color(
            "rgba(0, 0, 0, 0.0)"
        ) as transparent, Drawing() as draw:
            colors = [blue, red]
            boxes_per_page: DefaultDict[int, int] = defaultdict(int)
            boxes_by_page: DefaultDict[
                int, List[Tuple[int, int, int, int]]
            ] = defaultdict(list)
            for i, (page, top, bottom, left, right) in enumerate(boxes):
                boxes_per_page[page] += 1
                boxes_by_page[page].append((top, bottom, left, right))
            for i, page_num in enumerate(boxes_per_page.keys()):
                img = pdf_to_img(pdf_file, page_num)
                draw.fill_color = transparent
                for j, (top, bottom, left, right) in enumerate(boxes_by_page[page_num]):
                    draw.stroke_color = colors[j % 2] if alternate_colors else colors[0]
                    draw.rectangle(left=left, top=top, right=right, bottom=bottom)
                draw(img)
                imgs.append(img)
            return imgs 

def _get_table_cells(table: Table) -> DefaultDict[Cell, List[Sentence]]:
    """Cache table cells and the cells' sentences.

    This function significantly improves the speed of `get_row_ngrams`
    primarily by reducing the number of queries that are made (which were
    previously the bottleneck. Rather than taking a single mention, then its
    sentence, then its table, then all the cells in the table, then all the
    sentences in each cell, and performing operations on that series of
    queries, this performs a single query for all the sentences in a table and
    returns all of the cells and the cells sentences directly.

    :param table: the Table object to cache.
    :return: an iterator of (Cell, [Sentence._asdict(), ...]) tuples.
    """
    sent_map: DefaultDict[Cell, List[Sentence]] = defaultdict(list)
    for sent in table.sentences:
        sent_map[sent.cell].append(sent)
    return sent_map 

def _preprocess_visual_features(doc: Document) -> None:
    if hasattr(doc, "_visual_features"):
        return
    # cache flag
    doc._visual_features = True

    sentence_by_page: DefaultDict[str, List[Sentence]] = defaultdict(list)
    for sentence in doc.sentences:
        sentence_by_page[sentence.page[0]].append(sentence)
        sentence._aligned_lemmas = set()

    for page, sentences in sentence_by_page.items():
        # process per page alignments
        yc_aligned: DefaultDict[Any, List[Sentence]] = defaultdict(list)
        x0_aligned: DefaultDict[Any, List[Sentence]] = defaultdict(list)
        xc_aligned: DefaultDict[Any, List[Sentence]] = defaultdict(list)
        x1_aligned: DefaultDict[Any, List[Sentence]] = defaultdict(list)
        for sentence in sentences:
            sentence.bbox = sentence.get_bbox()
            sentence.yc = (sentence.bbox.top + sentence.bbox.bottom) / 2
            sentence.x0 = sentence.bbox.left
            sentence.x1 = sentence.bbox.right
            sentence.xc = (sentence.x0 + sentence.x1) / 2
            # index current sentence by different alignment keys
            yc_aligned[sentence.yc].append(sentence)
            x0_aligned[sentence.x0].append(sentence)
            x1_aligned[sentence.x1].append(sentence)
            xc_aligned[sentence.xc].append(sentence)
        for l in yc_aligned.values():
            l.sort(key=lambda p: p.xc)
        for l in x0_aligned.values():
            l.sort(key=lambda p: p.yc)
        for l in x1_aligned.values():
            l.sort(key=lambda p: p.yc)
        for l in xc_aligned.values():
            l.sort(key=lambda p: p.yc)
        _assign_alignment_features(yc_aligned, "Y_")
        _assign_alignment_features(x0_aligned, "LEFT_")
        _assign_alignment_features(x1_aligned, "RIGHT_")
        _assign_alignment_features(xc_aligned, "CENTER_") 

def _trip_counts_by_date(feed: Feed) -> Dict[datetime.date, int]:
    results: DefaultDict[datetime.date, int] = defaultdict(int)
    trips = feed.trips
    for service_ids, dates in _dates_by_service_ids(feed).items():
        trip_count = trips[trips.service_id.isin(service_ids)].shape[0]
        for date in dates:
            results[date] += trip_count
    return dict(results) 

def _dates_by_service_ids(feed: Feed) -> Dict[FrozenSet[str], FrozenSet[datetime.date]]:
    results: DefaultDict[FrozenSet[str], Set[datetime.date]] = defaultdict(set)
    for date, service_ids in _service_ids_by_date(feed).items():
        results[service_ids].add(date)
    return {k: frozenset(v) for k, v in results.items()} 

def edges(self) -> List[DefaultDict]:
        return [self.file_of, self.copied_to] 

def edges(self) -> List[DefaultDict]:
        return [self.alerted_on]