Python six.moves.reduce() Examples

def get_queryset(self):
        if not (self.request.user.is_authenticated and self.request.user.is_staff):
            raise Http404

        queryset = Plan.objects.exclude(enabled=False)

        if self.q:
            queryset = queryset.annotate(
                name_provider__name__company=Concat(
                    F("name"), Value(" "), F("provider__name"), Value(" "), F("provider__company")
                )
            )
            terms = self.q.split()

            query = reduce(
                operator.and_,
                (Q(name_provider__name__company__icontains=term) for term in terms)
            )

            queryset = queryset.filter(query)

        return queryset 

def __mul__(self, other):
        """
        Outer product of two cycles (`itertools.product`) or integer
        multiplication.

        Parameters
        ----------
        other : Cycler or int
           The second Cycler or integer
        """
        if isinstance(other, Cycler):
            return Cycler(self, other, product)
        elif isinstance(other, int):
            trans = self.by_key()
            return reduce(add, (_cycler(k, v*other)
                                for k, v in six.iteritems(trans)))
        else:
            return NotImplemented 

def simplify(self):
        """Simplify the Cycler

        Returned as a composition using only sums (no multiplications)

        Returns
        -------
        simple : Cycler
            An equivalent cycler using only summation"""
        # TODO: sort out if it is worth the effort to make sure this is
        # balanced.  Currently it is is
        # (((a + b) + c) + d) vs
        # ((a + b) + (c + d))
        # I would believe that there is some performance implications

        trans = self.by_key()
        return reduce(add, (_cycler(k, v) for k, v in six.iteritems(trans))) 

def values_eq_approx_high_tol(a, b):
    """
    This fct is needed to don't have DebugMode raise useless
    error due to ronding error.

    This happen as We reduce on the two last dimensions, so this
    can raise the absolute error if the number of element we
    reduce on is significant.

    """
    assert a.ndim == 4
    atol = None
    if a.shape[-1] * a.shape[-2] > 100:
        # For float32 the default atol is 1e-5
        atol = 3e-5
    return CudaNdarrayType.values_eq_approx(a, b, atol=atol) 

def _handle_license_list(self, lics_set, cls=None):
        """
        Return a license representing a `cls` object (LicenseConjunction
        or LicenseDisjunction) from a list of license resources or None.
        """
        licenses = []
        for _, _, lics_member in self.graph.triples(
            (lics_set, self.spdx_namespace['member'], None)):
            try:
                licenses.append(self.handle_lics(lics_member))
            except CardinalityError:
                self.value_error('LICS_LIST_MEMBER', lics_member)
                break
        if len(licenses) > 1:
            return reduce(lambda a, b: cls(a, b), licenses)
        else:
            self.value_error('PKG_CONC_LIST', '')
            return 

def get_gateway_address(self):
        """
        Perform several set of commands to obtain the gateway address
            * `adb getprop dhcp.wlan0.gateway`
            * `adb shell netcfg | grep wlan0`

        Returns:
            None if no gateway address has been found, otherwise return the gateway address

        """
        ip2int = lambda ip: reduce(lambda a, b: (a << 8) + b, map(int, ip.split('.')), 0)
        int2ip = lambda n: '.'.join([str(n >> (i << 3) & 0xFF) for i in range(0, 4)[::-1]])
        try:
            res = self.shell('getprop dhcp.wlan0.gateway')
        except AdbShellError:
            res = ''
        matcher = IP_PATTERN.search(res)
        if matcher:
            return matcher.group(0)
        ip = self.get_ip_address()
        if not ip:
            return None
        mask_len = self._get_subnet_mask_len()
        gateway = (ip2int(ip) & (((1 << mask_len) - 1) << (32 - mask_len))) + 1
        return int2ip(gateway) 

def osf_crawl(k, *pths, **kw):
    '''
    osf_crawl(k) crawls the osf repository k and returns a lazy nested map structure of the
      repository's files. Folders have values that are maps of their contents while files have
      values that are their download links.
    osf_crawl(k1, k2, ...) is equivalent to osf_crawl(posixpath.join(k1, k2...)).

    The optional named argument base (default: 'osfstorage') may be specified to search in a
    non-standard storage position in the OSF URL; e.g. in the github storage.
    '''
    from six.moves import reduce
    base = kw.pop('base', 'osfstorage')
    root = kw.pop('root', None)
    if len(kw) > 0: raise ValueError('Unknown optional parameters: %s' % (list(kw.keys()),))
    if k.lower().startswith('osf:'): k = k[4:]
    k = k.lstrip('/')
    pths = [p.lstrip('/') for p in (k.split('/') + list(pths))]
    (bpth, pths) = (pths[0].strip('/'), [p for p in pths[1:] if p != ''])
    if root is None: root = _osf_tree(bpth, base=base)
    return reduce(lambda m,k: m[k], pths, root) 

def values_eq_approx_high_tol(a, b):
    """
    This fct is needed to don't have DebugMode raise useless
    error due to ronding error.

    This happen as We reduce on the two last dimensions, so this
    can raise the absolute error if the number of element we
    reduce on is significant.

    """
    assert a.ndim == 4
    atol = None
    if a.shape[-1] * a.shape[-2] > 100:
        # For float32 the default atol is 1e-5
        atol = 3e-5
    return CudaNdarrayType.values_eq_approx(a, b, atol=atol) 

def get_queryset(self):
        if not (self.request.user.is_authenticated and self.request.user.is_staff):
            raise Http404

        queryset = Customer.objects.all()

        if self.q:
            queryset = queryset.annotate(
                first_last_company_name=Concat(
                    F("first_name"), Value(" "), F("last_name"), Value(" "), F("company")
                )
            )
            terms = self.q.split()

            query = reduce(
                operator.and_,
                (Q(first_last_company_name__icontains=term) for term in terms)
            )

            queryset = queryset.filter(query)

        return queryset 

def get_queryset(self):
        if not (self.request.user.is_authenticated and self.request.user.is_staff):
            raise Http404

        queryset = Provider.objects.all()

        if self.q:
            queryset = queryset.annotate(
                name_company=Concat(
                    F("name"), Value(" "), F("company")
                )
            )
            terms = self.q.split()

            query = reduce(
                operator.and_,
                (Q(name_company__icontains=term) for term in terms)
            )

            queryset = queryset.filter(query)

        return queryset 

def get_common_course_modes(course_runs):
    """
    Fake implementation returning common course modes.

    Arguments:
        course_run_ids(Iterable[str]): Target Course run IDs.

    Returns:
        set: course modes found in all given course runs
    """
    course_run_modes = [
        set(seat.get("type") for seat in course_run.get("seats"))
        for course_run in FAKE_COURSE_RUNS_RESPONSE
        if course_run.get("key") in course_runs
    ]

    return six_reduce(lambda left, right: left & right, course_run_modes) 

def crt(ak, nk):
  from six.moves import reduce
  """
  Chinese-Reminders-Theorem Implementation
  using Gauss's proof and generalization on gcd(n1, n2) != 1
  Should be len(ak) == len(nk)
  Original: https://gist.github.com/elliptic-shiho/901d223135965308a5f9ff0cf99dd7c8
  Explanation: http://elliptic-shiho.hatenablog.com/entry/2016/04/03/020117

  Args:
    ak: A Numbers [a1, a2, ..., ak]
    nk: A Modulus [n1, n2, ..., nk]
  """
  assert len(ak) == len(nk)
  N = reduce(lambda x, y: x * y, nk, 1)
  l = lcm(*nk)
  s = 0
  for n, a in zip(nk, ak):
    m = N // n
    g, x, y = egcd(m, n)
    s += (m // g) * x * a
    s %= l
  return s 

def _parse_reduction(self, operation, inames, red_exprs,
            allow_simultaneous=False):
        if isinstance(inames, p.Variable):
            inames = (inames,)

        if not isinstance(inames, (tuple)):
            raise TypeError("iname argument to reduce() must be a symbol "
                    "or a tuple of symbols")

        processed_inames = []
        for iname in inames:
            if not isinstance(iname, p.Variable):
                raise TypeError("iname argument to reduce() must be a symbol "
                        "or a tuple or a tuple of symbols")

            processed_inames.append(iname.name)

        if len(red_exprs) == 1:
            red_exprs = red_exprs[0]

        return Reduction(operation, tuple(processed_inames), red_exprs,
                allow_simultaneous=allow_simultaneous) 

def build_gemm_call(self):

        return reduce(str.__add__, (
            self.declare_NS,
            self.check_xyz_rank2,
            self.setup_z_Nz_Sz,
            self.check_xyz_double_or_float,
            self.check_ab_double_or_float,
            self.check_dims,
            self.check_strides,
            self.encode_strides_in_unit,
            self.compute_strides,
            self.begin_switch_typenum,
            self.case_float,
            self.case_float_ab_constants,
            self.case_float_gemm,
            self.case_double,
            self.case_double_ab_constants,
            self.case_double_gemm,
            self.end_switch_typenum), '') 

def build_gemm_call(self):

        return reduce(str.__add__, (
            self.declare_NS,
            self.check_xyz_rank2,
            self.setup_z_Nz_Sz,
            self.check_xyz_double_or_float,
            self.check_ab_double_or_float,
            self.check_dims,
            self.check_strides,
            self.encode_strides_in_unit,
            self.compute_strides,
            self.begin_switch_typenum,
            self.case_float,
            self.case_float_ab_constants,
            self.case_float_gemm,
            self.case_double,
            self.case_double_ab_constants,
            self.case_double_gemm,
            self.end_switch_typenum), '') 

def _zero_mantissa(dval):
    """Determine whether the mantissa bits of the given double are all
    zero."""
    bb = _double_as_bytes(dval)
    return ((bb[1] & 0x0f) | reduce(operator.or_, bb[2:])) == 0

##
# Functions to test for IEEE 754 special values
## 

def get_realtime_quotes(code_list, open_only=False):
    import tushare as ts

    max_len = 800
    loop_cnt = int(math.ceil(float(len(code_list)) / max_len))

    total_df = reduce(lambda df1, df2: df1.append(df2),
                      [ts.get_realtime_quotes([code for code in code_list[i::loop_cnt]])
                       for i in range(loop_cnt)])
    total_df["is_index"] = False

    index_symbol = ["sh", "sz", "hs300", "sz50", "zxb", "cyb"]
    index_df = ts.get_realtime_quotes(index_symbol)
    index_df["code"] = index_symbol
    index_df["is_index"] = True
    total_df = total_df.append(index_df)
    total_df = total_df.set_index("code").sort_index()

    columns = set(total_df.columns) - set(["name", "time", "date"])
    # columns = filter(lambda x: "_v" not in x, columns)
    for label in columns:
        total_df[label] = total_df[label].map(lambda x: 0 if str(x).strip() == "" else x)
        total_df[label] = total_df[label].astype(float)

    total_df["chg"] = total_df["price"] / total_df["pre_close"] - 1

    total_df["order_book_id"] = total_df.index
    total_df["order_book_id"] = total_df["order_book_id"].apply(tushare_code_2_order_book_id)

    total_df["datetime"] = total_df["date"] + " " + total_df["time"]
    total_df["datetime"] = total_df["datetime"].apply(lambda x: convert_dt_to_int(datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")))

    total_df["close"] = total_df["price"]

    if open_only:
        total_df = total_df[total_df.open > 0]

    return total_df 

def rectVolume(self):
        return reduce(operator.mul, self) 

def mac_aton(s):
    """Convert a Mac address to an integer."""
    try:
        mac = list(map(lambda x: int(x, 16), s.split(':')))
        mac = reduce(lambda a,b: a+b, [mac[i] << (5-i)*8 for i in range(6)])
    except (ValueError, IndexError):
        raise ValueError('illegal Mac: {0}'.format(s))
    return mac 

def ref_reduce(x_data_list, size, division):
    f = reduce(lambda x, y: x + y, np.arange(size)) + size
    results = []
    for x_data in x_data_list:
        result = x_data * f
        if division:
            result /= size
        results.append(result)
    return results 

def _calculate_flags(self):
        return reduce(lambda x, y: x | y, self.flags, 0) 

def cdbhash(s):
    return reduce(lambda h, c: (((h << 5) + h) ^ ord(c)) & 0xffffffff, s, 5381) 

def ref_reduce_scatter(x_data_list, size, division):
    f = reduce(lambda x, y: x + y, np.arange(size)) + size
    results = []
    for x_data in x_data_list:
        result = x_data * f
        if division:
            result /= size
        results.append(result)
    return results 

def ref_all_reduce(x_data_list, size, division):
    f = reduce(lambda x, y: x + y, np.arange(size)) + size
    results = []
    for x_data in x_data_list:
        result = x_data * f
        if division:
            result /= size
        results.append(result)
    return results 

def test_reduce(seed, dst, inplace, division, comm_nccl_opts):
    if comm_nccl_opts is None:
        pytest.skip(
            "Communicator test is disabled. You can turn it on by an option `--test-communicator`.")
    if len(comm_nccl_opts.devices) < 2:
        pytest.skip(
            "Communicator test is disabled. Use more than 1 gpus.")

    comm = comm_nccl_opts.comm
    device_id = int(comm_nccl_opts.device_id)
    n_devices = len(comm_nccl_opts.devices)
    mpi_rank = comm_nccl_opts.mpi_rank

    # Variables
    x_list = []
    x_data_list = []
    num_layers = 20
    rng = np.random.RandomState(seed)
    for l in range(num_layers):
        x_data = rng.rand(3, 4)
        x_data_list.append(x_data)
        x = nn.Variable(x_data.shape)
        x.d = x_data * (device_id + 1)
        x_list.append(x)

    # Reduce
    comm.reduce([x.data for x in x_list], dst,
                division=division, inplace=inplace)

    if mpi_rank == dst:
        # Ref Reduce
        refs = ref_reduce(x_data_list, n_devices, division)

        # Check
        for x, ref in zip(x_list, refs):
            assert_allclose(x.d, ref, rtol=1e-3, atol=1e-6) 

def _flatten_multiindex(m, join=' '):
    if m.nlevels <= 1: return m
    levels = map(m.get_level_values, range(m.nlevels))
    return reduce(lambda x, y: x+join+y, levels, next(levels)) 

def _calculate_flags(self):
        return reduce(lambda x, y: x | y, self.flags, 0) 

def chain_map(*args):
    return reduce(lambda x, y: x.update(y) or x, args, {}) 

def create_table(entries, columns=(), titles=(), csv=None, headers=True):
    table = [
        [
            reduce(
                lambda obj, key: obj.get(key, {}),
                column.split('.'),
                entry
            ) or ''
            for column in columns
        ]
        for entry in entries
    ]
    if headers:
        headers = [titles[i] if i < len(titles) and titles[i] else c for i, c in enumerate(columns)]
    else:
        headers = []
    output = ''
    if csv:
        if headers:
            output += ','.join(headers) + '\n'
        for entry in table:
            output += ','.join(map(str, entry)) + '\n'
    else:
        min_col_width = 3
        col_widths = [max(min_col_width, len(h)+1) for h in (headers or table[0])]
        for e in table:
            col_widths = list(map(max, zip(col_widths, [len(h)+1 for h in e])))

        output += '+-' + '+-'.join(['-' * c for c in col_widths]) + '-+' + '\n'
        if headers:
            output += '| ' + '| '.join(['{: <%d}' % c for c in col_widths]).format(*headers) + ' |' + '\n'

        output += '+-' + '+-'.join(['-' * c for c in col_widths]) + '-+' + '\n'

        for entry in table:
            line = map(str, entry)
            output += '| ' + '| '.join(['{: <%d}' % c for c in col_widths]).format(*line) + ' |' + '\n'

        output += '+-' + '+-'.join(['-' * c for c in col_widths]) + '-+' + '\n'
    return output 

def load_resources_from_files(file_path):
    resources = {}
    # http://code.activestate.com/recipes/577879-create-a-nested-dictionary-from-oswalk/
    rootdir = file_path.rstrip(os.sep)
    log.debug("Trying path {0}".format(rootdir))
    head = rootdir.rsplit(os.sep)[-1]
    start = rootdir.rfind(os.sep) + 1
    for path, dirs, files in os.walk(rootdir):
        log.debug("Trying path {0}".format(path))
        folders = path[start:].split(os.sep)
        subdir = dict.fromkeys(files)
        parent = reduce(dict.get, folders[:-1], resources)
        parent[folders[-1]] = subdir
        for file_name in subdir.keys():
            if file_name[0] == '.':
                log.debug("skipping dot file [%s]", file_name)
            else:
                log.debug("loading [%s]", os.path.join(path, file_name))
            if file_name.rsplit('.')[1] not in ['yaml', 'json']:
                subdir[file_name] = fs_read(os.path.join(path, file_name))
            else:
                # Valid yaml can't have tabs, only spaces
                # proactively replacing tabs as some tools do it wrong
                subdir[file_name] = load(
                    fs_read(os.path.join(
                        path, file_name
                    ))
                )
    return resources[head]