Python Code Examples for normalize

60 Python code examples are found related to "normalize". These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.
Example 1
Project: JapaneseTokenizers   Author: Kensuke-Mitsuzawa   File: text_preprocess.py    License: MIT License 6 votes vote down vote up
def normalize_text(input_text,
                   dictionary_mode='ipadic',
                   new_line_replaced='。',
                   is_replace_eos=True,
                   is_kana=True,
                   is_ascii=True,
                   is_digit=True):
    # type: (text_type,text_type,text_type,bool,bool,bool,bool)->text_type
    """* What you can do
    - It converts input-text into normalized-text which is good for tokenizer input.

    * Params
    - new_line_replaced: a string which replaces from \n string.
    """
    if is_replace_eos:
        without_new_line = input_text.replace('\n', new_line_replaced)
    else:
        without_new_line = new_line_replaced

    if dictionary_mode=='neologd' and is_neologdn_valid:
        return neologdn.normalize(normalize_text_normal_ipadic(without_new_line))
    elif dictionary_mode=='neologd' and is_neologdn_valid == False:
        raise Exception("You could not call neologd dictionary bacause you do NOT install the package neologdn.")
    else:
        return normalize_text_normal_ipadic(without_new_line, kana=is_kana, ascii=is_ascii, digit=is_digit) 
Example 2
Project: AmusingPythonCodes   Author: IsaacChanghau   File: evaluate-v1.1.py    License: MIT License 6 votes vote down vote up
def normalize_answer(s):
    """Lower text and remove punctuation, articles and extra whitespace."""
    def remove_articles(text):
        return re.sub(r'\b(a|an|the)\b', ' ', text)

    def white_space_fix(text):
        return ' '.join(text.split())

    def remove_punc(text):
        exclude = set(string.punctuation)
        return ''.join(ch for ch in text if ch not in exclude)

    def lower(text):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(s)))) 
Example 3
Project: pre-commit   Author: pre-commit   File: parse_shebang.py    License: MIT License 6 votes vote down vote up
def normalize_cmd(cmd: Tuple[str, ...]) -> Tuple[str, ...]:
    """Fixes for the following issues on windows
    - https://bugs.python.org/issue8557
    - windows does not parse shebangs

    This function also makes deep-path shebangs work just fine
    """
    # Use PATH to determine the executable
    exe = normexe(cmd[0])

    # Figure out the shebang from the resulting command
    cmd = parse_filename(exe) + (exe,) + cmd[1:]

    # This could have given us back another bare executable
    exe = normexe(cmd[0])

    return (exe,) + cmd[1:] 
Example 4
Project: bash-lambda-layer   Author: gkrizek   File: normalizers.py    License: MIT License 6 votes vote down vote up
def normalize_host(host):
    """Normalize a host string."""
    if misc.IPv6_MATCHER.match(host):
        percent = host.find('%')
        if percent != -1:
            percent_25 = host.find('%25')

            # Replace RFC 4007 IPv6 Zone ID delimiter '%' with '%25'
            # from RFC 6874. If the host is '[<IPv6 addr>%25]' then we
            # assume RFC 4007 and normalize to '[<IPV6 addr>%2525]'
            if percent_25 == -1 or percent < percent_25 or \
                    (percent == percent_25 and percent_25 == len(host) - 4):
                host = host.replace('%', '%25', 1)

            # Don't normalize the casing of the Zone ID
            return host[:percent].lower() + host[percent:]

    return host.lower() 
Example 5
Project: Safejumper-for-Desktop   Author: proxysh   File: __init__.py    License: GNU General Public License v2.0 6 votes vote down vote up
def normalize_encoding(encoding):

    """ Normalize an encoding name.

        Normalization works as follows: all non-alphanumeric
        characters except the dot used for Python package names are
        collapsed and replaced with a single underscore, e.g. '  -;#'
        becomes '_'. Leading and trailing underscores are removed.

        Note that encoding names should be ASCII only; if they do use
        non-ASCII characters, these must be Latin-1 compatible.

    """
    # Make sure we have an 8-bit string, because .translate() works
    # differently for Unicode strings.
    if hasattr(__builtin__, "unicode") and isinstance(encoding, unicode):
        # Note that .encode('latin-1') does *not* use the codec
        # registry, so this call doesn't recurse. (See unicodeobject.c
        # PyUnicode_AsEncodedString() for details)
        encoding = encoding.encode('latin-1')
    return '_'.join(encoding.translate(_norm_encoding_map).split()) 
Example 6
Project: neologdn   Author: ikegami-yukino   File: normalize_neologd.py    License: Apache License 2.0 6 votes vote down vote up
def normalize_neologd(s):
    s = s.strip()
    s = unicode_normalize('0−9A-Za-z。-゚', s)

    def maketrans(f, t):
        return {ord(x): ord(y) for x, y in zip(f, t)}

    s = s.translate(
        maketrans('!"#$%&\'()*+,-./:;<=>?@[\]^_`{|}~。、・「」',
                  '!”#$%&’()*+,−./:;<=>?@[¥]^_`{|}〜。、・「」'))
    s = re.sub('[˗֊‐‑‒–⁃⁻₋−]+', '-', s)  # normalize hyphens
    s = re.sub('[﹣-ー—―─━ー]+', 'ー', s)  # normalize choonpus
    s = re.sub('[~∼∾〜〰~]', '', s)  # remove tildes
    s = remove_extra_spaces(s)
    s = unicode_normalize('!”#$%&’()*+,−./:;<>?@[¥]^_`{|}〜', s)  # keep =,・,「,」
    return s 
Example 7
Project: mkdocs-pdf-export-plugin   Author: zhaoterryy   File: util.py    License: MIT License 6 votes vote down vote up
def normalize_href(href: str, rel_url: str):
    # foo/bar/baz/../../index.html -> foo/index.html
    def reduce_rel(x):
        try:
            i = x.index('..')
            if i is 0:
                return x

            del x[i]
            del x[i - 1]
            return reduce_rel(x)
        except ValueError:
            return x

    rel_dir = os.path.dirname(rel_url)
    href = str.split(os.path.join(rel_dir, href), '/')
    href = reduce_rel(href)
    href[-1], _ = os.path.splitext(href[-1])

    return os.path.join(*href) 
Example 8
Project: adviser   Author: DigitalPhonetics   File: cleaners.py    License: GNU General Public License v3.0 6 votes vote down vote up
def normalize_numbers(text):
    """
    Normalizes numbers in an utterance as preparation for TTS
    
    text (string): Text to be preprocessed
    """
    text = re.sub(_comma_number_re, _remove_commas, text)
    text = re.sub(_pounds_re, r'\1 pounds', text)
    text = re.sub(_dollars_re, _expand_dollars, text)
    text = re.sub(_decimal_number_re, _expand_decimal_point, text)
    text = re.sub(_ordinal_re, _expand_ordinal, text)
    text = re.sub(_ID_number_re, _expand_ID_number, text)
    text = re.sub(_number_re, _expand_number, text)
    return text


# Regular expression matching whitespace: 
Example 9
Project: cluster-loss-tensorflow   Author: CongWeilin   File: base_provider.py    License: BSD 2-Clause "Simplified" License 6 votes vote down vote up
def normalize_images(self, images, normalization_type):
        """
        Args:
            images: numpy 4D array
            normalization_type: `str`, available choices:
                - divide_255
                - divide_256
                - by_chanels
        """
        if normalization_type == 'divide_255':
            images = images / 255
        elif normalization_type == 'divide_256':
            images = images / 256
        elif normalization_type == 'by_chanels':
            images = images.astype('float64')
            # for every channel in image(assume this is last dimension)
            for i in range(images.shape[-1]):
                images[:, :, :, i] = ((images[:, :, :, i] - self.images_means[i]) /
                                       self.images_stds[i])
        else:
            raise Exception("Unknown type of normalization")
        return images 
Example 10
Project: layered-scene-inference   Author: google   File: homography.py    License: Apache License 2.0 6 votes vote down vote up
def normalize_homogeneous(pts_coords):
  """Converts homogeneous coordinates to regular coordinates.

  Args:
      pts_coords : [...] X n_dims_coords+1; Homogeneous coordinates.
  Returns:
      pts_coords_uv_norm : [...] X n_dims_coords;
          normal coordinates after dividing by the last entry.
  """
  with tf.name_scope('normalize_homogeneous'):
    pts_size = pts_coords.get_shape().as_list()
    n_dims = len(pts_size)
    n_dims_coords = pts_size[-1] - 1

    pts_coords_uv, pts_coords_norm = tf.split(
        pts_coords, [n_dims_coords, 1], axis=n_dims - 1)
    return nn_helpers.divide_safe(pts_coords_uv, pts_coords_norm) 
Example 11
Project: clickbait   Author: bhargaviparanjape   File: utility.py    License: MIT License 6 votes vote down vote up
def normalize_title(tags, words):
	norm = []
	i = 0
	while i < len(tags):
		if tags[i] == '``':
			while i < len(tags) and tags[i] != "''":
				i += 1
			if i < len(tags):
				norm.append("<QUOTE>")
			i += 1
		elif tags[i] == "CD":
			norm.append("<D>")
			i += 1
		else:
			norm.append(words[i])
			i += 1
	return ' '.join(norm) 
Example 12
Project: tw-rental-house-data   Author: g0v   File: export_uniq_house.py    License: MIT License 6 votes vote down vote up
def normalize_val(val, header, use_tf):
    json_val = val

    if 'fn' in header:
        val = header['fn'](val)
        json_val = val

    if type(val) is datetime:
        val = timezone.localtime(val).strftime('%Y-%m-%d %H:%M:%S %Z')
        json_val = val
    elif val is '' or val is None:
        val = '-'
        json_val = None
    elif val is True or val == 'true':
        val = 'T' if use_tf else 1
        json_val = True
    elif val is False or val == 'false':
        val = 'F' if use_tf else 0
        json_val = False

    return val, json_val 
Example 13
Project: armi   Author: terrapower   File: densityTools.py    License: Apache License 2.0 6 votes vote down vote up
def normalizeNuclideList(nuclideVector, normalization=1.0):
    """
    normalize the nuclide vector.

    Parameters
    ----------
    nuclideVector : dict
        dictionary of values -- e.g. floats, ints -- indexed by nuclide identifiers -- e.g. nucNames or nuclideBases

    normalization : float

    Returns
    -------
    nuclideVector : dict
        dictionary of values indexed by nuclide identifiers -- e.g. nucNames or nuclideBases
    """

    normalizationFactor = sum(nuclideVector.values()) / normalization

    for nucName, mFrac in nuclideVector.items():
        nuclideVector[nucName] = mFrac / normalizationFactor

    return nuclideVector 
Example 14
Project: OpenQA   Author: thunlp   File: utils.py    License: MIT License 6 votes vote down vote up
def normalize_answer(s):
    """Lower text and remove punctuation, articles and extra whitespace."""
    def remove_articles(text):
        return re.sub(r'\b(a|an|the)\b', ' ', text)

    def white_space_fix(text):
        return ' '.join(text.split())

    def remove_punc(text):
        exclude = set(string.punctuation)
        return ''.join(ch for ch in text if ch not in exclude)

    def lower(text):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(s)))) 
Example 15
Project: clusterfuzz   Author: google   File: source_mapper.py    License: Apache License 2.0 6 votes vote down vote up
def normalize_source_path(path):
  """Normalizes source path for comparison with component sources."""
  # Account for ../../ at start of path due to working directory
  # out/<build_dir>/ at time of build generation (chromium only).
  path = utils.remove_prefix(path, '../../')

  # Cross-platform way to determine path absoluteness.
  is_path_absolute = path.startswith('/') or DRIVE_LETTER_REGEX.match(path)

  # Normalize backslashes into slashes.
  normalized_path = path.replace('\\', '/')

  if is_path_absolute:
    source_start_id_index = normalized_path.find(SOURCE_START_ID)
    if source_start_id_index == -1:
      # This absolute path does not have source start id, so we cannot
      # figure out a relative path. Bail out.
      return None

    return normalized_path[source_start_id_index + len(SOURCE_START_ID):]

  return normalized_path 
Example 16
Project: codetransformer   Author: llllllllll   File: _343.py    License: GNU General Public License v2.0 6 votes vote down vote up
def normalize_tuple_slice(node):
    """
    Normalize an ast.Tuple node representing the internals of a slice.

    Returns the node wrapped in an ast.Index.
    Returns an ExtSlice node built from the tuple elements if there are any
    slices.
    """
    if not any(isinstance(elt, ast.Slice) for elt in node.elts):
        return ast.Index(value=node)

    return ast.ExtSlice(
        [
            # Wrap non-Slice nodes in Index nodes.
            elt if isinstance(elt, ast.Slice) else ast.Index(value=elt)
            for elt in node.elts
        ]
    ) 
Example 17
Project: bigbang   Author: datactive   File: mailman.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def normalize_archives_url(url):
    """
    Normalize url.

    will try to infer, find or guess the most useful archives URL, given a URL.

    Return normalized URL, or the original URL if no improvement is found.
    """
    # change new IETF mailarchive URLs to older, still available text .mail archives
    new_ietf_exp = re.compile('https://mailarchive\\.ietf\\.org/arch/search/'
                              '\\?email_list=(?P<list_name>[\\w-]+)')
    ietf_text_archives = r'https://www.ietf.org/mail-archive/text/\g<list_name>/'
    new_ietf_browse_exp = re.compile(r'https://mailarchive.ietf.org/arch/browse/(?P<list_name>[\w-]+)/?')

    match = new_ietf_exp.match(url)
    if match:
        return re.sub(new_ietf_exp, ietf_text_archives, url)
    
    match = new_ietf_browse_exp.match(url)
    if match:
        return re.sub(new_ietf_browse_exp, ietf_text_archives, url)

    return url # if no other change found, return the original URL 
Example 18
Project: ivre   Author: cea-sec   File: utils.py    License: GNU General Public License v3.0 6 votes vote down vote up
def normalize_props(props, braces=True):
    """Returns a normalized property list/dict so that (roughly):
        - a list gives {k: "{k}"} if braces=True, {k: "k"} otherwise
        - a dict gives {k: v if v is not None else "{%s}" % v} if braces=True,
                       {k: v if v is not Node else "%s" % v} otherwise
    """
    if not isinstance(props, dict):
        props = dict.fromkeys(props)
    # Remove braces if necessary
    if not braces:
        for key, value in viewitems(props):
            if (isinstance(value, basestring) and value.startswith('{') and
                    value.endswith('}')):
                props[key] = value[1:-1]
    form = "{%s}" if braces else "%s"
    props = dict(
        (key, (value if isinstance(value, basestring) else
               (form % key) if value is None else
               str(value))) for key, value in viewitems(props)
    )
    return props 
Example 19
Project: Deep-Learning-By-Example   Author: PacktPublishing   File: cifar10_objectDetection.py    License: MIT License 6 votes vote down vote up
def normalize_images(images):
    # initial zero ndarray
    normalized_images = np.zeros_like(images.astype(float))

    # The first images index is number of images where the other indices indicates
    # hieight, width and depth of the image
    num_images = images.shape[0]

    # Computing the minimum and maximum value of the input image to do the normalization based on them
    maximum_value, minimum_value = images.max(), images.min()

    # Normalize all the pixel values of the images to be from 0 to 1
    for img in range(num_images):
        normalized_images[img, ...] = (images[img, ...] - float(minimum_value)) / float(maximum_value - minimum_value)

    return normalized_images


# encoding the input images. Each image will be represented by a vector of zeros except for the class index of the image
# that this vector represents. The length of this vector depends on number of classes that we have
# the dataset which is 10 in CIFAR-10 
Example 20
Project: fontParts   Author: robotools   File: normalizers.py    License: MIT License 6 votes vote down vote up
def normalizeRotationAngle(value):
    """
    Normalizes an angle.

    * Value must be a :ref:`type-int-float`.
    * Value must be between -360 and 360.
    * If the value is negative, it is normalized by adding it to 360
    * Returned value is a ``float`` between 0 and 360.
    """
    if not isinstance(value, (int, float)):
        raise TypeError("Angle must be instances of "
                        ":ref:`type-int-float`, not %s."
                        % type(value).__name__)
    if abs(value) > 360:
        raise ValueError("Angle must be between -360 and 360.")
    if value < 0:
        value = value + 360
    return float(value)


# Color 
Example 21
Project: context_attentive_ir   Author: wasiahmad   File: squad_eval.py    License: MIT License 6 votes vote down vote up
def normalize_answer(s):
    """Lower text and remove punctuation, articles and extra whitespace."""

    def remove_articles(text):
        return re.sub(r'\b(a|an|the)\b', ' ', text)

    def white_space_fix(text):
        return ' '.join(text.split())

    def remove_punc(text):
        exclude = set(string.punctuation)
        return ''.join(ch for ch in text if ch not in exclude)

    def lower(text):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(s)))) 
Example 22
Project: unity-python   Author: exodrifter   File: __init__.py    License: MIT License 6 votes vote down vote up
def normalize_encoding(encoding):

    """ Normalize an encoding name.

        Normalization works as follows: all non-alphanumeric
        characters except the dot used for Python package names are
        collapsed and replaced with a single underscore, e.g. '  -;#'
        becomes '_'. Leading and trailing underscores are removed.

        Note that encoding names should be ASCII only; if they do use
        non-ASCII characters, these must be Latin-1 compatible.

    """
    # Make sure we have an 8-bit string, because .translate() works
    # differently for Unicode strings.
    if hasattr(__builtin__, "unicode") and isinstance(encoding, unicode):
        # Note that .encode('latin-1') does *not* use the codec
        # registry, so this call doesn't recurse. (See unicodeobject.c
        # PyUnicode_AsEncodedString() for details)
        encoding = encoding.encode('latin-1')
    return '_'.join(encoding.translate(_norm_encoding_map).split()) 
Example 23
Project: oss-ftp   Author: aliyun   File: patchcheck.py    License: MIT License 6 votes vote down vote up
def normalize_docs_whitespace(file_paths):
    fixed = []
    for path in file_paths:
        abspath = os.path.join(SRCDIR, path)
        try:
            with open(abspath, 'rb') as f:
                lines = f.readlines()
            new_lines = [ws_re.sub(br'\1', line) for line in lines]
            if new_lines != lines:
                shutil.copyfile(abspath, abspath + '.bak')
                with open(abspath, 'wb') as f:
                    f.writelines(new_lines)
                fixed.append(path)
        except Exception as err:
            print 'Cannot fix %s: %s' % (path, err)
    return fixed 
Example 24
Project: CatLearn   Author: SUNCAT-Center   File: scaling.py    License: GNU General Public License v3.0 6 votes vote down vote up
def target_normalize(target):
    """Return a list of normalized target values.

    Parameters
    ----------
    target : list
        A list of the target values.
    """
    target = np.asarray(target)

    data = defaultdict(list)
    data['mean'] = np.mean(target, axis=0)
    data['dif'] = np.max(target, axis=0) - np.min(target, axis=0)
    data['target'] = (target - data['mean']) / data['dif']

    return data 
Example 25
Project: SDNet   Author: microsoft   File: CoQAPreprocess.py    License: MIT License 6 votes vote down vote up
def normalize_answer(self, s):
        """Lower text and remove punctuation, storys and extra whitespace."""

        def remove_articles(text):
            regex = re.compile(r'\b(a|an|the)\b', re.UNICODE)
            return re.sub(regex, ' ', text)

        def white_space_fix(text):
            return ' '.join(text.split())

        def remove_punc(text):
            exclude = set(string.punctuation)
            return ''.join(ch for ch in text if ch not in exclude)

        def lower(text):
            return text.lower()

        return white_space_fix(remove_articles(remove_punc(lower(s))))


    # find the word id start and stop 
Example 26
Project: BiDAF   Author: jojonki   File: evaluate-v1.1.py    License: Apache License 2.0 6 votes vote down vote up
def normalize_answer(s):
    """Lower text and remove punctuation, articles and extra whitespace."""
    def remove_articles(text):
        return re.sub(r'\b(a|an|the)\b', ' ', text)

    def white_space_fix(text):
        return ' '.join(text.split())

    def remove_punc(text):
        exclude = set(string.punctuation)
        return ''.join(ch for ch in text if ch not in exclude)

    def lower(text):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(s)))) 
Example 27
Project: xlnet_extension_tf   Author: stevezheng23   File: eval_coqa.py    License: Apache License 2.0 6 votes vote down vote up
def normalize_answer(s):
        """Lower text and remove punctuation, storys and extra whitespace."""

        def remove_articles(text):
            regex = re.compile(r'\b(a|an|the)\b', re.UNICODE)
            return re.sub(regex, ' ', text)

        def white_space_fix(text):
            return ' '.join(text.split())

        def remove_punc(text):
            exclude = set(string.punctuation)
            return ''.join(ch for ch in text if ch not in exclude)

        def lower(text):
            return text.lower()

        return white_space_fix(remove_articles(remove_punc(lower(s)))) 
Example 28
Project: fontParts   Author: robotools   File: normalizers.py    License: MIT License 6 votes vote down vote up
def normalizeGlyphFormatVersion(value):
    """
    Normalizes glyph format version for saving to XML string.

    * **value** must be a :ref:`type-int-float` of either 1 or 2.
    * Returned value will be an int.
    """
    if not isinstance(value, (int, float)):
        raise TypeError("Glyph Format Version must be an "
                        ":ref:`type-int-float`, not %s."
                        % type(value).__name__)
    value = int(value)
    if value not in (1, 2):
        raise ValueError("Glyph Format Version must be either 1 or 2, not %s."
                         % value)
    return value

# -------
# Contour
# ------- 
Example 29
Project: swmmio   Author: aerispaha   File: sectionheaders.py    License: MIT License 6 votes vote down vote up
def normalize_inp_config(inp_obects):
    """
    Unpack the config details for each inp section and organize in a standard format.
    This allows the YAML file to be more short hand and human readable.
    :param inp_obects:
    :return:
    >>> from swmmio.defs import INP_OBJECTS
    >>> conf = normalize_inp_config(INP_OBJECTS)
    >>> print(conf['JUNCTIONS'])
    >>> print(conf)
    OrderedDict([('columns', ['Name', 'InvertElev', 'MaxDepth', 'InitDepth', 'SurchargeDepth', 'PondedArea'])])
    """
    normalized = OrderedDict()
    for sect, raw_conf in inp_obects.items():
        conf = parse_inp_section_config(raw_conf)
        normalized[sect] = conf

    return normalized 
Example 30
Project: ZOOpt   Author: eyounx   File: sparse_mse.py    License: MIT License 6 votes vote down vote up
def normalize_data(data_matrix):
        """
        Normalize data to have mean 0 and variance 1 for each column

        :param data_matrix: matrix of all data
        :return: normalized data
        """
        try:
            mat_size = np.shape(data_matrix)
            for i in range(0, mat_size[1]):
                the_column = data_matrix[:, i]
                column_mean = sum(the_column)/mat_size[0]
                minus_column = np.mat(the_column-column_mean)
                std = np.sqrt(np.transpose(minus_column)*minus_column/mat_size[0])
                data_matrix[:, i] = (the_column-column_mean)/std
            return data_matrix
        except Exception as e:
            print(e)
        finally:
            pass 
Example 31
Project: drf-reverse-proxy   Author: danpoland   File: utilites.py    License: Mozilla Public License 2.0 6 votes vote down vote up
def normalize_request_headers(request):
    """
    Function used to transform headers, replacing 'HTTP\_' to '' and replace '_' to '-'.

    :param request:  A HttpRequest that will be transformed
    :returns:        A dictionary with the normalized headers
    """

    norm_headers = {}

    for header, value in request.META.items():
        if required_header(header):
            norm_header = header.replace('HTTP_', '').title().replace('_', '-')
            norm_headers[norm_header] = value

    return norm_headers 
Example 32
Project: justcopy-backend   Author: ailabstw   File: utils.py    License: MIT License 6 votes vote down vote up
def normalize_answer(s):
    """Lower text and remove punctuation, articles and extra whitespace."""
    def remove_articles(text):
        return re.sub(r'\b(a|an|the)\b', ' ', text)

    def white_space_fix(text):
        return ' '.join(text.split())

    def remove_punc(text):
        exclude = set(string.punctuation)
        return ''.join(ch for ch in text if ch not in exclude)

    def lower(text):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(s)))) 
Example 33
Project: GEOMetrics   Author: EdwardSmith1884   File: utils.py    License: MIT License 6 votes vote down vote up
def normalize_adj(mx):
	rowsum = np.array(mx.sum(1))
	r_inv = np.power(rowsum, -1).flatten()
	r_inv[np.isinf(r_inv)] = 0.
	r_mat_inv = sp.diags(r_inv)

	mx = r_mat_inv.dot(mx)
	return mx

# calcualtes the a list of information for each face 
# the mesh is triangulated so each face is connected to 3 faces or neighbors
# we label the 0,1,2, and everyface poses a list of information about each neighbor 
# this information is: 
	#[ the index of the nighbor in the full list of faces (adj_info['faces']), 
	# what label this neighbor has given the face , 
	# its own face number in the list of faces ]
# face i stores info on the neighbor labelled j in face_list[i,j]
# only active faces are stored in the face_list 
Example 34
Project: bash-lambda-layer   Author: gkrizek   File: __init__.py    License: MIT License 6 votes vote down vote up
def normalize_language_tag(tag):
    """Return a list of normalized combinations for a `BCP 47` language tag.

    Example:

    >>> from docutils.utils import normalize_language_tag
    >>> normalize_language_tag('de_AT-1901')
    ['de-at-1901', 'de-at', 'de-1901', 'de']
    >>> normalize_language_tag('de-CH-x_altquot')
    ['de-ch-x-altquot', 'de-ch', 'de-x-altquot', 'de']

    """
    # normalize:
    tag = tag.lower().replace('-','_')
    # split (except singletons, which mark the following tag as non-standard):
    tag = re.sub(r'_([a-zA-Z0-9])_', r'_\1-', tag)
    subtags = [subtag for subtag in tag.split('_')]
    base_tag = [subtags.pop(0)]
    # find all combinations of subtags
    taglist = []
    for n in range(len(subtags), 0, -1):
        for tags in unique_combinations(subtags, n):
            taglist.append('-'.join(base_tag+tags))
    taglist += base_tag
    return taglist 
Example 35
Project: pywsd   Author: alvations   File: utils.py    License: MIT License 6 votes vote down vote up
def normalize_array(a,direction="column"):
    """
    Normalizes an array to sum to one, either column wise, 
    or row wise or the full array.
    
    *Directions*
        Column-wise - 0 default
        Row-wise - 1 default
        All - 2 default
    """
    b = a.copy()
    if(direction == "column"):
        sums = np.sum(b,0)
        return np.nan_to_num(b/sums)
    elif(direction == "row"):
        sums =np.sum(b,1)
        return  np.nan_to_num((b.transpose() / sums).transpose())
    elif(direction == "all"):
        sums = np.sum(b)
        return np.nan_to_num(b / sums)
    else:
        print "Error non existing normalization"
        return b 
Example 36
Project: fontParts   Author: robotools   File: normalizers.py    License: MIT License 6 votes vote down vote up
def normalizeFeatureText(value):
    """
    Normalizes feature text.

    * **value** must be a :ref:`type-string`.
    * Returned value will be an unencoded ``unicode`` string.
    """
    if not isinstance(value, str):
        raise TypeError("Feature text must be a string, not %s."
                        % type(value).__name__)
    return value


# ---
# Lib
# --- 
Example 37
Project: Blender-Metaverse-Addon   Author: Menithal   File: pose_helper.py    License: GNU General Public License v3.0 6 votes vote down vote up
def normalize_influence_for(constraints, of_type, amount=1.0):
    min_value = -1
    max_value = -1
    total_value = 0
    filtered_constraints = []
    for constraint in constraints:
        if constraint.type == of_type:
            print("Found")
            total_value += constraint.influence
            filtered_constraints.append(constraint)
    
    if len(filtered_constraints) <= 1:
        print("Not sufficient constraints to normalize")
        return False

    for constraint in filtered_constraints:
        constraint.influence = ((constraint.influence) / total_value) * amount 
Example 38
Project: chemprop   Author: wengong-jin   File: data.py    License: MIT License 6 votes vote down vote up
def normalize_features(self, scaler: StandardScaler = None, replace_nan_token: int = 0) -> StandardScaler:
        if len(self.data) == 0 or self.data[0].features is None:
            return None

        if scaler is not None:
            self.scaler = scaler

        elif self.scaler is None:
            features = np.vstack([d.features for d in self.data])
            self.scaler = StandardScaler(replace_nan_token=replace_nan_token)
            self.scaler.fit(features)

        for d in self.data:
            d.set_features(self.scaler.transform(d.features.reshape(1, -1))[0])

        return self.scaler 
Example 39
Project: semanticRetrievalMRS   Author: easonnie   File: squad_eval_v1.py    License: MIT License 6 votes vote down vote up
def normalize_answer(s):
    """Lower text and remove punctuation, articles and extra whitespace."""

    def remove_articles(text):
        regex = re.compile(r'\b(a|an|the)\b', re.UNICODE)
        return re.sub(regex, ' ', text)

    def white_space_fix(text):
        return ' '.join(text.split())

    def remove_punc(text):
        exclude = set(string.punctuation)
        return ''.join(ch for ch in text if ch not in exclude)

    def lower(text):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(s)))) 
Example 40
Project: openstax-cms   Author: openstax   File: search.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def normalize_query(query_string, findterms=re.compile(r'"([^"]+)"|(\S+)').findall, normspace=re.compile(r'\s{2,}').sub):
    """
    Splits the query string in individual keywords, getting rid of unnecessary spaces
    and grouping quoted words together.
    
    Example Input:
    normalize_query('  some random  words "with   quotes  " and   spaces')
    
    Response:
    ['some', 'random', 'words', 'with quotes', 'and', 'spaces']
    """

    # Remove common stopwords from the search query
    stopwords = ['of', 'is', 'a', 'at', 'is', 'the']
    querywords = query_string.split()
    resultwords = [word for word in querywords if word.lower() not in stopwords]
    result = ' '.join(resultwords)

    return [normspace(' ', (t[0] or t[1]).strip()) for t in findterms(result)] 
Example 41
Project: senlin   Author: openstack   File: base.py    License: Apache License 2.0 6 votes vote down vote up
def normalize_req(cls, name, req, key=None):
        result = {
            cls.OBJ_SERIAL_NAMESPACE + '.version': cls.VERSION,
            cls.OBJ_SERIAL_NAMESPACE + '.namespace': cls.OBJ_PROJECT_NAMESPACE,
            cls.OBJ_SERIAL_NAMESPACE + '.name': name,
        }
        if key is not None:
            if key not in req:
                raise ValueError(_("Request body missing '%s' key.") % key)

            result[cls.OBJ_SERIAL_NAMESPACE + '.data'] = {
                key: {
                    cls.OBJ_SERIAL_NAMESPACE + '.version': cls.VERSION,
                    cls.OBJ_SERIAL_NAMESPACE + '.namespace':
                        cls.OBJ_PROJECT_NAMESPACE,
                    cls.OBJ_SERIAL_NAMESPACE + '.name': name + 'Body',
                    cls.OBJ_SERIAL_NAMESPACE + '.data': req[key]
                }
            }
        else:
            result[cls.OBJ_SERIAL_NAMESPACE + '.data'] = req

        return result 
Example 42
Project: fontParts   Author: robotools   File: normalizers.py    License: MIT License 6 votes vote down vote up
def normalizeGlyphUnicodes(value):
    """
    Normalizes glyph unicodes.

    * **value** must be a ``list``.
    * **value** items must normalize as glyph unicodes with
      :func:`normalizeGlyphUnicode`.
    * **value** must not repeat unicode values.
    * Returned value will be a ``tuple`` of ints.
    """
    if not isinstance(value, (tuple, list)):
        raise TypeError("Glyph unicodes must be a list, not %s."
                        % type(value).__name__)
    values = [normalizeGlyphUnicode(v) for v in value]
    duplicates = [v for v, count in Counter(value).items() if count > 1]
    if len(duplicates) != 0:
        raise ValueError("Duplicate unicode values are not allowed.")
    return tuple(values) 
Example 43
Project: deepQuest   Author: sheffieldnlp   File: theano_backend.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def normalize_batch_in_training(x, gamma, beta,
                                reduction_axes, epsilon=1e-3):
    """Computes mean and std for batch then apply batch_normalization on batch.
    """
    # TODO remove this if statement when Theano without
    # T.nnet.bn.batch_normalization_train is deprecated
    if not hasattr(T.nnet.bn, 'batch_normalization_train'):
        return _old_normalize_batch_in_training(x, gamma, beta, reduction_axes, epsilon)

    if gamma is None:
        if beta is None:
            gamma = ones_like(x)
        else:
            gamma = ones_like(beta)
    if beta is None:
        if gamma is None:
            beta = zeros_like(x)
        beta = zeros_like(gamma)

    normed, mean, stdinv = T.nnet.bn.batch_normalization_train(
        x, gamma, beta, reduction_axes, epsilon)

    return normed, mean, T.inv(stdinv ** 2) 
Example 44
Project: pyKriging   Author: capaulson   File: regressionkrige.py    License: MIT License 6 votes vote down vote up
def normalizeData(self):
        '''
        This function is called when the initial data in the model is set.
        We find the max and min of each dimension and norm that axis to a range of [0,1]
        '''
        for i in range(self.k):
            self.normRange.append([min(self.X[:, i]), max(self.X[:, i])])

        for i in range(self.n):
            self.X[i] = self.normX(self.X[i])

        self.ynormRange.append(min(self.y))
        self.ynormRange.append(max(self.y))

        for i in range(self.n):
            self.y[i] = self.normy(self.y[i]) 
Example 45
Project: pipenv   Author: pypa   File: path.py    License: MIT License 6 votes vote down vote up
def normalize_path(path):
    # type: (TPath) -> Text
    """Return a case-normalized absolute variable-expanded path.

    :param str path: The non-normalized path
    :return: A normalized, expanded, case-normalized path
    :rtype: str
    """

    path = os.path.abspath(os.path.expandvars(os.path.expanduser(str(path))))
    if os.name == "nt" and os.path.exists(path):
        from ._winconsole import get_long_path

        path = get_long_path(path)

    return os.path.normpath(os.path.normcase(path)) 
Example 46
Project: CityEnergyAnalyst   Author: architecture-building-systems   File: __init__.py    License: MIT License 6 votes vote down vote up
def normalize_data_costs(self, data_processed, normalization, analysis_fields):
        if normalization == "gross floor area":
            data = pd.read_csv(self.locator.get_total_demand())
            normalization_factor = sum(data['GFA_m2'])
            data_processed = data_processed.apply(
                lambda x: x / normalization_factor if x.name in analysis_fields else x)
        elif normalization == "net floor area":
            data = pd.read_csv(self.locator.get_total_demand())
            normalization_factor = sum(data['Aocc_m2'])
            data_processed = data_processed.apply(
                lambda x: x / normalization_factor if x.name in analysis_fields else x)
        elif normalization == "air conditioned floor area":
            data = pd.read_csv(self.locator.get_total_demand())
            normalization_factor = sum(data['Af_m2'])
            data_processed = data_processed.apply(
                lambda x: x / normalization_factor if x.name in analysis_fields else x)
        elif normalization == "building occupancy":
            data = pd.read_csv(self.locator.get_total_demand())
            normalization_factor = sum(data['people0'])
            data_processed = data_processed.apply(
                lambda x: x / normalization_factor if x.name in analysis_fields else x)
        return data_processed 
Example 47
Project: cltk   Author: cltk   File: araby.py    License: MIT License 6 votes vote down vote up
def normalize_ligature(text):
    """Normalize Lam Alef ligatures into two letters (LAM and ALEF),
    and Tand return a result text.
    Some systems present lamAlef ligature as a single letter,
    this function convert it into two letters,
    The converted letters into  LAM and ALEF are :
        - LAM_ALEF,  LAM_ALEF_HAMZA_ABOVE,  LAM_ALEF_HAMZA_BELOW,  LAM_ALEF_MADDA_ABOVE

    @param text: arabic text.
    @type text: unicode.
    @return: return a converted text.
    @rtype: unicode.
    """
    if text:
        return LIGUATURES_PATTERN.sub('%s%s' % (LAM, ALEF), text)
    return text 
Example 48
Project: pytomo3d   Author: computational-seismology   File: window_weights.py    License: GNU Lesser General Public License v3.0 6 votes vote down vote up
def normalize_receiver_weights(points, wcounts):
    """
    Normalize the receiver weights

    :param points:
    :param rec_wcounts:
    :return:
    """
    wsum = 0
    nwins_total = 0
    for point in points:
        nwin = wcounts[point.tag]
        nwins_total += nwin
        wsum += point.weight * nwin
    norm_factor = nwins_total / wsum

    weights = {}
    for point in points:
        weights[point.tag] = point.weight * norm_factor

    return weights 
Example 49
Project: edr   Author: lekeno   File: edvehicles.py    License: Apache License 2.0 6 votes vote down vote up
def normalize_module_name(name):
        normalized = name.lower()
        
        # suffix _name or _name; is not used in loadout or afmurepair events 
        if normalized.endswith(u"_name"):
            useless_suffix_length = len(u"_name")
            normalized = normalized[:-useless_suffix_length]
        elif normalized.endswith(u"_name;"):
            useless_suffix_length = len(u"_name;")
            normalized = normalized[:-useless_suffix_length]

        if normalized.startswith(u"$"):
            normalized = normalized[1:]

        # just get rid of prefixes because sometimes int_ becomes ext_ depending on the event
        if normalized.startswith((u"int_", u"ext_", u"hpt_")):
            normalized = normalized[4:]
        return normalized 
Example 50
Project: bcbio-nextgen-vm   Author: bcbio   File: mounts.py    License: MIT License 6 votes vote down vote up
def normalize_config(config, fcdir=None):
    """Normalize sample configuration file to have absolute paths and collect directories.

    Prepares configuration for remapping directories into docker containers.
    """
    absdetails = []
    directories = []
    ignore = ["variantcaller", "realign", "recalibrate", "phasing", "svcaller"]
    for d in config["details"]:
        d = abs_file_paths(d, base_dirs=[fcdir] if fcdir else None,
                           ignore=["description", "analysis", "resources",
                                   "genome_build", "lane"])
        d["algorithm"] = abs_file_paths(d["algorithm"], base_dirs=[fcdir] if fcdir else None,
                                        ignore=ignore)
        absdetails.append(d)
        directories.extend(_get_directories(d, ignore))
    if config.get("upload", {}).get("dir"):
        config["upload"]["dir"] = os.path.normpath(os.path.realpath(
            os.path.join(os.getcwd(), config["upload"]["dir"])))
        if not os.path.exists(config["upload"]["dir"]):
            os.makedirs(config["upload"]["dir"])
    config["details"] = absdetails
    return config, directories 
Example 51
Project: wordinserter   Author: orf   File: list_elements.py    License: MIT License 6 votes vote down vote up
def normalize_list(op: BaseList):
    # If there are > 1 lists to move out then we need to insert it after previously moved ones,
    # instead of before. `moved` tracks this.
    children = list(op)

    for child in children:
        if isinstance(child, ListElement):
            moved = 0
            for element_child in child:
                if isinstance(element_child, BaseList):
                    moved += 1
                    # Move the list outside of the ListElement
                    child_index = op.child_index(child)
                    op.insert_child(child_index + moved, element_child)
                    child.remove_child(element_child)
                    normalize_list(element_child) 
Example 52
Project: holoviews   Author: holoviz   File: generate_modules.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def normalize_excludes(rootpath, excludes):
    """
    Normalize the excluded directory list:
    * must be either an absolute path or start with rootpath,
    * otherwise it is joined with rootpath
    * with trailing slash
    """
    sep = os.path.sep
    f_excludes = []
    for exclude in excludes:
        if not os.path.isabs(exclude) and not exclude.startswith(rootpath):
            exclude = os.path.join(rootpath, exclude)
        if not exclude.endswith(sep):
            exclude += sep
        f_excludes.append(exclude)
    return f_excludes 
Example 53
Project: cltk   Author: cltk   File: araby.py    License: MIT License 6 votes vote down vote up
def normalize_hamza(word):
    """Standardize the Hamzat into one form of hamza,
    replace Madda by hamza and alef.
    Replace the LamAlefs by simplified letters.

    @param word: arabic text.
    @type word: unicode.
    @return: return a converted text.
    @rtype: unicode.
    """
    if word.startswith(ALEF_MADDA):
        if len(word) >= 3 and (word[1] not in HARAKAT) and \
                (word[2] == SHADDA or len(word) == 3):
            word = HAMZA + ALEF + word[1:]
        else:
            word = HAMZA + HAMZA + word[1:]
    # convert all Hamza from into one form
    word = word.replace(ALEF_MADDA, HAMZA + HAMZA)
    word = HAMZAT_PATTERN.sub(HAMZA, word)
    return word 
Example 54
def normalize_pos_df(pos_df, dataset_kind):
        image_size = np.array(get_image_size(dataset_kind))

        xy = np.array(pos_df[["x", "y"]])
        # originally (0, 0) is the center of the frame,
        # therefore move (0, 0) to top-left
        xy += image_size / 2
        # clipping
        xy[:, 0] = np.clip(xy[:, 0], 0.0, image_size[0] - 1)
        xy[:, 1] = np.clip(xy[:, 1], 0.0, image_size[1] - 1)

        # normalize
        xy /= image_size

        # normalize position (x, y) respectively
        pos_df_norm = pd.DataFrame({
            "frame": pos_df["frame"],
            "id": pos_df["id"],
            "x": xy[:, 0],
            "y": xy[:, 1]
        })
        return pos_df_norm 
Example 55
Project: coa_tools   Author: ndee85   File: export_dragonbones.py    License: GNU General Public License v3.0 6 votes vote down vote up
def normalize_weights(obj, armature, threshold):
    for vert in obj.data.vertices:
        weight_total = 0.0
        groups = []
        for group in vert.groups:
            vgroup = obj.vertex_groups[group.group]
            group_name = vgroup.name
            if (group.weight > threshold and armature == None) or (group.weight > threshold and armature != None and group_name in armature.data.bones):
                groups.append(group)

        for group in groups:
            vgroup = obj.vertex_groups[group.group]
            weight_total += group.weight
        for group in groups:
            group.weight = 1*(group.weight/weight_total)

### get bone data 
Example 56
Project: aws-extender   Author: VirtueSecurity   File: __init__.py    License: MIT License 6 votes vote down vote up
def normalize_language_tag(tag):
    """Return a list of normalized combinations for a `BCP 47` language tag.

    Example:

    >>> from docutils.utils import normalize_language_tag
    >>> normalize_language_tag('de_AT-1901')
    ['de-at-1901', 'de-at', 'de-1901', 'de']
    >>> normalize_language_tag('de-CH-x_altquot')
    ['de-ch-x-altquot', 'de-ch', 'de-x-altquot', 'de']

    """
    # normalize:
    tag = tag.lower().replace('-','_')
    # split (except singletons, which mark the following tag as non-standard):
    tag = re.sub(r'_([a-zA-Z0-9])_', r'_\1-', tag)
    subtags = [subtag for subtag in tag.split('_')]
    base_tag = [subtags.pop(0)]
    # find all combinations of subtags
    taglist = []
    for n in range(len(subtags), 0, -1):
        for tags in unique_combinations(subtags, n):
            taglist.append('-'.join(base_tag+tags))
    taglist += base_tag
    return taglist 
Example 57
Project: Carnets   Author: holzschu   File: patchcheck.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def normalize_docs_whitespace(file_paths):
    fixed = []
    for path in file_paths:
        abspath = os.path.join(SRCDIR, path)
        try:
            with open(abspath, 'rb') as f:
                lines = f.readlines()
            new_lines = [ws_re.sub(br'\1', line) for line in lines]
            if new_lines != lines:
                shutil.copyfile(abspath, abspath + '.bak')
                with open(abspath, 'wb') as f:
                    f.writelines(new_lines)
                fixed.append(path)
        except Exception as err:
            print 'Cannot fix %s: %s' % (path, err)
    return fixed 
Example 58
Project: telegram-robot-rss   Author: cbrgm   File: __init__.py    License: Mozilla Public License 2.0 6 votes vote down vote up
def normalize_encoding(encoding):

    """ Normalize an encoding name.

        Normalization works as follows: all non-alphanumeric
        characters except the dot used for Python package names are
        collapsed and replaced with a single underscore, e.g. '  -;#'
        becomes '_'. Leading and trailing underscores are removed.

        Note that encoding names should be ASCII only; if they do use
        non-ASCII characters, these must be Latin-1 compatible.

    """
    # Make sure we have an 8-bit string, because .translate() works
    # differently for Unicode strings.
    if hasattr(__builtin__, "unicode") and isinstance(encoding, unicode):
        # Note that .encode('latin-1') does *not* use the codec
        # registry, so this call doesn't recurse. (See unicodeobject.c
        # PyUnicode_AsEncodedString() for details)
        encoding = encoding.encode('latin-1')
    return '_'.join(encoding.translate(_norm_encoding_map).split()) 
Example 59
Project: text-gcn-chainer   Author: koreyou   File: graphs.py    License: Creative Commons Zero v1.0 Universal 6 votes vote down vote up
def normalize_pygcn(a):
    """ normalize adjacency matrix with normalization-trick. This variant
    is proposed in https://github.com/tkipf/pygcn .
    Refer https://github.com/tkipf/pygcn/issues/11 for the author's comment.

    Arguments:
        a (scipy.sparse.coo_matrix): Unnormalied adjacency matrix

    Returns:
        scipy.sparse.coo_matrix: Normalized adjacency matrix
    """
    # no need to add identity matrix because self connection has already been added
    # a += sp.eye(a.shape[0])
    rowsum = np.array(a.sum(1))
    rowsum_inv = np.power(rowsum, -1).flatten()
    rowsum_inv[np.isinf(rowsum_inv)] = 0.
    # ~D in the GCN paper
    d_tilde = sp.diags(rowsum_inv)
    return d_tilde.dot(a) 
Example 60
Project: fontParts   Author: robotools   File: normalizers.py    License: MIT License 6 votes vote down vote up
def normalizeGlyphUnicode(value):
    """
    Normalizes glyph unicode.

    * **value** must be an int or hex (represented as a string).
    * **value** must be in a unicode range.
    * Returned value will be an ``int``.
    """
    if not isinstance(value, (int, str)) or isinstance(value, bool):
        raise TypeError("Glyph unicode must be a int or hex string, not %s."
                        % type(value).__name__)
    if isinstance(value, str):
        try:
            value = int(value, 16)
        except ValueError:
            raise ValueError("Glyph unicode hex must be a valid hex string.")
    if value < 0 or value > 1114111:
        raise ValueError("Glyph unicode must be in the Unicode range.")
    return value