Python itertools.ifilter() Examples

The following are code examples for showing how to use itertools.ifilter(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: pcfg-sampling   Author: wilkeraziz   File: agenda.py    Apache License 2.0 6 votes vote down vote up
def get_cfg(goal, root, fsa, agenda):
    """
    Constructs the CFG by visiting complete items in a top-down fashion.
    This is effectively a reachability test and it serves the purpose of filtering nonterminal symbols
    that could never be reached from the root.
    Note that bottom-up intersection typically does enumerate a lot of useless (unreachable) items.
    This is the recursive procedure described in the paper (Nederhof and Satta, 2008).
    """

    G = WCFG()
    processed = set()

    def make_rules(lhs, start, end):
        if (start, lhs, end) in processed:
            return
        processed.add((lhs, start, end))
        for item in agenda.itercomplete(lhs, start, end):
            G.add(get_intersected_rule(item))
            fsa_states = item.inner + (item.dot,)
            for i, sym in itertools.ifilter(lambda (_, s): is_nonterminal(s), enumerate(item.rule.rhs)):
                if (sym, fsa_states[i], fsa_states[
                        i + 1]) not in processed:  # Nederhof does not perform this test, but in python it turned out crucial
                    make_rules(sym, fsa_states[i], fsa_states[i + 1])

    # create goal items 
Example 2
Project: ngo-addons-backport   Author: camptocamp   File: models.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def _read_csv(self, record, options):
        """ Returns a CSV-parsed iterator of all empty lines in the file

        :throws csv.Error: if an error is detected during CSV parsing
        :throws UnicodeDecodeError: if ``options.encoding`` is incorrect
        """
        csv_iterator = csv.reader(
            StringIO(record.file),
            quotechar=str(options['quoting']),
            delimiter=str(options['separator']))
        csv_nonempty = itertools.ifilter(None, csv_iterator)
        # TODO: guess encoding with chardet? Or https://github.com/aadsm/jschardet
        encoding = options.get('encoding', 'utf-8')
        return itertools.imap(
            lambda row: [item.decode(encoding) for item in row],
            csv_nonempty) 
Example 3
Project: razzy-spinner   Author: rafasashi   File: compat.py    GNU General Public License v3.0 6 votes vote down vote up
def __and__(self, other):
                ''' Intersection is the minimum of corresponding counts.

                >>> Counter('abbb') & Counter('bcc')
                Counter({'b': 1})

                '''
                if not isinstance(other, Counter):
                    return NotImplemented
                _min = min
                result = Counter()
                if len(self) < len(other):
                    self, other = other, self
                for elem in ifilter(self.__contains__, other):
                    newcount = _min(self[elem], other[elem])
                    if newcount > 0:
                        result[elem] = newcount
                return result 
Example 4
Project: TornadoWeb   Author: VxCoder   File: backports.py    Apache License 2.0 6 votes vote down vote up
def __and__(self, other):
        ''' Intersection is the minimum of corresponding counts.

        >>> Counter('abbb') & Counter('bcc')
        Counter({'b': 1})

        '''
        if not isinstance(other, Counter):
            return NotImplemented
        _min = min
        result = Counter()
        if len(self) < len(other):
            self, other = other, self
        for elem in ifilter(self.__contains__, other):
            newcount = _min(self[elem], other[elem])
            if newcount > 0:
                result[elem] = newcount
        return result 
Example 5
Project: mixprint_addons   Author: jeffery9   File: models.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def _read_csv(self, record, options):
        """ Returns a CSV-parsed iterator of all empty lines in the file

        :throws csv.Error: if an error is detected during CSV parsing
        :throws UnicodeDecodeError: if ``options.encoding`` is incorrect
        """
        csv_iterator = csv.reader(
            StringIO(record.file),
            quotechar=str(options['quoting']),
            delimiter=str(options['separator']))
        csv_nonempty = itertools.ifilter(None, csv_iterator)
        # TODO: guess encoding with chardet? Or https://github.com/aadsm/jschardet
        encoding = options.get('encoding', 'utf-8')
        return itertools.imap(
            lambda row: [item.decode(encoding) for item in row],
            csv_nonempty) 
Example 6
Project: ckanutils   Author: reubano   File: ckanutils.py    MIT License 6 votes vote down vote up
def filter(self, items, tagged=None, named=None, updated=None):
        for i in items:
            if i['state'] != 'active':
                continue

            if updated and updated(self.get_update_date(i)):
                yield i
                continue

            if named and named.lower() in i['name'].lower():
                yield i
                continue

            tags = it.imap(itemgetter('name'), i['tags'])
            is_tagged = tagged and 'tags' in i

            if is_tagged and any(it.ifilter(lambda t: t == tagged, tags)):
                yield i
                continue

            if not (named or tagged or updated):
                yield i 
Example 7
Project: pyblish-win   Author: pyblish   File: filecmp.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def phase1(self): # Compute common names
        a = dict(izip(imap(os.path.normcase, self.left_list), self.left_list))
        b = dict(izip(imap(os.path.normcase, self.right_list), self.right_list))
        self.common = map(a.__getitem__, ifilter(b.__contains__, a))
        self.left_only = map(a.__getitem__, ifilterfalse(b.__contains__, a))
        self.right_only = map(b.__getitem__, ifilterfalse(a.__contains__, b)) 
Example 8
Project: pyblish-win   Author: pyblish   File: sets.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def intersection(self, other):
        """Return the intersection of two sets as a new set.

        (I.e. all elements that are in both sets.)
        """
        if not isinstance(other, BaseSet):
            other = Set(other)
        if len(self) <= len(other):
            little, big = self, other
        else:
            little, big = other, self
        common = ifilter(big._data.__contains__, little)
        return self.__class__(common) 
Example 9
Project: pyblish-win   Author: pyblish   File: sets.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def difference_update(self, other):
        """Remove all elements of another set from this set."""
        data = self._data
        if not isinstance(other, BaseSet):
            other = Set(other)
        if self is other:
            self.clear()
        for elt in ifilter(data.__contains__, other):
            del data[elt]

    # Python dict-like mass mutations: update, clear 
Example 10
Project: pyblish-win   Author: pyblish   File: test_future_builtins.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_itertools(self):
        from itertools import imap, izip, ifilter
        # We will assume that the itertools functions work, so provided
        # that we've got identical coppies, we will work!
        self.assertEqual(map, imap)
        self.assertEqual(zip, izip)
        self.assertEqual(filter, ifilter)
        # Testing that filter(None, stuff) raises a warning lives in
        # test_py3kwarn.py 
Example 11
Project: Splunk_CBER_App   Author: MHaggis   File: search_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            definitions = type(self).configuration_setting_definitions
            version = self.command.protocol_version
            return ifilter(
                lambda (name, value): value is not None, imap(
                    lambda setting: (setting.name, setting.__get__(self)), ifilter(
                        lambda setting: setting.is_supported_by_protocol(version), definitions))) 
Example 12
Project: Splunk_CBER_App   Author: MHaggis   File: streaming_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 1:
                if self.required_fields is None:
                    iteritems = ifilter(lambda (name, value): name != 'clear_required_fields', iteritems)
            else:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed:
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems

        # endregion 
Example 13
Project: Splunk_CBER_App   Author: MHaggis   File: generating_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 2:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed and self.type == 'streaming':
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems 
Example 14
Project: pcfg-sampling   Author: wilkeraziz   File: earley.py    Apache License 2.0 5 votes vote down vote up
def get_cfg(self, goal, root):
        """
        Constructs the CFG by visiting complete items in a top-down fashion.
        This is effectively a reachability test and it serves the purpose of filtering nonterminal symbols
        that could never be reached from the root.
        Note that bottom-up intersection typically does enumerate a lot of useless (unreachable) items.
        This is the recursive procedure described in the paper (Nederhof and Satta, 2008).
        """

        G = WCFG()
        processed = set()
        fsa = self._wfsa
        itergenerating = self._agenda.itergenerating
        itercomplete = self._agenda.itercomplete

        def make_rules(lhs, start, end):
            if (start, lhs, end) in processed:
                return
            processed.add((lhs, start, end))
            for item in itercomplete(lhs, start, end):
                G.add(self.get_intersected_rule(item))
                fsa_states = item.inner + (item.dot,)
                for i, sym in itertools.ifilter(lambda (_, s): is_nonterminal(s), enumerate(item.rule.rhs)):
                    if (sym, fsa_states[i], fsa_states[
                            i + 1]) not in processed:  # Nederhof does not perform this test, but in python it turned out crucial
                        make_rules(sym, fsa_states[i], fsa_states[i + 1])

        # create goal items 
Example 15
Project: pcfg-sampling   Author: wilkeraziz   File: cfgply.py    Apache License 2.0 5 votes vote down vote up
def parse(self, lines):
        for line in ifilter(None, lines):
            production = self.parser.parse(line, lexer=self.lexer)
            yield production 
Example 16
Project: pcfg-sampling   Author: wilkeraziz   File: sliced_earley.py    Apache License 2.0 5 votes vote down vote up
def get_cfg(self, goal, root):
        """
        Constructs the CFG by visiting complete items in a top-down fashion.
        This is effectively a reachability test and it serves the purpose of filtering nonterminal symbols
        that could never be reached from the root.
        Note that bottom-up intersection typically does enumerate a lot of useless (unreachable) items.
        This is the recursive procedure described in the paper (Nederhof and Satta, 2008).
        """

        G = WCFG()
        processed = set()
        fsa = self._wfsa
        itergenerating = self._agenda.itergenerating
        itercomplete = self._agenda.itercomplete

        def make_rules(lhs, start, end):
            if (start, lhs, end) in processed:
                return
            processed.add((lhs, start, end))
            for item in itercomplete(lhs, start, end):
                G.add(self.get_intersected_rule(item))
                fsa_states = item.inner + (item.dot,)
                for i, sym in itertools.ifilter(lambda (_, s): is_nonterminal(s), enumerate(item.rule.rhs)):
                    if (sym, fsa_states[i], fsa_states[
                            i + 1]) not in processed:  # Nederhof does not perform this test, but in python it turned out crucial
                        make_rules(sym, fsa_states[i], fsa_states[i + 1])

        # create goal items 
Example 17
Project: osg-configure   Author: opensciencegrid   File: reversevomap.py    Apache License 2.0 5 votes vote down vote up
def get_vos(mappings):
    """Get the VOs from a list of mappings (assumption is that the first VO group in the pattern matches the VO name)

    :return: Set of VOs
    """
    regex = re.compile("^/(\w+)/")
    patterns = (m.pattern for m in mappings)
    matches = ifilter(None, (regex.match(p) for p in patterns))
    vo_groups = set(m.group(1).lower() for m in matches)

    return vo_groups 
Example 18
Project: misp42splunk   Author: remg427   File: search_command.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def iteritems(self):
            definitions = type(self).configuration_setting_definitions
            version = self.command.protocol_version
            return ifilter(
                lambda (name, value): value is not None, imap(
                    lambda setting: (setting.name, setting.__get__(self)), ifilter(
                        lambda setting: setting.is_supported_by_protocol(version), definitions))) 
Example 19
Project: misp42splunk   Author: remg427   File: streaming_command.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 1:
                if self.required_fields is None:
                    iteritems = ifilter(lambda (name, value): name != 'clear_required_fields', iteritems)
            else:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed:
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems

        # endregion 
Example 20
Project: misp42splunk   Author: remg427   File: generating_command.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 2:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed and self.type == 'streaming':
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems 
Example 21
Project: whisperdb   Author: paritosh16   File: whisper-fill.py    MIT License 5 votes vote down vote up
def fill(src, dst, tstart, tstop):
    # fetch range start-stop from src, taking values from the highest
    # precision archive, thus optionally requiring multiple fetch + merges
    srcHeader = info(src)

    srcArchives = srcHeader['archives']
    srcArchives.sort(key=itemgetter('retention'))

    # find oldest point in time, stored by both files
    srcTime = int(time.time()) - srcHeader['maxRetention']

    if tstart < srcTime and tstop < srcTime:
        return

    # we want to retain as much precision as we can, hence we do backwards
    # walk in time

    # skip forward at max 'step' points at a time
    for archive in srcArchives:
        # skip over archives that don't have any data points
        rtime = time.time() - archive['retention']
        if tstop <= rtime:
            continue

        untilTime = tstop
        fromTime = rtime if rtime > tstart else tstart

        (timeInfo, values) = fetch(src, fromTime, untilTime)
        (start, end, archive_step) = timeInfo
        pointsToWrite = list(itertools.ifilter(
            lambda points: points[1] is not None,
            itertools.izip(xrange(start, end, archive_step), values)))
        # order points by timestamp, newest first
        pointsToWrite.sort(key=lambda p: p[0], reverse=True)
        update_many(dst, pointsToWrite)

        tstop = fromTime

        # can stop when there's nothing to fetch any more
        if tstart == tstop:
            return 
Example 22
Project: onering   Author: panyam   File: misc.py    Apache License 2.0 5 votes vote down vote up
def collect_files_by_extension(root_dir, ext):
    return ifilter(lambda path: path.endswith("." + ext) and os.path.isfile(path), collect_files(root_dir)) 
Example 23
Project: onering   Author: panyam   File: misc.py    Apache License 2.0 5 votes vote down vote up
def collect_jars(root_dir):
    def is_model_jar(name):
        return name.find("data-template") > 0 and name.find("SNAPSHOT") < 0 and name.endswith(".jar")
    return ifilter(is_model_jar, collect_files(root_dir)) 
Example 24
Project: onering   Author: panyam   File: misc.py    Apache License 2.0 5 votes vote down vote up
def collect_files_by_extension(root_dir, ext):
    return ifilter(lambda path: path.endswith("." + ext) and os.path.isfile(path), collect_files(root_dir)) 
Example 25
Project: onering   Author: panyam   File: misc.py    Apache License 2.0 5 votes vote down vote up
def collect_jars(root_dir):
    def is_model_jar(name):
        return name.find("data-template") > 0 and name.find("SNAPSHOT") < 0 and name.endswith(".jar")
    return ifilter(is_model_jar, collect_files(root_dir)) 
Example 26
Project: zendesk-help-center-localization   Author: mykola-mokhnach   File: zendesk_localization.py    Apache License 2.0 5 votes vote down vote up
def import_drafts_from_crowdin_to_zendesk(crowd_api, zen_api):
    all_categories = zen_api.list_categories()
    all_sections = reduce(lambda a, b: a + zen_api.list_sections(b['id']), all_categories, [])
    all_articles = reduce(lambda a, b: a + zen_api.list_articles(b['id']), all_sections, [])
    draft_articles = filter(_is_draft, all_articles)
    if not draft_articles:
        logger.info(u'No draft articles have been found. Nothing to import\n')
        return []
    processed_article_by_id = OrderedDict()
    crowd_api.export_translations()
    for dst_language_abbr in DST_LANGUAGE_ABBRS:
        language_abbr_in_crowdin = ZENDESK_TO_CROWDIN_LANGUGES_MAPPING.get(dst_language_abbr, dst_language_abbr)
        root = crowd_api.download_translations(language_abbr_in_crowdin)
        try:
            for current_root, dirs, files in os.walk(root):
                for fname in files:
                    full_path = os.path.join(current_root, fname)
                    article_id = _extract_article_id_from_filename(fname)
                    if article_id is None:
                        # logger.info('Cannot parse id from {}. Skipping...\n'.format(full_path))
                        continue
                    dst_article = next(itertools.ifilter(
                        lambda x: _extract_article_id_from_title(x) == article_id or long(x['id']) == article_id,
                        draft_articles), None)
                    if dst_article is None:
                        if dst_article is None:
                            logger.warning(
                                u'Cannot find Zendesk draft with id "{}" for {}. Skipping...\n'.format(article_id,
                                                                                                       full_path)
                            )
                            continue
                    logger.info(u'Importing {} (locale {})...'.format(full_path.replace(root, ''), dst_language_abbr))
                    _import_translation_to_zendesk(zen_api, dst_language_abbr, full_path, dst_article)
                    logger.info(u'Successfully updated draft article "{}" for locale "{}" at {}\n'.
                                format(dst_article['title'], dst_language_abbr, dst_article['html_url']))
                    processed_article_by_id[dst_article['id']] = dst_article
        finally:
            shutil.rmtree(root, ignore_errors=True)
    return processed_article_by_id.values() 
Example 27
Project: zendesk-help-center-localization   Author: mykola-mokhnach   File: zendesk_localization.py    Apache License 2.0 5 votes vote down vote up
def _find_original_article(draft_article, all_articles):
    match = CLONED_DRAFT_TITLE_PATTERN.search(draft_article['title'])
    if match is not None:
        draft_id = long(match.group(1))
        return next(itertools.ifilter(lambda x: long(x['id']) == draft_id, all_articles), None)
    return None 
Example 28
Project: grand_central   Author: amiracle   File: search_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            definitions = type(self).configuration_setting_definitions
            version = self.command.protocol_version
            return ifilter(
                lambda (name, value): value is not None, imap(
                    lambda setting: (setting.name, setting.__get__(self)), ifilter(
                        lambda setting: setting.is_supported_by_protocol(version), definitions))) 
Example 29
Project: grand_central   Author: amiracle   File: streaming_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 1:
                if self.required_fields is None:
                    iteritems = ifilter(lambda (name, value): name != 'clear_required_fields', iteritems)
            else:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed:
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems

        # endregion 
Example 30
Project: grand_central   Author: amiracle   File: generating_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 2:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed and self.type == 'streaming':
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems 
Example 31
Project: SplunkForPCAP   Author: DanielSchwartz1   File: search_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            definitions = type(self).configuration_setting_definitions
            version = self.command.protocol_version
            return ifilter(
                lambda (name, value): value is not None, imap(
                    lambda setting: (setting.name, setting.__get__(self)), ifilter(
                        lambda setting: setting.is_supported_by_protocol(version), definitions))) 
Example 32
Project: SplunkForPCAP   Author: DanielSchwartz1   File: streaming_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 1:
                if self.required_fields is None:
                    iteritems = ifilter(lambda (name, value): name != 'clear_required_fields', iteritems)
            else:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed:
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems

        # endregion 
Example 33
Project: SplunkForPCAP   Author: DanielSchwartz1   File: generating_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 2:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed and self.type == 'streaming':
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems 
Example 34
Project: foreman-yml   Author: adfinis-sygroup   File: voluptuous.py    GNU General Public License v3.0 5 votes vote down vote up
def _compile_object(self, schema):
        """Validate an object.

        Has the same behavior as dictionary validator but work with object
        attributes.

        For example:

            >>> class Structure(object):
            ...     def __init__(self, one=None, three=None):
            ...         self.one = one
            ...         self.three = three
            ...
            >>> validate = Schema(Object({'one': 'two', 'three': 'four'}, cls=Structure))
            >>> with raises(MultipleInvalid, "not a valid value for object value @ data['one']"):
            ...   validate(Structure(one='three'))

        """
        base_validate = self._compile_mapping(
            schema, invalid_msg='object value')

        def validate_object(path, data):
            if (schema.cls is not UNDEFINED
                    and not isinstance(data, schema.cls)):
                raise ObjectInvalid('expected a {0!r}'.format(schema.cls), path)
            iterable = _iterate_object(data)
            iterable = ifilter(lambda item: item[1] is not None, iterable)
            out = base_validate(path, iterable, {})
            return type(data)(**out)

        return validate_object 
Example 35
Project: ironpython2   Author: IronLanguages   File: filecmp.py    Apache License 2.0 5 votes vote down vote up
def phase1(self): # Compute common names
        a = dict(izip(imap(os.path.normcase, self.left_list), self.left_list))
        b = dict(izip(imap(os.path.normcase, self.right_list), self.right_list))
        self.common = map(a.__getitem__, ifilter(b.__contains__, a))
        self.left_only = map(a.__getitem__, ifilterfalse(b.__contains__, a))
        self.right_only = map(b.__getitem__, ifilterfalse(a.__contains__, b)) 
Example 36
Project: ironpython2   Author: IronLanguages   File: sets.py    Apache License 2.0 5 votes vote down vote up
def intersection(self, other):
        """Return the intersection of two sets as a new set.

        (I.e. all elements that are in both sets.)
        """
        if not isinstance(other, BaseSet):
            other = Set(other)
        if len(self) <= len(other):
            little, big = self, other
        else:
            little, big = other, self
        common = ifilter(big._data.__contains__, little)
        return self.__class__(common) 
Example 37
Project: ironpython2   Author: IronLanguages   File: sets.py    Apache License 2.0 5 votes vote down vote up
def difference_update(self, other):
        """Remove all elements of another set from this set."""
        data = self._data
        if not isinstance(other, BaseSet):
            other = Set(other)
        if self is other:
            self.clear()
        for elt in ifilter(data.__contains__, other):
            del data[elt]

    # Python dict-like mass mutations: update, clear 
Example 38
Project: ironpython2   Author: IronLanguages   File: test_future_builtins.py    Apache License 2.0 5 votes vote down vote up
def test_itertools(self):
        from itertools import imap, izip, ifilter
        # We will assume that the itertools functions work, so provided
        # that we've got identical coppies, we will work!
        self.assertEqual(map, imap)
        self.assertEqual(zip, izip)
        self.assertEqual(filter, ifilter)
        # Testing that filter(None, stuff) raises a warning lives in
        # test_py3kwarn.py 
Example 39
Project: TA-Send_to_HEC   Author: georgestarcher   File: search_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            definitions = type(self).configuration_setting_definitions
            version = self.command.protocol_version
            return ifilter(
                lambda (name, value): value is not None, imap(
                    lambda setting: (setting.name, setting.__get__(self)), ifilter(
                        lambda setting: setting.is_supported_by_protocol(version), definitions))) 
Example 40
Project: TA-Send_to_HEC   Author: georgestarcher   File: streaming_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 1:
                if self.required_fields is None:
                    iteritems = ifilter(lambda (name, value): name != 'clear_required_fields', iteritems)
            else:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed:
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems

        # endregion 
Example 41
Project: TA-Send_to_HEC   Author: georgestarcher   File: generating_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 2:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed and self.type == 'streaming':
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems 
Example 42
Project: minemeld-core   Author: PaloAltoNetworks   File: csv.py    Apache License 2.0 5 votes vote down vote up
def _build_iterator(self, now):
        _session = requests.Session()

        prepreq = self._build_request(now)

        # this is to honour the proxy environment variables
        rkwargs = _session.merge_environment_settings(
            prepreq.url,
            {}, None, None, None  # defaults
        )
        rkwargs['stream'] = True
        rkwargs['verify'] = self.verify_cert
        rkwargs['timeout'] = self.polling_timeout

        r = _session.send(prepreq, **rkwargs)

        try:
            r.raise_for_status()
        except:
            LOG.debug('%s - exception in request: %s %s',
                      self.name, r.status_code, r.content)
            raise

        response = r.raw
        if self.ignore_regex is not None:
            response = itertools.ifilter(
                lambda x: self.ignore_regex.match(x) is None,
                r.raw
            )

        csvreader = csv.DictReader(
            response,
            fieldnames=self.fieldnames,
            **self.dialect
        )

        return csvreader 
Example 43
Project: SA-dmarc   Author: aholzel   File: search_command.py    GNU General Public License v3.0 5 votes vote down vote up
def iteritems(self):
            definitions = type(self).configuration_setting_definitions
            version = self.command.protocol_version
            return ifilter(
                lambda (name, value): value is not None, imap(
                    lambda setting: (setting.name, setting.__get__(self)), ifilter(
                        lambda setting: setting.is_supported_by_protocol(version), definitions))) 
Example 44
Project: SA-dmarc   Author: aholzel   File: streaming_command.py    GNU General Public License v3.0 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 1:
                if self.required_fields is None:
                    iteritems = ifilter(lambda (name, value): name != 'clear_required_fields', iteritems)
            else:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed:
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems

        # endregion 
Example 45
Project: SA-dmarc   Author: aholzel   File: generating_command.py    GNU General Public License v3.0 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 2:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed and self.type == 'streaming':
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems 
Example 46
Project: code-inspector   Author: maxim5   File: 234a016ad1efa9f0f75909da98986d10_algorithm.py    Apache License 2.0 5 votes vote down vote up
def _create_data_generator(self, source_filter):
        """
        Create a merged data generator using the sources and
        transforms attached to this algorithm.

        ::source_filter:: is a method that receives events in date
        sorted order, and returns True for those events that should be
        processed by the zipline, and False for those that should be
        skipped.
        """
        benchmark_return_source = [
            Event({'dt': ret.date,
                   'returns': ret.returns,
                   'type': zipline.protocol.DATASOURCE_TYPE.BENCHMARK,
                   'source_id': 'benchmarks'})
            for ret in trading.environment.benchmark_returns
            if ret.date.date() >= self.sim_params.period_start.date()
            and ret.date.date() <= self.sim_params.period_end.date()
        ]

        date_sorted = date_sorted_sources(*self.sources)

        if source_filter:
            date_sorted = ifilter(source_filter, date_sorted)

        with_tnfms = sequential_transforms(date_sorted,
                                           *self.transforms)
        with_alias_dt = alias_dt(with_tnfms)

        with_benchmarks = date_sorted_sources(benchmark_return_source,
                                              with_alias_dt)

        # Group together events with the same dt field. This depends on the
        # events already being sorted.
        return groupby(with_benchmarks, attrgetter('dt')) 
Example 47
Project: code-inspector   Author: maxim5   File: bbe49301808cd5c07dbe3f6ef159241f_algorithm.py    Apache License 2.0 5 votes vote down vote up
def _create_data_generator(self, source_filter, sim_params):
        """
        Create a merged data generator using the sources and
        transforms attached to this algorithm.

        ::source_filter:: is a method that receives events in date
        sorted order, and returns True for those events that should be
        processed by the zipline, and False for those that should be
        skipped.
        """
        if self.benchmark_return_source is None:
            benchmark_return_source = [
                Event({'dt': ret.date,
                       'returns': ret.returns,
                       'type': zipline.protocol.DATASOURCE_TYPE.BENCHMARK,
                       'source_id': 'benchmarks'})
                for ret in trading.environment.benchmark_returns
                if ret.date.date() >= sim_params.period_start.date()
                and ret.date.date() <= sim_params.period_end.date()
            ]
        else:
            benchmark_return_source = self.benchmark_return_source

        date_sorted = date_sorted_sources(*self.sources)

        if source_filter:
            date_sorted = ifilter(source_filter, date_sorted)

        with_tnfms = sequential_transforms(date_sorted,
                                           *self.transforms)
        with_alias_dt = alias_dt(with_tnfms)

        with_benchmarks = date_sorted_sources(benchmark_return_source,
                                              with_alias_dt)

        # Group together events with the same dt field. This depends on the
        # events already being sorted.
        return groupby(with_benchmarks, attrgetter('dt')) 
Example 48
Project: git_for_splunk   Author: ChrisYounger   File: search_command.py    Apache License 2.0 5 votes vote down vote up
def iteritems(self):
            definitions = type(self).configuration_setting_definitions
            version = self.command.protocol_version
            return ifilter(
                lambda (name, value): value is not None, imap(
                    lambda setting: (setting.name, setting.__get__(self)), ifilter(
                        lambda setting: setting.is_supported_by_protocol(version), definitions))) 
Example 49
Project: git_for_splunk   Author: ChrisYounger   File: streaming_command.py    Apache License 2.0 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 1:
                if self.required_fields is None:
                    iteritems = ifilter(lambda (name, value): name != 'clear_required_fields', iteritems)
            else:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed:
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems

        # endregion 
Example 50
Project: git_for_splunk   Author: ChrisYounger   File: generating_command.py    Apache License 2.0 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 2:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed and self.type == 'streaming':
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems 
Example 51
Project: git_for_splunk   Author: ChrisYounger   File: search_command.py    Apache License 2.0 5 votes vote down vote up
def iteritems(self):
            definitions = type(self).configuration_setting_definitions
            version = self.command.protocol_version
            return ifilter(
                lambda (name, value): value is not None, imap(
                    lambda setting: (setting.name, setting.__get__(self)), ifilter(
                        lambda setting: setting.is_supported_by_protocol(version), definitions))) 
Example 52
Project: git_for_splunk   Author: ChrisYounger   File: streaming_command.py    Apache License 2.0 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 1:
                if self.required_fields is None:
                    iteritems = ifilter(lambda (name, value): name != 'clear_required_fields', iteritems)
            else:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed:
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems

        # endregion 
Example 53
Project: Library-Identification   Author: Riscure   File: library_identification.py    GNU General Public License v3.0 5 votes vote down vote up
def get_strings(self, section=None, minLength=5):
        # fast generator-friendly version of uniq+sort
        # from http://stackoverflow.com/questions/2931672/
        def sort_uniq(sequence):
            return itertools.imap(
                operator.itemgetter(0),
                itertools.groupby(sorted(sequence)))

        return sort_uniq(itertools.ifilter(lambda s: len(s) >= minLength,
                           self.read_strings(section=section)))


    # Returns strings that look like a version number 
Example 54
Project: sndlatr   Author: Schibum   File: voluptuous.py    Apache License 2.0 5 votes vote down vote up
def _compile_object(self, schema):
        """Validate an object.

        Has the same behavior as dictionary validator but work with object
        attributes.

        For example:

            >>> class Structure(object):
            ...     def __init__(self, one=None, three=None):
            ...         self.one = one
            ...         self.three = three
            ...
            >>> validate = Schema(Object({'one': 'two', 'three': 'four'}, cls=Structure))
            >>> with raises(MultipleInvalid, "not a valid value for object value @ data['one']"):
            ...   validate(Structure(one='three'))

        """
        base_validate = self._compile_mapping(schema,
            invalid_msg='for object value')

        def validate_object(path, data):
            if schema.cls is not UNDEFINED and not isinstance(data, schema.cls):
                raise Invalid('expected a {0!r}'.format(schema.cls), path)
            iterable = _iterate_object(data)
            iterable = ifilter(lambda item: item[1] is not None, iterable)
            out = base_validate(path, iterable, {})
            return type(data)(**out)

        return validate_object 
Example 55
Project: TA-esreplacefields   Author: georgestarcher   File: search_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            definitions = type(self).configuration_setting_definitions
            version = self.command.protocol_version
            return ifilter(
                lambda (name, value): value is not None, imap(
                    lambda setting: (setting.name, setting.__get__(self)), ifilter(
                        lambda setting: setting.is_supported_by_protocol(version), definitions))) 
Example 56
Project: TA-esreplacefields   Author: georgestarcher   File: streaming_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 1:
                if self.required_fields is None:
                    iteritems = ifilter(lambda (name, value): name != 'clear_required_fields', iteritems)
            else:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed:
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems

        # endregion 
Example 57
Project: TA-esreplacefields   Author: georgestarcher   File: generating_command.py    MIT License 5 votes vote down vote up
def iteritems(self):
            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
            version = self.command.protocol_version
            if version == 2:
                iteritems = ifilter(lambda (name, value): name != 'distributed', iteritems)
                if self.distributed and self.type == 'streaming':
                    iteritems = imap(
                        lambda (name, value): (name, 'stateful') if name == 'type' else (name, value), iteritems)
            return iteritems 
Example 58
Project: IronHydra   Author: microdee   File: filecmp.py    MIT License 5 votes vote down vote up
def phase1(self): # Compute common names
        a = dict(izip(imap(os.path.normcase, self.left_list), self.left_list))
        b = dict(izip(imap(os.path.normcase, self.right_list), self.right_list))
        self.common = map(a.__getitem__, ifilter(b.__contains__, a))
        self.left_only = map(a.__getitem__, ifilterfalse(b.__contains__, a))
        self.right_only = map(b.__getitem__, ifilterfalse(a.__contains__, b)) 
Example 59
Project: IronHydra   Author: microdee   File: sets.py    MIT License 5 votes vote down vote up
def intersection(self, other):
        """Return the intersection of two sets as a new set.

        (I.e. all elements that are in both sets.)
        """
        if not isinstance(other, BaseSet):
            other = Set(other)
        if len(self) <= len(other):
            little, big = self, other
        else:
            little, big = other, self
        common = ifilter(big._data.__contains__, little)
        return self.__class__(common) 
Example 60
Project: IronHydra   Author: microdee   File: sets.py    MIT License 5 votes vote down vote up
def difference_update(self, other):
        """Remove all elements of another set from this set."""
        data = self._data
        if not isinstance(other, BaseSet):
            other = Set(other)
        if self is other:
            self.clear()
        for elt in ifilter(data.__contains__, other):
            del data[elt]

    # Python dict-like mass mutations: update, clear 
Example 61
Project: closure-linter   Author: google   File: scopeutil.py    Apache License 2.0 5 votes vote down vote up
def _IsFunctionLiteralBlock(block_context):
  """Check if a context is a function literal block (without parameters).

  Example function literal block: 'function() {}'

  Args:
    block_context: An EcmaContext of type block.

  Returns:
    Whether this context is a function literal block.
  """

  previous_code_tokens_iter = itertools.ifilter(
      lambda token: token not in JavaScriptTokenType.NON_CODE_TYPES,
      reversed(block_context.start_token))

  # Ignore the current token
  next(previous_code_tokens_iter, None)

  # Grab the previous three tokens and put them in correct order.
  previous_code_tokens = list(itertools.islice(previous_code_tokens_iter, 3))
  previous_code_tokens.reverse()

  # There aren't three previous tokens.
  if len(previous_code_tokens) is not 3:
    return False

  # Check that the previous three code tokens are "function ()"
  previous_code_token_types = [token.type for token in previous_code_tokens]
  if (previous_code_token_types == [
      JavaScriptTokenType.FUNCTION_DECLARATION,
      JavaScriptTokenType.START_PARAMETERS,
      JavaScriptTokenType.END_PARAMETERS]):
    return True

  return False 
Example 62
Project: open_dnsdb   Author: qunarcorp   File: log.py    Apache License 2.0 5 votes vote down vote up
def formatException(self, ei, strip_newlines=True):
        lines = traceback.format_exception(*ei)
        if strip_newlines:
            lines = [itertools.ifilter(
                lambda x: x,
                line.rstrip().splitlines()) for line in lines]
            lines = list(itertools.chain(*lines))
        return lines 
Example 63
Project: Computable   Author: ktraunmueller   File: filecmp.py    MIT License 5 votes vote down vote up
def phase1(self): # Compute common names
        a = dict(izip(imap(os.path.normcase, self.left_list), self.left_list))
        b = dict(izip(imap(os.path.normcase, self.right_list), self.right_list))
        self.common = map(a.__getitem__, ifilter(b.__contains__, a))
        self.left_only = map(a.__getitem__, ifilterfalse(b.__contains__, a))
        self.right_only = map(b.__getitem__, ifilterfalse(a.__contains__, b)) 
Example 64
Project: ocr-process-service   Author: nfscan   File: postprocessing.py    MIT License 5 votes vote down vote up
def approximate_match(word_re, lines, fuzziness='e<=1'):
        logger = logging.getLogger(__name__)
        logger.debug('Looking for %s with fuzziness: %s' % (word_re, fuzziness))
        best_partial_matches = []
        search = re.compile(
            ur'(' + word_re + '){' + fuzziness + '}',
            flags=re.BESTMATCH | re.IGNORECASE).search
        for m in ifilter(None, imap(search, lines)):
            logger.debug('%s %s' % (m.span(), m[0]))
            best_partial_matches.append(m[0])
        return best_partial_matches 
Example 65
Project: oss-ftp   Author: aliyun   File: filecmp.py    MIT License 5 votes vote down vote up
def phase1(self): # Compute common names
        a = dict(izip(imap(os.path.normcase, self.left_list), self.left_list))
        b = dict(izip(imap(os.path.normcase, self.right_list), self.right_list))
        self.common = map(a.__getitem__, ifilter(b.__contains__, a))
        self.left_only = map(a.__getitem__, ifilterfalse(b.__contains__, a))
        self.right_only = map(b.__getitem__, ifilterfalse(a.__contains__, b)) 
Example 66
Project: oss-ftp   Author: aliyun   File: sets.py    MIT License 5 votes vote down vote up
def intersection(self, other):
        """Return the intersection of two sets as a new set.

        (I.e. all elements that are in both sets.)
        """
        if not isinstance(other, BaseSet):
            other = Set(other)
        if len(self) <= len(other):
            little, big = self, other
        else:
            little, big = other, self
        common = ifilter(big._data.__contains__, little)
        return self.__class__(common) 
Example 67
Project: oss-ftp   Author: aliyun   File: sets.py    MIT License 5 votes vote down vote up
def difference_update(self, other):
        """Remove all elements of another set from this set."""
        data = self._data
        if not isinstance(other, BaseSet):
            other = Set(other)
        if self is other:
            self.clear()
        for elt in ifilter(data.__contains__, other):
            del data[elt]

    # Python dict-like mass mutations: update, clear 
Example 68
Project: oss-ftp   Author: aliyun   File: test_future_builtins.py    MIT License 5 votes vote down vote up
def test_itertools(self):
        from itertools import imap, izip, ifilter
        # We will assume that the itertools functions work, so provided
        # that we've got identical coppies, we will work!
        self.assertEqual(map, imap)
        self.assertEqual(zip, izip)
        self.assertEqual(filter, ifilter)
        # Testing that filter(None, stuff) raises a warning lives in
        # test_py3kwarn.py 
Example 69
Project: pivy   Author: FreeCAD   File: _scons_sets.py    ISC License 5 votes vote down vote up
def ifilter(predicate, iterable):
        if predicate is None:
            def predicate(x):
                return x
        for x in iterable:
            if predicate(x):
                yield x 
Example 70
Project: pivy   Author: FreeCAD   File: _scons_sets.py    ISC License 5 votes vote down vote up
def intersection(self, other):
        """Return the intersection of two sets as a new set.

        (I.e. all elements that are in both sets.)
        """
        if not isinstance(other, BaseSet):
            other = Set(other)
        if len(self) <= len(other):
            little, big = self, other
        else:
            little, big = other, self
        common = ifilter(big._data.has_key, little)
        return self.__class__(common) 
Example 71
Project: pivy   Author: FreeCAD   File: _scons_sets.py    ISC License 5 votes vote down vote up
def difference_update(self, other):
        """Remove all elements of another set from this set."""
        data = self._data
        if not isinstance(other, BaseSet):
            other = Set(other)
        if self is other:
            self.clear()
        for elt in ifilter(data.has_key, other):
            del data[elt]

    # Python dict-like mass mutations: update, clear 
Example 72
Project: AGEpy   Author: mpg-age-bioinformatics   File: blast.py    MIT License 5 votes vote down vote up
def variablename(var):
    """
    Returns the string of a variable name.
    """
    s=[tpl[0] for tpl in itertools.ifilter(lambda x: var is x[1], globals().items())]
    s=s[0].upper()
    return s 
Example 73
Project: pyxtension   Author: asuiu   File: streams.py    GNU General Public License v2.0 5 votes vote down vote up
def filter(self, predicate=None):
        """
        :param predicate: If predicate is None, return the items that are true.
        :type predicate: None|(T) -> bool
        :rtype: stream
        """
        return stream(ItrFromFunc(lambda: ifilter(predicate, self))) 
Example 74
Project: pyxtension   Author: asuiu   File: test_Streams.py    GNU General Public License v2.0 5 votes vote down vote up
def testStream(self):
        s = self.s
        self.assertEquals(list(ifilter(lambda i: i % 2 == 0, s())), [2])
        self.assertEquals(list(s().filter(lambda i: i % 2 == 0)), [2])
        self.assertEquals(s().filter(lambda i: i % 2 == 0).toList(), [2])
        self.assertEquals(s()[1], 2)
        self.assertEquals(s()[1:].toList(), [2, 3])
        self.assertEqual(s().take(2).toList(), [1, 2])
        self.assertAlmostEqual(stream((0, 1, 2, 3)).filter(lambda x: x > 0).entropy(), 1.4591479)
        self.assertEquals(stream([(1, 2), (3, 4)]).zip().toList(), [(1, 3), (2, 4)]) 
Example 75
Project: xserverpy   Author: nsomar   File: bots.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def get_named(self, bot_name):
        bots = self.get_all()
        try:
            return list(itertools.ifilter(lambda x: x.name == bot_name, bots))[0]
        except:
            return None 
Example 76
Project: openhatch   Author: campbe13   File: search.py    GNU Affero General Public License v3.0 4 votes vote down vote up
def searchexcerpt(text, phrases, context_words=None, ignore_case=None, word_boundary=None):
    if isinstance(phrases, basestring):
        phrases = [phrases]
    if context_words is None:
        context_words = get_setting('CONTEXT_WORDS')
    if ignore_case is None:
        ignore_case = get_setting('IGNORE_CASE')
    if word_boundary is None:
        word_boundary = get_setting('WORD_BOUNDARY')

    phrases = map(re.escape, phrases)
    flags = ignore_case and re.I or 0
    exprs = [re.compile(r"^%s$" % p, flags) for p in phrases]
    whitespace = re.compile(r'\s+')

    re_template = word_boundary and r"\b(%s)\b" or r"(%s)"
    pieces = re.compile(re_template % "|".join(phrases), flags).split(text)
    matches = {}
    word_lists = []
    index = {}
    for i, piece in enumerate(pieces):
        word_lists.append(whitespace.split(piece))
        if i % 2:
            index[i] = expr = ifilter(lambda e: e.match(piece), exprs).next()
            matches.setdefault(expr, []).append(i)

    def merge(lists):
        merged = []
        for words in lists:
            if merged:
                merged[-1] += words[0]
                del words[0]
            merged.extend(words)
        return merged

    i = 0
    merged = []
    for j in map(min, matches.itervalues()):
        merged.append(merge(word_lists[i:j]))
        merged.append(word_lists[j])
        i = j + 1
    merged.append(merge(word_lists[i:]))

    output = []
    for i, words in enumerate(merged):
        omit = None
        if i == len(merged) - 1:
            omit = slice(max(1, 2 - i) * context_words + 1, None)
        elif i == 0:
            omit = slice(-context_words - 1)
        elif not i % 2:
            omit = slice(context_words + 1, -context_words - 1)
        if omit and words[omit]:
            words[omit] = ["..."]
        output.append(" ".join(words))

    return dict(original=text, excerpt="".join(output), hits=len(index)) 
Example 77
Project: minemeld-core   Author: PaloAltoNetworks   File: http.py    Apache License 2.0 4 votes vote down vote up
def _build_iterator(self, now):
        rkwargs = dict(
            stream=True,
            verify=self.verify_cert,
            timeout=self.polling_timeout
        )

        if self.user_agent is not None:
            if self.user_agent == 'MineMeld':
                rkwargs['headers'] = {
                    'User-Agent': 'MineMeld/%s' % MM_VERSION
                }

            else:
                rkwargs['headers'] = {
                    'User-Agent': self.user_agent
                }

        if self.username is not None and self.password is not None:
            rkwargs['auth'] = (self.username, self.password)

        r = requests.get(
            self.url,
            **rkwargs
        )

        try:
            r.raise_for_status()
        except:
            LOG.debug('%s - exception in request: %s %s',
                      self.name, r.status_code, r.content)
            raise

        result = r.iter_lines()
        if self.ignore_regex is not None:
            result = itertools.ifilter(
                lambda x: self.ignore_regex.match(x) is None,
                result
            )
        if self.encoding is not None:
            result = itertools.imap(
                lambda x: x.decode(self.encoding).encode('utf_8'),
                result
            )

        return result 
Example 78
Project: minemeld-core   Author: PaloAltoNetworks   File: auscert.py    Apache License 2.0 4 votes vote down vote up
def _build_iterator(self, now):
        if self.api_key is None:
            raise RuntimeError(
                '{} - API Key not set, '
                'poll not performed'.format(self.name)
            )

        rkwargs = dict(
            stream=True,
            verify=self.verify_cert,
            timeout=self.polling_timeout
        )
        
        rkwargs["headers"] = { 
            'API-Key': self.api_key 
        }

        session = requests.Session()

        r = session.get(
            self.url,
            **rkwargs
        )

        # if api_key is wrong we'll get a 403 response code
        if r.status_code == 403: 
            raise RuntimeError(
                '{} - not authorized (Invalid API Key?)'.format(self.name)
            )

        try:
            r.raise_for_status()
        except:
            LOG.debug('%s - exception in request: %s %s',
                      self.name, r.status_code, r.content)
            raise

        result = r.iter_lines()
        if self.ignore_regex is not None:
            result = itertools.ifilter(
                lambda x: self.ignore_regex.match(x) is None,
                result
            )

        return result 
Example 79
Project: minemeld-core   Author: PaloAltoNetworks   File: phishme.py    Apache License 2.0 4 votes vote down vote up
def _threathq_update(self, now):
        changelog_size = 1000
        while changelog_size == 1000:
            if self.position is not None:
                payload = dict(position=self.position)
            else:
                payload = dict(timestamp=int(now/1000.0))

            rkwargs = dict(
                stream=True,
                verify=self.verify_cert,
                timeout=self.polling_timeout,
                params=payload,
                auth=(self.username, self.api_key),
                headers=self.headers
            )

            r = requests.post(
                _API_BASE+_API_THREAT_UPDATE,
                **rkwargs
            )

            try:
                r.raise_for_status()
            except:
                LOG.error(
                    '%s - exception in request: %s %s',
                    self.name, r.status_code, r.content
                )
                raise

            cjson = r.json()

            data = cjson.get('data', None)
            if data is None:
                LOG.error('%s - no "data" in update request', self.name)
                return

            changelog = data.get('changelog', None)
            if changelog is not None:
                changelog_size = len(changelog)

            else:
                LOG.info('%s - no "changelog" in update request', self.name)
                changelog_size = 0
                changelog = []

            thgen = self._retrieve_threats(
                self._group_changes_in_pages(
                    itertools.ifilter(self._filter_changes, changelog)
                )
            )
            for t in thgen:
                yield t

            next_position = data.get('nextPosition', None)
            if next_position is None:
                LOG.error('%s - no nextPosition in update request', self.name)
            else:
                self.position = next_position 
Example 80
Project: Computable   Author: ktraunmueller   File: type1font.py    MIT License 4 votes vote down vote up
def _parse(self):
        """
        Find the values of various font properties. This limited kind
        of parsing is described in Chapter 10 "Adobe Type Manager
        Compatibility" of the Type-1 spec.
        """
        # Start with reasonable defaults
        prop = {'weight': 'Regular', 'ItalicAngle': 0.0, 'isFixedPitch': False,
                'UnderlinePosition': -100, 'UnderlineThickness': 50}
        tokenizer = self._tokens(self.parts[0])
        filtered = itertools.ifilter(lambda x: x[0] != 'whitespace', tokenizer)
        for token, value in filtered:
            if token == b'name' and value.startswith(b'/'):
                key = value[1:]
                token, value = next(filtered)
                if token == b'name':
                    if value in (b'true', b'false'):
                        value = value == b'true'
                    else:
                        value = value.lstrip(b'/')
                elif token == b'string':
                    value = value.lstrip(b'(').rstrip(b')')
                elif token == b'number':
                    if b'.' in value:
                        value = float(value)
                    else:
                        value = int(value)
                else:  # more complicated value such as an array
                    value = None
                if key != b'FontInfo' and value is not None:
                    prop[key] = value

        # Fill in the various *Name properties
        if 'FontName' not in prop:
            prop['FontName'] = (prop.get('FullName') or
                                prop.get('FamilyName') or
                                'Unknown')
        if 'FullName' not in prop:
            prop['FullName'] = prop['FontName']
        if 'FamilyName' not in prop:
            extras = r'(?i)([ -](regular|plain|italic|oblique|(semi)?bold|(ultra)?light|extra|condensed))+$'
            prop['FamilyName'] = re.sub(extras, '', prop['FullName'])

        self.prop = prop