Python toolz.sliding_window() Examples
The following are 10
code examples of toolz.sliding_window().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
toolz
, or try the search function
.
.
Example #1
| Source File: bpe_vocab.py From NLP_Toolkit with Apache License 2.0 | 6 votes |
|
def byte_pair_counts(self, words):
# type: (Encoder, Iterable[str]) -> Iterable[Counter]
""" Counts space separated token character pairs:
[('T h i s </w>', 4}] -> {'Th': 4, 'hi': 4, 'is': 4}
"""
for token, count in self._progress_bar(self.count_tokens(words).items()):
bp_counts = Counter() # type: Counter
for ngram in token.split(' '):
bp_counts[ngram] += count
for ngram_size in range(self.ngram_min, min([self.ngram_max, len(token)]) + 1):
ngrams = [''.join(ngram) for ngram in toolz.sliding_window(ngram_size, token.split(' '))]
for ngram in ngrams:
bp_counts[''.join(ngram)] += count
yield bp_counts
Example #2
| Source File: bpe_vocab.py From NLP_Toolkit with Apache License 2.0 | 6 votes |
|
def byte_pair_counts(self, words):
# type: (Encoder, Iterable[str]) -> Iterable[Counter]
""" Counts space separated token character pairs:
[('T h i s </w>', 4}] -> {'Th': 4, 'hi': 4, 'is': 4}
"""
for token, count in self._progress_bar(self.count_tokens(words).items()):
bp_counts = Counter() # type: Counter
for ngram in token.split(' '):
bp_counts[ngram] += count
for ngram_size in range(self.ngram_min, min([self.ngram_max, len(token)]) + 1):
ngrams = [''.join(ngram) for ngram in toolz.sliding_window(ngram_size, token.split(' '))]
for ngram in ngrams:
bp_counts[''.join(ngram)] += count
yield bp_counts
Example #3
| Source File: bpe_vocab.py From NLP_Toolkit with Apache License 2.0 | 6 votes |
|
def byte_pair_counts(self, words):
# type: (Encoder, Iterable[str]) -> Iterable[Counter]
""" Counts space separated token character pairs:
[('T h i s </w>', 4}] -> {'Th': 4, 'hi': 4, 'is': 4}
"""
for token, count in self._progress_bar(self.count_tokens(words).items()):
bp_counts = Counter() # type: Counter
for ngram in token.split(' '):
bp_counts[ngram] += count
for ngram_size in range(self.ngram_min, min([self.ngram_max, len(token)]) + 1):
ngrams = [''.join(ngram) for ngram in toolz.sliding_window(ngram_size, token.split(' '))]
for ngram in ngrams:
bp_counts[''.join(ngram)] += count
yield bp_counts
Example #4
| Source File: bpe_vocab.py From NLP_Toolkit with Apache License 2.0 | 6 votes |
|
def byte_pair_counts(self, words):
# type: (Encoder, Iterable[str]) -> Iterable[Counter]
""" Counts space separated token character pairs:
[('T h i s </w>', 4}] -> {'Th': 4, 'hi': 4, 'is': 4}
"""
for token, count in self._progress_bar(self.count_tokens(words).items()):
bp_counts = Counter() # type: Counter
for ngram in token.split(' '):
bp_counts[ngram] += count
for ngram_size in range(self.ngram_min, min([self.ngram_max, len(token)]) + 1):
ngrams = [''.join(ngram) for ngram in toolz.sliding_window(ngram_size, token.split(' '))]
for ngram in ngrams:
bp_counts[''.join(ngram)] += count
yield bp_counts
Example #5
| Source File: exchange_calendar_xhkg.py From trading_calendars with Apache License 2.0 | 6 votes |
|
def special_closes(self):
return [
(
time,
HolidayCalendar([
new_years_eve(
start_date=start,
end_date=end,
days_of_week=weekdays,
),
christmas_eve(
start_date=start,
end_date=end,
days_of_week=weekdays
),
]),
)
for (start, time), (end, _) in toolz.sliding_window(
2,
toolz.concatv(self.regular_early_close_times, [(None, None)]),
)
]
Example #6
| Source File: _343.py From codetransformer with GNU General Public License v2.0 | 6 votes |
|
def pycode_to_body(co, context):
"""
Convert a Python code object to a list of AST body elements.
"""
code = Code.from_pycode(co)
# On each instruction, temporarily store all the jumps to the **next**
# instruction. This is used in _make_expr to determine when an expression
# is part of a short-circuiting expression.
for a, b in sliding_window(2, code.instrs):
a._next_target_of = b._target_of
b._next_target_of = set()
try:
body = instrs_to_body(deque(code.instrs), context)
if context.in_function_block:
return make_global_and_nonlocal_decls(code.instrs) + body
return body
finally:
# Clean up jump target data.
for i in code.instrs:
del i._next_target_of
Example #7
| Source File: encoder.py From python-bpe with MIT License | 6 votes |
|
def byte_pair_counts(self, words):
# type: (Encoder, Iterable[str]) -> Iterable[Counter]
""" Counts space separated token character pairs:
[('T h i s </w>', 4}] -> {'Th': 4, 'hi': 4, 'is': 4}
"""
for token, count in self._progress_bar(self.count_tokens(words).items()):
bp_counts = Counter() # type: Counter
for ngram in token.split(' '):
bp_counts[ngram] += count
for ngram_size in range(self.ngram_min, min([self.ngram_max, len(token)]) + 1):
ngrams = [''.join(ngram) for ngram in toolz.sliding_window(ngram_size, token.split(' '))]
for ngram in ngrams:
bp_counts[''.join(ngram)] += count
yield bp_counts
Example #8
| Source File: trading_calendar.py From trading_calendars with Apache License 2.0 | 5 votes |
|
def _group_times(all_days, times, tz, offset):
elements = [
days_at_time(
selection(all_days, start, end),
time,
tz,
offset
)
for (start, time), (end, _) in toolz.sliding_window(
2,
toolz.concatv(times, [(None, None)])
)
]
return elements[0].append(elements[1:])
Example #9
| Source File: exchange_calendar_xhkg.py From trading_calendars with Apache License 2.0 | 5 votes |
|
def special_closes_adhoc(self):
lunar_new_years_eve = (
chinese_lunar_new_year_dates - pd.Timedelta(days=1)
)[
np.in1d(
chinese_lunar_new_year_dates.weekday,
[TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY],
) & (chinese_lunar_new_year_dates.year >= 2013)
].values
def selection(arr, start, end):
predicates = []
if start is not None:
predicates.append(start.asm8 <= arr)
if end is not None:
predicates.append(arr < end.asm8)
if not predicates:
return arr
return arr[np.all(predicates, axis=0)]
return [
(time, selection(lunar_new_years_eve, start, end))
for (start, time), (end, _) in toolz.sliding_window(
2,
toolz.concatv(self.regular_early_close_times, [(None, None)]),
)
]
Example #10
| Source File: history_loader.py From catalyst with Apache License 2.0 | 4 votes |
|
def _get_adjustments_in_range(self, cf, dts, field):
if field == 'volume' or field == 'sid':
return {}
if cf.adjustment is None:
return {}
rf = self._roll_finders[cf.roll_style]
partitions = []
rolls = rf.get_rolls(cf.root_symbol, dts[0], dts[-1],
cf.offset)
tc = self._trading_calendar
adjs = {}
for front, back in sliding_window(2, rolls):
front_sid, roll_dt = front
back_sid = back[0]
dt = tc.previous_session_label(roll_dt)
if self._frequency == 'minute':
dt = tc.open_and_close_for_session(dt)[1]
roll_dt = tc.open_and_close_for_session(roll_dt)[0]
partitions.append((front_sid,
back_sid,
dt,
roll_dt))
for partition in partitions:
front_sid, back_sid, dt, roll_dt = partition
last_front_dt = self._bar_reader.get_last_traded_dt(
self._asset_finder.retrieve_asset(front_sid), dt)
last_back_dt = self._bar_reader.get_last_traded_dt(
self._asset_finder.retrieve_asset(back_sid), dt)
if isnull(last_front_dt) or isnull(last_back_dt):
continue
front_close = self._bar_reader.get_value(
front_sid, last_front_dt, 'close')
back_close = self._bar_reader.get_value(
back_sid, last_back_dt, 'close')
adj_loc = dts.searchsorted(roll_dt)
end_loc = adj_loc - 1
adj = self._make_adjustment(cf.adjustment,
front_close,
back_close,
end_loc)
try:
adjs[adj_loc].append(adj)
except KeyError:
adjs[adj_loc] = [adj]
return adjs
