Python pandas.Series() Examples

def average_true_range(df, n):
    """
    
    :param df: pandas.DataFrame
    :param n: 
    :return: pandas.DataFrame
    """
    i = 0
    TR_l = [0]
    while i < df.index[-1]:
        TR = max(df.loc[i + 1, 'High'], df.loc[i, 'Close']) - min(df.loc[i + 1, 'Low'], df.loc[i, 'Close'])
        TR_l.append(TR)
        i = i + 1
    TR_s = pd.Series(TR_l)
    ATR = pd.Series(TR_s.ewm(span=n, min_periods=n).mean(), name='ATR_' + str(n))
    df = df.join(ATR)
    return df 

def get_topn_topm(self, s1, s2, n=10, m=3):
        s1_sorted=s1.sort_values(ascending=False)
        s1topn_index=s1_sorted.index[:n]
        d=dict()
        for i in s1topn_index:
            d[i[:-3]]=s2[i[:-3]+'wn']
        s=pd.Series(d)
        s_sorted=s.sort_values(ascending=False)
        l=len(s_sorted[s_sorted!=0])
        if(l==0):
            index=[]
            for i in range(m):
                index.append(s1topn_index[i][:-3])
            return index
        elif(l<m):
            return s_sorted.index[:l]
        else:
            return s_sorted.index[:m] 

def create_scats(self, varieties):
        self.scats = pd.DataFrame(columns=["x", "y", "color", "marker", "var"])
        for i, var in enumerate(varieties):
            self.legend.append(var)
            (x_array, y_array) = self.get_arrays(varieties, var)
            if len(x_array) <= 0:  # no data to graph!
                '''
                I am creating a single "position" for an agent that cannot
                be seen. This seems to fix the issue of colors being
                missmatched in the occasion that a group has no agents.
                '''
                x_array = [-1]
                y_array = [-1]
            elif len(x_array) != len(y_array):
                logging.debug("Array length mismatch in scatter plot")
                return
            color = get_color(varieties[var], i)
            marker = get_marker(varieties[var], i)
            scat = pd.DataFrame({"x": pd.Series(x_array),
                                 "y": pd.Series(y_array),
                                 "color": color,
                                 "marker": marker,
                                 "var": var})
            self.scats = self.scats.append(scat, ignore_index=True,
                                           sort=False) 

def money_flow_index(df, n):
    """Calculate Money Flow Index and Ratio for given data.
    
    :param df: pandas.DataFrame
    :param n: 
    :return: pandas.DataFrame
    """
    PP = (df['High'] + df['Low'] + df['Close']) / 3
    i = 0
    PosMF = [0]
    while i < df.index[-1]:
        if PP[i + 1] > PP[i]:
            PosMF.append(PP[i + 1] * df.loc[i + 1, 'Volume'])
        else:
            PosMF.append(0)
        i = i + 1
    PosMF = pd.Series(PosMF)
    TotMF = PP * df['Volume']
    MFR = pd.Series(PosMF / TotMF)
    MFI = pd.Series(MFR.rolling(n, min_periods=n).mean(), name='MFI_' + str(n))
    df = df.join(MFI)
    return df 

def trix(df, n):
    """Calculate TRIX for given data.
    
    :param df: pandas.DataFrame
    :param n: 
    :return: pandas.DataFrame
    """
    EX1 = df['Close'].ewm(span=n, min_periods=n).mean()
    EX2 = EX1.ewm(span=n, min_periods=n).mean()
    EX3 = EX2.ewm(span=n, min_periods=n).mean()
    i = 0
    ROC_l = [np.nan]
    while i + 1 <= df.index[-1]:
        ROC = (EX3[i + 1] - EX3[i]) / EX3[i]
        ROC_l.append(ROC)
        i = i + 1
    Trix = pd.Series(ROC_l, name='Trix_' + str(n))
    df = df.join(Trix)
    return df 

def ppsr(df):
    """Calculate Pivot Points, Supports and Resistances for given data
    
    :param df: pandas.DataFrame
    :return: pandas.DataFrame
    """
    PP = pd.Series((df['High'] + df['Low'] + df['Close']) / 3)
    R1 = pd.Series(2 * PP - df['Low'])
    S1 = pd.Series(2 * PP - df['High'])
    R2 = pd.Series(PP + df['High'] - df['Low'])
    S2 = pd.Series(PP - df['High'] + df['Low'])
    R3 = pd.Series(df['High'] + 2 * (PP - df['Low']))
    S3 = pd.Series(df['Low'] - 2 * (df['High'] - PP))
    psr = {'PP': PP, 'R1': R1, 'S1': S1, 'R2': R2, 'S2': S2, 'R3': R3, 'S3': S3}
    PSR = pd.DataFrame(psr)
    df = df.join(PSR)
    return df 

def vortex_indicator(df, n):
    """Calculate the Vortex Indicator for given data.
    
    Vortex Indicator described here:
        http://www.vortexindicator.com/VFX_VORTEX.PDF
    :param df: pandas.DataFrame
    :param n: 
    :return: pandas.DataFrame
    """
    i = 0
    TR = [0]
    while i < df.index[-1]:
        Range = max(df.loc[i + 1, 'High'], df.loc[i, 'Close']) - min(df.loc[i + 1, 'Low'], df.loc[i, 'Close'])
        TR.append(Range)
        i = i + 1
    i = 0
    VM = [0]
    while i < df.index[-1]:
        Range = abs(df.loc[i + 1, 'High'] - df.loc[i, 'Low']) - abs(df.loc[i + 1, 'Low'] - df.loc[i, 'High'])
        VM.append(Range)
        i = i + 1
    VI = pd.Series(pd.Series(VM).rolling(n).sum() / pd.Series(TR).rolling(n).sum(), name='Vortex_' + str(n))
    df = df.join(VI)
    return df 

def bootstrapped_split(car_ids, seed=args.seed):
    """
    # Arguments
        metadata: metadata.csv provided by Carvana (should include
        `train` column).

    # Returns
        A tuple (train_ids, test_ids)
    """
    all_ids = pd.Series(car_ids)
    train_ids, valid_ids = train_test_split(car_ids, test_size=args.test_size_float,
                                                     random_state=seed)

    np.random.seed(seed)
    bootstrapped_idx = np.random.random_integers(0, len(train_ids))
    bootstrapped_train_ids = train_ids[bootstrapped_idx]

    return generate_filenames(bootstrapped_train_ids.values), generate_filenames(valid_ids) 

def checkAndSetTransmissionLosses(losses, distances, locationalEligibility):
    """
    Check if the type of the losses are valid (i.e. a number, pandas DataFrame or a pandas Series),
    and if the given values for the losses of the transmission component are valid (i.e. between 0 and 1).
    """
    if not (isinstance(losses, int) or isinstance(losses, float) or isinstance(losses, pd.DataFrame)
            or isinstance(losses, pd.Series)):
        raise TypeError('The input data has to be a number, a pandas DataFrame or a pandas Series.')

    if isinstance(losses, int) or isinstance(losses, float):
        if losses < 0 or losses > 1:
            raise ValueError('Losses have to be values between 0 <= losses <= 1.')
        return pd.Series([float(losses) for loc in locationalEligibility.index], index=locationalEligibility.index)
    checkConnectionIndex(losses, locationalEligibility)

    losses = losses.astype(float)
    if losses.isnull().any():
        raise ValueError('The losses parameter contains values which are not a number.')
    if (losses < 0).any() or (losses > 1).any():
            raise ValueError('Losses have to be values between 0 <= losses <= 1.')
    if (1-losses*distances < 0).any():
        raise ValueError('The losses per distance multiplied with the distances result in negative values.')

    return losses 

def fit(self, X, y=None):
        """Encode categorical columns into frequency.

        Args:
            X (pandas.DataFrame): categorical columns to encode
            y (pandas.Series, optional): the target column

        Returns:
            (pandas.DataFrame): encoded columns
        """
        self.frequency_encoders = [None] * X.shape[1]

        for i, col in enumerate(X.columns):
            if self.cv is None:
                self.frequency_encoders[i] = self._get_frequency_encoder(X[col])
            else:
                self.frequency_encoders[i] = []
                for i_cv, (i_trn, i_val) in enumerate(self.cv.split(X[col]), 1):
                    self.frequency_encoders[i].append(self._get_frequency_encoder(X.loc[i_trn, col]))

        return self 

def on_balance_volume(df, n):
    """Calculate On-Balance Volume for given data.
    
    :param df: pandas.DataFrame
    :param n: 
    :return: pandas.DataFrame
    """
    i = 0
    OBV = [0]
    while i < df.index[-1]:
        if df.loc[i + 1, 'Close'] - df.loc[i, 'Close'] > 0:
            OBV.append(df.loc[i + 1, 'Volume'])
        if df.loc[i + 1, 'Close'] - df.loc[i, 'Close'] == 0:
            OBV.append(0)
        if df.loc[i + 1, 'Close'] - df.loc[i, 'Close'] < 0:
            OBV.append(-df.loc[i + 1, 'Volume'])
        i = i + 1
    OBV = pd.Series(OBV)
    OBV_ma = pd.Series(OBV.rolling(n, min_periods=n).mean(), name='OBV_' + str(n))
    df = df.join(OBV_ma)
    return df 

def coppock_curve(df, n):
    """Calculate Coppock Curve for given data.
    
    :param df: pandas.DataFrame
    :param n: 
    :return: pandas.DataFrame
    """
    M = df['Close'].diff(int(n * 11 / 10) - 1)
    N = df['Close'].shift(int(n * 11 / 10) - 1)
    ROC1 = M / N
    M = df['Close'].diff(int(n * 14 / 10) - 1)
    N = df['Close'].shift(int(n * 14 / 10) - 1)
    ROC2 = M / N
    Copp = pd.Series((ROC1 + ROC2).ewm(span=n, min_periods=n).mean(), name='Copp_' + str(n))
    df = df.join(Copp)
    return df 

def ultimate_oscillator(df):
    """Calculate Ultimate Oscillator for given data.
    
    :param df: pandas.DataFrame
    :return: pandas.DataFrame
    """
    i = 0
    TR_l = [0]
    BP_l = [0]
    while i < df.index[-1]:
        TR = max(df.loc[i + 1, 'High'], df.loc[i, 'Close']) - min(df.loc[i + 1, 'Low'], df.loc[i, 'Close'])
        TR_l.append(TR)
        BP = df.loc[i + 1, 'Close'] - min(df.loc[i + 1, 'Low'], df.loc[i, 'Close'])
        BP_l.append(BP)
        i = i + 1
    UltO = pd.Series((4 * pd.Series(BP_l).rolling(7).sum() / pd.Series(TR_l).rolling(7).sum()) + (
                2 * pd.Series(BP_l).rolling(14).sum() / pd.Series(TR_l).rolling(14).sum()) + (
                                 pd.Series(BP_l).rolling(28).sum() / pd.Series(TR_l).rolling(28).sum()),
                     name='Ultimate_Osc')
    df = df.join(UltO)
    return df 

def donchian_channel(df, n):
    """Calculate donchian channel of given pandas data frame.
    :param df: pandas.DataFrame
    :param n:
    :return: pandas.DataFrame
    """
    i = 0
    dc_l = []
    while i < n - 1:
        dc_l.append(0)
        i += 1

    i = 0
    while i + n - 1 < df.index[-1]:
        dc = max(df['High'].ix[i:i + n - 1]) - min(df['Low'].ix[i:i + n - 1])
        dc_l.append(dc)
        i += 1

    donchian_chan = pd.Series(dc_l, name='Donchian_' + str(n))
    donchian_chan = donchian_chan.shift(n - 1)
    return df.join(donchian_chan) 

def check_1d(seq: Sequence) -> Sequence:
    """Check given seq is one-dimensional. Raise error if can't be easily transformed."""
    e = ValueError("Too many dimensions.")
    for ndx, el in enumerate(seq):
        if isinstance(el, (list, tuple)):
            if len(el) > 1:
                raise e
            seq[ndx] = el[0]
        elif isinstance(el, (np.ndarray, pd.Series)):
            if el.size > 1:
                raise e
            if isinstance(el, np.ndarray):
                seq[ndx] = el.reshape(1, -1)[0][0]
            else:
                seq[ndx] = el[0]
    return seq 

def keltner_channel(df, n):
    """Calculate Keltner Channel for given data.
    
    :param df: pandas.DataFrame
    :param n: 
    :return: pandas.DataFrame
    """
    KelChM = pd.Series(((df['High'] + df['Low'] + df['Close']) / 3).rolling(n, min_periods=n).mean(),
                       name='KelChM_' + str(n))
    KelChU = pd.Series(((4 * df['High'] - 2 * df['Low'] + df['Close']) / 3).rolling(n, min_periods=n).mean(),
                       name='KelChU_' + str(n))
    KelChD = pd.Series(((-2 * df['High'] + 4 * df['Low'] + df['Close']) / 3).rolling(n, min_periods=n).mean(),
                       name='KelChD_' + str(n))
    df = df.join(KelChM)
    df = df.join(KelChU)
    df = df.join(KelChD)
    return df 

def macd(self, fast_window=12, slow_window=26, signal_window=9, col="netvalue"):
        """
        指数平滑异同移动平均线
        give the MACD index as three new columns 'MACD_DIFF/DEM/OSC' in the price table, return None

        :param fast_window: int,
        :param slow_window: int,
        :param signal_window: int, the ema window of the signal line
        :param col: string, column name in dataframe you want to calculate
        """
        EMAfast = pd.Series(self.price[col].ewm(span=fast_window).mean())
        EMAslow = pd.Series(self.price[col].ewm(span=slow_window).mean())
        # 短期ema和长期ema的差
        MACDDiff = pd.Series(EMAfast - EMAslow)
        # 该差的再次 ema 平均
        MACDDem = pd.Series(MACDDiff.ewm(span=signal_window).mean())
        # ema平均过的差和原来差的差
        MACDOsc = pd.Series(MACDDiff - MACDDem)
        self.price["MACD_DIFF_" + str(fast_window) + "_" + str(slow_window)] = MACDDiff
        self.price["MACD_DEM_" + str(fast_window) + "_" + str(slow_window)] = MACDDem
        self.price["MACD_OSC_" + str(fast_window) + "_" + str(slow_window)] = MACDOsc 

def finish(self):
        """ Compute the aggregated value in each scenario based on the parent `EventRecorder` events """
        events = self.event_recorder.events
        # Return NaN if no events found
        if len(events) == 0:
            return

        scen_id = np.empty(len(events), dtype=np.int)
        values = np.empty_like(scen_id, dtype=np.float64)

        for i, evt in enumerate(events):
            scen_id[i] = evt.scenario_index.global_id
            values[i] = pandas.Series(evt.values).aggregate(self.event_agg_func)

        df = pandas.DataFrame({'scenario_id': scen_id, 'value': values})

        # Group by scenario ...
        grouped = df.groupby('scenario_id').agg(self.recorder_agg_func)
        # ... and update the internal values
        for index, row in grouped.iterrows():
            self._values[index] = row['value'] 

def fit_transform(self, X, y=None):
        """Encode categorical columns into feature frequency counts.

        Args:
            X (pandas.DataFrame): categorical columns to encode
            y (pandas.Series, optional): the target column
        """
        self.frequency_encoders = [None] * X.shape[1]

        for i, col in enumerate(X.columns):
            if self.cv is None:
                self.frequency_encoders[i] = self._get_frequency_encoder(X[col])

                X.loc[:, col] = X[col].fillna('NaN').map(self.frequency_encoders[i]).fillna(0)
            else:
                self.frequency_encoders[i] = []
                for i_cv, (i_trn, i_val) in enumerate(self.cv.split(X[col]), 1):
                    frequency_encoder = self._get_frequency_encoder(X.loc[i_trn, col])

                    X.loc[i_val, col] = X.loc[i_val, col].fillna('NaN').map(frequency_encoder).fillna(0)
                    self.frequency_encoders[i].append(frequency_encoder)

        return X 

def fit(self, X, y):
        """Encode categorical columns into average target values.

        Args:
            X (pandas.DataFrame): categorical columns to encode
            y (pandas.Series): the target column

        Returns:
            (pandas.DataFrame): encoded columns
        """
        self.target_encoders = [None] * X.shape[1]
        self.target_mean = y.mean()

        for i, col in enumerate(X.columns):
            if self.cv is None:
                self.target_encoders[i] = self._get_target_encoder(X[col], y)
            else:
                self.target_encoders[i] = []
                for i_cv, (i_trn, i_val) in enumerate(self.cv.split(X[col], y), 1):
                    self.target_encoders[i].append(self._get_target_encoder(X.loc[i_trn, col], y[i_trn]))

        return self 

def _get_target_encoder(self, x, y):
        """Return a mapping from categories to average target values.

        Args:
            x (pandas.Series): a categorical column to encode.
            y (pandas.Series): the target column

        Returns:
            (dict): mapping from categories to average target values
        """

        assert len(x) == len(y)

        # NaN cannot be used as a key for dict. So replace it with a random
        # integer
        mean_count = pd.DataFrame({y.name: y, x.name: x.fillna(NAN_INT)}).groupby(x.name)[y.name].agg(['mean', 'count'])
        smoothing = 1 / (1 + np.exp(-(mean_count['count'] - self.min_samples) / self.smoothing))

        mean_count[y.name] = self.target_mean * (1 - smoothing) + mean_count['mean'] * smoothing
        return mean_count[y.name].to_dict() 

def checkAndSetDistances(distances, locationalEligibility, esM):
    """
    Check if the given values for the distances are valid (i.e. positive). If the distances parameter is None,
    the distances for the eligible connections are set to 1.
    """
    if distances is None:
        output('The distances of a component are set to a normalized value of 1.', esM.verbose, 0)
        distances = pd.Series([1 for loc in locationalEligibility.index], index=locationalEligibility.index)
    else:
        if not isinstance(distances, pd.Series):
            raise TypeError('Input data has to be a pandas DataFrame or Series')
        if (distances < 0).any():
            raise ValueError('Distance values smaller than 0 were detected.')
        checkConnectionIndex(distances, locationalEligibility)
    return distances 

def castToSeries(data, esM):
    if data is None:
        return None
    elif isinstance(data, pd.Series):
        return data
    isPositiveNumber(data)
    return pd.Series(data, index=list(esM.locations)) 

def test_automl():
    X, y = make_regression(n_samples=N_OBS,
                           n_features=N_FEATURE,
                           n_informative=N_IMP_FEATURE,
                           random_state=RANDOM_SEED)
    X = pd.DataFrame(X, columns=['x{}'.format(i) for i in range(X.shape[1])])
    y = pd.Series(y)
    logging.info(X.shape, y.shape)

    X_trn, X_tst, y_trn, y_tst = train_test_split(X, y, test_size=.2, random_state=RANDOM_SEED)

    model = AutoLGB(objective='regression', metric='l1')
    model.tune(X_trn, y_trn)
    model.fit(X_trn, y_trn)
    p = model.predict(X_tst)
    r = (np.random.rand(X_tst.shape[0]) * (y_trn.max() - y_trn.min()) + y_trn.min())
    logging.info('MAE (LGB): {:.4f}'.format(mae(y_tst, p)))
    assert mae(y_tst, p) < mae(y_tst, r)

    model = AutoXGB(objective='reg:linear', metric='rmse')
    model.tune(X_trn, y_trn)
    model.fit(X_trn, y_trn)
    p = model.predict(X_tst)
    r = (np.random.rand(X_tst.shape[0]) * (y_trn.max() - y_trn.min()) + y_trn.min())
    logging.info('MAE (XGB): {:.4f}'.format(mae(y_tst, p)))
    assert mae(y_tst, p) < mae(y_tst, r) 

def get_holdings_dict(code, aim=95):
    """
    通过天天基金的股票持仓数据来生成实时预测所需的持仓字典，不保证稳定性和可靠性以及 API 的连续性，慎用

    :param code:
    :param aim:
    :return:
    """
    df = get_fund_holdings(code)
    if df.ratio.sum() < 60:
        d = dt.datetime.now()
        if d.month > 3 and d.month < 8:
            year = d.year - 1
            season = 4
        elif d.month <= 3:
            year = d.year - 1
            season = 2
        else:
            year = d.year
            season = 2
        # season 只选 2，4, 具有更详细的持仓信息
        df = get_fund_holdings(code, year, season)
        if df is None:
            if season == 4:
                season = 2
            else:
                year -= 1
                season = 4
            df = get_fund_holdings(code, year, season)
    df["scode"] = df["code"].apply(ttjjcode)
    d = pd.Series(df.ratio.values, index=df.scode).to_dict()
    d = scale_dict(d, aim=aim)
    return d 

def net_worths():
    return pd.Series([100, 400, 350, 450, 200, 400, 330, 560], name="net_worth") 

def roc(self, window=10, col="netvalue"):
        """
        变动率指标
        give the ROC as a new column 'ROC' in the price table, return None, the ROC is in the unit of 1 instead of 1%

        :param window: int, the change rate between price now and window days ago
        :param col: string, column name in dataframe you want to calculate
        """
        abdiff = self.price[col].diff(window)
        deno = self.price[col].shift(window)
        reladiff = pd.Series(abdiff / deno)
        self.price["ROC" + str(window)] = reladiff 

def rsi(self, window=14, col="netvalue"):
        """
        相对强弱指标
        give the rsi as RSI column in price table

        :param window: int, MA_window
        :param col: string, column name in dataframe you want to calculate
        """
        i = 0
        UpI = [0]
        DoI = [0]
        while i + 1 <= len(self.price) - 1:
            Move = self.price.loc[i + 1, col] - self.price.loc[i, col]
            if Move > 0:
                UpD = Move
                DoD = 0
            else:
                UpD = 0
                DoD = -Move
            UpI.append(UpD)
            DoI.append(DoD)
            i = i + 1

        UpI = pd.Series(UpI)
        DoI = pd.Series(DoI)
        PosDI = pd.Series(UpI.ewm(span=window).mean())
        NegDI = pd.Series(DoI.ewm(span=window).mean())
        self.price["RSI" + str(window)] = pd.Series(PosDI / (PosDI + NegDI)) 

def _transform(self, x):
        features = {}
        features['text'] = x
        features['char_count'] = char_count(x)
        features['word_count'] = word_count(x)
        features['punctuation_count'] = punctuation_count(x)
        features['upper_case_count'] = upper_case_count(x)
        features['lower_case_count'] = lower_case_count(x)
        features['digit_count'] = digit_count(x)
        features['space_count'] = space_count(x)
        features['newline_count'] = newline_count(x)
        return pd.Series(features) 

def _check_target_shape_and_type(self, target, name):
        if not any([isinstance(target, obj_type) for obj_type in [pd.Series, np.ndarray, list]]):
            msg = '"target" must be "numpy.ndarray" or "Pandas.Series" or "list", got {} instead.'.format(type(target))
            raise SteppyToolkitError(msg)
        if not isinstance(target, list):
            assert len(target.shape) == 1, '"{}" must be 1-D. It is {}-D instead.'.format(name, len(target.shape))