Rec popular Karol

import math
import pandas as pd
import numpy as np
import time

GR_COLS = ["user_id", "session_id", "timestamp", "step"]


def get_submission_target(df):
    """Identify target rows with missing click outs."""

    mask = df["reference"].isnull() & (df["action_type"] == "clickout item")
    df_out = df[mask]

    return df_out


def get_popularity(df):
    start_time = time.time()

    # rozdziel wyswietlone hotele i ceny
    action_type = df['action_type'] == "clickout item"
    clicks = df.loc[action_type]

    impressions_explode = explode(clicks, "impressions")
    prices_explode = explode(clicks, "prices")

    # zlicz klikniecia w kazdy hotel
    clicks_count = clicks['reference'].value_counts()
    clicks_count.index = clicks_count.index.astype(int)

    # zlicz ile razy hotel byl wyswietlony
    views_count = impressions_explode['impressions'].value_counts()
    ctr = clicks_count.divide(views_count)

    # polacz hotele z ich cenami
    prices_temp = {'reference': impressions_explode.impressions, 'Price': prices_explode.prices}
    prices = pd.DataFrame(prices_temp)

    # prices.set_index('reference')

    # wyrzuc wszystkie duplikujace sie wiersze z cenami hotelow
    prices = prices.drop_duplicates('reference')

    # polacz klikniecia, wyswietlenia i CTR hotelu w jedno
    df_combined = {'Clicks': clicks_count, 'Views': views_count, "CTR": ctr}
    result = pd.DataFrame(df_combined)
    result = result.rename_axis('reference').reset_index()
    result = result.drop_duplicates('reference')
    result = result.dropna()

    # polacz poprzedni dataframe z cenami
    result2 = pd.merge(result, prices, on='reference')
    result2 = result2.sort_values('Clicks', ascending=False)
    avg = result2["Price"].mean()
   # print(avg)
   # print(result2.shape)

    # wez wyniki gdzie CTR jest wieksze od 0.5 i cena jest ponizej sredniej
    result2 = result2.loc[(result2.CTR > 0.50) & (result2.Price <= avg)]
   # print(result2.shape)
    print(result2)
    print(str(time.time() - start_time))

    return result2


def string_to_array(s):
    """Convert pipe separated string to array."""

    if isinstance(s, str):
        out = s.split("|")
    elif math.isnan(s):
        out = []
    else:
        raise ValueError("Value must be either string of nan")
    return out


def explode(df_in, col_expl):
    """Explode column col_expl of array type into multiple rows."""

    df = df_in.copy()
    df.loc[:, col_expl] = df[col_expl].apply(string_to_array)

    df_out = pd.DataFrame(
        {col: np.repeat(df[col].values,
                        df[col_expl].str.len())
         for col in df.columns.drop(col_expl)}
    )

    df_out.loc[:, col_expl] = np.concatenate(df[col_expl].values)
    df_out.loc[:, col_expl] = df_out[col_expl].apply(int)

    return df_out


def group_concat(df, gr_cols, col_concat):
    """Concatenate multiple rows into one."""

    df_out = (
        df
            .groupby(gr_cols)[col_concat]
            .apply(lambda x: ' '.join(x))
            .to_frame()
            .reset_index()
    )

    return df_out


def calc_recommendation(df_expl, df_pop):
    """Calculate recommendations based on popularity of items.

    The final data frame will have an impression list sorted according to the number of clicks per item in a reference data frame.

    :param df_expl: Data frame with exploded impression list
    :param df_pop: Data frame with items and number of clicks
    :return: Data frame with sorted impression list according to popularity in df_pop
    """

    df_expl_clicks = (
        df_expl[GR_COLS + ["impressions"]]
            .merge(df_pop,
                   left_on="impressions",
                   right_on="reference",
                   how="left")
    )

    df_out = (
        df_expl_clicks
            .assign(impressions=lambda x: x["impressions"].apply(str))
            .sort_values(GR_COLS + ["Clicks"],
                         ascending=[True, True, True, True, False])
    )

    df_out = group_concat(df_out, GR_COLS, "impressions")
    df_out.rename(columns={'impressions': 'item_recommendations'}, inplace=True)

    return df_out


def main():
    train_csv = '../../data/newsmall/train.csv'
    test_csv = '../../data/newsmall/test.csv'
    subm_csv = '../../data/newsmall/submission_popular.csv'

    # print(f"Reading {train_csv} ...")
    df_train = pd.read_csv(train_csv)
    #  print(f"Reading {test_csv} ...")
    df_test = pd.read_csv(test_csv)

    print("Get popular items...")
    df_popular = get_popularity(df_train)

    print("Identify target rows...")
    df_target = get_submission_target(df_test)

    print("Get recommendations...")
    df_expl = explode(df_target, "impressions")
    df_out = calc_recommendation(df_expl, df_popular)

    # print(f"Writing {subm_csv}...")
    df_out.to_csv(subm_csv, index=False)

    print("Finished calculating recommendations.")


if __name__ == '__main__':
    main()