DVC final (I hope)

Author: Mikikrus

cluster_PCA.py

@@ -3,6 +3,7 @@
 from sklearn.cluster import KMeans
 import pandas as pd
 import numpy as np
+import pickle
 import sys
 import yaml
 from pathlib import Path
@@ -22,4 +23,11 @@
 pca_df['clustering'] = clustering.labels_
 pca_df['clustering'] = pca_df['clustering'].astype(str)
 
-pca_df.to_csv('target/pca.csv', header=None)
+pca_df.to_csv('target/pca.csv', index=False)
+
+outfile = open('target/pca.pkl','wb')
+pickle.dump(pca,outfile)
+outfile.close()
+outfile = open('target/clustering.pkl','wb')
+pickle.dump(clustering,outfile)
+outfile.close()

cluster_TSNA.py

@@ -21,17 +21,20 @@
 input_file = sys.argv[1]
 #Path(output_dir).mkdir(exist_ok=True)
 df = pd.read_csv(input_file, sep=',')
-
+df = df.iloc[:-3,:]
 tsne =  TSNE(n_components=n_components,n_jobs=n_jobs,random_state=random_state)
 #X_tsne = tsne.fit_transform(df)
 X_tsne = pd.read_csv('X_tsneL=3U=2000.csv')
+X_tsne.drop(columns=['Unnamed: 0'],inplace=True)
+
 clustering = DBSCAN(eps=2, min_samples=8,n_jobs=-1).fit(X_tsne)
 X_tsne = pd.DataFrame(X_tsne,columns=['component1','component2','component3'])
 X_tsne['clustering'] = clustering.labels_
 X_tsne['clustering'] = X_tsne['clustering'].astype(str)
 X_tsne = X_tsne[X_tsne['clustering'] !='-1']
 
 clustered_useres_dicts = {}
+print(df.shape, X_tsne.shape)
 df.loc[:,'clustering'] = clustering.labels_
 clustered_users = df.groupby(by=df['clustering']).sum()
 clustered_users_matrix = clustered_users.to_numpy().astype(int)
@@ -49,4 +52,4 @@
 pickle.dump(X_tsne,outfile)
 outfile.close()
 
-clustered_users.to_csv('target/tsna.csv', header=None)
+clustered_users.to_csv('target/clustered_users.csv')

dvc.yaml

@@ -40,7 +40,7 @@ stages:
       - cluster_TSNA.n_jobs
       - cluster_TSNA.random_state
     outs:
-      - target/tsna.csv
+      - target/clustered_users.csv
       - target/tsna.pkl
   scrape_user:
     cmd: python scrape_user.py
@@ -49,9 +49,12 @@ stages:
     outs:
       - target/user.json
   recommend:
-    cmd: python recommend.py target/arules-10000-00035.json target/user.json target/pca.csv
+    cmd: python recommend.py target/arules-10000-00035.json target/user.json target/pca.csv prepared/matrix.csv target/pca.pkl target/clustering.pkl
     deps:
       - recommend.py
       - target/arules-10000-00035.json
       - target/user.json
-      - target/pca.csv
+      - target/pca.csv
+      - prepared/matrix.csv
+      - target/pca.pkl
+      - target/clustering.pkl

generate_association_rules_final.py

@@ -19,7 +19,6 @@
 #Path(output_dir).mkdir(exist_ok=True)
 
 df_bool = pd.read_csv(input_path, sep=',')
-df_bool = df_bool.iloc[: , 1:]
 frequent_itemsets = apriori(df_bool, min_support=min_support,
                             use_colnames=True, low_memory=True,
                             verbose=2, max_len=7)

params.yaml

@@ -12,4 +12,4 @@ cluster_PCA:
   n_components: 200
   n_clusters: 700
 scrape_user:
-  username: FirstGalacticEmpire
+  username: FirstGalacticEmpire

prepare.py

@@ -12,6 +12,7 @@
 import plotly.express as px
 import sys
 import yaml
+
 from pathlib import Path
 
 params = yaml.safe_load(open('params.yaml'))['prepare']
@@ -34,8 +35,7 @@ def create_matrix(data,matrix_width,subreddit_index):
         for key,value in redditor.items():
             matrix[idx,subreddit_index[key]] = value
     return matrix
-
-def filter_matrix(matrix,threshold,index_subreddit):
+def filter_matrix(matrix,threshold,indexsubreddit):
     mask = np.where(matrix>threshold,True,False)
     rows = ~np.all(mask==False,axis=1)
     columns = ~np.all(mask==False,axis=0)
@@ -46,23 +46,28 @@ def filter_matrix(matrix,threshold,index_subreddit):
     del data,columns
     df.rename(columns=index_subreddit,inplace=True)
     return df
-
-def extract_most_popular_subreddits(df,lower_limit,upper_limit):
+def extract_most_popular_subreddits(df,lower_limit,upper_limit,clear_zero_rows=True):
     most_popular_reddits = df.sum(axis=0).sort_values(ascending=False)[lower_limit:upper_limit].index
     column_base_order = dict(zip(df.columns,range(len(df.columns))))
     column_indexes = [column_base_order[i] for i in most_popular_reddits]
     X_np = df.to_numpy()[:, column_indexes]
     del df,column_base_order,column_indexes
     zero_rows = np.where(X_np.sum(axis=1) == 0)[0]
     X_np= np.delete(X_np, zero_rows, axis=0)
-    return pd.DataFrame(X_np,columns=most_popular_reddits).drop_duplicates()
+    if clear_zero_rows:
+        return pd.DataFrame(X_np,columns=most_popular_reddits).drop_duplicates()
+    else:
+        return pd.DataFrame(df,columns=most_popular_reddits).drop_duplicates()
+
 
 matrix = create_matrix(data,len(subreddit_names_list),subreddit_index)
+print(matrix.shape)
 df = filter_matrix(matrix,5,index_subreddit)
+del matrix
 df = extract_most_popular_subreddits(df,lower_limit,upper_limit)
-df.to_csv('prepared/matrix.csv', header=None)
+df.to_csv('prepared/matrix.csv', index=False)
 print("Almost done...")
 #df = filter_matrix(matrix,2,index_subreddit)
 df = df.astype(bool).astype(int)
 df.rename(columns=index_subreddit,inplace=True)
-df.to_csv('prepared/matrix_bool.csv', header=None)
+df.to_csv('prepared/matrix_bool.csv', index=False)

recommend.py

@@ -3,19 +3,32 @@
 import json
 import pickle
 import yaml
+import numpy as np
 from pathlib import Path
+from sklearn.neighbors import BallTree
 
-user = None
+params = yaml.safe_load(open('params.yaml'))['scrape_user']
+username = params['username']
 
 
 
 rules = pd.read_json(sys.argv[1])
-user = json.load(sys.argv[2])
-tsna = pd.read_csv(sys.argv[3], sep=',')
-infile = open(sys.argv[4],'rb')
-tsna_model = pickle.load(infile)
+f = open(sys.argv[2])
+user = json.load(f)
+f.close()
+#tsna = pd.read_csv(sys.argv[3], sep=',')
+#infile = open(sys.argv[4],'rb')
+#tsna_model = pickle.load(infile)
+#infile.close()
+pca_df = pd.read_csv(sys.argv[3], sep=',')
+df = pd.read_csv(sys.argv[4], sep=',')
+infile = open(sys.argv[5],'rb')
+pca = pickle.load(infile)
 infile.close()
-pca = pd.read_csv(sys.argv[5], sep=',')
+infile = open(sys.argv[6],'rb')
+clustering = pickle.load(infile)
+infile.close()
+
 
 rules["antecedents"] = rules["antecedents"].apply(lambda x: frozenset(x))
 rules["consequents"] = rules["consequents"].apply(lambda x: frozenset(x))
@@ -42,4 +55,43 @@ def get_rules(set_of_subreddits, top_n=100):
 print("User likes:", sub_red)
 print("User should like:",get_rules(sub_red,10))
 
-user2 = {'user' : user}
+
+
+user = {username : user}
+def reshape_new_user(new_user,df):
+    base_dict = dict(zip(df.columns,[0]*df.columns.shape[0]))
+    for key,value in list(new_user.values())[0].items():
+        if key in base_dict:
+            base_dict[key] = value
+    base_dict ={list(new_user.keys())[0]:base_dict}
+    new_user_df = pd.DataFrame(base_dict).T
+    new_user_df = new_user_df[df.columns.to_list()]
+    return new_user_df
+
+def get_cluster_index(pca_new_user,pca_df,NN=10):
+    tree = BallTree(pca_df.iloc[:,:-1])
+    dist, ind = tree.query(pca_new_user,k=NN)
+    new_user_cluster = pca_df.iloc[ind[0],pca_df.columns.get_loc("clustering")]\
+    .value_counts().sort_values(ascending=False).index[0]
+    return int(new_user_cluster)
+
+def get_clustered_subreddits(df,labels):
+    clustered_useres_dicts = {}
+    df.loc[:,'clustering'] = clustering.labels_
+    clustered_users = df.groupby(by=df['clustering']).sum()
+    clustered_users_matrix = clustered_users.to_numpy().astype(int)
+    for i in range(clustered_users.to_numpy().shape[0]):
+        mask = np.where(clustered_users_matrix[i,:] >0,True,False)
+        clustered_useres_dicts[clustered_users.iloc[i].name] = \
+        dict(zip(clustered_users.columns[mask],clustered_users_matrix[i,:][mask]))
+    df.drop(columns=['clustering'],inplace=True)
+    return clustered_useres_dicts
+
+new_user_df = reshape_new_user(user,df)
+pca_new_user = pca.transform(new_user_df)
+new_user_cluster_index = get_cluster_index(pca_new_user,pca_df)
+clustered_useres_dicts = get_clustered_subreddits(df,clustering.labels_)
+user_cluster = clustered_useres_dicts[new_user_cluster_index]
+user_cluster = {i[0]:i[1] for i in user_cluster.items() if i[0] not in list(user.values())[0].keys()}
+print(new_user_cluster_index,user,pd.Series(user_cluster).sort_values(ascending=False).head(20))
+