DVC pipeline #2

TSNA does not work yet and recommend is unfinished

Author: Mikikrus

cluster_PCA.py

@@ -0,0 +1,25 @@
+from sklearn.decomposition import PCA
+from sklearn.neighbors import BallTree
+from sklearn.cluster import KMeans
+import pandas as pd
+import numpy as np
+import sys
+import yaml
+from pathlib import Path
+
+params = yaml.safe_load(open('params.yaml'))['cluster_PCA']
+n_components = params['n_components']
+n_clusters = params['n_clusters']
+input_file = sys.argv[1]
+Path('target').mkdir(exist_ok=True)
+df = pd.read_csv(input_file, sep=',')
+
+pca = PCA(n_components=n_components).fit(df)
+pca_df = pd.DataFrame(pca.transform(df))
+
+pca_df['clustering'] = None
+clustering = KMeans(n_clusters=n_clusters).fit(pca_df.iloc[:,:-1])
+pca_df['clustering'] = clustering.labels_
+pca_df['clustering'] = pca_df['clustering'].astype(str)
+
+pca_df.to_csv('target/pca.csv', header=None)

cluster_TSNA.py

@@ -0,0 +1,52 @@
+from sklearn.metrics import silhouette_samples, silhouette_score
+from sklearn.manifold import TSNE
+from sklearn.cluster import DBSCAN
+import matplotlib.pyplot as plt
+import matplotlib.cm as cm
+from itertools import product
+import plotly.express as px
+from tqdm.notebook import tqdm
+import pandas as pd
+import numpy as np
+import sys
+import yaml
+from pathlib import Path
+import pickle
+
+params = yaml.safe_load(open('params.yaml'))['cluster_TSNA']
+n_components = params['n_components']
+n_jobs = params['n_jobs']
+random_state = params['random_state']
+
+input_file = sys.argv[1]
+#Path(output_dir).mkdir(exist_ok=True)
+df = pd.read_csv(input_file, sep=',')
+
+tsne =  TSNE(n_components=n_components,n_jobs=n_jobs,random_state=random_state)
+#X_tsne = tsne.fit_transform(df)
+X_tsne = pd.read_csv('X_tsneL=3U=2000.csv')
+clustering = DBSCAN(eps=2, min_samples=8,n_jobs=-1).fit(X_tsne)
+X_tsne = pd.DataFrame(X_tsne,columns=['component1','component2','component3'])
+X_tsne['clustering'] = clustering.labels_
+X_tsne['clustering'] = X_tsne['clustering'].astype(str)
+X_tsne = X_tsne[X_tsne['clustering'] !='-1']
+
+clustered_useres_dicts = {}
+df.loc[:,'clustering'] = clustering.labels_
+clustered_users = df.groupby(by=df['clustering']).sum()
+clustered_users_matrix = clustered_users.to_numpy().astype(int)
+for i in range(clustered_users.to_numpy().shape[0]):
+    mask = np.where(clustered_users_matrix[i,:] >0,True,False)
+    clustered_useres_dicts[clustered_users.iloc[i].name] = \
+    dict(zip(clustered_users.columns[mask],clustered_users_matrix[i,:][mask]))
+
+#for i in clustered_useres_dicts:
+#    x = clustered_useres_dicts[i]
+#    print(sorted(x.items(),key=lambda item: item[1],reverse=True)[:5])
+
+clustered_useres_dicts[df.loc[5,"clustering"]]
+outfile = open('target/tsna.pkl','wb')
+pickle.dump(X_tsne,outfile)
+outfile.close()
+
+clustered_users.to_csv('target/tsna.csv', header=None)

dvc.yaml

@@ -0,0 +1,59 @@
+stages:
+  prepare:
+    cmd: python prepare.py reddit_scrapper/data/scrapped_data.json reddit_scrapper/data/list_of_unique_subreddits.json
+    deps:
+    - prepare.py
+    - reddit_scrapper/data/list_of_unique_subreddits.json
+    - reddit_scrapper/data/scrapped_data.json
+    params:
+    - prepare.lower_limit
+    - prepare.upper_limit
+    outs:
+    - prepared/matrix.csv
+    - prepared/matrix_bool.csv
+  generate_association_rules_final:
+    cmd: python generate_association_rules_final.py prepared/matrix_bool.csv
+    deps:
+      - generate_association_rules_final.py
+      - prepared/matrix_bool.csv
+    params:
+      - generate_association_rules_final.min_support
+    outs:
+      - target/arules-10000-00035.json
+  cluster_PCA:
+    cmd: python cluster_PCA.py prepared/matrix.csv
+    deps:
+      - cluster_PCA.py
+      - prepared/matrix.csv
+    params:
+      - cluster_PCA.n_components
+      - cluster_PCA.n_clusters
+    outs:
+      - target/pca.csv
+  cluster_TSNA:
+    cmd: python cluster_TSNA.py prepared/matrix.csv
+    deps:
+      - cluster_TSNA.py
+      - prepared/matrix.csv
+    params:
+      - cluster_TSNA.n_components
+      - cluster_TSNA.n_jobs
+      - cluster_TSNA.random_state
+    outs:
+      - target/tsna.csv
+      - target/tsna.pkl
+  scrape_user:
+    cmd: python scrape_user.py
+    params:
+      - scrape_user.username
+    outs:
+      - target/user.json
+  recommend:
+    cmd: python recommend.py target/arules-10000-00035.json target/user.json target/tsna.csv target/tsna.pkl target/pca.csv
+    deps:
+      - recommend.py
+      - target/arules-10000-00035.json
+      - target/user.json
+      - target/tsna.csv
+      - target/tsna.pkl
+      - target/pca.csv

generate_association_rules_final.py

@@ -0,0 +1,40 @@
+import json
+import numpy as np
+import pandas as pd
+import plotly.express as px
+import mlxtend as mlx
+from tqdm.notebook import tqdm, trange
+from itertools import chain
+import time
+from mlxtend.preprocessing import TransactionEncoder
+from mlxtend.frequent_patterns import apriori
+from mlxtend.frequent_patterns import association_rules
+import yaml
+import sys
+from pathlib import Path
+
+params = yaml.safe_load(open('params.yaml'))['generate_association_rules_final']
+min_support = params['min_support']
+input_path = sys.argv[1]
+#Path(output_dir).mkdir(exist_ok=True)
+
+df_bool = pd.read_csv(input_path, sep=',')
+df_bool = df_bool.iloc[: , 1:]
+frequent_itemsets = apriori(df_bool, min_support=min_support,
+                            use_colnames=True, low_memory=True,
+                            verbose=2, max_len=7)
+rules = association_rules(frequent_itemsets,
+                  metric='lift',
+                  min_threshold=1.01)
+del frequent_itemsets
+rules.to_json("target/arules-10000-00035.json")
+del rules
+
+
+
+#frequent_itemsets = apriori(df_bool, min_support=min_support, use_colnames=True)
+#rules = association_rules(frequent_itemsets,
+#                  metric='confidence',
+#                  min_threshold=0.7)
+#
+#rules.to_json(output_dir + "arules.json")

params.yaml

@@ -1,3 +1,15 @@
-prepare:
-  upper_limit: 2000
-  lower_limit: 3
+prepare:
+  upper_limit: 2000
+  lower_limit: 3
+generate_association_rules_final:
+  upper_limit: 5000
+  min_support: 0.00035
+cluster_TSNA:
+  n_components: 3
+  n_jobs: -1
+  random_state: 42
+cluster_PCA:
+  n_components: 200
+  n_clusters: 700
+scrape_user:
+  username: FirstGalacticEmpire

prepare.py

@@ -1,78 +1,68 @@
-import mlxtend as mlx
-from tqdm.notebook import tqdm, trange
-from itertools import chain,product
-import pandas as pd
-import numpy as np
-from sklearn.cluster import DBSCAN
-import json
-from sklearn.metrics import silhouette_samples, silhouette_score
-from sklearn.manifold import TSNE
-import matplotlib.pyplot as plt
-import matplotlib.cm as cm
-import plotly.express as px
-import sys
-import yaml
-from pathlib import Path
-
-params = yaml.safe_load(open('params.yaml'))['prepare']
-upper_limit = params['upper_limit']
-lower_limit = params['lower_limit']
-
-input_file = Path(sys.argv[1]) #'reddit_scrapper/data/scrapped_data.json'
-input_index = Path(sys.argv[2]) #'reddit_scrapper/data/list_of_unique_subreddits.json'
-int_output = Path('data') / 'matrix.csv'
-bool_output = Path('data') / 'matrix_bool.csv'
-data = json.load(open('reddit_scrapper/data/scrapped_data.json','r+'))
-subreddit_names_list = json.load(open('reddit_scrapper/data/list_of_unique_subreddits.json','r+'))
-subreddit_index = dict(zip(subreddit_names_list,range(len(subreddit_names_list))))
-index_subreddit =  dict(zip(range(len(subreddit_names_list)),subreddit_names_list))
-
-def create_matrix(data,matrix_width,subreddit_index):
-    """ Creates matrix filled with zeros and iterates over it filling the cells based on 
-        the subreddit-index dictionary"""
-    matrix = np.zeros(shape=(len(data),matrix_width))
-    for idx,redditor in enumerate(data.values()):
-        for key,value in redditor.items():
-            matrix[idx,subreddit_index[key]] = value
-    return matrix
-
-def filter_matrix(matrix,threshold,index_subreddit):
-    mask = np.where(matrix>threshold,True,False)
-    rows = ~np.all(mask==False,axis=1)
-    columns = ~np.all(mask==False,axis=0)
-    del mask
-    data = matrix[np.ix_(rows,columns)]
-    del rows
-    df = pd.DataFrame(data,columns=np.squeeze(np.argwhere(columns)))
-    del data,columns
-    df.rename(columns=index_subreddit,inplace=True)
-    return df
-
-def extract_most_popular_subreddits(df,lower_limit,upper_limit):
-    most_popular_reddits = df.sum(axis=0).sort_values(ascending=False)[lower_limit:upper_limit].index
-    column_base_order = dict(zip(df.columns,range(len(df.columns))))
-    column_indexes = [column_base_order[i] for i in most_popular_reddits]
-    X_np = df.to_numpy()[:, column_indexes]
-    del df,column_base_order,column_indexes
-    zero_rows = np.where(X_np.sum(axis=1) == 0)[0]
-    X_np= np.delete(X_np, zero_rows, axis=0)
-    return pd.DataFrame(X_np,columns=most_popular_reddits).drop_duplicates()
-
-matrix = create_matrix(data,len(subreddit_names_list),subreddit_index)
-df = filter_matrix(matrix,5,index_subreddit)
-df_bool = df.astype(bool).astype(int)
-df = extract_most_popular_subreddits(df,lower_limit,upper_limit)
-df.to_csv(int_output, header=None)
-
-mask = np.where(matrix>2,True,False)
-rows = ~np.all(mask==False,axis=1)
-columns = ~np.all(mask==False,axis=0)
-del mask
-data = matrix[np.ix_(rows,columns)]
-df = pd.DataFrame(data,columns=np.squeeze(np.argwhere(columns)))
-del rows
-del columns
-del data
-del matrix
-df.rename(columns=index_subreddit,inplace=True)
-df.to_csv(bool_output, header=None)
+import mlxtend as mlx
+from tqdm.notebook import tqdm, trange
+from itertools import chain,product
+import pandas as pd
+import numpy as np
+from sklearn.cluster import DBSCAN
+import json
+from sklearn.metrics import silhouette_samples, silhouette_score
+from sklearn.manifold import TSNE
+import matplotlib.pyplot as plt
+import matplotlib.cm as cm
+import plotly.express as px
+import sys
+import yaml
+from pathlib import Path
+
+params = yaml.safe_load(open('params.yaml'))['prepare']
+upper_limit = params['upper_limit']
+lower_limit = params['lower_limit']
+
+input_file = Path(sys.argv[1]) #'reddit_scrapper/data/scrapped_data.json'
+input_index = Path(sys.argv[2]) #'reddit_scrapper/data/list_of_unique_subreddits.json'
+#Path('prepared').mkdir(parents=True, exist_ok=True)
+data = json.load(open(input_file,'r+'))
+subreddit_names_list = json.load(open(input_index,'r+'))
+subreddit_index = dict(zip(subreddit_names_list,range(len(subreddit_names_list))))
+index_subreddit =  dict(zip(range(len(subreddit_names_list)),subreddit_names_list))
+
+def create_matrix(data,matrix_width,subreddit_index):
+    """ Creates matrix filled with zeros and iterates over it filling the cells based on 
+        the subreddit-index dictionary"""
+    matrix = np.zeros(shape=(len(data),matrix_width))
+    for idx,redditor in enumerate(data.values()):
+        for key,value in redditor.items():
+            matrix[idx,subreddit_index[key]] = value
+    return matrix
+
+def filter_matrix(matrix,threshold,index_subreddit):
+    mask = np.where(matrix>threshold,True,False)
+    rows = ~np.all(mask==False,axis=1)
+    columns = ~np.all(mask==False,axis=0)
+    del mask
+    data = matrix[np.ix_(rows,columns)]
+    del rows
+    df = pd.DataFrame(data,columns=np.squeeze(np.argwhere(columns)))
+    del data,columns
+    df.rename(columns=index_subreddit,inplace=True)
+    return df
+
+def extract_most_popular_subreddits(df,lower_limit,upper_limit):
+    most_popular_reddits = df.sum(axis=0).sort_values(ascending=False)[lower_limit:upper_limit].index
+    column_base_order = dict(zip(df.columns,range(len(df.columns))))
+    column_indexes = [column_base_order[i] for i in most_popular_reddits]
+    X_np = df.to_numpy()[:, column_indexes]
+    del df,column_base_order,column_indexes
+    zero_rows = np.where(X_np.sum(axis=1) == 0)[0]
+    X_np= np.delete(X_np, zero_rows, axis=0)
+    return pd.DataFrame(X_np,columns=most_popular_reddits).drop_duplicates()
+
+matrix = create_matrix(data,len(subreddit_names_list),subreddit_index)
+df = filter_matrix(matrix,5,index_subreddit)
+df = extract_most_popular_subreddits(df,lower_limit,upper_limit)
+df.to_csv('prepared/matrix.csv', header=None)
+print("Almost done...")
+#df = filter_matrix(matrix,2,index_subreddit)
+df = df.astype(bool).astype(int)
+df.rename(columns=index_subreddit,inplace=True)
+df.to_csv('prepared/matrix_bool.csv', header=None)

recommend.py

@@ -0,0 +1,45 @@
+import pandas as pd
+import sys
+import json
+import pickle
+import yaml
+from pathlib import Path
+
+user = None
+
+
+
+rules = pd.read_json(sys.argv[1])
+user = json.load(sys.argv[2])
+tsna = pd.read_csv(sys.argv[3], sep=',')
+infile = open(sys.argv[4],'rb')
+tsna_model = pickle.load(infile)
+infile.close()
+pca = pd.read_csv(sys.argv[5], sep=',')
+
+rules["antecedents"] = rules["antecedents"].apply(lambda x: frozenset(x))
+rules["consequents"] = rules["consequents"].apply(lambda x: frozenset(x))
+
+def get_rules(set_of_subreddits, top_n=100):
+    antecedents_rule =  rules['antecedents'].apply(lambda x: set_of_subreddits.issuperset(x))
+    new_rules =  rules[antecedents_rule].copy()
+#     new_rules["Coefficient"] = new_rules["confidence"] + new_rules["support"]
+    new_rules["Coefficient"] = new_rules["lift"]
+#     new_rules["Coefficient"] = new_rules["confidence"] - new_rules["support"] + new_rules["lift"] + new_rules["leverage"]+ new_rules["conviction"]
+#     new_rules["consequents"] = new_rules["consequents"].apply(lambda x: x - set_of_subreddits - OBVIOUS_SUBREDDITS)
+    new_rules["consequents"] = new_rules["consequents"].apply(lambda x: x - set_of_subreddits)
+    new_rules = new_rules[new_rules["consequents"].apply(lambda x: len(x) > 0)]
+    if len(new_rules) == 0:
+        return []
+    new_rules = new_rules[['consequents', "Coefficient"]]
+    new_rules = new_rules.explode("consequents")
+    new_rules["consequents"] = new_rules["consequents"].apply(lambda x:list(x)[0])
+    new_rules = new_rules.groupby("consequents")["Coefficient"].max().reset_index()
+    return list(new_rules.nlargest(top_n, "Coefficient")["consequents"])
+
+
+sub_red = {k for k,v in user.items()}
+print("User likes:", sub_red)
+print("User should like:",get_rules(sub_red,10))
+
+user2 = {'user' : user}