DVC increased matrix_bool size

Author: Mikikrus

params.yaml

@@ -1,8 +1,9 @@
 prepare:
   upper_limit: 2000
   lower_limit: 3
+  upper_limit_arules: 10000
+  lower_limit_arules: 20
 generate_association_rules_final:
-  upper_limit: 5000
   min_support: 0.00035
 cluster_TSNA:
   n_components: 3

prepare.py

@@ -18,10 +18,13 @@
 params = yaml.safe_load(open('params.yaml'))['prepare']
 upper_limit = params['upper_limit']
 lower_limit = params['lower_limit']
+upper_limit_arules = params['upper_limit_arules']
+lower_limit_arules = params['lower_limit_arules']
 
 input_file = Path(sys.argv[1]) #'reddit_scrapper/data/scrapped_data.json'
 input_index = Path(sys.argv[2]) #'reddit_scrapper/data/list_of_unique_subreddits.json'
-#Path('prepared').mkdir(parents=True, exist_ok=True)
+Path('prepared').mkdir(parents=True, exist_ok=True)
+Path('target').mkdir(parents=True, exist_ok=True)
 data = json.load(open(input_file,'r+'))
 subreddit_names_list = json.load(open(input_index,'r+'))
 subreddit_index = dict(zip(subreddit_names_list,range(len(subreddit_names_list))))
@@ -61,13 +64,14 @@ def extract_most_popular_subreddits(df,lower_limit,upper_limit,clear_zero_rows=T
 
 
 matrix = create_matrix(data,len(subreddit_names_list),subreddit_index)
-print(matrix.shape)
 df = filter_matrix(matrix,5,index_subreddit)
 del matrix
+df.rename(columns=index_subreddit,inplace=True)
+most_popular_reddits = df.sum(axis=0).sort_values(ascending=False)[lower_limit_arules:upper_limit_arules].index
+df_bool = df.loc[:,most_popular_reddits].astype(bool).astype(int)
 df = extract_most_popular_subreddits(df,lower_limit,upper_limit)
+print(df.shape)
 df.to_csv('prepared/matrix.csv', index=False)
 print("Almost done...")
-#df = filter_matrix(matrix,2,index_subreddit)
-df = df.astype(bool).astype(int)
-df.rename(columns=index_subreddit,inplace=True)
-df.to_csv('prepared/matrix_bool.csv', index=False)
+print(df_bool.shape)
+df_bool.to_csv('prepared/matrix_bool.csv', index=False)