From 23058de6454a9adfe0ca884a79bc0243a8b3eb41 Mon Sep 17 00:00:00 2001 From: knowledge27 <84104476+KT-27@users.noreply.github.com> Date: Sat, 12 Jun 2021 07:23:53 +0530 Subject: [PATCH 1/4] Update get_accuracy.py --- Model Training/get_accuracy.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Model Training/get_accuracy.py b/Model Training/get_accuracy.py index 684b125..9aa7213 100644 --- a/Model Training/get_accuracy.py +++ b/Model Training/get_accuracy.py @@ -117,10 +117,13 @@ def get_recommendations(username, prev_sub, no_recomm, no_test): precision = tp/(tp + fp) recall = tp/(tp + fn) f1_score = 2 * precision * recall / (precision + recall) - + Specificity = tn / tn + fp + False-Positive Rate = fp / fp + tn + True-Negative Rate = tn / tn + fp + False-Negative Rate = fn / fn + tp print "Precision - " + str(precision) print "Recall - " + str(recall) print "F1score - " + str(f1_score) - dat = [prev_sub, no_recomm, no_test, tp, tn, fp, fn, precision, recall, f1_score] + dat = [prev_sub, no_recomm, no_test, tp, tn, fp, fn, precision, recall, f1_score,Specificity,False-Positive Rate,True-Negative Rate,False-Negative Rate] writer.writerow(dat) From 2bba77ebc4473557becee73438eec27559ba571d Mon Sep 17 00:00:00 2001 From: knowledge27 <84104476+KT-27@users.noreply.github.com> Date: Sat, 12 Jun 2021 07:32:44 +0530 Subject: [PATCH 2/4] added more parameters to get the complete idea --- Model Training/get_accuracy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Model Training/get_accuracy.py b/Model Training/get_accuracy.py index 9aa7213..b734a07 100644 --- a/Model Training/get_accuracy.py +++ b/Model Training/get_accuracy.py @@ -118,12 +118,12 @@ def get_recommendations(username, prev_sub, no_recomm, no_test): recall = tp/(tp + fn) f1_score = 2 * precision * recall / (precision + recall) Specificity = tn / tn + fp - False-Positive Rate = fp / fp + tn - True-Negative Rate = tn / tn + fp - False-Negative Rate = fn / fn + tp + False_Positive_Rate = fp / fp + tn + True_Negative_Rate = tn / tn + fp + False_Negative_Rate = fn / fn + tp print "Precision - " + str(precision) print "Recall - " + str(recall) print "F1score - " + str(f1_score) - dat = [prev_sub, no_recomm, no_test, tp, tn, fp, fn, precision, recall, f1_score,Specificity,False-Positive Rate,True-Negative Rate,False-Negative Rate] + dat = [prev_sub, no_recomm, no_test, tp, tn, fp, fn, precision, recall, f1_score,Specificity,False_Positive_Rate,True_Negative_Rate,False_Negative_Rate] writer.writerow(dat) From 9d934a876f245f67478c2b6ac510ab48c13d4f12 Mon Sep 17 00:00:00 2001 From: knowledge27 <84104476+KT-27@users.noreply.github.com> Date: Sat, 12 Jun 2021 07:34:08 +0530 Subject: [PATCH 3/4] Update get_accuracy.py --- Model Training/get_accuracy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Model Training/get_accuracy.py b/Model Training/get_accuracy.py index b734a07..b7f2eb0 100644 --- a/Model Training/get_accuracy.py +++ b/Model Training/get_accuracy.py @@ -96,7 +96,7 @@ def get_recommendations(username, prev_sub, no_recomm, no_test): with open('accuracy_val.csv', 'w') as f: writer = csv.writer(f) - writer.writerow(['prev_sub', 'no_recomm', 'no_test', 'tp', 'tn', 'fp', 'fn', 'precision', 'recall', 'f1_score']) + writer.writerow(['prev_sub', 'no_recomm', 'no_test', 'tp', 'tn', 'fp', 'fn', 'precision', 'recall', 'f1_score','Specificity','False_Positive_Rate','True_Negative_Rate','False_Negative_Rate']) prev_sub = 5 no_recomm = 10 From 2383259fce733aa794fbdb9b2e49bfbd573277e6 Mon Sep 17 00:00:00 2001 From: knowledge27 <84104476+KT-27@users.noreply.github.com> Date: Sat, 12 Jun 2021 07:43:16 +0530 Subject: [PATCH 4/4] added some cleaning steps --- .../codechef_problem/transform_description.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Data Transformation/codechef_problem/transform_description.py b/Data Transformation/codechef_problem/transform_description.py index 88055fc..938a702 100644 --- a/Data Transformation/codechef_problem/transform_description.py +++ b/Data Transformation/codechef_problem/transform_description.py @@ -1,11 +1,17 @@ __author__ = 'Pranay' import lemmatization, minimize_desc, remove_nondecodable_chars, removePunctuation - +from nltk.corpus import stopwords +from nltk.tokenize import word_tokenize + def transform(desc): print 'Description Transformation Started: ' desc = remove_nondecodable_chars.removeNondecodableChars(desc) desc = minimize_desc.minimizeDescription(desc) desc = removePunctuation.removePunctuation(desc) desc = lemmatization.lemmatizeDescription(desc) + stop_words = set(stopwords.words('english')) + word_tokens = word_tokenize(desc) + filtered_sentence = [w for w in word_tokens if not w.lower() in stop_words] + desc=' '.join(filtered_sentence) # print 'Description Transformation Ended ' return desc