diff --git a/code/Functions.py b/code/Functions.py index 4f290d9..e8de7d0 100644 --- a/code/Functions.py +++ b/code/Functions.py @@ -7,46 +7,55 @@ import torch import torch.nn as nn import torch.nn.functional as F -from sklearn.metrics import roc_curve, auc, precision_score, recall_score, f1_score +from sklearn.metrics import roc_curve, auc, f1_score, classification_report from sentence_transformers import SentenceTransformer from StoreDataset import FileLoader, retrieve_to class LinearConfig: - pos_weight = torch.tensor(0.502773) + pos_weight = torch.tensor(0.65) criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight) train_p = 0.8 val_p = 0.1 - d_model = 128 + d_model_choices = [32, 64, 128, 256] num_epochs = 50 batch_num = 32 - learning_rate = 0.01 + learning_rate_choices = [0.1, 0.05, 0.01, 0.005, 0.001] stop_patience = 10 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + learning_rate = 0.1 + d_model = 32 + class LogitConfig: - pos_weight = torch.tensor(0.5) + pos_weight = torch.tensor(0.65) criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight) + add_conv_choices = [False, True] train_p = 0.8 val_p = 0.1 - d_model = 128 - conv_ch = 32 + d_model_choices = [32, 64, 128, 256] + conv_ch_choices = [32, 64] drop_out = 0.1 - num_epochs = 50 + num_epochs = 100 batch_num = 32 - learning_rate = 0.01 + learning_rate_choices = [0.1, 0.05, 0.01, 0.005, 0.001] patience = 10 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + add_conv = False + d_model = 64 + conv_ch = 64 + learning_rate = 0.05 + def count_files(dir): files = [f for f in os.listdir(dir)] return len(files) @@ -201,7 +210,7 @@ def run_batch(config, return total_loss -def plot_loss(history, is_linear): +def plot_loss(history, is_linear, show, length=None): plt.figure(figsize=(8, 5)) plt.plot(history["running_loss"], label='running_loss', color='blue') plt.plot(history["val_loss"], label='val_loss', color='red', linestyle="dashed") @@ -211,18 +220,21 @@ def plot_loss(history, is_linear): plt.legend() if is_linear: - length = count_files(r"test_results\loss\linear") - plt.savefig(fr"test_results\loss\linear\linear_{length + 1}.pdf") + if not length: + length = count_files(r"test_results\loss\linear") + 1 + plt.savefig(fr"test_results\loss\linear\linear_{length}.pdf") else: - length = count_files(r"test_results\loss\logprob") - plt.savefig(fr"test_results\loss\logprob\logprob_{length + 1}.pdf") - plt.show() + if not length: + length = count_files(r"test_results\loss\logprob") + 1 + plt.savefig(fr"test_results\loss\logprob\logprob_{length}.pdf") -def graph_roc_curve(config, test_loader, model, parameter_path: str): + if show: + plt.show() +def test_model(config, test_loader, model, parameter_path): model.load_state_dict(torch.load(os.path.join("models", parameter_path))) - results, targets = run_batch( + y_pred, y_true = run_batch( config, test_loader, model, @@ -230,15 +242,16 @@ def graph_roc_curve(config, test_loader, model, parameter_path: str): "test", ) - fpr, tpr, threshold = roc_curve(targets, results) + fpr, tpr, threshold = roc_curve(y_true, y_pred) auroc = auc(fpr, tpr) - results = [0 if result < 0.5 else 1 for result in results] - f1 = f1_score(targets, results, average="binary") - prec = precision_score(targets, results) - recall = recall_score(targets, results) - print(f1) - print(prec) - print(recall) + y_pred = [0 if result < 0.5 else 1 for result in y_pred] + f1 = f1_score(y_true, y_pred) + report = classification_report(y_true, y_pred, zero_division=0) + report_dict = classification_report(y_true, y_pred, output_dict=True, zero_division=0) + + return auroc, f1, fpr, tpr, report, report_dict + +def graph_roc_curve(auroc, fpr, tpr, is_linear, length=None): plt.figure() plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % auroc) @@ -246,17 +259,17 @@ def graph_roc_curve(config, test_loader, model, parameter_path: str): plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') - plt.title(f'ROC Curve for {"linear model" if model.is_linear else "logprob model"}') + plt.title(f'ROC Curve for {"linear model" if is_linear else "logprob model"}') plt.legend() - if model.is_linear: - length = count_files(r"test_results\ROC\linear") - plt.savefig(fr"test_results\ROC\linear\{length + 1}.pdf") + if is_linear: + if not length: + length = count_files(r"test_results\ROC\linear") + 1 + plt.savefig(fr"test_results\ROC\linear\{length}.pdf") else: - length = count_files(r"test_results\ROC\logprob") - plt.savefig(fr"test_results\ROC\logprob\{length + 1}.pdf") - - plt.show() + if not length: + length = count_files(r"test_results\ROC\logprob") + 1 + plt.savefig(fr"test_results\ROC\logprob\{length}.pdf") if __name__ == "__main__": store_data(ans_dataset="TruthfulDataset.jsonl", cal_dataset="CalDataset.jsonl") \ No newline at end of file diff --git a/code/Layers.py b/code/Layers.py index 07bc891..dfe9722 100644 --- a/code/Layers.py +++ b/code/Layers.py @@ -21,7 +21,8 @@ def __init__(self): self.register_buffer("pe", pe) def forward(self, x): - return x + self.pe[:x.size(1), :].unsqueeze(0) + res = x + self.pe[:x.size(1), :].unsqueeze(0) + return res class Attention(nn.Module): def __init__(self, inner_feature): @@ -54,20 +55,26 @@ class LogitModel(nn.Module): def __init__(self, config): super().__init__() + self.con = config self.is_linear = False self.pe = PositionalEncoding() self.self_attention = Attention(inner_feature=config.d_model) - self.pooling_layer = nn.AdaptiveAvgPool1d(1) - self.dropout = nn.Dropout(p=config.drop_out) - self.fc_attention = nn.Linear(in_features=config.d_model, out_features=1) + if config.add_conv and config.conv_ch: + self.conv = nn.Conv2d(1, config.conv_ch, kernel_size=(3, 3)) + self.pooling_layer = nn.AdaptiveMaxPool2d((1, 1)) + self.dropout = nn.Dropout(p=config.drop_out) + self.fc_attention = nn.Linear(in_features=config.conv_ch, out_features=1) + elif not config.add_conv: + self.pooling_layer = nn.AdaptiveAvgPool1d(1) + self.dropout = nn.Dropout(p=config.drop_out) + self.fc_attention = nn.Linear(in_features=config.d_model, out_features=1) + else: + raise RuntimeError("add_conv not properly structured") + self.fc_final = nn.Linear(in_features=3, out_features=1) - # self.conv = nn.Conv2d(1, config.conv_ch, kernel_size=(3, 3)) - # self.pooling_layer = nn.AdaptiveMaxPool2d((1, 1)) - # self.dropout = nn.Dropout(p=config.drop_out) - # self.fc_attention = nn.Linear(in_features=config.conv_ch, out_features=1) def forward(self, resp1_logits, resp2_logits, cosine_sim, entropy): position1 = self.pe(resp1_logits) @@ -78,12 +85,15 @@ def forward(self, resp1_logits, resp2_logits, cosine_sim, entropy): position2, position2, ) - - pooled = self.pooling_layer(self_attention_values) - pooled = pooled.view(pooled.size(0), -1) - # conv_output = self.conv(self_attention_values.unsqueeze(1)) - # pooled = self.pooling_layer(conv_output).view(conv_output.size(0), conv_output.size(1)) + if self.con.add_conv: + conv_output = self.conv(self_attention_values.unsqueeze(1)) + pooled = self.pooling_layer(conv_output).view(conv_output.size(0), conv_output.size(1)) + elif not self.con.add_conv: + pooled = self.pooling_layer(self_attention_values) + pooled = pooled.view(pooled.size(0), -1) + else: + raise RuntimeError("add_conv not properly structured") dropped = self.dropout(pooled) attention_score = self.fc_attention(dropped) diff --git a/code/LinearTrain.py b/code/LinearTrain.py index 2d5d52b..9ce13ac 100644 --- a/code/LinearTrain.py +++ b/code/LinearTrain.py @@ -1,49 +1,60 @@ import torch import torch.optim as optim from Layers import LinearModel -from DatasetLoader import linear_train_loader, linear_val_loader -from Functions import LinearConfig, plot_loss, run_batch, count_files - -config = LinearConfig() -device = config.device -length = count_files(r"models\linear") - -model = LinearModel(config) -optimizer = optim.Adam(model.parameters(), lr=config.learning_rate) - -prev_loss = float('inf') -history = {'running_loss': [], 'val_loss': []} - -for epoch in range(config.num_epochs): - - running_loss = run_batch(config=config, - loader=linear_train_loader, - model=model, - optimizer=optimizer, - type="train", - epoch=epoch - ) - - val_loss = run_batch(config=config, - loader=linear_val_loader, - model=model, - optimizer=optimizer, - type="val", - ) - - running_loss /= len(linear_train_loader.dataset) - val_loss /= len(linear_val_loader.dataset) - history['running_loss'].append(running_loss) - history['val_loss'].append(val_loss) - - if val_loss < prev_loss: - prev_loss = val_loss - patience_count = 0 - torch.save(model.state_dict(), fr'models\linear\{length + 1}.pth') - else: - patience_count += 1 - if patience_count >= config.stop_patience: - print(f'Early stopping at epoch {epoch + 1}') - break - -plot_loss(history, is_linear=True) +from DatasetLoader import linear_train_loader, linear_val_loader, linear_test_loader +from Functions import LinearConfig, plot_loss, run_batch, test_model, graph_roc_curve + +def linear_train(config, model_pth): + + model = LinearModel(config) + optimizer = optim.Adam(model.parameters(), lr=config.learning_rate) + + prev_loss = float('inf') + history = {'running_loss': [], 'val_loss': []} + + for epoch in range(config.num_epochs): + + running_loss = run_batch(config=config, + loader=linear_train_loader, + model=model, + optimizer=optimizer, + type="train", + epoch=epoch + ) + + val_loss = run_batch(config=config, + loader=linear_val_loader, + model=model, + optimizer=optimizer, + type="val", + ) + + running_loss /= len(linear_train_loader.dataset) + val_loss /= len(linear_val_loader.dataset) + history['running_loss'].append(running_loss) + history['val_loss'].append(val_loss) + + if val_loss < prev_loss: + prev_loss = val_loss + patience_count = 0 + torch.save(model.state_dict(), fr'models\linear\{model_pth}') + else: + patience_count += 1 + if patience_count >= config.stop_patience: + print(f'Early stopping at epoch {epoch + 1}') + break + + return history + + +if __name__ == "__main__": + model_pth = input("model's parameter path: ") + + config = LinearConfig() + history = linear_train(config, model_pth) + model = LinearModel(config) + + plot_loss(history, is_linear=True, show=True) + + auroc, f1, fpr, tpr = test_model(config, linear_test_loader, model, fr"linear\{model_pth}") + graph_roc_curve(auroc, f1, fpr, tpr, is_linear=True) \ No newline at end of file diff --git a/code/LogprobTrain.py b/code/LogprobTrain.py index b4b6143..b519f95 100644 --- a/code/LogprobTrain.py +++ b/code/LogprobTrain.py @@ -1,52 +1,73 @@ import torch import torch.optim as optim +from torch.optim.lr_scheduler import ReduceLROnPlateau from Layers import LogitModel -from Functions import LogitConfig, plot_loss, run_batch, count_files -from DatasetLoader import logprob_train_loader, logprob_val_loader +from Functions import LogitConfig, plot_loss, run_batch, test_model, graph_roc_curve +from DatasetLoader import logprob_train_loader, logprob_val_loader, logprob_test_loader -config = LogitConfig() -device = config.device -length = count_files(r"models\logprob") +def logprob_train(config, model_pth): -model = LogitModel(config) -optimizer = optim.Adam(model.parameters(), lr=config.learning_rate) + model = LogitModel(config) + optimizer = optim.Adam(model.parameters(), lr=config.learning_rate) + scheduler = ReduceLROnPlateau( + optimizer, + mode='min', + patience=5, + factor=0.5 + ) -history = {"running_loss": [], "val_loss": []} -prev_loss = float("inf") + history = {"running_loss": [], "val_loss": []} + prev_loss = float("inf") -for epoch in range(config.num_epochs): - model.train() + for epoch in range(config.num_epochs): + model.train() - running_loss = run_batch( - config=config, - loader=logprob_train_loader, - model=model, - optimizer=optimizer, - type="train", - epoch=epoch - ) + running_loss = run_batch( + config=config, + loader=logprob_train_loader, + model=model, + optimizer=optimizer, + type="train", + epoch=epoch + ) - val_loss = run_batch( - config=config, - loader=logprob_val_loader, - model=model, - optimizer=optimizer, - type="val", - ) + val_loss = run_batch( + config=config, + loader=logprob_val_loader, + model=model, + optimizer=optimizer, + type="val", + ) + + running_loss /= len(logprob_train_loader.dataset) + val_loss /= len(logprob_val_loader.dataset) + history["running_loss"].append(running_loss) + history["val_loss"].append(val_loss) + + scheduler.step(val_loss) + + if val_loss < prev_loss: + prev_loss = val_loss + patience_count = 0 + torch.save(model.state_dict(), fr'models\logprob\{model_pth}') + else: + patience_count += 1 + if patience_count >= config.patience: + print(f"model stopped at {epoch} epochs.") + break + + return history + +if __name__ == "__main__": + model_pth = input("model's parameter path: ") + + config = LogitConfig() + history = logprob_train(config, model_pth) + model = LogitModel(config) + + plot_loss(history, is_linear=False, show=True) + + auroc, f1, fpr, tpr = test_model(config, logprob_test_loader, model, fr"logprob\{model_pth}") + graph_roc_curve(auroc, f1, fpr, tpr, is_linear=False) - running_loss /= len(logprob_train_loader.dataset) - val_loss /= len(logprob_val_loader.dataset) - history["running_loss"].append(running_loss) - history["val_loss"].append(val_loss) - - if val_loss < prev_loss: - prev_loss = val_loss - patience_count = 0 - torch.save(model.state_dict(), fr'models\logprob\{length + 1}.pth') - else: - patience_count += 1 - if patience_count >= config.patience: - print(f"model stopped at {epoch} epochs.") - break - -plot_loss(history, is_linear=False) \ No newline at end of file + \ No newline at end of file diff --git a/code/Main.py b/code/Main.py new file mode 100644 index 0000000..64b0869 --- /dev/null +++ b/code/Main.py @@ -0,0 +1,126 @@ +import os +import json +from LinearTrain import linear_train +from LogprobTrain import logprob_train +from Functions import LogitConfig, LinearConfig, test_model, plot_loss, graph_roc_curve +from DatasetLoader import logprob_test_loader, linear_test_loader +from Layers import LogitModel, LinearModel + +def logprob_train_model(config, model, count): + history = logprob_train(config, f"sec_try{count}.pth") + auroc, f1, fpr, tpr, report, report_dict = test_model(config, logprob_test_loader, model, fr"logprob\sec_try{count}.pth") + + print(f"\n{report}\nauroc: {auroc}") + + with open(r"history\logprob_history.txt", "a", encoding="utf-8") as f: + f.write(f"\n{count}\n dimension: {config.d_model} lr: {config.learning_rate}{f" CNN channel: {config.conv_ch}" if config.add_conv else ""} auroc: {auroc} f1: {f1}") + + if auroc >= 0.60 and f1 >= 0.70: + plot_loss(history, is_linear=False, show=False) + graph_roc_curve( + auroc=auroc, + fpr=fpr, + tpr=tpr, + is_linear=False, + length=count + ) + + report_dict = report_dict["1.0"] + del report_dict["support"] + report_dict["ID"] = count + report_dict["auroc"] = auroc + report_dict["d_model"] = config.d_model + report_dict["learning_rate"] = config.learning_rate + report_dict["conv_ch"] = config.conv_ch if config.add_conv else None + + with open(r"history\logprob_best.jsonl", "a", encoding="utf-8") as f: + json.dump(report_dict, f, ensure_ascii=False) + f.write("\n") + + else: + os.remove(fr"models\logprob\sec_try{count}.pth") + +def linear_train_model(config, model, count): + history = linear_train(config, f"sec_try{count}.pth") + auroc, f1, fpr, tpr, report, report_dict = test_model(config, linear_test_loader, model, fr"linear\sec_try{count}.pth") + print(f"\n{report}\nauroc: {auroc}") + + with open(r"history\linear_history.txt", "a", encoding="utf-8") as f: + f.write(f"\n{count}\n dimension: {config.d_model} lr: {config.learning_rate} auroc: {auroc} f1: {f1}") + + if auroc >= 0.60 and f1 >= 0.70: + plot_loss(history, is_linear=True, show=False) + graph_roc_curve( + auroc=auroc, + fpr=fpr, + tpr=tpr, + is_linear=True + ) + + report_dict = report_dict["1.0"] + del report_dict["support"] + report_dict["ID"] = count + report_dict["auroc"] = auroc + report_dict["d_model"] = config.d_model + report_dict["learning_rate"] = config.learning_rate + + with open(r"history\linear_best.jsonl", "a", encoding="utf-8") as f: + json.dump(report_dict, f, ensure_ascii=False) + f.write("\n") + + else: + os.remove(fr"models\linear\sec_try{count}.pth") + +if __name__ == "__main__": + while True: + type = input("linear or logporb --> ") + + if type == "linear": + config = LinearConfig() + count = 1 + total = len(config.d_model_choices) * len (config.learning_rate_choices) + + for dimension in config.d_model_choices: + config.d_model = dimension + for lr in config.learning_rate_choices: + config.learning_rate = lr + print(f"{count} / {total}\n") + print(f"model dimension: {config.d_model} learning rate: {config.learning_rate}") + model = LinearModel(config) + linear_train_model(config, model, count) + count += 1 + break + + elif type == "logprob": + config = LogitConfig() + count = 1 + total = len(config.d_model_choices) * len(config.learning_rate_choices) * (1 + len(config.conv_ch_choices) if len(config.add_conv_choices) == 2 else 1) + + for is_conv in config.add_conv_choices: + print("----With CNN layer----\n" if config.add_conv else "----No CNN layer----\n") + config.add_conv = is_conv + for item in config.d_model_choices: + config.d_model = item + for lr in config.learning_rate_choices: + config.learning_rate = lr + if config.add_conv: + for i in config.conv_ch_choices: + config.conv_ch = i + + print(f"{count} / {total}\n") + print(f"model dimension: {config.d_model} learning rate: {config.learning_rate} CNN channel: {config.conv_ch}") + model = LogitModel(config) + logprob_train_model(config, model, count) + count += 1 + + else: + print(f"{count} / {total}") + print(f"model dimension: {config.d_model} learning rate: {config.learning_rate}") + model = LogitModel(config) + logprob_train_model(config, model, count) + count += 1 + break + + else: + print("\n---------input [linear] or [logprob]----------\n") + diff --git a/code/TestModel.py b/code/TestModel.py deleted file mode 100644 index 8284810..0000000 --- a/code/TestModel.py +++ /dev/null @@ -1,23 +0,0 @@ -from Functions import LogitConfig, LinearConfig, graph_roc_curve, count_files -from Layers import LogitModel, LinearModel -from DatasetLoader import logprob_test_loader, linear_test_loader - -config = LogitConfig() -model = LogitModel(config) -length = count_files(r"models\logprob") -graph_roc_curve( - config, - logprob_test_loader, - model, - fr"logprob\{length}.pth", -) - -config = LinearConfig() -model = LinearModel(config) -length = count_files(r"models\linear") -graph_roc_curve( - config, - linear_test_loader, - model, - fr"linear\{length}.pth", -) \ No newline at end of file diff --git a/code/__pycache__/Functions.cpython-314.pyc b/code/__pycache__/Functions.cpython-314.pyc index 547e8e6..27ba93c 100644 Binary files a/code/__pycache__/Functions.cpython-314.pyc and b/code/__pycache__/Functions.cpython-314.pyc differ diff --git a/code/__pycache__/Layers.cpython-314.pyc b/code/__pycache__/Layers.cpython-314.pyc index 7dd9135..66636e0 100644 Binary files a/code/__pycache__/Layers.cpython-314.pyc and b/code/__pycache__/Layers.cpython-314.pyc differ diff --git a/code/__pycache__/LinearTrain.cpython-314.pyc b/code/__pycache__/LinearTrain.cpython-314.pyc new file mode 100644 index 0000000..c9bfcbf Binary files /dev/null and b/code/__pycache__/LinearTrain.cpython-314.pyc differ diff --git a/code/__pycache__/LogprobTrain.cpython-314.pyc b/code/__pycache__/LogprobTrain.cpython-314.pyc new file mode 100644 index 0000000..367ed42 Binary files /dev/null and b/code/__pycache__/LogprobTrain.cpython-314.pyc differ diff --git a/code/test.py b/code/test.py new file mode 100644 index 0000000..e22ca48 --- /dev/null +++ b/code/test.py @@ -0,0 +1,203 @@ +import torch +import os +import matplotlib.pyplot as plt +from Functions import test_model, LogitConfig, LinearConfig +from Layers import LogitModel, LinearModel +from DatasetLoader import logprob_test_loader, linear_test_loader +from StoreDataset import FileLoader + +COLORS = ["b", "g", "r", "c", "m", "y", "k"] + +if __name__ == "__main__": + mode = input("logprob or linear or both --> ") + + if mode == "logprob": + + compare = False + color = 0 + + plt.figure() + plt.plot([0, 1], color="grey", linestyle="dashed", label="random model") + + while True: + + config = LogitConfig() + + location = int(input("ID of model to test -->")) + file = FileLoader(rf"..\history\logprob_best.jsonl") + + parameters = None + for item in file: + if item["ID"] == location: + parameters = item + break + if not parameters: + raise RuntimeError("ID not in file") + + if parameters["conv_ch"]: + config.add_conv = True + config.conv_ch = parameters["conv_ch"] + else: + config.add_conv = False + + config.d_model = parameters["d_model"] + + model = LogitModel(config) + + auroc, f1, fpr, tpr, report, report_dict = test_model(config, logprob_test_loader, model, parameter_path=fr"logprob\sec_try{location}.pth") + label = input("model label: ") + plt.plot(fpr, tpr, label=f'{label} (AUC=%0.2f)' % auroc, color=f"{COLORS[color]}") + + print(report) + print(auroc) + + compare = input("Compare between models? [y/n] ") == "y" + color += 1 + if color >= 7: + color = 0 + + if not compare: + break + + plt.xlim([0.0, 1.0]) + plt.ylim([0.0, 1.05]) + plt.xlabel('False Positive Rate') + plt.ylabel('True Positive Rate') + plt.title(f'ROC Curves') + plt.legend() + plt.show() + + elif mode == "linear": + + compare = False + + plt.figure() + plt.plot([0, 1], color="grey", linestyle="dashed", label="random model") + + while True: + config = LinearConfig() + + location = int(input("ID of model to test -->")) + file = FileLoader(rf"..\history\linear_best.jsonl") + + parameters = None + for item in file: + if item["ID"] == location: + parameters = item + break + if not parameters: + raise RuntimeError("ID not in file") + + config.d_model = parameters["d_model"] + + model = LinearModel(config) + + auroc, f1, fpr, tpr, report, report_dict = test_model(config, linear_test_loader, model, parameter_path=fr"linear\sec_try{location}.pth") + label = input("model label: ") + plt.plot(fpr, tpr, label=f'{label} (AUC=%0.2f)' % auroc) + + print(report) + print(auroc) + + compare = input("Compare between models? [y/n] ") == "y" + + if not compare: + break + + plt.xlim([0.0, 1.0]) + plt.ylim([0.0, 1.05]) + plt.xlabel('False Positive Rate') + plt.ylabel('True Positive Rate') + plt.title(f'ROC Curves') + plt.legend() + plt.show() + + + elif mode == "both": + + color = 0 + compare = False + plt.figure() + plt.plot([0, 1], color="grey", linestyle="dashed", label="random model") + + while True: + + config = LogitConfig() + + location = int(input("ID of model to test -->")) + file = FileLoader(rf"..\history\logprob_best.jsonl") + + parameters = None + for item in file: + if item["ID"] == location: + parameters = item + break + if not parameters: + raise RuntimeError("ID not in file") + + if parameters["conv_ch"]: + config.add_conv = True + config.conv_ch = parameters["conv_ch"] + else: + config.add_conv = False + + config.d_model = parameters["d_model"] + + model = LogitModel(config) + + auroc, f1, fpr, tpr, report, report_dict = test_model(config, logprob_test_loader, model, parameter_path=fr"logprob\sec_try{location}.pth") + label = input("model label: ") + plt.plot(fpr, tpr, label=f'{label} (AUC=%0.2f)' % auroc, color=f"{COLORS[color]}") + + print(report) + print(auroc) + + compare = input("Compare between logprob models? [y/n] ") == "y" + color += 1 + if color >= 7: + color = 0 + + if not compare: + break + + while True: + config = LinearConfig() + + location = int(input("ID of model to test for hybrid model -->")) + file = FileLoader(rf"..\history\linear_best.jsonl") + + parameters = None + for item in file: + if item["ID"] == location: + parameters = item + break + if not parameters: + raise RuntimeError("ID not in file") + + config.d_model = parameters["d_model"] + + model = LinearModel(config) + + auroc, f1, fpr, tpr, report, report_dict = test_model(config, linear_test_loader, model, parameter_path=fr"linear\sec_try{location}.pth") + label = input("model label: ") + plt.plot(fpr, tpr, label=f'{label} (AUC=%0.2f)' % auroc) + + print(report) + print(auroc) + + compare = input("Compare between models? [y/n] ") == "y" + + if not compare: + break + + plt.xlim([0.0, 1.0]) + plt.ylim([0.0, 1.05]) + plt.xlabel('False Positive Rate') + plt.ylabel('True Positive Rate') + plt.title(f'ROC Curves') + plt.legend() + plt.show() + + else: + raise TypeError("Mode not logprob or linear or all") + diff --git a/history.txt b/history.txt deleted file mode 100644 index 8c8be64..0000000 --- a/history.txt +++ /dev/null @@ -1,172 +0,0 @@ -hallucination count = 541 -non hallucination count = 275 -weight = 1: 1.96727 - -Trial 1: - -class LinearConfig: - - criterion = nn.BCELoss() - - train_p = 0.75 - val_p = 0.125 - - d_model = 32 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - stop_patience = 10 - -class LogitConfig: - - criterion = nn.BCELoss() - - train_p = 0.75 - val_p = 0.125 - - d_model = 32 - drop_out = 0.2 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - patience = 10 - - -trial 2: - -class LinearConfig: - - criterion = nn.BCELoss() - - train_p = 0.75 - val_p = 0.125 - - d_model = 64 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - stop_patience = 10 - -class LogitConfig: - - criterion = nn.BCELoss() - - train_p = 0.75 - val_p = 0.125 - - d_model = 64 - drop_out = 0.2 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - patience = 10 - -trial 3: - -class LinearConfig: - - criterion = nn.BCELoss() - - train_p = 0.75 - val_p = 0.125 - - d_model = 64 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.05 - stop_patience = 10 - - -class LogitConfig: - - criterion = nn.BCELoss() - - train_p = 0.75 - val_p = 0.125 - - d_model = 64 - drop_out = 0.1 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.05 - patience = 10 - -trial 4: - criterion = nn.BCELoss() - - train_p = 0.8 - val_p = 0.1 - - d_model = 64 - drop_out = 0.0 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - patience = 10 - -trial 5: - -class LinearConfig: - - pos_weight = torch.tensor(0.502773) - criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight) - - train_p = 0.75 - val_p = 0.125 - - d_model = 64 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.05 - stop_patience = 10 - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - -class LogitConfig: - - pos_weight = torch.tensor(0.502773) - criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight) - - train_p = 0.8 - val_p = 0.1 - - d_model = 64 - drop_out = 0.0 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - patience = 10 - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - -trial 6: - -this time with convolutional network - -class LogitConfig: - - pos_weight = torch.tensor(0.5) - criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight) - - train_p = 0.8 - val_p = 0.1 - - d_model = 128 ---> conv_ch = 32 - drop_out = 0.1 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - patience = 10 - -AUROC of 0.53 not good \ No newline at end of file diff --git a/history/linear_best.jsonl b/history/linear_best.jsonl new file mode 100644 index 0000000..a187cad --- /dev/null +++ b/history/linear_best.jsonl @@ -0,0 +1,6 @@ +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": 1, "auroc": 0.6343702192758797, "d_model": 32, "learning_rate": 0.1} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": 2, "auroc": 0.6583375828658847, "d_model": 32, "learning_rate": 0.05} +{"precision": 0.7209302325581395, "recall": 0.8378378378378378, "f1-score": 0.775, "ID": 3, "auroc": 0.6591874893761687, "d_model": 32, "learning_rate": 0.01} +{"precision": 0.7205882352941176, "recall": 0.8828828828828829, "f1-score": 0.7935222672064778, "ID": 4, "auroc": 0.6554479007309196, "d_model": 32, "learning_rate": 0.005} +{"precision": 0.71900826446281, "recall": 0.7837837837837838, "f1-score": 0.75, "ID": 5, "auroc": 0.6568077511473739, "d_model": 32, "learning_rate": 0.001} +{"precision": 0.7266187050359713, "recall": 0.9099099099099099, "f1-score": 0.808, "ID": 6, "auroc": 0.65068842427333, "d_model": 64, "learning_rate": 0.1} diff --git a/history/linear_best.txt b/history/linear_best.txt new file mode 100644 index 0000000..fbb6299 --- /dev/null +++ b/history/linear_best.txt @@ -0,0 +1,39 @@ + +5 + dimension: 32 lr: 0.1 +6 + dimension: 32 lr: 0.05 +7 + dimension: 32 lr: 0.01 +8 + dimension: 32 lr: 0.005 +9 + dimension: 32 lr: 0.001 +10 + dimension: 64 lr: 0.05 +11 + dimension: 64 lr: 0.01 +12 + dimension: 64 lr: 0.005 +13 + dimension: 64 lr: 0.001 +14 + dimension: 128 lr: 0.1 +15 + dimension: 128 lr: 0.05 +16 + dimension: 128 lr: 0.01 +17 + dimension: 128 lr: 0.005 +18 + dimension: 128 lr: 0.001 +19 + dimension: 256 lr: 0.1 +20 + dimension: 256 lr: 0.05 +21 + dimension: 256 lr: 0.01 +22 + dimension: 256 lr: 0.005 +23 + dimension: 256 lr: 0.001 \ No newline at end of file diff --git a/history/linear_history.txt b/history/linear_history.txt new file mode 100644 index 0000000..908d898 --- /dev/null +++ b/history/linear_history.txt @@ -0,0 +1,55 @@ + +1 + dimension: 32 lr: 0.1 auroc: 0.6195818459969403 f1: 0.6730769230769231 +2 + dimension: 32 lr: 0.05 auroc: 0.668706442291348 f1: 0.6926829268292682 +3 + dimension: 32 lr: 0.01 auroc: 0.6583375828658848 f1: 0.6382978723404256 +4 + dimension: 32 lr: 0.005 auroc: 0.6408295087540371 f1: 0.6321243523316062 +5 + dimension: 32 lr: 0.001 auroc: 0.6527281998980113 f1: 0.6382978723404256 +6 + dimension: 64 lr: 0.1 auroc: 0.5884752677205507 f1: 0.8072727272727273 +7 + dimension: 64 lr: 0.05 auroc: 0.6423593404725482 f1: 0.7368421052631579 +8 + dimension: 64 lr: 0.01 auroc: 0.6338602753697095 f1: 0.6421052631578947 +9 + dimension: 64 lr: 0.005 auroc: 0.6226415094339622 f1: 0.6428571428571429 +10 + dimension: 64 lr: 0.001 auroc: 0.6610572836987931 f1: 0.6829268292682927 +11 + dimension: 128 lr: 0.1 auroc: 0.6656467788543261 f1: 0.7239819004524887 +12 + dimension: 128 lr: 0.05 auroc: 0.6692163861975183 f1: 0.7207207207207207 +13 + dimension: 128 lr: 0.01 auroc: 0.6421893591704912 f1: 0.6321243523316062 +14 + dimension: 128 lr: 0.005 auroc: 0.6688764235934048 f1: 0.6926829268292682 +15 + dimension: 128 lr: 0.001 auroc: 0.6447390787013428 f1: 0.6288659793814433 +16 + dimension: 256 lr: 0.1 auroc: 0.6600373958864526 f1: 0.7985347985347986 +17 + dimension: 256 lr: 0.05 auroc: 0.6357300696923339 f1: 0.7985347985347986 +18 + dimension: 256 lr: 0.01 auroc: 0.6450790413054565 f1: 0.6338797814207651 +19 + dimension: 256 lr: 0.005 auroc: 0.6693863674995751 f1: 0.6111111111111112 +20 + dimension: 256 lr: 0.001 auroc: 0.6342002379738229 f1: 0.6288659793814433 + +second try: +1 + dimension: 32 lr: 0.1 auroc: 0.6343702192758797 f1: 0.8072727272727273 +2 + dimension: 32 lr: 0.05 auroc: 0.6583375828658847 f1: 0.8072727272727273 +3 + dimension: 32 lr: 0.01 auroc: 0.6591874893761687 f1: 0.775 +4 + dimension: 32 lr: 0.005 auroc: 0.6554479007309196 f1: 0.7935222672064778 +5 + dimension: 32 lr: 0.001 auroc: 0.6568077511473739 f1: 0.75 +6 + dimension: 64 lr: 0.1 auroc: 0.65068842427333 f1: 0.808 \ No newline at end of file diff --git a/history/logprob_best.jsonl b/history/logprob_best.jsonl new file mode 100644 index 0000000..13a6da6 --- /dev/null +++ b/history/logprob_best.jsonl @@ -0,0 +1,3 @@ +{"precision": 0.7014925373134329, "recall": 0.8545454545454545, "f1-score": 0.7704918032786885, "ID": 1, "auroc": 0.6409090909090909, "d_model": 32, "learning_rate": 0.1, "conv_ch": null} +{"precision": 0.6626506024096386, "recall": 1.0, "f1-score": 0.7971014492753623, "ID": 14, "auroc": 0.605844155844156, "d_model": 128, "learning_rate": 0.005, "conv_ch": null} +{"precision": 0.6666666666666666, "recall": 0.9090909090909091, "f1-score": 0.7692307692307693, "ID": 39, "auroc": 0.6233766233766234, "d_model": 64, "learning_rate": 0.001, "conv_ch": 32} diff --git a/history/logprob_best_first_try.txt b/history/logprob_best_first_try.txt new file mode 100644 index 0000000..458a31a --- /dev/null +++ b/history/logprob_best_first_try.txt @@ -0,0 +1,9 @@ + +1 + dimension: 64 lr: 0.005 auroc: 0.612987012987013 f1: 0.6407766990291263 +2 + dimension: 128 lr: 0.005 auroc: 0.6071428571428571 f1: 0.7704918032786885 +3 + dimension: 32 lr: 0.005 CNN channel: 64 auroc: 0.6032467532467531 f1: 0.66 +4 + dimension: 64 lr: 0.0001 CNN channel: 64 auroc: 0.6142857142857142 f1: 0.47368421052631576 \ No newline at end of file diff --git a/history/logprob_history.txt b/history/logprob_history.txt new file mode 100644 index 0000000..0e8b5e2 --- /dev/null +++ b/history/logprob_history.txt @@ -0,0 +1,123 @@ + +1 + dimension: 32 lr: 0.1 auroc: 0.6058441558441559 f1: 0.7971014492753623 +1 + dimension: 32 lr: 0.1 auroc: 0.6409090909090909 f1: 0.7704918032786885 +2 + dimension: 32 lr: 0.05 auroc: 0.5285714285714286 f1: 0.7971014492753623 +3 + dimension: 32 lr: 0.01 auroc: 0.5 f1: 0.7971014492753623 +4 + dimension: 32 lr: 0.005 auroc: 0.525974025974026 f1: 0.7611940298507462 +5 + dimension: 32 lr: 0.001 auroc: 0.5753246753246752 f1: 0.8029197080291971 +6 + dimension: 64 lr: 0.1 auroc: 0.5376623376623377 f1: 0.7619047619047619 +7 + dimension: 64 lr: 0.05 auroc: 0.5467532467532467 f1: 0.7941176470588235 +8 + dimension: 64 lr: 0.01 auroc: 0.4792207792207792 f1: 0.7971014492753623 +9 + dimension: 64 lr: 0.005 auroc: 0.5064935064935064 f1: 0.8120300751879699 +10 + dimension: 64 lr: 0.001 auroc: 0.5993506493506493 f1: 0.7971014492753623 +11 + dimension: 128 lr: 0.1 auroc: 0.5922077922077922 f1: 0.7559055118110236 +12 + dimension: 128 lr: 0.05 auroc: 0.5623376623376624 f1: 0.7846153846153846 +13 + dimension: 128 lr: 0.01 auroc: 0.5149350649350649 f1: 0.7751937984496124 +14 + dimension: 128 lr: 0.005 auroc: 0.605844155844156 f1: 0.7971014492753623 +15 + dimension: 128 lr: 0.001 auroc: 0.4474025974025974 f1: 0.7971014492753623 +16 + dimension: 256 lr: 0.1 auroc: 0.5272727272727272 f1: 0.7971014492753623 +17 + dimension: 256 lr: 0.05 auroc: 0.5850649350649351 f1: 0.7424242424242424 +18 + dimension: 256 lr: 0.01 auroc: 0.5240259740259741 f1: 0.7794117647058824 +19 + dimension: 256 lr: 0.005 auroc: 0.5915584415584416 f1: 0.7971014492753623 +20 + dimension: 256 lr: 0.001 auroc: 0.5220779220779221 f1: 0.7971014492753623 +21 + dimension: 32 lr: 0.1 CNN channel: 32 auroc: 0.5363636363636364 f1: 0.8 +22 + dimension: 32 lr: 0.1 CNN channel: 64 auroc: 0.5461038961038961 f1: 0.7751937984496124 +23 + dimension: 32 lr: 0.05 CNN channel: 32 auroc: 0.4980519480519481 f1: 0.7703703703703704 +24 + dimension: 32 lr: 0.05 CNN channel: 64 auroc: 0.5201298701298701 f1: 0.7794117647058824 +25 + dimension: 32 lr: 0.01 CNN channel: 32 auroc: 0.5116883116883116 f1: 0.7883211678832117 +26 + dimension: 32 lr: 0.01 CNN channel: 64 auroc: 0.4831168831168831 f1: 0.7971014492753623 +27 + dimension: 32 lr: 0.005 CNN channel: 32 auroc: 0.5285714285714286 f1: 0.7971014492753623 +28 + dimension: 32 lr: 0.005 CNN channel: 64 auroc: 0.44999999999999996 f1: 0.7971014492753623 +29 + dimension: 32 lr: 0.001 CNN channel: 32 auroc: 0.5740259740259741 f1: 0.7883211678832117 +30 + dimension: 32 lr: 0.001 CNN channel: 64 auroc: 0.4850649350649351 f1: 0.7971014492753623 +31 + dimension: 64 lr: 0.1 CNN channel: 32 auroc: 0.5285714285714285 f1: 0.7971014492753623 +32 + dimension: 64 lr: 0.1 CNN channel: 64 auroc: 0.5487012987012987 f1: 0.7424242424242424 +33 + dimension: 64 lr: 0.05 CNN channel: 32 auroc: 0.5272727272727273 f1: 0.7971014492753623 +34 + dimension: 64 lr: 0.05 CNN channel: 64 auroc: 0.5292207792207793 f1: 0.7971014492753623 +35 + dimension: 64 lr: 0.01 CNN channel: 32 auroc: 0.5525974025974025 f1: 0.7751937984496124 +36 + dimension: 64 lr: 0.01 CNN channel: 64 auroc: 0.5233766233766234 f1: 0.7971014492753623 +37 + dimension: 64 lr: 0.005 CNN channel: 32 auroc: 0.5428571428571429 f1: 0.7971014492753623 +38 + dimension: 64 lr: 0.005 CNN channel: 64 auroc: 0.5142857142857142 f1: 0.8059701492537313 +39 + dimension: 64 lr: 0.001 CNN channel: 32 auroc: 0.6233766233766234 f1: 0.7692307692307693 +40 + dimension: 64 lr: 0.001 CNN channel: 64 auroc: 0.5889610389610389 f1: 0.7971014492753623 +41 + dimension: 128 lr: 0.1 CNN channel: 32 auroc: 0.5285714285714285 f1: 0.7611940298507462 +42 + dimension: 128 lr: 0.1 CNN channel: 64 auroc: 0.4967532467532467 f1: 0.7971014492753623 +43 + dimension: 128 lr: 0.05 CNN channel: 32 auroc: 0.5454545454545454 f1: 0.7971014492753623 +44 + dimension: 128 lr: 0.05 CNN channel: 64 auroc: 0.5175324675324675 f1: 0.7971014492753623 +45 + dimension: 128 lr: 0.01 CNN channel: 32 auroc: 0.5493506493506494 f1: 0.7703703703703704 +46 + dimension: 128 lr: 0.01 CNN channel: 64 auroc: 0.5642857142857143 f1: 0.7971014492753623 +47 + dimension: 128 lr: 0.005 CNN channel: 32 auroc: 0.49155844155844164 f1: 0.7971014492753623 +48 + dimension: 128 lr: 0.005 CNN channel: 64 auroc: 0.5474025974025973 f1: 0.7971014492753623 +49 + dimension: 128 lr: 0.001 CNN channel: 32 auroc: 0.5279220779220779 f1: 0.7971014492753623 +50 + dimension: 128 lr: 0.001 CNN channel: 64 auroc: 0.49870129870129865 f1: 0.7883211678832117 +51 + dimension: 256 lr: 0.1 CNN channel: 32 auroc: 0.5889610389610389 f1: 0.752 +52 + dimension: 256 lr: 0.1 CNN channel: 64 auroc: 0.5344155844155845 f1: 0.765625 +53 + dimension: 256 lr: 0.05 CNN channel: 32 auroc: 0.557142857142857 f1: 0.7971014492753623 +54 + dimension: 256 lr: 0.05 CNN channel: 64 auroc: 0.5480519480519481 f1: 0.765625 +55 + dimension: 256 lr: 0.01 CNN channel: 32 auroc: 0.5506493506493506 f1: 0.7971014492753623 +56 + dimension: 256 lr: 0.01 CNN channel: 64 auroc: 0.5318181818181819 f1: 0.7971014492753623 +57 + dimension: 256 lr: 0.005 CNN channel: 32 auroc: 0.5318181818181819 f1: 0.7971014492753623 +58 + dimension: 256 lr: 0.005 CNN channel: 64 auroc: 0.588961038961039 f1: 0.7971014492753623 +59 + dimension: 256 lr: 0.001 CNN channel: 32 auroc: 0.5233766233766234 f1: 0.7971014492753623 +60 + dimension: 256 lr: 0.001 CNN channel: 64 auroc: 0.5038961038961038 f1: 0.7971014492753623 \ No newline at end of file diff --git a/models/linear/1.pth b/models/linear/1.pth deleted file mode 100644 index 37bb028..0000000 Binary files a/models/linear/1.pth and /dev/null differ diff --git a/models/linear/10.pth b/models/linear/10.pth new file mode 100644 index 0000000..dcbdfd0 Binary files /dev/null and b/models/linear/10.pth differ diff --git a/models/linear/11.pth b/models/linear/11.pth new file mode 100644 index 0000000..129f022 Binary files /dev/null and b/models/linear/11.pth differ diff --git a/models/linear/12.pth b/models/linear/12.pth new file mode 100644 index 0000000..b5a8ab2 Binary files /dev/null and b/models/linear/12.pth differ diff --git a/models/linear/13.pth b/models/linear/13.pth new file mode 100644 index 0000000..5f23e20 Binary files /dev/null and b/models/linear/13.pth differ diff --git a/models/linear/14.pth b/models/linear/14.pth new file mode 100644 index 0000000..f5af553 Binary files /dev/null and b/models/linear/14.pth differ diff --git a/models/linear/15.pth b/models/linear/15.pth new file mode 100644 index 0000000..7526426 Binary files /dev/null and b/models/linear/15.pth differ diff --git a/models/linear/16.pth b/models/linear/16.pth new file mode 100644 index 0000000..d76a862 Binary files /dev/null and b/models/linear/16.pth differ diff --git a/models/linear/17.pth b/models/linear/17.pth new file mode 100644 index 0000000..2995256 Binary files /dev/null and b/models/linear/17.pth differ diff --git a/models/linear/18.pth b/models/linear/18.pth new file mode 100644 index 0000000..b5dd88f Binary files /dev/null and b/models/linear/18.pth differ diff --git a/models/linear/19.pth b/models/linear/19.pth new file mode 100644 index 0000000..0dd9b28 Binary files /dev/null and b/models/linear/19.pth differ diff --git a/models/linear/2.pth b/models/linear/2.pth deleted file mode 100644 index 5ca2c8a..0000000 Binary files a/models/linear/2.pth and /dev/null differ diff --git a/models/linear/20.pth b/models/linear/20.pth new file mode 100644 index 0000000..07c5026 Binary files /dev/null and b/models/linear/20.pth differ diff --git a/models/linear/21.pth b/models/linear/21.pth new file mode 100644 index 0000000..c8a8a89 Binary files /dev/null and b/models/linear/21.pth differ diff --git a/models/linear/22.pth b/models/linear/22.pth new file mode 100644 index 0000000..89aca1d Binary files /dev/null and b/models/linear/22.pth differ diff --git a/models/linear/23.pth b/models/linear/23.pth new file mode 100644 index 0000000..856c855 Binary files /dev/null and b/models/linear/23.pth differ diff --git a/models/linear/3.pth b/models/linear/3.pth deleted file mode 100644 index 49d389b..0000000 Binary files a/models/linear/3.pth and /dev/null differ diff --git a/models/linear/4.pth b/models/linear/4.pth deleted file mode 100644 index f764a3e..0000000 Binary files a/models/linear/4.pth and /dev/null differ diff --git a/models/linear/5.pth b/models/linear/5.pth new file mode 100644 index 0000000..2bee7cc Binary files /dev/null and b/models/linear/5.pth differ diff --git a/models/linear/6.pth b/models/linear/6.pth new file mode 100644 index 0000000..09e4fef Binary files /dev/null and b/models/linear/6.pth differ diff --git a/models/linear/7.pth b/models/linear/7.pth new file mode 100644 index 0000000..26145ae Binary files /dev/null and b/models/linear/7.pth differ diff --git a/models/linear/8.pth b/models/linear/8.pth new file mode 100644 index 0000000..cc8d770 Binary files /dev/null and b/models/linear/8.pth differ diff --git a/models/linear/9.pth b/models/linear/9.pth new file mode 100644 index 0000000..be53f13 Binary files /dev/null and b/models/linear/9.pth differ diff --git a/models/linear/sec_try1.pth b/models/linear/sec_try1.pth new file mode 100644 index 0000000..7ac2259 Binary files /dev/null and b/models/linear/sec_try1.pth differ diff --git a/models/linear/sec_try2.pth b/models/linear/sec_try2.pth new file mode 100644 index 0000000..4338e93 Binary files /dev/null and b/models/linear/sec_try2.pth differ diff --git a/models/linear/sec_try3.pth b/models/linear/sec_try3.pth new file mode 100644 index 0000000..1deed4d Binary files /dev/null and b/models/linear/sec_try3.pth differ diff --git a/models/linear/sec_try4.pth b/models/linear/sec_try4.pth new file mode 100644 index 0000000..a1a7c7b Binary files /dev/null and b/models/linear/sec_try4.pth differ diff --git a/models/linear/sec_try5.pth b/models/linear/sec_try5.pth new file mode 100644 index 0000000..6fd0a08 Binary files /dev/null and b/models/linear/sec_try5.pth differ diff --git a/models/linear/sec_try6.pth b/models/linear/sec_try6.pth new file mode 100644 index 0000000..514017b Binary files /dev/null and b/models/linear/sec_try6.pth differ diff --git a/models/linear/sec_try7.pth b/models/linear/sec_try7.pth new file mode 100644 index 0000000..44106d7 Binary files /dev/null and b/models/linear/sec_try7.pth differ diff --git a/models/logprob/1.pth b/models/logprob/1.pth index 36d8551..0320e22 100644 Binary files a/models/logprob/1.pth and b/models/logprob/1.pth differ diff --git a/models/logprob/2.pth b/models/logprob/2.pth index 4a40ef0..dac323a 100644 Binary files a/models/logprob/2.pth and b/models/logprob/2.pth differ diff --git a/models/logprob/3.pth b/models/logprob/3.pth index dc50c16..4918e10 100644 Binary files a/models/logprob/3.pth and b/models/logprob/3.pth differ diff --git a/models/logprob/4.pth b/models/logprob/4.pth index 464e4ba..7a6bac2 100644 Binary files a/models/logprob/4.pth and b/models/logprob/4.pth differ diff --git a/models/logprob/5.pth b/models/logprob/5.pth index 185e456..f54d415 100644 Binary files a/models/logprob/5.pth and b/models/logprob/5.pth differ diff --git a/models/logprob/6.pth b/models/logprob/6.pth deleted file mode 100644 index c851135..0000000 Binary files a/models/logprob/6.pth and /dev/null differ diff --git a/models/logprob/7.pth b/models/logprob/7.pth deleted file mode 100644 index 21440b6..0000000 Binary files a/models/logprob/7.pth and /dev/null differ diff --git a/models/logprob/sec_try1.pth b/models/logprob/sec_try1.pth new file mode 100644 index 0000000..eeedb4b Binary files /dev/null and b/models/logprob/sec_try1.pth differ diff --git a/models/logprob/sec_try14.pth b/models/logprob/sec_try14.pth new file mode 100644 index 0000000..4522e6f Binary files /dev/null and b/models/logprob/sec_try14.pth differ diff --git a/models/logprob/sec_try39.pth b/models/logprob/sec_try39.pth new file mode 100644 index 0000000..20d03cc Binary files /dev/null and b/models/logprob/sec_try39.pth differ diff --git a/test_results/ROC/linear/10.pdf b/test_results/ROC/linear/10.pdf new file mode 100644 index 0000000..37fbd5d Binary files /dev/null and b/test_results/ROC/linear/10.pdf differ diff --git a/test_results/ROC/logprob/6.pdf b/test_results/ROC/linear/11.pdf similarity index 73% rename from test_results/ROC/logprob/6.pdf rename to test_results/ROC/linear/11.pdf index 7bd3ede..c479307 100644 Binary files a/test_results/ROC/logprob/6.pdf and b/test_results/ROC/linear/11.pdf differ diff --git a/test_results/ROC/linear/12.pdf b/test_results/ROC/linear/12.pdf new file mode 100644 index 0000000..4b56d2f Binary files /dev/null and b/test_results/ROC/linear/12.pdf differ diff --git a/test_results/ROC/linear/13.pdf b/test_results/ROC/linear/13.pdf new file mode 100644 index 0000000..4b3c80a Binary files /dev/null and b/test_results/ROC/linear/13.pdf differ diff --git a/test_results/ROC/linear/14.pdf b/test_results/ROC/linear/14.pdf new file mode 100644 index 0000000..d401237 Binary files /dev/null and b/test_results/ROC/linear/14.pdf differ diff --git a/test_results/ROC/linear/15.pdf b/test_results/ROC/linear/15.pdf new file mode 100644 index 0000000..8cf3f29 Binary files /dev/null and b/test_results/ROC/linear/15.pdf differ diff --git a/test_results/ROC/linear/16.pdf b/test_results/ROC/linear/16.pdf new file mode 100644 index 0000000..59c1fcd Binary files /dev/null and b/test_results/ROC/linear/16.pdf differ diff --git a/test_results/ROC/linear/17.pdf b/test_results/ROC/linear/17.pdf new file mode 100644 index 0000000..339e148 Binary files /dev/null and b/test_results/ROC/linear/17.pdf differ diff --git a/test_results/ROC/linear/18.pdf b/test_results/ROC/linear/18.pdf new file mode 100644 index 0000000..dd51e57 Binary files /dev/null and b/test_results/ROC/linear/18.pdf differ diff --git a/test_results/ROC/linear/19.pdf b/test_results/ROC/linear/19.pdf new file mode 100644 index 0000000..ddd8f53 Binary files /dev/null and b/test_results/ROC/linear/19.pdf differ diff --git a/test_results/ROC/linear/20.pdf b/test_results/ROC/linear/20.pdf new file mode 100644 index 0000000..3ad7a43 Binary files /dev/null and b/test_results/ROC/linear/20.pdf differ diff --git a/test_results/ROC/linear/21.pdf b/test_results/ROC/linear/21.pdf new file mode 100644 index 0000000..b9876cf Binary files /dev/null and b/test_results/ROC/linear/21.pdf differ diff --git a/test_results/ROC/linear/22.pdf b/test_results/ROC/linear/22.pdf new file mode 100644 index 0000000..6e4e5b8 Binary files /dev/null and b/test_results/ROC/linear/22.pdf differ diff --git a/test_results/ROC/linear/23.pdf b/test_results/ROC/linear/23.pdf new file mode 100644 index 0000000..95e69fe Binary files /dev/null and b/test_results/ROC/linear/23.pdf differ diff --git a/test_results/ROC/linear/24.pdf b/test_results/ROC/linear/24.pdf new file mode 100644 index 0000000..00c7c1a Binary files /dev/null and b/test_results/ROC/linear/24.pdf differ diff --git a/test_results/ROC/linear/25.pdf b/test_results/ROC/linear/25.pdf new file mode 100644 index 0000000..3ce266c Binary files /dev/null and b/test_results/ROC/linear/25.pdf differ diff --git a/test_results/ROC/linear/26.pdf b/test_results/ROC/linear/26.pdf new file mode 100644 index 0000000..05857f3 Binary files /dev/null and b/test_results/ROC/linear/26.pdf differ diff --git a/test_results/ROC/linear/27.pdf b/test_results/ROC/linear/27.pdf new file mode 100644 index 0000000..bbdfaf5 Binary files /dev/null and b/test_results/ROC/linear/27.pdf differ diff --git a/test_results/ROC/linear/28.pdf b/test_results/ROC/linear/28.pdf new file mode 100644 index 0000000..0c7542f Binary files /dev/null and b/test_results/ROC/linear/28.pdf differ diff --git a/test_results/ROC/linear/29.pdf b/test_results/ROC/linear/29.pdf new file mode 100644 index 0000000..ed9d7b4 Binary files /dev/null and b/test_results/ROC/linear/29.pdf differ diff --git a/test_results/ROC/linear/30.pdf b/test_results/ROC/linear/30.pdf new file mode 100644 index 0000000..b045ebe Binary files /dev/null and b/test_results/ROC/linear/30.pdf differ diff --git a/test_results/ROC/linear/5.pdf b/test_results/ROC/linear/5.pdf new file mode 100644 index 0000000..c5c5871 Binary files /dev/null and b/test_results/ROC/linear/5.pdf differ diff --git a/test_results/ROC/linear/6.pdf b/test_results/ROC/linear/6.pdf new file mode 100644 index 0000000..518de06 Binary files /dev/null and b/test_results/ROC/linear/6.pdf differ diff --git a/test_results/ROC/linear/7.pdf b/test_results/ROC/linear/7.pdf new file mode 100644 index 0000000..21a88ae Binary files /dev/null and b/test_results/ROC/linear/7.pdf differ diff --git a/test_results/ROC/linear/8.pdf b/test_results/ROC/linear/8.pdf new file mode 100644 index 0000000..747db69 Binary files /dev/null and b/test_results/ROC/linear/8.pdf differ diff --git a/test_results/ROC/linear/9.pdf b/test_results/ROC/linear/9.pdf new file mode 100644 index 0000000..9a15801 Binary files /dev/null and b/test_results/ROC/linear/9.pdf differ diff --git a/test_results/ROC/logprob/1.pdf b/test_results/ROC/logprob/1.pdf index eb8a3aa..e42220f 100644 Binary files a/test_results/ROC/logprob/1.pdf and b/test_results/ROC/logprob/1.pdf differ diff --git a/test_results/ROC/logprob/8.pdf b/test_results/ROC/logprob/14.pdf similarity index 80% rename from test_results/ROC/logprob/8.pdf rename to test_results/ROC/logprob/14.pdf index 671ee18..d424d2c 100644 Binary files a/test_results/ROC/logprob/8.pdf and b/test_results/ROC/logprob/14.pdf differ diff --git a/test_results/ROC/logprob/2.pdf b/test_results/ROC/logprob/2.pdf index 8fe1b6e..93dc28c 100644 Binary files a/test_results/ROC/logprob/2.pdf and b/test_results/ROC/logprob/2.pdf differ diff --git a/test_results/ROC/logprob/3.pdf b/test_results/ROC/logprob/3.pdf index 7b4e42e..350ee95 100644 Binary files a/test_results/ROC/logprob/3.pdf and b/test_results/ROC/logprob/3.pdf differ diff --git a/test_results/ROC/logprob/7.pdf b/test_results/ROC/logprob/39.pdf similarity index 80% rename from test_results/ROC/logprob/7.pdf rename to test_results/ROC/logprob/39.pdf index 31183bc..c79cfc1 100644 Binary files a/test_results/ROC/logprob/7.pdf and b/test_results/ROC/logprob/39.pdf differ diff --git a/test_results/ROC/logprob/4.pdf b/test_results/ROC/logprob/4.pdf index 12d73f3..891305b 100644 Binary files a/test_results/ROC/logprob/4.pdf and b/test_results/ROC/logprob/4.pdf differ diff --git a/test_results/ROC/logprob/5.pdf b/test_results/ROC/logprob/auroc 0.57 f1 0.62.pdf similarity index 82% rename from test_results/ROC/logprob/5.pdf rename to test_results/ROC/logprob/auroc 0.57 f1 0.62.pdf index e13784b..241573b 100644 Binary files a/test_results/ROC/logprob/5.pdf and b/test_results/ROC/logprob/auroc 0.57 f1 0.62.pdf differ diff --git a/test_results/loss/linear/linear_10.pdf b/test_results/loss/linear/linear_10.pdf new file mode 100644 index 0000000..a72dabe Binary files /dev/null and b/test_results/loss/linear/linear_10.pdf differ diff --git a/test_results/loss/linear/linear_11.pdf b/test_results/loss/linear/linear_11.pdf new file mode 100644 index 0000000..e32a3d9 Binary files /dev/null and b/test_results/loss/linear/linear_11.pdf differ diff --git a/test_results/loss/linear/linear_12.pdf b/test_results/loss/linear/linear_12.pdf new file mode 100644 index 0000000..0435e33 Binary files /dev/null and b/test_results/loss/linear/linear_12.pdf differ diff --git a/test_results/loss/linear/linear_13.pdf b/test_results/loss/linear/linear_13.pdf new file mode 100644 index 0000000..06112f8 Binary files /dev/null and b/test_results/loss/linear/linear_13.pdf differ diff --git a/test_results/loss/linear/linear_14.pdf b/test_results/loss/linear/linear_14.pdf new file mode 100644 index 0000000..26086f2 Binary files /dev/null and b/test_results/loss/linear/linear_14.pdf differ diff --git a/test_results/loss/linear/linear_15.pdf b/test_results/loss/linear/linear_15.pdf new file mode 100644 index 0000000..468d65c Binary files /dev/null and b/test_results/loss/linear/linear_15.pdf differ diff --git a/test_results/loss/linear/linear_16.pdf b/test_results/loss/linear/linear_16.pdf new file mode 100644 index 0000000..332b8ca Binary files /dev/null and b/test_results/loss/linear/linear_16.pdf differ diff --git a/test_results/loss/linear/linear_17.pdf b/test_results/loss/linear/linear_17.pdf new file mode 100644 index 0000000..19ba6dd Binary files /dev/null and b/test_results/loss/linear/linear_17.pdf differ diff --git a/test_results/loss/linear/linear_18.pdf b/test_results/loss/linear/linear_18.pdf new file mode 100644 index 0000000..6ad30b1 Binary files /dev/null and b/test_results/loss/linear/linear_18.pdf differ diff --git a/test_results/loss/linear/linear_19.pdf b/test_results/loss/linear/linear_19.pdf new file mode 100644 index 0000000..bb0b80b Binary files /dev/null and b/test_results/loss/linear/linear_19.pdf differ diff --git a/test_results/loss/linear/linear_20.pdf b/test_results/loss/linear/linear_20.pdf new file mode 100644 index 0000000..3c111a3 Binary files /dev/null and b/test_results/loss/linear/linear_20.pdf differ diff --git a/test_results/loss/linear/linear_21.pdf b/test_results/loss/linear/linear_21.pdf new file mode 100644 index 0000000..d96be6e Binary files /dev/null and b/test_results/loss/linear/linear_21.pdf differ diff --git a/test_results/loss/linear/linear_22.pdf b/test_results/loss/linear/linear_22.pdf new file mode 100644 index 0000000..4972554 Binary files /dev/null and b/test_results/loss/linear/linear_22.pdf differ diff --git a/test_results/loss/logprob/logprob_5.pdf b/test_results/loss/linear/linear_23.pdf similarity index 79% rename from test_results/loss/logprob/logprob_5.pdf rename to test_results/loss/linear/linear_23.pdf index 7e0653a..b48130a 100644 Binary files a/test_results/loss/logprob/logprob_5.pdf and b/test_results/loss/linear/linear_23.pdf differ diff --git a/test_results/loss/linear/linear_24.pdf b/test_results/loss/linear/linear_24.pdf new file mode 100644 index 0000000..787a7ae Binary files /dev/null and b/test_results/loss/linear/linear_24.pdf differ diff --git a/test_results/loss/linear/linear_25.pdf b/test_results/loss/linear/linear_25.pdf new file mode 100644 index 0000000..62c1b1d Binary files /dev/null and b/test_results/loss/linear/linear_25.pdf differ diff --git a/test_results/loss/linear/linear_26.pdf b/test_results/loss/linear/linear_26.pdf new file mode 100644 index 0000000..c8ebe61 Binary files /dev/null and b/test_results/loss/linear/linear_26.pdf differ diff --git a/test_results/loss/linear/linear_27.pdf b/test_results/loss/linear/linear_27.pdf new file mode 100644 index 0000000..b439b4c Binary files /dev/null and b/test_results/loss/linear/linear_27.pdf differ diff --git a/test_results/loss/linear/linear_28.pdf b/test_results/loss/linear/linear_28.pdf new file mode 100644 index 0000000..e10078a Binary files /dev/null and b/test_results/loss/linear/linear_28.pdf differ diff --git a/test_results/loss/linear/linear_29.pdf b/test_results/loss/linear/linear_29.pdf new file mode 100644 index 0000000..b9f5a7e Binary files /dev/null and b/test_results/loss/linear/linear_29.pdf differ diff --git a/test_results/loss/linear/linear_4.pdf b/test_results/loss/linear/linear_4.pdf new file mode 100644 index 0000000..c9e45e2 Binary files /dev/null and b/test_results/loss/linear/linear_4.pdf differ diff --git a/test_results/loss/linear/linear_5.pdf b/test_results/loss/linear/linear_5.pdf new file mode 100644 index 0000000..2161c50 Binary files /dev/null and b/test_results/loss/linear/linear_5.pdf differ diff --git a/test_results/loss/linear/linear_6.pdf b/test_results/loss/linear/linear_6.pdf new file mode 100644 index 0000000..5fa0fa5 Binary files /dev/null and b/test_results/loss/linear/linear_6.pdf differ diff --git a/test_results/loss/linear/linear_7.pdf b/test_results/loss/linear/linear_7.pdf new file mode 100644 index 0000000..b7951bc Binary files /dev/null and b/test_results/loss/linear/linear_7.pdf differ diff --git a/test_results/loss/linear/linear_8.pdf b/test_results/loss/linear/linear_8.pdf new file mode 100644 index 0000000..b2ca015 Binary files /dev/null and b/test_results/loss/linear/linear_8.pdf differ diff --git a/test_results/loss/linear/linear_9.pdf b/test_results/loss/linear/linear_9.pdf new file mode 100644 index 0000000..c7fb01a Binary files /dev/null and b/test_results/loss/linear/linear_9.pdf differ diff --git a/test_results/loss/logprob/d_model 256 conv 32 lr 0.0001.pdf b/test_results/loss/logprob/d_model 256 conv 32 lr 0.0001.pdf new file mode 100644 index 0000000..b433150 Binary files /dev/null and b/test_results/loss/logprob/d_model 256 conv 32 lr 0.0001.pdf differ diff --git a/test_results/loss/logprob/logprob_1.pdf b/test_results/loss/logprob/logprob_1.pdf index 66d6315..ced2897 100644 Binary files a/test_results/loss/logprob/logprob_1.pdf and b/test_results/loss/logprob/logprob_1.pdf differ diff --git a/test_results/loss/logprob/logprob_2.pdf b/test_results/loss/logprob/logprob_2.pdf index 7d07b57..c777238 100644 Binary files a/test_results/loss/logprob/logprob_2.pdf and b/test_results/loss/logprob/logprob_2.pdf differ diff --git a/test_results/loss/logprob/logprob_3.pdf b/test_results/loss/logprob/logprob_3.pdf index 132469d..27ef9a2 100644 Binary files a/test_results/loss/logprob/logprob_3.pdf and b/test_results/loss/logprob/logprob_3.pdf differ diff --git a/test_results/loss/logprob/logprob_4.pdf b/test_results/loss/logprob/logprob_4.pdf index 49b4846..bac613d 100644 Binary files a/test_results/loss/logprob/logprob_4.pdf and b/test_results/loss/logprob/logprob_4.pdf differ diff --git a/test_results/loss/logprob/logprob_6.pdf b/test_results/loss/logprob/logprob_6.pdf index af8fe3d..9cf72e7 100644 Binary files a/test_results/loss/logprob/logprob_6.pdf and b/test_results/loss/logprob/logprob_6.pdf differ diff --git a/test_results/loss/logprob/logprob_7.pdf b/test_results/loss/logprob/logprob_7.pdf new file mode 100644 index 0000000..3fcb0c1 Binary files /dev/null and b/test_results/loss/logprob/logprob_7.pdf differ diff --git a/test_results/loss/logprob/logprob_8.pdf b/test_results/loss/logprob/logprob_8.pdf new file mode 100644 index 0000000..081606e Binary files /dev/null and b/test_results/loss/logprob/logprob_8.pdf differ diff --git a/test_results/loss/logprob/logprob_9.pdf b/test_results/loss/logprob/logprob_9.pdf new file mode 100644 index 0000000..4a561fe Binary files /dev/null and b/test_results/loss/logprob/logprob_9.pdf differ