change configuration

Author: hyunwoongko

.idea/workspace.xml

@@ -352,12 +352,12 @@ I follow original paper's parameter settings. (below) <br>
 * n_heads = 8
 * ffn_hidden = 2048
 * drop_prob = 0.1
-* init_lr = 1e-6
+* init_lr = 1e-5
 * factor = 0.8
 * min_lr = init_lr * 1e-12
 * patience = 7
 * warmup = 100
-* weight_decay = 1e-4
+* weight_decay = 1e-3
 * epoch = 1000
 * clip = 1
 
@@ -389,12 +389,12 @@ I follow original paper's parameter settings. (below) <br>
 * n_heads = 16
 * ffn_hidden = 4096
 * drop_prob = 0.3
-* init_lr = 1e-6
+* init_lr = 1e-5
 * factor = 0.8
 * min_lr = init_lr * 1e-12
 * patience = 7
 * warmup = 300
-* weight_decay = 1e-4
+* weight_decay = 1e-3
 * epoch = 3000
 * clip = 1
 

README.md

@@ -18,12 +18,12 @@
 drop_prob = 0.1
 
 # optimizer parameter setting
-init_lr = 1e-6
+init_lr = 1e-5
 factor = 0.8
 min_lr = init_lr * 1e-12
 patience = 7
 warmup = 100
-weight_decay = 1e-4
+weight_decay = 1e-3
 epoch = 1000
 clip = 1
 inf = float('inf')

__pycache__/conf.cpython-36.pyc

@@ -24,16 +24,18 @@ def draw(mode):
         test = read('./result/test_loss.txt')
         plt.plot(train, 'r', label='train')
         plt.plot(test, 'b', label='validation')
+        plt.legend(loc='lower left')
+
 
     elif mode == 'bleu':
         bleu = read('./result/bleu.txt')
         plt.plot(bleu, 'b', label='bleu score')
+        plt.legend(loc='lower right')
 
     plt.xlabel('epoch')
     plt.ylabel(mode)
     plt.title('training result')
     plt.grid(True, which='both', axis='both')
-    plt.legend(loc='lower left')
     plt.show()
 
 

conf.py

@@ -1 +1 @@
-[0.2501438161546564, 0.13948406463256255]
+[0.07622405577310074, 4.063269288356802]

graph.py

@@ -1 +1 @@
-[8.388485550880432, 7.465105414390564]
+[6.120936930179596, 5.668061196804047]

result/bleu.txt

@@ -1 +1 @@
-[9.24096984275112, 8.002939659068238]
+[7.244170436775106, 6.017328854699492]

result/test_loss.txt

@@ -151,7 +151,7 @@ def run(total_epoch, best_loss):
         f.write(str(test_losses))
         f.close()
 
-        print(f'Epoch: {step + 1} | Time: {epoch_mins}m {epoch_secs}s')
+        print(f'Epoch: {step + 1 + train_count} | Time: {epoch_mins}m {epoch_secs}s')
         print(f'\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')
         print(f'\tVal Loss: {valid_loss:.3f} |  Val PPL: {math.exp(valid_loss):7.3f}')
         print(f'\tBLEU Score: {bleu:.3f}')