Removing SOS and EOS

Author: PonteIneptique

boudams/cli.py

@@ -168,14 +168,12 @@ def train(config_files, epochs, batch_size, device, debug):
     for config_file in config_files:
         config = json.load(config_file)
 
-        masked = config["model"].startswith("linear")
         train_path, dev_path, test_path = config["datasets"]["train"],\
                                           config["datasets"]["dev"],\
                                           config["datasets"]["test"]
 
         vocabulary = LabelEncoder(
             maximum_length=config.get("max_sentence_size", None),
-            masked=masked,
             remove_diacriticals=config["label_encoder"].get("normalize", True),
             lower=config["label_encoder"].get("lower", True)
         )

boudams/encoder.py

@@ -145,71 +145,51 @@ def iterable():
 
 class LabelEncoder:
     def __init__(self,
-                 init_token=DEFAULT_INIT_TOKEN,
-                 eos_token=DEFAULT_EOS_TOKEN,
                  pad_token=DEFAULT_PAD_TOKEN,
                  unk_token=DEFAULT_UNK_TOKEN,
                  mask_token=DEFAULT_MASK_TOKEN,
                  maximum_length: int = None,
                  lower: bool = True,
-                 remove_diacriticals: bool = True,
-                 masked: bool = False
+                 remove_diacriticals: bool = True
                  ):
 
-        self.masked: bool = masked
-        self.init_token: str = init_token
-        self.eos_token: str = eos_token
         self.pad_token: str = pad_token
         self.unk_token: str = unk_token
         self.mask_token: str = mask_token
         self.space_token: str = " "
 
-        self.init_token_index: int = 0
-        self.eos_token_index: int = 1
         self.pad_token_index: int = 2
-        self.space_token_index: int = 3
-        self.mask_token_index: int = 4
-        self.unk_token_index: int = 5  # Put here because it isn't used in masked
+        self.space_token_index: int = 1
+        self.mask_token_index: int = 0
+        self.unk_token_index: int = 0  # Put here because it isn't used in masked
 
         self.max_len: Optional[int] = maximum_length
         self.lower = lower
         self.remove_diacriticals = remove_diacriticals
 
         self.itos: Dict[int, str] = {
-            self.init_token_index: self.init_token,
-            self.eos_token_index: self.eos_token,
             self.pad_token_index: self.pad_token,
-            self.unk_token_index: self.unk_token
+            self.unk_token_index: self.unk_token,
+            self.space_token_index: self.space_token
         }  # Id to string for reversal
 
         self.stoi: Dict[str, int] = {
-            self.init_token: self.init_token_index,
-            self.eos_token: self.eos_token_index,
             self.pad_token: self.pad_token_index,
-            self.unk_token: self.unk_token_index
+            self.unk_token: self.unk_token_index,
+            self.space_token: self.space_token_index
         }  # String to ID
 
         # Mask dictionaries
         self.itom: Dict[int, str] = {
-            self.init_token_index: self.init_token,
-            self.eos_token_index: self.eos_token,
             self.pad_token_index: self.pad_token,
             self.mask_token_index: self.mask_token,
             self.space_token_index: self.space_token
         }
         self.mtoi: Dict[str, int] = {
-            self.init_token: self.init_token_index,
-            self.eos_token: self.eos_token_index,
             self.pad_token: self.pad_token_index,
             self.mask_token: self.mask_token_index,
             self.space_token: self.space_token_index
         }
-        self.use_init = True
-        self.use_eos = True
-
-    def encoding_parameters(self, use_init, use_eos):
-        self.use_init = use_init
-        self.use_eos = use_eos
 
     def __len__(self):
         return len(self.stoi)
@@ -279,6 +259,7 @@ def pad_and_tensorize(
 
         :param sentences: List of sentences where characters have been separated into a list and index encoded
         :param padding: padding required (None if every sentence in the same size)
+        :param reorder: List of index to reorder the sequence
         :param device: Torch device
         :return: Transformed batch into tensor
         """
@@ -310,36 +291,26 @@ def pad_and_tensorize(
     def gt_to_numerical(self, sentence: Sequence[str]) -> Tuple[List[int], int]:
         """ Transform GT to numerical
 
-        :param sentence: Sequence of characters (can be a straight string)
-        :return: List of character indexes
+        :param sentence: Sequence of characters (can be a straight string) with spaces
+        :return: List of mask indexes
         """
-        if not self.masked:
-            return self.inp_to_numerical(sentence)
-        else:
-            obligatory_tokens = int(self.use_init) + int(self.use_eos)  # Tokens for init and end of string
-            init = [self.init_token_index] if self.use_init else []
-            eos = [self.eos_token_index] if self.use_eos else []
-            numericals = init + [
-                    self.mask_token_index if ngram[1] != " " else self.space_token_index
-                    for ngram in zip(*[sentence[i:] for i in range(2)])
-                    if ngram[0] != " "
-                ] + [self.space_token_index] + eos
+        numericals = [
+                self.mask_token_index if ngram[1] != " " else self.space_token_index
+                for ngram in zip(*[sentence[i:] for i in range(2)])
+                if ngram[0] != " "
+            ] + [self.space_token_index]
 
-            return numericals, len(sentence) - sentence.count(" ") + obligatory_tokens
+        return numericals, len(sentence) - sentence.count(" ")
 
     def inp_to_numerical(self, sentence: Sequence[str]) -> Tuple[List[int], int]:
-        """ Transform GT to numerical
+        """ Transform input sentence to numerical
 
-        :param sentence: Sequence of characters (can be a straight string)
+        :param sentence: Sequence of characters (can be a straight string) without spaces
         :return: List of character indexes
         """
-        obligatory_tokens = int(self.use_init) + int(self.use_eos)  # Tokens for init and end of string
-        init = [self.init_token_index] if self.use_init else []
-        eos = [self.eos_token_index] if self.use_eos else []
-
         return (
-            init + [self.stoi.get(char, self.unk_token_index) for char in sentence] + eos,
-            len(sentence) + obligatory_tokens
+            [self.stoi.get(char, self.unk_token_index) for char in sentence],
+            len(sentence)
         )
 
     def reverse_batch(
@@ -355,9 +326,8 @@ def reverse_batch(
             with torch.cuda.device_of(batch):
                 batch = batch.tolist()
 
-        if self.masked is True and masked is not None:
+        if masked is not None:
             if not isinstance(masked, list):
-
                 with torch.cuda.device_of(masked):
                     masked = masked.tolist()
 
@@ -371,9 +341,11 @@ def reverse_batch(
                 ]
             else:
                 masked = [
-                    [self.init_token_index] + list(sentence) + [self.eos_token_index]
+                    list(sentence)
                     for sentence in masked
                 ]
+            print(ignore)
+
             return [
                 [
                     tok
@@ -405,8 +377,7 @@ def reverse_batch(
 
     def transcribe_batch(self, batch: List[List[str]]):
         for sentence in batch:
-            end = len(sentence) if self.eos_token not in sentence else sentence.index(self.eos_token)
-            yield "".join(sentence[1:end])  # Remove SOS
+            yield "".join(sentence).strip()  # Remove SOS
 
     def get_dataset(self, path, **kwargs):
         """
@@ -431,14 +402,11 @@ def dump(self) -> str:
             "itos": self.itos,
             "stoi": self.stoi,
             "params": {
-                "init_token": self.init_token,
-                "eos_token": self.eos_token,
                 "pad_token": self.pad_token,
                 "unk_token": self.unk_token,
                 "mask_token": self.mask_token,
                 "remove_diacriticals": self.remove_diacriticals,
-                "lower": self.lower,
-                "masked": self.masked
+                "lower": self.lower
             }
         })
 

boudams/model/linear.py

@@ -149,7 +149,7 @@ def __init__(
         self,
         encoder: CNNEncoder, decoder: LinearDecoder,
         device: str,
-        pad_idx: int, sos_idx: int, eos_idx: int,
+        pad_idx: int,
         pos: bool = False,
         **kwargs
     ):
@@ -160,8 +160,6 @@ def __init__(
         self.pos = pos
 
         self.pad_idx = pad_idx
-        self.sos_idx = sos_idx
-        self.eos_idx = eos_idx
         self.device = device
 
         # nll weight
@@ -211,7 +209,7 @@ def predict(self, src, src_len, label_encoder: "LabelEncoder",
         return label_encoder.reverse_batch(
             logits,
             masked=override_src or src,
-            ignore=(self.pad_idx, self.eos_idx, self.sos_idx)
+            ignore=(self.pad_idx, )
         )
 
     def gradient(

boudams/tagger.py

@@ -47,7 +47,6 @@ def __init__(
 
         self.vocabulary: LabelEncoder = vocabulary
         self.vocabulary_dimension: int = len(self.vocabulary)
-        self.masked: bool = self.vocabulary.masked
 
         self.device: str = device
         self.enc_hid_dim = self.dec_hid_dim = self.hidden_size = hidden_size
@@ -64,16 +63,12 @@ def __init__(
         self.system: str = system
 
         # Based on self.masked, decoder dimension can be drastically different
-        self.dec_dim: int = self.vocabulary_dimension
-        if self.masked:
-            self.dec_dim = len(self.vocabulary.itom)
+        self.dec_dim = len(self.vocabulary.itom)
 
         self.mask_token = self.vocabulary.mask_token
 
         seq2seq_shared_params = {
             "pad_idx": self.padtoken,
-            "sos_idx": self.sostoken,
-            "eos_idx": self.eostoken,
             "device": self.device,
             "out_max_sentence_length": self.out_max_sentence_length
         }
@@ -128,14 +123,6 @@ def __init__(
     def padtoken(self):
         return self.vocabulary.pad_token_index
 
-    @property
-    def sostoken(self):
-        return self.vocabulary.init_token_index
-
-    @property
-    def eostoken(self):
-        return self.vocabulary.eos_token_index
-
     @property
     def settings(self):
         return {

boudams/trainer.py

@@ -128,9 +128,10 @@ def register_batch(self, hypotheses, targets, src):
                 src = src.tolist()
 
         for y_true, y_pred, x in zip(exp, out, src):
-            self.trues.append(y_true)
-            self.preds.append(y_pred)
-            self.srcs.append(x)
+            stop = x.index(self.tagger.padtoken) if self.tagger.padtoken in x else len(x)
+            self.trues.append(y_true[:stop])
+            self.preds.append(y_pred[:stop])
+            self.srcs.append(x[:stop])
 
 
 class LRScheduler(object):