Moved sigmoid and log functions inside Model class

Summary:
Moved the sigmoid and log functions inside the Model class.
No more need for the thread-unsafe `fasttext::utils::initTables()` function.

Reviewed By: ajoulin

Differential Revision: D4052203

fbshipit-source-id: ca593fdc83c1ed96abc5db5f09b8db2154062efa

Author: Edouard Grave

src/main.cc

@@ -129,7 +129,6 @@ void train(int argc, char** argv) {
 }
 
 int main(int argc, char** argv) {
-  utils::initTables();
   if (argc < 2) {
     printUsage();
     exit(EXIT_FAILURE);
@@ -147,6 +146,5 @@ int main(int argc, char** argv) {
     printUsage();
     exit(EXIT_FAILURE);
   }
-  utils::freeTables();
   return 0;
 }

src/model.cc

@@ -32,17 +32,24 @@ Model::Model(std::shared_ptr<Matrix> wi,
   negpos = 0;
   loss_ = 0.0;
   nexamples_ = 1;
+  initSigmoid();
+  initLog();
+}
+
+Model::~Model() {
+  delete[] t_sigmoid;
+  delete[] t_log;
 }
 
 real Model::binaryLogistic(int32_t target, bool label, real lr) {
-  real score = utils::sigmoid(wo_->dotRow(hidden_, target));
+  real score = sigmoid(wo_->dotRow(hidden_, target));
   real alpha = lr * (real(label) - score);
   grad_.addRow(*wo_, target, alpha);
   wo_->addRow(hidden_, target, alpha);
   if (label) {
-    return -utils::log(score);
+    return -log(score);
   } else {
-    return -utils::log(1.0 - score);
+    return -log(1.0 - score);
   }
 }
 
@@ -98,7 +105,7 @@ real Model::softmax(int32_t target, real lr) {
     grad_.addRow(*wo_, i, alpha);
     wo_->addRow(hidden_, i, alpha);
   }
-  return -utils::log(output_[target]);
+  return -log(output_[target]);
 }
 
 void Model::computeHidden(const std::vector<int32_t>& input, Vector& hidden) const {
@@ -138,10 +145,10 @@ void Model::findKBest(int32_t k, std::vector<std::pair<real, int32_t>>& heap,
                       Vector& hidden, Vector& output) const {
   computeOutputSoftmax(hidden, output);
   for (int32_t i = 0; i < osz_; i++) {
-    if (heap.size() == k && utils::log(output[i]) < heap.front().first) {
+    if (heap.size() == k && log(output[i]) < heap.front().first) {
       continue;
     }
-    heap.push_back(std::make_pair(utils::log(output[i]), i));
+    heap.push_back(std::make_pair(log(output[i]), i));
     std::push_heap(heap.begin(), heap.end(), comparePairs);
     if (heap.size() > k) {
       std::pop_heap(heap.begin(), heap.end(), comparePairs);
@@ -167,9 +174,9 @@ void Model::dfs(int32_t k, int32_t node, real score,
     return;
   }
 
-  real f = utils::sigmoid(wo_->dotRow(hidden, node - osz_));
-  dfs(k, tree[node].left, score + utils::log(1.0 - f), heap, hidden);
-  dfs(k, tree[node].right, score + utils::log(f), heap, hidden);
+  real f = sigmoid(wo_->dotRow(hidden, node - osz_));
+  dfs(k, tree[node].left, score + log(1.0 - f), heap, hidden);
+  dfs(k, tree[node].right, score + log(f), heap, hidden);
 }
 
 void Model::update(const std::vector<int32_t>& input, int32_t target, real lr) {
@@ -275,4 +282,39 @@ real Model::getLoss() const {
   return loss_ / nexamples_;
 }
 
+void Model::initSigmoid() {
+  t_sigmoid = new real[SIGMOID_TABLE_SIZE + 1];
+  for (int i = 0; i < SIGMOID_TABLE_SIZE + 1; i++) {
+    real x = real(i * 2 * MAX_SIGMOID) / SIGMOID_TABLE_SIZE - MAX_SIGMOID;
+    t_sigmoid[i] = 1.0 / (1.0 + std::exp(-x));
+  }
+}
+
+void Model::initLog() {
+  t_log = new real[LOG_TABLE_SIZE + 1];
+  for (int i = 0; i < LOG_TABLE_SIZE + 1; i++) {
+    real x = (real(i) + 1e-5) / LOG_TABLE_SIZE;
+    t_log[i] = std::log(x);
+  }
+}
+
+real Model::log(real x) const {
+  if (x > 1.0) {
+    return 0.0;
+  }
+  int i = int(x * LOG_TABLE_SIZE);
+  return t_log[i];
+}
+
+real Model::sigmoid(real x) const {
+  if (x < -MAX_SIGMOID) {
+    return 0.0;
+  } else if (x > MAX_SIGMOID) {
+    return 1.0;
+  } else {
+    int i = int((x + MAX_SIGMOID) * SIGMOID_TABLE_SIZE / MAX_SIGMOID / 2);
+    return t_sigmoid[i];
+  }
+}
+
 }

src/model.h

@@ -20,6 +20,10 @@
 #include "vector.h"
 #include "real.h"
 
+#define SIGMOID_TABLE_SIZE 512
+#define MAX_SIGMOID 8
+#define LOG_TABLE_SIZE 512
+
 namespace fasttext {
 
 struct Node {
@@ -43,24 +47,29 @@ class Model {
     int32_t osz_;
     real loss_;
     int64_t nexamples_;
-
-    static bool comparePairs(const std::pair<real, int32_t>&,
-                             const std::pair<real, int32_t>&);
-
+    real* t_sigmoid;
+    real* t_log;
+    // used for negative sampling:
     std::vector<int32_t> negatives;
     size_t negpos;
-
-    int32_t getNegative(int32_t target);
-
+    // used for hierarchical softmax:
     std::vector< std::vector<int32_t> > paths;
     std::vector< std::vector<bool> > codes;
     std::vector<Node> tree;
 
+    static bool comparePairs(const std::pair<real, int32_t>&,
+                             const std::pair<real, int32_t>&);
+
+    int32_t getNegative(int32_t target);
+    void initSigmoid();
+    void initLog();
+
     static const int32_t NEGATIVE_TABLE_SIZE = 10000000;
 
   public:
     Model(std::shared_ptr<Matrix>, std::shared_ptr<Matrix>,
           std::shared_ptr<Args>, int32_t);
+    ~Model();
 
     real binaryLogistic(int32_t, bool, real);
     real negativeSampling(int32_t, real);
@@ -86,6 +95,8 @@ class Model {
     void initTableNegatives(const std::vector<int64_t>&);
     void buildTree(const std::vector<int64_t>&);
     real getLoss() const;
+    real sigmoid(real) const;
+    real log(real) const;
 
     std::minstd_rand rng;
 };

src/utils.cc

@@ -15,57 +15,6 @@
 namespace fasttext {
 
 namespace utils {
-  real* t_sigmoid = nullptr;
-  real* t_log = nullptr;
-
-  real log(real x) {
-    if (x > 1.0) {
-      return 0.0;
-    }
-    int i = int(x * LOG_TABLE_SIZE);
-    return t_log[i];
-  }
-
-  real sigmoid(real x) {
-    if (x < -MAX_SIGMOID) {
-      return 0.0;
-    } else if (x > MAX_SIGMOID) {
-      return 1.0;
-    } else {
-      int i = int((x + MAX_SIGMOID) * SIGMOID_TABLE_SIZE / MAX_SIGMOID / 2);
-      return t_sigmoid[i];
-    }
-  }
-
-  void initTables() {
-    initSigmoid();
-    initLog();
-  }
-
-  void initSigmoid() {
-    if (t_sigmoid != nullptr) return;
-    t_sigmoid = new real[SIGMOID_TABLE_SIZE + 1];
-    for (int i = 0; i < SIGMOID_TABLE_SIZE + 1; i++) {
-      real x = real(i * 2 * MAX_SIGMOID) / SIGMOID_TABLE_SIZE - MAX_SIGMOID;
-      t_sigmoid[i] = 1.0 / (1.0 + std::exp(-x));
-    }
-  }
-
-  void initLog() {
-    if (t_log != nullptr) return;
-    t_log = new real[LOG_TABLE_SIZE + 1];
-    for (int i = 0; i < LOG_TABLE_SIZE + 1; i++) {
-      real x = (real(i) + 1e-5) / LOG_TABLE_SIZE;
-      t_log[i] = std::log(x);
-    }
-  }
-
-  void freeTables() {
-    delete[] t_sigmoid;
-    delete[] t_log;
-    t_sigmoid = nullptr;
-    t_log = nullptr;
-  }
 
   int64_t size(std::ifstream& ifs) {
     ifs.seekg(std::streamoff(0), std::ios::end);

src/utils.h

@@ -12,24 +12,10 @@
 
 #include <fstream>
 
-#include "real.h"
-
-#define SIGMOID_TABLE_SIZE 512
-#define MAX_SIGMOID 8
-#define LOG_TABLE_SIZE 512
-
 namespace fasttext {
 
 namespace utils {
 
-  real log(real);
-  real sigmoid(real);
-
-  void initTables();
-  void initSigmoid();
-  void initLog();
-  void freeTables();
-
   int64_t size(std::ifstream&);
   void seek(std::ifstream&, int64_t);
 }